1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
  4 * reserved.
  5 */
  6
  7/*
  8 * SN Platform Special Memory (mspec) Support
  9 *
 10 * This driver exports the SN special memory (mspec) facility to user
 11 * processes.
 12 * There are two types of memory made available thru this driver:
 13 * uncached and cached.
 14 *
 15 * Uncached are used for memory write combining feature of the ia64
 16 * cpu.
 17 *
 18 * Cached are used for areas of memory that are used as cached addresses
 19 * on our partition and used as uncached addresses from other partitions.
 20 * Due to a design constraint of the SN2 Shub, you can not have processors
 21 * on the same FSB perform both a cached and uncached reference to the
 22 * same cache line.  These special memory cached regions prevent the
 23 * kernel from ever dropping in a TLB entry and therefore prevent the
 24 * processor from ever speculating a cache line from this page.
 25 */
 26
 27#include <linux/types.h>
 28#include <linux/kernel.h>
 29#include <linux/module.h>
 30#include <linux/init.h>
 31#include <linux/errno.h>
 32#include <linux/miscdevice.h>
 33#include <linux/spinlock.h>
 34#include <linux/mm.h>
 35#include <linux/fs.h>
 36#include <linux/vmalloc.h>
 37#include <linux/string.h>
 38#include <linux/slab.h>
 39#include <linux/numa.h>
 40#include <linux/refcount.h>
 41#include <asm/page.h>
 42#include <linux/atomic.h>
 43#include <asm/tlbflush.h>
 44#include <asm/uncached.h>
 45
 46
 47#define CACHED_ID	"Cached,"
 48#define UNCACHED_ID	"Uncached"
 49#define REVISION	"4.0"
 50#define MSPEC_BASENAME	"mspec"
 51
 52/*
 53 * Page types allocated by the device.
 54 */
 55enum mspec_page_type {
 56	MSPEC_CACHED = 2,
 57	MSPEC_UNCACHED
 58};
 59
 60/*
 61 * One of these structures is allocated when an mspec region is mmaped. The
 62 * structure is pointed to by the vma->vm_private_data field in the vma struct.
 63 * This structure is used to record the addresses of the mspec pages.
 64 * This structure is shared by all vma's that are split off from the
 65 * original vma when split_vma()'s are done.
 66 *
 67 * The refcnt is incremented atomically because mm->mmap_lock does not
 68 * protect in fork case where multiple tasks share the vma_data.
 69 */
 70struct vma_data {
 71	refcount_t refcnt;	/* Number of vmas sharing the data. */
 72	spinlock_t lock;	/* Serialize access to this structure. */
 73	int count;		/* Number of pages allocated. */
 74	enum mspec_page_type type; /* Type of pages allocated. */
 75	unsigned long vm_start;	/* Original (unsplit) base. */
 76	unsigned long vm_end;	/* Original (unsplit) end. */
 77	unsigned long maddr[];	/* Array of MSPEC addresses. */
 78};
 79
 80/*
 81 * mspec_open
 82 *
 83 * Called when a device mapping is created by a means other than mmap
 84 * (via fork, munmap, etc.).  Increments the reference count on the
 85 * underlying mspec data so it is not freed prematurely.
 86 */
 87static void
 88mspec_open(struct vm_area_struct *vma)
 89{
 90	struct vma_data *vdata;
 91
 92	vdata = vma->vm_private_data;
 93	refcount_inc(&vdata->refcnt);
 94}
 95
 96/*
 97 * mspec_close
 98 *
 99 * Called when unmapping a device mapping. Frees all mspec pages
100 * belonging to all the vma's sharing this vma_data structure.
101 */
102static void
103mspec_close(struct vm_area_struct *vma)
104{
105	struct vma_data *vdata;
106	int index, last_index;
107	unsigned long my_page;
108
109	vdata = vma->vm_private_data;
110
111	if (!refcount_dec_and_test(&vdata->refcnt))
112		return;
113
114	last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT;
115	for (index = 0; index < last_index; index++) {
116		if (vdata->maddr[index] == 0)
117			continue;
118		/*
119		 * Clear the page before sticking it back
120		 * into the pool.
121		 */
122		my_page = vdata->maddr[index];
123		vdata->maddr[index] = 0;
124		memset((char *)my_page, 0, PAGE_SIZE);
125		uncached_free_page(my_page, 1);
126	}
127
128	kvfree(vdata);
129}
130
131/*
132 * mspec_fault
133 *
134 * Creates a mspec page and maps it to user space.
135 */
136static vm_fault_t
137mspec_fault(struct vm_fault *vmf)
138{
139	unsigned long paddr, maddr;
140	unsigned long pfn;
141	pgoff_t index = vmf->pgoff;
142	struct vma_data *vdata = vmf->vma->vm_private_data;
143
144	maddr = (volatile unsigned long) vdata->maddr[index];
145	if (maddr == 0) {
146		maddr = uncached_alloc_page(numa_node_id(), 1);
147		if (maddr == 0)
148			return VM_FAULT_OOM;
149
150		spin_lock(&vdata->lock);
151		if (vdata->maddr[index] == 0) {
152			vdata->count++;
153			vdata->maddr[index] = maddr;
154		} else {
155			uncached_free_page(maddr, 1);
156			maddr = vdata->maddr[index];
157		}
158		spin_unlock(&vdata->lock);
159	}
160
161	paddr = maddr & ~__IA64_UNCACHED_OFFSET;
162	pfn = paddr >> PAGE_SHIFT;
163
164	return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
165}
166
167static const struct vm_operations_struct mspec_vm_ops = {
168	.open = mspec_open,
169	.close = mspec_close,
170	.fault = mspec_fault,
171};
172
173/*
174 * mspec_mmap
175 *
176 * Called when mmapping the device.  Initializes the vma with a fault handler
177 * and private data structure necessary to allocate, track, and free the
178 * underlying pages.
179 */
180static int
181mspec_mmap(struct file *file, struct vm_area_struct *vma,
182					enum mspec_page_type type)
183{
184	struct vma_data *vdata;
185	int pages, vdata_size;
186
187	if (vma->vm_pgoff != 0)
188		return -EINVAL;
189
190	if ((vma->vm_flags & VM_SHARED) == 0)
191		return -EINVAL;
192
193	if ((vma->vm_flags & VM_WRITE) == 0)
194		return -EPERM;
195
196	pages = vma_pages(vma);
197	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
198	vdata = kvzalloc(vdata_size, GFP_KERNEL);
199	if (!vdata)
200		return -ENOMEM;
201
202	vdata->vm_start = vma->vm_start;
203	vdata->vm_end = vma->vm_end;
204	vdata->type = type;
205	spin_lock_init(&vdata->lock);
206	refcount_set(&vdata->refcnt, 1);
207	vma->vm_private_data = vdata;
208
209	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
210	if (vdata->type == MSPEC_UNCACHED)
211		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
212	vma->vm_ops = &mspec_vm_ops;
213
214	return 0;
215}
216
217static int
218cached_mmap(struct file *file, struct vm_area_struct *vma)
219{
220	return mspec_mmap(file, vma, MSPEC_CACHED);
221}
222
223static int
224uncached_mmap(struct file *file, struct vm_area_struct *vma)
225{
226	return mspec_mmap(file, vma, MSPEC_UNCACHED);
227}
228
229static const struct file_operations cached_fops = {
230	.owner = THIS_MODULE,
231	.mmap = cached_mmap,
232	.llseek = noop_llseek,
233};
234
235static struct miscdevice cached_miscdev = {
236	.minor = MISC_DYNAMIC_MINOR,
237	.name = "mspec_cached",
238	.fops = &cached_fops
239};
240
241static const struct file_operations uncached_fops = {
242	.owner = THIS_MODULE,
243	.mmap = uncached_mmap,
244	.llseek = noop_llseek,
245};
246
247static struct miscdevice uncached_miscdev = {
248	.minor = MISC_DYNAMIC_MINOR,
249	.name = "mspec_uncached",
250	.fops = &uncached_fops
251};
252
253/*
254 * mspec_init
255 *
256 * Called at boot time to initialize the mspec facility.
257 */
258static int __init
259mspec_init(void)
260{
261	int ret;
262
263	ret = misc_register(&cached_miscdev);
264	if (ret) {
265		printk(KERN_ERR "%s: failed to register device %i\n",
266		       CACHED_ID, ret);
267		return ret;
268	}
269	ret = misc_register(&uncached_miscdev);
270	if (ret) {
271		printk(KERN_ERR "%s: failed to register device %i\n",
272		       UNCACHED_ID, ret);
273		misc_deregister(&cached_miscdev);
274		return ret;
275	}
276
277	printk(KERN_INFO "%s %s initialized devices: %s %s\n",
278	       MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID);
279
280	return 0;
281}
282
283static void __exit
284mspec_exit(void)
285{
286	misc_deregister(&uncached_miscdev);
287	misc_deregister(&cached_miscdev);
288}
289
290module_init(mspec_init);
291module_exit(mspec_exit);
292
293MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
294MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
295MODULE_LICENSE("GPL");