1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights 4 * reserved. 5 */ 6 7 /* 8 * SN Platform Special Memory (mspec) Support 9 * 10 * This driver exports the SN special memory (mspec) facility to user 11 * processes. 12 * There are two types of memory made available thru this driver: 13 * uncached and cached. 14 * 15 * Uncached are used for memory write combining feature of the ia64 16 * cpu. 17 * 18 * Cached are used for areas of memory that are used as cached addresses 19 * on our partition and used as uncached addresses from other partitions. 20 * Due to a design constraint of the SN2 Shub, you can not have processors 21 * on the same FSB perform both a cached and uncached reference to the 22 * same cache line. These special memory cached regions prevent the 23 * kernel from ever dropping in a TLB entry and therefore prevent the 24 * processor from ever speculating a cache line from this page. 25 */ 26 27 #include <linux/types.h> 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/init.h> 31 #include <linux/errno.h> 32 #include <linux/miscdevice.h> 33 #include <linux/spinlock.h> 34 #include <linux/mm.h> 35 #include <linux/fs.h> 36 #include <linux/vmalloc.h> 37 #include <linux/string.h> 38 #include <linux/slab.h> 39 #include <linux/numa.h> 40 #include <linux/refcount.h> 41 #include <asm/page.h> 42 #include <linux/atomic.h> 43 #include <asm/tlbflush.h> 44 #include <asm/uncached.h> 45 46 47 #define CACHED_ID "Cached," 48 #define UNCACHED_ID "Uncached" 49 #define REVISION "4.0" 50 #define MSPEC_BASENAME "mspec" 51 52 /* 53 * Page types allocated by the device. 54 */ 55 enum mspec_page_type { 56 MSPEC_CACHED = 2, 57 MSPEC_UNCACHED 58 }; 59 60 /* 61 * One of these structures is allocated when an mspec region is mmaped. The 62 * structure is pointed to by the vma->vm_private_data field in the vma struct. 63 * This structure is used to record the addresses of the mspec pages. 64 * This structure is shared by all vma's that are split off from the 65 * original vma when split_vma()'s are done. 66 * 67 * The refcnt is incremented atomically because mm->mmap_lock does not 68 * protect in fork case where multiple tasks share the vma_data. 69 */ 70 struct vma_data { 71 refcount_t refcnt; /* Number of vmas sharing the data. */ 72 spinlock_t lock; /* Serialize access to this structure. */ 73 int count; /* Number of pages allocated. */ 74 enum mspec_page_type type; /* Type of pages allocated. */ 75 unsigned long vm_start; /* Original (unsplit) base. */ 76 unsigned long vm_end; /* Original (unsplit) end. */ 77 unsigned long maddr[]; /* Array of MSPEC addresses. */ 78 }; 79 80 /* 81 * mspec_open 82 * 83 * Called when a device mapping is created by a means other than mmap 84 * (via fork, munmap, etc.). Increments the reference count on the 85 * underlying mspec data so it is not freed prematurely. 86 */ 87 static void 88 mspec_open(struct vm_area_struct *vma) 89 { 90 struct vma_data *vdata; 91 92 vdata = vma->vm_private_data; 93 refcount_inc(&vdata->refcnt); 94 } 95 96 /* 97 * mspec_close 98 * 99 * Called when unmapping a device mapping. Frees all mspec pages 100 * belonging to all the vma's sharing this vma_data structure. 101 */ 102 static void 103 mspec_close(struct vm_area_struct *vma) 104 { 105 struct vma_data *vdata; 106 int index, last_index; 107 unsigned long my_page; 108 109 vdata = vma->vm_private_data; 110 111 if (!refcount_dec_and_test(&vdata->refcnt)) 112 return; 113 114 last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT; 115 for (index = 0; index < last_index; index++) { 116 if (vdata->maddr[index] == 0) 117 continue; 118 /* 119 * Clear the page before sticking it back 120 * into the pool. 121 */ 122 my_page = vdata->maddr[index]; 123 vdata->maddr[index] = 0; 124 memset((char *)my_page, 0, PAGE_SIZE); 125 uncached_free_page(my_page, 1); 126 } 127 128 kvfree(vdata); 129 } 130 131 /* 132 * mspec_fault 133 * 134 * Creates a mspec page and maps it to user space. 135 */ 136 static vm_fault_t 137 mspec_fault(struct vm_fault *vmf) 138 { 139 unsigned long paddr, maddr; 140 unsigned long pfn; 141 pgoff_t index = vmf->pgoff; 142 struct vma_data *vdata = vmf->vma->vm_private_data; 143 144 maddr = (volatile unsigned long) vdata->maddr[index]; 145 if (maddr == 0) { 146 maddr = uncached_alloc_page(numa_node_id(), 1); 147 if (maddr == 0) 148 return VM_FAULT_OOM; 149 150 spin_lock(&vdata->lock); 151 if (vdata->maddr[index] == 0) { 152 vdata->count++; 153 vdata->maddr[index] = maddr; 154 } else { 155 uncached_free_page(maddr, 1); 156 maddr = vdata->maddr[index]; 157 } 158 spin_unlock(&vdata->lock); 159 } 160 161 paddr = maddr & ~__IA64_UNCACHED_OFFSET; 162 pfn = paddr >> PAGE_SHIFT; 163 164 return vmf_insert_pfn(vmf->vma, vmf->address, pfn); 165 } 166 167 static const struct vm_operations_struct mspec_vm_ops = { 168 .open = mspec_open, 169 .close = mspec_close, 170 .fault = mspec_fault, 171 }; 172 173 /* 174 * mspec_mmap 175 * 176 * Called when mmapping the device. Initializes the vma with a fault handler 177 * and private data structure necessary to allocate, track, and free the 178 * underlying pages. 179 */ 180 static int 181 mspec_mmap(struct file *file, struct vm_area_struct *vma, 182 enum mspec_page_type type) 183 { 184 struct vma_data *vdata; 185 int pages, vdata_size; 186 187 if (vma->vm_pgoff != 0) 188 return -EINVAL; 189 190 if ((vma->vm_flags & VM_SHARED) == 0) 191 return -EINVAL; 192 193 if ((vma->vm_flags & VM_WRITE) == 0) 194 return -EPERM; 195 196 pages = vma_pages(vma); 197 vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 198 vdata = kvzalloc(vdata_size, GFP_KERNEL); 199 if (!vdata) 200 return -ENOMEM; 201 202 vdata->vm_start = vma->vm_start; 203 vdata->vm_end = vma->vm_end; 204 vdata->type = type; 205 spin_lock_init(&vdata->lock); 206 refcount_set(&vdata->refcnt, 1); 207 vma->vm_private_data = vdata; 208 209 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 210 if (vdata->type == MSPEC_UNCACHED) 211 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 212 vma->vm_ops = &mspec_vm_ops; 213 214 return 0; 215 } 216 217 static int 218 cached_mmap(struct file *file, struct vm_area_struct *vma) 219 { 220 return mspec_mmap(file, vma, MSPEC_CACHED); 221 } 222 223 static int 224 uncached_mmap(struct file *file, struct vm_area_struct *vma) 225 { 226 return mspec_mmap(file, vma, MSPEC_UNCACHED); 227 } 228 229 static const struct file_operations cached_fops = { 230 .owner = THIS_MODULE, 231 .mmap = cached_mmap, 232 .llseek = noop_llseek, 233 }; 234 235 static struct miscdevice cached_miscdev = { 236 .minor = MISC_DYNAMIC_MINOR, 237 .name = "mspec_cached", 238 .fops = &cached_fops 239 }; 240 241 static const struct file_operations uncached_fops = { 242 .owner = THIS_MODULE, 243 .mmap = uncached_mmap, 244 .llseek = noop_llseek, 245 }; 246 247 static struct miscdevice uncached_miscdev = { 248 .minor = MISC_DYNAMIC_MINOR, 249 .name = "mspec_uncached", 250 .fops = &uncached_fops 251 }; 252 253 /* 254 * mspec_init 255 * 256 * Called at boot time to initialize the mspec facility. 257 */ 258 static int __init 259 mspec_init(void) 260 { 261 int ret; 262 263 ret = misc_register(&cached_miscdev); 264 if (ret) { 265 printk(KERN_ERR "%s: failed to register device %i\n", 266 CACHED_ID, ret); 267 return ret; 268 } 269 ret = misc_register(&uncached_miscdev); 270 if (ret) { 271 printk(KERN_ERR "%s: failed to register device %i\n", 272 UNCACHED_ID, ret); 273 misc_deregister(&cached_miscdev); 274 return ret; 275 } 276 277 printk(KERN_INFO "%s %s initialized devices: %s %s\n", 278 MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID); 279 280 return 0; 281 } 282 283 static void __exit 284 mspec_exit(void) 285 { 286 misc_deregister(&uncached_miscdev); 287 misc_deregister(&cached_miscdev); 288 } 289 290 module_init(mspec_init); 291 module_exit(mspec_exit); 292 293 MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>"); 294 MODULE_DESCRIPTION("Driver for SGI SN special memory operations"); 295 MODULE_LICENSE("GPL"); 296