1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights 4 * reserved. 5 */ 6 7 /* 8 * SN Platform Special Memory (mspec) Support 9 * 10 * This driver exports the SN special memory (mspec) facility to user 11 * processes. 12 * There are two types of memory made available thru this driver: 13 * uncached and cached. 14 * 15 * Uncached are used for memory write combining feature of the ia64 16 * cpu. 17 * 18 * Cached are used for areas of memory that are used as cached addresses 19 * on our partition and used as uncached addresses from other partitions. 20 * Due to a design constraint of the SN2 Shub, you can not have processors 21 * on the same FSB perform both a cached and uncached reference to the 22 * same cache line. These special memory cached regions prevent the 23 * kernel from ever dropping in a TLB entry and therefore prevent the 24 * processor from ever speculating a cache line from this page. 25 */ 26 27 #include <linux/types.h> 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/init.h> 31 #include <linux/errno.h> 32 #include <linux/miscdevice.h> 33 #include <linux/spinlock.h> 34 #include <linux/mm.h> 35 #include <linux/fs.h> 36 #include <linux/vmalloc.h> 37 #include <linux/string.h> 38 #include <linux/slab.h> 39 #include <linux/numa.h> 40 #include <linux/refcount.h> 41 #include <asm/page.h> 42 #include <asm/pgtable.h> 43 #include <linux/atomic.h> 44 #include <asm/tlbflush.h> 45 #include <asm/uncached.h> 46 47 48 #define CACHED_ID "Cached," 49 #define UNCACHED_ID "Uncached" 50 #define REVISION "4.0" 51 #define MSPEC_BASENAME "mspec" 52 53 /* 54 * Page types allocated by the device. 55 */ 56 enum mspec_page_type { 57 MSPEC_CACHED = 2, 58 MSPEC_UNCACHED 59 }; 60 61 /* 62 * One of these structures is allocated when an mspec region is mmaped. The 63 * structure is pointed to by the vma->vm_private_data field in the vma struct. 64 * This structure is used to record the addresses of the mspec pages. 65 * This structure is shared by all vma's that are split off from the 66 * original vma when split_vma()'s are done. 67 * 68 * The refcnt is incremented atomically because mm->mmap_sem does not 69 * protect in fork case where multiple tasks share the vma_data. 70 */ 71 struct vma_data { 72 refcount_t refcnt; /* Number of vmas sharing the data. */ 73 spinlock_t lock; /* Serialize access to this structure. */ 74 int count; /* Number of pages allocated. */ 75 enum mspec_page_type type; /* Type of pages allocated. */ 76 unsigned long vm_start; /* Original (unsplit) base. */ 77 unsigned long vm_end; /* Original (unsplit) end. */ 78 unsigned long maddr[]; /* Array of MSPEC addresses. */ 79 }; 80 81 /* 82 * mspec_open 83 * 84 * Called when a device mapping is created by a means other than mmap 85 * (via fork, munmap, etc.). Increments the reference count on the 86 * underlying mspec data so it is not freed prematurely. 87 */ 88 static void 89 mspec_open(struct vm_area_struct *vma) 90 { 91 struct vma_data *vdata; 92 93 vdata = vma->vm_private_data; 94 refcount_inc(&vdata->refcnt); 95 } 96 97 /* 98 * mspec_close 99 * 100 * Called when unmapping a device mapping. Frees all mspec pages 101 * belonging to all the vma's sharing this vma_data structure. 102 */ 103 static void 104 mspec_close(struct vm_area_struct *vma) 105 { 106 struct vma_data *vdata; 107 int index, last_index; 108 unsigned long my_page; 109 110 vdata = vma->vm_private_data; 111 112 if (!refcount_dec_and_test(&vdata->refcnt)) 113 return; 114 115 last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT; 116 for (index = 0; index < last_index; index++) { 117 if (vdata->maddr[index] == 0) 118 continue; 119 /* 120 * Clear the page before sticking it back 121 * into the pool. 122 */ 123 my_page = vdata->maddr[index]; 124 vdata->maddr[index] = 0; 125 memset((char *)my_page, 0, PAGE_SIZE); 126 uncached_free_page(my_page, 1); 127 } 128 129 kvfree(vdata); 130 } 131 132 /* 133 * mspec_fault 134 * 135 * Creates a mspec page and maps it to user space. 136 */ 137 static vm_fault_t 138 mspec_fault(struct vm_fault *vmf) 139 { 140 unsigned long paddr, maddr; 141 unsigned long pfn; 142 pgoff_t index = vmf->pgoff; 143 struct vma_data *vdata = vmf->vma->vm_private_data; 144 145 maddr = (volatile unsigned long) vdata->maddr[index]; 146 if (maddr == 0) { 147 maddr = uncached_alloc_page(numa_node_id(), 1); 148 if (maddr == 0) 149 return VM_FAULT_OOM; 150 151 spin_lock(&vdata->lock); 152 if (vdata->maddr[index] == 0) { 153 vdata->count++; 154 vdata->maddr[index] = maddr; 155 } else { 156 uncached_free_page(maddr, 1); 157 maddr = vdata->maddr[index]; 158 } 159 spin_unlock(&vdata->lock); 160 } 161 162 paddr = maddr & ~__IA64_UNCACHED_OFFSET; 163 pfn = paddr >> PAGE_SHIFT; 164 165 return vmf_insert_pfn(vmf->vma, vmf->address, pfn); 166 } 167 168 static const struct vm_operations_struct mspec_vm_ops = { 169 .open = mspec_open, 170 .close = mspec_close, 171 .fault = mspec_fault, 172 }; 173 174 /* 175 * mspec_mmap 176 * 177 * Called when mmapping the device. Initializes the vma with a fault handler 178 * and private data structure necessary to allocate, track, and free the 179 * underlying pages. 180 */ 181 static int 182 mspec_mmap(struct file *file, struct vm_area_struct *vma, 183 enum mspec_page_type type) 184 { 185 struct vma_data *vdata; 186 int pages, vdata_size; 187 188 if (vma->vm_pgoff != 0) 189 return -EINVAL; 190 191 if ((vma->vm_flags & VM_SHARED) == 0) 192 return -EINVAL; 193 194 if ((vma->vm_flags & VM_WRITE) == 0) 195 return -EPERM; 196 197 pages = vma_pages(vma); 198 vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 199 if (vdata_size <= PAGE_SIZE) 200 vdata = kzalloc(vdata_size, GFP_KERNEL); 201 else 202 vdata = vzalloc(vdata_size); 203 if (!vdata) 204 return -ENOMEM; 205 206 vdata->vm_start = vma->vm_start; 207 vdata->vm_end = vma->vm_end; 208 vdata->type = type; 209 spin_lock_init(&vdata->lock); 210 refcount_set(&vdata->refcnt, 1); 211 vma->vm_private_data = vdata; 212 213 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 214 if (vdata->type == MSPEC_UNCACHED) 215 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 216 vma->vm_ops = &mspec_vm_ops; 217 218 return 0; 219 } 220 221 static int 222 cached_mmap(struct file *file, struct vm_area_struct *vma) 223 { 224 return mspec_mmap(file, vma, MSPEC_CACHED); 225 } 226 227 static int 228 uncached_mmap(struct file *file, struct vm_area_struct *vma) 229 { 230 return mspec_mmap(file, vma, MSPEC_UNCACHED); 231 } 232 233 static const struct file_operations cached_fops = { 234 .owner = THIS_MODULE, 235 .mmap = cached_mmap, 236 .llseek = noop_llseek, 237 }; 238 239 static struct miscdevice cached_miscdev = { 240 .minor = MISC_DYNAMIC_MINOR, 241 .name = "mspec_cached", 242 .fops = &cached_fops 243 }; 244 245 static const struct file_operations uncached_fops = { 246 .owner = THIS_MODULE, 247 .mmap = uncached_mmap, 248 .llseek = noop_llseek, 249 }; 250 251 static struct miscdevice uncached_miscdev = { 252 .minor = MISC_DYNAMIC_MINOR, 253 .name = "mspec_uncached", 254 .fops = &uncached_fops 255 }; 256 257 /* 258 * mspec_init 259 * 260 * Called at boot time to initialize the mspec facility. 261 */ 262 static int __init 263 mspec_init(void) 264 { 265 int ret; 266 267 ret = misc_register(&cached_miscdev); 268 if (ret) { 269 printk(KERN_ERR "%s: failed to register device %i\n", 270 CACHED_ID, ret); 271 return ret; 272 } 273 ret = misc_register(&uncached_miscdev); 274 if (ret) { 275 printk(KERN_ERR "%s: failed to register device %i\n", 276 UNCACHED_ID, ret); 277 misc_deregister(&cached_miscdev); 278 return ret; 279 } 280 281 printk(KERN_INFO "%s %s initialized devices: %s %s\n", 282 MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID); 283 284 return 0; 285 } 286 287 static void __exit 288 mspec_exit(void) 289 { 290 misc_deregister(&uncached_miscdev); 291 misc_deregister(&cached_miscdev); 292 } 293 294 module_init(mspec_init); 295 module_exit(mspec_exit); 296 297 MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>"); 298 MODULE_DESCRIPTION("Driver for SGI SN special memory operations"); 299 MODULE_LICENSE("GPL"); 300