125763b3cSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
217a3b050SJes Sorensen /*
317a3b050SJes Sorensen * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights
417a3b050SJes Sorensen * reserved.
517a3b050SJes Sorensen */
617a3b050SJes Sorensen
717a3b050SJes Sorensen /*
817a3b050SJes Sorensen * SN Platform Special Memory (mspec) Support
917a3b050SJes Sorensen *
1017a3b050SJes Sorensen * This driver exports the SN special memory (mspec) facility to user
1117a3b050SJes Sorensen * processes.
120fef2532SChristoph Hellwig * There are two types of memory made available thru this driver:
130fef2532SChristoph Hellwig * uncached and cached.
1417a3b050SJes Sorensen *
1517a3b050SJes Sorensen * Uncached are used for memory write combining feature of the ia64
1617a3b050SJes Sorensen * cpu.
1717a3b050SJes Sorensen *
1817a3b050SJes Sorensen * Cached are used for areas of memory that are used as cached addresses
1917a3b050SJes Sorensen * on our partition and used as uncached addresses from other partitions.
2017a3b050SJes Sorensen * Due to a design constraint of the SN2 Shub, you can not have processors
2117a3b050SJes Sorensen * on the same FSB perform both a cached and uncached reference to the
2217a3b050SJes Sorensen * same cache line. These special memory cached regions prevent the
2317a3b050SJes Sorensen * kernel from ever dropping in a TLB entry and therefore prevent the
2417a3b050SJes Sorensen * processor from ever speculating a cache line from this page.
2517a3b050SJes Sorensen */
2617a3b050SJes Sorensen
2717a3b050SJes Sorensen #include <linux/types.h>
2817a3b050SJes Sorensen #include <linux/kernel.h>
2917a3b050SJes Sorensen #include <linux/module.h>
3017a3b050SJes Sorensen #include <linux/init.h>
3117a3b050SJes Sorensen #include <linux/errno.h>
3217a3b050SJes Sorensen #include <linux/miscdevice.h>
3317a3b050SJes Sorensen #include <linux/spinlock.h>
3417a3b050SJes Sorensen #include <linux/mm.h>
354e950f6fSAlexey Dobriyan #include <linux/fs.h>
3617a3b050SJes Sorensen #include <linux/vmalloc.h>
3717a3b050SJes Sorensen #include <linux/string.h>
3817a3b050SJes Sorensen #include <linux/slab.h>
3917a3b050SJes Sorensen #include <linux/numa.h>
40f7d88d24SElena Reshetova #include <linux/refcount.h>
4117a3b050SJes Sorensen #include <asm/page.h>
4260063497SArun Sharma #include <linux/atomic.h>
4317a3b050SJes Sorensen #include <asm/tlbflush.h>
4417a3b050SJes Sorensen #include <asm/uncached.h>
4517a3b050SJes Sorensen
4617a3b050SJes Sorensen
4717a3b050SJes Sorensen #define CACHED_ID "Cached,"
4817a3b050SJes Sorensen #define UNCACHED_ID "Uncached"
4917a3b050SJes Sorensen #define REVISION "4.0"
5017a3b050SJes Sorensen #define MSPEC_BASENAME "mspec"
5117a3b050SJes Sorensen
5217a3b050SJes Sorensen /*
5317a3b050SJes Sorensen * Page types allocated by the device.
5417a3b050SJes Sorensen */
554191ba26SCliff Wickman enum mspec_page_type {
560fef2532SChristoph Hellwig MSPEC_CACHED = 2,
5717a3b050SJes Sorensen MSPEC_UNCACHED
5817a3b050SJes Sorensen };
5917a3b050SJes Sorensen
6017a3b050SJes Sorensen /*
6117a3b050SJes Sorensen * One of these structures is allocated when an mspec region is mmaped. The
6217a3b050SJes Sorensen * structure is pointed to by the vma->vm_private_data field in the vma struct.
6317a3b050SJes Sorensen * This structure is used to record the addresses of the mspec pages.
644191ba26SCliff Wickman * This structure is shared by all vma's that are split off from the
654191ba26SCliff Wickman * original vma when split_vma()'s are done.
664191ba26SCliff Wickman *
67c1e8d7c6SMichel Lespinasse * The refcnt is incremented atomically because mm->mmap_lock does not
684191ba26SCliff Wickman * protect in fork case where multiple tasks share the vma_data.
6917a3b050SJes Sorensen */
7017a3b050SJes Sorensen struct vma_data {
71f7d88d24SElena Reshetova refcount_t refcnt; /* Number of vmas sharing the data. */
724191ba26SCliff Wickman spinlock_t lock; /* Serialize access to this structure. */
7317a3b050SJes Sorensen int count; /* Number of pages allocated. */
744191ba26SCliff Wickman enum mspec_page_type type; /* Type of pages allocated. */
754191ba26SCliff Wickman unsigned long vm_start; /* Original (unsplit) base. */
764191ba26SCliff Wickman unsigned long vm_end; /* Original (unsplit) end. */
773c2faf61SGustavo A. R. Silva unsigned long maddr[]; /* Array of MSPEC addresses. */
7817a3b050SJes Sorensen };
7917a3b050SJes Sorensen
8017a3b050SJes Sorensen /*
8117a3b050SJes Sorensen * mspec_open
8217a3b050SJes Sorensen *
8317a3b050SJes Sorensen * Called when a device mapping is created by a means other than mmap
844191ba26SCliff Wickman * (via fork, munmap, etc.). Increments the reference count on the
854191ba26SCliff Wickman * underlying mspec data so it is not freed prematurely.
8617a3b050SJes Sorensen */
8717a3b050SJes Sorensen static void
mspec_open(struct vm_area_struct * vma)8817a3b050SJes Sorensen mspec_open(struct vm_area_struct *vma)
8917a3b050SJes Sorensen {
9017a3b050SJes Sorensen struct vma_data *vdata;
9117a3b050SJes Sorensen
9217a3b050SJes Sorensen vdata = vma->vm_private_data;
93f7d88d24SElena Reshetova refcount_inc(&vdata->refcnt);
9417a3b050SJes Sorensen }
9517a3b050SJes Sorensen
9617a3b050SJes Sorensen /*
9717a3b050SJes Sorensen * mspec_close
9817a3b050SJes Sorensen *
9917a3b050SJes Sorensen * Called when unmapping a device mapping. Frees all mspec pages
100afa684f6SCliff Wickman * belonging to all the vma's sharing this vma_data structure.
10117a3b050SJes Sorensen */
10217a3b050SJes Sorensen static void
mspec_close(struct vm_area_struct * vma)10317a3b050SJes Sorensen mspec_close(struct vm_area_struct *vma)
10417a3b050SJes Sorensen {
10517a3b050SJes Sorensen struct vma_data *vdata;
106afa684f6SCliff Wickman int index, last_index;
1074191ba26SCliff Wickman unsigned long my_page;
10817a3b050SJes Sorensen
10917a3b050SJes Sorensen vdata = vma->vm_private_data;
11017a3b050SJes Sorensen
111f7d88d24SElena Reshetova if (!refcount_dec_and_test(&vdata->refcnt))
112afa684f6SCliff Wickman return;
1134191ba26SCliff Wickman
114afa684f6SCliff Wickman last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT;
115afa684f6SCliff Wickman for (index = 0; index < last_index; index++) {
1164191ba26SCliff Wickman if (vdata->maddr[index] == 0)
11717a3b050SJes Sorensen continue;
11817a3b050SJes Sorensen /*
11917a3b050SJes Sorensen * Clear the page before sticking it back
12017a3b050SJes Sorensen * into the pool.
12117a3b050SJes Sorensen */
1224191ba26SCliff Wickman my_page = vdata->maddr[index];
1234191ba26SCliff Wickman vdata->maddr[index] = 0;
1240fef2532SChristoph Hellwig memset((char *)my_page, 0, PAGE_SIZE);
125e4a064dfSDean Nelson uncached_free_page(my_page, 1);
12617a3b050SJes Sorensen }
1274191ba26SCliff Wickman
1281d5cfdb0STetsuo Handa kvfree(vdata);
12917a3b050SJes Sorensen }
13017a3b050SJes Sorensen
13117a3b050SJes Sorensen /*
132efe9e779SNick Piggin * mspec_fault
13317a3b050SJes Sorensen *
13417a3b050SJes Sorensen * Creates a mspec page and maps it to user space.
13517a3b050SJes Sorensen */
1363eb87d4eSSouptick Joarder static vm_fault_t
mspec_fault(struct vm_fault * vmf)13711bac800SDave Jiang mspec_fault(struct vm_fault *vmf)
13817a3b050SJes Sorensen {
13917a3b050SJes Sorensen unsigned long paddr, maddr;
14017a3b050SJes Sorensen unsigned long pfn;
141efe9e779SNick Piggin pgoff_t index = vmf->pgoff;
14211bac800SDave Jiang struct vma_data *vdata = vmf->vma->vm_private_data;
14317a3b050SJes Sorensen
14417a3b050SJes Sorensen maddr = (volatile unsigned long) vdata->maddr[index];
14517a3b050SJes Sorensen if (maddr == 0) {
146e4a064dfSDean Nelson maddr = uncached_alloc_page(numa_node_id(), 1);
14717a3b050SJes Sorensen if (maddr == 0)
148efe9e779SNick Piggin return VM_FAULT_OOM;
14917a3b050SJes Sorensen
15017a3b050SJes Sorensen spin_lock(&vdata->lock);
15117a3b050SJes Sorensen if (vdata->maddr[index] == 0) {
15217a3b050SJes Sorensen vdata->count++;
15317a3b050SJes Sorensen vdata->maddr[index] = maddr;
15417a3b050SJes Sorensen } else {
155e4a064dfSDean Nelson uncached_free_page(maddr, 1);
15617a3b050SJes Sorensen maddr = vdata->maddr[index];
15717a3b050SJes Sorensen }
15817a3b050SJes Sorensen spin_unlock(&vdata->lock);
15917a3b050SJes Sorensen }
16017a3b050SJes Sorensen
1611a4b0fc5SJes Sorensen paddr = maddr & ~__IA64_UNCACHED_OFFSET;
16217a3b050SJes Sorensen pfn = paddr >> PAGE_SHIFT;
16317a3b050SJes Sorensen
1643eb87d4eSSouptick Joarder return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
16517a3b050SJes Sorensen }
16617a3b050SJes Sorensen
167f0f37e2fSAlexey Dobriyan static const struct vm_operations_struct mspec_vm_ops = {
16817a3b050SJes Sorensen .open = mspec_open,
16917a3b050SJes Sorensen .close = mspec_close,
170efe9e779SNick Piggin .fault = mspec_fault,
17117a3b050SJes Sorensen };
17217a3b050SJes Sorensen
17317a3b050SJes Sorensen /*
17417a3b050SJes Sorensen * mspec_mmap
17517a3b050SJes Sorensen *
176af901ca1SAndré Goddard Rosa * Called when mmapping the device. Initializes the vma with a fault handler
17717a3b050SJes Sorensen * and private data structure necessary to allocate, track, and free the
17817a3b050SJes Sorensen * underlying pages.
17917a3b050SJes Sorensen */
18017a3b050SJes Sorensen static int
mspec_mmap(struct file * file,struct vm_area_struct * vma,enum mspec_page_type type)1814191ba26SCliff Wickman mspec_mmap(struct file *file, struct vm_area_struct *vma,
1824191ba26SCliff Wickman enum mspec_page_type type)
18317a3b050SJes Sorensen {
18417a3b050SJes Sorensen struct vma_data *vdata;
1851d5cfdb0STetsuo Handa int pages, vdata_size;
18617a3b050SJes Sorensen
18717a3b050SJes Sorensen if (vma->vm_pgoff != 0)
18817a3b050SJes Sorensen return -EINVAL;
18917a3b050SJes Sorensen
19017a3b050SJes Sorensen if ((vma->vm_flags & VM_SHARED) == 0)
19117a3b050SJes Sorensen return -EINVAL;
19217a3b050SJes Sorensen
19317a3b050SJes Sorensen if ((vma->vm_flags & VM_WRITE) == 0)
19417a3b050SJes Sorensen return -EPERM;
19517a3b050SJes Sorensen
196a0ea59d5SLibin pages = vma_pages(vma);
19717a3b050SJes Sorensen vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
198fe69795eSDenis Efremov vdata = kvzalloc(vdata_size, GFP_KERNEL);
19917a3b050SJes Sorensen if (!vdata)
20017a3b050SJes Sorensen return -ENOMEM;
20117a3b050SJes Sorensen
2024191ba26SCliff Wickman vdata->vm_start = vma->vm_start;
2034191ba26SCliff Wickman vdata->vm_end = vma->vm_end;
20417a3b050SJes Sorensen vdata->type = type;
20517a3b050SJes Sorensen spin_lock_init(&vdata->lock);
206f7d88d24SElena Reshetova refcount_set(&vdata->refcnt, 1);
20717a3b050SJes Sorensen vma->vm_private_data = vdata;
20817a3b050SJes Sorensen
209*1c71222eSSuren Baghdasaryan vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
2100fef2532SChristoph Hellwig if (vdata->type == MSPEC_UNCACHED)
21117a3b050SJes Sorensen vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
21217a3b050SJes Sorensen vma->vm_ops = &mspec_vm_ops;
21317a3b050SJes Sorensen
21417a3b050SJes Sorensen return 0;
21517a3b050SJes Sorensen }
21617a3b050SJes Sorensen
21717a3b050SJes Sorensen static int
cached_mmap(struct file * file,struct vm_area_struct * vma)21817a3b050SJes Sorensen cached_mmap(struct file *file, struct vm_area_struct *vma)
21917a3b050SJes Sorensen {
22017a3b050SJes Sorensen return mspec_mmap(file, vma, MSPEC_CACHED);
22117a3b050SJes Sorensen }
22217a3b050SJes Sorensen
22317a3b050SJes Sorensen static int
uncached_mmap(struct file * file,struct vm_area_struct * vma)22417a3b050SJes Sorensen uncached_mmap(struct file *file, struct vm_area_struct *vma)
22517a3b050SJes Sorensen {
22617a3b050SJes Sorensen return mspec_mmap(file, vma, MSPEC_UNCACHED);
22717a3b050SJes Sorensen }
22817a3b050SJes Sorensen
2292b8693c0SArjan van de Ven static const struct file_operations cached_fops = {
23017a3b050SJes Sorensen .owner = THIS_MODULE,
2316038f373SArnd Bergmann .mmap = cached_mmap,
2326038f373SArnd Bergmann .llseek = noop_llseek,
23317a3b050SJes Sorensen };
23417a3b050SJes Sorensen
23517a3b050SJes Sorensen static struct miscdevice cached_miscdev = {
23617a3b050SJes Sorensen .minor = MISC_DYNAMIC_MINOR,
23717a3b050SJes Sorensen .name = "mspec_cached",
23817a3b050SJes Sorensen .fops = &cached_fops
23917a3b050SJes Sorensen };
24017a3b050SJes Sorensen
2412b8693c0SArjan van de Ven static const struct file_operations uncached_fops = {
24217a3b050SJes Sorensen .owner = THIS_MODULE,
2436038f373SArnd Bergmann .mmap = uncached_mmap,
2446038f373SArnd Bergmann .llseek = noop_llseek,
24517a3b050SJes Sorensen };
24617a3b050SJes Sorensen
24717a3b050SJes Sorensen static struct miscdevice uncached_miscdev = {
24817a3b050SJes Sorensen .minor = MISC_DYNAMIC_MINOR,
24917a3b050SJes Sorensen .name = "mspec_uncached",
25017a3b050SJes Sorensen .fops = &uncached_fops
25117a3b050SJes Sorensen };
25217a3b050SJes Sorensen
25317a3b050SJes Sorensen /*
25417a3b050SJes Sorensen * mspec_init
25517a3b050SJes Sorensen *
25617a3b050SJes Sorensen * Called at boot time to initialize the mspec facility.
25717a3b050SJes Sorensen */
25817a3b050SJes Sorensen static int __init
mspec_init(void)25917a3b050SJes Sorensen mspec_init(void)
26017a3b050SJes Sorensen {
26117a3b050SJes Sorensen int ret;
26217a3b050SJes Sorensen
26317a3b050SJes Sorensen ret = misc_register(&cached_miscdev);
26417a3b050SJes Sorensen if (ret) {
26517a3b050SJes Sorensen printk(KERN_ERR "%s: failed to register device %i\n",
26617a3b050SJes Sorensen CACHED_ID, ret);
2670fef2532SChristoph Hellwig return ret;
26817a3b050SJes Sorensen }
26917a3b050SJes Sorensen ret = misc_register(&uncached_miscdev);
27017a3b050SJes Sorensen if (ret) {
27117a3b050SJes Sorensen printk(KERN_ERR "%s: failed to register device %i\n",
27217a3b050SJes Sorensen UNCACHED_ID, ret);
27317a3b050SJes Sorensen misc_deregister(&cached_miscdev);
2740fef2532SChristoph Hellwig return ret;
27517a3b050SJes Sorensen }
27617a3b050SJes Sorensen
2770fef2532SChristoph Hellwig printk(KERN_INFO "%s %s initialized devices: %s %s\n",
2780fef2532SChristoph Hellwig MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID);
27917a3b050SJes Sorensen
28017a3b050SJes Sorensen return 0;
28117a3b050SJes Sorensen }
28217a3b050SJes Sorensen
28317a3b050SJes Sorensen static void __exit
mspec_exit(void)28417a3b050SJes Sorensen mspec_exit(void)
28517a3b050SJes Sorensen {
28617a3b050SJes Sorensen misc_deregister(&uncached_miscdev);
28717a3b050SJes Sorensen misc_deregister(&cached_miscdev);
28817a3b050SJes Sorensen }
28917a3b050SJes Sorensen
29017a3b050SJes Sorensen module_init(mspec_init);
29117a3b050SJes Sorensen module_exit(mspec_exit);
29217a3b050SJes Sorensen
29317a3b050SJes Sorensen MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
29417a3b050SJes Sorensen MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
29517a3b050SJes Sorensen MODULE_LICENSE("GPL");
296