1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Resource Director Technology (RDT)
4  *
5  * Pseudo-locking support built on top of Cache Allocation Technology (CAT)
6  *
7  * Copyright (C) 2018 Intel Corporation
8  *
9  * Author: Reinette Chatre <reinette.chatre@intel.com>
10  */
11 
12 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
13 
14 #include <linux/cacheinfo.h>
15 #include <linux/cpu.h>
16 #include <linux/cpumask.h>
17 #include <linux/debugfs.h>
18 #include <linux/kthread.h>
19 #include <linux/mman.h>
20 #include <linux/perf_event.h>
21 #include <linux/pm_qos.h>
22 #include <linux/slab.h>
23 #include <linux/uaccess.h>
24 
25 #include <asm/cacheflush.h>
26 #include <asm/intel-family.h>
27 #include <asm/resctrl.h>
28 #include <asm/perf_event.h>
29 
30 #include "../../events/perf_event.h" /* For X86_CONFIG() */
31 #include "internal.h"
32 
33 #define CREATE_TRACE_POINTS
34 #include "pseudo_lock_event.h"
35 
36 /*
37  * The bits needed to disable hardware prefetching varies based on the
38  * platform. During initialization we will discover which bits to use.
39  */
40 static u64 prefetch_disable_bits;
41 
42 /*
43  * Major number assigned to and shared by all devices exposing
44  * pseudo-locked regions.
45  */
46 static unsigned int pseudo_lock_major;
47 static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0);
48 static struct class *pseudo_lock_class;
49 
50 /**
51  * get_prefetch_disable_bits - prefetch disable bits of supported platforms
52  * @void: It takes no parameters.
53  *
54  * Capture the list of platforms that have been validated to support
55  * pseudo-locking. This includes testing to ensure pseudo-locked regions
56  * with low cache miss rates can be created under variety of load conditions
57  * as well as that these pseudo-locked regions can maintain their low cache
58  * miss rates under variety of load conditions for significant lengths of time.
59  *
60  * After a platform has been validated to support pseudo-locking its
61  * hardware prefetch disable bits are included here as they are documented
62  * in the SDM.
63  *
64  * When adding a platform here also add support for its cache events to
65  * measure_cycles_perf_fn()
66  *
67  * Return:
68  * If platform is supported, the bits to disable hardware prefetchers, 0
69  * if platform is not supported.
70  */
71 static u64 get_prefetch_disable_bits(void)
72 {
73 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
74 	    boot_cpu_data.x86 != 6)
75 		return 0;
76 
77 	switch (boot_cpu_data.x86_model) {
78 	case INTEL_FAM6_BROADWELL_X:
79 		/*
80 		 * SDM defines bits of MSR_MISC_FEATURE_CONTROL register
81 		 * as:
82 		 * 0    L2 Hardware Prefetcher Disable (R/W)
83 		 * 1    L2 Adjacent Cache Line Prefetcher Disable (R/W)
84 		 * 2    DCU Hardware Prefetcher Disable (R/W)
85 		 * 3    DCU IP Prefetcher Disable (R/W)
86 		 * 63:4 Reserved
87 		 */
88 		return 0xF;
89 	case INTEL_FAM6_ATOM_GOLDMONT:
90 	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
91 		/*
92 		 * SDM defines bits of MSR_MISC_FEATURE_CONTROL register
93 		 * as:
94 		 * 0     L2 Hardware Prefetcher Disable (R/W)
95 		 * 1     Reserved
96 		 * 2     DCU Hardware Prefetcher Disable (R/W)
97 		 * 63:3  Reserved
98 		 */
99 		return 0x5;
100 	}
101 
102 	return 0;
103 }
104 
105 /**
106  * pseudo_lock_minor_get - Obtain available minor number
107  * @minor: Pointer to where new minor number will be stored
108  *
109  * A bitmask is used to track available minor numbers. Here the next free
110  * minor number is marked as unavailable and returned.
111  *
112  * Return: 0 on success, <0 on failure.
113  */
114 static int pseudo_lock_minor_get(unsigned int *minor)
115 {
116 	unsigned long first_bit;
117 
118 	first_bit = find_first_bit(&pseudo_lock_minor_avail, MINORBITS);
119 
120 	if (first_bit == MINORBITS)
121 		return -ENOSPC;
122 
123 	__clear_bit(first_bit, &pseudo_lock_minor_avail);
124 	*minor = first_bit;
125 
126 	return 0;
127 }
128 
129 /**
130  * pseudo_lock_minor_release - Return minor number to available
131  * @minor: The minor number made available
132  */
133 static void pseudo_lock_minor_release(unsigned int minor)
134 {
135 	__set_bit(minor, &pseudo_lock_minor_avail);
136 }
137 
138 /**
139  * region_find_by_minor - Locate a pseudo-lock region by inode minor number
140  * @minor: The minor number of the device representing pseudo-locked region
141  *
142  * When the character device is accessed we need to determine which
143  * pseudo-locked region it belongs to. This is done by matching the minor
144  * number of the device to the pseudo-locked region it belongs.
145  *
146  * Minor numbers are assigned at the time a pseudo-locked region is associated
147  * with a cache instance.
148  *
149  * Return: On success return pointer to resource group owning the pseudo-locked
150  *         region, NULL on failure.
151  */
152 static struct rdtgroup *region_find_by_minor(unsigned int minor)
153 {
154 	struct rdtgroup *rdtgrp, *rdtgrp_match = NULL;
155 
156 	list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) {
157 		if (rdtgrp->plr && rdtgrp->plr->minor == minor) {
158 			rdtgrp_match = rdtgrp;
159 			break;
160 		}
161 	}
162 	return rdtgrp_match;
163 }
164 
165 /**
166  * struct pseudo_lock_pm_req - A power management QoS request list entry
167  * @list:	Entry within the @pm_reqs list for a pseudo-locked region
168  * @req:	PM QoS request
169  */
170 struct pseudo_lock_pm_req {
171 	struct list_head list;
172 	struct dev_pm_qos_request req;
173 };
174 
175 static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr)
176 {
177 	struct pseudo_lock_pm_req *pm_req, *next;
178 
179 	list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) {
180 		dev_pm_qos_remove_request(&pm_req->req);
181 		list_del(&pm_req->list);
182 		kfree(pm_req);
183 	}
184 }
185 
186 /**
187  * pseudo_lock_cstates_constrain - Restrict cores from entering C6
188  * @plr: Pseudo-locked region
189  *
190  * To prevent the cache from being affected by power management entering
191  * C6 has to be avoided. This is accomplished by requesting a latency
192  * requirement lower than lowest C6 exit latency of all supported
193  * platforms as found in the cpuidle state tables in the intel_idle driver.
194  * At this time it is possible to do so with a single latency requirement
195  * for all supported platforms.
196  *
197  * Since Goldmont is supported, which is affected by X86_BUG_MONITOR,
198  * the ACPI latencies need to be considered while keeping in mind that C2
199  * may be set to map to deeper sleep states. In this case the latency
200  * requirement needs to prevent entering C2 also.
201  *
202  * Return: 0 on success, <0 on failure
203  */
204 static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr)
205 {
206 	struct pseudo_lock_pm_req *pm_req;
207 	int cpu;
208 	int ret;
209 
210 	for_each_cpu(cpu, &plr->d->cpu_mask) {
211 		pm_req = kzalloc(sizeof(*pm_req), GFP_KERNEL);
212 		if (!pm_req) {
213 			rdt_last_cmd_puts("Failure to allocate memory for PM QoS\n");
214 			ret = -ENOMEM;
215 			goto out_err;
216 		}
217 		ret = dev_pm_qos_add_request(get_cpu_device(cpu),
218 					     &pm_req->req,
219 					     DEV_PM_QOS_RESUME_LATENCY,
220 					     30);
221 		if (ret < 0) {
222 			rdt_last_cmd_printf("Failed to add latency req CPU%d\n",
223 					    cpu);
224 			kfree(pm_req);
225 			ret = -1;
226 			goto out_err;
227 		}
228 		list_add(&pm_req->list, &plr->pm_reqs);
229 	}
230 
231 	return 0;
232 
233 out_err:
234 	pseudo_lock_cstates_relax(plr);
235 	return ret;
236 }
237 
238 /**
239  * pseudo_lock_region_clear - Reset pseudo-lock region data
240  * @plr: pseudo-lock region
241  *
242  * All content of the pseudo-locked region is reset - any memory allocated
243  * freed.
244  *
245  * Return: void
246  */
247 static void pseudo_lock_region_clear(struct pseudo_lock_region *plr)
248 {
249 	plr->size = 0;
250 	plr->line_size = 0;
251 	kfree(plr->kmem);
252 	plr->kmem = NULL;
253 	plr->s = NULL;
254 	if (plr->d)
255 		plr->d->plr = NULL;
256 	plr->d = NULL;
257 	plr->cbm = 0;
258 	plr->debugfs_dir = NULL;
259 }
260 
261 /**
262  * pseudo_lock_region_init - Initialize pseudo-lock region information
263  * @plr: pseudo-lock region
264  *
265  * Called after user provided a schemata to be pseudo-locked. From the
266  * schemata the &struct pseudo_lock_region is on entry already initialized
267  * with the resource, domain, and capacity bitmask. Here the information
268  * required for pseudo-locking is deduced from this data and &struct
269  * pseudo_lock_region initialized further. This information includes:
270  * - size in bytes of the region to be pseudo-locked
271  * - cache line size to know the stride with which data needs to be accessed
272  *   to be pseudo-locked
273  * - a cpu associated with the cache instance on which the pseudo-locking
274  *   flow can be executed
275  *
276  * Return: 0 on success, <0 on failure. Descriptive error will be written
277  * to last_cmd_status buffer.
278  */
279 static int pseudo_lock_region_init(struct pseudo_lock_region *plr)
280 {
281 	struct cpu_cacheinfo *ci;
282 	int ret;
283 	int i;
284 
285 	/* Pick the first cpu we find that is associated with the cache. */
286 	plr->cpu = cpumask_first(&plr->d->cpu_mask);
287 
288 	if (!cpu_online(plr->cpu)) {
289 		rdt_last_cmd_printf("CPU %u associated with cache not online\n",
290 				    plr->cpu);
291 		ret = -ENODEV;
292 		goto out_region;
293 	}
294 
295 	ci = get_cpu_cacheinfo(plr->cpu);
296 
297 	plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm);
298 
299 	for (i = 0; i < ci->num_leaves; i++) {
300 		if (ci->info_list[i].level == plr->s->res->cache_level) {
301 			plr->line_size = ci->info_list[i].coherency_line_size;
302 			return 0;
303 		}
304 	}
305 
306 	ret = -1;
307 	rdt_last_cmd_puts("Unable to determine cache line size\n");
308 out_region:
309 	pseudo_lock_region_clear(plr);
310 	return ret;
311 }
312 
313 /**
314  * pseudo_lock_init - Initialize a pseudo-lock region
315  * @rdtgrp: resource group to which new pseudo-locked region will belong
316  *
317  * A pseudo-locked region is associated with a resource group. When this
318  * association is created the pseudo-locked region is initialized. The
319  * details of the pseudo-locked region are not known at this time so only
320  * allocation is done and association established.
321  *
322  * Return: 0 on success, <0 on failure
323  */
324 static int pseudo_lock_init(struct rdtgroup *rdtgrp)
325 {
326 	struct pseudo_lock_region *plr;
327 
328 	plr = kzalloc(sizeof(*plr), GFP_KERNEL);
329 	if (!plr)
330 		return -ENOMEM;
331 
332 	init_waitqueue_head(&plr->lock_thread_wq);
333 	INIT_LIST_HEAD(&plr->pm_reqs);
334 	rdtgrp->plr = plr;
335 	return 0;
336 }
337 
338 /**
339  * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked
340  * @plr: pseudo-lock region
341  *
342  * Initialize the details required to set up the pseudo-locked region and
343  * allocate the contiguous memory that will be pseudo-locked to the cache.
344  *
345  * Return: 0 on success, <0 on failure.  Descriptive error will be written
346  * to last_cmd_status buffer.
347  */
348 static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr)
349 {
350 	int ret;
351 
352 	ret = pseudo_lock_region_init(plr);
353 	if (ret < 0)
354 		return ret;
355 
356 	/*
357 	 * We do not yet support contiguous regions larger than
358 	 * KMALLOC_MAX_SIZE.
359 	 */
360 	if (plr->size > KMALLOC_MAX_SIZE) {
361 		rdt_last_cmd_puts("Requested region exceeds maximum size\n");
362 		ret = -E2BIG;
363 		goto out_region;
364 	}
365 
366 	plr->kmem = kzalloc(plr->size, GFP_KERNEL);
367 	if (!plr->kmem) {
368 		rdt_last_cmd_puts("Unable to allocate memory\n");
369 		ret = -ENOMEM;
370 		goto out_region;
371 	}
372 
373 	ret = 0;
374 	goto out;
375 out_region:
376 	pseudo_lock_region_clear(plr);
377 out:
378 	return ret;
379 }
380 
381 /**
382  * pseudo_lock_free - Free a pseudo-locked region
383  * @rdtgrp: resource group to which pseudo-locked region belonged
384  *
385  * The pseudo-locked region's resources have already been released, or not
386  * yet created at this point. Now it can be freed and disassociated from the
387  * resource group.
388  *
389  * Return: void
390  */
391 static void pseudo_lock_free(struct rdtgroup *rdtgrp)
392 {
393 	pseudo_lock_region_clear(rdtgrp->plr);
394 	kfree(rdtgrp->plr);
395 	rdtgrp->plr = NULL;
396 }
397 
398 /**
399  * pseudo_lock_fn - Load kernel memory into cache
400  * @_rdtgrp: resource group to which pseudo-lock region belongs
401  *
402  * This is the core pseudo-locking flow.
403  *
404  * First we ensure that the kernel memory cannot be found in the cache.
405  * Then, while taking care that there will be as little interference as
406  * possible, the memory to be loaded is accessed while core is running
407  * with class of service set to the bitmask of the pseudo-locked region.
408  * After this is complete no future CAT allocations will be allowed to
409  * overlap with this bitmask.
410  *
411  * Local register variables are utilized to ensure that the memory region
412  * to be locked is the only memory access made during the critical locking
413  * loop.
414  *
415  * Return: 0. Waiter on waitqueue will be woken on completion.
416  */
417 static int pseudo_lock_fn(void *_rdtgrp)
418 {
419 	struct rdtgroup *rdtgrp = _rdtgrp;
420 	struct pseudo_lock_region *plr = rdtgrp->plr;
421 	u32 rmid_p, closid_p;
422 	unsigned long i;
423 	u64 saved_msr;
424 #ifdef CONFIG_KASAN
425 	/*
426 	 * The registers used for local register variables are also used
427 	 * when KASAN is active. When KASAN is active we use a regular
428 	 * variable to ensure we always use a valid pointer, but the cost
429 	 * is that this variable will enter the cache through evicting the
430 	 * memory we are trying to lock into the cache. Thus expect lower
431 	 * pseudo-locking success rate when KASAN is active.
432 	 */
433 	unsigned int line_size;
434 	unsigned int size;
435 	void *mem_r;
436 #else
437 	register unsigned int line_size asm("esi");
438 	register unsigned int size asm("edi");
439 	register void *mem_r asm(_ASM_BX);
440 #endif /* CONFIG_KASAN */
441 
442 	/*
443 	 * Make sure none of the allocated memory is cached. If it is we
444 	 * will get a cache hit in below loop from outside of pseudo-locked
445 	 * region.
446 	 * wbinvd (as opposed to clflush/clflushopt) is required to
447 	 * increase likelihood that allocated cache portion will be filled
448 	 * with associated memory.
449 	 */
450 	native_wbinvd();
451 
452 	/*
453 	 * Always called with interrupts enabled. By disabling interrupts
454 	 * ensure that we will not be preempted during this critical section.
455 	 */
456 	local_irq_disable();
457 
458 	/*
459 	 * Call wrmsr and rdmsr as directly as possible to avoid tracing
460 	 * clobbering local register variables or affecting cache accesses.
461 	 *
462 	 * Disable the hardware prefetcher so that when the end of the memory
463 	 * being pseudo-locked is reached the hardware will not read beyond
464 	 * the buffer and evict pseudo-locked memory read earlier from the
465 	 * cache.
466 	 */
467 	saved_msr = __rdmsr(MSR_MISC_FEATURE_CONTROL);
468 	__wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
469 	closid_p = this_cpu_read(pqr_state.cur_closid);
470 	rmid_p = this_cpu_read(pqr_state.cur_rmid);
471 	mem_r = plr->kmem;
472 	size = plr->size;
473 	line_size = plr->line_size;
474 	/*
475 	 * Critical section begin: start by writing the closid associated
476 	 * with the capacity bitmask of the cache region being
477 	 * pseudo-locked followed by reading of kernel memory to load it
478 	 * into the cache.
479 	 */
480 	__wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, rdtgrp->closid);
481 	/*
482 	 * Cache was flushed earlier. Now access kernel memory to read it
483 	 * into cache region associated with just activated plr->closid.
484 	 * Loop over data twice:
485 	 * - In first loop the cache region is shared with the page walker
486 	 *   as it populates the paging structure caches (including TLB).
487 	 * - In the second loop the paging structure caches are used and
488 	 *   cache region is populated with the memory being referenced.
489 	 */
490 	for (i = 0; i < size; i += PAGE_SIZE) {
491 		/*
492 		 * Add a barrier to prevent speculative execution of this
493 		 * loop reading beyond the end of the buffer.
494 		 */
495 		rmb();
496 		asm volatile("mov (%0,%1,1), %%eax\n\t"
497 			:
498 			: "r" (mem_r), "r" (i)
499 			: "%eax", "memory");
500 	}
501 	for (i = 0; i < size; i += line_size) {
502 		/*
503 		 * Add a barrier to prevent speculative execution of this
504 		 * loop reading beyond the end of the buffer.
505 		 */
506 		rmb();
507 		asm volatile("mov (%0,%1,1), %%eax\n\t"
508 			:
509 			: "r" (mem_r), "r" (i)
510 			: "%eax", "memory");
511 	}
512 	/*
513 	 * Critical section end: restore closid with capacity bitmask that
514 	 * does not overlap with pseudo-locked region.
515 	 */
516 	__wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, closid_p);
517 
518 	/* Re-enable the hardware prefetcher(s) */
519 	wrmsrl(MSR_MISC_FEATURE_CONTROL, saved_msr);
520 	local_irq_enable();
521 
522 	plr->thread_done = 1;
523 	wake_up_interruptible(&plr->lock_thread_wq);
524 	return 0;
525 }
526 
527 /**
528  * rdtgroup_monitor_in_progress - Test if monitoring in progress
529  * @rdtgrp: resource group being queried
530  *
531  * Return: 1 if monitor groups have been created for this resource
532  * group, 0 otherwise.
533  */
534 static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp)
535 {
536 	return !list_empty(&rdtgrp->mon.crdtgrp_list);
537 }
538 
539 /**
540  * rdtgroup_locksetup_user_restrict - Restrict user access to group
541  * @rdtgrp: resource group needing access restricted
542  *
543  * A resource group used for cache pseudo-locking cannot have cpus or tasks
544  * assigned to it. This is communicated to the user by restricting access
545  * to all the files that can be used to make such changes.
546  *
547  * Permissions restored with rdtgroup_locksetup_user_restore()
548  *
549  * Return: 0 on success, <0 on failure. If a failure occurs during the
550  * restriction of access an attempt will be made to restore permissions but
551  * the state of the mode of these files will be uncertain when a failure
552  * occurs.
553  */
554 static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp)
555 {
556 	int ret;
557 
558 	ret = rdtgroup_kn_mode_restrict(rdtgrp, "tasks");
559 	if (ret)
560 		return ret;
561 
562 	ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus");
563 	if (ret)
564 		goto err_tasks;
565 
566 	ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list");
567 	if (ret)
568 		goto err_cpus;
569 
570 	if (rdt_mon_capable) {
571 		ret = rdtgroup_kn_mode_restrict(rdtgrp, "mon_groups");
572 		if (ret)
573 			goto err_cpus_list;
574 	}
575 
576 	ret = 0;
577 	goto out;
578 
579 err_cpus_list:
580 	rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777);
581 err_cpus:
582 	rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777);
583 err_tasks:
584 	rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777);
585 out:
586 	return ret;
587 }
588 
589 /**
590  * rdtgroup_locksetup_user_restore - Restore user access to group
591  * @rdtgrp: resource group needing access restored
592  *
593  * Restore all file access previously removed using
594  * rdtgroup_locksetup_user_restrict()
595  *
596  * Return: 0 on success, <0 on failure.  If a failure occurs during the
597  * restoration of access an attempt will be made to restrict permissions
598  * again but the state of the mode of these files will be uncertain when
599  * a failure occurs.
600  */
601 static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp)
602 {
603 	int ret;
604 
605 	ret = rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777);
606 	if (ret)
607 		return ret;
608 
609 	ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777);
610 	if (ret)
611 		goto err_tasks;
612 
613 	ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777);
614 	if (ret)
615 		goto err_cpus;
616 
617 	if (rdt_mon_capable) {
618 		ret = rdtgroup_kn_mode_restore(rdtgrp, "mon_groups", 0777);
619 		if (ret)
620 			goto err_cpus_list;
621 	}
622 
623 	ret = 0;
624 	goto out;
625 
626 err_cpus_list:
627 	rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list");
628 err_cpus:
629 	rdtgroup_kn_mode_restrict(rdtgrp, "cpus");
630 err_tasks:
631 	rdtgroup_kn_mode_restrict(rdtgrp, "tasks");
632 out:
633 	return ret;
634 }
635 
636 /**
637  * rdtgroup_locksetup_enter - Resource group enters locksetup mode
638  * @rdtgrp: resource group requested to enter locksetup mode
639  *
640  * A resource group enters locksetup mode to reflect that it would be used
641  * to represent a pseudo-locked region and is in the process of being set
642  * up to do so. A resource group used for a pseudo-locked region would
643  * lose the closid associated with it so we cannot allow it to have any
644  * tasks or cpus assigned nor permit tasks or cpus to be assigned in the
645  * future. Monitoring of a pseudo-locked region is not allowed either.
646  *
647  * The above and more restrictions on a pseudo-locked region are checked
648  * for and enforced before the resource group enters the locksetup mode.
649  *
650  * Returns: 0 if the resource group successfully entered locksetup mode, <0
651  * on failure. On failure the last_cmd_status buffer is updated with text to
652  * communicate details of failure to the user.
653  */
654 int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp)
655 {
656 	int ret;
657 
658 	/*
659 	 * The default resource group can neither be removed nor lose the
660 	 * default closid associated with it.
661 	 */
662 	if (rdtgrp == &rdtgroup_default) {
663 		rdt_last_cmd_puts("Cannot pseudo-lock default group\n");
664 		return -EINVAL;
665 	}
666 
667 	/*
668 	 * Cache Pseudo-locking not supported when CDP is enabled.
669 	 *
670 	 * Some things to consider if you would like to enable this
671 	 * support (using L3 CDP as example):
672 	 * - When CDP is enabled two separate resources are exposed,
673 	 *   L3DATA and L3CODE, but they are actually on the same cache.
674 	 *   The implication for pseudo-locking is that if a
675 	 *   pseudo-locked region is created on a domain of one
676 	 *   resource (eg. L3CODE), then a pseudo-locked region cannot
677 	 *   be created on that same domain of the other resource
678 	 *   (eg. L3DATA). This is because the creation of a
679 	 *   pseudo-locked region involves a call to wbinvd that will
680 	 *   affect all cache allocations on particular domain.
681 	 * - Considering the previous, it may be possible to only
682 	 *   expose one of the CDP resources to pseudo-locking and
683 	 *   hide the other. For example, we could consider to only
684 	 *   expose L3DATA and since the L3 cache is unified it is
685 	 *   still possible to place instructions there are execute it.
686 	 * - If only one region is exposed to pseudo-locking we should
687 	 *   still keep in mind that availability of a portion of cache
688 	 *   for pseudo-locking should take into account both resources.
689 	 *   Similarly, if a pseudo-locked region is created in one
690 	 *   resource, the portion of cache used by it should be made
691 	 *   unavailable to all future allocations from both resources.
692 	 */
693 	if (resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L3) ||
694 	    resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L2)) {
695 		rdt_last_cmd_puts("CDP enabled\n");
696 		return -EINVAL;
697 	}
698 
699 	/*
700 	 * Not knowing the bits to disable prefetching implies that this
701 	 * platform does not support Cache Pseudo-Locking.
702 	 */
703 	prefetch_disable_bits = get_prefetch_disable_bits();
704 	if (prefetch_disable_bits == 0) {
705 		rdt_last_cmd_puts("Pseudo-locking not supported\n");
706 		return -EINVAL;
707 	}
708 
709 	if (rdtgroup_monitor_in_progress(rdtgrp)) {
710 		rdt_last_cmd_puts("Monitoring in progress\n");
711 		return -EINVAL;
712 	}
713 
714 	if (rdtgroup_tasks_assigned(rdtgrp)) {
715 		rdt_last_cmd_puts("Tasks assigned to resource group\n");
716 		return -EINVAL;
717 	}
718 
719 	if (!cpumask_empty(&rdtgrp->cpu_mask)) {
720 		rdt_last_cmd_puts("CPUs assigned to resource group\n");
721 		return -EINVAL;
722 	}
723 
724 	if (rdtgroup_locksetup_user_restrict(rdtgrp)) {
725 		rdt_last_cmd_puts("Unable to modify resctrl permissions\n");
726 		return -EIO;
727 	}
728 
729 	ret = pseudo_lock_init(rdtgrp);
730 	if (ret) {
731 		rdt_last_cmd_puts("Unable to init pseudo-lock region\n");
732 		goto out_release;
733 	}
734 
735 	/*
736 	 * If this system is capable of monitoring a rmid would have been
737 	 * allocated when the control group was created. This is not needed
738 	 * anymore when this group would be used for pseudo-locking. This
739 	 * is safe to call on platforms not capable of monitoring.
740 	 */
741 	free_rmid(rdtgrp->mon.rmid);
742 
743 	ret = 0;
744 	goto out;
745 
746 out_release:
747 	rdtgroup_locksetup_user_restore(rdtgrp);
748 out:
749 	return ret;
750 }
751 
752 /**
753  * rdtgroup_locksetup_exit - resource group exist locksetup mode
754  * @rdtgrp: resource group
755  *
756  * When a resource group exits locksetup mode the earlier restrictions are
757  * lifted.
758  *
759  * Return: 0 on success, <0 on failure
760  */
761 int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp)
762 {
763 	int ret;
764 
765 	if (rdt_mon_capable) {
766 		ret = alloc_rmid();
767 		if (ret < 0) {
768 			rdt_last_cmd_puts("Out of RMIDs\n");
769 			return ret;
770 		}
771 		rdtgrp->mon.rmid = ret;
772 	}
773 
774 	ret = rdtgroup_locksetup_user_restore(rdtgrp);
775 	if (ret) {
776 		free_rmid(rdtgrp->mon.rmid);
777 		return ret;
778 	}
779 
780 	pseudo_lock_free(rdtgrp);
781 	return 0;
782 }
783 
784 /**
785  * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked
786  * @d: RDT domain
787  * @cbm: CBM to test
788  *
789  * @d represents a cache instance and @cbm a capacity bitmask that is
790  * considered for it. Determine if @cbm overlaps with any existing
791  * pseudo-locked region on @d.
792  *
793  * @cbm is unsigned long, even if only 32 bits are used, to make the
794  * bitmap functions work correctly.
795  *
796  * Return: true if @cbm overlaps with pseudo-locked region on @d, false
797  * otherwise.
798  */
799 bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm)
800 {
801 	unsigned int cbm_len;
802 	unsigned long cbm_b;
803 
804 	if (d->plr) {
805 		cbm_len = d->plr->s->res->cache.cbm_len;
806 		cbm_b = d->plr->cbm;
807 		if (bitmap_intersects(&cbm, &cbm_b, cbm_len))
808 			return true;
809 	}
810 	return false;
811 }
812 
813 /**
814  * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy
815  * @d: RDT domain under test
816  *
817  * The setup of a pseudo-locked region affects all cache instances within
818  * the hierarchy of the region. It is thus essential to know if any
819  * pseudo-locked regions exist within a cache hierarchy to prevent any
820  * attempts to create new pseudo-locked regions in the same hierarchy.
821  *
822  * Return: true if a pseudo-locked region exists in the hierarchy of @d or
823  *         if it is not possible to test due to memory allocation issue,
824  *         false otherwise.
825  */
826 bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d)
827 {
828 	cpumask_var_t cpu_with_psl;
829 	struct rdt_resource *r;
830 	struct rdt_domain *d_i;
831 	bool ret = false;
832 
833 	if (!zalloc_cpumask_var(&cpu_with_psl, GFP_KERNEL))
834 		return true;
835 
836 	/*
837 	 * First determine which cpus have pseudo-locked regions
838 	 * associated with them.
839 	 */
840 	for_each_alloc_capable_rdt_resource(r) {
841 		list_for_each_entry(d_i, &r->domains, list) {
842 			if (d_i->plr)
843 				cpumask_or(cpu_with_psl, cpu_with_psl,
844 					   &d_i->cpu_mask);
845 		}
846 	}
847 
848 	/*
849 	 * Next test if new pseudo-locked region would intersect with
850 	 * existing region.
851 	 */
852 	if (cpumask_intersects(&d->cpu_mask, cpu_with_psl))
853 		ret = true;
854 
855 	free_cpumask_var(cpu_with_psl);
856 	return ret;
857 }
858 
859 /**
860  * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory
861  * @_plr: pseudo-lock region to measure
862  *
863  * There is no deterministic way to test if a memory region is cached. One
864  * way is to measure how long it takes to read the memory, the speed of
865  * access is a good way to learn how close to the cpu the data was. Even
866  * more, if the prefetcher is disabled and the memory is read at a stride
867  * of half the cache line, then a cache miss will be easy to spot since the
868  * read of the first half would be significantly slower than the read of
869  * the second half.
870  *
871  * Return: 0. Waiter on waitqueue will be woken on completion.
872  */
873 static int measure_cycles_lat_fn(void *_plr)
874 {
875 	struct pseudo_lock_region *plr = _plr;
876 	u32 saved_low, saved_high;
877 	unsigned long i;
878 	u64 start, end;
879 	void *mem_r;
880 
881 	local_irq_disable();
882 	/*
883 	 * Disable hardware prefetchers.
884 	 */
885 	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
886 	wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
887 	mem_r = READ_ONCE(plr->kmem);
888 	/*
889 	 * Dummy execute of the time measurement to load the needed
890 	 * instructions into the L1 instruction cache.
891 	 */
892 	start = rdtsc_ordered();
893 	for (i = 0; i < plr->size; i += 32) {
894 		start = rdtsc_ordered();
895 		asm volatile("mov (%0,%1,1), %%eax\n\t"
896 			     :
897 			     : "r" (mem_r), "r" (i)
898 			     : "%eax", "memory");
899 		end = rdtsc_ordered();
900 		trace_pseudo_lock_mem_latency((u32)(end - start));
901 	}
902 	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
903 	local_irq_enable();
904 	plr->thread_done = 1;
905 	wake_up_interruptible(&plr->lock_thread_wq);
906 	return 0;
907 }
908 
909 /*
910  * Create a perf_event_attr for the hit and miss perf events that will
911  * be used during the performance measurement. A perf_event maintains
912  * a pointer to its perf_event_attr so a unique attribute structure is
913  * created for each perf_event.
914  *
915  * The actual configuration of the event is set right before use in order
916  * to use the X86_CONFIG macro.
917  */
918 static struct perf_event_attr perf_miss_attr = {
919 	.type		= PERF_TYPE_RAW,
920 	.size		= sizeof(struct perf_event_attr),
921 	.pinned		= 1,
922 	.disabled	= 0,
923 	.exclude_user	= 1,
924 };
925 
926 static struct perf_event_attr perf_hit_attr = {
927 	.type		= PERF_TYPE_RAW,
928 	.size		= sizeof(struct perf_event_attr),
929 	.pinned		= 1,
930 	.disabled	= 0,
931 	.exclude_user	= 1,
932 };
933 
934 struct residency_counts {
935 	u64 miss_before, hits_before;
936 	u64 miss_after,  hits_after;
937 };
938 
939 static int measure_residency_fn(struct perf_event_attr *miss_attr,
940 				struct perf_event_attr *hit_attr,
941 				struct pseudo_lock_region *plr,
942 				struct residency_counts *counts)
943 {
944 	u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0;
945 	struct perf_event *miss_event, *hit_event;
946 	int hit_pmcnum, miss_pmcnum;
947 	u32 saved_low, saved_high;
948 	unsigned int line_size;
949 	unsigned int size;
950 	unsigned long i;
951 	void *mem_r;
952 	u64 tmp;
953 
954 	miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu,
955 						      NULL, NULL, NULL);
956 	if (IS_ERR(miss_event))
957 		goto out;
958 
959 	hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu,
960 						     NULL, NULL, NULL);
961 	if (IS_ERR(hit_event))
962 		goto out_miss;
963 
964 	local_irq_disable();
965 	/*
966 	 * Check any possible error state of events used by performing
967 	 * one local read.
968 	 */
969 	if (perf_event_read_local(miss_event, &tmp, NULL, NULL)) {
970 		local_irq_enable();
971 		goto out_hit;
972 	}
973 	if (perf_event_read_local(hit_event, &tmp, NULL, NULL)) {
974 		local_irq_enable();
975 		goto out_hit;
976 	}
977 
978 	/*
979 	 * Disable hardware prefetchers.
980 	 */
981 	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
982 	wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
983 
984 	/* Initialize rest of local variables */
985 	/*
986 	 * Performance event has been validated right before this with
987 	 * interrupts disabled - it is thus safe to read the counter index.
988 	 */
989 	miss_pmcnum = x86_perf_rdpmc_index(miss_event);
990 	hit_pmcnum = x86_perf_rdpmc_index(hit_event);
991 	line_size = READ_ONCE(plr->line_size);
992 	mem_r = READ_ONCE(plr->kmem);
993 	size = READ_ONCE(plr->size);
994 
995 	/*
996 	 * Read counter variables twice - first to load the instructions
997 	 * used in L1 cache, second to capture accurate value that does not
998 	 * include cache misses incurred because of instruction loads.
999 	 */
1000 	rdpmcl(hit_pmcnum, hits_before);
1001 	rdpmcl(miss_pmcnum, miss_before);
1002 	/*
1003 	 * From SDM: Performing back-to-back fast reads are not guaranteed
1004 	 * to be monotonic.
1005 	 * Use LFENCE to ensure all previous instructions are retired
1006 	 * before proceeding.
1007 	 */
1008 	rmb();
1009 	rdpmcl(hit_pmcnum, hits_before);
1010 	rdpmcl(miss_pmcnum, miss_before);
1011 	/*
1012 	 * Use LFENCE to ensure all previous instructions are retired
1013 	 * before proceeding.
1014 	 */
1015 	rmb();
1016 	for (i = 0; i < size; i += line_size) {
1017 		/*
1018 		 * Add a barrier to prevent speculative execution of this
1019 		 * loop reading beyond the end of the buffer.
1020 		 */
1021 		rmb();
1022 		asm volatile("mov (%0,%1,1), %%eax\n\t"
1023 			     :
1024 			     : "r" (mem_r), "r" (i)
1025 			     : "%eax", "memory");
1026 	}
1027 	/*
1028 	 * Use LFENCE to ensure all previous instructions are retired
1029 	 * before proceeding.
1030 	 */
1031 	rmb();
1032 	rdpmcl(hit_pmcnum, hits_after);
1033 	rdpmcl(miss_pmcnum, miss_after);
1034 	/*
1035 	 * Use LFENCE to ensure all previous instructions are retired
1036 	 * before proceeding.
1037 	 */
1038 	rmb();
1039 	/* Re-enable hardware prefetchers */
1040 	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
1041 	local_irq_enable();
1042 out_hit:
1043 	perf_event_release_kernel(hit_event);
1044 out_miss:
1045 	perf_event_release_kernel(miss_event);
1046 out:
1047 	/*
1048 	 * All counts will be zero on failure.
1049 	 */
1050 	counts->miss_before = miss_before;
1051 	counts->hits_before = hits_before;
1052 	counts->miss_after  = miss_after;
1053 	counts->hits_after  = hits_after;
1054 	return 0;
1055 }
1056 
1057 static int measure_l2_residency(void *_plr)
1058 {
1059 	struct pseudo_lock_region *plr = _plr;
1060 	struct residency_counts counts = {0};
1061 
1062 	/*
1063 	 * Non-architectural event for the Goldmont Microarchitecture
1064 	 * from Intel x86 Architecture Software Developer Manual (SDM):
1065 	 * MEM_LOAD_UOPS_RETIRED D1H (event number)
1066 	 * Umask values:
1067 	 *     L2_HIT   02H
1068 	 *     L2_MISS  10H
1069 	 */
1070 	switch (boot_cpu_data.x86_model) {
1071 	case INTEL_FAM6_ATOM_GOLDMONT:
1072 	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1073 		perf_miss_attr.config = X86_CONFIG(.event = 0xd1,
1074 						   .umask = 0x10);
1075 		perf_hit_attr.config = X86_CONFIG(.event = 0xd1,
1076 						  .umask = 0x2);
1077 		break;
1078 	default:
1079 		goto out;
1080 	}
1081 
1082 	measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts);
1083 	/*
1084 	 * If a failure prevented the measurements from succeeding
1085 	 * tracepoints will still be written and all counts will be zero.
1086 	 */
1087 	trace_pseudo_lock_l2(counts.hits_after - counts.hits_before,
1088 			     counts.miss_after - counts.miss_before);
1089 out:
1090 	plr->thread_done = 1;
1091 	wake_up_interruptible(&plr->lock_thread_wq);
1092 	return 0;
1093 }
1094 
1095 static int measure_l3_residency(void *_plr)
1096 {
1097 	struct pseudo_lock_region *plr = _plr;
1098 	struct residency_counts counts = {0};
1099 
1100 	/*
1101 	 * On Broadwell Microarchitecture the MEM_LOAD_UOPS_RETIRED event
1102 	 * has two "no fix" errata associated with it: BDM35 and BDM100. On
1103 	 * this platform the following events are used instead:
1104 	 * LONGEST_LAT_CACHE 2EH (Documented in SDM)
1105 	 *       REFERENCE 4FH
1106 	 *       MISS      41H
1107 	 */
1108 
1109 	switch (boot_cpu_data.x86_model) {
1110 	case INTEL_FAM6_BROADWELL_X:
1111 		/* On BDW the hit event counts references, not hits */
1112 		perf_hit_attr.config = X86_CONFIG(.event = 0x2e,
1113 						  .umask = 0x4f);
1114 		perf_miss_attr.config = X86_CONFIG(.event = 0x2e,
1115 						   .umask = 0x41);
1116 		break;
1117 	default:
1118 		goto out;
1119 	}
1120 
1121 	measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts);
1122 	/*
1123 	 * If a failure prevented the measurements from succeeding
1124 	 * tracepoints will still be written and all counts will be zero.
1125 	 */
1126 
1127 	counts.miss_after -= counts.miss_before;
1128 	if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) {
1129 		/*
1130 		 * On BDW references and misses are counted, need to adjust.
1131 		 * Sometimes the "hits" counter is a bit more than the
1132 		 * references, for example, x references but x + 1 hits.
1133 		 * To not report invalid hit values in this case we treat
1134 		 * that as misses equal to references.
1135 		 */
1136 		/* First compute the number of cache references measured */
1137 		counts.hits_after -= counts.hits_before;
1138 		/* Next convert references to cache hits */
1139 		counts.hits_after -= min(counts.miss_after, counts.hits_after);
1140 	} else {
1141 		counts.hits_after -= counts.hits_before;
1142 	}
1143 
1144 	trace_pseudo_lock_l3(counts.hits_after, counts.miss_after);
1145 out:
1146 	plr->thread_done = 1;
1147 	wake_up_interruptible(&plr->lock_thread_wq);
1148 	return 0;
1149 }
1150 
1151 /**
1152  * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region
1153  * @rdtgrp: Resource group to which the pseudo-locked region belongs.
1154  * @sel: Selector of which measurement to perform on a pseudo-locked region.
1155  *
1156  * The measurement of latency to access a pseudo-locked region should be
1157  * done from a cpu that is associated with that pseudo-locked region.
1158  * Determine which cpu is associated with this region and start a thread on
1159  * that cpu to perform the measurement, wait for that thread to complete.
1160  *
1161  * Return: 0 on success, <0 on failure
1162  */
1163 static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel)
1164 {
1165 	struct pseudo_lock_region *plr = rdtgrp->plr;
1166 	struct task_struct *thread;
1167 	unsigned int cpu;
1168 	int ret = -1;
1169 
1170 	cpus_read_lock();
1171 	mutex_lock(&rdtgroup_mutex);
1172 
1173 	if (rdtgrp->flags & RDT_DELETED) {
1174 		ret = -ENODEV;
1175 		goto out;
1176 	}
1177 
1178 	if (!plr->d) {
1179 		ret = -ENODEV;
1180 		goto out;
1181 	}
1182 
1183 	plr->thread_done = 0;
1184 	cpu = cpumask_first(&plr->d->cpu_mask);
1185 	if (!cpu_online(cpu)) {
1186 		ret = -ENODEV;
1187 		goto out;
1188 	}
1189 
1190 	plr->cpu = cpu;
1191 
1192 	if (sel == 1)
1193 		thread = kthread_create_on_node(measure_cycles_lat_fn, plr,
1194 						cpu_to_node(cpu),
1195 						"pseudo_lock_measure/%u",
1196 						cpu);
1197 	else if (sel == 2)
1198 		thread = kthread_create_on_node(measure_l2_residency, plr,
1199 						cpu_to_node(cpu),
1200 						"pseudo_lock_measure/%u",
1201 						cpu);
1202 	else if (sel == 3)
1203 		thread = kthread_create_on_node(measure_l3_residency, plr,
1204 						cpu_to_node(cpu),
1205 						"pseudo_lock_measure/%u",
1206 						cpu);
1207 	else
1208 		goto out;
1209 
1210 	if (IS_ERR(thread)) {
1211 		ret = PTR_ERR(thread);
1212 		goto out;
1213 	}
1214 	kthread_bind(thread, cpu);
1215 	wake_up_process(thread);
1216 
1217 	ret = wait_event_interruptible(plr->lock_thread_wq,
1218 				       plr->thread_done == 1);
1219 	if (ret < 0)
1220 		goto out;
1221 
1222 	ret = 0;
1223 
1224 out:
1225 	mutex_unlock(&rdtgroup_mutex);
1226 	cpus_read_unlock();
1227 	return ret;
1228 }
1229 
1230 static ssize_t pseudo_lock_measure_trigger(struct file *file,
1231 					   const char __user *user_buf,
1232 					   size_t count, loff_t *ppos)
1233 {
1234 	struct rdtgroup *rdtgrp = file->private_data;
1235 	size_t buf_size;
1236 	char buf[32];
1237 	int ret;
1238 	int sel;
1239 
1240 	buf_size = min(count, (sizeof(buf) - 1));
1241 	if (copy_from_user(buf, user_buf, buf_size))
1242 		return -EFAULT;
1243 
1244 	buf[buf_size] = '\0';
1245 	ret = kstrtoint(buf, 10, &sel);
1246 	if (ret == 0) {
1247 		if (sel != 1 && sel != 2 && sel != 3)
1248 			return -EINVAL;
1249 		ret = debugfs_file_get(file->f_path.dentry);
1250 		if (ret)
1251 			return ret;
1252 		ret = pseudo_lock_measure_cycles(rdtgrp, sel);
1253 		if (ret == 0)
1254 			ret = count;
1255 		debugfs_file_put(file->f_path.dentry);
1256 	}
1257 
1258 	return ret;
1259 }
1260 
1261 static const struct file_operations pseudo_measure_fops = {
1262 	.write = pseudo_lock_measure_trigger,
1263 	.open = simple_open,
1264 	.llseek = default_llseek,
1265 };
1266 
1267 /**
1268  * rdtgroup_pseudo_lock_create - Create a pseudo-locked region
1269  * @rdtgrp: resource group to which pseudo-lock region belongs
1270  *
1271  * Called when a resource group in the pseudo-locksetup mode receives a
1272  * valid schemata that should be pseudo-locked. Since the resource group is
1273  * in pseudo-locksetup mode the &struct pseudo_lock_region has already been
1274  * allocated and initialized with the essential information. If a failure
1275  * occurs the resource group remains in the pseudo-locksetup mode with the
1276  * &struct pseudo_lock_region associated with it, but cleared from all
1277  * information and ready for the user to re-attempt pseudo-locking by
1278  * writing the schemata again.
1279  *
1280  * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0
1281  * on failure. Descriptive error will be written to last_cmd_status buffer.
1282  */
1283 int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp)
1284 {
1285 	struct pseudo_lock_region *plr = rdtgrp->plr;
1286 	struct task_struct *thread;
1287 	unsigned int new_minor;
1288 	struct device *dev;
1289 	int ret;
1290 
1291 	ret = pseudo_lock_region_alloc(plr);
1292 	if (ret < 0)
1293 		return ret;
1294 
1295 	ret = pseudo_lock_cstates_constrain(plr);
1296 	if (ret < 0) {
1297 		ret = -EINVAL;
1298 		goto out_region;
1299 	}
1300 
1301 	plr->thread_done = 0;
1302 
1303 	thread = kthread_create_on_node(pseudo_lock_fn, rdtgrp,
1304 					cpu_to_node(plr->cpu),
1305 					"pseudo_lock/%u", plr->cpu);
1306 	if (IS_ERR(thread)) {
1307 		ret = PTR_ERR(thread);
1308 		rdt_last_cmd_printf("Locking thread returned error %d\n", ret);
1309 		goto out_cstates;
1310 	}
1311 
1312 	kthread_bind(thread, plr->cpu);
1313 	wake_up_process(thread);
1314 
1315 	ret = wait_event_interruptible(plr->lock_thread_wq,
1316 				       plr->thread_done == 1);
1317 	if (ret < 0) {
1318 		/*
1319 		 * If the thread does not get on the CPU for whatever
1320 		 * reason and the process which sets up the region is
1321 		 * interrupted then this will leave the thread in runnable
1322 		 * state and once it gets on the CPU it will dereference
1323 		 * the cleared, but not freed, plr struct resulting in an
1324 		 * empty pseudo-locking loop.
1325 		 */
1326 		rdt_last_cmd_puts("Locking thread interrupted\n");
1327 		goto out_cstates;
1328 	}
1329 
1330 	ret = pseudo_lock_minor_get(&new_minor);
1331 	if (ret < 0) {
1332 		rdt_last_cmd_puts("Unable to obtain a new minor number\n");
1333 		goto out_cstates;
1334 	}
1335 
1336 	/*
1337 	 * Unlock access but do not release the reference. The
1338 	 * pseudo-locked region will still be here on return.
1339 	 *
1340 	 * The mutex has to be released temporarily to avoid a potential
1341 	 * deadlock with the mm->mmap_lock which is obtained in the
1342 	 * device_create() and debugfs_create_dir() callpath below as well as
1343 	 * before the mmap() callback is called.
1344 	 */
1345 	mutex_unlock(&rdtgroup_mutex);
1346 
1347 	if (!IS_ERR_OR_NULL(debugfs_resctrl)) {
1348 		plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name,
1349 						      debugfs_resctrl);
1350 		if (!IS_ERR_OR_NULL(plr->debugfs_dir))
1351 			debugfs_create_file("pseudo_lock_measure", 0200,
1352 					    plr->debugfs_dir, rdtgrp,
1353 					    &pseudo_measure_fops);
1354 	}
1355 
1356 	dev = device_create(pseudo_lock_class, NULL,
1357 			    MKDEV(pseudo_lock_major, new_minor),
1358 			    rdtgrp, "%s", rdtgrp->kn->name);
1359 
1360 	mutex_lock(&rdtgroup_mutex);
1361 
1362 	if (IS_ERR(dev)) {
1363 		ret = PTR_ERR(dev);
1364 		rdt_last_cmd_printf("Failed to create character device: %d\n",
1365 				    ret);
1366 		goto out_debugfs;
1367 	}
1368 
1369 	/* We released the mutex - check if group was removed while we did so */
1370 	if (rdtgrp->flags & RDT_DELETED) {
1371 		ret = -ENODEV;
1372 		goto out_device;
1373 	}
1374 
1375 	plr->minor = new_minor;
1376 
1377 	rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED;
1378 	closid_free(rdtgrp->closid);
1379 	rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0444);
1380 	rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0444);
1381 
1382 	ret = 0;
1383 	goto out;
1384 
1385 out_device:
1386 	device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, new_minor));
1387 out_debugfs:
1388 	debugfs_remove_recursive(plr->debugfs_dir);
1389 	pseudo_lock_minor_release(new_minor);
1390 out_cstates:
1391 	pseudo_lock_cstates_relax(plr);
1392 out_region:
1393 	pseudo_lock_region_clear(plr);
1394 out:
1395 	return ret;
1396 }
1397 
1398 /**
1399  * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region
1400  * @rdtgrp: resource group to which the pseudo-locked region belongs
1401  *
1402  * The removal of a pseudo-locked region can be initiated when the resource
1403  * group is removed from user space via a "rmdir" from userspace or the
1404  * unmount of the resctrl filesystem. On removal the resource group does
1405  * not go back to pseudo-locksetup mode before it is removed, instead it is
1406  * removed directly. There is thus asymmetry with the creation where the
1407  * &struct pseudo_lock_region is removed here while it was not created in
1408  * rdtgroup_pseudo_lock_create().
1409  *
1410  * Return: void
1411  */
1412 void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp)
1413 {
1414 	struct pseudo_lock_region *plr = rdtgrp->plr;
1415 
1416 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1417 		/*
1418 		 * Default group cannot be a pseudo-locked region so we can
1419 		 * free closid here.
1420 		 */
1421 		closid_free(rdtgrp->closid);
1422 		goto free;
1423 	}
1424 
1425 	pseudo_lock_cstates_relax(plr);
1426 	debugfs_remove_recursive(rdtgrp->plr->debugfs_dir);
1427 	device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor));
1428 	pseudo_lock_minor_release(plr->minor);
1429 
1430 free:
1431 	pseudo_lock_free(rdtgrp);
1432 }
1433 
1434 static int pseudo_lock_dev_open(struct inode *inode, struct file *filp)
1435 {
1436 	struct rdtgroup *rdtgrp;
1437 
1438 	mutex_lock(&rdtgroup_mutex);
1439 
1440 	rdtgrp = region_find_by_minor(iminor(inode));
1441 	if (!rdtgrp) {
1442 		mutex_unlock(&rdtgroup_mutex);
1443 		return -ENODEV;
1444 	}
1445 
1446 	filp->private_data = rdtgrp;
1447 	atomic_inc(&rdtgrp->waitcount);
1448 	/* Perform a non-seekable open - llseek is not supported */
1449 	filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
1450 
1451 	mutex_unlock(&rdtgroup_mutex);
1452 
1453 	return 0;
1454 }
1455 
1456 static int pseudo_lock_dev_release(struct inode *inode, struct file *filp)
1457 {
1458 	struct rdtgroup *rdtgrp;
1459 
1460 	mutex_lock(&rdtgroup_mutex);
1461 	rdtgrp = filp->private_data;
1462 	WARN_ON(!rdtgrp);
1463 	if (!rdtgrp) {
1464 		mutex_unlock(&rdtgroup_mutex);
1465 		return -ENODEV;
1466 	}
1467 	filp->private_data = NULL;
1468 	atomic_dec(&rdtgrp->waitcount);
1469 	mutex_unlock(&rdtgroup_mutex);
1470 	return 0;
1471 }
1472 
1473 static int pseudo_lock_dev_mremap(struct vm_area_struct *area)
1474 {
1475 	/* Not supported */
1476 	return -EINVAL;
1477 }
1478 
1479 static const struct vm_operations_struct pseudo_mmap_ops = {
1480 	.mremap = pseudo_lock_dev_mremap,
1481 };
1482 
1483 static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma)
1484 {
1485 	unsigned long vsize = vma->vm_end - vma->vm_start;
1486 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
1487 	struct pseudo_lock_region *plr;
1488 	struct rdtgroup *rdtgrp;
1489 	unsigned long physical;
1490 	unsigned long psize;
1491 
1492 	mutex_lock(&rdtgroup_mutex);
1493 
1494 	rdtgrp = filp->private_data;
1495 	WARN_ON(!rdtgrp);
1496 	if (!rdtgrp) {
1497 		mutex_unlock(&rdtgroup_mutex);
1498 		return -ENODEV;
1499 	}
1500 
1501 	plr = rdtgrp->plr;
1502 
1503 	if (!plr->d) {
1504 		mutex_unlock(&rdtgroup_mutex);
1505 		return -ENODEV;
1506 	}
1507 
1508 	/*
1509 	 * Task is required to run with affinity to the cpus associated
1510 	 * with the pseudo-locked region. If this is not the case the task
1511 	 * may be scheduled elsewhere and invalidate entries in the
1512 	 * pseudo-locked region.
1513 	 */
1514 	if (!cpumask_subset(current->cpus_ptr, &plr->d->cpu_mask)) {
1515 		mutex_unlock(&rdtgroup_mutex);
1516 		return -EINVAL;
1517 	}
1518 
1519 	physical = __pa(plr->kmem) >> PAGE_SHIFT;
1520 	psize = plr->size - off;
1521 
1522 	if (off > plr->size) {
1523 		mutex_unlock(&rdtgroup_mutex);
1524 		return -ENOSPC;
1525 	}
1526 
1527 	/*
1528 	 * Ensure changes are carried directly to the memory being mapped,
1529 	 * do not allow copy-on-write mapping.
1530 	 */
1531 	if (!(vma->vm_flags & VM_SHARED)) {
1532 		mutex_unlock(&rdtgroup_mutex);
1533 		return -EINVAL;
1534 	}
1535 
1536 	if (vsize > psize) {
1537 		mutex_unlock(&rdtgroup_mutex);
1538 		return -ENOSPC;
1539 	}
1540 
1541 	memset(plr->kmem + off, 0, vsize);
1542 
1543 	if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff,
1544 			    vsize, vma->vm_page_prot)) {
1545 		mutex_unlock(&rdtgroup_mutex);
1546 		return -EAGAIN;
1547 	}
1548 	vma->vm_ops = &pseudo_mmap_ops;
1549 	mutex_unlock(&rdtgroup_mutex);
1550 	return 0;
1551 }
1552 
1553 static const struct file_operations pseudo_lock_dev_fops = {
1554 	.owner =	THIS_MODULE,
1555 	.llseek =	no_llseek,
1556 	.read =		NULL,
1557 	.write =	NULL,
1558 	.open =		pseudo_lock_dev_open,
1559 	.release =	pseudo_lock_dev_release,
1560 	.mmap =		pseudo_lock_dev_mmap,
1561 };
1562 
1563 static char *pseudo_lock_devnode(const struct device *dev, umode_t *mode)
1564 {
1565 	const struct rdtgroup *rdtgrp;
1566 
1567 	rdtgrp = dev_get_drvdata(dev);
1568 	if (mode)
1569 		*mode = 0600;
1570 	return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name);
1571 }
1572 
1573 int rdt_pseudo_lock_init(void)
1574 {
1575 	int ret;
1576 
1577 	ret = register_chrdev(0, "pseudo_lock", &pseudo_lock_dev_fops);
1578 	if (ret < 0)
1579 		return ret;
1580 
1581 	pseudo_lock_major = ret;
1582 
1583 	pseudo_lock_class = class_create("pseudo_lock");
1584 	if (IS_ERR(pseudo_lock_class)) {
1585 		ret = PTR_ERR(pseudo_lock_class);
1586 		unregister_chrdev(pseudo_lock_major, "pseudo_lock");
1587 		return ret;
1588 	}
1589 
1590 	pseudo_lock_class->devnode = pseudo_lock_devnode;
1591 	return 0;
1592 }
1593 
1594 void rdt_pseudo_lock_release(void)
1595 {
1596 	class_destroy(pseudo_lock_class);
1597 	pseudo_lock_class = NULL;
1598 	unregister_chrdev(pseudo_lock_major, "pseudo_lock");
1599 	pseudo_lock_major = 0;
1600 }
1601