// SPDX-License-Identifier: GPL-2.0 /* * Resource Director Technology (RDT) * * Pseudo-locking support built on top of Cache Allocation Technology (CAT) * * Copyright (C) 2018 Intel Corporation * * Author: Reinette Chatre */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../events/perf_event.h" /* For X86_CONFIG() */ #include "internal.h" #define CREATE_TRACE_POINTS #include "pseudo_lock_event.h" /* * The bits needed to disable hardware prefetching varies based on the * platform. During initialization we will discover which bits to use. */ static u64 prefetch_disable_bits; /* * Major number assigned to and shared by all devices exposing * pseudo-locked regions. */ static unsigned int pseudo_lock_major; static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0); static struct class *pseudo_lock_class; /** * get_prefetch_disable_bits - prefetch disable bits of supported platforms * * Capture the list of platforms that have been validated to support * pseudo-locking. This includes testing to ensure pseudo-locked regions * with low cache miss rates can be created under variety of load conditions * as well as that these pseudo-locked regions can maintain their low cache * miss rates under variety of load conditions for significant lengths of time. * * After a platform has been validated to support pseudo-locking its * hardware prefetch disable bits are included here as they are documented * in the SDM. * * When adding a platform here also add support for its cache events to * measure_cycles_perf_fn() * * Return: * If platform is supported, the bits to disable hardware prefetchers, 0 * if platform is not supported. */ static u64 get_prefetch_disable_bits(void) { if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || boot_cpu_data.x86 != 6) return 0; switch (boot_cpu_data.x86_model) { case INTEL_FAM6_BROADWELL_X: /* * SDM defines bits of MSR_MISC_FEATURE_CONTROL register * as: * 0 L2 Hardware Prefetcher Disable (R/W) * 1 L2 Adjacent Cache Line Prefetcher Disable (R/W) * 2 DCU Hardware Prefetcher Disable (R/W) * 3 DCU IP Prefetcher Disable (R/W) * 63:4 Reserved */ return 0xF; case INTEL_FAM6_ATOM_GOLDMONT: case INTEL_FAM6_ATOM_GOLDMONT_PLUS: /* * SDM defines bits of MSR_MISC_FEATURE_CONTROL register * as: * 0 L2 Hardware Prefetcher Disable (R/W) * 1 Reserved * 2 DCU Hardware Prefetcher Disable (R/W) * 63:3 Reserved */ return 0x5; } return 0; } /** * pseudo_lock_minor_get - Obtain available minor number * @minor: Pointer to where new minor number will be stored * * A bitmask is used to track available minor numbers. Here the next free * minor number is marked as unavailable and returned. * * Return: 0 on success, <0 on failure. */ static int pseudo_lock_minor_get(unsigned int *minor) { unsigned long first_bit; first_bit = find_first_bit(&pseudo_lock_minor_avail, MINORBITS); if (first_bit == MINORBITS) return -ENOSPC; __clear_bit(first_bit, &pseudo_lock_minor_avail); *minor = first_bit; return 0; } /** * pseudo_lock_minor_release - Return minor number to available * @minor: The minor number made available */ static void pseudo_lock_minor_release(unsigned int minor) { __set_bit(minor, &pseudo_lock_minor_avail); } /** * region_find_by_minor - Locate a pseudo-lock region by inode minor number * @minor: The minor number of the device representing pseudo-locked region * * When the character device is accessed we need to determine which * pseudo-locked region it belongs to. This is done by matching the minor * number of the device to the pseudo-locked region it belongs. * * Minor numbers are assigned at the time a pseudo-locked region is associated * with a cache instance. * * Return: On success return pointer to resource group owning the pseudo-locked * region, NULL on failure. */ static struct rdtgroup *region_find_by_minor(unsigned int minor) { struct rdtgroup *rdtgrp, *rdtgrp_match = NULL; list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { if (rdtgrp->plr && rdtgrp->plr->minor == minor) { rdtgrp_match = rdtgrp; break; } } return rdtgrp_match; } /** * pseudo_lock_pm_req - A power management QoS request list entry * @list: Entry within the @pm_reqs list for a pseudo-locked region * @req: PM QoS request */ struct pseudo_lock_pm_req { struct list_head list; struct dev_pm_qos_request req; }; static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) { struct pseudo_lock_pm_req *pm_req, *next; list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) { dev_pm_qos_remove_request(&pm_req->req); list_del(&pm_req->list); kfree(pm_req); } } /** * pseudo_lock_cstates_constrain - Restrict cores from entering C6 * * To prevent the cache from being affected by power management entering * C6 has to be avoided. This is accomplished by requesting a latency * requirement lower than lowest C6 exit latency of all supported * platforms as found in the cpuidle state tables in the intel_idle driver. * At this time it is possible to do so with a single latency requirement * for all supported platforms. * * Since Goldmont is supported, which is affected by X86_BUG_MONITOR, * the ACPI latencies need to be considered while keeping in mind that C2 * may be set to map to deeper sleep states. In this case the latency * requirement needs to prevent entering C2 also. */ static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) { struct pseudo_lock_pm_req *pm_req; int cpu; int ret; for_each_cpu(cpu, &plr->d->cpu_mask) { pm_req = kzalloc(sizeof(*pm_req), GFP_KERNEL); if (!pm_req) { rdt_last_cmd_puts("Failure to allocate memory for PM QoS\n"); ret = -ENOMEM; goto out_err; } ret = dev_pm_qos_add_request(get_cpu_device(cpu), &pm_req->req, DEV_PM_QOS_RESUME_LATENCY, 30); if (ret < 0) { rdt_last_cmd_printf("Failed to add latency req CPU%d\n", cpu); kfree(pm_req); ret = -1; goto out_err; } list_add(&pm_req->list, &plr->pm_reqs); } return 0; out_err: pseudo_lock_cstates_relax(plr); return ret; } /** * pseudo_lock_region_clear - Reset pseudo-lock region data * @plr: pseudo-lock region * * All content of the pseudo-locked region is reset - any memory allocated * freed. * * Return: void */ static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) { plr->size = 0; plr->line_size = 0; kfree(plr->kmem); plr->kmem = NULL; plr->r = NULL; if (plr->d) plr->d->plr = NULL; plr->d = NULL; plr->cbm = 0; plr->debugfs_dir = NULL; } /** * pseudo_lock_region_init - Initialize pseudo-lock region information * @plr: pseudo-lock region * * Called after user provided a schemata to be pseudo-locked. From the * schemata the &struct pseudo_lock_region is on entry already initialized * with the resource, domain, and capacity bitmask. Here the information * required for pseudo-locking is deduced from this data and &struct * pseudo_lock_region initialized further. This information includes: * - size in bytes of the region to be pseudo-locked * - cache line size to know the stride with which data needs to be accessed * to be pseudo-locked * - a cpu associated with the cache instance on which the pseudo-locking * flow can be executed * * Return: 0 on success, <0 on failure. Descriptive error will be written * to last_cmd_status buffer. */ static int pseudo_lock_region_init(struct pseudo_lock_region *plr) { struct cpu_cacheinfo *ci; int ret; int i; /* Pick the first cpu we find that is associated with the cache. */ plr->cpu = cpumask_first(&plr->d->cpu_mask); if (!cpu_online(plr->cpu)) { rdt_last_cmd_printf("CPU %u associated with cache not online\n", plr->cpu); ret = -ENODEV; goto out_region; } ci = get_cpu_cacheinfo(plr->cpu); plr->size = rdtgroup_cbm_to_size(plr->r, plr->d, plr->cbm); for (i = 0; i < ci->num_leaves; i++) { if (ci->info_list[i].level == plr->r->cache_level) { plr->line_size = ci->info_list[i].coherency_line_size; return 0; } } ret = -1; rdt_last_cmd_puts("Unable to determine cache line size\n"); out_region: pseudo_lock_region_clear(plr); return ret; } /** * pseudo_lock_init - Initialize a pseudo-lock region * @rdtgrp: resource group to which new pseudo-locked region will belong * * A pseudo-locked region is associated with a resource group. When this * association is created the pseudo-locked region is initialized. The * details of the pseudo-locked region are not known at this time so only * allocation is done and association established. * * Return: 0 on success, <0 on failure */ static int pseudo_lock_init(struct rdtgroup *rdtgrp) { struct pseudo_lock_region *plr; plr = kzalloc(sizeof(*plr), GFP_KERNEL); if (!plr) return -ENOMEM; init_waitqueue_head(&plr->lock_thread_wq); INIT_LIST_HEAD(&plr->pm_reqs); rdtgrp->plr = plr; return 0; } /** * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked * @plr: pseudo-lock region * * Initialize the details required to set up the pseudo-locked region and * allocate the contiguous memory that will be pseudo-locked to the cache. * * Return: 0 on success, <0 on failure. Descriptive error will be written * to last_cmd_status buffer. */ static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr) { int ret; ret = pseudo_lock_region_init(plr); if (ret < 0) return ret; /* * We do not yet support contiguous regions larger than * KMALLOC_MAX_SIZE. */ if (plr->size > KMALLOC_MAX_SIZE) { rdt_last_cmd_puts("Requested region exceeds maximum size\n"); ret = -E2BIG; goto out_region; } plr->kmem = kzalloc(plr->size, GFP_KERNEL); if (!plr->kmem) { rdt_last_cmd_puts("Unable to allocate memory\n"); ret = -ENOMEM; goto out_region; } ret = 0; goto out; out_region: pseudo_lock_region_clear(plr); out: return ret; } /** * pseudo_lock_free - Free a pseudo-locked region * @rdtgrp: resource group to which pseudo-locked region belonged * * The pseudo-locked region's resources have already been released, or not * yet created at this point. Now it can be freed and disassociated from the * resource group. * * Return: void */ static void pseudo_lock_free(struct rdtgroup *rdtgrp) { pseudo_lock_region_clear(rdtgrp->plr); kfree(rdtgrp->plr); rdtgrp->plr = NULL; } /** * pseudo_lock_fn - Load kernel memory into cache * @_rdtgrp: resource group to which pseudo-lock region belongs * * This is the core pseudo-locking flow. * * First we ensure that the kernel memory cannot be found in the cache. * Then, while taking care that there will be as little interference as * possible, the memory to be loaded is accessed while core is running * with class of service set to the bitmask of the pseudo-locked region. * After this is complete no future CAT allocations will be allowed to * overlap with this bitmask. * * Local register variables are utilized to ensure that the memory region * to be locked is the only memory access made during the critical locking * loop. * * Return: 0. Waiter on waitqueue will be woken on completion. */ static int pseudo_lock_fn(void *_rdtgrp) { struct rdtgroup *rdtgrp = _rdtgrp; struct pseudo_lock_region *plr = rdtgrp->plr; u32 rmid_p, closid_p; unsigned long i; #ifdef CONFIG_KASAN /* * The registers used for local register variables are also used * when KASAN is active. When KASAN is active we use a regular * variable to ensure we always use a valid pointer, but the cost * is that this variable will enter the cache through evicting the * memory we are trying to lock into the cache. Thus expect lower * pseudo-locking success rate when KASAN is active. */ unsigned int line_size; unsigned int size; void *mem_r; #else register unsigned int line_size asm("esi"); register unsigned int size asm("edi"); register void *mem_r asm(_ASM_BX); #endif /* CONFIG_KASAN */ /* * Make sure none of the allocated memory is cached. If it is we * will get a cache hit in below loop from outside of pseudo-locked * region. * wbinvd (as opposed to clflush/clflushopt) is required to * increase likelihood that allocated cache portion will be filled * with associated memory. */ native_wbinvd(); /* * Always called with interrupts enabled. By disabling interrupts * ensure that we will not be preempted during this critical section. */ local_irq_disable(); /* * Call wrmsr and rdmsr as directly as possible to avoid tracing * clobbering local register variables or affecting cache accesses. * * Disable the hardware prefetcher so that when the end of the memory * being pseudo-locked is reached the hardware will not read beyond * the buffer and evict pseudo-locked memory read earlier from the * cache. */ __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); closid_p = this_cpu_read(pqr_state.cur_closid); rmid_p = this_cpu_read(pqr_state.cur_rmid); mem_r = plr->kmem; size = plr->size; line_size = plr->line_size; /* * Critical section begin: start by writing the closid associated * with the capacity bitmask of the cache region being * pseudo-locked followed by reading of kernel memory to load it * into the cache. */ __wrmsr(IA32_PQR_ASSOC, rmid_p, rdtgrp->closid); /* * Cache was flushed earlier. Now access kernel memory to read it * into cache region associated with just activated plr->closid. * Loop over data twice: * - In first loop the cache region is shared with the page walker * as it populates the paging structure caches (including TLB). * - In the second loop the paging structure caches are used and * cache region is populated with the memory being referenced. */ for (i = 0; i < size; i += PAGE_SIZE) { /* * Add a barrier to prevent speculative execution of this * loop reading beyond the end of the buffer. */ rmb(); asm volatile("mov (%0,%1,1), %%eax\n\t" : : "r" (mem_r), "r" (i) : "%eax", "memory"); } for (i = 0; i < size; i += line_size) { /* * Add a barrier to prevent speculative execution of this * loop reading beyond the end of the buffer. */ rmb(); asm volatile("mov (%0,%1,1), %%eax\n\t" : : "r" (mem_r), "r" (i) : "%eax", "memory"); } /* * Critical section end: restore closid with capacity bitmask that * does not overlap with pseudo-locked region. */ __wrmsr(IA32_PQR_ASSOC, rmid_p, closid_p); /* Re-enable the hardware prefetcher(s) */ wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); return 0; } /** * rdtgroup_monitor_in_progress - Test if monitoring in progress * @r: resource group being queried * * Return: 1 if monitor groups have been created for this resource * group, 0 otherwise. */ static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp) { return !list_empty(&rdtgrp->mon.crdtgrp_list); } /** * rdtgroup_locksetup_user_restrict - Restrict user access to group * @rdtgrp: resource group needing access restricted * * A resource group used for cache pseudo-locking cannot have cpus or tasks * assigned to it. This is communicated to the user by restricting access * to all the files that can be used to make such changes. * * Permissions restored with rdtgroup_locksetup_user_restore() * * Return: 0 on success, <0 on failure. If a failure occurs during the * restriction of access an attempt will be made to restore permissions but * the state of the mode of these files will be uncertain when a failure * occurs. */ static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp) { int ret; ret = rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); if (ret) return ret; ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); if (ret) goto err_tasks; ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); if (ret) goto err_cpus; if (rdt_mon_capable) { ret = rdtgroup_kn_mode_restrict(rdtgrp, "mon_groups"); if (ret) goto err_cpus_list; } ret = 0; goto out; err_cpus_list: rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); err_cpus: rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); err_tasks: rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); out: return ret; } /** * rdtgroup_locksetup_user_restore - Restore user access to group * @rdtgrp: resource group needing access restored * * Restore all file access previously removed using * rdtgroup_locksetup_user_restrict() * * Return: 0 on success, <0 on failure. If a failure occurs during the * restoration of access an attempt will be made to restrict permissions * again but the state of the mode of these files will be uncertain when * a failure occurs. */ static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp) { int ret; ret = rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); if (ret) return ret; ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); if (ret) goto err_tasks; ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); if (ret) goto err_cpus; if (rdt_mon_capable) { ret = rdtgroup_kn_mode_restore(rdtgrp, "mon_groups", 0777); if (ret) goto err_cpus_list; } ret = 0; goto out; err_cpus_list: rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); err_cpus: rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); err_tasks: rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); out: return ret; } /** * rdtgroup_locksetup_enter - Resource group enters locksetup mode * @rdtgrp: resource group requested to enter locksetup mode * * A resource group enters locksetup mode to reflect that it would be used * to represent a pseudo-locked region and is in the process of being set * up to do so. A resource group used for a pseudo-locked region would * lose the closid associated with it so we cannot allow it to have any * tasks or cpus assigned nor permit tasks or cpus to be assigned in the * future. Monitoring of a pseudo-locked region is not allowed either. * * The above and more restrictions on a pseudo-locked region are checked * for and enforced before the resource group enters the locksetup mode. * * Returns: 0 if the resource group successfully entered locksetup mode, <0 * on failure. On failure the last_cmd_status buffer is updated with text to * communicate details of failure to the user. */ int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) { int ret; /* * The default resource group can neither be removed nor lose the * default closid associated with it. */ if (rdtgrp == &rdtgroup_default) { rdt_last_cmd_puts("Cannot pseudo-lock default group\n"); return -EINVAL; } /* * Cache Pseudo-locking not supported when CDP is enabled. * * Some things to consider if you would like to enable this * support (using L3 CDP as example): * - When CDP is enabled two separate resources are exposed, * L3DATA and L3CODE, but they are actually on the same cache. * The implication for pseudo-locking is that if a * pseudo-locked region is created on a domain of one * resource (eg. L3CODE), then a pseudo-locked region cannot * be created on that same domain of the other resource * (eg. L3DATA). This is because the creation of a * pseudo-locked region involves a call to wbinvd that will * affect all cache allocations on particular domain. * - Considering the previous, it may be possible to only * expose one of the CDP resources to pseudo-locking and * hide the other. For example, we could consider to only * expose L3DATA and since the L3 cache is unified it is * still possible to place instructions there are execute it. * - If only one region is exposed to pseudo-locking we should * still keep in mind that availability of a portion of cache * for pseudo-locking should take into account both resources. * Similarly, if a pseudo-locked region is created in one * resource, the portion of cache used by it should be made * unavailable to all future allocations from both resources. */ if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled || rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled) { rdt_last_cmd_puts("CDP enabled\n"); return -EINVAL; } /* * Not knowing the bits to disable prefetching implies that this * platform does not support Cache Pseudo-Locking. */ prefetch_disable_bits = get_prefetch_disable_bits(); if (prefetch_disable_bits == 0) { rdt_last_cmd_puts("Pseudo-locking not supported\n"); return -EINVAL; } if (rdtgroup_monitor_in_progress(rdtgrp)) { rdt_last_cmd_puts("Monitoring in progress\n"); return -EINVAL; } if (rdtgroup_tasks_assigned(rdtgrp)) { rdt_last_cmd_puts("Tasks assigned to resource group\n"); return -EINVAL; } if (!cpumask_empty(&rdtgrp->cpu_mask)) { rdt_last_cmd_puts("CPUs assigned to resource group\n"); return -EINVAL; } if (rdtgroup_locksetup_user_restrict(rdtgrp)) { rdt_last_cmd_puts("Unable to modify resctrl permissions\n"); return -EIO; } ret = pseudo_lock_init(rdtgrp); if (ret) { rdt_last_cmd_puts("Unable to init pseudo-lock region\n"); goto out_release; } /* * If this system is capable of monitoring a rmid would have been * allocated when the control group was created. This is not needed * anymore when this group would be used for pseudo-locking. This * is safe to call on platforms not capable of monitoring. */ free_rmid(rdtgrp->mon.rmid); ret = 0; goto out; out_release: rdtgroup_locksetup_user_restore(rdtgrp); out: return ret; } /** * rdtgroup_locksetup_exit - resource group exist locksetup mode * @rdtgrp: resource group * * When a resource group exits locksetup mode the earlier restrictions are * lifted. * * Return: 0 on success, <0 on failure */ int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) { int ret; if (rdt_mon_capable) { ret = alloc_rmid(); if (ret < 0) { rdt_last_cmd_puts("Out of RMIDs\n"); return ret; } rdtgrp->mon.rmid = ret; } ret = rdtgroup_locksetup_user_restore(rdtgrp); if (ret) { free_rmid(rdtgrp->mon.rmid); return ret; } pseudo_lock_free(rdtgrp); return 0; } /** * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked * @d: RDT domain * @cbm: CBM to test * * @d represents a cache instance and @cbm a capacity bitmask that is * considered for it. Determine if @cbm overlaps with any existing * pseudo-locked region on @d. * * @cbm is unsigned long, even if only 32 bits are used, to make the * bitmap functions work correctly. * * Return: true if @cbm overlaps with pseudo-locked region on @d, false * otherwise. */ bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm) { unsigned int cbm_len; unsigned long cbm_b; if (d->plr) { cbm_len = d->plr->r->cache.cbm_len; cbm_b = d->plr->cbm; if (bitmap_intersects(&cbm, &cbm_b, cbm_len)) return true; } return false; } /** * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy * @d: RDT domain under test * * The setup of a pseudo-locked region affects all cache instances within * the hierarchy of the region. It is thus essential to know if any * pseudo-locked regions exist within a cache hierarchy to prevent any * attempts to create new pseudo-locked regions in the same hierarchy. * * Return: true if a pseudo-locked region exists in the hierarchy of @d or * if it is not possible to test due to memory allocation issue, * false otherwise. */ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) { cpumask_var_t cpu_with_psl; struct rdt_resource *r; struct rdt_domain *d_i; bool ret = false; if (!zalloc_cpumask_var(&cpu_with_psl, GFP_KERNEL)) return true; /* * First determine which cpus have pseudo-locked regions * associated with them. */ for_each_alloc_enabled_rdt_resource(r) { list_for_each_entry(d_i, &r->domains, list) { if (d_i->plr) cpumask_or(cpu_with_psl, cpu_with_psl, &d_i->cpu_mask); } } /* * Next test if new pseudo-locked region would intersect with * existing region. */ if (cpumask_intersects(&d->cpu_mask, cpu_with_psl)) ret = true; free_cpumask_var(cpu_with_psl); return ret; } /** * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory * @_plr: pseudo-lock region to measure * * There is no deterministic way to test if a memory region is cached. One * way is to measure how long it takes to read the memory, the speed of * access is a good way to learn how close to the cpu the data was. Even * more, if the prefetcher is disabled and the memory is read at a stride * of half the cache line, then a cache miss will be easy to spot since the * read of the first half would be significantly slower than the read of * the second half. * * Return: 0. Waiter on waitqueue will be woken on completion. */ static int measure_cycles_lat_fn(void *_plr) { struct pseudo_lock_region *plr = _plr; unsigned long i; u64 start, end; void *mem_r; local_irq_disable(); /* * Disable hardware prefetchers. */ wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); mem_r = READ_ONCE(plr->kmem); /* * Dummy execute of the time measurement to load the needed * instructions into the L1 instruction cache. */ start = rdtsc_ordered(); for (i = 0; i < plr->size; i += 32) { start = rdtsc_ordered(); asm volatile("mov (%0,%1,1), %%eax\n\t" : : "r" (mem_r), "r" (i) : "%eax", "memory"); end = rdtsc_ordered(); trace_pseudo_lock_mem_latency((u32)(end - start)); } wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); return 0; } /* * Create a perf_event_attr for the hit and miss perf events that will * be used during the performance measurement. A perf_event maintains * a pointer to its perf_event_attr so a unique attribute structure is * created for each perf_event. * * The actual configuration of the event is set right before use in order * to use the X86_CONFIG macro. */ static struct perf_event_attr perf_miss_attr = { .type = PERF_TYPE_RAW, .size = sizeof(struct perf_event_attr), .pinned = 1, .disabled = 0, .exclude_user = 1, }; static struct perf_event_attr perf_hit_attr = { .type = PERF_TYPE_RAW, .size = sizeof(struct perf_event_attr), .pinned = 1, .disabled = 0, .exclude_user = 1, }; struct residency_counts { u64 miss_before, hits_before; u64 miss_after, hits_after; }; static int measure_residency_fn(struct perf_event_attr *miss_attr, struct perf_event_attr *hit_attr, struct pseudo_lock_region *plr, struct residency_counts *counts) { u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; struct perf_event *miss_event, *hit_event; int hit_pmcnum, miss_pmcnum; unsigned int line_size; unsigned int size; unsigned long i; void *mem_r; u64 tmp; miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, NULL, NULL, NULL); if (IS_ERR(miss_event)) goto out; hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu, NULL, NULL, NULL); if (IS_ERR(hit_event)) goto out_miss; local_irq_disable(); /* * Check any possible error state of events used by performing * one local read. */ if (perf_event_read_local(miss_event, &tmp, NULL, NULL)) { local_irq_enable(); goto out_hit; } if (perf_event_read_local(hit_event, &tmp, NULL, NULL)) { local_irq_enable(); goto out_hit; } /* * Disable hardware prefetchers. */ wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); /* Initialize rest of local variables */ /* * Performance event has been validated right before this with * interrupts disabled - it is thus safe to read the counter index. */ miss_pmcnum = x86_perf_rdpmc_index(miss_event); hit_pmcnum = x86_perf_rdpmc_index(hit_event); line_size = READ_ONCE(plr->line_size); mem_r = READ_ONCE(plr->kmem); size = READ_ONCE(plr->size); /* * Read counter variables twice - first to load the instructions * used in L1 cache, second to capture accurate value that does not * include cache misses incurred because of instruction loads. */ rdpmcl(hit_pmcnum, hits_before); rdpmcl(miss_pmcnum, miss_before); /* * From SDM: Performing back-to-back fast reads are not guaranteed * to be monotonic. * Use LFENCE to ensure all previous instructions are retired * before proceeding. */ rmb(); rdpmcl(hit_pmcnum, hits_before); rdpmcl(miss_pmcnum, miss_before); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. */ rmb(); for (i = 0; i < size; i += line_size) { /* * Add a barrier to prevent speculative execution of this * loop reading beyond the end of the buffer. */ rmb(); asm volatile("mov (%0,%1,1), %%eax\n\t" : : "r" (mem_r), "r" (i) : "%eax", "memory"); } /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. */ rmb(); rdpmcl(hit_pmcnum, hits_after); rdpmcl(miss_pmcnum, miss_after); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. */ rmb(); /* Re-enable hardware prefetchers */ wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); local_irq_enable(); out_hit: perf_event_release_kernel(hit_event); out_miss: perf_event_release_kernel(miss_event); out: /* * All counts will be zero on failure. */ counts->miss_before = miss_before; counts->hits_before = hits_before; counts->miss_after = miss_after; counts->hits_after = hits_after; return 0; } static int measure_l2_residency(void *_plr) { struct pseudo_lock_region *plr = _plr; struct residency_counts counts = {0}; /* * Non-architectural event for the Goldmont Microarchitecture * from Intel x86 Architecture Software Developer Manual (SDM): * MEM_LOAD_UOPS_RETIRED D1H (event number) * Umask values: * L2_HIT 02H * L2_MISS 10H */ switch (boot_cpu_data.x86_model) { case INTEL_FAM6_ATOM_GOLDMONT: case INTEL_FAM6_ATOM_GOLDMONT_PLUS: perf_miss_attr.config = X86_CONFIG(.event = 0xd1, .umask = 0x10); perf_hit_attr.config = X86_CONFIG(.event = 0xd1, .umask = 0x2); break; default: goto out; } measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts); /* * If a failure prevented the measurements from succeeding * tracepoints will still be written and all counts will be zero. */ trace_pseudo_lock_l2(counts.hits_after - counts.hits_before, counts.miss_after - counts.miss_before); out: plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); return 0; } static int measure_l3_residency(void *_plr) { struct pseudo_lock_region *plr = _plr; struct residency_counts counts = {0}; /* * On Broadwell Microarchitecture the MEM_LOAD_UOPS_RETIRED event * has two "no fix" errata associated with it: BDM35 and BDM100. On * this platform the following events are used instead: * LONGEST_LAT_CACHE 2EH (Documented in SDM) * REFERENCE 4FH * MISS 41H */ switch (boot_cpu_data.x86_model) { case INTEL_FAM6_BROADWELL_X: /* On BDW the hit event counts references, not hits */ perf_hit_attr.config = X86_CONFIG(.event = 0x2e, .umask = 0x4f); perf_miss_attr.config = X86_CONFIG(.event = 0x2e, .umask = 0x41); break; default: goto out; } measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts); /* * If a failure prevented the measurements from succeeding * tracepoints will still be written and all counts will be zero. */ counts.miss_after -= counts.miss_before; if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) { /* * On BDW references and misses are counted, need to adjust. * Sometimes the "hits" counter is a bit more than the * references, for example, x references but x + 1 hits. * To not report invalid hit values in this case we treat * that as misses equal to references. */ /* First compute the number of cache references measured */ counts.hits_after -= counts.hits_before; /* Next convert references to cache hits */ counts.hits_after -= min(counts.miss_after, counts.hits_after); } else { counts.hits_after -= counts.hits_before; } trace_pseudo_lock_l3(counts.hits_after, counts.miss_after); out: plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); return 0; } /** * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region * * The measurement of latency to access a pseudo-locked region should be * done from a cpu that is associated with that pseudo-locked region. * Determine which cpu is associated with this region and start a thread on * that cpu to perform the measurement, wait for that thread to complete. * * Return: 0 on success, <0 on failure */ static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) { struct pseudo_lock_region *plr = rdtgrp->plr; struct task_struct *thread; unsigned int cpu; int ret = -1; cpus_read_lock(); mutex_lock(&rdtgroup_mutex); if (rdtgrp->flags & RDT_DELETED) { ret = -ENODEV; goto out; } if (!plr->d) { ret = -ENODEV; goto out; } plr->thread_done = 0; cpu = cpumask_first(&plr->d->cpu_mask); if (!cpu_online(cpu)) { ret = -ENODEV; goto out; } plr->cpu = cpu; if (sel == 1) thread = kthread_create_on_node(measure_cycles_lat_fn, plr, cpu_to_node(cpu), "pseudo_lock_measure/%u", cpu); else if (sel == 2) thread = kthread_create_on_node(measure_l2_residency, plr, cpu_to_node(cpu), "pseudo_lock_measure/%u", cpu); else if (sel == 3) thread = kthread_create_on_node(measure_l3_residency, plr, cpu_to_node(cpu), "pseudo_lock_measure/%u", cpu); else goto out; if (IS_ERR(thread)) { ret = PTR_ERR(thread); goto out; } kthread_bind(thread, cpu); wake_up_process(thread); ret = wait_event_interruptible(plr->lock_thread_wq, plr->thread_done == 1); if (ret < 0) goto out; ret = 0; out: mutex_unlock(&rdtgroup_mutex); cpus_read_unlock(); return ret; } static ssize_t pseudo_lock_measure_trigger(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { struct rdtgroup *rdtgrp = file->private_data; size_t buf_size; char buf[32]; int ret; int sel; buf_size = min(count, (sizeof(buf) - 1)); if (copy_from_user(buf, user_buf, buf_size)) return -EFAULT; buf[buf_size] = '\0'; ret = kstrtoint(buf, 10, &sel); if (ret == 0) { if (sel != 1 && sel != 2 && sel != 3) return -EINVAL; ret = debugfs_file_get(file->f_path.dentry); if (ret) return ret; ret = pseudo_lock_measure_cycles(rdtgrp, sel); if (ret == 0) ret = count; debugfs_file_put(file->f_path.dentry); } return ret; } static const struct file_operations pseudo_measure_fops = { .write = pseudo_lock_measure_trigger, .open = simple_open, .llseek = default_llseek, }; /** * rdtgroup_pseudo_lock_create - Create a pseudo-locked region * @rdtgrp: resource group to which pseudo-lock region belongs * * Called when a resource group in the pseudo-locksetup mode receives a * valid schemata that should be pseudo-locked. Since the resource group is * in pseudo-locksetup mode the &struct pseudo_lock_region has already been * allocated and initialized with the essential information. If a failure * occurs the resource group remains in the pseudo-locksetup mode with the * &struct pseudo_lock_region associated with it, but cleared from all * information and ready for the user to re-attempt pseudo-locking by * writing the schemata again. * * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0 * on failure. Descriptive error will be written to last_cmd_status buffer. */ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) { struct pseudo_lock_region *plr = rdtgrp->plr; struct task_struct *thread; unsigned int new_minor; struct device *dev; int ret; ret = pseudo_lock_region_alloc(plr); if (ret < 0) return ret; ret = pseudo_lock_cstates_constrain(plr); if (ret < 0) { ret = -EINVAL; goto out_region; } plr->thread_done = 0; thread = kthread_create_on_node(pseudo_lock_fn, rdtgrp, cpu_to_node(plr->cpu), "pseudo_lock/%u", plr->cpu); if (IS_ERR(thread)) { ret = PTR_ERR(thread); rdt_last_cmd_printf("Locking thread returned error %d\n", ret); goto out_cstates; } kthread_bind(thread, plr->cpu); wake_up_process(thread); ret = wait_event_interruptible(plr->lock_thread_wq, plr->thread_done == 1); if (ret < 0) { /* * If the thread does not get on the CPU for whatever * reason and the process which sets up the region is * interrupted then this will leave the thread in runnable * state and once it gets on the CPU it will derefence * the cleared, but not freed, plr struct resulting in an * empty pseudo-locking loop. */ rdt_last_cmd_puts("Locking thread interrupted\n"); goto out_cstates; } ret = pseudo_lock_minor_get(&new_minor); if (ret < 0) { rdt_last_cmd_puts("Unable to obtain a new minor number\n"); goto out_cstates; } /* * Unlock access but do not release the reference. The * pseudo-locked region will still be here on return. * * The mutex has to be released temporarily to avoid a potential * deadlock with the mm->mmap_lock which is obtained in the * device_create() and debugfs_create_dir() callpath below as well as * before the mmap() callback is called. */ mutex_unlock(&rdtgroup_mutex); if (!IS_ERR_OR_NULL(debugfs_resctrl)) { plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name, debugfs_resctrl); if (!IS_ERR_OR_NULL(plr->debugfs_dir)) debugfs_create_file("pseudo_lock_measure", 0200, plr->debugfs_dir, rdtgrp, &pseudo_measure_fops); } dev = device_create(pseudo_lock_class, NULL, MKDEV(pseudo_lock_major, new_minor), rdtgrp, "%s", rdtgrp->kn->name); mutex_lock(&rdtgroup_mutex); if (IS_ERR(dev)) { ret = PTR_ERR(dev); rdt_last_cmd_printf("Failed to create character device: %d\n", ret); goto out_debugfs; } /* We released the mutex - check if group was removed while we did so */ if (rdtgrp->flags & RDT_DELETED) { ret = -ENODEV; goto out_device; } plr->minor = new_minor; rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED; closid_free(rdtgrp->closid); rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0444); rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0444); ret = 0; goto out; out_device: device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, new_minor)); out_debugfs: debugfs_remove_recursive(plr->debugfs_dir); pseudo_lock_minor_release(new_minor); out_cstates: pseudo_lock_cstates_relax(plr); out_region: pseudo_lock_region_clear(plr); out: return ret; } /** * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region * @rdtgrp: resource group to which the pseudo-locked region belongs * * The removal of a pseudo-locked region can be initiated when the resource * group is removed from user space via a "rmdir" from userspace or the * unmount of the resctrl filesystem. On removal the resource group does * not go back to pseudo-locksetup mode before it is removed, instead it is * removed directly. There is thus assymmetry with the creation where the * &struct pseudo_lock_region is removed here while it was not created in * rdtgroup_pseudo_lock_create(). * * Return: void */ void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) { struct pseudo_lock_region *plr = rdtgrp->plr; if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { /* * Default group cannot be a pseudo-locked region so we can * free closid here. */ closid_free(rdtgrp->closid); goto free; } pseudo_lock_cstates_relax(plr); debugfs_remove_recursive(rdtgrp->plr->debugfs_dir); device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor)); pseudo_lock_minor_release(plr->minor); free: pseudo_lock_free(rdtgrp); } static int pseudo_lock_dev_open(struct inode *inode, struct file *filp) { struct rdtgroup *rdtgrp; mutex_lock(&rdtgroup_mutex); rdtgrp = region_find_by_minor(iminor(inode)); if (!rdtgrp) { mutex_unlock(&rdtgroup_mutex); return -ENODEV; } filp->private_data = rdtgrp; atomic_inc(&rdtgrp->waitcount); /* Perform a non-seekable open - llseek is not supported */ filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); mutex_unlock(&rdtgroup_mutex); return 0; } static int pseudo_lock_dev_release(struct inode *inode, struct file *filp) { struct rdtgroup *rdtgrp; mutex_lock(&rdtgroup_mutex); rdtgrp = filp->private_data; WARN_ON(!rdtgrp); if (!rdtgrp) { mutex_unlock(&rdtgroup_mutex); return -ENODEV; } filp->private_data = NULL; atomic_dec(&rdtgrp->waitcount); mutex_unlock(&rdtgroup_mutex); return 0; } static int pseudo_lock_dev_mremap(struct vm_area_struct *area) { /* Not supported */ return -EINVAL; } static const struct vm_operations_struct pseudo_mmap_ops = { .mremap = pseudo_lock_dev_mremap, }; static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma) { unsigned long vsize = vma->vm_end - vma->vm_start; unsigned long off = vma->vm_pgoff << PAGE_SHIFT; struct pseudo_lock_region *plr; struct rdtgroup *rdtgrp; unsigned long physical; unsigned long psize; mutex_lock(&rdtgroup_mutex); rdtgrp = filp->private_data; WARN_ON(!rdtgrp); if (!rdtgrp) { mutex_unlock(&rdtgroup_mutex); return -ENODEV; } plr = rdtgrp->plr; if (!plr->d) { mutex_unlock(&rdtgroup_mutex); return -ENODEV; } /* * Task is required to run with affinity to the cpus associated * with the pseudo-locked region. If this is not the case the task * may be scheduled elsewhere and invalidate entries in the * pseudo-locked region. */ if (!cpumask_subset(current->cpus_ptr, &plr->d->cpu_mask)) { mutex_unlock(&rdtgroup_mutex); return -EINVAL; } physical = __pa(plr->kmem) >> PAGE_SHIFT; psize = plr->size - off; if (off > plr->size) { mutex_unlock(&rdtgroup_mutex); return -ENOSPC; } /* * Ensure changes are carried directly to the memory being mapped, * do not allow copy-on-write mapping. */ if (!(vma->vm_flags & VM_SHARED)) { mutex_unlock(&rdtgroup_mutex); return -EINVAL; } if (vsize > psize) { mutex_unlock(&rdtgroup_mutex); return -ENOSPC; } memset(plr->kmem + off, 0, vsize); if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff, vsize, vma->vm_page_prot)) { mutex_unlock(&rdtgroup_mutex); return -EAGAIN; } vma->vm_ops = &pseudo_mmap_ops; mutex_unlock(&rdtgroup_mutex); return 0; } static const struct file_operations pseudo_lock_dev_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = NULL, .write = NULL, .open = pseudo_lock_dev_open, .release = pseudo_lock_dev_release, .mmap = pseudo_lock_dev_mmap, }; static char *pseudo_lock_devnode(struct device *dev, umode_t *mode) { struct rdtgroup *rdtgrp; rdtgrp = dev_get_drvdata(dev); if (mode) *mode = 0600; return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name); } int rdt_pseudo_lock_init(void) { int ret; ret = register_chrdev(0, "pseudo_lock", &pseudo_lock_dev_fops); if (ret < 0) return ret; pseudo_lock_major = ret; pseudo_lock_class = class_create(THIS_MODULE, "pseudo_lock"); if (IS_ERR(pseudo_lock_class)) { ret = PTR_ERR(pseudo_lock_class); unregister_chrdev(pseudo_lock_major, "pseudo_lock"); return ret; } pseudo_lock_class->devnode = pseudo_lock_devnode; return 0; } void rdt_pseudo_lock_release(void) { class_destroy(pseudo_lock_class); pseudo_lock_class = NULL; unregister_chrdev(pseudo_lock_major, "pseudo_lock"); pseudo_lock_major = 0; }