1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Resource Director Technology(RDT)
4  * - Cache Allocation code.
5  *
6  * Copyright (C) 2016 Intel Corporation
7  *
8  * Authors:
9  *    Fenghua Yu <fenghua.yu@intel.com>
10  *    Tony Luck <tony.luck@intel.com>
11  *
12  * More information about RDT be found in the Intel (R) x86 Architecture
13  * Software Developer Manual June 2016, volume 3, section 17.17.
14  */
15 
16 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
17 
18 #include <linux/cpu.h>
19 #include <linux/kernfs.h>
20 #include <linux/seq_file.h>
21 #include <linux/slab.h>
22 #include "internal.h"
23 
24 /*
25  * Check whether MBA bandwidth percentage value is correct. The value is
26  * checked against the minimum and max bandwidth values specified by the
27  * hardware. The allocated bandwidth percentage is rounded to the next
28  * control step available on the hardware.
29  */
30 static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r)
31 {
32 	unsigned long bw;
33 	int ret;
34 
35 	/*
36 	 * Only linear delay values is supported for current Intel SKUs.
37 	 */
38 	if (!r->membw.delay_linear && r->membw.arch_needs_linear) {
39 		rdt_last_cmd_puts("No support for non-linear MB domains\n");
40 		return false;
41 	}
42 
43 	ret = kstrtoul(buf, 10, &bw);
44 	if (ret) {
45 		rdt_last_cmd_printf("Non-decimal digit in MB value %s\n", buf);
46 		return false;
47 	}
48 
49 	if ((bw < r->membw.min_bw || bw > r->default_ctrl) &&
50 	    !is_mba_sc(r)) {
51 		rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw,
52 				    r->membw.min_bw, r->default_ctrl);
53 		return false;
54 	}
55 
56 	*data = roundup(bw, (unsigned long)r->membw.bw_gran);
57 	return true;
58 }
59 
60 int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
61 	     struct rdt_domain *d)
62 {
63 	unsigned long bw_val;
64 
65 	if (d->have_new_ctrl) {
66 		rdt_last_cmd_printf("Duplicate domain %d\n", d->id);
67 		return -EINVAL;
68 	}
69 
70 	if (!bw_validate(data->buf, &bw_val, r))
71 		return -EINVAL;
72 	d->new_ctrl = bw_val;
73 	d->have_new_ctrl = true;
74 
75 	return 0;
76 }
77 
78 /*
79  * Check whether a cache bit mask is valid.
80  * For Intel the SDM says:
81  *	Please note that all (and only) contiguous '1' combinations
82  *	are allowed (e.g. FFFFH, 0FF0H, 003CH, etc.).
83  * Additionally Haswell requires at least two bits set.
84  * AMD allows non-contiguous bitmasks.
85  */
86 static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
87 {
88 	unsigned long first_bit, zero_bit, val;
89 	unsigned int cbm_len = r->cache.cbm_len;
90 	int ret;
91 
92 	ret = kstrtoul(buf, 16, &val);
93 	if (ret) {
94 		rdt_last_cmd_printf("Non-hex character in the mask %s\n", buf);
95 		return false;
96 	}
97 
98 	if ((!r->cache.arch_has_empty_bitmaps && val == 0) ||
99 	    val > r->default_ctrl) {
100 		rdt_last_cmd_puts("Mask out of range\n");
101 		return false;
102 	}
103 
104 	first_bit = find_first_bit(&val, cbm_len);
105 	zero_bit = find_next_zero_bit(&val, cbm_len, first_bit);
106 
107 	/* Are non-contiguous bitmaps allowed? */
108 	if (!r->cache.arch_has_sparse_bitmaps &&
109 	    (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) {
110 		rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val);
111 		return false;
112 	}
113 
114 	if ((zero_bit - first_bit) < r->cache.min_cbm_bits) {
115 		rdt_last_cmd_printf("Need at least %d bits in the mask\n",
116 				    r->cache.min_cbm_bits);
117 		return false;
118 	}
119 
120 	*data = val;
121 	return true;
122 }
123 
124 /*
125  * Read one cache bit mask (hex). Check that it is valid for the current
126  * resource type.
127  */
128 int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
129 	      struct rdt_domain *d)
130 {
131 	struct rdtgroup *rdtgrp = data->rdtgrp;
132 	u32 cbm_val;
133 
134 	if (d->have_new_ctrl) {
135 		rdt_last_cmd_printf("Duplicate domain %d\n", d->id);
136 		return -EINVAL;
137 	}
138 
139 	/*
140 	 * Cannot set up more than one pseudo-locked region in a cache
141 	 * hierarchy.
142 	 */
143 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP &&
144 	    rdtgroup_pseudo_locked_in_hierarchy(d)) {
145 		rdt_last_cmd_puts("Pseudo-locked region in hierarchy\n");
146 		return -EINVAL;
147 	}
148 
149 	if (!cbm_validate(data->buf, &cbm_val, r))
150 		return -EINVAL;
151 
152 	if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE ||
153 	     rdtgrp->mode == RDT_MODE_SHAREABLE) &&
154 	    rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) {
155 		rdt_last_cmd_puts("CBM overlaps with pseudo-locked region\n");
156 		return -EINVAL;
157 	}
158 
159 	/*
160 	 * The CBM may not overlap with the CBM of another closid if
161 	 * either is exclusive.
162 	 */
163 	if (rdtgroup_cbm_overlaps(r, d, cbm_val, rdtgrp->closid, true)) {
164 		rdt_last_cmd_puts("Overlaps with exclusive group\n");
165 		return -EINVAL;
166 	}
167 
168 	if (rdtgroup_cbm_overlaps(r, d, cbm_val, rdtgrp->closid, false)) {
169 		if (rdtgrp->mode == RDT_MODE_EXCLUSIVE ||
170 		    rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
171 			rdt_last_cmd_puts("Overlaps with other group\n");
172 			return -EINVAL;
173 		}
174 	}
175 
176 	d->new_ctrl = cbm_val;
177 	d->have_new_ctrl = true;
178 
179 	return 0;
180 }
181 
182 /*
183  * For each domain in this resource we expect to find a series of:
184  *	id=mask
185  * separated by ";". The "id" is in decimal, and must match one of
186  * the "id"s for this resource.
187  */
188 static int parse_line(char *line, struct rdt_resource *r,
189 		      struct rdtgroup *rdtgrp)
190 {
191 	struct rdt_parse_data data;
192 	char *dom = NULL, *id;
193 	struct rdt_domain *d;
194 	unsigned long dom_id;
195 
196 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP &&
197 	    r->rid == RDT_RESOURCE_MBA) {
198 		rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n");
199 		return -EINVAL;
200 	}
201 
202 next:
203 	if (!line || line[0] == '\0')
204 		return 0;
205 	dom = strsep(&line, ";");
206 	id = strsep(&dom, "=");
207 	if (!dom || kstrtoul(id, 10, &dom_id)) {
208 		rdt_last_cmd_puts("Missing '=' or non-numeric domain\n");
209 		return -EINVAL;
210 	}
211 	dom = strim(dom);
212 	list_for_each_entry(d, &r->domains, list) {
213 		if (d->id == dom_id) {
214 			data.buf = dom;
215 			data.rdtgrp = rdtgrp;
216 			if (r->parse_ctrlval(&data, r, d))
217 				return -EINVAL;
218 			if (rdtgrp->mode ==  RDT_MODE_PSEUDO_LOCKSETUP) {
219 				/*
220 				 * In pseudo-locking setup mode and just
221 				 * parsed a valid CBM that should be
222 				 * pseudo-locked. Only one locked region per
223 				 * resource group and domain so just do
224 				 * the required initialization for single
225 				 * region and return.
226 				 */
227 				rdtgrp->plr->r = r;
228 				rdtgrp->plr->d = d;
229 				rdtgrp->plr->cbm = d->new_ctrl;
230 				d->plr = rdtgrp->plr;
231 				return 0;
232 			}
233 			goto next;
234 		}
235 	}
236 	return -EINVAL;
237 }
238 
239 int update_domains(struct rdt_resource *r, int closid)
240 {
241 	struct msr_param msr_param;
242 	cpumask_var_t cpu_mask;
243 	struct rdt_domain *d;
244 	bool mba_sc;
245 	u32 *dc;
246 	int cpu;
247 
248 	if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
249 		return -ENOMEM;
250 
251 	msr_param.low = closid;
252 	msr_param.high = msr_param.low + 1;
253 	msr_param.res = r;
254 
255 	mba_sc = is_mba_sc(r);
256 	list_for_each_entry(d, &r->domains, list) {
257 		dc = !mba_sc ? d->ctrl_val : d->mbps_val;
258 		if (d->have_new_ctrl && d->new_ctrl != dc[closid]) {
259 			cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
260 			dc[closid] = d->new_ctrl;
261 		}
262 	}
263 
264 	/*
265 	 * Avoid writing the control msr with control values when
266 	 * MBA software controller is enabled
267 	 */
268 	if (cpumask_empty(cpu_mask) || mba_sc)
269 		goto done;
270 	cpu = get_cpu();
271 	/* Update resource control msr on this CPU if it's in cpu_mask. */
272 	if (cpumask_test_cpu(cpu, cpu_mask))
273 		rdt_ctrl_update(&msr_param);
274 	/* Update resource control msr on other CPUs. */
275 	smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1);
276 	put_cpu();
277 
278 done:
279 	free_cpumask_var(cpu_mask);
280 
281 	return 0;
282 }
283 
284 static int rdtgroup_parse_resource(char *resname, char *tok,
285 				   struct rdtgroup *rdtgrp)
286 {
287 	struct rdt_resource *r;
288 
289 	for_each_alloc_enabled_rdt_resource(r) {
290 		if (!strcmp(resname, r->name) && rdtgrp->closid < r->num_closid)
291 			return parse_line(tok, r, rdtgrp);
292 	}
293 	rdt_last_cmd_printf("Unknown or unsupported resource name '%s'\n", resname);
294 	return -EINVAL;
295 }
296 
297 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
298 				char *buf, size_t nbytes, loff_t off)
299 {
300 	struct rdtgroup *rdtgrp;
301 	struct rdt_domain *dom;
302 	struct rdt_resource *r;
303 	char *tok, *resname;
304 	int ret = 0;
305 
306 	/* Valid input requires a trailing newline */
307 	if (nbytes == 0 || buf[nbytes - 1] != '\n')
308 		return -EINVAL;
309 	buf[nbytes - 1] = '\0';
310 
311 	cpus_read_lock();
312 	rdtgrp = rdtgroup_kn_lock_live(of->kn);
313 	if (!rdtgrp) {
314 		rdtgroup_kn_unlock(of->kn);
315 		cpus_read_unlock();
316 		return -ENOENT;
317 	}
318 	rdt_last_cmd_clear();
319 
320 	/*
321 	 * No changes to pseudo-locked region allowed. It has to be removed
322 	 * and re-created instead.
323 	 */
324 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
325 		ret = -EINVAL;
326 		rdt_last_cmd_puts("Resource group is pseudo-locked\n");
327 		goto out;
328 	}
329 
330 	for_each_alloc_enabled_rdt_resource(r) {
331 		list_for_each_entry(dom, &r->domains, list)
332 			dom->have_new_ctrl = false;
333 	}
334 
335 	while ((tok = strsep(&buf, "\n")) != NULL) {
336 		resname = strim(strsep(&tok, ":"));
337 		if (!tok) {
338 			rdt_last_cmd_puts("Missing ':'\n");
339 			ret = -EINVAL;
340 			goto out;
341 		}
342 		if (tok[0] == '\0') {
343 			rdt_last_cmd_printf("Missing '%s' value\n", resname);
344 			ret = -EINVAL;
345 			goto out;
346 		}
347 		ret = rdtgroup_parse_resource(resname, tok, rdtgrp);
348 		if (ret)
349 			goto out;
350 	}
351 
352 	for_each_alloc_enabled_rdt_resource(r) {
353 		ret = update_domains(r, rdtgrp->closid);
354 		if (ret)
355 			goto out;
356 	}
357 
358 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
359 		/*
360 		 * If pseudo-locking fails we keep the resource group in
361 		 * mode RDT_MODE_PSEUDO_LOCKSETUP with its class of service
362 		 * active and updated for just the domain the pseudo-locked
363 		 * region was requested for.
364 		 */
365 		ret = rdtgroup_pseudo_lock_create(rdtgrp);
366 	}
367 
368 out:
369 	rdtgroup_kn_unlock(of->kn);
370 	cpus_read_unlock();
371 	return ret ?: nbytes;
372 }
373 
374 static void show_doms(struct seq_file *s, struct rdt_resource *r, int closid)
375 {
376 	struct rdt_domain *dom;
377 	bool sep = false;
378 	u32 ctrl_val;
379 
380 	seq_printf(s, "%*s:", max_name_width, r->name);
381 	list_for_each_entry(dom, &r->domains, list) {
382 		if (sep)
383 			seq_puts(s, ";");
384 
385 		ctrl_val = (!is_mba_sc(r) ? dom->ctrl_val[closid] :
386 			    dom->mbps_val[closid]);
387 		seq_printf(s, r->format_str, dom->id, max_data_width,
388 			   ctrl_val);
389 		sep = true;
390 	}
391 	seq_puts(s, "\n");
392 }
393 
394 int rdtgroup_schemata_show(struct kernfs_open_file *of,
395 			   struct seq_file *s, void *v)
396 {
397 	struct rdtgroup *rdtgrp;
398 	struct rdt_resource *r;
399 	int ret = 0;
400 	u32 closid;
401 
402 	rdtgrp = rdtgroup_kn_lock_live(of->kn);
403 	if (rdtgrp) {
404 		if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
405 			for_each_alloc_enabled_rdt_resource(r)
406 				seq_printf(s, "%s:uninitialized\n", r->name);
407 		} else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
408 			if (!rdtgrp->plr->d) {
409 				rdt_last_cmd_clear();
410 				rdt_last_cmd_puts("Cache domain offline\n");
411 				ret = -ENODEV;
412 			} else {
413 				seq_printf(s, "%s:%d=%x\n",
414 					   rdtgrp->plr->r->name,
415 					   rdtgrp->plr->d->id,
416 					   rdtgrp->plr->cbm);
417 			}
418 		} else {
419 			closid = rdtgrp->closid;
420 			for_each_alloc_enabled_rdt_resource(r) {
421 				if (closid < r->num_closid)
422 					show_doms(s, r, closid);
423 			}
424 		}
425 	} else {
426 		ret = -ENOENT;
427 	}
428 	rdtgroup_kn_unlock(of->kn);
429 	return ret;
430 }
431 
432 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
433 		    struct rdt_domain *d, struct rdtgroup *rdtgrp,
434 		    int evtid, int first)
435 {
436 	/*
437 	 * setup the parameters to send to the IPI to read the data.
438 	 */
439 	rr->rgrp = rdtgrp;
440 	rr->evtid = evtid;
441 	rr->r = r;
442 	rr->d = d;
443 	rr->val = 0;
444 	rr->first = first;
445 
446 	smp_call_function_any(&d->cpu_mask, mon_event_count, rr, 1);
447 }
448 
449 int rdtgroup_mondata_show(struct seq_file *m, void *arg)
450 {
451 	struct kernfs_open_file *of = m->private;
452 	u32 resid, evtid, domid;
453 	struct rdtgroup *rdtgrp;
454 	struct rdt_resource *r;
455 	union mon_data_bits md;
456 	struct rdt_domain *d;
457 	struct rmid_read rr;
458 	int ret = 0;
459 
460 	rdtgrp = rdtgroup_kn_lock_live(of->kn);
461 	if (!rdtgrp) {
462 		ret = -ENOENT;
463 		goto out;
464 	}
465 
466 	md.priv = of->kn->priv;
467 	resid = md.u.rid;
468 	domid = md.u.domid;
469 	evtid = md.u.evtid;
470 
471 	r = &rdt_resources_all[resid];
472 	d = rdt_find_domain(r, domid, NULL);
473 	if (IS_ERR_OR_NULL(d)) {
474 		ret = -ENOENT;
475 		goto out;
476 	}
477 
478 	mon_event_read(&rr, r, d, rdtgrp, evtid, false);
479 
480 	if (rr.val & RMID_VAL_ERROR)
481 		seq_puts(m, "Error\n");
482 	else if (rr.val & RMID_VAL_UNAVAIL)
483 		seq_puts(m, "Unavailable\n");
484 	else
485 		seq_printf(m, "%llu\n", rr.val * r->mon_scale);
486 
487 out:
488 	rdtgroup_kn_unlock(of->kn);
489 	return ret;
490 }
491