1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Resource Director Technology(RDT) 4 * - Cache Allocation code. 5 * 6 * Copyright (C) 2016 Intel Corporation 7 * 8 * Authors: 9 * Fenghua Yu <fenghua.yu@intel.com> 10 * Tony Luck <tony.luck@intel.com> 11 * 12 * More information about RDT be found in the Intel (R) x86 Architecture 13 * Software Developer Manual June 2016, volume 3, section 17.17. 14 */ 15 16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 17 18 #include <linux/cpu.h> 19 #include <linux/kernfs.h> 20 #include <linux/seq_file.h> 21 #include <linux/slab.h> 22 #include "internal.h" 23 24 /* 25 * Check whether MBA bandwidth percentage value is correct. The value is 26 * checked against the minimum and max bandwidth values specified by the 27 * hardware. The allocated bandwidth percentage is rounded to the next 28 * control step available on the hardware. 29 */ 30 static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r) 31 { 32 unsigned long bw; 33 int ret; 34 35 /* 36 * Only linear delay values is supported for current Intel SKUs. 37 */ 38 if (!r->membw.delay_linear && r->membw.arch_needs_linear) { 39 rdt_last_cmd_puts("No support for non-linear MB domains\n"); 40 return false; 41 } 42 43 ret = kstrtoul(buf, 10, &bw); 44 if (ret) { 45 rdt_last_cmd_printf("Non-decimal digit in MB value %s\n", buf); 46 return false; 47 } 48 49 if ((bw < r->membw.min_bw || bw > r->default_ctrl) && 50 !is_mba_sc(r)) { 51 rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw, 52 r->membw.min_bw, r->default_ctrl); 53 return false; 54 } 55 56 *data = roundup(bw, (unsigned long)r->membw.bw_gran); 57 return true; 58 } 59 60 int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, 61 struct rdt_domain *d) 62 { 63 struct resctrl_staged_config *cfg; 64 u32 closid = data->rdtgrp->closid; 65 struct rdt_resource *r = s->res; 66 unsigned long bw_val; 67 68 cfg = &d->staged_config[s->conf_type]; 69 if (cfg->have_new_ctrl) { 70 rdt_last_cmd_printf("Duplicate domain %d\n", d->id); 71 return -EINVAL; 72 } 73 74 if (!bw_validate(data->buf, &bw_val, r)) 75 return -EINVAL; 76 77 if (is_mba_sc(r)) { 78 d->mbps_val[closid] = bw_val; 79 return 0; 80 } 81 82 cfg->new_ctrl = bw_val; 83 cfg->have_new_ctrl = true; 84 85 return 0; 86 } 87 88 /* 89 * Check whether a cache bit mask is valid. 90 * For Intel the SDM says: 91 * Please note that all (and only) contiguous '1' combinations 92 * are allowed (e.g. FFFFH, 0FF0H, 003CH, etc.). 93 * Additionally Haswell requires at least two bits set. 94 * AMD allows non-contiguous bitmasks. 95 */ 96 static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) 97 { 98 unsigned long first_bit, zero_bit, val; 99 unsigned int cbm_len = r->cache.cbm_len; 100 int ret; 101 102 ret = kstrtoul(buf, 16, &val); 103 if (ret) { 104 rdt_last_cmd_printf("Non-hex character in the mask %s\n", buf); 105 return false; 106 } 107 108 if ((!r->cache.arch_has_empty_bitmaps && val == 0) || 109 val > r->default_ctrl) { 110 rdt_last_cmd_puts("Mask out of range\n"); 111 return false; 112 } 113 114 first_bit = find_first_bit(&val, cbm_len); 115 zero_bit = find_next_zero_bit(&val, cbm_len, first_bit); 116 117 /* Are non-contiguous bitmaps allowed? */ 118 if (!r->cache.arch_has_sparse_bitmaps && 119 (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) { 120 rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val); 121 return false; 122 } 123 124 if ((zero_bit - first_bit) < r->cache.min_cbm_bits) { 125 rdt_last_cmd_printf("Need at least %d bits in the mask\n", 126 r->cache.min_cbm_bits); 127 return false; 128 } 129 130 *data = val; 131 return true; 132 } 133 134 /* 135 * Read one cache bit mask (hex). Check that it is valid for the current 136 * resource type. 137 */ 138 int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, 139 struct rdt_domain *d) 140 { 141 struct rdtgroup *rdtgrp = data->rdtgrp; 142 struct resctrl_staged_config *cfg; 143 struct rdt_resource *r = s->res; 144 u32 cbm_val; 145 146 cfg = &d->staged_config[s->conf_type]; 147 if (cfg->have_new_ctrl) { 148 rdt_last_cmd_printf("Duplicate domain %d\n", d->id); 149 return -EINVAL; 150 } 151 152 /* 153 * Cannot set up more than one pseudo-locked region in a cache 154 * hierarchy. 155 */ 156 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && 157 rdtgroup_pseudo_locked_in_hierarchy(d)) { 158 rdt_last_cmd_puts("Pseudo-locked region in hierarchy\n"); 159 return -EINVAL; 160 } 161 162 if (!cbm_validate(data->buf, &cbm_val, r)) 163 return -EINVAL; 164 165 if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE || 166 rdtgrp->mode == RDT_MODE_SHAREABLE) && 167 rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) { 168 rdt_last_cmd_puts("CBM overlaps with pseudo-locked region\n"); 169 return -EINVAL; 170 } 171 172 /* 173 * The CBM may not overlap with the CBM of another closid if 174 * either is exclusive. 175 */ 176 if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, true)) { 177 rdt_last_cmd_puts("Overlaps with exclusive group\n"); 178 return -EINVAL; 179 } 180 181 if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, false)) { 182 if (rdtgrp->mode == RDT_MODE_EXCLUSIVE || 183 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { 184 rdt_last_cmd_puts("Overlaps with other group\n"); 185 return -EINVAL; 186 } 187 } 188 189 cfg->new_ctrl = cbm_val; 190 cfg->have_new_ctrl = true; 191 192 return 0; 193 } 194 195 /* 196 * For each domain in this resource we expect to find a series of: 197 * id=mask 198 * separated by ";". The "id" is in decimal, and must match one of 199 * the "id"s for this resource. 200 */ 201 static int parse_line(char *line, struct resctrl_schema *s, 202 struct rdtgroup *rdtgrp) 203 { 204 enum resctrl_conf_type t = s->conf_type; 205 struct resctrl_staged_config *cfg; 206 struct rdt_resource *r = s->res; 207 struct rdt_parse_data data; 208 char *dom = NULL, *id; 209 struct rdt_domain *d; 210 unsigned long dom_id; 211 212 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && 213 r->rid == RDT_RESOURCE_MBA) { 214 rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n"); 215 return -EINVAL; 216 } 217 218 next: 219 if (!line || line[0] == '\0') 220 return 0; 221 dom = strsep(&line, ";"); 222 id = strsep(&dom, "="); 223 if (!dom || kstrtoul(id, 10, &dom_id)) { 224 rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); 225 return -EINVAL; 226 } 227 dom = strim(dom); 228 list_for_each_entry(d, &r->domains, list) { 229 if (d->id == dom_id) { 230 data.buf = dom; 231 data.rdtgrp = rdtgrp; 232 if (r->parse_ctrlval(&data, s, d)) 233 return -EINVAL; 234 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { 235 cfg = &d->staged_config[t]; 236 /* 237 * In pseudo-locking setup mode and just 238 * parsed a valid CBM that should be 239 * pseudo-locked. Only one locked region per 240 * resource group and domain so just do 241 * the required initialization for single 242 * region and return. 243 */ 244 rdtgrp->plr->s = s; 245 rdtgrp->plr->d = d; 246 rdtgrp->plr->cbm = cfg->new_ctrl; 247 d->plr = rdtgrp->plr; 248 return 0; 249 } 250 goto next; 251 } 252 } 253 return -EINVAL; 254 } 255 256 static u32 get_config_index(u32 closid, enum resctrl_conf_type type) 257 { 258 switch (type) { 259 default: 260 case CDP_NONE: 261 return closid; 262 case CDP_CODE: 263 return closid * 2 + 1; 264 case CDP_DATA: 265 return closid * 2; 266 } 267 } 268 269 static bool apply_config(struct rdt_hw_domain *hw_dom, 270 struct resctrl_staged_config *cfg, u32 idx, 271 cpumask_var_t cpu_mask) 272 { 273 struct rdt_domain *dom = &hw_dom->d_resctrl; 274 275 if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) { 276 cpumask_set_cpu(cpumask_any(&dom->cpu_mask), cpu_mask); 277 hw_dom->ctrl_val[idx] = cfg->new_ctrl; 278 279 return true; 280 } 281 282 return false; 283 } 284 285 int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, 286 u32 closid, enum resctrl_conf_type t, u32 cfg_val) 287 { 288 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 289 struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); 290 u32 idx = get_config_index(closid, t); 291 struct msr_param msr_param; 292 293 if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) 294 return -EINVAL; 295 296 hw_dom->ctrl_val[idx] = cfg_val; 297 298 msr_param.res = r; 299 msr_param.low = idx; 300 msr_param.high = idx + 1; 301 hw_res->msr_update(d, &msr_param, r); 302 303 return 0; 304 } 305 306 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) 307 { 308 struct resctrl_staged_config *cfg; 309 struct rdt_hw_domain *hw_dom; 310 struct msr_param msr_param; 311 enum resctrl_conf_type t; 312 cpumask_var_t cpu_mask; 313 struct rdt_domain *d; 314 int cpu; 315 u32 idx; 316 317 if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) 318 return -ENOMEM; 319 320 msr_param.res = NULL; 321 list_for_each_entry(d, &r->domains, list) { 322 hw_dom = resctrl_to_arch_dom(d); 323 for (t = 0; t < CDP_NUM_TYPES; t++) { 324 cfg = &hw_dom->d_resctrl.staged_config[t]; 325 if (!cfg->have_new_ctrl) 326 continue; 327 328 idx = get_config_index(closid, t); 329 if (!apply_config(hw_dom, cfg, idx, cpu_mask)) 330 continue; 331 332 if (!msr_param.res) { 333 msr_param.low = idx; 334 msr_param.high = msr_param.low + 1; 335 msr_param.res = r; 336 } else { 337 msr_param.low = min(msr_param.low, idx); 338 msr_param.high = max(msr_param.high, idx + 1); 339 } 340 } 341 } 342 343 if (cpumask_empty(cpu_mask)) 344 goto done; 345 cpu = get_cpu(); 346 /* Update resource control msr on this CPU if it's in cpu_mask. */ 347 if (cpumask_test_cpu(cpu, cpu_mask)) 348 rdt_ctrl_update(&msr_param); 349 /* Update resource control msr on other CPUs. */ 350 smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1); 351 put_cpu(); 352 353 done: 354 free_cpumask_var(cpu_mask); 355 356 return 0; 357 } 358 359 static int rdtgroup_parse_resource(char *resname, char *tok, 360 struct rdtgroup *rdtgrp) 361 { 362 struct resctrl_schema *s; 363 364 list_for_each_entry(s, &resctrl_schema_all, list) { 365 if (!strcmp(resname, s->name) && rdtgrp->closid < s->num_closid) 366 return parse_line(tok, s, rdtgrp); 367 } 368 rdt_last_cmd_printf("Unknown or unsupported resource name '%s'\n", resname); 369 return -EINVAL; 370 } 371 372 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, 373 char *buf, size_t nbytes, loff_t off) 374 { 375 struct resctrl_schema *s; 376 struct rdtgroup *rdtgrp; 377 struct rdt_domain *dom; 378 struct rdt_resource *r; 379 char *tok, *resname; 380 int ret = 0; 381 382 /* Valid input requires a trailing newline */ 383 if (nbytes == 0 || buf[nbytes - 1] != '\n') 384 return -EINVAL; 385 buf[nbytes - 1] = '\0'; 386 387 cpus_read_lock(); 388 rdtgrp = rdtgroup_kn_lock_live(of->kn); 389 if (!rdtgrp) { 390 rdtgroup_kn_unlock(of->kn); 391 cpus_read_unlock(); 392 return -ENOENT; 393 } 394 rdt_last_cmd_clear(); 395 396 /* 397 * No changes to pseudo-locked region allowed. It has to be removed 398 * and re-created instead. 399 */ 400 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { 401 ret = -EINVAL; 402 rdt_last_cmd_puts("Resource group is pseudo-locked\n"); 403 goto out; 404 } 405 406 list_for_each_entry(s, &resctrl_schema_all, list) { 407 list_for_each_entry(dom, &s->res->domains, list) 408 memset(dom->staged_config, 0, sizeof(dom->staged_config)); 409 } 410 411 while ((tok = strsep(&buf, "\n")) != NULL) { 412 resname = strim(strsep(&tok, ":")); 413 if (!tok) { 414 rdt_last_cmd_puts("Missing ':'\n"); 415 ret = -EINVAL; 416 goto out; 417 } 418 if (tok[0] == '\0') { 419 rdt_last_cmd_printf("Missing '%s' value\n", resname); 420 ret = -EINVAL; 421 goto out; 422 } 423 ret = rdtgroup_parse_resource(resname, tok, rdtgrp); 424 if (ret) 425 goto out; 426 } 427 428 list_for_each_entry(s, &resctrl_schema_all, list) { 429 r = s->res; 430 431 /* 432 * Writes to mba_sc resources update the software controller, 433 * not the control MSR. 434 */ 435 if (is_mba_sc(r)) 436 continue; 437 438 ret = resctrl_arch_update_domains(r, rdtgrp->closid); 439 if (ret) 440 goto out; 441 } 442 443 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { 444 /* 445 * If pseudo-locking fails we keep the resource group in 446 * mode RDT_MODE_PSEUDO_LOCKSETUP with its class of service 447 * active and updated for just the domain the pseudo-locked 448 * region was requested for. 449 */ 450 ret = rdtgroup_pseudo_lock_create(rdtgrp); 451 } 452 453 out: 454 rdtgroup_kn_unlock(of->kn); 455 cpus_read_unlock(); 456 return ret ?: nbytes; 457 } 458 459 u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, 460 u32 closid, enum resctrl_conf_type type) 461 { 462 struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); 463 u32 idx = get_config_index(closid, type); 464 465 return hw_dom->ctrl_val[idx]; 466 } 467 468 static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int closid) 469 { 470 struct rdt_resource *r = schema->res; 471 struct rdt_domain *dom; 472 bool sep = false; 473 u32 ctrl_val; 474 475 seq_printf(s, "%*s:", max_name_width, schema->name); 476 list_for_each_entry(dom, &r->domains, list) { 477 if (sep) 478 seq_puts(s, ";"); 479 480 if (is_mba_sc(r)) 481 ctrl_val = dom->mbps_val[closid]; 482 else 483 ctrl_val = resctrl_arch_get_config(r, dom, closid, 484 schema->conf_type); 485 486 seq_printf(s, r->format_str, dom->id, max_data_width, 487 ctrl_val); 488 sep = true; 489 } 490 seq_puts(s, "\n"); 491 } 492 493 int rdtgroup_schemata_show(struct kernfs_open_file *of, 494 struct seq_file *s, void *v) 495 { 496 struct resctrl_schema *schema; 497 struct rdtgroup *rdtgrp; 498 int ret = 0; 499 u32 closid; 500 501 rdtgrp = rdtgroup_kn_lock_live(of->kn); 502 if (rdtgrp) { 503 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { 504 list_for_each_entry(schema, &resctrl_schema_all, list) { 505 seq_printf(s, "%s:uninitialized\n", schema->name); 506 } 507 } else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { 508 if (!rdtgrp->plr->d) { 509 rdt_last_cmd_clear(); 510 rdt_last_cmd_puts("Cache domain offline\n"); 511 ret = -ENODEV; 512 } else { 513 seq_printf(s, "%s:%d=%x\n", 514 rdtgrp->plr->s->res->name, 515 rdtgrp->plr->d->id, 516 rdtgrp->plr->cbm); 517 } 518 } else { 519 closid = rdtgrp->closid; 520 list_for_each_entry(schema, &resctrl_schema_all, list) { 521 if (closid < schema->num_closid) 522 show_doms(s, schema, closid); 523 } 524 } 525 } else { 526 ret = -ENOENT; 527 } 528 rdtgroup_kn_unlock(of->kn); 529 return ret; 530 } 531 532 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, 533 struct rdt_domain *d, struct rdtgroup *rdtgrp, 534 int evtid, int first) 535 { 536 /* 537 * setup the parameters to send to the IPI to read the data. 538 */ 539 rr->rgrp = rdtgrp; 540 rr->evtid = evtid; 541 rr->r = r; 542 rr->d = d; 543 rr->val = 0; 544 rr->first = first; 545 546 smp_call_function_any(&d->cpu_mask, mon_event_count, rr, 1); 547 } 548 549 int rdtgroup_mondata_show(struct seq_file *m, void *arg) 550 { 551 struct kernfs_open_file *of = m->private; 552 u32 resid, evtid, domid; 553 struct rdtgroup *rdtgrp; 554 struct rdt_resource *r; 555 union mon_data_bits md; 556 struct rdt_domain *d; 557 struct rmid_read rr; 558 int ret = 0; 559 560 rdtgrp = rdtgroup_kn_lock_live(of->kn); 561 if (!rdtgrp) { 562 ret = -ENOENT; 563 goto out; 564 } 565 566 md.priv = of->kn->priv; 567 resid = md.u.rid; 568 domid = md.u.domid; 569 evtid = md.u.evtid; 570 571 r = &rdt_resources_all[resid].r_resctrl; 572 d = rdt_find_domain(r, domid, NULL); 573 if (IS_ERR_OR_NULL(d)) { 574 ret = -ENOENT; 575 goto out; 576 } 577 578 mon_event_read(&rr, r, d, rdtgrp, evtid, false); 579 580 if (rr.err == -EIO) 581 seq_puts(m, "Error\n"); 582 else if (rr.err == -EINVAL) 583 seq_puts(m, "Unavailable\n"); 584 else 585 seq_printf(m, "%llu\n", rr.val); 586 587 out: 588 rdtgroup_kn_unlock(of->kn); 589 return ret; 590 } 591