1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/io-64-nonatomic-lo-hi.h> 9 #include <uapi/linux/idxd.h> 10 #include "registers.h" 11 #include "idxd.h" 12 13 static char *idxd_wq_type_names[] = { 14 [IDXD_WQT_NONE] = "none", 15 [IDXD_WQT_KERNEL] = "kernel", 16 [IDXD_WQT_USER] = "user", 17 }; 18 19 /* IDXD engine attributes */ 20 static ssize_t engine_group_id_show(struct device *dev, 21 struct device_attribute *attr, char *buf) 22 { 23 struct idxd_engine *engine = confdev_to_engine(dev); 24 25 if (engine->group) 26 return sysfs_emit(buf, "%d\n", engine->group->id); 27 else 28 return sysfs_emit(buf, "%d\n", -1); 29 } 30 31 static ssize_t engine_group_id_store(struct device *dev, 32 struct device_attribute *attr, 33 const char *buf, size_t count) 34 { 35 struct idxd_engine *engine = confdev_to_engine(dev); 36 struct idxd_device *idxd = engine->idxd; 37 long id; 38 int rc; 39 struct idxd_group *prevg; 40 41 rc = kstrtol(buf, 10, &id); 42 if (rc < 0) 43 return -EINVAL; 44 45 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 46 return -EPERM; 47 48 if (id > idxd->max_groups - 1 || id < -1) 49 return -EINVAL; 50 51 if (id == -1) { 52 if (engine->group) { 53 engine->group->num_engines--; 54 engine->group = NULL; 55 } 56 return count; 57 } 58 59 prevg = engine->group; 60 61 if (prevg) 62 prevg->num_engines--; 63 engine->group = idxd->groups[id]; 64 engine->group->num_engines++; 65 66 return count; 67 } 68 69 static struct device_attribute dev_attr_engine_group = 70 __ATTR(group_id, 0644, engine_group_id_show, 71 engine_group_id_store); 72 73 static struct attribute *idxd_engine_attributes[] = { 74 &dev_attr_engine_group.attr, 75 NULL, 76 }; 77 78 static const struct attribute_group idxd_engine_attribute_group = { 79 .attrs = idxd_engine_attributes, 80 }; 81 82 static const struct attribute_group *idxd_engine_attribute_groups[] = { 83 &idxd_engine_attribute_group, 84 NULL, 85 }; 86 87 static void idxd_conf_engine_release(struct device *dev) 88 { 89 struct idxd_engine *engine = confdev_to_engine(dev); 90 91 kfree(engine); 92 } 93 94 struct device_type idxd_engine_device_type = { 95 .name = "engine", 96 .release = idxd_conf_engine_release, 97 .groups = idxd_engine_attribute_groups, 98 }; 99 100 /* Group attributes */ 101 102 static void idxd_set_free_rdbufs(struct idxd_device *idxd) 103 { 104 int i, rdbufs; 105 106 for (i = 0, rdbufs = 0; i < idxd->max_groups; i++) { 107 struct idxd_group *g = idxd->groups[i]; 108 109 rdbufs += g->rdbufs_reserved; 110 } 111 112 idxd->nr_rdbufs = idxd->max_rdbufs - rdbufs; 113 } 114 115 static ssize_t group_read_buffers_reserved_show(struct device *dev, 116 struct device_attribute *attr, 117 char *buf) 118 { 119 struct idxd_group *group = confdev_to_group(dev); 120 121 return sysfs_emit(buf, "%u\n", group->rdbufs_reserved); 122 } 123 124 static ssize_t group_tokens_reserved_show(struct device *dev, 125 struct device_attribute *attr, 126 char *buf) 127 { 128 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n"); 129 return group_read_buffers_reserved_show(dev, attr, buf); 130 } 131 132 static ssize_t group_read_buffers_reserved_store(struct device *dev, 133 struct device_attribute *attr, 134 const char *buf, size_t count) 135 { 136 struct idxd_group *group = confdev_to_group(dev); 137 struct idxd_device *idxd = group->idxd; 138 unsigned long val; 139 int rc; 140 141 rc = kstrtoul(buf, 10, &val); 142 if (rc < 0) 143 return -EINVAL; 144 145 if (idxd->data->type == IDXD_TYPE_IAX) 146 return -EOPNOTSUPP; 147 148 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 149 return -EPERM; 150 151 if (idxd->state == IDXD_DEV_ENABLED) 152 return -EPERM; 153 154 if (val > idxd->max_rdbufs) 155 return -EINVAL; 156 157 if (val > idxd->nr_rdbufs + group->rdbufs_reserved) 158 return -EINVAL; 159 160 group->rdbufs_reserved = val; 161 idxd_set_free_rdbufs(idxd); 162 return count; 163 } 164 165 static ssize_t group_tokens_reserved_store(struct device *dev, 166 struct device_attribute *attr, 167 const char *buf, size_t count) 168 { 169 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n"); 170 return group_read_buffers_reserved_store(dev, attr, buf, count); 171 } 172 173 static struct device_attribute dev_attr_group_tokens_reserved = 174 __ATTR(tokens_reserved, 0644, group_tokens_reserved_show, 175 group_tokens_reserved_store); 176 177 static struct device_attribute dev_attr_group_read_buffers_reserved = 178 __ATTR(read_buffers_reserved, 0644, group_read_buffers_reserved_show, 179 group_read_buffers_reserved_store); 180 181 static ssize_t group_read_buffers_allowed_show(struct device *dev, 182 struct device_attribute *attr, 183 char *buf) 184 { 185 struct idxd_group *group = confdev_to_group(dev); 186 187 return sysfs_emit(buf, "%u\n", group->rdbufs_allowed); 188 } 189 190 static ssize_t group_tokens_allowed_show(struct device *dev, 191 struct device_attribute *attr, 192 char *buf) 193 { 194 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n"); 195 return group_read_buffers_allowed_show(dev, attr, buf); 196 } 197 198 static ssize_t group_read_buffers_allowed_store(struct device *dev, 199 struct device_attribute *attr, 200 const char *buf, size_t count) 201 { 202 struct idxd_group *group = confdev_to_group(dev); 203 struct idxd_device *idxd = group->idxd; 204 unsigned long val; 205 int rc; 206 207 rc = kstrtoul(buf, 10, &val); 208 if (rc < 0) 209 return -EINVAL; 210 211 if (idxd->data->type == IDXD_TYPE_IAX) 212 return -EOPNOTSUPP; 213 214 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 215 return -EPERM; 216 217 if (idxd->state == IDXD_DEV_ENABLED) 218 return -EPERM; 219 220 if (val < 4 * group->num_engines || 221 val > group->rdbufs_reserved + idxd->nr_rdbufs) 222 return -EINVAL; 223 224 group->rdbufs_allowed = val; 225 return count; 226 } 227 228 static ssize_t group_tokens_allowed_store(struct device *dev, 229 struct device_attribute *attr, 230 const char *buf, size_t count) 231 { 232 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n"); 233 return group_read_buffers_allowed_store(dev, attr, buf, count); 234 } 235 236 static struct device_attribute dev_attr_group_tokens_allowed = 237 __ATTR(tokens_allowed, 0644, group_tokens_allowed_show, 238 group_tokens_allowed_store); 239 240 static struct device_attribute dev_attr_group_read_buffers_allowed = 241 __ATTR(read_buffers_allowed, 0644, group_read_buffers_allowed_show, 242 group_read_buffers_allowed_store); 243 244 static ssize_t group_use_read_buffer_limit_show(struct device *dev, 245 struct device_attribute *attr, 246 char *buf) 247 { 248 struct idxd_group *group = confdev_to_group(dev); 249 250 return sysfs_emit(buf, "%u\n", group->use_rdbuf_limit); 251 } 252 253 static ssize_t group_use_token_limit_show(struct device *dev, 254 struct device_attribute *attr, 255 char *buf) 256 { 257 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n"); 258 return group_use_read_buffer_limit_show(dev, attr, buf); 259 } 260 261 static ssize_t group_use_read_buffer_limit_store(struct device *dev, 262 struct device_attribute *attr, 263 const char *buf, size_t count) 264 { 265 struct idxd_group *group = confdev_to_group(dev); 266 struct idxd_device *idxd = group->idxd; 267 unsigned long val; 268 int rc; 269 270 rc = kstrtoul(buf, 10, &val); 271 if (rc < 0) 272 return -EINVAL; 273 274 if (idxd->data->type == IDXD_TYPE_IAX) 275 return -EOPNOTSUPP; 276 277 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 278 return -EPERM; 279 280 if (idxd->state == IDXD_DEV_ENABLED) 281 return -EPERM; 282 283 if (idxd->rdbuf_limit == 0) 284 return -EPERM; 285 286 group->use_rdbuf_limit = !!val; 287 return count; 288 } 289 290 static ssize_t group_use_token_limit_store(struct device *dev, 291 struct device_attribute *attr, 292 const char *buf, size_t count) 293 { 294 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n"); 295 return group_use_read_buffer_limit_store(dev, attr, buf, count); 296 } 297 298 static struct device_attribute dev_attr_group_use_token_limit = 299 __ATTR(use_token_limit, 0644, group_use_token_limit_show, 300 group_use_token_limit_store); 301 302 static struct device_attribute dev_attr_group_use_read_buffer_limit = 303 __ATTR(use_read_buffer_limit, 0644, group_use_read_buffer_limit_show, 304 group_use_read_buffer_limit_store); 305 306 static ssize_t group_engines_show(struct device *dev, 307 struct device_attribute *attr, char *buf) 308 { 309 struct idxd_group *group = confdev_to_group(dev); 310 int i, rc = 0; 311 struct idxd_device *idxd = group->idxd; 312 313 for (i = 0; i < idxd->max_engines; i++) { 314 struct idxd_engine *engine = idxd->engines[i]; 315 316 if (!engine->group) 317 continue; 318 319 if (engine->group->id == group->id) 320 rc += sysfs_emit_at(buf, rc, "engine%d.%d ", idxd->id, engine->id); 321 } 322 323 if (!rc) 324 return 0; 325 rc--; 326 rc += sysfs_emit_at(buf, rc, "\n"); 327 328 return rc; 329 } 330 331 static struct device_attribute dev_attr_group_engines = 332 __ATTR(engines, 0444, group_engines_show, NULL); 333 334 static ssize_t group_work_queues_show(struct device *dev, 335 struct device_attribute *attr, char *buf) 336 { 337 struct idxd_group *group = confdev_to_group(dev); 338 int i, rc = 0; 339 struct idxd_device *idxd = group->idxd; 340 341 for (i = 0; i < idxd->max_wqs; i++) { 342 struct idxd_wq *wq = idxd->wqs[i]; 343 344 if (!wq->group) 345 continue; 346 347 if (wq->group->id == group->id) 348 rc += sysfs_emit_at(buf, rc, "wq%d.%d ", idxd->id, wq->id); 349 } 350 351 if (!rc) 352 return 0; 353 rc--; 354 rc += sysfs_emit_at(buf, rc, "\n"); 355 356 return rc; 357 } 358 359 static struct device_attribute dev_attr_group_work_queues = 360 __ATTR(work_queues, 0444, group_work_queues_show, NULL); 361 362 static ssize_t group_traffic_class_a_show(struct device *dev, 363 struct device_attribute *attr, 364 char *buf) 365 { 366 struct idxd_group *group = confdev_to_group(dev); 367 368 return sysfs_emit(buf, "%d\n", group->tc_a); 369 } 370 371 static ssize_t group_traffic_class_a_store(struct device *dev, 372 struct device_attribute *attr, 373 const char *buf, size_t count) 374 { 375 struct idxd_group *group = confdev_to_group(dev); 376 struct idxd_device *idxd = group->idxd; 377 long val; 378 int rc; 379 380 rc = kstrtol(buf, 10, &val); 381 if (rc < 0) 382 return -EINVAL; 383 384 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 385 return -EPERM; 386 387 if (idxd->state == IDXD_DEV_ENABLED) 388 return -EPERM; 389 390 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) 391 return -EPERM; 392 393 if (val < 0 || val > 7) 394 return -EINVAL; 395 396 group->tc_a = val; 397 return count; 398 } 399 400 static struct device_attribute dev_attr_group_traffic_class_a = 401 __ATTR(traffic_class_a, 0644, group_traffic_class_a_show, 402 group_traffic_class_a_store); 403 404 static ssize_t group_traffic_class_b_show(struct device *dev, 405 struct device_attribute *attr, 406 char *buf) 407 { 408 struct idxd_group *group = confdev_to_group(dev); 409 410 return sysfs_emit(buf, "%d\n", group->tc_b); 411 } 412 413 static ssize_t group_traffic_class_b_store(struct device *dev, 414 struct device_attribute *attr, 415 const char *buf, size_t count) 416 { 417 struct idxd_group *group = confdev_to_group(dev); 418 struct idxd_device *idxd = group->idxd; 419 long val; 420 int rc; 421 422 rc = kstrtol(buf, 10, &val); 423 if (rc < 0) 424 return -EINVAL; 425 426 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 427 return -EPERM; 428 429 if (idxd->state == IDXD_DEV_ENABLED) 430 return -EPERM; 431 432 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) 433 return -EPERM; 434 435 if (val < 0 || val > 7) 436 return -EINVAL; 437 438 group->tc_b = val; 439 return count; 440 } 441 442 static struct device_attribute dev_attr_group_traffic_class_b = 443 __ATTR(traffic_class_b, 0644, group_traffic_class_b_show, 444 group_traffic_class_b_store); 445 446 static ssize_t group_desc_progress_limit_show(struct device *dev, 447 struct device_attribute *attr, 448 char *buf) 449 { 450 struct idxd_group *group = confdev_to_group(dev); 451 452 return sysfs_emit(buf, "%d\n", group->desc_progress_limit); 453 } 454 455 static ssize_t group_desc_progress_limit_store(struct device *dev, 456 struct device_attribute *attr, 457 const char *buf, size_t count) 458 { 459 struct idxd_group *group = confdev_to_group(dev); 460 int val, rc; 461 462 rc = kstrtoint(buf, 10, &val); 463 if (rc < 0) 464 return -EINVAL; 465 466 if (val & ~GENMASK(1, 0)) 467 return -EINVAL; 468 469 group->desc_progress_limit = val; 470 return count; 471 } 472 473 static struct device_attribute dev_attr_group_desc_progress_limit = 474 __ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show, 475 group_desc_progress_limit_store); 476 477 static ssize_t group_batch_progress_limit_show(struct device *dev, 478 struct device_attribute *attr, 479 char *buf) 480 { 481 struct idxd_group *group = confdev_to_group(dev); 482 483 return sysfs_emit(buf, "%d\n", group->batch_progress_limit); 484 } 485 486 static ssize_t group_batch_progress_limit_store(struct device *dev, 487 struct device_attribute *attr, 488 const char *buf, size_t count) 489 { 490 struct idxd_group *group = confdev_to_group(dev); 491 int val, rc; 492 493 rc = kstrtoint(buf, 10, &val); 494 if (rc < 0) 495 return -EINVAL; 496 497 if (val & ~GENMASK(1, 0)) 498 return -EINVAL; 499 500 group->batch_progress_limit = val; 501 return count; 502 } 503 504 static struct device_attribute dev_attr_group_batch_progress_limit = 505 __ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show, 506 group_batch_progress_limit_store); 507 static struct attribute *idxd_group_attributes[] = { 508 &dev_attr_group_work_queues.attr, 509 &dev_attr_group_engines.attr, 510 &dev_attr_group_use_token_limit.attr, 511 &dev_attr_group_use_read_buffer_limit.attr, 512 &dev_attr_group_tokens_allowed.attr, 513 &dev_attr_group_read_buffers_allowed.attr, 514 &dev_attr_group_tokens_reserved.attr, 515 &dev_attr_group_read_buffers_reserved.attr, 516 &dev_attr_group_traffic_class_a.attr, 517 &dev_attr_group_traffic_class_b.attr, 518 &dev_attr_group_desc_progress_limit.attr, 519 &dev_attr_group_batch_progress_limit.attr, 520 NULL, 521 }; 522 523 static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr, 524 struct idxd_device *idxd) 525 { 526 return (attr == &dev_attr_group_desc_progress_limit.attr || 527 attr == &dev_attr_group_batch_progress_limit.attr) && 528 !idxd->hw.group_cap.progress_limit; 529 } 530 531 static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr, 532 struct idxd_device *idxd) 533 { 534 /* 535 * Intel IAA does not support Read Buffer allocation control, 536 * make these attributes invisible. 537 */ 538 return (attr == &dev_attr_group_use_token_limit.attr || 539 attr == &dev_attr_group_use_read_buffer_limit.attr || 540 attr == &dev_attr_group_tokens_allowed.attr || 541 attr == &dev_attr_group_read_buffers_allowed.attr || 542 attr == &dev_attr_group_tokens_reserved.attr || 543 attr == &dev_attr_group_read_buffers_reserved.attr) && 544 idxd->data->type == IDXD_TYPE_IAX; 545 } 546 547 static umode_t idxd_group_attr_visible(struct kobject *kobj, 548 struct attribute *attr, int n) 549 { 550 struct device *dev = container_of(kobj, struct device, kobj); 551 struct idxd_group *group = confdev_to_group(dev); 552 struct idxd_device *idxd = group->idxd; 553 554 if (idxd_group_attr_progress_limit_invisible(attr, idxd)) 555 return 0; 556 557 if (idxd_group_attr_read_buffers_invisible(attr, idxd)) 558 return 0; 559 560 return attr->mode; 561 } 562 563 static const struct attribute_group idxd_group_attribute_group = { 564 .attrs = idxd_group_attributes, 565 .is_visible = idxd_group_attr_visible, 566 }; 567 568 static const struct attribute_group *idxd_group_attribute_groups[] = { 569 &idxd_group_attribute_group, 570 NULL, 571 }; 572 573 static void idxd_conf_group_release(struct device *dev) 574 { 575 struct idxd_group *group = confdev_to_group(dev); 576 577 kfree(group); 578 } 579 580 struct device_type idxd_group_device_type = { 581 .name = "group", 582 .release = idxd_conf_group_release, 583 .groups = idxd_group_attribute_groups, 584 }; 585 586 /* IDXD work queue attribs */ 587 static ssize_t wq_clients_show(struct device *dev, 588 struct device_attribute *attr, char *buf) 589 { 590 struct idxd_wq *wq = confdev_to_wq(dev); 591 592 return sysfs_emit(buf, "%d\n", wq->client_count); 593 } 594 595 static struct device_attribute dev_attr_wq_clients = 596 __ATTR(clients, 0444, wq_clients_show, NULL); 597 598 static ssize_t wq_state_show(struct device *dev, 599 struct device_attribute *attr, char *buf) 600 { 601 struct idxd_wq *wq = confdev_to_wq(dev); 602 603 switch (wq->state) { 604 case IDXD_WQ_DISABLED: 605 return sysfs_emit(buf, "disabled\n"); 606 case IDXD_WQ_ENABLED: 607 return sysfs_emit(buf, "enabled\n"); 608 } 609 610 return sysfs_emit(buf, "unknown\n"); 611 } 612 613 static struct device_attribute dev_attr_wq_state = 614 __ATTR(state, 0444, wq_state_show, NULL); 615 616 static ssize_t wq_group_id_show(struct device *dev, 617 struct device_attribute *attr, char *buf) 618 { 619 struct idxd_wq *wq = confdev_to_wq(dev); 620 621 if (wq->group) 622 return sysfs_emit(buf, "%u\n", wq->group->id); 623 else 624 return sysfs_emit(buf, "-1\n"); 625 } 626 627 static ssize_t wq_group_id_store(struct device *dev, 628 struct device_attribute *attr, 629 const char *buf, size_t count) 630 { 631 struct idxd_wq *wq = confdev_to_wq(dev); 632 struct idxd_device *idxd = wq->idxd; 633 long id; 634 int rc; 635 struct idxd_group *prevg, *group; 636 637 rc = kstrtol(buf, 10, &id); 638 if (rc < 0) 639 return -EINVAL; 640 641 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 642 return -EPERM; 643 644 if (wq->state != IDXD_WQ_DISABLED) 645 return -EPERM; 646 647 if (id > idxd->max_groups - 1 || id < -1) 648 return -EINVAL; 649 650 if (id == -1) { 651 if (wq->group) { 652 wq->group->num_wqs--; 653 wq->group = NULL; 654 } 655 return count; 656 } 657 658 group = idxd->groups[id]; 659 prevg = wq->group; 660 661 if (prevg) 662 prevg->num_wqs--; 663 wq->group = group; 664 group->num_wqs++; 665 return count; 666 } 667 668 static struct device_attribute dev_attr_wq_group_id = 669 __ATTR(group_id, 0644, wq_group_id_show, wq_group_id_store); 670 671 static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr, 672 char *buf) 673 { 674 struct idxd_wq *wq = confdev_to_wq(dev); 675 676 return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared"); 677 } 678 679 static ssize_t wq_mode_store(struct device *dev, 680 struct device_attribute *attr, const char *buf, 681 size_t count) 682 { 683 struct idxd_wq *wq = confdev_to_wq(dev); 684 struct idxd_device *idxd = wq->idxd; 685 686 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 687 return -EPERM; 688 689 if (wq->state != IDXD_WQ_DISABLED) 690 return -EPERM; 691 692 if (sysfs_streq(buf, "dedicated")) { 693 set_bit(WQ_FLAG_DEDICATED, &wq->flags); 694 wq->threshold = 0; 695 } else if (sysfs_streq(buf, "shared")) { 696 clear_bit(WQ_FLAG_DEDICATED, &wq->flags); 697 } else { 698 return -EINVAL; 699 } 700 701 return count; 702 } 703 704 static struct device_attribute dev_attr_wq_mode = 705 __ATTR(mode, 0644, wq_mode_show, wq_mode_store); 706 707 static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, 708 char *buf) 709 { 710 struct idxd_wq *wq = confdev_to_wq(dev); 711 712 return sysfs_emit(buf, "%u\n", wq->size); 713 } 714 715 static int total_claimed_wq_size(struct idxd_device *idxd) 716 { 717 int i; 718 int wq_size = 0; 719 720 for (i = 0; i < idxd->max_wqs; i++) { 721 struct idxd_wq *wq = idxd->wqs[i]; 722 723 wq_size += wq->size; 724 } 725 726 return wq_size; 727 } 728 729 static ssize_t wq_size_store(struct device *dev, 730 struct device_attribute *attr, const char *buf, 731 size_t count) 732 { 733 struct idxd_wq *wq = confdev_to_wq(dev); 734 unsigned long size; 735 struct idxd_device *idxd = wq->idxd; 736 int rc; 737 738 rc = kstrtoul(buf, 10, &size); 739 if (rc < 0) 740 return -EINVAL; 741 742 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 743 return -EPERM; 744 745 if (idxd->state == IDXD_DEV_ENABLED) 746 return -EPERM; 747 748 if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size) 749 return -EINVAL; 750 751 wq->size = size; 752 return count; 753 } 754 755 static struct device_attribute dev_attr_wq_size = 756 __ATTR(size, 0644, wq_size_show, wq_size_store); 757 758 static ssize_t wq_priority_show(struct device *dev, 759 struct device_attribute *attr, char *buf) 760 { 761 struct idxd_wq *wq = confdev_to_wq(dev); 762 763 return sysfs_emit(buf, "%u\n", wq->priority); 764 } 765 766 static ssize_t wq_priority_store(struct device *dev, 767 struct device_attribute *attr, 768 const char *buf, size_t count) 769 { 770 struct idxd_wq *wq = confdev_to_wq(dev); 771 unsigned long prio; 772 struct idxd_device *idxd = wq->idxd; 773 int rc; 774 775 rc = kstrtoul(buf, 10, &prio); 776 if (rc < 0) 777 return -EINVAL; 778 779 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 780 return -EPERM; 781 782 if (wq->state != IDXD_WQ_DISABLED) 783 return -EPERM; 784 785 if (prio > IDXD_MAX_PRIORITY) 786 return -EINVAL; 787 788 wq->priority = prio; 789 return count; 790 } 791 792 static struct device_attribute dev_attr_wq_priority = 793 __ATTR(priority, 0644, wq_priority_show, wq_priority_store); 794 795 static ssize_t wq_block_on_fault_show(struct device *dev, 796 struct device_attribute *attr, char *buf) 797 { 798 struct idxd_wq *wq = confdev_to_wq(dev); 799 800 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); 801 } 802 803 static ssize_t wq_block_on_fault_store(struct device *dev, 804 struct device_attribute *attr, 805 const char *buf, size_t count) 806 { 807 struct idxd_wq *wq = confdev_to_wq(dev); 808 struct idxd_device *idxd = wq->idxd; 809 bool bof; 810 int rc; 811 812 if (!idxd->hw.gen_cap.block_on_fault) 813 return -EOPNOTSUPP; 814 815 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 816 return -EPERM; 817 818 if (wq->state != IDXD_WQ_DISABLED) 819 return -ENXIO; 820 821 rc = kstrtobool(buf, &bof); 822 if (rc < 0) 823 return rc; 824 825 if (bof) { 826 if (test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags)) 827 return -EOPNOTSUPP; 828 829 set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); 830 } else { 831 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); 832 } 833 834 return count; 835 } 836 837 static struct device_attribute dev_attr_wq_block_on_fault = 838 __ATTR(block_on_fault, 0644, wq_block_on_fault_show, 839 wq_block_on_fault_store); 840 841 static ssize_t wq_threshold_show(struct device *dev, 842 struct device_attribute *attr, char *buf) 843 { 844 struct idxd_wq *wq = confdev_to_wq(dev); 845 846 return sysfs_emit(buf, "%u\n", wq->threshold); 847 } 848 849 static ssize_t wq_threshold_store(struct device *dev, 850 struct device_attribute *attr, 851 const char *buf, size_t count) 852 { 853 struct idxd_wq *wq = confdev_to_wq(dev); 854 struct idxd_device *idxd = wq->idxd; 855 unsigned int val; 856 int rc; 857 858 rc = kstrtouint(buf, 0, &val); 859 if (rc < 0) 860 return -EINVAL; 861 862 if (val > wq->size || val <= 0) 863 return -EINVAL; 864 865 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 866 return -EPERM; 867 868 if (wq->state != IDXD_WQ_DISABLED) 869 return -ENXIO; 870 871 if (test_bit(WQ_FLAG_DEDICATED, &wq->flags)) 872 return -EINVAL; 873 874 wq->threshold = val; 875 876 return count; 877 } 878 879 static struct device_attribute dev_attr_wq_threshold = 880 __ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store); 881 882 static ssize_t wq_type_show(struct device *dev, 883 struct device_attribute *attr, char *buf) 884 { 885 struct idxd_wq *wq = confdev_to_wq(dev); 886 887 switch (wq->type) { 888 case IDXD_WQT_KERNEL: 889 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_KERNEL]); 890 case IDXD_WQT_USER: 891 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_USER]); 892 case IDXD_WQT_NONE: 893 default: 894 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_NONE]); 895 } 896 897 return -EINVAL; 898 } 899 900 static ssize_t wq_type_store(struct device *dev, 901 struct device_attribute *attr, const char *buf, 902 size_t count) 903 { 904 struct idxd_wq *wq = confdev_to_wq(dev); 905 enum idxd_wq_type old_type; 906 907 if (wq->state != IDXD_WQ_DISABLED) 908 return -EPERM; 909 910 old_type = wq->type; 911 if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE])) 912 wq->type = IDXD_WQT_NONE; 913 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL])) 914 wq->type = IDXD_WQT_KERNEL; 915 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER])) 916 wq->type = IDXD_WQT_USER; 917 else 918 return -EINVAL; 919 920 /* If we are changing queue type, clear the name */ 921 if (wq->type != old_type) 922 memset(wq->name, 0, WQ_NAME_SIZE + 1); 923 924 return count; 925 } 926 927 static struct device_attribute dev_attr_wq_type = 928 __ATTR(type, 0644, wq_type_show, wq_type_store); 929 930 static ssize_t wq_name_show(struct device *dev, 931 struct device_attribute *attr, char *buf) 932 { 933 struct idxd_wq *wq = confdev_to_wq(dev); 934 935 return sysfs_emit(buf, "%s\n", wq->name); 936 } 937 938 static ssize_t wq_name_store(struct device *dev, 939 struct device_attribute *attr, const char *buf, 940 size_t count) 941 { 942 struct idxd_wq *wq = confdev_to_wq(dev); 943 char *input, *pos; 944 945 if (wq->state != IDXD_WQ_DISABLED) 946 return -EPERM; 947 948 if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0) 949 return -EINVAL; 950 951 /* 952 * This is temporarily placed here until we have SVM support for 953 * dmaengine. 954 */ 955 if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd)) 956 return -EOPNOTSUPP; 957 958 input = kstrndup(buf, count, GFP_KERNEL); 959 if (!input) 960 return -ENOMEM; 961 962 pos = strim(input); 963 memset(wq->name, 0, WQ_NAME_SIZE + 1); 964 sprintf(wq->name, "%s", pos); 965 kfree(input); 966 return count; 967 } 968 969 static struct device_attribute dev_attr_wq_name = 970 __ATTR(name, 0644, wq_name_show, wq_name_store); 971 972 static ssize_t wq_cdev_minor_show(struct device *dev, 973 struct device_attribute *attr, char *buf) 974 { 975 struct idxd_wq *wq = confdev_to_wq(dev); 976 int minor = -1; 977 978 mutex_lock(&wq->wq_lock); 979 if (wq->idxd_cdev) 980 minor = wq->idxd_cdev->minor; 981 mutex_unlock(&wq->wq_lock); 982 983 if (minor == -1) 984 return -ENXIO; 985 return sysfs_emit(buf, "%d\n", minor); 986 } 987 988 static struct device_attribute dev_attr_wq_cdev_minor = 989 __ATTR(cdev_minor, 0444, wq_cdev_minor_show, NULL); 990 991 static int __get_sysfs_u64(const char *buf, u64 *val) 992 { 993 int rc; 994 995 rc = kstrtou64(buf, 0, val); 996 if (rc < 0) 997 return -EINVAL; 998 999 if (*val == 0) 1000 return -EINVAL; 1001 1002 *val = roundup_pow_of_two(*val); 1003 return 0; 1004 } 1005 1006 static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr, 1007 char *buf) 1008 { 1009 struct idxd_wq *wq = confdev_to_wq(dev); 1010 1011 return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes); 1012 } 1013 1014 static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr, 1015 const char *buf, size_t count) 1016 { 1017 struct idxd_wq *wq = confdev_to_wq(dev); 1018 struct idxd_device *idxd = wq->idxd; 1019 u64 xfer_size; 1020 int rc; 1021 1022 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 1023 return -EPERM; 1024 1025 if (wq->state != IDXD_WQ_DISABLED) 1026 return -EPERM; 1027 1028 rc = __get_sysfs_u64(buf, &xfer_size); 1029 if (rc < 0) 1030 return rc; 1031 1032 if (xfer_size > idxd->max_xfer_bytes) 1033 return -EINVAL; 1034 1035 wq->max_xfer_bytes = xfer_size; 1036 1037 return count; 1038 } 1039 1040 static struct device_attribute dev_attr_wq_max_transfer_size = 1041 __ATTR(max_transfer_size, 0644, 1042 wq_max_transfer_size_show, wq_max_transfer_size_store); 1043 1044 static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) 1045 { 1046 struct idxd_wq *wq = confdev_to_wq(dev); 1047 1048 return sysfs_emit(buf, "%u\n", wq->max_batch_size); 1049 } 1050 1051 static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr, 1052 const char *buf, size_t count) 1053 { 1054 struct idxd_wq *wq = confdev_to_wq(dev); 1055 struct idxd_device *idxd = wq->idxd; 1056 u64 batch_size; 1057 int rc; 1058 1059 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 1060 return -EPERM; 1061 1062 if (wq->state != IDXD_WQ_DISABLED) 1063 return -EPERM; 1064 1065 rc = __get_sysfs_u64(buf, &batch_size); 1066 if (rc < 0) 1067 return rc; 1068 1069 if (batch_size > idxd->max_batch_size) 1070 return -EINVAL; 1071 1072 idxd_wq_set_max_batch_size(idxd->data->type, wq, (u32)batch_size); 1073 1074 return count; 1075 } 1076 1077 static struct device_attribute dev_attr_wq_max_batch_size = 1078 __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store); 1079 1080 static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf) 1081 { 1082 struct idxd_wq *wq = confdev_to_wq(dev); 1083 1084 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags)); 1085 } 1086 1087 static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, 1088 const char *buf, size_t count) 1089 { 1090 struct idxd_wq *wq = confdev_to_wq(dev); 1091 struct idxd_device *idxd = wq->idxd; 1092 bool ats_dis; 1093 int rc; 1094 1095 if (wq->state != IDXD_WQ_DISABLED) 1096 return -EPERM; 1097 1098 if (!idxd->hw.wq_cap.wq_ats_support) 1099 return -EOPNOTSUPP; 1100 1101 rc = kstrtobool(buf, &ats_dis); 1102 if (rc < 0) 1103 return rc; 1104 1105 if (ats_dis) 1106 set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); 1107 else 1108 clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); 1109 1110 return count; 1111 } 1112 1113 static struct device_attribute dev_attr_wq_ats_disable = 1114 __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store); 1115 1116 static ssize_t wq_prs_disable_show(struct device *dev, struct device_attribute *attr, char *buf) 1117 { 1118 struct idxd_wq *wq = confdev_to_wq(dev); 1119 1120 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags)); 1121 } 1122 1123 static ssize_t wq_prs_disable_store(struct device *dev, struct device_attribute *attr, 1124 const char *buf, size_t count) 1125 { 1126 struct idxd_wq *wq = confdev_to_wq(dev); 1127 struct idxd_device *idxd = wq->idxd; 1128 bool prs_dis; 1129 int rc; 1130 1131 if (wq->state != IDXD_WQ_DISABLED) 1132 return -EPERM; 1133 1134 if (!idxd->hw.wq_cap.wq_prs_support) 1135 return -EOPNOTSUPP; 1136 1137 rc = kstrtobool(buf, &prs_dis); 1138 if (rc < 0) 1139 return rc; 1140 1141 if (prs_dis) { 1142 set_bit(WQ_FLAG_PRS_DISABLE, &wq->flags); 1143 /* when PRS is disabled, BOF needs to be off as well */ 1144 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); 1145 } else { 1146 clear_bit(WQ_FLAG_PRS_DISABLE, &wq->flags); 1147 } 1148 return count; 1149 } 1150 1151 static struct device_attribute dev_attr_wq_prs_disable = 1152 __ATTR(prs_disable, 0644, wq_prs_disable_show, wq_prs_disable_store); 1153 1154 static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf) 1155 { 1156 struct idxd_wq *wq = confdev_to_wq(dev); 1157 struct idxd_device *idxd = wq->idxd; 1158 u32 occup, offset; 1159 1160 if (!idxd->hw.wq_cap.occupancy) 1161 return -EOPNOTSUPP; 1162 1163 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX); 1164 occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK; 1165 1166 return sysfs_emit(buf, "%u\n", occup); 1167 } 1168 1169 static struct device_attribute dev_attr_wq_occupancy = 1170 __ATTR(occupancy, 0444, wq_occupancy_show, NULL); 1171 1172 static ssize_t wq_enqcmds_retries_show(struct device *dev, 1173 struct device_attribute *attr, char *buf) 1174 { 1175 struct idxd_wq *wq = confdev_to_wq(dev); 1176 1177 if (wq_dedicated(wq)) 1178 return -EOPNOTSUPP; 1179 1180 return sysfs_emit(buf, "%u\n", wq->enqcmds_retries); 1181 } 1182 1183 static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attribute *attr, 1184 const char *buf, size_t count) 1185 { 1186 struct idxd_wq *wq = confdev_to_wq(dev); 1187 int rc; 1188 unsigned int retries; 1189 1190 if (wq_dedicated(wq)) 1191 return -EOPNOTSUPP; 1192 1193 rc = kstrtouint(buf, 10, &retries); 1194 if (rc < 0) 1195 return rc; 1196 1197 if (retries > IDXD_ENQCMDS_MAX_RETRIES) 1198 retries = IDXD_ENQCMDS_MAX_RETRIES; 1199 1200 wq->enqcmds_retries = retries; 1201 return count; 1202 } 1203 1204 static struct device_attribute dev_attr_wq_enqcmds_retries = 1205 __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store); 1206 1207 static ssize_t wq_op_config_show(struct device *dev, 1208 struct device_attribute *attr, char *buf) 1209 { 1210 struct idxd_wq *wq = confdev_to_wq(dev); 1211 1212 return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap); 1213 } 1214 1215 static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask) 1216 { 1217 int bit; 1218 1219 /* 1220 * The OPCAP is defined as 256 bits that represents each operation the device 1221 * supports per bit. Iterate through all the bits and check if the input mask 1222 * is set for bits that are not set in the OPCAP for the device. If no OPCAP 1223 * bit is set and input mask has the bit set, then return error. 1224 */ 1225 for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) { 1226 if (!test_bit(bit, idxd->opcap_bmap)) 1227 return -EINVAL; 1228 } 1229 1230 return 0; 1231 } 1232 1233 static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr, 1234 const char *buf, size_t count) 1235 { 1236 struct idxd_wq *wq = confdev_to_wq(dev); 1237 struct idxd_device *idxd = wq->idxd; 1238 unsigned long *opmask; 1239 int rc; 1240 1241 if (wq->state != IDXD_WQ_DISABLED) 1242 return -EPERM; 1243 1244 opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 1245 if (!opmask) 1246 return -ENOMEM; 1247 1248 rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS); 1249 if (rc < 0) 1250 goto err; 1251 1252 rc = idxd_verify_supported_opcap(idxd, opmask); 1253 if (rc < 0) 1254 goto err; 1255 1256 bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS); 1257 1258 bitmap_free(opmask); 1259 return count; 1260 1261 err: 1262 bitmap_free(opmask); 1263 return rc; 1264 } 1265 1266 static struct device_attribute dev_attr_wq_op_config = 1267 __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store); 1268 1269 static struct attribute *idxd_wq_attributes[] = { 1270 &dev_attr_wq_clients.attr, 1271 &dev_attr_wq_state.attr, 1272 &dev_attr_wq_group_id.attr, 1273 &dev_attr_wq_mode.attr, 1274 &dev_attr_wq_size.attr, 1275 &dev_attr_wq_priority.attr, 1276 &dev_attr_wq_block_on_fault.attr, 1277 &dev_attr_wq_threshold.attr, 1278 &dev_attr_wq_type.attr, 1279 &dev_attr_wq_name.attr, 1280 &dev_attr_wq_cdev_minor.attr, 1281 &dev_attr_wq_max_transfer_size.attr, 1282 &dev_attr_wq_max_batch_size.attr, 1283 &dev_attr_wq_ats_disable.attr, 1284 &dev_attr_wq_prs_disable.attr, 1285 &dev_attr_wq_occupancy.attr, 1286 &dev_attr_wq_enqcmds_retries.attr, 1287 &dev_attr_wq_op_config.attr, 1288 NULL, 1289 }; 1290 1291 static bool idxd_wq_attr_op_config_invisible(struct attribute *attr, 1292 struct idxd_device *idxd) 1293 { 1294 return attr == &dev_attr_wq_op_config.attr && 1295 !idxd->hw.wq_cap.op_config; 1296 } 1297 1298 static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, 1299 struct idxd_device *idxd) 1300 { 1301 /* Intel IAA does not support batch processing, make it invisible */ 1302 return attr == &dev_attr_wq_max_batch_size.attr && 1303 idxd->data->type == IDXD_TYPE_IAX; 1304 } 1305 1306 static bool idxd_wq_attr_wq_prs_disable_invisible(struct attribute *attr, 1307 struct idxd_device *idxd) 1308 { 1309 return attr == &dev_attr_wq_prs_disable.attr && 1310 !idxd->hw.wq_cap.wq_prs_support; 1311 } 1312 1313 static umode_t idxd_wq_attr_visible(struct kobject *kobj, 1314 struct attribute *attr, int n) 1315 { 1316 struct device *dev = container_of(kobj, struct device, kobj); 1317 struct idxd_wq *wq = confdev_to_wq(dev); 1318 struct idxd_device *idxd = wq->idxd; 1319 1320 if (idxd_wq_attr_op_config_invisible(attr, idxd)) 1321 return 0; 1322 1323 if (idxd_wq_attr_max_batch_size_invisible(attr, idxd)) 1324 return 0; 1325 1326 if (idxd_wq_attr_wq_prs_disable_invisible(attr, idxd)) 1327 return 0; 1328 1329 return attr->mode; 1330 } 1331 1332 static const struct attribute_group idxd_wq_attribute_group = { 1333 .attrs = idxd_wq_attributes, 1334 .is_visible = idxd_wq_attr_visible, 1335 }; 1336 1337 static const struct attribute_group *idxd_wq_attribute_groups[] = { 1338 &idxd_wq_attribute_group, 1339 NULL, 1340 }; 1341 1342 static void idxd_conf_wq_release(struct device *dev) 1343 { 1344 struct idxd_wq *wq = confdev_to_wq(dev); 1345 1346 bitmap_free(wq->opcap_bmap); 1347 kfree(wq->wqcfg); 1348 xa_destroy(&wq->upasid_xa); 1349 kfree(wq); 1350 } 1351 1352 struct device_type idxd_wq_device_type = { 1353 .name = "wq", 1354 .release = idxd_conf_wq_release, 1355 .groups = idxd_wq_attribute_groups, 1356 }; 1357 1358 /* IDXD device attribs */ 1359 static ssize_t version_show(struct device *dev, struct device_attribute *attr, 1360 char *buf) 1361 { 1362 struct idxd_device *idxd = confdev_to_idxd(dev); 1363 1364 return sysfs_emit(buf, "%#x\n", idxd->hw.version); 1365 } 1366 static DEVICE_ATTR_RO(version); 1367 1368 static ssize_t max_work_queues_size_show(struct device *dev, 1369 struct device_attribute *attr, 1370 char *buf) 1371 { 1372 struct idxd_device *idxd = confdev_to_idxd(dev); 1373 1374 return sysfs_emit(buf, "%u\n", idxd->max_wq_size); 1375 } 1376 static DEVICE_ATTR_RO(max_work_queues_size); 1377 1378 static ssize_t max_groups_show(struct device *dev, 1379 struct device_attribute *attr, char *buf) 1380 { 1381 struct idxd_device *idxd = confdev_to_idxd(dev); 1382 1383 return sysfs_emit(buf, "%u\n", idxd->max_groups); 1384 } 1385 static DEVICE_ATTR_RO(max_groups); 1386 1387 static ssize_t max_work_queues_show(struct device *dev, 1388 struct device_attribute *attr, char *buf) 1389 { 1390 struct idxd_device *idxd = confdev_to_idxd(dev); 1391 1392 return sysfs_emit(buf, "%u\n", idxd->max_wqs); 1393 } 1394 static DEVICE_ATTR_RO(max_work_queues); 1395 1396 static ssize_t max_engines_show(struct device *dev, 1397 struct device_attribute *attr, char *buf) 1398 { 1399 struct idxd_device *idxd = confdev_to_idxd(dev); 1400 1401 return sysfs_emit(buf, "%u\n", idxd->max_engines); 1402 } 1403 static DEVICE_ATTR_RO(max_engines); 1404 1405 static ssize_t numa_node_show(struct device *dev, 1406 struct device_attribute *attr, char *buf) 1407 { 1408 struct idxd_device *idxd = confdev_to_idxd(dev); 1409 1410 return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); 1411 } 1412 static DEVICE_ATTR_RO(numa_node); 1413 1414 static ssize_t max_batch_size_show(struct device *dev, 1415 struct device_attribute *attr, char *buf) 1416 { 1417 struct idxd_device *idxd = confdev_to_idxd(dev); 1418 1419 return sysfs_emit(buf, "%u\n", idxd->max_batch_size); 1420 } 1421 static DEVICE_ATTR_RO(max_batch_size); 1422 1423 static ssize_t max_transfer_size_show(struct device *dev, 1424 struct device_attribute *attr, 1425 char *buf) 1426 { 1427 struct idxd_device *idxd = confdev_to_idxd(dev); 1428 1429 return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes); 1430 } 1431 static DEVICE_ATTR_RO(max_transfer_size); 1432 1433 static ssize_t op_cap_show(struct device *dev, 1434 struct device_attribute *attr, char *buf) 1435 { 1436 struct idxd_device *idxd = confdev_to_idxd(dev); 1437 1438 return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap); 1439 } 1440 static DEVICE_ATTR_RO(op_cap); 1441 1442 static ssize_t gen_cap_show(struct device *dev, 1443 struct device_attribute *attr, char *buf) 1444 { 1445 struct idxd_device *idxd = confdev_to_idxd(dev); 1446 1447 return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits); 1448 } 1449 static DEVICE_ATTR_RO(gen_cap); 1450 1451 static ssize_t configurable_show(struct device *dev, 1452 struct device_attribute *attr, char *buf) 1453 { 1454 struct idxd_device *idxd = confdev_to_idxd(dev); 1455 1456 return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); 1457 } 1458 static DEVICE_ATTR_RO(configurable); 1459 1460 static ssize_t clients_show(struct device *dev, 1461 struct device_attribute *attr, char *buf) 1462 { 1463 struct idxd_device *idxd = confdev_to_idxd(dev); 1464 int count = 0, i; 1465 1466 spin_lock(&idxd->dev_lock); 1467 for (i = 0; i < idxd->max_wqs; i++) { 1468 struct idxd_wq *wq = idxd->wqs[i]; 1469 1470 count += wq->client_count; 1471 } 1472 spin_unlock(&idxd->dev_lock); 1473 1474 return sysfs_emit(buf, "%d\n", count); 1475 } 1476 static DEVICE_ATTR_RO(clients); 1477 1478 static ssize_t pasid_enabled_show(struct device *dev, 1479 struct device_attribute *attr, char *buf) 1480 { 1481 struct idxd_device *idxd = confdev_to_idxd(dev); 1482 1483 return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); 1484 } 1485 static DEVICE_ATTR_RO(pasid_enabled); 1486 1487 static ssize_t state_show(struct device *dev, 1488 struct device_attribute *attr, char *buf) 1489 { 1490 struct idxd_device *idxd = confdev_to_idxd(dev); 1491 1492 switch (idxd->state) { 1493 case IDXD_DEV_DISABLED: 1494 return sysfs_emit(buf, "disabled\n"); 1495 case IDXD_DEV_ENABLED: 1496 return sysfs_emit(buf, "enabled\n"); 1497 case IDXD_DEV_HALTED: 1498 return sysfs_emit(buf, "halted\n"); 1499 } 1500 1501 return sysfs_emit(buf, "unknown\n"); 1502 } 1503 static DEVICE_ATTR_RO(state); 1504 1505 static ssize_t errors_show(struct device *dev, 1506 struct device_attribute *attr, char *buf) 1507 { 1508 struct idxd_device *idxd = confdev_to_idxd(dev); 1509 DECLARE_BITMAP(swerr_bmap, 256); 1510 1511 bitmap_zero(swerr_bmap, 256); 1512 spin_lock(&idxd->dev_lock); 1513 multi_u64_to_bmap(swerr_bmap, &idxd->sw_err.bits[0], 4); 1514 spin_unlock(&idxd->dev_lock); 1515 return sysfs_emit(buf, "%*pb\n", 256, swerr_bmap); 1516 } 1517 static DEVICE_ATTR_RO(errors); 1518 1519 static ssize_t max_read_buffers_show(struct device *dev, 1520 struct device_attribute *attr, char *buf) 1521 { 1522 struct idxd_device *idxd = confdev_to_idxd(dev); 1523 1524 return sysfs_emit(buf, "%u\n", idxd->max_rdbufs); 1525 } 1526 1527 static ssize_t max_tokens_show(struct device *dev, 1528 struct device_attribute *attr, char *buf) 1529 { 1530 dev_warn_once(dev, "attribute deprecated, see max_read_buffers.\n"); 1531 return max_read_buffers_show(dev, attr, buf); 1532 } 1533 1534 static DEVICE_ATTR_RO(max_tokens); /* deprecated */ 1535 static DEVICE_ATTR_RO(max_read_buffers); 1536 1537 static ssize_t read_buffer_limit_show(struct device *dev, 1538 struct device_attribute *attr, char *buf) 1539 { 1540 struct idxd_device *idxd = confdev_to_idxd(dev); 1541 1542 return sysfs_emit(buf, "%u\n", idxd->rdbuf_limit); 1543 } 1544 1545 static ssize_t token_limit_show(struct device *dev, 1546 struct device_attribute *attr, char *buf) 1547 { 1548 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit.\n"); 1549 return read_buffer_limit_show(dev, attr, buf); 1550 } 1551 1552 static ssize_t read_buffer_limit_store(struct device *dev, 1553 struct device_attribute *attr, 1554 const char *buf, size_t count) 1555 { 1556 struct idxd_device *idxd = confdev_to_idxd(dev); 1557 unsigned long val; 1558 int rc; 1559 1560 rc = kstrtoul(buf, 10, &val); 1561 if (rc < 0) 1562 return -EINVAL; 1563 1564 if (idxd->state == IDXD_DEV_ENABLED) 1565 return -EPERM; 1566 1567 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 1568 return -EPERM; 1569 1570 if (!idxd->hw.group_cap.rdbuf_limit) 1571 return -EPERM; 1572 1573 if (val > idxd->hw.group_cap.total_rdbufs) 1574 return -EINVAL; 1575 1576 idxd->rdbuf_limit = val; 1577 return count; 1578 } 1579 1580 static ssize_t token_limit_store(struct device *dev, 1581 struct device_attribute *attr, 1582 const char *buf, size_t count) 1583 { 1584 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit\n"); 1585 return read_buffer_limit_store(dev, attr, buf, count); 1586 } 1587 1588 static DEVICE_ATTR_RW(token_limit); /* deprecated */ 1589 static DEVICE_ATTR_RW(read_buffer_limit); 1590 1591 static ssize_t cdev_major_show(struct device *dev, 1592 struct device_attribute *attr, char *buf) 1593 { 1594 struct idxd_device *idxd = confdev_to_idxd(dev); 1595 1596 return sysfs_emit(buf, "%u\n", idxd->major); 1597 } 1598 static DEVICE_ATTR_RO(cdev_major); 1599 1600 static ssize_t cmd_status_show(struct device *dev, 1601 struct device_attribute *attr, char *buf) 1602 { 1603 struct idxd_device *idxd = confdev_to_idxd(dev); 1604 1605 return sysfs_emit(buf, "%#x\n", idxd->cmd_status); 1606 } 1607 1608 static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr, 1609 const char *buf, size_t count) 1610 { 1611 struct idxd_device *idxd = confdev_to_idxd(dev); 1612 1613 idxd->cmd_status = 0; 1614 return count; 1615 } 1616 static DEVICE_ATTR_RW(cmd_status); 1617 1618 static ssize_t iaa_cap_show(struct device *dev, 1619 struct device_attribute *attr, char *buf) 1620 { 1621 struct idxd_device *idxd = confdev_to_idxd(dev); 1622 1623 if (idxd->hw.version < DEVICE_VERSION_2) 1624 return -EOPNOTSUPP; 1625 1626 return sysfs_emit(buf, "%#llx\n", idxd->hw.iaa_cap.bits); 1627 } 1628 static DEVICE_ATTR_RO(iaa_cap); 1629 1630 static ssize_t event_log_size_show(struct device *dev, 1631 struct device_attribute *attr, char *buf) 1632 { 1633 struct idxd_device *idxd = confdev_to_idxd(dev); 1634 1635 if (!idxd->evl) 1636 return -EOPNOTSUPP; 1637 1638 return sysfs_emit(buf, "%u\n", idxd->evl->size); 1639 } 1640 1641 static ssize_t event_log_size_store(struct device *dev, 1642 struct device_attribute *attr, 1643 const char *buf, size_t count) 1644 { 1645 struct idxd_device *idxd = confdev_to_idxd(dev); 1646 unsigned long val; 1647 int rc; 1648 1649 if (!idxd->evl) 1650 return -EOPNOTSUPP; 1651 1652 rc = kstrtoul(buf, 10, &val); 1653 if (rc < 0) 1654 return -EINVAL; 1655 1656 if (idxd->state == IDXD_DEV_ENABLED) 1657 return -EPERM; 1658 1659 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 1660 return -EPERM; 1661 1662 if (val < IDXD_EVL_SIZE_MIN || val > IDXD_EVL_SIZE_MAX || 1663 (val * evl_ent_size(idxd) > ULONG_MAX - idxd->evl->dma)) 1664 return -EINVAL; 1665 1666 idxd->evl->size = val; 1667 return count; 1668 } 1669 static DEVICE_ATTR_RW(event_log_size); 1670 1671 static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr, 1672 struct idxd_device *idxd) 1673 { 1674 /* Intel IAA does not support batch processing, make it invisible */ 1675 return attr == &dev_attr_max_batch_size.attr && 1676 idxd->data->type == IDXD_TYPE_IAX; 1677 } 1678 1679 static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr, 1680 struct idxd_device *idxd) 1681 { 1682 /* 1683 * Intel IAA does not support Read Buffer allocation control, 1684 * make these attributes invisible. 1685 */ 1686 return (attr == &dev_attr_max_tokens.attr || 1687 attr == &dev_attr_max_read_buffers.attr || 1688 attr == &dev_attr_token_limit.attr || 1689 attr == &dev_attr_read_buffer_limit.attr) && 1690 idxd->data->type == IDXD_TYPE_IAX; 1691 } 1692 1693 static bool idxd_device_attr_iaa_cap_invisible(struct attribute *attr, 1694 struct idxd_device *idxd) 1695 { 1696 return attr == &dev_attr_iaa_cap.attr && 1697 (idxd->data->type != IDXD_TYPE_IAX || 1698 idxd->hw.version < DEVICE_VERSION_2); 1699 } 1700 1701 static bool idxd_device_attr_event_log_size_invisible(struct attribute *attr, 1702 struct idxd_device *idxd) 1703 { 1704 return (attr == &dev_attr_event_log_size.attr && 1705 !idxd->hw.gen_cap.evl_support); 1706 } 1707 1708 static umode_t idxd_device_attr_visible(struct kobject *kobj, 1709 struct attribute *attr, int n) 1710 { 1711 struct device *dev = container_of(kobj, struct device, kobj); 1712 struct idxd_device *idxd = confdev_to_idxd(dev); 1713 1714 if (idxd_device_attr_max_batch_size_invisible(attr, idxd)) 1715 return 0; 1716 1717 if (idxd_device_attr_read_buffers_invisible(attr, idxd)) 1718 return 0; 1719 1720 if (idxd_device_attr_iaa_cap_invisible(attr, idxd)) 1721 return 0; 1722 1723 if (idxd_device_attr_event_log_size_invisible(attr, idxd)) 1724 return 0; 1725 1726 return attr->mode; 1727 } 1728 1729 static struct attribute *idxd_device_attributes[] = { 1730 &dev_attr_version.attr, 1731 &dev_attr_max_groups.attr, 1732 &dev_attr_max_work_queues.attr, 1733 &dev_attr_max_work_queues_size.attr, 1734 &dev_attr_max_engines.attr, 1735 &dev_attr_numa_node.attr, 1736 &dev_attr_max_batch_size.attr, 1737 &dev_attr_max_transfer_size.attr, 1738 &dev_attr_op_cap.attr, 1739 &dev_attr_gen_cap.attr, 1740 &dev_attr_configurable.attr, 1741 &dev_attr_clients.attr, 1742 &dev_attr_pasid_enabled.attr, 1743 &dev_attr_state.attr, 1744 &dev_attr_errors.attr, 1745 &dev_attr_max_tokens.attr, 1746 &dev_attr_max_read_buffers.attr, 1747 &dev_attr_token_limit.attr, 1748 &dev_attr_read_buffer_limit.attr, 1749 &dev_attr_cdev_major.attr, 1750 &dev_attr_cmd_status.attr, 1751 &dev_attr_iaa_cap.attr, 1752 &dev_attr_event_log_size.attr, 1753 NULL, 1754 }; 1755 1756 static const struct attribute_group idxd_device_attribute_group = { 1757 .attrs = idxd_device_attributes, 1758 .is_visible = idxd_device_attr_visible, 1759 }; 1760 1761 static const struct attribute_group *idxd_attribute_groups[] = { 1762 &idxd_device_attribute_group, 1763 NULL, 1764 }; 1765 1766 static void idxd_conf_device_release(struct device *dev) 1767 { 1768 struct idxd_device *idxd = confdev_to_idxd(dev); 1769 1770 kfree(idxd->groups); 1771 bitmap_free(idxd->wq_enable_map); 1772 kfree(idxd->wqs); 1773 kfree(idxd->engines); 1774 kfree(idxd->evl); 1775 kmem_cache_destroy(idxd->evl_cache); 1776 ida_free(&idxd_ida, idxd->id); 1777 bitmap_free(idxd->opcap_bmap); 1778 kfree(idxd); 1779 } 1780 1781 struct device_type dsa_device_type = { 1782 .name = "dsa", 1783 .release = idxd_conf_device_release, 1784 .groups = idxd_attribute_groups, 1785 }; 1786 1787 struct device_type iax_device_type = { 1788 .name = "iax", 1789 .release = idxd_conf_device_release, 1790 .groups = idxd_attribute_groups, 1791 }; 1792 1793 static int idxd_register_engine_devices(struct idxd_device *idxd) 1794 { 1795 struct idxd_engine *engine; 1796 int i, j, rc; 1797 1798 for (i = 0; i < idxd->max_engines; i++) { 1799 engine = idxd->engines[i]; 1800 rc = device_add(engine_confdev(engine)); 1801 if (rc < 0) 1802 goto cleanup; 1803 } 1804 1805 return 0; 1806 1807 cleanup: 1808 j = i - 1; 1809 for (; i < idxd->max_engines; i++) { 1810 engine = idxd->engines[i]; 1811 put_device(engine_confdev(engine)); 1812 } 1813 1814 while (j--) { 1815 engine = idxd->engines[j]; 1816 device_unregister(engine_confdev(engine)); 1817 } 1818 return rc; 1819 } 1820 1821 static int idxd_register_group_devices(struct idxd_device *idxd) 1822 { 1823 struct idxd_group *group; 1824 int i, j, rc; 1825 1826 for (i = 0; i < idxd->max_groups; i++) { 1827 group = idxd->groups[i]; 1828 rc = device_add(group_confdev(group)); 1829 if (rc < 0) 1830 goto cleanup; 1831 } 1832 1833 return 0; 1834 1835 cleanup: 1836 j = i - 1; 1837 for (; i < idxd->max_groups; i++) { 1838 group = idxd->groups[i]; 1839 put_device(group_confdev(group)); 1840 } 1841 1842 while (j--) { 1843 group = idxd->groups[j]; 1844 device_unregister(group_confdev(group)); 1845 } 1846 return rc; 1847 } 1848 1849 static int idxd_register_wq_devices(struct idxd_device *idxd) 1850 { 1851 struct idxd_wq *wq; 1852 int i, rc, j; 1853 1854 for (i = 0; i < idxd->max_wqs; i++) { 1855 wq = idxd->wqs[i]; 1856 rc = device_add(wq_confdev(wq)); 1857 if (rc < 0) 1858 goto cleanup; 1859 } 1860 1861 return 0; 1862 1863 cleanup: 1864 j = i - 1; 1865 for (; i < idxd->max_wqs; i++) { 1866 wq = idxd->wqs[i]; 1867 put_device(wq_confdev(wq)); 1868 } 1869 1870 while (j--) { 1871 wq = idxd->wqs[j]; 1872 device_unregister(wq_confdev(wq)); 1873 } 1874 return rc; 1875 } 1876 1877 int idxd_register_devices(struct idxd_device *idxd) 1878 { 1879 struct device *dev = &idxd->pdev->dev; 1880 int rc, i; 1881 1882 rc = device_add(idxd_confdev(idxd)); 1883 if (rc < 0) 1884 return rc; 1885 1886 rc = idxd_register_wq_devices(idxd); 1887 if (rc < 0) { 1888 dev_dbg(dev, "WQ devices registering failed: %d\n", rc); 1889 goto err_wq; 1890 } 1891 1892 rc = idxd_register_engine_devices(idxd); 1893 if (rc < 0) { 1894 dev_dbg(dev, "Engine devices registering failed: %d\n", rc); 1895 goto err_engine; 1896 } 1897 1898 rc = idxd_register_group_devices(idxd); 1899 if (rc < 0) { 1900 dev_dbg(dev, "Group device registering failed: %d\n", rc); 1901 goto err_group; 1902 } 1903 1904 return 0; 1905 1906 err_group: 1907 for (i = 0; i < idxd->max_engines; i++) 1908 device_unregister(engine_confdev(idxd->engines[i])); 1909 err_engine: 1910 for (i = 0; i < idxd->max_wqs; i++) 1911 device_unregister(wq_confdev(idxd->wqs[i])); 1912 err_wq: 1913 device_del(idxd_confdev(idxd)); 1914 return rc; 1915 } 1916 1917 void idxd_unregister_devices(struct idxd_device *idxd) 1918 { 1919 int i; 1920 1921 for (i = 0; i < idxd->max_wqs; i++) { 1922 struct idxd_wq *wq = idxd->wqs[i]; 1923 1924 device_unregister(wq_confdev(wq)); 1925 } 1926 1927 for (i = 0; i < idxd->max_engines; i++) { 1928 struct idxd_engine *engine = idxd->engines[i]; 1929 1930 device_unregister(engine_confdev(engine)); 1931 } 1932 1933 for (i = 0; i < idxd->max_groups; i++) { 1934 struct idxd_group *group = idxd->groups[i]; 1935 1936 device_unregister(group_confdev(group)); 1937 } 1938 } 1939 1940 int idxd_register_bus_type(void) 1941 { 1942 return bus_register(&dsa_bus_type); 1943 } 1944 1945 void idxd_unregister_bus_type(void) 1946 { 1947 bus_unregister(&dsa_bus_type); 1948 } 1949