1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/io-64-nonatomic-lo-hi.h> 9 #include <uapi/linux/idxd.h> 10 #include "registers.h" 11 #include "idxd.h" 12 13 static char *idxd_wq_type_names[] = { 14 [IDXD_WQT_NONE] = "none", 15 [IDXD_WQT_KERNEL] = "kernel", 16 [IDXD_WQT_USER] = "user", 17 }; 18 19 /* IDXD engine attributes */ 20 static ssize_t engine_group_id_show(struct device *dev, 21 struct device_attribute *attr, char *buf) 22 { 23 struct idxd_engine *engine = confdev_to_engine(dev); 24 25 if (engine->group) 26 return sysfs_emit(buf, "%d\n", engine->group->id); 27 else 28 return sysfs_emit(buf, "%d\n", -1); 29 } 30 31 static ssize_t engine_group_id_store(struct device *dev, 32 struct device_attribute *attr, 33 const char *buf, size_t count) 34 { 35 struct idxd_engine *engine = confdev_to_engine(dev); 36 struct idxd_device *idxd = engine->idxd; 37 long id; 38 int rc; 39 struct idxd_group *prevg; 40 41 rc = kstrtol(buf, 10, &id); 42 if (rc < 0) 43 return -EINVAL; 44 45 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 46 return -EPERM; 47 48 if (id > idxd->max_groups - 1 || id < -1) 49 return -EINVAL; 50 51 if (id == -1) { 52 if (engine->group) { 53 engine->group->num_engines--; 54 engine->group = NULL; 55 } 56 return count; 57 } 58 59 prevg = engine->group; 60 61 if (prevg) 62 prevg->num_engines--; 63 engine->group = idxd->groups[id]; 64 engine->group->num_engines++; 65 66 return count; 67 } 68 69 static struct device_attribute dev_attr_engine_group = 70 __ATTR(group_id, 0644, engine_group_id_show, 71 engine_group_id_store); 72 73 static struct attribute *idxd_engine_attributes[] = { 74 &dev_attr_engine_group.attr, 75 NULL, 76 }; 77 78 static const struct attribute_group idxd_engine_attribute_group = { 79 .attrs = idxd_engine_attributes, 80 }; 81 82 static const struct attribute_group *idxd_engine_attribute_groups[] = { 83 &idxd_engine_attribute_group, 84 NULL, 85 }; 86 87 static void idxd_conf_engine_release(struct device *dev) 88 { 89 struct idxd_engine *engine = confdev_to_engine(dev); 90 91 kfree(engine); 92 } 93 94 struct device_type idxd_engine_device_type = { 95 .name = "engine", 96 .release = idxd_conf_engine_release, 97 .groups = idxd_engine_attribute_groups, 98 }; 99 100 /* Group attributes */ 101 102 static void idxd_set_free_tokens(struct idxd_device *idxd) 103 { 104 int i, tokens; 105 106 for (i = 0, tokens = 0; i < idxd->max_groups; i++) { 107 struct idxd_group *g = idxd->groups[i]; 108 109 tokens += g->tokens_reserved; 110 } 111 112 idxd->nr_tokens = idxd->max_tokens - tokens; 113 } 114 115 static ssize_t group_tokens_reserved_show(struct device *dev, 116 struct device_attribute *attr, 117 char *buf) 118 { 119 struct idxd_group *group = confdev_to_group(dev); 120 121 return sysfs_emit(buf, "%u\n", group->tokens_reserved); 122 } 123 124 static ssize_t group_tokens_reserved_store(struct device *dev, 125 struct device_attribute *attr, 126 const char *buf, size_t count) 127 { 128 struct idxd_group *group = confdev_to_group(dev); 129 struct idxd_device *idxd = group->idxd; 130 unsigned long val; 131 int rc; 132 133 rc = kstrtoul(buf, 10, &val); 134 if (rc < 0) 135 return -EINVAL; 136 137 if (idxd->data->type == IDXD_TYPE_IAX) 138 return -EOPNOTSUPP; 139 140 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 141 return -EPERM; 142 143 if (idxd->state == IDXD_DEV_ENABLED) 144 return -EPERM; 145 146 if (val > idxd->max_tokens) 147 return -EINVAL; 148 149 if (val > idxd->nr_tokens + group->tokens_reserved) 150 return -EINVAL; 151 152 group->tokens_reserved = val; 153 idxd_set_free_tokens(idxd); 154 return count; 155 } 156 157 static struct device_attribute dev_attr_group_tokens_reserved = 158 __ATTR(tokens_reserved, 0644, group_tokens_reserved_show, 159 group_tokens_reserved_store); 160 161 static ssize_t group_tokens_allowed_show(struct device *dev, 162 struct device_attribute *attr, 163 char *buf) 164 { 165 struct idxd_group *group = confdev_to_group(dev); 166 167 return sysfs_emit(buf, "%u\n", group->tokens_allowed); 168 } 169 170 static ssize_t group_tokens_allowed_store(struct device *dev, 171 struct device_attribute *attr, 172 const char *buf, size_t count) 173 { 174 struct idxd_group *group = confdev_to_group(dev); 175 struct idxd_device *idxd = group->idxd; 176 unsigned long val; 177 int rc; 178 179 rc = kstrtoul(buf, 10, &val); 180 if (rc < 0) 181 return -EINVAL; 182 183 if (idxd->data->type == IDXD_TYPE_IAX) 184 return -EOPNOTSUPP; 185 186 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 187 return -EPERM; 188 189 if (idxd->state == IDXD_DEV_ENABLED) 190 return -EPERM; 191 192 if (val < 4 * group->num_engines || 193 val > group->tokens_reserved + idxd->nr_tokens) 194 return -EINVAL; 195 196 group->tokens_allowed = val; 197 return count; 198 } 199 200 static struct device_attribute dev_attr_group_tokens_allowed = 201 __ATTR(tokens_allowed, 0644, group_tokens_allowed_show, 202 group_tokens_allowed_store); 203 204 static ssize_t group_use_token_limit_show(struct device *dev, 205 struct device_attribute *attr, 206 char *buf) 207 { 208 struct idxd_group *group = confdev_to_group(dev); 209 210 return sysfs_emit(buf, "%u\n", group->use_token_limit); 211 } 212 213 static ssize_t group_use_token_limit_store(struct device *dev, 214 struct device_attribute *attr, 215 const char *buf, size_t count) 216 { 217 struct idxd_group *group = confdev_to_group(dev); 218 struct idxd_device *idxd = group->idxd; 219 unsigned long val; 220 int rc; 221 222 rc = kstrtoul(buf, 10, &val); 223 if (rc < 0) 224 return -EINVAL; 225 226 if (idxd->data->type == IDXD_TYPE_IAX) 227 return -EOPNOTSUPP; 228 229 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 230 return -EPERM; 231 232 if (idxd->state == IDXD_DEV_ENABLED) 233 return -EPERM; 234 235 if (idxd->token_limit == 0) 236 return -EPERM; 237 238 group->use_token_limit = !!val; 239 return count; 240 } 241 242 static struct device_attribute dev_attr_group_use_token_limit = 243 __ATTR(use_token_limit, 0644, group_use_token_limit_show, 244 group_use_token_limit_store); 245 246 static ssize_t group_engines_show(struct device *dev, 247 struct device_attribute *attr, char *buf) 248 { 249 struct idxd_group *group = confdev_to_group(dev); 250 int i, rc = 0; 251 struct idxd_device *idxd = group->idxd; 252 253 for (i = 0; i < idxd->max_engines; i++) { 254 struct idxd_engine *engine = idxd->engines[i]; 255 256 if (!engine->group) 257 continue; 258 259 if (engine->group->id == group->id) 260 rc += sysfs_emit_at(buf, rc, "engine%d.%d ", idxd->id, engine->id); 261 } 262 263 if (!rc) 264 return 0; 265 rc--; 266 rc += sysfs_emit_at(buf, rc, "\n"); 267 268 return rc; 269 } 270 271 static struct device_attribute dev_attr_group_engines = 272 __ATTR(engines, 0444, group_engines_show, NULL); 273 274 static ssize_t group_work_queues_show(struct device *dev, 275 struct device_attribute *attr, char *buf) 276 { 277 struct idxd_group *group = confdev_to_group(dev); 278 int i, rc = 0; 279 struct idxd_device *idxd = group->idxd; 280 281 for (i = 0; i < idxd->max_wqs; i++) { 282 struct idxd_wq *wq = idxd->wqs[i]; 283 284 if (!wq->group) 285 continue; 286 287 if (wq->group->id == group->id) 288 rc += sysfs_emit_at(buf, rc, "wq%d.%d ", idxd->id, wq->id); 289 } 290 291 if (!rc) 292 return 0; 293 rc--; 294 rc += sysfs_emit_at(buf, rc, "\n"); 295 296 return rc; 297 } 298 299 static struct device_attribute dev_attr_group_work_queues = 300 __ATTR(work_queues, 0444, group_work_queues_show, NULL); 301 302 static ssize_t group_traffic_class_a_show(struct device *dev, 303 struct device_attribute *attr, 304 char *buf) 305 { 306 struct idxd_group *group = confdev_to_group(dev); 307 308 return sysfs_emit(buf, "%d\n", group->tc_a); 309 } 310 311 static ssize_t group_traffic_class_a_store(struct device *dev, 312 struct device_attribute *attr, 313 const char *buf, size_t count) 314 { 315 struct idxd_group *group = confdev_to_group(dev); 316 struct idxd_device *idxd = group->idxd; 317 long val; 318 int rc; 319 320 rc = kstrtol(buf, 10, &val); 321 if (rc < 0) 322 return -EINVAL; 323 324 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 325 return -EPERM; 326 327 if (idxd->state == IDXD_DEV_ENABLED) 328 return -EPERM; 329 330 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) 331 return -EPERM; 332 333 if (val < 0 || val > 7) 334 return -EINVAL; 335 336 group->tc_a = val; 337 return count; 338 } 339 340 static struct device_attribute dev_attr_group_traffic_class_a = 341 __ATTR(traffic_class_a, 0644, group_traffic_class_a_show, 342 group_traffic_class_a_store); 343 344 static ssize_t group_traffic_class_b_show(struct device *dev, 345 struct device_attribute *attr, 346 char *buf) 347 { 348 struct idxd_group *group = confdev_to_group(dev); 349 350 return sysfs_emit(buf, "%d\n", group->tc_b); 351 } 352 353 static ssize_t group_traffic_class_b_store(struct device *dev, 354 struct device_attribute *attr, 355 const char *buf, size_t count) 356 { 357 struct idxd_group *group = confdev_to_group(dev); 358 struct idxd_device *idxd = group->idxd; 359 long val; 360 int rc; 361 362 rc = kstrtol(buf, 10, &val); 363 if (rc < 0) 364 return -EINVAL; 365 366 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 367 return -EPERM; 368 369 if (idxd->state == IDXD_DEV_ENABLED) 370 return -EPERM; 371 372 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) 373 return -EPERM; 374 375 if (val < 0 || val > 7) 376 return -EINVAL; 377 378 group->tc_b = val; 379 return count; 380 } 381 382 static struct device_attribute dev_attr_group_traffic_class_b = 383 __ATTR(traffic_class_b, 0644, group_traffic_class_b_show, 384 group_traffic_class_b_store); 385 386 static struct attribute *idxd_group_attributes[] = { 387 &dev_attr_group_work_queues.attr, 388 &dev_attr_group_engines.attr, 389 &dev_attr_group_use_token_limit.attr, 390 &dev_attr_group_tokens_allowed.attr, 391 &dev_attr_group_tokens_reserved.attr, 392 &dev_attr_group_traffic_class_a.attr, 393 &dev_attr_group_traffic_class_b.attr, 394 NULL, 395 }; 396 397 static const struct attribute_group idxd_group_attribute_group = { 398 .attrs = idxd_group_attributes, 399 }; 400 401 static const struct attribute_group *idxd_group_attribute_groups[] = { 402 &idxd_group_attribute_group, 403 NULL, 404 }; 405 406 static void idxd_conf_group_release(struct device *dev) 407 { 408 struct idxd_group *group = confdev_to_group(dev); 409 410 kfree(group); 411 } 412 413 struct device_type idxd_group_device_type = { 414 .name = "group", 415 .release = idxd_conf_group_release, 416 .groups = idxd_group_attribute_groups, 417 }; 418 419 /* IDXD work queue attribs */ 420 static ssize_t wq_clients_show(struct device *dev, 421 struct device_attribute *attr, char *buf) 422 { 423 struct idxd_wq *wq = confdev_to_wq(dev); 424 425 return sysfs_emit(buf, "%d\n", wq->client_count); 426 } 427 428 static struct device_attribute dev_attr_wq_clients = 429 __ATTR(clients, 0444, wq_clients_show, NULL); 430 431 static ssize_t wq_state_show(struct device *dev, 432 struct device_attribute *attr, char *buf) 433 { 434 struct idxd_wq *wq = confdev_to_wq(dev); 435 436 switch (wq->state) { 437 case IDXD_WQ_DISABLED: 438 return sysfs_emit(buf, "disabled\n"); 439 case IDXD_WQ_ENABLED: 440 return sysfs_emit(buf, "enabled\n"); 441 } 442 443 return sysfs_emit(buf, "unknown\n"); 444 } 445 446 static struct device_attribute dev_attr_wq_state = 447 __ATTR(state, 0444, wq_state_show, NULL); 448 449 static ssize_t wq_group_id_show(struct device *dev, 450 struct device_attribute *attr, char *buf) 451 { 452 struct idxd_wq *wq = confdev_to_wq(dev); 453 454 if (wq->group) 455 return sysfs_emit(buf, "%u\n", wq->group->id); 456 else 457 return sysfs_emit(buf, "-1\n"); 458 } 459 460 static ssize_t wq_group_id_store(struct device *dev, 461 struct device_attribute *attr, 462 const char *buf, size_t count) 463 { 464 struct idxd_wq *wq = confdev_to_wq(dev); 465 struct idxd_device *idxd = wq->idxd; 466 long id; 467 int rc; 468 struct idxd_group *prevg, *group; 469 470 rc = kstrtol(buf, 10, &id); 471 if (rc < 0) 472 return -EINVAL; 473 474 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 475 return -EPERM; 476 477 if (wq->state != IDXD_WQ_DISABLED) 478 return -EPERM; 479 480 if (id > idxd->max_groups - 1 || id < -1) 481 return -EINVAL; 482 483 if (id == -1) { 484 if (wq->group) { 485 wq->group->num_wqs--; 486 wq->group = NULL; 487 } 488 return count; 489 } 490 491 group = idxd->groups[id]; 492 prevg = wq->group; 493 494 if (prevg) 495 prevg->num_wqs--; 496 wq->group = group; 497 group->num_wqs++; 498 return count; 499 } 500 501 static struct device_attribute dev_attr_wq_group_id = 502 __ATTR(group_id, 0644, wq_group_id_show, wq_group_id_store); 503 504 static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr, 505 char *buf) 506 { 507 struct idxd_wq *wq = confdev_to_wq(dev); 508 509 return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared"); 510 } 511 512 static ssize_t wq_mode_store(struct device *dev, 513 struct device_attribute *attr, const char *buf, 514 size_t count) 515 { 516 struct idxd_wq *wq = confdev_to_wq(dev); 517 struct idxd_device *idxd = wq->idxd; 518 519 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 520 return -EPERM; 521 522 if (wq->state != IDXD_WQ_DISABLED) 523 return -EPERM; 524 525 if (sysfs_streq(buf, "dedicated")) { 526 set_bit(WQ_FLAG_DEDICATED, &wq->flags); 527 wq->threshold = 0; 528 } else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) { 529 clear_bit(WQ_FLAG_DEDICATED, &wq->flags); 530 } else { 531 return -EINVAL; 532 } 533 534 return count; 535 } 536 537 static struct device_attribute dev_attr_wq_mode = 538 __ATTR(mode, 0644, wq_mode_show, wq_mode_store); 539 540 static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, 541 char *buf) 542 { 543 struct idxd_wq *wq = confdev_to_wq(dev); 544 545 return sysfs_emit(buf, "%u\n", wq->size); 546 } 547 548 static int total_claimed_wq_size(struct idxd_device *idxd) 549 { 550 int i; 551 int wq_size = 0; 552 553 for (i = 0; i < idxd->max_wqs; i++) { 554 struct idxd_wq *wq = idxd->wqs[i]; 555 556 wq_size += wq->size; 557 } 558 559 return wq_size; 560 } 561 562 static ssize_t wq_size_store(struct device *dev, 563 struct device_attribute *attr, const char *buf, 564 size_t count) 565 { 566 struct idxd_wq *wq = confdev_to_wq(dev); 567 unsigned long size; 568 struct idxd_device *idxd = wq->idxd; 569 int rc; 570 571 rc = kstrtoul(buf, 10, &size); 572 if (rc < 0) 573 return -EINVAL; 574 575 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 576 return -EPERM; 577 578 if (idxd->state == IDXD_DEV_ENABLED) 579 return -EPERM; 580 581 if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size) 582 return -EINVAL; 583 584 wq->size = size; 585 return count; 586 } 587 588 static struct device_attribute dev_attr_wq_size = 589 __ATTR(size, 0644, wq_size_show, wq_size_store); 590 591 static ssize_t wq_priority_show(struct device *dev, 592 struct device_attribute *attr, char *buf) 593 { 594 struct idxd_wq *wq = confdev_to_wq(dev); 595 596 return sysfs_emit(buf, "%u\n", wq->priority); 597 } 598 599 static ssize_t wq_priority_store(struct device *dev, 600 struct device_attribute *attr, 601 const char *buf, size_t count) 602 { 603 struct idxd_wq *wq = confdev_to_wq(dev); 604 unsigned long prio; 605 struct idxd_device *idxd = wq->idxd; 606 int rc; 607 608 rc = kstrtoul(buf, 10, &prio); 609 if (rc < 0) 610 return -EINVAL; 611 612 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 613 return -EPERM; 614 615 if (wq->state != IDXD_WQ_DISABLED) 616 return -EPERM; 617 618 if (prio > IDXD_MAX_PRIORITY) 619 return -EINVAL; 620 621 wq->priority = prio; 622 return count; 623 } 624 625 static struct device_attribute dev_attr_wq_priority = 626 __ATTR(priority, 0644, wq_priority_show, wq_priority_store); 627 628 static ssize_t wq_block_on_fault_show(struct device *dev, 629 struct device_attribute *attr, char *buf) 630 { 631 struct idxd_wq *wq = confdev_to_wq(dev); 632 633 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); 634 } 635 636 static ssize_t wq_block_on_fault_store(struct device *dev, 637 struct device_attribute *attr, 638 const char *buf, size_t count) 639 { 640 struct idxd_wq *wq = confdev_to_wq(dev); 641 struct idxd_device *idxd = wq->idxd; 642 bool bof; 643 int rc; 644 645 if (!idxd->hw.gen_cap.block_on_fault) 646 return -EOPNOTSUPP; 647 648 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 649 return -EPERM; 650 651 if (wq->state != IDXD_WQ_DISABLED) 652 return -ENXIO; 653 654 rc = kstrtobool(buf, &bof); 655 if (rc < 0) 656 return rc; 657 658 if (bof) 659 set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); 660 else 661 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); 662 663 return count; 664 } 665 666 static struct device_attribute dev_attr_wq_block_on_fault = 667 __ATTR(block_on_fault, 0644, wq_block_on_fault_show, 668 wq_block_on_fault_store); 669 670 static ssize_t wq_threshold_show(struct device *dev, 671 struct device_attribute *attr, char *buf) 672 { 673 struct idxd_wq *wq = confdev_to_wq(dev); 674 675 return sysfs_emit(buf, "%u\n", wq->threshold); 676 } 677 678 static ssize_t wq_threshold_store(struct device *dev, 679 struct device_attribute *attr, 680 const char *buf, size_t count) 681 { 682 struct idxd_wq *wq = confdev_to_wq(dev); 683 struct idxd_device *idxd = wq->idxd; 684 unsigned int val; 685 int rc; 686 687 rc = kstrtouint(buf, 0, &val); 688 if (rc < 0) 689 return -EINVAL; 690 691 if (val > wq->size || val <= 0) 692 return -EINVAL; 693 694 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 695 return -EPERM; 696 697 if (wq->state != IDXD_WQ_DISABLED) 698 return -ENXIO; 699 700 if (test_bit(WQ_FLAG_DEDICATED, &wq->flags)) 701 return -EINVAL; 702 703 wq->threshold = val; 704 705 return count; 706 } 707 708 static struct device_attribute dev_attr_wq_threshold = 709 __ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store); 710 711 static ssize_t wq_type_show(struct device *dev, 712 struct device_attribute *attr, char *buf) 713 { 714 struct idxd_wq *wq = confdev_to_wq(dev); 715 716 switch (wq->type) { 717 case IDXD_WQT_KERNEL: 718 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_KERNEL]); 719 case IDXD_WQT_USER: 720 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_USER]); 721 case IDXD_WQT_NONE: 722 default: 723 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_NONE]); 724 } 725 726 return -EINVAL; 727 } 728 729 static ssize_t wq_type_store(struct device *dev, 730 struct device_attribute *attr, const char *buf, 731 size_t count) 732 { 733 struct idxd_wq *wq = confdev_to_wq(dev); 734 enum idxd_wq_type old_type; 735 736 if (wq->state != IDXD_WQ_DISABLED) 737 return -EPERM; 738 739 old_type = wq->type; 740 if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE])) 741 wq->type = IDXD_WQT_NONE; 742 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL])) 743 wq->type = IDXD_WQT_KERNEL; 744 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER])) 745 wq->type = IDXD_WQT_USER; 746 else 747 return -EINVAL; 748 749 /* If we are changing queue type, clear the name */ 750 if (wq->type != old_type) 751 memset(wq->name, 0, WQ_NAME_SIZE + 1); 752 753 return count; 754 } 755 756 static struct device_attribute dev_attr_wq_type = 757 __ATTR(type, 0644, wq_type_show, wq_type_store); 758 759 static ssize_t wq_name_show(struct device *dev, 760 struct device_attribute *attr, char *buf) 761 { 762 struct idxd_wq *wq = confdev_to_wq(dev); 763 764 return sysfs_emit(buf, "%s\n", wq->name); 765 } 766 767 static ssize_t wq_name_store(struct device *dev, 768 struct device_attribute *attr, const char *buf, 769 size_t count) 770 { 771 struct idxd_wq *wq = confdev_to_wq(dev); 772 773 if (wq->state != IDXD_WQ_DISABLED) 774 return -EPERM; 775 776 if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0) 777 return -EINVAL; 778 779 /* 780 * This is temporarily placed here until we have SVM support for 781 * dmaengine. 782 */ 783 if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd)) 784 return -EOPNOTSUPP; 785 786 memset(wq->name, 0, WQ_NAME_SIZE + 1); 787 strncpy(wq->name, buf, WQ_NAME_SIZE); 788 strreplace(wq->name, '\n', '\0'); 789 return count; 790 } 791 792 static struct device_attribute dev_attr_wq_name = 793 __ATTR(name, 0644, wq_name_show, wq_name_store); 794 795 static ssize_t wq_cdev_minor_show(struct device *dev, 796 struct device_attribute *attr, char *buf) 797 { 798 struct idxd_wq *wq = confdev_to_wq(dev); 799 int minor = -1; 800 801 mutex_lock(&wq->wq_lock); 802 if (wq->idxd_cdev) 803 minor = wq->idxd_cdev->minor; 804 mutex_unlock(&wq->wq_lock); 805 806 if (minor == -1) 807 return -ENXIO; 808 return sysfs_emit(buf, "%d\n", minor); 809 } 810 811 static struct device_attribute dev_attr_wq_cdev_minor = 812 __ATTR(cdev_minor, 0444, wq_cdev_minor_show, NULL); 813 814 static int __get_sysfs_u64(const char *buf, u64 *val) 815 { 816 int rc; 817 818 rc = kstrtou64(buf, 0, val); 819 if (rc < 0) 820 return -EINVAL; 821 822 if (*val == 0) 823 return -EINVAL; 824 825 *val = roundup_pow_of_two(*val); 826 return 0; 827 } 828 829 static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr, 830 char *buf) 831 { 832 struct idxd_wq *wq = confdev_to_wq(dev); 833 834 return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes); 835 } 836 837 static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr, 838 const char *buf, size_t count) 839 { 840 struct idxd_wq *wq = confdev_to_wq(dev); 841 struct idxd_device *idxd = wq->idxd; 842 u64 xfer_size; 843 int rc; 844 845 if (wq->state != IDXD_WQ_DISABLED) 846 return -EPERM; 847 848 rc = __get_sysfs_u64(buf, &xfer_size); 849 if (rc < 0) 850 return rc; 851 852 if (xfer_size > idxd->max_xfer_bytes) 853 return -EINVAL; 854 855 wq->max_xfer_bytes = xfer_size; 856 857 return count; 858 } 859 860 static struct device_attribute dev_attr_wq_max_transfer_size = 861 __ATTR(max_transfer_size, 0644, 862 wq_max_transfer_size_show, wq_max_transfer_size_store); 863 864 static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) 865 { 866 struct idxd_wq *wq = confdev_to_wq(dev); 867 868 return sysfs_emit(buf, "%u\n", wq->max_batch_size); 869 } 870 871 static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr, 872 const char *buf, size_t count) 873 { 874 struct idxd_wq *wq = confdev_to_wq(dev); 875 struct idxd_device *idxd = wq->idxd; 876 u64 batch_size; 877 int rc; 878 879 if (wq->state != IDXD_WQ_DISABLED) 880 return -EPERM; 881 882 rc = __get_sysfs_u64(buf, &batch_size); 883 if (rc < 0) 884 return rc; 885 886 if (batch_size > idxd->max_batch_size) 887 return -EINVAL; 888 889 wq->max_batch_size = (u32)batch_size; 890 891 return count; 892 } 893 894 static struct device_attribute dev_attr_wq_max_batch_size = 895 __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store); 896 897 static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf) 898 { 899 struct idxd_wq *wq = confdev_to_wq(dev); 900 901 return sysfs_emit(buf, "%u\n", wq->ats_dis); 902 } 903 904 static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, 905 const char *buf, size_t count) 906 { 907 struct idxd_wq *wq = confdev_to_wq(dev); 908 struct idxd_device *idxd = wq->idxd; 909 bool ats_dis; 910 int rc; 911 912 if (wq->state != IDXD_WQ_DISABLED) 913 return -EPERM; 914 915 if (!idxd->hw.wq_cap.wq_ats_support) 916 return -EOPNOTSUPP; 917 918 rc = kstrtobool(buf, &ats_dis); 919 if (rc < 0) 920 return rc; 921 922 wq->ats_dis = ats_dis; 923 924 return count; 925 } 926 927 static struct device_attribute dev_attr_wq_ats_disable = 928 __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store); 929 930 static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf) 931 { 932 struct idxd_wq *wq = confdev_to_wq(dev); 933 struct idxd_device *idxd = wq->idxd; 934 u32 occup, offset; 935 936 if (!idxd->hw.wq_cap.occupancy) 937 return -EOPNOTSUPP; 938 939 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX); 940 occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK; 941 942 return sysfs_emit(buf, "%u\n", occup); 943 } 944 945 static struct device_attribute dev_attr_wq_occupancy = 946 __ATTR(occupancy, 0444, wq_occupancy_show, NULL); 947 948 static struct attribute *idxd_wq_attributes[] = { 949 &dev_attr_wq_clients.attr, 950 &dev_attr_wq_state.attr, 951 &dev_attr_wq_group_id.attr, 952 &dev_attr_wq_mode.attr, 953 &dev_attr_wq_size.attr, 954 &dev_attr_wq_priority.attr, 955 &dev_attr_wq_block_on_fault.attr, 956 &dev_attr_wq_threshold.attr, 957 &dev_attr_wq_type.attr, 958 &dev_attr_wq_name.attr, 959 &dev_attr_wq_cdev_minor.attr, 960 &dev_attr_wq_max_transfer_size.attr, 961 &dev_attr_wq_max_batch_size.attr, 962 &dev_attr_wq_ats_disable.attr, 963 &dev_attr_wq_occupancy.attr, 964 NULL, 965 }; 966 967 static const struct attribute_group idxd_wq_attribute_group = { 968 .attrs = idxd_wq_attributes, 969 }; 970 971 static const struct attribute_group *idxd_wq_attribute_groups[] = { 972 &idxd_wq_attribute_group, 973 NULL, 974 }; 975 976 static void idxd_conf_wq_release(struct device *dev) 977 { 978 struct idxd_wq *wq = confdev_to_wq(dev); 979 980 kfree(wq->wqcfg); 981 kfree(wq); 982 } 983 984 struct device_type idxd_wq_device_type = { 985 .name = "wq", 986 .release = idxd_conf_wq_release, 987 .groups = idxd_wq_attribute_groups, 988 }; 989 990 /* IDXD device attribs */ 991 static ssize_t version_show(struct device *dev, struct device_attribute *attr, 992 char *buf) 993 { 994 struct idxd_device *idxd = confdev_to_idxd(dev); 995 996 return sysfs_emit(buf, "%#x\n", idxd->hw.version); 997 } 998 static DEVICE_ATTR_RO(version); 999 1000 static ssize_t max_work_queues_size_show(struct device *dev, 1001 struct device_attribute *attr, 1002 char *buf) 1003 { 1004 struct idxd_device *idxd = confdev_to_idxd(dev); 1005 1006 return sysfs_emit(buf, "%u\n", idxd->max_wq_size); 1007 } 1008 static DEVICE_ATTR_RO(max_work_queues_size); 1009 1010 static ssize_t max_groups_show(struct device *dev, 1011 struct device_attribute *attr, char *buf) 1012 { 1013 struct idxd_device *idxd = confdev_to_idxd(dev); 1014 1015 return sysfs_emit(buf, "%u\n", idxd->max_groups); 1016 } 1017 static DEVICE_ATTR_RO(max_groups); 1018 1019 static ssize_t max_work_queues_show(struct device *dev, 1020 struct device_attribute *attr, char *buf) 1021 { 1022 struct idxd_device *idxd = confdev_to_idxd(dev); 1023 1024 return sysfs_emit(buf, "%u\n", idxd->max_wqs); 1025 } 1026 static DEVICE_ATTR_RO(max_work_queues); 1027 1028 static ssize_t max_engines_show(struct device *dev, 1029 struct device_attribute *attr, char *buf) 1030 { 1031 struct idxd_device *idxd = confdev_to_idxd(dev); 1032 1033 return sysfs_emit(buf, "%u\n", idxd->max_engines); 1034 } 1035 static DEVICE_ATTR_RO(max_engines); 1036 1037 static ssize_t numa_node_show(struct device *dev, 1038 struct device_attribute *attr, char *buf) 1039 { 1040 struct idxd_device *idxd = confdev_to_idxd(dev); 1041 1042 return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); 1043 } 1044 static DEVICE_ATTR_RO(numa_node); 1045 1046 static ssize_t max_batch_size_show(struct device *dev, 1047 struct device_attribute *attr, char *buf) 1048 { 1049 struct idxd_device *idxd = confdev_to_idxd(dev); 1050 1051 return sysfs_emit(buf, "%u\n", idxd->max_batch_size); 1052 } 1053 static DEVICE_ATTR_RO(max_batch_size); 1054 1055 static ssize_t max_transfer_size_show(struct device *dev, 1056 struct device_attribute *attr, 1057 char *buf) 1058 { 1059 struct idxd_device *idxd = confdev_to_idxd(dev); 1060 1061 return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes); 1062 } 1063 static DEVICE_ATTR_RO(max_transfer_size); 1064 1065 static ssize_t op_cap_show(struct device *dev, 1066 struct device_attribute *attr, char *buf) 1067 { 1068 struct idxd_device *idxd = confdev_to_idxd(dev); 1069 int i, rc = 0; 1070 1071 for (i = 0; i < 4; i++) 1072 rc += sysfs_emit_at(buf, rc, "%#llx ", idxd->hw.opcap.bits[i]); 1073 1074 rc--; 1075 rc += sysfs_emit_at(buf, rc, "\n"); 1076 return rc; 1077 } 1078 static DEVICE_ATTR_RO(op_cap); 1079 1080 static ssize_t gen_cap_show(struct device *dev, 1081 struct device_attribute *attr, char *buf) 1082 { 1083 struct idxd_device *idxd = confdev_to_idxd(dev); 1084 1085 return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits); 1086 } 1087 static DEVICE_ATTR_RO(gen_cap); 1088 1089 static ssize_t configurable_show(struct device *dev, 1090 struct device_attribute *attr, char *buf) 1091 { 1092 struct idxd_device *idxd = confdev_to_idxd(dev); 1093 1094 return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); 1095 } 1096 static DEVICE_ATTR_RO(configurable); 1097 1098 static ssize_t clients_show(struct device *dev, 1099 struct device_attribute *attr, char *buf) 1100 { 1101 struct idxd_device *idxd = confdev_to_idxd(dev); 1102 int count = 0, i; 1103 1104 spin_lock(&idxd->dev_lock); 1105 for (i = 0; i < idxd->max_wqs; i++) { 1106 struct idxd_wq *wq = idxd->wqs[i]; 1107 1108 count += wq->client_count; 1109 } 1110 spin_unlock(&idxd->dev_lock); 1111 1112 return sysfs_emit(buf, "%d\n", count); 1113 } 1114 static DEVICE_ATTR_RO(clients); 1115 1116 static ssize_t pasid_enabled_show(struct device *dev, 1117 struct device_attribute *attr, char *buf) 1118 { 1119 struct idxd_device *idxd = confdev_to_idxd(dev); 1120 1121 return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); 1122 } 1123 static DEVICE_ATTR_RO(pasid_enabled); 1124 1125 static ssize_t state_show(struct device *dev, 1126 struct device_attribute *attr, char *buf) 1127 { 1128 struct idxd_device *idxd = confdev_to_idxd(dev); 1129 1130 switch (idxd->state) { 1131 case IDXD_DEV_DISABLED: 1132 return sysfs_emit(buf, "disabled\n"); 1133 case IDXD_DEV_ENABLED: 1134 return sysfs_emit(buf, "enabled\n"); 1135 case IDXD_DEV_HALTED: 1136 return sysfs_emit(buf, "halted\n"); 1137 } 1138 1139 return sysfs_emit(buf, "unknown\n"); 1140 } 1141 static DEVICE_ATTR_RO(state); 1142 1143 static ssize_t errors_show(struct device *dev, 1144 struct device_attribute *attr, char *buf) 1145 { 1146 struct idxd_device *idxd = confdev_to_idxd(dev); 1147 int i, out = 0; 1148 1149 spin_lock(&idxd->dev_lock); 1150 for (i = 0; i < 4; i++) 1151 out += sysfs_emit_at(buf, out, "%#018llx ", idxd->sw_err.bits[i]); 1152 spin_unlock(&idxd->dev_lock); 1153 out--; 1154 out += sysfs_emit_at(buf, out, "\n"); 1155 return out; 1156 } 1157 static DEVICE_ATTR_RO(errors); 1158 1159 static ssize_t max_tokens_show(struct device *dev, 1160 struct device_attribute *attr, char *buf) 1161 { 1162 struct idxd_device *idxd = confdev_to_idxd(dev); 1163 1164 return sysfs_emit(buf, "%u\n", idxd->max_tokens); 1165 } 1166 static DEVICE_ATTR_RO(max_tokens); 1167 1168 static ssize_t token_limit_show(struct device *dev, 1169 struct device_attribute *attr, char *buf) 1170 { 1171 struct idxd_device *idxd = confdev_to_idxd(dev); 1172 1173 return sysfs_emit(buf, "%u\n", idxd->token_limit); 1174 } 1175 1176 static ssize_t token_limit_store(struct device *dev, 1177 struct device_attribute *attr, 1178 const char *buf, size_t count) 1179 { 1180 struct idxd_device *idxd = confdev_to_idxd(dev); 1181 unsigned long val; 1182 int rc; 1183 1184 rc = kstrtoul(buf, 10, &val); 1185 if (rc < 0) 1186 return -EINVAL; 1187 1188 if (idxd->state == IDXD_DEV_ENABLED) 1189 return -EPERM; 1190 1191 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) 1192 return -EPERM; 1193 1194 if (!idxd->hw.group_cap.token_limit) 1195 return -EPERM; 1196 1197 if (val > idxd->hw.group_cap.total_tokens) 1198 return -EINVAL; 1199 1200 idxd->token_limit = val; 1201 return count; 1202 } 1203 static DEVICE_ATTR_RW(token_limit); 1204 1205 static ssize_t cdev_major_show(struct device *dev, 1206 struct device_attribute *attr, char *buf) 1207 { 1208 struct idxd_device *idxd = confdev_to_idxd(dev); 1209 1210 return sysfs_emit(buf, "%u\n", idxd->major); 1211 } 1212 static DEVICE_ATTR_RO(cdev_major); 1213 1214 static ssize_t cmd_status_show(struct device *dev, 1215 struct device_attribute *attr, char *buf) 1216 { 1217 struct idxd_device *idxd = confdev_to_idxd(dev); 1218 1219 return sysfs_emit(buf, "%#x\n", idxd->cmd_status); 1220 } 1221 1222 static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr, 1223 const char *buf, size_t count) 1224 { 1225 struct idxd_device *idxd = confdev_to_idxd(dev); 1226 1227 idxd->cmd_status = 0; 1228 return count; 1229 } 1230 static DEVICE_ATTR_RW(cmd_status); 1231 1232 static struct attribute *idxd_device_attributes[] = { 1233 &dev_attr_version.attr, 1234 &dev_attr_max_groups.attr, 1235 &dev_attr_max_work_queues.attr, 1236 &dev_attr_max_work_queues_size.attr, 1237 &dev_attr_max_engines.attr, 1238 &dev_attr_numa_node.attr, 1239 &dev_attr_max_batch_size.attr, 1240 &dev_attr_max_transfer_size.attr, 1241 &dev_attr_op_cap.attr, 1242 &dev_attr_gen_cap.attr, 1243 &dev_attr_configurable.attr, 1244 &dev_attr_clients.attr, 1245 &dev_attr_pasid_enabled.attr, 1246 &dev_attr_state.attr, 1247 &dev_attr_errors.attr, 1248 &dev_attr_max_tokens.attr, 1249 &dev_attr_token_limit.attr, 1250 &dev_attr_cdev_major.attr, 1251 &dev_attr_cmd_status.attr, 1252 NULL, 1253 }; 1254 1255 static const struct attribute_group idxd_device_attribute_group = { 1256 .attrs = idxd_device_attributes, 1257 }; 1258 1259 static const struct attribute_group *idxd_attribute_groups[] = { 1260 &idxd_device_attribute_group, 1261 NULL, 1262 }; 1263 1264 static void idxd_conf_device_release(struct device *dev) 1265 { 1266 struct idxd_device *idxd = confdev_to_idxd(dev); 1267 1268 kfree(idxd->groups); 1269 kfree(idxd->wqs); 1270 kfree(idxd->engines); 1271 kfree(idxd->irq_entries); 1272 kfree(idxd->int_handles); 1273 ida_free(&idxd_ida, idxd->id); 1274 kfree(idxd); 1275 } 1276 1277 struct device_type dsa_device_type = { 1278 .name = "dsa", 1279 .release = idxd_conf_device_release, 1280 .groups = idxd_attribute_groups, 1281 }; 1282 1283 struct device_type iax_device_type = { 1284 .name = "iax", 1285 .release = idxd_conf_device_release, 1286 .groups = idxd_attribute_groups, 1287 }; 1288 1289 static int idxd_register_engine_devices(struct idxd_device *idxd) 1290 { 1291 struct idxd_engine *engine; 1292 int i, j, rc; 1293 1294 for (i = 0; i < idxd->max_engines; i++) { 1295 engine = idxd->engines[i]; 1296 rc = device_add(engine_confdev(engine)); 1297 if (rc < 0) 1298 goto cleanup; 1299 } 1300 1301 return 0; 1302 1303 cleanup: 1304 j = i - 1; 1305 for (; i < idxd->max_engines; i++) { 1306 engine = idxd->engines[i]; 1307 put_device(engine_confdev(engine)); 1308 } 1309 1310 while (j--) { 1311 engine = idxd->engines[j]; 1312 device_unregister(engine_confdev(engine)); 1313 } 1314 return rc; 1315 } 1316 1317 static int idxd_register_group_devices(struct idxd_device *idxd) 1318 { 1319 struct idxd_group *group; 1320 int i, j, rc; 1321 1322 for (i = 0; i < idxd->max_groups; i++) { 1323 group = idxd->groups[i]; 1324 rc = device_add(group_confdev(group)); 1325 if (rc < 0) 1326 goto cleanup; 1327 } 1328 1329 return 0; 1330 1331 cleanup: 1332 j = i - 1; 1333 for (; i < idxd->max_groups; i++) { 1334 group = idxd->groups[i]; 1335 put_device(group_confdev(group)); 1336 } 1337 1338 while (j--) { 1339 group = idxd->groups[j]; 1340 device_unregister(group_confdev(group)); 1341 } 1342 return rc; 1343 } 1344 1345 static int idxd_register_wq_devices(struct idxd_device *idxd) 1346 { 1347 struct idxd_wq *wq; 1348 int i, rc, j; 1349 1350 for (i = 0; i < idxd->max_wqs; i++) { 1351 wq = idxd->wqs[i]; 1352 rc = device_add(wq_confdev(wq)); 1353 if (rc < 0) 1354 goto cleanup; 1355 } 1356 1357 return 0; 1358 1359 cleanup: 1360 j = i - 1; 1361 for (; i < idxd->max_wqs; i++) { 1362 wq = idxd->wqs[i]; 1363 put_device(wq_confdev(wq)); 1364 } 1365 1366 while (j--) { 1367 wq = idxd->wqs[j]; 1368 device_unregister(wq_confdev(wq)); 1369 } 1370 return rc; 1371 } 1372 1373 int idxd_register_devices(struct idxd_device *idxd) 1374 { 1375 struct device *dev = &idxd->pdev->dev; 1376 int rc, i; 1377 1378 rc = device_add(idxd_confdev(idxd)); 1379 if (rc < 0) 1380 return rc; 1381 1382 rc = idxd_register_wq_devices(idxd); 1383 if (rc < 0) { 1384 dev_dbg(dev, "WQ devices registering failed: %d\n", rc); 1385 goto err_wq; 1386 } 1387 1388 rc = idxd_register_engine_devices(idxd); 1389 if (rc < 0) { 1390 dev_dbg(dev, "Engine devices registering failed: %d\n", rc); 1391 goto err_engine; 1392 } 1393 1394 rc = idxd_register_group_devices(idxd); 1395 if (rc < 0) { 1396 dev_dbg(dev, "Group device registering failed: %d\n", rc); 1397 goto err_group; 1398 } 1399 1400 return 0; 1401 1402 err_group: 1403 for (i = 0; i < idxd->max_engines; i++) 1404 device_unregister(engine_confdev(idxd->engines[i])); 1405 err_engine: 1406 for (i = 0; i < idxd->max_wqs; i++) 1407 device_unregister(wq_confdev(idxd->wqs[i])); 1408 err_wq: 1409 device_del(idxd_confdev(idxd)); 1410 return rc; 1411 } 1412 1413 void idxd_unregister_devices(struct idxd_device *idxd) 1414 { 1415 int i; 1416 1417 for (i = 0; i < idxd->max_wqs; i++) { 1418 struct idxd_wq *wq = idxd->wqs[i]; 1419 1420 device_unregister(wq_confdev(wq)); 1421 } 1422 1423 for (i = 0; i < idxd->max_engines; i++) { 1424 struct idxd_engine *engine = idxd->engines[i]; 1425 1426 device_unregister(engine_confdev(engine)); 1427 } 1428 1429 for (i = 0; i < idxd->max_groups; i++) { 1430 struct idxd_group *group = idxd->groups[i]; 1431 1432 device_unregister(group_confdev(group)); 1433 } 1434 } 1435 1436 int idxd_register_bus_type(void) 1437 { 1438 return bus_register(&dsa_bus_type); 1439 } 1440 1441 void idxd_unregister_bus_type(void) 1442 { 1443 bus_unregister(&dsa_bus_type); 1444 } 1445