1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Adjunct processor matrix VFIO device driver callbacks. 4 * 5 * Copyright IBM Corp. 2018 6 * 7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com> 8 * Halil Pasic <pasic@linux.ibm.com> 9 * Pierre Morel <pmorel@linux.ibm.com> 10 */ 11 #include <linux/string.h> 12 #include <linux/vfio.h> 13 #include <linux/device.h> 14 #include <linux/list.h> 15 #include <linux/ctype.h> 16 #include <asm/zcrypt.h> 17 18 #include "vfio_ap_private.h" 19 20 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough" 21 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device" 22 23 static void vfio_ap_matrix_init(struct ap_config_info *info, 24 struct ap_matrix *matrix) 25 { 26 matrix->apm_max = info->apxa ? info->Na : 63; 27 matrix->aqm_max = info->apxa ? info->Nd : 15; 28 matrix->adm_max = info->apxa ? info->Nd : 15; 29 } 30 31 static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev) 32 { 33 struct ap_matrix_mdev *matrix_mdev; 34 35 if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0)) 36 return -EPERM; 37 38 matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL); 39 if (!matrix_mdev) { 40 atomic_inc(&matrix_dev->available_instances); 41 return -ENOMEM; 42 } 43 44 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); 45 mdev_set_drvdata(mdev, matrix_mdev); 46 mutex_lock(&matrix_dev->lock); 47 list_add(&matrix_mdev->node, &matrix_dev->mdev_list); 48 mutex_unlock(&matrix_dev->lock); 49 50 return 0; 51 } 52 53 static int vfio_ap_mdev_remove(struct mdev_device *mdev) 54 { 55 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 56 57 mutex_lock(&matrix_dev->lock); 58 list_del(&matrix_mdev->node); 59 mutex_unlock(&matrix_dev->lock); 60 61 kfree(matrix_mdev); 62 mdev_set_drvdata(mdev, NULL); 63 atomic_inc(&matrix_dev->available_instances); 64 65 return 0; 66 } 67 68 static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf) 69 { 70 return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT); 71 } 72 73 MDEV_TYPE_ATTR_RO(name); 74 75 static ssize_t available_instances_show(struct kobject *kobj, 76 struct device *dev, char *buf) 77 { 78 return sprintf(buf, "%d\n", 79 atomic_read(&matrix_dev->available_instances)); 80 } 81 82 MDEV_TYPE_ATTR_RO(available_instances); 83 84 static ssize_t device_api_show(struct kobject *kobj, struct device *dev, 85 char *buf) 86 { 87 return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING); 88 } 89 90 MDEV_TYPE_ATTR_RO(device_api); 91 92 static struct attribute *vfio_ap_mdev_type_attrs[] = { 93 &mdev_type_attr_name.attr, 94 &mdev_type_attr_device_api.attr, 95 &mdev_type_attr_available_instances.attr, 96 NULL, 97 }; 98 99 static struct attribute_group vfio_ap_mdev_hwvirt_type_group = { 100 .name = VFIO_AP_MDEV_TYPE_HWVIRT, 101 .attrs = vfio_ap_mdev_type_attrs, 102 }; 103 104 static struct attribute_group *vfio_ap_mdev_type_groups[] = { 105 &vfio_ap_mdev_hwvirt_type_group, 106 NULL, 107 }; 108 109 struct vfio_ap_queue_reserved { 110 unsigned long *apid; 111 unsigned long *apqi; 112 bool reserved; 113 }; 114 115 /** 116 * vfio_ap_has_queue 117 * 118 * @dev: an AP queue device 119 * @data: a struct vfio_ap_queue_reserved reference 120 * 121 * Flags whether the AP queue device (@dev) has a queue ID containing the APQN, 122 * apid or apqi specified in @data: 123 * 124 * - If @data contains both an apid and apqi value, then @data will be flagged 125 * as reserved if the APID and APQI fields for the AP queue device matches 126 * 127 * - If @data contains only an apid value, @data will be flagged as 128 * reserved if the APID field in the AP queue device matches 129 * 130 * - If @data contains only an apqi value, @data will be flagged as 131 * reserved if the APQI field in the AP queue device matches 132 * 133 * Returns 0 to indicate the input to function succeeded. Returns -EINVAL if 134 * @data does not contain either an apid or apqi. 135 */ 136 static int vfio_ap_has_queue(struct device *dev, void *data) 137 { 138 struct vfio_ap_queue_reserved *qres = data; 139 struct ap_queue *ap_queue = to_ap_queue(dev); 140 ap_qid_t qid; 141 unsigned long id; 142 143 if (qres->apid && qres->apqi) { 144 qid = AP_MKQID(*qres->apid, *qres->apqi); 145 if (qid == ap_queue->qid) 146 qres->reserved = true; 147 } else if (qres->apid && !qres->apqi) { 148 id = AP_QID_CARD(ap_queue->qid); 149 if (id == *qres->apid) 150 qres->reserved = true; 151 } else if (!qres->apid && qres->apqi) { 152 id = AP_QID_QUEUE(ap_queue->qid); 153 if (id == *qres->apqi) 154 qres->reserved = true; 155 } else { 156 return -EINVAL; 157 } 158 159 return 0; 160 } 161 162 /** 163 * vfio_ap_verify_queue_reserved 164 * 165 * @matrix_dev: a mediated matrix device 166 * @apid: an AP adapter ID 167 * @apqi: an AP queue index 168 * 169 * Verifies that the AP queue with @apid/@apqi is reserved by the VFIO AP device 170 * driver according to the following rules: 171 * 172 * - If both @apid and @apqi are not NULL, then there must be an AP queue 173 * device bound to the vfio_ap driver with the APQN identified by @apid and 174 * @apqi 175 * 176 * - If only @apid is not NULL, then there must be an AP queue device bound 177 * to the vfio_ap driver with an APQN containing @apid 178 * 179 * - If only @apqi is not NULL, then there must be an AP queue device bound 180 * to the vfio_ap driver with an APQN containing @apqi 181 * 182 * Returns 0 if the AP queue is reserved; otherwise, returns -EADDRNOTAVAIL. 183 */ 184 static int vfio_ap_verify_queue_reserved(unsigned long *apid, 185 unsigned long *apqi) 186 { 187 int ret; 188 struct vfio_ap_queue_reserved qres; 189 190 qres.apid = apid; 191 qres.apqi = apqi; 192 qres.reserved = false; 193 194 ret = driver_for_each_device(matrix_dev->device.driver, NULL, &qres, 195 vfio_ap_has_queue); 196 if (ret) 197 return ret; 198 199 if (qres.reserved) 200 return 0; 201 202 return -EADDRNOTAVAIL; 203 } 204 205 static int 206 vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev, 207 unsigned long apid) 208 { 209 int ret; 210 unsigned long apqi; 211 unsigned long nbits = matrix_mdev->matrix.aqm_max + 1; 212 213 if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits) 214 return vfio_ap_verify_queue_reserved(&apid, NULL); 215 216 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) { 217 ret = vfio_ap_verify_queue_reserved(&apid, &apqi); 218 if (ret) 219 return ret; 220 } 221 222 return 0; 223 } 224 225 /** 226 * vfio_ap_mdev_verify_no_sharing 227 * 228 * Verifies that the APQNs derived from the cross product of the AP adapter IDs 229 * and AP queue indexes comprising the AP matrix are not configured for another 230 * mediated device. AP queue sharing is not allowed. 231 * 232 * @matrix_mdev: the mediated matrix device 233 * 234 * Returns 0 if the APQNs are not shared, otherwise; returns -EADDRINUSE. 235 */ 236 static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev) 237 { 238 struct ap_matrix_mdev *lstdev; 239 DECLARE_BITMAP(apm, AP_DEVICES); 240 DECLARE_BITMAP(aqm, AP_DOMAINS); 241 242 list_for_each_entry(lstdev, &matrix_dev->mdev_list, node) { 243 if (matrix_mdev == lstdev) 244 continue; 245 246 memset(apm, 0, sizeof(apm)); 247 memset(aqm, 0, sizeof(aqm)); 248 249 /* 250 * We work on full longs, as we can only exclude the leftover 251 * bits in non-inverse order. The leftover is all zeros. 252 */ 253 if (!bitmap_and(apm, matrix_mdev->matrix.apm, 254 lstdev->matrix.apm, AP_DEVICES)) 255 continue; 256 257 if (!bitmap_and(aqm, matrix_mdev->matrix.aqm, 258 lstdev->matrix.aqm, AP_DOMAINS)) 259 continue; 260 261 return -EADDRINUSE; 262 } 263 264 return 0; 265 } 266 267 /** 268 * assign_adapter_store 269 * 270 * @dev: the matrix device 271 * @attr: the mediated matrix device's assign_adapter attribute 272 * @buf: a buffer containing the AP adapter number (APID) to 273 * be assigned 274 * @count: the number of bytes in @buf 275 * 276 * Parses the APID from @buf and sets the corresponding bit in the mediated 277 * matrix device's APM. 278 * 279 * Returns the number of bytes processed if the APID is valid; otherwise, 280 * returns one of the following errors: 281 * 282 * 1. -EINVAL 283 * The APID is not a valid number 284 * 285 * 2. -ENODEV 286 * The APID exceeds the maximum value configured for the system 287 * 288 * 3. -EADDRNOTAVAIL 289 * An APQN derived from the cross product of the APID being assigned 290 * and the APQIs previously assigned is not bound to the vfio_ap device 291 * driver; or, if no APQIs have yet been assigned, the APID is not 292 * contained in an APQN bound to the vfio_ap device driver. 293 * 294 * 4. -EADDRINUSE 295 * An APQN derived from the cross product of the APID being assigned 296 * and the APQIs previously assigned is being used by another mediated 297 * matrix device 298 */ 299 static ssize_t assign_adapter_store(struct device *dev, 300 struct device_attribute *attr, 301 const char *buf, size_t count) 302 { 303 int ret; 304 unsigned long apid; 305 struct mdev_device *mdev = mdev_from_dev(dev); 306 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 307 308 ret = kstrtoul(buf, 0, &apid); 309 if (ret) 310 return ret; 311 312 if (apid > matrix_mdev->matrix.apm_max) 313 return -ENODEV; 314 315 /* 316 * Set the bit in the AP mask (APM) corresponding to the AP adapter 317 * number (APID). The bits in the mask, from most significant to least 318 * significant bit, correspond to APIDs 0-255. 319 */ 320 mutex_lock(&matrix_dev->lock); 321 322 ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid); 323 if (ret) 324 goto done; 325 326 set_bit_inv(apid, matrix_mdev->matrix.apm); 327 328 ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev); 329 if (ret) 330 goto share_err; 331 332 ret = count; 333 goto done; 334 335 share_err: 336 clear_bit_inv(apid, matrix_mdev->matrix.apm); 337 done: 338 mutex_unlock(&matrix_dev->lock); 339 340 return ret; 341 } 342 static DEVICE_ATTR_WO(assign_adapter); 343 344 /** 345 * unassign_adapter_store 346 * 347 * @dev: the matrix device 348 * @attr: the mediated matrix device's unassign_adapter attribute 349 * @buf: a buffer containing the adapter number (APID) to be unassigned 350 * @count: the number of bytes in @buf 351 * 352 * Parses the APID from @buf and clears the corresponding bit in the mediated 353 * matrix device's APM. 354 * 355 * Returns the number of bytes processed if the APID is valid; otherwise, 356 * returns one of the following errors: 357 * -EINVAL if the APID is not a number 358 * -ENODEV if the APID it exceeds the maximum value configured for the 359 * system 360 */ 361 static ssize_t unassign_adapter_store(struct device *dev, 362 struct device_attribute *attr, 363 const char *buf, size_t count) 364 { 365 int ret; 366 unsigned long apid; 367 struct mdev_device *mdev = mdev_from_dev(dev); 368 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 369 370 ret = kstrtoul(buf, 0, &apid); 371 if (ret) 372 return ret; 373 374 if (apid > matrix_mdev->matrix.apm_max) 375 return -ENODEV; 376 377 mutex_lock(&matrix_dev->lock); 378 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); 379 mutex_unlock(&matrix_dev->lock); 380 381 return count; 382 } 383 DEVICE_ATTR_WO(unassign_adapter); 384 385 static int 386 vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev, 387 unsigned long apqi) 388 { 389 int ret; 390 unsigned long apid; 391 unsigned long nbits = matrix_mdev->matrix.apm_max + 1; 392 393 if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits) 394 return vfio_ap_verify_queue_reserved(NULL, &apqi); 395 396 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) { 397 ret = vfio_ap_verify_queue_reserved(&apid, &apqi); 398 if (ret) 399 return ret; 400 } 401 402 return 0; 403 } 404 405 /** 406 * assign_domain_store 407 * 408 * @dev: the matrix device 409 * @attr: the mediated matrix device's assign_domain attribute 410 * @buf: a buffer containing the AP queue index (APQI) of the domain to 411 * be assigned 412 * @count: the number of bytes in @buf 413 * 414 * Parses the APQI from @buf and sets the corresponding bit in the mediated 415 * matrix device's AQM. 416 * 417 * Returns the number of bytes processed if the APQI is valid; otherwise returns 418 * one of the following errors: 419 * 420 * 1. -EINVAL 421 * The APQI is not a valid number 422 * 423 * 2. -ENODEV 424 * The APQI exceeds the maximum value configured for the system 425 * 426 * 3. -EADDRNOTAVAIL 427 * An APQN derived from the cross product of the APQI being assigned 428 * and the APIDs previously assigned is not bound to the vfio_ap device 429 * driver; or, if no APIDs have yet been assigned, the APQI is not 430 * contained in an APQN bound to the vfio_ap device driver. 431 * 432 * 4. -EADDRINUSE 433 * An APQN derived from the cross product of the APQI being assigned 434 * and the APIDs previously assigned is being used by another mediated 435 * matrix device 436 */ 437 static ssize_t assign_domain_store(struct device *dev, 438 struct device_attribute *attr, 439 const char *buf, size_t count) 440 { 441 int ret; 442 unsigned long apqi; 443 struct mdev_device *mdev = mdev_from_dev(dev); 444 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 445 unsigned long max_apqi = matrix_mdev->matrix.aqm_max; 446 447 ret = kstrtoul(buf, 0, &apqi); 448 if (ret) 449 return ret; 450 if (apqi > max_apqi) 451 return -ENODEV; 452 453 mutex_lock(&matrix_dev->lock); 454 455 ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi); 456 if (ret) 457 goto done; 458 459 set_bit_inv(apqi, matrix_mdev->matrix.aqm); 460 461 ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev); 462 if (ret) 463 goto share_err; 464 465 ret = count; 466 goto done; 467 468 share_err: 469 clear_bit_inv(apqi, matrix_mdev->matrix.aqm); 470 done: 471 mutex_unlock(&matrix_dev->lock); 472 473 return ret; 474 } 475 DEVICE_ATTR_WO(assign_domain); 476 477 478 /** 479 * unassign_domain_store 480 * 481 * @dev: the matrix device 482 * @attr: the mediated matrix device's unassign_domain attribute 483 * @buf: a buffer containing the AP queue index (APQI) of the domain to 484 * be unassigned 485 * @count: the number of bytes in @buf 486 * 487 * Parses the APQI from @buf and clears the corresponding bit in the 488 * mediated matrix device's AQM. 489 * 490 * Returns the number of bytes processed if the APQI is valid; otherwise, 491 * returns one of the following errors: 492 * -EINVAL if the APQI is not a number 493 * -ENODEV if the APQI exceeds the maximum value configured for the system 494 */ 495 static ssize_t unassign_domain_store(struct device *dev, 496 struct device_attribute *attr, 497 const char *buf, size_t count) 498 { 499 int ret; 500 unsigned long apqi; 501 struct mdev_device *mdev = mdev_from_dev(dev); 502 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 503 504 ret = kstrtoul(buf, 0, &apqi); 505 if (ret) 506 return ret; 507 508 if (apqi > matrix_mdev->matrix.aqm_max) 509 return -ENODEV; 510 511 mutex_lock(&matrix_dev->lock); 512 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); 513 mutex_unlock(&matrix_dev->lock); 514 515 return count; 516 } 517 DEVICE_ATTR_WO(unassign_domain); 518 519 /** 520 * assign_control_domain_store 521 * 522 * @dev: the matrix device 523 * @attr: the mediated matrix device's assign_control_domain attribute 524 * @buf: a buffer containing the domain ID to be assigned 525 * @count: the number of bytes in @buf 526 * 527 * Parses the domain ID from @buf and sets the corresponding bit in the mediated 528 * matrix device's ADM. 529 * 530 * Returns the number of bytes processed if the domain ID is valid; otherwise, 531 * returns one of the following errors: 532 * -EINVAL if the ID is not a number 533 * -ENODEV if the ID exceeds the maximum value configured for the system 534 */ 535 static ssize_t assign_control_domain_store(struct device *dev, 536 struct device_attribute *attr, 537 const char *buf, size_t count) 538 { 539 int ret; 540 unsigned long id; 541 struct mdev_device *mdev = mdev_from_dev(dev); 542 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 543 544 ret = kstrtoul(buf, 0, &id); 545 if (ret) 546 return ret; 547 548 if (id > matrix_mdev->matrix.adm_max) 549 return -ENODEV; 550 551 /* Set the bit in the ADM (bitmask) corresponding to the AP control 552 * domain number (id). The bits in the mask, from most significant to 553 * least significant, correspond to IDs 0 up to the one less than the 554 * number of control domains that can be assigned. 555 */ 556 mutex_lock(&matrix_dev->lock); 557 set_bit_inv(id, matrix_mdev->matrix.adm); 558 mutex_unlock(&matrix_dev->lock); 559 560 return count; 561 } 562 DEVICE_ATTR_WO(assign_control_domain); 563 564 /** 565 * unassign_control_domain_store 566 * 567 * @dev: the matrix device 568 * @attr: the mediated matrix device's unassign_control_domain attribute 569 * @buf: a buffer containing the domain ID to be unassigned 570 * @count: the number of bytes in @buf 571 * 572 * Parses the domain ID from @buf and clears the corresponding bit in the 573 * mediated matrix device's ADM. 574 * 575 * Returns the number of bytes processed if the domain ID is valid; otherwise, 576 * returns one of the following errors: 577 * -EINVAL if the ID is not a number 578 * -ENODEV if the ID exceeds the maximum value configured for the system 579 */ 580 static ssize_t unassign_control_domain_store(struct device *dev, 581 struct device_attribute *attr, 582 const char *buf, size_t count) 583 { 584 int ret; 585 unsigned long domid; 586 struct mdev_device *mdev = mdev_from_dev(dev); 587 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 588 unsigned long max_domid = matrix_mdev->matrix.adm_max; 589 590 ret = kstrtoul(buf, 0, &domid); 591 if (ret) 592 return ret; 593 if (domid > max_domid) 594 return -ENODEV; 595 596 mutex_lock(&matrix_dev->lock); 597 clear_bit_inv(domid, matrix_mdev->matrix.adm); 598 mutex_unlock(&matrix_dev->lock); 599 600 return count; 601 } 602 DEVICE_ATTR_WO(unassign_control_domain); 603 604 static ssize_t control_domains_show(struct device *dev, 605 struct device_attribute *dev_attr, 606 char *buf) 607 { 608 unsigned long id; 609 int nchars = 0; 610 int n; 611 char *bufpos = buf; 612 struct mdev_device *mdev = mdev_from_dev(dev); 613 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); 614 unsigned long max_domid = matrix_mdev->matrix.adm_max; 615 616 mutex_lock(&matrix_dev->lock); 617 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) { 618 n = sprintf(bufpos, "%04lx\n", id); 619 bufpos += n; 620 nchars += n; 621 } 622 mutex_unlock(&matrix_dev->lock); 623 624 return nchars; 625 } 626 DEVICE_ATTR_RO(control_domains); 627 628 static struct attribute *vfio_ap_mdev_attrs[] = { 629 &dev_attr_assign_adapter.attr, 630 &dev_attr_unassign_adapter.attr, 631 &dev_attr_assign_domain.attr, 632 &dev_attr_unassign_domain.attr, 633 &dev_attr_assign_control_domain.attr, 634 &dev_attr_unassign_control_domain.attr, 635 &dev_attr_control_domains.attr, 636 NULL, 637 }; 638 639 static struct attribute_group vfio_ap_mdev_attr_group = { 640 .attrs = vfio_ap_mdev_attrs 641 }; 642 643 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { 644 &vfio_ap_mdev_attr_group, 645 NULL 646 }; 647 648 static const struct mdev_parent_ops vfio_ap_matrix_ops = { 649 .owner = THIS_MODULE, 650 .supported_type_groups = vfio_ap_mdev_type_groups, 651 .mdev_attr_groups = vfio_ap_mdev_attr_groups, 652 .create = vfio_ap_mdev_create, 653 .remove = vfio_ap_mdev_remove, 654 }; 655 656 int vfio_ap_mdev_register(void) 657 { 658 atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT); 659 660 return mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops); 661 } 662 663 void vfio_ap_mdev_unregister(void) 664 { 665 mdev_unregister_device(&matrix_dev->device); 666 } 667