1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Adjunct processor matrix VFIO device driver callbacks. 4 * 5 * Copyright IBM Corp. 2018 6 * 7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com> 8 * Halil Pasic <pasic@linux.ibm.com> 9 * Pierre Morel <pmorel@linux.ibm.com> 10 */ 11 #include <linux/string.h> 12 #include <linux/vfio.h> 13 #include <linux/device.h> 14 #include <linux/list.h> 15 #include <linux/ctype.h> 16 #include <linux/bitops.h> 17 #include <linux/kvm_host.h> 18 #include <linux/module.h> 19 #include <linux/uuid.h> 20 #include <asm/kvm.h> 21 #include <asm/zcrypt.h> 22 23 #include "vfio_ap_private.h" 24 #include "vfio_ap_debug.h" 25 26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough" 27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device" 28 29 #define AP_QUEUE_ASSIGNED "assigned" 30 #define AP_QUEUE_UNASSIGNED "unassigned" 31 #define AP_QUEUE_IN_USE "in use" 32 33 #define AP_RESET_INTERVAL 20 /* Reset sleep interval (20ms) */ 34 35 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable); 36 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); 37 static const struct vfio_device_ops vfio_ap_matrix_dev_ops; 38 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); 39 40 /** 41 * get_update_locks_for_kvm: Acquire the locks required to dynamically update a 42 * KVM guest's APCB in the proper order. 43 * 44 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 45 * 46 * The proper locking order is: 47 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 48 * guest's APCB. 49 * 2. kvm->lock: required to update a guest's APCB 50 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 51 * 52 * Note: If @kvm is NULL, the KVM lock will not be taken. 53 */ 54 static inline void get_update_locks_for_kvm(struct kvm *kvm) 55 { 56 mutex_lock(&matrix_dev->guests_lock); 57 if (kvm) 58 mutex_lock(&kvm->lock); 59 mutex_lock(&matrix_dev->mdevs_lock); 60 } 61 62 /** 63 * release_update_locks_for_kvm: Release the locks used to dynamically update a 64 * KVM guest's APCB in the proper order. 65 * 66 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 67 * 68 * The proper unlocking order is: 69 * 1. matrix_dev->mdevs_lock 70 * 2. kvm->lock 71 * 3. matrix_dev->guests_lock 72 * 73 * Note: If @kvm is NULL, the KVM lock will not be released. 74 */ 75 static inline void release_update_locks_for_kvm(struct kvm *kvm) 76 { 77 mutex_unlock(&matrix_dev->mdevs_lock); 78 if (kvm) 79 mutex_unlock(&kvm->lock); 80 mutex_unlock(&matrix_dev->guests_lock); 81 } 82 83 /** 84 * get_update_locks_for_mdev: Acquire the locks required to dynamically update a 85 * KVM guest's APCB in the proper order. 86 * 87 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 88 * configuration data to use to update a KVM guest's APCB. 89 * 90 * The proper locking order is: 91 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 92 * guest's APCB. 93 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 94 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 95 * 96 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 97 * lock will not be taken. 98 */ 99 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 100 { 101 mutex_lock(&matrix_dev->guests_lock); 102 if (matrix_mdev && matrix_mdev->kvm) 103 mutex_lock(&matrix_mdev->kvm->lock); 104 mutex_lock(&matrix_dev->mdevs_lock); 105 } 106 107 /** 108 * release_update_locks_for_mdev: Release the locks used to dynamically update a 109 * KVM guest's APCB in the proper order. 110 * 111 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 112 * configuration data to use to update a KVM guest's APCB. 113 * 114 * The proper unlocking order is: 115 * 1. matrix_dev->mdevs_lock 116 * 2. matrix_mdev->kvm->lock 117 * 3. matrix_dev->guests_lock 118 * 119 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 120 * lock will not be released. 121 */ 122 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 123 { 124 mutex_unlock(&matrix_dev->mdevs_lock); 125 if (matrix_mdev && matrix_mdev->kvm) 126 mutex_unlock(&matrix_mdev->kvm->lock); 127 mutex_unlock(&matrix_dev->guests_lock); 128 } 129 130 /** 131 * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and 132 * acquire the locks required to update the APCB of 133 * the KVM guest to which the mdev is attached. 134 * 135 * @apqn: the APQN of a queue device. 136 * 137 * The proper locking order is: 138 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 139 * guest's APCB. 140 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 141 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 142 * 143 * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock 144 * will not be taken. 145 * 146 * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn 147 * is not assigned to an ap_matrix_mdev. 148 */ 149 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn) 150 { 151 struct ap_matrix_mdev *matrix_mdev; 152 153 mutex_lock(&matrix_dev->guests_lock); 154 155 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 156 if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) && 157 test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) { 158 if (matrix_mdev->kvm) 159 mutex_lock(&matrix_mdev->kvm->lock); 160 161 mutex_lock(&matrix_dev->mdevs_lock); 162 163 return matrix_mdev; 164 } 165 } 166 167 mutex_lock(&matrix_dev->mdevs_lock); 168 169 return NULL; 170 } 171 172 /** 173 * get_update_locks_for_queue: get the locks required to update the APCB of the 174 * KVM guest to which the matrix mdev linked to a 175 * vfio_ap_queue object is attached. 176 * 177 * @q: a pointer to a vfio_ap_queue object. 178 * 179 * The proper locking order is: 180 * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a 181 * KVM guest's APCB. 182 * 2. q->matrix_mdev->kvm->lock: required to update a guest's APCB 183 * 3. matrix_dev->mdevs_lock: required to access data stored in matrix_mdev 184 * 185 * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock 186 * will not be taken. 187 */ 188 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q) 189 { 190 mutex_lock(&matrix_dev->guests_lock); 191 if (q->matrix_mdev && q->matrix_mdev->kvm) 192 mutex_lock(&q->matrix_mdev->kvm->lock); 193 mutex_lock(&matrix_dev->mdevs_lock); 194 } 195 196 /** 197 * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a 198 * hash table of queues assigned to a matrix mdev 199 * @matrix_mdev: the matrix mdev 200 * @apqn: The APQN of a queue device 201 * 202 * Return: the pointer to the vfio_ap_queue struct representing the queue or 203 * NULL if the queue is not assigned to @matrix_mdev 204 */ 205 static struct vfio_ap_queue *vfio_ap_mdev_get_queue( 206 struct ap_matrix_mdev *matrix_mdev, 207 int apqn) 208 { 209 struct vfio_ap_queue *q; 210 211 hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode, 212 apqn) { 213 if (q && q->apqn == apqn) 214 return q; 215 } 216 217 return NULL; 218 } 219 220 /** 221 * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries 222 * @apqn: The AP Queue number 223 * 224 * Checks the IRQ bit for the status of this APQN using ap_tapq. 225 * Returns if the ap_tapq function succeeded and the bit is clear. 226 * Returns if ap_tapq function failed with invalid, deconfigured or 227 * checkstopped AP. 228 * Otherwise retries up to 5 times after waiting 20ms. 229 */ 230 static void vfio_ap_wait_for_irqclear(int apqn) 231 { 232 struct ap_queue_status status; 233 int retry = 5; 234 235 do { 236 status = ap_tapq(apqn, NULL); 237 switch (status.response_code) { 238 case AP_RESPONSE_NORMAL: 239 case AP_RESPONSE_RESET_IN_PROGRESS: 240 if (!status.irq_enabled) 241 return; 242 fallthrough; 243 case AP_RESPONSE_BUSY: 244 msleep(20); 245 break; 246 case AP_RESPONSE_Q_NOT_AVAIL: 247 case AP_RESPONSE_DECONFIGURED: 248 case AP_RESPONSE_CHECKSTOPPED: 249 default: 250 WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__, 251 status.response_code, apqn); 252 return; 253 } 254 } while (--retry); 255 256 WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n", 257 __func__, status.response_code, apqn); 258 } 259 260 /** 261 * vfio_ap_free_aqic_resources - free vfio_ap_queue resources 262 * @q: The vfio_ap_queue 263 * 264 * Unregisters the ISC in the GIB when the saved ISC not invalid. 265 * Unpins the guest's page holding the NIB when it exists. 266 * Resets the saved_iova and saved_isc to invalid values. 267 */ 268 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) 269 { 270 if (!q) 271 return; 272 if (q->saved_isc != VFIO_AP_ISC_INVALID && 273 !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { 274 kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); 275 q->saved_isc = VFIO_AP_ISC_INVALID; 276 } 277 if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { 278 vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1); 279 q->saved_iova = 0; 280 } 281 } 282 283 /** 284 * vfio_ap_irq_disable - disables and clears an ap_queue interrupt 285 * @q: The vfio_ap_queue 286 * 287 * Uses ap_aqic to disable the interruption and in case of success, reset 288 * in progress or IRQ disable command already proceeded: calls 289 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear 290 * and calls vfio_ap_free_aqic_resources() to free the resources associated 291 * with the AP interrupt handling. 292 * 293 * In the case the AP is busy, or a reset is in progress, 294 * retries after 20ms, up to 5 times. 295 * 296 * Returns if ap_aqic function failed with invalid, deconfigured or 297 * checkstopped AP. 298 * 299 * Return: &struct ap_queue_status 300 */ 301 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q) 302 { 303 union ap_qirq_ctrl aqic_gisa = { .value = 0 }; 304 struct ap_queue_status status; 305 int retries = 5; 306 307 do { 308 status = ap_aqic(q->apqn, aqic_gisa, 0); 309 switch (status.response_code) { 310 case AP_RESPONSE_OTHERWISE_CHANGED: 311 case AP_RESPONSE_NORMAL: 312 vfio_ap_wait_for_irqclear(q->apqn); 313 goto end_free; 314 case AP_RESPONSE_RESET_IN_PROGRESS: 315 case AP_RESPONSE_BUSY: 316 msleep(20); 317 break; 318 case AP_RESPONSE_Q_NOT_AVAIL: 319 case AP_RESPONSE_DECONFIGURED: 320 case AP_RESPONSE_CHECKSTOPPED: 321 case AP_RESPONSE_INVALID_ADDRESS: 322 default: 323 /* All cases in default means AP not operational */ 324 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 325 status.response_code); 326 goto end_free; 327 } 328 } while (retries--); 329 330 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 331 status.response_code); 332 end_free: 333 vfio_ap_free_aqic_resources(q); 334 return status; 335 } 336 337 /** 338 * vfio_ap_validate_nib - validate a notification indicator byte (nib) address. 339 * 340 * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction. 341 * @nib: the location for storing the nib address. 342 * 343 * When the PQAP(AQIC) instruction is executed, general register 2 contains the 344 * address of the notification indicator byte (nib) used for IRQ notification. 345 * This function parses and validates the nib from gr2. 346 * 347 * Return: returns zero if the nib address is a valid; otherwise, returns 348 * -EINVAL. 349 */ 350 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib) 351 { 352 *nib = vcpu->run->s.regs.gprs[2]; 353 354 if (!*nib) 355 return -EINVAL; 356 if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT))) 357 return -EINVAL; 358 359 return 0; 360 } 361 362 /** 363 * vfio_ap_irq_enable - Enable Interruption for a APQN 364 * 365 * @q: the vfio_ap_queue holding AQIC parameters 366 * @isc: the guest ISC to register with the GIB interface 367 * @vcpu: the vcpu object containing the registers specifying the parameters 368 * passed to the PQAP(AQIC) instruction. 369 * 370 * Pin the NIB saved in *q 371 * Register the guest ISC to GIB interface and retrieve the 372 * host ISC to issue the host side PQAP/AQIC 373 * 374 * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the 375 * vfio_pin_pages failed. 376 * 377 * Otherwise return the ap_queue_status returned by the ap_aqic(), 378 * all retry handling will be done by the guest. 379 * 380 * Return: &struct ap_queue_status 381 */ 382 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, 383 int isc, 384 struct kvm_vcpu *vcpu) 385 { 386 union ap_qirq_ctrl aqic_gisa = { .value = 0 }; 387 struct ap_queue_status status = {}; 388 struct kvm_s390_gisa *gisa; 389 struct page *h_page; 390 int nisc; 391 struct kvm *kvm; 392 phys_addr_t h_nib; 393 dma_addr_t nib; 394 int ret; 395 396 /* Verify that the notification indicator byte address is valid */ 397 if (vfio_ap_validate_nib(vcpu, &nib)) { 398 VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n", 399 __func__, &nib, q->apqn); 400 401 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 402 return status; 403 } 404 405 ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1, 406 IOMMU_READ | IOMMU_WRITE, &h_page); 407 switch (ret) { 408 case 1: 409 break; 410 default: 411 VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d," 412 "nib=%pad, apqn=%#04x\n", 413 __func__, ret, &nib, q->apqn); 414 415 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 416 return status; 417 } 418 419 kvm = q->matrix_mdev->kvm; 420 gisa = kvm->arch.gisa_int.origin; 421 422 h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK); 423 aqic_gisa.gisc = isc; 424 425 nisc = kvm_s390_gisc_register(kvm, isc); 426 if (nisc < 0) { 427 VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n", 428 __func__, nisc, isc, q->apqn); 429 430 status.response_code = AP_RESPONSE_INVALID_GISA; 431 return status; 432 } 433 434 aqic_gisa.isc = nisc; 435 aqic_gisa.ir = 1; 436 aqic_gisa.gisa = virt_to_phys(gisa) >> 4; 437 438 status = ap_aqic(q->apqn, aqic_gisa, h_nib); 439 switch (status.response_code) { 440 case AP_RESPONSE_NORMAL: 441 /* See if we did clear older IRQ configuration */ 442 vfio_ap_free_aqic_resources(q); 443 q->saved_iova = nib; 444 q->saved_isc = isc; 445 break; 446 case AP_RESPONSE_OTHERWISE_CHANGED: 447 /* We could not modify IRQ settings: clear new configuration */ 448 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); 449 kvm_s390_gisc_unregister(kvm, isc); 450 break; 451 default: 452 pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn, 453 status.response_code); 454 vfio_ap_irq_disable(q); 455 break; 456 } 457 458 if (status.response_code != AP_RESPONSE_NORMAL) { 459 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: " 460 "zone=%#x, ir=%#x, gisc=%#x, f=%#x," 461 "gisa=%#x, isc=%#x, apqn=%#04x\n", 462 __func__, status.response_code, 463 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc, 464 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc, 465 q->apqn); 466 } 467 468 return status; 469 } 470 471 /** 472 * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array 473 * of big endian elements that can be passed by 474 * value to an s390dbf sprintf event function to 475 * format a UUID string. 476 * 477 * @guid: the object containing the little endian guid 478 * @uuid: a six-element array of long values that can be passed by value as 479 * arguments for a formatting string specifying a UUID. 480 * 481 * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf 482 * event functions if the memory for the passed string is available as long as 483 * the debug feature exists. Since a mediated device can be removed at any 484 * time, it's name can not be used because %s passes the reference to the string 485 * in memory and the reference will go stale once the device is removed . 486 * 487 * The s390dbf string formatting function allows a maximum of 9 arguments for a 488 * message to be displayed in the 'sprintf' view. In order to use the bytes 489 * comprising the mediated device's UUID to display the mediated device name, 490 * they will have to be converted into an array whose elements can be passed by 491 * value to sprintf. For example: 492 * 493 * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 } 494 * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804 495 * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 } 496 * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx" 497 */ 498 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid) 499 { 500 /* 501 * The input guid is ordered in little endian, so it needs to be 502 * reordered for displaying a UUID as a string. This specifies the 503 * guid indices in proper order. 504 */ 505 uuid[0] = le32_to_cpup((__le32 *)guid); 506 uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]); 507 uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]); 508 uuid[3] = *((__u16 *)&guid->b[8]); 509 uuid[4] = *((__u16 *)&guid->b[10]); 510 uuid[5] = *((__u32 *)&guid->b[12]); 511 } 512 513 /** 514 * handle_pqap - PQAP instruction callback 515 * 516 * @vcpu: The vcpu on which we received the PQAP instruction 517 * 518 * Get the general register contents to initialize internal variables. 519 * REG[0]: APQN 520 * REG[1]: IR and ISC 521 * REG[2]: NIB 522 * 523 * Response.status may be set to following Response Code: 524 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available 525 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured 526 * - AP_RESPONSE_NORMAL (0) : in case of success 527 * Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC. 528 * We take the matrix_dev lock to ensure serialization on queues and 529 * mediated device access. 530 * 531 * Return: 0 if we could handle the request inside KVM. 532 * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault. 533 */ 534 static int handle_pqap(struct kvm_vcpu *vcpu) 535 { 536 uint64_t status; 537 uint16_t apqn; 538 unsigned long uuid[6]; 539 struct vfio_ap_queue *q; 540 struct ap_queue_status qstatus = { 541 .response_code = AP_RESPONSE_Q_NOT_AVAIL, }; 542 struct ap_matrix_mdev *matrix_mdev; 543 544 apqn = vcpu->run->s.regs.gprs[0] & 0xffff; 545 546 /* If we do not use the AIV facility just go to userland */ 547 if (!(vcpu->arch.sie_block->eca & ECA_AIV)) { 548 VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n", 549 __func__, apqn, vcpu->arch.sie_block->eca); 550 551 return -EOPNOTSUPP; 552 } 553 554 mutex_lock(&matrix_dev->mdevs_lock); 555 556 if (!vcpu->kvm->arch.crypto.pqap_hook) { 557 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n", 558 __func__, apqn); 559 560 goto out_unlock; 561 } 562 563 matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook, 564 struct ap_matrix_mdev, pqap_hook); 565 566 /* If the there is no guest using the mdev, there is nothing to do */ 567 if (!matrix_mdev->kvm) { 568 vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid); 569 VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n", 570 __func__, uuid[0], uuid[1], uuid[2], 571 uuid[3], uuid[4], uuid[5], apqn); 572 goto out_unlock; 573 } 574 575 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); 576 if (!q) { 577 VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n", 578 __func__, AP_QID_CARD(apqn), 579 AP_QID_QUEUE(apqn)); 580 goto out_unlock; 581 } 582 583 status = vcpu->run->s.regs.gprs[1]; 584 585 /* If IR bit(16) is set we enable the interrupt */ 586 if ((status >> (63 - 16)) & 0x01) 587 qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu); 588 else 589 qstatus = vfio_ap_irq_disable(q); 590 591 out_unlock: 592 memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus)); 593 vcpu->run->s.regs.gprs[1] >>= 32; 594 mutex_unlock(&matrix_dev->mdevs_lock); 595 return 0; 596 } 597 598 static void vfio_ap_matrix_init(struct ap_config_info *info, 599 struct ap_matrix *matrix) 600 { 601 matrix->apm_max = info->apxa ? info->na : 63; 602 matrix->aqm_max = info->apxa ? info->nd : 15; 603 matrix->adm_max = info->apxa ? info->nd : 15; 604 } 605 606 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev) 607 { 608 if (matrix_mdev->kvm) 609 kvm_arch_crypto_set_masks(matrix_mdev->kvm, 610 matrix_mdev->shadow_apcb.apm, 611 matrix_mdev->shadow_apcb.aqm, 612 matrix_mdev->shadow_apcb.adm); 613 } 614 615 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) 616 { 617 DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS); 618 619 bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS); 620 bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm, 621 (unsigned long *)matrix_dev->info.adm, AP_DOMAINS); 622 623 return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, 624 AP_DOMAINS); 625 } 626 627 /* 628 * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev 629 * to ensure no queue devices are passed through to 630 * the guest that are not bound to the vfio_ap 631 * device driver. 632 * 633 * @matrix_mdev: the matrix mdev whose matrix is to be filtered. 634 * 635 * Note: If an APQN referencing a queue device that is not bound to the vfio_ap 636 * driver, its APID will be filtered from the guest's APCB. The matrix 637 * structure precludes filtering an individual APQN, so its APID will be 638 * filtered. 639 * 640 * Return: a boolean value indicating whether the KVM guest's APCB was changed 641 * by the filtering or not. 642 */ 643 static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, 644 struct ap_matrix_mdev *matrix_mdev) 645 { 646 unsigned long apid, apqi, apqn; 647 DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); 648 DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); 649 struct vfio_ap_queue *q; 650 651 bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); 652 bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); 653 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 654 655 /* 656 * Copy the adapters, domains and control domains to the shadow_apcb 657 * from the matrix mdev, but only those that are assigned to the host's 658 * AP configuration. 659 */ 660 bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm, 661 (unsigned long *)matrix_dev->info.apm, AP_DEVICES); 662 bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm, 663 (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS); 664 665 for_each_set_bit_inv(apid, apm, AP_DEVICES) { 666 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { 667 /* 668 * If the APQN is not bound to the vfio_ap device 669 * driver, then we can't assign it to the guest's 670 * AP configuration. The AP architecture won't 671 * allow filtering of a single APQN, so let's filter 672 * the APID since an adapter represents a physical 673 * hardware device. 674 */ 675 apqn = AP_MKQID(apid, apqi); 676 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); 677 if (!q || q->reset_rc) { 678 clear_bit_inv(apid, 679 matrix_mdev->shadow_apcb.apm); 680 break; 681 } 682 } 683 } 684 685 return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, 686 AP_DEVICES) || 687 !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, 688 AP_DOMAINS); 689 } 690 691 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev) 692 { 693 struct ap_matrix_mdev *matrix_mdev = 694 container_of(vdev, struct ap_matrix_mdev, vdev); 695 696 matrix_mdev->mdev = to_mdev_device(vdev->dev); 697 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); 698 matrix_mdev->pqap_hook = handle_pqap; 699 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 700 hash_init(matrix_mdev->qtable.queues); 701 702 return 0; 703 } 704 705 static int vfio_ap_mdev_probe(struct mdev_device *mdev) 706 { 707 struct ap_matrix_mdev *matrix_mdev; 708 int ret; 709 710 matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev, 711 &vfio_ap_matrix_dev_ops); 712 if (IS_ERR(matrix_mdev)) 713 return PTR_ERR(matrix_mdev); 714 715 ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); 716 if (ret) 717 goto err_put_vdev; 718 matrix_mdev->req_trigger = NULL; 719 dev_set_drvdata(&mdev->dev, matrix_mdev); 720 mutex_lock(&matrix_dev->mdevs_lock); 721 list_add(&matrix_mdev->node, &matrix_dev->mdev_list); 722 mutex_unlock(&matrix_dev->mdevs_lock); 723 return 0; 724 725 err_put_vdev: 726 vfio_put_device(&matrix_mdev->vdev); 727 return ret; 728 } 729 730 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, 731 struct vfio_ap_queue *q) 732 { 733 if (q) { 734 q->matrix_mdev = matrix_mdev; 735 hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); 736 } 737 } 738 739 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) 740 { 741 struct vfio_ap_queue *q; 742 743 q = vfio_ap_find_queue(apqn); 744 vfio_ap_mdev_link_queue(matrix_mdev, q); 745 } 746 747 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q) 748 { 749 hash_del(&q->mdev_qnode); 750 } 751 752 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q) 753 { 754 q->matrix_mdev = NULL; 755 } 756 757 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) 758 { 759 struct vfio_ap_queue *q; 760 unsigned long apid, apqi; 761 762 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 763 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, 764 AP_DOMAINS) { 765 q = vfio_ap_mdev_get_queue(matrix_mdev, 766 AP_MKQID(apid, apqi)); 767 if (q) 768 q->matrix_mdev = NULL; 769 } 770 } 771 } 772 773 static void vfio_ap_mdev_remove(struct mdev_device *mdev) 774 { 775 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev); 776 777 vfio_unregister_group_dev(&matrix_mdev->vdev); 778 779 mutex_lock(&matrix_dev->guests_lock); 780 mutex_lock(&matrix_dev->mdevs_lock); 781 vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 782 vfio_ap_mdev_unlink_fr_queues(matrix_mdev); 783 list_del(&matrix_mdev->node); 784 mutex_unlock(&matrix_dev->mdevs_lock); 785 mutex_unlock(&matrix_dev->guests_lock); 786 vfio_put_device(&matrix_mdev->vdev); 787 } 788 789 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ 790 "already assigned to %s" 791 792 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, 793 unsigned long *apm, 794 unsigned long *aqm) 795 { 796 unsigned long apid, apqi; 797 const struct device *dev = mdev_dev(matrix_mdev->mdev); 798 const char *mdev_name = dev_name(dev); 799 800 for_each_set_bit_inv(apid, apm, AP_DEVICES) 801 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) 802 dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); 803 } 804 805 /** 806 * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs 807 * 808 * @mdev_apm: mask indicating the APIDs of the APQNs to be verified 809 * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified 810 * 811 * Verifies that each APQN derived from the Cartesian product of a bitmap of 812 * AP adapter IDs and AP queue indexes is not configured for any matrix 813 * mediated device. AP queue sharing is not allowed. 814 * 815 * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. 816 */ 817 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, 818 unsigned long *mdev_aqm) 819 { 820 struct ap_matrix_mdev *matrix_mdev; 821 DECLARE_BITMAP(apm, AP_DEVICES); 822 DECLARE_BITMAP(aqm, AP_DOMAINS); 823 824 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 825 /* 826 * If the input apm and aqm are fields of the matrix_mdev 827 * object, then move on to the next matrix_mdev. 828 */ 829 if (mdev_apm == matrix_mdev->matrix.apm && 830 mdev_aqm == matrix_mdev->matrix.aqm) 831 continue; 832 833 memset(apm, 0, sizeof(apm)); 834 memset(aqm, 0, sizeof(aqm)); 835 836 /* 837 * We work on full longs, as we can only exclude the leftover 838 * bits in non-inverse order. The leftover is all zeros. 839 */ 840 if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, 841 AP_DEVICES)) 842 continue; 843 844 if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, 845 AP_DOMAINS)) 846 continue; 847 848 vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); 849 850 return -EADDRINUSE; 851 } 852 853 return 0; 854 } 855 856 /** 857 * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are 858 * not reserved for the default zcrypt driver and 859 * are not assigned to another mdev. 860 * 861 * @matrix_mdev: the mdev to which the APQNs being validated are assigned. 862 * 863 * Return: One of the following values: 864 * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, 865 * most likely -EBUSY indicating the ap_perms_mutex lock is already held. 866 * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the 867 * zcrypt default driver. 868 * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev 869 * o A zero indicating validation succeeded. 870 */ 871 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) 872 { 873 if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm, 874 matrix_mdev->matrix.aqm)) 875 return -EADDRNOTAVAIL; 876 877 return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, 878 matrix_mdev->matrix.aqm); 879 } 880 881 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, 882 unsigned long apid) 883 { 884 unsigned long apqi; 885 886 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) 887 vfio_ap_mdev_link_apqn(matrix_mdev, 888 AP_MKQID(apid, apqi)); 889 } 890 891 /** 892 * assign_adapter_store - parses the APID from @buf and sets the 893 * corresponding bit in the mediated matrix device's APM 894 * 895 * @dev: the matrix device 896 * @attr: the mediated matrix device's assign_adapter attribute 897 * @buf: a buffer containing the AP adapter number (APID) to 898 * be assigned 899 * @count: the number of bytes in @buf 900 * 901 * Return: the number of bytes processed if the APID is valid; otherwise, 902 * returns one of the following errors: 903 * 904 * 1. -EINVAL 905 * The APID is not a valid number 906 * 907 * 2. -ENODEV 908 * The APID exceeds the maximum value configured for the system 909 * 910 * 3. -EADDRNOTAVAIL 911 * An APQN derived from the cross product of the APID being assigned 912 * and the APQIs previously assigned is not bound to the vfio_ap device 913 * driver; or, if no APQIs have yet been assigned, the APID is not 914 * contained in an APQN bound to the vfio_ap device driver. 915 * 916 * 4. -EADDRINUSE 917 * An APQN derived from the cross product of the APID being assigned 918 * and the APQIs previously assigned is being used by another mediated 919 * matrix device 920 * 921 * 5. -EAGAIN 922 * A lock required to validate the mdev's AP configuration could not 923 * be obtained. 924 */ 925 static ssize_t assign_adapter_store(struct device *dev, 926 struct device_attribute *attr, 927 const char *buf, size_t count) 928 { 929 int ret; 930 unsigned long apid; 931 DECLARE_BITMAP(apm_delta, AP_DEVICES); 932 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 933 934 mutex_lock(&ap_perms_mutex); 935 get_update_locks_for_mdev(matrix_mdev); 936 937 ret = kstrtoul(buf, 0, &apid); 938 if (ret) 939 goto done; 940 941 if (apid > matrix_mdev->matrix.apm_max) { 942 ret = -ENODEV; 943 goto done; 944 } 945 946 if (test_bit_inv(apid, matrix_mdev->matrix.apm)) { 947 ret = count; 948 goto done; 949 } 950 951 set_bit_inv(apid, matrix_mdev->matrix.apm); 952 953 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 954 if (ret) { 955 clear_bit_inv(apid, matrix_mdev->matrix.apm); 956 goto done; 957 } 958 959 vfio_ap_mdev_link_adapter(matrix_mdev, apid); 960 memset(apm_delta, 0, sizeof(apm_delta)); 961 set_bit_inv(apid, apm_delta); 962 963 if (vfio_ap_mdev_filter_matrix(apm_delta, 964 matrix_mdev->matrix.aqm, matrix_mdev)) 965 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 966 967 ret = count; 968 done: 969 release_update_locks_for_mdev(matrix_mdev); 970 mutex_unlock(&ap_perms_mutex); 971 972 return ret; 973 } 974 static DEVICE_ATTR_WO(assign_adapter); 975 976 static struct vfio_ap_queue 977 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev, 978 unsigned long apid, unsigned long apqi) 979 { 980 struct vfio_ap_queue *q = NULL; 981 982 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); 983 /* If the queue is assigned to the matrix mdev, unlink it. */ 984 if (q) 985 vfio_ap_unlink_queue_fr_mdev(q); 986 987 return q; 988 } 989 990 /** 991 * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned 992 * adapter from the matrix mdev to which the 993 * adapter was assigned. 994 * @matrix_mdev: the matrix mediated device to which the adapter was assigned. 995 * @apid: the APID of the unassigned adapter. 996 * @qtable: table for storing queues associated with unassigned adapter. 997 */ 998 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, 999 unsigned long apid, 1000 struct ap_queue_table *qtable) 1001 { 1002 unsigned long apqi; 1003 struct vfio_ap_queue *q; 1004 1005 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { 1006 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1007 1008 if (q && qtable) { 1009 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1010 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1011 hash_add(qtable->queues, &q->mdev_qnode, 1012 q->apqn); 1013 } 1014 } 1015 } 1016 1017 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, 1018 unsigned long apid) 1019 { 1020 int loop_cursor; 1021 struct vfio_ap_queue *q; 1022 struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); 1023 1024 hash_init(qtable->queues); 1025 vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable); 1026 1027 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { 1028 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 1029 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1030 } 1031 1032 vfio_ap_mdev_reset_queues(qtable); 1033 1034 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1035 vfio_ap_unlink_mdev_fr_queue(q); 1036 hash_del(&q->mdev_qnode); 1037 } 1038 1039 kfree(qtable); 1040 } 1041 1042 /** 1043 * unassign_adapter_store - parses the APID from @buf and clears the 1044 * corresponding bit in the mediated matrix device's APM 1045 * 1046 * @dev: the matrix device 1047 * @attr: the mediated matrix device's unassign_adapter attribute 1048 * @buf: a buffer containing the adapter number (APID) to be unassigned 1049 * @count: the number of bytes in @buf 1050 * 1051 * Return: the number of bytes processed if the APID is valid; otherwise, 1052 * returns one of the following errors: 1053 * -EINVAL if the APID is not a number 1054 * -ENODEV if the APID it exceeds the maximum value configured for the 1055 * system 1056 */ 1057 static ssize_t unassign_adapter_store(struct device *dev, 1058 struct device_attribute *attr, 1059 const char *buf, size_t count) 1060 { 1061 int ret; 1062 unsigned long apid; 1063 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1064 1065 get_update_locks_for_mdev(matrix_mdev); 1066 1067 ret = kstrtoul(buf, 0, &apid); 1068 if (ret) 1069 goto done; 1070 1071 if (apid > matrix_mdev->matrix.apm_max) { 1072 ret = -ENODEV; 1073 goto done; 1074 } 1075 1076 if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) { 1077 ret = count; 1078 goto done; 1079 } 1080 1081 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); 1082 vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid); 1083 ret = count; 1084 done: 1085 release_update_locks_for_mdev(matrix_mdev); 1086 return ret; 1087 } 1088 static DEVICE_ATTR_WO(unassign_adapter); 1089 1090 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, 1091 unsigned long apqi) 1092 { 1093 unsigned long apid; 1094 1095 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) 1096 vfio_ap_mdev_link_apqn(matrix_mdev, 1097 AP_MKQID(apid, apqi)); 1098 } 1099 1100 /** 1101 * assign_domain_store - parses the APQI from @buf and sets the 1102 * corresponding bit in the mediated matrix device's AQM 1103 * 1104 * @dev: the matrix device 1105 * @attr: the mediated matrix device's assign_domain attribute 1106 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1107 * be assigned 1108 * @count: the number of bytes in @buf 1109 * 1110 * Return: the number of bytes processed if the APQI is valid; otherwise returns 1111 * one of the following errors: 1112 * 1113 * 1. -EINVAL 1114 * The APQI is not a valid number 1115 * 1116 * 2. -ENODEV 1117 * The APQI exceeds the maximum value configured for the system 1118 * 1119 * 3. -EADDRNOTAVAIL 1120 * An APQN derived from the cross product of the APQI being assigned 1121 * and the APIDs previously assigned is not bound to the vfio_ap device 1122 * driver; or, if no APIDs have yet been assigned, the APQI is not 1123 * contained in an APQN bound to the vfio_ap device driver. 1124 * 1125 * 4. -EADDRINUSE 1126 * An APQN derived from the cross product of the APQI being assigned 1127 * and the APIDs previously assigned is being used by another mediated 1128 * matrix device 1129 * 1130 * 5. -EAGAIN 1131 * The lock required to validate the mdev's AP configuration could not 1132 * be obtained. 1133 */ 1134 static ssize_t assign_domain_store(struct device *dev, 1135 struct device_attribute *attr, 1136 const char *buf, size_t count) 1137 { 1138 int ret; 1139 unsigned long apqi; 1140 DECLARE_BITMAP(aqm_delta, AP_DOMAINS); 1141 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1142 1143 mutex_lock(&ap_perms_mutex); 1144 get_update_locks_for_mdev(matrix_mdev); 1145 1146 ret = kstrtoul(buf, 0, &apqi); 1147 if (ret) 1148 goto done; 1149 1150 if (apqi > matrix_mdev->matrix.aqm_max) { 1151 ret = -ENODEV; 1152 goto done; 1153 } 1154 1155 if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { 1156 ret = count; 1157 goto done; 1158 } 1159 1160 set_bit_inv(apqi, matrix_mdev->matrix.aqm); 1161 1162 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 1163 if (ret) { 1164 clear_bit_inv(apqi, matrix_mdev->matrix.aqm); 1165 goto done; 1166 } 1167 1168 vfio_ap_mdev_link_domain(matrix_mdev, apqi); 1169 memset(aqm_delta, 0, sizeof(aqm_delta)); 1170 set_bit_inv(apqi, aqm_delta); 1171 1172 if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, 1173 matrix_mdev)) 1174 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1175 1176 ret = count; 1177 done: 1178 release_update_locks_for_mdev(matrix_mdev); 1179 mutex_unlock(&ap_perms_mutex); 1180 1181 return ret; 1182 } 1183 static DEVICE_ATTR_WO(assign_domain); 1184 1185 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, 1186 unsigned long apqi, 1187 struct ap_queue_table *qtable) 1188 { 1189 unsigned long apid; 1190 struct vfio_ap_queue *q; 1191 1192 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 1193 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1194 1195 if (q && qtable) { 1196 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1197 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1198 hash_add(qtable->queues, &q->mdev_qnode, 1199 q->apqn); 1200 } 1201 } 1202 } 1203 1204 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, 1205 unsigned long apqi) 1206 { 1207 int loop_cursor; 1208 struct vfio_ap_queue *q; 1209 struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); 1210 1211 hash_init(qtable->queues); 1212 vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable); 1213 1214 if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 1215 clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); 1216 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1217 } 1218 1219 vfio_ap_mdev_reset_queues(qtable); 1220 1221 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1222 vfio_ap_unlink_mdev_fr_queue(q); 1223 hash_del(&q->mdev_qnode); 1224 } 1225 1226 kfree(qtable); 1227 } 1228 1229 /** 1230 * unassign_domain_store - parses the APQI from @buf and clears the 1231 * corresponding bit in the mediated matrix device's AQM 1232 * 1233 * @dev: the matrix device 1234 * @attr: the mediated matrix device's unassign_domain attribute 1235 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1236 * be unassigned 1237 * @count: the number of bytes in @buf 1238 * 1239 * Return: the number of bytes processed if the APQI is valid; otherwise, 1240 * returns one of the following errors: 1241 * -EINVAL if the APQI is not a number 1242 * -ENODEV if the APQI exceeds the maximum value configured for the system 1243 */ 1244 static ssize_t unassign_domain_store(struct device *dev, 1245 struct device_attribute *attr, 1246 const char *buf, size_t count) 1247 { 1248 int ret; 1249 unsigned long apqi; 1250 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1251 1252 get_update_locks_for_mdev(matrix_mdev); 1253 1254 ret = kstrtoul(buf, 0, &apqi); 1255 if (ret) 1256 goto done; 1257 1258 if (apqi > matrix_mdev->matrix.aqm_max) { 1259 ret = -ENODEV; 1260 goto done; 1261 } 1262 1263 if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { 1264 ret = count; 1265 goto done; 1266 } 1267 1268 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); 1269 vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi); 1270 ret = count; 1271 1272 done: 1273 release_update_locks_for_mdev(matrix_mdev); 1274 return ret; 1275 } 1276 static DEVICE_ATTR_WO(unassign_domain); 1277 1278 /** 1279 * assign_control_domain_store - parses the domain ID from @buf and sets 1280 * the corresponding bit in the mediated matrix device's ADM 1281 * 1282 * @dev: the matrix device 1283 * @attr: the mediated matrix device's assign_control_domain attribute 1284 * @buf: a buffer containing the domain ID to be assigned 1285 * @count: the number of bytes in @buf 1286 * 1287 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1288 * returns one of the following errors: 1289 * -EINVAL if the ID is not a number 1290 * -ENODEV if the ID exceeds the maximum value configured for the system 1291 */ 1292 static ssize_t assign_control_domain_store(struct device *dev, 1293 struct device_attribute *attr, 1294 const char *buf, size_t count) 1295 { 1296 int ret; 1297 unsigned long id; 1298 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1299 1300 get_update_locks_for_mdev(matrix_mdev); 1301 1302 ret = kstrtoul(buf, 0, &id); 1303 if (ret) 1304 goto done; 1305 1306 if (id > matrix_mdev->matrix.adm_max) { 1307 ret = -ENODEV; 1308 goto done; 1309 } 1310 1311 if (test_bit_inv(id, matrix_mdev->matrix.adm)) { 1312 ret = count; 1313 goto done; 1314 } 1315 1316 /* Set the bit in the ADM (bitmask) corresponding to the AP control 1317 * domain number (id). The bits in the mask, from most significant to 1318 * least significant, correspond to IDs 0 up to the one less than the 1319 * number of control domains that can be assigned. 1320 */ 1321 set_bit_inv(id, matrix_mdev->matrix.adm); 1322 if (vfio_ap_mdev_filter_cdoms(matrix_mdev)) 1323 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1324 1325 ret = count; 1326 done: 1327 release_update_locks_for_mdev(matrix_mdev); 1328 return ret; 1329 } 1330 static DEVICE_ATTR_WO(assign_control_domain); 1331 1332 /** 1333 * unassign_control_domain_store - parses the domain ID from @buf and 1334 * clears the corresponding bit in the mediated matrix device's ADM 1335 * 1336 * @dev: the matrix device 1337 * @attr: the mediated matrix device's unassign_control_domain attribute 1338 * @buf: a buffer containing the domain ID to be unassigned 1339 * @count: the number of bytes in @buf 1340 * 1341 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1342 * returns one of the following errors: 1343 * -EINVAL if the ID is not a number 1344 * -ENODEV if the ID exceeds the maximum value configured for the system 1345 */ 1346 static ssize_t unassign_control_domain_store(struct device *dev, 1347 struct device_attribute *attr, 1348 const char *buf, size_t count) 1349 { 1350 int ret; 1351 unsigned long domid; 1352 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1353 1354 get_update_locks_for_mdev(matrix_mdev); 1355 1356 ret = kstrtoul(buf, 0, &domid); 1357 if (ret) 1358 goto done; 1359 1360 if (domid > matrix_mdev->matrix.adm_max) { 1361 ret = -ENODEV; 1362 goto done; 1363 } 1364 1365 if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) { 1366 ret = count; 1367 goto done; 1368 } 1369 1370 clear_bit_inv(domid, matrix_mdev->matrix.adm); 1371 1372 if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) { 1373 clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm); 1374 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1375 } 1376 1377 ret = count; 1378 done: 1379 release_update_locks_for_mdev(matrix_mdev); 1380 return ret; 1381 } 1382 static DEVICE_ATTR_WO(unassign_control_domain); 1383 1384 static ssize_t control_domains_show(struct device *dev, 1385 struct device_attribute *dev_attr, 1386 char *buf) 1387 { 1388 unsigned long id; 1389 int nchars = 0; 1390 int n; 1391 char *bufpos = buf; 1392 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1393 unsigned long max_domid = matrix_mdev->matrix.adm_max; 1394 1395 mutex_lock(&matrix_dev->mdevs_lock); 1396 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) { 1397 n = sprintf(bufpos, "%04lx\n", id); 1398 bufpos += n; 1399 nchars += n; 1400 } 1401 mutex_unlock(&matrix_dev->mdevs_lock); 1402 1403 return nchars; 1404 } 1405 static DEVICE_ATTR_RO(control_domains); 1406 1407 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) 1408 { 1409 char *bufpos = buf; 1410 unsigned long apid; 1411 unsigned long apqi; 1412 unsigned long apid1; 1413 unsigned long apqi1; 1414 unsigned long napm_bits = matrix->apm_max + 1; 1415 unsigned long naqm_bits = matrix->aqm_max + 1; 1416 int nchars = 0; 1417 int n; 1418 1419 apid1 = find_first_bit_inv(matrix->apm, napm_bits); 1420 apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits); 1421 1422 if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { 1423 for_each_set_bit_inv(apid, matrix->apm, napm_bits) { 1424 for_each_set_bit_inv(apqi, matrix->aqm, 1425 naqm_bits) { 1426 n = sprintf(bufpos, "%02lx.%04lx\n", apid, 1427 apqi); 1428 bufpos += n; 1429 nchars += n; 1430 } 1431 } 1432 } else if (apid1 < napm_bits) { 1433 for_each_set_bit_inv(apid, matrix->apm, napm_bits) { 1434 n = sprintf(bufpos, "%02lx.\n", apid); 1435 bufpos += n; 1436 nchars += n; 1437 } 1438 } else if (apqi1 < naqm_bits) { 1439 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) { 1440 n = sprintf(bufpos, ".%04lx\n", apqi); 1441 bufpos += n; 1442 nchars += n; 1443 } 1444 } 1445 1446 return nchars; 1447 } 1448 1449 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, 1450 char *buf) 1451 { 1452 ssize_t nchars; 1453 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1454 1455 mutex_lock(&matrix_dev->mdevs_lock); 1456 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf); 1457 mutex_unlock(&matrix_dev->mdevs_lock); 1458 1459 return nchars; 1460 } 1461 static DEVICE_ATTR_RO(matrix); 1462 1463 static ssize_t guest_matrix_show(struct device *dev, 1464 struct device_attribute *attr, char *buf) 1465 { 1466 ssize_t nchars; 1467 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1468 1469 mutex_lock(&matrix_dev->mdevs_lock); 1470 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf); 1471 mutex_unlock(&matrix_dev->mdevs_lock); 1472 1473 return nchars; 1474 } 1475 static DEVICE_ATTR_RO(guest_matrix); 1476 1477 static struct attribute *vfio_ap_mdev_attrs[] = { 1478 &dev_attr_assign_adapter.attr, 1479 &dev_attr_unassign_adapter.attr, 1480 &dev_attr_assign_domain.attr, 1481 &dev_attr_unassign_domain.attr, 1482 &dev_attr_assign_control_domain.attr, 1483 &dev_attr_unassign_control_domain.attr, 1484 &dev_attr_control_domains.attr, 1485 &dev_attr_matrix.attr, 1486 &dev_attr_guest_matrix.attr, 1487 NULL, 1488 }; 1489 1490 static struct attribute_group vfio_ap_mdev_attr_group = { 1491 .attrs = vfio_ap_mdev_attrs 1492 }; 1493 1494 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { 1495 &vfio_ap_mdev_attr_group, 1496 NULL 1497 }; 1498 1499 /** 1500 * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed 1501 * to manage AP resources for the guest whose state is represented by @kvm 1502 * 1503 * @matrix_mdev: a mediated matrix device 1504 * @kvm: reference to KVM instance 1505 * 1506 * Return: 0 if no other mediated matrix device has a reference to @kvm; 1507 * otherwise, returns an -EPERM. 1508 */ 1509 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, 1510 struct kvm *kvm) 1511 { 1512 struct ap_matrix_mdev *m; 1513 1514 if (kvm->arch.crypto.crycbd) { 1515 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1516 kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; 1517 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1518 1519 get_update_locks_for_kvm(kvm); 1520 1521 list_for_each_entry(m, &matrix_dev->mdev_list, node) { 1522 if (m != matrix_mdev && m->kvm == kvm) { 1523 release_update_locks_for_kvm(kvm); 1524 return -EPERM; 1525 } 1526 } 1527 1528 kvm_get_kvm(kvm); 1529 matrix_mdev->kvm = kvm; 1530 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1531 1532 release_update_locks_for_kvm(kvm); 1533 } 1534 1535 return 0; 1536 } 1537 1538 static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length) 1539 { 1540 struct ap_queue_table *qtable = &matrix_mdev->qtable; 1541 struct vfio_ap_queue *q; 1542 int loop_cursor; 1543 1544 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1545 if (q->saved_iova >= iova && q->saved_iova < iova + length) 1546 vfio_ap_irq_disable(q); 1547 } 1548 } 1549 1550 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova, 1551 u64 length) 1552 { 1553 struct ap_matrix_mdev *matrix_mdev = 1554 container_of(vdev, struct ap_matrix_mdev, vdev); 1555 1556 mutex_lock(&matrix_dev->mdevs_lock); 1557 1558 unmap_iova(matrix_mdev, iova, length); 1559 1560 mutex_unlock(&matrix_dev->mdevs_lock); 1561 } 1562 1563 /** 1564 * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed 1565 * by @matrix_mdev. 1566 * 1567 * @matrix_mdev: a matrix mediated device 1568 */ 1569 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) 1570 { 1571 struct kvm *kvm = matrix_mdev->kvm; 1572 1573 if (kvm && kvm->arch.crypto.crycbd) { 1574 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1575 kvm->arch.crypto.pqap_hook = NULL; 1576 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1577 1578 get_update_locks_for_kvm(kvm); 1579 1580 kvm_arch_crypto_clear_masks(kvm); 1581 vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 1582 kvm_put_kvm(kvm); 1583 matrix_mdev->kvm = NULL; 1584 1585 release_update_locks_for_kvm(kvm); 1586 } 1587 } 1588 1589 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn) 1590 { 1591 struct ap_queue *queue; 1592 struct vfio_ap_queue *q = NULL; 1593 1594 queue = ap_get_qdev(apqn); 1595 if (!queue) 1596 return NULL; 1597 1598 if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver) 1599 q = dev_get_drvdata(&queue->ap_dev.device); 1600 1601 put_device(&queue->ap_dev.device); 1602 1603 return q; 1604 } 1605 1606 static int apq_status_check(int apqn, struct ap_queue_status *status) 1607 { 1608 switch (status->response_code) { 1609 case AP_RESPONSE_NORMAL: 1610 case AP_RESPONSE_DECONFIGURED: 1611 return 0; 1612 case AP_RESPONSE_RESET_IN_PROGRESS: 1613 case AP_RESPONSE_BUSY: 1614 return -EBUSY; 1615 default: 1616 WARN(true, 1617 "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n", 1618 AP_QID_CARD(apqn), AP_QID_QUEUE(apqn), 1619 status->response_code); 1620 return -EIO; 1621 } 1622 } 1623 1624 #define WAIT_MSG "Waited %dms for reset of queue %02x.%04x (%u, %u, %u)" 1625 1626 static int apq_reset_check(struct vfio_ap_queue *q) 1627 { 1628 int ret = -EBUSY, elapsed = 0; 1629 struct ap_queue_status status; 1630 1631 while (true) { 1632 msleep(AP_RESET_INTERVAL); 1633 elapsed += AP_RESET_INTERVAL; 1634 status = ap_tapq(q->apqn, NULL); 1635 ret = apq_status_check(q->apqn, &status); 1636 if (ret == -EIO) 1637 return ret; 1638 if (ret == -EBUSY) { 1639 pr_notice_ratelimited(WAIT_MSG, elapsed, 1640 AP_QID_CARD(q->apqn), 1641 AP_QID_QUEUE(q->apqn), 1642 status.response_code, 1643 status.queue_empty, 1644 status.irq_enabled); 1645 } else { 1646 if (q->reset_rc == AP_RESPONSE_RESET_IN_PROGRESS || 1647 q->reset_rc == AP_RESPONSE_BUSY) { 1648 status = ap_zapq(q->apqn, 0); 1649 q->reset_rc = status.response_code; 1650 continue; 1651 } 1652 /* 1653 * When an AP adapter is deconfigured, the associated 1654 * queues are reset, so let's set the status response 1655 * code to 0 so the queue may be passed through (i.e., 1656 * not filtered). 1657 */ 1658 if (q->reset_rc == AP_RESPONSE_DECONFIGURED) 1659 q->reset_rc = 0; 1660 if (q->saved_isc != VFIO_AP_ISC_INVALID) 1661 vfio_ap_free_aqic_resources(q); 1662 break; 1663 } 1664 } 1665 return ret; 1666 } 1667 1668 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) 1669 { 1670 struct ap_queue_status status; 1671 int ret = 0; 1672 1673 if (!q) 1674 return 0; 1675 status = ap_zapq(q->apqn, 0); 1676 q->reset_rc = status.response_code; 1677 switch (status.response_code) { 1678 case AP_RESPONSE_NORMAL: 1679 case AP_RESPONSE_RESET_IN_PROGRESS: 1680 case AP_RESPONSE_BUSY: 1681 /* Let's verify whether the ZAPQ completed successfully */ 1682 ret = apq_reset_check(q); 1683 break; 1684 case AP_RESPONSE_DECONFIGURED: 1685 /* 1686 * When an AP adapter is deconfigured, the associated 1687 * queues are reset, so let's set the status response code to 0 1688 * so the queue may be passed through (i.e., not filtered) and 1689 * return a value indicating the reset completed successfully. 1690 */ 1691 q->reset_rc = 0; 1692 vfio_ap_free_aqic_resources(q); 1693 break; 1694 default: 1695 WARN(true, 1696 "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n", 1697 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), 1698 status.response_code); 1699 return -EIO; 1700 } 1701 1702 return ret; 1703 } 1704 1705 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable) 1706 { 1707 int ret, loop_cursor, rc = 0; 1708 struct vfio_ap_queue *q; 1709 1710 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1711 ret = vfio_ap_mdev_reset_queue(q); 1712 /* 1713 * Regardless whether a queue turns out to be busy, or 1714 * is not operational, we need to continue resetting 1715 * the remaining queues. 1716 */ 1717 if (ret) 1718 rc = ret; 1719 } 1720 1721 return rc; 1722 } 1723 1724 static int vfio_ap_mdev_open_device(struct vfio_device *vdev) 1725 { 1726 struct ap_matrix_mdev *matrix_mdev = 1727 container_of(vdev, struct ap_matrix_mdev, vdev); 1728 1729 if (!vdev->kvm) 1730 return -EINVAL; 1731 1732 return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm); 1733 } 1734 1735 static void vfio_ap_mdev_close_device(struct vfio_device *vdev) 1736 { 1737 struct ap_matrix_mdev *matrix_mdev = 1738 container_of(vdev, struct ap_matrix_mdev, vdev); 1739 1740 vfio_ap_mdev_unset_kvm(matrix_mdev); 1741 } 1742 1743 static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count) 1744 { 1745 struct device *dev = vdev->dev; 1746 struct ap_matrix_mdev *matrix_mdev; 1747 1748 matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); 1749 1750 if (matrix_mdev->req_trigger) { 1751 if (!(count % 10)) 1752 dev_notice_ratelimited(dev, 1753 "Relaying device request to user (#%u)\n", 1754 count); 1755 1756 eventfd_signal(matrix_mdev->req_trigger, 1); 1757 } else if (count == 0) { 1758 dev_notice(dev, 1759 "No device request registered, blocked until released by user\n"); 1760 } 1761 } 1762 1763 static int vfio_ap_mdev_get_device_info(unsigned long arg) 1764 { 1765 unsigned long minsz; 1766 struct vfio_device_info info; 1767 1768 minsz = offsetofend(struct vfio_device_info, num_irqs); 1769 1770 if (copy_from_user(&info, (void __user *)arg, minsz)) 1771 return -EFAULT; 1772 1773 if (info.argsz < minsz) 1774 return -EINVAL; 1775 1776 info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET; 1777 info.num_regions = 0; 1778 info.num_irqs = VFIO_AP_NUM_IRQS; 1779 1780 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 1781 } 1782 1783 static ssize_t vfio_ap_get_irq_info(unsigned long arg) 1784 { 1785 unsigned long minsz; 1786 struct vfio_irq_info info; 1787 1788 minsz = offsetofend(struct vfio_irq_info, count); 1789 1790 if (copy_from_user(&info, (void __user *)arg, minsz)) 1791 return -EFAULT; 1792 1793 if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS) 1794 return -EINVAL; 1795 1796 switch (info.index) { 1797 case VFIO_AP_REQ_IRQ_INDEX: 1798 info.count = 1; 1799 info.flags = VFIO_IRQ_INFO_EVENTFD; 1800 break; 1801 default: 1802 return -EINVAL; 1803 } 1804 1805 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 1806 } 1807 1808 static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg) 1809 { 1810 int ret; 1811 size_t data_size; 1812 unsigned long minsz; 1813 1814 minsz = offsetofend(struct vfio_irq_set, count); 1815 1816 if (copy_from_user(irq_set, (void __user *)arg, minsz)) 1817 return -EFAULT; 1818 1819 ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS, 1820 &data_size); 1821 if (ret) 1822 return ret; 1823 1824 if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 1825 return -EINVAL; 1826 1827 return 0; 1828 } 1829 1830 static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev, 1831 unsigned long arg) 1832 { 1833 s32 fd; 1834 void __user *data; 1835 unsigned long minsz; 1836 struct eventfd_ctx *req_trigger; 1837 1838 minsz = offsetofend(struct vfio_irq_set, count); 1839 data = (void __user *)(arg + minsz); 1840 1841 if (get_user(fd, (s32 __user *)data)) 1842 return -EFAULT; 1843 1844 if (fd == -1) { 1845 if (matrix_mdev->req_trigger) 1846 eventfd_ctx_put(matrix_mdev->req_trigger); 1847 matrix_mdev->req_trigger = NULL; 1848 } else if (fd >= 0) { 1849 req_trigger = eventfd_ctx_fdget(fd); 1850 if (IS_ERR(req_trigger)) 1851 return PTR_ERR(req_trigger); 1852 1853 if (matrix_mdev->req_trigger) 1854 eventfd_ctx_put(matrix_mdev->req_trigger); 1855 1856 matrix_mdev->req_trigger = req_trigger; 1857 } else { 1858 return -EINVAL; 1859 } 1860 1861 return 0; 1862 } 1863 1864 static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev, 1865 unsigned long arg) 1866 { 1867 int ret; 1868 struct vfio_irq_set irq_set; 1869 1870 ret = vfio_ap_irq_set_init(&irq_set, arg); 1871 if (ret) 1872 return ret; 1873 1874 switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1875 case VFIO_IRQ_SET_DATA_EVENTFD: 1876 switch (irq_set.index) { 1877 case VFIO_AP_REQ_IRQ_INDEX: 1878 return vfio_ap_set_request_irq(matrix_mdev, arg); 1879 default: 1880 return -EINVAL; 1881 } 1882 default: 1883 return -EINVAL; 1884 } 1885 } 1886 1887 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, 1888 unsigned int cmd, unsigned long arg) 1889 { 1890 struct ap_matrix_mdev *matrix_mdev = 1891 container_of(vdev, struct ap_matrix_mdev, vdev); 1892 int ret; 1893 1894 mutex_lock(&matrix_dev->mdevs_lock); 1895 switch (cmd) { 1896 case VFIO_DEVICE_GET_INFO: 1897 ret = vfio_ap_mdev_get_device_info(arg); 1898 break; 1899 case VFIO_DEVICE_RESET: 1900 ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 1901 break; 1902 case VFIO_DEVICE_GET_IRQ_INFO: 1903 ret = vfio_ap_get_irq_info(arg); 1904 break; 1905 case VFIO_DEVICE_SET_IRQS: 1906 ret = vfio_ap_set_irqs(matrix_mdev, arg); 1907 break; 1908 default: 1909 ret = -EOPNOTSUPP; 1910 break; 1911 } 1912 mutex_unlock(&matrix_dev->mdevs_lock); 1913 1914 return ret; 1915 } 1916 1917 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) 1918 { 1919 struct ap_matrix_mdev *matrix_mdev; 1920 unsigned long apid = AP_QID_CARD(q->apqn); 1921 unsigned long apqi = AP_QID_QUEUE(q->apqn); 1922 1923 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 1924 if (test_bit_inv(apid, matrix_mdev->matrix.apm) && 1925 test_bit_inv(apqi, matrix_mdev->matrix.aqm)) 1926 return matrix_mdev; 1927 } 1928 1929 return NULL; 1930 } 1931 1932 static ssize_t status_show(struct device *dev, 1933 struct device_attribute *attr, 1934 char *buf) 1935 { 1936 ssize_t nchars = 0; 1937 struct vfio_ap_queue *q; 1938 struct ap_matrix_mdev *matrix_mdev; 1939 struct ap_device *apdev = to_ap_dev(dev); 1940 1941 mutex_lock(&matrix_dev->mdevs_lock); 1942 q = dev_get_drvdata(&apdev->device); 1943 matrix_mdev = vfio_ap_mdev_for_queue(q); 1944 1945 if (matrix_mdev) { 1946 if (matrix_mdev->kvm) 1947 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1948 AP_QUEUE_IN_USE); 1949 else 1950 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1951 AP_QUEUE_ASSIGNED); 1952 } else { 1953 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1954 AP_QUEUE_UNASSIGNED); 1955 } 1956 1957 mutex_unlock(&matrix_dev->mdevs_lock); 1958 1959 return nchars; 1960 } 1961 1962 static DEVICE_ATTR_RO(status); 1963 1964 static struct attribute *vfio_queue_attrs[] = { 1965 &dev_attr_status.attr, 1966 NULL, 1967 }; 1968 1969 static const struct attribute_group vfio_queue_attr_group = { 1970 .attrs = vfio_queue_attrs, 1971 }; 1972 1973 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { 1974 .init = vfio_ap_mdev_init_dev, 1975 .open_device = vfio_ap_mdev_open_device, 1976 .close_device = vfio_ap_mdev_close_device, 1977 .ioctl = vfio_ap_mdev_ioctl, 1978 .dma_unmap = vfio_ap_mdev_dma_unmap, 1979 .bind_iommufd = vfio_iommufd_emulated_bind, 1980 .unbind_iommufd = vfio_iommufd_emulated_unbind, 1981 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 1982 .request = vfio_ap_mdev_request 1983 }; 1984 1985 static struct mdev_driver vfio_ap_matrix_driver = { 1986 .device_api = VFIO_DEVICE_API_AP_STRING, 1987 .max_instances = MAX_ZDEV_ENTRIES_EXT, 1988 .driver = { 1989 .name = "vfio_ap_mdev", 1990 .owner = THIS_MODULE, 1991 .mod_name = KBUILD_MODNAME, 1992 .dev_groups = vfio_ap_mdev_attr_groups, 1993 }, 1994 .probe = vfio_ap_mdev_probe, 1995 .remove = vfio_ap_mdev_remove, 1996 }; 1997 1998 int vfio_ap_mdev_register(void) 1999 { 2000 int ret; 2001 2002 ret = mdev_register_driver(&vfio_ap_matrix_driver); 2003 if (ret) 2004 return ret; 2005 2006 matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT; 2007 matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT; 2008 matrix_dev->mdev_types[0] = &matrix_dev->mdev_type; 2009 ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device, 2010 &vfio_ap_matrix_driver, 2011 matrix_dev->mdev_types, 1); 2012 if (ret) 2013 goto err_driver; 2014 return 0; 2015 2016 err_driver: 2017 mdev_unregister_driver(&vfio_ap_matrix_driver); 2018 return ret; 2019 } 2020 2021 void vfio_ap_mdev_unregister(void) 2022 { 2023 mdev_unregister_parent(&matrix_dev->parent); 2024 mdev_unregister_driver(&vfio_ap_matrix_driver); 2025 } 2026 2027 int vfio_ap_mdev_probe_queue(struct ap_device *apdev) 2028 { 2029 int ret; 2030 struct vfio_ap_queue *q; 2031 struct ap_matrix_mdev *matrix_mdev; 2032 2033 ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); 2034 if (ret) 2035 return ret; 2036 2037 q = kzalloc(sizeof(*q), GFP_KERNEL); 2038 if (!q) { 2039 ret = -ENOMEM; 2040 goto err_remove_group; 2041 } 2042 2043 q->apqn = to_ap_queue(&apdev->device)->qid; 2044 q->saved_isc = VFIO_AP_ISC_INVALID; 2045 matrix_mdev = get_update_locks_by_apqn(q->apqn); 2046 2047 if (matrix_mdev) { 2048 vfio_ap_mdev_link_queue(matrix_mdev, q); 2049 2050 if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, 2051 matrix_mdev->matrix.aqm, 2052 matrix_mdev)) 2053 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2054 } 2055 dev_set_drvdata(&apdev->device, q); 2056 release_update_locks_for_mdev(matrix_mdev); 2057 2058 return 0; 2059 2060 err_remove_group: 2061 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); 2062 return ret; 2063 } 2064 2065 void vfio_ap_mdev_remove_queue(struct ap_device *apdev) 2066 { 2067 unsigned long apid, apqi; 2068 struct vfio_ap_queue *q; 2069 struct ap_matrix_mdev *matrix_mdev; 2070 2071 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); 2072 q = dev_get_drvdata(&apdev->device); 2073 get_update_locks_for_queue(q); 2074 matrix_mdev = q->matrix_mdev; 2075 2076 if (matrix_mdev) { 2077 vfio_ap_unlink_queue_fr_mdev(q); 2078 2079 apid = AP_QID_CARD(q->apqn); 2080 apqi = AP_QID_QUEUE(q->apqn); 2081 2082 /* 2083 * If the queue is assigned to the guest's APCB, then remove 2084 * the adapter's APID from the APCB and hot it into the guest. 2085 */ 2086 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 2087 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 2088 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 2089 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2090 } 2091 } 2092 2093 vfio_ap_mdev_reset_queue(q); 2094 dev_set_drvdata(&apdev->device, NULL); 2095 kfree(q); 2096 release_update_locks_for_mdev(matrix_mdev); 2097 } 2098 2099 /** 2100 * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is 2101 * assigned to a mediated device under the control 2102 * of the vfio_ap device driver. 2103 * 2104 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check. 2105 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check. 2106 * 2107 * Return: 2108 * * -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are 2109 * assigned to a mediated device under the control of the vfio_ap 2110 * device driver. 2111 * * Otherwise, return 0. 2112 */ 2113 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) 2114 { 2115 int ret; 2116 2117 mutex_lock(&matrix_dev->guests_lock); 2118 mutex_lock(&matrix_dev->mdevs_lock); 2119 ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); 2120 mutex_unlock(&matrix_dev->mdevs_lock); 2121 mutex_unlock(&matrix_dev->guests_lock); 2122 2123 return ret; 2124 } 2125 2126 /** 2127 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control 2128 * domains that have been removed from the host's 2129 * AP configuration from a guest. 2130 * 2131 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest. 2132 * @aprem: the adapters that have been removed from the host's AP configuration 2133 * @aqrem: the domains that have been removed from the host's AP configuration 2134 * @cdrem: the control domains that have been removed from the host's AP 2135 * configuration. 2136 */ 2137 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, 2138 unsigned long *aprem, 2139 unsigned long *aqrem, 2140 unsigned long *cdrem) 2141 { 2142 int do_hotplug = 0; 2143 2144 if (!bitmap_empty(aprem, AP_DEVICES)) { 2145 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, 2146 matrix_mdev->shadow_apcb.apm, 2147 aprem, AP_DEVICES); 2148 } 2149 2150 if (!bitmap_empty(aqrem, AP_DOMAINS)) { 2151 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, 2152 matrix_mdev->shadow_apcb.aqm, 2153 aqrem, AP_DEVICES); 2154 } 2155 2156 if (!bitmap_empty(cdrem, AP_DOMAINS)) 2157 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, 2158 matrix_mdev->shadow_apcb.adm, 2159 cdrem, AP_DOMAINS); 2160 2161 if (do_hotplug) 2162 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2163 } 2164 2165 /** 2166 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters, 2167 * domains and control domains that have been removed 2168 * from the host AP configuration and unplugs them 2169 * from those guests. 2170 * 2171 * @ap_remove: bitmap specifying which adapters have been removed from the host 2172 * config. 2173 * @aq_remove: bitmap specifying which domains have been removed from the host 2174 * config. 2175 * @cd_remove: bitmap specifying which control domains have been removed from 2176 * the host config. 2177 */ 2178 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, 2179 unsigned long *aq_remove, 2180 unsigned long *cd_remove) 2181 { 2182 struct ap_matrix_mdev *matrix_mdev; 2183 DECLARE_BITMAP(aprem, AP_DEVICES); 2184 DECLARE_BITMAP(aqrem, AP_DOMAINS); 2185 DECLARE_BITMAP(cdrem, AP_DOMAINS); 2186 int do_remove = 0; 2187 2188 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2189 mutex_lock(&matrix_mdev->kvm->lock); 2190 mutex_lock(&matrix_dev->mdevs_lock); 2191 2192 do_remove |= bitmap_and(aprem, ap_remove, 2193 matrix_mdev->matrix.apm, 2194 AP_DEVICES); 2195 do_remove |= bitmap_and(aqrem, aq_remove, 2196 matrix_mdev->matrix.aqm, 2197 AP_DOMAINS); 2198 do_remove |= bitmap_andnot(cdrem, cd_remove, 2199 matrix_mdev->matrix.adm, 2200 AP_DOMAINS); 2201 2202 if (do_remove) 2203 vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, 2204 cdrem); 2205 2206 mutex_unlock(&matrix_dev->mdevs_lock); 2207 mutex_unlock(&matrix_mdev->kvm->lock); 2208 } 2209 } 2210 2211 /** 2212 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and 2213 * control domains from the host AP configuration 2214 * by unplugging them from the guests that are 2215 * using them. 2216 * @cur_config_info: the current host AP configuration information 2217 * @prev_config_info: the previous host AP configuration information 2218 */ 2219 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info, 2220 struct ap_config_info *prev_config_info) 2221 { 2222 int do_remove; 2223 DECLARE_BITMAP(aprem, AP_DEVICES); 2224 DECLARE_BITMAP(aqrem, AP_DOMAINS); 2225 DECLARE_BITMAP(cdrem, AP_DOMAINS); 2226 2227 do_remove = bitmap_andnot(aprem, 2228 (unsigned long *)prev_config_info->apm, 2229 (unsigned long *)cur_config_info->apm, 2230 AP_DEVICES); 2231 do_remove |= bitmap_andnot(aqrem, 2232 (unsigned long *)prev_config_info->aqm, 2233 (unsigned long *)cur_config_info->aqm, 2234 AP_DEVICES); 2235 do_remove |= bitmap_andnot(cdrem, 2236 (unsigned long *)prev_config_info->adm, 2237 (unsigned long *)cur_config_info->adm, 2238 AP_DEVICES); 2239 2240 if (do_remove) 2241 vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem); 2242 } 2243 2244 /** 2245 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that 2246 * are older than AP type 10 (CEX4). 2247 * @apm: a bitmap of the APIDs to examine 2248 * @aqm: a bitmap of the APQIs of the queues to query for the AP type. 2249 */ 2250 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm) 2251 { 2252 bool apid_cleared; 2253 struct ap_queue_status status; 2254 unsigned long apid, apqi; 2255 struct ap_tapq_gr2 info; 2256 2257 for_each_set_bit_inv(apid, apm, AP_DEVICES) { 2258 apid_cleared = false; 2259 2260 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { 2261 status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info); 2262 switch (status.response_code) { 2263 /* 2264 * According to the architecture in each case 2265 * below, the queue's info should be filled. 2266 */ 2267 case AP_RESPONSE_NORMAL: 2268 case AP_RESPONSE_RESET_IN_PROGRESS: 2269 case AP_RESPONSE_DECONFIGURED: 2270 case AP_RESPONSE_CHECKSTOPPED: 2271 case AP_RESPONSE_BUSY: 2272 /* 2273 * The vfio_ap device driver only 2274 * supports CEX4 and newer adapters, so 2275 * remove the APID if the adapter is 2276 * older than a CEX4. 2277 */ 2278 if (info.at < AP_DEVICE_TYPE_CEX4) { 2279 clear_bit_inv(apid, apm); 2280 apid_cleared = true; 2281 } 2282 2283 break; 2284 2285 default: 2286 /* 2287 * If we don't know the adapter type, 2288 * clear its APID since it can't be 2289 * determined whether the vfio_ap 2290 * device driver supports it. 2291 */ 2292 clear_bit_inv(apid, apm); 2293 apid_cleared = true; 2294 break; 2295 } 2296 2297 /* 2298 * If we've already cleared the APID from the apm, there 2299 * is no need to continue examining the remainin AP 2300 * queues to determine the type of the adapter. 2301 */ 2302 if (apid_cleared) 2303 continue; 2304 } 2305 } 2306 } 2307 2308 /** 2309 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and 2310 * control domains that have been added to the host's 2311 * AP configuration for each matrix mdev to which they 2312 * are assigned. 2313 * 2314 * @apm_add: a bitmap specifying the adapters that have been added to the AP 2315 * configuration. 2316 * @aqm_add: a bitmap specifying the domains that have been added to the AP 2317 * configuration. 2318 * @adm_add: a bitmap specifying the control domains that have been added to the 2319 * AP configuration. 2320 */ 2321 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, 2322 unsigned long *adm_add) 2323 { 2324 struct ap_matrix_mdev *matrix_mdev; 2325 2326 if (list_empty(&matrix_dev->mdev_list)) 2327 return; 2328 2329 vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); 2330 2331 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2332 bitmap_and(matrix_mdev->apm_add, 2333 matrix_mdev->matrix.apm, apm_add, AP_DEVICES); 2334 bitmap_and(matrix_mdev->aqm_add, 2335 matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); 2336 bitmap_and(matrix_mdev->adm_add, 2337 matrix_mdev->matrix.adm, adm_add, AP_DEVICES); 2338 } 2339 } 2340 2341 /** 2342 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and 2343 * control domains to the host AP configuration 2344 * by updating the bitmaps that specify what adapters, 2345 * domains and control domains have been added so they 2346 * can be hot plugged into the guest when the AP bus 2347 * scan completes (see vfio_ap_on_scan_complete 2348 * function). 2349 * @cur_config_info: the current AP configuration information 2350 * @prev_config_info: the previous AP configuration information 2351 */ 2352 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info, 2353 struct ap_config_info *prev_config_info) 2354 { 2355 bool do_add; 2356 DECLARE_BITMAP(apm_add, AP_DEVICES); 2357 DECLARE_BITMAP(aqm_add, AP_DOMAINS); 2358 DECLARE_BITMAP(adm_add, AP_DOMAINS); 2359 2360 do_add = bitmap_andnot(apm_add, 2361 (unsigned long *)cur_config_info->apm, 2362 (unsigned long *)prev_config_info->apm, 2363 AP_DEVICES); 2364 do_add |= bitmap_andnot(aqm_add, 2365 (unsigned long *)cur_config_info->aqm, 2366 (unsigned long *)prev_config_info->aqm, 2367 AP_DOMAINS); 2368 do_add |= bitmap_andnot(adm_add, 2369 (unsigned long *)cur_config_info->adm, 2370 (unsigned long *)prev_config_info->adm, 2371 AP_DOMAINS); 2372 2373 if (do_add) 2374 vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add); 2375 } 2376 2377 /** 2378 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP 2379 * configuration. 2380 * 2381 * @cur_cfg_info: the current host AP configuration 2382 * @prev_cfg_info: the previous host AP configuration 2383 */ 2384 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, 2385 struct ap_config_info *prev_cfg_info) 2386 { 2387 if (!cur_cfg_info || !prev_cfg_info) 2388 return; 2389 2390 mutex_lock(&matrix_dev->guests_lock); 2391 2392 vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); 2393 vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info); 2394 memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info)); 2395 2396 mutex_unlock(&matrix_dev->guests_lock); 2397 } 2398 2399 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) 2400 { 2401 bool do_hotplug = false; 2402 int filter_domains = 0; 2403 int filter_adapters = 0; 2404 DECLARE_BITMAP(apm, AP_DEVICES); 2405 DECLARE_BITMAP(aqm, AP_DOMAINS); 2406 2407 mutex_lock(&matrix_mdev->kvm->lock); 2408 mutex_lock(&matrix_dev->mdevs_lock); 2409 2410 filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm, 2411 matrix_mdev->apm_add, AP_DEVICES); 2412 filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm, 2413 matrix_mdev->aqm_add, AP_DOMAINS); 2414 2415 if (filter_adapters && filter_domains) 2416 do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev); 2417 else if (filter_adapters) 2418 do_hotplug |= 2419 vfio_ap_mdev_filter_matrix(apm, 2420 matrix_mdev->shadow_apcb.aqm, 2421 matrix_mdev); 2422 else 2423 do_hotplug |= 2424 vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm, 2425 aqm, matrix_mdev); 2426 2427 if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add, 2428 AP_DOMAINS)) 2429 do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev); 2430 2431 if (do_hotplug) 2432 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2433 2434 mutex_unlock(&matrix_dev->mdevs_lock); 2435 mutex_unlock(&matrix_mdev->kvm->lock); 2436 } 2437 2438 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, 2439 struct ap_config_info *old_config_info) 2440 { 2441 struct ap_matrix_mdev *matrix_mdev; 2442 2443 mutex_lock(&matrix_dev->guests_lock); 2444 2445 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2446 if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && 2447 bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && 2448 bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) 2449 continue; 2450 2451 vfio_ap_mdev_hot_plug_cfg(matrix_mdev); 2452 bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); 2453 bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); 2454 bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); 2455 } 2456 2457 mutex_unlock(&matrix_dev->guests_lock); 2458 } 2459