1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Adjunct processor matrix VFIO device driver callbacks. 4 * 5 * Copyright IBM Corp. 2018 6 * 7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com> 8 * Halil Pasic <pasic@linux.ibm.com> 9 * Pierre Morel <pmorel@linux.ibm.com> 10 */ 11 #include <linux/string.h> 12 #include <linux/vfio.h> 13 #include <linux/device.h> 14 #include <linux/list.h> 15 #include <linux/ctype.h> 16 #include <linux/bitops.h> 17 #include <linux/kvm_host.h> 18 #include <linux/module.h> 19 #include <linux/uuid.h> 20 #include <asm/kvm.h> 21 #include <asm/zcrypt.h> 22 23 #include "vfio_ap_private.h" 24 #include "vfio_ap_debug.h" 25 26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough" 27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device" 28 29 #define AP_QUEUE_ASSIGNED "assigned" 30 #define AP_QUEUE_UNASSIGNED "unassigned" 31 #define AP_QUEUE_IN_USE "in use" 32 33 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable); 34 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); 35 static const struct vfio_device_ops vfio_ap_matrix_dev_ops; 36 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, unsigned int retry); 37 38 /** 39 * get_update_locks_for_kvm: Acquire the locks required to dynamically update a 40 * KVM guest's APCB in the proper order. 41 * 42 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 43 * 44 * The proper locking order is: 45 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 46 * guest's APCB. 47 * 2. kvm->lock: required to update a guest's APCB 48 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 49 * 50 * Note: If @kvm is NULL, the KVM lock will not be taken. 51 */ 52 static inline void get_update_locks_for_kvm(struct kvm *kvm) 53 { 54 mutex_lock(&matrix_dev->guests_lock); 55 if (kvm) 56 mutex_lock(&kvm->lock); 57 mutex_lock(&matrix_dev->mdevs_lock); 58 } 59 60 /** 61 * release_update_locks_for_kvm: Release the locks used to dynamically update a 62 * KVM guest's APCB in the proper order. 63 * 64 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 65 * 66 * The proper unlocking order is: 67 * 1. matrix_dev->mdevs_lock 68 * 2. kvm->lock 69 * 3. matrix_dev->guests_lock 70 * 71 * Note: If @kvm is NULL, the KVM lock will not be released. 72 */ 73 static inline void release_update_locks_for_kvm(struct kvm *kvm) 74 { 75 mutex_unlock(&matrix_dev->mdevs_lock); 76 if (kvm) 77 mutex_unlock(&kvm->lock); 78 mutex_unlock(&matrix_dev->guests_lock); 79 } 80 81 /** 82 * get_update_locks_for_mdev: Acquire the locks required to dynamically update a 83 * KVM guest's APCB in the proper order. 84 * 85 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 86 * configuration data to use to update a KVM guest's APCB. 87 * 88 * The proper locking order is: 89 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 90 * guest's APCB. 91 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 92 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 93 * 94 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 95 * lock will not be taken. 96 */ 97 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 98 { 99 mutex_lock(&matrix_dev->guests_lock); 100 if (matrix_mdev && matrix_mdev->kvm) 101 mutex_lock(&matrix_mdev->kvm->lock); 102 mutex_lock(&matrix_dev->mdevs_lock); 103 } 104 105 /** 106 * release_update_locks_for_mdev: Release the locks used to dynamically update a 107 * KVM guest's APCB in the proper order. 108 * 109 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 110 * configuration data to use to update a KVM guest's APCB. 111 * 112 * The proper unlocking order is: 113 * 1. matrix_dev->mdevs_lock 114 * 2. matrix_mdev->kvm->lock 115 * 3. matrix_dev->guests_lock 116 * 117 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 118 * lock will not be released. 119 */ 120 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 121 { 122 mutex_unlock(&matrix_dev->mdevs_lock); 123 if (matrix_mdev && matrix_mdev->kvm) 124 mutex_unlock(&matrix_mdev->kvm->lock); 125 mutex_unlock(&matrix_dev->guests_lock); 126 } 127 128 /** 129 * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and 130 * acquire the locks required to update the APCB of 131 * the KVM guest to which the mdev is attached. 132 * 133 * @apqn: the APQN of a queue device. 134 * 135 * The proper locking order is: 136 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 137 * guest's APCB. 138 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 139 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 140 * 141 * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock 142 * will not be taken. 143 * 144 * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn 145 * is not assigned to an ap_matrix_mdev. 146 */ 147 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn) 148 { 149 struct ap_matrix_mdev *matrix_mdev; 150 151 mutex_lock(&matrix_dev->guests_lock); 152 153 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 154 if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) && 155 test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) { 156 if (matrix_mdev->kvm) 157 mutex_lock(&matrix_mdev->kvm->lock); 158 159 mutex_lock(&matrix_dev->mdevs_lock); 160 161 return matrix_mdev; 162 } 163 } 164 165 mutex_lock(&matrix_dev->mdevs_lock); 166 167 return NULL; 168 } 169 170 /** 171 * get_update_locks_for_queue: get the locks required to update the APCB of the 172 * KVM guest to which the matrix mdev linked to a 173 * vfio_ap_queue object is attached. 174 * 175 * @q: a pointer to a vfio_ap_queue object. 176 * 177 * The proper locking order is: 178 * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a 179 * KVM guest's APCB. 180 * 2. q->matrix_mdev->kvm->lock: required to update a guest's APCB 181 * 3. matrix_dev->mdevs_lock: required to access data stored in matrix_mdev 182 * 183 * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock 184 * will not be taken. 185 */ 186 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q) 187 { 188 mutex_lock(&matrix_dev->guests_lock); 189 if (q->matrix_mdev && q->matrix_mdev->kvm) 190 mutex_lock(&q->matrix_mdev->kvm->lock); 191 mutex_lock(&matrix_dev->mdevs_lock); 192 } 193 194 /** 195 * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a 196 * hash table of queues assigned to a matrix mdev 197 * @matrix_mdev: the matrix mdev 198 * @apqn: The APQN of a queue device 199 * 200 * Return: the pointer to the vfio_ap_queue struct representing the queue or 201 * NULL if the queue is not assigned to @matrix_mdev 202 */ 203 static struct vfio_ap_queue *vfio_ap_mdev_get_queue( 204 struct ap_matrix_mdev *matrix_mdev, 205 int apqn) 206 { 207 struct vfio_ap_queue *q; 208 209 hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode, 210 apqn) { 211 if (q && q->apqn == apqn) 212 return q; 213 } 214 215 return NULL; 216 } 217 218 /** 219 * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries 220 * @apqn: The AP Queue number 221 * 222 * Checks the IRQ bit for the status of this APQN using ap_tapq. 223 * Returns if the ap_tapq function succeeded and the bit is clear. 224 * Returns if ap_tapq function failed with invalid, deconfigured or 225 * checkstopped AP. 226 * Otherwise retries up to 5 times after waiting 20ms. 227 */ 228 static void vfio_ap_wait_for_irqclear(int apqn) 229 { 230 struct ap_queue_status status; 231 int retry = 5; 232 233 do { 234 status = ap_tapq(apqn, NULL); 235 switch (status.response_code) { 236 case AP_RESPONSE_NORMAL: 237 case AP_RESPONSE_RESET_IN_PROGRESS: 238 if (!status.irq_enabled) 239 return; 240 fallthrough; 241 case AP_RESPONSE_BUSY: 242 msleep(20); 243 break; 244 case AP_RESPONSE_Q_NOT_AVAIL: 245 case AP_RESPONSE_DECONFIGURED: 246 case AP_RESPONSE_CHECKSTOPPED: 247 default: 248 WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__, 249 status.response_code, apqn); 250 return; 251 } 252 } while (--retry); 253 254 WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n", 255 __func__, status.response_code, apqn); 256 } 257 258 /** 259 * vfio_ap_free_aqic_resources - free vfio_ap_queue resources 260 * @q: The vfio_ap_queue 261 * 262 * Unregisters the ISC in the GIB when the saved ISC not invalid. 263 * Unpins the guest's page holding the NIB when it exists. 264 * Resets the saved_iova and saved_isc to invalid values. 265 */ 266 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) 267 { 268 if (!q) 269 return; 270 if (q->saved_isc != VFIO_AP_ISC_INVALID && 271 !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { 272 kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); 273 q->saved_isc = VFIO_AP_ISC_INVALID; 274 } 275 if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { 276 vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1); 277 q->saved_iova = 0; 278 } 279 } 280 281 /** 282 * vfio_ap_irq_disable - disables and clears an ap_queue interrupt 283 * @q: The vfio_ap_queue 284 * 285 * Uses ap_aqic to disable the interruption and in case of success, reset 286 * in progress or IRQ disable command already proceeded: calls 287 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear 288 * and calls vfio_ap_free_aqic_resources() to free the resources associated 289 * with the AP interrupt handling. 290 * 291 * In the case the AP is busy, or a reset is in progress, 292 * retries after 20ms, up to 5 times. 293 * 294 * Returns if ap_aqic function failed with invalid, deconfigured or 295 * checkstopped AP. 296 * 297 * Return: &struct ap_queue_status 298 */ 299 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q) 300 { 301 struct ap_qirq_ctrl aqic_gisa = {}; 302 struct ap_queue_status status; 303 int retries = 5; 304 305 do { 306 status = ap_aqic(q->apqn, aqic_gisa, 0); 307 switch (status.response_code) { 308 case AP_RESPONSE_OTHERWISE_CHANGED: 309 case AP_RESPONSE_NORMAL: 310 vfio_ap_wait_for_irqclear(q->apqn); 311 goto end_free; 312 case AP_RESPONSE_RESET_IN_PROGRESS: 313 case AP_RESPONSE_BUSY: 314 msleep(20); 315 break; 316 case AP_RESPONSE_Q_NOT_AVAIL: 317 case AP_RESPONSE_DECONFIGURED: 318 case AP_RESPONSE_CHECKSTOPPED: 319 case AP_RESPONSE_INVALID_ADDRESS: 320 default: 321 /* All cases in default means AP not operational */ 322 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 323 status.response_code); 324 goto end_free; 325 } 326 } while (retries--); 327 328 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 329 status.response_code); 330 end_free: 331 vfio_ap_free_aqic_resources(q); 332 return status; 333 } 334 335 /** 336 * vfio_ap_validate_nib - validate a notification indicator byte (nib) address. 337 * 338 * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction. 339 * @nib: the location for storing the nib address. 340 * 341 * When the PQAP(AQIC) instruction is executed, general register 2 contains the 342 * address of the notification indicator byte (nib) used for IRQ notification. 343 * This function parses and validates the nib from gr2. 344 * 345 * Return: returns zero if the nib address is a valid; otherwise, returns 346 * -EINVAL. 347 */ 348 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib) 349 { 350 *nib = vcpu->run->s.regs.gprs[2]; 351 352 if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT))) 353 return -EINVAL; 354 355 return 0; 356 } 357 358 /** 359 * vfio_ap_irq_enable - Enable Interruption for a APQN 360 * 361 * @q: the vfio_ap_queue holding AQIC parameters 362 * @isc: the guest ISC to register with the GIB interface 363 * @vcpu: the vcpu object containing the registers specifying the parameters 364 * passed to the PQAP(AQIC) instruction. 365 * 366 * Pin the NIB saved in *q 367 * Register the guest ISC to GIB interface and retrieve the 368 * host ISC to issue the host side PQAP/AQIC 369 * 370 * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the 371 * vfio_pin_pages failed. 372 * 373 * Otherwise return the ap_queue_status returned by the ap_aqic(), 374 * all retry handling will be done by the guest. 375 * 376 * Return: &struct ap_queue_status 377 */ 378 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, 379 int isc, 380 struct kvm_vcpu *vcpu) 381 { 382 struct ap_qirq_ctrl aqic_gisa = {}; 383 struct ap_queue_status status = {}; 384 struct kvm_s390_gisa *gisa; 385 struct page *h_page; 386 int nisc; 387 struct kvm *kvm; 388 phys_addr_t h_nib; 389 dma_addr_t nib; 390 int ret; 391 392 /* Verify that the notification indicator byte address is valid */ 393 if (vfio_ap_validate_nib(vcpu, &nib)) { 394 VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n", 395 __func__, &nib, q->apqn); 396 397 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 398 return status; 399 } 400 401 ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1, 402 IOMMU_READ | IOMMU_WRITE, &h_page); 403 switch (ret) { 404 case 1: 405 break; 406 default: 407 VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d," 408 "nib=%pad, apqn=%#04x\n", 409 __func__, ret, &nib, q->apqn); 410 411 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 412 return status; 413 } 414 415 kvm = q->matrix_mdev->kvm; 416 gisa = kvm->arch.gisa_int.origin; 417 418 h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK); 419 aqic_gisa.gisc = isc; 420 421 nisc = kvm_s390_gisc_register(kvm, isc); 422 if (nisc < 0) { 423 VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n", 424 __func__, nisc, isc, q->apqn); 425 426 status.response_code = AP_RESPONSE_INVALID_GISA; 427 return status; 428 } 429 430 aqic_gisa.isc = nisc; 431 aqic_gisa.ir = 1; 432 aqic_gisa.gisa = (uint64_t)gisa >> 4; 433 434 status = ap_aqic(q->apqn, aqic_gisa, h_nib); 435 switch (status.response_code) { 436 case AP_RESPONSE_NORMAL: 437 /* See if we did clear older IRQ configuration */ 438 vfio_ap_free_aqic_resources(q); 439 q->saved_iova = nib; 440 q->saved_isc = isc; 441 break; 442 case AP_RESPONSE_OTHERWISE_CHANGED: 443 /* We could not modify IRQ setings: clear new configuration */ 444 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); 445 kvm_s390_gisc_unregister(kvm, isc); 446 break; 447 default: 448 pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn, 449 status.response_code); 450 vfio_ap_irq_disable(q); 451 break; 452 } 453 454 if (status.response_code != AP_RESPONSE_NORMAL) { 455 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: " 456 "zone=%#x, ir=%#x, gisc=%#x, f=%#x," 457 "gisa=%#x, isc=%#x, apqn=%#04x\n", 458 __func__, status.response_code, 459 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc, 460 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc, 461 q->apqn); 462 } 463 464 return status; 465 } 466 467 /** 468 * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array 469 * of big endian elements that can be passed by 470 * value to an s390dbf sprintf event function to 471 * format a UUID string. 472 * 473 * @guid: the object containing the little endian guid 474 * @uuid: a six-element array of long values that can be passed by value as 475 * arguments for a formatting string specifying a UUID. 476 * 477 * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf 478 * event functions if the memory for the passed string is available as long as 479 * the debug feature exists. Since a mediated device can be removed at any 480 * time, it's name can not be used because %s passes the reference to the string 481 * in memory and the reference will go stale once the device is removed . 482 * 483 * The s390dbf string formatting function allows a maximum of 9 arguments for a 484 * message to be displayed in the 'sprintf' view. In order to use the bytes 485 * comprising the mediated device's UUID to display the mediated device name, 486 * they will have to be converted into an array whose elements can be passed by 487 * value to sprintf. For example: 488 * 489 * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 } 490 * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804 491 * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 } 492 * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx" 493 */ 494 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid) 495 { 496 /* 497 * The input guid is ordered in little endian, so it needs to be 498 * reordered for displaying a UUID as a string. This specifies the 499 * guid indices in proper order. 500 */ 501 uuid[0] = le32_to_cpup((__le32 *)guid); 502 uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]); 503 uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]); 504 uuid[3] = *((__u16 *)&guid->b[8]); 505 uuid[4] = *((__u16 *)&guid->b[10]); 506 uuid[5] = *((__u32 *)&guid->b[12]); 507 } 508 509 /** 510 * handle_pqap - PQAP instruction callback 511 * 512 * @vcpu: The vcpu on which we received the PQAP instruction 513 * 514 * Get the general register contents to initialize internal variables. 515 * REG[0]: APQN 516 * REG[1]: IR and ISC 517 * REG[2]: NIB 518 * 519 * Response.status may be set to following Response Code: 520 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available 521 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured 522 * - AP_RESPONSE_NORMAL (0) : in case of successs 523 * Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC. 524 * We take the matrix_dev lock to ensure serialization on queues and 525 * mediated device access. 526 * 527 * Return: 0 if we could handle the request inside KVM. 528 * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault. 529 */ 530 static int handle_pqap(struct kvm_vcpu *vcpu) 531 { 532 uint64_t status; 533 uint16_t apqn; 534 unsigned long uuid[6]; 535 struct vfio_ap_queue *q; 536 struct ap_queue_status qstatus = { 537 .response_code = AP_RESPONSE_Q_NOT_AVAIL, }; 538 struct ap_matrix_mdev *matrix_mdev; 539 540 apqn = vcpu->run->s.regs.gprs[0] & 0xffff; 541 542 /* If we do not use the AIV facility just go to userland */ 543 if (!(vcpu->arch.sie_block->eca & ECA_AIV)) { 544 VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n", 545 __func__, apqn, vcpu->arch.sie_block->eca); 546 547 return -EOPNOTSUPP; 548 } 549 550 mutex_lock(&matrix_dev->mdevs_lock); 551 552 if (!vcpu->kvm->arch.crypto.pqap_hook) { 553 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n", 554 __func__, apqn); 555 556 goto out_unlock; 557 } 558 559 matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook, 560 struct ap_matrix_mdev, pqap_hook); 561 562 /* If the there is no guest using the mdev, there is nothing to do */ 563 if (!matrix_mdev->kvm) { 564 vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid); 565 VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n", 566 __func__, uuid[0], uuid[1], uuid[2], 567 uuid[3], uuid[4], uuid[5], apqn); 568 goto out_unlock; 569 } 570 571 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); 572 if (!q) { 573 VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n", 574 __func__, AP_QID_CARD(apqn), 575 AP_QID_QUEUE(apqn)); 576 goto out_unlock; 577 } 578 579 status = vcpu->run->s.regs.gprs[1]; 580 581 /* If IR bit(16) is set we enable the interrupt */ 582 if ((status >> (63 - 16)) & 0x01) 583 qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu); 584 else 585 qstatus = vfio_ap_irq_disable(q); 586 587 out_unlock: 588 memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus)); 589 vcpu->run->s.regs.gprs[1] >>= 32; 590 mutex_unlock(&matrix_dev->mdevs_lock); 591 return 0; 592 } 593 594 static void vfio_ap_matrix_init(struct ap_config_info *info, 595 struct ap_matrix *matrix) 596 { 597 matrix->apm_max = info->apxa ? info->Na : 63; 598 matrix->aqm_max = info->apxa ? info->Nd : 15; 599 matrix->adm_max = info->apxa ? info->Nd : 15; 600 } 601 602 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev) 603 { 604 if (matrix_mdev->kvm) 605 kvm_arch_crypto_set_masks(matrix_mdev->kvm, 606 matrix_mdev->shadow_apcb.apm, 607 matrix_mdev->shadow_apcb.aqm, 608 matrix_mdev->shadow_apcb.adm); 609 } 610 611 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) 612 { 613 DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS); 614 615 bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS); 616 bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm, 617 (unsigned long *)matrix_dev->info.adm, AP_DOMAINS); 618 619 return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, 620 AP_DOMAINS); 621 } 622 623 /* 624 * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev 625 * to ensure no queue devices are passed through to 626 * the guest that are not bound to the vfio_ap 627 * device driver. 628 * 629 * @matrix_mdev: the matrix mdev whose matrix is to be filtered. 630 * 631 * Note: If an APQN referencing a queue device that is not bound to the vfio_ap 632 * driver, its APID will be filtered from the guest's APCB. The matrix 633 * structure precludes filtering an individual APQN, so its APID will be 634 * filtered. 635 * 636 * Return: a boolean value indicating whether the KVM guest's APCB was changed 637 * by the filtering or not. 638 */ 639 static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, 640 struct ap_matrix_mdev *matrix_mdev) 641 { 642 unsigned long apid, apqi, apqn; 643 DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); 644 DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); 645 struct vfio_ap_queue *q; 646 647 bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); 648 bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); 649 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 650 651 /* 652 * Copy the adapters, domains and control domains to the shadow_apcb 653 * from the matrix mdev, but only those that are assigned to the host's 654 * AP configuration. 655 */ 656 bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm, 657 (unsigned long *)matrix_dev->info.apm, AP_DEVICES); 658 bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm, 659 (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS); 660 661 for_each_set_bit_inv(apid, apm, AP_DEVICES) { 662 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { 663 /* 664 * If the APQN is not bound to the vfio_ap device 665 * driver, then we can't assign it to the guest's 666 * AP configuration. The AP architecture won't 667 * allow filtering of a single APQN, so let's filter 668 * the APID since an adapter represents a physical 669 * hardware device. 670 */ 671 apqn = AP_MKQID(apid, apqi); 672 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); 673 if (!q || q->reset_rc) { 674 clear_bit_inv(apid, 675 matrix_mdev->shadow_apcb.apm); 676 break; 677 } 678 } 679 } 680 681 return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, 682 AP_DEVICES) || 683 !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, 684 AP_DOMAINS); 685 } 686 687 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev) 688 { 689 struct ap_matrix_mdev *matrix_mdev = 690 container_of(vdev, struct ap_matrix_mdev, vdev); 691 692 if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0)) 693 return -EPERM; 694 695 matrix_mdev->mdev = to_mdev_device(vdev->dev); 696 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); 697 matrix_mdev->pqap_hook = handle_pqap; 698 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 699 hash_init(matrix_mdev->qtable.queues); 700 701 return 0; 702 } 703 704 static int vfio_ap_mdev_probe(struct mdev_device *mdev) 705 { 706 struct ap_matrix_mdev *matrix_mdev; 707 int ret; 708 709 matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev, 710 &vfio_ap_matrix_dev_ops); 711 if (IS_ERR(matrix_mdev)) 712 return PTR_ERR(matrix_mdev); 713 714 ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); 715 if (ret) 716 goto err_put_vdev; 717 dev_set_drvdata(&mdev->dev, matrix_mdev); 718 mutex_lock(&matrix_dev->mdevs_lock); 719 list_add(&matrix_mdev->node, &matrix_dev->mdev_list); 720 mutex_unlock(&matrix_dev->mdevs_lock); 721 return 0; 722 723 err_put_vdev: 724 vfio_put_device(&matrix_mdev->vdev); 725 return ret; 726 } 727 728 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, 729 struct vfio_ap_queue *q) 730 { 731 if (q) { 732 q->matrix_mdev = matrix_mdev; 733 hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); 734 } 735 } 736 737 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) 738 { 739 struct vfio_ap_queue *q; 740 741 q = vfio_ap_find_queue(apqn); 742 vfio_ap_mdev_link_queue(matrix_mdev, q); 743 } 744 745 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q) 746 { 747 hash_del(&q->mdev_qnode); 748 } 749 750 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q) 751 { 752 q->matrix_mdev = NULL; 753 } 754 755 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) 756 { 757 struct vfio_ap_queue *q; 758 unsigned long apid, apqi; 759 760 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 761 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, 762 AP_DOMAINS) { 763 q = vfio_ap_mdev_get_queue(matrix_mdev, 764 AP_MKQID(apid, apqi)); 765 if (q) 766 q->matrix_mdev = NULL; 767 } 768 } 769 } 770 771 static void vfio_ap_mdev_release_dev(struct vfio_device *vdev) 772 { 773 atomic_inc(&matrix_dev->available_instances); 774 vfio_free_device(vdev); 775 } 776 777 static void vfio_ap_mdev_remove(struct mdev_device *mdev) 778 { 779 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev); 780 781 vfio_unregister_group_dev(&matrix_mdev->vdev); 782 783 mutex_lock(&matrix_dev->guests_lock); 784 mutex_lock(&matrix_dev->mdevs_lock); 785 vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 786 vfio_ap_mdev_unlink_fr_queues(matrix_mdev); 787 list_del(&matrix_mdev->node); 788 mutex_unlock(&matrix_dev->mdevs_lock); 789 mutex_unlock(&matrix_dev->guests_lock); 790 vfio_put_device(&matrix_mdev->vdev); 791 } 792 793 static ssize_t name_show(struct mdev_type *mtype, 794 struct mdev_type_attribute *attr, char *buf) 795 { 796 return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT); 797 } 798 799 static MDEV_TYPE_ATTR_RO(name); 800 801 static ssize_t available_instances_show(struct mdev_type *mtype, 802 struct mdev_type_attribute *attr, 803 char *buf) 804 { 805 return sprintf(buf, "%d\n", 806 atomic_read(&matrix_dev->available_instances)); 807 } 808 809 static MDEV_TYPE_ATTR_RO(available_instances); 810 811 static const struct attribute *vfio_ap_mdev_type_attrs[] = { 812 &mdev_type_attr_name.attr, 813 &mdev_type_attr_available_instances.attr, 814 NULL, 815 }; 816 817 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ 818 "already assigned to %s" 819 820 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, 821 unsigned long *apm, 822 unsigned long *aqm) 823 { 824 unsigned long apid, apqi; 825 const struct device *dev = mdev_dev(matrix_mdev->mdev); 826 const char *mdev_name = dev_name(dev); 827 828 for_each_set_bit_inv(apid, apm, AP_DEVICES) 829 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) 830 dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); 831 } 832 833 /** 834 * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs 835 * 836 * @mdev_apm: mask indicating the APIDs of the APQNs to be verified 837 * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified 838 * 839 * Verifies that each APQN derived from the Cartesian product of a bitmap of 840 * AP adapter IDs and AP queue indexes is not configured for any matrix 841 * mediated device. AP queue sharing is not allowed. 842 * 843 * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. 844 */ 845 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, 846 unsigned long *mdev_aqm) 847 { 848 struct ap_matrix_mdev *matrix_mdev; 849 DECLARE_BITMAP(apm, AP_DEVICES); 850 DECLARE_BITMAP(aqm, AP_DOMAINS); 851 852 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 853 /* 854 * If the input apm and aqm are fields of the matrix_mdev 855 * object, then move on to the next matrix_mdev. 856 */ 857 if (mdev_apm == matrix_mdev->matrix.apm && 858 mdev_aqm == matrix_mdev->matrix.aqm) 859 continue; 860 861 memset(apm, 0, sizeof(apm)); 862 memset(aqm, 0, sizeof(aqm)); 863 864 /* 865 * We work on full longs, as we can only exclude the leftover 866 * bits in non-inverse order. The leftover is all zeros. 867 */ 868 if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, 869 AP_DEVICES)) 870 continue; 871 872 if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, 873 AP_DOMAINS)) 874 continue; 875 876 vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); 877 878 return -EADDRINUSE; 879 } 880 881 return 0; 882 } 883 884 /** 885 * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are 886 * not reserved for the default zcrypt driver and 887 * are not assigned to another mdev. 888 * 889 * @matrix_mdev: the mdev to which the APQNs being validated are assigned. 890 * 891 * Return: One of the following values: 892 * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, 893 * most likely -EBUSY indicating the ap_perms_mutex lock is already held. 894 * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the 895 * zcrypt default driver. 896 * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev 897 * o A zero indicating validation succeeded. 898 */ 899 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) 900 { 901 if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm, 902 matrix_mdev->matrix.aqm)) 903 return -EADDRNOTAVAIL; 904 905 return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, 906 matrix_mdev->matrix.aqm); 907 } 908 909 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, 910 unsigned long apid) 911 { 912 unsigned long apqi; 913 914 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) 915 vfio_ap_mdev_link_apqn(matrix_mdev, 916 AP_MKQID(apid, apqi)); 917 } 918 919 /** 920 * assign_adapter_store - parses the APID from @buf and sets the 921 * corresponding bit in the mediated matrix device's APM 922 * 923 * @dev: the matrix device 924 * @attr: the mediated matrix device's assign_adapter attribute 925 * @buf: a buffer containing the AP adapter number (APID) to 926 * be assigned 927 * @count: the number of bytes in @buf 928 * 929 * Return: the number of bytes processed if the APID is valid; otherwise, 930 * returns one of the following errors: 931 * 932 * 1. -EINVAL 933 * The APID is not a valid number 934 * 935 * 2. -ENODEV 936 * The APID exceeds the maximum value configured for the system 937 * 938 * 3. -EADDRNOTAVAIL 939 * An APQN derived from the cross product of the APID being assigned 940 * and the APQIs previously assigned is not bound to the vfio_ap device 941 * driver; or, if no APQIs have yet been assigned, the APID is not 942 * contained in an APQN bound to the vfio_ap device driver. 943 * 944 * 4. -EADDRINUSE 945 * An APQN derived from the cross product of the APID being assigned 946 * and the APQIs previously assigned is being used by another mediated 947 * matrix device 948 * 949 * 5. -EAGAIN 950 * A lock required to validate the mdev's AP configuration could not 951 * be obtained. 952 */ 953 static ssize_t assign_adapter_store(struct device *dev, 954 struct device_attribute *attr, 955 const char *buf, size_t count) 956 { 957 int ret; 958 unsigned long apid; 959 DECLARE_BITMAP(apm_delta, AP_DEVICES); 960 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 961 962 mutex_lock(&ap_perms_mutex); 963 get_update_locks_for_mdev(matrix_mdev); 964 965 ret = kstrtoul(buf, 0, &apid); 966 if (ret) 967 goto done; 968 969 if (apid > matrix_mdev->matrix.apm_max) { 970 ret = -ENODEV; 971 goto done; 972 } 973 974 set_bit_inv(apid, matrix_mdev->matrix.apm); 975 976 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 977 if (ret) { 978 clear_bit_inv(apid, matrix_mdev->matrix.apm); 979 goto done; 980 } 981 982 vfio_ap_mdev_link_adapter(matrix_mdev, apid); 983 memset(apm_delta, 0, sizeof(apm_delta)); 984 set_bit_inv(apid, apm_delta); 985 986 if (vfio_ap_mdev_filter_matrix(apm_delta, 987 matrix_mdev->matrix.aqm, matrix_mdev)) 988 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 989 990 ret = count; 991 done: 992 release_update_locks_for_mdev(matrix_mdev); 993 mutex_unlock(&ap_perms_mutex); 994 995 return ret; 996 } 997 static DEVICE_ATTR_WO(assign_adapter); 998 999 static struct vfio_ap_queue 1000 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev, 1001 unsigned long apid, unsigned long apqi) 1002 { 1003 struct vfio_ap_queue *q = NULL; 1004 1005 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); 1006 /* If the queue is assigned to the matrix mdev, unlink it. */ 1007 if (q) 1008 vfio_ap_unlink_queue_fr_mdev(q); 1009 1010 return q; 1011 } 1012 1013 /** 1014 * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned 1015 * adapter from the matrix mdev to which the 1016 * adapter was assigned. 1017 * @matrix_mdev: the matrix mediated device to which the adapter was assigned. 1018 * @apid: the APID of the unassigned adapter. 1019 * @qtable: table for storing queues associated with unassigned adapter. 1020 */ 1021 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, 1022 unsigned long apid, 1023 struct ap_queue_table *qtable) 1024 { 1025 unsigned long apqi; 1026 struct vfio_ap_queue *q; 1027 1028 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { 1029 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1030 1031 if (q && qtable) { 1032 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1033 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1034 hash_add(qtable->queues, &q->mdev_qnode, 1035 q->apqn); 1036 } 1037 } 1038 } 1039 1040 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, 1041 unsigned long apid) 1042 { 1043 int loop_cursor; 1044 struct vfio_ap_queue *q; 1045 struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); 1046 1047 hash_init(qtable->queues); 1048 vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable); 1049 1050 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { 1051 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 1052 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1053 } 1054 1055 vfio_ap_mdev_reset_queues(qtable); 1056 1057 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1058 vfio_ap_unlink_mdev_fr_queue(q); 1059 hash_del(&q->mdev_qnode); 1060 } 1061 1062 kfree(qtable); 1063 } 1064 1065 /** 1066 * unassign_adapter_store - parses the APID from @buf and clears the 1067 * corresponding bit in the mediated matrix device's APM 1068 * 1069 * @dev: the matrix device 1070 * @attr: the mediated matrix device's unassign_adapter attribute 1071 * @buf: a buffer containing the adapter number (APID) to be unassigned 1072 * @count: the number of bytes in @buf 1073 * 1074 * Return: the number of bytes processed if the APID is valid; otherwise, 1075 * returns one of the following errors: 1076 * -EINVAL if the APID is not a number 1077 * -ENODEV if the APID it exceeds the maximum value configured for the 1078 * system 1079 */ 1080 static ssize_t unassign_adapter_store(struct device *dev, 1081 struct device_attribute *attr, 1082 const char *buf, size_t count) 1083 { 1084 int ret; 1085 unsigned long apid; 1086 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1087 1088 get_update_locks_for_mdev(matrix_mdev); 1089 1090 ret = kstrtoul(buf, 0, &apid); 1091 if (ret) 1092 goto done; 1093 1094 if (apid > matrix_mdev->matrix.apm_max) { 1095 ret = -ENODEV; 1096 goto done; 1097 } 1098 1099 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); 1100 vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid); 1101 ret = count; 1102 done: 1103 release_update_locks_for_mdev(matrix_mdev); 1104 return ret; 1105 } 1106 static DEVICE_ATTR_WO(unassign_adapter); 1107 1108 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, 1109 unsigned long apqi) 1110 { 1111 unsigned long apid; 1112 1113 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) 1114 vfio_ap_mdev_link_apqn(matrix_mdev, 1115 AP_MKQID(apid, apqi)); 1116 } 1117 1118 /** 1119 * assign_domain_store - parses the APQI from @buf and sets the 1120 * corresponding bit in the mediated matrix device's AQM 1121 * 1122 * @dev: the matrix device 1123 * @attr: the mediated matrix device's assign_domain attribute 1124 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1125 * be assigned 1126 * @count: the number of bytes in @buf 1127 * 1128 * Return: the number of bytes processed if the APQI is valid; otherwise returns 1129 * one of the following errors: 1130 * 1131 * 1. -EINVAL 1132 * The APQI is not a valid number 1133 * 1134 * 2. -ENODEV 1135 * The APQI exceeds the maximum value configured for the system 1136 * 1137 * 3. -EADDRNOTAVAIL 1138 * An APQN derived from the cross product of the APQI being assigned 1139 * and the APIDs previously assigned is not bound to the vfio_ap device 1140 * driver; or, if no APIDs have yet been assigned, the APQI is not 1141 * contained in an APQN bound to the vfio_ap device driver. 1142 * 1143 * 4. -EADDRINUSE 1144 * An APQN derived from the cross product of the APQI being assigned 1145 * and the APIDs previously assigned is being used by another mediated 1146 * matrix device 1147 * 1148 * 5. -EAGAIN 1149 * The lock required to validate the mdev's AP configuration could not 1150 * be obtained. 1151 */ 1152 static ssize_t assign_domain_store(struct device *dev, 1153 struct device_attribute *attr, 1154 const char *buf, size_t count) 1155 { 1156 int ret; 1157 unsigned long apqi; 1158 DECLARE_BITMAP(aqm_delta, AP_DOMAINS); 1159 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1160 1161 mutex_lock(&ap_perms_mutex); 1162 get_update_locks_for_mdev(matrix_mdev); 1163 1164 ret = kstrtoul(buf, 0, &apqi); 1165 if (ret) 1166 goto done; 1167 1168 if (apqi > matrix_mdev->matrix.aqm_max) { 1169 ret = -ENODEV; 1170 goto done; 1171 } 1172 1173 set_bit_inv(apqi, matrix_mdev->matrix.aqm); 1174 1175 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 1176 if (ret) { 1177 clear_bit_inv(apqi, matrix_mdev->matrix.aqm); 1178 goto done; 1179 } 1180 1181 vfio_ap_mdev_link_domain(matrix_mdev, apqi); 1182 memset(aqm_delta, 0, sizeof(aqm_delta)); 1183 set_bit_inv(apqi, aqm_delta); 1184 1185 if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, 1186 matrix_mdev)) 1187 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1188 1189 ret = count; 1190 done: 1191 release_update_locks_for_mdev(matrix_mdev); 1192 mutex_unlock(&ap_perms_mutex); 1193 1194 return ret; 1195 } 1196 static DEVICE_ATTR_WO(assign_domain); 1197 1198 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, 1199 unsigned long apqi, 1200 struct ap_queue_table *qtable) 1201 { 1202 unsigned long apid; 1203 struct vfio_ap_queue *q; 1204 1205 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 1206 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1207 1208 if (q && qtable) { 1209 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1210 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1211 hash_add(qtable->queues, &q->mdev_qnode, 1212 q->apqn); 1213 } 1214 } 1215 } 1216 1217 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, 1218 unsigned long apqi) 1219 { 1220 int loop_cursor; 1221 struct vfio_ap_queue *q; 1222 struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); 1223 1224 hash_init(qtable->queues); 1225 vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable); 1226 1227 if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 1228 clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); 1229 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1230 } 1231 1232 vfio_ap_mdev_reset_queues(qtable); 1233 1234 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1235 vfio_ap_unlink_mdev_fr_queue(q); 1236 hash_del(&q->mdev_qnode); 1237 } 1238 1239 kfree(qtable); 1240 } 1241 1242 /** 1243 * unassign_domain_store - parses the APQI from @buf and clears the 1244 * corresponding bit in the mediated matrix device's AQM 1245 * 1246 * @dev: the matrix device 1247 * @attr: the mediated matrix device's unassign_domain attribute 1248 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1249 * be unassigned 1250 * @count: the number of bytes in @buf 1251 * 1252 * Return: the number of bytes processed if the APQI is valid; otherwise, 1253 * returns one of the following errors: 1254 * -EINVAL if the APQI is not a number 1255 * -ENODEV if the APQI exceeds the maximum value configured for the system 1256 */ 1257 static ssize_t unassign_domain_store(struct device *dev, 1258 struct device_attribute *attr, 1259 const char *buf, size_t count) 1260 { 1261 int ret; 1262 unsigned long apqi; 1263 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1264 1265 get_update_locks_for_mdev(matrix_mdev); 1266 1267 ret = kstrtoul(buf, 0, &apqi); 1268 if (ret) 1269 goto done; 1270 1271 if (apqi > matrix_mdev->matrix.aqm_max) { 1272 ret = -ENODEV; 1273 goto done; 1274 } 1275 1276 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); 1277 vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi); 1278 ret = count; 1279 1280 done: 1281 release_update_locks_for_mdev(matrix_mdev); 1282 return ret; 1283 } 1284 static DEVICE_ATTR_WO(unassign_domain); 1285 1286 /** 1287 * assign_control_domain_store - parses the domain ID from @buf and sets 1288 * the corresponding bit in the mediated matrix device's ADM 1289 * 1290 * @dev: the matrix device 1291 * @attr: the mediated matrix device's assign_control_domain attribute 1292 * @buf: a buffer containing the domain ID to be assigned 1293 * @count: the number of bytes in @buf 1294 * 1295 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1296 * returns one of the following errors: 1297 * -EINVAL if the ID is not a number 1298 * -ENODEV if the ID exceeds the maximum value configured for the system 1299 */ 1300 static ssize_t assign_control_domain_store(struct device *dev, 1301 struct device_attribute *attr, 1302 const char *buf, size_t count) 1303 { 1304 int ret; 1305 unsigned long id; 1306 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1307 1308 get_update_locks_for_mdev(matrix_mdev); 1309 1310 ret = kstrtoul(buf, 0, &id); 1311 if (ret) 1312 goto done; 1313 1314 if (id > matrix_mdev->matrix.adm_max) { 1315 ret = -ENODEV; 1316 goto done; 1317 } 1318 1319 /* Set the bit in the ADM (bitmask) corresponding to the AP control 1320 * domain number (id). The bits in the mask, from most significant to 1321 * least significant, correspond to IDs 0 up to the one less than the 1322 * number of control domains that can be assigned. 1323 */ 1324 set_bit_inv(id, matrix_mdev->matrix.adm); 1325 if (vfio_ap_mdev_filter_cdoms(matrix_mdev)) 1326 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1327 1328 ret = count; 1329 done: 1330 release_update_locks_for_mdev(matrix_mdev); 1331 return ret; 1332 } 1333 static DEVICE_ATTR_WO(assign_control_domain); 1334 1335 /** 1336 * unassign_control_domain_store - parses the domain ID from @buf and 1337 * clears the corresponding bit in the mediated matrix device's ADM 1338 * 1339 * @dev: the matrix device 1340 * @attr: the mediated matrix device's unassign_control_domain attribute 1341 * @buf: a buffer containing the domain ID to be unassigned 1342 * @count: the number of bytes in @buf 1343 * 1344 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1345 * returns one of the following errors: 1346 * -EINVAL if the ID is not a number 1347 * -ENODEV if the ID exceeds the maximum value configured for the system 1348 */ 1349 static ssize_t unassign_control_domain_store(struct device *dev, 1350 struct device_attribute *attr, 1351 const char *buf, size_t count) 1352 { 1353 int ret; 1354 unsigned long domid; 1355 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1356 1357 get_update_locks_for_mdev(matrix_mdev); 1358 1359 ret = kstrtoul(buf, 0, &domid); 1360 if (ret) 1361 goto done; 1362 1363 if (domid > matrix_mdev->matrix.adm_max) { 1364 ret = -ENODEV; 1365 goto done; 1366 } 1367 1368 clear_bit_inv(domid, matrix_mdev->matrix.adm); 1369 1370 if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) { 1371 clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm); 1372 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1373 } 1374 1375 ret = count; 1376 done: 1377 release_update_locks_for_mdev(matrix_mdev); 1378 return ret; 1379 } 1380 static DEVICE_ATTR_WO(unassign_control_domain); 1381 1382 static ssize_t control_domains_show(struct device *dev, 1383 struct device_attribute *dev_attr, 1384 char *buf) 1385 { 1386 unsigned long id; 1387 int nchars = 0; 1388 int n; 1389 char *bufpos = buf; 1390 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1391 unsigned long max_domid = matrix_mdev->matrix.adm_max; 1392 1393 mutex_lock(&matrix_dev->mdevs_lock); 1394 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) { 1395 n = sprintf(bufpos, "%04lx\n", id); 1396 bufpos += n; 1397 nchars += n; 1398 } 1399 mutex_unlock(&matrix_dev->mdevs_lock); 1400 1401 return nchars; 1402 } 1403 static DEVICE_ATTR_RO(control_domains); 1404 1405 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) 1406 { 1407 char *bufpos = buf; 1408 unsigned long apid; 1409 unsigned long apqi; 1410 unsigned long apid1; 1411 unsigned long apqi1; 1412 unsigned long napm_bits = matrix->apm_max + 1; 1413 unsigned long naqm_bits = matrix->aqm_max + 1; 1414 int nchars = 0; 1415 int n; 1416 1417 apid1 = find_first_bit_inv(matrix->apm, napm_bits); 1418 apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits); 1419 1420 if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { 1421 for_each_set_bit_inv(apid, matrix->apm, napm_bits) { 1422 for_each_set_bit_inv(apqi, matrix->aqm, 1423 naqm_bits) { 1424 n = sprintf(bufpos, "%02lx.%04lx\n", apid, 1425 apqi); 1426 bufpos += n; 1427 nchars += n; 1428 } 1429 } 1430 } else if (apid1 < napm_bits) { 1431 for_each_set_bit_inv(apid, matrix->apm, napm_bits) { 1432 n = sprintf(bufpos, "%02lx.\n", apid); 1433 bufpos += n; 1434 nchars += n; 1435 } 1436 } else if (apqi1 < naqm_bits) { 1437 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) { 1438 n = sprintf(bufpos, ".%04lx\n", apqi); 1439 bufpos += n; 1440 nchars += n; 1441 } 1442 } 1443 1444 return nchars; 1445 } 1446 1447 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, 1448 char *buf) 1449 { 1450 ssize_t nchars; 1451 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1452 1453 mutex_lock(&matrix_dev->mdevs_lock); 1454 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf); 1455 mutex_unlock(&matrix_dev->mdevs_lock); 1456 1457 return nchars; 1458 } 1459 static DEVICE_ATTR_RO(matrix); 1460 1461 static ssize_t guest_matrix_show(struct device *dev, 1462 struct device_attribute *attr, char *buf) 1463 { 1464 ssize_t nchars; 1465 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1466 1467 mutex_lock(&matrix_dev->mdevs_lock); 1468 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf); 1469 mutex_unlock(&matrix_dev->mdevs_lock); 1470 1471 return nchars; 1472 } 1473 static DEVICE_ATTR_RO(guest_matrix); 1474 1475 static struct attribute *vfio_ap_mdev_attrs[] = { 1476 &dev_attr_assign_adapter.attr, 1477 &dev_attr_unassign_adapter.attr, 1478 &dev_attr_assign_domain.attr, 1479 &dev_attr_unassign_domain.attr, 1480 &dev_attr_assign_control_domain.attr, 1481 &dev_attr_unassign_control_domain.attr, 1482 &dev_attr_control_domains.attr, 1483 &dev_attr_matrix.attr, 1484 &dev_attr_guest_matrix.attr, 1485 NULL, 1486 }; 1487 1488 static struct attribute_group vfio_ap_mdev_attr_group = { 1489 .attrs = vfio_ap_mdev_attrs 1490 }; 1491 1492 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { 1493 &vfio_ap_mdev_attr_group, 1494 NULL 1495 }; 1496 1497 /** 1498 * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed 1499 * to manage AP resources for the guest whose state is represented by @kvm 1500 * 1501 * @matrix_mdev: a mediated matrix device 1502 * @kvm: reference to KVM instance 1503 * 1504 * Return: 0 if no other mediated matrix device has a reference to @kvm; 1505 * otherwise, returns an -EPERM. 1506 */ 1507 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, 1508 struct kvm *kvm) 1509 { 1510 struct ap_matrix_mdev *m; 1511 1512 if (kvm->arch.crypto.crycbd) { 1513 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1514 kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; 1515 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1516 1517 get_update_locks_for_kvm(kvm); 1518 1519 list_for_each_entry(m, &matrix_dev->mdev_list, node) { 1520 if (m != matrix_mdev && m->kvm == kvm) { 1521 release_update_locks_for_kvm(kvm); 1522 return -EPERM; 1523 } 1524 } 1525 1526 kvm_get_kvm(kvm); 1527 matrix_mdev->kvm = kvm; 1528 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1529 1530 release_update_locks_for_kvm(kvm); 1531 } 1532 1533 return 0; 1534 } 1535 1536 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova, 1537 u64 length) 1538 { 1539 struct ap_matrix_mdev *matrix_mdev = 1540 container_of(vdev, struct ap_matrix_mdev, vdev); 1541 1542 vfio_unpin_pages(&matrix_mdev->vdev, iova, 1); 1543 } 1544 1545 /** 1546 * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed 1547 * by @matrix_mdev. 1548 * 1549 * @matrix_mdev: a matrix mediated device 1550 */ 1551 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) 1552 { 1553 struct kvm *kvm = matrix_mdev->kvm; 1554 1555 if (kvm && kvm->arch.crypto.crycbd) { 1556 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1557 kvm->arch.crypto.pqap_hook = NULL; 1558 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1559 1560 get_update_locks_for_kvm(kvm); 1561 1562 kvm_arch_crypto_clear_masks(kvm); 1563 vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 1564 kvm_put_kvm(kvm); 1565 matrix_mdev->kvm = NULL; 1566 1567 release_update_locks_for_kvm(kvm); 1568 } 1569 } 1570 1571 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn) 1572 { 1573 struct ap_queue *queue; 1574 struct vfio_ap_queue *q = NULL; 1575 1576 queue = ap_get_qdev(apqn); 1577 if (!queue) 1578 return NULL; 1579 1580 if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver) 1581 q = dev_get_drvdata(&queue->ap_dev.device); 1582 1583 put_device(&queue->ap_dev.device); 1584 1585 return q; 1586 } 1587 1588 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, 1589 unsigned int retry) 1590 { 1591 struct ap_queue_status status; 1592 int ret; 1593 int retry2 = 2; 1594 1595 if (!q) 1596 return 0; 1597 retry_zapq: 1598 status = ap_zapq(q->apqn); 1599 q->reset_rc = status.response_code; 1600 switch (status.response_code) { 1601 case AP_RESPONSE_NORMAL: 1602 ret = 0; 1603 break; 1604 case AP_RESPONSE_RESET_IN_PROGRESS: 1605 if (retry--) { 1606 msleep(20); 1607 goto retry_zapq; 1608 } 1609 ret = -EBUSY; 1610 break; 1611 case AP_RESPONSE_Q_NOT_AVAIL: 1612 case AP_RESPONSE_DECONFIGURED: 1613 case AP_RESPONSE_CHECKSTOPPED: 1614 WARN_ONCE(status.irq_enabled, 1615 "PQAP/ZAPQ for %02x.%04x failed with rc=%u while IRQ enabled", 1616 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), 1617 status.response_code); 1618 ret = -EBUSY; 1619 goto free_resources; 1620 default: 1621 /* things are really broken, give up */ 1622 WARN(true, 1623 "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n", 1624 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), 1625 status.response_code); 1626 return -EIO; 1627 } 1628 1629 /* wait for the reset to take effect */ 1630 while (retry2--) { 1631 if (status.queue_empty && !status.irq_enabled) 1632 break; 1633 msleep(20); 1634 status = ap_tapq(q->apqn, NULL); 1635 } 1636 WARN_ONCE(retry2 <= 0, "unable to verify reset of queue %02x.%04x", 1637 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn)); 1638 1639 free_resources: 1640 vfio_ap_free_aqic_resources(q); 1641 1642 return ret; 1643 } 1644 1645 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable) 1646 { 1647 int ret, loop_cursor, rc = 0; 1648 struct vfio_ap_queue *q; 1649 1650 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1651 ret = vfio_ap_mdev_reset_queue(q, 1); 1652 /* 1653 * Regardless whether a queue turns out to be busy, or 1654 * is not operational, we need to continue resetting 1655 * the remaining queues. 1656 */ 1657 if (ret) 1658 rc = ret; 1659 } 1660 1661 return rc; 1662 } 1663 1664 static int vfio_ap_mdev_open_device(struct vfio_device *vdev) 1665 { 1666 struct ap_matrix_mdev *matrix_mdev = 1667 container_of(vdev, struct ap_matrix_mdev, vdev); 1668 1669 if (!vdev->kvm) 1670 return -EINVAL; 1671 1672 return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm); 1673 } 1674 1675 static void vfio_ap_mdev_close_device(struct vfio_device *vdev) 1676 { 1677 struct ap_matrix_mdev *matrix_mdev = 1678 container_of(vdev, struct ap_matrix_mdev, vdev); 1679 1680 vfio_ap_mdev_unset_kvm(matrix_mdev); 1681 } 1682 1683 static int vfio_ap_mdev_get_device_info(unsigned long arg) 1684 { 1685 unsigned long minsz; 1686 struct vfio_device_info info; 1687 1688 minsz = offsetofend(struct vfio_device_info, num_irqs); 1689 1690 if (copy_from_user(&info, (void __user *)arg, minsz)) 1691 return -EFAULT; 1692 1693 if (info.argsz < minsz) 1694 return -EINVAL; 1695 1696 info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET; 1697 info.num_regions = 0; 1698 info.num_irqs = 0; 1699 1700 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 1701 } 1702 1703 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, 1704 unsigned int cmd, unsigned long arg) 1705 { 1706 struct ap_matrix_mdev *matrix_mdev = 1707 container_of(vdev, struct ap_matrix_mdev, vdev); 1708 int ret; 1709 1710 mutex_lock(&matrix_dev->mdevs_lock); 1711 switch (cmd) { 1712 case VFIO_DEVICE_GET_INFO: 1713 ret = vfio_ap_mdev_get_device_info(arg); 1714 break; 1715 case VFIO_DEVICE_RESET: 1716 ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); 1717 break; 1718 default: 1719 ret = -EOPNOTSUPP; 1720 break; 1721 } 1722 mutex_unlock(&matrix_dev->mdevs_lock); 1723 1724 return ret; 1725 } 1726 1727 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) 1728 { 1729 struct ap_matrix_mdev *matrix_mdev; 1730 unsigned long apid = AP_QID_CARD(q->apqn); 1731 unsigned long apqi = AP_QID_QUEUE(q->apqn); 1732 1733 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 1734 if (test_bit_inv(apid, matrix_mdev->matrix.apm) && 1735 test_bit_inv(apqi, matrix_mdev->matrix.aqm)) 1736 return matrix_mdev; 1737 } 1738 1739 return NULL; 1740 } 1741 1742 static ssize_t status_show(struct device *dev, 1743 struct device_attribute *attr, 1744 char *buf) 1745 { 1746 ssize_t nchars = 0; 1747 struct vfio_ap_queue *q; 1748 struct ap_matrix_mdev *matrix_mdev; 1749 struct ap_device *apdev = to_ap_dev(dev); 1750 1751 mutex_lock(&matrix_dev->mdevs_lock); 1752 q = dev_get_drvdata(&apdev->device); 1753 matrix_mdev = vfio_ap_mdev_for_queue(q); 1754 1755 if (matrix_mdev) { 1756 if (matrix_mdev->kvm) 1757 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1758 AP_QUEUE_IN_USE); 1759 else 1760 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1761 AP_QUEUE_ASSIGNED); 1762 } else { 1763 nchars = scnprintf(buf, PAGE_SIZE, "%s\n", 1764 AP_QUEUE_UNASSIGNED); 1765 } 1766 1767 mutex_unlock(&matrix_dev->mdevs_lock); 1768 1769 return nchars; 1770 } 1771 1772 static DEVICE_ATTR_RO(status); 1773 1774 static struct attribute *vfio_queue_attrs[] = { 1775 &dev_attr_status.attr, 1776 NULL, 1777 }; 1778 1779 static const struct attribute_group vfio_queue_attr_group = { 1780 .attrs = vfio_queue_attrs, 1781 }; 1782 1783 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { 1784 .init = vfio_ap_mdev_init_dev, 1785 .release = vfio_ap_mdev_release_dev, 1786 .open_device = vfio_ap_mdev_open_device, 1787 .close_device = vfio_ap_mdev_close_device, 1788 .ioctl = vfio_ap_mdev_ioctl, 1789 .dma_unmap = vfio_ap_mdev_dma_unmap, 1790 }; 1791 1792 static struct mdev_driver vfio_ap_matrix_driver = { 1793 .device_api = VFIO_DEVICE_API_AP_STRING, 1794 .driver = { 1795 .name = "vfio_ap_mdev", 1796 .owner = THIS_MODULE, 1797 .mod_name = KBUILD_MODNAME, 1798 .dev_groups = vfio_ap_mdev_attr_groups, 1799 }, 1800 .probe = vfio_ap_mdev_probe, 1801 .remove = vfio_ap_mdev_remove, 1802 .types_attrs = vfio_ap_mdev_type_attrs, 1803 }; 1804 1805 int vfio_ap_mdev_register(void) 1806 { 1807 int ret; 1808 1809 atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT); 1810 1811 ret = mdev_register_driver(&vfio_ap_matrix_driver); 1812 if (ret) 1813 return ret; 1814 1815 matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT; 1816 matrix_dev->mdev_types[0] = &matrix_dev->mdev_type; 1817 ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device, 1818 &vfio_ap_matrix_driver, 1819 matrix_dev->mdev_types, 1); 1820 if (ret) 1821 goto err_driver; 1822 return 0; 1823 1824 err_driver: 1825 mdev_unregister_driver(&vfio_ap_matrix_driver); 1826 return ret; 1827 } 1828 1829 void vfio_ap_mdev_unregister(void) 1830 { 1831 mdev_unregister_parent(&matrix_dev->parent); 1832 mdev_unregister_driver(&vfio_ap_matrix_driver); 1833 } 1834 1835 int vfio_ap_mdev_probe_queue(struct ap_device *apdev) 1836 { 1837 int ret; 1838 struct vfio_ap_queue *q; 1839 struct ap_matrix_mdev *matrix_mdev; 1840 1841 ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); 1842 if (ret) 1843 return ret; 1844 1845 q = kzalloc(sizeof(*q), GFP_KERNEL); 1846 if (!q) 1847 return -ENOMEM; 1848 1849 q->apqn = to_ap_queue(&apdev->device)->qid; 1850 q->saved_isc = VFIO_AP_ISC_INVALID; 1851 matrix_mdev = get_update_locks_by_apqn(q->apqn); 1852 1853 if (matrix_mdev) { 1854 vfio_ap_mdev_link_queue(matrix_mdev, q); 1855 1856 if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, 1857 matrix_mdev->matrix.aqm, 1858 matrix_mdev)) 1859 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1860 } 1861 dev_set_drvdata(&apdev->device, q); 1862 release_update_locks_for_mdev(matrix_mdev); 1863 1864 return 0; 1865 } 1866 1867 void vfio_ap_mdev_remove_queue(struct ap_device *apdev) 1868 { 1869 unsigned long apid, apqi; 1870 struct vfio_ap_queue *q; 1871 struct ap_matrix_mdev *matrix_mdev; 1872 1873 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); 1874 q = dev_get_drvdata(&apdev->device); 1875 get_update_locks_for_queue(q); 1876 matrix_mdev = q->matrix_mdev; 1877 1878 if (matrix_mdev) { 1879 vfio_ap_unlink_queue_fr_mdev(q); 1880 1881 apid = AP_QID_CARD(q->apqn); 1882 apqi = AP_QID_QUEUE(q->apqn); 1883 1884 /* 1885 * If the queue is assigned to the guest's APCB, then remove 1886 * the adapter's APID from the APCB and hot it into the guest. 1887 */ 1888 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1889 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 1890 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 1891 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1892 } 1893 } 1894 1895 vfio_ap_mdev_reset_queue(q, 1); 1896 dev_set_drvdata(&apdev->device, NULL); 1897 kfree(q); 1898 release_update_locks_for_mdev(matrix_mdev); 1899 } 1900 1901 /** 1902 * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is 1903 * assigned to a mediated device under the control 1904 * of the vfio_ap device driver. 1905 * 1906 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check. 1907 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check. 1908 * 1909 * Return: 1910 * * -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are 1911 * assigned to a mediated device under the control of the vfio_ap 1912 * device driver. 1913 * * Otherwise, return 0. 1914 */ 1915 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) 1916 { 1917 int ret; 1918 1919 mutex_lock(&matrix_dev->guests_lock); 1920 mutex_lock(&matrix_dev->mdevs_lock); 1921 ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); 1922 mutex_unlock(&matrix_dev->mdevs_lock); 1923 mutex_unlock(&matrix_dev->guests_lock); 1924 1925 return ret; 1926 } 1927 1928 /** 1929 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control 1930 * domains that have been removed from the host's 1931 * AP configuration from a guest. 1932 * 1933 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest. 1934 * @aprem: the adapters that have been removed from the host's AP configuration 1935 * @aqrem: the domains that have been removed from the host's AP configuration 1936 * @cdrem: the control domains that have been removed from the host's AP 1937 * configuration. 1938 */ 1939 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, 1940 unsigned long *aprem, 1941 unsigned long *aqrem, 1942 unsigned long *cdrem) 1943 { 1944 int do_hotplug = 0; 1945 1946 if (!bitmap_empty(aprem, AP_DEVICES)) { 1947 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, 1948 matrix_mdev->shadow_apcb.apm, 1949 aprem, AP_DEVICES); 1950 } 1951 1952 if (!bitmap_empty(aqrem, AP_DOMAINS)) { 1953 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, 1954 matrix_mdev->shadow_apcb.aqm, 1955 aqrem, AP_DEVICES); 1956 } 1957 1958 if (!bitmap_empty(cdrem, AP_DOMAINS)) 1959 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, 1960 matrix_mdev->shadow_apcb.adm, 1961 cdrem, AP_DOMAINS); 1962 1963 if (do_hotplug) 1964 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1965 } 1966 1967 /** 1968 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters, 1969 * domains and control domains that have been removed 1970 * from the host AP configuration and unplugs them 1971 * from those guests. 1972 * 1973 * @ap_remove: bitmap specifying which adapters have been removed from the host 1974 * config. 1975 * @aq_remove: bitmap specifying which domains have been removed from the host 1976 * config. 1977 * @cd_remove: bitmap specifying which control domains have been removed from 1978 * the host config. 1979 */ 1980 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, 1981 unsigned long *aq_remove, 1982 unsigned long *cd_remove) 1983 { 1984 struct ap_matrix_mdev *matrix_mdev; 1985 DECLARE_BITMAP(aprem, AP_DEVICES); 1986 DECLARE_BITMAP(aqrem, AP_DOMAINS); 1987 DECLARE_BITMAP(cdrem, AP_DOMAINS); 1988 int do_remove = 0; 1989 1990 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 1991 mutex_lock(&matrix_mdev->kvm->lock); 1992 mutex_lock(&matrix_dev->mdevs_lock); 1993 1994 do_remove |= bitmap_and(aprem, ap_remove, 1995 matrix_mdev->matrix.apm, 1996 AP_DEVICES); 1997 do_remove |= bitmap_and(aqrem, aq_remove, 1998 matrix_mdev->matrix.aqm, 1999 AP_DOMAINS); 2000 do_remove |= bitmap_andnot(cdrem, cd_remove, 2001 matrix_mdev->matrix.adm, 2002 AP_DOMAINS); 2003 2004 if (do_remove) 2005 vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, 2006 cdrem); 2007 2008 mutex_unlock(&matrix_dev->mdevs_lock); 2009 mutex_unlock(&matrix_mdev->kvm->lock); 2010 } 2011 } 2012 2013 /** 2014 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and 2015 * control domains from the host AP configuration 2016 * by unplugging them from the guests that are 2017 * using them. 2018 * @cur_config_info: the current host AP configuration information 2019 * @prev_config_info: the previous host AP configuration information 2020 */ 2021 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info, 2022 struct ap_config_info *prev_config_info) 2023 { 2024 int do_remove; 2025 DECLARE_BITMAP(aprem, AP_DEVICES); 2026 DECLARE_BITMAP(aqrem, AP_DOMAINS); 2027 DECLARE_BITMAP(cdrem, AP_DOMAINS); 2028 2029 do_remove = bitmap_andnot(aprem, 2030 (unsigned long *)prev_config_info->apm, 2031 (unsigned long *)cur_config_info->apm, 2032 AP_DEVICES); 2033 do_remove |= bitmap_andnot(aqrem, 2034 (unsigned long *)prev_config_info->aqm, 2035 (unsigned long *)cur_config_info->aqm, 2036 AP_DEVICES); 2037 do_remove |= bitmap_andnot(cdrem, 2038 (unsigned long *)prev_config_info->adm, 2039 (unsigned long *)cur_config_info->adm, 2040 AP_DEVICES); 2041 2042 if (do_remove) 2043 vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem); 2044 } 2045 2046 /** 2047 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that 2048 * are older than AP type 10 (CEX4). 2049 * @apm: a bitmap of the APIDs to examine 2050 * @aqm: a bitmap of the APQIs of the queues to query for the AP type. 2051 */ 2052 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm) 2053 { 2054 bool apid_cleared; 2055 struct ap_queue_status status; 2056 unsigned long apid, apqi, info; 2057 int qtype, qtype_mask = 0xff000000; 2058 2059 for_each_set_bit_inv(apid, apm, AP_DEVICES) { 2060 apid_cleared = false; 2061 2062 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { 2063 status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info); 2064 switch (status.response_code) { 2065 /* 2066 * According to the architecture in each case 2067 * below, the queue's info should be filled. 2068 */ 2069 case AP_RESPONSE_NORMAL: 2070 case AP_RESPONSE_RESET_IN_PROGRESS: 2071 case AP_RESPONSE_DECONFIGURED: 2072 case AP_RESPONSE_CHECKSTOPPED: 2073 case AP_RESPONSE_BUSY: 2074 qtype = info & qtype_mask; 2075 2076 /* 2077 * The vfio_ap device driver only 2078 * supports CEX4 and newer adapters, so 2079 * remove the APID if the adapter is 2080 * older than a CEX4. 2081 */ 2082 if (qtype < AP_DEVICE_TYPE_CEX4) { 2083 clear_bit_inv(apid, apm); 2084 apid_cleared = true; 2085 } 2086 2087 break; 2088 2089 default: 2090 /* 2091 * If we don't know the adapter type, 2092 * clear its APID since it can't be 2093 * determined whether the vfio_ap 2094 * device driver supports it. 2095 */ 2096 clear_bit_inv(apid, apm); 2097 apid_cleared = true; 2098 break; 2099 } 2100 2101 /* 2102 * If we've already cleared the APID from the apm, there 2103 * is no need to continue examining the remainin AP 2104 * queues to determine the type of the adapter. 2105 */ 2106 if (apid_cleared) 2107 continue; 2108 } 2109 } 2110 } 2111 2112 /** 2113 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and 2114 * control domains that have been added to the host's 2115 * AP configuration for each matrix mdev to which they 2116 * are assigned. 2117 * 2118 * @apm_add: a bitmap specifying the adapters that have been added to the AP 2119 * configuration. 2120 * @aqm_add: a bitmap specifying the domains that have been added to the AP 2121 * configuration. 2122 * @adm_add: a bitmap specifying the control domains that have been added to the 2123 * AP configuration. 2124 */ 2125 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, 2126 unsigned long *adm_add) 2127 { 2128 struct ap_matrix_mdev *matrix_mdev; 2129 2130 if (list_empty(&matrix_dev->mdev_list)) 2131 return; 2132 2133 vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); 2134 2135 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2136 bitmap_and(matrix_mdev->apm_add, 2137 matrix_mdev->matrix.apm, apm_add, AP_DEVICES); 2138 bitmap_and(matrix_mdev->aqm_add, 2139 matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); 2140 bitmap_and(matrix_mdev->adm_add, 2141 matrix_mdev->matrix.adm, adm_add, AP_DEVICES); 2142 } 2143 } 2144 2145 /** 2146 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and 2147 * control domains to the host AP configuration 2148 * by updating the bitmaps that specify what adapters, 2149 * domains and control domains have been added so they 2150 * can be hot plugged into the guest when the AP bus 2151 * scan completes (see vfio_ap_on_scan_complete 2152 * function). 2153 * @cur_config_info: the current AP configuration information 2154 * @prev_config_info: the previous AP configuration information 2155 */ 2156 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info, 2157 struct ap_config_info *prev_config_info) 2158 { 2159 bool do_add; 2160 DECLARE_BITMAP(apm_add, AP_DEVICES); 2161 DECLARE_BITMAP(aqm_add, AP_DOMAINS); 2162 DECLARE_BITMAP(adm_add, AP_DOMAINS); 2163 2164 do_add = bitmap_andnot(apm_add, 2165 (unsigned long *)cur_config_info->apm, 2166 (unsigned long *)prev_config_info->apm, 2167 AP_DEVICES); 2168 do_add |= bitmap_andnot(aqm_add, 2169 (unsigned long *)cur_config_info->aqm, 2170 (unsigned long *)prev_config_info->aqm, 2171 AP_DOMAINS); 2172 do_add |= bitmap_andnot(adm_add, 2173 (unsigned long *)cur_config_info->adm, 2174 (unsigned long *)prev_config_info->adm, 2175 AP_DOMAINS); 2176 2177 if (do_add) 2178 vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add); 2179 } 2180 2181 /** 2182 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP 2183 * configuration. 2184 * 2185 * @cur_cfg_info: the current host AP configuration 2186 * @prev_cfg_info: the previous host AP configuration 2187 */ 2188 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, 2189 struct ap_config_info *prev_cfg_info) 2190 { 2191 if (!cur_cfg_info || !prev_cfg_info) 2192 return; 2193 2194 mutex_lock(&matrix_dev->guests_lock); 2195 2196 vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); 2197 vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info); 2198 memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info)); 2199 2200 mutex_unlock(&matrix_dev->guests_lock); 2201 } 2202 2203 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) 2204 { 2205 bool do_hotplug = false; 2206 int filter_domains = 0; 2207 int filter_adapters = 0; 2208 DECLARE_BITMAP(apm, AP_DEVICES); 2209 DECLARE_BITMAP(aqm, AP_DOMAINS); 2210 2211 mutex_lock(&matrix_mdev->kvm->lock); 2212 mutex_lock(&matrix_dev->mdevs_lock); 2213 2214 filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm, 2215 matrix_mdev->apm_add, AP_DEVICES); 2216 filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm, 2217 matrix_mdev->aqm_add, AP_DOMAINS); 2218 2219 if (filter_adapters && filter_domains) 2220 do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev); 2221 else if (filter_adapters) 2222 do_hotplug |= 2223 vfio_ap_mdev_filter_matrix(apm, 2224 matrix_mdev->shadow_apcb.aqm, 2225 matrix_mdev); 2226 else 2227 do_hotplug |= 2228 vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm, 2229 aqm, matrix_mdev); 2230 2231 if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add, 2232 AP_DOMAINS)) 2233 do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev); 2234 2235 if (do_hotplug) 2236 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2237 2238 mutex_unlock(&matrix_dev->mdevs_lock); 2239 mutex_unlock(&matrix_mdev->kvm->lock); 2240 } 2241 2242 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, 2243 struct ap_config_info *old_config_info) 2244 { 2245 struct ap_matrix_mdev *matrix_mdev; 2246 2247 mutex_lock(&matrix_dev->guests_lock); 2248 2249 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2250 if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && 2251 bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && 2252 bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) 2253 continue; 2254 2255 vfio_ap_mdev_hot_plug_cfg(matrix_mdev); 2256 bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); 2257 bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); 2258 bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); 2259 } 2260 2261 mutex_unlock(&matrix_dev->guests_lock); 2262 } 2263