1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Adjunct processor matrix VFIO device driver callbacks.
4  *
5  * Copyright IBM Corp. 2018
6  *
7  * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
8  *	      Halil Pasic <pasic@linux.ibm.com>
9  *	      Pierre Morel <pmorel@linux.ibm.com>
10  */
11 #include <linux/string.h>
12 #include <linux/vfio.h>
13 #include <linux/device.h>
14 #include <linux/list.h>
15 #include <linux/ctype.h>
16 #include <linux/bitops.h>
17 #include <linux/kvm_host.h>
18 #include <linux/module.h>
19 #include <linux/uuid.h>
20 #include <asm/kvm.h>
21 #include <asm/zcrypt.h>
22 
23 #include "vfio_ap_private.h"
24 #include "vfio_ap_debug.h"
25 
26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
28 
29 #define AP_QUEUE_ASSIGNED "assigned"
30 #define AP_QUEUE_UNASSIGNED "unassigned"
31 #define AP_QUEUE_IN_USE "in use"
32 
33 #define MAX_RESET_CHECK_WAIT	200	/* Sleep max 200ms for reset check	*/
34 #define AP_RESET_INTERVAL		20	/* Reset sleep interval (20ms)		*/
35 
36 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable);
37 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
38 static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
39 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q);
40 
41 /**
42  * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
43  *			     KVM guest's APCB in the proper order.
44  *
45  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
46  *
47  * The proper locking order is:
48  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
49  *			       guest's APCB.
50  * 2. kvm->lock:	       required to update a guest's APCB
51  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
52  *
53  * Note: If @kvm is NULL, the KVM lock will not be taken.
54  */
55 static inline void get_update_locks_for_kvm(struct kvm *kvm)
56 {
57 	mutex_lock(&matrix_dev->guests_lock);
58 	if (kvm)
59 		mutex_lock(&kvm->lock);
60 	mutex_lock(&matrix_dev->mdevs_lock);
61 }
62 
63 /**
64  * release_update_locks_for_kvm: Release the locks used to dynamically update a
65  *				 KVM guest's APCB in the proper order.
66  *
67  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
68  *
69  * The proper unlocking order is:
70  * 1. matrix_dev->mdevs_lock
71  * 2. kvm->lock
72  * 3. matrix_dev->guests_lock
73  *
74  * Note: If @kvm is NULL, the KVM lock will not be released.
75  */
76 static inline void release_update_locks_for_kvm(struct kvm *kvm)
77 {
78 	mutex_unlock(&matrix_dev->mdevs_lock);
79 	if (kvm)
80 		mutex_unlock(&kvm->lock);
81 	mutex_unlock(&matrix_dev->guests_lock);
82 }
83 
84 /**
85  * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
86  *			      KVM guest's APCB in the proper order.
87  *
88  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
89  *		 configuration data to use to update a KVM guest's APCB.
90  *
91  * The proper locking order is:
92  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
93  *			       guest's APCB.
94  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
95  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
96  *
97  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
98  *	 lock will not be taken.
99  */
100 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
101 {
102 	mutex_lock(&matrix_dev->guests_lock);
103 	if (matrix_mdev && matrix_mdev->kvm)
104 		mutex_lock(&matrix_mdev->kvm->lock);
105 	mutex_lock(&matrix_dev->mdevs_lock);
106 }
107 
108 /**
109  * release_update_locks_for_mdev: Release the locks used to dynamically update a
110  *				  KVM guest's APCB in the proper order.
111  *
112  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
113  *		 configuration data to use to update a KVM guest's APCB.
114  *
115  * The proper unlocking order is:
116  * 1. matrix_dev->mdevs_lock
117  * 2. matrix_mdev->kvm->lock
118  * 3. matrix_dev->guests_lock
119  *
120  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
121  *	 lock will not be released.
122  */
123 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
124 {
125 	mutex_unlock(&matrix_dev->mdevs_lock);
126 	if (matrix_mdev && matrix_mdev->kvm)
127 		mutex_unlock(&matrix_mdev->kvm->lock);
128 	mutex_unlock(&matrix_dev->guests_lock);
129 }
130 
131 /**
132  * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
133  *			     acquire the locks required to update the APCB of
134  *			     the KVM guest to which the mdev is attached.
135  *
136  * @apqn: the APQN of a queue device.
137  *
138  * The proper locking order is:
139  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
140  *			       guest's APCB.
141  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
142  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
143  *
144  * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
145  *	 will not be taken.
146  *
147  * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
148  *	   is not assigned to an ap_matrix_mdev.
149  */
150 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
151 {
152 	struct ap_matrix_mdev *matrix_mdev;
153 
154 	mutex_lock(&matrix_dev->guests_lock);
155 
156 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
159 			if (matrix_mdev->kvm)
160 				mutex_lock(&matrix_mdev->kvm->lock);
161 
162 			mutex_lock(&matrix_dev->mdevs_lock);
163 
164 			return matrix_mdev;
165 		}
166 	}
167 
168 	mutex_lock(&matrix_dev->mdevs_lock);
169 
170 	return NULL;
171 }
172 
173 /**
174  * get_update_locks_for_queue: get the locks required to update the APCB of the
175  *			       KVM guest to which the matrix mdev linked to a
176  *			       vfio_ap_queue object is attached.
177  *
178  * @q: a pointer to a vfio_ap_queue object.
179  *
180  * The proper locking order is:
181  * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
182  *				  KVM guest's APCB.
183  * 2. q->matrix_mdev->kvm->lock:  required to update a guest's APCB
184  * 3. matrix_dev->mdevs_lock:	  required to access data stored in matrix_mdev
185  *
186  * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
187  *	  will not be taken.
188  */
189 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
190 {
191 	mutex_lock(&matrix_dev->guests_lock);
192 	if (q->matrix_mdev && q->matrix_mdev->kvm)
193 		mutex_lock(&q->matrix_mdev->kvm->lock);
194 	mutex_lock(&matrix_dev->mdevs_lock);
195 }
196 
197 /**
198  * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
201  * @apqn: The APQN of a queue device
202  *
203  * Return: the pointer to the vfio_ap_queue struct representing the queue or
204  *	   NULL if the queue is not assigned to @matrix_mdev
205  */
206 static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
207 					struct ap_matrix_mdev *matrix_mdev,
208 					int apqn)
209 {
210 	struct vfio_ap_queue *q;
211 
212 	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
213 			       apqn) {
214 		if (q && q->apqn == apqn)
215 			return q;
216 	}
217 
218 	return NULL;
219 }
220 
221 /**
222  * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
223  * @apqn: The AP Queue number
224  *
225  * Checks the IRQ bit for the status of this APQN using ap_tapq.
226  * Returns if the ap_tapq function succeeded and the bit is clear.
227  * Returns if ap_tapq function failed with invalid, deconfigured or
228  * checkstopped AP.
229  * Otherwise retries up to 5 times after waiting 20ms.
230  */
231 static void vfio_ap_wait_for_irqclear(int apqn)
232 {
233 	struct ap_queue_status status;
234 	int retry = 5;
235 
236 	do {
237 		status = ap_tapq(apqn, NULL);
238 		switch (status.response_code) {
239 		case AP_RESPONSE_NORMAL:
240 		case AP_RESPONSE_RESET_IN_PROGRESS:
241 			if (!status.irq_enabled)
242 				return;
243 			fallthrough;
244 		case AP_RESPONSE_BUSY:
245 			msleep(20);
246 			break;
247 		case AP_RESPONSE_Q_NOT_AVAIL:
248 		case AP_RESPONSE_DECONFIGURED:
249 		case AP_RESPONSE_CHECKSTOPPED:
250 		default:
251 			WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
252 				  status.response_code, apqn);
253 			return;
254 		}
255 	} while (--retry);
256 
257 	WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
258 		  __func__, status.response_code, apqn);
259 }
260 
261 /**
262  * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
263  * @q: The vfio_ap_queue
264  *
265  * Unregisters the ISC in the GIB when the saved ISC not invalid.
266  * Unpins the guest's page holding the NIB when it exists.
267  * Resets the saved_iova and saved_isc to invalid values.
268  */
269 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
270 {
271 	if (!q)
272 		return;
273 	if (q->saved_isc != VFIO_AP_ISC_INVALID &&
274 	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
275 		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
276 		q->saved_isc = VFIO_AP_ISC_INVALID;
277 	}
278 	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
279 		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
280 		q->saved_iova = 0;
281 	}
282 }
283 
284 /**
285  * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
286  * @q: The vfio_ap_queue
287  *
288  * Uses ap_aqic to disable the interruption and in case of success, reset
289  * in progress or IRQ disable command already proceeded: calls
290  * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
291  * and calls vfio_ap_free_aqic_resources() to free the resources associated
292  * with the AP interrupt handling.
293  *
294  * In the case the AP is busy, or a reset is in progress,
295  * retries after 20ms, up to 5 times.
296  *
297  * Returns if ap_aqic function failed with invalid, deconfigured or
298  * checkstopped AP.
299  *
300  * Return: &struct ap_queue_status
301  */
302 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
303 {
304 	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
305 	struct ap_queue_status status;
306 	int retries = 5;
307 
308 	do {
309 		status = ap_aqic(q->apqn, aqic_gisa, 0);
310 		switch (status.response_code) {
311 		case AP_RESPONSE_OTHERWISE_CHANGED:
312 		case AP_RESPONSE_NORMAL:
313 			vfio_ap_wait_for_irqclear(q->apqn);
314 			goto end_free;
315 		case AP_RESPONSE_RESET_IN_PROGRESS:
316 		case AP_RESPONSE_BUSY:
317 			msleep(20);
318 			break;
319 		case AP_RESPONSE_Q_NOT_AVAIL:
320 		case AP_RESPONSE_DECONFIGURED:
321 		case AP_RESPONSE_CHECKSTOPPED:
322 		case AP_RESPONSE_INVALID_ADDRESS:
323 		default:
324 			/* All cases in default means AP not operational */
325 			WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
326 				  status.response_code);
327 			goto end_free;
328 		}
329 	} while (retries--);
330 
331 	WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
332 		  status.response_code);
333 end_free:
334 	vfio_ap_free_aqic_resources(q);
335 	return status;
336 }
337 
338 /**
339  * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
340  *
341  * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
342  * @nib: the location for storing the nib address.
343  *
344  * When the PQAP(AQIC) instruction is executed, general register 2 contains the
345  * address of the notification indicator byte (nib) used for IRQ notification.
346  * This function parses and validates the nib from gr2.
347  *
348  * Return: returns zero if the nib address is a valid; otherwise, returns
349  *	   -EINVAL.
350  */
351 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
352 {
353 	*nib = vcpu->run->s.regs.gprs[2];
354 
355 	if (!*nib)
356 		return -EINVAL;
357 	if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT)))
358 		return -EINVAL;
359 
360 	return 0;
361 }
362 
363 /**
364  * vfio_ap_irq_enable - Enable Interruption for a APQN
365  *
366  * @q:	 the vfio_ap_queue holding AQIC parameters
367  * @isc: the guest ISC to register with the GIB interface
368  * @vcpu: the vcpu object containing the registers specifying the parameters
369  *	  passed to the PQAP(AQIC) instruction.
370  *
371  * Pin the NIB saved in *q
372  * Register the guest ISC to GIB interface and retrieve the
373  * host ISC to issue the host side PQAP/AQIC
374  *
375  * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the
376  * vfio_pin_pages failed.
377  *
378  * Otherwise return the ap_queue_status returned by the ap_aqic(),
379  * all retry handling will be done by the guest.
380  *
381  * Return: &struct ap_queue_status
382  */
383 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
384 						 int isc,
385 						 struct kvm_vcpu *vcpu)
386 {
387 	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
388 	struct ap_queue_status status = {};
389 	struct kvm_s390_gisa *gisa;
390 	struct page *h_page;
391 	int nisc;
392 	struct kvm *kvm;
393 	phys_addr_t h_nib;
394 	dma_addr_t nib;
395 	int ret;
396 
397 	/* Verify that the notification indicator byte address is valid */
398 	if (vfio_ap_validate_nib(vcpu, &nib)) {
399 		VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
400 				 __func__, &nib, q->apqn);
401 
402 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
403 		return status;
404 	}
405 
406 	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
407 			     IOMMU_READ | IOMMU_WRITE, &h_page);
408 	switch (ret) {
409 	case 1:
410 		break;
411 	default:
412 		VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
413 				 "nib=%pad, apqn=%#04x\n",
414 				 __func__, ret, &nib, q->apqn);
415 
416 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
417 		return status;
418 	}
419 
420 	kvm = q->matrix_mdev->kvm;
421 	gisa = kvm->arch.gisa_int.origin;
422 
423 	h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
424 	aqic_gisa.gisc = isc;
425 
426 	nisc = kvm_s390_gisc_register(kvm, isc);
427 	if (nisc < 0) {
428 		VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
429 				 __func__, nisc, isc, q->apqn);
430 
431 		status.response_code = AP_RESPONSE_INVALID_GISA;
432 		return status;
433 	}
434 
435 	aqic_gisa.isc = nisc;
436 	aqic_gisa.ir = 1;
437 	aqic_gisa.gisa = virt_to_phys(gisa) >> 4;
438 
439 	status = ap_aqic(q->apqn, aqic_gisa, h_nib);
440 	switch (status.response_code) {
441 	case AP_RESPONSE_NORMAL:
442 		/* See if we did clear older IRQ configuration */
443 		vfio_ap_free_aqic_resources(q);
444 		q->saved_iova = nib;
445 		q->saved_isc = isc;
446 		break;
447 	case AP_RESPONSE_OTHERWISE_CHANGED:
448 		/* We could not modify IRQ setings: clear new configuration */
449 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
450 		kvm_s390_gisc_unregister(kvm, isc);
451 		break;
452 	default:
453 		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
454 			status.response_code);
455 		vfio_ap_irq_disable(q);
456 		break;
457 	}
458 
459 	if (status.response_code != AP_RESPONSE_NORMAL) {
460 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
461 				 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
462 				 "gisa=%#x, isc=%#x, apqn=%#04x\n",
463 				 __func__, status.response_code,
464 				 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
465 				 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
466 				 q->apqn);
467 	}
468 
469 	return status;
470 }
471 
472 /**
473  * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
474  *				of big endian elements that can be passed by
475  *				value to an s390dbf sprintf event function to
476  *				format a UUID string.
477  *
478  * @guid: the object containing the little endian guid
479  * @uuid: a six-element array of long values that can be passed by value as
480  *	  arguments for a formatting string specifying a UUID.
481  *
482  * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
483  * event functions if the memory for the passed string is available as long as
484  * the debug feature exists. Since a mediated device can be removed at any
485  * time, it's name can not be used because %s passes the reference to the string
486  * in memory and the reference will go stale once the device is removed .
487  *
488  * The s390dbf string formatting function allows a maximum of 9 arguments for a
489  * message to be displayed in the 'sprintf' view. In order to use the bytes
490  * comprising the mediated device's UUID to display the mediated device name,
491  * they will have to be converted into an array whose elements can be passed by
492  * value to sprintf. For example:
493  *
494  * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
495  * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
496  * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
497  * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
498  */
499 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
500 {
501 	/*
502 	 * The input guid is ordered in little endian, so it needs to be
503 	 * reordered for displaying a UUID as a string. This specifies the
504 	 * guid indices in proper order.
505 	 */
506 	uuid[0] = le32_to_cpup((__le32 *)guid);
507 	uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
508 	uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
509 	uuid[3] = *((__u16 *)&guid->b[8]);
510 	uuid[4] = *((__u16 *)&guid->b[10]);
511 	uuid[5] = *((__u32 *)&guid->b[12]);
512 }
513 
514 /**
515  * handle_pqap - PQAP instruction callback
516  *
517  * @vcpu: The vcpu on which we received the PQAP instruction
518  *
519  * Get the general register contents to initialize internal variables.
520  * REG[0]: APQN
521  * REG[1]: IR and ISC
522  * REG[2]: NIB
523  *
524  * Response.status may be set to following Response Code:
525  * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
526  * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
527  * - AP_RESPONSE_NORMAL (0) : in case of successs
528  *   Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
529  * We take the matrix_dev lock to ensure serialization on queues and
530  * mediated device access.
531  *
532  * Return: 0 if we could handle the request inside KVM.
533  * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
534  */
535 static int handle_pqap(struct kvm_vcpu *vcpu)
536 {
537 	uint64_t status;
538 	uint16_t apqn;
539 	unsigned long uuid[6];
540 	struct vfio_ap_queue *q;
541 	struct ap_queue_status qstatus = {
542 			       .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
543 	struct ap_matrix_mdev *matrix_mdev;
544 
545 	apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
546 
547 	/* If we do not use the AIV facility just go to userland */
548 	if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
549 		VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
550 				 __func__, apqn, vcpu->arch.sie_block->eca);
551 
552 		return -EOPNOTSUPP;
553 	}
554 
555 	mutex_lock(&matrix_dev->mdevs_lock);
556 
557 	if (!vcpu->kvm->arch.crypto.pqap_hook) {
558 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
559 				 __func__, apqn);
560 
561 		goto out_unlock;
562 	}
563 
564 	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
565 				   struct ap_matrix_mdev, pqap_hook);
566 
567 	/* If the there is no guest using the mdev, there is nothing to do */
568 	if (!matrix_mdev->kvm) {
569 		vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
570 		VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
571 				 __func__, uuid[0],  uuid[1], uuid[2],
572 				 uuid[3], uuid[4], uuid[5], apqn);
573 		goto out_unlock;
574 	}
575 
576 	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
577 	if (!q) {
578 		VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
579 				 __func__, AP_QID_CARD(apqn),
580 				 AP_QID_QUEUE(apqn));
581 		goto out_unlock;
582 	}
583 
584 	status = vcpu->run->s.regs.gprs[1];
585 
586 	/* If IR bit(16) is set we enable the interrupt */
587 	if ((status >> (63 - 16)) & 0x01)
588 		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
589 	else
590 		qstatus = vfio_ap_irq_disable(q);
591 
592 out_unlock:
593 	memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
594 	vcpu->run->s.regs.gprs[1] >>= 32;
595 	mutex_unlock(&matrix_dev->mdevs_lock);
596 	return 0;
597 }
598 
599 static void vfio_ap_matrix_init(struct ap_config_info *info,
600 				struct ap_matrix *matrix)
601 {
602 	matrix->apm_max = info->apxa ? info->Na : 63;
603 	matrix->aqm_max = info->apxa ? info->Nd : 15;
604 	matrix->adm_max = info->apxa ? info->Nd : 15;
605 }
606 
607 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
608 {
609 	if (matrix_mdev->kvm)
610 		kvm_arch_crypto_set_masks(matrix_mdev->kvm,
611 					  matrix_mdev->shadow_apcb.apm,
612 					  matrix_mdev->shadow_apcb.aqm,
613 					  matrix_mdev->shadow_apcb.adm);
614 }
615 
616 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
617 {
618 	DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
619 
620 	bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
621 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
622 		   (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
623 
624 	return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
625 			     AP_DOMAINS);
626 }
627 
628 /*
629  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
630  *				to ensure no queue devices are passed through to
631  *				the guest that are not bound to the vfio_ap
632  *				device driver.
633  *
634  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
635  *
636  * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
637  *	 driver, its APID will be filtered from the guest's APCB. The matrix
638  *	 structure precludes filtering an individual APQN, so its APID will be
639  *	 filtered.
640  *
641  * Return: a boolean value indicating whether the KVM guest's APCB was changed
642  *	   by the filtering or not.
643  */
644 static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm,
645 				       struct ap_matrix_mdev *matrix_mdev)
646 {
647 	unsigned long apid, apqi, apqn;
648 	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
649 	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
650 	struct vfio_ap_queue *q;
651 
652 	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
653 	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
654 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
655 
656 	/*
657 	 * Copy the adapters, domains and control domains to the shadow_apcb
658 	 * from the matrix mdev, but only those that are assigned to the host's
659 	 * AP configuration.
660 	 */
661 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
662 		   (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
663 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
664 		   (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
665 
666 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
667 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
668 			/*
669 			 * If the APQN is not bound to the vfio_ap device
670 			 * driver, then we can't assign it to the guest's
671 			 * AP configuration. The AP architecture won't
672 			 * allow filtering of a single APQN, so let's filter
673 			 * the APID since an adapter represents a physical
674 			 * hardware device.
675 			 */
676 			apqn = AP_MKQID(apid, apqi);
677 			q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
678 			if (!q || q->reset_rc) {
679 				clear_bit_inv(apid,
680 					      matrix_mdev->shadow_apcb.apm);
681 				break;
682 			}
683 		}
684 	}
685 
686 	return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
687 			     AP_DEVICES) ||
688 	       !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
689 			     AP_DOMAINS);
690 }
691 
692 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
693 {
694 	struct ap_matrix_mdev *matrix_mdev =
695 		container_of(vdev, struct ap_matrix_mdev, vdev);
696 
697 	matrix_mdev->mdev = to_mdev_device(vdev->dev);
698 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
699 	matrix_mdev->pqap_hook = handle_pqap;
700 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
701 	hash_init(matrix_mdev->qtable.queues);
702 
703 	return 0;
704 }
705 
706 static int vfio_ap_mdev_probe(struct mdev_device *mdev)
707 {
708 	struct ap_matrix_mdev *matrix_mdev;
709 	int ret;
710 
711 	matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
712 					&vfio_ap_matrix_dev_ops);
713 	if (IS_ERR(matrix_mdev))
714 		return PTR_ERR(matrix_mdev);
715 
716 	ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
717 	if (ret)
718 		goto err_put_vdev;
719 	dev_set_drvdata(&mdev->dev, matrix_mdev);
720 	mutex_lock(&matrix_dev->mdevs_lock);
721 	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
722 	mutex_unlock(&matrix_dev->mdevs_lock);
723 	return 0;
724 
725 err_put_vdev:
726 	vfio_put_device(&matrix_mdev->vdev);
727 	return ret;
728 }
729 
730 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
731 				    struct vfio_ap_queue *q)
732 {
733 	if (q) {
734 		q->matrix_mdev = matrix_mdev;
735 		hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
736 	}
737 }
738 
739 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
740 {
741 	struct vfio_ap_queue *q;
742 
743 	q = vfio_ap_find_queue(apqn);
744 	vfio_ap_mdev_link_queue(matrix_mdev, q);
745 }
746 
747 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
748 {
749 	hash_del(&q->mdev_qnode);
750 }
751 
752 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
753 {
754 	q->matrix_mdev = NULL;
755 }
756 
757 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
758 {
759 	struct vfio_ap_queue *q;
760 	unsigned long apid, apqi;
761 
762 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
763 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
764 				     AP_DOMAINS) {
765 			q = vfio_ap_mdev_get_queue(matrix_mdev,
766 						   AP_MKQID(apid, apqi));
767 			if (q)
768 				q->matrix_mdev = NULL;
769 		}
770 	}
771 }
772 
773 static void vfio_ap_mdev_remove(struct mdev_device *mdev)
774 {
775 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
776 
777 	vfio_unregister_group_dev(&matrix_mdev->vdev);
778 
779 	mutex_lock(&matrix_dev->guests_lock);
780 	mutex_lock(&matrix_dev->mdevs_lock);
781 	vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
782 	vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
783 	list_del(&matrix_mdev->node);
784 	mutex_unlock(&matrix_dev->mdevs_lock);
785 	mutex_unlock(&matrix_dev->guests_lock);
786 	vfio_put_device(&matrix_mdev->vdev);
787 }
788 
789 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \
790 			 "already assigned to %s"
791 
792 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev,
793 					 unsigned long *apm,
794 					 unsigned long *aqm)
795 {
796 	unsigned long apid, apqi;
797 	const struct device *dev = mdev_dev(matrix_mdev->mdev);
798 	const char *mdev_name = dev_name(dev);
799 
800 	for_each_set_bit_inv(apid, apm, AP_DEVICES)
801 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
802 			dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name);
803 }
804 
805 /**
806  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
807  *
808  * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
809  * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
810  *
811  * Verifies that each APQN derived from the Cartesian product of a bitmap of
812  * AP adapter IDs and AP queue indexes is not configured for any matrix
813  * mediated device. AP queue sharing is not allowed.
814  *
815  * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
816  */
817 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm,
818 					  unsigned long *mdev_aqm)
819 {
820 	struct ap_matrix_mdev *matrix_mdev;
821 	DECLARE_BITMAP(apm, AP_DEVICES);
822 	DECLARE_BITMAP(aqm, AP_DOMAINS);
823 
824 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
825 		/*
826 		 * If the input apm and aqm are fields of the matrix_mdev
827 		 * object, then move on to the next matrix_mdev.
828 		 */
829 		if (mdev_apm == matrix_mdev->matrix.apm &&
830 		    mdev_aqm == matrix_mdev->matrix.aqm)
831 			continue;
832 
833 		memset(apm, 0, sizeof(apm));
834 		memset(aqm, 0, sizeof(aqm));
835 
836 		/*
837 		 * We work on full longs, as we can only exclude the leftover
838 		 * bits in non-inverse order. The leftover is all zeros.
839 		 */
840 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,
841 				AP_DEVICES))
842 			continue;
843 
844 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,
845 				AP_DOMAINS))
846 			continue;
847 
848 		vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm);
849 
850 		return -EADDRINUSE;
851 	}
852 
853 	return 0;
854 }
855 
856 /**
857  * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
858  *				 not reserved for the default zcrypt driver and
859  *				 are not assigned to another mdev.
860  *
861  * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
862  *
863  * Return: One of the following values:
864  * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
865  *   most likely -EBUSY indicating the ap_perms_mutex lock is already held.
866  * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
867  *		   zcrypt default driver.
868  * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
869  * o A zero indicating validation succeeded.
870  */
871 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
872 {
873 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
874 					       matrix_mdev->matrix.aqm))
875 		return -EADDRNOTAVAIL;
876 
877 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,
878 					      matrix_mdev->matrix.aqm);
879 }
880 
881 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
882 				      unsigned long apid)
883 {
884 	unsigned long apqi;
885 
886 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
887 		vfio_ap_mdev_link_apqn(matrix_mdev,
888 				       AP_MKQID(apid, apqi));
889 }
890 
891 /**
892  * assign_adapter_store - parses the APID from @buf and sets the
893  * corresponding bit in the mediated matrix device's APM
894  *
895  * @dev:	the matrix device
896  * @attr:	the mediated matrix device's assign_adapter attribute
897  * @buf:	a buffer containing the AP adapter number (APID) to
898  *		be assigned
899  * @count:	the number of bytes in @buf
900  *
901  * Return: the number of bytes processed if the APID is valid; otherwise,
902  * returns one of the following errors:
903  *
904  *	1. -EINVAL
905  *	   The APID is not a valid number
906  *
907  *	2. -ENODEV
908  *	   The APID exceeds the maximum value configured for the system
909  *
910  *	3. -EADDRNOTAVAIL
911  *	   An APQN derived from the cross product of the APID being assigned
912  *	   and the APQIs previously assigned is not bound to the vfio_ap device
913  *	   driver; or, if no APQIs have yet been assigned, the APID is not
914  *	   contained in an APQN bound to the vfio_ap device driver.
915  *
916  *	4. -EADDRINUSE
917  *	   An APQN derived from the cross product of the APID being assigned
918  *	   and the APQIs previously assigned is being used by another mediated
919  *	   matrix device
920  *
921  *	5. -EAGAIN
922  *	   A lock required to validate the mdev's AP configuration could not
923  *	   be obtained.
924  */
925 static ssize_t assign_adapter_store(struct device *dev,
926 				    struct device_attribute *attr,
927 				    const char *buf, size_t count)
928 {
929 	int ret;
930 	unsigned long apid;
931 	DECLARE_BITMAP(apm_delta, AP_DEVICES);
932 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
933 
934 	mutex_lock(&ap_perms_mutex);
935 	get_update_locks_for_mdev(matrix_mdev);
936 
937 	ret = kstrtoul(buf, 0, &apid);
938 	if (ret)
939 		goto done;
940 
941 	if (apid > matrix_mdev->matrix.apm_max) {
942 		ret = -ENODEV;
943 		goto done;
944 	}
945 
946 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
947 		ret = count;
948 		goto done;
949 	}
950 
951 	set_bit_inv(apid, matrix_mdev->matrix.apm);
952 
953 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
954 	if (ret) {
955 		clear_bit_inv(apid, matrix_mdev->matrix.apm);
956 		goto done;
957 	}
958 
959 	vfio_ap_mdev_link_adapter(matrix_mdev, apid);
960 	memset(apm_delta, 0, sizeof(apm_delta));
961 	set_bit_inv(apid, apm_delta);
962 
963 	if (vfio_ap_mdev_filter_matrix(apm_delta,
964 				       matrix_mdev->matrix.aqm, matrix_mdev))
965 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
966 
967 	ret = count;
968 done:
969 	release_update_locks_for_mdev(matrix_mdev);
970 	mutex_unlock(&ap_perms_mutex);
971 
972 	return ret;
973 }
974 static DEVICE_ATTR_WO(assign_adapter);
975 
976 static struct vfio_ap_queue
977 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
978 			     unsigned long apid, unsigned long apqi)
979 {
980 	struct vfio_ap_queue *q = NULL;
981 
982 	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
983 	/* If the queue is assigned to the matrix mdev, unlink it. */
984 	if (q)
985 		vfio_ap_unlink_queue_fr_mdev(q);
986 
987 	return q;
988 }
989 
990 /**
991  * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
992  *				 adapter from the matrix mdev to which the
993  *				 adapter was assigned.
994  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
995  * @apid: the APID of the unassigned adapter.
996  * @qtable: table for storing queues associated with unassigned adapter.
997  */
998 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
999 					unsigned long apid,
1000 					struct ap_queue_table *qtable)
1001 {
1002 	unsigned long apqi;
1003 	struct vfio_ap_queue *q;
1004 
1005 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
1006 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1007 
1008 		if (q && qtable) {
1009 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1010 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1011 				hash_add(qtable->queues, &q->mdev_qnode,
1012 					 q->apqn);
1013 		}
1014 	}
1015 }
1016 
1017 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
1018 					    unsigned long apid)
1019 {
1020 	int loop_cursor;
1021 	struct vfio_ap_queue *q;
1022 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1023 
1024 	hash_init(qtable->queues);
1025 	vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable);
1026 
1027 	if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
1028 		clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1029 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1030 	}
1031 
1032 	vfio_ap_mdev_reset_queues(qtable);
1033 
1034 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1035 		vfio_ap_unlink_mdev_fr_queue(q);
1036 		hash_del(&q->mdev_qnode);
1037 	}
1038 
1039 	kfree(qtable);
1040 }
1041 
1042 /**
1043  * unassign_adapter_store - parses the APID from @buf and clears the
1044  * corresponding bit in the mediated matrix device's APM
1045  *
1046  * @dev:	the matrix device
1047  * @attr:	the mediated matrix device's unassign_adapter attribute
1048  * @buf:	a buffer containing the adapter number (APID) to be unassigned
1049  * @count:	the number of bytes in @buf
1050  *
1051  * Return: the number of bytes processed if the APID is valid; otherwise,
1052  * returns one of the following errors:
1053  *	-EINVAL if the APID is not a number
1054  *	-ENODEV if the APID it exceeds the maximum value configured for the
1055  *		system
1056  */
1057 static ssize_t unassign_adapter_store(struct device *dev,
1058 				      struct device_attribute *attr,
1059 				      const char *buf, size_t count)
1060 {
1061 	int ret;
1062 	unsigned long apid;
1063 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1064 
1065 	get_update_locks_for_mdev(matrix_mdev);
1066 
1067 	ret = kstrtoul(buf, 0, &apid);
1068 	if (ret)
1069 		goto done;
1070 
1071 	if (apid > matrix_mdev->matrix.apm_max) {
1072 		ret = -ENODEV;
1073 		goto done;
1074 	}
1075 
1076 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1077 		ret = count;
1078 		goto done;
1079 	}
1080 
1081 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
1082 	vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
1083 	ret = count;
1084 done:
1085 	release_update_locks_for_mdev(matrix_mdev);
1086 	return ret;
1087 }
1088 static DEVICE_ATTR_WO(unassign_adapter);
1089 
1090 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
1091 				     unsigned long apqi)
1092 {
1093 	unsigned long apid;
1094 
1095 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
1096 		vfio_ap_mdev_link_apqn(matrix_mdev,
1097 				       AP_MKQID(apid, apqi));
1098 }
1099 
1100 /**
1101  * assign_domain_store - parses the APQI from @buf and sets the
1102  * corresponding bit in the mediated matrix device's AQM
1103  *
1104  * @dev:	the matrix device
1105  * @attr:	the mediated matrix device's assign_domain attribute
1106  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1107  *		be assigned
1108  * @count:	the number of bytes in @buf
1109  *
1110  * Return: the number of bytes processed if the APQI is valid; otherwise returns
1111  * one of the following errors:
1112  *
1113  *	1. -EINVAL
1114  *	   The APQI is not a valid number
1115  *
1116  *	2. -ENODEV
1117  *	   The APQI exceeds the maximum value configured for the system
1118  *
1119  *	3. -EADDRNOTAVAIL
1120  *	   An APQN derived from the cross product of the APQI being assigned
1121  *	   and the APIDs previously assigned is not bound to the vfio_ap device
1122  *	   driver; or, if no APIDs have yet been assigned, the APQI is not
1123  *	   contained in an APQN bound to the vfio_ap device driver.
1124  *
1125  *	4. -EADDRINUSE
1126  *	   An APQN derived from the cross product of the APQI being assigned
1127  *	   and the APIDs previously assigned is being used by another mediated
1128  *	   matrix device
1129  *
1130  *	5. -EAGAIN
1131  *	   The lock required to validate the mdev's AP configuration could not
1132  *	   be obtained.
1133  */
1134 static ssize_t assign_domain_store(struct device *dev,
1135 				   struct device_attribute *attr,
1136 				   const char *buf, size_t count)
1137 {
1138 	int ret;
1139 	unsigned long apqi;
1140 	DECLARE_BITMAP(aqm_delta, AP_DOMAINS);
1141 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1142 
1143 	mutex_lock(&ap_perms_mutex);
1144 	get_update_locks_for_mdev(matrix_mdev);
1145 
1146 	ret = kstrtoul(buf, 0, &apqi);
1147 	if (ret)
1148 		goto done;
1149 
1150 	if (apqi > matrix_mdev->matrix.aqm_max) {
1151 		ret = -ENODEV;
1152 		goto done;
1153 	}
1154 
1155 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1156 		ret = count;
1157 		goto done;
1158 	}
1159 
1160 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);
1161 
1162 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1163 	if (ret) {
1164 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
1165 		goto done;
1166 	}
1167 
1168 	vfio_ap_mdev_link_domain(matrix_mdev, apqi);
1169 	memset(aqm_delta, 0, sizeof(aqm_delta));
1170 	set_bit_inv(apqi, aqm_delta);
1171 
1172 	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,
1173 				       matrix_mdev))
1174 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1175 
1176 	ret = count;
1177 done:
1178 	release_update_locks_for_mdev(matrix_mdev);
1179 	mutex_unlock(&ap_perms_mutex);
1180 
1181 	return ret;
1182 }
1183 static DEVICE_ATTR_WO(assign_domain);
1184 
1185 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
1186 				       unsigned long apqi,
1187 				       struct ap_queue_table *qtable)
1188 {
1189 	unsigned long apid;
1190 	struct vfio_ap_queue *q;
1191 
1192 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
1193 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1194 
1195 		if (q && qtable) {
1196 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1197 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1198 				hash_add(qtable->queues, &q->mdev_qnode,
1199 					 q->apqn);
1200 		}
1201 	}
1202 }
1203 
1204 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
1205 					   unsigned long apqi)
1206 {
1207 	int loop_cursor;
1208 	struct vfio_ap_queue *q;
1209 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1210 
1211 	hash_init(qtable->queues);
1212 	vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable);
1213 
1214 	if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1215 		clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
1216 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1217 	}
1218 
1219 	vfio_ap_mdev_reset_queues(qtable);
1220 
1221 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1222 		vfio_ap_unlink_mdev_fr_queue(q);
1223 		hash_del(&q->mdev_qnode);
1224 	}
1225 
1226 	kfree(qtable);
1227 }
1228 
1229 /**
1230  * unassign_domain_store - parses the APQI from @buf and clears the
1231  * corresponding bit in the mediated matrix device's AQM
1232  *
1233  * @dev:	the matrix device
1234  * @attr:	the mediated matrix device's unassign_domain attribute
1235  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1236  *		be unassigned
1237  * @count:	the number of bytes in @buf
1238  *
1239  * Return: the number of bytes processed if the APQI is valid; otherwise,
1240  * returns one of the following errors:
1241  *	-EINVAL if the APQI is not a number
1242  *	-ENODEV if the APQI exceeds the maximum value configured for the system
1243  */
1244 static ssize_t unassign_domain_store(struct device *dev,
1245 				     struct device_attribute *attr,
1246 				     const char *buf, size_t count)
1247 {
1248 	int ret;
1249 	unsigned long apqi;
1250 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1251 
1252 	get_update_locks_for_mdev(matrix_mdev);
1253 
1254 	ret = kstrtoul(buf, 0, &apqi);
1255 	if (ret)
1256 		goto done;
1257 
1258 	if (apqi > matrix_mdev->matrix.aqm_max) {
1259 		ret = -ENODEV;
1260 		goto done;
1261 	}
1262 
1263 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1264 		ret = count;
1265 		goto done;
1266 	}
1267 
1268 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
1269 	vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
1270 	ret = count;
1271 
1272 done:
1273 	release_update_locks_for_mdev(matrix_mdev);
1274 	return ret;
1275 }
1276 static DEVICE_ATTR_WO(unassign_domain);
1277 
1278 /**
1279  * assign_control_domain_store - parses the domain ID from @buf and sets
1280  * the corresponding bit in the mediated matrix device's ADM
1281  *
1282  * @dev:	the matrix device
1283  * @attr:	the mediated matrix device's assign_control_domain attribute
1284  * @buf:	a buffer containing the domain ID to be assigned
1285  * @count:	the number of bytes in @buf
1286  *
1287  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1288  * returns one of the following errors:
1289  *	-EINVAL if the ID is not a number
1290  *	-ENODEV if the ID exceeds the maximum value configured for the system
1291  */
1292 static ssize_t assign_control_domain_store(struct device *dev,
1293 					   struct device_attribute *attr,
1294 					   const char *buf, size_t count)
1295 {
1296 	int ret;
1297 	unsigned long id;
1298 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1299 
1300 	get_update_locks_for_mdev(matrix_mdev);
1301 
1302 	ret = kstrtoul(buf, 0, &id);
1303 	if (ret)
1304 		goto done;
1305 
1306 	if (id > matrix_mdev->matrix.adm_max) {
1307 		ret = -ENODEV;
1308 		goto done;
1309 	}
1310 
1311 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
1312 		ret = count;
1313 		goto done;
1314 	}
1315 
1316 	/* Set the bit in the ADM (bitmask) corresponding to the AP control
1317 	 * domain number (id). The bits in the mask, from most significant to
1318 	 * least significant, correspond to IDs 0 up to the one less than the
1319 	 * number of control domains that can be assigned.
1320 	 */
1321 	set_bit_inv(id, matrix_mdev->matrix.adm);
1322 	if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
1323 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1324 
1325 	ret = count;
1326 done:
1327 	release_update_locks_for_mdev(matrix_mdev);
1328 	return ret;
1329 }
1330 static DEVICE_ATTR_WO(assign_control_domain);
1331 
1332 /**
1333  * unassign_control_domain_store - parses the domain ID from @buf and
1334  * clears the corresponding bit in the mediated matrix device's ADM
1335  *
1336  * @dev:	the matrix device
1337  * @attr:	the mediated matrix device's unassign_control_domain attribute
1338  * @buf:	a buffer containing the domain ID to be unassigned
1339  * @count:	the number of bytes in @buf
1340  *
1341  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1342  * returns one of the following errors:
1343  *	-EINVAL if the ID is not a number
1344  *	-ENODEV if the ID exceeds the maximum value configured for the system
1345  */
1346 static ssize_t unassign_control_domain_store(struct device *dev,
1347 					     struct device_attribute *attr,
1348 					     const char *buf, size_t count)
1349 {
1350 	int ret;
1351 	unsigned long domid;
1352 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1353 
1354 	get_update_locks_for_mdev(matrix_mdev);
1355 
1356 	ret = kstrtoul(buf, 0, &domid);
1357 	if (ret)
1358 		goto done;
1359 
1360 	if (domid > matrix_mdev->matrix.adm_max) {
1361 		ret = -ENODEV;
1362 		goto done;
1363 	}
1364 
1365 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
1366 		ret = count;
1367 		goto done;
1368 	}
1369 
1370 	clear_bit_inv(domid, matrix_mdev->matrix.adm);
1371 
1372 	if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
1373 		clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
1374 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1375 	}
1376 
1377 	ret = count;
1378 done:
1379 	release_update_locks_for_mdev(matrix_mdev);
1380 	return ret;
1381 }
1382 static DEVICE_ATTR_WO(unassign_control_domain);
1383 
1384 static ssize_t control_domains_show(struct device *dev,
1385 				    struct device_attribute *dev_attr,
1386 				    char *buf)
1387 {
1388 	unsigned long id;
1389 	int nchars = 0;
1390 	int n;
1391 	char *bufpos = buf;
1392 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1393 	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1394 
1395 	mutex_lock(&matrix_dev->mdevs_lock);
1396 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
1397 		n = sprintf(bufpos, "%04lx\n", id);
1398 		bufpos += n;
1399 		nchars += n;
1400 	}
1401 	mutex_unlock(&matrix_dev->mdevs_lock);
1402 
1403 	return nchars;
1404 }
1405 static DEVICE_ATTR_RO(control_domains);
1406 
1407 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
1408 {
1409 	char *bufpos = buf;
1410 	unsigned long apid;
1411 	unsigned long apqi;
1412 	unsigned long apid1;
1413 	unsigned long apqi1;
1414 	unsigned long napm_bits = matrix->apm_max + 1;
1415 	unsigned long naqm_bits = matrix->aqm_max + 1;
1416 	int nchars = 0;
1417 	int n;
1418 
1419 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);
1420 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
1421 
1422 	if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
1423 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1424 			for_each_set_bit_inv(apqi, matrix->aqm,
1425 					     naqm_bits) {
1426 				n = sprintf(bufpos, "%02lx.%04lx\n", apid,
1427 					    apqi);
1428 				bufpos += n;
1429 				nchars += n;
1430 			}
1431 		}
1432 	} else if (apid1 < napm_bits) {
1433 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1434 			n = sprintf(bufpos, "%02lx.\n", apid);
1435 			bufpos += n;
1436 			nchars += n;
1437 		}
1438 	} else if (apqi1 < naqm_bits) {
1439 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {
1440 			n = sprintf(bufpos, ".%04lx\n", apqi);
1441 			bufpos += n;
1442 			nchars += n;
1443 		}
1444 	}
1445 
1446 	return nchars;
1447 }
1448 
1449 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1450 			   char *buf)
1451 {
1452 	ssize_t nchars;
1453 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1454 
1455 	mutex_lock(&matrix_dev->mdevs_lock);
1456 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
1457 	mutex_unlock(&matrix_dev->mdevs_lock);
1458 
1459 	return nchars;
1460 }
1461 static DEVICE_ATTR_RO(matrix);
1462 
1463 static ssize_t guest_matrix_show(struct device *dev,
1464 				 struct device_attribute *attr, char *buf)
1465 {
1466 	ssize_t nchars;
1467 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1468 
1469 	mutex_lock(&matrix_dev->mdevs_lock);
1470 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
1471 	mutex_unlock(&matrix_dev->mdevs_lock);
1472 
1473 	return nchars;
1474 }
1475 static DEVICE_ATTR_RO(guest_matrix);
1476 
1477 static struct attribute *vfio_ap_mdev_attrs[] = {
1478 	&dev_attr_assign_adapter.attr,
1479 	&dev_attr_unassign_adapter.attr,
1480 	&dev_attr_assign_domain.attr,
1481 	&dev_attr_unassign_domain.attr,
1482 	&dev_attr_assign_control_domain.attr,
1483 	&dev_attr_unassign_control_domain.attr,
1484 	&dev_attr_control_domains.attr,
1485 	&dev_attr_matrix.attr,
1486 	&dev_attr_guest_matrix.attr,
1487 	NULL,
1488 };
1489 
1490 static struct attribute_group vfio_ap_mdev_attr_group = {
1491 	.attrs = vfio_ap_mdev_attrs
1492 };
1493 
1494 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1495 	&vfio_ap_mdev_attr_group,
1496 	NULL
1497 };
1498 
1499 /**
1500  * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
1501  * to manage AP resources for the guest whose state is represented by @kvm
1502  *
1503  * @matrix_mdev: a mediated matrix device
1504  * @kvm: reference to KVM instance
1505  *
1506  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1507  * otherwise, returns an -EPERM.
1508  */
1509 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1510 				struct kvm *kvm)
1511 {
1512 	struct ap_matrix_mdev *m;
1513 
1514 	if (kvm->arch.crypto.crycbd) {
1515 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1516 		kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1517 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1518 
1519 		get_update_locks_for_kvm(kvm);
1520 
1521 		list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1522 			if (m != matrix_mdev && m->kvm == kvm) {
1523 				release_update_locks_for_kvm(kvm);
1524 				return -EPERM;
1525 			}
1526 		}
1527 
1528 		kvm_get_kvm(kvm);
1529 		matrix_mdev->kvm = kvm;
1530 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1531 
1532 		release_update_locks_for_kvm(kvm);
1533 	}
1534 
1535 	return 0;
1536 }
1537 
1538 static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
1539 {
1540 	struct ap_queue_table *qtable = &matrix_mdev->qtable;
1541 	struct vfio_ap_queue *q;
1542 	int loop_cursor;
1543 
1544 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1545 		if (q->saved_iova >= iova && q->saved_iova < iova + length)
1546 			vfio_ap_irq_disable(q);
1547 	}
1548 }
1549 
1550 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
1551 				   u64 length)
1552 {
1553 	struct ap_matrix_mdev *matrix_mdev =
1554 		container_of(vdev, struct ap_matrix_mdev, vdev);
1555 
1556 	mutex_lock(&matrix_dev->mdevs_lock);
1557 
1558 	unmap_iova(matrix_mdev, iova, length);
1559 
1560 	mutex_unlock(&matrix_dev->mdevs_lock);
1561 }
1562 
1563 /**
1564  * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
1565  * by @matrix_mdev.
1566  *
1567  * @matrix_mdev: a matrix mediated device
1568  */
1569 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
1570 {
1571 	struct kvm *kvm = matrix_mdev->kvm;
1572 
1573 	if (kvm && kvm->arch.crypto.crycbd) {
1574 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1575 		kvm->arch.crypto.pqap_hook = NULL;
1576 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1577 
1578 		get_update_locks_for_kvm(kvm);
1579 
1580 		kvm_arch_crypto_clear_masks(kvm);
1581 		vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1582 		kvm_put_kvm(kvm);
1583 		matrix_mdev->kvm = NULL;
1584 
1585 		release_update_locks_for_kvm(kvm);
1586 	}
1587 }
1588 
1589 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
1590 {
1591 	struct ap_queue *queue;
1592 	struct vfio_ap_queue *q = NULL;
1593 
1594 	queue = ap_get_qdev(apqn);
1595 	if (!queue)
1596 		return NULL;
1597 
1598 	if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
1599 		q = dev_get_drvdata(&queue->ap_dev.device);
1600 
1601 	put_device(&queue->ap_dev.device);
1602 
1603 	return q;
1604 }
1605 
1606 static int apq_status_check(int apqn, struct ap_queue_status *status)
1607 {
1608 	switch (status->response_code) {
1609 	case AP_RESPONSE_NORMAL:
1610 	case AP_RESPONSE_RESET_IN_PROGRESS:
1611 		if (status->queue_empty && !status->irq_enabled)
1612 			return 0;
1613 		return -EBUSY;
1614 	case AP_RESPONSE_DECONFIGURED:
1615 		/*
1616 		 * If the AP queue is deconfigured, any subsequent AP command
1617 		 * targeting the queue will fail with the same response code. On the
1618 		 * other hand, when an AP adapter is deconfigured, the associated
1619 		 * queues are reset, so let's return a value indicating the reset
1620 		 * for which we're waiting completed successfully.
1621 		 */
1622 		return 0;
1623 	default:
1624 		WARN(true,
1625 		     "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n",
1626 		     AP_QID_CARD(apqn), AP_QID_QUEUE(apqn),
1627 		     status->response_code);
1628 		return -EIO;
1629 	}
1630 }
1631 
1632 static int apq_reset_check(struct vfio_ap_queue *q)
1633 {
1634 	int ret;
1635 	int iters = MAX_RESET_CHECK_WAIT / AP_RESET_INTERVAL;
1636 	struct ap_queue_status status;
1637 
1638 	for (; iters > 0; iters--) {
1639 		msleep(AP_RESET_INTERVAL);
1640 		status = ap_tapq(q->apqn, NULL);
1641 		ret = apq_status_check(q->apqn, &status);
1642 		if (ret != -EBUSY)
1643 			return ret;
1644 	}
1645 	WARN_ONCE(iters <= 0,
1646 		  "timeout verifying reset of queue %02x.%04x (%u, %u, %u)",
1647 		  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1648 		  status.queue_empty, status.irq_enabled, status.response_code);
1649 	return ret;
1650 }
1651 
1652 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)
1653 {
1654 	struct ap_queue_status status;
1655 	int ret;
1656 
1657 	if (!q)
1658 		return 0;
1659 retry_zapq:
1660 	status = ap_zapq(q->apqn);
1661 	q->reset_rc = status.response_code;
1662 	switch (status.response_code) {
1663 	case AP_RESPONSE_NORMAL:
1664 		ret = 0;
1665 		/* if the reset has not completed, wait for it to take effect */
1666 		if (!status.queue_empty || status.irq_enabled)
1667 			ret = apq_reset_check(q);
1668 		break;
1669 	case AP_RESPONSE_RESET_IN_PROGRESS:
1670 		/*
1671 		 * There is a reset issued by another process in progress. Let's wait
1672 		 * for that to complete. Since we have no idea whether it was a RAPQ or
1673 		 * ZAPQ, then if it completes successfully, let's issue the ZAPQ.
1674 		 */
1675 		ret = apq_reset_check(q);
1676 		if (ret)
1677 			break;
1678 		goto retry_zapq;
1679 	case AP_RESPONSE_DECONFIGURED:
1680 		/*
1681 		 * When an AP adapter is deconfigured, the associated
1682 		 * queues are reset, so let's return a value indicating the reset
1683 		 * completed successfully.
1684 		 */
1685 		ret = 0;
1686 		break;
1687 	default:
1688 		WARN(true,
1689 		     "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
1690 		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1691 		     status.response_code);
1692 		return -EIO;
1693 	}
1694 
1695 	vfio_ap_free_aqic_resources(q);
1696 
1697 	return ret;
1698 }
1699 
1700 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable)
1701 {
1702 	int ret, loop_cursor, rc = 0;
1703 	struct vfio_ap_queue *q;
1704 
1705 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1706 		ret = vfio_ap_mdev_reset_queue(q);
1707 		/*
1708 		 * Regardless whether a queue turns out to be busy, or
1709 		 * is not operational, we need to continue resetting
1710 		 * the remaining queues.
1711 		 */
1712 		if (ret)
1713 			rc = ret;
1714 	}
1715 
1716 	return rc;
1717 }
1718 
1719 static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1720 {
1721 	struct ap_matrix_mdev *matrix_mdev =
1722 		container_of(vdev, struct ap_matrix_mdev, vdev);
1723 
1724 	if (!vdev->kvm)
1725 		return -EINVAL;
1726 
1727 	return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
1728 }
1729 
1730 static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1731 {
1732 	struct ap_matrix_mdev *matrix_mdev =
1733 		container_of(vdev, struct ap_matrix_mdev, vdev);
1734 
1735 	vfio_ap_mdev_unset_kvm(matrix_mdev);
1736 }
1737 
1738 static int vfio_ap_mdev_get_device_info(unsigned long arg)
1739 {
1740 	unsigned long minsz;
1741 	struct vfio_device_info info;
1742 
1743 	minsz = offsetofend(struct vfio_device_info, num_irqs);
1744 
1745 	if (copy_from_user(&info, (void __user *)arg, minsz))
1746 		return -EFAULT;
1747 
1748 	if (info.argsz < minsz)
1749 		return -EINVAL;
1750 
1751 	info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
1752 	info.num_regions = 0;
1753 	info.num_irqs = 0;
1754 
1755 	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1756 }
1757 
1758 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1759 				    unsigned int cmd, unsigned long arg)
1760 {
1761 	struct ap_matrix_mdev *matrix_mdev =
1762 		container_of(vdev, struct ap_matrix_mdev, vdev);
1763 	int ret;
1764 
1765 	mutex_lock(&matrix_dev->mdevs_lock);
1766 	switch (cmd) {
1767 	case VFIO_DEVICE_GET_INFO:
1768 		ret = vfio_ap_mdev_get_device_info(arg);
1769 		break;
1770 	case VFIO_DEVICE_RESET:
1771 		ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1772 		break;
1773 	default:
1774 		ret = -EOPNOTSUPP;
1775 		break;
1776 	}
1777 	mutex_unlock(&matrix_dev->mdevs_lock);
1778 
1779 	return ret;
1780 }
1781 
1782 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
1783 {
1784 	struct ap_matrix_mdev *matrix_mdev;
1785 	unsigned long apid = AP_QID_CARD(q->apqn);
1786 	unsigned long apqi = AP_QID_QUEUE(q->apqn);
1787 
1788 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
1789 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
1790 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))
1791 			return matrix_mdev;
1792 	}
1793 
1794 	return NULL;
1795 }
1796 
1797 static ssize_t status_show(struct device *dev,
1798 			   struct device_attribute *attr,
1799 			   char *buf)
1800 {
1801 	ssize_t nchars = 0;
1802 	struct vfio_ap_queue *q;
1803 	struct ap_matrix_mdev *matrix_mdev;
1804 	struct ap_device *apdev = to_ap_dev(dev);
1805 
1806 	mutex_lock(&matrix_dev->mdevs_lock);
1807 	q = dev_get_drvdata(&apdev->device);
1808 	matrix_mdev = vfio_ap_mdev_for_queue(q);
1809 
1810 	if (matrix_mdev) {
1811 		if (matrix_mdev->kvm)
1812 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1813 					   AP_QUEUE_IN_USE);
1814 		else
1815 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1816 					   AP_QUEUE_ASSIGNED);
1817 	} else {
1818 		nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1819 				   AP_QUEUE_UNASSIGNED);
1820 	}
1821 
1822 	mutex_unlock(&matrix_dev->mdevs_lock);
1823 
1824 	return nchars;
1825 }
1826 
1827 static DEVICE_ATTR_RO(status);
1828 
1829 static struct attribute *vfio_queue_attrs[] = {
1830 	&dev_attr_status.attr,
1831 	NULL,
1832 };
1833 
1834 static const struct attribute_group vfio_queue_attr_group = {
1835 	.attrs = vfio_queue_attrs,
1836 };
1837 
1838 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
1839 	.init = vfio_ap_mdev_init_dev,
1840 	.open_device = vfio_ap_mdev_open_device,
1841 	.close_device = vfio_ap_mdev_close_device,
1842 	.ioctl = vfio_ap_mdev_ioctl,
1843 	.dma_unmap = vfio_ap_mdev_dma_unmap,
1844 	.bind_iommufd = vfio_iommufd_emulated_bind,
1845 	.unbind_iommufd = vfio_iommufd_emulated_unbind,
1846 	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
1847 };
1848 
1849 static struct mdev_driver vfio_ap_matrix_driver = {
1850 	.device_api = VFIO_DEVICE_API_AP_STRING,
1851 	.max_instances = MAX_ZDEV_ENTRIES_EXT,
1852 	.driver = {
1853 		.name = "vfio_ap_mdev",
1854 		.owner = THIS_MODULE,
1855 		.mod_name = KBUILD_MODNAME,
1856 		.dev_groups = vfio_ap_mdev_attr_groups,
1857 	},
1858 	.probe = vfio_ap_mdev_probe,
1859 	.remove = vfio_ap_mdev_remove,
1860 };
1861 
1862 int vfio_ap_mdev_register(void)
1863 {
1864 	int ret;
1865 
1866 	ret = mdev_register_driver(&vfio_ap_matrix_driver);
1867 	if (ret)
1868 		return ret;
1869 
1870 	matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
1871 	matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
1872 	matrix_dev->mdev_types[0] = &matrix_dev->mdev_type;
1873 	ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
1874 				   &vfio_ap_matrix_driver,
1875 				   matrix_dev->mdev_types, 1);
1876 	if (ret)
1877 		goto err_driver;
1878 	return 0;
1879 
1880 err_driver:
1881 	mdev_unregister_driver(&vfio_ap_matrix_driver);
1882 	return ret;
1883 }
1884 
1885 void vfio_ap_mdev_unregister(void)
1886 {
1887 	mdev_unregister_parent(&matrix_dev->parent);
1888 	mdev_unregister_driver(&vfio_ap_matrix_driver);
1889 }
1890 
1891 int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
1892 {
1893 	int ret;
1894 	struct vfio_ap_queue *q;
1895 	struct ap_matrix_mdev *matrix_mdev;
1896 
1897 	ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
1898 	if (ret)
1899 		return ret;
1900 
1901 	q = kzalloc(sizeof(*q), GFP_KERNEL);
1902 	if (!q) {
1903 		ret = -ENOMEM;
1904 		goto err_remove_group;
1905 	}
1906 
1907 	q->apqn = to_ap_queue(&apdev->device)->qid;
1908 	q->saved_isc = VFIO_AP_ISC_INVALID;
1909 	matrix_mdev = get_update_locks_by_apqn(q->apqn);
1910 
1911 	if (matrix_mdev) {
1912 		vfio_ap_mdev_link_queue(matrix_mdev, q);
1913 
1914 		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,
1915 					       matrix_mdev->matrix.aqm,
1916 					       matrix_mdev))
1917 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1918 	}
1919 	dev_set_drvdata(&apdev->device, q);
1920 	release_update_locks_for_mdev(matrix_mdev);
1921 
1922 	return 0;
1923 
1924 err_remove_group:
1925 	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
1926 	return ret;
1927 }
1928 
1929 void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
1930 {
1931 	unsigned long apid, apqi;
1932 	struct vfio_ap_queue *q;
1933 	struct ap_matrix_mdev *matrix_mdev;
1934 
1935 	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
1936 	q = dev_get_drvdata(&apdev->device);
1937 	get_update_locks_for_queue(q);
1938 	matrix_mdev = q->matrix_mdev;
1939 
1940 	if (matrix_mdev) {
1941 		vfio_ap_unlink_queue_fr_mdev(q);
1942 
1943 		apid = AP_QID_CARD(q->apqn);
1944 		apqi = AP_QID_QUEUE(q->apqn);
1945 
1946 		/*
1947 		 * If the queue is assigned to the guest's APCB, then remove
1948 		 * the adapter's APID from the APCB and hot it into the guest.
1949 		 */
1950 		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1951 		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1952 			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1953 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1954 		}
1955 	}
1956 
1957 	vfio_ap_mdev_reset_queue(q);
1958 	dev_set_drvdata(&apdev->device, NULL);
1959 	kfree(q);
1960 	release_update_locks_for_mdev(matrix_mdev);
1961 }
1962 
1963 /**
1964  * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
1965  *				 assigned to a mediated device under the control
1966  *				 of the vfio_ap device driver.
1967  *
1968  * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
1969  * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
1970  *
1971  * Return:
1972  *	* -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
1973  *	  assigned to a mediated device under the control of the vfio_ap
1974  *	  device driver.
1975  *	* Otherwise, return 0.
1976  */
1977 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
1978 {
1979 	int ret;
1980 
1981 	mutex_lock(&matrix_dev->guests_lock);
1982 	mutex_lock(&matrix_dev->mdevs_lock);
1983 	ret = vfio_ap_mdev_verify_no_sharing(apm, aqm);
1984 	mutex_unlock(&matrix_dev->mdevs_lock);
1985 	mutex_unlock(&matrix_dev->guests_lock);
1986 
1987 	return ret;
1988 }
1989 
1990 /**
1991  * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
1992  *				 domains that have been removed from the host's
1993  *				 AP configuration from a guest.
1994  *
1995  * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
1996  * @aprem: the adapters that have been removed from the host's AP configuration
1997  * @aqrem: the domains that have been removed from the host's AP configuration
1998  * @cdrem: the control domains that have been removed from the host's AP
1999  *	   configuration.
2000  */
2001 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
2002 					unsigned long *aprem,
2003 					unsigned long *aqrem,
2004 					unsigned long *cdrem)
2005 {
2006 	int do_hotplug = 0;
2007 
2008 	if (!bitmap_empty(aprem, AP_DEVICES)) {
2009 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,
2010 					    matrix_mdev->shadow_apcb.apm,
2011 					    aprem, AP_DEVICES);
2012 	}
2013 
2014 	if (!bitmap_empty(aqrem, AP_DOMAINS)) {
2015 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
2016 					    matrix_mdev->shadow_apcb.aqm,
2017 					    aqrem, AP_DEVICES);
2018 	}
2019 
2020 	if (!bitmap_empty(cdrem, AP_DOMAINS))
2021 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,
2022 					    matrix_mdev->shadow_apcb.adm,
2023 					    cdrem, AP_DOMAINS);
2024 
2025 	if (do_hotplug)
2026 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2027 }
2028 
2029 /**
2030  * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
2031  *			     domains and control domains that have been removed
2032  *			     from the host AP configuration and unplugs them
2033  *			     from those guests.
2034  *
2035  * @ap_remove:	bitmap specifying which adapters have been removed from the host
2036  *		config.
2037  * @aq_remove:	bitmap specifying which domains have been removed from the host
2038  *		config.
2039  * @cd_remove:	bitmap specifying which control domains have been removed from
2040  *		the host config.
2041  */
2042 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
2043 				    unsigned long *aq_remove,
2044 				    unsigned long *cd_remove)
2045 {
2046 	struct ap_matrix_mdev *matrix_mdev;
2047 	DECLARE_BITMAP(aprem, AP_DEVICES);
2048 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2049 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2050 	int do_remove = 0;
2051 
2052 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2053 		mutex_lock(&matrix_mdev->kvm->lock);
2054 		mutex_lock(&matrix_dev->mdevs_lock);
2055 
2056 		do_remove |= bitmap_and(aprem, ap_remove,
2057 					  matrix_mdev->matrix.apm,
2058 					  AP_DEVICES);
2059 		do_remove |= bitmap_and(aqrem, aq_remove,
2060 					  matrix_mdev->matrix.aqm,
2061 					  AP_DOMAINS);
2062 		do_remove |= bitmap_andnot(cdrem, cd_remove,
2063 					     matrix_mdev->matrix.adm,
2064 					     AP_DOMAINS);
2065 
2066 		if (do_remove)
2067 			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
2068 						    cdrem);
2069 
2070 		mutex_unlock(&matrix_dev->mdevs_lock);
2071 		mutex_unlock(&matrix_mdev->kvm->lock);
2072 	}
2073 }
2074 
2075 /**
2076  * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
2077  *				control domains from the host AP configuration
2078  *				by unplugging them from the guests that are
2079  *				using them.
2080  * @cur_config_info: the current host AP configuration information
2081  * @prev_config_info: the previous host AP configuration information
2082  */
2083 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
2084 				       struct ap_config_info *prev_config_info)
2085 {
2086 	int do_remove;
2087 	DECLARE_BITMAP(aprem, AP_DEVICES);
2088 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2089 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2090 
2091 	do_remove = bitmap_andnot(aprem,
2092 				  (unsigned long *)prev_config_info->apm,
2093 				  (unsigned long *)cur_config_info->apm,
2094 				  AP_DEVICES);
2095 	do_remove |= bitmap_andnot(aqrem,
2096 				   (unsigned long *)prev_config_info->aqm,
2097 				   (unsigned long *)cur_config_info->aqm,
2098 				   AP_DEVICES);
2099 	do_remove |= bitmap_andnot(cdrem,
2100 				   (unsigned long *)prev_config_info->adm,
2101 				   (unsigned long *)cur_config_info->adm,
2102 				   AP_DEVICES);
2103 
2104 	if (do_remove)
2105 		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
2106 }
2107 
2108 /**
2109  * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
2110  *				 are older than AP type 10 (CEX4).
2111  * @apm: a bitmap of the APIDs to examine
2112  * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
2113  */
2114 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
2115 {
2116 	bool apid_cleared;
2117 	struct ap_queue_status status;
2118 	unsigned long apid, apqi, info;
2119 	int qtype, qtype_mask = 0xff000000;
2120 
2121 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
2122 		apid_cleared = false;
2123 
2124 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
2125 			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
2126 			switch (status.response_code) {
2127 			/*
2128 			 * According to the architecture in each case
2129 			 * below, the queue's info should be filled.
2130 			 */
2131 			case AP_RESPONSE_NORMAL:
2132 			case AP_RESPONSE_RESET_IN_PROGRESS:
2133 			case AP_RESPONSE_DECONFIGURED:
2134 			case AP_RESPONSE_CHECKSTOPPED:
2135 			case AP_RESPONSE_BUSY:
2136 				qtype = info & qtype_mask;
2137 
2138 				/*
2139 				 * The vfio_ap device driver only
2140 				 * supports CEX4 and newer adapters, so
2141 				 * remove the APID if the adapter is
2142 				 * older than a CEX4.
2143 				 */
2144 				if (qtype < AP_DEVICE_TYPE_CEX4) {
2145 					clear_bit_inv(apid, apm);
2146 					apid_cleared = true;
2147 				}
2148 
2149 				break;
2150 
2151 			default:
2152 				/*
2153 				 * If we don't know the adapter type,
2154 				 * clear its APID since it can't be
2155 				 * determined whether the vfio_ap
2156 				 * device driver supports it.
2157 				 */
2158 				clear_bit_inv(apid, apm);
2159 				apid_cleared = true;
2160 				break;
2161 			}
2162 
2163 			/*
2164 			 * If we've already cleared the APID from the apm, there
2165 			 * is no need to continue examining the remainin AP
2166 			 * queues to determine the type of the adapter.
2167 			 */
2168 			if (apid_cleared)
2169 				continue;
2170 		}
2171 	}
2172 }
2173 
2174 /**
2175  * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
2176  *			  control domains that have been added to the host's
2177  *			  AP configuration for each matrix mdev to which they
2178  *			  are assigned.
2179  *
2180  * @apm_add: a bitmap specifying the adapters that have been added to the AP
2181  *	     configuration.
2182  * @aqm_add: a bitmap specifying the domains that have been added to the AP
2183  *	     configuration.
2184  * @adm_add: a bitmap specifying the control domains that have been added to the
2185  *	     AP configuration.
2186  */
2187 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
2188 				 unsigned long *adm_add)
2189 {
2190 	struct ap_matrix_mdev *matrix_mdev;
2191 
2192 	if (list_empty(&matrix_dev->mdev_list))
2193 		return;
2194 
2195 	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
2196 
2197 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2198 		bitmap_and(matrix_mdev->apm_add,
2199 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
2200 		bitmap_and(matrix_mdev->aqm_add,
2201 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
2202 		bitmap_and(matrix_mdev->adm_add,
2203 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
2204 	}
2205 }
2206 
2207 /**
2208  * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
2209  *			     control domains to the host AP configuration
2210  *			     by updating the bitmaps that specify what adapters,
2211  *			     domains and control domains have been added so they
2212  *			     can be hot plugged into the guest when the AP bus
2213  *			     scan completes (see vfio_ap_on_scan_complete
2214  *			     function).
2215  * @cur_config_info: the current AP configuration information
2216  * @prev_config_info: the previous AP configuration information
2217  */
2218 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
2219 				    struct ap_config_info *prev_config_info)
2220 {
2221 	bool do_add;
2222 	DECLARE_BITMAP(apm_add, AP_DEVICES);
2223 	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
2224 	DECLARE_BITMAP(adm_add, AP_DOMAINS);
2225 
2226 	do_add = bitmap_andnot(apm_add,
2227 			       (unsigned long *)cur_config_info->apm,
2228 			       (unsigned long *)prev_config_info->apm,
2229 			       AP_DEVICES);
2230 	do_add |= bitmap_andnot(aqm_add,
2231 				(unsigned long *)cur_config_info->aqm,
2232 				(unsigned long *)prev_config_info->aqm,
2233 				AP_DOMAINS);
2234 	do_add |= bitmap_andnot(adm_add,
2235 				(unsigned long *)cur_config_info->adm,
2236 				(unsigned long *)prev_config_info->adm,
2237 				AP_DOMAINS);
2238 
2239 	if (do_add)
2240 		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
2241 }
2242 
2243 /**
2244  * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
2245  *			    configuration.
2246  *
2247  * @cur_cfg_info: the current host AP configuration
2248  * @prev_cfg_info: the previous host AP configuration
2249  */
2250 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
2251 			    struct ap_config_info *prev_cfg_info)
2252 {
2253 	if (!cur_cfg_info || !prev_cfg_info)
2254 		return;
2255 
2256 	mutex_lock(&matrix_dev->guests_lock);
2257 
2258 	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
2259 	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
2260 	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
2261 
2262 	mutex_unlock(&matrix_dev->guests_lock);
2263 }
2264 
2265 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
2266 {
2267 	bool do_hotplug = false;
2268 	int filter_domains = 0;
2269 	int filter_adapters = 0;
2270 	DECLARE_BITMAP(apm, AP_DEVICES);
2271 	DECLARE_BITMAP(aqm, AP_DOMAINS);
2272 
2273 	mutex_lock(&matrix_mdev->kvm->lock);
2274 	mutex_lock(&matrix_dev->mdevs_lock);
2275 
2276 	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,
2277 				     matrix_mdev->apm_add, AP_DEVICES);
2278 	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,
2279 				    matrix_mdev->aqm_add, AP_DOMAINS);
2280 
2281 	if (filter_adapters && filter_domains)
2282 		do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev);
2283 	else if (filter_adapters)
2284 		do_hotplug |=
2285 			vfio_ap_mdev_filter_matrix(apm,
2286 						   matrix_mdev->shadow_apcb.aqm,
2287 						   matrix_mdev);
2288 	else
2289 		do_hotplug |=
2290 			vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm,
2291 						   aqm, matrix_mdev);
2292 
2293 	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,
2294 			      AP_DOMAINS))
2295 		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
2296 
2297 	if (do_hotplug)
2298 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2299 
2300 	mutex_unlock(&matrix_dev->mdevs_lock);
2301 	mutex_unlock(&matrix_mdev->kvm->lock);
2302 }
2303 
2304 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
2305 			      struct ap_config_info *old_config_info)
2306 {
2307 	struct ap_matrix_mdev *matrix_mdev;
2308 
2309 	mutex_lock(&matrix_dev->guests_lock);
2310 
2311 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2312 		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
2313 		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
2314 		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
2315 			continue;
2316 
2317 		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
2318 		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
2319 		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
2320 		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
2321 	}
2322 
2323 	mutex_unlock(&matrix_dev->guests_lock);
2324 }
2325