1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Adjunct processor matrix VFIO device driver callbacks.
4  *
5  * Copyright IBM Corp. 2018
6  *
7  * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
8  *	      Halil Pasic <pasic@linux.ibm.com>
9  *	      Pierre Morel <pmorel@linux.ibm.com>
10  */
11 #include <linux/string.h>
12 #include <linux/vfio.h>
13 #include <linux/device.h>
14 #include <linux/list.h>
15 #include <linux/ctype.h>
16 #include <linux/bitops.h>
17 #include <linux/kvm_host.h>
18 #include <linux/module.h>
19 #include <linux/uuid.h>
20 #include <asm/kvm.h>
21 #include <asm/zcrypt.h>
22 
23 #include "vfio_ap_private.h"
24 #include "vfio_ap_debug.h"
25 
26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
28 
29 #define AP_QUEUE_ASSIGNED "assigned"
30 #define AP_QUEUE_UNASSIGNED "unassigned"
31 #define AP_QUEUE_IN_USE "in use"
32 
33 #define MAX_RESET_CHECK_WAIT	200	/* Sleep max 200ms for reset check	*/
34 #define AP_RESET_INTERVAL		20	/* Reset sleep interval (20ms)		*/
35 
36 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable);
37 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
38 static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
39 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q);
40 
41 /**
42  * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
43  *			     KVM guest's APCB in the proper order.
44  *
45  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
46  *
47  * The proper locking order is:
48  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
49  *			       guest's APCB.
50  * 2. kvm->lock:	       required to update a guest's APCB
51  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
52  *
53  * Note: If @kvm is NULL, the KVM lock will not be taken.
54  */
55 static inline void get_update_locks_for_kvm(struct kvm *kvm)
56 {
57 	mutex_lock(&matrix_dev->guests_lock);
58 	if (kvm)
59 		mutex_lock(&kvm->lock);
60 	mutex_lock(&matrix_dev->mdevs_lock);
61 }
62 
63 /**
64  * release_update_locks_for_kvm: Release the locks used to dynamically update a
65  *				 KVM guest's APCB in the proper order.
66  *
67  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
68  *
69  * The proper unlocking order is:
70  * 1. matrix_dev->mdevs_lock
71  * 2. kvm->lock
72  * 3. matrix_dev->guests_lock
73  *
74  * Note: If @kvm is NULL, the KVM lock will not be released.
75  */
76 static inline void release_update_locks_for_kvm(struct kvm *kvm)
77 {
78 	mutex_unlock(&matrix_dev->mdevs_lock);
79 	if (kvm)
80 		mutex_unlock(&kvm->lock);
81 	mutex_unlock(&matrix_dev->guests_lock);
82 }
83 
84 /**
85  * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
86  *			      KVM guest's APCB in the proper order.
87  *
88  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
89  *		 configuration data to use to update a KVM guest's APCB.
90  *
91  * The proper locking order is:
92  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
93  *			       guest's APCB.
94  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
95  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
96  *
97  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
98  *	 lock will not be taken.
99  */
100 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
101 {
102 	mutex_lock(&matrix_dev->guests_lock);
103 	if (matrix_mdev && matrix_mdev->kvm)
104 		mutex_lock(&matrix_mdev->kvm->lock);
105 	mutex_lock(&matrix_dev->mdevs_lock);
106 }
107 
108 /**
109  * release_update_locks_for_mdev: Release the locks used to dynamically update a
110  *				  KVM guest's APCB in the proper order.
111  *
112  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
113  *		 configuration data to use to update a KVM guest's APCB.
114  *
115  * The proper unlocking order is:
116  * 1. matrix_dev->mdevs_lock
117  * 2. matrix_mdev->kvm->lock
118  * 3. matrix_dev->guests_lock
119  *
120  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
121  *	 lock will not be released.
122  */
123 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
124 {
125 	mutex_unlock(&matrix_dev->mdevs_lock);
126 	if (matrix_mdev && matrix_mdev->kvm)
127 		mutex_unlock(&matrix_mdev->kvm->lock);
128 	mutex_unlock(&matrix_dev->guests_lock);
129 }
130 
131 /**
132  * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
133  *			     acquire the locks required to update the APCB of
134  *			     the KVM guest to which the mdev is attached.
135  *
136  * @apqn: the APQN of a queue device.
137  *
138  * The proper locking order is:
139  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
140  *			       guest's APCB.
141  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
142  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
143  *
144  * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
145  *	 will not be taken.
146  *
147  * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
148  *	   is not assigned to an ap_matrix_mdev.
149  */
150 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
151 {
152 	struct ap_matrix_mdev *matrix_mdev;
153 
154 	mutex_lock(&matrix_dev->guests_lock);
155 
156 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
159 			if (matrix_mdev->kvm)
160 				mutex_lock(&matrix_mdev->kvm->lock);
161 
162 			mutex_lock(&matrix_dev->mdevs_lock);
163 
164 			return matrix_mdev;
165 		}
166 	}
167 
168 	mutex_lock(&matrix_dev->mdevs_lock);
169 
170 	return NULL;
171 }
172 
173 /**
174  * get_update_locks_for_queue: get the locks required to update the APCB of the
175  *			       KVM guest to which the matrix mdev linked to a
176  *			       vfio_ap_queue object is attached.
177  *
178  * @q: a pointer to a vfio_ap_queue object.
179  *
180  * The proper locking order is:
181  * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
182  *				  KVM guest's APCB.
183  * 2. q->matrix_mdev->kvm->lock:  required to update a guest's APCB
184  * 3. matrix_dev->mdevs_lock:	  required to access data stored in matrix_mdev
185  *
186  * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
187  *	  will not be taken.
188  */
189 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
190 {
191 	mutex_lock(&matrix_dev->guests_lock);
192 	if (q->matrix_mdev && q->matrix_mdev->kvm)
193 		mutex_lock(&q->matrix_mdev->kvm->lock);
194 	mutex_lock(&matrix_dev->mdevs_lock);
195 }
196 
197 /**
198  * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
201  * @apqn: The APQN of a queue device
202  *
203  * Return: the pointer to the vfio_ap_queue struct representing the queue or
204  *	   NULL if the queue is not assigned to @matrix_mdev
205  */
206 static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
207 					struct ap_matrix_mdev *matrix_mdev,
208 					int apqn)
209 {
210 	struct vfio_ap_queue *q;
211 
212 	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
213 			       apqn) {
214 		if (q && q->apqn == apqn)
215 			return q;
216 	}
217 
218 	return NULL;
219 }
220 
221 /**
222  * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
223  * @apqn: The AP Queue number
224  *
225  * Checks the IRQ bit for the status of this APQN using ap_tapq.
226  * Returns if the ap_tapq function succeeded and the bit is clear.
227  * Returns if ap_tapq function failed with invalid, deconfigured or
228  * checkstopped AP.
229  * Otherwise retries up to 5 times after waiting 20ms.
230  */
231 static void vfio_ap_wait_for_irqclear(int apqn)
232 {
233 	struct ap_queue_status status;
234 	int retry = 5;
235 
236 	do {
237 		status = ap_tapq(apqn, NULL);
238 		switch (status.response_code) {
239 		case AP_RESPONSE_NORMAL:
240 		case AP_RESPONSE_RESET_IN_PROGRESS:
241 			if (!status.irq_enabled)
242 				return;
243 			fallthrough;
244 		case AP_RESPONSE_BUSY:
245 			msleep(20);
246 			break;
247 		case AP_RESPONSE_Q_NOT_AVAIL:
248 		case AP_RESPONSE_DECONFIGURED:
249 		case AP_RESPONSE_CHECKSTOPPED:
250 		default:
251 			WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
252 				  status.response_code, apqn);
253 			return;
254 		}
255 	} while (--retry);
256 
257 	WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
258 		  __func__, status.response_code, apqn);
259 }
260 
261 /**
262  * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
263  * @q: The vfio_ap_queue
264  *
265  * Unregisters the ISC in the GIB when the saved ISC not invalid.
266  * Unpins the guest's page holding the NIB when it exists.
267  * Resets the saved_iova and saved_isc to invalid values.
268  */
269 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
270 {
271 	if (!q)
272 		return;
273 	if (q->saved_isc != VFIO_AP_ISC_INVALID &&
274 	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
275 		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
276 		q->saved_isc = VFIO_AP_ISC_INVALID;
277 	}
278 	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
279 		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
280 		q->saved_iova = 0;
281 	}
282 }
283 
284 /**
285  * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
286  * @q: The vfio_ap_queue
287  *
288  * Uses ap_aqic to disable the interruption and in case of success, reset
289  * in progress or IRQ disable command already proceeded: calls
290  * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
291  * and calls vfio_ap_free_aqic_resources() to free the resources associated
292  * with the AP interrupt handling.
293  *
294  * In the case the AP is busy, or a reset is in progress,
295  * retries after 20ms, up to 5 times.
296  *
297  * Returns if ap_aqic function failed with invalid, deconfigured or
298  * checkstopped AP.
299  *
300  * Return: &struct ap_queue_status
301  */
302 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
303 {
304 	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
305 	struct ap_queue_status status;
306 	int retries = 5;
307 
308 	do {
309 		status = ap_aqic(q->apqn, aqic_gisa, 0);
310 		switch (status.response_code) {
311 		case AP_RESPONSE_OTHERWISE_CHANGED:
312 		case AP_RESPONSE_NORMAL:
313 			vfio_ap_wait_for_irqclear(q->apqn);
314 			goto end_free;
315 		case AP_RESPONSE_RESET_IN_PROGRESS:
316 		case AP_RESPONSE_BUSY:
317 			msleep(20);
318 			break;
319 		case AP_RESPONSE_Q_NOT_AVAIL:
320 		case AP_RESPONSE_DECONFIGURED:
321 		case AP_RESPONSE_CHECKSTOPPED:
322 		case AP_RESPONSE_INVALID_ADDRESS:
323 		default:
324 			/* All cases in default means AP not operational */
325 			WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
326 				  status.response_code);
327 			goto end_free;
328 		}
329 	} while (retries--);
330 
331 	WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
332 		  status.response_code);
333 end_free:
334 	vfio_ap_free_aqic_resources(q);
335 	return status;
336 }
337 
338 /**
339  * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
340  *
341  * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
342  * @nib: the location for storing the nib address.
343  *
344  * When the PQAP(AQIC) instruction is executed, general register 2 contains the
345  * address of the notification indicator byte (nib) used for IRQ notification.
346  * This function parses and validates the nib from gr2.
347  *
348  * Return: returns zero if the nib address is a valid; otherwise, returns
349  *	   -EINVAL.
350  */
351 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
352 {
353 	*nib = vcpu->run->s.regs.gprs[2];
354 
355 	if (!*nib)
356 		return -EINVAL;
357 	if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT)))
358 		return -EINVAL;
359 
360 	return 0;
361 }
362 
363 /**
364  * vfio_ap_irq_enable - Enable Interruption for a APQN
365  *
366  * @q:	 the vfio_ap_queue holding AQIC parameters
367  * @isc: the guest ISC to register with the GIB interface
368  * @vcpu: the vcpu object containing the registers specifying the parameters
369  *	  passed to the PQAP(AQIC) instruction.
370  *
371  * Pin the NIB saved in *q
372  * Register the guest ISC to GIB interface and retrieve the
373  * host ISC to issue the host side PQAP/AQIC
374  *
375  * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the
376  * vfio_pin_pages failed.
377  *
378  * Otherwise return the ap_queue_status returned by the ap_aqic(),
379  * all retry handling will be done by the guest.
380  *
381  * Return: &struct ap_queue_status
382  */
383 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
384 						 int isc,
385 						 struct kvm_vcpu *vcpu)
386 {
387 	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
388 	struct ap_queue_status status = {};
389 	struct kvm_s390_gisa *gisa;
390 	struct page *h_page;
391 	int nisc;
392 	struct kvm *kvm;
393 	phys_addr_t h_nib;
394 	dma_addr_t nib;
395 	int ret;
396 
397 	/* Verify that the notification indicator byte address is valid */
398 	if (vfio_ap_validate_nib(vcpu, &nib)) {
399 		VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
400 				 __func__, &nib, q->apqn);
401 
402 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
403 		return status;
404 	}
405 
406 	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
407 			     IOMMU_READ | IOMMU_WRITE, &h_page);
408 	switch (ret) {
409 	case 1:
410 		break;
411 	default:
412 		VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
413 				 "nib=%pad, apqn=%#04x\n",
414 				 __func__, ret, &nib, q->apqn);
415 
416 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
417 		return status;
418 	}
419 
420 	kvm = q->matrix_mdev->kvm;
421 	gisa = kvm->arch.gisa_int.origin;
422 
423 	h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
424 	aqic_gisa.gisc = isc;
425 
426 	nisc = kvm_s390_gisc_register(kvm, isc);
427 	if (nisc < 0) {
428 		VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
429 				 __func__, nisc, isc, q->apqn);
430 
431 		status.response_code = AP_RESPONSE_INVALID_GISA;
432 		return status;
433 	}
434 
435 	aqic_gisa.isc = nisc;
436 	aqic_gisa.ir = 1;
437 	aqic_gisa.gisa = virt_to_phys(gisa) >> 4;
438 
439 	status = ap_aqic(q->apqn, aqic_gisa, h_nib);
440 	switch (status.response_code) {
441 	case AP_RESPONSE_NORMAL:
442 		/* See if we did clear older IRQ configuration */
443 		vfio_ap_free_aqic_resources(q);
444 		q->saved_iova = nib;
445 		q->saved_isc = isc;
446 		break;
447 	case AP_RESPONSE_OTHERWISE_CHANGED:
448 		/* We could not modify IRQ settings: clear new configuration */
449 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
450 		kvm_s390_gisc_unregister(kvm, isc);
451 		break;
452 	default:
453 		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
454 			status.response_code);
455 		vfio_ap_irq_disable(q);
456 		break;
457 	}
458 
459 	if (status.response_code != AP_RESPONSE_NORMAL) {
460 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
461 				 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
462 				 "gisa=%#x, isc=%#x, apqn=%#04x\n",
463 				 __func__, status.response_code,
464 				 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
465 				 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
466 				 q->apqn);
467 	}
468 
469 	return status;
470 }
471 
472 /**
473  * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
474  *				of big endian elements that can be passed by
475  *				value to an s390dbf sprintf event function to
476  *				format a UUID string.
477  *
478  * @guid: the object containing the little endian guid
479  * @uuid: a six-element array of long values that can be passed by value as
480  *	  arguments for a formatting string specifying a UUID.
481  *
482  * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
483  * event functions if the memory for the passed string is available as long as
484  * the debug feature exists. Since a mediated device can be removed at any
485  * time, it's name can not be used because %s passes the reference to the string
486  * in memory and the reference will go stale once the device is removed .
487  *
488  * The s390dbf string formatting function allows a maximum of 9 arguments for a
489  * message to be displayed in the 'sprintf' view. In order to use the bytes
490  * comprising the mediated device's UUID to display the mediated device name,
491  * they will have to be converted into an array whose elements can be passed by
492  * value to sprintf. For example:
493  *
494  * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
495  * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
496  * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
497  * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
498  */
499 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
500 {
501 	/*
502 	 * The input guid is ordered in little endian, so it needs to be
503 	 * reordered for displaying a UUID as a string. This specifies the
504 	 * guid indices in proper order.
505 	 */
506 	uuid[0] = le32_to_cpup((__le32 *)guid);
507 	uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
508 	uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
509 	uuid[3] = *((__u16 *)&guid->b[8]);
510 	uuid[4] = *((__u16 *)&guid->b[10]);
511 	uuid[5] = *((__u32 *)&guid->b[12]);
512 }
513 
514 /**
515  * handle_pqap - PQAP instruction callback
516  *
517  * @vcpu: The vcpu on which we received the PQAP instruction
518  *
519  * Get the general register contents to initialize internal variables.
520  * REG[0]: APQN
521  * REG[1]: IR and ISC
522  * REG[2]: NIB
523  *
524  * Response.status may be set to following Response Code:
525  * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
526  * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
527  * - AP_RESPONSE_NORMAL (0) : in case of success
528  *   Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
529  * We take the matrix_dev lock to ensure serialization on queues and
530  * mediated device access.
531  *
532  * Return: 0 if we could handle the request inside KVM.
533  * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
534  */
535 static int handle_pqap(struct kvm_vcpu *vcpu)
536 {
537 	uint64_t status;
538 	uint16_t apqn;
539 	unsigned long uuid[6];
540 	struct vfio_ap_queue *q;
541 	struct ap_queue_status qstatus = {
542 			       .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
543 	struct ap_matrix_mdev *matrix_mdev;
544 
545 	apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
546 
547 	/* If we do not use the AIV facility just go to userland */
548 	if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
549 		VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
550 				 __func__, apqn, vcpu->arch.sie_block->eca);
551 
552 		return -EOPNOTSUPP;
553 	}
554 
555 	mutex_lock(&matrix_dev->mdevs_lock);
556 
557 	if (!vcpu->kvm->arch.crypto.pqap_hook) {
558 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
559 				 __func__, apqn);
560 
561 		goto out_unlock;
562 	}
563 
564 	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
565 				   struct ap_matrix_mdev, pqap_hook);
566 
567 	/* If the there is no guest using the mdev, there is nothing to do */
568 	if (!matrix_mdev->kvm) {
569 		vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
570 		VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
571 				 __func__, uuid[0],  uuid[1], uuid[2],
572 				 uuid[3], uuid[4], uuid[5], apqn);
573 		goto out_unlock;
574 	}
575 
576 	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
577 	if (!q) {
578 		VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
579 				 __func__, AP_QID_CARD(apqn),
580 				 AP_QID_QUEUE(apqn));
581 		goto out_unlock;
582 	}
583 
584 	status = vcpu->run->s.regs.gprs[1];
585 
586 	/* If IR bit(16) is set we enable the interrupt */
587 	if ((status >> (63 - 16)) & 0x01)
588 		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
589 	else
590 		qstatus = vfio_ap_irq_disable(q);
591 
592 out_unlock:
593 	memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
594 	vcpu->run->s.regs.gprs[1] >>= 32;
595 	mutex_unlock(&matrix_dev->mdevs_lock);
596 	return 0;
597 }
598 
599 static void vfio_ap_matrix_init(struct ap_config_info *info,
600 				struct ap_matrix *matrix)
601 {
602 	matrix->apm_max = info->apxa ? info->na : 63;
603 	matrix->aqm_max = info->apxa ? info->nd : 15;
604 	matrix->adm_max = info->apxa ? info->nd : 15;
605 }
606 
607 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
608 {
609 	if (matrix_mdev->kvm)
610 		kvm_arch_crypto_set_masks(matrix_mdev->kvm,
611 					  matrix_mdev->shadow_apcb.apm,
612 					  matrix_mdev->shadow_apcb.aqm,
613 					  matrix_mdev->shadow_apcb.adm);
614 }
615 
616 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
617 {
618 	DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
619 
620 	bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
621 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
622 		   (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
623 
624 	return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
625 			     AP_DOMAINS);
626 }
627 
628 /*
629  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
630  *				to ensure no queue devices are passed through to
631  *				the guest that are not bound to the vfio_ap
632  *				device driver.
633  *
634  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
635  *
636  * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
637  *	 driver, its APID will be filtered from the guest's APCB. The matrix
638  *	 structure precludes filtering an individual APQN, so its APID will be
639  *	 filtered.
640  *
641  * Return: a boolean value indicating whether the KVM guest's APCB was changed
642  *	   by the filtering or not.
643  */
644 static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm,
645 				       struct ap_matrix_mdev *matrix_mdev)
646 {
647 	unsigned long apid, apqi, apqn;
648 	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
649 	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
650 	struct vfio_ap_queue *q;
651 
652 	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
653 	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
654 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
655 
656 	/*
657 	 * Copy the adapters, domains and control domains to the shadow_apcb
658 	 * from the matrix mdev, but only those that are assigned to the host's
659 	 * AP configuration.
660 	 */
661 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
662 		   (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
663 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
664 		   (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
665 
666 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
667 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
668 			/*
669 			 * If the APQN is not bound to the vfio_ap device
670 			 * driver, then we can't assign it to the guest's
671 			 * AP configuration. The AP architecture won't
672 			 * allow filtering of a single APQN, so let's filter
673 			 * the APID since an adapter represents a physical
674 			 * hardware device.
675 			 */
676 			apqn = AP_MKQID(apid, apqi);
677 			q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
678 			if (!q || q->reset_rc) {
679 				clear_bit_inv(apid,
680 					      matrix_mdev->shadow_apcb.apm);
681 				break;
682 			}
683 		}
684 	}
685 
686 	return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
687 			     AP_DEVICES) ||
688 	       !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
689 			     AP_DOMAINS);
690 }
691 
692 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
693 {
694 	struct ap_matrix_mdev *matrix_mdev =
695 		container_of(vdev, struct ap_matrix_mdev, vdev);
696 
697 	matrix_mdev->mdev = to_mdev_device(vdev->dev);
698 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
699 	matrix_mdev->pqap_hook = handle_pqap;
700 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
701 	hash_init(matrix_mdev->qtable.queues);
702 
703 	return 0;
704 }
705 
706 static int vfio_ap_mdev_probe(struct mdev_device *mdev)
707 {
708 	struct ap_matrix_mdev *matrix_mdev;
709 	int ret;
710 
711 	matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
712 					&vfio_ap_matrix_dev_ops);
713 	if (IS_ERR(matrix_mdev))
714 		return PTR_ERR(matrix_mdev);
715 
716 	ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
717 	if (ret)
718 		goto err_put_vdev;
719 	matrix_mdev->req_trigger = NULL;
720 	dev_set_drvdata(&mdev->dev, matrix_mdev);
721 	mutex_lock(&matrix_dev->mdevs_lock);
722 	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
723 	mutex_unlock(&matrix_dev->mdevs_lock);
724 	return 0;
725 
726 err_put_vdev:
727 	vfio_put_device(&matrix_mdev->vdev);
728 	return ret;
729 }
730 
731 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
732 				    struct vfio_ap_queue *q)
733 {
734 	if (q) {
735 		q->matrix_mdev = matrix_mdev;
736 		hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
737 	}
738 }
739 
740 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
741 {
742 	struct vfio_ap_queue *q;
743 
744 	q = vfio_ap_find_queue(apqn);
745 	vfio_ap_mdev_link_queue(matrix_mdev, q);
746 }
747 
748 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
749 {
750 	hash_del(&q->mdev_qnode);
751 }
752 
753 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
754 {
755 	q->matrix_mdev = NULL;
756 }
757 
758 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
759 {
760 	struct vfio_ap_queue *q;
761 	unsigned long apid, apqi;
762 
763 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
764 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
765 				     AP_DOMAINS) {
766 			q = vfio_ap_mdev_get_queue(matrix_mdev,
767 						   AP_MKQID(apid, apqi));
768 			if (q)
769 				q->matrix_mdev = NULL;
770 		}
771 	}
772 }
773 
774 static void vfio_ap_mdev_remove(struct mdev_device *mdev)
775 {
776 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
777 
778 	vfio_unregister_group_dev(&matrix_mdev->vdev);
779 
780 	mutex_lock(&matrix_dev->guests_lock);
781 	mutex_lock(&matrix_dev->mdevs_lock);
782 	vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
783 	vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
784 	list_del(&matrix_mdev->node);
785 	mutex_unlock(&matrix_dev->mdevs_lock);
786 	mutex_unlock(&matrix_dev->guests_lock);
787 	vfio_put_device(&matrix_mdev->vdev);
788 }
789 
790 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \
791 			 "already assigned to %s"
792 
793 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev,
794 					 unsigned long *apm,
795 					 unsigned long *aqm)
796 {
797 	unsigned long apid, apqi;
798 	const struct device *dev = mdev_dev(matrix_mdev->mdev);
799 	const char *mdev_name = dev_name(dev);
800 
801 	for_each_set_bit_inv(apid, apm, AP_DEVICES)
802 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
803 			dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name);
804 }
805 
806 /**
807  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
808  *
809  * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
810  * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
811  *
812  * Verifies that each APQN derived from the Cartesian product of a bitmap of
813  * AP adapter IDs and AP queue indexes is not configured for any matrix
814  * mediated device. AP queue sharing is not allowed.
815  *
816  * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
817  */
818 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm,
819 					  unsigned long *mdev_aqm)
820 {
821 	struct ap_matrix_mdev *matrix_mdev;
822 	DECLARE_BITMAP(apm, AP_DEVICES);
823 	DECLARE_BITMAP(aqm, AP_DOMAINS);
824 
825 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
826 		/*
827 		 * If the input apm and aqm are fields of the matrix_mdev
828 		 * object, then move on to the next matrix_mdev.
829 		 */
830 		if (mdev_apm == matrix_mdev->matrix.apm &&
831 		    mdev_aqm == matrix_mdev->matrix.aqm)
832 			continue;
833 
834 		memset(apm, 0, sizeof(apm));
835 		memset(aqm, 0, sizeof(aqm));
836 
837 		/*
838 		 * We work on full longs, as we can only exclude the leftover
839 		 * bits in non-inverse order. The leftover is all zeros.
840 		 */
841 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,
842 				AP_DEVICES))
843 			continue;
844 
845 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,
846 				AP_DOMAINS))
847 			continue;
848 
849 		vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm);
850 
851 		return -EADDRINUSE;
852 	}
853 
854 	return 0;
855 }
856 
857 /**
858  * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
859  *				 not reserved for the default zcrypt driver and
860  *				 are not assigned to another mdev.
861  *
862  * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
863  *
864  * Return: One of the following values:
865  * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
866  *   most likely -EBUSY indicating the ap_perms_mutex lock is already held.
867  * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
868  *		   zcrypt default driver.
869  * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
870  * o A zero indicating validation succeeded.
871  */
872 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
873 {
874 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
875 					       matrix_mdev->matrix.aqm))
876 		return -EADDRNOTAVAIL;
877 
878 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,
879 					      matrix_mdev->matrix.aqm);
880 }
881 
882 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
883 				      unsigned long apid)
884 {
885 	unsigned long apqi;
886 
887 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
888 		vfio_ap_mdev_link_apqn(matrix_mdev,
889 				       AP_MKQID(apid, apqi));
890 }
891 
892 /**
893  * assign_adapter_store - parses the APID from @buf and sets the
894  * corresponding bit in the mediated matrix device's APM
895  *
896  * @dev:	the matrix device
897  * @attr:	the mediated matrix device's assign_adapter attribute
898  * @buf:	a buffer containing the AP adapter number (APID) to
899  *		be assigned
900  * @count:	the number of bytes in @buf
901  *
902  * Return: the number of bytes processed if the APID is valid; otherwise,
903  * returns one of the following errors:
904  *
905  *	1. -EINVAL
906  *	   The APID is not a valid number
907  *
908  *	2. -ENODEV
909  *	   The APID exceeds the maximum value configured for the system
910  *
911  *	3. -EADDRNOTAVAIL
912  *	   An APQN derived from the cross product of the APID being assigned
913  *	   and the APQIs previously assigned is not bound to the vfio_ap device
914  *	   driver; or, if no APQIs have yet been assigned, the APID is not
915  *	   contained in an APQN bound to the vfio_ap device driver.
916  *
917  *	4. -EADDRINUSE
918  *	   An APQN derived from the cross product of the APID being assigned
919  *	   and the APQIs previously assigned is being used by another mediated
920  *	   matrix device
921  *
922  *	5. -EAGAIN
923  *	   A lock required to validate the mdev's AP configuration could not
924  *	   be obtained.
925  */
926 static ssize_t assign_adapter_store(struct device *dev,
927 				    struct device_attribute *attr,
928 				    const char *buf, size_t count)
929 {
930 	int ret;
931 	unsigned long apid;
932 	DECLARE_BITMAP(apm_delta, AP_DEVICES);
933 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
934 
935 	mutex_lock(&ap_perms_mutex);
936 	get_update_locks_for_mdev(matrix_mdev);
937 
938 	ret = kstrtoul(buf, 0, &apid);
939 	if (ret)
940 		goto done;
941 
942 	if (apid > matrix_mdev->matrix.apm_max) {
943 		ret = -ENODEV;
944 		goto done;
945 	}
946 
947 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
948 		ret = count;
949 		goto done;
950 	}
951 
952 	set_bit_inv(apid, matrix_mdev->matrix.apm);
953 
954 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
955 	if (ret) {
956 		clear_bit_inv(apid, matrix_mdev->matrix.apm);
957 		goto done;
958 	}
959 
960 	vfio_ap_mdev_link_adapter(matrix_mdev, apid);
961 	memset(apm_delta, 0, sizeof(apm_delta));
962 	set_bit_inv(apid, apm_delta);
963 
964 	if (vfio_ap_mdev_filter_matrix(apm_delta,
965 				       matrix_mdev->matrix.aqm, matrix_mdev))
966 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
967 
968 	ret = count;
969 done:
970 	release_update_locks_for_mdev(matrix_mdev);
971 	mutex_unlock(&ap_perms_mutex);
972 
973 	return ret;
974 }
975 static DEVICE_ATTR_WO(assign_adapter);
976 
977 static struct vfio_ap_queue
978 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
979 			     unsigned long apid, unsigned long apqi)
980 {
981 	struct vfio_ap_queue *q = NULL;
982 
983 	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
984 	/* If the queue is assigned to the matrix mdev, unlink it. */
985 	if (q)
986 		vfio_ap_unlink_queue_fr_mdev(q);
987 
988 	return q;
989 }
990 
991 /**
992  * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
993  *				 adapter from the matrix mdev to which the
994  *				 adapter was assigned.
995  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
996  * @apid: the APID of the unassigned adapter.
997  * @qtable: table for storing queues associated with unassigned adapter.
998  */
999 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
1000 					unsigned long apid,
1001 					struct ap_queue_table *qtable)
1002 {
1003 	unsigned long apqi;
1004 	struct vfio_ap_queue *q;
1005 
1006 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
1007 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1008 
1009 		if (q && qtable) {
1010 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1011 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1012 				hash_add(qtable->queues, &q->mdev_qnode,
1013 					 q->apqn);
1014 		}
1015 	}
1016 }
1017 
1018 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
1019 					    unsigned long apid)
1020 {
1021 	int loop_cursor;
1022 	struct vfio_ap_queue *q;
1023 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1024 
1025 	hash_init(qtable->queues);
1026 	vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable);
1027 
1028 	if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
1029 		clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1030 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1031 	}
1032 
1033 	vfio_ap_mdev_reset_queues(qtable);
1034 
1035 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1036 		vfio_ap_unlink_mdev_fr_queue(q);
1037 		hash_del(&q->mdev_qnode);
1038 	}
1039 
1040 	kfree(qtable);
1041 }
1042 
1043 /**
1044  * unassign_adapter_store - parses the APID from @buf and clears the
1045  * corresponding bit in the mediated matrix device's APM
1046  *
1047  * @dev:	the matrix device
1048  * @attr:	the mediated matrix device's unassign_adapter attribute
1049  * @buf:	a buffer containing the adapter number (APID) to be unassigned
1050  * @count:	the number of bytes in @buf
1051  *
1052  * Return: the number of bytes processed if the APID is valid; otherwise,
1053  * returns one of the following errors:
1054  *	-EINVAL if the APID is not a number
1055  *	-ENODEV if the APID it exceeds the maximum value configured for the
1056  *		system
1057  */
1058 static ssize_t unassign_adapter_store(struct device *dev,
1059 				      struct device_attribute *attr,
1060 				      const char *buf, size_t count)
1061 {
1062 	int ret;
1063 	unsigned long apid;
1064 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1065 
1066 	get_update_locks_for_mdev(matrix_mdev);
1067 
1068 	ret = kstrtoul(buf, 0, &apid);
1069 	if (ret)
1070 		goto done;
1071 
1072 	if (apid > matrix_mdev->matrix.apm_max) {
1073 		ret = -ENODEV;
1074 		goto done;
1075 	}
1076 
1077 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1078 		ret = count;
1079 		goto done;
1080 	}
1081 
1082 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
1083 	vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
1084 	ret = count;
1085 done:
1086 	release_update_locks_for_mdev(matrix_mdev);
1087 	return ret;
1088 }
1089 static DEVICE_ATTR_WO(unassign_adapter);
1090 
1091 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
1092 				     unsigned long apqi)
1093 {
1094 	unsigned long apid;
1095 
1096 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
1097 		vfio_ap_mdev_link_apqn(matrix_mdev,
1098 				       AP_MKQID(apid, apqi));
1099 }
1100 
1101 /**
1102  * assign_domain_store - parses the APQI from @buf and sets the
1103  * corresponding bit in the mediated matrix device's AQM
1104  *
1105  * @dev:	the matrix device
1106  * @attr:	the mediated matrix device's assign_domain attribute
1107  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1108  *		be assigned
1109  * @count:	the number of bytes in @buf
1110  *
1111  * Return: the number of bytes processed if the APQI is valid; otherwise returns
1112  * one of the following errors:
1113  *
1114  *	1. -EINVAL
1115  *	   The APQI is not a valid number
1116  *
1117  *	2. -ENODEV
1118  *	   The APQI exceeds the maximum value configured for the system
1119  *
1120  *	3. -EADDRNOTAVAIL
1121  *	   An APQN derived from the cross product of the APQI being assigned
1122  *	   and the APIDs previously assigned is not bound to the vfio_ap device
1123  *	   driver; or, if no APIDs have yet been assigned, the APQI is not
1124  *	   contained in an APQN bound to the vfio_ap device driver.
1125  *
1126  *	4. -EADDRINUSE
1127  *	   An APQN derived from the cross product of the APQI being assigned
1128  *	   and the APIDs previously assigned is being used by another mediated
1129  *	   matrix device
1130  *
1131  *	5. -EAGAIN
1132  *	   The lock required to validate the mdev's AP configuration could not
1133  *	   be obtained.
1134  */
1135 static ssize_t assign_domain_store(struct device *dev,
1136 				   struct device_attribute *attr,
1137 				   const char *buf, size_t count)
1138 {
1139 	int ret;
1140 	unsigned long apqi;
1141 	DECLARE_BITMAP(aqm_delta, AP_DOMAINS);
1142 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1143 
1144 	mutex_lock(&ap_perms_mutex);
1145 	get_update_locks_for_mdev(matrix_mdev);
1146 
1147 	ret = kstrtoul(buf, 0, &apqi);
1148 	if (ret)
1149 		goto done;
1150 
1151 	if (apqi > matrix_mdev->matrix.aqm_max) {
1152 		ret = -ENODEV;
1153 		goto done;
1154 	}
1155 
1156 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1157 		ret = count;
1158 		goto done;
1159 	}
1160 
1161 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);
1162 
1163 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1164 	if (ret) {
1165 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
1166 		goto done;
1167 	}
1168 
1169 	vfio_ap_mdev_link_domain(matrix_mdev, apqi);
1170 	memset(aqm_delta, 0, sizeof(aqm_delta));
1171 	set_bit_inv(apqi, aqm_delta);
1172 
1173 	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,
1174 				       matrix_mdev))
1175 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1176 
1177 	ret = count;
1178 done:
1179 	release_update_locks_for_mdev(matrix_mdev);
1180 	mutex_unlock(&ap_perms_mutex);
1181 
1182 	return ret;
1183 }
1184 static DEVICE_ATTR_WO(assign_domain);
1185 
1186 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
1187 				       unsigned long apqi,
1188 				       struct ap_queue_table *qtable)
1189 {
1190 	unsigned long apid;
1191 	struct vfio_ap_queue *q;
1192 
1193 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
1194 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1195 
1196 		if (q && qtable) {
1197 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1198 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1199 				hash_add(qtable->queues, &q->mdev_qnode,
1200 					 q->apqn);
1201 		}
1202 	}
1203 }
1204 
1205 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
1206 					   unsigned long apqi)
1207 {
1208 	int loop_cursor;
1209 	struct vfio_ap_queue *q;
1210 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1211 
1212 	hash_init(qtable->queues);
1213 	vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable);
1214 
1215 	if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1216 		clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
1217 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1218 	}
1219 
1220 	vfio_ap_mdev_reset_queues(qtable);
1221 
1222 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1223 		vfio_ap_unlink_mdev_fr_queue(q);
1224 		hash_del(&q->mdev_qnode);
1225 	}
1226 
1227 	kfree(qtable);
1228 }
1229 
1230 /**
1231  * unassign_domain_store - parses the APQI from @buf and clears the
1232  * corresponding bit in the mediated matrix device's AQM
1233  *
1234  * @dev:	the matrix device
1235  * @attr:	the mediated matrix device's unassign_domain attribute
1236  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1237  *		be unassigned
1238  * @count:	the number of bytes in @buf
1239  *
1240  * Return: the number of bytes processed if the APQI is valid; otherwise,
1241  * returns one of the following errors:
1242  *	-EINVAL if the APQI is not a number
1243  *	-ENODEV if the APQI exceeds the maximum value configured for the system
1244  */
1245 static ssize_t unassign_domain_store(struct device *dev,
1246 				     struct device_attribute *attr,
1247 				     const char *buf, size_t count)
1248 {
1249 	int ret;
1250 	unsigned long apqi;
1251 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1252 
1253 	get_update_locks_for_mdev(matrix_mdev);
1254 
1255 	ret = kstrtoul(buf, 0, &apqi);
1256 	if (ret)
1257 		goto done;
1258 
1259 	if (apqi > matrix_mdev->matrix.aqm_max) {
1260 		ret = -ENODEV;
1261 		goto done;
1262 	}
1263 
1264 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1265 		ret = count;
1266 		goto done;
1267 	}
1268 
1269 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
1270 	vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
1271 	ret = count;
1272 
1273 done:
1274 	release_update_locks_for_mdev(matrix_mdev);
1275 	return ret;
1276 }
1277 static DEVICE_ATTR_WO(unassign_domain);
1278 
1279 /**
1280  * assign_control_domain_store - parses the domain ID from @buf and sets
1281  * the corresponding bit in the mediated matrix device's ADM
1282  *
1283  * @dev:	the matrix device
1284  * @attr:	the mediated matrix device's assign_control_domain attribute
1285  * @buf:	a buffer containing the domain ID to be assigned
1286  * @count:	the number of bytes in @buf
1287  *
1288  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1289  * returns one of the following errors:
1290  *	-EINVAL if the ID is not a number
1291  *	-ENODEV if the ID exceeds the maximum value configured for the system
1292  */
1293 static ssize_t assign_control_domain_store(struct device *dev,
1294 					   struct device_attribute *attr,
1295 					   const char *buf, size_t count)
1296 {
1297 	int ret;
1298 	unsigned long id;
1299 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1300 
1301 	get_update_locks_for_mdev(matrix_mdev);
1302 
1303 	ret = kstrtoul(buf, 0, &id);
1304 	if (ret)
1305 		goto done;
1306 
1307 	if (id > matrix_mdev->matrix.adm_max) {
1308 		ret = -ENODEV;
1309 		goto done;
1310 	}
1311 
1312 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
1313 		ret = count;
1314 		goto done;
1315 	}
1316 
1317 	/* Set the bit in the ADM (bitmask) corresponding to the AP control
1318 	 * domain number (id). The bits in the mask, from most significant to
1319 	 * least significant, correspond to IDs 0 up to the one less than the
1320 	 * number of control domains that can be assigned.
1321 	 */
1322 	set_bit_inv(id, matrix_mdev->matrix.adm);
1323 	if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
1324 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1325 
1326 	ret = count;
1327 done:
1328 	release_update_locks_for_mdev(matrix_mdev);
1329 	return ret;
1330 }
1331 static DEVICE_ATTR_WO(assign_control_domain);
1332 
1333 /**
1334  * unassign_control_domain_store - parses the domain ID from @buf and
1335  * clears the corresponding bit in the mediated matrix device's ADM
1336  *
1337  * @dev:	the matrix device
1338  * @attr:	the mediated matrix device's unassign_control_domain attribute
1339  * @buf:	a buffer containing the domain ID to be unassigned
1340  * @count:	the number of bytes in @buf
1341  *
1342  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1343  * returns one of the following errors:
1344  *	-EINVAL if the ID is not a number
1345  *	-ENODEV if the ID exceeds the maximum value configured for the system
1346  */
1347 static ssize_t unassign_control_domain_store(struct device *dev,
1348 					     struct device_attribute *attr,
1349 					     const char *buf, size_t count)
1350 {
1351 	int ret;
1352 	unsigned long domid;
1353 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1354 
1355 	get_update_locks_for_mdev(matrix_mdev);
1356 
1357 	ret = kstrtoul(buf, 0, &domid);
1358 	if (ret)
1359 		goto done;
1360 
1361 	if (domid > matrix_mdev->matrix.adm_max) {
1362 		ret = -ENODEV;
1363 		goto done;
1364 	}
1365 
1366 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
1367 		ret = count;
1368 		goto done;
1369 	}
1370 
1371 	clear_bit_inv(domid, matrix_mdev->matrix.adm);
1372 
1373 	if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
1374 		clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
1375 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1376 	}
1377 
1378 	ret = count;
1379 done:
1380 	release_update_locks_for_mdev(matrix_mdev);
1381 	return ret;
1382 }
1383 static DEVICE_ATTR_WO(unassign_control_domain);
1384 
1385 static ssize_t control_domains_show(struct device *dev,
1386 				    struct device_attribute *dev_attr,
1387 				    char *buf)
1388 {
1389 	unsigned long id;
1390 	int nchars = 0;
1391 	int n;
1392 	char *bufpos = buf;
1393 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1394 	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1395 
1396 	mutex_lock(&matrix_dev->mdevs_lock);
1397 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
1398 		n = sprintf(bufpos, "%04lx\n", id);
1399 		bufpos += n;
1400 		nchars += n;
1401 	}
1402 	mutex_unlock(&matrix_dev->mdevs_lock);
1403 
1404 	return nchars;
1405 }
1406 static DEVICE_ATTR_RO(control_domains);
1407 
1408 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
1409 {
1410 	char *bufpos = buf;
1411 	unsigned long apid;
1412 	unsigned long apqi;
1413 	unsigned long apid1;
1414 	unsigned long apqi1;
1415 	unsigned long napm_bits = matrix->apm_max + 1;
1416 	unsigned long naqm_bits = matrix->aqm_max + 1;
1417 	int nchars = 0;
1418 	int n;
1419 
1420 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);
1421 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
1422 
1423 	if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
1424 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1425 			for_each_set_bit_inv(apqi, matrix->aqm,
1426 					     naqm_bits) {
1427 				n = sprintf(bufpos, "%02lx.%04lx\n", apid,
1428 					    apqi);
1429 				bufpos += n;
1430 				nchars += n;
1431 			}
1432 		}
1433 	} else if (apid1 < napm_bits) {
1434 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1435 			n = sprintf(bufpos, "%02lx.\n", apid);
1436 			bufpos += n;
1437 			nchars += n;
1438 		}
1439 	} else if (apqi1 < naqm_bits) {
1440 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {
1441 			n = sprintf(bufpos, ".%04lx\n", apqi);
1442 			bufpos += n;
1443 			nchars += n;
1444 		}
1445 	}
1446 
1447 	return nchars;
1448 }
1449 
1450 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1451 			   char *buf)
1452 {
1453 	ssize_t nchars;
1454 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1455 
1456 	mutex_lock(&matrix_dev->mdevs_lock);
1457 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
1458 	mutex_unlock(&matrix_dev->mdevs_lock);
1459 
1460 	return nchars;
1461 }
1462 static DEVICE_ATTR_RO(matrix);
1463 
1464 static ssize_t guest_matrix_show(struct device *dev,
1465 				 struct device_attribute *attr, char *buf)
1466 {
1467 	ssize_t nchars;
1468 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1469 
1470 	mutex_lock(&matrix_dev->mdevs_lock);
1471 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
1472 	mutex_unlock(&matrix_dev->mdevs_lock);
1473 
1474 	return nchars;
1475 }
1476 static DEVICE_ATTR_RO(guest_matrix);
1477 
1478 static struct attribute *vfio_ap_mdev_attrs[] = {
1479 	&dev_attr_assign_adapter.attr,
1480 	&dev_attr_unassign_adapter.attr,
1481 	&dev_attr_assign_domain.attr,
1482 	&dev_attr_unassign_domain.attr,
1483 	&dev_attr_assign_control_domain.attr,
1484 	&dev_attr_unassign_control_domain.attr,
1485 	&dev_attr_control_domains.attr,
1486 	&dev_attr_matrix.attr,
1487 	&dev_attr_guest_matrix.attr,
1488 	NULL,
1489 };
1490 
1491 static struct attribute_group vfio_ap_mdev_attr_group = {
1492 	.attrs = vfio_ap_mdev_attrs
1493 };
1494 
1495 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1496 	&vfio_ap_mdev_attr_group,
1497 	NULL
1498 };
1499 
1500 /**
1501  * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
1502  * to manage AP resources for the guest whose state is represented by @kvm
1503  *
1504  * @matrix_mdev: a mediated matrix device
1505  * @kvm: reference to KVM instance
1506  *
1507  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1508  * otherwise, returns an -EPERM.
1509  */
1510 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1511 				struct kvm *kvm)
1512 {
1513 	struct ap_matrix_mdev *m;
1514 
1515 	if (kvm->arch.crypto.crycbd) {
1516 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1517 		kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1518 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1519 
1520 		get_update_locks_for_kvm(kvm);
1521 
1522 		list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1523 			if (m != matrix_mdev && m->kvm == kvm) {
1524 				release_update_locks_for_kvm(kvm);
1525 				return -EPERM;
1526 			}
1527 		}
1528 
1529 		kvm_get_kvm(kvm);
1530 		matrix_mdev->kvm = kvm;
1531 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1532 
1533 		release_update_locks_for_kvm(kvm);
1534 	}
1535 
1536 	return 0;
1537 }
1538 
1539 static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
1540 {
1541 	struct ap_queue_table *qtable = &matrix_mdev->qtable;
1542 	struct vfio_ap_queue *q;
1543 	int loop_cursor;
1544 
1545 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1546 		if (q->saved_iova >= iova && q->saved_iova < iova + length)
1547 			vfio_ap_irq_disable(q);
1548 	}
1549 }
1550 
1551 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
1552 				   u64 length)
1553 {
1554 	struct ap_matrix_mdev *matrix_mdev =
1555 		container_of(vdev, struct ap_matrix_mdev, vdev);
1556 
1557 	mutex_lock(&matrix_dev->mdevs_lock);
1558 
1559 	unmap_iova(matrix_mdev, iova, length);
1560 
1561 	mutex_unlock(&matrix_dev->mdevs_lock);
1562 }
1563 
1564 /**
1565  * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
1566  * by @matrix_mdev.
1567  *
1568  * @matrix_mdev: a matrix mediated device
1569  */
1570 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
1571 {
1572 	struct kvm *kvm = matrix_mdev->kvm;
1573 
1574 	if (kvm && kvm->arch.crypto.crycbd) {
1575 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1576 		kvm->arch.crypto.pqap_hook = NULL;
1577 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1578 
1579 		get_update_locks_for_kvm(kvm);
1580 
1581 		kvm_arch_crypto_clear_masks(kvm);
1582 		vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1583 		kvm_put_kvm(kvm);
1584 		matrix_mdev->kvm = NULL;
1585 
1586 		release_update_locks_for_kvm(kvm);
1587 	}
1588 }
1589 
1590 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
1591 {
1592 	struct ap_queue *queue;
1593 	struct vfio_ap_queue *q = NULL;
1594 
1595 	queue = ap_get_qdev(apqn);
1596 	if (!queue)
1597 		return NULL;
1598 
1599 	if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
1600 		q = dev_get_drvdata(&queue->ap_dev.device);
1601 
1602 	put_device(&queue->ap_dev.device);
1603 
1604 	return q;
1605 }
1606 
1607 static int apq_status_check(int apqn, struct ap_queue_status *status)
1608 {
1609 	switch (status->response_code) {
1610 	case AP_RESPONSE_NORMAL:
1611 	case AP_RESPONSE_RESET_IN_PROGRESS:
1612 		if (status->queue_empty && !status->irq_enabled)
1613 			return 0;
1614 		return -EBUSY;
1615 	case AP_RESPONSE_DECONFIGURED:
1616 		/*
1617 		 * If the AP queue is deconfigured, any subsequent AP command
1618 		 * targeting the queue will fail with the same response code. On the
1619 		 * other hand, when an AP adapter is deconfigured, the associated
1620 		 * queues are reset, so let's return a value indicating the reset
1621 		 * for which we're waiting completed successfully.
1622 		 */
1623 		return 0;
1624 	default:
1625 		WARN(true,
1626 		     "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n",
1627 		     AP_QID_CARD(apqn), AP_QID_QUEUE(apqn),
1628 		     status->response_code);
1629 		return -EIO;
1630 	}
1631 }
1632 
1633 static int apq_reset_check(struct vfio_ap_queue *q)
1634 {
1635 	int ret;
1636 	int iters = MAX_RESET_CHECK_WAIT / AP_RESET_INTERVAL;
1637 	struct ap_queue_status status;
1638 
1639 	for (; iters > 0; iters--) {
1640 		msleep(AP_RESET_INTERVAL);
1641 		status = ap_tapq(q->apqn, NULL);
1642 		ret = apq_status_check(q->apqn, &status);
1643 		if (ret != -EBUSY)
1644 			return ret;
1645 	}
1646 	WARN_ONCE(iters <= 0,
1647 		  "timeout verifying reset of queue %02x.%04x (%u, %u, %u)",
1648 		  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1649 		  status.queue_empty, status.irq_enabled, status.response_code);
1650 	return ret;
1651 }
1652 
1653 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)
1654 {
1655 	struct ap_queue_status status;
1656 	int ret;
1657 
1658 	if (!q)
1659 		return 0;
1660 retry_zapq:
1661 	status = ap_zapq(q->apqn, 0);
1662 	q->reset_rc = status.response_code;
1663 	switch (status.response_code) {
1664 	case AP_RESPONSE_NORMAL:
1665 		ret = 0;
1666 		/* if the reset has not completed, wait for it to take effect */
1667 		if (!status.queue_empty || status.irq_enabled)
1668 			ret = apq_reset_check(q);
1669 		break;
1670 	case AP_RESPONSE_RESET_IN_PROGRESS:
1671 		/*
1672 		 * There is a reset issued by another process in progress. Let's wait
1673 		 * for that to complete. Since we have no idea whether it was a RAPQ or
1674 		 * ZAPQ, then if it completes successfully, let's issue the ZAPQ.
1675 		 */
1676 		ret = apq_reset_check(q);
1677 		if (ret)
1678 			break;
1679 		goto retry_zapq;
1680 	case AP_RESPONSE_DECONFIGURED:
1681 		/*
1682 		 * When an AP adapter is deconfigured, the associated
1683 		 * queues are reset, so let's return a value indicating the reset
1684 		 * completed successfully.
1685 		 */
1686 		ret = 0;
1687 		break;
1688 	default:
1689 		WARN(true,
1690 		     "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
1691 		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1692 		     status.response_code);
1693 		return -EIO;
1694 	}
1695 
1696 	vfio_ap_free_aqic_resources(q);
1697 
1698 	return ret;
1699 }
1700 
1701 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable)
1702 {
1703 	int ret, loop_cursor, rc = 0;
1704 	struct vfio_ap_queue *q;
1705 
1706 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1707 		ret = vfio_ap_mdev_reset_queue(q);
1708 		/*
1709 		 * Regardless whether a queue turns out to be busy, or
1710 		 * is not operational, we need to continue resetting
1711 		 * the remaining queues.
1712 		 */
1713 		if (ret)
1714 			rc = ret;
1715 	}
1716 
1717 	return rc;
1718 }
1719 
1720 static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1721 {
1722 	struct ap_matrix_mdev *matrix_mdev =
1723 		container_of(vdev, struct ap_matrix_mdev, vdev);
1724 
1725 	if (!vdev->kvm)
1726 		return -EINVAL;
1727 
1728 	return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
1729 }
1730 
1731 static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1732 {
1733 	struct ap_matrix_mdev *matrix_mdev =
1734 		container_of(vdev, struct ap_matrix_mdev, vdev);
1735 
1736 	vfio_ap_mdev_unset_kvm(matrix_mdev);
1737 }
1738 
1739 static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count)
1740 {
1741 	struct device *dev = vdev->dev;
1742 	struct ap_matrix_mdev *matrix_mdev;
1743 
1744 	matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev);
1745 
1746 	if (matrix_mdev->req_trigger) {
1747 		if (!(count % 10))
1748 			dev_notice_ratelimited(dev,
1749 					       "Relaying device request to user (#%u)\n",
1750 					       count);
1751 
1752 		eventfd_signal(matrix_mdev->req_trigger, 1);
1753 	} else if (count == 0) {
1754 		dev_notice(dev,
1755 			   "No device request registered, blocked until released by user\n");
1756 	}
1757 }
1758 
1759 static int vfio_ap_mdev_get_device_info(unsigned long arg)
1760 {
1761 	unsigned long minsz;
1762 	struct vfio_device_info info;
1763 
1764 	minsz = offsetofend(struct vfio_device_info, num_irqs);
1765 
1766 	if (copy_from_user(&info, (void __user *)arg, minsz))
1767 		return -EFAULT;
1768 
1769 	if (info.argsz < minsz)
1770 		return -EINVAL;
1771 
1772 	info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
1773 	info.num_regions = 0;
1774 	info.num_irqs = VFIO_AP_NUM_IRQS;
1775 
1776 	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1777 }
1778 
1779 static ssize_t vfio_ap_get_irq_info(unsigned long arg)
1780 {
1781 	unsigned long minsz;
1782 	struct vfio_irq_info info;
1783 
1784 	minsz = offsetofend(struct vfio_irq_info, count);
1785 
1786 	if (copy_from_user(&info, (void __user *)arg, minsz))
1787 		return -EFAULT;
1788 
1789 	if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS)
1790 		return -EINVAL;
1791 
1792 	switch (info.index) {
1793 	case VFIO_AP_REQ_IRQ_INDEX:
1794 		info.count = 1;
1795 		info.flags = VFIO_IRQ_INFO_EVENTFD;
1796 		break;
1797 	default:
1798 		return -EINVAL;
1799 	}
1800 
1801 	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1802 }
1803 
1804 static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg)
1805 {
1806 	int ret;
1807 	size_t data_size;
1808 	unsigned long minsz;
1809 
1810 	minsz = offsetofend(struct vfio_irq_set, count);
1811 
1812 	if (copy_from_user(irq_set, (void __user *)arg, minsz))
1813 		return -EFAULT;
1814 
1815 	ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS,
1816 						 &data_size);
1817 	if (ret)
1818 		return ret;
1819 
1820 	if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER))
1821 		return -EINVAL;
1822 
1823 	return 0;
1824 }
1825 
1826 static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev,
1827 				   unsigned long arg)
1828 {
1829 	s32 fd;
1830 	void __user *data;
1831 	unsigned long minsz;
1832 	struct eventfd_ctx *req_trigger;
1833 
1834 	minsz = offsetofend(struct vfio_irq_set, count);
1835 	data = (void __user *)(arg + minsz);
1836 
1837 	if (get_user(fd, (s32 __user *)data))
1838 		return -EFAULT;
1839 
1840 	if (fd == -1) {
1841 		if (matrix_mdev->req_trigger)
1842 			eventfd_ctx_put(matrix_mdev->req_trigger);
1843 		matrix_mdev->req_trigger = NULL;
1844 	} else if (fd >= 0) {
1845 		req_trigger = eventfd_ctx_fdget(fd);
1846 		if (IS_ERR(req_trigger))
1847 			return PTR_ERR(req_trigger);
1848 
1849 		if (matrix_mdev->req_trigger)
1850 			eventfd_ctx_put(matrix_mdev->req_trigger);
1851 
1852 		matrix_mdev->req_trigger = req_trigger;
1853 	} else {
1854 		return -EINVAL;
1855 	}
1856 
1857 	return 0;
1858 }
1859 
1860 static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev,
1861 			    unsigned long arg)
1862 {
1863 	int ret;
1864 	struct vfio_irq_set irq_set;
1865 
1866 	ret = vfio_ap_irq_set_init(&irq_set, arg);
1867 	if (ret)
1868 		return ret;
1869 
1870 	switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1871 	case VFIO_IRQ_SET_DATA_EVENTFD:
1872 		switch (irq_set.index) {
1873 		case VFIO_AP_REQ_IRQ_INDEX:
1874 			return vfio_ap_set_request_irq(matrix_mdev, arg);
1875 		default:
1876 			return -EINVAL;
1877 		}
1878 	default:
1879 		return -EINVAL;
1880 	}
1881 }
1882 
1883 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1884 				    unsigned int cmd, unsigned long arg)
1885 {
1886 	struct ap_matrix_mdev *matrix_mdev =
1887 		container_of(vdev, struct ap_matrix_mdev, vdev);
1888 	int ret;
1889 
1890 	mutex_lock(&matrix_dev->mdevs_lock);
1891 	switch (cmd) {
1892 	case VFIO_DEVICE_GET_INFO:
1893 		ret = vfio_ap_mdev_get_device_info(arg);
1894 		break;
1895 	case VFIO_DEVICE_RESET:
1896 		ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1897 		break;
1898 	case VFIO_DEVICE_GET_IRQ_INFO:
1899 			ret = vfio_ap_get_irq_info(arg);
1900 			break;
1901 	case VFIO_DEVICE_SET_IRQS:
1902 		ret = vfio_ap_set_irqs(matrix_mdev, arg);
1903 		break;
1904 	default:
1905 		ret = -EOPNOTSUPP;
1906 		break;
1907 	}
1908 	mutex_unlock(&matrix_dev->mdevs_lock);
1909 
1910 	return ret;
1911 }
1912 
1913 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
1914 {
1915 	struct ap_matrix_mdev *matrix_mdev;
1916 	unsigned long apid = AP_QID_CARD(q->apqn);
1917 	unsigned long apqi = AP_QID_QUEUE(q->apqn);
1918 
1919 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
1920 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
1921 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))
1922 			return matrix_mdev;
1923 	}
1924 
1925 	return NULL;
1926 }
1927 
1928 static ssize_t status_show(struct device *dev,
1929 			   struct device_attribute *attr,
1930 			   char *buf)
1931 {
1932 	ssize_t nchars = 0;
1933 	struct vfio_ap_queue *q;
1934 	struct ap_matrix_mdev *matrix_mdev;
1935 	struct ap_device *apdev = to_ap_dev(dev);
1936 
1937 	mutex_lock(&matrix_dev->mdevs_lock);
1938 	q = dev_get_drvdata(&apdev->device);
1939 	matrix_mdev = vfio_ap_mdev_for_queue(q);
1940 
1941 	if (matrix_mdev) {
1942 		if (matrix_mdev->kvm)
1943 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1944 					   AP_QUEUE_IN_USE);
1945 		else
1946 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1947 					   AP_QUEUE_ASSIGNED);
1948 	} else {
1949 		nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1950 				   AP_QUEUE_UNASSIGNED);
1951 	}
1952 
1953 	mutex_unlock(&matrix_dev->mdevs_lock);
1954 
1955 	return nchars;
1956 }
1957 
1958 static DEVICE_ATTR_RO(status);
1959 
1960 static struct attribute *vfio_queue_attrs[] = {
1961 	&dev_attr_status.attr,
1962 	NULL,
1963 };
1964 
1965 static const struct attribute_group vfio_queue_attr_group = {
1966 	.attrs = vfio_queue_attrs,
1967 };
1968 
1969 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
1970 	.init = vfio_ap_mdev_init_dev,
1971 	.open_device = vfio_ap_mdev_open_device,
1972 	.close_device = vfio_ap_mdev_close_device,
1973 	.ioctl = vfio_ap_mdev_ioctl,
1974 	.dma_unmap = vfio_ap_mdev_dma_unmap,
1975 	.bind_iommufd = vfio_iommufd_emulated_bind,
1976 	.unbind_iommufd = vfio_iommufd_emulated_unbind,
1977 	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
1978 	.detach_ioas = vfio_iommufd_emulated_detach_ioas,
1979 	.request = vfio_ap_mdev_request
1980 };
1981 
1982 static struct mdev_driver vfio_ap_matrix_driver = {
1983 	.device_api = VFIO_DEVICE_API_AP_STRING,
1984 	.max_instances = MAX_ZDEV_ENTRIES_EXT,
1985 	.driver = {
1986 		.name = "vfio_ap_mdev",
1987 		.owner = THIS_MODULE,
1988 		.mod_name = KBUILD_MODNAME,
1989 		.dev_groups = vfio_ap_mdev_attr_groups,
1990 	},
1991 	.probe = vfio_ap_mdev_probe,
1992 	.remove = vfio_ap_mdev_remove,
1993 };
1994 
1995 int vfio_ap_mdev_register(void)
1996 {
1997 	int ret;
1998 
1999 	ret = mdev_register_driver(&vfio_ap_matrix_driver);
2000 	if (ret)
2001 		return ret;
2002 
2003 	matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
2004 	matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
2005 	matrix_dev->mdev_types[0] = &matrix_dev->mdev_type;
2006 	ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
2007 				   &vfio_ap_matrix_driver,
2008 				   matrix_dev->mdev_types, 1);
2009 	if (ret)
2010 		goto err_driver;
2011 	return 0;
2012 
2013 err_driver:
2014 	mdev_unregister_driver(&vfio_ap_matrix_driver);
2015 	return ret;
2016 }
2017 
2018 void vfio_ap_mdev_unregister(void)
2019 {
2020 	mdev_unregister_parent(&matrix_dev->parent);
2021 	mdev_unregister_driver(&vfio_ap_matrix_driver);
2022 }
2023 
2024 int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
2025 {
2026 	int ret;
2027 	struct vfio_ap_queue *q;
2028 	struct ap_matrix_mdev *matrix_mdev;
2029 
2030 	ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
2031 	if (ret)
2032 		return ret;
2033 
2034 	q = kzalloc(sizeof(*q), GFP_KERNEL);
2035 	if (!q) {
2036 		ret = -ENOMEM;
2037 		goto err_remove_group;
2038 	}
2039 
2040 	q->apqn = to_ap_queue(&apdev->device)->qid;
2041 	q->saved_isc = VFIO_AP_ISC_INVALID;
2042 	matrix_mdev = get_update_locks_by_apqn(q->apqn);
2043 
2044 	if (matrix_mdev) {
2045 		vfio_ap_mdev_link_queue(matrix_mdev, q);
2046 
2047 		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,
2048 					       matrix_mdev->matrix.aqm,
2049 					       matrix_mdev))
2050 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2051 	}
2052 	dev_set_drvdata(&apdev->device, q);
2053 	release_update_locks_for_mdev(matrix_mdev);
2054 
2055 	return 0;
2056 
2057 err_remove_group:
2058 	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
2059 	return ret;
2060 }
2061 
2062 void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
2063 {
2064 	unsigned long apid, apqi;
2065 	struct vfio_ap_queue *q;
2066 	struct ap_matrix_mdev *matrix_mdev;
2067 
2068 	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
2069 	q = dev_get_drvdata(&apdev->device);
2070 	get_update_locks_for_queue(q);
2071 	matrix_mdev = q->matrix_mdev;
2072 
2073 	if (matrix_mdev) {
2074 		vfio_ap_unlink_queue_fr_mdev(q);
2075 
2076 		apid = AP_QID_CARD(q->apqn);
2077 		apqi = AP_QID_QUEUE(q->apqn);
2078 
2079 		/*
2080 		 * If the queue is assigned to the guest's APCB, then remove
2081 		 * the adapter's APID from the APCB and hot it into the guest.
2082 		 */
2083 		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
2084 		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
2085 			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
2086 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2087 		}
2088 	}
2089 
2090 	vfio_ap_mdev_reset_queue(q);
2091 	dev_set_drvdata(&apdev->device, NULL);
2092 	kfree(q);
2093 	release_update_locks_for_mdev(matrix_mdev);
2094 }
2095 
2096 /**
2097  * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
2098  *				 assigned to a mediated device under the control
2099  *				 of the vfio_ap device driver.
2100  *
2101  * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
2102  * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
2103  *
2104  * Return:
2105  *	* -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
2106  *	  assigned to a mediated device under the control of the vfio_ap
2107  *	  device driver.
2108  *	* Otherwise, return 0.
2109  */
2110 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
2111 {
2112 	int ret;
2113 
2114 	mutex_lock(&matrix_dev->guests_lock);
2115 	mutex_lock(&matrix_dev->mdevs_lock);
2116 	ret = vfio_ap_mdev_verify_no_sharing(apm, aqm);
2117 	mutex_unlock(&matrix_dev->mdevs_lock);
2118 	mutex_unlock(&matrix_dev->guests_lock);
2119 
2120 	return ret;
2121 }
2122 
2123 /**
2124  * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
2125  *				 domains that have been removed from the host's
2126  *				 AP configuration from a guest.
2127  *
2128  * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
2129  * @aprem: the adapters that have been removed from the host's AP configuration
2130  * @aqrem: the domains that have been removed from the host's AP configuration
2131  * @cdrem: the control domains that have been removed from the host's AP
2132  *	   configuration.
2133  */
2134 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
2135 					unsigned long *aprem,
2136 					unsigned long *aqrem,
2137 					unsigned long *cdrem)
2138 {
2139 	int do_hotplug = 0;
2140 
2141 	if (!bitmap_empty(aprem, AP_DEVICES)) {
2142 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,
2143 					    matrix_mdev->shadow_apcb.apm,
2144 					    aprem, AP_DEVICES);
2145 	}
2146 
2147 	if (!bitmap_empty(aqrem, AP_DOMAINS)) {
2148 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
2149 					    matrix_mdev->shadow_apcb.aqm,
2150 					    aqrem, AP_DEVICES);
2151 	}
2152 
2153 	if (!bitmap_empty(cdrem, AP_DOMAINS))
2154 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,
2155 					    matrix_mdev->shadow_apcb.adm,
2156 					    cdrem, AP_DOMAINS);
2157 
2158 	if (do_hotplug)
2159 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2160 }
2161 
2162 /**
2163  * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
2164  *			     domains and control domains that have been removed
2165  *			     from the host AP configuration and unplugs them
2166  *			     from those guests.
2167  *
2168  * @ap_remove:	bitmap specifying which adapters have been removed from the host
2169  *		config.
2170  * @aq_remove:	bitmap specifying which domains have been removed from the host
2171  *		config.
2172  * @cd_remove:	bitmap specifying which control domains have been removed from
2173  *		the host config.
2174  */
2175 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
2176 				    unsigned long *aq_remove,
2177 				    unsigned long *cd_remove)
2178 {
2179 	struct ap_matrix_mdev *matrix_mdev;
2180 	DECLARE_BITMAP(aprem, AP_DEVICES);
2181 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2182 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2183 	int do_remove = 0;
2184 
2185 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2186 		mutex_lock(&matrix_mdev->kvm->lock);
2187 		mutex_lock(&matrix_dev->mdevs_lock);
2188 
2189 		do_remove |= bitmap_and(aprem, ap_remove,
2190 					  matrix_mdev->matrix.apm,
2191 					  AP_DEVICES);
2192 		do_remove |= bitmap_and(aqrem, aq_remove,
2193 					  matrix_mdev->matrix.aqm,
2194 					  AP_DOMAINS);
2195 		do_remove |= bitmap_andnot(cdrem, cd_remove,
2196 					     matrix_mdev->matrix.adm,
2197 					     AP_DOMAINS);
2198 
2199 		if (do_remove)
2200 			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
2201 						    cdrem);
2202 
2203 		mutex_unlock(&matrix_dev->mdevs_lock);
2204 		mutex_unlock(&matrix_mdev->kvm->lock);
2205 	}
2206 }
2207 
2208 /**
2209  * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
2210  *				control domains from the host AP configuration
2211  *				by unplugging them from the guests that are
2212  *				using them.
2213  * @cur_config_info: the current host AP configuration information
2214  * @prev_config_info: the previous host AP configuration information
2215  */
2216 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
2217 				       struct ap_config_info *prev_config_info)
2218 {
2219 	int do_remove;
2220 	DECLARE_BITMAP(aprem, AP_DEVICES);
2221 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2222 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2223 
2224 	do_remove = bitmap_andnot(aprem,
2225 				  (unsigned long *)prev_config_info->apm,
2226 				  (unsigned long *)cur_config_info->apm,
2227 				  AP_DEVICES);
2228 	do_remove |= bitmap_andnot(aqrem,
2229 				   (unsigned long *)prev_config_info->aqm,
2230 				   (unsigned long *)cur_config_info->aqm,
2231 				   AP_DEVICES);
2232 	do_remove |= bitmap_andnot(cdrem,
2233 				   (unsigned long *)prev_config_info->adm,
2234 				   (unsigned long *)cur_config_info->adm,
2235 				   AP_DEVICES);
2236 
2237 	if (do_remove)
2238 		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
2239 }
2240 
2241 /**
2242  * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
2243  *				 are older than AP type 10 (CEX4).
2244  * @apm: a bitmap of the APIDs to examine
2245  * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
2246  */
2247 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
2248 {
2249 	bool apid_cleared;
2250 	struct ap_queue_status status;
2251 	unsigned long apid, apqi;
2252 	struct ap_tapq_gr2 info;
2253 
2254 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
2255 		apid_cleared = false;
2256 
2257 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
2258 			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
2259 			switch (status.response_code) {
2260 			/*
2261 			 * According to the architecture in each case
2262 			 * below, the queue's info should be filled.
2263 			 */
2264 			case AP_RESPONSE_NORMAL:
2265 			case AP_RESPONSE_RESET_IN_PROGRESS:
2266 			case AP_RESPONSE_DECONFIGURED:
2267 			case AP_RESPONSE_CHECKSTOPPED:
2268 			case AP_RESPONSE_BUSY:
2269 				/*
2270 				 * The vfio_ap device driver only
2271 				 * supports CEX4 and newer adapters, so
2272 				 * remove the APID if the adapter is
2273 				 * older than a CEX4.
2274 				 */
2275 				if (info.at < AP_DEVICE_TYPE_CEX4) {
2276 					clear_bit_inv(apid, apm);
2277 					apid_cleared = true;
2278 				}
2279 
2280 				break;
2281 
2282 			default:
2283 				/*
2284 				 * If we don't know the adapter type,
2285 				 * clear its APID since it can't be
2286 				 * determined whether the vfio_ap
2287 				 * device driver supports it.
2288 				 */
2289 				clear_bit_inv(apid, apm);
2290 				apid_cleared = true;
2291 				break;
2292 			}
2293 
2294 			/*
2295 			 * If we've already cleared the APID from the apm, there
2296 			 * is no need to continue examining the remainin AP
2297 			 * queues to determine the type of the adapter.
2298 			 */
2299 			if (apid_cleared)
2300 				continue;
2301 		}
2302 	}
2303 }
2304 
2305 /**
2306  * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
2307  *			  control domains that have been added to the host's
2308  *			  AP configuration for each matrix mdev to which they
2309  *			  are assigned.
2310  *
2311  * @apm_add: a bitmap specifying the adapters that have been added to the AP
2312  *	     configuration.
2313  * @aqm_add: a bitmap specifying the domains that have been added to the AP
2314  *	     configuration.
2315  * @adm_add: a bitmap specifying the control domains that have been added to the
2316  *	     AP configuration.
2317  */
2318 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
2319 				 unsigned long *adm_add)
2320 {
2321 	struct ap_matrix_mdev *matrix_mdev;
2322 
2323 	if (list_empty(&matrix_dev->mdev_list))
2324 		return;
2325 
2326 	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
2327 
2328 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2329 		bitmap_and(matrix_mdev->apm_add,
2330 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
2331 		bitmap_and(matrix_mdev->aqm_add,
2332 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
2333 		bitmap_and(matrix_mdev->adm_add,
2334 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
2335 	}
2336 }
2337 
2338 /**
2339  * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
2340  *			     control domains to the host AP configuration
2341  *			     by updating the bitmaps that specify what adapters,
2342  *			     domains and control domains have been added so they
2343  *			     can be hot plugged into the guest when the AP bus
2344  *			     scan completes (see vfio_ap_on_scan_complete
2345  *			     function).
2346  * @cur_config_info: the current AP configuration information
2347  * @prev_config_info: the previous AP configuration information
2348  */
2349 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
2350 				    struct ap_config_info *prev_config_info)
2351 {
2352 	bool do_add;
2353 	DECLARE_BITMAP(apm_add, AP_DEVICES);
2354 	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
2355 	DECLARE_BITMAP(adm_add, AP_DOMAINS);
2356 
2357 	do_add = bitmap_andnot(apm_add,
2358 			       (unsigned long *)cur_config_info->apm,
2359 			       (unsigned long *)prev_config_info->apm,
2360 			       AP_DEVICES);
2361 	do_add |= bitmap_andnot(aqm_add,
2362 				(unsigned long *)cur_config_info->aqm,
2363 				(unsigned long *)prev_config_info->aqm,
2364 				AP_DOMAINS);
2365 	do_add |= bitmap_andnot(adm_add,
2366 				(unsigned long *)cur_config_info->adm,
2367 				(unsigned long *)prev_config_info->adm,
2368 				AP_DOMAINS);
2369 
2370 	if (do_add)
2371 		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
2372 }
2373 
2374 /**
2375  * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
2376  *			    configuration.
2377  *
2378  * @cur_cfg_info: the current host AP configuration
2379  * @prev_cfg_info: the previous host AP configuration
2380  */
2381 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
2382 			    struct ap_config_info *prev_cfg_info)
2383 {
2384 	if (!cur_cfg_info || !prev_cfg_info)
2385 		return;
2386 
2387 	mutex_lock(&matrix_dev->guests_lock);
2388 
2389 	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
2390 	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
2391 	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
2392 
2393 	mutex_unlock(&matrix_dev->guests_lock);
2394 }
2395 
2396 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
2397 {
2398 	bool do_hotplug = false;
2399 	int filter_domains = 0;
2400 	int filter_adapters = 0;
2401 	DECLARE_BITMAP(apm, AP_DEVICES);
2402 	DECLARE_BITMAP(aqm, AP_DOMAINS);
2403 
2404 	mutex_lock(&matrix_mdev->kvm->lock);
2405 	mutex_lock(&matrix_dev->mdevs_lock);
2406 
2407 	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,
2408 				     matrix_mdev->apm_add, AP_DEVICES);
2409 	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,
2410 				    matrix_mdev->aqm_add, AP_DOMAINS);
2411 
2412 	if (filter_adapters && filter_domains)
2413 		do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev);
2414 	else if (filter_adapters)
2415 		do_hotplug |=
2416 			vfio_ap_mdev_filter_matrix(apm,
2417 						   matrix_mdev->shadow_apcb.aqm,
2418 						   matrix_mdev);
2419 	else
2420 		do_hotplug |=
2421 			vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm,
2422 						   aqm, matrix_mdev);
2423 
2424 	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,
2425 			      AP_DOMAINS))
2426 		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
2427 
2428 	if (do_hotplug)
2429 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2430 
2431 	mutex_unlock(&matrix_dev->mdevs_lock);
2432 	mutex_unlock(&matrix_mdev->kvm->lock);
2433 }
2434 
2435 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
2436 			      struct ap_config_info *old_config_info)
2437 {
2438 	struct ap_matrix_mdev *matrix_mdev;
2439 
2440 	mutex_lock(&matrix_dev->guests_lock);
2441 
2442 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2443 		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
2444 		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
2445 		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
2446 			continue;
2447 
2448 		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
2449 		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
2450 		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
2451 		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
2452 	}
2453 
2454 	mutex_unlock(&matrix_dev->guests_lock);
2455 }
2456