39 static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q);
189 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)  in get_update_locks_for_queue()  argument
192 	if (q->matrix_mdev && q->matrix_mdev->kvm)  in get_update_locks_for_queue()
193 		mutex_lock(&q->matrix_mdev->kvm->lock);  in get_update_locks_for_queue()
210 	struct vfio_ap_queue *q;  in vfio_ap_mdev_get_queue()  local
212 	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,  in vfio_ap_mdev_get_queue()
214 		if (q && q->apqn == apqn)  in vfio_ap_mdev_get_queue()
215 			return q;  in vfio_ap_mdev_get_queue()
269 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)  in vfio_ap_free_aqic_resources()  argument
271 	if (!q)  in vfio_ap_free_aqic_resources()
273 	if (q->saved_isc != VFIO_AP_ISC_INVALID &&  in vfio_ap_free_aqic_resources()
274 	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {  in vfio_ap_free_aqic_resources()
275 		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);  in vfio_ap_free_aqic_resources()
276 		q->saved_isc = VFIO_AP_ISC_INVALID;  in vfio_ap_free_aqic_resources()
278 	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {  in vfio_ap_free_aqic_resources()
279 		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);  in vfio_ap_free_aqic_resources()
280 		q->saved_iova = 0;  in vfio_ap_free_aqic_resources()
302 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)  in vfio_ap_irq_disable()  argument
309 		status = ap_aqic(q->apqn, aqic_gisa, 0);  in vfio_ap_irq_disable()
313 			vfio_ap_wait_for_irqclear(q->apqn);  in vfio_ap_irq_disable()
334 	vfio_ap_free_aqic_resources(q);  in vfio_ap_irq_disable()
405 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,  in vfio_ap_irq_enable()  argument
422 				 __func__, &nib, q->apqn);  in vfio_ap_irq_enable()
428 	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,  in vfio_ap_irq_enable()
436 				 __func__, ret, &nib, q->apqn);  in vfio_ap_irq_enable()
442 	kvm = q->matrix_mdev->kvm;  in vfio_ap_irq_enable()
451 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);  in vfio_ap_irq_enable()
459 				 __func__, nisc, isc, q->apqn);  in vfio_ap_irq_enable()
461 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);  in vfio_ap_irq_enable()
470 	status = ap_aqic(q->apqn, aqic_gisa, h_nib);  in vfio_ap_irq_enable()
474 		vfio_ap_free_aqic_resources(q);  in vfio_ap_irq_enable()
475 		q->saved_iova = nib;  in vfio_ap_irq_enable()
476 		q->saved_isc = isc;  in vfio_ap_irq_enable()
480 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);  in vfio_ap_irq_enable()
484 		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,  in vfio_ap_irq_enable()
486 		vfio_ap_irq_disable(q);  in vfio_ap_irq_enable()
497 				 q->apqn);  in vfio_ap_irq_enable()
571 	struct vfio_ap_queue *q;  in handle_pqap()  local
607 	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);  in handle_pqap()
608 	if (!q) {  in handle_pqap()
619 		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);  in handle_pqap()
621 		qstatus = vfio_ap_irq_disable(q);  in handle_pqap()
687 	struct vfio_ap_queue *q;  in vfio_ap_mdev_filter_matrix()  local
716 			q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);  in vfio_ap_mdev_filter_matrix()
717 			if (!q || q->reset_status.response_code) {  in vfio_ap_mdev_filter_matrix()
779 				    struct vfio_ap_queue *q)  in vfio_ap_mdev_link_queue()  argument
781 	if (q) {  in vfio_ap_mdev_link_queue()
782 		q->matrix_mdev = matrix_mdev;  in vfio_ap_mdev_link_queue()
783 		hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);  in vfio_ap_mdev_link_queue()
789 	struct vfio_ap_queue *q;  in vfio_ap_mdev_link_apqn()  local
791 	q = vfio_ap_find_queue(apqn);  in vfio_ap_mdev_link_apqn()
792 	vfio_ap_mdev_link_queue(matrix_mdev, q);  in vfio_ap_mdev_link_apqn()
795 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)  in vfio_ap_unlink_queue_fr_mdev()  argument
797 	hash_del(&q->mdev_qnode);  in vfio_ap_unlink_queue_fr_mdev()
800 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)  in vfio_ap_unlink_mdev_fr_queue()  argument
802 	q->matrix_mdev = NULL;  in vfio_ap_unlink_mdev_fr_queue()
807 	struct vfio_ap_queue *q;  in vfio_ap_mdev_unlink_fr_queues()  local
813 			q = vfio_ap_mdev_get_queue(matrix_mdev,  in vfio_ap_mdev_unlink_fr_queues()
815 			if (q)  in vfio_ap_mdev_unlink_fr_queues()
816 				q->matrix_mdev = NULL;  in vfio_ap_mdev_unlink_fr_queues()
943 	struct vfio_ap_queue *q;  in collect_queues_to_reset()  local
947 		q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));  in collect_queues_to_reset()
948 		if (q)  in collect_queues_to_reset()
949 			list_add_tail(&q->reset_qnode, qlist);  in collect_queues_to_reset()
1068 	struct vfio_ap_queue *q = NULL;  in vfio_ap_unlink_apqn_fr_mdev()  local
1070 	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));  in vfio_ap_unlink_apqn_fr_mdev()
1072 	if (q)  in vfio_ap_unlink_apqn_fr_mdev()
1073 		vfio_ap_unlink_queue_fr_mdev(q);  in vfio_ap_unlink_apqn_fr_mdev()
1075 	return q;  in vfio_ap_unlink_apqn_fr_mdev()
1092 	struct vfio_ap_queue *q;  in vfio_ap_mdev_unlink_adapter()  local
1095 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);  in vfio_ap_mdev_unlink_adapter()
1097 		if (q && qlist) {  in vfio_ap_mdev_unlink_adapter()
1100 				list_add_tail(&q->reset_qnode, qlist);  in vfio_ap_mdev_unlink_adapter()
1108 	struct vfio_ap_queue *q, *tmpq;  in vfio_ap_mdev_hot_unplug_adapter()  local
1121 	list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {  in vfio_ap_mdev_hot_unplug_adapter()
1122 		vfio_ap_unlink_mdev_fr_queue(q);  in vfio_ap_mdev_hot_unplug_adapter()
1123 		list_del(&q->reset_qnode);  in vfio_ap_mdev_hot_unplug_adapter()
1274 	struct vfio_ap_queue *q;  in vfio_ap_mdev_unlink_domain()  local
1277 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);  in vfio_ap_mdev_unlink_domain()
1279 		if (q && qlist) {  in vfio_ap_mdev_unlink_domain()
1282 				list_add_tail(&q->reset_qnode, qlist);  in vfio_ap_mdev_unlink_domain()
1290 	struct vfio_ap_queue *q, *tmpq;  in vfio_ap_mdev_hot_unplug_domain()  local
1303 	list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {  in vfio_ap_mdev_hot_unplug_domain()
1304 		vfio_ap_unlink_mdev_fr_queue(q);  in vfio_ap_mdev_hot_unplug_domain()
1305 		list_del(&q->reset_qnode);  in vfio_ap_mdev_hot_unplug_domain()
1621 	struct vfio_ap_queue *q;  in unmap_iova()  local
1624 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {  in unmap_iova()
1625 		if (q->saved_iova >= iova && q->saved_iova < iova + length)  in unmap_iova()
1626 			vfio_ap_irq_disable(q);  in unmap_iova()
1672 	struct vfio_ap_queue *q = NULL;  in vfio_ap_find_queue()  local
1679 		q = dev_get_drvdata(&queue->ap_dev.device);  in vfio_ap_find_queue()
1683 	return q;  in vfio_ap_find_queue()
1720 	struct vfio_ap_queue *q;  in apq_reset_check()  local
1722 	q = container_of(reset_work, struct vfio_ap_queue, reset_work);  in apq_reset_check()
1723 	memcpy(&status, &q->reset_status, sizeof(status));  in apq_reset_check()
1727 		status = ap_tapq(q->apqn, NULL);  in apq_reset_check()
1728 		ret = apq_status_check(q->apqn, &status);  in apq_reset_check()
1733 					      AP_QID_CARD(q->apqn),  in apq_reset_check()
1734 					      AP_QID_QUEUE(q->apqn),  in apq_reset_check()
1739 			if (q->reset_status.response_code == AP_RESPONSE_RESET_IN_PROGRESS ||  in apq_reset_check()
1740 			    q->reset_status.response_code == AP_RESPONSE_BUSY ||  in apq_reset_check()
1741 			    q->reset_status.response_code == AP_RESPONSE_STATE_CHANGE_IN_PROGRESS ||  in apq_reset_check()
1743 				status = ap_zapq(q->apqn, 0);  in apq_reset_check()
1744 				memcpy(&q->reset_status, &status, sizeof(status));  in apq_reset_check()
1754 				q->reset_status.response_code = 0;  in apq_reset_check()
1755 			if (q->saved_isc != VFIO_AP_ISC_INVALID)  in apq_reset_check()
1756 				vfio_ap_free_aqic_resources(q);  in apq_reset_check()
1762 static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)  in vfio_ap_mdev_reset_queue()  argument
1766 	if (!q)  in vfio_ap_mdev_reset_queue()
1768 	status = ap_zapq(q->apqn, 0);  in vfio_ap_mdev_reset_queue()
1769 	memcpy(&q->reset_status, &status, sizeof(status));  in vfio_ap_mdev_reset_queue()
1778 		queue_work(system_long_wq, &q->reset_work);  in vfio_ap_mdev_reset_queue()
1786 		q->reset_status.response_code = 0;  in vfio_ap_mdev_reset_queue()
1787 		vfio_ap_free_aqic_resources(q);  in vfio_ap_mdev_reset_queue()
1792 		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),  in vfio_ap_mdev_reset_queue()
1800 	struct vfio_ap_queue *q;  in vfio_ap_mdev_reset_queues()  local
1802 	hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode)  in vfio_ap_mdev_reset_queues()
1803 		vfio_ap_mdev_reset_queue(q);  in vfio_ap_mdev_reset_queues()
1805 	hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) {  in vfio_ap_mdev_reset_queues()
1806 		flush_work(&q->reset_work);  in vfio_ap_mdev_reset_queues()
1808 		if (q->reset_status.response_code)  in vfio_ap_mdev_reset_queues()
1818 	struct vfio_ap_queue *q;  in vfio_ap_mdev_reset_qlist()  local
1820 	list_for_each_entry(q, qlist, reset_qnode)  in vfio_ap_mdev_reset_qlist()
1821 		vfio_ap_mdev_reset_queue(q);  in vfio_ap_mdev_reset_qlist()
1823 	list_for_each_entry(q, qlist, reset_qnode) {  in vfio_ap_mdev_reset_qlist()
1824 		flush_work(&q->reset_work);  in vfio_ap_mdev_reset_qlist()
1826 		if (q->reset_status.response_code)  in vfio_ap_mdev_reset_qlist()
2026 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)  in vfio_ap_mdev_for_queue()  argument
2029 	unsigned long apid = AP_QID_CARD(q->apqn);  in vfio_ap_mdev_for_queue()
2030 	unsigned long apqi = AP_QID_QUEUE(q->apqn);  in vfio_ap_mdev_for_queue()
2046 	struct vfio_ap_queue *q;  in status_show()  local
2052 	q = dev_get_drvdata(&apdev->device);  in status_show()
2053 	matrix_mdev = vfio_ap_mdev_for_queue(q);  in status_show()
2060 		apid = AP_QID_CARD(q->apqn);  in status_show()
2061 		apqi = AP_QID_QUEUE(q->apqn);  in status_show()
2154 	struct vfio_ap_queue *q;  in vfio_ap_mdev_probe_queue()  local
2162 	q = kzalloc(sizeof(*q), GFP_KERNEL);  in vfio_ap_mdev_probe_queue()
2163 	if (!q) {  in vfio_ap_mdev_probe_queue()
2168 	q->apqn = to_ap_queue(&apdev->device)->qid;  in vfio_ap_mdev_probe_queue()
2169 	q->saved_isc = VFIO_AP_ISC_INVALID;  in vfio_ap_mdev_probe_queue()
2170 	memset(&q->reset_status, 0, sizeof(q->reset_status));  in vfio_ap_mdev_probe_queue()
2171 	INIT_WORK(&q->reset_work, apq_reset_check);  in vfio_ap_mdev_probe_queue()
2172 	matrix_mdev = get_update_locks_by_apqn(q->apqn);  in vfio_ap_mdev_probe_queue()
2175 		vfio_ap_mdev_link_queue(matrix_mdev, q);  in vfio_ap_mdev_probe_queue()
2195 	dev_set_drvdata(&apdev->device, q);  in vfio_ap_mdev_probe_queue()
2208 	struct vfio_ap_queue *q;  in vfio_ap_mdev_remove_queue()  local
2212 	q = dev_get_drvdata(&apdev->device);  in vfio_ap_mdev_remove_queue()
2213 	get_update_locks_for_queue(q);  in vfio_ap_mdev_remove_queue()
2214 	matrix_mdev = q->matrix_mdev;  in vfio_ap_mdev_remove_queue()
2215 	apid = AP_QID_CARD(q->apqn);  in vfio_ap_mdev_remove_queue()
2216 	apqi = AP_QID_QUEUE(q->apqn);  in vfio_ap_mdev_remove_queue()
2241 		vfio_ap_mdev_reset_queue(q);  in vfio_ap_mdev_remove_queue()
2242 		flush_work(&q->reset_work);  in vfio_ap_mdev_remove_queue()
2247 		vfio_ap_unlink_queue_fr_mdev(q);  in vfio_ap_mdev_remove_queue()
2250 	kfree(q);  in vfio_ap_mdev_remove_queue()