125 static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)  in get_request()  argument
127 	*req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);  in get_request()
128 	if (RING_FULL(&bedata->ring) ||  in get_request()
129 	    bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)  in get_request()
168 	struct pvcalls_bedata *bedata;  in pvcalls_front_event_handler()  local
177 	bedata = dev_get_drvdata(&dev->dev);  in pvcalls_front_event_handler()
178 	if (bedata == NULL) {  in pvcalls_front_event_handler()
184 	while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {  in pvcalls_front_event_handler()
185 		rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);  in pvcalls_front_event_handler()
203 			dst = (uint8_t *)&bedata->rsp[req_id] +  in pvcalls_front_event_handler()
212 			bedata->rsp[req_id].req_id = req_id;  in pvcalls_front_event_handler()
216 		bedata->ring.rsp_cons++;  in pvcalls_front_event_handler()
219 	RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);  in pvcalls_front_event_handler()
223 		wake_up(&bedata->inflight_req);  in pvcalls_front_event_handler()
230 static void pvcalls_front_destroy_active(struct pvcalls_bedata *bedata,  in pvcalls_front_destroy_active()  argument
237 	if (bedata) {  in pvcalls_front_destroy_active()
238 		spin_lock(&bedata->socket_lock);  in pvcalls_front_destroy_active()
241 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_destroy_active()
250 static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,  in pvcalls_front_free_map()  argument
253 	pvcalls_front_destroy_active(bedata, map);  in pvcalls_front_free_map()
272 	struct pvcalls_bedata *bedata;  in pvcalls_front_socket()  local
292 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_socket()
300 	spin_lock(&bedata->socket_lock);  in pvcalls_front_socket()
302 	ret = get_request(bedata, &req_id);  in pvcalls_front_socket()
305 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_socket()
317 	list_add_tail(&map->list, &bedata->socket_mappings);  in pvcalls_front_socket()
319 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_socket()
327 	bedata->ring.req_prod_pvt++;  in pvcalls_front_socket()
328 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_socket()
329 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_socket()
331 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_socket()
333 	wait_event(bedata->inflight_req,  in pvcalls_front_socket()
334 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_socket()
338 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_socket()
339 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_socket()
425 	struct pvcalls_bedata *bedata;  in pvcalls_front_connect()  local
438 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_connect()
451 	spin_lock(&bedata->socket_lock);  in pvcalls_front_connect()
452 	ret = get_request(bedata, &req_id);  in pvcalls_front_connect()
454 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_connect()
460 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_connect()
472 	bedata->ring.req_prod_pvt++;  in pvcalls_front_connect()
473 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_connect()
474 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_connect()
477 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_connect()
479 	wait_event(bedata->inflight_req,  in pvcalls_front_connect()
480 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_connect()
484 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_connect()
485 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_connect()
672 	struct pvcalls_bedata *bedata;  in pvcalls_front_bind()  local
683 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_bind()
685 	spin_lock(&bedata->socket_lock);  in pvcalls_front_bind()
686 	ret = get_request(bedata, &req_id);  in pvcalls_front_bind()
688 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_bind()
692 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_bind()
704 	bedata->ring.req_prod_pvt++;  in pvcalls_front_bind()
705 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_bind()
706 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_bind()
708 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_bind()
710 	wait_event(bedata->inflight_req,  in pvcalls_front_bind()
711 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_bind()
715 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_bind()
716 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_bind()
725 	struct pvcalls_bedata *bedata;  in pvcalls_front_listen()  local
733 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_listen()
740 	spin_lock(&bedata->socket_lock);  in pvcalls_front_listen()
741 	ret = get_request(bedata, &req_id);  in pvcalls_front_listen()
743 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_listen()
747 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_listen()
753 	bedata->ring.req_prod_pvt++;  in pvcalls_front_listen()
754 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_listen()
755 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_listen()
757 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_listen()
759 	wait_event(bedata->inflight_req,  in pvcalls_front_listen()
760 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_listen()
764 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_listen()
765 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_listen()
774 	struct pvcalls_bedata *bedata;  in pvcalls_front_accept()  local
784 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_accept()
800 		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {  in pvcalls_front_accept()
841 	spin_lock(&bedata->socket_lock);  in pvcalls_front_accept()
842 	ret = get_request(bedata, &req_id);  in pvcalls_front_accept()
846 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_accept()
847 		pvcalls_front_free_map(bedata, map2);  in pvcalls_front_accept()
852 	list_add_tail(&map2->list, &bedata->socket_mappings);  in pvcalls_front_accept()
854 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_accept()
863 	bedata->ring.req_prod_pvt++;  in pvcalls_front_accept()
864 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_accept()
865 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_accept()
867 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_accept()
875 	if (wait_event_interruptible(bedata->inflight_req,  in pvcalls_front_accept()
876 		READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {  in pvcalls_front_accept()
887 		bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_accept()
891 		pvcalls_front_free_map(bedata, map2);  in pvcalls_front_accept()
897 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_accept()
898 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_accept()
909 					       struct pvcalls_bedata *bedata,  in pvcalls_front_poll_passive()  argument
921 		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id)  in pvcalls_front_poll_passive()
940 		poll_wait(file, &bedata->inflight_req, wait);  in pvcalls_front_poll_passive()
944 	spin_lock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
945 	ret = get_request(bedata, &req_id);  in pvcalls_front_poll_passive()
947 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
950 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_poll_passive()
955 	bedata->ring.req_prod_pvt++;  in pvcalls_front_poll_passive()
956 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_poll_passive()
957 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
959 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_poll_passive()
961 	poll_wait(file, &bedata->inflight_req, wait);  in pvcalls_front_poll_passive()
966 					      struct pvcalls_bedata *bedata,  in pvcalls_front_poll_active()  argument
991 	struct pvcalls_bedata *bedata;  in pvcalls_front_poll()  local
998 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_poll()
1001 		ret = pvcalls_front_poll_active(file, bedata, map, wait);  in pvcalls_front_poll()
1003 		ret = pvcalls_front_poll_passive(file, bedata, map, wait);  in pvcalls_front_poll()
1010 	struct pvcalls_bedata *bedata;  in pvcalls_front_release()  local
1025 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_release()
1027 	spin_lock(&bedata->socket_lock);  in pvcalls_front_release()
1028 	ret = get_request(bedata, &req_id);  in pvcalls_front_release()
1030 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1036 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_release()
1041 	bedata->ring.req_prod_pvt++;  in pvcalls_front_release()
1042 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_release()
1043 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1045 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_release()
1047 	wait_event(bedata->inflight_req,  in pvcalls_front_release()
1048 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_release()
1067 		pvcalls_front_free_map(bedata, map);  in pvcalls_front_release()
1069 		wake_up(&bedata->inflight_req);  in pvcalls_front_release()
1075 		spin_lock(&bedata->socket_lock);  in pvcalls_front_release()
1077 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1080 			pvcalls_front_free_map(bedata,  in pvcalls_front_release()
1085 	WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);  in pvcalls_front_release()
1098 	struct pvcalls_bedata *bedata;  in pvcalls_front_remove()  local
1101 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_remove()
1104 	if (bedata->irq >= 0)  in pvcalls_front_remove()
1105 		unbind_from_irqhandler(bedata->irq, dev);  in pvcalls_front_remove()
1107 	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {  in pvcalls_front_remove()
1118 	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {  in pvcalls_front_remove()
1121 			pvcalls_front_free_map(bedata, map);  in pvcalls_front_remove()
1127 	if (bedata->ref != -1)  in pvcalls_front_remove()
1128 		gnttab_end_foreign_access(bedata->ref, NULL);  in pvcalls_front_remove()
1129 	kfree(bedata->ring.sring);  in pvcalls_front_remove()
1130 	kfree(bedata);  in pvcalls_front_remove()
1143 	struct pvcalls_bedata *bedata = NULL;  in pvcalls_front_probe()  local
1172 	bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);  in pvcalls_front_probe()
1173 	if (!bedata)  in pvcalls_front_probe()
1176 	dev_set_drvdata(&dev->dev, bedata);  in pvcalls_front_probe()
1178 	init_waitqueue_head(&bedata->inflight_req);  in pvcalls_front_probe()
1179 	INIT_LIST_HEAD(&bedata->socket_mappings);  in pvcalls_front_probe()
1180 	spin_lock_init(&bedata->socket_lock);  in pvcalls_front_probe()
1181 	bedata->irq = -1;  in pvcalls_front_probe()
1182 	bedata->ref = -1;  in pvcalls_front_probe()
1185 		bedata->rsp[i].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_probe()
1192 	FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);  in pvcalls_front_probe()
1198 	bedata->irq = bind_evtchn_to_irqhandler(evtchn,  in pvcalls_front_probe()
1201 	if (bedata->irq < 0) {  in pvcalls_front_probe()
1202 		ret = bedata->irq;  in pvcalls_front_probe()
1212 	bedata->ref = ret;  in pvcalls_front_probe()
1213 	gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,  in pvcalls_front_probe()
1225 	ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);  in pvcalls_front_probe()