Lines Matching refs:queue
218 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, in xennet_get_rx_skb() argument
222 struct sk_buff *skb = queue->rx_skbs[i]; in xennet_get_rx_skb()
223 queue->rx_skbs[i] = NULL; in xennet_get_rx_skb()
227 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, in xennet_get_rx_ref() argument
231 grant_ref_t ref = queue->grant_rx_ref[i]; in xennet_get_rx_ref()
232 queue->grant_rx_ref[i] = INVALID_GRANT_REF; in xennet_get_rx_ref()
248 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); in rx_refill_timeout() local
249 napi_schedule(&queue->napi); in rx_refill_timeout()
252 static int netfront_tx_slot_available(struct netfront_queue *queue) in netfront_tx_slot_available() argument
254 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < in netfront_tx_slot_available()
258 static void xennet_maybe_wake_tx(struct netfront_queue *queue) in xennet_maybe_wake_tx() argument
260 struct net_device *dev = queue->info->netdev; in xennet_maybe_wake_tx()
261 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); in xennet_maybe_wake_tx()
264 netfront_tx_slot_available(queue) && in xennet_maybe_wake_tx()
266 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_maybe_wake_tx()
270 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) in xennet_alloc_one_rx_buffer() argument
275 skb = __netdev_alloc_skb(queue->info->netdev, in xennet_alloc_one_rx_buffer()
281 page = page_pool_alloc_pages(queue->page_pool, in xennet_alloc_one_rx_buffer()
292 skb->dev = queue->info->netdev; in xennet_alloc_one_rx_buffer()
298 static void xennet_alloc_rx_buffers(struct netfront_queue *queue) in xennet_alloc_rx_buffers() argument
300 RING_IDX req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
304 if (unlikely(!netif_carrier_ok(queue->info->netdev))) in xennet_alloc_rx_buffers()
307 for (req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
308 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; in xennet_alloc_rx_buffers()
316 skb = xennet_alloc_one_rx_buffer(queue); in xennet_alloc_rx_buffers()
324 BUG_ON(queue->rx_skbs[id]); in xennet_alloc_rx_buffers()
325 queue->rx_skbs[id] = skb; in xennet_alloc_rx_buffers()
327 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); in xennet_alloc_rx_buffers()
329 queue->grant_rx_ref[id] = ref; in xennet_alloc_rx_buffers()
333 req = RING_GET_REQUEST(&queue->rx, req_prod); in xennet_alloc_rx_buffers()
335 queue->info->xbdev->otherend_id, in xennet_alloc_rx_buffers()
342 queue->rx.req_prod_pvt = req_prod; in xennet_alloc_rx_buffers()
349 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || in xennet_alloc_rx_buffers()
351 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); in xennet_alloc_rx_buffers()
355 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); in xennet_alloc_rx_buffers()
357 notify_remote_via_irq(queue->rx_irq); in xennet_alloc_rx_buffers()
365 struct netfront_queue *queue = NULL; in xennet_open() local
371 queue = &np->queues[i]; in xennet_open()
372 napi_enable(&queue->napi); in xennet_open()
374 spin_lock_bh(&queue->rx_lock); in xennet_open()
376 xennet_alloc_rx_buffers(queue); in xennet_open()
377 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; in xennet_open()
378 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) in xennet_open()
379 napi_schedule(&queue->napi); in xennet_open()
381 spin_unlock_bh(&queue->rx_lock); in xennet_open()
389 static bool xennet_tx_buf_gc(struct netfront_queue *queue) in xennet_tx_buf_gc() argument
396 const struct device *dev = &queue->info->netdev->dev; in xennet_tx_buf_gc()
398 BUG_ON(!netif_carrier_ok(queue->info->netdev)); in xennet_tx_buf_gc()
401 prod = queue->tx.sring->rsp_prod; in xennet_tx_buf_gc()
402 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { in xennet_tx_buf_gc()
404 prod - queue->tx.rsp_cons); in xennet_tx_buf_gc()
409 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { in xennet_tx_buf_gc()
414 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); in xennet_tx_buf_gc()
419 if (id >= RING_SIZE(&queue->tx)) { in xennet_tx_buf_gc()
425 if (queue->tx_link[id] != TX_PENDING) { in xennet_tx_buf_gc()
431 queue->tx_link[id] = TX_LINK_NONE; in xennet_tx_buf_gc()
432 skb = queue->tx_skbs[id]; in xennet_tx_buf_gc()
433 queue->tx_skbs[id] = NULL; in xennet_tx_buf_gc()
435 queue->grant_tx_ref[id]))) { in xennet_tx_buf_gc()
441 &queue->gref_tx_head, queue->grant_tx_ref[id]); in xennet_tx_buf_gc()
442 queue->grant_tx_ref[id] = INVALID_GRANT_REF; in xennet_tx_buf_gc()
443 queue->grant_tx_page[id] = NULL; in xennet_tx_buf_gc()
444 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); in xennet_tx_buf_gc()
448 queue->tx.rsp_cons = prod; in xennet_tx_buf_gc()
450 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); in xennet_tx_buf_gc()
453 xennet_maybe_wake_tx(queue); in xennet_tx_buf_gc()
458 queue->info->broken = true; in xennet_tx_buf_gc()
465 struct netfront_queue *queue; member
482 struct netfront_queue *queue = info->queue; in xennet_tx_setup_grant() local
485 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); in xennet_tx_setup_grant()
486 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_tx_setup_grant()
487 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); in xennet_tx_setup_grant()
490 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, in xennet_tx_setup_grant()
493 queue->tx_skbs[id] = skb; in xennet_tx_setup_grant()
494 queue->grant_tx_page[id] = page; in xennet_tx_setup_grant()
495 queue->grant_tx_ref[id] = ref; in xennet_tx_setup_grant()
509 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); in xennet_tx_setup_grant()
603 static void xennet_mark_tx_pending(struct netfront_queue *queue) in xennet_mark_tx_pending() argument
607 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != in xennet_mark_tx_pending()
609 queue->tx_link[i] = TX_PENDING; in xennet_mark_tx_pending()
613 struct netfront_queue *queue, in xennet_xdp_xmit_one() argument
619 .queue = queue, in xennet_xdp_xmit_one()
629 xennet_mark_tx_pending(queue); in xennet_xdp_xmit_one()
631 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_xdp_xmit_one()
633 notify_remote_via_irq(queue->tx_irq); in xennet_xdp_xmit_one()
640 xennet_tx_buf_gc(queue); in xennet_xdp_xmit_one()
650 struct netfront_queue *queue = NULL; in xennet_xdp_xmit() local
660 queue = &np->queues[smp_processor_id() % num_queues]; in xennet_xdp_xmit()
662 spin_lock_irqsave(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
668 if (xennet_xdp_xmit_one(dev, queue, xdpf)) in xennet_xdp_xmit()
672 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
719 struct netfront_queue *queue = NULL; in xennet_start_xmit() local
732 queue = &np->queues[queue_index]; in xennet_start_xmit()
774 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_start_xmit()
779 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
784 info.queue = queue; in xennet_start_xmit()
808 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_start_xmit()
840 xennet_mark_tx_pending(queue); in xennet_start_xmit()
842 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_start_xmit()
844 notify_remote_via_irq(queue->tx_irq); in xennet_start_xmit()
852 xennet_tx_buf_gc(queue); in xennet_start_xmit()
854 if (!netfront_tx_slot_available(queue)) in xennet_start_xmit()
855 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_start_xmit()
857 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
872 struct netfront_queue *queue; in xennet_close() local
875 queue = &np->queues[i]; in xennet_close()
876 napi_disable(&queue->napi); in xennet_close()
886 struct netfront_queue *queue = &info->queues[i]; in xennet_destroy_queues() local
889 napi_disable(&queue->napi); in xennet_destroy_queues()
890 netif_napi_del(&queue->napi); in xennet_destroy_queues()
903 static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val) in xennet_set_rx_rsp_cons() argument
907 spin_lock_irqsave(&queue->rx_cons_lock, flags); in xennet_set_rx_rsp_cons()
908 queue->rx.rsp_cons = val; in xennet_set_rx_rsp_cons()
909 queue->rx_rsp_unconsumed = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx); in xennet_set_rx_rsp_cons()
910 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_set_rx_rsp_cons()
913 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, in xennet_move_rx_slot() argument
916 int new = xennet_rxidx(queue->rx.req_prod_pvt); in xennet_move_rx_slot()
918 BUG_ON(queue->rx_skbs[new]); in xennet_move_rx_slot()
919 queue->rx_skbs[new] = skb; in xennet_move_rx_slot()
920 queue->grant_rx_ref[new] = ref; in xennet_move_rx_slot()
921 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; in xennet_move_rx_slot()
922 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; in xennet_move_rx_slot()
923 queue->rx.req_prod_pvt++; in xennet_move_rx_slot()
926 static int xennet_get_extras(struct netfront_queue *queue, in xennet_get_extras() argument
932 struct device *dev = &queue->info->netdev->dev; in xennet_get_extras()
933 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_extras()
947 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); in xennet_get_extras()
959 skb = xennet_get_rx_skb(queue, cons); in xennet_get_extras()
960 ref = xennet_get_rx_ref(queue, cons); in xennet_get_extras()
961 xennet_move_rx_slot(queue, skb, ref); in xennet_get_extras()
964 xennet_set_rx_rsp_cons(queue, cons); in xennet_get_extras()
968 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, in xennet_run_xdp() argument
978 &queue->xdp_rxq); in xennet_run_xdp()
987 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); in xennet_run_xdp()
991 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
995 err = xdp_do_redirect(queue->info->netdev, xdp, prog); in xennet_run_xdp()
998 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
1005 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
1009 bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act); in xennet_run_xdp()
1015 static int xennet_get_responses(struct netfront_queue *queue, in xennet_get_responses() argument
1022 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_responses()
1023 struct sk_buff *skb = xennet_get_rx_skb(queue, cons); in xennet_get_responses()
1025 grant_ref_t ref = xennet_get_rx_ref(queue, cons); in xennet_get_responses()
1026 struct device *dev = &queue->info->netdev->dev; in xennet_get_responses()
1034 err = xennet_get_extras(queue, extras, rp); in xennet_get_responses()
1043 cons = queue->rx.rsp_cons; in xennet_get_responses()
1065 xennet_move_rx_slot(queue, skb, ref); in xennet_get_responses()
1073 queue->info->broken = true; in xennet_get_responses()
1078 gnttab_release_grant_reference(&queue->gref_rx_head, ref); in xennet_get_responses()
1081 xdp_prog = rcu_dereference(queue->xdp_prog); in xennet_get_responses()
1085 verdict = xennet_run_xdp(queue, in xennet_get_responses()
1110 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); in xennet_get_responses()
1112 skb = xennet_get_rx_skb(queue, cons + slots); in xennet_get_responses()
1113 ref = xennet_get_rx_ref(queue, cons + slots); in xennet_get_responses()
1124 xennet_set_rx_rsp_cons(queue, cons + slots); in xennet_get_responses()
1158 static int xennet_fill_frags(struct netfront_queue *queue, in xennet_fill_frags() argument
1162 RING_IDX cons = queue->rx.rsp_cons; in xennet_fill_frags()
1169 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); in xennet_fill_frags()
1178 xennet_set_rx_rsp_cons(queue, in xennet_fill_frags()
1192 xennet_set_rx_rsp_cons(queue, cons); in xennet_fill_frags()
1221 static int handle_incoming_queue(struct netfront_queue *queue, in handle_incoming_queue() argument
1224 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); in handle_incoming_queue()
1235 skb->protocol = eth_type_trans(skb, queue->info->netdev); in handle_incoming_queue()
1238 if (checksum_setup(queue->info->netdev, skb)) { in handle_incoming_queue()
1241 queue->info->netdev->stats.rx_errors++; in handle_incoming_queue()
1251 napi_gro_receive(&queue->napi, skb); in handle_incoming_queue()
1259 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); in xennet_poll() local
1260 struct net_device *dev = queue->info->netdev; in xennet_poll()
1273 spin_lock(&queue->rx_lock); in xennet_poll()
1279 rp = queue->rx.sring->rsp_prod; in xennet_poll()
1280 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { in xennet_poll()
1282 rp - queue->rx.rsp_cons); in xennet_poll()
1283 queue->info->broken = true; in xennet_poll()
1284 spin_unlock(&queue->rx_lock); in xennet_poll()
1289 i = queue->rx.rsp_cons; in xennet_poll()
1292 RING_COPY_RESPONSE(&queue->rx, i, rx); in xennet_poll()
1295 err = xennet_get_responses(queue, &rinfo, rp, &tmpq, in xennet_poll()
1299 if (queue->info->broken) { in xennet_poll()
1300 spin_unlock(&queue->rx_lock); in xennet_poll()
1307 i = queue->rx.rsp_cons; in xennet_poll()
1319 xennet_set_rx_rsp_cons(queue, in xennet_poll()
1320 queue->rx.rsp_cons + in xennet_poll()
1335 if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) in xennet_poll()
1345 i = queue->rx.rsp_cons + 1; in xennet_poll()
1346 xennet_set_rx_rsp_cons(queue, i); in xennet_poll()
1354 work_done -= handle_incoming_queue(queue, &rxq); in xennet_poll()
1356 xennet_alloc_rx_buffers(queue); in xennet_poll()
1363 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); in xennet_poll()
1368 spin_unlock(&queue->rx_lock); in xennet_poll()
1417 static void xennet_release_tx_bufs(struct netfront_queue *queue) in xennet_release_tx_bufs() argument
1424 if (!queue->tx_skbs[i]) in xennet_release_tx_bufs()
1427 skb = queue->tx_skbs[i]; in xennet_release_tx_bufs()
1428 queue->tx_skbs[i] = NULL; in xennet_release_tx_bufs()
1429 get_page(queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1430 gnttab_end_foreign_access(queue->grant_tx_ref[i], in xennet_release_tx_bufs()
1431 queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1432 queue->grant_tx_page[i] = NULL; in xennet_release_tx_bufs()
1433 queue->grant_tx_ref[i] = INVALID_GRANT_REF; in xennet_release_tx_bufs()
1434 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); in xennet_release_tx_bufs()
1439 static void xennet_release_rx_bufs(struct netfront_queue *queue) in xennet_release_rx_bufs() argument
1443 spin_lock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1449 skb = queue->rx_skbs[id]; in xennet_release_rx_bufs()
1453 ref = queue->grant_rx_ref[id]; in xennet_release_rx_bufs()
1464 queue->grant_rx_ref[id] = INVALID_GRANT_REF; in xennet_release_rx_bufs()
1469 spin_unlock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1508 static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi) in xennet_handle_tx() argument
1512 if (unlikely(queue->info->broken)) in xennet_handle_tx()
1515 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_handle_tx()
1516 if (xennet_tx_buf_gc(queue)) in xennet_handle_tx()
1518 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_handle_tx()
1533 static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi) in xennet_handle_rx() argument
1538 if (unlikely(queue->info->broken)) in xennet_handle_rx()
1541 spin_lock_irqsave(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1542 work_queued = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx); in xennet_handle_rx()
1543 if (work_queued > queue->rx_rsp_unconsumed) { in xennet_handle_rx()
1544 queue->rx_rsp_unconsumed = work_queued; in xennet_handle_rx()
1546 } else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) { in xennet_handle_rx()
1547 const struct device *dev = &queue->info->netdev->dev; in xennet_handle_rx()
1549 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1552 queue->info->broken = true; in xennet_handle_rx()
1555 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1557 if (likely(netif_carrier_ok(queue->info->netdev) && work_queued)) in xennet_handle_rx()
1558 napi_schedule(&queue->napi); in xennet_handle_rx()
1817 struct netfront_queue *queue = &info->queues[i]; in xennet_disconnect_backend() local
1819 del_timer_sync(&queue->rx_refill_timer); in xennet_disconnect_backend()
1821 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) in xennet_disconnect_backend()
1822 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1823 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { in xennet_disconnect_backend()
1824 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1825 unbind_from_irqhandler(queue->rx_irq, queue); in xennet_disconnect_backend()
1827 queue->tx_evtchn = queue->rx_evtchn = 0; in xennet_disconnect_backend()
1828 queue->tx_irq = queue->rx_irq = 0; in xennet_disconnect_backend()
1831 napi_synchronize(&queue->napi); in xennet_disconnect_backend()
1833 xennet_release_tx_bufs(queue); in xennet_disconnect_backend()
1834 xennet_release_rx_bufs(queue); in xennet_disconnect_backend()
1835 gnttab_free_grant_references(queue->gref_tx_head); in xennet_disconnect_backend()
1836 gnttab_free_grant_references(queue->gref_rx_head); in xennet_disconnect_backend()
1839 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); in xennet_disconnect_backend()
1840 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); in xennet_disconnect_backend()
1842 queue->tx_ring_ref = INVALID_GRANT_REF; in xennet_disconnect_backend()
1843 queue->rx_ring_ref = INVALID_GRANT_REF; in xennet_disconnect_backend()
1844 queue->tx.sring = NULL; in xennet_disconnect_backend()
1845 queue->rx.sring = NULL; in xennet_disconnect_backend()
1847 page_pool_destroy(queue->page_pool); in xennet_disconnect_backend()
1899 static int setup_netfront_single(struct netfront_queue *queue) in setup_netfront_single() argument
1903 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_single()
1907 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, in setup_netfront_single()
1909 queue->info->netdev->name, in setup_netfront_single()
1910 queue); in setup_netfront_single()
1913 queue->rx_evtchn = queue->tx_evtchn; in setup_netfront_single()
1914 queue->rx_irq = queue->tx_irq = err; in setup_netfront_single()
1919 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_single()
1920 queue->tx_evtchn = 0; in setup_netfront_single()
1925 static int setup_netfront_split(struct netfront_queue *queue) in setup_netfront_split() argument
1929 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_split()
1932 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); in setup_netfront_split()
1936 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), in setup_netfront_split()
1937 "%s-tx", queue->name); in setup_netfront_split()
1938 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, in setup_netfront_split()
1940 queue->tx_irq_name, queue); in setup_netfront_split()
1943 queue->tx_irq = err; in setup_netfront_split()
1945 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), in setup_netfront_split()
1946 "%s-rx", queue->name); in setup_netfront_split()
1947 err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn, in setup_netfront_split()
1949 queue->rx_irq_name, queue); in setup_netfront_split()
1952 queue->rx_irq = err; in setup_netfront_split()
1957 unbind_from_irqhandler(queue->tx_irq, queue); in setup_netfront_split()
1958 queue->tx_irq = 0; in setup_netfront_split()
1960 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); in setup_netfront_split()
1961 queue->rx_evtchn = 0; in setup_netfront_split()
1963 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_split()
1964 queue->tx_evtchn = 0; in setup_netfront_split()
1970 struct netfront_queue *queue, unsigned int feature_split_evtchn) in setup_netfront() argument
1976 queue->tx_ring_ref = INVALID_GRANT_REF; in setup_netfront()
1977 queue->rx_ring_ref = INVALID_GRANT_REF; in setup_netfront()
1978 queue->rx.sring = NULL; in setup_netfront()
1979 queue->tx.sring = NULL; in setup_netfront()
1982 1, &queue->tx_ring_ref); in setup_netfront()
1986 XEN_FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); in setup_netfront()
1989 1, &queue->rx_ring_ref); in setup_netfront()
1993 XEN_FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); in setup_netfront()
1996 err = setup_netfront_split(queue); in setup_netfront()
2002 err = setup_netfront_single(queue); in setup_netfront()
2010 xenbus_teardown_ring((void **)&queue->rx.sring, 1, &queue->rx_ring_ref); in setup_netfront()
2011 xenbus_teardown_ring((void **)&queue->tx.sring, 1, &queue->tx_ring_ref); in setup_netfront()
2020 static int xennet_init_queue(struct netfront_queue *queue) in xennet_init_queue() argument
2026 spin_lock_init(&queue->tx_lock); in xennet_init_queue()
2027 spin_lock_init(&queue->rx_lock); in xennet_init_queue()
2028 spin_lock_init(&queue->rx_cons_lock); in xennet_init_queue()
2030 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); in xennet_init_queue()
2032 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; in xennet_init_queue()
2033 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", in xennet_init_queue()
2034 devid, queue->id); in xennet_init_queue()
2037 queue->tx_skb_freelist = 0; in xennet_init_queue()
2038 queue->tx_pend_queue = TX_LINK_NONE; in xennet_init_queue()
2040 queue->tx_link[i] = i + 1; in xennet_init_queue()
2041 queue->grant_tx_ref[i] = INVALID_GRANT_REF; in xennet_init_queue()
2042 queue->grant_tx_page[i] = NULL; in xennet_init_queue()
2044 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; in xennet_init_queue()
2048 queue->rx_skbs[i] = NULL; in xennet_init_queue()
2049 queue->grant_rx_ref[i] = INVALID_GRANT_REF; in xennet_init_queue()
2054 &queue->gref_tx_head) < 0) { in xennet_init_queue()
2062 &queue->gref_rx_head) < 0) { in xennet_init_queue()
2071 gnttab_free_grant_references(queue->gref_tx_head); in xennet_init_queue()
2076 static int write_queue_xenstore_keys(struct netfront_queue *queue, in write_queue_xenstore_keys() argument
2083 struct xenbus_device *dev = queue->info->xbdev; in write_queue_xenstore_keys()
2099 dev->nodename, queue->id); in write_queue_xenstore_keys()
2106 queue->tx_ring_ref); in write_queue_xenstore_keys()
2113 queue->rx_ring_ref); in write_queue_xenstore_keys()
2122 if (queue->tx_evtchn == queue->rx_evtchn) { in write_queue_xenstore_keys()
2125 "event-channel", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2133 "event-channel-tx", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2140 "event-channel-rx", "%u", queue->rx_evtchn); in write_queue_xenstore_keys()
2160 static int xennet_create_page_pool(struct netfront_queue *queue) in xennet_create_page_pool() argument
2168 .dev = &queue->info->netdev->dev, in xennet_create_page_pool()
2173 queue->page_pool = page_pool_create(&pp_params); in xennet_create_page_pool()
2174 if (IS_ERR(queue->page_pool)) { in xennet_create_page_pool()
2175 err = PTR_ERR(queue->page_pool); in xennet_create_page_pool()
2176 queue->page_pool = NULL; in xennet_create_page_pool()
2180 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, in xennet_create_page_pool()
2181 queue->id, 0); in xennet_create_page_pool()
2183 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); in xennet_create_page_pool()
2187 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, in xennet_create_page_pool()
2188 MEM_TYPE_PAGE_POOL, queue->page_pool); in xennet_create_page_pool()
2190 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); in xennet_create_page_pool()
2196 xdp_rxq_info_unreg(&queue->xdp_rxq); in xennet_create_page_pool()
2198 page_pool_destroy(queue->page_pool); in xennet_create_page_pool()
2199 queue->page_pool = NULL; in xennet_create_page_pool()
2215 struct netfront_queue *queue = &info->queues[i]; in xennet_create_queues() local
2217 queue->id = i; in xennet_create_queues()
2218 queue->info = info; in xennet_create_queues()
2220 ret = xennet_init_queue(queue); in xennet_create_queues()
2229 ret = xennet_create_page_pool(queue); in xennet_create_queues()
2236 netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll); in xennet_create_queues()
2238 napi_enable(&queue->napi); in xennet_create_queues()
2260 struct netfront_queue *queue = NULL; in talk_to_netback() local
2316 queue = &info->queues[i]; in talk_to_netback()
2317 err = setup_netfront(dev, queue, feature_split_evtchn); in talk_to_netback()
2347 queue = &info->queues[i]; in talk_to_netback()
2348 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ in talk_to_netback()
2424 struct netfront_queue *queue = NULL; in xennet_connect() local
2469 queue = &np->queues[j]; in xennet_connect()
2471 notify_remote_via_irq(queue->tx_irq); in xennet_connect()
2472 if (queue->tx_irq != queue->rx_irq) in xennet_connect()
2473 notify_remote_via_irq(queue->rx_irq); in xennet_connect()
2475 spin_lock_bh(&queue->rx_lock); in xennet_connect()
2476 xennet_alloc_rx_buffers(queue); in xennet_connect()
2477 spin_unlock_bh(&queue->rx_lock); in xennet_connect()