/openbmc/linux/drivers/net/ethernet/amazon/ena/ |
H A D | ena_xdp.h | 83 struct bpf_prog *xdp_prog; in ena_xdp_execute() local 88 xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog); in ena_xdp_execute() 90 if (!xdp_prog) in ena_xdp_execute() 93 verdict = bpf_prog_run_xdp(xdp_prog, xdp); in ena_xdp_execute() 99 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute() 120 if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) { in ena_xdp_execute() 125 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute() 130 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute() 143 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute()
|
/openbmc/linux/drivers/net/ethernet/microchip/lan966x/ |
H A D | lan966x_xdp.c | 24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup() 32 xchg(&port->xdp_prog, old_prog); in lan966x_xdp_setup() 77 struct bpf_prog *xdp_prog = port->xdp_prog; in lan966x_xdp_run() local 87 act = bpf_prog_run_xdp(xdp_prog, &xdp); in lan966x_xdp_run() 96 if (xdp_do_redirect(port->dev, &xdp, xdp_prog)) in lan966x_xdp_run() 101 bpf_warn_invalid_xdp_action(port->dev, xdp_prog, act); in lan966x_xdp_run() 104 trace_xdp_exception(port->dev, xdp_prog, act); in lan966x_xdp_run()
|
/openbmc/linux/kernel/bpf/ |
H A D | devmap.c | 61 struct bpf_prog *xdp_prog; member 68 struct bpf_prog *xdp_prog; member 215 if (dev->xdp_prog) in dev_map_free() 216 bpf_prog_put(dev->xdp_prog); in dev_map_free() 231 if (dev->xdp_prog) in dev_map_free() 232 bpf_prog_put(dev->xdp_prog); in dev_map_free() 323 static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, in dev_map_bpf_prog_run() argument 342 act = bpf_prog_run_xdp(xdp_prog, &xdp); in dev_map_bpf_prog_run() 352 bpf_warn_invalid_xdp_action(NULL, xdp_prog, act); in dev_map_bpf_prog_run() 355 trace_xdp_exception(tx_dev, xdp_prog, act); in dev_map_bpf_prog_run() [all …]
|
/openbmc/linux/drivers/net/ethernet/broadcom/bnxt/ |
H A D | bnxt_xdp.c | 177 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_xdp_attached() local 179 return !!xdp_prog; in bnxt_xdp_attached() 228 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_rx_xdp() local 239 if (!xdp_prog) in bnxt_rx_xdp() 249 act = bpf_prog_run_xdp(xdp_prog, &xdp); in bnxt_rx_xdp() 281 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp() 303 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp() 309 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { in bnxt_rx_xdp() 310 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp() 318 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act); in bnxt_rx_xdp() [all …]
|
/openbmc/linux/drivers/net/ethernet/sfc/siena/ |
H A D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local 263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp() 264 if (!xdp_prog) in efx_do_xdp() 294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp() 320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp() 334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
/openbmc/linux/drivers/net/ethernet/sfc/ |
H A D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local 263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp() 264 if (!xdp_prog) in efx_do_xdp() 294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp() 320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp() 334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp() 348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
/openbmc/linux/drivers/net/ethernet/freescale/dpaa2/ |
H A D | dpaa2-xsk.c | 38 struct bpf_prog *xdp_prog; in dpaa2_xsk_run_xdp() local 44 xdp_prog = READ_ONCE(ch->xdp.prog); in dpaa2_xsk_run_xdp() 45 if (!xdp_prog) in dpaa2_xsk_run_xdp() 59 xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff); in dpaa2_xsk_run_xdp() 66 err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog); in dpaa2_xsk_run_xdp() 85 bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); in dpaa2_xsk_run_xdp() 88 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); in dpaa2_xsk_run_xdp() 122 WARN_ON(priv->xdp_prog); in dpaa2_xsk_rx() 317 if (!priv->xdp_prog) in dpaa2_xsk_wakeup()
|
/openbmc/linux/drivers/net/vmxnet3/ |
H A D | vmxnet3_xdp.c | 330 struct bpf_prog *xdp_prog; in vmxnet3_process_xdp_small() local 349 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); in vmxnet3_process_xdp_small() 350 if (!xdp_prog) { in vmxnet3_process_xdp_small() 354 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); in vmxnet3_process_xdp_small() 375 struct bpf_prog *xdp_prog; in vmxnet3_process_xdp() local 393 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); in vmxnet3_process_xdp() 394 if (!xdp_prog) { in vmxnet3_process_xdp() 398 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); in vmxnet3_process_xdp()
|
/openbmc/linux/drivers/net/ethernet/intel/ixgbe/ |
H A D | ixgbe_xsk.c | 14 bool xdp_on = READ_ONCE(adapter->xdp_prog); in ixgbe_xsk_pool() 102 struct bpf_prog *xdp_prog; in ixgbe_run_xdp_zc() local 107 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ixgbe_run_xdp_zc() 108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc() 111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc() 141 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc() 146 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc() 527 if (!READ_ONCE(adapter->xdp_prog)) in ixgbe_xsk_wakeup()
|
/openbmc/linux/drivers/net/ethernet/netronome/nfp/nfd3/ |
H A D | xsk.c | 121 struct bpf_prog *xdp_prog; in nfp_nfd3_xsk_rx() local 125 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_xsk_rx() 219 act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp); in nfp_nfd3_xsk_rx() 237 if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) { in nfp_nfd3_xsk_rx() 245 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx() 248 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx()
|
H A D | dp.c | 553 if (!dp->xdp_prog) { in nfp_nfd3_napi_alloc_one() 568 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfd3_napi_alloc_one() 908 struct bpf_prog *xdp_prog; in nfp_nfd3_rx() local 915 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_rx() 916 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfd3_rx() 1004 if (xdp_prog && !meta.portid) { in nfp_nfd3_rx() 1014 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfd3_rx() 1032 xdp_prog, act); in nfp_nfd3_rx() 1035 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_rx() 1038 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_rx() [all …]
|
/openbmc/linux/drivers/net/ethernet/fungible/funeth/ |
H A D | funeth_rx.c | 144 struct bpf_prog *xdp_prog; in fun_run_xdp() local 155 xdp_prog = READ_ONCE(q->xdp_prog); in fun_run_xdp() 156 act = bpf_prog_run_xdp(xdp_prog, &xdp); in fun_run_xdp() 177 if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog))) in fun_run_xdp() 183 bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act); in fun_run_xdp() 186 trace_xdp_exception(q->netdev, xdp_prog, act); in fun_run_xdp() 608 WRITE_ONCE(q->xdp_prog, prog); in fun_rxq_set_bpf() 724 q->xdp_prog = fp->xdp_prog; in fun_rxq_create_dev() 725 q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM; in fun_rxq_create_dev()
|
/openbmc/linux/drivers/net/ethernet/intel/i40e/ |
H A D | i40e_xsk.c | 198 struct bpf_prog *xdp_prog) in i40e_run_xdp_zc() argument 204 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc() 207 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc() 230 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc() 235 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc() 440 struct bpf_prog *xdp_prog; in i40e_clean_rx_irq_zc() local 450 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in i40e_clean_rx_irq_zc() 500 xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog); in i40e_clean_rx_irq_zc()
|
/openbmc/linux/drivers/net/ |
H A D | veth.c | 63 struct bpf_prog __rcu *xdp_prog; member 584 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) in veth_xdp_flush() 614 struct bpf_prog *xdp_prog; in veth_xdp_rcv_one() local 617 xdp_prog = rcu_dereference(rq->xdp_prog); in veth_xdp_rcv_one() 618 if (likely(xdp_prog)) { in veth_xdp_rcv_one() 627 act = bpf_prog_run_xdp(xdp_prog, xdp); in veth_xdp_rcv_one() 638 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one() 649 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { in veth_xdp_rcv_one() 658 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); in veth_xdp_rcv_one() 661 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one() [all …]
|
H A D | tun.c | 208 struct bpf_prog __rcu *xdp_prog; member 829 if (rtnl_dereference(tun->xdp_prog)) in tun_attach() 1204 old_prog = rtnl_dereference(tun->xdp_prog); in tun_xdp_set() 1205 rcu_assign_pointer(tun->xdp_prog, prog); in tun_xdp_set() 1626 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, in tun_xdp_act() argument 1633 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); in tun_xdp_act() 1651 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); in tun_xdp_act() 1654 trace_xdp_exception(tun->dev, xdp_prog, act); in tun_xdp_act() 1671 struct bpf_prog *xdp_prog; in tun_build_skb() local 1679 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_build_skb() [all …]
|
H A D | virtio_net.c | 171 struct bpf_prog __rcu *xdp_prog; member 929 struct bpf_prog *xdp_prog; in virtnet_xdp_xmit() local 943 xdp_prog = rcu_access_pointer(rq->xdp_prog); in virtnet_xdp_xmit() 944 if (!xdp_prog) in virtnet_xdp_xmit() 1014 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, in virtnet_xdp_handler() argument 1023 act = bpf_prog_run_xdp(xdp_prog, xdp); in virtnet_xdp_handler() 1042 trace_xdp_exception(dev, xdp_prog, act); in virtnet_xdp_handler() 1050 err = xdp_do_redirect(dev, xdp, xdp_prog); in virtnet_xdp_handler() 1058 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); in virtnet_xdp_handler() 1061 trace_xdp_exception(dev, xdp_prog, act); in virtnet_xdp_handler() [all …]
|
/openbmc/linux/drivers/net/ethernet/aquantia/atlantic/ |
H A D | aq_main.c | 137 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_change_mtu() 229 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_fix_features() 231 aq_nic->xdp_prog && features & NETIF_F_LRO) { in aq_ndev_fix_features() 466 need_update = !!aq_nic->xdp_prog != !!prog; in aq_xdp_setup() 470 old_prog = xchg(&aq_nic->xdp_prog, prog); in aq_xdp_setup()
|
/openbmc/linux/drivers/net/ethernet/mellanox/mlx5/core/en/ |
H A D | params.c | 227 if (params->xdp_prog) in mlx5e_get_linear_rq_headroom() 270 return params->xdp_prog && sz < PAGE_SIZE ? PAGE_SIZE : sz; in mlx5e_rx_get_linear_stride_sz() 424 if (params->xdp_prog) in mlx5e_mpwqe_get_log_stride_size() 717 wqe_bulk_in_bytes = min_t(u32, MAX_WQE_BULK_BYTES(params->xdp_prog), in mlx5e_rx_compute_wqe_bulk_params() 726 split_factor = DIV_ROUND_UP(MAX_WQE_BULK_BYTES(params->xdp_prog), in mlx5e_rx_compute_wqe_bulk_params() 770 params->xdp_prog); in mlx5e_build_rq_frags_info() 771 if (byte_count > max_mtu || params->xdp_prog) { in mlx5e_build_rq_frags_info() 776 params->xdp_prog); in mlx5e_build_rq_frags_info() 796 if (params->xdp_prog) { in mlx5e_build_rq_frags_info() 850 *xdp_frag_size = info->num_frags > 1 && params->xdp_prog ? PAGE_SIZE : 0; in mlx5e_build_rq_frags_info() [all …]
|
H A D | xdp.h | 129 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_enable() 135 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_disable()
|
/openbmc/linux/drivers/net/ethernet/intel/ice/ |
H A D | ice_xsk.c | 764 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) in ice_run_xdp_zc() argument 769 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp_zc() 772 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ice_run_xdp_zc() 794 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc() 799 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc() 851 struct bpf_prog *xdp_prog; in ice_clean_rx_irq_zc() local 859 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq_zc() 909 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring); in ice_clean_rx_irq_zc()
|
H A D | ice_txrx.c | 459 WRITE_ONCE(rx_ring->xdp_prog, NULL); in ice_free_rx_ring() 514 WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog); in ice_setup_rx_ring() 536 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring, in ice_run_xdp() argument 542 if (!xdp_prog) in ice_run_xdp() 545 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp() 559 if (xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog)) in ice_run_xdp() 564 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp() 568 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp() 1123 struct bpf_prog *xdp_prog = NULL; in ice_clean_rx_irq() local 1131 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq() [all …]
|
/openbmc/linux/drivers/net/ethernet/stmicro/stmmac/ |
H A D | stmmac_xdp.c | 123 need_update = !!priv->xdp_prog != !!prog; in stmmac_xdp_set_prog() 127 old_prog = xchg(&priv->xdp_prog, prog); in stmmac_xdp_set_prog()
|
/openbmc/linux/drivers/net/ethernet/mellanox/mlx4/ |
H A D | en_rx.c | 438 ring->xdp_prog, in mlx4_en_destroy_rx_ring() 719 struct bpf_prog *xdp_prog; in mlx4_en_process_rx_cq() local 732 xdp_prog = rcu_dereference_bh(ring->xdp_prog); in mlx4_en_process_rx_cq() 818 if (xdp_prog) { in mlx4_en_process_rx_cq() 836 act = bpf_prog_run_xdp(xdp_prog, &mxbuf.xdp); in mlx4_en_process_rx_cq() 849 if (likely(!xdp_do_redirect(dev, &mxbuf.xdp, xdp_prog))) { in mlx4_en_process_rx_cq() 856 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq() 865 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq() 868 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); in mlx4_en_process_rx_cq() 871 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
|
/openbmc/linux/drivers/net/ethernet/netronome/nfp/nfdk/ |
H A D | dp.c | 573 if (!dp->xdp_prog) { in nfp_nfdk_napi_alloc_one() 588 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfdk_napi_alloc_one() 1029 struct bpf_prog *xdp_prog; in nfp_nfdk_rx() local 1037 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfdk_rx() 1038 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfdk_rx() 1123 if (xdp_prog && !meta.portid) { in nfp_nfdk_rx() 1133 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfdk_rx() 1151 xdp_prog, act); in nfp_nfdk_rx() 1154 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfdk_rx() 1157 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfdk_rx() [all …]
|
/openbmc/linux/drivers/net/ethernet/intel/igc/ |
H A D | igc_xdp.h | 14 return !!adapter->xdp_prog; in igc_xdp_is_enabled()
|