1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux driver for VMware's vmxnet3 ethernet NIC. 4 * Copyright (C) 2008-2023, VMware, Inc. All Rights Reserved. 5 * Maintained by: pv-drivers@vmware.com 6 * 7 */ 8 9 #include "vmxnet3_int.h" 10 #include "vmxnet3_xdp.h" 11 12 static void 13 vmxnet3_xdp_exchange_program(struct vmxnet3_adapter *adapter, 14 struct bpf_prog *prog) 15 { 16 rcu_assign_pointer(adapter->xdp_bpf_prog, prog); 17 } 18 19 static inline struct vmxnet3_tx_queue * 20 vmxnet3_xdp_get_tq(struct vmxnet3_adapter *adapter) 21 { 22 struct vmxnet3_tx_queue *tq; 23 int tq_number; 24 int cpu; 25 26 tq_number = adapter->num_tx_queues; 27 cpu = smp_processor_id(); 28 if (likely(cpu < tq_number)) 29 tq = &adapter->tx_queue[cpu]; 30 else 31 tq = &adapter->tx_queue[reciprocal_scale(cpu, tq_number)]; 32 33 return tq; 34 } 35 36 static int 37 vmxnet3_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf, 38 struct netlink_ext_ack *extack) 39 { 40 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 41 struct bpf_prog *new_bpf_prog = bpf->prog; 42 struct bpf_prog *old_bpf_prog; 43 bool need_update; 44 bool running; 45 int err; 46 47 if (new_bpf_prog && netdev->mtu > VMXNET3_XDP_MAX_MTU) { 48 NL_SET_ERR_MSG_FMT_MOD(extack, "MTU %u too large for XDP", 49 netdev->mtu); 50 return -EOPNOTSUPP; 51 } 52 53 if (adapter->netdev->features & NETIF_F_LRO) { 54 NL_SET_ERR_MSG_MOD(extack, "LRO is not supported with XDP"); 55 adapter->netdev->features &= ~NETIF_F_LRO; 56 } 57 58 old_bpf_prog = rcu_dereference(adapter->xdp_bpf_prog); 59 if (!new_bpf_prog && !old_bpf_prog) 60 return 0; 61 62 running = netif_running(netdev); 63 need_update = !!old_bpf_prog != !!new_bpf_prog; 64 65 if (running && need_update) 66 vmxnet3_quiesce_dev(adapter); 67 68 vmxnet3_xdp_exchange_program(adapter, new_bpf_prog); 69 if (old_bpf_prog) 70 bpf_prog_put(old_bpf_prog); 71 72 if (!running || !need_update) 73 return 0; 74 75 if (new_bpf_prog) 76 xdp_features_set_redirect_target(netdev, false); 77 else 78 xdp_features_clear_redirect_target(netdev); 79 80 vmxnet3_reset_dev(adapter); 81 vmxnet3_rq_destroy_all(adapter); 82 vmxnet3_adjust_rx_ring_size(adapter); 83 err = vmxnet3_rq_create_all(adapter); 84 if (err) { 85 NL_SET_ERR_MSG_MOD(extack, 86 "failed to re-create rx queues for XDP."); 87 return -EOPNOTSUPP; 88 } 89 err = vmxnet3_activate_dev(adapter); 90 if (err) { 91 NL_SET_ERR_MSG_MOD(extack, 92 "failed to activate device for XDP."); 93 return -EOPNOTSUPP; 94 } 95 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 96 97 return 0; 98 } 99 100 /* This is the main xdp call used by kernel to set/unset eBPF program. */ 101 int 102 vmxnet3_xdp(struct net_device *netdev, struct netdev_bpf *bpf) 103 { 104 switch (bpf->command) { 105 case XDP_SETUP_PROG: 106 return vmxnet3_xdp_set(netdev, bpf, bpf->extack); 107 default: 108 return -EINVAL; 109 } 110 111 return 0; 112 } 113 114 static int 115 vmxnet3_xdp_xmit_frame(struct vmxnet3_adapter *adapter, 116 struct xdp_frame *xdpf, 117 struct vmxnet3_tx_queue *tq, bool dma_map) 118 { 119 struct vmxnet3_tx_buf_info *tbi = NULL; 120 union Vmxnet3_GenericDesc *gdesc; 121 struct vmxnet3_tx_ctx ctx; 122 int tx_num_deferred; 123 struct page *page; 124 u32 buf_size; 125 u32 dw2; 126 127 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 128 dw2 |= xdpf->len; 129 ctx.sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 130 gdesc = ctx.sop_txd; 131 132 buf_size = xdpf->len; 133 tbi = tq->buf_info + tq->tx_ring.next2fill; 134 135 if (vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) == 0) { 136 tq->stats.tx_ring_full++; 137 return -ENOSPC; 138 } 139 140 tbi->map_type = VMXNET3_MAP_XDP; 141 if (dma_map) { /* ndo_xdp_xmit */ 142 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 143 xdpf->data, buf_size, 144 DMA_TO_DEVICE); 145 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 146 return -EFAULT; 147 tbi->map_type |= VMXNET3_MAP_SINGLE; 148 } else { /* XDP buffer from page pool */ 149 page = virt_to_page(xdpf->data); 150 tbi->dma_addr = page_pool_get_dma_addr(page) + 151 (xdpf->data - (void *)xdpf); 152 dma_sync_single_for_device(&adapter->pdev->dev, 153 tbi->dma_addr, buf_size, 154 DMA_TO_DEVICE); 155 } 156 tbi->xdpf = xdpf; 157 tbi->len = buf_size; 158 159 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 160 WARN_ON_ONCE(gdesc->txd.gen == tq->tx_ring.gen); 161 162 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 163 gdesc->dword[2] = cpu_to_le32(dw2); 164 165 /* Setup the EOP desc */ 166 gdesc->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 167 168 gdesc->txd.om = 0; 169 gdesc->txd.msscof = 0; 170 gdesc->txd.hlen = 0; 171 gdesc->txd.ti = 0; 172 173 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 174 le32_add_cpu(&tq->shared->txNumDeferred, 1); 175 tx_num_deferred++; 176 177 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 178 179 /* set the last buf_info for the pkt */ 180 tbi->sop_idx = ctx.sop_txd - tq->tx_ring.base; 181 182 dma_wmb(); 183 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 184 VMXNET3_TXD_GEN); 185 186 /* No need to handle the case when tx_num_deferred doesn't reach 187 * threshold. Backend driver at hypervisor side will poll and reset 188 * tq->shared->txNumDeferred to 0. 189 */ 190 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 191 tq->shared->txNumDeferred = 0; 192 VMXNET3_WRITE_BAR0_REG(adapter, 193 VMXNET3_REG_TXPROD + tq->qid * 8, 194 tq->tx_ring.next2fill); 195 } 196 197 return 0; 198 } 199 200 static int 201 vmxnet3_xdp_xmit_back(struct vmxnet3_adapter *adapter, 202 struct xdp_frame *xdpf) 203 { 204 struct vmxnet3_tx_queue *tq; 205 struct netdev_queue *nq; 206 int err; 207 208 tq = vmxnet3_xdp_get_tq(adapter); 209 if (tq->stopped) 210 return -ENETDOWN; 211 212 nq = netdev_get_tx_queue(adapter->netdev, tq->qid); 213 214 __netif_tx_lock(nq, smp_processor_id()); 215 err = vmxnet3_xdp_xmit_frame(adapter, xdpf, tq, false); 216 __netif_tx_unlock(nq); 217 218 return err; 219 } 220 221 /* ndo_xdp_xmit */ 222 int 223 vmxnet3_xdp_xmit(struct net_device *dev, 224 int n, struct xdp_frame **frames, u32 flags) 225 { 226 struct vmxnet3_adapter *adapter = netdev_priv(dev); 227 struct vmxnet3_tx_queue *tq; 228 int i; 229 230 if (unlikely(test_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))) 231 return -ENETDOWN; 232 if (unlikely(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))) 233 return -EINVAL; 234 235 tq = vmxnet3_xdp_get_tq(adapter); 236 if (tq->stopped) 237 return -ENETDOWN; 238 239 for (i = 0; i < n; i++) { 240 if (vmxnet3_xdp_xmit_frame(adapter, frames[i], tq, true)) { 241 tq->stats.xdp_xmit_err++; 242 break; 243 } 244 } 245 tq->stats.xdp_xmit += i; 246 247 return i; 248 } 249 250 static int 251 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp, 252 struct bpf_prog *prog) 253 { 254 struct xdp_frame *xdpf; 255 struct page *page; 256 int err; 257 u32 act; 258 259 rq->stats.xdp_packets++; 260 act = bpf_prog_run_xdp(prog, xdp); 261 page = virt_to_page(xdp->data_hard_start); 262 263 switch (act) { 264 case XDP_PASS: 265 return act; 266 case XDP_REDIRECT: 267 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog); 268 if (!err) { 269 rq->stats.xdp_redirects++; 270 } else { 271 rq->stats.xdp_drops++; 272 page_pool_recycle_direct(rq->page_pool, page); 273 } 274 return act; 275 case XDP_TX: 276 xdpf = xdp_convert_buff_to_frame(xdp); 277 if (unlikely(!xdpf || 278 vmxnet3_xdp_xmit_back(rq->adapter, xdpf))) { 279 rq->stats.xdp_drops++; 280 page_pool_recycle_direct(rq->page_pool, page); 281 } else { 282 rq->stats.xdp_tx++; 283 } 284 return act; 285 default: 286 bpf_warn_invalid_xdp_action(rq->adapter->netdev, prog, act); 287 fallthrough; 288 case XDP_ABORTED: 289 trace_xdp_exception(rq->adapter->netdev, prog, act); 290 rq->stats.xdp_aborted++; 291 break; 292 case XDP_DROP: 293 rq->stats.xdp_drops++; 294 break; 295 } 296 297 page_pool_recycle_direct(rq->page_pool, page); 298 299 return act; 300 } 301 302 static struct sk_buff * 303 vmxnet3_build_skb(struct vmxnet3_rx_queue *rq, struct page *page, 304 const struct xdp_buff *xdp) 305 { 306 struct sk_buff *skb; 307 308 skb = build_skb(page_address(page), PAGE_SIZE); 309 if (unlikely(!skb)) { 310 page_pool_recycle_direct(rq->page_pool, page); 311 rq->stats.rx_buf_alloc_failure++; 312 return NULL; 313 } 314 315 /* bpf prog might change len and data position. */ 316 skb_reserve(skb, xdp->data - xdp->data_hard_start); 317 skb_put(skb, xdp->data_end - xdp->data); 318 skb_mark_for_recycle(skb); 319 320 return skb; 321 } 322 323 /* Handle packets from DataRing. */ 324 int 325 vmxnet3_process_xdp_small(struct vmxnet3_adapter *adapter, 326 struct vmxnet3_rx_queue *rq, 327 void *data, int len, 328 struct sk_buff **skb_xdp_pass) 329 { 330 struct bpf_prog *xdp_prog; 331 struct xdp_buff xdp; 332 struct page *page; 333 int act; 334 335 page = page_pool_alloc_pages(rq->page_pool, GFP_ATOMIC); 336 if (unlikely(!page)) { 337 rq->stats.rx_buf_alloc_failure++; 338 return XDP_DROP; 339 } 340 341 xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq); 342 xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset, 343 len, false); 344 xdp_buff_clear_frags_flag(&xdp); 345 346 /* Must copy the data because it's at dataring. */ 347 memcpy(xdp.data, data, len); 348 349 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); 350 if (!xdp_prog) { 351 act = XDP_PASS; 352 goto out_skb; 353 } 354 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); 355 if (act != XDP_PASS) 356 return act; 357 358 out_skb: 359 *skb_xdp_pass = vmxnet3_build_skb(rq, page, &xdp); 360 if (!*skb_xdp_pass) 361 return XDP_DROP; 362 363 /* No need to refill. */ 364 return likely(*skb_xdp_pass) ? act : XDP_DROP; 365 } 366 367 int 368 vmxnet3_process_xdp(struct vmxnet3_adapter *adapter, 369 struct vmxnet3_rx_queue *rq, 370 struct Vmxnet3_RxCompDesc *rcd, 371 struct vmxnet3_rx_buf_info *rbi, 372 struct Vmxnet3_RxDesc *rxd, 373 struct sk_buff **skb_xdp_pass) 374 { 375 struct bpf_prog *xdp_prog; 376 dma_addr_t new_dma_addr; 377 struct xdp_buff xdp; 378 struct page *page; 379 void *new_data; 380 int act; 381 382 page = rbi->page; 383 dma_sync_single_for_cpu(&adapter->pdev->dev, 384 page_pool_get_dma_addr(page) + 385 rq->page_pool->p.offset, rbi->len, 386 page_pool_get_dma_dir(rq->page_pool)); 387 388 xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq); 389 xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset, 390 rbi->len, false); 391 xdp_buff_clear_frags_flag(&xdp); 392 393 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); 394 if (!xdp_prog) { 395 act = XDP_PASS; 396 goto out_skb; 397 } 398 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); 399 400 if (act == XDP_PASS) { 401 out_skb: 402 *skb_xdp_pass = vmxnet3_build_skb(rq, page, &xdp); 403 if (!*skb_xdp_pass) 404 act = XDP_DROP; 405 } 406 407 new_data = vmxnet3_pp_get_buff(rq->page_pool, &new_dma_addr, 408 GFP_ATOMIC); 409 if (!new_data) { 410 rq->stats.rx_buf_alloc_failure++; 411 return XDP_DROP; 412 } 413 rbi->page = virt_to_page(new_data); 414 rbi->dma_addr = new_dma_addr; 415 rxd->addr = cpu_to_le64(rbi->dma_addr); 416 rxd->len = rbi->len; 417 418 return act; 419 } 420