1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */ 3 4 #include "nfp_app.h" 5 #include "nfp_net_dp.h" 6 #include "nfp_net_xsk.h" 7 8 /** 9 * nfp_net_rx_alloc_one() - Allocate and map page frag for RX 10 * @dp: NFP Net data path struct 11 * @dma_addr: Pointer to storage for DMA address (output param) 12 * 13 * This function will allcate a new page frag, map it for DMA. 14 * 15 * Return: allocated page frag or NULL on failure. 16 */ 17 void *nfp_net_rx_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr) 18 { 19 void *frag; 20 21 if (!dp->xdp_prog) { 22 frag = netdev_alloc_frag(dp->fl_bufsz); 23 } else { 24 struct page *page; 25 26 page = alloc_page(GFP_KERNEL); 27 frag = page ? page_address(page) : NULL; 28 } 29 if (!frag) { 30 nn_dp_warn(dp, "Failed to alloc receive page frag\n"); 31 return NULL; 32 } 33 34 *dma_addr = nfp_net_dma_map_rx(dp, frag); 35 if (dma_mapping_error(dp->dev, *dma_addr)) { 36 nfp_net_free_frag(frag, dp->xdp_prog); 37 nn_dp_warn(dp, "Failed to map DMA RX buffer\n"); 38 return NULL; 39 } 40 41 return frag; 42 } 43 44 /** 45 * nfp_net_tx_ring_init() - Fill in the boilerplate for a TX ring 46 * @tx_ring: TX ring structure 47 * @dp: NFP Net data path struct 48 * @r_vec: IRQ vector servicing this ring 49 * @idx: Ring index 50 * @is_xdp: Is this an XDP TX ring? 51 */ 52 static void 53 nfp_net_tx_ring_init(struct nfp_net_tx_ring *tx_ring, struct nfp_net_dp *dp, 54 struct nfp_net_r_vector *r_vec, unsigned int idx, 55 bool is_xdp) 56 { 57 struct nfp_net *nn = r_vec->nfp_net; 58 59 tx_ring->idx = idx; 60 tx_ring->r_vec = r_vec; 61 tx_ring->is_xdp = is_xdp; 62 u64_stats_init(&tx_ring->r_vec->tx_sync); 63 64 tx_ring->qcidx = tx_ring->idx * nn->stride_tx; 65 tx_ring->txrwb = dp->txrwb ? &dp->txrwb[idx] : NULL; 66 tx_ring->qcp_q = nn->tx_bar + NFP_QCP_QUEUE_OFF(tx_ring->qcidx); 67 } 68 69 /** 70 * nfp_net_rx_ring_init() - Fill in the boilerplate for a RX ring 71 * @rx_ring: RX ring structure 72 * @r_vec: IRQ vector servicing this ring 73 * @idx: Ring index 74 */ 75 static void 76 nfp_net_rx_ring_init(struct nfp_net_rx_ring *rx_ring, 77 struct nfp_net_r_vector *r_vec, unsigned int idx) 78 { 79 struct nfp_net *nn = r_vec->nfp_net; 80 81 rx_ring->idx = idx; 82 rx_ring->r_vec = r_vec; 83 u64_stats_init(&rx_ring->r_vec->rx_sync); 84 85 rx_ring->fl_qcidx = rx_ring->idx * nn->stride_rx; 86 rx_ring->qcp_fl = nn->rx_bar + NFP_QCP_QUEUE_OFF(rx_ring->fl_qcidx); 87 } 88 89 /** 90 * nfp_net_rx_ring_reset() - Reflect in SW state of freelist after disable 91 * @rx_ring: RX ring structure 92 * 93 * Assumes that the device is stopped, must be idempotent. 94 */ 95 void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring) 96 { 97 unsigned int wr_idx, last_idx; 98 99 /* wr_p == rd_p means ring was never fed FL bufs. RX rings are always 100 * kept at cnt - 1 FL bufs. 101 */ 102 if (rx_ring->wr_p == 0 && rx_ring->rd_p == 0) 103 return; 104 105 /* Move the empty entry to the end of the list */ 106 wr_idx = D_IDX(rx_ring, rx_ring->wr_p); 107 last_idx = rx_ring->cnt - 1; 108 if (rx_ring->r_vec->xsk_pool) { 109 rx_ring->xsk_rxbufs[wr_idx] = rx_ring->xsk_rxbufs[last_idx]; 110 memset(&rx_ring->xsk_rxbufs[last_idx], 0, 111 sizeof(*rx_ring->xsk_rxbufs)); 112 } else { 113 rx_ring->rxbufs[wr_idx] = rx_ring->rxbufs[last_idx]; 114 memset(&rx_ring->rxbufs[last_idx], 0, sizeof(*rx_ring->rxbufs)); 115 } 116 117 memset(rx_ring->rxds, 0, rx_ring->size); 118 rx_ring->wr_p = 0; 119 rx_ring->rd_p = 0; 120 } 121 122 /** 123 * nfp_net_rx_ring_bufs_free() - Free any buffers currently on the RX ring 124 * @dp: NFP Net data path struct 125 * @rx_ring: RX ring to remove buffers from 126 * 127 * Assumes that the device is stopped and buffers are in [0, ring->cnt - 1) 128 * entries. After device is disabled nfp_net_rx_ring_reset() must be called 129 * to restore required ring geometry. 130 */ 131 static void 132 nfp_net_rx_ring_bufs_free(struct nfp_net_dp *dp, 133 struct nfp_net_rx_ring *rx_ring) 134 { 135 unsigned int i; 136 137 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) 138 return; 139 140 for (i = 0; i < rx_ring->cnt - 1; i++) { 141 /* NULL skb can only happen when initial filling of the ring 142 * fails to allocate enough buffers and calls here to free 143 * already allocated ones. 144 */ 145 if (!rx_ring->rxbufs[i].frag) 146 continue; 147 148 nfp_net_dma_unmap_rx(dp, rx_ring->rxbufs[i].dma_addr); 149 nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog); 150 rx_ring->rxbufs[i].dma_addr = 0; 151 rx_ring->rxbufs[i].frag = NULL; 152 } 153 } 154 155 /** 156 * nfp_net_rx_ring_bufs_alloc() - Fill RX ring with buffers (don't give to FW) 157 * @dp: NFP Net data path struct 158 * @rx_ring: RX ring to remove buffers from 159 */ 160 static int 161 nfp_net_rx_ring_bufs_alloc(struct nfp_net_dp *dp, 162 struct nfp_net_rx_ring *rx_ring) 163 { 164 struct nfp_net_rx_buf *rxbufs; 165 unsigned int i; 166 167 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) 168 return 0; 169 170 rxbufs = rx_ring->rxbufs; 171 172 for (i = 0; i < rx_ring->cnt - 1; i++) { 173 rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr); 174 if (!rxbufs[i].frag) { 175 nfp_net_rx_ring_bufs_free(dp, rx_ring); 176 return -ENOMEM; 177 } 178 } 179 180 return 0; 181 } 182 183 int nfp_net_tx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp) 184 { 185 unsigned int r; 186 187 dp->tx_rings = kcalloc(dp->num_tx_rings, sizeof(*dp->tx_rings), 188 GFP_KERNEL); 189 if (!dp->tx_rings) 190 return -ENOMEM; 191 192 if (dp->ctrl & NFP_NET_CFG_CTRL_TXRWB) { 193 dp->txrwb = dma_alloc_coherent(dp->dev, 194 dp->num_tx_rings * sizeof(u64), 195 &dp->txrwb_dma, GFP_KERNEL); 196 if (!dp->txrwb) 197 goto err_free_rings; 198 } 199 200 for (r = 0; r < dp->num_tx_rings; r++) { 201 int bias = 0; 202 203 if (r >= dp->num_stack_tx_rings) 204 bias = dp->num_stack_tx_rings; 205 206 nfp_net_tx_ring_init(&dp->tx_rings[r], dp, 207 &nn->r_vecs[r - bias], r, bias); 208 209 if (nfp_net_tx_ring_alloc(dp, &dp->tx_rings[r])) 210 goto err_free_prev; 211 212 if (nfp_net_tx_ring_bufs_alloc(dp, &dp->tx_rings[r])) 213 goto err_free_ring; 214 } 215 216 return 0; 217 218 err_free_prev: 219 while (r--) { 220 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]); 221 err_free_ring: 222 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]); 223 } 224 if (dp->txrwb) 225 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64), 226 dp->txrwb, dp->txrwb_dma); 227 err_free_rings: 228 kfree(dp->tx_rings); 229 return -ENOMEM; 230 } 231 232 void nfp_net_tx_rings_free(struct nfp_net_dp *dp) 233 { 234 unsigned int r; 235 236 for (r = 0; r < dp->num_tx_rings; r++) { 237 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]); 238 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]); 239 } 240 241 if (dp->txrwb) 242 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64), 243 dp->txrwb, dp->txrwb_dma); 244 kfree(dp->tx_rings); 245 } 246 247 /** 248 * nfp_net_rx_ring_free() - Free resources allocated to a RX ring 249 * @rx_ring: RX ring to free 250 */ 251 static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring) 252 { 253 struct nfp_net_r_vector *r_vec = rx_ring->r_vec; 254 struct nfp_net_dp *dp = &r_vec->nfp_net->dp; 255 256 if (dp->netdev) 257 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 258 259 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) 260 kvfree(rx_ring->xsk_rxbufs); 261 else 262 kvfree(rx_ring->rxbufs); 263 264 if (rx_ring->rxds) 265 dma_free_coherent(dp->dev, rx_ring->size, 266 rx_ring->rxds, rx_ring->dma); 267 268 rx_ring->cnt = 0; 269 rx_ring->rxbufs = NULL; 270 rx_ring->xsk_rxbufs = NULL; 271 rx_ring->rxds = NULL; 272 rx_ring->dma = 0; 273 rx_ring->size = 0; 274 } 275 276 /** 277 * nfp_net_rx_ring_alloc() - Allocate resource for a RX ring 278 * @dp: NFP Net data path struct 279 * @rx_ring: RX ring to allocate 280 * 281 * Return: 0 on success, negative errno otherwise. 282 */ 283 static int 284 nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring) 285 { 286 enum xdp_mem_type mem_type; 287 size_t rxbuf_sw_desc_sz; 288 int err; 289 290 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) { 291 mem_type = MEM_TYPE_XSK_BUFF_POOL; 292 rxbuf_sw_desc_sz = sizeof(*rx_ring->xsk_rxbufs); 293 } else { 294 mem_type = MEM_TYPE_PAGE_ORDER0; 295 rxbuf_sw_desc_sz = sizeof(*rx_ring->rxbufs); 296 } 297 298 if (dp->netdev) { 299 err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev, 300 rx_ring->idx, rx_ring->r_vec->napi.napi_id); 301 if (err < 0) 302 return err; 303 304 err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, mem_type, NULL); 305 if (err) 306 goto err_alloc; 307 } 308 309 rx_ring->cnt = dp->rxd_cnt; 310 rx_ring->size = array_size(rx_ring->cnt, sizeof(*rx_ring->rxds)); 311 rx_ring->rxds = dma_alloc_coherent(dp->dev, rx_ring->size, 312 &rx_ring->dma, 313 GFP_KERNEL | __GFP_NOWARN); 314 if (!rx_ring->rxds) { 315 netdev_warn(dp->netdev, "failed to allocate RX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n", 316 rx_ring->cnt); 317 goto err_alloc; 318 } 319 320 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) { 321 rx_ring->xsk_rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz, 322 GFP_KERNEL); 323 if (!rx_ring->xsk_rxbufs) 324 goto err_alloc; 325 } else { 326 rx_ring->rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz, 327 GFP_KERNEL); 328 if (!rx_ring->rxbufs) 329 goto err_alloc; 330 } 331 332 return 0; 333 334 err_alloc: 335 nfp_net_rx_ring_free(rx_ring); 336 return -ENOMEM; 337 } 338 339 int nfp_net_rx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp) 340 { 341 unsigned int r; 342 343 dp->rx_rings = kcalloc(dp->num_rx_rings, sizeof(*dp->rx_rings), 344 GFP_KERNEL); 345 if (!dp->rx_rings) 346 return -ENOMEM; 347 348 for (r = 0; r < dp->num_rx_rings; r++) { 349 nfp_net_rx_ring_init(&dp->rx_rings[r], &nn->r_vecs[r], r); 350 351 if (nfp_net_rx_ring_alloc(dp, &dp->rx_rings[r])) 352 goto err_free_prev; 353 354 if (nfp_net_rx_ring_bufs_alloc(dp, &dp->rx_rings[r])) 355 goto err_free_ring; 356 } 357 358 return 0; 359 360 err_free_prev: 361 while (r--) { 362 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]); 363 err_free_ring: 364 nfp_net_rx_ring_free(&dp->rx_rings[r]); 365 } 366 kfree(dp->rx_rings); 367 return -ENOMEM; 368 } 369 370 void nfp_net_rx_rings_free(struct nfp_net_dp *dp) 371 { 372 unsigned int r; 373 374 for (r = 0; r < dp->num_rx_rings; r++) { 375 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]); 376 nfp_net_rx_ring_free(&dp->rx_rings[r]); 377 } 378 379 kfree(dp->rx_rings); 380 } 381 382 void 383 nfp_net_rx_ring_hw_cfg_write(struct nfp_net *nn, 384 struct nfp_net_rx_ring *rx_ring, unsigned int idx) 385 { 386 /* Write the DMA address, size and MSI-X info to the device */ 387 nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), rx_ring->dma); 388 nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), ilog2(rx_ring->cnt)); 389 nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), rx_ring->r_vec->irq_entry); 390 } 391 392 void 393 nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn, 394 struct nfp_net_tx_ring *tx_ring, unsigned int idx) 395 { 396 nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), tx_ring->dma); 397 if (tx_ring->txrwb) { 398 *tx_ring->txrwb = 0; 399 nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), 400 nn->dp.txrwb_dma + idx * sizeof(u64)); 401 } 402 nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), ilog2(tx_ring->cnt)); 403 nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), tx_ring->r_vec->irq_entry); 404 } 405 406 void nfp_net_vec_clear_ring_data(struct nfp_net *nn, unsigned int idx) 407 { 408 nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), 0); 409 nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), 0); 410 nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), 0); 411 412 nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), 0); 413 nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), 0); 414 nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), 0); 415 nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), 0); 416 } 417 418 netdev_tx_t nfp_net_tx(struct sk_buff *skb, struct net_device *netdev) 419 { 420 struct nfp_net *nn = netdev_priv(netdev); 421 422 return nn->dp.ops->xmit(skb, netdev); 423 } 424 425 bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb) 426 { 427 struct nfp_net_r_vector *r_vec = &nn->r_vecs[0]; 428 429 return nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false); 430 } 431 432 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb) 433 { 434 struct nfp_net_r_vector *r_vec = &nn->r_vecs[0]; 435 bool ret; 436 437 spin_lock_bh(&r_vec->lock); 438 ret = nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false); 439 spin_unlock_bh(&r_vec->lock); 440 441 return ret; 442 } 443