1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/inetdevice.h> 7 #include <linux/etherdevice.h> 8 #include <linux/ethtool.h> 9 #include <linux/filter.h> 10 #include <linux/mm.h> 11 12 #include <net/checksum.h> 13 #include <net/ip6_checksum.h> 14 #include <net/page_pool/helpers.h> 15 #include <net/xdp.h> 16 17 #include <net/mana/mana.h> 18 #include <net/mana/mana_auxiliary.h> 19 20 static DEFINE_IDA(mana_adev_ida); 21 22 static int mana_adev_idx_alloc(void) 23 { 24 return ida_alloc(&mana_adev_ida, GFP_KERNEL); 25 } 26 27 static void mana_adev_idx_free(int idx) 28 { 29 ida_free(&mana_adev_ida, idx); 30 } 31 32 /* Microsoft Azure Network Adapter (MANA) functions */ 33 34 static int mana_open(struct net_device *ndev) 35 { 36 struct mana_port_context *apc = netdev_priv(ndev); 37 int err; 38 39 err = mana_alloc_queues(ndev); 40 if (err) 41 return err; 42 43 apc->port_is_up = true; 44 45 /* Ensure port state updated before txq state */ 46 smp_wmb(); 47 48 netif_carrier_on(ndev); 49 netif_tx_wake_all_queues(ndev); 50 51 return 0; 52 } 53 54 static int mana_close(struct net_device *ndev) 55 { 56 struct mana_port_context *apc = netdev_priv(ndev); 57 58 if (!apc->port_is_up) 59 return 0; 60 61 return mana_detach(ndev, true); 62 } 63 64 static bool mana_can_tx(struct gdma_queue *wq) 65 { 66 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 67 } 68 69 static unsigned int mana_checksum_info(struct sk_buff *skb) 70 { 71 if (skb->protocol == htons(ETH_P_IP)) { 72 struct iphdr *ip = ip_hdr(skb); 73 74 if (ip->protocol == IPPROTO_TCP) 75 return IPPROTO_TCP; 76 77 if (ip->protocol == IPPROTO_UDP) 78 return IPPROTO_UDP; 79 } else if (skb->protocol == htons(ETH_P_IPV6)) { 80 struct ipv6hdr *ip6 = ipv6_hdr(skb); 81 82 if (ip6->nexthdr == IPPROTO_TCP) 83 return IPPROTO_TCP; 84 85 if (ip6->nexthdr == IPPROTO_UDP) 86 return IPPROTO_UDP; 87 } 88 89 /* No csum offloading */ 90 return 0; 91 } 92 93 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 94 struct mana_tx_package *tp) 95 { 96 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 97 struct gdma_dev *gd = apc->ac->gdma_dev; 98 struct gdma_context *gc; 99 struct device *dev; 100 skb_frag_t *frag; 101 dma_addr_t da; 102 int i; 103 104 gc = gd->gdma_context; 105 dev = gc->dev; 106 da = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 107 108 if (dma_mapping_error(dev, da)) 109 return -ENOMEM; 110 111 ash->dma_handle[0] = da; 112 ash->size[0] = skb_headlen(skb); 113 114 tp->wqe_req.sgl[0].address = ash->dma_handle[0]; 115 tp->wqe_req.sgl[0].mem_key = gd->gpa_mkey; 116 tp->wqe_req.sgl[0].size = ash->size[0]; 117 118 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 119 frag = &skb_shinfo(skb)->frags[i]; 120 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 121 DMA_TO_DEVICE); 122 123 if (dma_mapping_error(dev, da)) 124 goto frag_err; 125 126 ash->dma_handle[i + 1] = da; 127 ash->size[i + 1] = skb_frag_size(frag); 128 129 tp->wqe_req.sgl[i + 1].address = ash->dma_handle[i + 1]; 130 tp->wqe_req.sgl[i + 1].mem_key = gd->gpa_mkey; 131 tp->wqe_req.sgl[i + 1].size = ash->size[i + 1]; 132 } 133 134 return 0; 135 136 frag_err: 137 for (i = i - 1; i >= 0; i--) 138 dma_unmap_page(dev, ash->dma_handle[i + 1], ash->size[i + 1], 139 DMA_TO_DEVICE); 140 141 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE); 142 143 return -ENOMEM; 144 } 145 146 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 147 { 148 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 149 struct mana_port_context *apc = netdev_priv(ndev); 150 u16 txq_idx = skb_get_queue_mapping(skb); 151 struct gdma_dev *gd = apc->ac->gdma_dev; 152 bool ipv4 = false, ipv6 = false; 153 struct mana_tx_package pkg = {}; 154 struct netdev_queue *net_txq; 155 struct mana_stats_tx *tx_stats; 156 struct gdma_queue *gdma_sq; 157 unsigned int csum_type; 158 struct mana_txq *txq; 159 struct mana_cq *cq; 160 int err, len; 161 u16 ihs; 162 163 if (unlikely(!apc->port_is_up)) 164 goto tx_drop; 165 166 if (skb_cow_head(skb, MANA_HEADROOM)) 167 goto tx_drop_count; 168 169 txq = &apc->tx_qp[txq_idx].txq; 170 gdma_sq = txq->gdma_sq; 171 cq = &apc->tx_qp[txq_idx].tx_cq; 172 tx_stats = &txq->stats; 173 174 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 175 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 176 177 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 178 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 179 pkt_fmt = MANA_LONG_PKT_FMT; 180 } else { 181 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 182 } 183 184 if (skb_vlan_tag_present(skb)) { 185 pkt_fmt = MANA_LONG_PKT_FMT; 186 pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; 187 pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); 188 pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); 189 pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); 190 } 191 192 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 193 194 if (pkt_fmt == MANA_SHORT_PKT_FMT) { 195 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 196 u64_stats_update_begin(&tx_stats->syncp); 197 tx_stats->short_pkt_fmt++; 198 u64_stats_update_end(&tx_stats->syncp); 199 } else { 200 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 201 u64_stats_update_begin(&tx_stats->syncp); 202 tx_stats->long_pkt_fmt++; 203 u64_stats_update_end(&tx_stats->syncp); 204 } 205 206 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 207 pkg.wqe_req.flags = 0; 208 pkg.wqe_req.client_data_unit = 0; 209 210 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 211 WARN_ON_ONCE(pkg.wqe_req.num_sge > MAX_TX_WQE_SGL_ENTRIES); 212 213 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 214 pkg.wqe_req.sgl = pkg.sgl_array; 215 } else { 216 pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge, 217 sizeof(struct gdma_sge), 218 GFP_ATOMIC); 219 if (!pkg.sgl_ptr) 220 goto tx_drop_count; 221 222 pkg.wqe_req.sgl = pkg.sgl_ptr; 223 } 224 225 if (skb->protocol == htons(ETH_P_IP)) 226 ipv4 = true; 227 else if (skb->protocol == htons(ETH_P_IPV6)) 228 ipv6 = true; 229 230 if (skb_is_gso(skb)) { 231 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 232 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 233 234 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 235 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 236 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 237 238 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 239 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 240 if (ipv4) { 241 ip_hdr(skb)->tot_len = 0; 242 ip_hdr(skb)->check = 0; 243 tcp_hdr(skb)->check = 244 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 245 ip_hdr(skb)->daddr, 0, 246 IPPROTO_TCP, 0); 247 } else { 248 ipv6_hdr(skb)->payload_len = 0; 249 tcp_hdr(skb)->check = 250 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 251 &ipv6_hdr(skb)->daddr, 0, 252 IPPROTO_TCP, 0); 253 } 254 255 if (skb->encapsulation) { 256 ihs = skb_inner_tcp_all_headers(skb); 257 u64_stats_update_begin(&tx_stats->syncp); 258 tx_stats->tso_inner_packets++; 259 tx_stats->tso_inner_bytes += skb->len - ihs; 260 u64_stats_update_end(&tx_stats->syncp); 261 } else { 262 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 263 ihs = skb_transport_offset(skb) + sizeof(struct udphdr); 264 } else { 265 ihs = skb_tcp_all_headers(skb); 266 if (ipv6_has_hopopt_jumbo(skb)) 267 ihs -= sizeof(struct hop_jumbo_hdr); 268 } 269 270 u64_stats_update_begin(&tx_stats->syncp); 271 tx_stats->tso_packets++; 272 tx_stats->tso_bytes += skb->len - ihs; 273 u64_stats_update_end(&tx_stats->syncp); 274 } 275 276 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 277 csum_type = mana_checksum_info(skb); 278 279 u64_stats_update_begin(&tx_stats->syncp); 280 tx_stats->csum_partial++; 281 u64_stats_update_end(&tx_stats->syncp); 282 283 if (csum_type == IPPROTO_TCP) { 284 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 285 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 286 287 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 288 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 289 290 } else if (csum_type == IPPROTO_UDP) { 291 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 292 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 293 294 pkg.tx_oob.s_oob.comp_udp_csum = 1; 295 } else { 296 /* Can't do offload of this type of checksum */ 297 if (skb_checksum_help(skb)) 298 goto free_sgl_ptr; 299 } 300 } 301 302 if (mana_map_skb(skb, apc, &pkg)) { 303 u64_stats_update_begin(&tx_stats->syncp); 304 tx_stats->mana_map_err++; 305 u64_stats_update_end(&tx_stats->syncp); 306 goto free_sgl_ptr; 307 } 308 309 skb_queue_tail(&txq->pending_skbs, skb); 310 311 len = skb->len; 312 net_txq = netdev_get_tx_queue(ndev, txq_idx); 313 314 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 315 (struct gdma_posted_wqe_info *)skb->cb); 316 if (!mana_can_tx(gdma_sq)) { 317 netif_tx_stop_queue(net_txq); 318 apc->eth_stats.stop_queue++; 319 } 320 321 if (err) { 322 (void)skb_dequeue_tail(&txq->pending_skbs); 323 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 324 err = NETDEV_TX_BUSY; 325 goto tx_busy; 326 } 327 328 err = NETDEV_TX_OK; 329 atomic_inc(&txq->pending_sends); 330 331 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 332 333 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 334 skb = NULL; 335 336 tx_stats = &txq->stats; 337 u64_stats_update_begin(&tx_stats->syncp); 338 tx_stats->packets++; 339 tx_stats->bytes += len; 340 u64_stats_update_end(&tx_stats->syncp); 341 342 tx_busy: 343 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 344 netif_tx_wake_queue(net_txq); 345 apc->eth_stats.wake_queue++; 346 } 347 348 kfree(pkg.sgl_ptr); 349 return err; 350 351 free_sgl_ptr: 352 kfree(pkg.sgl_ptr); 353 tx_drop_count: 354 ndev->stats.tx_dropped++; 355 tx_drop: 356 dev_kfree_skb_any(skb); 357 return NETDEV_TX_OK; 358 } 359 360 static void mana_get_stats64(struct net_device *ndev, 361 struct rtnl_link_stats64 *st) 362 { 363 struct mana_port_context *apc = netdev_priv(ndev); 364 unsigned int num_queues = apc->num_queues; 365 struct mana_stats_rx *rx_stats; 366 struct mana_stats_tx *tx_stats; 367 unsigned int start; 368 u64 packets, bytes; 369 int q; 370 371 if (!apc->port_is_up) 372 return; 373 374 netdev_stats_to_stats64(st, &ndev->stats); 375 376 for (q = 0; q < num_queues; q++) { 377 rx_stats = &apc->rxqs[q]->stats; 378 379 do { 380 start = u64_stats_fetch_begin(&rx_stats->syncp); 381 packets = rx_stats->packets; 382 bytes = rx_stats->bytes; 383 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 384 385 st->rx_packets += packets; 386 st->rx_bytes += bytes; 387 } 388 389 for (q = 0; q < num_queues; q++) { 390 tx_stats = &apc->tx_qp[q].txq.stats; 391 392 do { 393 start = u64_stats_fetch_begin(&tx_stats->syncp); 394 packets = tx_stats->packets; 395 bytes = tx_stats->bytes; 396 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 397 398 st->tx_packets += packets; 399 st->tx_bytes += bytes; 400 } 401 } 402 403 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 404 int old_q) 405 { 406 struct mana_port_context *apc = netdev_priv(ndev); 407 u32 hash = skb_get_hash(skb); 408 struct sock *sk = skb->sk; 409 int txq; 410 411 txq = apc->indir_table[hash & MANA_INDIRECT_TABLE_MASK]; 412 413 if (txq != old_q && sk && sk_fullsock(sk) && 414 rcu_access_pointer(sk->sk_dst_cache)) 415 sk_tx_queue_set(sk, txq); 416 417 return txq; 418 } 419 420 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 421 struct net_device *sb_dev) 422 { 423 int txq; 424 425 if (ndev->real_num_tx_queues == 1) 426 return 0; 427 428 txq = sk_tx_queue_get(skb->sk); 429 430 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 431 if (skb_rx_queue_recorded(skb)) 432 txq = skb_get_rx_queue(skb); 433 else 434 txq = mana_get_tx_queue(ndev, skb, txq); 435 } 436 437 return txq; 438 } 439 440 /* Release pre-allocated RX buffers */ 441 static void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) 442 { 443 struct device *dev; 444 int i; 445 446 dev = mpc->ac->gdma_dev->gdma_context->dev; 447 448 if (!mpc->rxbufs_pre) 449 goto out1; 450 451 if (!mpc->das_pre) 452 goto out2; 453 454 while (mpc->rxbpre_total) { 455 i = --mpc->rxbpre_total; 456 dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, 457 DMA_FROM_DEVICE); 458 put_page(virt_to_head_page(mpc->rxbufs_pre[i])); 459 } 460 461 kfree(mpc->das_pre); 462 mpc->das_pre = NULL; 463 464 out2: 465 kfree(mpc->rxbufs_pre); 466 mpc->rxbufs_pre = NULL; 467 468 out1: 469 mpc->rxbpre_datasize = 0; 470 mpc->rxbpre_alloc_size = 0; 471 mpc->rxbpre_headroom = 0; 472 } 473 474 /* Get a buffer from the pre-allocated RX buffers */ 475 static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) 476 { 477 struct net_device *ndev = rxq->ndev; 478 struct mana_port_context *mpc; 479 void *va; 480 481 mpc = netdev_priv(ndev); 482 483 if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { 484 netdev_err(ndev, "No RX pre-allocated bufs\n"); 485 return NULL; 486 } 487 488 /* Check sizes to catch unexpected coding error */ 489 if (mpc->rxbpre_datasize != rxq->datasize) { 490 netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", 491 mpc->rxbpre_datasize, rxq->datasize); 492 return NULL; 493 } 494 495 if (mpc->rxbpre_alloc_size != rxq->alloc_size) { 496 netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", 497 mpc->rxbpre_alloc_size, rxq->alloc_size); 498 return NULL; 499 } 500 501 if (mpc->rxbpre_headroom != rxq->headroom) { 502 netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", 503 mpc->rxbpre_headroom, rxq->headroom); 504 return NULL; 505 } 506 507 mpc->rxbpre_total--; 508 509 *da = mpc->das_pre[mpc->rxbpre_total]; 510 va = mpc->rxbufs_pre[mpc->rxbpre_total]; 511 mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; 512 513 /* Deallocate the array after all buffers are gone */ 514 if (!mpc->rxbpre_total) 515 mana_pre_dealloc_rxbufs(mpc); 516 517 return va; 518 } 519 520 /* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ 521 static void mana_get_rxbuf_cfg(int mtu, u32 *datasize, u32 *alloc_size, 522 u32 *headroom) 523 { 524 if (mtu > MANA_XDP_MTU_MAX) 525 *headroom = 0; /* no support for XDP */ 526 else 527 *headroom = XDP_PACKET_HEADROOM; 528 529 *alloc_size = mtu + MANA_RXBUF_PAD + *headroom; 530 531 *datasize = ALIGN(mtu + ETH_HLEN, MANA_RX_DATA_ALIGN); 532 } 533 534 static int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu) 535 { 536 struct device *dev; 537 struct page *page; 538 dma_addr_t da; 539 int num_rxb; 540 void *va; 541 int i; 542 543 mana_get_rxbuf_cfg(new_mtu, &mpc->rxbpre_datasize, 544 &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom); 545 546 dev = mpc->ac->gdma_dev->gdma_context->dev; 547 548 num_rxb = mpc->num_queues * RX_BUFFERS_PER_QUEUE; 549 550 WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); 551 mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); 552 if (!mpc->rxbufs_pre) 553 goto error; 554 555 mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL); 556 if (!mpc->das_pre) 557 goto error; 558 559 mpc->rxbpre_total = 0; 560 561 for (i = 0; i < num_rxb; i++) { 562 if (mpc->rxbpre_alloc_size > PAGE_SIZE) { 563 va = netdev_alloc_frag(mpc->rxbpre_alloc_size); 564 if (!va) 565 goto error; 566 567 page = virt_to_head_page(va); 568 /* Check if the frag falls back to single page */ 569 if (compound_order(page) < 570 get_order(mpc->rxbpre_alloc_size)) { 571 put_page(page); 572 goto error; 573 } 574 } else { 575 page = dev_alloc_page(); 576 if (!page) 577 goto error; 578 579 va = page_to_virt(page); 580 } 581 582 da = dma_map_single(dev, va + mpc->rxbpre_headroom, 583 mpc->rxbpre_datasize, DMA_FROM_DEVICE); 584 if (dma_mapping_error(dev, da)) { 585 put_page(virt_to_head_page(va)); 586 goto error; 587 } 588 589 mpc->rxbufs_pre[i] = va; 590 mpc->das_pre[i] = da; 591 mpc->rxbpre_total = i + 1; 592 } 593 594 return 0; 595 596 error: 597 mana_pre_dealloc_rxbufs(mpc); 598 return -ENOMEM; 599 } 600 601 static int mana_change_mtu(struct net_device *ndev, int new_mtu) 602 { 603 struct mana_port_context *mpc = netdev_priv(ndev); 604 unsigned int old_mtu = ndev->mtu; 605 int err; 606 607 /* Pre-allocate buffers to prevent failure in mana_attach later */ 608 err = mana_pre_alloc_rxbufs(mpc, new_mtu); 609 if (err) { 610 netdev_err(ndev, "Insufficient memory for new MTU\n"); 611 return err; 612 } 613 614 err = mana_detach(ndev, false); 615 if (err) { 616 netdev_err(ndev, "mana_detach failed: %d\n", err); 617 goto out; 618 } 619 620 ndev->mtu = new_mtu; 621 622 err = mana_attach(ndev); 623 if (err) { 624 netdev_err(ndev, "mana_attach failed: %d\n", err); 625 ndev->mtu = old_mtu; 626 } 627 628 out: 629 mana_pre_dealloc_rxbufs(mpc); 630 return err; 631 } 632 633 static const struct net_device_ops mana_devops = { 634 .ndo_open = mana_open, 635 .ndo_stop = mana_close, 636 .ndo_select_queue = mana_select_queue, 637 .ndo_start_xmit = mana_start_xmit, 638 .ndo_validate_addr = eth_validate_addr, 639 .ndo_get_stats64 = mana_get_stats64, 640 .ndo_bpf = mana_bpf, 641 .ndo_xdp_xmit = mana_xdp_xmit, 642 .ndo_change_mtu = mana_change_mtu, 643 }; 644 645 static void mana_cleanup_port_context(struct mana_port_context *apc) 646 { 647 kfree(apc->rxqs); 648 apc->rxqs = NULL; 649 } 650 651 static int mana_init_port_context(struct mana_port_context *apc) 652 { 653 apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *), 654 GFP_KERNEL); 655 656 return !apc->rxqs ? -ENOMEM : 0; 657 } 658 659 static int mana_send_request(struct mana_context *ac, void *in_buf, 660 u32 in_len, void *out_buf, u32 out_len) 661 { 662 struct gdma_context *gc = ac->gdma_dev->gdma_context; 663 struct gdma_resp_hdr *resp = out_buf; 664 struct gdma_req_hdr *req = in_buf; 665 struct device *dev = gc->dev; 666 static atomic_t activity_id; 667 int err; 668 669 req->dev_id = gc->mana.dev_id; 670 req->activity_id = atomic_inc_return(&activity_id); 671 672 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 673 out_buf); 674 if (err || resp->status) { 675 dev_err(dev, "Failed to send mana message: %d, 0x%x\n", 676 err, resp->status); 677 return err ? err : -EPROTO; 678 } 679 680 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 681 req->activity_id != resp->activity_id) { 682 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 683 req->dev_id.as_uint32, resp->dev_id.as_uint32, 684 req->activity_id, resp->activity_id); 685 return -EPROTO; 686 } 687 688 return 0; 689 } 690 691 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 692 const enum mana_command_code expected_code, 693 const u32 min_size) 694 { 695 if (resp_hdr->response.msg_type != expected_code) 696 return -EPROTO; 697 698 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 699 return -EPROTO; 700 701 if (resp_hdr->response.msg_size < min_size) 702 return -EPROTO; 703 704 return 0; 705 } 706 707 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 708 { 709 struct mana_register_hw_vport_resp resp = {}; 710 struct mana_register_hw_vport_req req = {}; 711 int err; 712 713 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 714 sizeof(req), sizeof(resp)); 715 req.attached_gfid = 1; 716 req.is_pf_default_vport = 1; 717 req.allow_all_ether_types = 1; 718 719 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 720 sizeof(resp)); 721 if (err) { 722 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 723 return err; 724 } 725 726 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 727 sizeof(resp)); 728 if (err || resp.hdr.status) { 729 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 730 err, resp.hdr.status); 731 return err ? err : -EPROTO; 732 } 733 734 apc->port_handle = resp.hw_vport_handle; 735 return 0; 736 } 737 738 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 739 { 740 struct mana_deregister_hw_vport_resp resp = {}; 741 struct mana_deregister_hw_vport_req req = {}; 742 int err; 743 744 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 745 sizeof(req), sizeof(resp)); 746 req.hw_vport_handle = apc->port_handle; 747 748 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 749 sizeof(resp)); 750 if (err) { 751 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 752 err); 753 return; 754 } 755 756 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 757 sizeof(resp)); 758 if (err || resp.hdr.status) 759 netdev_err(apc->ndev, 760 "Failed to deregister hw vPort: %d, 0x%x\n", 761 err, resp.hdr.status); 762 } 763 764 static int mana_pf_register_filter(struct mana_port_context *apc) 765 { 766 struct mana_register_filter_resp resp = {}; 767 struct mana_register_filter_req req = {}; 768 int err; 769 770 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 771 sizeof(req), sizeof(resp)); 772 req.vport = apc->port_handle; 773 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 774 775 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 776 sizeof(resp)); 777 if (err) { 778 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 779 return err; 780 } 781 782 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 783 sizeof(resp)); 784 if (err || resp.hdr.status) { 785 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 786 err, resp.hdr.status); 787 return err ? err : -EPROTO; 788 } 789 790 apc->pf_filter_handle = resp.filter_handle; 791 return 0; 792 } 793 794 static void mana_pf_deregister_filter(struct mana_port_context *apc) 795 { 796 struct mana_deregister_filter_resp resp = {}; 797 struct mana_deregister_filter_req req = {}; 798 int err; 799 800 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 801 sizeof(req), sizeof(resp)); 802 req.filter_handle = apc->pf_filter_handle; 803 804 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 805 sizeof(resp)); 806 if (err) { 807 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 808 err); 809 return; 810 } 811 812 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 813 sizeof(resp)); 814 if (err || resp.hdr.status) 815 netdev_err(apc->ndev, 816 "Failed to deregister filter: %d, 0x%x\n", 817 err, resp.hdr.status); 818 } 819 820 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, 821 u32 proto_minor_ver, u32 proto_micro_ver, 822 u16 *max_num_vports) 823 { 824 struct gdma_context *gc = ac->gdma_dev->gdma_context; 825 struct mana_query_device_cfg_resp resp = {}; 826 struct mana_query_device_cfg_req req = {}; 827 struct device *dev = gc->dev; 828 int err = 0; 829 830 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 831 sizeof(req), sizeof(resp)); 832 833 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 834 835 req.proto_major_ver = proto_major_ver; 836 req.proto_minor_ver = proto_minor_ver; 837 req.proto_micro_ver = proto_micro_ver; 838 839 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp)); 840 if (err) { 841 dev_err(dev, "Failed to query config: %d", err); 842 return err; 843 } 844 845 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 846 sizeof(resp)); 847 if (err || resp.hdr.status) { 848 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 849 resp.hdr.status); 850 if (!err) 851 err = -EPROTO; 852 return err; 853 } 854 855 *max_num_vports = resp.max_num_vports; 856 857 if (resp.hdr.response.msg_version == GDMA_MESSAGE_V2) 858 gc->adapter_mtu = resp.adapter_mtu; 859 else 860 gc->adapter_mtu = ETH_FRAME_LEN; 861 862 return 0; 863 } 864 865 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 866 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 867 { 868 struct mana_query_vport_cfg_resp resp = {}; 869 struct mana_query_vport_cfg_req req = {}; 870 int err; 871 872 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 873 sizeof(req), sizeof(resp)); 874 875 req.vport_index = vport_index; 876 877 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 878 sizeof(resp)); 879 if (err) 880 return err; 881 882 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 883 sizeof(resp)); 884 if (err) 885 return err; 886 887 if (resp.hdr.status) 888 return -EPROTO; 889 890 *max_sq = resp.max_num_sq; 891 *max_rq = resp.max_num_rq; 892 *num_indir_entry = resp.num_indirection_ent; 893 894 apc->port_handle = resp.vport; 895 ether_addr_copy(apc->mac_addr, resp.mac_addr); 896 897 return 0; 898 } 899 900 void mana_uncfg_vport(struct mana_port_context *apc) 901 { 902 mutex_lock(&apc->vport_mutex); 903 apc->vport_use_count--; 904 WARN_ON(apc->vport_use_count < 0); 905 mutex_unlock(&apc->vport_mutex); 906 } 907 EXPORT_SYMBOL_NS(mana_uncfg_vport, NET_MANA); 908 909 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 910 u32 doorbell_pg_id) 911 { 912 struct mana_config_vport_resp resp = {}; 913 struct mana_config_vport_req req = {}; 914 int err; 915 916 /* This function is used to program the Ethernet port in the hardware 917 * table. It can be called from the Ethernet driver or the RDMA driver. 918 * 919 * For Ethernet usage, the hardware supports only one active user on a 920 * physical port. The driver checks on the port usage before programming 921 * the hardware when creating the RAW QP (RDMA driver) or exposing the 922 * device to kernel NET layer (Ethernet driver). 923 * 924 * Because the RDMA driver doesn't know in advance which QP type the 925 * user will create, it exposes the device with all its ports. The user 926 * may not be able to create RAW QP on a port if this port is already 927 * in used by the Ethernet driver from the kernel. 928 * 929 * This physical port limitation only applies to the RAW QP. For RC QP, 930 * the hardware doesn't have this limitation. The user can create RC 931 * QPs on a physical port up to the hardware limits independent of the 932 * Ethernet usage on the same port. 933 */ 934 mutex_lock(&apc->vport_mutex); 935 if (apc->vport_use_count > 0) { 936 mutex_unlock(&apc->vport_mutex); 937 return -EBUSY; 938 } 939 apc->vport_use_count++; 940 mutex_unlock(&apc->vport_mutex); 941 942 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 943 sizeof(req), sizeof(resp)); 944 req.vport = apc->port_handle; 945 req.pdid = protection_dom_id; 946 req.doorbell_pageid = doorbell_pg_id; 947 948 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 949 sizeof(resp)); 950 if (err) { 951 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 952 goto out; 953 } 954 955 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 956 sizeof(resp)); 957 if (err || resp.hdr.status) { 958 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 959 err, resp.hdr.status); 960 if (!err) 961 err = -EPROTO; 962 963 goto out; 964 } 965 966 apc->tx_shortform_allowed = resp.short_form_allowed; 967 apc->tx_vp_offset = resp.tx_vport_offset; 968 969 netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n", 970 apc->port_handle, protection_dom_id, doorbell_pg_id); 971 out: 972 if (err) 973 mana_uncfg_vport(apc); 974 975 return err; 976 } 977 EXPORT_SYMBOL_NS(mana_cfg_vport, NET_MANA); 978 979 static int mana_cfg_vport_steering(struct mana_port_context *apc, 980 enum TRI_STATE rx, 981 bool update_default_rxobj, bool update_key, 982 bool update_tab) 983 { 984 u16 num_entries = MANA_INDIRECT_TABLE_SIZE; 985 struct mana_cfg_rx_steer_req_v2 *req; 986 struct mana_cfg_rx_steer_resp resp = {}; 987 struct net_device *ndev = apc->ndev; 988 mana_handle_t *req_indir_tab; 989 u32 req_buf_size; 990 int err; 991 992 req_buf_size = sizeof(*req) + sizeof(mana_handle_t) * num_entries; 993 req = kzalloc(req_buf_size, GFP_KERNEL); 994 if (!req) 995 return -ENOMEM; 996 997 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 998 sizeof(resp)); 999 1000 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 1001 1002 req->vport = apc->port_handle; 1003 req->num_indir_entries = num_entries; 1004 req->indir_tab_offset = sizeof(*req); 1005 req->rx_enable = rx; 1006 req->rss_enable = apc->rss_state; 1007 req->update_default_rxobj = update_default_rxobj; 1008 req->update_hashkey = update_key; 1009 req->update_indir_tab = update_tab; 1010 req->default_rxobj = apc->default_rxobj; 1011 req->cqe_coalescing_enable = 0; 1012 1013 if (update_key) 1014 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1015 1016 if (update_tab) { 1017 req_indir_tab = (mana_handle_t *)(req + 1); 1018 memcpy(req_indir_tab, apc->rxobj_table, 1019 req->num_indir_entries * sizeof(mana_handle_t)); 1020 } 1021 1022 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1023 sizeof(resp)); 1024 if (err) { 1025 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 1026 goto out; 1027 } 1028 1029 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1030 sizeof(resp)); 1031 if (err) { 1032 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 1033 goto out; 1034 } 1035 1036 if (resp.hdr.status) { 1037 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 1038 resp.hdr.status); 1039 err = -EPROTO; 1040 } 1041 1042 netdev_info(ndev, "Configured steering vPort %llu entries %u\n", 1043 apc->port_handle, num_entries); 1044 out: 1045 kfree(req); 1046 return err; 1047 } 1048 1049 int mana_create_wq_obj(struct mana_port_context *apc, 1050 mana_handle_t vport, 1051 u32 wq_type, struct mana_obj_spec *wq_spec, 1052 struct mana_obj_spec *cq_spec, 1053 mana_handle_t *wq_obj) 1054 { 1055 struct mana_create_wqobj_resp resp = {}; 1056 struct mana_create_wqobj_req req = {}; 1057 struct net_device *ndev = apc->ndev; 1058 int err; 1059 1060 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1061 sizeof(req), sizeof(resp)); 1062 req.vport = vport; 1063 req.wq_type = wq_type; 1064 req.wq_gdma_region = wq_spec->gdma_region; 1065 req.cq_gdma_region = cq_spec->gdma_region; 1066 req.wq_size = wq_spec->queue_size; 1067 req.cq_size = cq_spec->queue_size; 1068 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1069 req.cq_parent_qid = cq_spec->attached_eq; 1070 1071 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1072 sizeof(resp)); 1073 if (err) { 1074 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 1075 goto out; 1076 } 1077 1078 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1079 sizeof(resp)); 1080 if (err || resp.hdr.status) { 1081 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1082 resp.hdr.status); 1083 if (!err) 1084 err = -EPROTO; 1085 goto out; 1086 } 1087 1088 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1089 netdev_err(ndev, "Got an invalid WQ object handle\n"); 1090 err = -EPROTO; 1091 goto out; 1092 } 1093 1094 *wq_obj = resp.wq_obj; 1095 wq_spec->queue_index = resp.wq_id; 1096 cq_spec->queue_index = resp.cq_id; 1097 1098 return 0; 1099 out: 1100 return err; 1101 } 1102 EXPORT_SYMBOL_NS(mana_create_wq_obj, NET_MANA); 1103 1104 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1105 mana_handle_t wq_obj) 1106 { 1107 struct mana_destroy_wqobj_resp resp = {}; 1108 struct mana_destroy_wqobj_req req = {}; 1109 struct net_device *ndev = apc->ndev; 1110 int err; 1111 1112 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1113 sizeof(req), sizeof(resp)); 1114 req.wq_type = wq_type; 1115 req.wq_obj_handle = wq_obj; 1116 1117 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1118 sizeof(resp)); 1119 if (err) { 1120 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 1121 return; 1122 } 1123 1124 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1125 sizeof(resp)); 1126 if (err || resp.hdr.status) 1127 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 1128 resp.hdr.status); 1129 } 1130 EXPORT_SYMBOL_NS(mana_destroy_wq_obj, NET_MANA); 1131 1132 static void mana_destroy_eq(struct mana_context *ac) 1133 { 1134 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1135 struct gdma_queue *eq; 1136 int i; 1137 1138 if (!ac->eqs) 1139 return; 1140 1141 for (i = 0; i < gc->max_num_queues; i++) { 1142 eq = ac->eqs[i].eq; 1143 if (!eq) 1144 continue; 1145 1146 mana_gd_destroy_queue(gc, eq); 1147 } 1148 1149 kfree(ac->eqs); 1150 ac->eqs = NULL; 1151 } 1152 1153 static int mana_create_eq(struct mana_context *ac) 1154 { 1155 struct gdma_dev *gd = ac->gdma_dev; 1156 struct gdma_context *gc = gd->gdma_context; 1157 struct gdma_queue_spec spec = {}; 1158 int err; 1159 int i; 1160 1161 ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq), 1162 GFP_KERNEL); 1163 if (!ac->eqs) 1164 return -ENOMEM; 1165 1166 spec.type = GDMA_EQ; 1167 spec.monitor_avl_buf = false; 1168 spec.queue_size = EQ_SIZE; 1169 spec.eq.callback = NULL; 1170 spec.eq.context = ac->eqs; 1171 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1172 1173 for (i = 0; i < gc->max_num_queues; i++) { 1174 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); 1175 if (err) 1176 goto out; 1177 } 1178 1179 return 0; 1180 out: 1181 mana_destroy_eq(ac); 1182 return err; 1183 } 1184 1185 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1186 { 1187 struct mana_fence_rq_resp resp = {}; 1188 struct mana_fence_rq_req req = {}; 1189 int err; 1190 1191 init_completion(&rxq->fence_event); 1192 1193 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1194 sizeof(req), sizeof(resp)); 1195 req.wq_obj_handle = rxq->rxobj; 1196 1197 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1198 sizeof(resp)); 1199 if (err) { 1200 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 1201 rxq->rxq_idx, err); 1202 return err; 1203 } 1204 1205 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1206 if (err || resp.hdr.status) { 1207 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1208 rxq->rxq_idx, err, resp.hdr.status); 1209 if (!err) 1210 err = -EPROTO; 1211 1212 return err; 1213 } 1214 1215 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 1216 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 1217 rxq->rxq_idx); 1218 return -ETIMEDOUT; 1219 } 1220 1221 return 0; 1222 } 1223 1224 static void mana_fence_rqs(struct mana_port_context *apc) 1225 { 1226 unsigned int rxq_idx; 1227 struct mana_rxq *rxq; 1228 int err; 1229 1230 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1231 rxq = apc->rxqs[rxq_idx]; 1232 err = mana_fence_rq(apc, rxq); 1233 1234 /* In case of any error, use sleep instead. */ 1235 if (err) 1236 msleep(100); 1237 } 1238 } 1239 1240 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 1241 { 1242 u32 used_space_old; 1243 u32 used_space_new; 1244 1245 used_space_old = wq->head - wq->tail; 1246 used_space_new = wq->head - (wq->tail + num_units); 1247 1248 if (WARN_ON_ONCE(used_space_new > used_space_old)) 1249 return -ERANGE; 1250 1251 wq->tail += num_units; 1252 return 0; 1253 } 1254 1255 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 1256 { 1257 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 1258 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1259 struct device *dev = gc->dev; 1260 int i; 1261 1262 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE); 1263 1264 for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++) 1265 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 1266 DMA_TO_DEVICE); 1267 } 1268 1269 static void mana_poll_tx_cq(struct mana_cq *cq) 1270 { 1271 struct gdma_comp *completions = cq->gdma_comp_buf; 1272 struct gdma_posted_wqe_info *wqe_info; 1273 unsigned int pkt_transmitted = 0; 1274 unsigned int wqe_unit_cnt = 0; 1275 struct mana_txq *txq = cq->txq; 1276 struct mana_port_context *apc; 1277 struct netdev_queue *net_txq; 1278 struct gdma_queue *gdma_wq; 1279 unsigned int avail_space; 1280 struct net_device *ndev; 1281 struct sk_buff *skb; 1282 bool txq_stopped; 1283 int comp_read; 1284 int i; 1285 1286 ndev = txq->ndev; 1287 apc = netdev_priv(ndev); 1288 1289 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1290 CQE_POLLING_BUFFER); 1291 1292 if (comp_read < 1) 1293 return; 1294 1295 for (i = 0; i < comp_read; i++) { 1296 struct mana_tx_comp_oob *cqe_oob; 1297 1298 if (WARN_ON_ONCE(!completions[i].is_sq)) 1299 return; 1300 1301 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 1302 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 1303 MANA_CQE_COMPLETION)) 1304 return; 1305 1306 switch (cqe_oob->cqe_hdr.cqe_type) { 1307 case CQE_TX_OKAY: 1308 break; 1309 1310 case CQE_TX_SA_DROP: 1311 case CQE_TX_MTU_DROP: 1312 case CQE_TX_INVALID_OOB: 1313 case CQE_TX_INVALID_ETH_TYPE: 1314 case CQE_TX_HDR_PROCESSING_ERROR: 1315 case CQE_TX_VF_DISABLED: 1316 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1317 case CQE_TX_VPORT_DISABLED: 1318 case CQE_TX_VLAN_TAGGING_VIOLATION: 1319 WARN_ONCE(1, "TX: CQE error %d: ignored.\n", 1320 cqe_oob->cqe_hdr.cqe_type); 1321 apc->eth_stats.tx_cqe_err++; 1322 break; 1323 1324 default: 1325 /* If the CQE type is unexpected, log an error, assert, 1326 * and go through the error path. 1327 */ 1328 WARN_ONCE(1, "TX: Unexpected CQE type %d: HW BUG?\n", 1329 cqe_oob->cqe_hdr.cqe_type); 1330 apc->eth_stats.tx_cqe_unknown_type++; 1331 return; 1332 } 1333 1334 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 1335 return; 1336 1337 skb = skb_dequeue(&txq->pending_skbs); 1338 if (WARN_ON_ONCE(!skb)) 1339 return; 1340 1341 wqe_info = (struct gdma_posted_wqe_info *)skb->cb; 1342 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1343 1344 mana_unmap_skb(skb, apc); 1345 1346 napi_consume_skb(skb, cq->budget); 1347 1348 pkt_transmitted++; 1349 } 1350 1351 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 1352 return; 1353 1354 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1355 1356 gdma_wq = txq->gdma_sq; 1357 avail_space = mana_gd_wq_avail_space(gdma_wq); 1358 1359 /* Ensure tail updated before checking q stop */ 1360 smp_mb(); 1361 1362 net_txq = txq->net_txq; 1363 txq_stopped = netif_tx_queue_stopped(net_txq); 1364 1365 /* Ensure checking txq_stopped before apc->port_is_up. */ 1366 smp_rmb(); 1367 1368 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1369 netif_tx_wake_queue(net_txq); 1370 apc->eth_stats.wake_queue++; 1371 } 1372 1373 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1374 WARN_ON_ONCE(1); 1375 1376 cq->work_done = pkt_transmitted; 1377 } 1378 1379 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 1380 { 1381 struct mana_recv_buf_oob *recv_buf_oob; 1382 u32 curr_index; 1383 int err; 1384 1385 curr_index = rxq->buf_index++; 1386 if (rxq->buf_index == rxq->num_rx_buf) 1387 rxq->buf_index = 0; 1388 1389 recv_buf_oob = &rxq->rx_oobs[curr_index]; 1390 1391 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1392 &recv_buf_oob->wqe_inf); 1393 if (WARN_ON_ONCE(err)) 1394 return; 1395 1396 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 1397 } 1398 1399 static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, 1400 uint pkt_len, struct xdp_buff *xdp) 1401 { 1402 struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); 1403 1404 if (!skb) 1405 return NULL; 1406 1407 if (xdp->data_hard_start) { 1408 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1409 skb_put(skb, xdp->data_end - xdp->data); 1410 return skb; 1411 } 1412 1413 skb_reserve(skb, rxq->headroom); 1414 skb_put(skb, pkt_len); 1415 1416 return skb; 1417 } 1418 1419 static void mana_rx_skb(void *buf_va, bool from_pool, 1420 struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) 1421 { 1422 struct mana_stats_rx *rx_stats = &rxq->stats; 1423 struct net_device *ndev = rxq->ndev; 1424 uint pkt_len = cqe->ppi[0].pkt_len; 1425 u16 rxq_idx = rxq->rxq_idx; 1426 struct napi_struct *napi; 1427 struct xdp_buff xdp = {}; 1428 struct sk_buff *skb; 1429 u32 hash_value; 1430 u32 act; 1431 1432 rxq->rx_cq.work_done++; 1433 napi = &rxq->rx_cq.napi; 1434 1435 if (!buf_va) { 1436 ++ndev->stats.rx_dropped; 1437 return; 1438 } 1439 1440 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 1441 1442 if (act == XDP_REDIRECT && !rxq->xdp_rc) 1443 return; 1444 1445 if (act != XDP_PASS && act != XDP_TX) 1446 goto drop_xdp; 1447 1448 skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); 1449 1450 if (!skb) 1451 goto drop; 1452 1453 if (from_pool) 1454 skb_mark_for_recycle(skb); 1455 1456 skb->dev = napi->dev; 1457 1458 skb->protocol = eth_type_trans(skb, ndev); 1459 skb_checksum_none_assert(skb); 1460 skb_record_rx_queue(skb, rxq_idx); 1461 1462 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 1463 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 1464 skb->ip_summed = CHECKSUM_UNNECESSARY; 1465 } 1466 1467 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 1468 hash_value = cqe->ppi[0].pkt_hash; 1469 1470 if (cqe->rx_hashtype & MANA_HASH_L4) 1471 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); 1472 else 1473 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); 1474 } 1475 1476 if (cqe->rx_vlantag_present) { 1477 u16 vlan_tci = cqe->rx_vlan_id; 1478 1479 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 1480 } 1481 1482 u64_stats_update_begin(&rx_stats->syncp); 1483 rx_stats->packets++; 1484 rx_stats->bytes += pkt_len; 1485 1486 if (act == XDP_TX) 1487 rx_stats->xdp_tx++; 1488 u64_stats_update_end(&rx_stats->syncp); 1489 1490 if (act == XDP_TX) { 1491 skb_set_queue_mapping(skb, rxq_idx); 1492 mana_xdp_tx(skb, ndev); 1493 return; 1494 } 1495 1496 napi_gro_receive(napi, skb); 1497 1498 return; 1499 1500 drop_xdp: 1501 u64_stats_update_begin(&rx_stats->syncp); 1502 rx_stats->xdp_drop++; 1503 u64_stats_update_end(&rx_stats->syncp); 1504 1505 drop: 1506 if (from_pool) { 1507 page_pool_recycle_direct(rxq->page_pool, 1508 virt_to_head_page(buf_va)); 1509 } else { 1510 WARN_ON_ONCE(rxq->xdp_save_va); 1511 /* Save for reuse */ 1512 rxq->xdp_save_va = buf_va; 1513 } 1514 1515 ++ndev->stats.rx_dropped; 1516 1517 return; 1518 } 1519 1520 static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, 1521 dma_addr_t *da, bool *from_pool, bool is_napi) 1522 { 1523 struct page *page; 1524 void *va; 1525 1526 *from_pool = false; 1527 1528 /* Reuse XDP dropped page if available */ 1529 if (rxq->xdp_save_va) { 1530 va = rxq->xdp_save_va; 1531 rxq->xdp_save_va = NULL; 1532 } else if (rxq->alloc_size > PAGE_SIZE) { 1533 if (is_napi) 1534 va = napi_alloc_frag(rxq->alloc_size); 1535 else 1536 va = netdev_alloc_frag(rxq->alloc_size); 1537 1538 if (!va) 1539 return NULL; 1540 1541 page = virt_to_head_page(va); 1542 /* Check if the frag falls back to single page */ 1543 if (compound_order(page) < get_order(rxq->alloc_size)) { 1544 put_page(page); 1545 return NULL; 1546 } 1547 } else { 1548 page = page_pool_dev_alloc_pages(rxq->page_pool); 1549 if (!page) 1550 return NULL; 1551 1552 *from_pool = true; 1553 va = page_to_virt(page); 1554 } 1555 1556 *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, 1557 DMA_FROM_DEVICE); 1558 if (dma_mapping_error(dev, *da)) { 1559 if (*from_pool) 1560 page_pool_put_full_page(rxq->page_pool, page, false); 1561 else 1562 put_page(virt_to_head_page(va)); 1563 1564 return NULL; 1565 } 1566 1567 return va; 1568 } 1569 1570 /* Allocate frag for rx buffer, and save the old buf */ 1571 static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, 1572 struct mana_recv_buf_oob *rxoob, void **old_buf, 1573 bool *old_fp) 1574 { 1575 bool from_pool; 1576 dma_addr_t da; 1577 void *va; 1578 1579 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, true); 1580 if (!va) 1581 return; 1582 1583 dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, 1584 DMA_FROM_DEVICE); 1585 *old_buf = rxoob->buf_va; 1586 *old_fp = rxoob->from_pool; 1587 1588 rxoob->buf_va = va; 1589 rxoob->sgl[0].address = da; 1590 rxoob->from_pool = from_pool; 1591 } 1592 1593 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 1594 struct gdma_comp *cqe) 1595 { 1596 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 1597 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 1598 struct net_device *ndev = rxq->ndev; 1599 struct mana_recv_buf_oob *rxbuf_oob; 1600 struct mana_port_context *apc; 1601 struct device *dev = gc->dev; 1602 void *old_buf = NULL; 1603 u32 curr, pktlen; 1604 bool old_fp; 1605 1606 apc = netdev_priv(ndev); 1607 1608 switch (oob->cqe_hdr.cqe_type) { 1609 case CQE_RX_OKAY: 1610 break; 1611 1612 case CQE_RX_TRUNCATED: 1613 ++ndev->stats.rx_dropped; 1614 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 1615 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 1616 goto drop; 1617 1618 case CQE_RX_COALESCED_4: 1619 netdev_err(ndev, "RX coalescing is unsupported\n"); 1620 apc->eth_stats.rx_coalesced_err++; 1621 return; 1622 1623 case CQE_RX_OBJECT_FENCE: 1624 complete(&rxq->fence_event); 1625 return; 1626 1627 default: 1628 netdev_err(ndev, "Unknown RX CQE type = %d\n", 1629 oob->cqe_hdr.cqe_type); 1630 apc->eth_stats.rx_cqe_unknown_type++; 1631 return; 1632 } 1633 1634 pktlen = oob->ppi[0].pkt_len; 1635 1636 if (pktlen == 0) { 1637 /* data packets should never have packetlength of zero */ 1638 netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 1639 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 1640 return; 1641 } 1642 1643 curr = rxq->buf_index; 1644 rxbuf_oob = &rxq->rx_oobs[curr]; 1645 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 1646 1647 mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); 1648 1649 /* Unsuccessful refill will have old_buf == NULL. 1650 * In this case, mana_rx_skb() will drop the packet. 1651 */ 1652 mana_rx_skb(old_buf, old_fp, oob, rxq); 1653 1654 drop: 1655 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 1656 1657 mana_post_pkt_rxq(rxq); 1658 } 1659 1660 static void mana_poll_rx_cq(struct mana_cq *cq) 1661 { 1662 struct gdma_comp *comp = cq->gdma_comp_buf; 1663 struct mana_rxq *rxq = cq->rxq; 1664 int comp_read, i; 1665 1666 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); 1667 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 1668 1669 rxq->xdp_flush = false; 1670 1671 for (i = 0; i < comp_read; i++) { 1672 if (WARN_ON_ONCE(comp[i].is_sq)) 1673 return; 1674 1675 /* verify recv cqe references the right rxq */ 1676 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 1677 return; 1678 1679 mana_process_rx_cqe(rxq, cq, &comp[i]); 1680 } 1681 1682 if (comp_read > 0) { 1683 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 1684 1685 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 1686 } 1687 1688 if (rxq->xdp_flush) 1689 xdp_do_flush(); 1690 } 1691 1692 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 1693 { 1694 struct mana_cq *cq = context; 1695 u8 arm_bit; 1696 int w; 1697 1698 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 1699 1700 if (cq->type == MANA_CQ_TYPE_RX) 1701 mana_poll_rx_cq(cq); 1702 else 1703 mana_poll_tx_cq(cq); 1704 1705 w = cq->work_done; 1706 1707 if (w < cq->budget && 1708 napi_complete_done(&cq->napi, w)) { 1709 arm_bit = SET_ARM_BIT; 1710 } else { 1711 arm_bit = 0; 1712 } 1713 1714 mana_gd_ring_cq(gdma_queue, arm_bit); 1715 1716 return w; 1717 } 1718 1719 static int mana_poll(struct napi_struct *napi, int budget) 1720 { 1721 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 1722 int w; 1723 1724 cq->work_done = 0; 1725 cq->budget = budget; 1726 1727 w = mana_cq_handler(cq, cq->gdma_cq); 1728 1729 return min(w, budget); 1730 } 1731 1732 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 1733 { 1734 struct mana_cq *cq = context; 1735 1736 napi_schedule_irqoff(&cq->napi); 1737 } 1738 1739 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 1740 { 1741 struct gdma_dev *gd = apc->ac->gdma_dev; 1742 1743 if (!cq->gdma_cq) 1744 return; 1745 1746 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 1747 } 1748 1749 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 1750 { 1751 struct gdma_dev *gd = apc->ac->gdma_dev; 1752 1753 if (!txq->gdma_sq) 1754 return; 1755 1756 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 1757 } 1758 1759 static void mana_destroy_txq(struct mana_port_context *apc) 1760 { 1761 struct napi_struct *napi; 1762 int i; 1763 1764 if (!apc->tx_qp) 1765 return; 1766 1767 for (i = 0; i < apc->num_queues; i++) { 1768 napi = &apc->tx_qp[i].tx_cq.napi; 1769 napi_synchronize(napi); 1770 napi_disable(napi); 1771 netif_napi_del(napi); 1772 1773 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); 1774 1775 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); 1776 1777 mana_deinit_txq(apc, &apc->tx_qp[i].txq); 1778 } 1779 1780 kfree(apc->tx_qp); 1781 apc->tx_qp = NULL; 1782 } 1783 1784 static int mana_create_txq(struct mana_port_context *apc, 1785 struct net_device *net) 1786 { 1787 struct mana_context *ac = apc->ac; 1788 struct gdma_dev *gd = ac->gdma_dev; 1789 struct mana_obj_spec wq_spec; 1790 struct mana_obj_spec cq_spec; 1791 struct gdma_queue_spec spec; 1792 struct gdma_context *gc; 1793 struct mana_txq *txq; 1794 struct mana_cq *cq; 1795 u32 txq_size; 1796 u32 cq_size; 1797 int err; 1798 int i; 1799 1800 apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp), 1801 GFP_KERNEL); 1802 if (!apc->tx_qp) 1803 return -ENOMEM; 1804 1805 /* The minimum size of the WQE is 32 bytes, hence 1806 * MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs 1807 * the SQ can store. This value is then used to size other queues 1808 * to prevent overflow. 1809 */ 1810 txq_size = MAX_SEND_BUFFERS_PER_QUEUE * 32; 1811 BUILD_BUG_ON(!PAGE_ALIGNED(txq_size)); 1812 1813 cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE; 1814 cq_size = PAGE_ALIGN(cq_size); 1815 1816 gc = gd->gdma_context; 1817 1818 for (i = 0; i < apc->num_queues; i++) { 1819 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE; 1820 1821 /* Create SQ */ 1822 txq = &apc->tx_qp[i].txq; 1823 1824 u64_stats_init(&txq->stats.syncp); 1825 txq->ndev = net; 1826 txq->net_txq = netdev_get_tx_queue(net, i); 1827 txq->vp_offset = apc->tx_vp_offset; 1828 skb_queue_head_init(&txq->pending_skbs); 1829 1830 memset(&spec, 0, sizeof(spec)); 1831 spec.type = GDMA_SQ; 1832 spec.monitor_avl_buf = true; 1833 spec.queue_size = txq_size; 1834 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 1835 if (err) 1836 goto out; 1837 1838 /* Create SQ's CQ */ 1839 cq = &apc->tx_qp[i].tx_cq; 1840 cq->type = MANA_CQ_TYPE_TX; 1841 1842 cq->txq = txq; 1843 1844 memset(&spec, 0, sizeof(spec)); 1845 spec.type = GDMA_CQ; 1846 spec.monitor_avl_buf = false; 1847 spec.queue_size = cq_size; 1848 spec.cq.callback = mana_schedule_napi; 1849 spec.cq.parent_eq = ac->eqs[i].eq; 1850 spec.cq.context = cq; 1851 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 1852 if (err) 1853 goto out; 1854 1855 memset(&wq_spec, 0, sizeof(wq_spec)); 1856 memset(&cq_spec, 0, sizeof(cq_spec)); 1857 1858 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 1859 wq_spec.queue_size = txq->gdma_sq->queue_size; 1860 1861 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 1862 cq_spec.queue_size = cq->gdma_cq->queue_size; 1863 cq_spec.modr_ctx_id = 0; 1864 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 1865 1866 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 1867 &wq_spec, &cq_spec, 1868 &apc->tx_qp[i].tx_object); 1869 1870 if (err) 1871 goto out; 1872 1873 txq->gdma_sq->id = wq_spec.queue_index; 1874 cq->gdma_cq->id = cq_spec.queue_index; 1875 1876 txq->gdma_sq->mem_info.dma_region_handle = 1877 GDMA_INVALID_DMA_REGION; 1878 cq->gdma_cq->mem_info.dma_region_handle = 1879 GDMA_INVALID_DMA_REGION; 1880 1881 txq->gdma_txq_id = txq->gdma_sq->id; 1882 1883 cq->gdma_id = cq->gdma_cq->id; 1884 1885 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 1886 err = -EINVAL; 1887 goto out; 1888 } 1889 1890 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 1891 1892 netif_napi_add_tx(net, &cq->napi, mana_poll); 1893 napi_enable(&cq->napi); 1894 1895 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 1896 } 1897 1898 return 0; 1899 out: 1900 mana_destroy_txq(apc); 1901 return err; 1902 } 1903 1904 static void mana_destroy_rxq(struct mana_port_context *apc, 1905 struct mana_rxq *rxq, bool validate_state) 1906 1907 { 1908 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1909 struct mana_recv_buf_oob *rx_oob; 1910 struct device *dev = gc->dev; 1911 struct napi_struct *napi; 1912 struct page *page; 1913 int i; 1914 1915 if (!rxq) 1916 return; 1917 1918 napi = &rxq->rx_cq.napi; 1919 1920 if (validate_state) 1921 napi_synchronize(napi); 1922 1923 napi_disable(napi); 1924 1925 xdp_rxq_info_unreg(&rxq->xdp_rxq); 1926 1927 netif_napi_del(napi); 1928 1929 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 1930 1931 mana_deinit_cq(apc, &rxq->rx_cq); 1932 1933 if (rxq->xdp_save_va) 1934 put_page(virt_to_head_page(rxq->xdp_save_va)); 1935 1936 for (i = 0; i < rxq->num_rx_buf; i++) { 1937 rx_oob = &rxq->rx_oobs[i]; 1938 1939 if (!rx_oob->buf_va) 1940 continue; 1941 1942 dma_unmap_single(dev, rx_oob->sgl[0].address, 1943 rx_oob->sgl[0].size, DMA_FROM_DEVICE); 1944 1945 page = virt_to_head_page(rx_oob->buf_va); 1946 1947 if (rx_oob->from_pool) 1948 page_pool_put_full_page(rxq->page_pool, page, false); 1949 else 1950 put_page(page); 1951 1952 rx_oob->buf_va = NULL; 1953 } 1954 1955 page_pool_destroy(rxq->page_pool); 1956 1957 if (rxq->gdma_rq) 1958 mana_gd_destroy_queue(gc, rxq->gdma_rq); 1959 1960 kfree(rxq); 1961 } 1962 1963 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, 1964 struct mana_rxq *rxq, struct device *dev) 1965 { 1966 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 1967 bool from_pool = false; 1968 dma_addr_t da; 1969 void *va; 1970 1971 if (mpc->rxbufs_pre) 1972 va = mana_get_rxbuf_pre(rxq, &da); 1973 else 1974 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, false); 1975 1976 if (!va) 1977 return -ENOMEM; 1978 1979 rx_oob->buf_va = va; 1980 rx_oob->from_pool = from_pool; 1981 1982 rx_oob->sgl[0].address = da; 1983 rx_oob->sgl[0].size = rxq->datasize; 1984 rx_oob->sgl[0].mem_key = mem_key; 1985 1986 return 0; 1987 } 1988 1989 #define MANA_WQE_HEADER_SIZE 16 1990 #define MANA_WQE_SGE_SIZE 16 1991 1992 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 1993 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 1994 { 1995 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1996 struct mana_recv_buf_oob *rx_oob; 1997 struct device *dev = gc->dev; 1998 u32 buf_idx; 1999 int ret; 2000 2001 WARN_ON(rxq->datasize == 0); 2002 2003 *rxq_size = 0; 2004 *cq_size = 0; 2005 2006 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2007 rx_oob = &rxq->rx_oobs[buf_idx]; 2008 memset(rx_oob, 0, sizeof(*rx_oob)); 2009 2010 rx_oob->num_sge = 1; 2011 2012 ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, 2013 dev); 2014 if (ret) 2015 return ret; 2016 2017 rx_oob->wqe_req.sgl = rx_oob->sgl; 2018 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 2019 rx_oob->wqe_req.inline_oob_size = 0; 2020 rx_oob->wqe_req.inline_oob_data = NULL; 2021 rx_oob->wqe_req.flags = 0; 2022 rx_oob->wqe_req.client_data_unit = 0; 2023 2024 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 2025 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 2026 *cq_size += COMP_ENTRY_SIZE; 2027 } 2028 2029 return 0; 2030 } 2031 2032 static int mana_push_wqe(struct mana_rxq *rxq) 2033 { 2034 struct mana_recv_buf_oob *rx_oob; 2035 u32 buf_idx; 2036 int err; 2037 2038 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2039 rx_oob = &rxq->rx_oobs[buf_idx]; 2040 2041 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 2042 &rx_oob->wqe_inf); 2043 if (err) 2044 return -ENOSPC; 2045 } 2046 2047 return 0; 2048 } 2049 2050 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) 2051 { 2052 struct page_pool_params pprm = {}; 2053 int ret; 2054 2055 pprm.pool_size = RX_BUFFERS_PER_QUEUE; 2056 pprm.nid = gc->numa_node; 2057 pprm.napi = &rxq->rx_cq.napi; 2058 2059 rxq->page_pool = page_pool_create(&pprm); 2060 2061 if (IS_ERR(rxq->page_pool)) { 2062 ret = PTR_ERR(rxq->page_pool); 2063 rxq->page_pool = NULL; 2064 return ret; 2065 } 2066 2067 return 0; 2068 } 2069 2070 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 2071 u32 rxq_idx, struct mana_eq *eq, 2072 struct net_device *ndev) 2073 { 2074 struct gdma_dev *gd = apc->ac->gdma_dev; 2075 struct mana_obj_spec wq_spec; 2076 struct mana_obj_spec cq_spec; 2077 struct gdma_queue_spec spec; 2078 struct mana_cq *cq = NULL; 2079 struct gdma_context *gc; 2080 u32 cq_size, rq_size; 2081 struct mana_rxq *rxq; 2082 int err; 2083 2084 gc = gd->gdma_context; 2085 2086 rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE), 2087 GFP_KERNEL); 2088 if (!rxq) 2089 return NULL; 2090 2091 rxq->ndev = ndev; 2092 rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE; 2093 rxq->rxq_idx = rxq_idx; 2094 rxq->rxobj = INVALID_MANA_HANDLE; 2095 2096 mana_get_rxbuf_cfg(ndev->mtu, &rxq->datasize, &rxq->alloc_size, 2097 &rxq->headroom); 2098 2099 /* Create page pool for RX queue */ 2100 err = mana_create_page_pool(rxq, gc); 2101 if (err) { 2102 netdev_err(ndev, "Create page pool err:%d\n", err); 2103 goto out; 2104 } 2105 2106 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 2107 if (err) 2108 goto out; 2109 2110 rq_size = PAGE_ALIGN(rq_size); 2111 cq_size = PAGE_ALIGN(cq_size); 2112 2113 /* Create RQ */ 2114 memset(&spec, 0, sizeof(spec)); 2115 spec.type = GDMA_RQ; 2116 spec.monitor_avl_buf = true; 2117 spec.queue_size = rq_size; 2118 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 2119 if (err) 2120 goto out; 2121 2122 /* Create RQ's CQ */ 2123 cq = &rxq->rx_cq; 2124 cq->type = MANA_CQ_TYPE_RX; 2125 cq->rxq = rxq; 2126 2127 memset(&spec, 0, sizeof(spec)); 2128 spec.type = GDMA_CQ; 2129 spec.monitor_avl_buf = false; 2130 spec.queue_size = cq_size; 2131 spec.cq.callback = mana_schedule_napi; 2132 spec.cq.parent_eq = eq->eq; 2133 spec.cq.context = cq; 2134 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2135 if (err) 2136 goto out; 2137 2138 memset(&wq_spec, 0, sizeof(wq_spec)); 2139 memset(&cq_spec, 0, sizeof(cq_spec)); 2140 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 2141 wq_spec.queue_size = rxq->gdma_rq->queue_size; 2142 2143 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2144 cq_spec.queue_size = cq->gdma_cq->queue_size; 2145 cq_spec.modr_ctx_id = 0; 2146 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2147 2148 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 2149 &wq_spec, &cq_spec, &rxq->rxobj); 2150 if (err) 2151 goto out; 2152 2153 rxq->gdma_rq->id = wq_spec.queue_index; 2154 cq->gdma_cq->id = cq_spec.queue_index; 2155 2156 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2157 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2158 2159 rxq->gdma_id = rxq->gdma_rq->id; 2160 cq->gdma_id = cq->gdma_cq->id; 2161 2162 err = mana_push_wqe(rxq); 2163 if (err) 2164 goto out; 2165 2166 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2167 err = -EINVAL; 2168 goto out; 2169 } 2170 2171 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2172 2173 netif_napi_add_weight(ndev, &cq->napi, mana_poll, 1); 2174 2175 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 2176 cq->napi.napi_id)); 2177 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, 2178 rxq->page_pool)); 2179 2180 napi_enable(&cq->napi); 2181 2182 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2183 out: 2184 if (!err) 2185 return rxq; 2186 2187 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 2188 2189 mana_destroy_rxq(apc, rxq, false); 2190 2191 if (cq) 2192 mana_deinit_cq(apc, cq); 2193 2194 return NULL; 2195 } 2196 2197 static int mana_add_rx_queues(struct mana_port_context *apc, 2198 struct net_device *ndev) 2199 { 2200 struct mana_context *ac = apc->ac; 2201 struct mana_rxq *rxq; 2202 int err = 0; 2203 int i; 2204 2205 for (i = 0; i < apc->num_queues; i++) { 2206 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev); 2207 if (!rxq) { 2208 err = -ENOMEM; 2209 goto out; 2210 } 2211 2212 u64_stats_init(&rxq->stats.syncp); 2213 2214 apc->rxqs[i] = rxq; 2215 } 2216 2217 apc->default_rxobj = apc->rxqs[0]->rxobj; 2218 out: 2219 return err; 2220 } 2221 2222 static void mana_destroy_vport(struct mana_port_context *apc) 2223 { 2224 struct gdma_dev *gd = apc->ac->gdma_dev; 2225 struct mana_rxq *rxq; 2226 u32 rxq_idx; 2227 2228 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 2229 rxq = apc->rxqs[rxq_idx]; 2230 if (!rxq) 2231 continue; 2232 2233 mana_destroy_rxq(apc, rxq, true); 2234 apc->rxqs[rxq_idx] = NULL; 2235 } 2236 2237 mana_destroy_txq(apc); 2238 mana_uncfg_vport(apc); 2239 2240 if (gd->gdma_context->is_pf) 2241 mana_pf_deregister_hw_vport(apc); 2242 } 2243 2244 static int mana_create_vport(struct mana_port_context *apc, 2245 struct net_device *net) 2246 { 2247 struct gdma_dev *gd = apc->ac->gdma_dev; 2248 int err; 2249 2250 apc->default_rxobj = INVALID_MANA_HANDLE; 2251 2252 if (gd->gdma_context->is_pf) { 2253 err = mana_pf_register_hw_vport(apc); 2254 if (err) 2255 return err; 2256 } 2257 2258 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell); 2259 if (err) 2260 return err; 2261 2262 return mana_create_txq(apc, net); 2263 } 2264 2265 static void mana_rss_table_init(struct mana_port_context *apc) 2266 { 2267 int i; 2268 2269 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) 2270 apc->indir_table[i] = 2271 ethtool_rxfh_indir_default(i, apc->num_queues); 2272 } 2273 2274 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 2275 bool update_hash, bool update_tab) 2276 { 2277 u32 queue_idx; 2278 int err; 2279 int i; 2280 2281 if (update_tab) { 2282 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) { 2283 queue_idx = apc->indir_table[i]; 2284 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 2285 } 2286 } 2287 2288 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 2289 if (err) 2290 return err; 2291 2292 mana_fence_rqs(apc); 2293 2294 return 0; 2295 } 2296 2297 static int mana_init_port(struct net_device *ndev) 2298 { 2299 struct mana_port_context *apc = netdev_priv(ndev); 2300 u32 max_txq, max_rxq, max_queues; 2301 int port_idx = apc->port_idx; 2302 u32 num_indirect_entries; 2303 int err; 2304 2305 err = mana_init_port_context(apc); 2306 if (err) 2307 return err; 2308 2309 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 2310 &num_indirect_entries); 2311 if (err) { 2312 netdev_err(ndev, "Failed to query info for vPort %d\n", 2313 port_idx); 2314 goto reset_apc; 2315 } 2316 2317 max_queues = min_t(u32, max_txq, max_rxq); 2318 if (apc->max_queues > max_queues) 2319 apc->max_queues = max_queues; 2320 2321 if (apc->num_queues > apc->max_queues) 2322 apc->num_queues = apc->max_queues; 2323 2324 eth_hw_addr_set(ndev, apc->mac_addr); 2325 2326 return 0; 2327 2328 reset_apc: 2329 kfree(apc->rxqs); 2330 apc->rxqs = NULL; 2331 return err; 2332 } 2333 2334 int mana_alloc_queues(struct net_device *ndev) 2335 { 2336 struct mana_port_context *apc = netdev_priv(ndev); 2337 struct gdma_dev *gd = apc->ac->gdma_dev; 2338 int err; 2339 2340 err = mana_create_vport(apc, ndev); 2341 if (err) 2342 return err; 2343 2344 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 2345 if (err) 2346 goto destroy_vport; 2347 2348 err = mana_add_rx_queues(apc, ndev); 2349 if (err) 2350 goto destroy_vport; 2351 2352 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 2353 2354 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 2355 if (err) 2356 goto destroy_vport; 2357 2358 mana_rss_table_init(apc); 2359 2360 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 2361 if (err) 2362 goto destroy_vport; 2363 2364 if (gd->gdma_context->is_pf) { 2365 err = mana_pf_register_filter(apc); 2366 if (err) 2367 goto destroy_vport; 2368 } 2369 2370 mana_chn_setxdp(apc, mana_xdp_get(apc)); 2371 2372 return 0; 2373 2374 destroy_vport: 2375 mana_destroy_vport(apc); 2376 return err; 2377 } 2378 2379 int mana_attach(struct net_device *ndev) 2380 { 2381 struct mana_port_context *apc = netdev_priv(ndev); 2382 int err; 2383 2384 ASSERT_RTNL(); 2385 2386 err = mana_init_port(ndev); 2387 if (err) 2388 return err; 2389 2390 if (apc->port_st_save) { 2391 err = mana_alloc_queues(ndev); 2392 if (err) { 2393 mana_cleanup_port_context(apc); 2394 return err; 2395 } 2396 } 2397 2398 apc->port_is_up = apc->port_st_save; 2399 2400 /* Ensure port state updated before txq state */ 2401 smp_wmb(); 2402 2403 if (apc->port_is_up) 2404 netif_carrier_on(ndev); 2405 2406 netif_device_attach(ndev); 2407 2408 return 0; 2409 } 2410 2411 static int mana_dealloc_queues(struct net_device *ndev) 2412 { 2413 struct mana_port_context *apc = netdev_priv(ndev); 2414 struct gdma_dev *gd = apc->ac->gdma_dev; 2415 struct mana_txq *txq; 2416 int i, err; 2417 2418 if (apc->port_is_up) 2419 return -EINVAL; 2420 2421 mana_chn_setxdp(apc, NULL); 2422 2423 if (gd->gdma_context->is_pf) 2424 mana_pf_deregister_filter(apc); 2425 2426 /* No packet can be transmitted now since apc->port_is_up is false. 2427 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 2428 * a txq because it may not timely see apc->port_is_up being cleared 2429 * to false, but it doesn't matter since mana_start_xmit() drops any 2430 * new packets due to apc->port_is_up being false. 2431 * 2432 * Drain all the in-flight TX packets 2433 */ 2434 for (i = 0; i < apc->num_queues; i++) { 2435 txq = &apc->tx_qp[i].txq; 2436 2437 while (atomic_read(&txq->pending_sends) > 0) 2438 usleep_range(1000, 2000); 2439 } 2440 2441 /* We're 100% sure the queues can no longer be woken up, because 2442 * we're sure now mana_poll_tx_cq() can't be running. 2443 */ 2444 2445 apc->rss_state = TRI_STATE_FALSE; 2446 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); 2447 if (err) { 2448 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 2449 return err; 2450 } 2451 2452 mana_destroy_vport(apc); 2453 2454 return 0; 2455 } 2456 2457 int mana_detach(struct net_device *ndev, bool from_close) 2458 { 2459 struct mana_port_context *apc = netdev_priv(ndev); 2460 int err; 2461 2462 ASSERT_RTNL(); 2463 2464 apc->port_st_save = apc->port_is_up; 2465 apc->port_is_up = false; 2466 2467 /* Ensure port state updated before txq state */ 2468 smp_wmb(); 2469 2470 netif_tx_disable(ndev); 2471 netif_carrier_off(ndev); 2472 2473 if (apc->port_st_save) { 2474 err = mana_dealloc_queues(ndev); 2475 if (err) 2476 return err; 2477 } 2478 2479 if (!from_close) { 2480 netif_device_detach(ndev); 2481 mana_cleanup_port_context(apc); 2482 } 2483 2484 return 0; 2485 } 2486 2487 static int mana_probe_port(struct mana_context *ac, int port_idx, 2488 struct net_device **ndev_storage) 2489 { 2490 struct gdma_context *gc = ac->gdma_dev->gdma_context; 2491 struct mana_port_context *apc; 2492 struct net_device *ndev; 2493 int err; 2494 2495 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 2496 gc->max_num_queues); 2497 if (!ndev) 2498 return -ENOMEM; 2499 2500 *ndev_storage = ndev; 2501 2502 apc = netdev_priv(ndev); 2503 apc->ac = ac; 2504 apc->ndev = ndev; 2505 apc->max_queues = gc->max_num_queues; 2506 apc->num_queues = gc->max_num_queues; 2507 apc->port_handle = INVALID_MANA_HANDLE; 2508 apc->pf_filter_handle = INVALID_MANA_HANDLE; 2509 apc->port_idx = port_idx; 2510 2511 mutex_init(&apc->vport_mutex); 2512 apc->vport_use_count = 0; 2513 2514 ndev->netdev_ops = &mana_devops; 2515 ndev->ethtool_ops = &mana_ethtool_ops; 2516 ndev->mtu = ETH_DATA_LEN; 2517 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; 2518 ndev->min_mtu = ETH_MIN_MTU; 2519 ndev->needed_headroom = MANA_HEADROOM; 2520 ndev->dev_port = port_idx; 2521 SET_NETDEV_DEV(ndev, gc->dev); 2522 2523 netif_carrier_off(ndev); 2524 2525 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 2526 2527 err = mana_init_port(ndev); 2528 if (err) 2529 goto free_net; 2530 2531 netdev_lockdep_set_classes(ndev); 2532 2533 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 2534 ndev->hw_features |= NETIF_F_RXCSUM; 2535 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 2536 ndev->hw_features |= NETIF_F_RXHASH; 2537 ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | 2538 NETIF_F_HW_VLAN_CTAG_RX; 2539 ndev->vlan_features = ndev->features; 2540 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2541 NETDEV_XDP_ACT_NDO_XMIT; 2542 2543 err = register_netdev(ndev); 2544 if (err) { 2545 netdev_err(ndev, "Unable to register netdev.\n"); 2546 goto reset_apc; 2547 } 2548 2549 return 0; 2550 2551 reset_apc: 2552 kfree(apc->rxqs); 2553 apc->rxqs = NULL; 2554 free_net: 2555 *ndev_storage = NULL; 2556 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 2557 free_netdev(ndev); 2558 return err; 2559 } 2560 2561 static void adev_release(struct device *dev) 2562 { 2563 struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); 2564 2565 kfree(madev); 2566 } 2567 2568 static void remove_adev(struct gdma_dev *gd) 2569 { 2570 struct auxiliary_device *adev = gd->adev; 2571 int id = adev->id; 2572 2573 auxiliary_device_delete(adev); 2574 auxiliary_device_uninit(adev); 2575 2576 mana_adev_idx_free(id); 2577 gd->adev = NULL; 2578 } 2579 2580 static int add_adev(struct gdma_dev *gd) 2581 { 2582 struct auxiliary_device *adev; 2583 struct mana_adev *madev; 2584 int ret; 2585 2586 madev = kzalloc(sizeof(*madev), GFP_KERNEL); 2587 if (!madev) 2588 return -ENOMEM; 2589 2590 adev = &madev->adev; 2591 ret = mana_adev_idx_alloc(); 2592 if (ret < 0) 2593 goto idx_fail; 2594 adev->id = ret; 2595 2596 adev->name = "rdma"; 2597 adev->dev.parent = gd->gdma_context->dev; 2598 adev->dev.release = adev_release; 2599 madev->mdev = gd; 2600 2601 ret = auxiliary_device_init(adev); 2602 if (ret) 2603 goto init_fail; 2604 2605 ret = auxiliary_device_add(adev); 2606 if (ret) 2607 goto add_fail; 2608 2609 gd->adev = adev; 2610 return 0; 2611 2612 add_fail: 2613 auxiliary_device_uninit(adev); 2614 2615 init_fail: 2616 mana_adev_idx_free(adev->id); 2617 2618 idx_fail: 2619 kfree(madev); 2620 2621 return ret; 2622 } 2623 2624 int mana_probe(struct gdma_dev *gd, bool resuming) 2625 { 2626 struct gdma_context *gc = gd->gdma_context; 2627 struct mana_context *ac = gd->driver_data; 2628 struct device *dev = gc->dev; 2629 u16 num_ports = 0; 2630 int err; 2631 int i; 2632 2633 dev_info(dev, 2634 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 2635 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 2636 2637 err = mana_gd_register_device(gd); 2638 if (err) 2639 return err; 2640 2641 if (!resuming) { 2642 ac = kzalloc(sizeof(*ac), GFP_KERNEL); 2643 if (!ac) 2644 return -ENOMEM; 2645 2646 ac->gdma_dev = gd; 2647 gd->driver_data = ac; 2648 } 2649 2650 err = mana_create_eq(ac); 2651 if (err) 2652 goto out; 2653 2654 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, 2655 MANA_MICRO_VERSION, &num_ports); 2656 if (err) 2657 goto out; 2658 2659 if (!resuming) { 2660 ac->num_ports = num_ports; 2661 } else { 2662 if (ac->num_ports != num_ports) { 2663 dev_err(dev, "The number of vPorts changed: %d->%d\n", 2664 ac->num_ports, num_ports); 2665 err = -EPROTO; 2666 goto out; 2667 } 2668 } 2669 2670 if (ac->num_ports == 0) 2671 dev_err(dev, "Failed to detect any vPort\n"); 2672 2673 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 2674 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 2675 2676 if (!resuming) { 2677 for (i = 0; i < ac->num_ports; i++) { 2678 err = mana_probe_port(ac, i, &ac->ports[i]); 2679 if (err) 2680 break; 2681 } 2682 } else { 2683 for (i = 0; i < ac->num_ports; i++) { 2684 rtnl_lock(); 2685 err = mana_attach(ac->ports[i]); 2686 rtnl_unlock(); 2687 if (err) 2688 break; 2689 } 2690 } 2691 2692 err = add_adev(gd); 2693 out: 2694 if (err) 2695 mana_remove(gd, false); 2696 2697 return err; 2698 } 2699 2700 void mana_remove(struct gdma_dev *gd, bool suspending) 2701 { 2702 struct gdma_context *gc = gd->gdma_context; 2703 struct mana_context *ac = gd->driver_data; 2704 struct device *dev = gc->dev; 2705 struct net_device *ndev; 2706 int err; 2707 int i; 2708 2709 /* adev currently doesn't support suspending, always remove it */ 2710 if (gd->adev) 2711 remove_adev(gd); 2712 2713 for (i = 0; i < ac->num_ports; i++) { 2714 ndev = ac->ports[i]; 2715 if (!ndev) { 2716 if (i == 0) 2717 dev_err(dev, "No net device to remove\n"); 2718 goto out; 2719 } 2720 2721 /* All cleanup actions should stay after rtnl_lock(), otherwise 2722 * other functions may access partially cleaned up data. 2723 */ 2724 rtnl_lock(); 2725 2726 err = mana_detach(ndev, false); 2727 if (err) 2728 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 2729 i, err); 2730 2731 if (suspending) { 2732 /* No need to unregister the ndev. */ 2733 rtnl_unlock(); 2734 continue; 2735 } 2736 2737 unregister_netdevice(ndev); 2738 2739 rtnl_unlock(); 2740 2741 free_netdev(ndev); 2742 } 2743 2744 mana_destroy_eq(ac); 2745 out: 2746 mana_gd_deregister_device(gd); 2747 2748 if (suspending) 2749 return; 2750 2751 gd->driver_data = NULL; 2752 gd->gdma_context = NULL; 2753 kfree(ac); 2754 } 2755