1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Ethernet driver 3 * 4 * Copyright (C) 2020 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/interrupt.h> 12 #include <linux/pci.h> 13 #include <net/tso.h> 14 15 #include "otx2_reg.h" 16 #include "otx2_common.h" 17 #include "otx2_struct.h" 18 #include "cn10k.h" 19 20 static void otx2_nix_rq_op_stats(struct queue_stats *stats, 21 struct otx2_nic *pfvf, int qidx) 22 { 23 u64 incr = (u64)qidx << 32; 24 u64 *ptr; 25 26 ptr = (u64 *)otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_OCTS); 27 stats->bytes = otx2_atomic64_add(incr, ptr); 28 29 ptr = (u64 *)otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_PKTS); 30 stats->pkts = otx2_atomic64_add(incr, ptr); 31 } 32 33 static void otx2_nix_sq_op_stats(struct queue_stats *stats, 34 struct otx2_nic *pfvf, int qidx) 35 { 36 u64 incr = (u64)qidx << 32; 37 u64 *ptr; 38 39 ptr = (u64 *)otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_OCTS); 40 stats->bytes = otx2_atomic64_add(incr, ptr); 41 42 ptr = (u64 *)otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_PKTS); 43 stats->pkts = otx2_atomic64_add(incr, ptr); 44 } 45 46 void otx2_update_lmac_stats(struct otx2_nic *pfvf) 47 { 48 struct msg_req *req; 49 50 if (!netif_running(pfvf->netdev)) 51 return; 52 53 mutex_lock(&pfvf->mbox.lock); 54 req = otx2_mbox_alloc_msg_cgx_stats(&pfvf->mbox); 55 if (!req) { 56 mutex_unlock(&pfvf->mbox.lock); 57 return; 58 } 59 60 otx2_sync_mbox_msg(&pfvf->mbox); 61 mutex_unlock(&pfvf->mbox.lock); 62 } 63 64 void otx2_update_lmac_fec_stats(struct otx2_nic *pfvf) 65 { 66 struct msg_req *req; 67 68 if (!netif_running(pfvf->netdev)) 69 return; 70 mutex_lock(&pfvf->mbox.lock); 71 req = otx2_mbox_alloc_msg_cgx_fec_stats(&pfvf->mbox); 72 if (req) 73 otx2_sync_mbox_msg(&pfvf->mbox); 74 mutex_unlock(&pfvf->mbox.lock); 75 } 76 77 int otx2_update_rq_stats(struct otx2_nic *pfvf, int qidx) 78 { 79 struct otx2_rcv_queue *rq = &pfvf->qset.rq[qidx]; 80 81 if (!pfvf->qset.rq) 82 return 0; 83 84 otx2_nix_rq_op_stats(&rq->stats, pfvf, qidx); 85 return 1; 86 } 87 88 int otx2_update_sq_stats(struct otx2_nic *pfvf, int qidx) 89 { 90 struct otx2_snd_queue *sq = &pfvf->qset.sq[qidx]; 91 92 if (!pfvf->qset.sq) 93 return 0; 94 95 otx2_nix_sq_op_stats(&sq->stats, pfvf, qidx); 96 return 1; 97 } 98 99 void otx2_get_dev_stats(struct otx2_nic *pfvf) 100 { 101 struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; 102 103 #define OTX2_GET_RX_STATS(reg) \ 104 otx2_read64(pfvf, NIX_LF_RX_STATX(reg)) 105 #define OTX2_GET_TX_STATS(reg) \ 106 otx2_read64(pfvf, NIX_LF_TX_STATX(reg)) 107 108 dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); 109 dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); 110 dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); 111 dev_stats->rx_mcast_frames = OTX2_GET_RX_STATS(RX_MCAST); 112 dev_stats->rx_ucast_frames = OTX2_GET_RX_STATS(RX_UCAST); 113 dev_stats->rx_frames = dev_stats->rx_bcast_frames + 114 dev_stats->rx_mcast_frames + 115 dev_stats->rx_ucast_frames; 116 117 dev_stats->tx_bytes = OTX2_GET_TX_STATS(TX_OCTS); 118 dev_stats->tx_drops = OTX2_GET_TX_STATS(TX_DROP); 119 dev_stats->tx_bcast_frames = OTX2_GET_TX_STATS(TX_BCAST); 120 dev_stats->tx_mcast_frames = OTX2_GET_TX_STATS(TX_MCAST); 121 dev_stats->tx_ucast_frames = OTX2_GET_TX_STATS(TX_UCAST); 122 dev_stats->tx_frames = dev_stats->tx_bcast_frames + 123 dev_stats->tx_mcast_frames + 124 dev_stats->tx_ucast_frames; 125 } 126 127 void otx2_get_stats64(struct net_device *netdev, 128 struct rtnl_link_stats64 *stats) 129 { 130 struct otx2_nic *pfvf = netdev_priv(netdev); 131 struct otx2_dev_stats *dev_stats; 132 133 otx2_get_dev_stats(pfvf); 134 135 dev_stats = &pfvf->hw.dev_stats; 136 stats->rx_bytes = dev_stats->rx_bytes; 137 stats->rx_packets = dev_stats->rx_frames; 138 stats->rx_dropped = dev_stats->rx_drops; 139 stats->multicast = dev_stats->rx_mcast_frames; 140 141 stats->tx_bytes = dev_stats->tx_bytes; 142 stats->tx_packets = dev_stats->tx_frames; 143 stats->tx_dropped = dev_stats->tx_drops; 144 } 145 EXPORT_SYMBOL(otx2_get_stats64); 146 147 /* Sync MAC address with RVU AF */ 148 static int otx2_hw_set_mac_addr(struct otx2_nic *pfvf, u8 *mac) 149 { 150 struct nix_set_mac_addr *req; 151 int err; 152 153 mutex_lock(&pfvf->mbox.lock); 154 req = otx2_mbox_alloc_msg_nix_set_mac_addr(&pfvf->mbox); 155 if (!req) { 156 mutex_unlock(&pfvf->mbox.lock); 157 return -ENOMEM; 158 } 159 160 ether_addr_copy(req->mac_addr, mac); 161 162 err = otx2_sync_mbox_msg(&pfvf->mbox); 163 mutex_unlock(&pfvf->mbox.lock); 164 return err; 165 } 166 167 static int otx2_hw_get_mac_addr(struct otx2_nic *pfvf, 168 struct net_device *netdev) 169 { 170 struct nix_get_mac_addr_rsp *rsp; 171 struct mbox_msghdr *msghdr; 172 struct msg_req *req; 173 int err; 174 175 mutex_lock(&pfvf->mbox.lock); 176 req = otx2_mbox_alloc_msg_nix_get_mac_addr(&pfvf->mbox); 177 if (!req) { 178 mutex_unlock(&pfvf->mbox.lock); 179 return -ENOMEM; 180 } 181 182 err = otx2_sync_mbox_msg(&pfvf->mbox); 183 if (err) { 184 mutex_unlock(&pfvf->mbox.lock); 185 return err; 186 } 187 188 msghdr = otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 189 if (IS_ERR(msghdr)) { 190 mutex_unlock(&pfvf->mbox.lock); 191 return PTR_ERR(msghdr); 192 } 193 rsp = (struct nix_get_mac_addr_rsp *)msghdr; 194 ether_addr_copy(netdev->dev_addr, rsp->mac_addr); 195 mutex_unlock(&pfvf->mbox.lock); 196 197 return 0; 198 } 199 200 int otx2_set_mac_address(struct net_device *netdev, void *p) 201 { 202 struct otx2_nic *pfvf = netdev_priv(netdev); 203 struct sockaddr *addr = p; 204 205 if (!is_valid_ether_addr(addr->sa_data)) 206 return -EADDRNOTAVAIL; 207 208 if (!otx2_hw_set_mac_addr(pfvf, addr->sa_data)) { 209 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 210 /* update dmac field in vlan offload rule */ 211 if (pfvf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) 212 otx2_install_rxvlan_offload_flow(pfvf); 213 } else { 214 return -EPERM; 215 } 216 217 return 0; 218 } 219 EXPORT_SYMBOL(otx2_set_mac_address); 220 221 int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) 222 { 223 struct nix_frs_cfg *req; 224 int err; 225 226 mutex_lock(&pfvf->mbox.lock); 227 req = otx2_mbox_alloc_msg_nix_set_hw_frs(&pfvf->mbox); 228 if (!req) { 229 mutex_unlock(&pfvf->mbox.lock); 230 return -ENOMEM; 231 } 232 233 req->maxlen = pfvf->max_frs; 234 235 err = otx2_sync_mbox_msg(&pfvf->mbox); 236 mutex_unlock(&pfvf->mbox.lock); 237 return err; 238 } 239 240 int otx2_config_pause_frm(struct otx2_nic *pfvf) 241 { 242 struct cgx_pause_frm_cfg *req; 243 int err; 244 245 if (is_otx2_lbkvf(pfvf->pdev)) 246 return 0; 247 248 mutex_lock(&pfvf->mbox.lock); 249 req = otx2_mbox_alloc_msg_cgx_cfg_pause_frm(&pfvf->mbox); 250 if (!req) { 251 err = -ENOMEM; 252 goto unlock; 253 } 254 255 req->rx_pause = !!(pfvf->flags & OTX2_FLAG_RX_PAUSE_ENABLED); 256 req->tx_pause = !!(pfvf->flags & OTX2_FLAG_TX_PAUSE_ENABLED); 257 req->set = 1; 258 259 err = otx2_sync_mbox_msg(&pfvf->mbox); 260 unlock: 261 mutex_unlock(&pfvf->mbox.lock); 262 return err; 263 } 264 265 int otx2_set_flowkey_cfg(struct otx2_nic *pfvf) 266 { 267 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 268 struct nix_rss_flowkey_cfg *req; 269 int err; 270 271 mutex_lock(&pfvf->mbox.lock); 272 req = otx2_mbox_alloc_msg_nix_rss_flowkey_cfg(&pfvf->mbox); 273 if (!req) { 274 mutex_unlock(&pfvf->mbox.lock); 275 return -ENOMEM; 276 } 277 req->mcam_index = -1; /* Default or reserved index */ 278 req->flowkey_cfg = rss->flowkey_cfg; 279 req->group = DEFAULT_RSS_CONTEXT_GROUP; 280 281 err = otx2_sync_mbox_msg(&pfvf->mbox); 282 mutex_unlock(&pfvf->mbox.lock); 283 return err; 284 } 285 286 int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id) 287 { 288 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 289 const int index = rss->rss_size * ctx_id; 290 struct mbox *mbox = &pfvf->mbox; 291 struct otx2_rss_ctx *rss_ctx; 292 struct nix_aq_enq_req *aq; 293 int idx, err; 294 295 mutex_lock(&mbox->lock); 296 rss_ctx = rss->rss_ctx[ctx_id]; 297 /* Get memory to put this msg */ 298 for (idx = 0; idx < rss->rss_size; idx++) { 299 aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); 300 if (!aq) { 301 /* The shared memory buffer can be full. 302 * Flush it and retry 303 */ 304 err = otx2_sync_mbox_msg(mbox); 305 if (err) { 306 mutex_unlock(&mbox->lock); 307 return err; 308 } 309 aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); 310 if (!aq) { 311 mutex_unlock(&mbox->lock); 312 return -ENOMEM; 313 } 314 } 315 316 aq->rss.rq = rss_ctx->ind_tbl[idx]; 317 318 /* Fill AQ info */ 319 aq->qidx = index + idx; 320 aq->ctype = NIX_AQ_CTYPE_RSS; 321 aq->op = NIX_AQ_INSTOP_INIT; 322 } 323 err = otx2_sync_mbox_msg(mbox); 324 mutex_unlock(&mbox->lock); 325 return err; 326 } 327 328 void otx2_set_rss_key(struct otx2_nic *pfvf) 329 { 330 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 331 u64 *key = (u64 *)&rss->key[4]; 332 int idx; 333 334 /* 352bit or 44byte key needs to be configured as below 335 * NIX_LF_RX_SECRETX0 = key<351:288> 336 * NIX_LF_RX_SECRETX1 = key<287:224> 337 * NIX_LF_RX_SECRETX2 = key<223:160> 338 * NIX_LF_RX_SECRETX3 = key<159:96> 339 * NIX_LF_RX_SECRETX4 = key<95:32> 340 * NIX_LF_RX_SECRETX5<63:32> = key<31:0> 341 */ 342 otx2_write64(pfvf, NIX_LF_RX_SECRETX(5), 343 (u64)(*((u32 *)&rss->key)) << 32); 344 idx = sizeof(rss->key) / sizeof(u64); 345 while (idx > 0) { 346 idx--; 347 otx2_write64(pfvf, NIX_LF_RX_SECRETX(idx), *key++); 348 } 349 } 350 351 int otx2_rss_init(struct otx2_nic *pfvf) 352 { 353 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 354 struct otx2_rss_ctx *rss_ctx; 355 int idx, ret = 0; 356 357 rss->rss_size = sizeof(*rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]); 358 359 /* Init RSS key if it is not setup already */ 360 if (!rss->enable) 361 netdev_rss_key_fill(rss->key, sizeof(rss->key)); 362 otx2_set_rss_key(pfvf); 363 364 if (!netif_is_rxfh_configured(pfvf->netdev)) { 365 /* Set RSS group 0 as default indirection table */ 366 rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP] = kzalloc(rss->rss_size, 367 GFP_KERNEL); 368 if (!rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]) 369 return -ENOMEM; 370 371 rss_ctx = rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]; 372 for (idx = 0; idx < rss->rss_size; idx++) 373 rss_ctx->ind_tbl[idx] = 374 ethtool_rxfh_indir_default(idx, 375 pfvf->hw.rx_queues); 376 } 377 ret = otx2_set_rss_table(pfvf, DEFAULT_RSS_CONTEXT_GROUP); 378 if (ret) 379 return ret; 380 381 /* Flowkey or hash config to be used for generating flow tag */ 382 rss->flowkey_cfg = rss->enable ? rss->flowkey_cfg : 383 NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6 | 384 NIX_FLOW_KEY_TYPE_TCP | NIX_FLOW_KEY_TYPE_UDP | 385 NIX_FLOW_KEY_TYPE_SCTP | NIX_FLOW_KEY_TYPE_VLAN | 386 NIX_FLOW_KEY_TYPE_IPV4_PROTO; 387 388 ret = otx2_set_flowkey_cfg(pfvf); 389 if (ret) 390 return ret; 391 392 rss->enable = true; 393 return 0; 394 } 395 396 /* Setup UDP segmentation algorithm in HW */ 397 static void otx2_setup_udp_segmentation(struct nix_lso_format_cfg *lso, bool v4) 398 { 399 struct nix_lso_format *field; 400 401 field = (struct nix_lso_format *)&lso->fields[0]; 402 lso->field_mask = GENMASK(18, 0); 403 404 /* IP's Length field */ 405 field->layer = NIX_TXLAYER_OL3; 406 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ 407 field->offset = v4 ? 2 : 4; 408 field->sizem1 = 1; /* i.e 2 bytes */ 409 field->alg = NIX_LSOALG_ADD_PAYLEN; 410 field++; 411 412 /* No ID field in IPv6 header */ 413 if (v4) { 414 /* Increment IPID */ 415 field->layer = NIX_TXLAYER_OL3; 416 field->offset = 4; 417 field->sizem1 = 1; /* i.e 2 bytes */ 418 field->alg = NIX_LSOALG_ADD_SEGNUM; 419 field++; 420 } 421 422 /* Update length in UDP header */ 423 field->layer = NIX_TXLAYER_OL4; 424 field->offset = 4; 425 field->sizem1 = 1; 426 field->alg = NIX_LSOALG_ADD_PAYLEN; 427 } 428 429 /* Setup segmentation algorithms in HW and retrieve algorithm index */ 430 void otx2_setup_segmentation(struct otx2_nic *pfvf) 431 { 432 struct nix_lso_format_cfg_rsp *rsp; 433 struct nix_lso_format_cfg *lso; 434 struct otx2_hw *hw = &pfvf->hw; 435 int err; 436 437 mutex_lock(&pfvf->mbox.lock); 438 439 /* UDPv4 segmentation */ 440 lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); 441 if (!lso) 442 goto fail; 443 444 /* Setup UDP/IP header fields that HW should update per segment */ 445 otx2_setup_udp_segmentation(lso, true); 446 447 err = otx2_sync_mbox_msg(&pfvf->mbox); 448 if (err) 449 goto fail; 450 451 rsp = (struct nix_lso_format_cfg_rsp *) 452 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); 453 if (IS_ERR(rsp)) 454 goto fail; 455 456 hw->lso_udpv4_idx = rsp->lso_format_idx; 457 458 /* UDPv6 segmentation */ 459 lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); 460 if (!lso) 461 goto fail; 462 463 /* Setup UDP/IP header fields that HW should update per segment */ 464 otx2_setup_udp_segmentation(lso, false); 465 466 err = otx2_sync_mbox_msg(&pfvf->mbox); 467 if (err) 468 goto fail; 469 470 rsp = (struct nix_lso_format_cfg_rsp *) 471 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); 472 if (IS_ERR(rsp)) 473 goto fail; 474 475 hw->lso_udpv6_idx = rsp->lso_format_idx; 476 mutex_unlock(&pfvf->mbox.lock); 477 return; 478 fail: 479 mutex_unlock(&pfvf->mbox.lock); 480 netdev_info(pfvf->netdev, 481 "Failed to get LSO index for UDP GSO offload, disabling\n"); 482 pfvf->netdev->hw_features &= ~NETIF_F_GSO_UDP_L4; 483 } 484 485 void otx2_config_irq_coalescing(struct otx2_nic *pfvf, int qidx) 486 { 487 /* Configure CQE interrupt coalescing parameters 488 * 489 * HW triggers an irq when ECOUNT > cq_ecount_wait, hence 490 * set 1 less than cq_ecount_wait. And cq_time_wait is in 491 * usecs, convert that to 100ns count. 492 */ 493 otx2_write64(pfvf, NIX_LF_CINTX_WAIT(qidx), 494 ((u64)(pfvf->hw.cq_time_wait * 10) << 48) | 495 ((u64)pfvf->hw.cq_qcount_wait << 32) | 496 (pfvf->hw.cq_ecount_wait - 1)); 497 } 498 499 int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 500 dma_addr_t *dma) 501 { 502 u8 *buf; 503 504 buf = napi_alloc_frag_align(pool->rbsize, OTX2_ALIGN); 505 if (unlikely(!buf)) 506 return -ENOMEM; 507 508 *dma = dma_map_single_attrs(pfvf->dev, buf, pool->rbsize, 509 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 510 if (unlikely(dma_mapping_error(pfvf->dev, *dma))) { 511 page_frag_free(buf); 512 return -ENOMEM; 513 } 514 515 return 0; 516 } 517 518 static int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 519 dma_addr_t *dma) 520 { 521 int ret; 522 523 local_bh_disable(); 524 ret = __otx2_alloc_rbuf(pfvf, pool, dma); 525 local_bh_enable(); 526 return ret; 527 } 528 529 int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, 530 dma_addr_t *dma) 531 { 532 if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma))) { 533 struct refill_work *work; 534 struct delayed_work *dwork; 535 536 work = &pfvf->refill_wrk[cq->cq_idx]; 537 dwork = &work->pool_refill_work; 538 /* Schedule a task if no other task is running */ 539 if (!cq->refill_task_sched) { 540 cq->refill_task_sched = true; 541 schedule_delayed_work(dwork, 542 msecs_to_jiffies(100)); 543 } 544 return -ENOMEM; 545 } 546 return 0; 547 } 548 549 void otx2_tx_timeout(struct net_device *netdev, unsigned int txq) 550 { 551 struct otx2_nic *pfvf = netdev_priv(netdev); 552 553 schedule_work(&pfvf->reset_task); 554 } 555 EXPORT_SYMBOL(otx2_tx_timeout); 556 557 void otx2_get_mac_from_af(struct net_device *netdev) 558 { 559 struct otx2_nic *pfvf = netdev_priv(netdev); 560 int err; 561 562 err = otx2_hw_get_mac_addr(pfvf, netdev); 563 if (err) 564 dev_warn(pfvf->dev, "Failed to read mac from hardware\n"); 565 566 /* If AF doesn't provide a valid MAC, generate a random one */ 567 if (!is_valid_ether_addr(netdev->dev_addr)) 568 eth_hw_addr_random(netdev); 569 } 570 EXPORT_SYMBOL(otx2_get_mac_from_af); 571 572 static int otx2_get_link(struct otx2_nic *pfvf) 573 { 574 int link = 0; 575 u16 map; 576 577 /* cgx lmac link */ 578 if (pfvf->hw.tx_chan_base >= CGX_CHAN_BASE) { 579 map = pfvf->hw.tx_chan_base & 0x7FF; 580 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF); 581 } 582 /* LBK channel */ 583 if (pfvf->hw.tx_chan_base < SDP_CHAN_BASE) { 584 map = pfvf->hw.tx_chan_base & 0x7FF; 585 link = pfvf->hw.cgx_links | ((map >> 8) & 0xF); 586 } 587 588 return link; 589 } 590 591 int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) 592 { 593 struct otx2_hw *hw = &pfvf->hw; 594 struct nix_txschq_config *req; 595 u64 schq, parent; 596 597 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); 598 if (!req) 599 return -ENOMEM; 600 601 req->lvl = lvl; 602 req->num_regs = 1; 603 604 schq = hw->txschq_list[lvl][0]; 605 /* Set topology e.t.c configuration */ 606 if (lvl == NIX_TXSCH_LVL_SMQ) { 607 req->reg[0] = NIX_AF_SMQX_CFG(schq); 608 req->regval[0] = ((pfvf->netdev->max_mtu + OTX2_ETH_HLEN) << 8) 609 | OTX2_MIN_MTU; 610 611 req->regval[0] |= (0x20ULL << 51) | (0x80ULL << 39) | 612 (0x2ULL << 36); 613 req->num_regs++; 614 /* MDQ config */ 615 parent = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; 616 req->reg[1] = NIX_AF_MDQX_PARENT(schq); 617 req->regval[1] = parent << 16; 618 req->num_regs++; 619 /* Set DWRR quantum */ 620 req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); 621 req->regval[2] = DFLT_RR_QTM; 622 } else if (lvl == NIX_TXSCH_LVL_TL4) { 623 parent = hw->txschq_list[NIX_TXSCH_LVL_TL3][0]; 624 req->reg[0] = NIX_AF_TL4X_PARENT(schq); 625 req->regval[0] = parent << 16; 626 req->num_regs++; 627 req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); 628 req->regval[1] = DFLT_RR_QTM; 629 } else if (lvl == NIX_TXSCH_LVL_TL3) { 630 parent = hw->txschq_list[NIX_TXSCH_LVL_TL2][0]; 631 req->reg[0] = NIX_AF_TL3X_PARENT(schq); 632 req->regval[0] = parent << 16; 633 req->num_regs++; 634 req->reg[1] = NIX_AF_TL3X_SCHEDULE(schq); 635 req->regval[1] = DFLT_RR_QTM; 636 } else if (lvl == NIX_TXSCH_LVL_TL2) { 637 parent = hw->txschq_list[NIX_TXSCH_LVL_TL1][0]; 638 req->reg[0] = NIX_AF_TL2X_PARENT(schq); 639 req->regval[0] = parent << 16; 640 641 req->num_regs++; 642 req->reg[1] = NIX_AF_TL2X_SCHEDULE(schq); 643 req->regval[1] = TXSCH_TL1_DFLT_RR_PRIO << 24 | DFLT_RR_QTM; 644 645 req->num_regs++; 646 req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, 647 otx2_get_link(pfvf)); 648 /* Enable this queue and backpressure */ 649 req->regval[2] = BIT_ULL(13) | BIT_ULL(12); 650 651 } else if (lvl == NIX_TXSCH_LVL_TL1) { 652 /* Default config for TL1. 653 * For VF this is always ignored. 654 */ 655 656 /* Set DWRR quantum */ 657 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq); 658 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM; 659 660 req->num_regs++; 661 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq); 662 req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1); 663 664 req->num_regs++; 665 req->reg[2] = NIX_AF_TL1X_CIR(schq); 666 req->regval[2] = 0; 667 } 668 669 return otx2_sync_mbox_msg(&pfvf->mbox); 670 } 671 672 int otx2_txsch_alloc(struct otx2_nic *pfvf) 673 { 674 struct nix_txsch_alloc_req *req; 675 int lvl; 676 677 /* Get memory to put this msg */ 678 req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); 679 if (!req) 680 return -ENOMEM; 681 682 /* Request one schq per level */ 683 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) 684 req->schq[lvl] = 1; 685 686 return otx2_sync_mbox_msg(&pfvf->mbox); 687 } 688 689 int otx2_txschq_stop(struct otx2_nic *pfvf) 690 { 691 struct nix_txsch_free_req *free_req; 692 int lvl, schq, err; 693 694 mutex_lock(&pfvf->mbox.lock); 695 /* Free the transmit schedulers */ 696 free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); 697 if (!free_req) { 698 mutex_unlock(&pfvf->mbox.lock); 699 return -ENOMEM; 700 } 701 702 free_req->flags = TXSCHQ_FREE_ALL; 703 err = otx2_sync_mbox_msg(&pfvf->mbox); 704 mutex_unlock(&pfvf->mbox.lock); 705 706 /* Clear the txschq list */ 707 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 708 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) 709 pfvf->hw.txschq_list[lvl][schq] = 0; 710 } 711 return err; 712 } 713 714 void otx2_sqb_flush(struct otx2_nic *pfvf) 715 { 716 int qidx, sqe_tail, sqe_head; 717 u64 incr, *ptr, val; 718 int timeout = 1000; 719 720 ptr = (u64 *)otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_STATUS); 721 for (qidx = 0; qidx < pfvf->hw.tx_queues; qidx++) { 722 incr = (u64)qidx << 32; 723 while (timeout) { 724 val = otx2_atomic64_add(incr, ptr); 725 sqe_head = (val >> 20) & 0x3F; 726 sqe_tail = (val >> 28) & 0x3F; 727 if (sqe_head == sqe_tail) 728 break; 729 usleep_range(1, 3); 730 timeout--; 731 } 732 } 733 } 734 735 /* RED and drop levels of CQ on packet reception. 736 * For CQ level is measure of emptiness ( 0x0 = full, 255 = empty). 737 */ 738 #define RQ_PASS_LVL_CQ(skid, qsize) ((((skid) + 16) * 256) / (qsize)) 739 #define RQ_DROP_LVL_CQ(skid, qsize) (((skid) * 256) / (qsize)) 740 741 /* RED and drop levels of AURA for packet reception. 742 * For AURA level is measure of fullness (0x0 = empty, 255 = full). 743 * Eg: For RQ length 1K, for pass/drop level 204/230. 744 * RED accepts pkts if free pointers > 102 & <= 205. 745 * Drops pkts if free pointers < 102. 746 */ 747 #define RQ_BP_LVL_AURA (255 - ((85 * 256) / 100)) /* BP when 85% is full */ 748 #define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ 749 #define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ 750 751 static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) 752 { 753 struct otx2_qset *qset = &pfvf->qset; 754 struct nix_aq_enq_req *aq; 755 756 /* Get memory to put this msg */ 757 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 758 if (!aq) 759 return -ENOMEM; 760 761 aq->rq.cq = qidx; 762 aq->rq.ena = 1; 763 aq->rq.pb_caching = 1; 764 aq->rq.lpb_aura = lpb_aura; /* Use large packet buffer aura */ 765 aq->rq.lpb_sizem1 = (DMA_BUFFER_LEN(pfvf->rbsize) / 8) - 1; 766 aq->rq.xqe_imm_size = 0; /* Copying of packet to CQE not needed */ 767 aq->rq.flow_tagw = 32; /* Copy full 32bit flow_tag to CQE header */ 768 aq->rq.qint_idx = 0; 769 aq->rq.lpb_drop_ena = 1; /* Enable RED dropping for AURA */ 770 aq->rq.xqe_drop_ena = 1; /* Enable RED dropping for CQ/SSO */ 771 aq->rq.xqe_pass = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 772 aq->rq.xqe_drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 773 aq->rq.lpb_aura_pass = RQ_PASS_LVL_AURA; 774 aq->rq.lpb_aura_drop = RQ_DROP_LVL_AURA; 775 776 /* Fill AQ info */ 777 aq->qidx = qidx; 778 aq->ctype = NIX_AQ_CTYPE_RQ; 779 aq->op = NIX_AQ_INSTOP_INIT; 780 781 return otx2_sync_mbox_msg(&pfvf->mbox); 782 } 783 784 int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) 785 { 786 struct otx2_nic *pfvf = dev; 787 struct otx2_snd_queue *sq; 788 struct nix_aq_enq_req *aq; 789 790 sq = &pfvf->qset.sq[qidx]; 791 sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); 792 /* Get memory to put this msg */ 793 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 794 if (!aq) 795 return -ENOMEM; 796 797 aq->sq.cq = pfvf->hw.rx_queues + qidx; 798 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 799 aq->sq.cq_ena = 1; 800 aq->sq.ena = 1; 801 /* Only one SMQ is allocated, map all SQ's to that SMQ */ 802 aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; 803 aq->sq.smq_rr_quantum = DFLT_RR_QTM; 804 aq->sq.default_chan = pfvf->hw.tx_chan_base; 805 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 806 aq->sq.sqb_aura = sqb_aura; 807 aq->sq.sq_int_ena = NIX_SQINT_BITS; 808 aq->sq.qint_idx = 0; 809 /* Due pipelining impact minimum 2000 unused SQ CQE's 810 * need to maintain to avoid CQ overflow. 811 */ 812 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 813 814 /* Fill AQ info */ 815 aq->qidx = qidx; 816 aq->ctype = NIX_AQ_CTYPE_SQ; 817 aq->op = NIX_AQ_INSTOP_INIT; 818 819 return otx2_sync_mbox_msg(&pfvf->mbox); 820 } 821 822 static int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) 823 { 824 struct otx2_qset *qset = &pfvf->qset; 825 struct otx2_snd_queue *sq; 826 struct otx2_pool *pool; 827 int err; 828 829 pool = &pfvf->qset.pool[sqb_aura]; 830 sq = &qset->sq[qidx]; 831 sq->sqe_size = NIX_SQESZ_W16 ? 64 : 128; 832 sq->sqe_cnt = qset->sqe_cnt; 833 834 err = qmem_alloc(pfvf->dev, &sq->sqe, 1, sq->sqe_size); 835 if (err) 836 return err; 837 838 err = qmem_alloc(pfvf->dev, &sq->tso_hdrs, qset->sqe_cnt, 839 TSO_HEADER_SIZE); 840 if (err) 841 return err; 842 843 sq->sqe_base = sq->sqe->base; 844 sq->sg = kcalloc(qset->sqe_cnt, sizeof(struct sg_list), GFP_KERNEL); 845 if (!sq->sg) 846 return -ENOMEM; 847 848 if (pfvf->ptp) { 849 err = qmem_alloc(pfvf->dev, &sq->timestamps, qset->sqe_cnt, 850 sizeof(*sq->timestamps)); 851 if (err) 852 return err; 853 } 854 855 sq->head = 0; 856 sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; 857 sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; 858 /* Set SQE threshold to 10% of total SQEs */ 859 sq->sqe_thresh = ((sq->num_sqbs * sq->sqe_per_sqb) * 10) / 100; 860 sq->aura_id = sqb_aura; 861 sq->aura_fc_addr = pool->fc_addr->base; 862 sq->io_addr = (__force u64)otx2_get_regaddr(pfvf, NIX_LF_OP_SENDX(0)); 863 864 sq->stats.bytes = 0; 865 sq->stats.pkts = 0; 866 867 return pfvf->hw_ops->sq_aq_init(pfvf, qidx, sqb_aura); 868 869 } 870 871 static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) 872 { 873 struct otx2_qset *qset = &pfvf->qset; 874 struct nix_aq_enq_req *aq; 875 struct otx2_cq_queue *cq; 876 int err, pool_id; 877 878 cq = &qset->cq[qidx]; 879 cq->cq_idx = qidx; 880 if (qidx < pfvf->hw.rx_queues) { 881 cq->cq_type = CQ_RX; 882 cq->cint_idx = qidx; 883 cq->cqe_cnt = qset->rqe_cnt; 884 } else { 885 cq->cq_type = CQ_TX; 886 cq->cint_idx = qidx - pfvf->hw.rx_queues; 887 cq->cqe_cnt = qset->sqe_cnt; 888 } 889 cq->cqe_size = pfvf->qset.xqe_size; 890 891 /* Allocate memory for CQEs */ 892 err = qmem_alloc(pfvf->dev, &cq->cqe, cq->cqe_cnt, cq->cqe_size); 893 if (err) 894 return err; 895 896 /* Save CQE CPU base for faster reference */ 897 cq->cqe_base = cq->cqe->base; 898 /* In case where all RQs auras point to single pool, 899 * all CQs receive buffer pool also point to same pool. 900 */ 901 pool_id = ((cq->cq_type == CQ_RX) && 902 (pfvf->hw.rqpool_cnt != pfvf->hw.rx_queues)) ? 0 : qidx; 903 cq->rbpool = &qset->pool[pool_id]; 904 cq->refill_task_sched = false; 905 906 /* Get memory to put this msg */ 907 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 908 if (!aq) 909 return -ENOMEM; 910 911 aq->cq.ena = 1; 912 aq->cq.qsize = Q_SIZE(cq->cqe_cnt, 4); 913 aq->cq.caching = 1; 914 aq->cq.base = cq->cqe->iova; 915 aq->cq.cint_idx = cq->cint_idx; 916 aq->cq.cq_err_int_ena = NIX_CQERRINT_BITS; 917 aq->cq.qint_idx = 0; 918 aq->cq.avg_level = 255; 919 920 if (qidx < pfvf->hw.rx_queues) { 921 aq->cq.drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, cq->cqe_cnt); 922 aq->cq.drop_ena = 1; 923 924 /* Enable receive CQ backpressure */ 925 aq->cq.bp_ena = 1; 926 aq->cq.bpid = pfvf->bpid[0]; 927 928 /* Set backpressure level is same as cq pass level */ 929 aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 930 } 931 932 /* Fill AQ info */ 933 aq->qidx = qidx; 934 aq->ctype = NIX_AQ_CTYPE_CQ; 935 aq->op = NIX_AQ_INSTOP_INIT; 936 937 return otx2_sync_mbox_msg(&pfvf->mbox); 938 } 939 940 static void otx2_pool_refill_task(struct work_struct *work) 941 { 942 struct otx2_cq_queue *cq; 943 struct otx2_pool *rbpool; 944 struct refill_work *wrk; 945 int qidx, free_ptrs = 0; 946 struct otx2_nic *pfvf; 947 dma_addr_t bufptr; 948 949 wrk = container_of(work, struct refill_work, pool_refill_work.work); 950 pfvf = wrk->pf; 951 qidx = wrk - pfvf->refill_wrk; 952 cq = &pfvf->qset.cq[qidx]; 953 rbpool = cq->rbpool; 954 free_ptrs = cq->pool_ptrs; 955 956 while (cq->pool_ptrs) { 957 if (otx2_alloc_rbuf(pfvf, rbpool, &bufptr)) { 958 /* Schedule a WQ if we fails to free atleast half of the 959 * pointers else enable napi for this RQ. 960 */ 961 if (!((free_ptrs - cq->pool_ptrs) > free_ptrs / 2)) { 962 struct delayed_work *dwork; 963 964 dwork = &wrk->pool_refill_work; 965 schedule_delayed_work(dwork, 966 msecs_to_jiffies(100)); 967 } else { 968 cq->refill_task_sched = false; 969 } 970 return; 971 } 972 pfvf->hw_ops->aura_freeptr(pfvf, qidx, bufptr + OTX2_HEAD_ROOM); 973 cq->pool_ptrs--; 974 } 975 cq->refill_task_sched = false; 976 } 977 978 int otx2_config_nix_queues(struct otx2_nic *pfvf) 979 { 980 int qidx, err; 981 982 /* Initialize RX queues */ 983 for (qidx = 0; qidx < pfvf->hw.rx_queues; qidx++) { 984 u16 lpb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx); 985 986 err = otx2_rq_init(pfvf, qidx, lpb_aura); 987 if (err) 988 return err; 989 } 990 991 /* Initialize TX queues */ 992 for (qidx = 0; qidx < pfvf->hw.tx_queues; qidx++) { 993 u16 sqb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 994 995 err = otx2_sq_init(pfvf, qidx, sqb_aura); 996 if (err) 997 return err; 998 } 999 1000 /* Initialize completion queues */ 1001 for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { 1002 err = otx2_cq_init(pfvf, qidx); 1003 if (err) 1004 return err; 1005 } 1006 1007 /* Initialize work queue for receive buffer refill */ 1008 pfvf->refill_wrk = devm_kcalloc(pfvf->dev, pfvf->qset.cq_cnt, 1009 sizeof(struct refill_work), GFP_KERNEL); 1010 if (!pfvf->refill_wrk) 1011 return -ENOMEM; 1012 1013 for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { 1014 pfvf->refill_wrk[qidx].pf = pfvf; 1015 INIT_DELAYED_WORK(&pfvf->refill_wrk[qidx].pool_refill_work, 1016 otx2_pool_refill_task); 1017 } 1018 return 0; 1019 } 1020 1021 int otx2_config_nix(struct otx2_nic *pfvf) 1022 { 1023 struct nix_lf_alloc_req *nixlf; 1024 struct nix_lf_alloc_rsp *rsp; 1025 int err; 1026 1027 pfvf->qset.xqe_size = NIX_XQESZ_W16 ? 128 : 512; 1028 1029 /* Get memory to put this msg */ 1030 nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(&pfvf->mbox); 1031 if (!nixlf) 1032 return -ENOMEM; 1033 1034 /* Set RQ/SQ/CQ counts */ 1035 nixlf->rq_cnt = pfvf->hw.rx_queues; 1036 nixlf->sq_cnt = pfvf->hw.tx_queues; 1037 nixlf->cq_cnt = pfvf->qset.cq_cnt; 1038 nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE; 1039 nixlf->rss_grps = MAX_RSS_GROUPS; 1040 nixlf->xqe_sz = NIX_XQESZ_W16; 1041 /* We don't know absolute NPA LF idx attached. 1042 * AF will replace 'RVU_DEFAULT_PF_FUNC' with 1043 * NPA LF attached to this RVU PF/VF. 1044 */ 1045 nixlf->npa_func = RVU_DEFAULT_PF_FUNC; 1046 /* Disable alignment pad, enable L2 length check, 1047 * enable L4 TCP/UDP checksum verification. 1048 */ 1049 nixlf->rx_cfg = BIT_ULL(33) | BIT_ULL(35) | BIT_ULL(37); 1050 1051 err = otx2_sync_mbox_msg(&pfvf->mbox); 1052 if (err) 1053 return err; 1054 1055 rsp = (struct nix_lf_alloc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, 1056 &nixlf->hdr); 1057 if (IS_ERR(rsp)) 1058 return PTR_ERR(rsp); 1059 1060 if (rsp->qints < 1) 1061 return -ENXIO; 1062 1063 return rsp->hdr.rc; 1064 } 1065 1066 void otx2_sq_free_sqbs(struct otx2_nic *pfvf) 1067 { 1068 struct otx2_qset *qset = &pfvf->qset; 1069 struct otx2_hw *hw = &pfvf->hw; 1070 struct otx2_snd_queue *sq; 1071 int sqb, qidx; 1072 u64 iova, pa; 1073 1074 for (qidx = 0; qidx < hw->tx_queues; qidx++) { 1075 sq = &qset->sq[qidx]; 1076 if (!sq->sqb_ptrs) 1077 continue; 1078 for (sqb = 0; sqb < sq->sqb_count; sqb++) { 1079 if (!sq->sqb_ptrs[sqb]) 1080 continue; 1081 iova = sq->sqb_ptrs[sqb]; 1082 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 1083 dma_unmap_page_attrs(pfvf->dev, iova, hw->sqb_size, 1084 DMA_FROM_DEVICE, 1085 DMA_ATTR_SKIP_CPU_SYNC); 1086 put_page(virt_to_page(phys_to_virt(pa))); 1087 } 1088 sq->sqb_count = 0; 1089 } 1090 } 1091 1092 void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) 1093 { 1094 int pool_id, pool_start = 0, pool_end = 0, size = 0; 1095 u64 iova, pa; 1096 1097 if (type == AURA_NIX_SQ) { 1098 pool_start = otx2_get_pool_idx(pfvf, type, 0); 1099 pool_end = pool_start + pfvf->hw.sqpool_cnt; 1100 size = pfvf->hw.sqb_size; 1101 } 1102 if (type == AURA_NIX_RQ) { 1103 pool_start = otx2_get_pool_idx(pfvf, type, 0); 1104 pool_end = pfvf->hw.rqpool_cnt; 1105 size = pfvf->rbsize; 1106 } 1107 1108 /* Free SQB and RQB pointers from the aura pool */ 1109 for (pool_id = pool_start; pool_id < pool_end; pool_id++) { 1110 iova = otx2_aura_allocptr(pfvf, pool_id); 1111 while (iova) { 1112 if (type == AURA_NIX_RQ) 1113 iova -= OTX2_HEAD_ROOM; 1114 1115 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 1116 dma_unmap_page_attrs(pfvf->dev, iova, size, 1117 DMA_FROM_DEVICE, 1118 DMA_ATTR_SKIP_CPU_SYNC); 1119 put_page(virt_to_page(phys_to_virt(pa))); 1120 iova = otx2_aura_allocptr(pfvf, pool_id); 1121 } 1122 } 1123 } 1124 1125 void otx2_aura_pool_free(struct otx2_nic *pfvf) 1126 { 1127 struct otx2_pool *pool; 1128 int pool_id; 1129 1130 if (!pfvf->qset.pool) 1131 return; 1132 1133 for (pool_id = 0; pool_id < pfvf->hw.pool_cnt; pool_id++) { 1134 pool = &pfvf->qset.pool[pool_id]; 1135 qmem_free(pfvf->dev, pool->stack); 1136 qmem_free(pfvf->dev, pool->fc_addr); 1137 } 1138 devm_kfree(pfvf->dev, pfvf->qset.pool); 1139 pfvf->qset.pool = NULL; 1140 } 1141 1142 static int otx2_aura_init(struct otx2_nic *pfvf, int aura_id, 1143 int pool_id, int numptrs) 1144 { 1145 struct npa_aq_enq_req *aq; 1146 struct otx2_pool *pool; 1147 int err; 1148 1149 pool = &pfvf->qset.pool[pool_id]; 1150 1151 /* Allocate memory for HW to update Aura count. 1152 * Alloc one cache line, so that it fits all FC_STYPE modes. 1153 */ 1154 if (!pool->fc_addr) { 1155 err = qmem_alloc(pfvf->dev, &pool->fc_addr, 1, OTX2_ALIGN); 1156 if (err) 1157 return err; 1158 } 1159 1160 /* Initialize this aura's context via AF */ 1161 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1162 if (!aq) { 1163 /* Shared mbox memory buffer is full, flush it and retry */ 1164 err = otx2_sync_mbox_msg(&pfvf->mbox); 1165 if (err) 1166 return err; 1167 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1168 if (!aq) 1169 return -ENOMEM; 1170 } 1171 1172 aq->aura_id = aura_id; 1173 /* Will be filled by AF with correct pool context address */ 1174 aq->aura.pool_addr = pool_id; 1175 aq->aura.pool_caching = 1; 1176 aq->aura.shift = ilog2(numptrs) - 8; 1177 aq->aura.count = numptrs; 1178 aq->aura.limit = numptrs; 1179 aq->aura.avg_level = 255; 1180 aq->aura.ena = 1; 1181 aq->aura.fc_ena = 1; 1182 aq->aura.fc_addr = pool->fc_addr->iova; 1183 aq->aura.fc_hyst_bits = 0; /* Store count on all updates */ 1184 1185 /* Enable backpressure for RQ aura */ 1186 if (aura_id < pfvf->hw.rqpool_cnt) { 1187 aq->aura.bp_ena = 0; 1188 aq->aura.nix0_bpid = pfvf->bpid[0]; 1189 /* Set backpressure level for RQ's Aura */ 1190 aq->aura.bp = RQ_BP_LVL_AURA; 1191 } 1192 1193 /* Fill AQ info */ 1194 aq->ctype = NPA_AQ_CTYPE_AURA; 1195 aq->op = NPA_AQ_INSTOP_INIT; 1196 1197 return 0; 1198 } 1199 1200 static int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, 1201 int stack_pages, int numptrs, int buf_size) 1202 { 1203 struct npa_aq_enq_req *aq; 1204 struct otx2_pool *pool; 1205 int err; 1206 1207 pool = &pfvf->qset.pool[pool_id]; 1208 /* Alloc memory for stack which is used to store buffer pointers */ 1209 err = qmem_alloc(pfvf->dev, &pool->stack, 1210 stack_pages, pfvf->hw.stack_pg_bytes); 1211 if (err) 1212 return err; 1213 1214 pool->rbsize = buf_size; 1215 1216 /* Set LMTST addr for NPA batch free */ 1217 if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) 1218 pool->lmt_addr = (__force u64 *)((u64)pfvf->hw.npa_lmt_base + 1219 (pool_id * LMT_LINE_SIZE)); 1220 1221 /* Initialize this pool's context via AF */ 1222 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1223 if (!aq) { 1224 /* Shared mbox memory buffer is full, flush it and retry */ 1225 err = otx2_sync_mbox_msg(&pfvf->mbox); 1226 if (err) { 1227 qmem_free(pfvf->dev, pool->stack); 1228 return err; 1229 } 1230 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1231 if (!aq) { 1232 qmem_free(pfvf->dev, pool->stack); 1233 return -ENOMEM; 1234 } 1235 } 1236 1237 aq->aura_id = pool_id; 1238 aq->pool.stack_base = pool->stack->iova; 1239 aq->pool.stack_caching = 1; 1240 aq->pool.ena = 1; 1241 aq->pool.buf_size = buf_size / 128; 1242 aq->pool.stack_max_pages = stack_pages; 1243 aq->pool.shift = ilog2(numptrs) - 8; 1244 aq->pool.ptr_start = 0; 1245 aq->pool.ptr_end = ~0ULL; 1246 1247 /* Fill AQ info */ 1248 aq->ctype = NPA_AQ_CTYPE_POOL; 1249 aq->op = NPA_AQ_INSTOP_INIT; 1250 1251 return 0; 1252 } 1253 1254 int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) 1255 { 1256 int qidx, pool_id, stack_pages, num_sqbs; 1257 struct otx2_qset *qset = &pfvf->qset; 1258 struct otx2_hw *hw = &pfvf->hw; 1259 struct otx2_snd_queue *sq; 1260 struct otx2_pool *pool; 1261 dma_addr_t bufptr; 1262 int err, ptr; 1263 1264 /* Calculate number of SQBs needed. 1265 * 1266 * For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. 1267 * Last SQE is used for pointing to next SQB. 1268 */ 1269 num_sqbs = (hw->sqb_size / 128) - 1; 1270 num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; 1271 1272 /* Get no of stack pages needed */ 1273 stack_pages = 1274 (num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; 1275 1276 for (qidx = 0; qidx < hw->tx_queues; qidx++) { 1277 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 1278 /* Initialize aura context */ 1279 err = otx2_aura_init(pfvf, pool_id, pool_id, num_sqbs); 1280 if (err) 1281 goto fail; 1282 1283 /* Initialize pool context */ 1284 err = otx2_pool_init(pfvf, pool_id, stack_pages, 1285 num_sqbs, hw->sqb_size); 1286 if (err) 1287 goto fail; 1288 } 1289 1290 /* Flush accumulated messages */ 1291 err = otx2_sync_mbox_msg(&pfvf->mbox); 1292 if (err) 1293 goto fail; 1294 1295 /* Allocate pointers and free them to aura/pool */ 1296 for (qidx = 0; qidx < hw->tx_queues; qidx++) { 1297 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 1298 pool = &pfvf->qset.pool[pool_id]; 1299 1300 sq = &qset->sq[qidx]; 1301 sq->sqb_count = 0; 1302 sq->sqb_ptrs = kcalloc(num_sqbs, sizeof(*sq->sqb_ptrs), GFP_KERNEL); 1303 if (!sq->sqb_ptrs) 1304 return -ENOMEM; 1305 1306 for (ptr = 0; ptr < num_sqbs; ptr++) { 1307 if (otx2_alloc_rbuf(pfvf, pool, &bufptr)) 1308 return -ENOMEM; 1309 pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); 1310 sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; 1311 } 1312 } 1313 1314 return 0; 1315 fail: 1316 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 1317 otx2_aura_pool_free(pfvf); 1318 return err; 1319 } 1320 1321 int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) 1322 { 1323 struct otx2_hw *hw = &pfvf->hw; 1324 int stack_pages, pool_id, rq; 1325 struct otx2_pool *pool; 1326 int err, ptr, num_ptrs; 1327 dma_addr_t bufptr; 1328 1329 num_ptrs = pfvf->qset.rqe_cnt; 1330 1331 stack_pages = 1332 (num_ptrs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; 1333 1334 for (rq = 0; rq < hw->rx_queues; rq++) { 1335 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, rq); 1336 /* Initialize aura context */ 1337 err = otx2_aura_init(pfvf, pool_id, pool_id, num_ptrs); 1338 if (err) 1339 goto fail; 1340 } 1341 for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { 1342 err = otx2_pool_init(pfvf, pool_id, stack_pages, 1343 num_ptrs, pfvf->rbsize); 1344 if (err) 1345 goto fail; 1346 } 1347 1348 /* Flush accumulated messages */ 1349 err = otx2_sync_mbox_msg(&pfvf->mbox); 1350 if (err) 1351 goto fail; 1352 1353 /* Allocate pointers and free them to aura/pool */ 1354 for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { 1355 pool = &pfvf->qset.pool[pool_id]; 1356 for (ptr = 0; ptr < num_ptrs; ptr++) { 1357 if (otx2_alloc_rbuf(pfvf, pool, &bufptr)) 1358 return -ENOMEM; 1359 pfvf->hw_ops->aura_freeptr(pfvf, pool_id, 1360 bufptr + OTX2_HEAD_ROOM); 1361 } 1362 } 1363 1364 return 0; 1365 fail: 1366 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 1367 otx2_aura_pool_free(pfvf); 1368 return err; 1369 } 1370 1371 int otx2_config_npa(struct otx2_nic *pfvf) 1372 { 1373 struct otx2_qset *qset = &pfvf->qset; 1374 struct npa_lf_alloc_req *npalf; 1375 struct otx2_hw *hw = &pfvf->hw; 1376 int aura_cnt; 1377 1378 /* Pool - Stack of free buffer pointers 1379 * Aura - Alloc/frees pointers from/to pool for NIX DMA. 1380 */ 1381 1382 if (!hw->pool_cnt) 1383 return -EINVAL; 1384 1385 qset->pool = devm_kcalloc(pfvf->dev, hw->pool_cnt, 1386 sizeof(struct otx2_pool), GFP_KERNEL); 1387 if (!qset->pool) 1388 return -ENOMEM; 1389 1390 /* Get memory to put this msg */ 1391 npalf = otx2_mbox_alloc_msg_npa_lf_alloc(&pfvf->mbox); 1392 if (!npalf) 1393 return -ENOMEM; 1394 1395 /* Set aura and pool counts */ 1396 npalf->nr_pools = hw->pool_cnt; 1397 aura_cnt = ilog2(roundup_pow_of_two(hw->pool_cnt)); 1398 npalf->aura_sz = (aura_cnt >= ilog2(128)) ? (aura_cnt - 6) : 1; 1399 1400 return otx2_sync_mbox_msg(&pfvf->mbox); 1401 } 1402 1403 int otx2_detach_resources(struct mbox *mbox) 1404 { 1405 struct rsrc_detach *detach; 1406 1407 mutex_lock(&mbox->lock); 1408 detach = otx2_mbox_alloc_msg_detach_resources(mbox); 1409 if (!detach) { 1410 mutex_unlock(&mbox->lock); 1411 return -ENOMEM; 1412 } 1413 1414 /* detach all */ 1415 detach->partial = false; 1416 1417 /* Send detach request to AF */ 1418 otx2_mbox_msg_send(&mbox->mbox, 0); 1419 mutex_unlock(&mbox->lock); 1420 return 0; 1421 } 1422 EXPORT_SYMBOL(otx2_detach_resources); 1423 1424 int otx2_attach_npa_nix(struct otx2_nic *pfvf) 1425 { 1426 struct rsrc_attach *attach; 1427 struct msg_req *msix; 1428 int err; 1429 1430 mutex_lock(&pfvf->mbox.lock); 1431 /* Get memory to put this msg */ 1432 attach = otx2_mbox_alloc_msg_attach_resources(&pfvf->mbox); 1433 if (!attach) { 1434 mutex_unlock(&pfvf->mbox.lock); 1435 return -ENOMEM; 1436 } 1437 1438 attach->npalf = true; 1439 attach->nixlf = true; 1440 1441 /* Send attach request to AF */ 1442 err = otx2_sync_mbox_msg(&pfvf->mbox); 1443 if (err) { 1444 mutex_unlock(&pfvf->mbox.lock); 1445 return err; 1446 } 1447 1448 pfvf->nix_blkaddr = BLKADDR_NIX0; 1449 1450 /* If the platform has two NIX blocks then LF may be 1451 * allocated from NIX1. 1452 */ 1453 if (otx2_read64(pfvf, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_NIX1)) & 0x1FFULL) 1454 pfvf->nix_blkaddr = BLKADDR_NIX1; 1455 1456 /* Get NPA and NIX MSIX vector offsets */ 1457 msix = otx2_mbox_alloc_msg_msix_offset(&pfvf->mbox); 1458 if (!msix) { 1459 mutex_unlock(&pfvf->mbox.lock); 1460 return -ENOMEM; 1461 } 1462 1463 err = otx2_sync_mbox_msg(&pfvf->mbox); 1464 if (err) { 1465 mutex_unlock(&pfvf->mbox.lock); 1466 return err; 1467 } 1468 mutex_unlock(&pfvf->mbox.lock); 1469 1470 if (pfvf->hw.npa_msixoff == MSIX_VECTOR_INVALID || 1471 pfvf->hw.nix_msixoff == MSIX_VECTOR_INVALID) { 1472 dev_err(pfvf->dev, 1473 "RVUPF: Invalid MSIX vector offset for NPA/NIX\n"); 1474 return -EINVAL; 1475 } 1476 1477 return 0; 1478 } 1479 EXPORT_SYMBOL(otx2_attach_npa_nix); 1480 1481 void otx2_ctx_disable(struct mbox *mbox, int type, bool npa) 1482 { 1483 struct hwctx_disable_req *req; 1484 1485 mutex_lock(&mbox->lock); 1486 /* Request AQ to disable this context */ 1487 if (npa) 1488 req = otx2_mbox_alloc_msg_npa_hwctx_disable(mbox); 1489 else 1490 req = otx2_mbox_alloc_msg_nix_hwctx_disable(mbox); 1491 1492 if (!req) { 1493 mutex_unlock(&mbox->lock); 1494 return; 1495 } 1496 1497 req->ctype = type; 1498 1499 if (otx2_sync_mbox_msg(mbox)) 1500 dev_err(mbox->pfvf->dev, "%s failed to disable context\n", 1501 __func__); 1502 1503 mutex_unlock(&mbox->lock); 1504 } 1505 1506 int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable) 1507 { 1508 struct nix_bp_cfg_req *req; 1509 1510 if (enable) 1511 req = otx2_mbox_alloc_msg_nix_bp_enable(&pfvf->mbox); 1512 else 1513 req = otx2_mbox_alloc_msg_nix_bp_disable(&pfvf->mbox); 1514 1515 if (!req) 1516 return -ENOMEM; 1517 1518 req->chan_base = 0; 1519 req->chan_cnt = 1; 1520 req->bpid_per_chan = 0; 1521 1522 return otx2_sync_mbox_msg(&pfvf->mbox); 1523 } 1524 1525 /* Mbox message handlers */ 1526 void mbox_handler_cgx_stats(struct otx2_nic *pfvf, 1527 struct cgx_stats_rsp *rsp) 1528 { 1529 int id; 1530 1531 for (id = 0; id < CGX_RX_STATS_COUNT; id++) 1532 pfvf->hw.cgx_rx_stats[id] = rsp->rx_stats[id]; 1533 for (id = 0; id < CGX_TX_STATS_COUNT; id++) 1534 pfvf->hw.cgx_tx_stats[id] = rsp->tx_stats[id]; 1535 } 1536 1537 void mbox_handler_cgx_fec_stats(struct otx2_nic *pfvf, 1538 struct cgx_fec_stats_rsp *rsp) 1539 { 1540 pfvf->hw.cgx_fec_corr_blks += rsp->fec_corr_blks; 1541 pfvf->hw.cgx_fec_uncorr_blks += rsp->fec_uncorr_blks; 1542 } 1543 1544 void mbox_handler_nix_txsch_alloc(struct otx2_nic *pf, 1545 struct nix_txsch_alloc_rsp *rsp) 1546 { 1547 int lvl, schq; 1548 1549 /* Setup transmit scheduler list */ 1550 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) 1551 for (schq = 0; schq < rsp->schq[lvl]; schq++) 1552 pf->hw.txschq_list[lvl][schq] = 1553 rsp->schq_list[lvl][schq]; 1554 } 1555 EXPORT_SYMBOL(mbox_handler_nix_txsch_alloc); 1556 1557 void mbox_handler_npa_lf_alloc(struct otx2_nic *pfvf, 1558 struct npa_lf_alloc_rsp *rsp) 1559 { 1560 pfvf->hw.stack_pg_ptrs = rsp->stack_pg_ptrs; 1561 pfvf->hw.stack_pg_bytes = rsp->stack_pg_bytes; 1562 } 1563 EXPORT_SYMBOL(mbox_handler_npa_lf_alloc); 1564 1565 void mbox_handler_nix_lf_alloc(struct otx2_nic *pfvf, 1566 struct nix_lf_alloc_rsp *rsp) 1567 { 1568 pfvf->hw.sqb_size = rsp->sqb_size; 1569 pfvf->hw.rx_chan_base = rsp->rx_chan_base; 1570 pfvf->hw.tx_chan_base = rsp->tx_chan_base; 1571 pfvf->hw.lso_tsov4_idx = rsp->lso_tsov4_idx; 1572 pfvf->hw.lso_tsov6_idx = rsp->lso_tsov6_idx; 1573 pfvf->hw.cgx_links = rsp->cgx_links; 1574 pfvf->hw.lbk_links = rsp->lbk_links; 1575 } 1576 EXPORT_SYMBOL(mbox_handler_nix_lf_alloc); 1577 1578 void mbox_handler_msix_offset(struct otx2_nic *pfvf, 1579 struct msix_offset_rsp *rsp) 1580 { 1581 pfvf->hw.npa_msixoff = rsp->npa_msixoff; 1582 pfvf->hw.nix_msixoff = rsp->nix_msixoff; 1583 } 1584 EXPORT_SYMBOL(mbox_handler_msix_offset); 1585 1586 void mbox_handler_nix_bp_enable(struct otx2_nic *pfvf, 1587 struct nix_bp_cfg_rsp *rsp) 1588 { 1589 int chan, chan_id; 1590 1591 for (chan = 0; chan < rsp->chan_cnt; chan++) { 1592 chan_id = ((rsp->chan_bpid[chan] >> 10) & 0x7F); 1593 pfvf->bpid[chan_id] = rsp->chan_bpid[chan] & 0x3FF; 1594 } 1595 } 1596 EXPORT_SYMBOL(mbox_handler_nix_bp_enable); 1597 1598 void otx2_free_cints(struct otx2_nic *pfvf, int n) 1599 { 1600 struct otx2_qset *qset = &pfvf->qset; 1601 struct otx2_hw *hw = &pfvf->hw; 1602 int irq, qidx; 1603 1604 for (qidx = 0, irq = hw->nix_msixoff + NIX_LF_CINT_VEC_START; 1605 qidx < n; 1606 qidx++, irq++) { 1607 int vector = pci_irq_vector(pfvf->pdev, irq); 1608 1609 irq_set_affinity_hint(vector, NULL); 1610 free_cpumask_var(hw->affinity_mask[irq]); 1611 free_irq(vector, &qset->napi[qidx]); 1612 } 1613 } 1614 1615 void otx2_set_cints_affinity(struct otx2_nic *pfvf) 1616 { 1617 struct otx2_hw *hw = &pfvf->hw; 1618 int vec, cpu, irq, cint; 1619 1620 vec = hw->nix_msixoff + NIX_LF_CINT_VEC_START; 1621 cpu = cpumask_first(cpu_online_mask); 1622 1623 /* CQ interrupts */ 1624 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++, vec++) { 1625 if (!alloc_cpumask_var(&hw->affinity_mask[vec], GFP_KERNEL)) 1626 return; 1627 1628 cpumask_set_cpu(cpu, hw->affinity_mask[vec]); 1629 1630 irq = pci_irq_vector(pfvf->pdev, vec); 1631 irq_set_affinity_hint(irq, hw->affinity_mask[vec]); 1632 1633 cpu = cpumask_next(cpu, cpu_online_mask); 1634 if (unlikely(cpu >= nr_cpu_ids)) 1635 cpu = 0; 1636 } 1637 } 1638 1639 u16 otx2_get_max_mtu(struct otx2_nic *pfvf) 1640 { 1641 struct nix_hw_info *rsp; 1642 struct msg_req *req; 1643 u16 max_mtu; 1644 int rc; 1645 1646 mutex_lock(&pfvf->mbox.lock); 1647 1648 req = otx2_mbox_alloc_msg_nix_get_hw_info(&pfvf->mbox); 1649 if (!req) { 1650 rc = -ENOMEM; 1651 goto out; 1652 } 1653 1654 rc = otx2_sync_mbox_msg(&pfvf->mbox); 1655 if (!rc) { 1656 rsp = (struct nix_hw_info *) 1657 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 1658 1659 /* HW counts VLAN insertion bytes (8 for double tag) 1660 * irrespective of whether SQE is requesting to insert VLAN 1661 * in the packet or not. Hence these 8 bytes have to be 1662 * discounted from max packet size otherwise HW will throw 1663 * SMQ errors 1664 */ 1665 max_mtu = rsp->max_mtu - 8 - OTX2_ETH_HLEN; 1666 } 1667 1668 out: 1669 mutex_unlock(&pfvf->mbox.lock); 1670 if (rc) { 1671 dev_warn(pfvf->dev, 1672 "Failed to get MTU from hardware setting default value(1500)\n"); 1673 max_mtu = 1500; 1674 } 1675 return max_mtu; 1676 } 1677 EXPORT_SYMBOL(otx2_get_max_mtu); 1678 1679 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ 1680 int __weak \ 1681 otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ 1682 struct _req_type *req, \ 1683 struct _rsp_type *rsp) \ 1684 { \ 1685 /* Nothing to do here */ \ 1686 return 0; \ 1687 } \ 1688 EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); 1689 MBOX_UP_CGX_MESSAGES 1690 #undef M 1691