1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * aQuantia Corporation Network Driver 4 * Copyright (C) 2014-2019 aQuantia Corporation. All rights reserved 5 */ 6 7 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 8 9 #include "aq_ring.h" 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 15 #include <linux/netdevice.h> 16 #include <linux/etherdevice.h> 17 18 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 19 { 20 unsigned int len = PAGE_SIZE << rxpage->order; 21 22 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 23 24 /* Drop the ref for being in the ring. */ 25 __free_pages(rxpage->page, rxpage->order); 26 rxpage->page = NULL; 27 } 28 29 static int aq_get_rxpage(struct aq_rxpage *rxpage, unsigned int order, 30 struct device *dev) 31 { 32 struct page *page; 33 int ret = -ENOMEM; 34 dma_addr_t daddr; 35 36 page = dev_alloc_pages(order); 37 if (unlikely(!page)) 38 goto err_exit; 39 40 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 41 DMA_FROM_DEVICE); 42 43 if (unlikely(dma_mapping_error(dev, daddr))) 44 goto free_page; 45 46 rxpage->page = page; 47 rxpage->daddr = daddr; 48 rxpage->order = order; 49 rxpage->pg_off = 0; 50 51 return 0; 52 53 free_page: 54 __free_pages(page, order); 55 56 err_exit: 57 return ret; 58 } 59 60 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf, 61 int order) 62 { 63 int ret; 64 65 if (rxbuf->rxdata.page) { 66 /* One means ring is the only user and can reuse */ 67 if (page_ref_count(rxbuf->rxdata.page) > 1) { 68 /* Try reuse buffer */ 69 rxbuf->rxdata.pg_off += AQ_CFG_RX_FRAME_MAX; 70 if (rxbuf->rxdata.pg_off + AQ_CFG_RX_FRAME_MAX <= 71 (PAGE_SIZE << order)) { 72 self->stats.rx.pg_flips++; 73 } else { 74 /* Buffer exhausted. We have other users and 75 * should release this page and realloc 76 */ 77 aq_free_rxpage(&rxbuf->rxdata, 78 aq_nic_get_dev(self->aq_nic)); 79 self->stats.rx.pg_losts++; 80 } 81 } else { 82 rxbuf->rxdata.pg_off = 0; 83 self->stats.rx.pg_reuses++; 84 } 85 } 86 87 if (!rxbuf->rxdata.page) { 88 ret = aq_get_rxpage(&rxbuf->rxdata, order, 89 aq_nic_get_dev(self->aq_nic)); 90 return ret; 91 } 92 93 return 0; 94 } 95 96 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 97 struct aq_nic_s *aq_nic) 98 { 99 int err = 0; 100 101 self->buff_ring = 102 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 103 104 if (!self->buff_ring) { 105 err = -ENOMEM; 106 goto err_exit; 107 } 108 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 109 self->size * self->dx_size, 110 &self->dx_ring_pa, GFP_KERNEL); 111 if (!self->dx_ring) { 112 err = -ENOMEM; 113 goto err_exit; 114 } 115 116 err_exit: 117 if (err < 0) { 118 aq_ring_free(self); 119 self = NULL; 120 } 121 122 return self; 123 } 124 125 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 126 struct aq_nic_s *aq_nic, 127 unsigned int idx, 128 struct aq_nic_cfg_s *aq_nic_cfg) 129 { 130 int err = 0; 131 132 self->aq_nic = aq_nic; 133 self->idx = idx; 134 self->size = aq_nic_cfg->txds; 135 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 136 137 self = aq_ring_alloc(self, aq_nic); 138 if (!self) { 139 err = -ENOMEM; 140 goto err_exit; 141 } 142 143 err_exit: 144 if (err < 0) { 145 aq_ring_free(self); 146 self = NULL; 147 } 148 149 return self; 150 } 151 152 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 153 struct aq_nic_s *aq_nic, 154 unsigned int idx, 155 struct aq_nic_cfg_s *aq_nic_cfg) 156 { 157 int err = 0; 158 159 self->aq_nic = aq_nic; 160 self->idx = idx; 161 self->size = aq_nic_cfg->rxds; 162 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 163 self->page_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE + 164 (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1; 165 166 if (aq_nic_cfg->rxpageorder > self->page_order) 167 self->page_order = aq_nic_cfg->rxpageorder; 168 169 self = aq_ring_alloc(self, aq_nic); 170 if (!self) { 171 err = -ENOMEM; 172 goto err_exit; 173 } 174 175 err_exit: 176 if (err < 0) { 177 aq_ring_free(self); 178 self = NULL; 179 } 180 181 return self; 182 } 183 184 struct aq_ring_s * 185 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 186 unsigned int idx, unsigned int size, unsigned int dx_size) 187 { 188 struct device *dev = aq_nic_get_dev(aq_nic); 189 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 190 191 memset(self, 0, sizeof(*self)); 192 193 self->aq_nic = aq_nic; 194 self->idx = idx; 195 self->size = size; 196 self->dx_size = dx_size; 197 198 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 199 GFP_KERNEL); 200 if (!self->dx_ring) { 201 aq_ring_free(self); 202 return NULL; 203 } 204 205 return self; 206 } 207 208 int aq_ring_init(struct aq_ring_s *self) 209 { 210 self->hw_head = 0; 211 self->sw_head = 0; 212 self->sw_tail = 0; 213 214 return 0; 215 } 216 217 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 218 unsigned int t) 219 { 220 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 221 } 222 223 void aq_ring_update_queue_state(struct aq_ring_s *ring) 224 { 225 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 226 aq_ring_queue_stop(ring); 227 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 228 aq_ring_queue_wake(ring); 229 } 230 231 void aq_ring_queue_wake(struct aq_ring_s *ring) 232 { 233 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 234 235 if (__netif_subqueue_stopped(ndev, 236 AQ_NIC_RING2QMAP(ring->aq_nic, 237 ring->idx))) { 238 netif_wake_subqueue(ndev, 239 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 240 ring->stats.tx.queue_restarts++; 241 } 242 } 243 244 void aq_ring_queue_stop(struct aq_ring_s *ring) 245 { 246 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 247 248 if (!__netif_subqueue_stopped(ndev, 249 AQ_NIC_RING2QMAP(ring->aq_nic, 250 ring->idx))) 251 netif_stop_subqueue(ndev, 252 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 253 } 254 255 bool aq_ring_tx_clean(struct aq_ring_s *self) 256 { 257 struct device *dev = aq_nic_get_dev(self->aq_nic); 258 unsigned int budget; 259 260 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 261 budget && self->sw_head != self->hw_head; budget--) { 262 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 263 264 if (likely(buff->is_mapped)) { 265 if (unlikely(buff->is_sop)) { 266 if (!buff->is_eop && 267 buff->eop_index != 0xffffU && 268 (!aq_ring_dx_in_range(self->sw_head, 269 buff->eop_index, 270 self->hw_head))) 271 break; 272 273 dma_unmap_single(dev, buff->pa, buff->len, 274 DMA_TO_DEVICE); 275 } else { 276 dma_unmap_page(dev, buff->pa, buff->len, 277 DMA_TO_DEVICE); 278 } 279 } 280 281 if (unlikely(buff->is_eop)) { 282 ++self->stats.rx.packets; 283 self->stats.tx.bytes += buff->skb->len; 284 285 dev_kfree_skb_any(buff->skb); 286 } 287 buff->pa = 0U; 288 buff->eop_index = 0xffffU; 289 self->sw_head = aq_ring_next_dx(self, self->sw_head); 290 } 291 292 return !!budget; 293 } 294 295 static void aq_rx_checksum(struct aq_ring_s *self, 296 struct aq_ring_buff_s *buff, 297 struct sk_buff *skb) 298 { 299 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 300 return; 301 302 if (unlikely(buff->is_cso_err)) { 303 ++self->stats.rx.errors; 304 skb->ip_summed = CHECKSUM_NONE; 305 return; 306 } 307 if (buff->is_ip_cso) { 308 __skb_incr_checksum_unnecessary(skb); 309 } else { 310 skb->ip_summed = CHECKSUM_NONE; 311 } 312 313 if (buff->is_udp_cso || buff->is_tcp_cso) 314 __skb_incr_checksum_unnecessary(skb); 315 } 316 317 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 318 int aq_ring_rx_clean(struct aq_ring_s *self, 319 struct napi_struct *napi, 320 int *work_done, 321 int budget) 322 { 323 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 324 bool is_rsc_completed = true; 325 int err = 0; 326 327 for (; (self->sw_head != self->hw_head) && budget; 328 self->sw_head = aq_ring_next_dx(self, self->sw_head), 329 --budget, ++(*work_done)) { 330 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 331 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 332 struct aq_ring_buff_s *buff_ = NULL; 333 struct sk_buff *skb = NULL; 334 unsigned int next_ = 0U; 335 unsigned int i = 0U; 336 u16 hdr_len; 337 338 if (buff->is_cleaned) 339 continue; 340 341 if (!buff->is_eop) { 342 buff_ = buff; 343 do { 344 next_ = buff_->next, 345 buff_ = &self->buff_ring[next_]; 346 is_rsc_completed = 347 aq_ring_dx_in_range(self->sw_head, 348 next_, 349 self->hw_head); 350 351 if (unlikely(!is_rsc_completed)) 352 break; 353 354 buff->is_error |= buff_->is_error; 355 buff->is_cso_err |= buff_->is_cso_err; 356 357 } while (!buff_->is_eop); 358 359 if (!is_rsc_completed) { 360 err = 0; 361 goto err_exit; 362 } 363 if (buff->is_error || 364 (buff->is_lro && buff->is_cso_err)) { 365 buff_ = buff; 366 do { 367 next_ = buff_->next, 368 buff_ = &self->buff_ring[next_]; 369 370 buff_->is_cleaned = true; 371 } while (!buff_->is_eop); 372 373 ++self->stats.rx.errors; 374 continue; 375 } 376 } 377 378 if (buff->is_error) { 379 ++self->stats.rx.errors; 380 continue; 381 } 382 383 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 384 buff->rxdata.daddr, 385 buff->rxdata.pg_off, 386 buff->len, DMA_FROM_DEVICE); 387 388 /* for single fragment packets use build_skb() */ 389 if (buff->is_eop && 390 buff->len <= AQ_CFG_RX_FRAME_MAX - AQ_SKB_ALIGN) { 391 skb = build_skb(aq_buf_vaddr(&buff->rxdata), 392 AQ_CFG_RX_FRAME_MAX); 393 if (unlikely(!skb)) { 394 err = -ENOMEM; 395 goto err_exit; 396 } 397 if (is_ptp_ring) 398 buff->len -= 399 aq_ptp_extract_ts(self->aq_nic, skb, 400 aq_buf_vaddr(&buff->rxdata), 401 buff->len); 402 skb_put(skb, buff->len); 403 page_ref_inc(buff->rxdata.page); 404 } else { 405 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 406 if (unlikely(!skb)) { 407 err = -ENOMEM; 408 goto err_exit; 409 } 410 if (is_ptp_ring) 411 buff->len -= 412 aq_ptp_extract_ts(self->aq_nic, skb, 413 aq_buf_vaddr(&buff->rxdata), 414 buff->len); 415 416 hdr_len = buff->len; 417 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 418 hdr_len = eth_get_headlen(skb->dev, 419 aq_buf_vaddr(&buff->rxdata), 420 AQ_CFG_RX_HDR_SIZE); 421 422 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 423 ALIGN(hdr_len, sizeof(long))); 424 425 if (buff->len - hdr_len > 0) { 426 skb_add_rx_frag(skb, 0, buff->rxdata.page, 427 buff->rxdata.pg_off + hdr_len, 428 buff->len - hdr_len, 429 AQ_CFG_RX_FRAME_MAX); 430 page_ref_inc(buff->rxdata.page); 431 } 432 433 if (!buff->is_eop) { 434 buff_ = buff; 435 i = 1U; 436 do { 437 next_ = buff_->next, 438 buff_ = &self->buff_ring[next_]; 439 440 dma_sync_single_range_for_cpu( 441 aq_nic_get_dev(self->aq_nic), 442 buff_->rxdata.daddr, 443 buff_->rxdata.pg_off, 444 buff_->len, 445 DMA_FROM_DEVICE); 446 skb_add_rx_frag(skb, i++, 447 buff_->rxdata.page, 448 buff_->rxdata.pg_off, 449 buff_->len, 450 AQ_CFG_RX_FRAME_MAX); 451 page_ref_inc(buff_->rxdata.page); 452 buff_->is_cleaned = 1; 453 454 buff->is_ip_cso &= buff_->is_ip_cso; 455 buff->is_udp_cso &= buff_->is_udp_cso; 456 buff->is_tcp_cso &= buff_->is_tcp_cso; 457 buff->is_cso_err |= buff_->is_cso_err; 458 459 } while (!buff_->is_eop); 460 } 461 } 462 463 if (buff->is_vlan) 464 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 465 buff->vlan_rx_tag); 466 467 skb->protocol = eth_type_trans(skb, ndev); 468 469 aq_rx_checksum(self, buff, skb); 470 471 skb_set_hash(skb, buff->rss_hash, 472 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 473 PKT_HASH_TYPE_NONE); 474 /* Send all PTP traffic to 0 queue */ 475 skb_record_rx_queue(skb, 476 is_ptp_ring ? 0 477 : AQ_NIC_RING2QMAP(self->aq_nic, 478 self->idx)); 479 480 ++self->stats.rx.packets; 481 self->stats.rx.bytes += skb->len; 482 483 napi_gro_receive(napi, skb); 484 } 485 486 err_exit: 487 return err; 488 } 489 490 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 491 { 492 while (self->sw_head != self->hw_head) { 493 u64 ns; 494 495 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 496 self->dx_ring + 497 (self->sw_head * self->dx_size), 498 self->dx_size, &ns); 499 aq_ptp_tx_hwtstamp(aq_nic, ns); 500 501 self->sw_head = aq_ring_next_dx(self, self->sw_head); 502 } 503 } 504 505 int aq_ring_rx_fill(struct aq_ring_s *self) 506 { 507 unsigned int page_order = self->page_order; 508 struct aq_ring_buff_s *buff = NULL; 509 int err = 0; 510 int i = 0; 511 512 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 513 self->size / 2)) 514 return err; 515 516 for (i = aq_ring_avail_dx(self); i--; 517 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 518 buff = &self->buff_ring[self->sw_tail]; 519 520 buff->flags = 0U; 521 buff->len = AQ_CFG_RX_FRAME_MAX; 522 523 err = aq_get_rxpages(self, buff, page_order); 524 if (err) 525 goto err_exit; 526 527 buff->pa = aq_buf_daddr(&buff->rxdata); 528 buff = NULL; 529 } 530 531 err_exit: 532 return err; 533 } 534 535 void aq_ring_rx_deinit(struct aq_ring_s *self) 536 { 537 if (!self) 538 goto err_exit; 539 540 for (; self->sw_head != self->sw_tail; 541 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 542 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 543 544 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 545 } 546 547 err_exit:; 548 } 549 550 void aq_ring_free(struct aq_ring_s *self) 551 { 552 if (!self) 553 goto err_exit; 554 555 kfree(self->buff_ring); 556 557 if (self->dx_ring) 558 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 559 self->size * self->dx_size, self->dx_ring, 560 self->dx_ring_pa); 561 562 err_exit:; 563 } 564