1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/pci.h> 10 #include <linux/device.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/vmalloc.h> 13 #include <linux/errno.h> 14 #include <linux/sizes.h> 15 #include <linux/atomic.h> 16 #include <linux/skbuff.h> 17 #include <linux/io.h> 18 #include <asm/barrier.h> 19 #include <asm/byteorder.h> 20 21 #include "hinic_common.h" 22 #include "hinic_hw_if.h" 23 #include "hinic_hw_wqe.h" 24 #include "hinic_hw_wq.h" 25 #include "hinic_hw_qp_ctxt.h" 26 #include "hinic_hw_qp.h" 27 #include "hinic_hw_io.h" 28 29 #define SQ_DB_OFF SZ_2K 30 31 /* The number of cache line to prefetch Until threshold state */ 32 #define WQ_PREFETCH_MAX 2 33 /* The number of cache line to prefetch After threshold state */ 34 #define WQ_PREFETCH_MIN 1 35 /* Threshold state */ 36 #define WQ_PREFETCH_THRESHOLD 256 37 38 /* sizes of the SQ/RQ ctxt */ 39 #define Q_CTXT_SIZE 48 40 #define CTXT_RSVD 240 41 42 #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \ 43 (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE) 44 45 #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \ 46 (((max_rqs) + (max_sqs)) * CTXT_RSVD + \ 47 (max_sqs + (q_id)) * Q_CTXT_SIZE) 48 49 #define SIZE_16BYTES(size) (ALIGN(size, 16) >> 4) 50 #define SIZE_8BYTES(size) (ALIGN(size, 8) >> 3) 51 #define SECT_SIZE_FROM_8BYTES(size) ((size) << 3) 52 53 #define SQ_DB_PI_HI_SHIFT 8 54 #define SQ_DB_PI_HI(prod_idx) ((prod_idx) >> SQ_DB_PI_HI_SHIFT) 55 56 #define SQ_DB_PI_LOW_MASK 0xFF 57 #define SQ_DB_PI_LOW(prod_idx) ((prod_idx) & SQ_DB_PI_LOW_MASK) 58 59 #define SQ_DB_ADDR(sq, pi) ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi)) 60 61 #define SQ_MASKED_IDX(sq, idx) ((idx) & (sq)->wq->mask) 62 #define RQ_MASKED_IDX(rq, idx) ((idx) & (rq)->wq->mask) 63 64 enum sq_wqe_type { 65 SQ_NORMAL_WQE = 0, 66 }; 67 68 enum rq_completion_fmt { 69 RQ_COMPLETE_SGE = 1 70 }; 71 72 void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr, 73 enum hinic_qp_ctxt_type ctxt_type, 74 u16 num_queues, u16 max_queues) 75 { 76 u16 max_sqs = max_queues; 77 u16 max_rqs = max_queues; 78 79 qp_ctxt_hdr->num_queues = num_queues; 80 qp_ctxt_hdr->queue_type = ctxt_type; 81 82 if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ) 83 qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0); 84 else 85 qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0); 86 87 qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset); 88 89 hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr)); 90 } 91 92 void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt, 93 struct hinic_sq *sq, u16 global_qid) 94 { 95 u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo; 96 u64 wq_page_addr, wq_page_pfn, wq_block_pfn; 97 u16 pi_start, ci_start; 98 struct hinic_wq *wq; 99 100 wq = sq->wq; 101 ci_start = atomic_read(&wq->cons_idx); 102 pi_start = atomic_read(&wq->prod_idx); 103 104 /* Read the first page paddr from the WQ page paddr ptrs */ 105 wq_page_addr = be64_to_cpu(*wq->block_vaddr); 106 107 wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr); 108 wq_page_pfn_hi = upper_32_bits(wq_page_pfn); 109 wq_page_pfn_lo = lower_32_bits(wq_page_pfn); 110 111 /* If only one page, use 0-level CLA */ 112 if (wq->num_q_pages == 1) 113 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr); 114 else 115 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr); 116 117 wq_block_pfn_hi = upper_32_bits(wq_block_pfn); 118 wq_block_pfn_lo = lower_32_bits(wq_block_pfn); 119 120 sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid, 121 GLOBAL_SQ_ID) | 122 HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN); 123 124 sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) | 125 HINIC_SQ_CTXT_CI_SET(1, WRAPPED); 126 127 sq_ctxt->wq_hi_pfn_pi = 128 HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) | 129 HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI); 130 131 sq_ctxt->wq_lo_pfn = wq_page_pfn_lo; 132 133 sq_ctxt->pref_cache = 134 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) | 135 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) | 136 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD); 137 138 sq_ctxt->pref_wrapped = 1; 139 140 sq_ctxt->pref_wq_hi_pfn_ci = 141 HINIC_SQ_CTXT_PREF_SET(ci_start, CI) | 142 HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN); 143 144 sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo; 145 146 sq_ctxt->wq_block_hi_pfn = 147 HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN); 148 149 sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo; 150 151 hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt)); 152 } 153 154 void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt, 155 struct hinic_rq *rq, u16 global_qid) 156 { 157 u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo; 158 u64 wq_page_addr, wq_page_pfn, wq_block_pfn; 159 u16 pi_start, ci_start; 160 struct hinic_wq *wq; 161 162 wq = rq->wq; 163 ci_start = atomic_read(&wq->cons_idx); 164 pi_start = atomic_read(&wq->prod_idx); 165 166 /* Read the first page paddr from the WQ page paddr ptrs */ 167 wq_page_addr = be64_to_cpu(*wq->block_vaddr); 168 169 wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr); 170 wq_page_pfn_hi = upper_32_bits(wq_page_pfn); 171 wq_page_pfn_lo = lower_32_bits(wq_page_pfn); 172 173 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr); 174 wq_block_pfn_hi = upper_32_bits(wq_block_pfn); 175 wq_block_pfn_lo = lower_32_bits(wq_block_pfn); 176 177 rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) | 178 HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED); 179 180 rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) | 181 HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR); 182 183 rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, 184 HI_PFN) | 185 HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI); 186 187 rq_ctxt->wq_lo_pfn = wq_page_pfn_lo; 188 189 rq_ctxt->pref_cache = 190 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) | 191 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) | 192 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD); 193 194 rq_ctxt->pref_wrapped = 1; 195 196 rq_ctxt->pref_wq_hi_pfn_ci = 197 HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) | 198 HINIC_RQ_CTXT_PREF_SET(ci_start, CI); 199 200 rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo; 201 202 rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr); 203 rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr); 204 205 rq_ctxt->wq_block_hi_pfn = 206 HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN); 207 208 rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo; 209 210 hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt)); 211 } 212 213 /** 214 * alloc_sq_skb_arr - allocate sq array for saved skb 215 * @sq: HW Send Queue 216 * 217 * Return 0 - Success, negative - Failure 218 **/ 219 static int alloc_sq_skb_arr(struct hinic_sq *sq) 220 { 221 struct hinic_wq *wq = sq->wq; 222 size_t skb_arr_size; 223 224 skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb); 225 sq->saved_skb = vzalloc(skb_arr_size); 226 if (!sq->saved_skb) 227 return -ENOMEM; 228 229 return 0; 230 } 231 232 /** 233 * free_sq_skb_arr - free sq array for saved skb 234 * @sq: HW Send Queue 235 **/ 236 static void free_sq_skb_arr(struct hinic_sq *sq) 237 { 238 vfree(sq->saved_skb); 239 } 240 241 /** 242 * alloc_rq_skb_arr - allocate rq array for saved skb 243 * @rq: HW Receive Queue 244 * 245 * Return 0 - Success, negative - Failure 246 **/ 247 static int alloc_rq_skb_arr(struct hinic_rq *rq) 248 { 249 struct hinic_wq *wq = rq->wq; 250 size_t skb_arr_size; 251 252 skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb); 253 rq->saved_skb = vzalloc(skb_arr_size); 254 if (!rq->saved_skb) 255 return -ENOMEM; 256 257 return 0; 258 } 259 260 /** 261 * free_rq_skb_arr - free rq array for saved skb 262 * @rq: HW Receive Queue 263 **/ 264 static void free_rq_skb_arr(struct hinic_rq *rq) 265 { 266 vfree(rq->saved_skb); 267 } 268 269 /** 270 * hinic_init_sq - Initialize HW Send Queue 271 * @sq: HW Send Queue 272 * @hwif: HW Interface for accessing HW 273 * @wq: Work Queue for the data of the SQ 274 * @entry: msix entry for sq 275 * @ci_addr: address for reading the current HW consumer index 276 * @ci_dma_addr: dma address for reading the current HW consumer index 277 * @db_base: doorbell base address 278 * 279 * Return 0 - Success, negative - Failure 280 **/ 281 int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif, 282 struct hinic_wq *wq, struct msix_entry *entry, 283 void *ci_addr, dma_addr_t ci_dma_addr, 284 void __iomem *db_base) 285 { 286 sq->hwif = hwif; 287 288 sq->wq = wq; 289 290 sq->irq = entry->vector; 291 sq->msix_entry = entry->entry; 292 293 sq->hw_ci_addr = ci_addr; 294 sq->hw_ci_dma_addr = ci_dma_addr; 295 296 sq->db_base = db_base + SQ_DB_OFF; 297 298 return alloc_sq_skb_arr(sq); 299 } 300 301 /** 302 * hinic_clean_sq - Clean HW Send Queue's Resources 303 * @sq: Send Queue 304 **/ 305 void hinic_clean_sq(struct hinic_sq *sq) 306 { 307 free_sq_skb_arr(sq); 308 } 309 310 /** 311 * alloc_rq_cqe - allocate rq completion queue elements 312 * @rq: HW Receive Queue 313 * 314 * Return 0 - Success, negative - Failure 315 **/ 316 static int alloc_rq_cqe(struct hinic_rq *rq) 317 { 318 struct hinic_hwif *hwif = rq->hwif; 319 struct pci_dev *pdev = hwif->pdev; 320 size_t cqe_dma_size, cqe_size; 321 struct hinic_wq *wq = rq->wq; 322 int j, i; 323 324 cqe_size = wq->q_depth * sizeof(*rq->cqe); 325 rq->cqe = vzalloc(cqe_size); 326 if (!rq->cqe) 327 return -ENOMEM; 328 329 cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma); 330 rq->cqe_dma = vzalloc(cqe_dma_size); 331 if (!rq->cqe_dma) 332 goto err_cqe_dma_arr_alloc; 333 334 for (i = 0; i < wq->q_depth; i++) { 335 rq->cqe[i] = dma_alloc_coherent(&pdev->dev, 336 sizeof(*rq->cqe[i]), 337 &rq->cqe_dma[i], GFP_KERNEL); 338 if (!rq->cqe[i]) 339 goto err_cqe_alloc; 340 } 341 342 return 0; 343 344 err_cqe_alloc: 345 for (j = 0; j < i; j++) 346 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j], 347 rq->cqe_dma[j]); 348 349 vfree(rq->cqe_dma); 350 351 err_cqe_dma_arr_alloc: 352 vfree(rq->cqe); 353 return -ENOMEM; 354 } 355 356 /** 357 * free_rq_cqe - free rq completion queue elements 358 * @rq: HW Receive Queue 359 **/ 360 static void free_rq_cqe(struct hinic_rq *rq) 361 { 362 struct hinic_hwif *hwif = rq->hwif; 363 struct pci_dev *pdev = hwif->pdev; 364 struct hinic_wq *wq = rq->wq; 365 int i; 366 367 for (i = 0; i < wq->q_depth; i++) 368 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i], 369 rq->cqe_dma[i]); 370 371 vfree(rq->cqe_dma); 372 vfree(rq->cqe); 373 } 374 375 /** 376 * hinic_init_rq - Initialize HW Receive Queue 377 * @rq: HW Receive Queue 378 * @hwif: HW Interface for accessing HW 379 * @wq: Work Queue for the data of the RQ 380 * @entry: msix entry for rq 381 * 382 * Return 0 - Success, negative - Failure 383 **/ 384 int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif, 385 struct hinic_wq *wq, struct msix_entry *entry) 386 { 387 struct pci_dev *pdev = hwif->pdev; 388 size_t pi_size; 389 int err; 390 391 rq->hwif = hwif; 392 393 rq->wq = wq; 394 395 rq->irq = entry->vector; 396 rq->msix_entry = entry->entry; 397 398 rq->buf_sz = HINIC_RX_BUF_SZ; 399 400 err = alloc_rq_skb_arr(rq); 401 if (err) { 402 dev_err(&pdev->dev, "Failed to allocate rq priv data\n"); 403 return err; 404 } 405 406 err = alloc_rq_cqe(rq); 407 if (err) { 408 dev_err(&pdev->dev, "Failed to allocate rq cqe\n"); 409 goto err_alloc_rq_cqe; 410 } 411 412 /* HW requirements: Must be at least 32 bit */ 413 pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32)); 414 rq->pi_virt_addr = dma_alloc_coherent(&pdev->dev, pi_size, 415 &rq->pi_dma_addr, GFP_KERNEL); 416 if (!rq->pi_virt_addr) { 417 err = -ENOMEM; 418 goto err_pi_virt; 419 } 420 421 return 0; 422 423 err_pi_virt: 424 free_rq_cqe(rq); 425 426 err_alloc_rq_cqe: 427 free_rq_skb_arr(rq); 428 return err; 429 } 430 431 /** 432 * hinic_clean_rq - Clean HW Receive Queue's Resources 433 * @rq: HW Receive Queue 434 **/ 435 void hinic_clean_rq(struct hinic_rq *rq) 436 { 437 struct hinic_hwif *hwif = rq->hwif; 438 struct pci_dev *pdev = hwif->pdev; 439 size_t pi_size; 440 441 pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32)); 442 dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr, 443 rq->pi_dma_addr); 444 445 free_rq_cqe(rq); 446 free_rq_skb_arr(rq); 447 } 448 449 /** 450 * hinic_get_sq_free_wqebbs - return number of free wqebbs for use 451 * @sq: send queue 452 * 453 * Return number of free wqebbs 454 **/ 455 int hinic_get_sq_free_wqebbs(struct hinic_sq *sq) 456 { 457 struct hinic_wq *wq = sq->wq; 458 459 return atomic_read(&wq->delta) - 1; 460 } 461 462 /** 463 * hinic_get_rq_free_wqebbs - return number of free wqebbs for use 464 * @rq: recv queue 465 * 466 * Return number of free wqebbs 467 **/ 468 int hinic_get_rq_free_wqebbs(struct hinic_rq *rq) 469 { 470 struct hinic_wq *wq = rq->wq; 471 472 return atomic_read(&wq->delta) - 1; 473 } 474 475 static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs) 476 { 477 u32 ctrl_size, task_size, bufdesc_size; 478 479 ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl)); 480 task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task)); 481 bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc); 482 bufdesc_size = SIZE_8BYTES(bufdesc_size); 483 484 ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) | 485 HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN) | 486 HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | 487 HINIC_SQ_CTRL_SET(ctrl_size, LEN); 488 489 ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT, 490 QUEUE_INFO_MSS) | 491 HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC); 492 } 493 494 static void sq_prepare_task(struct hinic_sq_task *task) 495 { 496 task->pkt_info0 = 0; 497 task->pkt_info1 = 0; 498 task->pkt_info2 = 0; 499 500 task->ufo_v6_identify = 0; 501 502 task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE); 503 504 task->zero_pad = 0; 505 } 506 507 void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len) 508 { 509 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN); 510 } 511 512 void hinic_task_set_outter_l3(struct hinic_sq_task *task, 513 enum hinic_l3_offload_type l3_type, 514 u32 network_len) 515 { 516 task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) | 517 HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN); 518 } 519 520 void hinic_task_set_inner_l3(struct hinic_sq_task *task, 521 enum hinic_l3_offload_type l3_type, 522 u32 network_len) 523 { 524 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE); 525 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN); 526 } 527 528 void hinic_task_set_tunnel_l4(struct hinic_sq_task *task, 529 enum hinic_l4_tunnel_type l4_type, 530 u32 tunnel_len) 531 { 532 task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) | 533 HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN); 534 } 535 536 void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info, 537 enum hinic_l4_offload_type l4_offload, 538 u32 l4_len, u32 offset) 539 { 540 u32 tcp_udp_cs = 0, sctp = 0; 541 u32 mss = HINIC_MSS_DEFAULT; 542 543 if (l4_offload == TCP_OFFLOAD_ENABLE || 544 l4_offload == UDP_OFFLOAD_ENABLE) 545 tcp_udp_cs = 1; 546 else if (l4_offload == SCTP_OFFLOAD_ENABLE) 547 sctp = 1; 548 549 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD); 550 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN); 551 552 *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) | 553 HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) | 554 HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP); 555 556 *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS); 557 *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS); 558 } 559 560 void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info, 561 enum hinic_l4_offload_type l4_offload, 562 u32 l4_len, u32 offset, u32 ip_ident, u32 mss) 563 { 564 u32 tso = 0, ufo = 0; 565 566 if (l4_offload == TCP_OFFLOAD_ENABLE) 567 tso = 1; 568 else if (l4_offload == UDP_OFFLOAD_ENABLE) 569 ufo = 1; 570 571 task->ufo_v6_identify = ip_ident; 572 573 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD); 574 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG); 575 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN); 576 577 *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) | 578 HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) | 579 HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) | 580 HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS); 581 582 /* set MSS value */ 583 *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS); 584 *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS); 585 } 586 587 /** 588 * hinic_sq_prepare_wqe - prepare wqe before insert to the queue 589 * @sq: send queue 590 * @sq_wqe: wqe to prepare 591 * @sges: sges for use by the wqe for send for buf addresses 592 * @nr_sges: number of sges 593 **/ 594 void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe, 595 struct hinic_sge *sges, int nr_sges) 596 { 597 int i; 598 599 sq_prepare_ctrl(&sq_wqe->ctrl, nr_sges); 600 601 sq_prepare_task(&sq_wqe->task); 602 603 for (i = 0; i < nr_sges; i++) 604 sq_wqe->buf_descs[i].sge = sges[i]; 605 } 606 607 /** 608 * sq_prepare_db - prepare doorbell to write 609 * @sq: send queue 610 * @prod_idx: pi value for the doorbell 611 * @cos: cos of the doorbell 612 * 613 * Return db value 614 **/ 615 static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos) 616 { 617 struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); 618 u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx)); 619 620 /* Data should be written to HW in Big Endian Format */ 621 return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI) | 622 HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) | 623 HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH) | 624 HINIC_SQ_DB_INFO_SET(cos, COS) | 625 HINIC_SQ_DB_INFO_SET(qp->q_id, QID)); 626 } 627 628 /** 629 * hinic_sq_write_db- write doorbell 630 * @sq: send queue 631 * @prod_idx: pi value for the doorbell 632 * @wqe_size: wqe size 633 * @cos: cos of the wqe 634 **/ 635 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size, 636 unsigned int cos) 637 { 638 struct hinic_wq *wq = sq->wq; 639 640 /* increment prod_idx to the next */ 641 prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 642 prod_idx = SQ_MASKED_IDX(sq, prod_idx); 643 644 wmb(); /* Write all before the doorbell */ 645 646 writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx)); 647 } 648 649 /** 650 * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi 651 * @sq: sq to get wqe from 652 * @wqe_size: wqe size 653 * @prod_idx: returned pi 654 * 655 * Return wqe pointer 656 **/ 657 struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq, 658 unsigned int wqe_size, u16 *prod_idx) 659 { 660 struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size, 661 prod_idx); 662 663 if (IS_ERR(hw_wqe)) 664 return NULL; 665 666 return &hw_wqe->sq_wqe; 667 } 668 669 /** 670 * hinic_sq_return_wqe - return the wqe to the sq 671 * @sq: send queue 672 * @wqe_size: the size of the wqe 673 **/ 674 void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size) 675 { 676 hinic_return_wqe(sq->wq, wqe_size); 677 } 678 679 /** 680 * hinic_sq_write_wqe - write the wqe to the sq 681 * @sq: send queue 682 * @prod_idx: pi of the wqe 683 * @sq_wqe: the wqe to write 684 * @skb: skb to save 685 * @wqe_size: the size of the wqe 686 **/ 687 void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx, 688 struct hinic_sq_wqe *sq_wqe, 689 struct sk_buff *skb, unsigned int wqe_size) 690 { 691 struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe; 692 693 sq->saved_skb[prod_idx] = skb; 694 695 /* The data in the HW should be in Big Endian Format */ 696 hinic_cpu_to_be32(sq_wqe, wqe_size); 697 698 hinic_write_wqe(sq->wq, hw_wqe, wqe_size); 699 } 700 701 /** 702 * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the 703 * wqe only have one wqebb 704 * @sq: send queue 705 * @skb: return skb that was saved 706 * @wqe_size: the wqe size ptr 707 * @cons_idx: consumer index of the wqe 708 * 709 * Return wqe in ci position 710 **/ 711 struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq, 712 struct sk_buff **skb, 713 unsigned int *wqe_size, u16 *cons_idx) 714 { 715 struct hinic_hw_wqe *hw_wqe; 716 struct hinic_sq_wqe *sq_wqe; 717 struct hinic_sq_ctrl *ctrl; 718 unsigned int buf_sect_len; 719 u32 ctrl_info; 720 721 /* read the ctrl section for getting wqe size */ 722 hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx); 723 if (IS_ERR(hw_wqe)) 724 return NULL; 725 726 *skb = sq->saved_skb[*cons_idx]; 727 728 sq_wqe = &hw_wqe->sq_wqe; 729 ctrl = &sq_wqe->ctrl; 730 ctrl_info = be32_to_cpu(ctrl->ctrl_info); 731 buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN); 732 733 *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task); 734 *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len); 735 *wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size); 736 737 return &hw_wqe->sq_wqe; 738 } 739 740 /** 741 * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci 742 * @sq: send queue 743 * @skb: return skb that was saved 744 * @wqe_size: the size of the wqe 745 * @cons_idx: consumer index of the wqe 746 * 747 * Return wqe in ci position 748 **/ 749 struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq, 750 struct sk_buff **skb, 751 unsigned int wqe_size, u16 *cons_idx) 752 { 753 struct hinic_hw_wqe *hw_wqe; 754 755 hw_wqe = hinic_read_wqe(sq->wq, wqe_size, cons_idx); 756 *skb = sq->saved_skb[*cons_idx]; 757 758 return &hw_wqe->sq_wqe; 759 } 760 761 /** 762 * hinic_sq_put_wqe - release the ci for new wqes 763 * @sq: send queue 764 * @wqe_size: the size of the wqe 765 **/ 766 void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size) 767 { 768 hinic_put_wqe(sq->wq, wqe_size); 769 } 770 771 /** 772 * hinic_sq_get_sges - get sges from the wqe 773 * @sq_wqe: wqe to get the sges from its buffer addresses 774 * @sges: returned sges 775 * @nr_sges: number sges to return 776 **/ 777 void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges, 778 int nr_sges) 779 { 780 int i; 781 782 for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) { 783 sges[i] = sq_wqe->buf_descs[i].sge; 784 hinic_be32_to_cpu(&sges[i], sizeof(sges[i])); 785 } 786 } 787 788 /** 789 * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi 790 * @rq: rq to get wqe from 791 * @wqe_size: wqe size 792 * @prod_idx: returned pi 793 * 794 * Return wqe pointer 795 **/ 796 struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq, 797 unsigned int wqe_size, u16 *prod_idx) 798 { 799 struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size, 800 prod_idx); 801 802 if (IS_ERR(hw_wqe)) 803 return NULL; 804 805 return &hw_wqe->rq_wqe; 806 } 807 808 /** 809 * hinic_rq_write_wqe - write the wqe to the rq 810 * @rq: recv queue 811 * @prod_idx: pi of the wqe 812 * @rq_wqe: the wqe to write 813 * @skb: skb to save 814 **/ 815 void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx, 816 struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb) 817 { 818 struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe; 819 820 rq->saved_skb[prod_idx] = skb; 821 822 /* The data in the HW should be in Big Endian Format */ 823 hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe)); 824 825 hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe)); 826 } 827 828 /** 829 * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci 830 * @rq: recv queue 831 * @wqe_size: the size of the wqe 832 * @skb: return saved skb 833 * @cons_idx: consumer index of the wqe 834 * 835 * Return wqe in ci position 836 **/ 837 struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq, 838 unsigned int wqe_size, 839 struct sk_buff **skb, u16 *cons_idx) 840 { 841 struct hinic_hw_wqe *hw_wqe; 842 struct hinic_rq_cqe *cqe; 843 int rx_done; 844 u32 status; 845 846 hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx); 847 if (IS_ERR(hw_wqe)) 848 return NULL; 849 850 cqe = rq->cqe[*cons_idx]; 851 852 status = be32_to_cpu(cqe->status); 853 854 rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE); 855 if (!rx_done) 856 return NULL; 857 858 *skb = rq->saved_skb[*cons_idx]; 859 860 return &hw_wqe->rq_wqe; 861 } 862 863 /** 864 * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position 865 * @rq: recv queue 866 * @wqe_size: the size of the wqe 867 * @skb: return saved skb 868 * @cons_idx: consumer index in the wq 869 * 870 * Return wqe in incremented ci position 871 **/ 872 struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq, 873 unsigned int wqe_size, 874 struct sk_buff **skb, 875 u16 *cons_idx) 876 { 877 struct hinic_wq *wq = rq->wq; 878 struct hinic_hw_wqe *hw_wqe; 879 unsigned int num_wqebbs; 880 881 wqe_size = ALIGN(wqe_size, wq->wqebb_size); 882 num_wqebbs = wqe_size / wq->wqebb_size; 883 884 *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs); 885 886 *skb = rq->saved_skb[*cons_idx]; 887 888 hw_wqe = hinic_read_wqe_direct(wq, *cons_idx); 889 890 return &hw_wqe->rq_wqe; 891 } 892 893 /** 894 * hinic_rq_put_wqe - release the ci for new wqes 895 * @rq: recv queue 896 * @cons_idx: consumer index of the wqe 897 * @wqe_size: the size of the wqe 898 **/ 899 void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx, 900 unsigned int wqe_size) 901 { 902 struct hinic_rq_cqe *cqe = rq->cqe[cons_idx]; 903 u32 status = be32_to_cpu(cqe->status); 904 905 status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE); 906 907 /* Rx WQE size is 1 WQEBB, no wq shadow*/ 908 cqe->status = cpu_to_be32(status); 909 910 wmb(); /* clear done flag */ 911 912 hinic_put_wqe(rq->wq, wqe_size); 913 } 914 915 /** 916 * hinic_rq_get_sge - get sge from the wqe 917 * @rq: recv queue 918 * @rq_wqe: wqe to get the sge from its buf address 919 * @cons_idx: consumer index 920 * @sge: returned sge 921 **/ 922 void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe, 923 u16 cons_idx, struct hinic_sge *sge) 924 { 925 struct hinic_rq_cqe *cqe = rq->cqe[cons_idx]; 926 u32 len = be32_to_cpu(cqe->len); 927 928 sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr); 929 sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr); 930 sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN); 931 } 932 933 /** 934 * hinic_rq_prepare_wqe - prepare wqe before insert to the queue 935 * @rq: recv queue 936 * @prod_idx: pi value 937 * @rq_wqe: the wqe 938 * @sge: sge for use by the wqe for recv buf address 939 **/ 940 void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx, 941 struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge) 942 { 943 struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect; 944 struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc; 945 struct hinic_rq_cqe *cqe = rq->cqe[prod_idx]; 946 struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl; 947 dma_addr_t cqe_dma = rq->cqe_dma[prod_idx]; 948 949 ctrl->ctrl_info = 950 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) | 951 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), 952 COMPLETE_LEN) | 953 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), 954 BUFDESC_SECT_LEN) | 955 HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT); 956 957 hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe)); 958 959 buf_desc->hi_addr = sge->hi_addr; 960 buf_desc->lo_addr = sge->lo_addr; 961 } 962 963 /** 964 * hinic_rq_update - update pi of the rq 965 * @rq: recv queue 966 * @prod_idx: pi value 967 **/ 968 void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx) 969 { 970 *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1)); 971 } 972