1 /* 2 * Huawei HiNIC PCI Express Linux driver 3 * Copyright(c) 2017 Huawei Technologies Co., Ltd 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * for more details. 13 * 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/errno.h> 19 #include <linux/pci.h> 20 #include <linux/device.h> 21 #include <linux/workqueue.h> 22 #include <linux/interrupt.h> 23 #include <linux/slab.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/log2.h> 26 #include <asm/byteorder.h> 27 #include <asm/barrier.h> 28 29 #include "hinic_hw_csr.h" 30 #include "hinic_hw_if.h" 31 #include "hinic_hw_eqs.h" 32 33 #define HINIC_EQS_WQ_NAME "hinic_eqs" 34 35 #define GET_EQ_NUM_PAGES(eq, pg_size) \ 36 (ALIGN((eq)->q_len * (eq)->elem_size, pg_size) / (pg_size)) 37 38 #define GET_EQ_NUM_ELEMS_IN_PG(eq, pg_size) ((pg_size) / (eq)->elem_size) 39 40 #define EQ_CONS_IDX_REG_ADDR(eq) (((eq)->type == HINIC_AEQ) ? \ 41 HINIC_CSR_AEQ_CONS_IDX_ADDR((eq)->q_id) : \ 42 HINIC_CSR_CEQ_CONS_IDX_ADDR((eq)->q_id)) 43 44 #define EQ_PROD_IDX_REG_ADDR(eq) (((eq)->type == HINIC_AEQ) ? \ 45 HINIC_CSR_AEQ_PROD_IDX_ADDR((eq)->q_id) : \ 46 HINIC_CSR_CEQ_PROD_IDX_ADDR((eq)->q_id)) 47 48 #define EQ_HI_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \ 49 HINIC_CSR_AEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num) : \ 50 HINIC_CSR_CEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num)) 51 52 #define EQ_LO_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \ 53 HINIC_CSR_AEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num) : \ 54 HINIC_CSR_CEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num)) 55 56 #define GET_EQ_ELEMENT(eq, idx) \ 57 ((eq)->virt_addr[(idx) / (eq)->num_elem_in_pg] + \ 58 (((idx) & ((eq)->num_elem_in_pg - 1)) * (eq)->elem_size)) 59 60 #define GET_AEQ_ELEM(eq, idx) ((struct hinic_aeq_elem *) \ 61 GET_EQ_ELEMENT(eq, idx)) 62 63 #define GET_CEQ_ELEM(eq, idx) ((u32 *) \ 64 GET_EQ_ELEMENT(eq, idx)) 65 66 #define GET_CURR_AEQ_ELEM(eq) GET_AEQ_ELEM(eq, (eq)->cons_idx) 67 68 #define GET_CURR_CEQ_ELEM(eq) GET_CEQ_ELEM(eq, (eq)->cons_idx) 69 70 #define PAGE_IN_4K(page_size) ((page_size) >> 12) 71 #define EQ_SET_HW_PAGE_SIZE_VAL(eq) (ilog2(PAGE_IN_4K((eq)->page_size))) 72 73 #define ELEMENT_SIZE_IN_32B(eq) (((eq)->elem_size) >> 5) 74 #define EQ_SET_HW_ELEM_SIZE_VAL(eq) (ilog2(ELEMENT_SIZE_IN_32B(eq))) 75 76 #define EQ_MAX_PAGES 8 77 78 #define CEQE_TYPE_SHIFT 23 79 #define CEQE_TYPE_MASK 0x7 80 81 #define CEQE_TYPE(ceqe) (((ceqe) >> CEQE_TYPE_SHIFT) & \ 82 CEQE_TYPE_MASK) 83 84 #define CEQE_DATA_MASK 0x3FFFFFF 85 #define CEQE_DATA(ceqe) ((ceqe) & CEQE_DATA_MASK) 86 87 #define aeq_to_aeqs(eq) \ 88 container_of((eq) - (eq)->q_id, struct hinic_aeqs, aeq[0]) 89 90 #define ceq_to_ceqs(eq) \ 91 container_of((eq) - (eq)->q_id, struct hinic_ceqs, ceq[0]) 92 93 #define work_to_aeq_work(work) \ 94 container_of(work, struct hinic_eq_work, work) 95 96 #define DMA_ATTR_AEQ_DEFAULT 0 97 #define DMA_ATTR_CEQ_DEFAULT 0 98 99 /* No coalescence */ 100 #define THRESH_CEQ_DEFAULT 0 101 102 enum eq_int_mode { 103 EQ_INT_MODE_ARMED, 104 EQ_INT_MODE_ALWAYS 105 }; 106 107 enum eq_arm_state { 108 EQ_NOT_ARMED, 109 EQ_ARMED 110 }; 111 112 /** 113 * hinic_aeq_register_hw_cb - register AEQ callback for specific event 114 * @aeqs: pointer to Async eqs of the chip 115 * @event: aeq event to register callback for it 116 * @handle: private data will be used by the callback 117 * @hw_handler: callback function 118 **/ 119 void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs, 120 enum hinic_aeq_type event, void *handle, 121 void (*hwe_handler)(void *handle, void *data, 122 u8 size)) 123 { 124 struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event]; 125 126 hwe_cb->hwe_handler = hwe_handler; 127 hwe_cb->handle = handle; 128 hwe_cb->hwe_state = HINIC_EQE_ENABLED; 129 } 130 131 /** 132 * hinic_aeq_unregister_hw_cb - unregister the AEQ callback for specific event 133 * @aeqs: pointer to Async eqs of the chip 134 * @event: aeq event to unregister callback for it 135 **/ 136 void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs, 137 enum hinic_aeq_type event) 138 { 139 struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event]; 140 141 hwe_cb->hwe_state &= ~HINIC_EQE_ENABLED; 142 143 while (hwe_cb->hwe_state & HINIC_EQE_RUNNING) 144 schedule(); 145 146 hwe_cb->hwe_handler = NULL; 147 } 148 149 /** 150 * hinic_ceq_register_cb - register CEQ callback for specific event 151 * @ceqs: pointer to Completion eqs part of the chip 152 * @event: ceq event to register callback for it 153 * @handle: private data will be used by the callback 154 * @handler: callback function 155 **/ 156 void hinic_ceq_register_cb(struct hinic_ceqs *ceqs, 157 enum hinic_ceq_type event, void *handle, 158 void (*handler)(void *handle, u32 ceqe_data)) 159 { 160 struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event]; 161 162 ceq_cb->handler = handler; 163 ceq_cb->handle = handle; 164 ceq_cb->ceqe_state = HINIC_EQE_ENABLED; 165 } 166 167 /** 168 * hinic_ceq_unregister_cb - unregister the CEQ callback for specific event 169 * @ceqs: pointer to Completion eqs part of the chip 170 * @event: ceq event to unregister callback for it 171 **/ 172 void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs, 173 enum hinic_ceq_type event) 174 { 175 struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event]; 176 177 ceq_cb->ceqe_state &= ~HINIC_EQE_ENABLED; 178 179 while (ceq_cb->ceqe_state & HINIC_EQE_RUNNING) 180 schedule(); 181 182 ceq_cb->handler = NULL; 183 } 184 185 static u8 eq_cons_idx_checksum_set(u32 val) 186 { 187 u8 checksum = 0; 188 int idx; 189 190 for (idx = 0; idx < 32; idx += 4) 191 checksum ^= ((val >> idx) & 0xF); 192 193 return (checksum & 0xF); 194 } 195 196 /** 197 * eq_update_ci - update the HW cons idx of event queue 198 * @eq: the event queue to update the cons idx for 199 **/ 200 static void eq_update_ci(struct hinic_eq *eq) 201 { 202 u32 val, addr = EQ_CONS_IDX_REG_ADDR(eq); 203 204 /* Read Modify Write */ 205 val = hinic_hwif_read_reg(eq->hwif, addr); 206 207 val = HINIC_EQ_CI_CLEAR(val, IDX) & 208 HINIC_EQ_CI_CLEAR(val, WRAPPED) & 209 HINIC_EQ_CI_CLEAR(val, INT_ARMED) & 210 HINIC_EQ_CI_CLEAR(val, XOR_CHKSUM); 211 212 val |= HINIC_EQ_CI_SET(eq->cons_idx, IDX) | 213 HINIC_EQ_CI_SET(eq->wrapped, WRAPPED) | 214 HINIC_EQ_CI_SET(EQ_ARMED, INT_ARMED); 215 216 val |= HINIC_EQ_CI_SET(eq_cons_idx_checksum_set(val), XOR_CHKSUM); 217 218 hinic_hwif_write_reg(eq->hwif, addr, val); 219 } 220 221 /** 222 * aeq_irq_handler - handler for the AEQ event 223 * @eq: the Async Event Queue that received the event 224 **/ 225 static void aeq_irq_handler(struct hinic_eq *eq) 226 { 227 struct hinic_aeqs *aeqs = aeq_to_aeqs(eq); 228 struct hinic_hwif *hwif = aeqs->hwif; 229 struct pci_dev *pdev = hwif->pdev; 230 struct hinic_aeq_elem *aeqe_curr; 231 struct hinic_hw_event_cb *hwe_cb; 232 enum hinic_aeq_type event; 233 unsigned long eqe_state; 234 u32 aeqe_desc; 235 int i, size; 236 237 for (i = 0; i < eq->q_len; i++) { 238 aeqe_curr = GET_CURR_AEQ_ELEM(eq); 239 240 /* Data in HW is in Big endian Format */ 241 aeqe_desc = be32_to_cpu(aeqe_curr->desc); 242 243 /* HW toggles the wrapped bit, when it adds eq element */ 244 if (HINIC_EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED) == eq->wrapped) 245 break; 246 247 event = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, TYPE); 248 if (event >= HINIC_MAX_AEQ_EVENTS) { 249 dev_err(&pdev->dev, "Unknown AEQ Event %d\n", event); 250 return; 251 } 252 253 if (!HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SRC)) { 254 hwe_cb = &aeqs->hwe_cb[event]; 255 256 size = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SIZE); 257 258 eqe_state = cmpxchg(&hwe_cb->hwe_state, 259 HINIC_EQE_ENABLED, 260 HINIC_EQE_ENABLED | 261 HINIC_EQE_RUNNING); 262 if ((eqe_state == HINIC_EQE_ENABLED) && 263 (hwe_cb->hwe_handler)) 264 hwe_cb->hwe_handler(hwe_cb->handle, 265 aeqe_curr->data, size); 266 else 267 dev_err(&pdev->dev, "Unhandled AEQ Event %d\n", 268 event); 269 270 hwe_cb->hwe_state &= ~HINIC_EQE_RUNNING; 271 } 272 273 eq->cons_idx++; 274 275 if (eq->cons_idx == eq->q_len) { 276 eq->cons_idx = 0; 277 eq->wrapped = !eq->wrapped; 278 } 279 } 280 } 281 282 /** 283 * ceq_event_handler - handler for the ceq events 284 * @ceqs: ceqs part of the chip 285 * @ceqe: ceq element that describes the event 286 **/ 287 static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe) 288 { 289 struct hinic_hwif *hwif = ceqs->hwif; 290 struct pci_dev *pdev = hwif->pdev; 291 struct hinic_ceq_cb *ceq_cb; 292 enum hinic_ceq_type event; 293 unsigned long eqe_state; 294 295 event = CEQE_TYPE(ceqe); 296 if (event >= HINIC_MAX_CEQ_EVENTS) { 297 dev_err(&pdev->dev, "Unknown CEQ event, event = %d\n", event); 298 return; 299 } 300 301 ceq_cb = &ceqs->ceq_cb[event]; 302 303 eqe_state = cmpxchg(&ceq_cb->ceqe_state, 304 HINIC_EQE_ENABLED, 305 HINIC_EQE_ENABLED | HINIC_EQE_RUNNING); 306 307 if ((eqe_state == HINIC_EQE_ENABLED) && (ceq_cb->handler)) 308 ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe)); 309 else 310 dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event); 311 312 ceq_cb->ceqe_state &= ~HINIC_EQE_RUNNING; 313 } 314 315 /** 316 * ceq_irq_handler - handler for the CEQ event 317 * @eq: the Completion Event Queue that received the event 318 **/ 319 static void ceq_irq_handler(struct hinic_eq *eq) 320 { 321 struct hinic_ceqs *ceqs = ceq_to_ceqs(eq); 322 u32 ceqe; 323 int i; 324 325 for (i = 0; i < eq->q_len; i++) { 326 ceqe = *(GET_CURR_CEQ_ELEM(eq)); 327 328 /* Data in HW is in Big endian Format */ 329 ceqe = be32_to_cpu(ceqe); 330 331 /* HW toggles the wrapped bit, when it adds eq element event */ 332 if (HINIC_EQ_ELEM_DESC_GET(ceqe, WRAPPED) == eq->wrapped) 333 break; 334 335 ceq_event_handler(ceqs, ceqe); 336 337 eq->cons_idx++; 338 339 if (eq->cons_idx == eq->q_len) { 340 eq->cons_idx = 0; 341 eq->wrapped = !eq->wrapped; 342 } 343 } 344 } 345 346 /** 347 * eq_irq_handler - handler for the EQ event 348 * @data: the Event Queue that received the event 349 **/ 350 static void eq_irq_handler(void *data) 351 { 352 struct hinic_eq *eq = data; 353 354 if (eq->type == HINIC_AEQ) 355 aeq_irq_handler(eq); 356 else if (eq->type == HINIC_CEQ) 357 ceq_irq_handler(eq); 358 359 eq_update_ci(eq); 360 } 361 362 /** 363 * eq_irq_work - the work of the EQ that received the event 364 * @work: the work struct that is associated with the EQ 365 **/ 366 static void eq_irq_work(struct work_struct *work) 367 { 368 struct hinic_eq_work *aeq_work = work_to_aeq_work(work); 369 struct hinic_eq *aeq; 370 371 aeq = aeq_work->data; 372 eq_irq_handler(aeq); 373 } 374 375 /** 376 * ceq_tasklet - the tasklet of the EQ that received the event 377 * @ceq_data: the eq 378 **/ 379 static void ceq_tasklet(unsigned long ceq_data) 380 { 381 struct hinic_eq *ceq = (struct hinic_eq *)ceq_data; 382 383 eq_irq_handler(ceq); 384 } 385 386 /** 387 * aeq_interrupt - aeq interrupt handler 388 * @irq: irq number 389 * @data: the Async Event Queue that collected the event 390 **/ 391 static irqreturn_t aeq_interrupt(int irq, void *data) 392 { 393 struct hinic_eq_work *aeq_work; 394 struct hinic_eq *aeq = data; 395 struct hinic_aeqs *aeqs; 396 397 /* clear resend timer cnt register */ 398 hinic_msix_attr_cnt_clear(aeq->hwif, aeq->msix_entry.entry); 399 400 aeq_work = &aeq->aeq_work; 401 aeq_work->data = aeq; 402 403 aeqs = aeq_to_aeqs(aeq); 404 queue_work(aeqs->workq, &aeq_work->work); 405 406 return IRQ_HANDLED; 407 } 408 409 /** 410 * ceq_interrupt - ceq interrupt handler 411 * @irq: irq number 412 * @data: the Completion Event Queue that collected the event 413 **/ 414 static irqreturn_t ceq_interrupt(int irq, void *data) 415 { 416 struct hinic_eq *ceq = data; 417 418 /* clear resend timer cnt register */ 419 hinic_msix_attr_cnt_clear(ceq->hwif, ceq->msix_entry.entry); 420 421 tasklet_schedule(&ceq->ceq_tasklet); 422 423 return IRQ_HANDLED; 424 } 425 426 static void set_ctrl0(struct hinic_eq *eq) 427 { 428 struct msix_entry *msix_entry = &eq->msix_entry; 429 enum hinic_eq_type type = eq->type; 430 u32 addr, val, ctrl0; 431 432 if (type == HINIC_AEQ) { 433 /* RMW Ctrl0 */ 434 addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id); 435 436 val = hinic_hwif_read_reg(eq->hwif, addr); 437 438 val = HINIC_AEQ_CTRL_0_CLEAR(val, INT_IDX) & 439 HINIC_AEQ_CTRL_0_CLEAR(val, DMA_ATTR) & 440 HINIC_AEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) & 441 HINIC_AEQ_CTRL_0_CLEAR(val, INT_MODE); 442 443 ctrl0 = HINIC_AEQ_CTRL_0_SET(msix_entry->entry, INT_IDX) | 444 HINIC_AEQ_CTRL_0_SET(DMA_ATTR_AEQ_DEFAULT, DMA_ATTR) | 445 HINIC_AEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif), 446 PCI_INTF_IDX) | 447 HINIC_AEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INT_MODE); 448 449 val |= ctrl0; 450 451 hinic_hwif_write_reg(eq->hwif, addr, val); 452 } else if (type == HINIC_CEQ) { 453 /* RMW Ctrl0 */ 454 addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id); 455 456 val = hinic_hwif_read_reg(eq->hwif, addr); 457 458 val = HINIC_CEQ_CTRL_0_CLEAR(val, INTR_IDX) & 459 HINIC_CEQ_CTRL_0_CLEAR(val, DMA_ATTR) & 460 HINIC_CEQ_CTRL_0_CLEAR(val, KICK_THRESH) & 461 HINIC_CEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) & 462 HINIC_CEQ_CTRL_0_CLEAR(val, INTR_MODE); 463 464 ctrl0 = HINIC_CEQ_CTRL_0_SET(msix_entry->entry, INTR_IDX) | 465 HINIC_CEQ_CTRL_0_SET(DMA_ATTR_CEQ_DEFAULT, DMA_ATTR) | 466 HINIC_CEQ_CTRL_0_SET(THRESH_CEQ_DEFAULT, KICK_THRESH) | 467 HINIC_CEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif), 468 PCI_INTF_IDX) | 469 HINIC_CEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INTR_MODE); 470 471 val |= ctrl0; 472 473 hinic_hwif_write_reg(eq->hwif, addr, val); 474 } 475 } 476 477 static void set_ctrl1(struct hinic_eq *eq) 478 { 479 enum hinic_eq_type type = eq->type; 480 u32 page_size_val, elem_size; 481 u32 addr, val, ctrl1; 482 483 if (type == HINIC_AEQ) { 484 /* RMW Ctrl1 */ 485 addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id); 486 487 page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq); 488 elem_size = EQ_SET_HW_ELEM_SIZE_VAL(eq); 489 490 val = hinic_hwif_read_reg(eq->hwif, addr); 491 492 val = HINIC_AEQ_CTRL_1_CLEAR(val, LEN) & 493 HINIC_AEQ_CTRL_1_CLEAR(val, ELEM_SIZE) & 494 HINIC_AEQ_CTRL_1_CLEAR(val, PAGE_SIZE); 495 496 ctrl1 = HINIC_AEQ_CTRL_1_SET(eq->q_len, LEN) | 497 HINIC_AEQ_CTRL_1_SET(elem_size, ELEM_SIZE) | 498 HINIC_AEQ_CTRL_1_SET(page_size_val, PAGE_SIZE); 499 500 val |= ctrl1; 501 502 hinic_hwif_write_reg(eq->hwif, addr, val); 503 } else if (type == HINIC_CEQ) { 504 /* RMW Ctrl1 */ 505 addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id); 506 507 page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq); 508 509 val = hinic_hwif_read_reg(eq->hwif, addr); 510 511 val = HINIC_CEQ_CTRL_1_CLEAR(val, LEN) & 512 HINIC_CEQ_CTRL_1_CLEAR(val, PAGE_SIZE); 513 514 ctrl1 = HINIC_CEQ_CTRL_1_SET(eq->q_len, LEN) | 515 HINIC_CEQ_CTRL_1_SET(page_size_val, PAGE_SIZE); 516 517 val |= ctrl1; 518 519 hinic_hwif_write_reg(eq->hwif, addr, val); 520 } 521 } 522 523 /** 524 * set_eq_ctrls - setting eq's ctrl registers 525 * @eq: the Event Queue for setting 526 **/ 527 static void set_eq_ctrls(struct hinic_eq *eq) 528 { 529 set_ctrl0(eq); 530 set_ctrl1(eq); 531 } 532 533 /** 534 * aeq_elements_init - initialize all the elements in the aeq 535 * @eq: the Async Event Queue 536 * @init_val: value to initialize the elements with it 537 **/ 538 static void aeq_elements_init(struct hinic_eq *eq, u32 init_val) 539 { 540 struct hinic_aeq_elem *aeqe; 541 int i; 542 543 for (i = 0; i < eq->q_len; i++) { 544 aeqe = GET_AEQ_ELEM(eq, i); 545 aeqe->desc = cpu_to_be32(init_val); 546 } 547 548 wmb(); /* Write the initilzation values */ 549 } 550 551 /** 552 * ceq_elements_init - Initialize all the elements in the ceq 553 * @eq: the event queue 554 * @init_val: value to init with it the elements 555 **/ 556 static void ceq_elements_init(struct hinic_eq *eq, u32 init_val) 557 { 558 u32 *ceqe; 559 int i; 560 561 for (i = 0; i < eq->q_len; i++) { 562 ceqe = GET_CEQ_ELEM(eq, i); 563 *(ceqe) = cpu_to_be32(init_val); 564 } 565 566 wmb(); /* Write the initilzation values */ 567 } 568 569 /** 570 * alloc_eq_pages - allocate the pages for the queue 571 * @eq: the event queue 572 * 573 * Return 0 - Success, Negative - Failure 574 **/ 575 static int alloc_eq_pages(struct hinic_eq *eq) 576 { 577 struct hinic_hwif *hwif = eq->hwif; 578 struct pci_dev *pdev = hwif->pdev; 579 u32 init_val, addr, val; 580 size_t addr_size; 581 int err, pg; 582 583 addr_size = eq->num_pages * sizeof(*eq->dma_addr); 584 eq->dma_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL); 585 if (!eq->dma_addr) 586 return -ENOMEM; 587 588 addr_size = eq->num_pages * sizeof(*eq->virt_addr); 589 eq->virt_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL); 590 if (!eq->virt_addr) { 591 err = -ENOMEM; 592 goto err_virt_addr_alloc; 593 } 594 595 for (pg = 0; pg < eq->num_pages; pg++) { 596 eq->virt_addr[pg] = dma_zalloc_coherent(&pdev->dev, 597 eq->page_size, 598 &eq->dma_addr[pg], 599 GFP_KERNEL); 600 if (!eq->virt_addr[pg]) { 601 err = -ENOMEM; 602 goto err_dma_alloc; 603 } 604 605 addr = EQ_HI_PHYS_ADDR_REG(eq, pg); 606 val = upper_32_bits(eq->dma_addr[pg]); 607 608 hinic_hwif_write_reg(hwif, addr, val); 609 610 addr = EQ_LO_PHYS_ADDR_REG(eq, pg); 611 val = lower_32_bits(eq->dma_addr[pg]); 612 613 hinic_hwif_write_reg(hwif, addr, val); 614 } 615 616 init_val = HINIC_EQ_ELEM_DESC_SET(eq->wrapped, WRAPPED); 617 618 if (eq->type == HINIC_AEQ) 619 aeq_elements_init(eq, init_val); 620 else if (eq->type == HINIC_CEQ) 621 ceq_elements_init(eq, init_val); 622 623 return 0; 624 625 err_dma_alloc: 626 while (--pg >= 0) 627 dma_free_coherent(&pdev->dev, eq->page_size, 628 eq->virt_addr[pg], 629 eq->dma_addr[pg]); 630 631 devm_kfree(&pdev->dev, eq->virt_addr); 632 633 err_virt_addr_alloc: 634 devm_kfree(&pdev->dev, eq->dma_addr); 635 return err; 636 } 637 638 /** 639 * free_eq_pages - free the pages of the queue 640 * @eq: the Event Queue 641 **/ 642 static void free_eq_pages(struct hinic_eq *eq) 643 { 644 struct hinic_hwif *hwif = eq->hwif; 645 struct pci_dev *pdev = hwif->pdev; 646 int pg; 647 648 for (pg = 0; pg < eq->num_pages; pg++) 649 dma_free_coherent(&pdev->dev, eq->page_size, 650 eq->virt_addr[pg], 651 eq->dma_addr[pg]); 652 653 devm_kfree(&pdev->dev, eq->virt_addr); 654 devm_kfree(&pdev->dev, eq->dma_addr); 655 } 656 657 /** 658 * init_eq - initialize Event Queue 659 * @eq: the event queue 660 * @hwif: the HW interface of a PCI function device 661 * @type: the type of the event queue, aeq or ceq 662 * @q_id: Queue id number 663 * @q_len: the number of EQ elements 664 * @page_size: the page size of the pages in the event queue 665 * @entry: msix entry associated with the event queue 666 * 667 * Return 0 - Success, Negative - Failure 668 **/ 669 static int init_eq(struct hinic_eq *eq, struct hinic_hwif *hwif, 670 enum hinic_eq_type type, int q_id, u32 q_len, u32 page_size, 671 struct msix_entry entry) 672 { 673 struct pci_dev *pdev = hwif->pdev; 674 int err; 675 676 eq->hwif = hwif; 677 eq->type = type; 678 eq->q_id = q_id; 679 eq->q_len = q_len; 680 eq->page_size = page_size; 681 682 /* Clear PI and CI, also clear the ARM bit */ 683 hinic_hwif_write_reg(eq->hwif, EQ_CONS_IDX_REG_ADDR(eq), 0); 684 hinic_hwif_write_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq), 0); 685 686 eq->cons_idx = 0; 687 eq->wrapped = 0; 688 689 if (type == HINIC_AEQ) { 690 eq->elem_size = HINIC_AEQE_SIZE; 691 } else if (type == HINIC_CEQ) { 692 eq->elem_size = HINIC_CEQE_SIZE; 693 } else { 694 dev_err(&pdev->dev, "Invalid EQ type\n"); 695 return -EINVAL; 696 } 697 698 eq->num_pages = GET_EQ_NUM_PAGES(eq, page_size); 699 eq->num_elem_in_pg = GET_EQ_NUM_ELEMS_IN_PG(eq, page_size); 700 701 eq->msix_entry = entry; 702 703 if (eq->num_elem_in_pg & (eq->num_elem_in_pg - 1)) { 704 dev_err(&pdev->dev, "num elements in eq page != power of 2\n"); 705 return -EINVAL; 706 } 707 708 if (eq->num_pages > EQ_MAX_PAGES) { 709 dev_err(&pdev->dev, "too many pages for eq\n"); 710 return -EINVAL; 711 } 712 713 set_eq_ctrls(eq); 714 eq_update_ci(eq); 715 716 err = alloc_eq_pages(eq); 717 if (err) { 718 dev_err(&pdev->dev, "Failed to allocate pages for eq\n"); 719 return err; 720 } 721 722 if (type == HINIC_AEQ) { 723 struct hinic_eq_work *aeq_work = &eq->aeq_work; 724 725 INIT_WORK(&aeq_work->work, eq_irq_work); 726 } else if (type == HINIC_CEQ) { 727 tasklet_init(&eq->ceq_tasklet, ceq_tasklet, 728 (unsigned long)eq); 729 } 730 731 /* set the attributes of the msix entry */ 732 hinic_msix_attr_set(eq->hwif, eq->msix_entry.entry, 733 HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT, 734 HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT, 735 HINIC_EQ_MSIX_LLI_TIMER_DEFAULT, 736 HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT, 737 HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT); 738 739 if (type == HINIC_AEQ) 740 err = request_irq(entry.vector, aeq_interrupt, 0, 741 "hinic_aeq", eq); 742 else if (type == HINIC_CEQ) 743 err = request_irq(entry.vector, ceq_interrupt, 0, 744 "hinic_ceq", eq); 745 746 if (err) { 747 dev_err(&pdev->dev, "Failed to request irq for the EQ\n"); 748 goto err_req_irq; 749 } 750 751 return 0; 752 753 err_req_irq: 754 free_eq_pages(eq); 755 return err; 756 } 757 758 /** 759 * remove_eq - remove Event Queue 760 * @eq: the event queue 761 **/ 762 static void remove_eq(struct hinic_eq *eq) 763 { 764 struct msix_entry *entry = &eq->msix_entry; 765 766 free_irq(entry->vector, eq); 767 768 if (eq->type == HINIC_AEQ) { 769 struct hinic_eq_work *aeq_work = &eq->aeq_work; 770 771 cancel_work_sync(&aeq_work->work); 772 } else if (eq->type == HINIC_CEQ) { 773 tasklet_kill(&eq->ceq_tasklet); 774 } 775 776 free_eq_pages(eq); 777 } 778 779 /** 780 * hinic_aeqs_init - initialize all the aeqs 781 * @aeqs: pointer to Async eqs of the chip 782 * @hwif: the HW interface of a PCI function device 783 * @num_aeqs: number of AEQs 784 * @q_len: number of EQ elements 785 * @page_size: the page size of the pages in the event queue 786 * @msix_entries: msix entries associated with the event queues 787 * 788 * Return 0 - Success, negative - Failure 789 **/ 790 int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif, 791 int num_aeqs, u32 q_len, u32 page_size, 792 struct msix_entry *msix_entries) 793 { 794 struct pci_dev *pdev = hwif->pdev; 795 int err, i, q_id; 796 797 aeqs->workq = create_singlethread_workqueue(HINIC_EQS_WQ_NAME); 798 if (!aeqs->workq) 799 return -ENOMEM; 800 801 aeqs->hwif = hwif; 802 aeqs->num_aeqs = num_aeqs; 803 804 for (q_id = 0; q_id < num_aeqs; q_id++) { 805 err = init_eq(&aeqs->aeq[q_id], hwif, HINIC_AEQ, q_id, q_len, 806 page_size, msix_entries[q_id]); 807 if (err) { 808 dev_err(&pdev->dev, "Failed to init aeq %d\n", q_id); 809 goto err_init_aeq; 810 } 811 } 812 813 return 0; 814 815 err_init_aeq: 816 for (i = 0; i < q_id; i++) 817 remove_eq(&aeqs->aeq[i]); 818 819 destroy_workqueue(aeqs->workq); 820 return err; 821 } 822 823 /** 824 * hinic_aeqs_free - free all the aeqs 825 * @aeqs: pointer to Async eqs of the chip 826 **/ 827 void hinic_aeqs_free(struct hinic_aeqs *aeqs) 828 { 829 int q_id; 830 831 for (q_id = 0; q_id < aeqs->num_aeqs ; q_id++) 832 remove_eq(&aeqs->aeq[q_id]); 833 834 destroy_workqueue(aeqs->workq); 835 } 836 837 /** 838 * hinic_ceqs_init - init all the ceqs 839 * @ceqs: ceqs part of the chip 840 * @hwif: the hardware interface of a pci function device 841 * @num_ceqs: number of CEQs 842 * @q_len: number of EQ elements 843 * @page_size: the page size of the event queue 844 * @msix_entries: msix entries associated with the event queues 845 * 846 * Return 0 - Success, Negative - Failure 847 **/ 848 int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif, 849 int num_ceqs, u32 q_len, u32 page_size, 850 struct msix_entry *msix_entries) 851 { 852 struct pci_dev *pdev = hwif->pdev; 853 int i, q_id, err; 854 855 ceqs->hwif = hwif; 856 ceqs->num_ceqs = num_ceqs; 857 858 for (q_id = 0; q_id < num_ceqs; q_id++) { 859 err = init_eq(&ceqs->ceq[q_id], hwif, HINIC_CEQ, q_id, q_len, 860 page_size, msix_entries[q_id]); 861 if (err) { 862 dev_err(&pdev->dev, "Failed to init ceq %d\n", q_id); 863 goto err_init_ceq; 864 } 865 } 866 867 return 0; 868 869 err_init_ceq: 870 for (i = 0; i < q_id; i++) 871 remove_eq(&ceqs->ceq[i]); 872 873 return err; 874 } 875 876 /** 877 * hinic_ceqs_free - free all the ceqs 878 * @ceqs: ceqs part of the chip 879 **/ 880 void hinic_ceqs_free(struct hinic_ceqs *ceqs) 881 { 882 int q_id; 883 884 for (q_id = 0; q_id < ceqs->num_ceqs; q_id++) 885 remove_eq(&ceqs->ceq[q_id]); 886 } 887