1 /* 2 * Copyright (C) 2016 Cavium, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License 6 * as published by the Free Software Foundation. 7 */ 8 9 #include <linux/interrupt.h> 10 #include <linux/module.h> 11 12 #include "cptvf.h" 13 14 #define DRV_NAME "thunder-cptvf" 15 #define DRV_VERSION "1.0" 16 17 struct cptvf_wqe { 18 struct tasklet_struct twork; 19 void *cptvf; 20 u32 qno; 21 }; 22 23 struct cptvf_wqe_info { 24 struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF]; 25 }; 26 27 static void vq_work_handler(unsigned long data) 28 { 29 struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data; 30 struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0]; 31 32 vq_post_process(cwqe->cptvf, cwqe->qno); 33 } 34 35 static int init_worker_threads(struct cpt_vf *cptvf) 36 { 37 struct pci_dev *pdev = cptvf->pdev; 38 struct cptvf_wqe_info *cwqe_info; 39 int i; 40 41 cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL); 42 if (!cwqe_info) 43 return -ENOMEM; 44 45 if (cptvf->nr_queues) { 46 dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n", 47 cptvf->nr_queues); 48 } 49 50 for (i = 0; i < cptvf->nr_queues; i++) { 51 tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler, 52 (u64)cwqe_info); 53 cwqe_info->vq_wqe[i].qno = i; 54 cwqe_info->vq_wqe[i].cptvf = cptvf; 55 } 56 57 cptvf->wqe_info = cwqe_info; 58 59 return 0; 60 } 61 62 static void cleanup_worker_threads(struct cpt_vf *cptvf) 63 { 64 struct cptvf_wqe_info *cwqe_info; 65 struct pci_dev *pdev = cptvf->pdev; 66 int i; 67 68 cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; 69 if (!cwqe_info) 70 return; 71 72 if (cptvf->nr_queues) { 73 dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n", 74 cptvf->nr_queues); 75 } 76 77 for (i = 0; i < cptvf->nr_queues; i++) 78 tasklet_kill(&cwqe_info->vq_wqe[i].twork); 79 80 kzfree(cwqe_info); 81 cptvf->wqe_info = NULL; 82 } 83 84 static void free_pending_queues(struct pending_qinfo *pqinfo) 85 { 86 int i; 87 struct pending_queue *queue; 88 89 for_each_pending_queue(pqinfo, queue, i) { 90 if (!queue->head) 91 continue; 92 93 /* free single queue */ 94 kzfree((queue->head)); 95 96 queue->front = 0; 97 queue->rear = 0; 98 99 return; 100 } 101 102 pqinfo->qlen = 0; 103 pqinfo->nr_queues = 0; 104 } 105 106 static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, 107 u32 nr_queues) 108 { 109 u32 i; 110 size_t size; 111 int ret; 112 struct pending_queue *queue = NULL; 113 114 pqinfo->nr_queues = nr_queues; 115 pqinfo->qlen = qlen; 116 117 size = (qlen * sizeof(struct pending_entry)); 118 119 for_each_pending_queue(pqinfo, queue, i) { 120 queue->head = kzalloc((size), GFP_KERNEL); 121 if (!queue->head) { 122 ret = -ENOMEM; 123 goto pending_qfail; 124 } 125 126 queue->front = 0; 127 queue->rear = 0; 128 atomic64_set((&queue->pending_count), (0)); 129 130 /* init queue spin lock */ 131 spin_lock_init(&queue->lock); 132 } 133 134 return 0; 135 136 pending_qfail: 137 free_pending_queues(pqinfo); 138 139 return ret; 140 } 141 142 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) 143 { 144 struct pci_dev *pdev = cptvf->pdev; 145 int ret; 146 147 if (!nr_queues) 148 return 0; 149 150 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); 151 if (ret) { 152 dev_err(&pdev->dev, "failed to setup pending queues (%u)\n", 153 nr_queues); 154 return ret; 155 } 156 157 return 0; 158 } 159 160 static void cleanup_pending_queues(struct cpt_vf *cptvf) 161 { 162 struct pci_dev *pdev = cptvf->pdev; 163 164 if (!cptvf->nr_queues) 165 return; 166 167 dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n", 168 cptvf->nr_queues); 169 free_pending_queues(&cptvf->pqinfo); 170 } 171 172 static void free_command_queues(struct cpt_vf *cptvf, 173 struct command_qinfo *cqinfo) 174 { 175 int i; 176 struct command_queue *queue = NULL; 177 struct command_chunk *chunk = NULL; 178 struct pci_dev *pdev = cptvf->pdev; 179 struct hlist_node *node; 180 181 /* clean up for each queue */ 182 for (i = 0; i < cptvf->nr_queues; i++) { 183 queue = &cqinfo->queue[i]; 184 if (hlist_empty(&cqinfo->queue[i].chead)) 185 continue; 186 187 hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, 188 nextchunk) { 189 dma_free_coherent(&pdev->dev, chunk->size, 190 chunk->head, 191 chunk->dma_addr); 192 chunk->head = NULL; 193 chunk->dma_addr = 0; 194 hlist_del(&chunk->nextchunk); 195 kzfree(chunk); 196 } 197 198 queue->nchunks = 0; 199 queue->idx = 0; 200 } 201 202 /* common cleanup */ 203 cqinfo->cmd_size = 0; 204 } 205 206 static int alloc_command_queues(struct cpt_vf *cptvf, 207 struct command_qinfo *cqinfo, size_t cmd_size, 208 u32 qlen) 209 { 210 int i; 211 size_t q_size; 212 struct command_queue *queue = NULL; 213 struct pci_dev *pdev = cptvf->pdev; 214 215 /* common init */ 216 cqinfo->cmd_size = cmd_size; 217 /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ 218 cptvf->qsize = min(qlen, cqinfo->qchunksize) * 219 CPT_NEXT_CHUNK_PTR_SIZE + 1; 220 /* Qsize in bytes to create space for alignment */ 221 q_size = qlen * cqinfo->cmd_size; 222 223 /* per queue initialization */ 224 for (i = 0; i < cptvf->nr_queues; i++) { 225 size_t c_size = 0; 226 size_t rem_q_size = q_size; 227 struct command_chunk *curr = NULL, *first = NULL, *last = NULL; 228 u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size; 229 230 queue = &cqinfo->queue[i]; 231 INIT_HLIST_HEAD(&cqinfo->queue[i].chead); 232 do { 233 curr = kzalloc(sizeof(*curr), GFP_KERNEL); 234 if (!curr) 235 goto cmd_qfail; 236 237 c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : 238 rem_q_size; 239 curr->head = (u8 *)dma_alloc_coherent(&pdev->dev, 240 c_size + CPT_NEXT_CHUNK_PTR_SIZE, 241 &curr->dma_addr, 242 GFP_KERNEL); 243 if (!curr->head) { 244 dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", 245 i, queue->nchunks); 246 kfree(curr); 247 goto cmd_qfail; 248 } 249 250 curr->size = c_size; 251 if (queue->nchunks == 0) { 252 hlist_add_head(&curr->nextchunk, 253 &cqinfo->queue[i].chead); 254 first = curr; 255 } else { 256 hlist_add_behind(&curr->nextchunk, 257 &last->nextchunk); 258 } 259 260 queue->nchunks++; 261 rem_q_size -= c_size; 262 if (last) 263 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; 264 265 last = curr; 266 } while (rem_q_size); 267 268 /* Make the queue circular */ 269 /* Tie back last chunk entry to head */ 270 curr = first; 271 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; 272 queue->qhead = curr; 273 spin_lock_init(&queue->lock); 274 } 275 return 0; 276 277 cmd_qfail: 278 free_command_queues(cptvf, cqinfo); 279 return -ENOMEM; 280 } 281 282 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) 283 { 284 struct pci_dev *pdev = cptvf->pdev; 285 int ret; 286 287 /* setup AE command queues */ 288 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, 289 qlen); 290 if (ret) { 291 dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n", 292 cptvf->nr_queues); 293 return ret; 294 } 295 296 return ret; 297 } 298 299 static void cleanup_command_queues(struct cpt_vf *cptvf) 300 { 301 struct pci_dev *pdev = cptvf->pdev; 302 303 if (!cptvf->nr_queues) 304 return; 305 306 dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n", 307 cptvf->nr_queues); 308 free_command_queues(cptvf, &cptvf->cqinfo); 309 } 310 311 static void cptvf_sw_cleanup(struct cpt_vf *cptvf) 312 { 313 cleanup_worker_threads(cptvf); 314 cleanup_pending_queues(cptvf); 315 cleanup_command_queues(cptvf); 316 } 317 318 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) 319 { 320 struct pci_dev *pdev = cptvf->pdev; 321 int ret = 0; 322 u32 max_dev_queues = 0; 323 324 max_dev_queues = CPT_NUM_QS_PER_VF; 325 /* possible cpus */ 326 nr_queues = min_t(u32, nr_queues, max_dev_queues); 327 cptvf->nr_queues = nr_queues; 328 329 ret = init_command_queues(cptvf, qlen); 330 if (ret) { 331 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n", 332 nr_queues); 333 return ret; 334 } 335 336 ret = init_pending_queues(cptvf, qlen, nr_queues); 337 if (ret) { 338 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", 339 nr_queues); 340 goto setup_pqfail; 341 } 342 343 /* Create worker threads for BH processing */ 344 ret = init_worker_threads(cptvf); 345 if (ret) { 346 dev_err(&pdev->dev, "Failed to setup worker threads\n"); 347 goto init_work_fail; 348 } 349 350 return 0; 351 352 init_work_fail: 353 cleanup_worker_threads(cptvf); 354 cleanup_pending_queues(cptvf); 355 356 setup_pqfail: 357 cleanup_command_queues(cptvf); 358 359 return ret; 360 } 361 362 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) 363 { 364 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); 365 free_cpumask_var(cptvf->affinity_mask[vec]); 366 } 367 368 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) 369 { 370 union cptx_vqx_ctl vqx_ctl; 371 372 vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); 373 vqx_ctl.s.ena = val; 374 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); 375 } 376 377 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) 378 { 379 union cptx_vqx_doorbell vqx_dbell; 380 381 vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, 382 CPTX_VQX_DOORBELL(0, 0)); 383 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ 384 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), 385 vqx_dbell.u); 386 } 387 388 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) 389 { 390 union cptx_vqx_inprog vqx_inprg; 391 392 vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); 393 vqx_inprg.s.inflight = val; 394 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); 395 } 396 397 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) 398 { 399 union cptx_vqx_done_wait vqx_dwait; 400 401 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, 402 CPTX_VQX_DONE_WAIT(0, 0)); 403 vqx_dwait.s.num_wait = val; 404 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), 405 vqx_dwait.u); 406 } 407 408 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) 409 { 410 union cptx_vqx_done_wait vqx_dwait; 411 412 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, 413 CPTX_VQX_DONE_WAIT(0, 0)); 414 vqx_dwait.s.time_wait = time; 415 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), 416 vqx_dwait.u); 417 } 418 419 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) 420 { 421 union cptx_vqx_misc_ena_w1s vqx_misc_ena; 422 423 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, 424 CPTX_VQX_MISC_ENA_W1S(0, 0)); 425 /* Set mbox(0) interupts for the requested vf */ 426 vqx_misc_ena.s.swerr = 1; 427 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), 428 vqx_misc_ena.u); 429 } 430 431 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) 432 { 433 union cptx_vqx_misc_ena_w1s vqx_misc_ena; 434 435 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, 436 CPTX_VQX_MISC_ENA_W1S(0, 0)); 437 /* Set mbox(0) interupts for the requested vf */ 438 vqx_misc_ena.s.mbox = 1; 439 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), 440 vqx_misc_ena.u); 441 } 442 443 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) 444 { 445 union cptx_vqx_done_ena_w1s vqx_done_ena; 446 447 vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, 448 CPTX_VQX_DONE_ENA_W1S(0, 0)); 449 /* Set DONE interrupt for the requested vf */ 450 vqx_done_ena.s.done = 1; 451 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), 452 vqx_done_ena.u); 453 } 454 455 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) 456 { 457 union cptx_vqx_misc_int vqx_misc_int; 458 459 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 460 CPTX_VQX_MISC_INT(0, 0)); 461 /* W1C for the VF */ 462 vqx_misc_int.s.dovf = 1; 463 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 464 vqx_misc_int.u); 465 } 466 467 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) 468 { 469 union cptx_vqx_misc_int vqx_misc_int; 470 471 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 472 CPTX_VQX_MISC_INT(0, 0)); 473 /* W1C for the VF */ 474 vqx_misc_int.s.irde = 1; 475 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 476 vqx_misc_int.u); 477 } 478 479 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) 480 { 481 union cptx_vqx_misc_int vqx_misc_int; 482 483 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 484 CPTX_VQX_MISC_INT(0, 0)); 485 /* W1C for the VF */ 486 vqx_misc_int.s.nwrp = 1; 487 cpt_write_csr64(cptvf->reg_base, 488 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 489 } 490 491 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) 492 { 493 union cptx_vqx_misc_int vqx_misc_int; 494 495 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 496 CPTX_VQX_MISC_INT(0, 0)); 497 /* W1C for the VF */ 498 vqx_misc_int.s.mbox = 1; 499 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 500 vqx_misc_int.u); 501 } 502 503 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) 504 { 505 union cptx_vqx_misc_int vqx_misc_int; 506 507 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 508 CPTX_VQX_MISC_INT(0, 0)); 509 /* W1C for the VF */ 510 vqx_misc_int.s.swerr = 1; 511 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 512 vqx_misc_int.u); 513 } 514 515 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) 516 { 517 return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); 518 } 519 520 static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) 521 { 522 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; 523 struct pci_dev *pdev = cptvf->pdev; 524 u64 intr; 525 526 intr = cptvf_read_vf_misc_intr_status(cptvf); 527 /*Check for MISC interrupt types*/ 528 if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { 529 dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", 530 intr, cptvf->vfid); 531 cptvf_handle_mbox_intr(cptvf); 532 cptvf_clear_mbox_intr(cptvf); 533 } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { 534 cptvf_clear_dovf_intr(cptvf); 535 /*Clear doorbell count*/ 536 cptvf_write_vq_doorbell(cptvf, 0); 537 dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n", 538 intr, cptvf->vfid); 539 } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { 540 cptvf_clear_irde_intr(cptvf); 541 dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n", 542 intr, cptvf->vfid); 543 } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { 544 cptvf_clear_nwrp_intr(cptvf); 545 dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n", 546 intr, cptvf->vfid); 547 } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) { 548 cptvf_clear_swerr_intr(cptvf); 549 dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n", 550 intr, cptvf->vfid); 551 } else { 552 dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n", 553 cptvf->vfid); 554 } 555 556 return IRQ_HANDLED; 557 } 558 559 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, 560 int qno) 561 { 562 struct cptvf_wqe_info *nwqe_info; 563 564 if (unlikely(qno >= cptvf->nr_queues)) 565 return NULL; 566 nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; 567 568 return &nwqe_info->vq_wqe[qno]; 569 } 570 571 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) 572 { 573 union cptx_vqx_done vqx_done; 574 575 vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); 576 return vqx_done.s.done; 577 } 578 579 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, 580 u32 ackcnt) 581 { 582 union cptx_vqx_done_ack vqx_dack_cnt; 583 584 vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, 585 CPTX_VQX_DONE_ACK(0, 0)); 586 vqx_dack_cnt.s.done_ack = ackcnt; 587 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), 588 vqx_dack_cnt.u); 589 } 590 591 static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq) 592 { 593 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; 594 struct pci_dev *pdev = cptvf->pdev; 595 /* Read the number of completions */ 596 u32 intr = cptvf_read_vq_done_count(cptvf); 597 598 if (intr) { 599 struct cptvf_wqe *wqe; 600 601 /* Acknowledge the number of 602 * scheduled completions for processing 603 */ 604 cptvf_write_vq_done_ack(cptvf, intr); 605 wqe = get_cptvf_vq_wqe(cptvf, 0); 606 if (unlikely(!wqe)) { 607 dev_err(&pdev->dev, "No work to schedule for VF (%d)", 608 cptvf->vfid); 609 return IRQ_NONE; 610 } 611 tasklet_hi_schedule(&wqe->twork); 612 } 613 614 return IRQ_HANDLED; 615 } 616 617 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) 618 { 619 struct pci_dev *pdev = cptvf->pdev; 620 int cpu; 621 622 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], 623 GFP_KERNEL)) { 624 dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d", 625 cptvf->vfid); 626 return; 627 } 628 629 cpu = cptvf->vfid % num_online_cpus(); 630 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), 631 cptvf->affinity_mask[vec]); 632 irq_set_affinity_hint(pci_irq_vector(pdev, vec), 633 cptvf->affinity_mask[vec]); 634 } 635 636 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) 637 { 638 union cptx_vqx_saddr vqx_saddr; 639 640 vqx_saddr.u = val; 641 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); 642 } 643 644 static void cptvf_device_init(struct cpt_vf *cptvf) 645 { 646 u64 base_addr = 0; 647 648 /* Disable the VQ */ 649 cptvf_write_vq_ctl(cptvf, 0); 650 /* Reset the doorbell */ 651 cptvf_write_vq_doorbell(cptvf, 0); 652 /* Clear inflight */ 653 cptvf_write_vq_inprog(cptvf, 0); 654 /* Write VQ SADDR */ 655 /* TODO: for now only one queue, so hard coded */ 656 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); 657 cptvf_write_vq_saddr(cptvf, base_addr); 658 /* Configure timerhold / coalescence */ 659 cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); 660 cptvf_write_vq_done_numwait(cptvf, 1); 661 /* Enable the VQ */ 662 cptvf_write_vq_ctl(cptvf, 1); 663 /* Flag the VF ready */ 664 cptvf->flags |= CPT_FLAG_DEVICE_READY; 665 } 666 667 static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 668 { 669 struct device *dev = &pdev->dev; 670 struct cpt_vf *cptvf; 671 int err; 672 673 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); 674 if (!cptvf) 675 return -ENOMEM; 676 677 pci_set_drvdata(pdev, cptvf); 678 cptvf->pdev = pdev; 679 err = pci_enable_device(pdev); 680 if (err) { 681 dev_err(dev, "Failed to enable PCI device\n"); 682 pci_set_drvdata(pdev, NULL); 683 return err; 684 } 685 686 err = pci_request_regions(pdev, DRV_NAME); 687 if (err) { 688 dev_err(dev, "PCI request regions failed 0x%x\n", err); 689 goto cptvf_err_disable_device; 690 } 691 /* Mark as VF driver */ 692 cptvf->flags |= CPT_FLAG_VF_DRIVER; 693 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); 694 if (err) { 695 dev_err(dev, "Unable to get usable DMA configuration\n"); 696 goto cptvf_err_release_regions; 697 } 698 699 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); 700 if (err) { 701 dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); 702 goto cptvf_err_release_regions; 703 } 704 705 /* MAP PF's configuration registers */ 706 cptvf->reg_base = pcim_iomap(pdev, 0, 0); 707 if (!cptvf->reg_base) { 708 dev_err(dev, "Cannot map config register space, aborting\n"); 709 err = -ENOMEM; 710 goto cptvf_err_release_regions; 711 } 712 713 cptvf->node = dev_to_node(&pdev->dev); 714 err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS, 715 CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); 716 if (err < 0) { 717 dev_err(dev, "Request for #%d msix vectors failed\n", 718 CPT_VF_MSIX_VECTORS); 719 goto cptvf_err_release_regions; 720 } 721 722 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), 723 cptvf_misc_intr_handler, 0, "CPT VF misc intr", 724 cptvf); 725 if (err) { 726 dev_err(dev, "Request misc irq failed"); 727 goto cptvf_free_vectors; 728 } 729 730 /* Enable mailbox interrupt */ 731 cptvf_enable_mbox_interrupts(cptvf); 732 cptvf_enable_swerr_interrupts(cptvf); 733 734 /* Check ready with PF */ 735 /* Gets chip ID / device Id from PF if ready */ 736 err = cptvf_check_pf_ready(cptvf); 737 if (err) { 738 dev_err(dev, "PF not responding to READY msg"); 739 goto cptvf_free_misc_irq; 740 } 741 742 /* CPT VF software resources initialization */ 743 cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; 744 err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); 745 if (err) { 746 dev_err(dev, "cptvf_sw_init() failed"); 747 goto cptvf_free_misc_irq; 748 } 749 /* Convey VQ LEN to PF */ 750 err = cptvf_send_vq_size_msg(cptvf); 751 if (err) { 752 dev_err(dev, "PF not responding to QLEN msg"); 753 goto cptvf_free_misc_irq; 754 } 755 756 /* CPT VF device initialization */ 757 cptvf_device_init(cptvf); 758 /* Send msg to PF to assign currnet Q to required group */ 759 cptvf->vfgrp = 1; 760 err = cptvf_send_vf_to_grp_msg(cptvf); 761 if (err) { 762 dev_err(dev, "PF not responding to VF_GRP msg"); 763 goto cptvf_free_misc_irq; 764 } 765 766 cptvf->priority = 1; 767 err = cptvf_send_vf_priority_msg(cptvf); 768 if (err) { 769 dev_err(dev, "PF not responding to VF_PRIO msg"); 770 goto cptvf_free_misc_irq; 771 } 772 773 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), 774 cptvf_done_intr_handler, 0, "CPT VF done intr", 775 cptvf); 776 if (err) { 777 dev_err(dev, "Request done irq failed\n"); 778 goto cptvf_free_misc_irq; 779 } 780 781 /* Enable mailbox interrupt */ 782 cptvf_enable_done_interrupts(cptvf); 783 784 /* Set irq affinity masks */ 785 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 786 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 787 788 err = cptvf_send_vf_up(cptvf); 789 if (err) { 790 dev_err(dev, "PF not responding to UP msg"); 791 goto cptvf_free_irq_affinity; 792 } 793 err = cvm_crypto_init(cptvf); 794 if (err) { 795 dev_err(dev, "Algorithm register failed\n"); 796 goto cptvf_free_irq_affinity; 797 } 798 return 0; 799 800 cptvf_free_irq_affinity: 801 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 802 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 803 cptvf_free_misc_irq: 804 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); 805 cptvf_free_vectors: 806 pci_free_irq_vectors(cptvf->pdev); 807 cptvf_err_release_regions: 808 pci_release_regions(pdev); 809 cptvf_err_disable_device: 810 pci_disable_device(pdev); 811 pci_set_drvdata(pdev, NULL); 812 813 return err; 814 } 815 816 static void cptvf_remove(struct pci_dev *pdev) 817 { 818 struct cpt_vf *cptvf = pci_get_drvdata(pdev); 819 820 if (!cptvf) { 821 dev_err(&pdev->dev, "Invalid CPT-VF device\n"); 822 return; 823 } 824 825 /* Convey DOWN to PF */ 826 if (cptvf_send_vf_down(cptvf)) { 827 dev_err(&pdev->dev, "PF not responding to DOWN msg"); 828 } else { 829 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 830 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 831 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); 832 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); 833 pci_free_irq_vectors(cptvf->pdev); 834 cptvf_sw_cleanup(cptvf); 835 pci_set_drvdata(pdev, NULL); 836 pci_release_regions(pdev); 837 pci_disable_device(pdev); 838 cvm_crypto_exit(); 839 } 840 } 841 842 static void cptvf_shutdown(struct pci_dev *pdev) 843 { 844 cptvf_remove(pdev); 845 } 846 847 /* Supported devices */ 848 static const struct pci_device_id cptvf_id_table[] = { 849 {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0}, 850 { 0, } /* end of table */ 851 }; 852 853 static struct pci_driver cptvf_pci_driver = { 854 .name = DRV_NAME, 855 .id_table = cptvf_id_table, 856 .probe = cptvf_probe, 857 .remove = cptvf_remove, 858 .shutdown = cptvf_shutdown, 859 }; 860 861 module_pci_driver(cptvf_pci_driver); 862 863 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>"); 864 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver"); 865 MODULE_LICENSE("GPL v2"); 866 MODULE_VERSION(DRV_VERSION); 867 MODULE_DEVICE_TABLE(pci, cptvf_id_table); 868