1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <uapi/linux/idxd.h> 10 #include "../dmaengine.h" 11 #include "idxd.h" 12 #include "registers.h" 13 14 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout); 15 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand); 16 17 /* Interrupt control bits */ 18 int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) 19 { 20 struct pci_dev *pdev = idxd->pdev; 21 int msixcnt = pci_msix_vec_count(pdev); 22 union msix_perm perm; 23 u32 offset; 24 25 if (vec_id < 0 || vec_id >= msixcnt) 26 return -EINVAL; 27 28 offset = idxd->msix_perm_offset + vec_id * 8; 29 perm.bits = ioread32(idxd->reg_base + offset); 30 perm.ignore = 1; 31 iowrite32(perm.bits, idxd->reg_base + offset); 32 33 return 0; 34 } 35 36 void idxd_mask_msix_vectors(struct idxd_device *idxd) 37 { 38 struct pci_dev *pdev = idxd->pdev; 39 int msixcnt = pci_msix_vec_count(pdev); 40 int i, rc; 41 42 for (i = 0; i < msixcnt; i++) { 43 rc = idxd_mask_msix_vector(idxd, i); 44 if (rc < 0) 45 dev_warn(&pdev->dev, 46 "Failed disabling msix vec %d\n", i); 47 } 48 } 49 50 int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) 51 { 52 struct pci_dev *pdev = idxd->pdev; 53 int msixcnt = pci_msix_vec_count(pdev); 54 union msix_perm perm; 55 u32 offset; 56 57 if (vec_id < 0 || vec_id >= msixcnt) 58 return -EINVAL; 59 60 offset = idxd->msix_perm_offset + vec_id * 8; 61 perm.bits = ioread32(idxd->reg_base + offset); 62 perm.ignore = 0; 63 iowrite32(perm.bits, idxd->reg_base + offset); 64 65 /* 66 * A readback from the device ensures that any previously generated 67 * completion record writes are visible to software based on PCI 68 * ordering rules. 69 */ 70 perm.bits = ioread32(idxd->reg_base + offset); 71 72 return 0; 73 } 74 75 void idxd_unmask_error_interrupts(struct idxd_device *idxd) 76 { 77 union genctrl_reg genctrl; 78 79 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); 80 genctrl.softerr_int_en = 1; 81 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); 82 } 83 84 void idxd_mask_error_interrupts(struct idxd_device *idxd) 85 { 86 union genctrl_reg genctrl; 87 88 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); 89 genctrl.softerr_int_en = 0; 90 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); 91 } 92 93 static void free_hw_descs(struct idxd_wq *wq) 94 { 95 int i; 96 97 for (i = 0; i < wq->num_descs; i++) 98 kfree(wq->hw_descs[i]); 99 100 kfree(wq->hw_descs); 101 } 102 103 static int alloc_hw_descs(struct idxd_wq *wq, int num) 104 { 105 struct device *dev = &wq->idxd->pdev->dev; 106 int i; 107 int node = dev_to_node(dev); 108 109 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *), 110 GFP_KERNEL, node); 111 if (!wq->hw_descs) 112 return -ENOMEM; 113 114 for (i = 0; i < num; i++) { 115 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]), 116 GFP_KERNEL, node); 117 if (!wq->hw_descs[i]) { 118 free_hw_descs(wq); 119 return -ENOMEM; 120 } 121 } 122 123 return 0; 124 } 125 126 static void free_descs(struct idxd_wq *wq) 127 { 128 int i; 129 130 for (i = 0; i < wq->num_descs; i++) 131 kfree(wq->descs[i]); 132 133 kfree(wq->descs); 134 } 135 136 static int alloc_descs(struct idxd_wq *wq, int num) 137 { 138 struct device *dev = &wq->idxd->pdev->dev; 139 int i; 140 int node = dev_to_node(dev); 141 142 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *), 143 GFP_KERNEL, node); 144 if (!wq->descs) 145 return -ENOMEM; 146 147 for (i = 0; i < num; i++) { 148 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]), 149 GFP_KERNEL, node); 150 if (!wq->descs[i]) { 151 free_descs(wq); 152 return -ENOMEM; 153 } 154 } 155 156 return 0; 157 } 158 159 /* WQ control bits */ 160 int idxd_wq_alloc_resources(struct idxd_wq *wq) 161 { 162 struct idxd_device *idxd = wq->idxd; 163 struct idxd_group *group = wq->group; 164 struct device *dev = &idxd->pdev->dev; 165 int rc, num_descs, i; 166 167 if (wq->type != IDXD_WQT_KERNEL) 168 return 0; 169 170 num_descs = wq->size + 171 idxd->hw.gen_cap.max_descs_per_engine * group->num_engines; 172 wq->num_descs = num_descs; 173 174 rc = alloc_hw_descs(wq, num_descs); 175 if (rc < 0) 176 return rc; 177 178 wq->compls_size = num_descs * sizeof(struct dsa_completion_record); 179 wq->compls = dma_alloc_coherent(dev, wq->compls_size, 180 &wq->compls_addr, GFP_KERNEL); 181 if (!wq->compls) { 182 rc = -ENOMEM; 183 goto fail_alloc_compls; 184 } 185 186 rc = alloc_descs(wq, num_descs); 187 if (rc < 0) 188 goto fail_alloc_descs; 189 190 rc = sbitmap_init_node(&wq->sbmap, num_descs, -1, GFP_KERNEL, 191 dev_to_node(dev)); 192 if (rc < 0) 193 goto fail_sbitmap_init; 194 195 for (i = 0; i < num_descs; i++) { 196 struct idxd_desc *desc = wq->descs[i]; 197 198 desc->hw = wq->hw_descs[i]; 199 desc->completion = &wq->compls[i]; 200 desc->compl_dma = wq->compls_addr + 201 sizeof(struct dsa_completion_record) * i; 202 desc->id = i; 203 desc->wq = wq; 204 205 dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan); 206 desc->txd.tx_submit = idxd_dma_tx_submit; 207 } 208 209 return 0; 210 211 fail_sbitmap_init: 212 free_descs(wq); 213 fail_alloc_descs: 214 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); 215 fail_alloc_compls: 216 free_hw_descs(wq); 217 return rc; 218 } 219 220 void idxd_wq_free_resources(struct idxd_wq *wq) 221 { 222 struct device *dev = &wq->idxd->pdev->dev; 223 224 if (wq->type != IDXD_WQT_KERNEL) 225 return; 226 227 free_hw_descs(wq); 228 free_descs(wq); 229 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); 230 sbitmap_free(&wq->sbmap); 231 } 232 233 int idxd_wq_enable(struct idxd_wq *wq) 234 { 235 struct idxd_device *idxd = wq->idxd; 236 struct device *dev = &idxd->pdev->dev; 237 u32 status; 238 int rc; 239 240 lockdep_assert_held(&idxd->dev_lock); 241 242 if (wq->state == IDXD_WQ_ENABLED) { 243 dev_dbg(dev, "WQ %d already enabled\n", wq->id); 244 return -ENXIO; 245 } 246 247 rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_WQ, wq->id); 248 if (rc < 0) 249 return rc; 250 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 251 if (rc < 0) 252 return rc; 253 254 if (status != IDXD_CMDSTS_SUCCESS && 255 status != IDXD_CMDSTS_ERR_WQ_ENABLED) { 256 dev_dbg(dev, "WQ enable failed: %#x\n", status); 257 return -ENXIO; 258 } 259 260 wq->state = IDXD_WQ_ENABLED; 261 dev_dbg(dev, "WQ %d enabled\n", wq->id); 262 return 0; 263 } 264 265 int idxd_wq_disable(struct idxd_wq *wq) 266 { 267 struct idxd_device *idxd = wq->idxd; 268 struct device *dev = &idxd->pdev->dev; 269 u32 status, operand; 270 int rc; 271 272 lockdep_assert_held(&idxd->dev_lock); 273 dev_dbg(dev, "Disabling WQ %d\n", wq->id); 274 275 if (wq->state != IDXD_WQ_ENABLED) { 276 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); 277 return 0; 278 } 279 280 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); 281 rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_WQ, operand); 282 if (rc < 0) 283 return rc; 284 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 285 if (rc < 0) 286 return rc; 287 288 if (status != IDXD_CMDSTS_SUCCESS) { 289 dev_dbg(dev, "WQ disable failed: %#x\n", status); 290 return -ENXIO; 291 } 292 293 wq->state = IDXD_WQ_DISABLED; 294 dev_dbg(dev, "WQ %d disabled\n", wq->id); 295 return 0; 296 } 297 298 int idxd_wq_map_portal(struct idxd_wq *wq) 299 { 300 struct idxd_device *idxd = wq->idxd; 301 struct pci_dev *pdev = idxd->pdev; 302 struct device *dev = &pdev->dev; 303 resource_size_t start; 304 305 start = pci_resource_start(pdev, IDXD_WQ_BAR); 306 start = start + wq->id * IDXD_PORTAL_SIZE; 307 308 wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE); 309 if (!wq->dportal) 310 return -ENOMEM; 311 dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal); 312 313 return 0; 314 } 315 316 void idxd_wq_unmap_portal(struct idxd_wq *wq) 317 { 318 struct device *dev = &wq->idxd->pdev->dev; 319 320 devm_iounmap(dev, wq->dportal); 321 } 322 323 /* Device control bits */ 324 static inline bool idxd_is_enabled(struct idxd_device *idxd) 325 { 326 union gensts_reg gensts; 327 328 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 329 330 if (gensts.state == IDXD_DEVICE_STATE_ENABLED) 331 return true; 332 return false; 333 } 334 335 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout) 336 { 337 u32 sts, to = timeout; 338 339 lockdep_assert_held(&idxd->dev_lock); 340 sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); 341 while (sts & IDXD_CMDSTS_ACTIVE && --to) { 342 cpu_relax(); 343 sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); 344 } 345 346 if (to == 0 && sts & IDXD_CMDSTS_ACTIVE) { 347 dev_warn(&idxd->pdev->dev, "%s timed out!\n", __func__); 348 *status = 0; 349 return -EBUSY; 350 } 351 352 *status = sts; 353 return 0; 354 } 355 356 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand) 357 { 358 union idxd_command_reg cmd; 359 int rc; 360 u32 status; 361 362 lockdep_assert_held(&idxd->dev_lock); 363 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 364 if (rc < 0) 365 return rc; 366 367 memset(&cmd, 0, sizeof(cmd)); 368 cmd.cmd = cmd_code; 369 cmd.operand = operand; 370 dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n", 371 __func__, cmd_code, operand); 372 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); 373 374 return 0; 375 } 376 377 int idxd_device_enable(struct idxd_device *idxd) 378 { 379 struct device *dev = &idxd->pdev->dev; 380 int rc; 381 u32 status; 382 383 lockdep_assert_held(&idxd->dev_lock); 384 if (idxd_is_enabled(idxd)) { 385 dev_dbg(dev, "Device already enabled\n"); 386 return -ENXIO; 387 } 388 389 rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_DEVICE, 0); 390 if (rc < 0) 391 return rc; 392 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 393 if (rc < 0) 394 return rc; 395 396 /* If the command is successful or if the device was enabled */ 397 if (status != IDXD_CMDSTS_SUCCESS && 398 status != IDXD_CMDSTS_ERR_DEV_ENABLED) { 399 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status); 400 return -ENXIO; 401 } 402 403 idxd->state = IDXD_DEV_ENABLED; 404 return 0; 405 } 406 407 int idxd_device_disable(struct idxd_device *idxd) 408 { 409 struct device *dev = &idxd->pdev->dev; 410 int rc; 411 u32 status; 412 413 lockdep_assert_held(&idxd->dev_lock); 414 if (!idxd_is_enabled(idxd)) { 415 dev_dbg(dev, "Device is not enabled\n"); 416 return 0; 417 } 418 419 rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_DEVICE, 0); 420 if (rc < 0) 421 return rc; 422 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 423 if (rc < 0) 424 return rc; 425 426 /* If the command is successful or if the device was disabled */ 427 if (status != IDXD_CMDSTS_SUCCESS && 428 !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) { 429 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status); 430 rc = -ENXIO; 431 return rc; 432 } 433 434 idxd->state = IDXD_DEV_CONF_READY; 435 return 0; 436 } 437 438 int __idxd_device_reset(struct idxd_device *idxd) 439 { 440 u32 status; 441 int rc; 442 443 rc = idxd_cmd_send(idxd, IDXD_CMD_RESET_DEVICE, 0); 444 if (rc < 0) 445 return rc; 446 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT); 447 if (rc < 0) 448 return rc; 449 450 return 0; 451 } 452 453 int idxd_device_reset(struct idxd_device *idxd) 454 { 455 unsigned long flags; 456 int rc; 457 458 spin_lock_irqsave(&idxd->dev_lock, flags); 459 rc = __idxd_device_reset(idxd); 460 spin_unlock_irqrestore(&idxd->dev_lock, flags); 461 return rc; 462 } 463 464 /* Device configuration bits */ 465 static void idxd_group_config_write(struct idxd_group *group) 466 { 467 struct idxd_device *idxd = group->idxd; 468 struct device *dev = &idxd->pdev->dev; 469 int i; 470 u32 grpcfg_offset; 471 472 dev_dbg(dev, "Writing group %d cfg registers\n", group->id); 473 474 /* setup GRPWQCFG */ 475 for (i = 0; i < 4; i++) { 476 grpcfg_offset = idxd->grpcfg_offset + 477 group->id * 64 + i * sizeof(u64); 478 iowrite64(group->grpcfg.wqs[i], 479 idxd->reg_base + grpcfg_offset); 480 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", 481 group->id, i, grpcfg_offset, 482 ioread64(idxd->reg_base + grpcfg_offset)); 483 } 484 485 /* setup GRPENGCFG */ 486 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32; 487 iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset); 488 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, 489 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); 490 491 /* setup GRPFLAGS */ 492 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40; 493 iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); 494 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", 495 group->id, grpcfg_offset, 496 ioread32(idxd->reg_base + grpcfg_offset)); 497 } 498 499 static int idxd_groups_config_write(struct idxd_device *idxd) 500 501 { 502 union gencfg_reg reg; 503 int i; 504 struct device *dev = &idxd->pdev->dev; 505 506 /* Setup bandwidth token limit */ 507 if (idxd->token_limit) { 508 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 509 reg.token_limit = idxd->token_limit; 510 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 511 } 512 513 dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET, 514 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET)); 515 516 for (i = 0; i < idxd->max_groups; i++) { 517 struct idxd_group *group = &idxd->groups[i]; 518 519 idxd_group_config_write(group); 520 } 521 522 return 0; 523 } 524 525 static int idxd_wq_config_write(struct idxd_wq *wq) 526 { 527 struct idxd_device *idxd = wq->idxd; 528 struct device *dev = &idxd->pdev->dev; 529 u32 wq_offset; 530 int i; 531 532 if (!wq->group) 533 return 0; 534 535 memset(&wq->wqcfg, 0, sizeof(union wqcfg)); 536 537 /* byte 0-3 */ 538 wq->wqcfg.wq_size = wq->size; 539 540 if (wq->size == 0) { 541 dev_warn(dev, "Incorrect work queue size: 0\n"); 542 return -EINVAL; 543 } 544 545 /* bytes 4-7 */ 546 wq->wqcfg.wq_thresh = wq->threshold; 547 548 /* byte 8-11 */ 549 wq->wqcfg.priv = !!(wq->type == IDXD_WQT_KERNEL); 550 wq->wqcfg.mode = 1; 551 552 wq->wqcfg.priority = wq->priority; 553 554 /* bytes 12-15 */ 555 wq->wqcfg.max_xfer_shift = idxd->hw.gen_cap.max_xfer_shift; 556 wq->wqcfg.max_batch_shift = idxd->hw.gen_cap.max_batch_shift; 557 558 dev_dbg(dev, "WQ %d CFGs\n", wq->id); 559 for (i = 0; i < 8; i++) { 560 wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32); 561 iowrite32(wq->wqcfg.bits[i], idxd->reg_base + wq_offset); 562 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", 563 wq->id, i, wq_offset, 564 ioread32(idxd->reg_base + wq_offset)); 565 } 566 567 return 0; 568 } 569 570 static int idxd_wqs_config_write(struct idxd_device *idxd) 571 { 572 int i, rc; 573 574 for (i = 0; i < idxd->max_wqs; i++) { 575 struct idxd_wq *wq = &idxd->wqs[i]; 576 577 rc = idxd_wq_config_write(wq); 578 if (rc < 0) 579 return rc; 580 } 581 582 return 0; 583 } 584 585 static void idxd_group_flags_setup(struct idxd_device *idxd) 586 { 587 int i; 588 589 /* TC-A 0 and TC-B 1 should be defaults */ 590 for (i = 0; i < idxd->max_groups; i++) { 591 struct idxd_group *group = &idxd->groups[i]; 592 593 if (group->tc_a == -1) 594 group->tc_a = group->grpcfg.flags.tc_a = 0; 595 else 596 group->grpcfg.flags.tc_a = group->tc_a; 597 if (group->tc_b == -1) 598 group->tc_b = group->grpcfg.flags.tc_b = 1; 599 else 600 group->grpcfg.flags.tc_b = group->tc_b; 601 group->grpcfg.flags.use_token_limit = group->use_token_limit; 602 group->grpcfg.flags.tokens_reserved = group->tokens_reserved; 603 if (group->tokens_allowed) 604 group->grpcfg.flags.tokens_allowed = 605 group->tokens_allowed; 606 else 607 group->grpcfg.flags.tokens_allowed = idxd->max_tokens; 608 } 609 } 610 611 static int idxd_engines_setup(struct idxd_device *idxd) 612 { 613 int i, engines = 0; 614 struct idxd_engine *eng; 615 struct idxd_group *group; 616 617 for (i = 0; i < idxd->max_groups; i++) { 618 group = &idxd->groups[i]; 619 group->grpcfg.engines = 0; 620 } 621 622 for (i = 0; i < idxd->max_engines; i++) { 623 eng = &idxd->engines[i]; 624 group = eng->group; 625 626 if (!group) 627 continue; 628 629 group->grpcfg.engines |= BIT(eng->id); 630 engines++; 631 } 632 633 if (!engines) 634 return -EINVAL; 635 636 return 0; 637 } 638 639 static int idxd_wqs_setup(struct idxd_device *idxd) 640 { 641 struct idxd_wq *wq; 642 struct idxd_group *group; 643 int i, j, configured = 0; 644 struct device *dev = &idxd->pdev->dev; 645 646 for (i = 0; i < idxd->max_groups; i++) { 647 group = &idxd->groups[i]; 648 for (j = 0; j < 4; j++) 649 group->grpcfg.wqs[j] = 0; 650 } 651 652 for (i = 0; i < idxd->max_wqs; i++) { 653 wq = &idxd->wqs[i]; 654 group = wq->group; 655 656 if (!wq->group) 657 continue; 658 if (!wq->size) 659 continue; 660 661 if (!wq_dedicated(wq)) { 662 dev_warn(dev, "No shared workqueue support.\n"); 663 return -EINVAL; 664 } 665 666 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64); 667 configured++; 668 } 669 670 if (configured == 0) 671 return -EINVAL; 672 673 return 0; 674 } 675 676 int idxd_device_config(struct idxd_device *idxd) 677 { 678 int rc; 679 680 lockdep_assert_held(&idxd->dev_lock); 681 rc = idxd_wqs_setup(idxd); 682 if (rc < 0) 683 return rc; 684 685 rc = idxd_engines_setup(idxd); 686 if (rc < 0) 687 return rc; 688 689 idxd_group_flags_setup(idxd); 690 691 rc = idxd_wqs_config_write(idxd); 692 if (rc < 0) 693 return rc; 694 695 rc = idxd_groups_config_write(idxd); 696 if (rc < 0) 697 return rc; 698 699 return 0; 700 } 701