1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <linux/irq.h> 10 #include <linux/msi.h> 11 #include <uapi/linux/idxd.h> 12 #include "../dmaengine.h" 13 #include "idxd.h" 14 #include "registers.h" 15 16 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, 17 u32 *status); 18 19 /* Interrupt control bits */ 20 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) 21 { 22 struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); 23 24 pci_msi_mask_irq(data); 25 } 26 27 void idxd_mask_msix_vectors(struct idxd_device *idxd) 28 { 29 struct pci_dev *pdev = idxd->pdev; 30 int msixcnt = pci_msix_vec_count(pdev); 31 int i; 32 33 for (i = 0; i < msixcnt; i++) 34 idxd_mask_msix_vector(idxd, i); 35 } 36 37 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) 38 { 39 struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); 40 41 pci_msi_unmask_irq(data); 42 } 43 44 void idxd_unmask_error_interrupts(struct idxd_device *idxd) 45 { 46 union genctrl_reg genctrl; 47 48 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); 49 genctrl.softerr_int_en = 1; 50 genctrl.halt_int_en = 1; 51 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); 52 } 53 54 void idxd_mask_error_interrupts(struct idxd_device *idxd) 55 { 56 union genctrl_reg genctrl; 57 58 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); 59 genctrl.softerr_int_en = 0; 60 genctrl.halt_int_en = 0; 61 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); 62 } 63 64 static void free_hw_descs(struct idxd_wq *wq) 65 { 66 int i; 67 68 for (i = 0; i < wq->num_descs; i++) 69 kfree(wq->hw_descs[i]); 70 71 kfree(wq->hw_descs); 72 } 73 74 static int alloc_hw_descs(struct idxd_wq *wq, int num) 75 { 76 struct device *dev = &wq->idxd->pdev->dev; 77 int i; 78 int node = dev_to_node(dev); 79 80 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *), 81 GFP_KERNEL, node); 82 if (!wq->hw_descs) 83 return -ENOMEM; 84 85 for (i = 0; i < num; i++) { 86 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]), 87 GFP_KERNEL, node); 88 if (!wq->hw_descs[i]) { 89 free_hw_descs(wq); 90 return -ENOMEM; 91 } 92 } 93 94 return 0; 95 } 96 97 static void free_descs(struct idxd_wq *wq) 98 { 99 int i; 100 101 for (i = 0; i < wq->num_descs; i++) 102 kfree(wq->descs[i]); 103 104 kfree(wq->descs); 105 } 106 107 static int alloc_descs(struct idxd_wq *wq, int num) 108 { 109 struct device *dev = &wq->idxd->pdev->dev; 110 int i; 111 int node = dev_to_node(dev); 112 113 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *), 114 GFP_KERNEL, node); 115 if (!wq->descs) 116 return -ENOMEM; 117 118 for (i = 0; i < num; i++) { 119 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]), 120 GFP_KERNEL, node); 121 if (!wq->descs[i]) { 122 free_descs(wq); 123 return -ENOMEM; 124 } 125 } 126 127 return 0; 128 } 129 130 /* WQ control bits */ 131 int idxd_wq_alloc_resources(struct idxd_wq *wq) 132 { 133 struct idxd_device *idxd = wq->idxd; 134 struct device *dev = &idxd->pdev->dev; 135 int rc, num_descs, i; 136 int align; 137 u64 tmp; 138 139 if (wq->type != IDXD_WQT_KERNEL) 140 return 0; 141 142 wq->num_descs = wq->size; 143 num_descs = wq->size; 144 145 rc = alloc_hw_descs(wq, num_descs); 146 if (rc < 0) 147 return rc; 148 149 align = idxd->data->align; 150 wq->compls_size = num_descs * idxd->data->compl_size + align; 151 wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size, 152 &wq->compls_addr_raw, GFP_KERNEL); 153 if (!wq->compls_raw) { 154 rc = -ENOMEM; 155 goto fail_alloc_compls; 156 } 157 158 /* Adjust alignment */ 159 wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1); 160 tmp = (u64)wq->compls_raw; 161 tmp = (tmp + (align - 1)) & ~(align - 1); 162 wq->compls = (struct dsa_completion_record *)tmp; 163 164 rc = alloc_descs(wq, num_descs); 165 if (rc < 0) 166 goto fail_alloc_descs; 167 168 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL, 169 dev_to_node(dev)); 170 if (rc < 0) 171 goto fail_sbitmap_init; 172 173 for (i = 0; i < num_descs; i++) { 174 struct idxd_desc *desc = wq->descs[i]; 175 176 desc->hw = wq->hw_descs[i]; 177 if (idxd->data->type == IDXD_TYPE_DSA) 178 desc->completion = &wq->compls[i]; 179 else if (idxd->data->type == IDXD_TYPE_IAX) 180 desc->iax_completion = &wq->iax_compls[i]; 181 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i; 182 desc->id = i; 183 desc->wq = wq; 184 desc->cpu = -1; 185 } 186 187 return 0; 188 189 fail_sbitmap_init: 190 free_descs(wq); 191 fail_alloc_descs: 192 dma_free_coherent(dev, wq->compls_size, wq->compls_raw, 193 wq->compls_addr_raw); 194 fail_alloc_compls: 195 free_hw_descs(wq); 196 return rc; 197 } 198 199 void idxd_wq_free_resources(struct idxd_wq *wq) 200 { 201 struct device *dev = &wq->idxd->pdev->dev; 202 203 if (wq->type != IDXD_WQT_KERNEL) 204 return; 205 206 free_hw_descs(wq); 207 free_descs(wq); 208 dma_free_coherent(dev, wq->compls_size, wq->compls_raw, 209 wq->compls_addr_raw); 210 sbitmap_queue_free(&wq->sbq); 211 } 212 213 int idxd_wq_enable(struct idxd_wq *wq) 214 { 215 struct idxd_device *idxd = wq->idxd; 216 struct device *dev = &idxd->pdev->dev; 217 u32 status; 218 219 if (wq->state == IDXD_WQ_ENABLED) { 220 dev_dbg(dev, "WQ %d already enabled\n", wq->id); 221 return -ENXIO; 222 } 223 224 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status); 225 226 if (status != IDXD_CMDSTS_SUCCESS && 227 status != IDXD_CMDSTS_ERR_WQ_ENABLED) { 228 dev_dbg(dev, "WQ enable failed: %#x\n", status); 229 return -ENXIO; 230 } 231 232 wq->state = IDXD_WQ_ENABLED; 233 dev_dbg(dev, "WQ %d enabled\n", wq->id); 234 return 0; 235 } 236 237 int idxd_wq_disable(struct idxd_wq *wq) 238 { 239 struct idxd_device *idxd = wq->idxd; 240 struct device *dev = &idxd->pdev->dev; 241 u32 status, operand; 242 243 dev_dbg(dev, "Disabling WQ %d\n", wq->id); 244 245 if (wq->state != IDXD_WQ_ENABLED) { 246 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); 247 return 0; 248 } 249 250 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); 251 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status); 252 253 if (status != IDXD_CMDSTS_SUCCESS) { 254 dev_dbg(dev, "WQ disable failed: %#x\n", status); 255 return -ENXIO; 256 } 257 258 wq->state = IDXD_WQ_DISABLED; 259 dev_dbg(dev, "WQ %d disabled\n", wq->id); 260 return 0; 261 } 262 263 void idxd_wq_drain(struct idxd_wq *wq) 264 { 265 struct idxd_device *idxd = wq->idxd; 266 struct device *dev = &idxd->pdev->dev; 267 u32 operand; 268 269 if (wq->state != IDXD_WQ_ENABLED) { 270 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); 271 return; 272 } 273 274 dev_dbg(dev, "Draining WQ %d\n", wq->id); 275 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); 276 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL); 277 } 278 279 void idxd_wq_reset(struct idxd_wq *wq) 280 { 281 struct idxd_device *idxd = wq->idxd; 282 struct device *dev = &idxd->pdev->dev; 283 u32 operand; 284 285 if (wq->state != IDXD_WQ_ENABLED) { 286 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); 287 return; 288 } 289 290 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); 291 idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL); 292 wq->state = IDXD_WQ_DISABLED; 293 } 294 295 int idxd_wq_map_portal(struct idxd_wq *wq) 296 { 297 struct idxd_device *idxd = wq->idxd; 298 struct pci_dev *pdev = idxd->pdev; 299 struct device *dev = &pdev->dev; 300 resource_size_t start; 301 302 start = pci_resource_start(pdev, IDXD_WQ_BAR); 303 start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED); 304 305 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE); 306 if (!wq->portal) 307 return -ENOMEM; 308 309 return 0; 310 } 311 312 void idxd_wq_unmap_portal(struct idxd_wq *wq) 313 { 314 struct device *dev = &wq->idxd->pdev->dev; 315 316 devm_iounmap(dev, wq->portal); 317 wq->portal = NULL; 318 } 319 320 void idxd_wqs_unmap_portal(struct idxd_device *idxd) 321 { 322 int i; 323 324 for (i = 0; i < idxd->max_wqs; i++) { 325 struct idxd_wq *wq = idxd->wqs[i]; 326 327 if (wq->portal) 328 idxd_wq_unmap_portal(wq); 329 } 330 } 331 332 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) 333 { 334 struct idxd_device *idxd = wq->idxd; 335 int rc; 336 union wqcfg wqcfg; 337 unsigned int offset; 338 unsigned long flags; 339 340 rc = idxd_wq_disable(wq); 341 if (rc < 0) 342 return rc; 343 344 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); 345 spin_lock_irqsave(&idxd->dev_lock, flags); 346 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); 347 wqcfg.pasid_en = 1; 348 wqcfg.pasid = pasid; 349 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); 350 spin_unlock_irqrestore(&idxd->dev_lock, flags); 351 352 rc = idxd_wq_enable(wq); 353 if (rc < 0) 354 return rc; 355 356 return 0; 357 } 358 359 int idxd_wq_disable_pasid(struct idxd_wq *wq) 360 { 361 struct idxd_device *idxd = wq->idxd; 362 int rc; 363 union wqcfg wqcfg; 364 unsigned int offset; 365 unsigned long flags; 366 367 rc = idxd_wq_disable(wq); 368 if (rc < 0) 369 return rc; 370 371 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); 372 spin_lock_irqsave(&idxd->dev_lock, flags); 373 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); 374 wqcfg.pasid_en = 0; 375 wqcfg.pasid = 0; 376 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); 377 spin_unlock_irqrestore(&idxd->dev_lock, flags); 378 379 rc = idxd_wq_enable(wq); 380 if (rc < 0) 381 return rc; 382 383 return 0; 384 } 385 386 void idxd_wq_disable_cleanup(struct idxd_wq *wq) 387 { 388 struct idxd_device *idxd = wq->idxd; 389 390 lockdep_assert_held(&idxd->dev_lock); 391 memset(wq->wqcfg, 0, idxd->wqcfg_size); 392 wq->type = IDXD_WQT_NONE; 393 wq->size = 0; 394 wq->group = NULL; 395 wq->threshold = 0; 396 wq->priority = 0; 397 wq->ats_dis = 0; 398 clear_bit(WQ_FLAG_DEDICATED, &wq->flags); 399 memset(wq->name, 0, WQ_NAME_SIZE); 400 } 401 402 static void idxd_wq_ref_release(struct percpu_ref *ref) 403 { 404 struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active); 405 406 complete(&wq->wq_dead); 407 } 408 409 int idxd_wq_init_percpu_ref(struct idxd_wq *wq) 410 { 411 int rc; 412 413 memset(&wq->wq_active, 0, sizeof(wq->wq_active)); 414 rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL); 415 if (rc < 0) 416 return rc; 417 reinit_completion(&wq->wq_dead); 418 return 0; 419 } 420 421 void idxd_wq_quiesce(struct idxd_wq *wq) 422 { 423 percpu_ref_kill(&wq->wq_active); 424 wait_for_completion(&wq->wq_dead); 425 percpu_ref_exit(&wq->wq_active); 426 } 427 428 /* Device control bits */ 429 static inline bool idxd_is_enabled(struct idxd_device *idxd) 430 { 431 union gensts_reg gensts; 432 433 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 434 435 if (gensts.state == IDXD_DEVICE_STATE_ENABLED) 436 return true; 437 return false; 438 } 439 440 static inline bool idxd_device_is_halted(struct idxd_device *idxd) 441 { 442 union gensts_reg gensts; 443 444 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 445 446 return (gensts.state == IDXD_DEVICE_STATE_HALT); 447 } 448 449 /* 450 * This is function is only used for reset during probe and will 451 * poll for completion. Once the device is setup with interrupts, 452 * all commands will be done via interrupt completion. 453 */ 454 int idxd_device_init_reset(struct idxd_device *idxd) 455 { 456 struct device *dev = &idxd->pdev->dev; 457 union idxd_command_reg cmd; 458 unsigned long flags; 459 460 if (idxd_device_is_halted(idxd)) { 461 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); 462 return -ENXIO; 463 } 464 465 memset(&cmd, 0, sizeof(cmd)); 466 cmd.cmd = IDXD_CMD_RESET_DEVICE; 467 dev_dbg(dev, "%s: sending reset for init.\n", __func__); 468 spin_lock_irqsave(&idxd->cmd_lock, flags); 469 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); 470 471 while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & 472 IDXD_CMDSTS_ACTIVE) 473 cpu_relax(); 474 spin_unlock_irqrestore(&idxd->cmd_lock, flags); 475 return 0; 476 } 477 478 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, 479 u32 *status) 480 { 481 union idxd_command_reg cmd; 482 DECLARE_COMPLETION_ONSTACK(done); 483 unsigned long flags; 484 485 if (idxd_device_is_halted(idxd)) { 486 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); 487 if (status) 488 *status = IDXD_CMDSTS_HW_ERR; 489 return; 490 } 491 492 memset(&cmd, 0, sizeof(cmd)); 493 cmd.cmd = cmd_code; 494 cmd.operand = operand; 495 cmd.int_req = 1; 496 497 spin_lock_irqsave(&idxd->cmd_lock, flags); 498 wait_event_lock_irq(idxd->cmd_waitq, 499 !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags), 500 idxd->cmd_lock); 501 502 dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n", 503 __func__, cmd_code, operand); 504 505 idxd->cmd_status = 0; 506 __set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); 507 idxd->cmd_done = &done; 508 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); 509 510 /* 511 * After command submitted, release lock and go to sleep until 512 * the command completes via interrupt. 513 */ 514 spin_unlock_irqrestore(&idxd->cmd_lock, flags); 515 wait_for_completion(&done); 516 spin_lock_irqsave(&idxd->cmd_lock, flags); 517 if (status) { 518 *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); 519 idxd->cmd_status = *status & GENMASK(7, 0); 520 } 521 522 __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); 523 /* Wake up other pending commands */ 524 wake_up(&idxd->cmd_waitq); 525 spin_unlock_irqrestore(&idxd->cmd_lock, flags); 526 } 527 528 int idxd_device_enable(struct idxd_device *idxd) 529 { 530 struct device *dev = &idxd->pdev->dev; 531 u32 status; 532 533 if (idxd_is_enabled(idxd)) { 534 dev_dbg(dev, "Device already enabled\n"); 535 return -ENXIO; 536 } 537 538 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status); 539 540 /* If the command is successful or if the device was enabled */ 541 if (status != IDXD_CMDSTS_SUCCESS && 542 status != IDXD_CMDSTS_ERR_DEV_ENABLED) { 543 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status); 544 return -ENXIO; 545 } 546 547 idxd->state = IDXD_DEV_ENABLED; 548 return 0; 549 } 550 551 void idxd_device_wqs_clear_state(struct idxd_device *idxd) 552 { 553 int i; 554 555 lockdep_assert_held(&idxd->dev_lock); 556 557 for (i = 0; i < idxd->max_wqs; i++) { 558 struct idxd_wq *wq = idxd->wqs[i]; 559 560 if (wq->state == IDXD_WQ_ENABLED) { 561 idxd_wq_disable_cleanup(wq); 562 wq->state = IDXD_WQ_DISABLED; 563 } 564 } 565 } 566 567 int idxd_device_disable(struct idxd_device *idxd) 568 { 569 struct device *dev = &idxd->pdev->dev; 570 u32 status; 571 unsigned long flags; 572 573 if (!idxd_is_enabled(idxd)) { 574 dev_dbg(dev, "Device is not enabled\n"); 575 return 0; 576 } 577 578 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status); 579 580 /* If the command is successful or if the device was disabled */ 581 if (status != IDXD_CMDSTS_SUCCESS && 582 !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) { 583 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status); 584 return -ENXIO; 585 } 586 587 spin_lock_irqsave(&idxd->dev_lock, flags); 588 idxd_device_wqs_clear_state(idxd); 589 idxd->state = IDXD_DEV_CONF_READY; 590 spin_unlock_irqrestore(&idxd->dev_lock, flags); 591 return 0; 592 } 593 594 void idxd_device_reset(struct idxd_device *idxd) 595 { 596 unsigned long flags; 597 598 idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL); 599 spin_lock_irqsave(&idxd->dev_lock, flags); 600 idxd_device_wqs_clear_state(idxd); 601 idxd->state = IDXD_DEV_CONF_READY; 602 spin_unlock_irqrestore(&idxd->dev_lock, flags); 603 } 604 605 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid) 606 { 607 struct device *dev = &idxd->pdev->dev; 608 u32 operand; 609 610 operand = pasid; 611 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand); 612 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL); 613 dev_dbg(dev, "pasid %d drained\n", pasid); 614 } 615 616 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 617 enum idxd_interrupt_type irq_type) 618 { 619 struct device *dev = &idxd->pdev->dev; 620 u32 operand, status; 621 622 if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE))) 623 return -EOPNOTSUPP; 624 625 dev_dbg(dev, "get int handle, idx %d\n", idx); 626 627 operand = idx & GENMASK(15, 0); 628 if (irq_type == IDXD_IRQ_IMS) 629 operand |= CMD_INT_HANDLE_IMS; 630 631 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE, operand); 632 633 idxd_cmd_exec(idxd, IDXD_CMD_REQUEST_INT_HANDLE, operand, &status); 634 635 if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { 636 dev_dbg(dev, "request int handle failed: %#x\n", status); 637 return -ENXIO; 638 } 639 640 *handle = (status >> IDXD_CMDSTS_RES_SHIFT) & GENMASK(15, 0); 641 642 dev_dbg(dev, "int handle acquired: %u\n", *handle); 643 return 0; 644 } 645 646 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 647 enum idxd_interrupt_type irq_type) 648 { 649 struct device *dev = &idxd->pdev->dev; 650 u32 operand, status; 651 union idxd_command_reg cmd; 652 unsigned long flags; 653 654 if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))) 655 return -EOPNOTSUPP; 656 657 dev_dbg(dev, "release int handle, handle %d\n", handle); 658 659 memset(&cmd, 0, sizeof(cmd)); 660 operand = handle & GENMASK(15, 0); 661 662 if (irq_type == IDXD_IRQ_IMS) 663 operand |= CMD_INT_HANDLE_IMS; 664 665 cmd.cmd = IDXD_CMD_RELEASE_INT_HANDLE; 666 cmd.operand = operand; 667 668 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand); 669 670 spin_lock_irqsave(&idxd->cmd_lock, flags); 671 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); 672 673 while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) 674 cpu_relax(); 675 status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); 676 spin_unlock_irqrestore(&idxd->cmd_lock, flags); 677 678 if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { 679 dev_dbg(dev, "release int handle failed: %#x\n", status); 680 return -ENXIO; 681 } 682 683 dev_dbg(dev, "int handle released.\n"); 684 return 0; 685 } 686 687 /* Device configuration bits */ 688 void idxd_msix_perm_setup(struct idxd_device *idxd) 689 { 690 union msix_perm mperm; 691 int i, msixcnt; 692 693 msixcnt = pci_msix_vec_count(idxd->pdev); 694 if (msixcnt < 0) 695 return; 696 697 mperm.bits = 0; 698 mperm.pasid = idxd->pasid; 699 mperm.pasid_en = device_pasid_enabled(idxd); 700 for (i = 1; i < msixcnt; i++) 701 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); 702 } 703 704 void idxd_msix_perm_clear(struct idxd_device *idxd) 705 { 706 union msix_perm mperm; 707 int i, msixcnt; 708 709 msixcnt = pci_msix_vec_count(idxd->pdev); 710 if (msixcnt < 0) 711 return; 712 713 mperm.bits = 0; 714 for (i = 1; i < msixcnt; i++) 715 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); 716 } 717 718 static void idxd_group_config_write(struct idxd_group *group) 719 { 720 struct idxd_device *idxd = group->idxd; 721 struct device *dev = &idxd->pdev->dev; 722 int i; 723 u32 grpcfg_offset; 724 725 dev_dbg(dev, "Writing group %d cfg registers\n", group->id); 726 727 /* setup GRPWQCFG */ 728 for (i = 0; i < GRPWQCFG_STRIDES; i++) { 729 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i); 730 iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset); 731 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", 732 group->id, i, grpcfg_offset, 733 ioread64(idxd->reg_base + grpcfg_offset)); 734 } 735 736 /* setup GRPENGCFG */ 737 grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id); 738 iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset); 739 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, 740 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); 741 742 /* setup GRPFLAGS */ 743 grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); 744 iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); 745 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", 746 group->id, grpcfg_offset, 747 ioread32(idxd->reg_base + grpcfg_offset)); 748 } 749 750 static int idxd_groups_config_write(struct idxd_device *idxd) 751 752 { 753 union gencfg_reg reg; 754 int i; 755 struct device *dev = &idxd->pdev->dev; 756 757 /* Setup bandwidth token limit */ 758 if (idxd->token_limit) { 759 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 760 reg.token_limit = idxd->token_limit; 761 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 762 } 763 764 dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET, 765 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET)); 766 767 for (i = 0; i < idxd->max_groups; i++) { 768 struct idxd_group *group = idxd->groups[i]; 769 770 idxd_group_config_write(group); 771 } 772 773 return 0; 774 } 775 776 static int idxd_wq_config_write(struct idxd_wq *wq) 777 { 778 struct idxd_device *idxd = wq->idxd; 779 struct device *dev = &idxd->pdev->dev; 780 u32 wq_offset; 781 int i; 782 783 if (!wq->group) 784 return 0; 785 786 /* 787 * Instead of memset the entire shadow copy of WQCFG, copy from the hardware after 788 * wq reset. This will copy back the sticky values that are present on some devices. 789 */ 790 for (i = 0; i < WQCFG_STRIDES(idxd); i++) { 791 wq_offset = WQCFG_OFFSET(idxd, wq->id, i); 792 wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset); 793 } 794 795 /* byte 0-3 */ 796 wq->wqcfg->wq_size = wq->size; 797 798 if (wq->size == 0) { 799 dev_warn(dev, "Incorrect work queue size: 0\n"); 800 return -EINVAL; 801 } 802 803 /* bytes 4-7 */ 804 wq->wqcfg->wq_thresh = wq->threshold; 805 806 /* byte 8-11 */ 807 wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); 808 if (wq_dedicated(wq)) 809 wq->wqcfg->mode = 1; 810 811 if (device_pasid_enabled(idxd)) { 812 wq->wqcfg->pasid_en = 1; 813 if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq)) 814 wq->wqcfg->pasid = idxd->pasid; 815 } 816 817 wq->wqcfg->priority = wq->priority; 818 819 if (idxd->hw.gen_cap.block_on_fault && 820 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)) 821 wq->wqcfg->bof = 1; 822 823 if (idxd->hw.wq_cap.wq_ats_support) 824 wq->wqcfg->wq_ats_disable = wq->ats_dis; 825 826 /* bytes 12-15 */ 827 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); 828 wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); 829 830 dev_dbg(dev, "WQ %d CFGs\n", wq->id); 831 for (i = 0; i < WQCFG_STRIDES(idxd); i++) { 832 wq_offset = WQCFG_OFFSET(idxd, wq->id, i); 833 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset); 834 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", 835 wq->id, i, wq_offset, 836 ioread32(idxd->reg_base + wq_offset)); 837 } 838 839 return 0; 840 } 841 842 static int idxd_wqs_config_write(struct idxd_device *idxd) 843 { 844 int i, rc; 845 846 for (i = 0; i < idxd->max_wqs; i++) { 847 struct idxd_wq *wq = idxd->wqs[i]; 848 849 rc = idxd_wq_config_write(wq); 850 if (rc < 0) 851 return rc; 852 } 853 854 return 0; 855 } 856 857 static void idxd_group_flags_setup(struct idxd_device *idxd) 858 { 859 int i; 860 861 /* TC-A 0 and TC-B 1 should be defaults */ 862 for (i = 0; i < idxd->max_groups; i++) { 863 struct idxd_group *group = idxd->groups[i]; 864 865 if (group->tc_a == -1) 866 group->tc_a = group->grpcfg.flags.tc_a = 0; 867 else 868 group->grpcfg.flags.tc_a = group->tc_a; 869 if (group->tc_b == -1) 870 group->tc_b = group->grpcfg.flags.tc_b = 1; 871 else 872 group->grpcfg.flags.tc_b = group->tc_b; 873 group->grpcfg.flags.use_token_limit = group->use_token_limit; 874 group->grpcfg.flags.tokens_reserved = group->tokens_reserved; 875 if (group->tokens_allowed) 876 group->grpcfg.flags.tokens_allowed = 877 group->tokens_allowed; 878 else 879 group->grpcfg.flags.tokens_allowed = idxd->max_tokens; 880 } 881 } 882 883 static int idxd_engines_setup(struct idxd_device *idxd) 884 { 885 int i, engines = 0; 886 struct idxd_engine *eng; 887 struct idxd_group *group; 888 889 for (i = 0; i < idxd->max_groups; i++) { 890 group = idxd->groups[i]; 891 group->grpcfg.engines = 0; 892 } 893 894 for (i = 0; i < idxd->max_engines; i++) { 895 eng = idxd->engines[i]; 896 group = eng->group; 897 898 if (!group) 899 continue; 900 901 group->grpcfg.engines |= BIT(eng->id); 902 engines++; 903 } 904 905 if (!engines) 906 return -EINVAL; 907 908 return 0; 909 } 910 911 static int idxd_wqs_setup(struct idxd_device *idxd) 912 { 913 struct idxd_wq *wq; 914 struct idxd_group *group; 915 int i, j, configured = 0; 916 struct device *dev = &idxd->pdev->dev; 917 918 for (i = 0; i < idxd->max_groups; i++) { 919 group = idxd->groups[i]; 920 for (j = 0; j < 4; j++) 921 group->grpcfg.wqs[j] = 0; 922 } 923 924 for (i = 0; i < idxd->max_wqs; i++) { 925 wq = idxd->wqs[i]; 926 group = wq->group; 927 928 if (!wq->group) 929 continue; 930 if (!wq->size) 931 continue; 932 933 if (wq_shared(wq) && !device_swq_supported(idxd)) { 934 dev_warn(dev, "No shared wq support but configured.\n"); 935 return -EINVAL; 936 } 937 938 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64); 939 configured++; 940 } 941 942 if (configured == 0) 943 return -EINVAL; 944 945 return 0; 946 } 947 948 int idxd_device_config(struct idxd_device *idxd) 949 { 950 int rc; 951 952 lockdep_assert_held(&idxd->dev_lock); 953 rc = idxd_wqs_setup(idxd); 954 if (rc < 0) 955 return rc; 956 957 rc = idxd_engines_setup(idxd); 958 if (rc < 0) 959 return rc; 960 961 idxd_group_flags_setup(idxd); 962 963 rc = idxd_wqs_config_write(idxd); 964 if (rc < 0) 965 return rc; 966 967 rc = idxd_groups_config_write(idxd); 968 if (rc < 0) 969 return rc; 970 971 return 0; 972 } 973 974 static int idxd_wq_load_config(struct idxd_wq *wq) 975 { 976 struct idxd_device *idxd = wq->idxd; 977 struct device *dev = &idxd->pdev->dev; 978 int wqcfg_offset; 979 int i; 980 981 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0); 982 memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size); 983 984 wq->size = wq->wqcfg->wq_size; 985 wq->threshold = wq->wqcfg->wq_thresh; 986 if (wq->wqcfg->priv) 987 wq->type = IDXD_WQT_KERNEL; 988 989 /* The driver does not support shared WQ mode in read-only config yet */ 990 if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en) 991 return -EOPNOTSUPP; 992 993 set_bit(WQ_FLAG_DEDICATED, &wq->flags); 994 995 wq->priority = wq->wqcfg->priority; 996 997 for (i = 0; i < WQCFG_STRIDES(idxd); i++) { 998 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i); 999 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]); 1000 } 1001 1002 return 0; 1003 } 1004 1005 static void idxd_group_load_config(struct idxd_group *group) 1006 { 1007 struct idxd_device *idxd = group->idxd; 1008 struct device *dev = &idxd->pdev->dev; 1009 int i, j, grpcfg_offset; 1010 1011 /* 1012 * Load WQS bit fields 1013 * Iterate through all 256 bits 64 bits at a time 1014 */ 1015 for (i = 0; i < GRPWQCFG_STRIDES; i++) { 1016 struct idxd_wq *wq; 1017 1018 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i); 1019 group->grpcfg.wqs[i] = ioread64(idxd->reg_base + grpcfg_offset); 1020 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", 1021 group->id, i, grpcfg_offset, group->grpcfg.wqs[i]); 1022 1023 if (i * 64 >= idxd->max_wqs) 1024 break; 1025 1026 /* Iterate through all 64 bits and check for wq set */ 1027 for (j = 0; j < 64; j++) { 1028 int id = i * 64 + j; 1029 1030 /* No need to check beyond max wqs */ 1031 if (id >= idxd->max_wqs) 1032 break; 1033 1034 /* Set group assignment for wq if wq bit is set */ 1035 if (group->grpcfg.wqs[i] & BIT(j)) { 1036 wq = idxd->wqs[id]; 1037 wq->group = group; 1038 } 1039 } 1040 } 1041 1042 grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id); 1043 group->grpcfg.engines = ioread64(idxd->reg_base + grpcfg_offset); 1044 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, 1045 grpcfg_offset, group->grpcfg.engines); 1046 1047 /* Iterate through all 64 bits to check engines set */ 1048 for (i = 0; i < 64; i++) { 1049 if (i >= idxd->max_engines) 1050 break; 1051 1052 if (group->grpcfg.engines & BIT(i)) { 1053 struct idxd_engine *engine = idxd->engines[i]; 1054 1055 engine->group = group; 1056 } 1057 } 1058 1059 grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); 1060 group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset); 1061 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", 1062 group->id, grpcfg_offset, group->grpcfg.flags.bits); 1063 } 1064 1065 int idxd_device_load_config(struct idxd_device *idxd) 1066 { 1067 union gencfg_reg reg; 1068 int i, rc; 1069 1070 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 1071 idxd->token_limit = reg.token_limit; 1072 1073 for (i = 0; i < idxd->max_groups; i++) { 1074 struct idxd_group *group = idxd->groups[i]; 1075 1076 idxd_group_load_config(group); 1077 } 1078 1079 for (i = 0; i < idxd->max_wqs; i++) { 1080 struct idxd_wq *wq = idxd->wqs[i]; 1081 1082 rc = idxd_wq_load_config(wq); 1083 if (rc < 0) 1084 return rc; 1085 } 1086 1087 return 0; 1088 } 1089