1 /* 2 * drivers/pci/iov.c 3 * 4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 5 * 6 * PCI Express I/O Virtualization (IOV) support. 7 * Single Root IOV 1.0 8 * Address Translation Service 1.0 9 */ 10 11 #include <linux/pci.h> 12 #include <linux/mutex.h> 13 #include <linux/string.h> 14 #include <linux/delay.h> 15 #include "pci.h" 16 17 #define VIRTFN_ID_LEN 16 18 19 static inline u8 virtfn_bus(struct pci_dev *dev, int id) 20 { 21 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 22 dev->sriov->stride * id) >> 8); 23 } 24 25 static inline u8 virtfn_devfn(struct pci_dev *dev, int id) 26 { 27 return (dev->devfn + dev->sriov->offset + 28 dev->sriov->stride * id) & 0xff; 29 } 30 31 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 32 { 33 int rc; 34 struct pci_bus *child; 35 36 if (bus->number == busnr) 37 return bus; 38 39 child = pci_find_bus(pci_domain_nr(bus), busnr); 40 if (child) 41 return child; 42 43 child = pci_add_new_bus(bus, NULL, busnr); 44 if (!child) 45 return NULL; 46 47 child->subordinate = busnr; 48 child->dev.parent = bus->bridge; 49 rc = pci_bus_add_child(child); 50 if (rc) { 51 pci_remove_bus(child); 52 return NULL; 53 } 54 55 return child; 56 } 57 58 static void virtfn_remove_bus(struct pci_bus *bus, int busnr) 59 { 60 struct pci_bus *child; 61 62 if (bus->number == busnr) 63 return; 64 65 child = pci_find_bus(pci_domain_nr(bus), busnr); 66 BUG_ON(!child); 67 68 if (list_empty(&child->devices)) 69 pci_remove_bus(child); 70 } 71 72 static int virtfn_add(struct pci_dev *dev, int id, int reset) 73 { 74 int i; 75 int rc; 76 u64 size; 77 char buf[VIRTFN_ID_LEN]; 78 struct pci_dev *virtfn; 79 struct resource *res; 80 struct pci_sriov *iov = dev->sriov; 81 82 virtfn = alloc_pci_dev(); 83 if (!virtfn) 84 return -ENOMEM; 85 86 mutex_lock(&iov->dev->sriov->lock); 87 virtfn->bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id)); 88 if (!virtfn->bus) { 89 kfree(virtfn); 90 mutex_unlock(&iov->dev->sriov->lock); 91 return -ENOMEM; 92 } 93 virtfn->devfn = virtfn_devfn(dev, id); 94 virtfn->vendor = dev->vendor; 95 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device); 96 pci_setup_device(virtfn); 97 virtfn->dev.parent = dev->dev.parent; 98 99 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 100 res = dev->resource + PCI_IOV_RESOURCES + i; 101 if (!res->parent) 102 continue; 103 virtfn->resource[i].name = pci_name(virtfn); 104 virtfn->resource[i].flags = res->flags; 105 size = resource_size(res); 106 do_div(size, iov->total); 107 virtfn->resource[i].start = res->start + size * id; 108 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 109 rc = request_resource(res, &virtfn->resource[i]); 110 BUG_ON(rc); 111 } 112 113 if (reset) 114 __pci_reset_function(virtfn); 115 116 pci_device_add(virtfn, virtfn->bus); 117 mutex_unlock(&iov->dev->sriov->lock); 118 119 virtfn->physfn = pci_dev_get(dev); 120 virtfn->is_virtfn = 1; 121 122 rc = pci_bus_add_device(virtfn); 123 if (rc) 124 goto failed1; 125 sprintf(buf, "virtfn%u", id); 126 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 127 if (rc) 128 goto failed1; 129 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 130 if (rc) 131 goto failed2; 132 133 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 134 135 return 0; 136 137 failed2: 138 sysfs_remove_link(&dev->dev.kobj, buf); 139 failed1: 140 pci_dev_put(dev); 141 mutex_lock(&iov->dev->sriov->lock); 142 pci_remove_bus_device(virtfn); 143 virtfn_remove_bus(dev->bus, virtfn_bus(dev, id)); 144 mutex_unlock(&iov->dev->sriov->lock); 145 146 return rc; 147 } 148 149 static void virtfn_remove(struct pci_dev *dev, int id, int reset) 150 { 151 char buf[VIRTFN_ID_LEN]; 152 struct pci_bus *bus; 153 struct pci_dev *virtfn; 154 struct pci_sriov *iov = dev->sriov; 155 156 bus = pci_find_bus(pci_domain_nr(dev->bus), virtfn_bus(dev, id)); 157 if (!bus) 158 return; 159 160 virtfn = pci_get_slot(bus, virtfn_devfn(dev, id)); 161 if (!virtfn) 162 return; 163 164 pci_dev_put(virtfn); 165 166 if (reset) { 167 device_release_driver(&virtfn->dev); 168 __pci_reset_function(virtfn); 169 } 170 171 sprintf(buf, "virtfn%u", id); 172 sysfs_remove_link(&dev->dev.kobj, buf); 173 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 174 175 mutex_lock(&iov->dev->sriov->lock); 176 pci_remove_bus_device(virtfn); 177 virtfn_remove_bus(dev->bus, virtfn_bus(dev, id)); 178 mutex_unlock(&iov->dev->sriov->lock); 179 180 pci_dev_put(dev); 181 } 182 183 static int sriov_migration(struct pci_dev *dev) 184 { 185 u16 status; 186 struct pci_sriov *iov = dev->sriov; 187 188 if (!iov->nr_virtfn) 189 return 0; 190 191 if (!(iov->cap & PCI_SRIOV_CAP_VFM)) 192 return 0; 193 194 pci_read_config_word(dev, iov->pos + PCI_SRIOV_STATUS, &status); 195 if (!(status & PCI_SRIOV_STATUS_VFM)) 196 return 0; 197 198 schedule_work(&iov->mtask); 199 200 return 1; 201 } 202 203 static void sriov_migration_task(struct work_struct *work) 204 { 205 int i; 206 u8 state; 207 u16 status; 208 struct pci_sriov *iov = container_of(work, struct pci_sriov, mtask); 209 210 for (i = iov->initial; i < iov->nr_virtfn; i++) { 211 state = readb(iov->mstate + i); 212 if (state == PCI_SRIOV_VFM_MI) { 213 writeb(PCI_SRIOV_VFM_AV, iov->mstate + i); 214 state = readb(iov->mstate + i); 215 if (state == PCI_SRIOV_VFM_AV) 216 virtfn_add(iov->self, i, 1); 217 } else if (state == PCI_SRIOV_VFM_MO) { 218 virtfn_remove(iov->self, i, 1); 219 writeb(PCI_SRIOV_VFM_UA, iov->mstate + i); 220 state = readb(iov->mstate + i); 221 if (state == PCI_SRIOV_VFM_AV) 222 virtfn_add(iov->self, i, 0); 223 } 224 } 225 226 pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status); 227 status &= ~PCI_SRIOV_STATUS_VFM; 228 pci_write_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, status); 229 } 230 231 static int sriov_enable_migration(struct pci_dev *dev, int nr_virtfn) 232 { 233 int bir; 234 u32 table; 235 resource_size_t pa; 236 struct pci_sriov *iov = dev->sriov; 237 238 if (nr_virtfn <= iov->initial) 239 return 0; 240 241 pci_read_config_dword(dev, iov->pos + PCI_SRIOV_VFM, &table); 242 bir = PCI_SRIOV_VFM_BIR(table); 243 if (bir > PCI_STD_RESOURCE_END) 244 return -EIO; 245 246 table = PCI_SRIOV_VFM_OFFSET(table); 247 if (table + nr_virtfn > pci_resource_len(dev, bir)) 248 return -EIO; 249 250 pa = pci_resource_start(dev, bir) + table; 251 iov->mstate = ioremap(pa, nr_virtfn); 252 if (!iov->mstate) 253 return -ENOMEM; 254 255 INIT_WORK(&iov->mtask, sriov_migration_task); 256 257 iov->ctrl |= PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR; 258 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 259 260 return 0; 261 } 262 263 static void sriov_disable_migration(struct pci_dev *dev) 264 { 265 struct pci_sriov *iov = dev->sriov; 266 267 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR); 268 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 269 270 cancel_work_sync(&iov->mtask); 271 iounmap(iov->mstate); 272 } 273 274 static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 275 { 276 int rc; 277 int i, j; 278 int nres; 279 u16 offset, stride, initial; 280 struct resource *res; 281 struct pci_dev *pdev; 282 struct pci_sriov *iov = dev->sriov; 283 284 if (!nr_virtfn) 285 return 0; 286 287 if (iov->nr_virtfn) 288 return -EINVAL; 289 290 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 291 if (initial > iov->total || 292 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total))) 293 return -EIO; 294 295 if (nr_virtfn < 0 || nr_virtfn > iov->total || 296 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 297 return -EINVAL; 298 299 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 300 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset); 301 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride); 302 if (!offset || (nr_virtfn > 1 && !stride)) 303 return -EIO; 304 305 nres = 0; 306 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 307 res = dev->resource + PCI_IOV_RESOURCES + i; 308 if (res->parent) 309 nres++; 310 } 311 if (nres != iov->nres) { 312 dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n"); 313 return -ENOMEM; 314 } 315 316 iov->offset = offset; 317 iov->stride = stride; 318 319 if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->subordinate) { 320 dev_err(&dev->dev, "SR-IOV: bus number out of range\n"); 321 return -ENOMEM; 322 } 323 324 if (iov->link != dev->devfn) { 325 pdev = pci_get_slot(dev->bus, iov->link); 326 if (!pdev) 327 return -ENODEV; 328 329 pci_dev_put(pdev); 330 331 if (!pdev->is_physfn) 332 return -ENODEV; 333 334 rc = sysfs_create_link(&dev->dev.kobj, 335 &pdev->dev.kobj, "dep_link"); 336 if (rc) 337 return rc; 338 } 339 340 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 341 pci_block_user_cfg_access(dev); 342 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 343 msleep(100); 344 pci_unblock_user_cfg_access(dev); 345 346 iov->initial = initial; 347 if (nr_virtfn < initial) 348 initial = nr_virtfn; 349 350 for (i = 0; i < initial; i++) { 351 rc = virtfn_add(dev, i, 0); 352 if (rc) 353 goto failed; 354 } 355 356 if (iov->cap & PCI_SRIOV_CAP_VFM) { 357 rc = sriov_enable_migration(dev, nr_virtfn); 358 if (rc) 359 goto failed; 360 } 361 362 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 363 iov->nr_virtfn = nr_virtfn; 364 365 return 0; 366 367 failed: 368 for (j = 0; j < i; j++) 369 virtfn_remove(dev, j, 0); 370 371 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 372 pci_block_user_cfg_access(dev); 373 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 374 ssleep(1); 375 pci_unblock_user_cfg_access(dev); 376 377 if (iov->link != dev->devfn) 378 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 379 380 return rc; 381 } 382 383 static void sriov_disable(struct pci_dev *dev) 384 { 385 int i; 386 struct pci_sriov *iov = dev->sriov; 387 388 if (!iov->nr_virtfn) 389 return; 390 391 if (iov->cap & PCI_SRIOV_CAP_VFM) 392 sriov_disable_migration(dev); 393 394 for (i = 0; i < iov->nr_virtfn; i++) 395 virtfn_remove(dev, i, 0); 396 397 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 398 pci_block_user_cfg_access(dev); 399 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 400 ssleep(1); 401 pci_unblock_user_cfg_access(dev); 402 403 if (iov->link != dev->devfn) 404 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 405 406 iov->nr_virtfn = 0; 407 } 408 409 static int sriov_init(struct pci_dev *dev, int pos) 410 { 411 int i; 412 int rc; 413 int nres; 414 u32 pgsz; 415 u16 ctrl, total, offset, stride; 416 struct pci_sriov *iov; 417 struct resource *res; 418 struct pci_dev *pdev; 419 420 if (dev->pcie_type != PCI_EXP_TYPE_RC_END && 421 dev->pcie_type != PCI_EXP_TYPE_ENDPOINT) 422 return -ENODEV; 423 424 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 425 if (ctrl & PCI_SRIOV_CTRL_VFE) { 426 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 427 ssleep(1); 428 } 429 430 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 431 if (!total) 432 return 0; 433 434 ctrl = 0; 435 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 436 if (pdev->is_physfn) 437 goto found; 438 439 pdev = NULL; 440 if (pci_ari_enabled(dev->bus)) 441 ctrl |= PCI_SRIOV_CTRL_ARI; 442 443 found: 444 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 445 pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, total); 446 pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset); 447 pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride); 448 if (!offset || (total > 1 && !stride)) 449 return -EIO; 450 451 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 452 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 453 pgsz &= ~((1 << i) - 1); 454 if (!pgsz) 455 return -EIO; 456 457 pgsz &= ~(pgsz - 1); 458 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 459 460 nres = 0; 461 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 462 res = dev->resource + PCI_IOV_RESOURCES + i; 463 i += __pci_read_base(dev, pci_bar_unknown, res, 464 pos + PCI_SRIOV_BAR + i * 4); 465 if (!res->flags) 466 continue; 467 if (resource_size(res) & (PAGE_SIZE - 1)) { 468 rc = -EIO; 469 goto failed; 470 } 471 res->end = res->start + resource_size(res) * total - 1; 472 nres++; 473 } 474 475 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 476 if (!iov) { 477 rc = -ENOMEM; 478 goto failed; 479 } 480 481 iov->pos = pos; 482 iov->nres = nres; 483 iov->ctrl = ctrl; 484 iov->total = total; 485 iov->offset = offset; 486 iov->stride = stride; 487 iov->pgsz = pgsz; 488 iov->self = dev; 489 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 490 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 491 if (dev->pcie_type == PCI_EXP_TYPE_RC_END) 492 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 493 494 if (pdev) 495 iov->dev = pci_dev_get(pdev); 496 else 497 iov->dev = dev; 498 499 mutex_init(&iov->lock); 500 501 dev->sriov = iov; 502 dev->is_physfn = 1; 503 504 return 0; 505 506 failed: 507 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 508 res = dev->resource + PCI_IOV_RESOURCES + i; 509 res->flags = 0; 510 } 511 512 return rc; 513 } 514 515 static void sriov_release(struct pci_dev *dev) 516 { 517 BUG_ON(dev->sriov->nr_virtfn); 518 519 if (dev != dev->sriov->dev) 520 pci_dev_put(dev->sriov->dev); 521 522 mutex_destroy(&dev->sriov->lock); 523 524 kfree(dev->sriov); 525 dev->sriov = NULL; 526 } 527 528 static void sriov_restore_state(struct pci_dev *dev) 529 { 530 int i; 531 u16 ctrl; 532 struct pci_sriov *iov = dev->sriov; 533 534 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 535 if (ctrl & PCI_SRIOV_CTRL_VFE) 536 return; 537 538 for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) 539 pci_update_resource(dev, i); 540 541 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 542 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, iov->nr_virtfn); 543 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 544 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 545 msleep(100); 546 } 547 548 /** 549 * pci_iov_init - initialize the IOV capability 550 * @dev: the PCI device 551 * 552 * Returns 0 on success, or negative on failure. 553 */ 554 int pci_iov_init(struct pci_dev *dev) 555 { 556 int pos; 557 558 if (!dev->is_pcie) 559 return -ENODEV; 560 561 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 562 if (pos) 563 return sriov_init(dev, pos); 564 565 return -ENODEV; 566 } 567 568 /** 569 * pci_iov_release - release resources used by the IOV capability 570 * @dev: the PCI device 571 */ 572 void pci_iov_release(struct pci_dev *dev) 573 { 574 if (dev->is_physfn) 575 sriov_release(dev); 576 } 577 578 /** 579 * pci_iov_resource_bar - get position of the SR-IOV BAR 580 * @dev: the PCI device 581 * @resno: the resource number 582 * @type: the BAR type to be filled in 583 * 584 * Returns position of the BAR encapsulated in the SR-IOV capability. 585 */ 586 int pci_iov_resource_bar(struct pci_dev *dev, int resno, 587 enum pci_bar_type *type) 588 { 589 if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END) 590 return 0; 591 592 BUG_ON(!dev->is_physfn); 593 594 *type = pci_bar_unknown; 595 596 return dev->sriov->pos + PCI_SRIOV_BAR + 597 4 * (resno - PCI_IOV_RESOURCES); 598 } 599 600 /** 601 * pci_restore_iov_state - restore the state of the IOV capability 602 * @dev: the PCI device 603 */ 604 void pci_restore_iov_state(struct pci_dev *dev) 605 { 606 if (dev->is_physfn) 607 sriov_restore_state(dev); 608 } 609 610 /** 611 * pci_iov_bus_range - find bus range used by Virtual Function 612 * @bus: the PCI bus 613 * 614 * Returns max number of buses (exclude current one) used by Virtual 615 * Functions. 616 */ 617 int pci_iov_bus_range(struct pci_bus *bus) 618 { 619 int max = 0; 620 u8 busnr; 621 struct pci_dev *dev; 622 623 list_for_each_entry(dev, &bus->devices, bus_list) { 624 if (!dev->is_physfn) 625 continue; 626 busnr = virtfn_bus(dev, dev->sriov->total - 1); 627 if (busnr > max) 628 max = busnr; 629 } 630 631 return max ? max - bus->number : 0; 632 } 633 634 /** 635 * pci_enable_sriov - enable the SR-IOV capability 636 * @dev: the PCI device 637 * @nr_virtfn: number of virtual functions to enable 638 * 639 * Returns 0 on success, or negative on failure. 640 */ 641 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 642 { 643 might_sleep(); 644 645 if (!dev->is_physfn) 646 return -ENODEV; 647 648 return sriov_enable(dev, nr_virtfn); 649 } 650 EXPORT_SYMBOL_GPL(pci_enable_sriov); 651 652 /** 653 * pci_disable_sriov - disable the SR-IOV capability 654 * @dev: the PCI device 655 */ 656 void pci_disable_sriov(struct pci_dev *dev) 657 { 658 might_sleep(); 659 660 if (!dev->is_physfn) 661 return; 662 663 sriov_disable(dev); 664 } 665 EXPORT_SYMBOL_GPL(pci_disable_sriov); 666 667 /** 668 * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration 669 * @dev: the PCI device 670 * 671 * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not. 672 * 673 * Physical Function driver is responsible to register IRQ handler using 674 * VF Migration Interrupt Message Number, and call this function when the 675 * interrupt is generated by the hardware. 676 */ 677 irqreturn_t pci_sriov_migration(struct pci_dev *dev) 678 { 679 if (!dev->is_physfn) 680 return IRQ_NONE; 681 682 return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE; 683 } 684 EXPORT_SYMBOL_GPL(pci_sriov_migration); 685 686 static int ats_alloc_one(struct pci_dev *dev, int ps) 687 { 688 int pos; 689 u16 cap; 690 struct pci_ats *ats; 691 692 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 693 if (!pos) 694 return -ENODEV; 695 696 ats = kzalloc(sizeof(*ats), GFP_KERNEL); 697 if (!ats) 698 return -ENOMEM; 699 700 ats->pos = pos; 701 ats->stu = ps; 702 pci_read_config_word(dev, pos + PCI_ATS_CAP, &cap); 703 ats->qdep = PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : 704 PCI_ATS_MAX_QDEP; 705 dev->ats = ats; 706 707 return 0; 708 } 709 710 static void ats_free_one(struct pci_dev *dev) 711 { 712 kfree(dev->ats); 713 dev->ats = NULL; 714 } 715 716 /** 717 * pci_enable_ats - enable the ATS capability 718 * @dev: the PCI device 719 * @ps: the IOMMU page shift 720 * 721 * Returns 0 on success, or negative on failure. 722 */ 723 int pci_enable_ats(struct pci_dev *dev, int ps) 724 { 725 int rc; 726 u16 ctrl; 727 728 BUG_ON(dev->ats && dev->ats->is_enabled); 729 730 if (ps < PCI_ATS_MIN_STU) 731 return -EINVAL; 732 733 if (dev->is_physfn || dev->is_virtfn) { 734 struct pci_dev *pdev = dev->is_physfn ? dev : dev->physfn; 735 736 mutex_lock(&pdev->sriov->lock); 737 if (pdev->ats) 738 rc = pdev->ats->stu == ps ? 0 : -EINVAL; 739 else 740 rc = ats_alloc_one(pdev, ps); 741 742 if (!rc) 743 pdev->ats->ref_cnt++; 744 mutex_unlock(&pdev->sriov->lock); 745 if (rc) 746 return rc; 747 } 748 749 if (!dev->is_physfn) { 750 rc = ats_alloc_one(dev, ps); 751 if (rc) 752 return rc; 753 } 754 755 ctrl = PCI_ATS_CTRL_ENABLE; 756 if (!dev->is_virtfn) 757 ctrl |= PCI_ATS_CTRL_STU(ps - PCI_ATS_MIN_STU); 758 pci_write_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, ctrl); 759 760 dev->ats->is_enabled = 1; 761 762 return 0; 763 } 764 765 /** 766 * pci_disable_ats - disable the ATS capability 767 * @dev: the PCI device 768 */ 769 void pci_disable_ats(struct pci_dev *dev) 770 { 771 u16 ctrl; 772 773 BUG_ON(!dev->ats || !dev->ats->is_enabled); 774 775 pci_read_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, &ctrl); 776 ctrl &= ~PCI_ATS_CTRL_ENABLE; 777 pci_write_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, ctrl); 778 779 dev->ats->is_enabled = 0; 780 781 if (dev->is_physfn || dev->is_virtfn) { 782 struct pci_dev *pdev = dev->is_physfn ? dev : dev->physfn; 783 784 mutex_lock(&pdev->sriov->lock); 785 pdev->ats->ref_cnt--; 786 if (!pdev->ats->ref_cnt) 787 ats_free_one(pdev); 788 mutex_unlock(&pdev->sriov->lock); 789 } 790 791 if (!dev->is_physfn) 792 ats_free_one(dev); 793 } 794 795 /** 796 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 797 * @dev: the PCI device 798 * 799 * Returns the queue depth on success, or negative on failure. 800 * 801 * The ATS spec uses 0 in the Invalidate Queue Depth field to 802 * indicate that the function can accept 32 Invalidate Request. 803 * But here we use the `real' values (i.e. 1~32) for the Queue 804 * Depth; and 0 indicates the function shares the Queue with 805 * other functions (doesn't exclusively own a Queue). 806 */ 807 int pci_ats_queue_depth(struct pci_dev *dev) 808 { 809 int pos; 810 u16 cap; 811 812 if (dev->is_virtfn) 813 return 0; 814 815 if (dev->ats) 816 return dev->ats->qdep; 817 818 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 819 if (!pos) 820 return -ENODEV; 821 822 pci_read_config_word(dev, pos + PCI_ATS_CAP, &cap); 823 824 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : 825 PCI_ATS_MAX_QDEP; 826 } 827