1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pci.h> 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include <linux/ioport.h> 6 #include <linux/wait.h> 7 8 #include "pci.h" 9 10 /* 11 * This interrupt-safe spinlock protects all accesses to PCI 12 * configuration space. 13 */ 14 15 DEFINE_RAW_SPINLOCK(pci_lock); 16 17 /* 18 * Wrappers for all PCI configuration access functions. They just check 19 * alignment, do locking and call the low-level functions pointed to 20 * by pci_dev->ops. 21 */ 22 23 #define PCI_byte_BAD 0 24 #define PCI_word_BAD (pos & 1) 25 #define PCI_dword_BAD (pos & 3) 26 27 #ifdef CONFIG_PCI_LOCKLESS_CONFIG 28 # define pci_lock_config(f) do { (void)(f); } while (0) 29 # define pci_unlock_config(f) do { (void)(f); } while (0) 30 #else 31 # define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f) 32 # define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f) 33 #endif 34 35 #define PCI_OP_READ(size, type, len) \ 36 int noinline pci_bus_read_config_##size \ 37 (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ 38 { \ 39 int res; \ 40 unsigned long flags; \ 41 u32 data = 0; \ 42 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 43 pci_lock_config(flags); \ 44 res = bus->ops->read(bus, devfn, pos, len, &data); \ 45 *value = (type)data; \ 46 pci_unlock_config(flags); \ 47 return res; \ 48 } 49 50 #define PCI_OP_WRITE(size, type, len) \ 51 int noinline pci_bus_write_config_##size \ 52 (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ 53 { \ 54 int res; \ 55 unsigned long flags; \ 56 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 57 pci_lock_config(flags); \ 58 res = bus->ops->write(bus, devfn, pos, len, value); \ 59 pci_unlock_config(flags); \ 60 return res; \ 61 } 62 63 PCI_OP_READ(byte, u8, 1) 64 PCI_OP_READ(word, u16, 2) 65 PCI_OP_READ(dword, u32, 4) 66 PCI_OP_WRITE(byte, u8, 1) 67 PCI_OP_WRITE(word, u16, 2) 68 PCI_OP_WRITE(dword, u32, 4) 69 70 EXPORT_SYMBOL(pci_bus_read_config_byte); 71 EXPORT_SYMBOL(pci_bus_read_config_word); 72 EXPORT_SYMBOL(pci_bus_read_config_dword); 73 EXPORT_SYMBOL(pci_bus_write_config_byte); 74 EXPORT_SYMBOL(pci_bus_write_config_word); 75 EXPORT_SYMBOL(pci_bus_write_config_dword); 76 77 int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, 78 int where, int size, u32 *val) 79 { 80 void __iomem *addr; 81 82 addr = bus->ops->map_bus(bus, devfn, where); 83 if (!addr) { 84 *val = ~0; 85 return PCIBIOS_DEVICE_NOT_FOUND; 86 } 87 88 if (size == 1) 89 *val = readb(addr); 90 else if (size == 2) 91 *val = readw(addr); 92 else 93 *val = readl(addr); 94 95 return PCIBIOS_SUCCESSFUL; 96 } 97 EXPORT_SYMBOL_GPL(pci_generic_config_read); 98 99 int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, 100 int where, int size, u32 val) 101 { 102 void __iomem *addr; 103 104 addr = bus->ops->map_bus(bus, devfn, where); 105 if (!addr) 106 return PCIBIOS_DEVICE_NOT_FOUND; 107 108 if (size == 1) 109 writeb(val, addr); 110 else if (size == 2) 111 writew(val, addr); 112 else 113 writel(val, addr); 114 115 return PCIBIOS_SUCCESSFUL; 116 } 117 EXPORT_SYMBOL_GPL(pci_generic_config_write); 118 119 int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, 120 int where, int size, u32 *val) 121 { 122 void __iomem *addr; 123 124 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 125 if (!addr) { 126 *val = ~0; 127 return PCIBIOS_DEVICE_NOT_FOUND; 128 } 129 130 *val = readl(addr); 131 132 if (size <= 2) 133 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 134 135 return PCIBIOS_SUCCESSFUL; 136 } 137 EXPORT_SYMBOL_GPL(pci_generic_config_read32); 138 139 int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, 140 int where, int size, u32 val) 141 { 142 void __iomem *addr; 143 u32 mask, tmp; 144 145 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 146 if (!addr) 147 return PCIBIOS_DEVICE_NOT_FOUND; 148 149 if (size == 4) { 150 writel(val, addr); 151 return PCIBIOS_SUCCESSFUL; 152 } 153 154 /* 155 * In general, hardware that supports only 32-bit writes on PCI is 156 * not spec-compliant. For example, software may perform a 16-bit 157 * write. If the hardware only supports 32-bit accesses, we must 158 * do a 32-bit read, merge in the 16 bits we intend to write, 159 * followed by a 32-bit write. If the 16 bits we *don't* intend to 160 * write happen to have any RW1C (write-one-to-clear) bits set, we 161 * just inadvertently cleared something we shouldn't have. 162 */ 163 dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n", 164 size, pci_domain_nr(bus), bus->number, 165 PCI_SLOT(devfn), PCI_FUNC(devfn), where); 166 167 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 168 tmp = readl(addr) & mask; 169 tmp |= val << ((where & 0x3) * 8); 170 writel(tmp, addr); 171 172 return PCIBIOS_SUCCESSFUL; 173 } 174 EXPORT_SYMBOL_GPL(pci_generic_config_write32); 175 176 /** 177 * pci_bus_set_ops - Set raw operations of pci bus 178 * @bus: pci bus struct 179 * @ops: new raw operations 180 * 181 * Return previous raw operations 182 */ 183 struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) 184 { 185 struct pci_ops *old_ops; 186 unsigned long flags; 187 188 raw_spin_lock_irqsave(&pci_lock, flags); 189 old_ops = bus->ops; 190 bus->ops = ops; 191 raw_spin_unlock_irqrestore(&pci_lock, flags); 192 return old_ops; 193 } 194 EXPORT_SYMBOL(pci_bus_set_ops); 195 196 /* 197 * The following routines are to prevent the user from accessing PCI config 198 * space when it's unsafe to do so. Some devices require this during BIST and 199 * we're required to prevent it during D-state transitions. 200 * 201 * We have a bit per device to indicate it's blocked and a global wait queue 202 * for callers to sleep on until devices are unblocked. 203 */ 204 static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); 205 206 static noinline void pci_wait_cfg(struct pci_dev *dev) 207 { 208 DECLARE_WAITQUEUE(wait, current); 209 210 __add_wait_queue(&pci_cfg_wait, &wait); 211 do { 212 set_current_state(TASK_UNINTERRUPTIBLE); 213 raw_spin_unlock_irq(&pci_lock); 214 schedule(); 215 raw_spin_lock_irq(&pci_lock); 216 } while (dev->block_cfg_access); 217 __remove_wait_queue(&pci_cfg_wait, &wait); 218 } 219 220 /* Returns 0 on success, negative values indicate error. */ 221 #define PCI_USER_READ_CONFIG(size, type) \ 222 int pci_user_read_config_##size \ 223 (struct pci_dev *dev, int pos, type *val) \ 224 { \ 225 int ret = PCIBIOS_SUCCESSFUL; \ 226 u32 data = -1; \ 227 if (PCI_##size##_BAD) \ 228 return -EINVAL; \ 229 raw_spin_lock_irq(&pci_lock); \ 230 if (unlikely(dev->block_cfg_access)) \ 231 pci_wait_cfg(dev); \ 232 ret = dev->bus->ops->read(dev->bus, dev->devfn, \ 233 pos, sizeof(type), &data); \ 234 raw_spin_unlock_irq(&pci_lock); \ 235 *val = (type)data; \ 236 return pcibios_err_to_errno(ret); \ 237 } \ 238 EXPORT_SYMBOL_GPL(pci_user_read_config_##size); 239 240 /* Returns 0 on success, negative values indicate error. */ 241 #define PCI_USER_WRITE_CONFIG(size, type) \ 242 int pci_user_write_config_##size \ 243 (struct pci_dev *dev, int pos, type val) \ 244 { \ 245 int ret = PCIBIOS_SUCCESSFUL; \ 246 if (PCI_##size##_BAD) \ 247 return -EINVAL; \ 248 raw_spin_lock_irq(&pci_lock); \ 249 if (unlikely(dev->block_cfg_access)) \ 250 pci_wait_cfg(dev); \ 251 ret = dev->bus->ops->write(dev->bus, dev->devfn, \ 252 pos, sizeof(type), val); \ 253 raw_spin_unlock_irq(&pci_lock); \ 254 return pcibios_err_to_errno(ret); \ 255 } \ 256 EXPORT_SYMBOL_GPL(pci_user_write_config_##size); 257 258 PCI_USER_READ_CONFIG(byte, u8) 259 PCI_USER_READ_CONFIG(word, u16) 260 PCI_USER_READ_CONFIG(dword, u32) 261 PCI_USER_WRITE_CONFIG(byte, u8) 262 PCI_USER_WRITE_CONFIG(word, u16) 263 PCI_USER_WRITE_CONFIG(dword, u32) 264 265 /** 266 * pci_cfg_access_lock - Lock PCI config reads/writes 267 * @dev: pci device struct 268 * 269 * When access is locked, any userspace reads or writes to config 270 * space and concurrent lock requests will sleep until access is 271 * allowed via pci_cfg_access_unlock() again. 272 */ 273 void pci_cfg_access_lock(struct pci_dev *dev) 274 { 275 might_sleep(); 276 277 raw_spin_lock_irq(&pci_lock); 278 if (dev->block_cfg_access) 279 pci_wait_cfg(dev); 280 dev->block_cfg_access = 1; 281 raw_spin_unlock_irq(&pci_lock); 282 } 283 EXPORT_SYMBOL_GPL(pci_cfg_access_lock); 284 285 /** 286 * pci_cfg_access_trylock - try to lock PCI config reads/writes 287 * @dev: pci device struct 288 * 289 * Same as pci_cfg_access_lock, but will return 0 if access is 290 * already locked, 1 otherwise. This function can be used from 291 * atomic contexts. 292 */ 293 bool pci_cfg_access_trylock(struct pci_dev *dev) 294 { 295 unsigned long flags; 296 bool locked = true; 297 298 raw_spin_lock_irqsave(&pci_lock, flags); 299 if (dev->block_cfg_access) 300 locked = false; 301 else 302 dev->block_cfg_access = 1; 303 raw_spin_unlock_irqrestore(&pci_lock, flags); 304 305 return locked; 306 } 307 EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); 308 309 /** 310 * pci_cfg_access_unlock - Unlock PCI config reads/writes 311 * @dev: pci device struct 312 * 313 * This function allows PCI config accesses to resume. 314 */ 315 void pci_cfg_access_unlock(struct pci_dev *dev) 316 { 317 unsigned long flags; 318 319 raw_spin_lock_irqsave(&pci_lock, flags); 320 321 /* 322 * This indicates a problem in the caller, but we don't need 323 * to kill them, unlike a double-block above. 324 */ 325 WARN_ON(!dev->block_cfg_access); 326 327 dev->block_cfg_access = 0; 328 raw_spin_unlock_irqrestore(&pci_lock, flags); 329 330 wake_up_all(&pci_cfg_wait); 331 } 332 EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); 333 334 static inline int pcie_cap_version(const struct pci_dev *dev) 335 { 336 return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; 337 } 338 339 static bool pcie_downstream_port(const struct pci_dev *dev) 340 { 341 int type = pci_pcie_type(dev); 342 343 return type == PCI_EXP_TYPE_ROOT_PORT || 344 type == PCI_EXP_TYPE_DOWNSTREAM || 345 type == PCI_EXP_TYPE_PCIE_BRIDGE; 346 } 347 348 bool pcie_cap_has_lnkctl(const struct pci_dev *dev) 349 { 350 int type = pci_pcie_type(dev); 351 352 return type == PCI_EXP_TYPE_ENDPOINT || 353 type == PCI_EXP_TYPE_LEG_END || 354 type == PCI_EXP_TYPE_ROOT_PORT || 355 type == PCI_EXP_TYPE_UPSTREAM || 356 type == PCI_EXP_TYPE_DOWNSTREAM || 357 type == PCI_EXP_TYPE_PCI_BRIDGE || 358 type == PCI_EXP_TYPE_PCIE_BRIDGE; 359 } 360 361 static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) 362 { 363 return pcie_downstream_port(dev) && 364 pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; 365 } 366 367 static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev) 368 { 369 int type = pci_pcie_type(dev); 370 371 return type == PCI_EXP_TYPE_ROOT_PORT || 372 type == PCI_EXP_TYPE_RC_EC; 373 } 374 375 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) 376 { 377 if (!pci_is_pcie(dev)) 378 return false; 379 380 switch (pos) { 381 case PCI_EXP_FLAGS: 382 return true; 383 case PCI_EXP_DEVCAP: 384 case PCI_EXP_DEVCTL: 385 case PCI_EXP_DEVSTA: 386 return true; 387 case PCI_EXP_LNKCAP: 388 case PCI_EXP_LNKCTL: 389 case PCI_EXP_LNKSTA: 390 return pcie_cap_has_lnkctl(dev); 391 case PCI_EXP_SLTCAP: 392 case PCI_EXP_SLTCTL: 393 case PCI_EXP_SLTSTA: 394 return pcie_cap_has_sltctl(dev); 395 case PCI_EXP_RTCTL: 396 case PCI_EXP_RTCAP: 397 case PCI_EXP_RTSTA: 398 return pcie_cap_has_rtctl(dev); 399 case PCI_EXP_DEVCAP2: 400 case PCI_EXP_DEVCTL2: 401 case PCI_EXP_LNKCAP2: 402 case PCI_EXP_LNKCTL2: 403 case PCI_EXP_LNKSTA2: 404 return pcie_cap_version(dev) > 1; 405 default: 406 return false; 407 } 408 } 409 410 /* 411 * Note that these accessor functions are only for the "PCI Express 412 * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the 413 * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) 414 */ 415 int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) 416 { 417 int ret; 418 419 *val = 0; 420 if (pos & 1) 421 return -EINVAL; 422 423 if (pcie_capability_reg_implemented(dev, pos)) { 424 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); 425 /* 426 * Reset *val to 0 if pci_read_config_word() fails, it may 427 * have been written as 0xFFFF if hardware error happens 428 * during pci_read_config_word(). 429 */ 430 if (ret) 431 *val = 0; 432 return ret; 433 } 434 435 /* 436 * For Functions that do not implement the Slot Capabilities, 437 * Slot Status, and Slot Control registers, these spaces must 438 * be hardwired to 0b, with the exception of the Presence Detect 439 * State bit in the Slot Status register of Downstream Ports, 440 * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) 441 */ 442 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 443 pos == PCI_EXP_SLTSTA) 444 *val = PCI_EXP_SLTSTA_PDS; 445 446 return 0; 447 } 448 EXPORT_SYMBOL(pcie_capability_read_word); 449 450 int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) 451 { 452 int ret; 453 454 *val = 0; 455 if (pos & 3) 456 return -EINVAL; 457 458 if (pcie_capability_reg_implemented(dev, pos)) { 459 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); 460 /* 461 * Reset *val to 0 if pci_read_config_dword() fails, it may 462 * have been written as 0xFFFFFFFF if hardware error happens 463 * during pci_read_config_dword(). 464 */ 465 if (ret) 466 *val = 0; 467 return ret; 468 } 469 470 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 471 pos == PCI_EXP_SLTSTA) 472 *val = PCI_EXP_SLTSTA_PDS; 473 474 return 0; 475 } 476 EXPORT_SYMBOL(pcie_capability_read_dword); 477 478 int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) 479 { 480 if (pos & 1) 481 return -EINVAL; 482 483 if (!pcie_capability_reg_implemented(dev, pos)) 484 return 0; 485 486 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); 487 } 488 EXPORT_SYMBOL(pcie_capability_write_word); 489 490 int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) 491 { 492 if (pos & 3) 493 return -EINVAL; 494 495 if (!pcie_capability_reg_implemented(dev, pos)) 496 return 0; 497 498 return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); 499 } 500 EXPORT_SYMBOL(pcie_capability_write_dword); 501 502 int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, 503 u16 clear, u16 set) 504 { 505 int ret; 506 u16 val; 507 508 ret = pcie_capability_read_word(dev, pos, &val); 509 if (!ret) { 510 val &= ~clear; 511 val |= set; 512 ret = pcie_capability_write_word(dev, pos, val); 513 } 514 515 return ret; 516 } 517 EXPORT_SYMBOL(pcie_capability_clear_and_set_word); 518 519 int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, 520 u32 clear, u32 set) 521 { 522 int ret; 523 u32 val; 524 525 ret = pcie_capability_read_dword(dev, pos, &val); 526 if (!ret) { 527 val &= ~clear; 528 val |= set; 529 ret = pcie_capability_write_dword(dev, pos, val); 530 } 531 532 return ret; 533 } 534 EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); 535 536 int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val) 537 { 538 if (pci_dev_is_disconnected(dev)) { 539 *val = ~0; 540 return PCIBIOS_DEVICE_NOT_FOUND; 541 } 542 return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val); 543 } 544 EXPORT_SYMBOL(pci_read_config_byte); 545 546 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val) 547 { 548 if (pci_dev_is_disconnected(dev)) { 549 *val = ~0; 550 return PCIBIOS_DEVICE_NOT_FOUND; 551 } 552 return pci_bus_read_config_word(dev->bus, dev->devfn, where, val); 553 } 554 EXPORT_SYMBOL(pci_read_config_word); 555 556 int pci_read_config_dword(const struct pci_dev *dev, int where, 557 u32 *val) 558 { 559 if (pci_dev_is_disconnected(dev)) { 560 *val = ~0; 561 return PCIBIOS_DEVICE_NOT_FOUND; 562 } 563 return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val); 564 } 565 EXPORT_SYMBOL(pci_read_config_dword); 566 567 int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val) 568 { 569 if (pci_dev_is_disconnected(dev)) 570 return PCIBIOS_DEVICE_NOT_FOUND; 571 return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val); 572 } 573 EXPORT_SYMBOL(pci_write_config_byte); 574 575 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val) 576 { 577 if (pci_dev_is_disconnected(dev)) 578 return PCIBIOS_DEVICE_NOT_FOUND; 579 return pci_bus_write_config_word(dev->bus, dev->devfn, where, val); 580 } 581 EXPORT_SYMBOL(pci_write_config_word); 582 583 int pci_write_config_dword(const struct pci_dev *dev, int where, 584 u32 val) 585 { 586 if (pci_dev_is_disconnected(dev)) 587 return PCIBIOS_DEVICE_NOT_FOUND; 588 return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val); 589 } 590 EXPORT_SYMBOL(pci_write_config_dword); 591