1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pci.h> 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include <linux/ioport.h> 6 #include <linux/wait.h> 7 8 #include "pci.h" 9 10 /* 11 * This interrupt-safe spinlock protects all accesses to PCI 12 * configuration space. 13 */ 14 15 DEFINE_RAW_SPINLOCK(pci_lock); 16 17 /* 18 * Wrappers for all PCI configuration access functions. They just check 19 * alignment, do locking and call the low-level functions pointed to 20 * by pci_dev->ops. 21 */ 22 23 #define PCI_byte_BAD 0 24 #define PCI_word_BAD (pos & 1) 25 #define PCI_dword_BAD (pos & 3) 26 27 #ifdef CONFIG_PCI_LOCKLESS_CONFIG 28 # define pci_lock_config(f) do { (void)(f); } while (0) 29 # define pci_unlock_config(f) do { (void)(f); } while (0) 30 #else 31 # define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f) 32 # define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f) 33 #endif 34 35 #define PCI_OP_READ(size, type, len) \ 36 int noinline pci_bus_read_config_##size \ 37 (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ 38 { \ 39 int res; \ 40 unsigned long flags; \ 41 u32 data = 0; \ 42 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 43 pci_lock_config(flags); \ 44 res = bus->ops->read(bus, devfn, pos, len, &data); \ 45 if (res) \ 46 PCI_SET_ERROR_RESPONSE(value); \ 47 else \ 48 *value = (type)data; \ 49 pci_unlock_config(flags); \ 50 return res; \ 51 } 52 53 #define PCI_OP_WRITE(size, type, len) \ 54 int noinline pci_bus_write_config_##size \ 55 (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ 56 { \ 57 int res; \ 58 unsigned long flags; \ 59 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 60 pci_lock_config(flags); \ 61 res = bus->ops->write(bus, devfn, pos, len, value); \ 62 pci_unlock_config(flags); \ 63 return res; \ 64 } 65 66 PCI_OP_READ(byte, u8, 1) 67 PCI_OP_READ(word, u16, 2) 68 PCI_OP_READ(dword, u32, 4) 69 PCI_OP_WRITE(byte, u8, 1) 70 PCI_OP_WRITE(word, u16, 2) 71 PCI_OP_WRITE(dword, u32, 4) 72 73 EXPORT_SYMBOL(pci_bus_read_config_byte); 74 EXPORT_SYMBOL(pci_bus_read_config_word); 75 EXPORT_SYMBOL(pci_bus_read_config_dword); 76 EXPORT_SYMBOL(pci_bus_write_config_byte); 77 EXPORT_SYMBOL(pci_bus_write_config_word); 78 EXPORT_SYMBOL(pci_bus_write_config_dword); 79 80 int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, 81 int where, int size, u32 *val) 82 { 83 void __iomem *addr; 84 85 addr = bus->ops->map_bus(bus, devfn, where); 86 if (!addr) 87 return PCIBIOS_DEVICE_NOT_FOUND; 88 89 if (size == 1) 90 *val = readb(addr); 91 else if (size == 2) 92 *val = readw(addr); 93 else 94 *val = readl(addr); 95 96 return PCIBIOS_SUCCESSFUL; 97 } 98 EXPORT_SYMBOL_GPL(pci_generic_config_read); 99 100 int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, 101 int where, int size, u32 val) 102 { 103 void __iomem *addr; 104 105 addr = bus->ops->map_bus(bus, devfn, where); 106 if (!addr) 107 return PCIBIOS_DEVICE_NOT_FOUND; 108 109 if (size == 1) 110 writeb(val, addr); 111 else if (size == 2) 112 writew(val, addr); 113 else 114 writel(val, addr); 115 116 return PCIBIOS_SUCCESSFUL; 117 } 118 EXPORT_SYMBOL_GPL(pci_generic_config_write); 119 120 int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, 121 int where, int size, u32 *val) 122 { 123 void __iomem *addr; 124 125 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 126 if (!addr) 127 return PCIBIOS_DEVICE_NOT_FOUND; 128 129 *val = readl(addr); 130 131 if (size <= 2) 132 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 133 134 return PCIBIOS_SUCCESSFUL; 135 } 136 EXPORT_SYMBOL_GPL(pci_generic_config_read32); 137 138 int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, 139 int where, int size, u32 val) 140 { 141 void __iomem *addr; 142 u32 mask, tmp; 143 144 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 145 if (!addr) 146 return PCIBIOS_DEVICE_NOT_FOUND; 147 148 if (size == 4) { 149 writel(val, addr); 150 return PCIBIOS_SUCCESSFUL; 151 } 152 153 /* 154 * In general, hardware that supports only 32-bit writes on PCI is 155 * not spec-compliant. For example, software may perform a 16-bit 156 * write. If the hardware only supports 32-bit accesses, we must 157 * do a 32-bit read, merge in the 16 bits we intend to write, 158 * followed by a 32-bit write. If the 16 bits we *don't* intend to 159 * write happen to have any RW1C (write-one-to-clear) bits set, we 160 * just inadvertently cleared something we shouldn't have. 161 */ 162 dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n", 163 size, pci_domain_nr(bus), bus->number, 164 PCI_SLOT(devfn), PCI_FUNC(devfn), where); 165 166 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 167 tmp = readl(addr) & mask; 168 tmp |= val << ((where & 0x3) * 8); 169 writel(tmp, addr); 170 171 return PCIBIOS_SUCCESSFUL; 172 } 173 EXPORT_SYMBOL_GPL(pci_generic_config_write32); 174 175 /** 176 * pci_bus_set_ops - Set raw operations of pci bus 177 * @bus: pci bus struct 178 * @ops: new raw operations 179 * 180 * Return previous raw operations 181 */ 182 struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) 183 { 184 struct pci_ops *old_ops; 185 unsigned long flags; 186 187 raw_spin_lock_irqsave(&pci_lock, flags); 188 old_ops = bus->ops; 189 bus->ops = ops; 190 raw_spin_unlock_irqrestore(&pci_lock, flags); 191 return old_ops; 192 } 193 EXPORT_SYMBOL(pci_bus_set_ops); 194 195 /* 196 * The following routines are to prevent the user from accessing PCI config 197 * space when it's unsafe to do so. Some devices require this during BIST and 198 * we're required to prevent it during D-state transitions. 199 * 200 * We have a bit per device to indicate it's blocked and a global wait queue 201 * for callers to sleep on until devices are unblocked. 202 */ 203 static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); 204 205 static noinline void pci_wait_cfg(struct pci_dev *dev) 206 __must_hold(&pci_lock) 207 { 208 do { 209 raw_spin_unlock_irq(&pci_lock); 210 wait_event(pci_cfg_wait, !dev->block_cfg_access); 211 raw_spin_lock_irq(&pci_lock); 212 } while (dev->block_cfg_access); 213 } 214 215 /* Returns 0 on success, negative values indicate error. */ 216 #define PCI_USER_READ_CONFIG(size, type) \ 217 int pci_user_read_config_##size \ 218 (struct pci_dev *dev, int pos, type *val) \ 219 { \ 220 int ret = PCIBIOS_SUCCESSFUL; \ 221 u32 data = -1; \ 222 if (PCI_##size##_BAD) \ 223 return -EINVAL; \ 224 raw_spin_lock_irq(&pci_lock); \ 225 if (unlikely(dev->block_cfg_access)) \ 226 pci_wait_cfg(dev); \ 227 ret = dev->bus->ops->read(dev->bus, dev->devfn, \ 228 pos, sizeof(type), &data); \ 229 raw_spin_unlock_irq(&pci_lock); \ 230 if (ret) \ 231 PCI_SET_ERROR_RESPONSE(val); \ 232 else \ 233 *val = (type)data; \ 234 return pcibios_err_to_errno(ret); \ 235 } \ 236 EXPORT_SYMBOL_GPL(pci_user_read_config_##size); 237 238 /* Returns 0 on success, negative values indicate error. */ 239 #define PCI_USER_WRITE_CONFIG(size, type) \ 240 int pci_user_write_config_##size \ 241 (struct pci_dev *dev, int pos, type val) \ 242 { \ 243 int ret = PCIBIOS_SUCCESSFUL; \ 244 if (PCI_##size##_BAD) \ 245 return -EINVAL; \ 246 raw_spin_lock_irq(&pci_lock); \ 247 if (unlikely(dev->block_cfg_access)) \ 248 pci_wait_cfg(dev); \ 249 ret = dev->bus->ops->write(dev->bus, dev->devfn, \ 250 pos, sizeof(type), val); \ 251 raw_spin_unlock_irq(&pci_lock); \ 252 return pcibios_err_to_errno(ret); \ 253 } \ 254 EXPORT_SYMBOL_GPL(pci_user_write_config_##size); 255 256 PCI_USER_READ_CONFIG(byte, u8) 257 PCI_USER_READ_CONFIG(word, u16) 258 PCI_USER_READ_CONFIG(dword, u32) 259 PCI_USER_WRITE_CONFIG(byte, u8) 260 PCI_USER_WRITE_CONFIG(word, u16) 261 PCI_USER_WRITE_CONFIG(dword, u32) 262 263 /** 264 * pci_cfg_access_lock - Lock PCI config reads/writes 265 * @dev: pci device struct 266 * 267 * When access is locked, any userspace reads or writes to config 268 * space and concurrent lock requests will sleep until access is 269 * allowed via pci_cfg_access_unlock() again. 270 */ 271 void pci_cfg_access_lock(struct pci_dev *dev) 272 { 273 might_sleep(); 274 275 raw_spin_lock_irq(&pci_lock); 276 if (dev->block_cfg_access) 277 pci_wait_cfg(dev); 278 dev->block_cfg_access = 1; 279 raw_spin_unlock_irq(&pci_lock); 280 } 281 EXPORT_SYMBOL_GPL(pci_cfg_access_lock); 282 283 /** 284 * pci_cfg_access_trylock - try to lock PCI config reads/writes 285 * @dev: pci device struct 286 * 287 * Same as pci_cfg_access_lock, but will return 0 if access is 288 * already locked, 1 otherwise. This function can be used from 289 * atomic contexts. 290 */ 291 bool pci_cfg_access_trylock(struct pci_dev *dev) 292 { 293 unsigned long flags; 294 bool locked = true; 295 296 raw_spin_lock_irqsave(&pci_lock, flags); 297 if (dev->block_cfg_access) 298 locked = false; 299 else 300 dev->block_cfg_access = 1; 301 raw_spin_unlock_irqrestore(&pci_lock, flags); 302 303 return locked; 304 } 305 EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); 306 307 /** 308 * pci_cfg_access_unlock - Unlock PCI config reads/writes 309 * @dev: pci device struct 310 * 311 * This function allows PCI config accesses to resume. 312 */ 313 void pci_cfg_access_unlock(struct pci_dev *dev) 314 { 315 unsigned long flags; 316 317 raw_spin_lock_irqsave(&pci_lock, flags); 318 319 /* 320 * This indicates a problem in the caller, but we don't need 321 * to kill them, unlike a double-block above. 322 */ 323 WARN_ON(!dev->block_cfg_access); 324 325 dev->block_cfg_access = 0; 326 raw_spin_unlock_irqrestore(&pci_lock, flags); 327 328 wake_up_all(&pci_cfg_wait); 329 } 330 EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); 331 332 static inline int pcie_cap_version(const struct pci_dev *dev) 333 { 334 return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; 335 } 336 337 bool pcie_cap_has_lnkctl(const struct pci_dev *dev) 338 { 339 int type = pci_pcie_type(dev); 340 341 return type == PCI_EXP_TYPE_ENDPOINT || 342 type == PCI_EXP_TYPE_LEG_END || 343 type == PCI_EXP_TYPE_ROOT_PORT || 344 type == PCI_EXP_TYPE_UPSTREAM || 345 type == PCI_EXP_TYPE_DOWNSTREAM || 346 type == PCI_EXP_TYPE_PCI_BRIDGE || 347 type == PCI_EXP_TYPE_PCIE_BRIDGE; 348 } 349 350 static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) 351 { 352 return pcie_downstream_port(dev) && 353 pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; 354 } 355 356 bool pcie_cap_has_rtctl(const struct pci_dev *dev) 357 { 358 int type = pci_pcie_type(dev); 359 360 return type == PCI_EXP_TYPE_ROOT_PORT || 361 type == PCI_EXP_TYPE_RC_EC; 362 } 363 364 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) 365 { 366 if (!pci_is_pcie(dev)) 367 return false; 368 369 switch (pos) { 370 case PCI_EXP_FLAGS: 371 return true; 372 case PCI_EXP_DEVCAP: 373 case PCI_EXP_DEVCTL: 374 case PCI_EXP_DEVSTA: 375 return true; 376 case PCI_EXP_LNKCAP: 377 case PCI_EXP_LNKCTL: 378 case PCI_EXP_LNKSTA: 379 return pcie_cap_has_lnkctl(dev); 380 case PCI_EXP_SLTCAP: 381 case PCI_EXP_SLTCTL: 382 case PCI_EXP_SLTSTA: 383 return pcie_cap_has_sltctl(dev); 384 case PCI_EXP_RTCTL: 385 case PCI_EXP_RTCAP: 386 case PCI_EXP_RTSTA: 387 return pcie_cap_has_rtctl(dev); 388 case PCI_EXP_DEVCAP2: 389 case PCI_EXP_DEVCTL2: 390 case PCI_EXP_LNKCAP2: 391 case PCI_EXP_LNKCTL2: 392 case PCI_EXP_LNKSTA2: 393 return pcie_cap_version(dev) > 1; 394 default: 395 return false; 396 } 397 } 398 399 /* 400 * Note that these accessor functions are only for the "PCI Express 401 * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the 402 * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) 403 */ 404 int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) 405 { 406 int ret; 407 408 *val = 0; 409 if (pos & 1) 410 return PCIBIOS_BAD_REGISTER_NUMBER; 411 412 if (pcie_capability_reg_implemented(dev, pos)) { 413 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); 414 /* 415 * Reset *val to 0 if pci_read_config_word() fails; it may 416 * have been written as 0xFFFF (PCI_ERROR_RESPONSE) if the 417 * config read failed on PCI. 418 */ 419 if (ret) 420 *val = 0; 421 return ret; 422 } 423 424 /* 425 * For Functions that do not implement the Slot Capabilities, 426 * Slot Status, and Slot Control registers, these spaces must 427 * be hardwired to 0b, with the exception of the Presence Detect 428 * State bit in the Slot Status register of Downstream Ports, 429 * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) 430 */ 431 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 432 pos == PCI_EXP_SLTSTA) 433 *val = PCI_EXP_SLTSTA_PDS; 434 435 return 0; 436 } 437 EXPORT_SYMBOL(pcie_capability_read_word); 438 439 int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) 440 { 441 int ret; 442 443 *val = 0; 444 if (pos & 3) 445 return PCIBIOS_BAD_REGISTER_NUMBER; 446 447 if (pcie_capability_reg_implemented(dev, pos)) { 448 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); 449 /* 450 * Reset *val to 0 if pci_read_config_dword() fails; it may 451 * have been written as 0xFFFFFFFF (PCI_ERROR_RESPONSE) if 452 * the config read failed on PCI. 453 */ 454 if (ret) 455 *val = 0; 456 return ret; 457 } 458 459 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 460 pos == PCI_EXP_SLTSTA) 461 *val = PCI_EXP_SLTSTA_PDS; 462 463 return 0; 464 } 465 EXPORT_SYMBOL(pcie_capability_read_dword); 466 467 int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) 468 { 469 if (pos & 1) 470 return PCIBIOS_BAD_REGISTER_NUMBER; 471 472 if (!pcie_capability_reg_implemented(dev, pos)) 473 return 0; 474 475 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); 476 } 477 EXPORT_SYMBOL(pcie_capability_write_word); 478 479 int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) 480 { 481 if (pos & 3) 482 return PCIBIOS_BAD_REGISTER_NUMBER; 483 484 if (!pcie_capability_reg_implemented(dev, pos)) 485 return 0; 486 487 return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); 488 } 489 EXPORT_SYMBOL(pcie_capability_write_dword); 490 491 int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, 492 u16 clear, u16 set) 493 { 494 int ret; 495 u16 val; 496 497 ret = pcie_capability_read_word(dev, pos, &val); 498 if (!ret) { 499 val &= ~clear; 500 val |= set; 501 ret = pcie_capability_write_word(dev, pos, val); 502 } 503 504 return ret; 505 } 506 EXPORT_SYMBOL(pcie_capability_clear_and_set_word); 507 508 int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, 509 u32 clear, u32 set) 510 { 511 int ret; 512 u32 val; 513 514 ret = pcie_capability_read_dword(dev, pos, &val); 515 if (!ret) { 516 val &= ~clear; 517 val |= set; 518 ret = pcie_capability_write_dword(dev, pos, val); 519 } 520 521 return ret; 522 } 523 EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); 524 525 int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val) 526 { 527 if (pci_dev_is_disconnected(dev)) { 528 PCI_SET_ERROR_RESPONSE(val); 529 return PCIBIOS_DEVICE_NOT_FOUND; 530 } 531 return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val); 532 } 533 EXPORT_SYMBOL(pci_read_config_byte); 534 535 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val) 536 { 537 if (pci_dev_is_disconnected(dev)) { 538 PCI_SET_ERROR_RESPONSE(val); 539 return PCIBIOS_DEVICE_NOT_FOUND; 540 } 541 return pci_bus_read_config_word(dev->bus, dev->devfn, where, val); 542 } 543 EXPORT_SYMBOL(pci_read_config_word); 544 545 int pci_read_config_dword(const struct pci_dev *dev, int where, 546 u32 *val) 547 { 548 if (pci_dev_is_disconnected(dev)) { 549 PCI_SET_ERROR_RESPONSE(val); 550 return PCIBIOS_DEVICE_NOT_FOUND; 551 } 552 return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val); 553 } 554 EXPORT_SYMBOL(pci_read_config_dword); 555 556 int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val) 557 { 558 if (pci_dev_is_disconnected(dev)) 559 return PCIBIOS_DEVICE_NOT_FOUND; 560 return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val); 561 } 562 EXPORT_SYMBOL(pci_write_config_byte); 563 564 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val) 565 { 566 if (pci_dev_is_disconnected(dev)) 567 return PCIBIOS_DEVICE_NOT_FOUND; 568 return pci_bus_write_config_word(dev->bus, dev->devfn, where, val); 569 } 570 EXPORT_SYMBOL(pci_write_config_word); 571 572 int pci_write_config_dword(const struct pci_dev *dev, int where, 573 u32 val) 574 { 575 if (pci_dev_is_disconnected(dev)) 576 return PCIBIOS_DEVICE_NOT_FOUND; 577 return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val); 578 } 579 EXPORT_SYMBOL(pci_write_config_dword); 580