1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_ats_supported - check if the device can use ATS 35 * @dev: the PCI device 36 * 37 * Returns true if the device supports ATS and is allowed to use it, false 38 * otherwise. 39 */ 40 bool pci_ats_supported(struct pci_dev *dev) 41 { 42 if (!dev->ats_cap) 43 return false; 44 45 return (dev->untrusted == 0); 46 } 47 EXPORT_SYMBOL_GPL(pci_ats_supported); 48 49 /** 50 * pci_enable_ats - enable the ATS capability 51 * @dev: the PCI device 52 * @ps: the IOMMU page shift 53 * 54 * Returns 0 on success, or negative on failure. 55 */ 56 int pci_enable_ats(struct pci_dev *dev, int ps) 57 { 58 u16 ctrl; 59 struct pci_dev *pdev; 60 61 if (!pci_ats_supported(dev)) 62 return -EINVAL; 63 64 if (WARN_ON(dev->ats_enabled)) 65 return -EBUSY; 66 67 if (ps < PCI_ATS_MIN_STU) 68 return -EINVAL; 69 70 /* 71 * Note that enabling ATS on a VF fails unless it's already enabled 72 * with the same STU on the PF. 73 */ 74 ctrl = PCI_ATS_CTRL_ENABLE; 75 if (dev->is_virtfn) { 76 pdev = pci_physfn(dev); 77 if (pdev->ats_stu != ps) 78 return -EINVAL; 79 } else { 80 dev->ats_stu = ps; 81 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 82 } 83 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 84 85 dev->ats_enabled = 1; 86 return 0; 87 } 88 EXPORT_SYMBOL_GPL(pci_enable_ats); 89 90 /** 91 * pci_disable_ats - disable the ATS capability 92 * @dev: the PCI device 93 */ 94 void pci_disable_ats(struct pci_dev *dev) 95 { 96 u16 ctrl; 97 98 if (WARN_ON(!dev->ats_enabled)) 99 return; 100 101 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 102 ctrl &= ~PCI_ATS_CTRL_ENABLE; 103 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 104 105 dev->ats_enabled = 0; 106 } 107 EXPORT_SYMBOL_GPL(pci_disable_ats); 108 109 void pci_restore_ats_state(struct pci_dev *dev) 110 { 111 u16 ctrl; 112 113 if (!dev->ats_enabled) 114 return; 115 116 ctrl = PCI_ATS_CTRL_ENABLE; 117 if (!dev->is_virtfn) 118 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 119 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 120 } 121 122 /** 123 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 124 * @dev: the PCI device 125 * 126 * Returns the queue depth on success, or negative on failure. 127 * 128 * The ATS spec uses 0 in the Invalidate Queue Depth field to 129 * indicate that the function can accept 32 Invalidate Request. 130 * But here we use the `real' values (i.e. 1~32) for the Queue 131 * Depth; and 0 indicates the function shares the Queue with 132 * other functions (doesn't exclusively own a Queue). 133 */ 134 int pci_ats_queue_depth(struct pci_dev *dev) 135 { 136 u16 cap; 137 138 if (!dev->ats_cap) 139 return -EINVAL; 140 141 if (dev->is_virtfn) 142 return 0; 143 144 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 145 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 146 } 147 148 /** 149 * pci_ats_page_aligned - Return Page Aligned Request bit status. 150 * @pdev: the PCI device 151 * 152 * Returns 1, if the Untranslated Addresses generated by the device 153 * are always aligned or 0 otherwise. 154 * 155 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 156 * is set, it indicates the Untranslated Addresses generated by the 157 * device are always aligned to a 4096 byte boundary. 158 */ 159 int pci_ats_page_aligned(struct pci_dev *pdev) 160 { 161 u16 cap; 162 163 if (!pdev->ats_cap) 164 return 0; 165 166 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 167 168 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 169 return 1; 170 171 return 0; 172 } 173 174 #ifdef CONFIG_PCI_PRI 175 void pci_pri_init(struct pci_dev *pdev) 176 { 177 u16 status; 178 179 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 180 181 if (!pdev->pri_cap) 182 return; 183 184 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 185 if (status & PCI_PRI_STATUS_PASID) 186 pdev->pasid_required = 1; 187 } 188 189 /** 190 * pci_enable_pri - Enable PRI capability 191 * @pdev: PCI device structure 192 * @reqs: outstanding requests 193 * 194 * Returns 0 on success, negative value on error 195 */ 196 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 197 { 198 u16 control, status; 199 u32 max_requests; 200 int pri = pdev->pri_cap; 201 202 /* 203 * VFs must not implement the PRI Capability. If their PF 204 * implements PRI, it is shared by the VFs, so if the PF PRI is 205 * enabled, it is also enabled for the VF. 206 */ 207 if (pdev->is_virtfn) { 208 if (pci_physfn(pdev)->pri_enabled) 209 return 0; 210 return -EINVAL; 211 } 212 213 if (WARN_ON(pdev->pri_enabled)) 214 return -EBUSY; 215 216 if (!pri) 217 return -EINVAL; 218 219 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 220 if (!(status & PCI_PRI_STATUS_STOPPED)) 221 return -EBUSY; 222 223 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 224 reqs = min(max_requests, reqs); 225 pdev->pri_reqs_alloc = reqs; 226 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 227 228 control = PCI_PRI_CTRL_ENABLE; 229 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 230 231 pdev->pri_enabled = 1; 232 233 return 0; 234 } 235 236 /** 237 * pci_disable_pri - Disable PRI capability 238 * @pdev: PCI device structure 239 * 240 * Only clears the enabled-bit, regardless of its former value 241 */ 242 void pci_disable_pri(struct pci_dev *pdev) 243 { 244 u16 control; 245 int pri = pdev->pri_cap; 246 247 /* VFs share the PF PRI */ 248 if (pdev->is_virtfn) 249 return; 250 251 if (WARN_ON(!pdev->pri_enabled)) 252 return; 253 254 if (!pri) 255 return; 256 257 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 258 control &= ~PCI_PRI_CTRL_ENABLE; 259 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 260 261 pdev->pri_enabled = 0; 262 } 263 EXPORT_SYMBOL_GPL(pci_disable_pri); 264 265 /** 266 * pci_restore_pri_state - Restore PRI 267 * @pdev: PCI device structure 268 */ 269 void pci_restore_pri_state(struct pci_dev *pdev) 270 { 271 u16 control = PCI_PRI_CTRL_ENABLE; 272 u32 reqs = pdev->pri_reqs_alloc; 273 int pri = pdev->pri_cap; 274 275 if (pdev->is_virtfn) 276 return; 277 278 if (!pdev->pri_enabled) 279 return; 280 281 if (!pri) 282 return; 283 284 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 285 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 286 } 287 288 /** 289 * pci_reset_pri - Resets device's PRI state 290 * @pdev: PCI device structure 291 * 292 * The PRI capability must be disabled before this function is called. 293 * Returns 0 on success, negative value on error. 294 */ 295 int pci_reset_pri(struct pci_dev *pdev) 296 { 297 u16 control; 298 int pri = pdev->pri_cap; 299 300 if (pdev->is_virtfn) 301 return 0; 302 303 if (WARN_ON(pdev->pri_enabled)) 304 return -EBUSY; 305 306 if (!pri) 307 return -EINVAL; 308 309 control = PCI_PRI_CTRL_RESET; 310 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 311 312 return 0; 313 } 314 315 /** 316 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 317 * status. 318 * @pdev: PCI device structure 319 * 320 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 321 */ 322 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 323 { 324 if (pdev->is_virtfn) 325 pdev = pci_physfn(pdev); 326 327 return pdev->pasid_required; 328 } 329 330 /** 331 * pci_pri_supported - Check if PRI is supported. 332 * @pdev: PCI device structure 333 * 334 * Returns true if PRI capability is present, false otherwise. 335 */ 336 bool pci_pri_supported(struct pci_dev *pdev) 337 { 338 /* VFs share the PF PRI */ 339 if (pci_physfn(pdev)->pri_cap) 340 return true; 341 return false; 342 } 343 EXPORT_SYMBOL_GPL(pci_pri_supported); 344 #endif /* CONFIG_PCI_PRI */ 345 346 #ifdef CONFIG_PCI_PASID 347 void pci_pasid_init(struct pci_dev *pdev) 348 { 349 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 350 } 351 352 /** 353 * pci_enable_pasid - Enable the PASID capability 354 * @pdev: PCI device structure 355 * @features: Features to enable 356 * 357 * Returns 0 on success, negative value on error. This function checks 358 * whether the features are actually supported by the device and returns 359 * an error if not. 360 */ 361 int pci_enable_pasid(struct pci_dev *pdev, int features) 362 { 363 u16 control, supported; 364 int pasid = pdev->pasid_cap; 365 366 /* 367 * VFs must not implement the PASID Capability, but if a PF 368 * supports PASID, its VFs share the PF PASID configuration. 369 */ 370 if (pdev->is_virtfn) { 371 if (pci_physfn(pdev)->pasid_enabled) 372 return 0; 373 return -EINVAL; 374 } 375 376 if (WARN_ON(pdev->pasid_enabled)) 377 return -EBUSY; 378 379 if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp) 380 return -EINVAL; 381 382 if (!pasid) 383 return -EINVAL; 384 385 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 386 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 387 388 /* User wants to enable anything unsupported? */ 389 if ((supported & features) != features) 390 return -EINVAL; 391 392 control = PCI_PASID_CTRL_ENABLE | features; 393 pdev->pasid_features = features; 394 395 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 396 397 pdev->pasid_enabled = 1; 398 399 return 0; 400 } 401 EXPORT_SYMBOL_GPL(pci_enable_pasid); 402 403 /** 404 * pci_disable_pasid - Disable the PASID capability 405 * @pdev: PCI device structure 406 */ 407 void pci_disable_pasid(struct pci_dev *pdev) 408 { 409 u16 control = 0; 410 int pasid = pdev->pasid_cap; 411 412 /* VFs share the PF PASID configuration */ 413 if (pdev->is_virtfn) 414 return; 415 416 if (WARN_ON(!pdev->pasid_enabled)) 417 return; 418 419 if (!pasid) 420 return; 421 422 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 423 424 pdev->pasid_enabled = 0; 425 } 426 EXPORT_SYMBOL_GPL(pci_disable_pasid); 427 428 /** 429 * pci_restore_pasid_state - Restore PASID capabilities 430 * @pdev: PCI device structure 431 */ 432 void pci_restore_pasid_state(struct pci_dev *pdev) 433 { 434 u16 control; 435 int pasid = pdev->pasid_cap; 436 437 if (pdev->is_virtfn) 438 return; 439 440 if (!pdev->pasid_enabled) 441 return; 442 443 if (!pasid) 444 return; 445 446 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 447 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 448 } 449 450 /** 451 * pci_pasid_features - Check which PASID features are supported 452 * @pdev: PCI device structure 453 * 454 * Returns a negative value when no PASI capability is present. 455 * Otherwise is returns a bitmask with supported features. Current 456 * features reported are: 457 * PCI_PASID_CAP_EXEC - Execute permission supported 458 * PCI_PASID_CAP_PRIV - Privileged mode supported 459 */ 460 int pci_pasid_features(struct pci_dev *pdev) 461 { 462 u16 supported; 463 int pasid; 464 465 if (pdev->is_virtfn) 466 pdev = pci_physfn(pdev); 467 468 pasid = pdev->pasid_cap; 469 if (!pasid) 470 return -EINVAL; 471 472 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 473 474 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 475 476 return supported; 477 } 478 EXPORT_SYMBOL_GPL(pci_pasid_features); 479 480 #define PASID_NUMBER_SHIFT 8 481 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 482 /** 483 * pci_max_pasids - Get maximum number of PASIDs supported by device 484 * @pdev: PCI device structure 485 * 486 * Returns negative value when PASID capability is not present. 487 * Otherwise it returns the number of supported PASIDs. 488 */ 489 int pci_max_pasids(struct pci_dev *pdev) 490 { 491 u16 supported; 492 int pasid; 493 494 if (pdev->is_virtfn) 495 pdev = pci_physfn(pdev); 496 497 pasid = pdev->pasid_cap; 498 if (!pasid) 499 return -EINVAL; 500 501 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 502 503 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 504 505 return (1 << supported); 506 } 507 EXPORT_SYMBOL_GPL(pci_max_pasids); 508 #endif /* CONFIG_PCI_PASID */ 509