1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_ats_supported - check if the device can use ATS 35 * @dev: the PCI device 36 * 37 * Returns true if the device supports ATS and is allowed to use it, false 38 * otherwise. 39 */ 40 bool pci_ats_supported(struct pci_dev *dev) 41 { 42 if (!dev->ats_cap) 43 return false; 44 45 return (dev->untrusted == 0); 46 } 47 EXPORT_SYMBOL_GPL(pci_ats_supported); 48 49 /** 50 * pci_enable_ats - enable the ATS capability 51 * @dev: the PCI device 52 * @ps: the IOMMU page shift 53 * 54 * Returns 0 on success, or negative on failure. 55 */ 56 int pci_enable_ats(struct pci_dev *dev, int ps) 57 { 58 u16 ctrl; 59 struct pci_dev *pdev; 60 61 if (!pci_ats_supported(dev)) 62 return -EINVAL; 63 64 if (WARN_ON(dev->ats_enabled)) 65 return -EBUSY; 66 67 if (ps < PCI_ATS_MIN_STU) 68 return -EINVAL; 69 70 /* 71 * Note that enabling ATS on a VF fails unless it's already enabled 72 * with the same STU on the PF. 73 */ 74 ctrl = PCI_ATS_CTRL_ENABLE; 75 if (dev->is_virtfn) { 76 pdev = pci_physfn(dev); 77 if (pdev->ats_stu != ps) 78 return -EINVAL; 79 } else { 80 dev->ats_stu = ps; 81 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 82 } 83 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 84 85 dev->ats_enabled = 1; 86 return 0; 87 } 88 EXPORT_SYMBOL_GPL(pci_enable_ats); 89 90 /** 91 * pci_disable_ats - disable the ATS capability 92 * @dev: the PCI device 93 */ 94 void pci_disable_ats(struct pci_dev *dev) 95 { 96 u16 ctrl; 97 98 if (WARN_ON(!dev->ats_enabled)) 99 return; 100 101 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 102 ctrl &= ~PCI_ATS_CTRL_ENABLE; 103 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 104 105 dev->ats_enabled = 0; 106 } 107 EXPORT_SYMBOL_GPL(pci_disable_ats); 108 109 void pci_restore_ats_state(struct pci_dev *dev) 110 { 111 u16 ctrl; 112 113 if (!dev->ats_enabled) 114 return; 115 116 ctrl = PCI_ATS_CTRL_ENABLE; 117 if (!dev->is_virtfn) 118 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 119 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 120 } 121 122 /** 123 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 124 * @dev: the PCI device 125 * 126 * Returns the queue depth on success, or negative on failure. 127 * 128 * The ATS spec uses 0 in the Invalidate Queue Depth field to 129 * indicate that the function can accept 32 Invalidate Request. 130 * But here we use the `real' values (i.e. 1~32) for the Queue 131 * Depth; and 0 indicates the function shares the Queue with 132 * other functions (doesn't exclusively own a Queue). 133 */ 134 int pci_ats_queue_depth(struct pci_dev *dev) 135 { 136 u16 cap; 137 138 if (!dev->ats_cap) 139 return -EINVAL; 140 141 if (dev->is_virtfn) 142 return 0; 143 144 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 145 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 146 } 147 148 /** 149 * pci_ats_page_aligned - Return Page Aligned Request bit status. 150 * @pdev: the PCI device 151 * 152 * Returns 1, if the Untranslated Addresses generated by the device 153 * are always aligned or 0 otherwise. 154 * 155 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 156 * is set, it indicates the Untranslated Addresses generated by the 157 * device are always aligned to a 4096 byte boundary. 158 */ 159 int pci_ats_page_aligned(struct pci_dev *pdev) 160 { 161 u16 cap; 162 163 if (!pdev->ats_cap) 164 return 0; 165 166 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 167 168 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 169 return 1; 170 171 return 0; 172 } 173 174 #ifdef CONFIG_PCI_PRI 175 void pci_pri_init(struct pci_dev *pdev) 176 { 177 u16 status; 178 179 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 180 181 if (!pdev->pri_cap) 182 return; 183 184 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 185 if (status & PCI_PRI_STATUS_PASID) 186 pdev->pasid_required = 1; 187 } 188 189 /** 190 * pci_enable_pri - Enable PRI capability 191 * @ pdev: PCI device structure 192 * 193 * Returns 0 on success, negative value on error 194 */ 195 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 196 { 197 u16 control, status; 198 u32 max_requests; 199 int pri = pdev->pri_cap; 200 201 /* 202 * VFs must not implement the PRI Capability. If their PF 203 * implements PRI, it is shared by the VFs, so if the PF PRI is 204 * enabled, it is also enabled for the VF. 205 */ 206 if (pdev->is_virtfn) { 207 if (pci_physfn(pdev)->pri_enabled) 208 return 0; 209 return -EINVAL; 210 } 211 212 if (WARN_ON(pdev->pri_enabled)) 213 return -EBUSY; 214 215 if (!pri) 216 return -EINVAL; 217 218 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 219 if (!(status & PCI_PRI_STATUS_STOPPED)) 220 return -EBUSY; 221 222 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 223 reqs = min(max_requests, reqs); 224 pdev->pri_reqs_alloc = reqs; 225 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 226 227 control = PCI_PRI_CTRL_ENABLE; 228 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 229 230 pdev->pri_enabled = 1; 231 232 return 0; 233 } 234 235 /** 236 * pci_disable_pri - Disable PRI capability 237 * @pdev: PCI device structure 238 * 239 * Only clears the enabled-bit, regardless of its former value 240 */ 241 void pci_disable_pri(struct pci_dev *pdev) 242 { 243 u16 control; 244 int pri = pdev->pri_cap; 245 246 /* VFs share the PF PRI */ 247 if (pdev->is_virtfn) 248 return; 249 250 if (WARN_ON(!pdev->pri_enabled)) 251 return; 252 253 if (!pri) 254 return; 255 256 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 257 control &= ~PCI_PRI_CTRL_ENABLE; 258 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 259 260 pdev->pri_enabled = 0; 261 } 262 EXPORT_SYMBOL_GPL(pci_disable_pri); 263 264 /** 265 * pci_restore_pri_state - Restore PRI 266 * @pdev: PCI device structure 267 */ 268 void pci_restore_pri_state(struct pci_dev *pdev) 269 { 270 u16 control = PCI_PRI_CTRL_ENABLE; 271 u32 reqs = pdev->pri_reqs_alloc; 272 int pri = pdev->pri_cap; 273 274 if (pdev->is_virtfn) 275 return; 276 277 if (!pdev->pri_enabled) 278 return; 279 280 if (!pri) 281 return; 282 283 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 284 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 285 } 286 287 /** 288 * pci_reset_pri - Resets device's PRI state 289 * @pdev: PCI device structure 290 * 291 * The PRI capability must be disabled before this function is called. 292 * Returns 0 on success, negative value on error. 293 */ 294 int pci_reset_pri(struct pci_dev *pdev) 295 { 296 u16 control; 297 int pri = pdev->pri_cap; 298 299 if (pdev->is_virtfn) 300 return 0; 301 302 if (WARN_ON(pdev->pri_enabled)) 303 return -EBUSY; 304 305 if (!pri) 306 return -EINVAL; 307 308 control = PCI_PRI_CTRL_RESET; 309 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 310 311 return 0; 312 } 313 314 /** 315 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 316 * status. 317 * @pdev: PCI device structure 318 * 319 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 320 */ 321 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 322 { 323 if (pdev->is_virtfn) 324 pdev = pci_physfn(pdev); 325 326 return pdev->pasid_required; 327 } 328 #endif /* CONFIG_PCI_PRI */ 329 330 #ifdef CONFIG_PCI_PASID 331 void pci_pasid_init(struct pci_dev *pdev) 332 { 333 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 334 } 335 336 /** 337 * pci_enable_pasid - Enable the PASID capability 338 * @pdev: PCI device structure 339 * @features: Features to enable 340 * 341 * Returns 0 on success, negative value on error. This function checks 342 * whether the features are actually supported by the device and returns 343 * an error if not. 344 */ 345 int pci_enable_pasid(struct pci_dev *pdev, int features) 346 { 347 u16 control, supported; 348 int pasid = pdev->pasid_cap; 349 350 /* 351 * VFs must not implement the PASID Capability, but if a PF 352 * supports PASID, its VFs share the PF PASID configuration. 353 */ 354 if (pdev->is_virtfn) { 355 if (pci_physfn(pdev)->pasid_enabled) 356 return 0; 357 return -EINVAL; 358 } 359 360 if (WARN_ON(pdev->pasid_enabled)) 361 return -EBUSY; 362 363 if (!pdev->eetlp_prefix_path) 364 return -EINVAL; 365 366 if (!pasid) 367 return -EINVAL; 368 369 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 370 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 371 372 /* User wants to enable anything unsupported? */ 373 if ((supported & features) != features) 374 return -EINVAL; 375 376 control = PCI_PASID_CTRL_ENABLE | features; 377 pdev->pasid_features = features; 378 379 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 380 381 pdev->pasid_enabled = 1; 382 383 return 0; 384 } 385 EXPORT_SYMBOL_GPL(pci_enable_pasid); 386 387 /** 388 * pci_disable_pasid - Disable the PASID capability 389 * @pdev: PCI device structure 390 */ 391 void pci_disable_pasid(struct pci_dev *pdev) 392 { 393 u16 control = 0; 394 int pasid = pdev->pasid_cap; 395 396 /* VFs share the PF PASID configuration */ 397 if (pdev->is_virtfn) 398 return; 399 400 if (WARN_ON(!pdev->pasid_enabled)) 401 return; 402 403 if (!pasid) 404 return; 405 406 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 407 408 pdev->pasid_enabled = 0; 409 } 410 EXPORT_SYMBOL_GPL(pci_disable_pasid); 411 412 /** 413 * pci_restore_pasid_state - Restore PASID capabilities 414 * @pdev: PCI device structure 415 */ 416 void pci_restore_pasid_state(struct pci_dev *pdev) 417 { 418 u16 control; 419 int pasid = pdev->pasid_cap; 420 421 if (pdev->is_virtfn) 422 return; 423 424 if (!pdev->pasid_enabled) 425 return; 426 427 if (!pasid) 428 return; 429 430 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 431 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 432 } 433 434 /** 435 * pci_pasid_features - Check which PASID features are supported 436 * @pdev: PCI device structure 437 * 438 * Returns a negative value when no PASI capability is present. 439 * Otherwise is returns a bitmask with supported features. Current 440 * features reported are: 441 * PCI_PASID_CAP_EXEC - Execute permission supported 442 * PCI_PASID_CAP_PRIV - Privileged mode supported 443 */ 444 int pci_pasid_features(struct pci_dev *pdev) 445 { 446 u16 supported; 447 int pasid; 448 449 if (pdev->is_virtfn) 450 pdev = pci_physfn(pdev); 451 452 pasid = pdev->pasid_cap; 453 if (!pasid) 454 return -EINVAL; 455 456 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 457 458 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 459 460 return supported; 461 } 462 EXPORT_SYMBOL_GPL(pci_pasid_features); 463 464 #define PASID_NUMBER_SHIFT 8 465 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 466 /** 467 * pci_max_pasid - Get maximum number of PASIDs supported by device 468 * @pdev: PCI device structure 469 * 470 * Returns negative value when PASID capability is not present. 471 * Otherwise it returns the number of supported PASIDs. 472 */ 473 int pci_max_pasids(struct pci_dev *pdev) 474 { 475 u16 supported; 476 int pasid; 477 478 if (pdev->is_virtfn) 479 pdev = pci_physfn(pdev); 480 481 pasid = pdev->pasid_cap; 482 if (!pasid) 483 return -EINVAL; 484 485 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 486 487 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 488 489 return (1 << supported); 490 } 491 EXPORT_SYMBOL_GPL(pci_max_pasids); 492 #endif /* CONFIG_PCI_PASID */ 493