1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_enable_ats - enable the ATS capability 35 * @dev: the PCI device 36 * @ps: the IOMMU page shift 37 * 38 * Returns 0 on success, or negative on failure. 39 */ 40 int pci_enable_ats(struct pci_dev *dev, int ps) 41 { 42 u16 ctrl; 43 struct pci_dev *pdev; 44 45 if (!dev->ats_cap) 46 return -EINVAL; 47 48 if (WARN_ON(dev->ats_enabled)) 49 return -EBUSY; 50 51 if (ps < PCI_ATS_MIN_STU) 52 return -EINVAL; 53 54 /* 55 * Note that enabling ATS on a VF fails unless it's already enabled 56 * with the same STU on the PF. 57 */ 58 ctrl = PCI_ATS_CTRL_ENABLE; 59 if (dev->is_virtfn) { 60 pdev = pci_physfn(dev); 61 if (pdev->ats_stu != ps) 62 return -EINVAL; 63 } else { 64 dev->ats_stu = ps; 65 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 66 } 67 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 68 69 dev->ats_enabled = 1; 70 return 0; 71 } 72 73 /** 74 * pci_disable_ats - disable the ATS capability 75 * @dev: the PCI device 76 */ 77 void pci_disable_ats(struct pci_dev *dev) 78 { 79 u16 ctrl; 80 81 if (WARN_ON(!dev->ats_enabled)) 82 return; 83 84 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 85 ctrl &= ~PCI_ATS_CTRL_ENABLE; 86 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 87 88 dev->ats_enabled = 0; 89 } 90 91 void pci_restore_ats_state(struct pci_dev *dev) 92 { 93 u16 ctrl; 94 95 if (!dev->ats_enabled) 96 return; 97 98 ctrl = PCI_ATS_CTRL_ENABLE; 99 if (!dev->is_virtfn) 100 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 101 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 102 } 103 104 /** 105 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 106 * @dev: the PCI device 107 * 108 * Returns the queue depth on success, or negative on failure. 109 * 110 * The ATS spec uses 0 in the Invalidate Queue Depth field to 111 * indicate that the function can accept 32 Invalidate Request. 112 * But here we use the `real' values (i.e. 1~32) for the Queue 113 * Depth; and 0 indicates the function shares the Queue with 114 * other functions (doesn't exclusively own a Queue). 115 */ 116 int pci_ats_queue_depth(struct pci_dev *dev) 117 { 118 u16 cap; 119 120 if (!dev->ats_cap) 121 return -EINVAL; 122 123 if (dev->is_virtfn) 124 return 0; 125 126 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 127 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 128 } 129 130 /** 131 * pci_ats_page_aligned - Return Page Aligned Request bit status. 132 * @pdev: the PCI device 133 * 134 * Returns 1, if the Untranslated Addresses generated by the device 135 * are always aligned or 0 otherwise. 136 * 137 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 138 * is set, it indicates the Untranslated Addresses generated by the 139 * device are always aligned to a 4096 byte boundary. 140 */ 141 int pci_ats_page_aligned(struct pci_dev *pdev) 142 { 143 u16 cap; 144 145 if (!pdev->ats_cap) 146 return 0; 147 148 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 149 150 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 151 return 1; 152 153 return 0; 154 } 155 156 #ifdef CONFIG_PCI_PRI 157 void pci_pri_init(struct pci_dev *pdev) 158 { 159 u16 status; 160 161 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 162 163 if (!pdev->pri_cap) 164 return; 165 166 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 167 if (status & PCI_PRI_STATUS_PASID) 168 pdev->pasid_required = 1; 169 } 170 171 /** 172 * pci_enable_pri - Enable PRI capability 173 * @ pdev: PCI device structure 174 * 175 * Returns 0 on success, negative value on error 176 */ 177 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 178 { 179 u16 control, status; 180 u32 max_requests; 181 int pri = pdev->pri_cap; 182 183 /* 184 * VFs must not implement the PRI Capability. If their PF 185 * implements PRI, it is shared by the VFs, so if the PF PRI is 186 * enabled, it is also enabled for the VF. 187 */ 188 if (pdev->is_virtfn) { 189 if (pci_physfn(pdev)->pri_enabled) 190 return 0; 191 return -EINVAL; 192 } 193 194 if (WARN_ON(pdev->pri_enabled)) 195 return -EBUSY; 196 197 if (!pri) 198 return -EINVAL; 199 200 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 201 if (!(status & PCI_PRI_STATUS_STOPPED)) 202 return -EBUSY; 203 204 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 205 reqs = min(max_requests, reqs); 206 pdev->pri_reqs_alloc = reqs; 207 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 208 209 control = PCI_PRI_CTRL_ENABLE; 210 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 211 212 pdev->pri_enabled = 1; 213 214 return 0; 215 } 216 217 /** 218 * pci_disable_pri - Disable PRI capability 219 * @pdev: PCI device structure 220 * 221 * Only clears the enabled-bit, regardless of its former value 222 */ 223 void pci_disable_pri(struct pci_dev *pdev) 224 { 225 u16 control; 226 int pri = pdev->pri_cap; 227 228 /* VFs share the PF PRI */ 229 if (pdev->is_virtfn) 230 return; 231 232 if (WARN_ON(!pdev->pri_enabled)) 233 return; 234 235 if (!pri) 236 return; 237 238 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 239 control &= ~PCI_PRI_CTRL_ENABLE; 240 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 241 242 pdev->pri_enabled = 0; 243 } 244 EXPORT_SYMBOL_GPL(pci_disable_pri); 245 246 /** 247 * pci_restore_pri_state - Restore PRI 248 * @pdev: PCI device structure 249 */ 250 void pci_restore_pri_state(struct pci_dev *pdev) 251 { 252 u16 control = PCI_PRI_CTRL_ENABLE; 253 u32 reqs = pdev->pri_reqs_alloc; 254 int pri = pdev->pri_cap; 255 256 if (pdev->is_virtfn) 257 return; 258 259 if (!pdev->pri_enabled) 260 return; 261 262 if (!pri) 263 return; 264 265 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 266 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 267 } 268 269 /** 270 * pci_reset_pri - Resets device's PRI state 271 * @pdev: PCI device structure 272 * 273 * The PRI capability must be disabled before this function is called. 274 * Returns 0 on success, negative value on error. 275 */ 276 int pci_reset_pri(struct pci_dev *pdev) 277 { 278 u16 control; 279 int pri = pdev->pri_cap; 280 281 if (pdev->is_virtfn) 282 return 0; 283 284 if (WARN_ON(pdev->pri_enabled)) 285 return -EBUSY; 286 287 if (!pri) 288 return -EINVAL; 289 290 control = PCI_PRI_CTRL_RESET; 291 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 292 293 return 0; 294 } 295 296 /** 297 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 298 * status. 299 * @pdev: PCI device structure 300 * 301 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 302 */ 303 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 304 { 305 if (pdev->is_virtfn) 306 pdev = pci_physfn(pdev); 307 308 return pdev->pasid_required; 309 } 310 #endif /* CONFIG_PCI_PRI */ 311 312 #ifdef CONFIG_PCI_PASID 313 void pci_pasid_init(struct pci_dev *pdev) 314 { 315 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 316 } 317 318 /** 319 * pci_enable_pasid - Enable the PASID capability 320 * @pdev: PCI device structure 321 * @features: Features to enable 322 * 323 * Returns 0 on success, negative value on error. This function checks 324 * whether the features are actually supported by the device and returns 325 * an error if not. 326 */ 327 int pci_enable_pasid(struct pci_dev *pdev, int features) 328 { 329 u16 control, supported; 330 int pasid = pdev->pasid_cap; 331 332 /* 333 * VFs must not implement the PASID Capability, but if a PF 334 * supports PASID, its VFs share the PF PASID configuration. 335 */ 336 if (pdev->is_virtfn) { 337 if (pci_physfn(pdev)->pasid_enabled) 338 return 0; 339 return -EINVAL; 340 } 341 342 if (WARN_ON(pdev->pasid_enabled)) 343 return -EBUSY; 344 345 if (!pdev->eetlp_prefix_path) 346 return -EINVAL; 347 348 if (!pasid) 349 return -EINVAL; 350 351 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 352 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 353 354 /* User wants to enable anything unsupported? */ 355 if ((supported & features) != features) 356 return -EINVAL; 357 358 control = PCI_PASID_CTRL_ENABLE | features; 359 pdev->pasid_features = features; 360 361 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 362 363 pdev->pasid_enabled = 1; 364 365 return 0; 366 } 367 368 /** 369 * pci_disable_pasid - Disable the PASID capability 370 * @pdev: PCI device structure 371 */ 372 void pci_disable_pasid(struct pci_dev *pdev) 373 { 374 u16 control = 0; 375 int pasid = pdev->pasid_cap; 376 377 /* VFs share the PF PASID configuration */ 378 if (pdev->is_virtfn) 379 return; 380 381 if (WARN_ON(!pdev->pasid_enabled)) 382 return; 383 384 if (!pasid) 385 return; 386 387 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 388 389 pdev->pasid_enabled = 0; 390 } 391 392 /** 393 * pci_restore_pasid_state - Restore PASID capabilities 394 * @pdev: PCI device structure 395 */ 396 void pci_restore_pasid_state(struct pci_dev *pdev) 397 { 398 u16 control; 399 int pasid = pdev->pasid_cap; 400 401 if (pdev->is_virtfn) 402 return; 403 404 if (!pdev->pasid_enabled) 405 return; 406 407 if (!pasid) 408 return; 409 410 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 411 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 412 } 413 414 /** 415 * pci_pasid_features - Check which PASID features are supported 416 * @pdev: PCI device structure 417 * 418 * Returns a negative value when no PASI capability is present. 419 * Otherwise is returns a bitmask with supported features. Current 420 * features reported are: 421 * PCI_PASID_CAP_EXEC - Execute permission supported 422 * PCI_PASID_CAP_PRIV - Privileged mode supported 423 */ 424 int pci_pasid_features(struct pci_dev *pdev) 425 { 426 u16 supported; 427 int pasid = pdev->pasid_cap; 428 429 if (pdev->is_virtfn) 430 pdev = pci_physfn(pdev); 431 432 if (!pasid) 433 return -EINVAL; 434 435 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 436 437 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 438 439 return supported; 440 } 441 442 #define PASID_NUMBER_SHIFT 8 443 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 444 /** 445 * pci_max_pasid - Get maximum number of PASIDs supported by device 446 * @pdev: PCI device structure 447 * 448 * Returns negative value when PASID capability is not present. 449 * Otherwise it returns the number of supported PASIDs. 450 */ 451 int pci_max_pasids(struct pci_dev *pdev) 452 { 453 u16 supported; 454 int pasid = pdev->pasid_cap; 455 456 if (pdev->is_virtfn) 457 pdev = pci_physfn(pdev); 458 459 if (!pasid) 460 return -EINVAL; 461 462 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 463 464 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 465 466 return (1 << supported); 467 } 468 #endif /* CONFIG_PCI_PASID */ 469