1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_enable_ats - enable the ATS capability 35 * @dev: the PCI device 36 * @ps: the IOMMU page shift 37 * 38 * Returns 0 on success, or negative on failure. 39 */ 40 int pci_enable_ats(struct pci_dev *dev, int ps) 41 { 42 u16 ctrl; 43 struct pci_dev *pdev; 44 45 if (!dev->ats_cap) 46 return -EINVAL; 47 48 if (WARN_ON(dev->ats_enabled)) 49 return -EBUSY; 50 51 if (ps < PCI_ATS_MIN_STU) 52 return -EINVAL; 53 54 /* 55 * Note that enabling ATS on a VF fails unless it's already enabled 56 * with the same STU on the PF. 57 */ 58 ctrl = PCI_ATS_CTRL_ENABLE; 59 if (dev->is_virtfn) { 60 pdev = pci_physfn(dev); 61 if (pdev->ats_stu != ps) 62 return -EINVAL; 63 64 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */ 65 } else { 66 dev->ats_stu = ps; 67 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 68 } 69 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 70 71 dev->ats_enabled = 1; 72 return 0; 73 } 74 EXPORT_SYMBOL_GPL(pci_enable_ats); 75 76 /** 77 * pci_disable_ats - disable the ATS capability 78 * @dev: the PCI device 79 */ 80 void pci_disable_ats(struct pci_dev *dev) 81 { 82 struct pci_dev *pdev; 83 u16 ctrl; 84 85 if (WARN_ON(!dev->ats_enabled)) 86 return; 87 88 if (atomic_read(&dev->ats_ref_cnt)) 89 return; /* VFs still enabled */ 90 91 if (dev->is_virtfn) { 92 pdev = pci_physfn(dev); 93 atomic_dec(&pdev->ats_ref_cnt); 94 } 95 96 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 97 ctrl &= ~PCI_ATS_CTRL_ENABLE; 98 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 99 100 dev->ats_enabled = 0; 101 } 102 EXPORT_SYMBOL_GPL(pci_disable_ats); 103 104 void pci_restore_ats_state(struct pci_dev *dev) 105 { 106 u16 ctrl; 107 108 if (!dev->ats_enabled) 109 return; 110 111 ctrl = PCI_ATS_CTRL_ENABLE; 112 if (!dev->is_virtfn) 113 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 114 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 115 } 116 EXPORT_SYMBOL_GPL(pci_restore_ats_state); 117 118 /** 119 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 120 * @dev: the PCI device 121 * 122 * Returns the queue depth on success, or negative on failure. 123 * 124 * The ATS spec uses 0 in the Invalidate Queue Depth field to 125 * indicate that the function can accept 32 Invalidate Request. 126 * But here we use the `real' values (i.e. 1~32) for the Queue 127 * Depth; and 0 indicates the function shares the Queue with 128 * other functions (doesn't exclusively own a Queue). 129 */ 130 int pci_ats_queue_depth(struct pci_dev *dev) 131 { 132 u16 cap; 133 134 if (!dev->ats_cap) 135 return -EINVAL; 136 137 if (dev->is_virtfn) 138 return 0; 139 140 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 141 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 142 } 143 EXPORT_SYMBOL_GPL(pci_ats_queue_depth); 144 145 /** 146 * pci_ats_page_aligned - Return Page Aligned Request bit status. 147 * @pdev: the PCI device 148 * 149 * Returns 1, if the Untranslated Addresses generated by the device 150 * are always aligned or 0 otherwise. 151 * 152 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 153 * is set, it indicates the Untranslated Addresses generated by the 154 * device are always aligned to a 4096 byte boundary. 155 */ 156 int pci_ats_page_aligned(struct pci_dev *pdev) 157 { 158 u16 cap; 159 160 if (!pdev->ats_cap) 161 return 0; 162 163 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 164 165 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 166 return 1; 167 168 return 0; 169 } 170 EXPORT_SYMBOL_GPL(pci_ats_page_aligned); 171 172 #ifdef CONFIG_PCI_PRI 173 /** 174 * pci_enable_pri - Enable PRI capability 175 * @ pdev: PCI device structure 176 * 177 * Returns 0 on success, negative value on error 178 */ 179 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 180 { 181 u16 control, status; 182 u32 max_requests; 183 int pos; 184 185 if (WARN_ON(pdev->pri_enabled)) 186 return -EBUSY; 187 188 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 189 if (!pos) 190 return -EINVAL; 191 192 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); 193 if (!(status & PCI_PRI_STATUS_STOPPED)) 194 return -EBUSY; 195 196 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests); 197 reqs = min(max_requests, reqs); 198 pdev->pri_reqs_alloc = reqs; 199 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 200 201 control = PCI_PRI_CTRL_ENABLE; 202 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 203 204 pdev->pri_enabled = 1; 205 206 return 0; 207 } 208 EXPORT_SYMBOL_GPL(pci_enable_pri); 209 210 /** 211 * pci_disable_pri - Disable PRI capability 212 * @pdev: PCI device structure 213 * 214 * Only clears the enabled-bit, regardless of its former value 215 */ 216 void pci_disable_pri(struct pci_dev *pdev) 217 { 218 u16 control; 219 int pos; 220 221 if (WARN_ON(!pdev->pri_enabled)) 222 return; 223 224 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 225 if (!pos) 226 return; 227 228 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control); 229 control &= ~PCI_PRI_CTRL_ENABLE; 230 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 231 232 pdev->pri_enabled = 0; 233 } 234 EXPORT_SYMBOL_GPL(pci_disable_pri); 235 236 /** 237 * pci_restore_pri_state - Restore PRI 238 * @pdev: PCI device structure 239 */ 240 void pci_restore_pri_state(struct pci_dev *pdev) 241 { 242 u16 control = PCI_PRI_CTRL_ENABLE; 243 u32 reqs = pdev->pri_reqs_alloc; 244 int pos; 245 246 if (!pdev->pri_enabled) 247 return; 248 249 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 250 if (!pos) 251 return; 252 253 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 254 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 255 } 256 EXPORT_SYMBOL_GPL(pci_restore_pri_state); 257 258 /** 259 * pci_reset_pri - Resets device's PRI state 260 * @pdev: PCI device structure 261 * 262 * The PRI capability must be disabled before this function is called. 263 * Returns 0 on success, negative value on error. 264 */ 265 int pci_reset_pri(struct pci_dev *pdev) 266 { 267 u16 control; 268 int pos; 269 270 if (WARN_ON(pdev->pri_enabled)) 271 return -EBUSY; 272 273 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 274 if (!pos) 275 return -EINVAL; 276 277 control = PCI_PRI_CTRL_RESET; 278 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 279 280 return 0; 281 } 282 EXPORT_SYMBOL_GPL(pci_reset_pri); 283 #endif /* CONFIG_PCI_PRI */ 284 285 #ifdef CONFIG_PCI_PASID 286 /** 287 * pci_enable_pasid - Enable the PASID capability 288 * @pdev: PCI device structure 289 * @features: Features to enable 290 * 291 * Returns 0 on success, negative value on error. This function checks 292 * whether the features are actually supported by the device and returns 293 * an error if not. 294 */ 295 int pci_enable_pasid(struct pci_dev *pdev, int features) 296 { 297 u16 control, supported; 298 int pos; 299 300 if (WARN_ON(pdev->pasid_enabled)) 301 return -EBUSY; 302 303 if (!pdev->eetlp_prefix_path) 304 return -EINVAL; 305 306 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 307 if (!pos) 308 return -EINVAL; 309 310 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 311 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 312 313 /* User wants to enable anything unsupported? */ 314 if ((supported & features) != features) 315 return -EINVAL; 316 317 control = PCI_PASID_CTRL_ENABLE | features; 318 pdev->pasid_features = features; 319 320 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 321 322 pdev->pasid_enabled = 1; 323 324 return 0; 325 } 326 EXPORT_SYMBOL_GPL(pci_enable_pasid); 327 328 /** 329 * pci_disable_pasid - Disable the PASID capability 330 * @pdev: PCI device structure 331 */ 332 void pci_disable_pasid(struct pci_dev *pdev) 333 { 334 u16 control = 0; 335 int pos; 336 337 if (WARN_ON(!pdev->pasid_enabled)) 338 return; 339 340 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 341 if (!pos) 342 return; 343 344 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 345 346 pdev->pasid_enabled = 0; 347 } 348 EXPORT_SYMBOL_GPL(pci_disable_pasid); 349 350 /** 351 * pci_restore_pasid_state - Restore PASID capabilities 352 * @pdev: PCI device structure 353 */ 354 void pci_restore_pasid_state(struct pci_dev *pdev) 355 { 356 u16 control; 357 int pos; 358 359 if (!pdev->pasid_enabled) 360 return; 361 362 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 363 if (!pos) 364 return; 365 366 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 367 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 368 } 369 EXPORT_SYMBOL_GPL(pci_restore_pasid_state); 370 371 /** 372 * pci_pasid_features - Check which PASID features are supported 373 * @pdev: PCI device structure 374 * 375 * Returns a negative value when no PASI capability is present. 376 * Otherwise is returns a bitmask with supported features. Current 377 * features reported are: 378 * PCI_PASID_CAP_EXEC - Execute permission supported 379 * PCI_PASID_CAP_PRIV - Privileged mode supported 380 */ 381 int pci_pasid_features(struct pci_dev *pdev) 382 { 383 u16 supported; 384 int pos; 385 386 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 387 if (!pos) 388 return -EINVAL; 389 390 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 391 392 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 393 394 return supported; 395 } 396 EXPORT_SYMBOL_GPL(pci_pasid_features); 397 398 /** 399 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 400 * status. 401 * @pdev: PCI device structure 402 * 403 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 404 * 405 * Even though the PRG response PASID status is read from PRI Status 406 * Register, since this API will mainly be used by PASID users, this 407 * function is defined within #ifdef CONFIG_PCI_PASID instead of 408 * CONFIG_PCI_PRI. 409 */ 410 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 411 { 412 u16 status; 413 int pos; 414 415 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 416 if (!pos) 417 return 0; 418 419 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); 420 421 if (status & PCI_PRI_STATUS_PASID) 422 return 1; 423 424 return 0; 425 } 426 EXPORT_SYMBOL_GPL(pci_prg_resp_pasid_required); 427 428 #define PASID_NUMBER_SHIFT 8 429 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 430 /** 431 * pci_max_pasid - Get maximum number of PASIDs supported by device 432 * @pdev: PCI device structure 433 * 434 * Returns negative value when PASID capability is not present. 435 * Otherwise it returns the number of supported PASIDs. 436 */ 437 int pci_max_pasids(struct pci_dev *pdev) 438 { 439 u16 supported; 440 int pos; 441 442 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 443 if (!pos) 444 return -EINVAL; 445 446 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 447 448 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 449 450 return (1 << supported); 451 } 452 EXPORT_SYMBOL_GPL(pci_max_pasids); 453 #endif /* CONFIG_PCI_PASID */ 454