1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 24 if (!pos) 25 return; 26 27 dev->ats_cap = pos; 28 } 29 30 /** 31 * pci_enable_ats - enable the ATS capability 32 * @dev: the PCI device 33 * @ps: the IOMMU page shift 34 * 35 * Returns 0 on success, or negative on failure. 36 */ 37 int pci_enable_ats(struct pci_dev *dev, int ps) 38 { 39 u16 ctrl; 40 struct pci_dev *pdev; 41 42 if (!dev->ats_cap) 43 return -EINVAL; 44 45 if (WARN_ON(dev->ats_enabled)) 46 return -EBUSY; 47 48 if (ps < PCI_ATS_MIN_STU) 49 return -EINVAL; 50 51 /* 52 * Note that enabling ATS on a VF fails unless it's already enabled 53 * with the same STU on the PF. 54 */ 55 ctrl = PCI_ATS_CTRL_ENABLE; 56 if (dev->is_virtfn) { 57 pdev = pci_physfn(dev); 58 if (pdev->ats_stu != ps) 59 return -EINVAL; 60 61 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */ 62 } else { 63 dev->ats_stu = ps; 64 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 65 } 66 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 67 68 dev->ats_enabled = 1; 69 return 0; 70 } 71 EXPORT_SYMBOL_GPL(pci_enable_ats); 72 73 /** 74 * pci_disable_ats - disable the ATS capability 75 * @dev: the PCI device 76 */ 77 void pci_disable_ats(struct pci_dev *dev) 78 { 79 struct pci_dev *pdev; 80 u16 ctrl; 81 82 if (WARN_ON(!dev->ats_enabled)) 83 return; 84 85 if (atomic_read(&dev->ats_ref_cnt)) 86 return; /* VFs still enabled */ 87 88 if (dev->is_virtfn) { 89 pdev = pci_physfn(dev); 90 atomic_dec(&pdev->ats_ref_cnt); 91 } 92 93 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 94 ctrl &= ~PCI_ATS_CTRL_ENABLE; 95 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 96 97 dev->ats_enabled = 0; 98 } 99 EXPORT_SYMBOL_GPL(pci_disable_ats); 100 101 void pci_restore_ats_state(struct pci_dev *dev) 102 { 103 u16 ctrl; 104 105 if (!dev->ats_enabled) 106 return; 107 108 ctrl = PCI_ATS_CTRL_ENABLE; 109 if (!dev->is_virtfn) 110 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 111 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 112 } 113 EXPORT_SYMBOL_GPL(pci_restore_ats_state); 114 115 /** 116 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 117 * @dev: the PCI device 118 * 119 * Returns the queue depth on success, or negative on failure. 120 * 121 * The ATS spec uses 0 in the Invalidate Queue Depth field to 122 * indicate that the function can accept 32 Invalidate Request. 123 * But here we use the `real' values (i.e. 1~32) for the Queue 124 * Depth; and 0 indicates the function shares the Queue with 125 * other functions (doesn't exclusively own a Queue). 126 */ 127 int pci_ats_queue_depth(struct pci_dev *dev) 128 { 129 u16 cap; 130 131 if (!dev->ats_cap) 132 return -EINVAL; 133 134 if (dev->is_virtfn) 135 return 0; 136 137 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 138 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 139 } 140 EXPORT_SYMBOL_GPL(pci_ats_queue_depth); 141 142 #ifdef CONFIG_PCI_PRI 143 /** 144 * pci_enable_pri - Enable PRI capability 145 * @ pdev: PCI device structure 146 * 147 * Returns 0 on success, negative value on error 148 */ 149 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 150 { 151 u16 control, status; 152 u32 max_requests; 153 int pos; 154 155 if (WARN_ON(pdev->pri_enabled)) 156 return -EBUSY; 157 158 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 159 if (!pos) 160 return -EINVAL; 161 162 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); 163 if (!(status & PCI_PRI_STATUS_STOPPED)) 164 return -EBUSY; 165 166 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests); 167 reqs = min(max_requests, reqs); 168 pdev->pri_reqs_alloc = reqs; 169 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 170 171 control = PCI_PRI_CTRL_ENABLE; 172 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 173 174 pdev->pri_enabled = 1; 175 176 return 0; 177 } 178 EXPORT_SYMBOL_GPL(pci_enable_pri); 179 180 /** 181 * pci_disable_pri - Disable PRI capability 182 * @pdev: PCI device structure 183 * 184 * Only clears the enabled-bit, regardless of its former value 185 */ 186 void pci_disable_pri(struct pci_dev *pdev) 187 { 188 u16 control; 189 int pos; 190 191 if (WARN_ON(!pdev->pri_enabled)) 192 return; 193 194 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 195 if (!pos) 196 return; 197 198 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control); 199 control &= ~PCI_PRI_CTRL_ENABLE; 200 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 201 202 pdev->pri_enabled = 0; 203 } 204 EXPORT_SYMBOL_GPL(pci_disable_pri); 205 206 /** 207 * pci_restore_pri_state - Restore PRI 208 * @pdev: PCI device structure 209 */ 210 void pci_restore_pri_state(struct pci_dev *pdev) 211 { 212 u16 control = PCI_PRI_CTRL_ENABLE; 213 u32 reqs = pdev->pri_reqs_alloc; 214 int pos; 215 216 if (!pdev->pri_enabled) 217 return; 218 219 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 220 if (!pos) 221 return; 222 223 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 224 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 225 } 226 EXPORT_SYMBOL_GPL(pci_restore_pri_state); 227 228 /** 229 * pci_reset_pri - Resets device's PRI state 230 * @pdev: PCI device structure 231 * 232 * The PRI capability must be disabled before this function is called. 233 * Returns 0 on success, negative value on error. 234 */ 235 int pci_reset_pri(struct pci_dev *pdev) 236 { 237 u16 control; 238 int pos; 239 240 if (WARN_ON(pdev->pri_enabled)) 241 return -EBUSY; 242 243 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 244 if (!pos) 245 return -EINVAL; 246 247 control = PCI_PRI_CTRL_RESET; 248 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 249 250 return 0; 251 } 252 EXPORT_SYMBOL_GPL(pci_reset_pri); 253 #endif /* CONFIG_PCI_PRI */ 254 255 #ifdef CONFIG_PCI_PASID 256 /** 257 * pci_enable_pasid - Enable the PASID capability 258 * @pdev: PCI device structure 259 * @features: Features to enable 260 * 261 * Returns 0 on success, negative value on error. This function checks 262 * whether the features are actually supported by the device and returns 263 * an error if not. 264 */ 265 int pci_enable_pasid(struct pci_dev *pdev, int features) 266 { 267 u16 control, supported; 268 int pos; 269 270 if (WARN_ON(pdev->pasid_enabled)) 271 return -EBUSY; 272 273 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 274 if (!pos) 275 return -EINVAL; 276 277 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 278 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 279 280 /* User wants to enable anything unsupported? */ 281 if ((supported & features) != features) 282 return -EINVAL; 283 284 control = PCI_PASID_CTRL_ENABLE | features; 285 pdev->pasid_features = features; 286 287 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 288 289 pdev->pasid_enabled = 1; 290 291 return 0; 292 } 293 EXPORT_SYMBOL_GPL(pci_enable_pasid); 294 295 /** 296 * pci_disable_pasid - Disable the PASID capability 297 * @pdev: PCI device structure 298 */ 299 void pci_disable_pasid(struct pci_dev *pdev) 300 { 301 u16 control = 0; 302 int pos; 303 304 if (WARN_ON(!pdev->pasid_enabled)) 305 return; 306 307 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 308 if (!pos) 309 return; 310 311 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 312 313 pdev->pasid_enabled = 0; 314 } 315 EXPORT_SYMBOL_GPL(pci_disable_pasid); 316 317 /** 318 * pci_restore_pasid_state - Restore PASID capabilities 319 * @pdev: PCI device structure 320 */ 321 void pci_restore_pasid_state(struct pci_dev *pdev) 322 { 323 u16 control; 324 int pos; 325 326 if (!pdev->pasid_enabled) 327 return; 328 329 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 330 if (!pos) 331 return; 332 333 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 334 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 335 } 336 EXPORT_SYMBOL_GPL(pci_restore_pasid_state); 337 338 /** 339 * pci_pasid_features - Check which PASID features are supported 340 * @pdev: PCI device structure 341 * 342 * Returns a negative value when no PASI capability is present. 343 * Otherwise is returns a bitmask with supported features. Current 344 * features reported are: 345 * PCI_PASID_CAP_EXEC - Execute permission supported 346 * PCI_PASID_CAP_PRIV - Privileged mode supported 347 */ 348 int pci_pasid_features(struct pci_dev *pdev) 349 { 350 u16 supported; 351 int pos; 352 353 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 354 if (!pos) 355 return -EINVAL; 356 357 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 358 359 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 360 361 return supported; 362 } 363 EXPORT_SYMBOL_GPL(pci_pasid_features); 364 365 #define PASID_NUMBER_SHIFT 8 366 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 367 /** 368 * pci_max_pasid - Get maximum number of PASIDs supported by device 369 * @pdev: PCI device structure 370 * 371 * Returns negative value when PASID capability is not present. 372 * Otherwise it returns the numer of supported PASIDs. 373 */ 374 int pci_max_pasids(struct pci_dev *pdev) 375 { 376 u16 supported; 377 int pos; 378 379 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 380 if (!pos) 381 return -EINVAL; 382 383 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 384 385 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 386 387 return (1 << supported); 388 } 389 EXPORT_SYMBOL_GPL(pci_max_pasids); 390 #endif /* CONFIG_PCI_PASID */ 391