1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_enable_ats - enable the ATS capability 35 * @dev: the PCI device 36 * @ps: the IOMMU page shift 37 * 38 * Returns 0 on success, or negative on failure. 39 */ 40 int pci_enable_ats(struct pci_dev *dev, int ps) 41 { 42 u16 ctrl; 43 struct pci_dev *pdev; 44 45 if (!dev->ats_cap) 46 return -EINVAL; 47 48 if (WARN_ON(dev->ats_enabled)) 49 return -EBUSY; 50 51 if (ps < PCI_ATS_MIN_STU) 52 return -EINVAL; 53 54 /* 55 * Note that enabling ATS on a VF fails unless it's already enabled 56 * with the same STU on the PF. 57 */ 58 ctrl = PCI_ATS_CTRL_ENABLE; 59 if (dev->is_virtfn) { 60 pdev = pci_physfn(dev); 61 if (pdev->ats_stu != ps) 62 return -EINVAL; 63 64 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */ 65 } else { 66 dev->ats_stu = ps; 67 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 68 } 69 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 70 71 dev->ats_enabled = 1; 72 return 0; 73 } 74 EXPORT_SYMBOL_GPL(pci_enable_ats); 75 76 /** 77 * pci_disable_ats - disable the ATS capability 78 * @dev: the PCI device 79 */ 80 void pci_disable_ats(struct pci_dev *dev) 81 { 82 struct pci_dev *pdev; 83 u16 ctrl; 84 85 if (WARN_ON(!dev->ats_enabled)) 86 return; 87 88 if (atomic_read(&dev->ats_ref_cnt)) 89 return; /* VFs still enabled */ 90 91 if (dev->is_virtfn) { 92 pdev = pci_physfn(dev); 93 atomic_dec(&pdev->ats_ref_cnt); 94 } 95 96 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 97 ctrl &= ~PCI_ATS_CTRL_ENABLE; 98 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 99 100 dev->ats_enabled = 0; 101 } 102 EXPORT_SYMBOL_GPL(pci_disable_ats); 103 104 void pci_restore_ats_state(struct pci_dev *dev) 105 { 106 u16 ctrl; 107 108 if (!dev->ats_enabled) 109 return; 110 111 ctrl = PCI_ATS_CTRL_ENABLE; 112 if (!dev->is_virtfn) 113 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 114 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 115 } 116 EXPORT_SYMBOL_GPL(pci_restore_ats_state); 117 118 /** 119 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 120 * @dev: the PCI device 121 * 122 * Returns the queue depth on success, or negative on failure. 123 * 124 * The ATS spec uses 0 in the Invalidate Queue Depth field to 125 * indicate that the function can accept 32 Invalidate Request. 126 * But here we use the `real' values (i.e. 1~32) for the Queue 127 * Depth; and 0 indicates the function shares the Queue with 128 * other functions (doesn't exclusively own a Queue). 129 */ 130 int pci_ats_queue_depth(struct pci_dev *dev) 131 { 132 u16 cap; 133 134 if (!dev->ats_cap) 135 return -EINVAL; 136 137 if (dev->is_virtfn) 138 return 0; 139 140 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 141 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 142 } 143 EXPORT_SYMBOL_GPL(pci_ats_queue_depth); 144 145 #ifdef CONFIG_PCI_PRI 146 /** 147 * pci_enable_pri - Enable PRI capability 148 * @ pdev: PCI device structure 149 * 150 * Returns 0 on success, negative value on error 151 */ 152 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 153 { 154 u16 control, status; 155 u32 max_requests; 156 int pos; 157 158 if (WARN_ON(pdev->pri_enabled)) 159 return -EBUSY; 160 161 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 162 if (!pos) 163 return -EINVAL; 164 165 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); 166 if (!(status & PCI_PRI_STATUS_STOPPED)) 167 return -EBUSY; 168 169 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests); 170 reqs = min(max_requests, reqs); 171 pdev->pri_reqs_alloc = reqs; 172 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 173 174 control = PCI_PRI_CTRL_ENABLE; 175 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 176 177 pdev->pri_enabled = 1; 178 179 return 0; 180 } 181 EXPORT_SYMBOL_GPL(pci_enable_pri); 182 183 /** 184 * pci_disable_pri - Disable PRI capability 185 * @pdev: PCI device structure 186 * 187 * Only clears the enabled-bit, regardless of its former value 188 */ 189 void pci_disable_pri(struct pci_dev *pdev) 190 { 191 u16 control; 192 int pos; 193 194 if (WARN_ON(!pdev->pri_enabled)) 195 return; 196 197 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 198 if (!pos) 199 return; 200 201 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control); 202 control &= ~PCI_PRI_CTRL_ENABLE; 203 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 204 205 pdev->pri_enabled = 0; 206 } 207 EXPORT_SYMBOL_GPL(pci_disable_pri); 208 209 /** 210 * pci_restore_pri_state - Restore PRI 211 * @pdev: PCI device structure 212 */ 213 void pci_restore_pri_state(struct pci_dev *pdev) 214 { 215 u16 control = PCI_PRI_CTRL_ENABLE; 216 u32 reqs = pdev->pri_reqs_alloc; 217 int pos; 218 219 if (!pdev->pri_enabled) 220 return; 221 222 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 223 if (!pos) 224 return; 225 226 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 227 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 228 } 229 EXPORT_SYMBOL_GPL(pci_restore_pri_state); 230 231 /** 232 * pci_reset_pri - Resets device's PRI state 233 * @pdev: PCI device structure 234 * 235 * The PRI capability must be disabled before this function is called. 236 * Returns 0 on success, negative value on error. 237 */ 238 int pci_reset_pri(struct pci_dev *pdev) 239 { 240 u16 control; 241 int pos; 242 243 if (WARN_ON(pdev->pri_enabled)) 244 return -EBUSY; 245 246 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 247 if (!pos) 248 return -EINVAL; 249 250 control = PCI_PRI_CTRL_RESET; 251 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 252 253 return 0; 254 } 255 EXPORT_SYMBOL_GPL(pci_reset_pri); 256 #endif /* CONFIG_PCI_PRI */ 257 258 #ifdef CONFIG_PCI_PASID 259 /** 260 * pci_enable_pasid - Enable the PASID capability 261 * @pdev: PCI device structure 262 * @features: Features to enable 263 * 264 * Returns 0 on success, negative value on error. This function checks 265 * whether the features are actually supported by the device and returns 266 * an error if not. 267 */ 268 int pci_enable_pasid(struct pci_dev *pdev, int features) 269 { 270 u16 control, supported; 271 int pos; 272 273 if (WARN_ON(pdev->pasid_enabled)) 274 return -EBUSY; 275 276 if (!pdev->eetlp_prefix_path) 277 return -EINVAL; 278 279 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 280 if (!pos) 281 return -EINVAL; 282 283 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 284 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 285 286 /* User wants to enable anything unsupported? */ 287 if ((supported & features) != features) 288 return -EINVAL; 289 290 control = PCI_PASID_CTRL_ENABLE | features; 291 pdev->pasid_features = features; 292 293 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 294 295 pdev->pasid_enabled = 1; 296 297 return 0; 298 } 299 EXPORT_SYMBOL_GPL(pci_enable_pasid); 300 301 /** 302 * pci_disable_pasid - Disable the PASID capability 303 * @pdev: PCI device structure 304 */ 305 void pci_disable_pasid(struct pci_dev *pdev) 306 { 307 u16 control = 0; 308 int pos; 309 310 if (WARN_ON(!pdev->pasid_enabled)) 311 return; 312 313 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 314 if (!pos) 315 return; 316 317 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 318 319 pdev->pasid_enabled = 0; 320 } 321 EXPORT_SYMBOL_GPL(pci_disable_pasid); 322 323 /** 324 * pci_restore_pasid_state - Restore PASID capabilities 325 * @pdev: PCI device structure 326 */ 327 void pci_restore_pasid_state(struct pci_dev *pdev) 328 { 329 u16 control; 330 int pos; 331 332 if (!pdev->pasid_enabled) 333 return; 334 335 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 336 if (!pos) 337 return; 338 339 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 340 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 341 } 342 EXPORT_SYMBOL_GPL(pci_restore_pasid_state); 343 344 /** 345 * pci_pasid_features - Check which PASID features are supported 346 * @pdev: PCI device structure 347 * 348 * Returns a negative value when no PASI capability is present. 349 * Otherwise is returns a bitmask with supported features. Current 350 * features reported are: 351 * PCI_PASID_CAP_EXEC - Execute permission supported 352 * PCI_PASID_CAP_PRIV - Privileged mode supported 353 */ 354 int pci_pasid_features(struct pci_dev *pdev) 355 { 356 u16 supported; 357 int pos; 358 359 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 360 if (!pos) 361 return -EINVAL; 362 363 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 364 365 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 366 367 return supported; 368 } 369 EXPORT_SYMBOL_GPL(pci_pasid_features); 370 371 #define PASID_NUMBER_SHIFT 8 372 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 373 /** 374 * pci_max_pasid - Get maximum number of PASIDs supported by device 375 * @pdev: PCI device structure 376 * 377 * Returns negative value when PASID capability is not present. 378 * Otherwise it returns the numer of supported PASIDs. 379 */ 380 int pci_max_pasids(struct pci_dev *pdev) 381 { 382 u16 supported; 383 int pos; 384 385 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 386 if (!pos) 387 return -EINVAL; 388 389 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 390 391 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 392 393 return (1 << supported); 394 } 395 EXPORT_SYMBOL_GPL(pci_max_pasids); 396 #endif /* CONFIG_PCI_PASID */ 397