1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/export.h> 13 #include <linux/pci-ats.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 17 #include "pci.h" 18 19 void pci_ats_init(struct pci_dev *dev) 20 { 21 int pos; 22 23 if (pci_ats_disabled()) 24 return; 25 26 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 27 if (!pos) 28 return; 29 30 dev->ats_cap = pos; 31 } 32 33 /** 34 * pci_enable_ats - enable the ATS capability 35 * @dev: the PCI device 36 * @ps: the IOMMU page shift 37 * 38 * Returns 0 on success, or negative on failure. 39 */ 40 int pci_enable_ats(struct pci_dev *dev, int ps) 41 { 42 u16 ctrl; 43 struct pci_dev *pdev; 44 45 if (!dev->ats_cap) 46 return -EINVAL; 47 48 if (WARN_ON(dev->ats_enabled)) 49 return -EBUSY; 50 51 if (ps < PCI_ATS_MIN_STU) 52 return -EINVAL; 53 54 /* 55 * Note that enabling ATS on a VF fails unless it's already enabled 56 * with the same STU on the PF. 57 */ 58 ctrl = PCI_ATS_CTRL_ENABLE; 59 if (dev->is_virtfn) { 60 pdev = pci_physfn(dev); 61 if (pdev->ats_stu != ps) 62 return -EINVAL; 63 64 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */ 65 } else { 66 dev->ats_stu = ps; 67 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 68 } 69 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 70 71 dev->ats_enabled = 1; 72 return 0; 73 } 74 EXPORT_SYMBOL_GPL(pci_enable_ats); 75 76 /** 77 * pci_disable_ats - disable the ATS capability 78 * @dev: the PCI device 79 */ 80 void pci_disable_ats(struct pci_dev *dev) 81 { 82 struct pci_dev *pdev; 83 u16 ctrl; 84 85 if (WARN_ON(!dev->ats_enabled)) 86 return; 87 88 if (atomic_read(&dev->ats_ref_cnt)) 89 return; /* VFs still enabled */ 90 91 if (dev->is_virtfn) { 92 pdev = pci_physfn(dev); 93 atomic_dec(&pdev->ats_ref_cnt); 94 } 95 96 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 97 ctrl &= ~PCI_ATS_CTRL_ENABLE; 98 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 99 100 dev->ats_enabled = 0; 101 } 102 EXPORT_SYMBOL_GPL(pci_disable_ats); 103 104 void pci_restore_ats_state(struct pci_dev *dev) 105 { 106 u16 ctrl; 107 108 if (!dev->ats_enabled) 109 return; 110 111 ctrl = PCI_ATS_CTRL_ENABLE; 112 if (!dev->is_virtfn) 113 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 114 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 115 } 116 EXPORT_SYMBOL_GPL(pci_restore_ats_state); 117 118 /** 119 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 120 * @dev: the PCI device 121 * 122 * Returns the queue depth on success, or negative on failure. 123 * 124 * The ATS spec uses 0 in the Invalidate Queue Depth field to 125 * indicate that the function can accept 32 Invalidate Request. 126 * But here we use the `real' values (i.e. 1~32) for the Queue 127 * Depth; and 0 indicates the function shares the Queue with 128 * other functions (doesn't exclusively own a Queue). 129 */ 130 int pci_ats_queue_depth(struct pci_dev *dev) 131 { 132 u16 cap; 133 134 if (!dev->ats_cap) 135 return -EINVAL; 136 137 if (dev->is_virtfn) 138 return 0; 139 140 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 141 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 142 } 143 EXPORT_SYMBOL_GPL(pci_ats_queue_depth); 144 145 #ifdef CONFIG_PCI_PRI 146 /** 147 * pci_enable_pri - Enable PRI capability 148 * @ pdev: PCI device structure 149 * 150 * Returns 0 on success, negative value on error 151 */ 152 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 153 { 154 u16 control, status; 155 u32 max_requests; 156 int pos; 157 158 if (WARN_ON(pdev->pri_enabled)) 159 return -EBUSY; 160 161 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 162 if (!pos) 163 return -EINVAL; 164 165 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); 166 if (!(status & PCI_PRI_STATUS_STOPPED)) 167 return -EBUSY; 168 169 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests); 170 reqs = min(max_requests, reqs); 171 pdev->pri_reqs_alloc = reqs; 172 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 173 174 control = PCI_PRI_CTRL_ENABLE; 175 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 176 177 pdev->pri_enabled = 1; 178 179 return 0; 180 } 181 EXPORT_SYMBOL_GPL(pci_enable_pri); 182 183 /** 184 * pci_disable_pri - Disable PRI capability 185 * @pdev: PCI device structure 186 * 187 * Only clears the enabled-bit, regardless of its former value 188 */ 189 void pci_disable_pri(struct pci_dev *pdev) 190 { 191 u16 control; 192 int pos; 193 194 if (WARN_ON(!pdev->pri_enabled)) 195 return; 196 197 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 198 if (!pos) 199 return; 200 201 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control); 202 control &= ~PCI_PRI_CTRL_ENABLE; 203 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 204 205 pdev->pri_enabled = 0; 206 } 207 EXPORT_SYMBOL_GPL(pci_disable_pri); 208 209 /** 210 * pci_restore_pri_state - Restore PRI 211 * @pdev: PCI device structure 212 */ 213 void pci_restore_pri_state(struct pci_dev *pdev) 214 { 215 u16 control = PCI_PRI_CTRL_ENABLE; 216 u32 reqs = pdev->pri_reqs_alloc; 217 int pos; 218 219 if (!pdev->pri_enabled) 220 return; 221 222 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 223 if (!pos) 224 return; 225 226 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs); 227 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 228 } 229 EXPORT_SYMBOL_GPL(pci_restore_pri_state); 230 231 /** 232 * pci_reset_pri - Resets device's PRI state 233 * @pdev: PCI device structure 234 * 235 * The PRI capability must be disabled before this function is called. 236 * Returns 0 on success, negative value on error. 237 */ 238 int pci_reset_pri(struct pci_dev *pdev) 239 { 240 u16 control; 241 int pos; 242 243 if (WARN_ON(pdev->pri_enabled)) 244 return -EBUSY; 245 246 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 247 if (!pos) 248 return -EINVAL; 249 250 control = PCI_PRI_CTRL_RESET; 251 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); 252 253 return 0; 254 } 255 EXPORT_SYMBOL_GPL(pci_reset_pri); 256 #endif /* CONFIG_PCI_PRI */ 257 258 #ifdef CONFIG_PCI_PASID 259 /** 260 * pci_enable_pasid - Enable the PASID capability 261 * @pdev: PCI device structure 262 * @features: Features to enable 263 * 264 * Returns 0 on success, negative value on error. This function checks 265 * whether the features are actually supported by the device and returns 266 * an error if not. 267 */ 268 int pci_enable_pasid(struct pci_dev *pdev, int features) 269 { 270 u16 control, supported; 271 int pos; 272 273 if (WARN_ON(pdev->pasid_enabled)) 274 return -EBUSY; 275 276 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 277 if (!pos) 278 return -EINVAL; 279 280 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 281 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 282 283 /* User wants to enable anything unsupported? */ 284 if ((supported & features) != features) 285 return -EINVAL; 286 287 control = PCI_PASID_CTRL_ENABLE | features; 288 pdev->pasid_features = features; 289 290 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 291 292 pdev->pasid_enabled = 1; 293 294 return 0; 295 } 296 EXPORT_SYMBOL_GPL(pci_enable_pasid); 297 298 /** 299 * pci_disable_pasid - Disable the PASID capability 300 * @pdev: PCI device structure 301 */ 302 void pci_disable_pasid(struct pci_dev *pdev) 303 { 304 u16 control = 0; 305 int pos; 306 307 if (WARN_ON(!pdev->pasid_enabled)) 308 return; 309 310 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 311 if (!pos) 312 return; 313 314 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 315 316 pdev->pasid_enabled = 0; 317 } 318 EXPORT_SYMBOL_GPL(pci_disable_pasid); 319 320 /** 321 * pci_restore_pasid_state - Restore PASID capabilities 322 * @pdev: PCI device structure 323 */ 324 void pci_restore_pasid_state(struct pci_dev *pdev) 325 { 326 u16 control; 327 int pos; 328 329 if (!pdev->pasid_enabled) 330 return; 331 332 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 333 if (!pos) 334 return; 335 336 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 337 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control); 338 } 339 EXPORT_SYMBOL_GPL(pci_restore_pasid_state); 340 341 /** 342 * pci_pasid_features - Check which PASID features are supported 343 * @pdev: PCI device structure 344 * 345 * Returns a negative value when no PASI capability is present. 346 * Otherwise is returns a bitmask with supported features. Current 347 * features reported are: 348 * PCI_PASID_CAP_EXEC - Execute permission supported 349 * PCI_PASID_CAP_PRIV - Privileged mode supported 350 */ 351 int pci_pasid_features(struct pci_dev *pdev) 352 { 353 u16 supported; 354 int pos; 355 356 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 357 if (!pos) 358 return -EINVAL; 359 360 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 361 362 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 363 364 return supported; 365 } 366 EXPORT_SYMBOL_GPL(pci_pasid_features); 367 368 #define PASID_NUMBER_SHIFT 8 369 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) 370 /** 371 * pci_max_pasid - Get maximum number of PASIDs supported by device 372 * @pdev: PCI device structure 373 * 374 * Returns negative value when PASID capability is not present. 375 * Otherwise it returns the numer of supported PASIDs. 376 */ 377 int pci_max_pasids(struct pci_dev *pdev) 378 { 379 u16 supported; 380 int pos; 381 382 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 383 if (!pos) 384 return -EINVAL; 385 386 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported); 387 388 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT; 389 390 return (1 << supported); 391 } 392 EXPORT_SYMBOL_GPL(pci_max_pasids); 393 #endif /* CONFIG_PCI_PASID */ 394