1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO PCI config space virtualization 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 /* 14 * This code handles reading and writing of PCI configuration registers. 15 * This is hairy because we want to allow a lot of flexibility to the 16 * user driver, but cannot trust it with all of the config fields. 17 * Tables determine which fields can be read and written, as well as 18 * which fields are 'virtualized' - special actions and translations to 19 * make it appear to the user that he has control, when in fact things 20 * must be negotiated with the underlying OS. 21 */ 22 23 #include <linux/fs.h> 24 #include <linux/pci.h> 25 #include <linux/uaccess.h> 26 #include <linux/vfio.h> 27 #include <linux/slab.h> 28 29 #include "vfio_pci_priv.h" 30 31 /* Fake capability ID for standard config space */ 32 #define PCI_CAP_ID_BASIC 0 33 34 #define is_bar(offset) \ 35 ((offset >= PCI_BASE_ADDRESS_0 && offset < PCI_BASE_ADDRESS_5 + 4) || \ 36 (offset >= PCI_ROM_ADDRESS && offset < PCI_ROM_ADDRESS + 4)) 37 38 /* 39 * Lengths of PCI Config Capabilities 40 * 0: Removed from the user visible capability list 41 * FF: Variable length 42 */ 43 static const u8 pci_cap_length[PCI_CAP_ID_MAX + 1] = { 44 [PCI_CAP_ID_BASIC] = PCI_STD_HEADER_SIZEOF, /* pci config header */ 45 [PCI_CAP_ID_PM] = PCI_PM_SIZEOF, 46 [PCI_CAP_ID_AGP] = PCI_AGP_SIZEOF, 47 [PCI_CAP_ID_VPD] = PCI_CAP_VPD_SIZEOF, 48 [PCI_CAP_ID_SLOTID] = 0, /* bridge - don't care */ 49 [PCI_CAP_ID_MSI] = 0xFF, /* 10, 14, 20, or 24 */ 50 [PCI_CAP_ID_CHSWP] = 0, /* cpci - not yet */ 51 [PCI_CAP_ID_PCIX] = 0xFF, /* 8 or 24 */ 52 [PCI_CAP_ID_HT] = 0xFF, /* hypertransport */ 53 [PCI_CAP_ID_VNDR] = 0xFF, /* variable */ 54 [PCI_CAP_ID_DBG] = 0, /* debug - don't care */ 55 [PCI_CAP_ID_CCRC] = 0, /* cpci - not yet */ 56 [PCI_CAP_ID_SHPC] = 0, /* hotswap - not yet */ 57 [PCI_CAP_ID_SSVID] = 0, /* bridge - don't care */ 58 [PCI_CAP_ID_AGP3] = 0, /* AGP8x - not yet */ 59 [PCI_CAP_ID_SECDEV] = 0, /* secure device not yet */ 60 [PCI_CAP_ID_EXP] = 0xFF, /* 20 or 44 */ 61 [PCI_CAP_ID_MSIX] = PCI_CAP_MSIX_SIZEOF, 62 [PCI_CAP_ID_SATA] = 0xFF, 63 [PCI_CAP_ID_AF] = PCI_CAP_AF_SIZEOF, 64 }; 65 66 /* 67 * Lengths of PCIe/PCI-X Extended Config Capabilities 68 * 0: Removed or masked from the user visible capability list 69 * FF: Variable length 70 */ 71 static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = { 72 [PCI_EXT_CAP_ID_ERR] = PCI_ERR_ROOT_COMMAND, 73 [PCI_EXT_CAP_ID_VC] = 0xFF, 74 [PCI_EXT_CAP_ID_DSN] = PCI_EXT_CAP_DSN_SIZEOF, 75 [PCI_EXT_CAP_ID_PWR] = PCI_EXT_CAP_PWR_SIZEOF, 76 [PCI_EXT_CAP_ID_RCLD] = 0, /* root only - don't care */ 77 [PCI_EXT_CAP_ID_RCILC] = 0, /* root only - don't care */ 78 [PCI_EXT_CAP_ID_RCEC] = 0, /* root only - don't care */ 79 [PCI_EXT_CAP_ID_MFVC] = 0xFF, 80 [PCI_EXT_CAP_ID_VC9] = 0xFF, /* same as CAP_ID_VC */ 81 [PCI_EXT_CAP_ID_RCRB] = 0, /* root only - don't care */ 82 [PCI_EXT_CAP_ID_VNDR] = 0xFF, 83 [PCI_EXT_CAP_ID_CAC] = 0, /* obsolete */ 84 [PCI_EXT_CAP_ID_ACS] = 0xFF, 85 [PCI_EXT_CAP_ID_ARI] = PCI_EXT_CAP_ARI_SIZEOF, 86 [PCI_EXT_CAP_ID_ATS] = PCI_EXT_CAP_ATS_SIZEOF, 87 [PCI_EXT_CAP_ID_SRIOV] = PCI_EXT_CAP_SRIOV_SIZEOF, 88 [PCI_EXT_CAP_ID_MRIOV] = 0, /* not yet */ 89 [PCI_EXT_CAP_ID_MCAST] = PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF, 90 [PCI_EXT_CAP_ID_PRI] = PCI_EXT_CAP_PRI_SIZEOF, 91 [PCI_EXT_CAP_ID_AMD_XXX] = 0, /* not yet */ 92 [PCI_EXT_CAP_ID_REBAR] = 0xFF, 93 [PCI_EXT_CAP_ID_DPA] = 0xFF, 94 [PCI_EXT_CAP_ID_TPH] = 0xFF, 95 [PCI_EXT_CAP_ID_LTR] = PCI_EXT_CAP_LTR_SIZEOF, 96 [PCI_EXT_CAP_ID_SECPCI] = 0, /* not yet */ 97 [PCI_EXT_CAP_ID_PMUX] = 0, /* not yet */ 98 [PCI_EXT_CAP_ID_PASID] = 0, /* not yet */ 99 [PCI_EXT_CAP_ID_DVSEC] = 0xFF, 100 }; 101 102 /* 103 * Read/Write Permission Bits - one bit for each bit in capability 104 * Any field can be read if it exists, but what is read depends on 105 * whether the field is 'virtualized', or just pass through to the 106 * hardware. Any virtualized field is also virtualized for writes. 107 * Writes are only permitted if they have a 1 bit here. 108 */ 109 struct perm_bits { 110 u8 *virt; /* read/write virtual data, not hw */ 111 u8 *write; /* writeable bits */ 112 int (*readfn)(struct vfio_pci_core_device *vdev, int pos, int count, 113 struct perm_bits *perm, int offset, __le32 *val); 114 int (*writefn)(struct vfio_pci_core_device *vdev, int pos, int count, 115 struct perm_bits *perm, int offset, __le32 val); 116 }; 117 118 #define NO_VIRT 0 119 #define ALL_VIRT 0xFFFFFFFFU 120 #define NO_WRITE 0 121 #define ALL_WRITE 0xFFFFFFFFU 122 123 static int vfio_user_config_read(struct pci_dev *pdev, int offset, 124 __le32 *val, int count) 125 { 126 int ret = -EINVAL; 127 u32 tmp_val = 0; 128 129 switch (count) { 130 case 1: 131 { 132 u8 tmp; 133 ret = pci_user_read_config_byte(pdev, offset, &tmp); 134 tmp_val = tmp; 135 break; 136 } 137 case 2: 138 { 139 u16 tmp; 140 ret = pci_user_read_config_word(pdev, offset, &tmp); 141 tmp_val = tmp; 142 break; 143 } 144 case 4: 145 ret = pci_user_read_config_dword(pdev, offset, &tmp_val); 146 break; 147 } 148 149 *val = cpu_to_le32(tmp_val); 150 151 return ret; 152 } 153 154 static int vfio_user_config_write(struct pci_dev *pdev, int offset, 155 __le32 val, int count) 156 { 157 int ret = -EINVAL; 158 u32 tmp_val = le32_to_cpu(val); 159 160 switch (count) { 161 case 1: 162 ret = pci_user_write_config_byte(pdev, offset, tmp_val); 163 break; 164 case 2: 165 ret = pci_user_write_config_word(pdev, offset, tmp_val); 166 break; 167 case 4: 168 ret = pci_user_write_config_dword(pdev, offset, tmp_val); 169 break; 170 } 171 172 return ret; 173 } 174 175 static int vfio_default_config_read(struct vfio_pci_core_device *vdev, int pos, 176 int count, struct perm_bits *perm, 177 int offset, __le32 *val) 178 { 179 __le32 virt = 0; 180 181 memcpy(val, vdev->vconfig + pos, count); 182 183 memcpy(&virt, perm->virt + offset, count); 184 185 /* Any non-virtualized bits? */ 186 if (cpu_to_le32(~0U >> (32 - (count * 8))) != virt) { 187 struct pci_dev *pdev = vdev->pdev; 188 __le32 phys_val = 0; 189 int ret; 190 191 ret = vfio_user_config_read(pdev, pos, &phys_val, count); 192 if (ret) 193 return ret; 194 195 *val = (phys_val & ~virt) | (*val & virt); 196 } 197 198 return count; 199 } 200 201 static int vfio_default_config_write(struct vfio_pci_core_device *vdev, int pos, 202 int count, struct perm_bits *perm, 203 int offset, __le32 val) 204 { 205 __le32 virt = 0, write = 0; 206 207 memcpy(&write, perm->write + offset, count); 208 209 if (!write) 210 return count; /* drop, no writable bits */ 211 212 memcpy(&virt, perm->virt + offset, count); 213 214 /* Virtualized and writable bits go to vconfig */ 215 if (write & virt) { 216 __le32 virt_val = 0; 217 218 memcpy(&virt_val, vdev->vconfig + pos, count); 219 220 virt_val &= ~(write & virt); 221 virt_val |= (val & (write & virt)); 222 223 memcpy(vdev->vconfig + pos, &virt_val, count); 224 } 225 226 /* Non-virtualized and writable bits go to hardware */ 227 if (write & ~virt) { 228 struct pci_dev *pdev = vdev->pdev; 229 __le32 phys_val = 0; 230 int ret; 231 232 ret = vfio_user_config_read(pdev, pos, &phys_val, count); 233 if (ret) 234 return ret; 235 236 phys_val &= ~(write & ~virt); 237 phys_val |= (val & (write & ~virt)); 238 239 ret = vfio_user_config_write(pdev, pos, phys_val, count); 240 if (ret) 241 return ret; 242 } 243 244 return count; 245 } 246 247 /* Allow direct read from hardware, except for capability next pointer */ 248 static int vfio_direct_config_read(struct vfio_pci_core_device *vdev, int pos, 249 int count, struct perm_bits *perm, 250 int offset, __le32 *val) 251 { 252 int ret; 253 254 ret = vfio_user_config_read(vdev->pdev, pos, val, count); 255 if (ret) 256 return ret; 257 258 if (pos >= PCI_CFG_SPACE_SIZE) { /* Extended cap header mangling */ 259 if (offset < 4) 260 memcpy(val, vdev->vconfig + pos, count); 261 } else if (pos >= PCI_STD_HEADER_SIZEOF) { /* Std cap mangling */ 262 if (offset == PCI_CAP_LIST_ID && count > 1) 263 memcpy(val, vdev->vconfig + pos, 264 min(PCI_CAP_FLAGS, count)); 265 else if (offset == PCI_CAP_LIST_NEXT) 266 memcpy(val, vdev->vconfig + pos, 1); 267 } 268 269 return count; 270 } 271 272 /* Raw access skips any kind of virtualization */ 273 static int vfio_raw_config_write(struct vfio_pci_core_device *vdev, int pos, 274 int count, struct perm_bits *perm, 275 int offset, __le32 val) 276 { 277 int ret; 278 279 ret = vfio_user_config_write(vdev->pdev, pos, val, count); 280 if (ret) 281 return ret; 282 283 return count; 284 } 285 286 static int vfio_raw_config_read(struct vfio_pci_core_device *vdev, int pos, 287 int count, struct perm_bits *perm, 288 int offset, __le32 *val) 289 { 290 int ret; 291 292 ret = vfio_user_config_read(vdev->pdev, pos, val, count); 293 if (ret) 294 return ret; 295 296 return count; 297 } 298 299 /* Virt access uses only virtualization */ 300 static int vfio_virt_config_write(struct vfio_pci_core_device *vdev, int pos, 301 int count, struct perm_bits *perm, 302 int offset, __le32 val) 303 { 304 memcpy(vdev->vconfig + pos, &val, count); 305 return count; 306 } 307 308 static int vfio_virt_config_read(struct vfio_pci_core_device *vdev, int pos, 309 int count, struct perm_bits *perm, 310 int offset, __le32 *val) 311 { 312 memcpy(val, vdev->vconfig + pos, count); 313 return count; 314 } 315 316 static struct perm_bits direct_ro_perms = { 317 .readfn = vfio_direct_config_read, 318 }; 319 320 /* Default capability regions to read-only, no-virtualization */ 321 static struct perm_bits cap_perms[PCI_CAP_ID_MAX + 1] = { 322 [0 ... PCI_CAP_ID_MAX] = { .readfn = vfio_direct_config_read } 323 }; 324 static struct perm_bits ecap_perms[PCI_EXT_CAP_ID_MAX + 1] = { 325 [0 ... PCI_EXT_CAP_ID_MAX] = { .readfn = vfio_direct_config_read } 326 }; 327 /* 328 * Default unassigned regions to raw read-write access. Some devices 329 * require this to function as they hide registers between the gaps in 330 * config space (be2net). Like MMIO and I/O port registers, we have 331 * to trust the hardware isolation. 332 */ 333 static struct perm_bits unassigned_perms = { 334 .readfn = vfio_raw_config_read, 335 .writefn = vfio_raw_config_write 336 }; 337 338 static struct perm_bits virt_perms = { 339 .readfn = vfio_virt_config_read, 340 .writefn = vfio_virt_config_write 341 }; 342 343 static void free_perm_bits(struct perm_bits *perm) 344 { 345 kfree(perm->virt); 346 kfree(perm->write); 347 perm->virt = NULL; 348 perm->write = NULL; 349 } 350 351 static int alloc_perm_bits(struct perm_bits *perm, int size) 352 { 353 /* 354 * Round up all permission bits to the next dword, this lets us 355 * ignore whether a read/write exceeds the defined capability 356 * structure. We can do this because: 357 * - Standard config space is already dword aligned 358 * - Capabilities are all dword aligned (bits 0:1 of next reserved) 359 * - Express capabilities defined as dword aligned 360 */ 361 size = round_up(size, 4); 362 363 /* 364 * Zero state is 365 * - All Readable, None Writeable, None Virtualized 366 */ 367 perm->virt = kzalloc(size, GFP_KERNEL); 368 perm->write = kzalloc(size, GFP_KERNEL); 369 if (!perm->virt || !perm->write) { 370 free_perm_bits(perm); 371 return -ENOMEM; 372 } 373 374 perm->readfn = vfio_default_config_read; 375 perm->writefn = vfio_default_config_write; 376 377 return 0; 378 } 379 380 /* 381 * Helper functions for filling in permission tables 382 */ 383 static inline void p_setb(struct perm_bits *p, int off, u8 virt, u8 write) 384 { 385 p->virt[off] = virt; 386 p->write[off] = write; 387 } 388 389 /* Handle endian-ness - pci and tables are little-endian */ 390 static inline void p_setw(struct perm_bits *p, int off, u16 virt, u16 write) 391 { 392 *(__le16 *)(&p->virt[off]) = cpu_to_le16(virt); 393 *(__le16 *)(&p->write[off]) = cpu_to_le16(write); 394 } 395 396 /* Handle endian-ness - pci and tables are little-endian */ 397 static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write) 398 { 399 *(__le32 *)(&p->virt[off]) = cpu_to_le32(virt); 400 *(__le32 *)(&p->write[off]) = cpu_to_le32(write); 401 } 402 403 /* Caller should hold memory_lock semaphore */ 404 bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev) 405 { 406 struct pci_dev *pdev = vdev->pdev; 407 u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); 408 409 /* 410 * Memory region cannot be accessed if device power state is D3. 411 * 412 * SR-IOV VF memory enable is handled by the MSE bit in the 413 * PF SR-IOV capability, there's therefore no need to trigger 414 * faults based on the virtual value. 415 */ 416 return pdev->current_state < PCI_D3hot && 417 (pdev->no_command_memory || (cmd & PCI_COMMAND_MEMORY)); 418 } 419 420 /* 421 * Restore the *real* BARs after we detect a FLR or backdoor reset. 422 * (backdoor = some device specific technique that we didn't catch) 423 */ 424 static void vfio_bar_restore(struct vfio_pci_core_device *vdev) 425 { 426 struct pci_dev *pdev = vdev->pdev; 427 u32 *rbar = vdev->rbar; 428 u16 cmd; 429 int i; 430 431 if (pdev->is_virtfn) 432 return; 433 434 pci_info(pdev, "%s: reset recovery - restoring BARs\n", __func__); 435 436 for (i = PCI_BASE_ADDRESS_0; i <= PCI_BASE_ADDRESS_5; i += 4, rbar++) 437 pci_user_write_config_dword(pdev, i, *rbar); 438 439 pci_user_write_config_dword(pdev, PCI_ROM_ADDRESS, *rbar); 440 441 if (vdev->nointx) { 442 pci_user_read_config_word(pdev, PCI_COMMAND, &cmd); 443 cmd |= PCI_COMMAND_INTX_DISABLE; 444 pci_user_write_config_word(pdev, PCI_COMMAND, cmd); 445 } 446 } 447 448 static __le32 vfio_generate_bar_flags(struct pci_dev *pdev, int bar) 449 { 450 unsigned long flags = pci_resource_flags(pdev, bar); 451 u32 val; 452 453 if (flags & IORESOURCE_IO) 454 return cpu_to_le32(PCI_BASE_ADDRESS_SPACE_IO); 455 456 val = PCI_BASE_ADDRESS_SPACE_MEMORY; 457 458 if (flags & IORESOURCE_PREFETCH) 459 val |= PCI_BASE_ADDRESS_MEM_PREFETCH; 460 461 if (flags & IORESOURCE_MEM_64) 462 val |= PCI_BASE_ADDRESS_MEM_TYPE_64; 463 464 return cpu_to_le32(val); 465 } 466 467 /* 468 * Pretend we're hardware and tweak the values of the *virtual* PCI BARs 469 * to reflect the hardware capabilities. This implements BAR sizing. 470 */ 471 static void vfio_bar_fixup(struct vfio_pci_core_device *vdev) 472 { 473 struct pci_dev *pdev = vdev->pdev; 474 int i; 475 __le32 *vbar; 476 u64 mask; 477 478 if (!vdev->bardirty) 479 return; 480 481 vbar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0]; 482 483 for (i = 0; i < PCI_STD_NUM_BARS; i++, vbar++) { 484 int bar = i + PCI_STD_RESOURCES; 485 486 if (!pci_resource_start(pdev, bar)) { 487 *vbar = 0; /* Unmapped by host = unimplemented to user */ 488 continue; 489 } 490 491 mask = ~(pci_resource_len(pdev, bar) - 1); 492 493 *vbar &= cpu_to_le32((u32)mask); 494 *vbar |= vfio_generate_bar_flags(pdev, bar); 495 496 if (*vbar & cpu_to_le32(PCI_BASE_ADDRESS_MEM_TYPE_64)) { 497 vbar++; 498 *vbar &= cpu_to_le32((u32)(mask >> 32)); 499 i++; 500 } 501 } 502 503 vbar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS]; 504 505 /* 506 * NB. REGION_INFO will have reported zero size if we weren't able 507 * to read the ROM, but we still return the actual BAR size here if 508 * it exists (or the shadow ROM space). 509 */ 510 if (pci_resource_start(pdev, PCI_ROM_RESOURCE)) { 511 mask = ~(pci_resource_len(pdev, PCI_ROM_RESOURCE) - 1); 512 mask |= PCI_ROM_ADDRESS_ENABLE; 513 *vbar &= cpu_to_le32((u32)mask); 514 } else if (pdev->resource[PCI_ROM_RESOURCE].flags & 515 IORESOURCE_ROM_SHADOW) { 516 mask = ~(0x20000 - 1); 517 mask |= PCI_ROM_ADDRESS_ENABLE; 518 *vbar &= cpu_to_le32((u32)mask); 519 } else 520 *vbar = 0; 521 522 vdev->bardirty = false; 523 } 524 525 static int vfio_basic_config_read(struct vfio_pci_core_device *vdev, int pos, 526 int count, struct perm_bits *perm, 527 int offset, __le32 *val) 528 { 529 if (is_bar(offset)) /* pos == offset for basic config */ 530 vfio_bar_fixup(vdev); 531 532 count = vfio_default_config_read(vdev, pos, count, perm, offset, val); 533 534 /* Mask in virtual memory enable */ 535 if (offset == PCI_COMMAND && vdev->pdev->no_command_memory) { 536 u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); 537 u32 tmp_val = le32_to_cpu(*val); 538 539 tmp_val |= cmd & PCI_COMMAND_MEMORY; 540 *val = cpu_to_le32(tmp_val); 541 } 542 543 return count; 544 } 545 546 /* Test whether BARs match the value we think they should contain */ 547 static bool vfio_need_bar_restore(struct vfio_pci_core_device *vdev) 548 { 549 int i = 0, pos = PCI_BASE_ADDRESS_0, ret; 550 u32 bar; 551 552 for (; pos <= PCI_BASE_ADDRESS_5; i++, pos += 4) { 553 if (vdev->rbar[i]) { 554 ret = pci_user_read_config_dword(vdev->pdev, pos, &bar); 555 if (ret || vdev->rbar[i] != bar) 556 return true; 557 } 558 } 559 560 return false; 561 } 562 563 static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos, 564 int count, struct perm_bits *perm, 565 int offset, __le32 val) 566 { 567 struct pci_dev *pdev = vdev->pdev; 568 __le16 *virt_cmd; 569 u16 new_cmd = 0; 570 int ret; 571 572 virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND]; 573 574 if (offset == PCI_COMMAND) { 575 bool phys_mem, virt_mem, new_mem, phys_io, virt_io, new_io; 576 u16 phys_cmd; 577 578 ret = pci_user_read_config_word(pdev, PCI_COMMAND, &phys_cmd); 579 if (ret) 580 return ret; 581 582 new_cmd = le32_to_cpu(val); 583 584 phys_io = !!(phys_cmd & PCI_COMMAND_IO); 585 virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO); 586 new_io = !!(new_cmd & PCI_COMMAND_IO); 587 588 phys_mem = !!(phys_cmd & PCI_COMMAND_MEMORY); 589 virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY); 590 new_mem = !!(new_cmd & PCI_COMMAND_MEMORY); 591 592 if (!new_mem) 593 vfio_pci_zap_and_down_write_memory_lock(vdev); 594 else 595 down_write(&vdev->memory_lock); 596 597 /* 598 * If the user is writing mem/io enable (new_mem/io) and we 599 * think it's already enabled (virt_mem/io), but the hardware 600 * shows it disabled (phys_mem/io, then the device has 601 * undergone some kind of backdoor reset and needs to be 602 * restored before we allow it to enable the bars. 603 * SR-IOV devices will trigger this - for mem enable let's 604 * catch this now and for io enable it will be caught later 605 */ 606 if ((new_mem && virt_mem && !phys_mem && 607 !pdev->no_command_memory) || 608 (new_io && virt_io && !phys_io) || 609 vfio_need_bar_restore(vdev)) 610 vfio_bar_restore(vdev); 611 } 612 613 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 614 if (count < 0) { 615 if (offset == PCI_COMMAND) 616 up_write(&vdev->memory_lock); 617 return count; 618 } 619 620 /* 621 * Save current memory/io enable bits in vconfig to allow for 622 * the test above next time. 623 */ 624 if (offset == PCI_COMMAND) { 625 u16 mask = PCI_COMMAND_MEMORY | PCI_COMMAND_IO; 626 627 *virt_cmd &= cpu_to_le16(~mask); 628 *virt_cmd |= cpu_to_le16(new_cmd & mask); 629 630 up_write(&vdev->memory_lock); 631 } 632 633 /* Emulate INTx disable */ 634 if (offset >= PCI_COMMAND && offset <= PCI_COMMAND + 1) { 635 bool virt_intx_disable; 636 637 virt_intx_disable = !!(le16_to_cpu(*virt_cmd) & 638 PCI_COMMAND_INTX_DISABLE); 639 640 if (virt_intx_disable && !vdev->virq_disabled) { 641 vdev->virq_disabled = true; 642 vfio_pci_intx_mask(vdev); 643 } else if (!virt_intx_disable && vdev->virq_disabled) { 644 vdev->virq_disabled = false; 645 vfio_pci_intx_unmask(vdev); 646 } 647 } 648 649 if (is_bar(offset)) 650 vdev->bardirty = true; 651 652 return count; 653 } 654 655 /* Permissions for the Basic PCI Header */ 656 static int __init init_pci_cap_basic_perm(struct perm_bits *perm) 657 { 658 if (alloc_perm_bits(perm, PCI_STD_HEADER_SIZEOF)) 659 return -ENOMEM; 660 661 perm->readfn = vfio_basic_config_read; 662 perm->writefn = vfio_basic_config_write; 663 664 /* Virtualized for SR-IOV functions, which just have FFFF */ 665 p_setw(perm, PCI_VENDOR_ID, (u16)ALL_VIRT, NO_WRITE); 666 p_setw(perm, PCI_DEVICE_ID, (u16)ALL_VIRT, NO_WRITE); 667 668 /* 669 * Virtualize INTx disable, we use it internally for interrupt 670 * control and can emulate it for non-PCI 2.3 devices. 671 */ 672 p_setw(perm, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE, (u16)ALL_WRITE); 673 674 /* Virtualize capability list, we might want to skip/disable */ 675 p_setw(perm, PCI_STATUS, PCI_STATUS_CAP_LIST, NO_WRITE); 676 677 /* No harm to write */ 678 p_setb(perm, PCI_CACHE_LINE_SIZE, NO_VIRT, (u8)ALL_WRITE); 679 p_setb(perm, PCI_LATENCY_TIMER, NO_VIRT, (u8)ALL_WRITE); 680 p_setb(perm, PCI_BIST, NO_VIRT, (u8)ALL_WRITE); 681 682 /* Virtualize all bars, can't touch the real ones */ 683 p_setd(perm, PCI_BASE_ADDRESS_0, ALL_VIRT, ALL_WRITE); 684 p_setd(perm, PCI_BASE_ADDRESS_1, ALL_VIRT, ALL_WRITE); 685 p_setd(perm, PCI_BASE_ADDRESS_2, ALL_VIRT, ALL_WRITE); 686 p_setd(perm, PCI_BASE_ADDRESS_3, ALL_VIRT, ALL_WRITE); 687 p_setd(perm, PCI_BASE_ADDRESS_4, ALL_VIRT, ALL_WRITE); 688 p_setd(perm, PCI_BASE_ADDRESS_5, ALL_VIRT, ALL_WRITE); 689 p_setd(perm, PCI_ROM_ADDRESS, ALL_VIRT, ALL_WRITE); 690 691 /* Allow us to adjust capability chain */ 692 p_setb(perm, PCI_CAPABILITY_LIST, (u8)ALL_VIRT, NO_WRITE); 693 694 /* Sometimes used by sw, just virtualize */ 695 p_setb(perm, PCI_INTERRUPT_LINE, (u8)ALL_VIRT, (u8)ALL_WRITE); 696 697 /* Virtualize interrupt pin to allow hiding INTx */ 698 p_setb(perm, PCI_INTERRUPT_PIN, (u8)ALL_VIRT, (u8)NO_WRITE); 699 700 return 0; 701 } 702 703 /* 704 * It takes all the required locks to protect the access of power related 705 * variables and then invokes vfio_pci_set_power_state(). 706 */ 707 static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev, 708 pci_power_t state) 709 { 710 if (state >= PCI_D3hot) 711 vfio_pci_zap_and_down_write_memory_lock(vdev); 712 else 713 down_write(&vdev->memory_lock); 714 715 vfio_pci_set_power_state(vdev, state); 716 up_write(&vdev->memory_lock); 717 } 718 719 static int vfio_pm_config_write(struct vfio_pci_core_device *vdev, int pos, 720 int count, struct perm_bits *perm, 721 int offset, __le32 val) 722 { 723 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 724 if (count < 0) 725 return count; 726 727 if (offset == PCI_PM_CTRL) { 728 pci_power_t state; 729 730 switch (le32_to_cpu(val) & PCI_PM_CTRL_STATE_MASK) { 731 case 0: 732 state = PCI_D0; 733 break; 734 case 1: 735 state = PCI_D1; 736 break; 737 case 2: 738 state = PCI_D2; 739 break; 740 case 3: 741 state = PCI_D3hot; 742 break; 743 } 744 745 vfio_lock_and_set_power_state(vdev, state); 746 } 747 748 return count; 749 } 750 751 /* Permissions for the Power Management capability */ 752 static int __init init_pci_cap_pm_perm(struct perm_bits *perm) 753 { 754 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_PM])) 755 return -ENOMEM; 756 757 perm->writefn = vfio_pm_config_write; 758 759 /* 760 * We always virtualize the next field so we can remove 761 * capabilities from the chain if we want to. 762 */ 763 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 764 765 /* 766 * The guests can't process PME events. If any PME event will be 767 * generated, then it will be mostly handled in the host and the 768 * host will clear the PME_STATUS. So virtualize PME_Support bits. 769 * The vconfig bits will be cleared during device capability 770 * initialization. 771 */ 772 p_setw(perm, PCI_PM_PMC, PCI_PM_CAP_PME_MASK, NO_WRITE); 773 774 /* 775 * Power management is defined *per function*, so we can let 776 * the user change power state, but we trap and initiate the 777 * change ourselves, so the state bits are read-only. 778 * 779 * The guest can't process PME from D3cold so virtualize PME_Status 780 * and PME_En bits. The vconfig bits will be cleared during device 781 * capability initialization. 782 */ 783 p_setd(perm, PCI_PM_CTRL, 784 PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS, 785 ~(PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS | 786 PCI_PM_CTRL_STATE_MASK)); 787 788 return 0; 789 } 790 791 static int vfio_vpd_config_write(struct vfio_pci_core_device *vdev, int pos, 792 int count, struct perm_bits *perm, 793 int offset, __le32 val) 794 { 795 struct pci_dev *pdev = vdev->pdev; 796 __le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR); 797 __le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA); 798 u16 addr; 799 u32 data; 800 801 /* 802 * Write through to emulation. If the write includes the upper byte 803 * of PCI_VPD_ADDR, then the PCI_VPD_ADDR_F bit is written and we 804 * have work to do. 805 */ 806 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 807 if (count < 0 || offset > PCI_VPD_ADDR + 1 || 808 offset + count <= PCI_VPD_ADDR + 1) 809 return count; 810 811 addr = le16_to_cpu(*paddr); 812 813 if (addr & PCI_VPD_ADDR_F) { 814 data = le32_to_cpu(*pdata); 815 if (pci_write_vpd(pdev, addr & ~PCI_VPD_ADDR_F, 4, &data) != 4) 816 return count; 817 } else { 818 data = 0; 819 if (pci_read_vpd(pdev, addr, 4, &data) < 0) 820 return count; 821 *pdata = cpu_to_le32(data); 822 } 823 824 /* 825 * Toggle PCI_VPD_ADDR_F in the emulated PCI_VPD_ADDR register to 826 * signal completion. If an error occurs above, we assume that not 827 * toggling this bit will induce a driver timeout. 828 */ 829 addr ^= PCI_VPD_ADDR_F; 830 *paddr = cpu_to_le16(addr); 831 832 return count; 833 } 834 835 /* Permissions for Vital Product Data capability */ 836 static int __init init_pci_cap_vpd_perm(struct perm_bits *perm) 837 { 838 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_VPD])) 839 return -ENOMEM; 840 841 perm->writefn = vfio_vpd_config_write; 842 843 /* 844 * We always virtualize the next field so we can remove 845 * capabilities from the chain if we want to. 846 */ 847 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 848 849 /* 850 * Both the address and data registers are virtualized to 851 * enable access through the pci_vpd_read/write functions 852 */ 853 p_setw(perm, PCI_VPD_ADDR, (u16)ALL_VIRT, (u16)ALL_WRITE); 854 p_setd(perm, PCI_VPD_DATA, ALL_VIRT, ALL_WRITE); 855 856 return 0; 857 } 858 859 /* Permissions for PCI-X capability */ 860 static int __init init_pci_cap_pcix_perm(struct perm_bits *perm) 861 { 862 /* Alloc 24, but only 8 are used in v0 */ 863 if (alloc_perm_bits(perm, PCI_CAP_PCIX_SIZEOF_V2)) 864 return -ENOMEM; 865 866 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 867 868 p_setw(perm, PCI_X_CMD, NO_VIRT, (u16)ALL_WRITE); 869 p_setd(perm, PCI_X_ECC_CSR, NO_VIRT, ALL_WRITE); 870 return 0; 871 } 872 873 static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos, 874 int count, struct perm_bits *perm, 875 int offset, __le32 val) 876 { 877 __le16 *ctrl = (__le16 *)(vdev->vconfig + pos - 878 offset + PCI_EXP_DEVCTL); 879 int readrq = le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ; 880 881 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 882 if (count < 0) 883 return count; 884 885 /* 886 * The FLR bit is virtualized, if set and the device supports PCIe 887 * FLR, issue a reset_function. Regardless, clear the bit, the spec 888 * requires it to be always read as zero. NB, reset_function might 889 * not use a PCIe FLR, we don't have that level of granularity. 890 */ 891 if (*ctrl & cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR)) { 892 u32 cap; 893 int ret; 894 895 *ctrl &= ~cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR); 896 897 ret = pci_user_read_config_dword(vdev->pdev, 898 pos - offset + PCI_EXP_DEVCAP, 899 &cap); 900 901 if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) { 902 vfio_pci_zap_and_down_write_memory_lock(vdev); 903 pci_try_reset_function(vdev->pdev); 904 up_write(&vdev->memory_lock); 905 } 906 } 907 908 /* 909 * MPS is virtualized to the user, writes do not change the physical 910 * register since determining a proper MPS value requires a system wide 911 * device view. The MRRS is largely independent of MPS, but since the 912 * user does not have that system-wide view, they might set a safe, but 913 * inefficiently low value. Here we allow writes through to hardware, 914 * but we set the floor to the physical device MPS setting, so that 915 * we can at least use full TLPs, as defined by the MPS value. 916 * 917 * NB, if any devices actually depend on an artificially low MRRS 918 * setting, this will need to be revisited, perhaps with a quirk 919 * though pcie_set_readrq(). 920 */ 921 if (readrq != (le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ)) { 922 readrq = 128 << 923 ((le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ) >> 12); 924 readrq = max(readrq, pcie_get_mps(vdev->pdev)); 925 926 pcie_set_readrq(vdev->pdev, readrq); 927 } 928 929 return count; 930 } 931 932 /* Permissions for PCI Express capability */ 933 static int __init init_pci_cap_exp_perm(struct perm_bits *perm) 934 { 935 /* Alloc largest of possible sizes */ 936 if (alloc_perm_bits(perm, PCI_CAP_EXP_ENDPOINT_SIZEOF_V2)) 937 return -ENOMEM; 938 939 perm->writefn = vfio_exp_config_write; 940 941 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 942 943 /* 944 * Allow writes to device control fields, except devctl_phantom, 945 * which could confuse IOMMU, MPS, which can break communication 946 * with other physical devices, and the ARI bit in devctl2, which 947 * is set at probe time. FLR and MRRS get virtualized via our 948 * writefn. 949 */ 950 p_setw(perm, PCI_EXP_DEVCTL, 951 PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_PAYLOAD | 952 PCI_EXP_DEVCTL_READRQ, ~PCI_EXP_DEVCTL_PHANTOM); 953 p_setw(perm, PCI_EXP_DEVCTL2, NO_VIRT, ~PCI_EXP_DEVCTL2_ARI); 954 return 0; 955 } 956 957 static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos, 958 int count, struct perm_bits *perm, 959 int offset, __le32 val) 960 { 961 u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL; 962 963 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 964 if (count < 0) 965 return count; 966 967 /* 968 * The FLR bit is virtualized, if set and the device supports AF 969 * FLR, issue a reset_function. Regardless, clear the bit, the spec 970 * requires it to be always read as zero. NB, reset_function might 971 * not use an AF FLR, we don't have that level of granularity. 972 */ 973 if (*ctrl & PCI_AF_CTRL_FLR) { 974 u8 cap; 975 int ret; 976 977 *ctrl &= ~PCI_AF_CTRL_FLR; 978 979 ret = pci_user_read_config_byte(vdev->pdev, 980 pos - offset + PCI_AF_CAP, 981 &cap); 982 983 if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) { 984 vfio_pci_zap_and_down_write_memory_lock(vdev); 985 pci_try_reset_function(vdev->pdev); 986 up_write(&vdev->memory_lock); 987 } 988 } 989 990 return count; 991 } 992 993 /* Permissions for Advanced Function capability */ 994 static int __init init_pci_cap_af_perm(struct perm_bits *perm) 995 { 996 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_AF])) 997 return -ENOMEM; 998 999 perm->writefn = vfio_af_config_write; 1000 1001 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 1002 p_setb(perm, PCI_AF_CTRL, PCI_AF_CTRL_FLR, PCI_AF_CTRL_FLR); 1003 return 0; 1004 } 1005 1006 /* Permissions for Advanced Error Reporting extended capability */ 1007 static int __init init_pci_ext_cap_err_perm(struct perm_bits *perm) 1008 { 1009 u32 mask; 1010 1011 if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_ERR])) 1012 return -ENOMEM; 1013 1014 /* 1015 * Virtualize the first dword of all express capabilities 1016 * because it includes the next pointer. This lets us later 1017 * remove capabilities from the chain if we need to. 1018 */ 1019 p_setd(perm, 0, ALL_VIRT, NO_WRITE); 1020 1021 /* Writable bits mask */ 1022 mask = PCI_ERR_UNC_UND | /* Undefined */ 1023 PCI_ERR_UNC_DLP | /* Data Link Protocol */ 1024 PCI_ERR_UNC_SURPDN | /* Surprise Down */ 1025 PCI_ERR_UNC_POISON_TLP | /* Poisoned TLP */ 1026 PCI_ERR_UNC_FCP | /* Flow Control Protocol */ 1027 PCI_ERR_UNC_COMP_TIME | /* Completion Timeout */ 1028 PCI_ERR_UNC_COMP_ABORT | /* Completer Abort */ 1029 PCI_ERR_UNC_UNX_COMP | /* Unexpected Completion */ 1030 PCI_ERR_UNC_RX_OVER | /* Receiver Overflow */ 1031 PCI_ERR_UNC_MALF_TLP | /* Malformed TLP */ 1032 PCI_ERR_UNC_ECRC | /* ECRC Error Status */ 1033 PCI_ERR_UNC_UNSUP | /* Unsupported Request */ 1034 PCI_ERR_UNC_ACSV | /* ACS Violation */ 1035 PCI_ERR_UNC_INTN | /* internal error */ 1036 PCI_ERR_UNC_MCBTLP | /* MC blocked TLP */ 1037 PCI_ERR_UNC_ATOMEG | /* Atomic egress blocked */ 1038 PCI_ERR_UNC_TLPPRE; /* TLP prefix blocked */ 1039 p_setd(perm, PCI_ERR_UNCOR_STATUS, NO_VIRT, mask); 1040 p_setd(perm, PCI_ERR_UNCOR_MASK, NO_VIRT, mask); 1041 p_setd(perm, PCI_ERR_UNCOR_SEVER, NO_VIRT, mask); 1042 1043 mask = PCI_ERR_COR_RCVR | /* Receiver Error Status */ 1044 PCI_ERR_COR_BAD_TLP | /* Bad TLP Status */ 1045 PCI_ERR_COR_BAD_DLLP | /* Bad DLLP Status */ 1046 PCI_ERR_COR_REP_ROLL | /* REPLAY_NUM Rollover */ 1047 PCI_ERR_COR_REP_TIMER | /* Replay Timer Timeout */ 1048 PCI_ERR_COR_ADV_NFAT | /* Advisory Non-Fatal */ 1049 PCI_ERR_COR_INTERNAL | /* Corrected Internal */ 1050 PCI_ERR_COR_LOG_OVER; /* Header Log Overflow */ 1051 p_setd(perm, PCI_ERR_COR_STATUS, NO_VIRT, mask); 1052 p_setd(perm, PCI_ERR_COR_MASK, NO_VIRT, mask); 1053 1054 mask = PCI_ERR_CAP_ECRC_GENE | /* ECRC Generation Enable */ 1055 PCI_ERR_CAP_ECRC_CHKE; /* ECRC Check Enable */ 1056 p_setd(perm, PCI_ERR_CAP, NO_VIRT, mask); 1057 return 0; 1058 } 1059 1060 /* Permissions for Power Budgeting extended capability */ 1061 static int __init init_pci_ext_cap_pwr_perm(struct perm_bits *perm) 1062 { 1063 if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_PWR])) 1064 return -ENOMEM; 1065 1066 p_setd(perm, 0, ALL_VIRT, NO_WRITE); 1067 1068 /* Writing the data selector is OK, the info is still read-only */ 1069 p_setb(perm, PCI_PWR_DATA, NO_VIRT, (u8)ALL_WRITE); 1070 return 0; 1071 } 1072 1073 /* 1074 * Initialize the shared permission tables 1075 */ 1076 void vfio_pci_uninit_perm_bits(void) 1077 { 1078 free_perm_bits(&cap_perms[PCI_CAP_ID_BASIC]); 1079 1080 free_perm_bits(&cap_perms[PCI_CAP_ID_PM]); 1081 free_perm_bits(&cap_perms[PCI_CAP_ID_VPD]); 1082 free_perm_bits(&cap_perms[PCI_CAP_ID_PCIX]); 1083 free_perm_bits(&cap_perms[PCI_CAP_ID_EXP]); 1084 free_perm_bits(&cap_perms[PCI_CAP_ID_AF]); 1085 1086 free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_ERR]); 1087 free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_PWR]); 1088 } 1089 1090 int __init vfio_pci_init_perm_bits(void) 1091 { 1092 int ret; 1093 1094 /* Basic config space */ 1095 ret = init_pci_cap_basic_perm(&cap_perms[PCI_CAP_ID_BASIC]); 1096 1097 /* Capabilities */ 1098 ret |= init_pci_cap_pm_perm(&cap_perms[PCI_CAP_ID_PM]); 1099 ret |= init_pci_cap_vpd_perm(&cap_perms[PCI_CAP_ID_VPD]); 1100 ret |= init_pci_cap_pcix_perm(&cap_perms[PCI_CAP_ID_PCIX]); 1101 cap_perms[PCI_CAP_ID_VNDR].writefn = vfio_raw_config_write; 1102 ret |= init_pci_cap_exp_perm(&cap_perms[PCI_CAP_ID_EXP]); 1103 ret |= init_pci_cap_af_perm(&cap_perms[PCI_CAP_ID_AF]); 1104 1105 /* Extended capabilities */ 1106 ret |= init_pci_ext_cap_err_perm(&ecap_perms[PCI_EXT_CAP_ID_ERR]); 1107 ret |= init_pci_ext_cap_pwr_perm(&ecap_perms[PCI_EXT_CAP_ID_PWR]); 1108 ecap_perms[PCI_EXT_CAP_ID_VNDR].writefn = vfio_raw_config_write; 1109 ecap_perms[PCI_EXT_CAP_ID_DVSEC].writefn = vfio_raw_config_write; 1110 1111 if (ret) 1112 vfio_pci_uninit_perm_bits(); 1113 1114 return ret; 1115 } 1116 1117 static int vfio_find_cap_start(struct vfio_pci_core_device *vdev, int pos) 1118 { 1119 u8 cap; 1120 int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE : 1121 PCI_STD_HEADER_SIZEOF; 1122 cap = vdev->pci_config_map[pos]; 1123 1124 if (cap == PCI_CAP_ID_BASIC) 1125 return 0; 1126 1127 /* XXX Can we have to abutting capabilities of the same type? */ 1128 while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap) 1129 pos--; 1130 1131 return pos; 1132 } 1133 1134 static int vfio_msi_config_read(struct vfio_pci_core_device *vdev, int pos, 1135 int count, struct perm_bits *perm, 1136 int offset, __le32 *val) 1137 { 1138 /* Update max available queue size from msi_qmax */ 1139 if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) { 1140 __le16 *flags; 1141 int start; 1142 1143 start = vfio_find_cap_start(vdev, pos); 1144 1145 flags = (__le16 *)&vdev->vconfig[start]; 1146 1147 *flags &= cpu_to_le16(~PCI_MSI_FLAGS_QMASK); 1148 *flags |= cpu_to_le16(vdev->msi_qmax << 1); 1149 } 1150 1151 return vfio_default_config_read(vdev, pos, count, perm, offset, val); 1152 } 1153 1154 static int vfio_msi_config_write(struct vfio_pci_core_device *vdev, int pos, 1155 int count, struct perm_bits *perm, 1156 int offset, __le32 val) 1157 { 1158 count = vfio_default_config_write(vdev, pos, count, perm, offset, val); 1159 if (count < 0) 1160 return count; 1161 1162 /* Fixup and write configured queue size and enable to hardware */ 1163 if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) { 1164 __le16 *pflags; 1165 u16 flags; 1166 int start, ret; 1167 1168 start = vfio_find_cap_start(vdev, pos); 1169 1170 pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS]; 1171 1172 flags = le16_to_cpu(*pflags); 1173 1174 /* MSI is enabled via ioctl */ 1175 if (vdev->irq_type != VFIO_PCI_MSI_IRQ_INDEX) 1176 flags &= ~PCI_MSI_FLAGS_ENABLE; 1177 1178 /* Check queue size */ 1179 if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) { 1180 flags &= ~PCI_MSI_FLAGS_QSIZE; 1181 flags |= vdev->msi_qmax << 4; 1182 } 1183 1184 /* Write back to virt and to hardware */ 1185 *pflags = cpu_to_le16(flags); 1186 ret = pci_user_write_config_word(vdev->pdev, 1187 start + PCI_MSI_FLAGS, 1188 flags); 1189 if (ret) 1190 return ret; 1191 } 1192 1193 return count; 1194 } 1195 1196 /* 1197 * MSI determination is per-device, so this routine gets used beyond 1198 * initialization time. Don't add __init 1199 */ 1200 static int init_pci_cap_msi_perm(struct perm_bits *perm, int len, u16 flags) 1201 { 1202 if (alloc_perm_bits(perm, len)) 1203 return -ENOMEM; 1204 1205 perm->readfn = vfio_msi_config_read; 1206 perm->writefn = vfio_msi_config_write; 1207 1208 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); 1209 1210 /* 1211 * The upper byte of the control register is reserved, 1212 * just setup the lower byte. 1213 */ 1214 p_setb(perm, PCI_MSI_FLAGS, (u8)ALL_VIRT, (u8)ALL_WRITE); 1215 p_setd(perm, PCI_MSI_ADDRESS_LO, ALL_VIRT, ALL_WRITE); 1216 if (flags & PCI_MSI_FLAGS_64BIT) { 1217 p_setd(perm, PCI_MSI_ADDRESS_HI, ALL_VIRT, ALL_WRITE); 1218 p_setw(perm, PCI_MSI_DATA_64, (u16)ALL_VIRT, (u16)ALL_WRITE); 1219 if (flags & PCI_MSI_FLAGS_MASKBIT) { 1220 p_setd(perm, PCI_MSI_MASK_64, NO_VIRT, ALL_WRITE); 1221 p_setd(perm, PCI_MSI_PENDING_64, NO_VIRT, ALL_WRITE); 1222 } 1223 } else { 1224 p_setw(perm, PCI_MSI_DATA_32, (u16)ALL_VIRT, (u16)ALL_WRITE); 1225 if (flags & PCI_MSI_FLAGS_MASKBIT) { 1226 p_setd(perm, PCI_MSI_MASK_32, NO_VIRT, ALL_WRITE); 1227 p_setd(perm, PCI_MSI_PENDING_32, NO_VIRT, ALL_WRITE); 1228 } 1229 } 1230 return 0; 1231 } 1232 1233 /* Determine MSI CAP field length; initialize msi_perms on 1st call per vdev */ 1234 static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos) 1235 { 1236 struct pci_dev *pdev = vdev->pdev; 1237 int len, ret; 1238 u16 flags; 1239 1240 ret = pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &flags); 1241 if (ret) 1242 return pcibios_err_to_errno(ret); 1243 1244 len = 10; /* Minimum size */ 1245 if (flags & PCI_MSI_FLAGS_64BIT) 1246 len += 4; 1247 if (flags & PCI_MSI_FLAGS_MASKBIT) 1248 len += 10; 1249 1250 if (vdev->msi_perm) 1251 return len; 1252 1253 vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL_ACCOUNT); 1254 if (!vdev->msi_perm) 1255 return -ENOMEM; 1256 1257 ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags); 1258 if (ret) { 1259 kfree(vdev->msi_perm); 1260 return ret; 1261 } 1262 1263 return len; 1264 } 1265 1266 /* Determine extended capability length for VC (2 & 9) and MFVC */ 1267 static int vfio_vc_cap_len(struct vfio_pci_core_device *vdev, u16 pos) 1268 { 1269 struct pci_dev *pdev = vdev->pdev; 1270 u32 tmp; 1271 int ret, evcc, phases, vc_arb; 1272 int len = PCI_CAP_VC_BASE_SIZEOF; 1273 1274 ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP1, &tmp); 1275 if (ret) 1276 return pcibios_err_to_errno(ret); 1277 1278 evcc = tmp & PCI_VC_CAP1_EVCC; /* extended vc count */ 1279 ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP2, &tmp); 1280 if (ret) 1281 return pcibios_err_to_errno(ret); 1282 1283 if (tmp & PCI_VC_CAP2_128_PHASE) 1284 phases = 128; 1285 else if (tmp & PCI_VC_CAP2_64_PHASE) 1286 phases = 64; 1287 else if (tmp & PCI_VC_CAP2_32_PHASE) 1288 phases = 32; 1289 else 1290 phases = 0; 1291 1292 vc_arb = phases * 4; 1293 1294 /* 1295 * Port arbitration tables are root & switch only; 1296 * function arbitration tables are function 0 only. 1297 * In either case, we'll never let user write them so 1298 * we don't care how big they are 1299 */ 1300 len += (1 + evcc) * PCI_CAP_VC_PER_VC_SIZEOF; 1301 if (vc_arb) { 1302 len = round_up(len, 16); 1303 len += vc_arb / 8; 1304 } 1305 return len; 1306 } 1307 1308 static int vfio_cap_len(struct vfio_pci_core_device *vdev, u8 cap, u8 pos) 1309 { 1310 struct pci_dev *pdev = vdev->pdev; 1311 u32 dword; 1312 u16 word; 1313 u8 byte; 1314 int ret; 1315 1316 switch (cap) { 1317 case PCI_CAP_ID_MSI: 1318 return vfio_msi_cap_len(vdev, pos); 1319 case PCI_CAP_ID_PCIX: 1320 ret = pci_read_config_word(pdev, pos + PCI_X_CMD, &word); 1321 if (ret) 1322 return pcibios_err_to_errno(ret); 1323 1324 if (PCI_X_CMD_VERSION(word)) { 1325 if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) { 1326 /* Test for extended capabilities */ 1327 pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, 1328 &dword); 1329 vdev->extended_caps = (dword != 0); 1330 } 1331 return PCI_CAP_PCIX_SIZEOF_V2; 1332 } else 1333 return PCI_CAP_PCIX_SIZEOF_V0; 1334 case PCI_CAP_ID_VNDR: 1335 /* length follows next field */ 1336 ret = pci_read_config_byte(pdev, pos + PCI_CAP_FLAGS, &byte); 1337 if (ret) 1338 return pcibios_err_to_errno(ret); 1339 1340 return byte; 1341 case PCI_CAP_ID_EXP: 1342 if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) { 1343 /* Test for extended capabilities */ 1344 pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &dword); 1345 vdev->extended_caps = (dword != 0); 1346 } 1347 1348 /* length based on version and type */ 1349 if ((pcie_caps_reg(pdev) & PCI_EXP_FLAGS_VERS) == 1) { 1350 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END) 1351 return 0xc; /* "All Devices" only, no link */ 1352 return PCI_CAP_EXP_ENDPOINT_SIZEOF_V1; 1353 } else { 1354 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END) 1355 return 0x2c; /* No link */ 1356 return PCI_CAP_EXP_ENDPOINT_SIZEOF_V2; 1357 } 1358 case PCI_CAP_ID_HT: 1359 ret = pci_read_config_byte(pdev, pos + 3, &byte); 1360 if (ret) 1361 return pcibios_err_to_errno(ret); 1362 1363 return (byte & HT_3BIT_CAP_MASK) ? 1364 HT_CAP_SIZEOF_SHORT : HT_CAP_SIZEOF_LONG; 1365 case PCI_CAP_ID_SATA: 1366 ret = pci_read_config_byte(pdev, pos + PCI_SATA_REGS, &byte); 1367 if (ret) 1368 return pcibios_err_to_errno(ret); 1369 1370 byte &= PCI_SATA_REGS_MASK; 1371 if (byte == PCI_SATA_REGS_INLINE) 1372 return PCI_SATA_SIZEOF_LONG; 1373 else 1374 return PCI_SATA_SIZEOF_SHORT; 1375 default: 1376 pci_warn(pdev, "%s: unknown length for PCI cap %#x@%#x\n", 1377 __func__, cap, pos); 1378 } 1379 1380 return 0; 1381 } 1382 1383 static int vfio_ext_cap_len(struct vfio_pci_core_device *vdev, u16 ecap, u16 epos) 1384 { 1385 struct pci_dev *pdev = vdev->pdev; 1386 u8 byte; 1387 u32 dword; 1388 int ret; 1389 1390 switch (ecap) { 1391 case PCI_EXT_CAP_ID_VNDR: 1392 ret = pci_read_config_dword(pdev, epos + PCI_VSEC_HDR, &dword); 1393 if (ret) 1394 return pcibios_err_to_errno(ret); 1395 1396 return dword >> PCI_VSEC_HDR_LEN_SHIFT; 1397 case PCI_EXT_CAP_ID_VC: 1398 case PCI_EXT_CAP_ID_VC9: 1399 case PCI_EXT_CAP_ID_MFVC: 1400 return vfio_vc_cap_len(vdev, epos); 1401 case PCI_EXT_CAP_ID_ACS: 1402 ret = pci_read_config_byte(pdev, epos + PCI_ACS_CAP, &byte); 1403 if (ret) 1404 return pcibios_err_to_errno(ret); 1405 1406 if (byte & PCI_ACS_EC) { 1407 int bits; 1408 1409 ret = pci_read_config_byte(pdev, 1410 epos + PCI_ACS_EGRESS_BITS, 1411 &byte); 1412 if (ret) 1413 return pcibios_err_to_errno(ret); 1414 1415 bits = byte ? round_up(byte, 32) : 256; 1416 return 8 + (bits / 8); 1417 } 1418 return 8; 1419 1420 case PCI_EXT_CAP_ID_REBAR: 1421 ret = pci_read_config_byte(pdev, epos + PCI_REBAR_CTRL, &byte); 1422 if (ret) 1423 return pcibios_err_to_errno(ret); 1424 1425 byte &= PCI_REBAR_CTRL_NBAR_MASK; 1426 byte >>= PCI_REBAR_CTRL_NBAR_SHIFT; 1427 1428 return 4 + (byte * 8); 1429 case PCI_EXT_CAP_ID_DPA: 1430 ret = pci_read_config_byte(pdev, epos + PCI_DPA_CAP, &byte); 1431 if (ret) 1432 return pcibios_err_to_errno(ret); 1433 1434 byte &= PCI_DPA_CAP_SUBSTATE_MASK; 1435 return PCI_DPA_BASE_SIZEOF + byte + 1; 1436 case PCI_EXT_CAP_ID_TPH: 1437 ret = pci_read_config_dword(pdev, epos + PCI_TPH_CAP, &dword); 1438 if (ret) 1439 return pcibios_err_to_errno(ret); 1440 1441 if ((dword & PCI_TPH_CAP_LOC_MASK) == PCI_TPH_LOC_CAP) { 1442 int sts; 1443 1444 sts = dword & PCI_TPH_CAP_ST_MASK; 1445 sts >>= PCI_TPH_CAP_ST_SHIFT; 1446 return PCI_TPH_BASE_SIZEOF + (sts * 2) + 2; 1447 } 1448 return PCI_TPH_BASE_SIZEOF; 1449 case PCI_EXT_CAP_ID_DVSEC: 1450 ret = pci_read_config_dword(pdev, epos + PCI_DVSEC_HEADER1, &dword); 1451 if (ret) 1452 return pcibios_err_to_errno(ret); 1453 return PCI_DVSEC_HEADER1_LEN(dword); 1454 default: 1455 pci_warn(pdev, "%s: unknown length for PCI ecap %#x@%#x\n", 1456 __func__, ecap, epos); 1457 } 1458 1459 return 0; 1460 } 1461 1462 static void vfio_update_pm_vconfig_bytes(struct vfio_pci_core_device *vdev, 1463 int offset) 1464 { 1465 __le16 *pmc = (__le16 *)&vdev->vconfig[offset + PCI_PM_PMC]; 1466 __le16 *ctrl = (__le16 *)&vdev->vconfig[offset + PCI_PM_CTRL]; 1467 1468 /* Clear vconfig PME_Support, PME_Status, and PME_En bits */ 1469 *pmc &= ~cpu_to_le16(PCI_PM_CAP_PME_MASK); 1470 *ctrl &= ~cpu_to_le16(PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS); 1471 } 1472 1473 static int vfio_fill_vconfig_bytes(struct vfio_pci_core_device *vdev, 1474 int offset, int size) 1475 { 1476 struct pci_dev *pdev = vdev->pdev; 1477 int ret = 0; 1478 1479 /* 1480 * We try to read physical config space in the largest chunks 1481 * we can, assuming that all of the fields support dword access. 1482 * pci_save_state() makes this same assumption and seems to do ok. 1483 */ 1484 while (size) { 1485 int filled; 1486 1487 if (size >= 4 && !(offset % 4)) { 1488 __le32 *dwordp = (__le32 *)&vdev->vconfig[offset]; 1489 u32 dword; 1490 1491 ret = pci_read_config_dword(pdev, offset, &dword); 1492 if (ret) 1493 return ret; 1494 *dwordp = cpu_to_le32(dword); 1495 filled = 4; 1496 } else if (size >= 2 && !(offset % 2)) { 1497 __le16 *wordp = (__le16 *)&vdev->vconfig[offset]; 1498 u16 word; 1499 1500 ret = pci_read_config_word(pdev, offset, &word); 1501 if (ret) 1502 return ret; 1503 *wordp = cpu_to_le16(word); 1504 filled = 2; 1505 } else { 1506 u8 *byte = &vdev->vconfig[offset]; 1507 ret = pci_read_config_byte(pdev, offset, byte); 1508 if (ret) 1509 return ret; 1510 filled = 1; 1511 } 1512 1513 offset += filled; 1514 size -= filled; 1515 } 1516 1517 return ret; 1518 } 1519 1520 static int vfio_cap_init(struct vfio_pci_core_device *vdev) 1521 { 1522 struct pci_dev *pdev = vdev->pdev; 1523 u8 *map = vdev->pci_config_map; 1524 u16 status; 1525 u8 pos, *prev, cap; 1526 int loops, ret, caps = 0; 1527 1528 /* Any capabilities? */ 1529 ret = pci_read_config_word(pdev, PCI_STATUS, &status); 1530 if (ret) 1531 return ret; 1532 1533 if (!(status & PCI_STATUS_CAP_LIST)) 1534 return 0; /* Done */ 1535 1536 ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos); 1537 if (ret) 1538 return ret; 1539 1540 /* Mark the previous position in case we want to skip a capability */ 1541 prev = &vdev->vconfig[PCI_CAPABILITY_LIST]; 1542 1543 /* We can bound our loop, capabilities are dword aligned */ 1544 loops = (PCI_CFG_SPACE_SIZE - PCI_STD_HEADER_SIZEOF) / PCI_CAP_SIZEOF; 1545 while (pos && loops--) { 1546 u8 next; 1547 int i, len = 0; 1548 1549 ret = pci_read_config_byte(pdev, pos, &cap); 1550 if (ret) 1551 return ret; 1552 1553 ret = pci_read_config_byte(pdev, 1554 pos + PCI_CAP_LIST_NEXT, &next); 1555 if (ret) 1556 return ret; 1557 1558 /* 1559 * ID 0 is a NULL capability, conflicting with our fake 1560 * PCI_CAP_ID_BASIC. As it has no content, consider it 1561 * hidden for now. 1562 */ 1563 if (cap && cap <= PCI_CAP_ID_MAX) { 1564 len = pci_cap_length[cap]; 1565 if (len == 0xFF) { /* Variable length */ 1566 len = vfio_cap_len(vdev, cap, pos); 1567 if (len < 0) 1568 return len; 1569 } 1570 } 1571 1572 if (!len) { 1573 pci_dbg(pdev, "%s: hiding cap %#x@%#x\n", __func__, 1574 cap, pos); 1575 *prev = next; 1576 pos = next; 1577 continue; 1578 } 1579 1580 /* Sanity check, do we overlap other capabilities? */ 1581 for (i = 0; i < len; i++) { 1582 if (likely(map[pos + i] == PCI_CAP_ID_INVALID)) 1583 continue; 1584 1585 pci_warn(pdev, "%s: PCI config conflict @%#x, was cap %#x now cap %#x\n", 1586 __func__, pos + i, map[pos + i], cap); 1587 } 1588 1589 BUILD_BUG_ON(PCI_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT); 1590 1591 memset(map + pos, cap, len); 1592 ret = vfio_fill_vconfig_bytes(vdev, pos, len); 1593 if (ret) 1594 return ret; 1595 1596 if (cap == PCI_CAP_ID_PM) 1597 vfio_update_pm_vconfig_bytes(vdev, pos); 1598 1599 prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT]; 1600 pos = next; 1601 caps++; 1602 } 1603 1604 /* If we didn't fill any capabilities, clear the status flag */ 1605 if (!caps) { 1606 __le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS]; 1607 *vstatus &= ~cpu_to_le16(PCI_STATUS_CAP_LIST); 1608 } 1609 1610 return 0; 1611 } 1612 1613 static int vfio_ecap_init(struct vfio_pci_core_device *vdev) 1614 { 1615 struct pci_dev *pdev = vdev->pdev; 1616 u8 *map = vdev->pci_config_map; 1617 u16 epos; 1618 __le32 *prev = NULL; 1619 int loops, ret, ecaps = 0; 1620 1621 if (!vdev->extended_caps) 1622 return 0; 1623 1624 epos = PCI_CFG_SPACE_SIZE; 1625 1626 loops = (pdev->cfg_size - PCI_CFG_SPACE_SIZE) / PCI_CAP_SIZEOF; 1627 1628 while (loops-- && epos >= PCI_CFG_SPACE_SIZE) { 1629 u32 header; 1630 u16 ecap; 1631 int i, len = 0; 1632 bool hidden = false; 1633 1634 ret = pci_read_config_dword(pdev, epos, &header); 1635 if (ret) 1636 return ret; 1637 1638 ecap = PCI_EXT_CAP_ID(header); 1639 1640 if (ecap <= PCI_EXT_CAP_ID_MAX) { 1641 len = pci_ext_cap_length[ecap]; 1642 if (len == 0xFF) { 1643 len = vfio_ext_cap_len(vdev, ecap, epos); 1644 if (len < 0) 1645 return len; 1646 } 1647 } 1648 1649 if (!len) { 1650 pci_dbg(pdev, "%s: hiding ecap %#x@%#x\n", 1651 __func__, ecap, epos); 1652 1653 /* If not the first in the chain, we can skip over it */ 1654 if (prev) { 1655 u32 val = epos = PCI_EXT_CAP_NEXT(header); 1656 *prev &= cpu_to_le32(~(0xffcU << 20)); 1657 *prev |= cpu_to_le32(val << 20); 1658 continue; 1659 } 1660 1661 /* 1662 * Otherwise, fill in a placeholder, the direct 1663 * readfn will virtualize this automatically 1664 */ 1665 len = PCI_CAP_SIZEOF; 1666 hidden = true; 1667 } 1668 1669 for (i = 0; i < len; i++) { 1670 if (likely(map[epos + i] == PCI_CAP_ID_INVALID)) 1671 continue; 1672 1673 pci_warn(pdev, "%s: PCI config conflict @%#x, was ecap %#x now ecap %#x\n", 1674 __func__, epos + i, map[epos + i], ecap); 1675 } 1676 1677 /* 1678 * Even though ecap is 2 bytes, we're currently a long way 1679 * from exceeding 1 byte capabilities. If we ever make it 1680 * up to 0xFE we'll need to up this to a two-byte, byte map. 1681 */ 1682 BUILD_BUG_ON(PCI_EXT_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT); 1683 1684 memset(map + epos, ecap, len); 1685 ret = vfio_fill_vconfig_bytes(vdev, epos, len); 1686 if (ret) 1687 return ret; 1688 1689 /* 1690 * If we're just using this capability to anchor the list, 1691 * hide the real ID. Only count real ecaps. XXX PCI spec 1692 * indicates to use cap id = 0, version = 0, next = 0 if 1693 * ecaps are absent, hope users check all the way to next. 1694 */ 1695 if (hidden) 1696 *(__le32 *)&vdev->vconfig[epos] &= 1697 cpu_to_le32((0xffcU << 20)); 1698 else 1699 ecaps++; 1700 1701 prev = (__le32 *)&vdev->vconfig[epos]; 1702 epos = PCI_EXT_CAP_NEXT(header); 1703 } 1704 1705 if (!ecaps) 1706 *(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0; 1707 1708 return 0; 1709 } 1710 1711 /* 1712 * Nag about hardware bugs, hopefully to have vendors fix them, but at least 1713 * to collect a list of dependencies for the VF INTx pin quirk below. 1714 */ 1715 static const struct pci_device_id known_bogus_vf_intx_pin[] = { 1716 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x270c) }, 1717 {} 1718 }; 1719 1720 /* 1721 * For each device we allocate a pci_config_map that indicates the 1722 * capability occupying each dword and thus the struct perm_bits we 1723 * use for read and write. We also allocate a virtualized config 1724 * space which tracks reads and writes to bits that we emulate for 1725 * the user. Initial values filled from device. 1726 * 1727 * Using shared struct perm_bits between all vfio-pci devices saves 1728 * us from allocating cfg_size buffers for virt and write for every 1729 * device. We could remove vconfig and allocate individual buffers 1730 * for each area requiring emulated bits, but the array of pointers 1731 * would be comparable in size (at least for standard config space). 1732 */ 1733 int vfio_config_init(struct vfio_pci_core_device *vdev) 1734 { 1735 struct pci_dev *pdev = vdev->pdev; 1736 u8 *map, *vconfig; 1737 int ret; 1738 1739 /* 1740 * Config space, caps and ecaps are all dword aligned, so we could 1741 * use one byte per dword to record the type. However, there are 1742 * no requirements on the length of a capability, so the gap between 1743 * capabilities needs byte granularity. 1744 */ 1745 map = kmalloc(pdev->cfg_size, GFP_KERNEL_ACCOUNT); 1746 if (!map) 1747 return -ENOMEM; 1748 1749 vconfig = kmalloc(pdev->cfg_size, GFP_KERNEL_ACCOUNT); 1750 if (!vconfig) { 1751 kfree(map); 1752 return -ENOMEM; 1753 } 1754 1755 vdev->pci_config_map = map; 1756 vdev->vconfig = vconfig; 1757 1758 memset(map, PCI_CAP_ID_BASIC, PCI_STD_HEADER_SIZEOF); 1759 memset(map + PCI_STD_HEADER_SIZEOF, PCI_CAP_ID_INVALID, 1760 pdev->cfg_size - PCI_STD_HEADER_SIZEOF); 1761 1762 ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF); 1763 if (ret) 1764 goto out; 1765 1766 vdev->bardirty = true; 1767 1768 /* 1769 * XXX can we just pci_load_saved_state/pci_restore_state? 1770 * may need to rebuild vconfig after that 1771 */ 1772 1773 /* For restore after reset */ 1774 vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]); 1775 vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]); 1776 vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]); 1777 vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]); 1778 vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]); 1779 vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]); 1780 vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]); 1781 1782 if (pdev->is_virtfn) { 1783 *(__le16 *)&vconfig[PCI_VENDOR_ID] = cpu_to_le16(pdev->vendor); 1784 *(__le16 *)&vconfig[PCI_DEVICE_ID] = cpu_to_le16(pdev->device); 1785 1786 /* 1787 * Per SR-IOV spec rev 1.1, 3.4.1.18 the interrupt pin register 1788 * does not apply to VFs and VFs must implement this register 1789 * as read-only with value zero. Userspace is not readily able 1790 * to identify whether a device is a VF and thus that the pin 1791 * definition on the device is bogus should it violate this 1792 * requirement. We already virtualize the pin register for 1793 * other purposes, so we simply need to replace the bogus value 1794 * and consider VFs when we determine INTx IRQ count. 1795 */ 1796 if (vconfig[PCI_INTERRUPT_PIN] && 1797 !pci_match_id(known_bogus_vf_intx_pin, pdev)) 1798 pci_warn(pdev, 1799 "Hardware bug: VF reports bogus INTx pin %d\n", 1800 vconfig[PCI_INTERRUPT_PIN]); 1801 1802 vconfig[PCI_INTERRUPT_PIN] = 0; /* Gratuitous for good VFs */ 1803 } 1804 if (pdev->no_command_memory) { 1805 /* 1806 * VFs and devices that set pdev->no_command_memory do not 1807 * implement the memory enable bit of the COMMAND register 1808 * therefore we'll not have it set in our initial copy of 1809 * config space after pci_enable_device(). For consistency 1810 * with PFs, set the virtual enable bit here. 1811 */ 1812 *(__le16 *)&vconfig[PCI_COMMAND] |= 1813 cpu_to_le16(PCI_COMMAND_MEMORY); 1814 } 1815 1816 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx) 1817 vconfig[PCI_INTERRUPT_PIN] = 0; 1818 1819 ret = vfio_cap_init(vdev); 1820 if (ret) 1821 goto out; 1822 1823 ret = vfio_ecap_init(vdev); 1824 if (ret) 1825 goto out; 1826 1827 return 0; 1828 1829 out: 1830 kfree(map); 1831 vdev->pci_config_map = NULL; 1832 kfree(vconfig); 1833 vdev->vconfig = NULL; 1834 return pcibios_err_to_errno(ret); 1835 } 1836 1837 void vfio_config_free(struct vfio_pci_core_device *vdev) 1838 { 1839 kfree(vdev->vconfig); 1840 vdev->vconfig = NULL; 1841 kfree(vdev->pci_config_map); 1842 vdev->pci_config_map = NULL; 1843 if (vdev->msi_perm) { 1844 free_perm_bits(vdev->msi_perm); 1845 kfree(vdev->msi_perm); 1846 vdev->msi_perm = NULL; 1847 } 1848 } 1849 1850 /* 1851 * Find the remaining number of bytes in a dword that match the given 1852 * position. Stop at either the end of the capability or the dword boundary. 1853 */ 1854 static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_core_device *vdev, 1855 loff_t pos) 1856 { 1857 u8 cap = vdev->pci_config_map[pos]; 1858 size_t i; 1859 1860 for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++) 1861 /* nop */; 1862 1863 return i; 1864 } 1865 1866 static ssize_t vfio_config_do_rw(struct vfio_pci_core_device *vdev, char __user *buf, 1867 size_t count, loff_t *ppos, bool iswrite) 1868 { 1869 struct pci_dev *pdev = vdev->pdev; 1870 struct perm_bits *perm; 1871 __le32 val = 0; 1872 int cap_start = 0, offset; 1873 u8 cap_id; 1874 ssize_t ret; 1875 1876 if (*ppos < 0 || *ppos >= pdev->cfg_size || 1877 *ppos + count > pdev->cfg_size) 1878 return -EFAULT; 1879 1880 /* 1881 * Chop accesses into aligned chunks containing no more than a 1882 * single capability. Caller increments to the next chunk. 1883 */ 1884 count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos)); 1885 if (count >= 4 && !(*ppos % 4)) 1886 count = 4; 1887 else if (count >= 2 && !(*ppos % 2)) 1888 count = 2; 1889 else 1890 count = 1; 1891 1892 ret = count; 1893 1894 cap_id = vdev->pci_config_map[*ppos]; 1895 1896 if (cap_id == PCI_CAP_ID_INVALID) { 1897 perm = &unassigned_perms; 1898 cap_start = *ppos; 1899 } else if (cap_id == PCI_CAP_ID_INVALID_VIRT) { 1900 perm = &virt_perms; 1901 cap_start = *ppos; 1902 } else { 1903 if (*ppos >= PCI_CFG_SPACE_SIZE) { 1904 /* 1905 * We can get a cap_id that exceeds PCI_EXT_CAP_ID_MAX 1906 * if we're hiding an unknown capability at the start 1907 * of the extended capability list. Use default, ro 1908 * access, which will virtualize the id and next values. 1909 */ 1910 if (cap_id > PCI_EXT_CAP_ID_MAX) 1911 perm = &direct_ro_perms; 1912 else 1913 perm = &ecap_perms[cap_id]; 1914 1915 cap_start = vfio_find_cap_start(vdev, *ppos); 1916 } else { 1917 WARN_ON(cap_id > PCI_CAP_ID_MAX); 1918 1919 perm = &cap_perms[cap_id]; 1920 1921 if (cap_id == PCI_CAP_ID_MSI) 1922 perm = vdev->msi_perm; 1923 1924 if (cap_id > PCI_CAP_ID_BASIC) 1925 cap_start = vfio_find_cap_start(vdev, *ppos); 1926 } 1927 } 1928 1929 WARN_ON(!cap_start && cap_id != PCI_CAP_ID_BASIC); 1930 WARN_ON(cap_start > *ppos); 1931 1932 offset = *ppos - cap_start; 1933 1934 if (iswrite) { 1935 if (!perm->writefn) 1936 return ret; 1937 1938 if (copy_from_user(&val, buf, count)) 1939 return -EFAULT; 1940 1941 ret = perm->writefn(vdev, *ppos, count, perm, offset, val); 1942 } else { 1943 if (perm->readfn) { 1944 ret = perm->readfn(vdev, *ppos, count, 1945 perm, offset, &val); 1946 if (ret < 0) 1947 return ret; 1948 } 1949 1950 if (copy_to_user(buf, &val, count)) 1951 return -EFAULT; 1952 } 1953 1954 return ret; 1955 } 1956 1957 ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, char __user *buf, 1958 size_t count, loff_t *ppos, bool iswrite) 1959 { 1960 size_t done = 0; 1961 int ret = 0; 1962 loff_t pos = *ppos; 1963 1964 pos &= VFIO_PCI_OFFSET_MASK; 1965 1966 while (count) { 1967 ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite); 1968 if (ret < 0) 1969 return ret; 1970 1971 count -= ret; 1972 done += ret; 1973 buf += ret; 1974 pos += ret; 1975 } 1976 1977 *ppos += done; 1978 1979 return done; 1980 } 1981