1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Virtual Channel support 4 * 5 * Copyright (C) 2013 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 */ 8 9 #include <linux/device.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/pci_regs.h> 14 #include <linux/types.h> 15 16 /** 17 * pci_vc_save_restore_dwords - Save or restore a series of dwords 18 * @dev: device 19 * @pos: starting config space position 20 * @buf: buffer to save to or restore from 21 * @dwords: number of dwords to save/restore 22 * @save: whether to save or restore 23 */ 24 static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, 25 u32 *buf, int dwords, bool save) 26 { 27 int i; 28 29 for (i = 0; i < dwords; i++, buf++) { 30 if (save) 31 pci_read_config_dword(dev, pos + (i * 4), buf); 32 else 33 pci_write_config_dword(dev, pos + (i * 4), *buf); 34 } 35 } 36 37 /** 38 * pci_vc_load_arb_table - load and wait for VC arbitration table 39 * @dev: device 40 * @pos: starting position of VC capability (VC/VC9/MFVC) 41 * 42 * Set Load VC Arbitration Table bit requesting hardware to apply the VC 43 * Arbitration Table (previously loaded). When the VC Arbitration Table 44 * Status clears, hardware has latched the table into VC arbitration logic. 45 */ 46 static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) 47 { 48 u16 ctrl; 49 50 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); 51 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 52 ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); 53 if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, 54 PCI_VC_PORT_STATUS_TABLE)) 55 return; 56 57 pci_err(dev, "VC arbitration table failed to load\n"); 58 } 59 60 /** 61 * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table 62 * @dev: device 63 * @pos: starting position of VC capability (VC/VC9/MFVC) 64 * @res: VC resource number, ie. VCn (0-7) 65 * 66 * Set Load Port Arbitration Table bit requesting hardware to apply the Port 67 * Arbitration Table (previously loaded). When the Port Arbitration Table 68 * Status clears, hardware has latched the table into port arbitration logic. 69 */ 70 static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) 71 { 72 int ctrl_pos, status_pos; 73 u32 ctrl; 74 75 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 76 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 77 78 pci_read_config_dword(dev, ctrl_pos, &ctrl); 79 pci_write_config_dword(dev, ctrl_pos, 80 ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); 81 82 if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) 83 return; 84 85 pci_err(dev, "VC%d port arbitration table failed to load\n", res); 86 } 87 88 /** 89 * pci_vc_enable - Enable virtual channel 90 * @dev: device 91 * @pos: starting position of VC capability (VC/VC9/MFVC) 92 * @res: VC res number, ie. VCn (0-7) 93 * 94 * A VC is enabled by setting the enable bit in matching resource control 95 * registers on both sides of a link. We therefore need to find the opposite 96 * end of the link. To keep this simple we enable from the downstream device. 97 * RC devices do not have an upstream device, nor does it seem that VC9 do 98 * (spec is unclear). Once we find the upstream device, match the VC ID to 99 * get the correct resource, disable and enable on both ends. 100 */ 101 static void pci_vc_enable(struct pci_dev *dev, int pos, int res) 102 { 103 int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; 104 u32 ctrl, header, cap1, ctrl2; 105 struct pci_dev *link = NULL; 106 107 /* Enable VCs from the downstream device */ 108 if (!dev->has_secondary_link) 109 return; 110 111 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 112 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 113 114 pci_read_config_dword(dev, ctrl_pos, &ctrl); 115 id = ctrl & PCI_VC_RES_CTRL_ID; 116 117 pci_read_config_dword(dev, pos, &header); 118 119 /* If there is no opposite end of the link, skip to enable */ 120 if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || 121 pci_is_root_bus(dev->bus)) 122 goto enable; 123 124 pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); 125 if (!pos2) 126 goto enable; 127 128 pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); 129 evcc = cap1 & PCI_VC_CAP1_EVCC; 130 131 /* VC0 is hardwired enabled, so we can start with 1 */ 132 for (i = 1; i < evcc + 1; i++) { 133 ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + 134 (i * PCI_CAP_VC_PER_VC_SIZEOF); 135 status_pos2 = pos2 + PCI_VC_RES_STATUS + 136 (i * PCI_CAP_VC_PER_VC_SIZEOF); 137 pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); 138 if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { 139 link = dev->bus->self; 140 break; 141 } 142 } 143 144 if (!link) 145 goto enable; 146 147 /* Disable if enabled */ 148 if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { 149 ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; 150 pci_write_config_dword(link, ctrl_pos2, ctrl2); 151 } 152 153 /* Enable on both ends */ 154 ctrl2 |= PCI_VC_RES_CTRL_ENABLE; 155 pci_write_config_dword(link, ctrl_pos2, ctrl2); 156 enable: 157 ctrl |= PCI_VC_RES_CTRL_ENABLE; 158 pci_write_config_dword(dev, ctrl_pos, ctrl); 159 160 if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) 161 pci_err(dev, "VC%d negotiation stuck pending\n", id); 162 163 if (link && !pci_wait_for_pending(link, status_pos2, 164 PCI_VC_RES_STATUS_NEGO)) 165 pci_err(link, "VC%d negotiation stuck pending\n", id); 166 } 167 168 /** 169 * pci_vc_do_save_buffer - Size, save, or restore VC state 170 * @dev: device 171 * @pos: starting position of VC capability (VC/VC9/MFVC) 172 * @save_state: buffer for save/restore 173 * @name: for error message 174 * @save: if provided a buffer, this indicates what to do with it 175 * 176 * Walking Virtual Channel config space to size, save, or restore it 177 * is complicated, so we do it all from one function to reduce code and 178 * guarantee ordering matches in the buffer. When called with NULL 179 * @save_state, return the size of the necessary save buffer. When called 180 * with a non-NULL @save_state, @save determines whether we save to the 181 * buffer or restore from it. 182 */ 183 static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, 184 struct pci_cap_saved_state *save_state, 185 bool save) 186 { 187 u32 cap1; 188 char evcc, lpevcc, parb_size; 189 int i, len = 0; 190 u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; 191 192 /* Sanity check buffer size for save/restore */ 193 if (buf && save_state->cap.size != 194 pci_vc_do_save_buffer(dev, pos, NULL, save)) { 195 pci_err(dev, "VC save buffer size does not match @0x%x\n", pos); 196 return -ENOMEM; 197 } 198 199 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); 200 /* Extended VC Count (not counting VC0) */ 201 evcc = cap1 & PCI_VC_CAP1_EVCC; 202 /* Low Priority Extended VC Count (not counting VC0) */ 203 lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; 204 /* Port Arbitration Table Entry Size (bits) */ 205 parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); 206 207 /* 208 * Port VC Control Register contains VC Arbitration Select, which 209 * cannot be modified when more than one LPVC is in operation. We 210 * therefore save/restore it first, as only VC0 should be enabled 211 * after device reset. 212 */ 213 if (buf) { 214 if (save) 215 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, 216 (u16 *)buf); 217 else 218 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 219 *(u16 *)buf); 220 buf += 4; 221 } 222 len += 4; 223 224 /* 225 * If we have any Low Priority VCs and a VC Arbitration Table Offset 226 * in Port VC Capability Register 2 then save/restore it next. 227 */ 228 if (lpevcc) { 229 u32 cap2; 230 int vcarb_offset; 231 232 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); 233 vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; 234 235 if (vcarb_offset) { 236 int size, vcarb_phases = 0; 237 238 if (cap2 & PCI_VC_CAP2_128_PHASE) 239 vcarb_phases = 128; 240 else if (cap2 & PCI_VC_CAP2_64_PHASE) 241 vcarb_phases = 64; 242 else if (cap2 & PCI_VC_CAP2_32_PHASE) 243 vcarb_phases = 32; 244 245 /* Fixed 4 bits per phase per lpevcc (plus VC0) */ 246 size = ((lpevcc + 1) * vcarb_phases * 4) / 8; 247 248 if (size && buf) { 249 pci_vc_save_restore_dwords(dev, 250 pos + vcarb_offset, 251 (u32 *)buf, 252 size / 4, save); 253 /* 254 * On restore, we need to signal hardware to 255 * re-load the VC Arbitration Table. 256 */ 257 if (!save) 258 pci_vc_load_arb_table(dev, pos); 259 260 buf += size; 261 } 262 len += size; 263 } 264 } 265 266 /* 267 * In addition to each VC Resource Control Register, we may have a 268 * Port Arbitration Table attached to each VC. The Port Arbitration 269 * Table Offset in each VC Resource Capability Register tells us if 270 * it exists. The entry size is global from the Port VC Capability 271 * Register1 above. The number of phases is determined per VC. 272 */ 273 for (i = 0; i < evcc + 1; i++) { 274 u32 cap; 275 int parb_offset; 276 277 pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + 278 (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); 279 parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; 280 if (parb_offset) { 281 int size, parb_phases = 0; 282 283 if (cap & PCI_VC_RES_CAP_256_PHASE) 284 parb_phases = 256; 285 else if (cap & (PCI_VC_RES_CAP_128_PHASE | 286 PCI_VC_RES_CAP_128_PHASE_TB)) 287 parb_phases = 128; 288 else if (cap & PCI_VC_RES_CAP_64_PHASE) 289 parb_phases = 64; 290 else if (cap & PCI_VC_RES_CAP_32_PHASE) 291 parb_phases = 32; 292 293 size = (parb_size * parb_phases) / 8; 294 295 if (size && buf) { 296 pci_vc_save_restore_dwords(dev, 297 pos + parb_offset, 298 (u32 *)buf, 299 size / 4, save); 300 buf += size; 301 } 302 len += size; 303 } 304 305 /* VC Resource Control Register */ 306 if (buf) { 307 int ctrl_pos = pos + PCI_VC_RES_CTRL + 308 (i * PCI_CAP_VC_PER_VC_SIZEOF); 309 if (save) 310 pci_read_config_dword(dev, ctrl_pos, 311 (u32 *)buf); 312 else { 313 u32 tmp, ctrl = *(u32 *)buf; 314 /* 315 * For an FLR case, the VC config may remain. 316 * Preserve enable bit, restore the rest. 317 */ 318 pci_read_config_dword(dev, ctrl_pos, &tmp); 319 tmp &= PCI_VC_RES_CTRL_ENABLE; 320 tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; 321 pci_write_config_dword(dev, ctrl_pos, tmp); 322 /* Load port arbitration table if used */ 323 if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) 324 pci_vc_load_port_arb_table(dev, pos, i); 325 /* Re-enable if needed */ 326 if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) 327 pci_vc_enable(dev, pos, i); 328 } 329 buf += 4; 330 } 331 len += 4; 332 } 333 334 return buf ? 0 : len; 335 } 336 337 static struct { 338 u16 id; 339 const char *name; 340 } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, 341 { PCI_EXT_CAP_ID_VC, "VC" }, 342 { PCI_EXT_CAP_ID_VC9, "VC9" } }; 343 344 /** 345 * pci_save_vc_state - Save VC state to pre-allocate save buffer 346 * @dev: device 347 * 348 * For each type of VC capability, VC/VC9/MFVC, find the capability and 349 * save it to the pre-allocated save buffer. 350 */ 351 int pci_save_vc_state(struct pci_dev *dev) 352 { 353 int i; 354 355 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 356 int pos, ret; 357 struct pci_cap_saved_state *save_state; 358 359 pos = pci_find_ext_capability(dev, vc_caps[i].id); 360 if (!pos) 361 continue; 362 363 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 364 if (!save_state) { 365 pci_err(dev, "%s buffer not found in %s\n", 366 vc_caps[i].name, __func__); 367 return -ENOMEM; 368 } 369 370 ret = pci_vc_do_save_buffer(dev, pos, save_state, true); 371 if (ret) { 372 pci_err(dev, "%s save unsuccessful %s\n", 373 vc_caps[i].name, __func__); 374 return ret; 375 } 376 } 377 378 return 0; 379 } 380 381 /** 382 * pci_restore_vc_state - Restore VC state from save buffer 383 * @dev: device 384 * 385 * For each type of VC capability, VC/VC9/MFVC, find the capability and 386 * restore it from the previously saved buffer. 387 */ 388 void pci_restore_vc_state(struct pci_dev *dev) 389 { 390 int i; 391 392 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 393 int pos; 394 struct pci_cap_saved_state *save_state; 395 396 pos = pci_find_ext_capability(dev, vc_caps[i].id); 397 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 398 if (!save_state || !pos) 399 continue; 400 401 pci_vc_do_save_buffer(dev, pos, save_state, false); 402 } 403 } 404 405 /** 406 * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps 407 * @dev: device 408 * 409 * For each type of VC capability, VC/VC9/MFVC, find the capability, size 410 * it, and allocate a buffer for save/restore. 411 */ 412 413 void pci_allocate_vc_save_buffers(struct pci_dev *dev) 414 { 415 int i; 416 417 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 418 int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); 419 420 if (!pos) 421 continue; 422 423 len = pci_vc_do_save_buffer(dev, pos, NULL, false); 424 if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) 425 pci_err(dev, "unable to preallocate %s save buffer\n", 426 vc_caps[i].name); 427 } 428 } 429