1 /* 2 * PCI Virtual Channel support 3 * 4 * Copyright (C) 2013 Red Hat, Inc. All rights reserved. 5 * Author: Alex Williamson <alex.williamson@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/pci.h> 16 #include <linux/pci_regs.h> 17 #include <linux/types.h> 18 19 /** 20 * pci_vc_save_restore_dwords - Save or restore a series of dwords 21 * @dev: device 22 * @pos: starting config space position 23 * @buf: buffer to save to or restore from 24 * @dwords: number of dwords to save/restore 25 * @save: whether to save or restore 26 */ 27 static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, 28 u32 *buf, int dwords, bool save) 29 { 30 int i; 31 32 for (i = 0; i < dwords; i++, buf++) { 33 if (save) 34 pci_read_config_dword(dev, pos + (i * 4), buf); 35 else 36 pci_write_config_dword(dev, pos + (i * 4), *buf); 37 } 38 } 39 40 /** 41 * pci_vc_load_arb_table - load and wait for VC arbitration table 42 * @dev: device 43 * @pos: starting position of VC capability (VC/VC9/MFVC) 44 * 45 * Set Load VC Arbitration Table bit requesting hardware to apply the VC 46 * Arbitration Table (previously loaded). When the VC Arbitration Table 47 * Status clears, hardware has latched the table into VC arbitration logic. 48 */ 49 static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) 50 { 51 u16 ctrl; 52 53 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); 54 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 55 ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); 56 if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, 57 PCI_VC_PORT_STATUS_TABLE)) 58 return; 59 60 dev_err(&dev->dev, "VC arbitration table failed to load\n"); 61 } 62 63 /** 64 * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table 65 * @dev: device 66 * @pos: starting position of VC capability (VC/VC9/MFVC) 67 * @res: VC resource number, ie. VCn (0-7) 68 * 69 * Set Load Port Arbitration Table bit requesting hardware to apply the Port 70 * Arbitration Table (previously loaded). When the Port Arbitration Table 71 * Status clears, hardware has latched the table into port arbitration logic. 72 */ 73 static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) 74 { 75 int ctrl_pos, status_pos; 76 u32 ctrl; 77 78 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 79 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 80 81 pci_read_config_dword(dev, ctrl_pos, &ctrl); 82 pci_write_config_dword(dev, ctrl_pos, 83 ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); 84 85 if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) 86 return; 87 88 dev_err(&dev->dev, "VC%d port arbitration table failed to load\n", res); 89 } 90 91 /** 92 * pci_vc_enable - Enable virtual channel 93 * @dev: device 94 * @pos: starting position of VC capability (VC/VC9/MFVC) 95 * @res: VC res number, ie. VCn (0-7) 96 * 97 * A VC is enabled by setting the enable bit in matching resource control 98 * registers on both sides of a link. We therefore need to find the opposite 99 * end of the link. To keep this simple we enable from the downstream device. 100 * RC devices do not have an upstream device, nor does it seem that VC9 do 101 * (spec is unclear). Once we find the upstream device, match the VC ID to 102 * get the correct resource, disable and enable on both ends. 103 */ 104 static void pci_vc_enable(struct pci_dev *dev, int pos, int res) 105 { 106 int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; 107 u32 ctrl, header, cap1, ctrl2; 108 struct pci_dev *link = NULL; 109 110 /* Enable VCs from the downstream device */ 111 if (!dev->has_secondary_link) 112 return; 113 114 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 115 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 116 117 pci_read_config_dword(dev, ctrl_pos, &ctrl); 118 id = ctrl & PCI_VC_RES_CTRL_ID; 119 120 pci_read_config_dword(dev, pos, &header); 121 122 /* If there is no opposite end of the link, skip to enable */ 123 if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || 124 pci_is_root_bus(dev->bus)) 125 goto enable; 126 127 pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); 128 if (!pos2) 129 goto enable; 130 131 pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); 132 evcc = cap1 & PCI_VC_CAP1_EVCC; 133 134 /* VC0 is hardwired enabled, so we can start with 1 */ 135 for (i = 1; i < evcc + 1; i++) { 136 ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + 137 (i * PCI_CAP_VC_PER_VC_SIZEOF); 138 status_pos2 = pos2 + PCI_VC_RES_STATUS + 139 (i * PCI_CAP_VC_PER_VC_SIZEOF); 140 pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); 141 if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { 142 link = dev->bus->self; 143 break; 144 } 145 } 146 147 if (!link) 148 goto enable; 149 150 /* Disable if enabled */ 151 if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { 152 ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; 153 pci_write_config_dword(link, ctrl_pos2, ctrl2); 154 } 155 156 /* Enable on both ends */ 157 ctrl2 |= PCI_VC_RES_CTRL_ENABLE; 158 pci_write_config_dword(link, ctrl_pos2, ctrl2); 159 enable: 160 ctrl |= PCI_VC_RES_CTRL_ENABLE; 161 pci_write_config_dword(dev, ctrl_pos, ctrl); 162 163 if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) 164 dev_err(&dev->dev, "VC%d negotiation stuck pending\n", id); 165 166 if (link && !pci_wait_for_pending(link, status_pos2, 167 PCI_VC_RES_STATUS_NEGO)) 168 dev_err(&link->dev, "VC%d negotiation stuck pending\n", id); 169 } 170 171 /** 172 * pci_vc_do_save_buffer - Size, save, or restore VC state 173 * @dev: device 174 * @pos: starting position of VC capability (VC/VC9/MFVC) 175 * @save_state: buffer for save/restore 176 * @name: for error message 177 * @save: if provided a buffer, this indicates what to do with it 178 * 179 * Walking Virtual Channel config space to size, save, or restore it 180 * is complicated, so we do it all from one function to reduce code and 181 * guarantee ordering matches in the buffer. When called with NULL 182 * @save_state, return the size of the necessary save buffer. When called 183 * with a non-NULL @save_state, @save determines whether we save to the 184 * buffer or restore from it. 185 */ 186 static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, 187 struct pci_cap_saved_state *save_state, 188 bool save) 189 { 190 u32 cap1; 191 char evcc, lpevcc, parb_size; 192 int i, len = 0; 193 u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; 194 195 /* Sanity check buffer size for save/restore */ 196 if (buf && save_state->cap.size != 197 pci_vc_do_save_buffer(dev, pos, NULL, save)) { 198 dev_err(&dev->dev, 199 "VC save buffer size does not match @0x%x\n", pos); 200 return -ENOMEM; 201 } 202 203 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); 204 /* Extended VC Count (not counting VC0) */ 205 evcc = cap1 & PCI_VC_CAP1_EVCC; 206 /* Low Priority Extended VC Count (not counting VC0) */ 207 lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; 208 /* Port Arbitration Table Entry Size (bits) */ 209 parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); 210 211 /* 212 * Port VC Control Register contains VC Arbitration Select, which 213 * cannot be modified when more than one LPVC is in operation. We 214 * therefore save/restore it first, as only VC0 should be enabled 215 * after device reset. 216 */ 217 if (buf) { 218 if (save) 219 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, 220 (u16 *)buf); 221 else 222 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 223 *(u16 *)buf); 224 buf += 4; 225 } 226 len += 4; 227 228 /* 229 * If we have any Low Priority VCs and a VC Arbitration Table Offset 230 * in Port VC Capability Register 2 then save/restore it next. 231 */ 232 if (lpevcc) { 233 u32 cap2; 234 int vcarb_offset; 235 236 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); 237 vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; 238 239 if (vcarb_offset) { 240 int size, vcarb_phases = 0; 241 242 if (cap2 & PCI_VC_CAP2_128_PHASE) 243 vcarb_phases = 128; 244 else if (cap2 & PCI_VC_CAP2_64_PHASE) 245 vcarb_phases = 64; 246 else if (cap2 & PCI_VC_CAP2_32_PHASE) 247 vcarb_phases = 32; 248 249 /* Fixed 4 bits per phase per lpevcc (plus VC0) */ 250 size = ((lpevcc + 1) * vcarb_phases * 4) / 8; 251 252 if (size && buf) { 253 pci_vc_save_restore_dwords(dev, 254 pos + vcarb_offset, 255 (u32 *)buf, 256 size / 4, save); 257 /* 258 * On restore, we need to signal hardware to 259 * re-load the VC Arbitration Table. 260 */ 261 if (!save) 262 pci_vc_load_arb_table(dev, pos); 263 264 buf += size; 265 } 266 len += size; 267 } 268 } 269 270 /* 271 * In addition to each VC Resource Control Register, we may have a 272 * Port Arbitration Table attached to each VC. The Port Arbitration 273 * Table Offset in each VC Resource Capability Register tells us if 274 * it exists. The entry size is global from the Port VC Capability 275 * Register1 above. The number of phases is determined per VC. 276 */ 277 for (i = 0; i < evcc + 1; i++) { 278 u32 cap; 279 int parb_offset; 280 281 pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + 282 (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); 283 parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; 284 if (parb_offset) { 285 int size, parb_phases = 0; 286 287 if (cap & PCI_VC_RES_CAP_256_PHASE) 288 parb_phases = 256; 289 else if (cap & (PCI_VC_RES_CAP_128_PHASE | 290 PCI_VC_RES_CAP_128_PHASE_TB)) 291 parb_phases = 128; 292 else if (cap & PCI_VC_RES_CAP_64_PHASE) 293 parb_phases = 64; 294 else if (cap & PCI_VC_RES_CAP_32_PHASE) 295 parb_phases = 32; 296 297 size = (parb_size * parb_phases) / 8; 298 299 if (size && buf) { 300 pci_vc_save_restore_dwords(dev, 301 pos + parb_offset, 302 (u32 *)buf, 303 size / 4, save); 304 buf += size; 305 } 306 len += size; 307 } 308 309 /* VC Resource Control Register */ 310 if (buf) { 311 int ctrl_pos = pos + PCI_VC_RES_CTRL + 312 (i * PCI_CAP_VC_PER_VC_SIZEOF); 313 if (save) 314 pci_read_config_dword(dev, ctrl_pos, 315 (u32 *)buf); 316 else { 317 u32 tmp, ctrl = *(u32 *)buf; 318 /* 319 * For an FLR case, the VC config may remain. 320 * Preserve enable bit, restore the rest. 321 */ 322 pci_read_config_dword(dev, ctrl_pos, &tmp); 323 tmp &= PCI_VC_RES_CTRL_ENABLE; 324 tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; 325 pci_write_config_dword(dev, ctrl_pos, tmp); 326 /* Load port arbitration table if used */ 327 if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) 328 pci_vc_load_port_arb_table(dev, pos, i); 329 /* Re-enable if needed */ 330 if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) 331 pci_vc_enable(dev, pos, i); 332 } 333 buf += 4; 334 } 335 len += 4; 336 } 337 338 return buf ? 0 : len; 339 } 340 341 static struct { 342 u16 id; 343 const char *name; 344 } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, 345 { PCI_EXT_CAP_ID_VC, "VC" }, 346 { PCI_EXT_CAP_ID_VC9, "VC9" } }; 347 348 /** 349 * pci_save_vc_state - Save VC state to pre-allocate save buffer 350 * @dev: device 351 * 352 * For each type of VC capability, VC/VC9/MFVC, find the capability and 353 * save it to the pre-allocated save buffer. 354 */ 355 int pci_save_vc_state(struct pci_dev *dev) 356 { 357 int i; 358 359 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 360 int pos, ret; 361 struct pci_cap_saved_state *save_state; 362 363 pos = pci_find_ext_capability(dev, vc_caps[i].id); 364 if (!pos) 365 continue; 366 367 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 368 if (!save_state) { 369 dev_err(&dev->dev, "%s buffer not found in %s\n", 370 vc_caps[i].name, __func__); 371 return -ENOMEM; 372 } 373 374 ret = pci_vc_do_save_buffer(dev, pos, save_state, true); 375 if (ret) { 376 dev_err(&dev->dev, "%s save unsuccessful %s\n", 377 vc_caps[i].name, __func__); 378 return ret; 379 } 380 } 381 382 return 0; 383 } 384 385 /** 386 * pci_restore_vc_state - Restore VC state from save buffer 387 * @dev: device 388 * 389 * For each type of VC capability, VC/VC9/MFVC, find the capability and 390 * restore it from the previously saved buffer. 391 */ 392 void pci_restore_vc_state(struct pci_dev *dev) 393 { 394 int i; 395 396 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 397 int pos; 398 struct pci_cap_saved_state *save_state; 399 400 pos = pci_find_ext_capability(dev, vc_caps[i].id); 401 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 402 if (!save_state || !pos) 403 continue; 404 405 pci_vc_do_save_buffer(dev, pos, save_state, false); 406 } 407 } 408 409 /** 410 * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps 411 * @dev: device 412 * 413 * For each type of VC capability, VC/VC9/MFVC, find the capability, size 414 * it, and allocate a buffer for save/restore. 415 */ 416 417 void pci_allocate_vc_save_buffers(struct pci_dev *dev) 418 { 419 int i; 420 421 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 422 int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); 423 424 if (!pos) 425 continue; 426 427 len = pci_vc_do_save_buffer(dev, pos, NULL, false); 428 if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) 429 dev_err(&dev->dev, 430 "unable to preallocate %s save buffer\n", 431 vc_caps[i].name); 432 } 433 } 434