1 /* 2 * Copyright © 2014-2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "display/intel_dp.h" 25 26 #include "intel_display_types.h" 27 #include "intel_dpio_phy.h" 28 #include "intel_sideband.h" 29 30 /** 31 * DOC: DPIO 32 * 33 * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI 34 * ports. DPIO is the name given to such a display PHY. These PHYs 35 * don't follow the standard programming model using direct MMIO 36 * registers, and instead their registers must be accessed trough IOSF 37 * sideband. VLV has one such PHY for driving ports B and C, and CHV 38 * adds another PHY for driving port D. Each PHY responds to specific 39 * IOSF-SB port. 40 * 41 * Each display PHY is made up of one or two channels. Each channel 42 * houses a common lane part which contains the PLL and other common 43 * logic. CH0 common lane also contains the IOSF-SB logic for the 44 * Common Register Interface (CRI) ie. the DPIO registers. CRI clock 45 * must be running when any DPIO registers are accessed. 46 * 47 * In addition to having their own registers, the PHYs are also 48 * controlled through some dedicated signals from the display 49 * controller. These include PLL reference clock enable, PLL enable, 50 * and CRI clock selection, for example. 51 * 52 * Eeach channel also has two splines (also called data lanes), and 53 * each spline is made up of one Physical Access Coding Sub-Layer 54 * (PCS) block and two TX lanes. So each channel has two PCS blocks 55 * and four TX lanes. The TX lanes are used as DP lanes or TMDS 56 * data/clock pairs depending on the output type. 57 * 58 * Additionally the PHY also contains an AUX lane with AUX blocks 59 * for each channel. This is used for DP AUX communication, but 60 * this fact isn't really relevant for the driver since AUX is 61 * controlled from the display controller side. No DPIO registers 62 * need to be accessed during AUX communication, 63 * 64 * Generally on VLV/CHV the common lane corresponds to the pipe and 65 * the spline (PCS/TX) corresponds to the port. 66 * 67 * For dual channel PHY (VLV/CHV): 68 * 69 * pipe A == CMN/PLL/REF CH0 70 * 71 * pipe B == CMN/PLL/REF CH1 72 * 73 * port B == PCS/TX CH0 74 * 75 * port C == PCS/TX CH1 76 * 77 * This is especially important when we cross the streams 78 * ie. drive port B with pipe B, or port C with pipe A. 79 * 80 * For single channel PHY (CHV): 81 * 82 * pipe C == CMN/PLL/REF CH0 83 * 84 * port D == PCS/TX CH0 85 * 86 * On BXT the entire PHY channel corresponds to the port. That means 87 * the PLL is also now associated with the port rather than the pipe, 88 * and so the clock needs to be routed to the appropriate transcoder. 89 * Port A PLL is directly connected to transcoder EDP and port B/C 90 * PLLs can be routed to any transcoder A/B/C. 91 * 92 * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is 93 * digital port D (CHV) or port A (BXT). :: 94 * 95 * 96 * Dual channel PHY (VLV/CHV/BXT) 97 * --------------------------------- 98 * | CH0 | CH1 | 99 * | CMN/PLL/REF | CMN/PLL/REF | 100 * |---------------|---------------| Display PHY 101 * | PCS01 | PCS23 | PCS01 | PCS23 | 102 * |-------|-------|-------|-------| 103 * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| 104 * --------------------------------- 105 * | DDI0 | DDI1 | DP/HDMI ports 106 * --------------------------------- 107 * 108 * Single channel PHY (CHV/BXT) 109 * ----------------- 110 * | CH0 | 111 * | CMN/PLL/REF | 112 * |---------------| Display PHY 113 * | PCS01 | PCS23 | 114 * |-------|-------| 115 * |TX0|TX1|TX2|TX3| 116 * ----------------- 117 * | DDI2 | DP/HDMI port 118 * ----------------- 119 */ 120 121 /** 122 * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy 123 */ 124 struct bxt_ddi_phy_info { 125 /** 126 * @dual_channel: true if this phy has a second channel. 127 */ 128 bool dual_channel; 129 130 /** 131 * @rcomp_phy: If -1, indicates this phy has its own rcomp resistor. 132 * Otherwise the GRC value will be copied from the phy indicated by 133 * this field. 134 */ 135 enum dpio_phy rcomp_phy; 136 137 /** 138 * @reset_delay: delay in us to wait before setting the common reset 139 * bit in BXT_PHY_CTL_FAMILY, which effectively enables the phy. 140 */ 141 int reset_delay; 142 143 /** 144 * @pwron_mask: Mask with the appropriate bit set that would cause the 145 * punit to power this phy if written to BXT_P_CR_GT_DISP_PWRON. 146 */ 147 u32 pwron_mask; 148 149 /** 150 * @channel: struct containing per channel information. 151 */ 152 struct { 153 /** 154 * @channel.port: which port maps to this channel. 155 */ 156 enum port port; 157 } channel[2]; 158 }; 159 160 static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = { 161 [DPIO_PHY0] = { 162 .dual_channel = true, 163 .rcomp_phy = DPIO_PHY1, 164 .pwron_mask = BIT(0), 165 166 .channel = { 167 [DPIO_CH0] = { .port = PORT_B }, 168 [DPIO_CH1] = { .port = PORT_C }, 169 } 170 }, 171 [DPIO_PHY1] = { 172 .dual_channel = false, 173 .rcomp_phy = -1, 174 .pwron_mask = BIT(1), 175 176 .channel = { 177 [DPIO_CH0] = { .port = PORT_A }, 178 } 179 }, 180 }; 181 182 static const struct bxt_ddi_phy_info glk_ddi_phy_info[] = { 183 [DPIO_PHY0] = { 184 .dual_channel = false, 185 .rcomp_phy = DPIO_PHY1, 186 .pwron_mask = BIT(0), 187 .reset_delay = 20, 188 189 .channel = { 190 [DPIO_CH0] = { .port = PORT_B }, 191 } 192 }, 193 [DPIO_PHY1] = { 194 .dual_channel = false, 195 .rcomp_phy = -1, 196 .pwron_mask = BIT(3), 197 .reset_delay = 20, 198 199 .channel = { 200 [DPIO_CH0] = { .port = PORT_A }, 201 } 202 }, 203 [DPIO_PHY2] = { 204 .dual_channel = false, 205 .rcomp_phy = DPIO_PHY1, 206 .pwron_mask = BIT(1), 207 .reset_delay = 20, 208 209 .channel = { 210 [DPIO_CH0] = { .port = PORT_C }, 211 } 212 }, 213 }; 214 215 static const struct bxt_ddi_phy_info * 216 bxt_get_phy_list(struct drm_i915_private *dev_priv, int *count) 217 { 218 if (IS_GEMINILAKE(dev_priv)) { 219 *count = ARRAY_SIZE(glk_ddi_phy_info); 220 return glk_ddi_phy_info; 221 } else { 222 *count = ARRAY_SIZE(bxt_ddi_phy_info); 223 return bxt_ddi_phy_info; 224 } 225 } 226 227 static const struct bxt_ddi_phy_info * 228 bxt_get_phy_info(struct drm_i915_private *dev_priv, enum dpio_phy phy) 229 { 230 int count; 231 const struct bxt_ddi_phy_info *phy_list = 232 bxt_get_phy_list(dev_priv, &count); 233 234 return &phy_list[phy]; 235 } 236 237 void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, 238 enum dpio_phy *phy, enum dpio_channel *ch) 239 { 240 const struct bxt_ddi_phy_info *phy_info, *phys; 241 int i, count; 242 243 phys = bxt_get_phy_list(dev_priv, &count); 244 245 for (i = 0; i < count; i++) { 246 phy_info = &phys[i]; 247 248 if (port == phy_info->channel[DPIO_CH0].port) { 249 *phy = i; 250 *ch = DPIO_CH0; 251 return; 252 } 253 254 if (phy_info->dual_channel && 255 port == phy_info->channel[DPIO_CH1].port) { 256 *phy = i; 257 *ch = DPIO_CH1; 258 return; 259 } 260 } 261 262 WARN(1, "PHY not found for PORT %c", port_name(port)); 263 *phy = DPIO_PHY0; 264 *ch = DPIO_CH0; 265 } 266 267 void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, 268 enum port port, u32 margin, u32 scale, 269 u32 enable, u32 deemphasis) 270 { 271 u32 val; 272 enum dpio_phy phy; 273 enum dpio_channel ch; 274 275 bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); 276 277 /* 278 * While we write to the group register to program all lanes at once we 279 * can read only lane registers and we pick lanes 0/1 for that. 280 */ 281 val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); 282 val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); 283 I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); 284 285 val = I915_READ(BXT_PORT_TX_DW2_LN0(phy, ch)); 286 val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); 287 val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT; 288 I915_WRITE(BXT_PORT_TX_DW2_GRP(phy, ch), val); 289 290 val = I915_READ(BXT_PORT_TX_DW3_LN0(phy, ch)); 291 val &= ~SCALE_DCOMP_METHOD; 292 if (enable) 293 val |= SCALE_DCOMP_METHOD; 294 295 if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) 296 DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); 297 298 I915_WRITE(BXT_PORT_TX_DW3_GRP(phy, ch), val); 299 300 val = I915_READ(BXT_PORT_TX_DW4_LN0(phy, ch)); 301 val &= ~DE_EMPHASIS; 302 val |= deemphasis << DEEMPH_SHIFT; 303 I915_WRITE(BXT_PORT_TX_DW4_GRP(phy, ch), val); 304 305 val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); 306 val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; 307 I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); 308 } 309 310 bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, 311 enum dpio_phy phy) 312 { 313 const struct bxt_ddi_phy_info *phy_info; 314 315 phy_info = bxt_get_phy_info(dev_priv, phy); 316 317 if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & phy_info->pwron_mask)) 318 return false; 319 320 if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & 321 (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { 322 DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", 323 phy); 324 325 return false; 326 } 327 328 if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { 329 DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", 330 phy); 331 332 return false; 333 } 334 335 return true; 336 } 337 338 static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) 339 { 340 u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); 341 342 return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; 343 } 344 345 static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, 346 enum dpio_phy phy) 347 { 348 if (intel_de_wait_for_set(dev_priv, BXT_PORT_REF_DW3(phy), 349 GRC_DONE, 10)) 350 DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); 351 } 352 353 static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, 354 enum dpio_phy phy) 355 { 356 const struct bxt_ddi_phy_info *phy_info; 357 u32 val; 358 359 phy_info = bxt_get_phy_info(dev_priv, phy); 360 361 if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { 362 /* Still read out the GRC value for state verification */ 363 if (phy_info->rcomp_phy != -1) 364 dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); 365 366 if (bxt_ddi_phy_verify_state(dev_priv, phy)) { 367 DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " 368 "won't reprogram it\n", phy); 369 return; 370 } 371 372 DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " 373 "force reprogramming it\n", phy); 374 } 375 376 val = I915_READ(BXT_P_CR_GT_DISP_PWRON); 377 val |= phy_info->pwron_mask; 378 I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); 379 380 /* 381 * The PHY registers start out inaccessible and respond to reads with 382 * all 1s. Eventually they become accessible as they power up, then 383 * the reserved bit will give the default 0. Poll on the reserved bit 384 * becoming 0 to find when the PHY is accessible. 385 * The flag should get set in 100us according to the HW team, but 386 * use 1ms due to occasional timeouts observed with that. 387 */ 388 if (intel_wait_for_register_fw(&dev_priv->uncore, 389 BXT_PORT_CL1CM_DW0(phy), 390 PHY_RESERVED | PHY_POWER_GOOD, 391 PHY_POWER_GOOD, 392 1)) 393 DRM_ERROR("timeout during PHY%d power on\n", phy); 394 395 /* Program PLL Rcomp code offset */ 396 val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); 397 val &= ~IREF0RC_OFFSET_MASK; 398 val |= 0xE4 << IREF0RC_OFFSET_SHIFT; 399 I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); 400 401 val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); 402 val &= ~IREF1RC_OFFSET_MASK; 403 val |= 0xE4 << IREF1RC_OFFSET_SHIFT; 404 I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); 405 406 /* Program power gating */ 407 val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); 408 val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | 409 SUS_CLK_CONFIG; 410 I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); 411 412 if (phy_info->dual_channel) { 413 val = I915_READ(BXT_PORT_CL2CM_DW6(phy)); 414 val |= DW6_OLDO_DYN_PWR_DOWN_EN; 415 I915_WRITE(BXT_PORT_CL2CM_DW6(phy), val); 416 } 417 418 if (phy_info->rcomp_phy != -1) { 419 u32 grc_code; 420 421 bxt_phy_wait_grc_done(dev_priv, phy_info->rcomp_phy); 422 423 /* 424 * PHY0 isn't connected to an RCOMP resistor so copy over 425 * the corresponding calibrated value from PHY1, and disable 426 * the automatic calibration on PHY0. 427 */ 428 val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, 429 phy_info->rcomp_phy); 430 grc_code = val << GRC_CODE_FAST_SHIFT | 431 val << GRC_CODE_SLOW_SHIFT | 432 val; 433 I915_WRITE(BXT_PORT_REF_DW6(phy), grc_code); 434 435 val = I915_READ(BXT_PORT_REF_DW8(phy)); 436 val |= GRC_DIS | GRC_RDY_OVRD; 437 I915_WRITE(BXT_PORT_REF_DW8(phy), val); 438 } 439 440 if (phy_info->reset_delay) 441 udelay(phy_info->reset_delay); 442 443 val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); 444 val |= COMMON_RESET_DIS; 445 I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); 446 } 447 448 void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) 449 { 450 const struct bxt_ddi_phy_info *phy_info; 451 u32 val; 452 453 phy_info = bxt_get_phy_info(dev_priv, phy); 454 455 val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); 456 val &= ~COMMON_RESET_DIS; 457 I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); 458 459 val = I915_READ(BXT_P_CR_GT_DISP_PWRON); 460 val &= ~phy_info->pwron_mask; 461 I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); 462 } 463 464 void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) 465 { 466 const struct bxt_ddi_phy_info *phy_info = 467 bxt_get_phy_info(dev_priv, phy); 468 enum dpio_phy rcomp_phy = phy_info->rcomp_phy; 469 bool was_enabled; 470 471 lockdep_assert_held(&dev_priv->power_domains.lock); 472 473 was_enabled = true; 474 if (rcomp_phy != -1) 475 was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy); 476 477 /* 478 * We need to copy the GRC calibration value from rcomp_phy, 479 * so make sure it's powered up. 480 */ 481 if (!was_enabled) 482 _bxt_ddi_phy_init(dev_priv, rcomp_phy); 483 484 _bxt_ddi_phy_init(dev_priv, phy); 485 486 if (!was_enabled) 487 bxt_ddi_phy_uninit(dev_priv, rcomp_phy); 488 } 489 490 static bool __printf(6, 7) 491 __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, 492 i915_reg_t reg, u32 mask, u32 expected, 493 const char *reg_fmt, ...) 494 { 495 struct va_format vaf; 496 va_list args; 497 u32 val; 498 499 val = I915_READ(reg); 500 if ((val & mask) == expected) 501 return true; 502 503 va_start(args, reg_fmt); 504 vaf.fmt = reg_fmt; 505 vaf.va = &args; 506 507 DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " 508 "current %08x, expected %08x (mask %08x)\n", 509 phy, &vaf, reg.reg, val, (val & ~mask) | expected, 510 mask); 511 512 va_end(args); 513 514 return false; 515 } 516 517 bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, 518 enum dpio_phy phy) 519 { 520 const struct bxt_ddi_phy_info *phy_info; 521 u32 mask; 522 bool ok; 523 524 phy_info = bxt_get_phy_info(dev_priv, phy); 525 526 #define _CHK(reg, mask, exp, fmt, ...) \ 527 __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ 528 ## __VA_ARGS__) 529 530 if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) 531 return false; 532 533 ok = true; 534 535 /* PLL Rcomp code offset */ 536 ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), 537 IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, 538 "BXT_PORT_CL1CM_DW9(%d)", phy); 539 ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), 540 IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, 541 "BXT_PORT_CL1CM_DW10(%d)", phy); 542 543 /* Power gating */ 544 mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; 545 ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, 546 "BXT_PORT_CL1CM_DW28(%d)", phy); 547 548 if (phy_info->dual_channel) 549 ok &= _CHK(BXT_PORT_CL2CM_DW6(phy), 550 DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, 551 "BXT_PORT_CL2CM_DW6(%d)", phy); 552 553 if (phy_info->rcomp_phy != -1) { 554 u32 grc_code = dev_priv->bxt_phy_grc; 555 556 grc_code = grc_code << GRC_CODE_FAST_SHIFT | 557 grc_code << GRC_CODE_SLOW_SHIFT | 558 grc_code; 559 mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | 560 GRC_CODE_NOM_MASK; 561 ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code, 562 "BXT_PORT_REF_DW6(%d)", phy); 563 564 mask = GRC_DIS | GRC_RDY_OVRD; 565 ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask, 566 "BXT_PORT_REF_DW8(%d)", phy); 567 } 568 569 return ok; 570 #undef _CHK 571 } 572 573 u8 574 bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count) 575 { 576 switch (lane_count) { 577 case 1: 578 return 0; 579 case 2: 580 return BIT(2) | BIT(0); 581 case 4: 582 return BIT(3) | BIT(2) | BIT(0); 583 default: 584 MISSING_CASE(lane_count); 585 586 return 0; 587 } 588 } 589 590 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, 591 u8 lane_lat_optim_mask) 592 { 593 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 594 enum port port = encoder->port; 595 enum dpio_phy phy; 596 enum dpio_channel ch; 597 int lane; 598 599 bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); 600 601 for (lane = 0; lane < 4; lane++) { 602 u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); 603 604 /* 605 * Note that on CHV this flag is called UPAR, but has 606 * the same function. 607 */ 608 val &= ~LATENCY_OPTIM; 609 if (lane_lat_optim_mask & BIT(lane)) 610 val |= LATENCY_OPTIM; 611 612 I915_WRITE(BXT_PORT_TX_DW14_LN(phy, ch, lane), val); 613 } 614 } 615 616 u8 617 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) 618 { 619 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 620 enum port port = encoder->port; 621 enum dpio_phy phy; 622 enum dpio_channel ch; 623 int lane; 624 u8 mask; 625 626 bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); 627 628 mask = 0; 629 for (lane = 0; lane < 4; lane++) { 630 u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); 631 632 if (val & LATENCY_OPTIM) 633 mask |= BIT(lane); 634 } 635 636 return mask; 637 } 638 639 640 void chv_set_phy_signal_level(struct intel_encoder *encoder, 641 u32 deemph_reg_value, u32 margin_reg_value, 642 bool uniq_trans_scale) 643 { 644 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 645 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 646 struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); 647 enum dpio_channel ch = vlv_dport_to_channel(dport); 648 enum pipe pipe = intel_crtc->pipe; 649 u32 val; 650 int i; 651 652 vlv_dpio_get(dev_priv); 653 654 /* Clear calc init */ 655 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); 656 val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); 657 val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); 658 val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; 659 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); 660 661 if (intel_crtc->config->lane_count > 2) { 662 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); 663 val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); 664 val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); 665 val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; 666 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); 667 } 668 669 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); 670 val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); 671 val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; 672 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); 673 674 if (intel_crtc->config->lane_count > 2) { 675 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); 676 val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); 677 val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; 678 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); 679 } 680 681 /* Program swing deemph */ 682 for (i = 0; i < intel_crtc->config->lane_count; i++) { 683 val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); 684 val &= ~DPIO_SWING_DEEMPH9P5_MASK; 685 val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; 686 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); 687 } 688 689 /* Program swing margin */ 690 for (i = 0; i < intel_crtc->config->lane_count; i++) { 691 val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); 692 693 val &= ~DPIO_SWING_MARGIN000_MASK; 694 val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; 695 696 /* 697 * Supposedly this value shouldn't matter when unique transition 698 * scale is disabled, but in fact it does matter. Let's just 699 * always program the same value and hope it's OK. 700 */ 701 val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); 702 val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; 703 704 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); 705 } 706 707 /* 708 * The document said it needs to set bit 27 for ch0 and bit 26 709 * for ch1. Might be a typo in the doc. 710 * For now, for this unique transition scale selection, set bit 711 * 27 for ch0 and ch1. 712 */ 713 for (i = 0; i < intel_crtc->config->lane_count; i++) { 714 val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); 715 if (uniq_trans_scale) 716 val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; 717 else 718 val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; 719 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); 720 } 721 722 /* Start swing calculation */ 723 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); 724 val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; 725 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); 726 727 if (intel_crtc->config->lane_count > 2) { 728 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); 729 val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; 730 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); 731 } 732 733 vlv_dpio_put(dev_priv); 734 } 735 736 void chv_data_lane_soft_reset(struct intel_encoder *encoder, 737 const struct intel_crtc_state *crtc_state, 738 bool reset) 739 { 740 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 741 enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); 742 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 743 enum pipe pipe = crtc->pipe; 744 u32 val; 745 746 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); 747 if (reset) 748 val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); 749 else 750 val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; 751 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); 752 753 if (crtc_state->lane_count > 2) { 754 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); 755 if (reset) 756 val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); 757 else 758 val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; 759 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); 760 } 761 762 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); 763 val |= CHV_PCS_REQ_SOFTRESET_EN; 764 if (reset) 765 val &= ~DPIO_PCS_CLK_SOFT_RESET; 766 else 767 val |= DPIO_PCS_CLK_SOFT_RESET; 768 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); 769 770 if (crtc_state->lane_count > 2) { 771 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); 772 val |= CHV_PCS_REQ_SOFTRESET_EN; 773 if (reset) 774 val &= ~DPIO_PCS_CLK_SOFT_RESET; 775 else 776 val |= DPIO_PCS_CLK_SOFT_RESET; 777 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); 778 } 779 } 780 781 void chv_phy_pre_pll_enable(struct intel_encoder *encoder, 782 const struct intel_crtc_state *crtc_state) 783 { 784 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 785 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 786 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 787 enum dpio_channel ch = vlv_dport_to_channel(dport); 788 enum pipe pipe = crtc->pipe; 789 unsigned int lane_mask = 790 intel_dp_unused_lane_mask(crtc_state->lane_count); 791 u32 val; 792 793 /* 794 * Must trick the second common lane into life. 795 * Otherwise we can't even access the PLL. 796 */ 797 if (ch == DPIO_CH0 && pipe == PIPE_B) 798 dport->release_cl2_override = 799 !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); 800 801 chv_phy_powergate_lanes(encoder, true, lane_mask); 802 803 vlv_dpio_get(dev_priv); 804 805 /* Assert data lane reset */ 806 chv_data_lane_soft_reset(encoder, crtc_state, true); 807 808 /* program left/right clock distribution */ 809 if (pipe != PIPE_B) { 810 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); 811 val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); 812 if (ch == DPIO_CH0) 813 val |= CHV_BUFLEFTENA1_FORCE; 814 if (ch == DPIO_CH1) 815 val |= CHV_BUFRIGHTENA1_FORCE; 816 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); 817 } else { 818 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); 819 val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); 820 if (ch == DPIO_CH0) 821 val |= CHV_BUFLEFTENA2_FORCE; 822 if (ch == DPIO_CH1) 823 val |= CHV_BUFRIGHTENA2_FORCE; 824 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); 825 } 826 827 /* program clock channel usage */ 828 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch)); 829 val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; 830 if (pipe != PIPE_B) 831 val &= ~CHV_PCS_USEDCLKCHANNEL; 832 else 833 val |= CHV_PCS_USEDCLKCHANNEL; 834 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); 835 836 if (crtc_state->lane_count > 2) { 837 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); 838 val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; 839 if (pipe != PIPE_B) 840 val &= ~CHV_PCS_USEDCLKCHANNEL; 841 else 842 val |= CHV_PCS_USEDCLKCHANNEL; 843 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); 844 } 845 846 /* 847 * This a a bit weird since generally CL 848 * matches the pipe, but here we need to 849 * pick the CL based on the port. 850 */ 851 val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch)); 852 if (pipe != PIPE_B) 853 val &= ~CHV_CMN_USEDCLKCHANNEL; 854 else 855 val |= CHV_CMN_USEDCLKCHANNEL; 856 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val); 857 858 vlv_dpio_put(dev_priv); 859 } 860 861 void chv_phy_pre_encoder_enable(struct intel_encoder *encoder, 862 const struct intel_crtc_state *crtc_state) 863 { 864 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 865 struct intel_digital_port *dport = dp_to_dig_port(intel_dp); 866 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 867 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 868 enum dpio_channel ch = vlv_dport_to_channel(dport); 869 enum pipe pipe = crtc->pipe; 870 int data, i, stagger; 871 u32 val; 872 873 vlv_dpio_get(dev_priv); 874 875 /* allow hardware to manage TX FIFO reset source */ 876 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); 877 val &= ~DPIO_LANEDESKEW_STRAP_OVRD; 878 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); 879 880 if (crtc_state->lane_count > 2) { 881 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); 882 val &= ~DPIO_LANEDESKEW_STRAP_OVRD; 883 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); 884 } 885 886 /* Program Tx lane latency optimal setting*/ 887 for (i = 0; i < crtc_state->lane_count; i++) { 888 /* Set the upar bit */ 889 if (crtc_state->lane_count == 1) 890 data = 0x0; 891 else 892 data = (i == 1) ? 0x0 : 0x1; 893 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), 894 data << DPIO_UPAR_SHIFT); 895 } 896 897 /* Data lane stagger programming */ 898 if (crtc_state->port_clock > 270000) 899 stagger = 0x18; 900 else if (crtc_state->port_clock > 135000) 901 stagger = 0xd; 902 else if (crtc_state->port_clock > 67500) 903 stagger = 0x7; 904 else if (crtc_state->port_clock > 33750) 905 stagger = 0x4; 906 else 907 stagger = 0x2; 908 909 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); 910 val |= DPIO_TX2_STAGGER_MASK(0x1f); 911 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); 912 913 if (crtc_state->lane_count > 2) { 914 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); 915 val |= DPIO_TX2_STAGGER_MASK(0x1f); 916 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); 917 } 918 919 vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), 920 DPIO_LANESTAGGER_STRAP(stagger) | 921 DPIO_LANESTAGGER_STRAP_OVRD | 922 DPIO_TX1_STAGGER_MASK(0x1f) | 923 DPIO_TX1_STAGGER_MULT(6) | 924 DPIO_TX2_STAGGER_MULT(0)); 925 926 if (crtc_state->lane_count > 2) { 927 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), 928 DPIO_LANESTAGGER_STRAP(stagger) | 929 DPIO_LANESTAGGER_STRAP_OVRD | 930 DPIO_TX1_STAGGER_MASK(0x1f) | 931 DPIO_TX1_STAGGER_MULT(7) | 932 DPIO_TX2_STAGGER_MULT(5)); 933 } 934 935 /* Deassert data lane reset */ 936 chv_data_lane_soft_reset(encoder, crtc_state, false); 937 938 vlv_dpio_put(dev_priv); 939 } 940 941 void chv_phy_release_cl2_override(struct intel_encoder *encoder) 942 { 943 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 944 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 945 946 if (dport->release_cl2_override) { 947 chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); 948 dport->release_cl2_override = false; 949 } 950 } 951 952 void chv_phy_post_pll_disable(struct intel_encoder *encoder, 953 const struct intel_crtc_state *old_crtc_state) 954 { 955 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 956 enum pipe pipe = to_intel_crtc(old_crtc_state->base.crtc)->pipe; 957 u32 val; 958 959 vlv_dpio_get(dev_priv); 960 961 /* disable left/right clock distribution */ 962 if (pipe != PIPE_B) { 963 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); 964 val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); 965 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); 966 } else { 967 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); 968 val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); 969 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); 970 } 971 972 vlv_dpio_put(dev_priv); 973 974 /* 975 * Leave the power down bit cleared for at least one 976 * lane so that chv_powergate_phy_ch() will power 977 * on something when the channel is otherwise unused. 978 * When the port is off and the override is removed 979 * the lanes power down anyway, so otherwise it doesn't 980 * really matter what the state of power down bits is 981 * after this. 982 */ 983 chv_phy_powergate_lanes(encoder, false, 0x0); 984 } 985 986 void vlv_set_phy_signal_level(struct intel_encoder *encoder, 987 u32 demph_reg_value, u32 preemph_reg_value, 988 u32 uniqtranscale_reg_value, u32 tx3_demph) 989 { 990 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 991 struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); 992 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 993 enum dpio_channel port = vlv_dport_to_channel(dport); 994 enum pipe pipe = intel_crtc->pipe; 995 996 vlv_dpio_get(dev_priv); 997 998 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000); 999 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value); 1000 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), 1001 uniqtranscale_reg_value); 1002 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040); 1003 1004 if (tx3_demph) 1005 vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), tx3_demph); 1006 1007 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000); 1008 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value); 1009 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN); 1010 1011 vlv_dpio_put(dev_priv); 1012 } 1013 1014 void vlv_phy_pre_pll_enable(struct intel_encoder *encoder, 1015 const struct intel_crtc_state *crtc_state) 1016 { 1017 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 1018 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 1019 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 1020 enum dpio_channel port = vlv_dport_to_channel(dport); 1021 enum pipe pipe = crtc->pipe; 1022 1023 /* Program Tx lane resets to default */ 1024 vlv_dpio_get(dev_priv); 1025 1026 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 1027 DPIO_PCS_TX_LANE2_RESET | 1028 DPIO_PCS_TX_LANE1_RESET); 1029 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 1030 DPIO_PCS_CLK_CRI_RXEB_EIOS_EN | 1031 DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN | 1032 (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) | 1033 DPIO_PCS_CLK_SOFT_RESET); 1034 1035 /* Fix up inter-pair skew failure */ 1036 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00); 1037 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500); 1038 vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000); 1039 1040 vlv_dpio_put(dev_priv); 1041 } 1042 1043 void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder, 1044 const struct intel_crtc_state *crtc_state) 1045 { 1046 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 1047 struct intel_digital_port *dport = dp_to_dig_port(intel_dp); 1048 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 1049 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 1050 enum dpio_channel port = vlv_dport_to_channel(dport); 1051 enum pipe pipe = crtc->pipe; 1052 u32 val; 1053 1054 vlv_dpio_get(dev_priv); 1055 1056 /* Enable clock channels for this port */ 1057 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); 1058 val = 0; 1059 if (pipe) 1060 val |= (1<<21); 1061 else 1062 val &= ~(1<<21); 1063 val |= 0x001000c4; 1064 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val); 1065 1066 /* Program lane clock */ 1067 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018); 1068 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); 1069 1070 vlv_dpio_put(dev_priv); 1071 } 1072 1073 void vlv_phy_reset_lanes(struct intel_encoder *encoder, 1074 const struct intel_crtc_state *old_crtc_state) 1075 { 1076 struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); 1077 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 1078 struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); 1079 enum dpio_channel port = vlv_dport_to_channel(dport); 1080 enum pipe pipe = crtc->pipe; 1081 1082 vlv_dpio_get(dev_priv); 1083 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000); 1084 vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060); 1085 vlv_dpio_put(dev_priv); 1086 } 1087