1 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*- 2 */ 3 /* 4 * 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 */ 29 30 #ifndef _I915_DRV_H_ 31 #define _I915_DRV_H_ 32 33 #include <uapi/drm/i915_drm.h> 34 #include <uapi/drm/drm_fourcc.h> 35 36 #include <linux/io-mapping.h> 37 #include <linux/i2c.h> 38 #include <linux/i2c-algo-bit.h> 39 #include <linux/backlight.h> 40 #include <linux/hash.h> 41 #include <linux/intel-iommu.h> 42 #include <linux/kref.h> 43 #include <linux/pm_qos.h> 44 #include <linux/reservation.h> 45 #include <linux/shmem_fs.h> 46 47 #include <drm/drmP.h> 48 #include <drm/intel-gtt.h> 49 #include <drm/drm_legacy.h> /* for struct drm_dma_handle */ 50 #include <drm/drm_gem.h> 51 #include <drm/drm_auth.h> 52 #include <drm/drm_cache.h> 53 54 #include "i915_params.h" 55 #include "i915_reg.h" 56 #include "i915_utils.h" 57 58 #include "intel_uncore.h" 59 #include "intel_bios.h" 60 #include "intel_dpll_mgr.h" 61 #include "intel_uc.h" 62 #include "intel_lrc.h" 63 #include "intel_ringbuffer.h" 64 65 #include "i915_gem.h" 66 #include "i915_gem_context.h" 67 #include "i915_gem_fence_reg.h" 68 #include "i915_gem_object.h" 69 #include "i915_gem_gtt.h" 70 #include "i915_gem_render_state.h" 71 #include "i915_gem_request.h" 72 #include "i915_gem_timeline.h" 73 74 #include "i915_vma.h" 75 76 #include "intel_gvt.h" 77 78 /* General customization: 79 */ 80 81 #define DRIVER_NAME "i915" 82 #define DRIVER_DESC "Intel Graphics" 83 #define DRIVER_DATE "20170619" 84 #define DRIVER_TIMESTAMP 1497857498 85 86 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and 87 * WARN_ON()) for hw state sanity checks to check for unexpected conditions 88 * which may not necessarily be a user visible problem. This will either 89 * WARN() or DRM_ERROR() depending on the verbose_checks moduleparam, to 90 * enable distros and users to tailor their preferred amount of i915 abrt 91 * spam. 92 */ 93 #define I915_STATE_WARN(condition, format...) ({ \ 94 int __ret_warn_on = !!(condition); \ 95 if (unlikely(__ret_warn_on)) \ 96 if (!WARN(i915.verbose_state_checks, format)) \ 97 DRM_ERROR(format); \ 98 unlikely(__ret_warn_on); \ 99 }) 100 101 #define I915_STATE_WARN_ON(x) \ 102 I915_STATE_WARN((x), "%s", "WARN_ON(" __stringify(x) ")") 103 104 bool __i915_inject_load_failure(const char *func, int line); 105 #define i915_inject_load_failure() \ 106 __i915_inject_load_failure(__func__, __LINE__) 107 108 typedef struct { 109 uint32_t val; 110 } uint_fixed_16_16_t; 111 112 #define FP_16_16_MAX ({ \ 113 uint_fixed_16_16_t fp; \ 114 fp.val = UINT_MAX; \ 115 fp; \ 116 }) 117 118 static inline bool is_fixed16_zero(uint_fixed_16_16_t val) 119 { 120 if (val.val == 0) 121 return true; 122 return false; 123 } 124 125 static inline uint_fixed_16_16_t u32_to_fixed_16_16(uint32_t val) 126 { 127 uint_fixed_16_16_t fp; 128 129 WARN_ON(val >> 16); 130 131 fp.val = val << 16; 132 return fp; 133 } 134 135 static inline uint32_t fixed_16_16_to_u32_round_up(uint_fixed_16_16_t fp) 136 { 137 return DIV_ROUND_UP(fp.val, 1 << 16); 138 } 139 140 static inline uint32_t fixed_16_16_to_u32(uint_fixed_16_16_t fp) 141 { 142 return fp.val >> 16; 143 } 144 145 static inline uint_fixed_16_16_t min_fixed_16_16(uint_fixed_16_16_t min1, 146 uint_fixed_16_16_t min2) 147 { 148 uint_fixed_16_16_t min; 149 150 min.val = min(min1.val, min2.val); 151 return min; 152 } 153 154 static inline uint_fixed_16_16_t max_fixed_16_16(uint_fixed_16_16_t max1, 155 uint_fixed_16_16_t max2) 156 { 157 uint_fixed_16_16_t max; 158 159 max.val = max(max1.val, max2.val); 160 return max; 161 } 162 163 static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val, 164 uint_fixed_16_16_t d) 165 { 166 return DIV_ROUND_UP(val.val, d.val); 167 } 168 169 static inline uint32_t mul_round_up_u32_fixed16(uint32_t val, 170 uint_fixed_16_16_t mul) 171 { 172 uint64_t intermediate_val; 173 uint32_t result; 174 175 intermediate_val = (uint64_t) val * mul.val; 176 intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16); 177 WARN_ON(intermediate_val >> 32); 178 result = clamp_t(uint32_t, intermediate_val, 0, ~0); 179 return result; 180 } 181 182 static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val, 183 uint_fixed_16_16_t mul) 184 { 185 uint64_t intermediate_val; 186 uint_fixed_16_16_t fp; 187 188 intermediate_val = (uint64_t) val.val * mul.val; 189 intermediate_val = intermediate_val >> 16; 190 WARN_ON(intermediate_val >> 32); 191 fp.val = clamp_t(uint32_t, intermediate_val, 0, ~0); 192 return fp; 193 } 194 195 static inline uint_fixed_16_16_t fixed_16_16_div(uint32_t val, uint32_t d) 196 { 197 uint_fixed_16_16_t fp, res; 198 199 fp = u32_to_fixed_16_16(val); 200 res.val = DIV_ROUND_UP(fp.val, d); 201 return res; 202 } 203 204 static inline uint_fixed_16_16_t fixed_16_16_div_u64(uint32_t val, uint32_t d) 205 { 206 uint_fixed_16_16_t res; 207 uint64_t interm_val; 208 209 interm_val = (uint64_t)val << 16; 210 interm_val = DIV_ROUND_UP_ULL(interm_val, d); 211 WARN_ON(interm_val >> 32); 212 res.val = (uint32_t) interm_val; 213 214 return res; 215 } 216 217 static inline uint32_t div_round_up_u32_fixed16(uint32_t val, 218 uint_fixed_16_16_t d) 219 { 220 uint64_t interm_val; 221 222 interm_val = (uint64_t)val << 16; 223 interm_val = DIV_ROUND_UP_ULL(interm_val, d.val); 224 WARN_ON(interm_val >> 32); 225 return clamp_t(uint32_t, interm_val, 0, ~0); 226 } 227 228 static inline uint_fixed_16_16_t mul_u32_fixed_16_16(uint32_t val, 229 uint_fixed_16_16_t mul) 230 { 231 uint64_t intermediate_val; 232 uint_fixed_16_16_t fp; 233 234 intermediate_val = (uint64_t) val * mul.val; 235 WARN_ON(intermediate_val >> 32); 236 fp.val = (uint32_t) intermediate_val; 237 return fp; 238 } 239 240 static inline const char *yesno(bool v) 241 { 242 return v ? "yes" : "no"; 243 } 244 245 static inline const char *onoff(bool v) 246 { 247 return v ? "on" : "off"; 248 } 249 250 static inline const char *enableddisabled(bool v) 251 { 252 return v ? "enabled" : "disabled"; 253 } 254 255 enum pipe { 256 INVALID_PIPE = -1, 257 PIPE_A = 0, 258 PIPE_B, 259 PIPE_C, 260 _PIPE_EDP, 261 I915_MAX_PIPES = _PIPE_EDP 262 }; 263 #define pipe_name(p) ((p) + 'A') 264 265 enum transcoder { 266 TRANSCODER_A = 0, 267 TRANSCODER_B, 268 TRANSCODER_C, 269 TRANSCODER_EDP, 270 TRANSCODER_DSI_A, 271 TRANSCODER_DSI_C, 272 I915_MAX_TRANSCODERS 273 }; 274 275 static inline const char *transcoder_name(enum transcoder transcoder) 276 { 277 switch (transcoder) { 278 case TRANSCODER_A: 279 return "A"; 280 case TRANSCODER_B: 281 return "B"; 282 case TRANSCODER_C: 283 return "C"; 284 case TRANSCODER_EDP: 285 return "EDP"; 286 case TRANSCODER_DSI_A: 287 return "DSI A"; 288 case TRANSCODER_DSI_C: 289 return "DSI C"; 290 default: 291 return "<invalid>"; 292 } 293 } 294 295 static inline bool transcoder_is_dsi(enum transcoder transcoder) 296 { 297 return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C; 298 } 299 300 /* 301 * Global legacy plane identifier. Valid only for primary/sprite 302 * planes on pre-g4x, and only for primary planes on g4x+. 303 */ 304 enum plane { 305 PLANE_A, 306 PLANE_B, 307 PLANE_C, 308 }; 309 #define plane_name(p) ((p) + 'A') 310 311 #define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') 312 313 /* 314 * Per-pipe plane identifier. 315 * I915_MAX_PLANES in the enum below is the maximum (across all platforms) 316 * number of planes per CRTC. Not all platforms really have this many planes, 317 * which means some arrays of size I915_MAX_PLANES may have unused entries 318 * between the topmost sprite plane and the cursor plane. 319 * 320 * This is expected to be passed to various register macros 321 * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care. 322 */ 323 enum plane_id { 324 PLANE_PRIMARY, 325 PLANE_SPRITE0, 326 PLANE_SPRITE1, 327 PLANE_SPRITE2, 328 PLANE_CURSOR, 329 I915_MAX_PLANES, 330 }; 331 332 #define for_each_plane_id_on_crtc(__crtc, __p) \ 333 for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \ 334 for_each_if ((__crtc)->plane_ids_mask & BIT(__p)) 335 336 enum port { 337 PORT_NONE = -1, 338 PORT_A = 0, 339 PORT_B, 340 PORT_C, 341 PORT_D, 342 PORT_E, 343 I915_MAX_PORTS 344 }; 345 #define port_name(p) ((p) + 'A') 346 347 #define I915_NUM_PHYS_VLV 2 348 349 enum dpio_channel { 350 DPIO_CH0, 351 DPIO_CH1 352 }; 353 354 enum dpio_phy { 355 DPIO_PHY0, 356 DPIO_PHY1, 357 DPIO_PHY2, 358 }; 359 360 enum intel_display_power_domain { 361 POWER_DOMAIN_PIPE_A, 362 POWER_DOMAIN_PIPE_B, 363 POWER_DOMAIN_PIPE_C, 364 POWER_DOMAIN_PIPE_A_PANEL_FITTER, 365 POWER_DOMAIN_PIPE_B_PANEL_FITTER, 366 POWER_DOMAIN_PIPE_C_PANEL_FITTER, 367 POWER_DOMAIN_TRANSCODER_A, 368 POWER_DOMAIN_TRANSCODER_B, 369 POWER_DOMAIN_TRANSCODER_C, 370 POWER_DOMAIN_TRANSCODER_EDP, 371 POWER_DOMAIN_TRANSCODER_DSI_A, 372 POWER_DOMAIN_TRANSCODER_DSI_C, 373 POWER_DOMAIN_PORT_DDI_A_LANES, 374 POWER_DOMAIN_PORT_DDI_B_LANES, 375 POWER_DOMAIN_PORT_DDI_C_LANES, 376 POWER_DOMAIN_PORT_DDI_D_LANES, 377 POWER_DOMAIN_PORT_DDI_E_LANES, 378 POWER_DOMAIN_PORT_DDI_A_IO, 379 POWER_DOMAIN_PORT_DDI_B_IO, 380 POWER_DOMAIN_PORT_DDI_C_IO, 381 POWER_DOMAIN_PORT_DDI_D_IO, 382 POWER_DOMAIN_PORT_DDI_E_IO, 383 POWER_DOMAIN_PORT_DSI, 384 POWER_DOMAIN_PORT_CRT, 385 POWER_DOMAIN_PORT_OTHER, 386 POWER_DOMAIN_VGA, 387 POWER_DOMAIN_AUDIO, 388 POWER_DOMAIN_PLLS, 389 POWER_DOMAIN_AUX_A, 390 POWER_DOMAIN_AUX_B, 391 POWER_DOMAIN_AUX_C, 392 POWER_DOMAIN_AUX_D, 393 POWER_DOMAIN_GMBUS, 394 POWER_DOMAIN_MODESET, 395 POWER_DOMAIN_INIT, 396 397 POWER_DOMAIN_NUM, 398 }; 399 400 #define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A) 401 #define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \ 402 ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER) 403 #define POWER_DOMAIN_TRANSCODER(tran) \ 404 ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \ 405 (tran) + POWER_DOMAIN_TRANSCODER_A) 406 407 enum hpd_pin { 408 HPD_NONE = 0, 409 HPD_TV = HPD_NONE, /* TV is known to be unreliable */ 410 HPD_CRT, 411 HPD_SDVO_B, 412 HPD_SDVO_C, 413 HPD_PORT_A, 414 HPD_PORT_B, 415 HPD_PORT_C, 416 HPD_PORT_D, 417 HPD_PORT_E, 418 HPD_NUM_PINS 419 }; 420 421 #define for_each_hpd_pin(__pin) \ 422 for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++) 423 424 #define HPD_STORM_DEFAULT_THRESHOLD 5 425 426 struct i915_hotplug { 427 struct work_struct hotplug_work; 428 429 struct { 430 unsigned long last_jiffies; 431 int count; 432 enum { 433 HPD_ENABLED = 0, 434 HPD_DISABLED = 1, 435 HPD_MARK_DISABLED = 2 436 } state; 437 } stats[HPD_NUM_PINS]; 438 u32 event_bits; 439 struct delayed_work reenable_work; 440 441 struct intel_digital_port *irq_port[I915_MAX_PORTS]; 442 u32 long_port_mask; 443 u32 short_port_mask; 444 struct work_struct dig_port_work; 445 446 struct work_struct poll_init_work; 447 bool poll_enabled; 448 449 unsigned int hpd_storm_threshold; 450 451 /* 452 * if we get a HPD irq from DP and a HPD irq from non-DP 453 * the non-DP HPD could block the workqueue on a mode config 454 * mutex getting, that userspace may have taken. However 455 * userspace is waiting on the DP workqueue to run which is 456 * blocked behind the non-DP one. 457 */ 458 struct workqueue_struct *dp_wq; 459 }; 460 461 #define I915_GEM_GPU_DOMAINS \ 462 (I915_GEM_DOMAIN_RENDER | \ 463 I915_GEM_DOMAIN_SAMPLER | \ 464 I915_GEM_DOMAIN_COMMAND | \ 465 I915_GEM_DOMAIN_INSTRUCTION | \ 466 I915_GEM_DOMAIN_VERTEX) 467 468 #define for_each_pipe(__dev_priv, __p) \ 469 for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) 470 #define for_each_pipe_masked(__dev_priv, __p, __mask) \ 471 for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ 472 for_each_if ((__mask) & (1 << (__p))) 473 #define for_each_universal_plane(__dev_priv, __pipe, __p) \ 474 for ((__p) = 0; \ 475 (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ 476 (__p)++) 477 #define for_each_sprite(__dev_priv, __p, __s) \ 478 for ((__s) = 0; \ 479 (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)]; \ 480 (__s)++) 481 482 #define for_each_port_masked(__port, __ports_mask) \ 483 for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ 484 for_each_if ((__ports_mask) & (1 << (__port))) 485 486 #define for_each_crtc(dev, crtc) \ 487 list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head) 488 489 #define for_each_intel_plane(dev, intel_plane) \ 490 list_for_each_entry(intel_plane, \ 491 &(dev)->mode_config.plane_list, \ 492 base.head) 493 494 #define for_each_intel_plane_mask(dev, intel_plane, plane_mask) \ 495 list_for_each_entry(intel_plane, \ 496 &(dev)->mode_config.plane_list, \ 497 base.head) \ 498 for_each_if ((plane_mask) & \ 499 (1 << drm_plane_index(&intel_plane->base))) 500 501 #define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) \ 502 list_for_each_entry(intel_plane, \ 503 &(dev)->mode_config.plane_list, \ 504 base.head) \ 505 for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe) 506 507 #define for_each_intel_crtc(dev, intel_crtc) \ 508 list_for_each_entry(intel_crtc, \ 509 &(dev)->mode_config.crtc_list, \ 510 base.head) 511 512 #define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask) \ 513 list_for_each_entry(intel_crtc, \ 514 &(dev)->mode_config.crtc_list, \ 515 base.head) \ 516 for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base))) 517 518 #define for_each_intel_encoder(dev, intel_encoder) \ 519 list_for_each_entry(intel_encoder, \ 520 &(dev)->mode_config.encoder_list, \ 521 base.head) 522 523 #define for_each_intel_connector_iter(intel_connector, iter) \ 524 while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter)))) 525 526 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ 527 list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ 528 for_each_if ((intel_encoder)->base.crtc == (__crtc)) 529 530 #define for_each_connector_on_encoder(dev, __encoder, intel_connector) \ 531 list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \ 532 for_each_if ((intel_connector)->base.encoder == (__encoder)) 533 534 #define for_each_power_domain(domain, mask) \ 535 for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ 536 for_each_if (BIT_ULL(domain) & (mask)) 537 538 #define for_each_power_well(__dev_priv, __power_well) \ 539 for ((__power_well) = (__dev_priv)->power_domains.power_wells; \ 540 (__power_well) - (__dev_priv)->power_domains.power_wells < \ 541 (__dev_priv)->power_domains.power_well_count; \ 542 (__power_well)++) 543 544 #define for_each_power_well_rev(__dev_priv, __power_well) \ 545 for ((__power_well) = (__dev_priv)->power_domains.power_wells + \ 546 (__dev_priv)->power_domains.power_well_count - 1; \ 547 (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \ 548 (__power_well)--) 549 550 #define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask) \ 551 for_each_power_well(__dev_priv, __power_well) \ 552 for_each_if ((__power_well)->domains & (__domain_mask)) 553 554 #define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \ 555 for_each_power_well_rev(__dev_priv, __power_well) \ 556 for_each_if ((__power_well)->domains & (__domain_mask)) 557 558 #define for_each_intel_plane_in_state(__state, plane, plane_state, __i) \ 559 for ((__i) = 0; \ 560 (__i) < (__state)->base.dev->mode_config.num_total_plane && \ 561 ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ 562 (plane_state) = to_intel_plane_state((__state)->base.planes[__i].state), 1); \ 563 (__i)++) \ 564 for_each_if (plane_state) 565 566 struct drm_i915_private; 567 struct i915_mm_struct; 568 struct i915_mmu_object; 569 570 struct drm_i915_file_private { 571 struct drm_i915_private *dev_priv; 572 struct drm_file *file; 573 574 struct { 575 spinlock_t lock; 576 struct list_head request_list; 577 /* 20ms is a fairly arbitrary limit (greater than the average frame time) 578 * chosen to prevent the CPU getting more than a frame ahead of the GPU 579 * (when using lax throttling for the frontbuffer). We also use it to 580 * offer free GPU waitboosts for severely congested workloads. 581 */ 582 #define DRM_I915_THROTTLE_JIFFIES msecs_to_jiffies(20) 583 } mm; 584 struct idr context_idr; 585 586 struct intel_rps_client { 587 struct list_head link; 588 unsigned boosts; 589 } rps; 590 591 unsigned int bsd_engine; 592 593 /* Client can have a maximum of 3 contexts banned before 594 * it is denied of creating new contexts. As one context 595 * ban needs 4 consecutive hangs, and more if there is 596 * progress in between, this is a last resort stop gap measure 597 * to limit the badly behaving clients access to gpu. 598 */ 599 #define I915_MAX_CLIENT_CONTEXT_BANS 3 600 int context_bans; 601 }; 602 603 /* Used by dp and fdi links */ 604 struct intel_link_m_n { 605 uint32_t tu; 606 uint32_t gmch_m; 607 uint32_t gmch_n; 608 uint32_t link_m; 609 uint32_t link_n; 610 }; 611 612 void intel_link_compute_m_n(int bpp, int nlanes, 613 int pixel_clock, int link_clock, 614 struct intel_link_m_n *m_n, 615 bool reduce_m_n); 616 617 /* Interface history: 618 * 619 * 1.1: Original. 620 * 1.2: Add Power Management 621 * 1.3: Add vblank support 622 * 1.4: Fix cmdbuffer path, add heap destroy 623 * 1.5: Add vblank pipe configuration 624 * 1.6: - New ioctl for scheduling buffer swaps on vertical blank 625 * - Support vertical blank on secondary display pipe 626 */ 627 #define DRIVER_MAJOR 1 628 #define DRIVER_MINOR 6 629 #define DRIVER_PATCHLEVEL 0 630 631 struct opregion_header; 632 struct opregion_acpi; 633 struct opregion_swsci; 634 struct opregion_asle; 635 636 struct intel_opregion { 637 struct opregion_header *header; 638 struct opregion_acpi *acpi; 639 struct opregion_swsci *swsci; 640 u32 swsci_gbda_sub_functions; 641 u32 swsci_sbcb_sub_functions; 642 struct opregion_asle *asle; 643 void *rvda; 644 const void *vbt; 645 u32 vbt_size; 646 u32 *lid_state; 647 struct work_struct asle_work; 648 }; 649 #define OPREGION_SIZE (8*1024) 650 651 struct intel_overlay; 652 struct intel_overlay_error_state; 653 654 struct sdvo_device_mapping { 655 u8 initialized; 656 u8 dvo_port; 657 u8 slave_addr; 658 u8 dvo_wiring; 659 u8 i2c_pin; 660 u8 ddc_pin; 661 }; 662 663 struct intel_connector; 664 struct intel_encoder; 665 struct intel_atomic_state; 666 struct intel_crtc_state; 667 struct intel_initial_plane_config; 668 struct intel_crtc; 669 struct intel_limit; 670 struct dpll; 671 struct intel_cdclk_state; 672 673 struct drm_i915_display_funcs { 674 void (*get_cdclk)(struct drm_i915_private *dev_priv, 675 struct intel_cdclk_state *cdclk_state); 676 void (*set_cdclk)(struct drm_i915_private *dev_priv, 677 const struct intel_cdclk_state *cdclk_state); 678 int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane); 679 int (*compute_pipe_wm)(struct intel_crtc_state *cstate); 680 int (*compute_intermediate_wm)(struct drm_device *dev, 681 struct intel_crtc *intel_crtc, 682 struct intel_crtc_state *newstate); 683 void (*initial_watermarks)(struct intel_atomic_state *state, 684 struct intel_crtc_state *cstate); 685 void (*atomic_update_watermarks)(struct intel_atomic_state *state, 686 struct intel_crtc_state *cstate); 687 void (*optimize_watermarks)(struct intel_atomic_state *state, 688 struct intel_crtc_state *cstate); 689 int (*compute_global_watermarks)(struct drm_atomic_state *state); 690 void (*update_wm)(struct intel_crtc *crtc); 691 int (*modeset_calc_cdclk)(struct drm_atomic_state *state); 692 /* Returns the active state of the crtc, and if the crtc is active, 693 * fills out the pipe-config with the hw state. */ 694 bool (*get_pipe_config)(struct intel_crtc *, 695 struct intel_crtc_state *); 696 void (*get_initial_plane_config)(struct intel_crtc *, 697 struct intel_initial_plane_config *); 698 int (*crtc_compute_clock)(struct intel_crtc *crtc, 699 struct intel_crtc_state *crtc_state); 700 void (*crtc_enable)(struct intel_crtc_state *pipe_config, 701 struct drm_atomic_state *old_state); 702 void (*crtc_disable)(struct intel_crtc_state *old_crtc_state, 703 struct drm_atomic_state *old_state); 704 void (*update_crtcs)(struct drm_atomic_state *state, 705 unsigned int *crtc_vblank_mask); 706 void (*audio_codec_enable)(struct drm_connector *connector, 707 struct intel_encoder *encoder, 708 const struct drm_display_mode *adjusted_mode); 709 void (*audio_codec_disable)(struct intel_encoder *encoder); 710 void (*fdi_link_train)(struct intel_crtc *crtc, 711 const struct intel_crtc_state *crtc_state); 712 void (*init_clock_gating)(struct drm_i915_private *dev_priv); 713 int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, 714 struct drm_framebuffer *fb, 715 struct drm_i915_gem_object *obj, 716 struct drm_i915_gem_request *req, 717 uint32_t flags); 718 void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); 719 /* clock updates for mode set */ 720 /* cursor updates */ 721 /* render clock increase/decrease */ 722 /* display clock increase/decrease */ 723 /* pll clock increase/decrease */ 724 725 void (*load_csc_matrix)(struct drm_crtc_state *crtc_state); 726 void (*load_luts)(struct drm_crtc_state *crtc_state); 727 }; 728 729 #define CSR_VERSION(major, minor) ((major) << 16 | (minor)) 730 #define CSR_VERSION_MAJOR(version) ((version) >> 16) 731 #define CSR_VERSION_MINOR(version) ((version) & 0xffff) 732 733 struct intel_csr { 734 struct work_struct work; 735 const char *fw_path; 736 uint32_t *dmc_payload; 737 uint32_t dmc_fw_size; 738 uint32_t version; 739 uint32_t mmio_count; 740 i915_reg_t mmioaddr[8]; 741 uint32_t mmiodata[8]; 742 uint32_t dc_state; 743 uint32_t allowed_dc_mask; 744 }; 745 746 #define DEV_INFO_FOR_EACH_FLAG(func) \ 747 func(is_mobile); \ 748 func(is_lp); \ 749 func(is_alpha_support); \ 750 /* Keep has_* in alphabetical order */ \ 751 func(has_64bit_reloc); \ 752 func(has_aliasing_ppgtt); \ 753 func(has_csr); \ 754 func(has_ddi); \ 755 func(has_dp_mst); \ 756 func(has_fbc); \ 757 func(has_fpga_dbg); \ 758 func(has_full_ppgtt); \ 759 func(has_full_48bit_ppgtt); \ 760 func(has_gmbus_irq); \ 761 func(has_gmch_display); \ 762 func(has_guc); \ 763 func(has_guc_ct); \ 764 func(has_hotplug); \ 765 func(has_l3_dpf); \ 766 func(has_llc); \ 767 func(has_logical_ring_contexts); \ 768 func(has_overlay); \ 769 func(has_pipe_cxsr); \ 770 func(has_pooled_eu); \ 771 func(has_psr); \ 772 func(has_rc6); \ 773 func(has_rc6p); \ 774 func(has_resource_streamer); \ 775 func(has_runtime_pm); \ 776 func(has_snoop); \ 777 func(unfenced_needs_alignment); \ 778 func(cursor_needs_physical); \ 779 func(hws_needs_physical); \ 780 func(overlay_needs_physical); \ 781 func(supports_tv); 782 783 struct sseu_dev_info { 784 u8 slice_mask; 785 u8 subslice_mask; 786 u8 eu_total; 787 u8 eu_per_subslice; 788 u8 min_eu_in_pool; 789 /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */ 790 u8 subslice_7eu[3]; 791 u8 has_slice_pg:1; 792 u8 has_subslice_pg:1; 793 u8 has_eu_pg:1; 794 }; 795 796 static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu) 797 { 798 return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask); 799 } 800 801 /* Keep in gen based order, and chronological order within a gen */ 802 enum intel_platform { 803 INTEL_PLATFORM_UNINITIALIZED = 0, 804 INTEL_I830, 805 INTEL_I845G, 806 INTEL_I85X, 807 INTEL_I865G, 808 INTEL_I915G, 809 INTEL_I915GM, 810 INTEL_I945G, 811 INTEL_I945GM, 812 INTEL_G33, 813 INTEL_PINEVIEW, 814 INTEL_I965G, 815 INTEL_I965GM, 816 INTEL_G45, 817 INTEL_GM45, 818 INTEL_IRONLAKE, 819 INTEL_SANDYBRIDGE, 820 INTEL_IVYBRIDGE, 821 INTEL_VALLEYVIEW, 822 INTEL_HASWELL, 823 INTEL_BROADWELL, 824 INTEL_CHERRYVIEW, 825 INTEL_SKYLAKE, 826 INTEL_BROXTON, 827 INTEL_KABYLAKE, 828 INTEL_GEMINILAKE, 829 INTEL_COFFEELAKE, 830 INTEL_CANNONLAKE, 831 INTEL_MAX_PLATFORMS 832 }; 833 834 struct intel_device_info { 835 u32 display_mmio_offset; 836 u16 device_id; 837 u8 num_pipes; 838 u8 num_sprites[I915_MAX_PIPES]; 839 u8 num_scalers[I915_MAX_PIPES]; 840 u8 gen; 841 u16 gen_mask; 842 enum intel_platform platform; 843 u8 ring_mask; /* Rings supported by the HW */ 844 u8 num_rings; 845 #define DEFINE_FLAG(name) u8 name:1 846 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); 847 #undef DEFINE_FLAG 848 u16 ddb_size; /* in blocks */ 849 /* Register offsets for the various display pipes and transcoders */ 850 int pipe_offsets[I915_MAX_TRANSCODERS]; 851 int trans_offsets[I915_MAX_TRANSCODERS]; 852 int palette_offsets[I915_MAX_PIPES]; 853 int cursor_offsets[I915_MAX_PIPES]; 854 855 /* Slice/subslice/EU info */ 856 struct sseu_dev_info sseu; 857 858 struct color_luts { 859 u16 degamma_lut_size; 860 u16 gamma_lut_size; 861 } color; 862 }; 863 864 struct intel_display_error_state; 865 866 struct i915_gpu_state { 867 struct kref ref; 868 struct timeval time; 869 struct timeval boottime; 870 struct timeval uptime; 871 872 struct drm_i915_private *i915; 873 874 char error_msg[128]; 875 bool simulated; 876 bool awake; 877 bool wakelock; 878 bool suspended; 879 int iommu; 880 u32 reset_count; 881 u32 suspend_count; 882 struct intel_device_info device_info; 883 struct i915_params params; 884 885 /* Generic register state */ 886 u32 eir; 887 u32 pgtbl_er; 888 u32 ier; 889 u32 gtier[4], ngtier; 890 u32 ccid; 891 u32 derrmr; 892 u32 forcewake; 893 u32 error; /* gen6+ */ 894 u32 err_int; /* gen7 */ 895 u32 fault_data0; /* gen8, gen9 */ 896 u32 fault_data1; /* gen8, gen9 */ 897 u32 done_reg; 898 u32 gac_eco; 899 u32 gam_ecochk; 900 u32 gab_ctl; 901 u32 gfx_mode; 902 903 u32 nfence; 904 u64 fence[I915_MAX_NUM_FENCES]; 905 struct intel_overlay_error_state *overlay; 906 struct intel_display_error_state *display; 907 struct drm_i915_error_object *semaphore; 908 struct drm_i915_error_object *guc_log; 909 910 struct drm_i915_error_engine { 911 int engine_id; 912 /* Software tracked state */ 913 bool waiting; 914 int num_waiters; 915 unsigned long hangcheck_timestamp; 916 bool hangcheck_stalled; 917 enum intel_engine_hangcheck_action hangcheck_action; 918 struct i915_address_space *vm; 919 int num_requests; 920 921 /* position of active request inside the ring */ 922 u32 rq_head, rq_post, rq_tail; 923 924 /* our own tracking of ring head and tail */ 925 u32 cpu_ring_head; 926 u32 cpu_ring_tail; 927 928 u32 last_seqno; 929 930 /* Register state */ 931 u32 start; 932 u32 tail; 933 u32 head; 934 u32 ctl; 935 u32 mode; 936 u32 hws; 937 u32 ipeir; 938 u32 ipehr; 939 u32 bbstate; 940 u32 instpm; 941 u32 instps; 942 u32 seqno; 943 u64 bbaddr; 944 u64 acthd; 945 u32 fault_reg; 946 u64 faddr; 947 u32 rc_psmi; /* sleep state */ 948 u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; 949 struct intel_instdone instdone; 950 951 struct drm_i915_error_context { 952 char comm[TASK_COMM_LEN]; 953 pid_t pid; 954 u32 handle; 955 u32 hw_id; 956 int ban_score; 957 int active; 958 int guilty; 959 } context; 960 961 struct drm_i915_error_object { 962 u64 gtt_offset; 963 u64 gtt_size; 964 int page_count; 965 int unused; 966 u32 *pages[0]; 967 } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; 968 969 struct drm_i915_error_object **user_bo; 970 long user_bo_count; 971 972 struct drm_i915_error_object *wa_ctx; 973 974 struct drm_i915_error_request { 975 long jiffies; 976 pid_t pid; 977 u32 context; 978 int ban_score; 979 u32 seqno; 980 u32 head; 981 u32 tail; 982 } *requests, execlist[2]; 983 984 struct drm_i915_error_waiter { 985 char comm[TASK_COMM_LEN]; 986 pid_t pid; 987 u32 seqno; 988 } *waiters; 989 990 struct { 991 u32 gfx_mode; 992 union { 993 u64 pdp[4]; 994 u32 pp_dir_base; 995 }; 996 } vm_info; 997 } engine[I915_NUM_ENGINES]; 998 999 struct drm_i915_error_buffer { 1000 u32 size; 1001 u32 name; 1002 u32 rseqno[I915_NUM_ENGINES], wseqno; 1003 u64 gtt_offset; 1004 u32 read_domains; 1005 u32 write_domain; 1006 s32 fence_reg:I915_MAX_NUM_FENCE_BITS; 1007 u32 tiling:2; 1008 u32 dirty:1; 1009 u32 purgeable:1; 1010 u32 userptr:1; 1011 s32 engine:4; 1012 u32 cache_level:3; 1013 } *active_bo[I915_NUM_ENGINES], *pinned_bo; 1014 u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; 1015 struct i915_address_space *active_vm[I915_NUM_ENGINES]; 1016 }; 1017 1018 enum i915_cache_level { 1019 I915_CACHE_NONE = 0, 1020 I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */ 1021 I915_CACHE_L3_LLC, /* gen7+, L3 sits between the domain specifc 1022 caches, eg sampler/render caches, and the 1023 large Last-Level-Cache. LLC is coherent with 1024 the CPU, but L3 is only visible to the GPU. */ 1025 I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */ 1026 }; 1027 1028 #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ 1029 1030 enum fb_op_origin { 1031 ORIGIN_GTT, 1032 ORIGIN_CPU, 1033 ORIGIN_CS, 1034 ORIGIN_FLIP, 1035 ORIGIN_DIRTYFB, 1036 }; 1037 1038 struct intel_fbc { 1039 /* This is always the inner lock when overlapping with struct_mutex and 1040 * it's the outer lock when overlapping with stolen_lock. */ 1041 struct mutex lock; 1042 unsigned threshold; 1043 unsigned int possible_framebuffer_bits; 1044 unsigned int busy_bits; 1045 unsigned int visible_pipes_mask; 1046 struct intel_crtc *crtc; 1047 1048 struct drm_mm_node compressed_fb; 1049 struct drm_mm_node *compressed_llb; 1050 1051 bool false_color; 1052 1053 bool enabled; 1054 bool active; 1055 1056 bool underrun_detected; 1057 struct work_struct underrun_work; 1058 1059 struct intel_fbc_state_cache { 1060 struct i915_vma *vma; 1061 1062 struct { 1063 unsigned int mode_flags; 1064 uint32_t hsw_bdw_pixel_rate; 1065 } crtc; 1066 1067 struct { 1068 unsigned int rotation; 1069 int src_w; 1070 int src_h; 1071 bool visible; 1072 } plane; 1073 1074 struct { 1075 const struct drm_format_info *format; 1076 unsigned int stride; 1077 } fb; 1078 } state_cache; 1079 1080 struct intel_fbc_reg_params { 1081 struct i915_vma *vma; 1082 1083 struct { 1084 enum pipe pipe; 1085 enum plane plane; 1086 unsigned int fence_y_offset; 1087 } crtc; 1088 1089 struct { 1090 const struct drm_format_info *format; 1091 unsigned int stride; 1092 } fb; 1093 1094 int cfb_size; 1095 } params; 1096 1097 struct intel_fbc_work { 1098 bool scheduled; 1099 u32 scheduled_vblank; 1100 struct work_struct work; 1101 } work; 1102 1103 const char *no_fbc_reason; 1104 }; 1105 1106 /* 1107 * HIGH_RR is the highest eDP panel refresh rate read from EDID 1108 * LOW_RR is the lowest eDP panel refresh rate found from EDID 1109 * parsing for same resolution. 1110 */ 1111 enum drrs_refresh_rate_type { 1112 DRRS_HIGH_RR, 1113 DRRS_LOW_RR, 1114 DRRS_MAX_RR, /* RR count */ 1115 }; 1116 1117 enum drrs_support_type { 1118 DRRS_NOT_SUPPORTED = 0, 1119 STATIC_DRRS_SUPPORT = 1, 1120 SEAMLESS_DRRS_SUPPORT = 2 1121 }; 1122 1123 struct intel_dp; 1124 struct i915_drrs { 1125 struct mutex mutex; 1126 struct delayed_work work; 1127 struct intel_dp *dp; 1128 unsigned busy_frontbuffer_bits; 1129 enum drrs_refresh_rate_type refresh_rate_type; 1130 enum drrs_support_type type; 1131 }; 1132 1133 struct i915_psr { 1134 struct mutex lock; 1135 bool sink_support; 1136 bool source_ok; 1137 struct intel_dp *enabled; 1138 bool active; 1139 struct delayed_work work; 1140 unsigned busy_frontbuffer_bits; 1141 bool psr2_support; 1142 bool aux_frame_sync; 1143 bool link_standby; 1144 bool y_cord_support; 1145 bool colorimetry_support; 1146 bool alpm; 1147 }; 1148 1149 enum intel_pch { 1150 PCH_NONE = 0, /* No PCH present */ 1151 PCH_IBX, /* Ibexpeak PCH */ 1152 PCH_CPT, /* Cougarpoint PCH */ 1153 PCH_LPT, /* Lynxpoint PCH */ 1154 PCH_SPT, /* Sunrisepoint PCH */ 1155 PCH_KBP, /* Kabypoint PCH */ 1156 PCH_CNP, /* Cannonpoint PCH */ 1157 PCH_NOP, 1158 }; 1159 1160 enum intel_sbi_destination { 1161 SBI_ICLK, 1162 SBI_MPHY, 1163 }; 1164 1165 #define QUIRK_LVDS_SSC_DISABLE (1<<1) 1166 #define QUIRK_INVERT_BRIGHTNESS (1<<2) 1167 #define QUIRK_BACKLIGHT_PRESENT (1<<3) 1168 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5) 1169 1170 struct intel_fbdev; 1171 struct intel_fbc_work; 1172 1173 struct intel_gmbus { 1174 struct i2c_adapter adapter; 1175 #define GMBUS_FORCE_BIT_RETRY (1U << 31) 1176 u32 force_bit; 1177 u32 reg0; 1178 i915_reg_t gpio_reg; 1179 struct i2c_algo_bit_data bit_algo; 1180 struct drm_i915_private *dev_priv; 1181 }; 1182 1183 struct i915_suspend_saved_registers { 1184 u32 saveDSPARB; 1185 u32 saveFBC_CONTROL; 1186 u32 saveCACHE_MODE_0; 1187 u32 saveMI_ARB_STATE; 1188 u32 saveSWF0[16]; 1189 u32 saveSWF1[16]; 1190 u32 saveSWF3[3]; 1191 uint64_t saveFENCE[I915_MAX_NUM_FENCES]; 1192 u32 savePCH_PORT_HOTPLUG; 1193 u16 saveGCDGMBUS; 1194 }; 1195 1196 struct vlv_s0ix_state { 1197 /* GAM */ 1198 u32 wr_watermark; 1199 u32 gfx_prio_ctrl; 1200 u32 arb_mode; 1201 u32 gfx_pend_tlb0; 1202 u32 gfx_pend_tlb1; 1203 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; 1204 u32 media_max_req_count; 1205 u32 gfx_max_req_count; 1206 u32 render_hwsp; 1207 u32 ecochk; 1208 u32 bsd_hwsp; 1209 u32 blt_hwsp; 1210 u32 tlb_rd_addr; 1211 1212 /* MBC */ 1213 u32 g3dctl; 1214 u32 gsckgctl; 1215 u32 mbctl; 1216 1217 /* GCP */ 1218 u32 ucgctl1; 1219 u32 ucgctl3; 1220 u32 rcgctl1; 1221 u32 rcgctl2; 1222 u32 rstctl; 1223 u32 misccpctl; 1224 1225 /* GPM */ 1226 u32 gfxpause; 1227 u32 rpdeuhwtc; 1228 u32 rpdeuc; 1229 u32 ecobus; 1230 u32 pwrdwnupctl; 1231 u32 rp_down_timeout; 1232 u32 rp_deucsw; 1233 u32 rcubmabdtmr; 1234 u32 rcedata; 1235 u32 spare2gh; 1236 1237 /* Display 1 CZ domain */ 1238 u32 gt_imr; 1239 u32 gt_ier; 1240 u32 pm_imr; 1241 u32 pm_ier; 1242 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; 1243 1244 /* GT SA CZ domain */ 1245 u32 tilectl; 1246 u32 gt_fifoctl; 1247 u32 gtlc_wake_ctrl; 1248 u32 gtlc_survive; 1249 u32 pmwgicz; 1250 1251 /* Display 2 CZ domain */ 1252 u32 gu_ctl0; 1253 u32 gu_ctl1; 1254 u32 pcbr; 1255 u32 clock_gate_dis2; 1256 }; 1257 1258 struct intel_rps_ei { 1259 ktime_t ktime; 1260 u32 render_c0; 1261 u32 media_c0; 1262 }; 1263 1264 struct intel_gen6_power_mgmt { 1265 /* 1266 * work, interrupts_enabled and pm_iir are protected by 1267 * dev_priv->irq_lock 1268 */ 1269 struct work_struct work; 1270 bool interrupts_enabled; 1271 u32 pm_iir; 1272 1273 /* PM interrupt bits that should never be masked */ 1274 u32 pm_intrmsk_mbz; 1275 1276 /* Frequencies are stored in potentially platform dependent multiples. 1277 * In other words, *_freq needs to be multiplied by X to be interesting. 1278 * Soft limits are those which are used for the dynamic reclocking done 1279 * by the driver (raise frequencies under heavy loads, and lower for 1280 * lighter loads). Hard limits are those imposed by the hardware. 1281 * 1282 * A distinction is made for overclocking, which is never enabled by 1283 * default, and is considered to be above the hard limit if it's 1284 * possible at all. 1285 */ 1286 u8 cur_freq; /* Current frequency (cached, may not == HW) */ 1287 u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ 1288 u8 max_freq_softlimit; /* Max frequency permitted by the driver */ 1289 u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ 1290 u8 min_freq; /* AKA RPn. Minimum frequency */ 1291 u8 boost_freq; /* Frequency to request when wait boosting */ 1292 u8 idle_freq; /* Frequency to request when we are idle */ 1293 u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ 1294 u8 rp1_freq; /* "less than" RP0 power/freqency */ 1295 u8 rp0_freq; /* Non-overclocked max frequency. */ 1296 u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ 1297 1298 u8 up_threshold; /* Current %busy required to uplock */ 1299 u8 down_threshold; /* Current %busy required to downclock */ 1300 1301 int last_adj; 1302 enum { LOW_POWER, BETWEEN, HIGH_POWER } power; 1303 1304 spinlock_t client_lock; 1305 struct list_head clients; 1306 bool client_boost; 1307 1308 bool enabled; 1309 struct delayed_work autoenable_work; 1310 unsigned boosts; 1311 1312 /* manual wa residency calculations */ 1313 struct intel_rps_ei ei; 1314 1315 /* 1316 * Protects RPS/RC6 register access and PCU communication. 1317 * Must be taken after struct_mutex if nested. Note that 1318 * this lock may be held for long periods of time when 1319 * talking to hw - so only take it when talking to hw! 1320 */ 1321 struct mutex hw_lock; 1322 }; 1323 1324 /* defined intel_pm.c */ 1325 extern spinlock_t mchdev_lock; 1326 1327 struct intel_ilk_power_mgmt { 1328 u8 cur_delay; 1329 u8 min_delay; 1330 u8 max_delay; 1331 u8 fmax; 1332 u8 fstart; 1333 1334 u64 last_count1; 1335 unsigned long last_time1; 1336 unsigned long chipset_power; 1337 u64 last_count2; 1338 u64 last_time2; 1339 unsigned long gfx_power; 1340 u8 corr; 1341 1342 int c_m; 1343 int r_t; 1344 }; 1345 1346 struct drm_i915_private; 1347 struct i915_power_well; 1348 1349 struct i915_power_well_ops { 1350 /* 1351 * Synchronize the well's hw state to match the current sw state, for 1352 * example enable/disable it based on the current refcount. Called 1353 * during driver init and resume time, possibly after first calling 1354 * the enable/disable handlers. 1355 */ 1356 void (*sync_hw)(struct drm_i915_private *dev_priv, 1357 struct i915_power_well *power_well); 1358 /* 1359 * Enable the well and resources that depend on it (for example 1360 * interrupts located on the well). Called after the 0->1 refcount 1361 * transition. 1362 */ 1363 void (*enable)(struct drm_i915_private *dev_priv, 1364 struct i915_power_well *power_well); 1365 /* 1366 * Disable the well and resources that depend on it. Called after 1367 * the 1->0 refcount transition. 1368 */ 1369 void (*disable)(struct drm_i915_private *dev_priv, 1370 struct i915_power_well *power_well); 1371 /* Returns the hw enabled state. */ 1372 bool (*is_enabled)(struct drm_i915_private *dev_priv, 1373 struct i915_power_well *power_well); 1374 }; 1375 1376 /* Power well structure for haswell */ 1377 struct i915_power_well { 1378 const char *name; 1379 bool always_on; 1380 /* power well enable/disable usage count */ 1381 int count; 1382 /* cached hw enabled state */ 1383 bool hw_enabled; 1384 u64 domains; 1385 /* unique identifier for this power well */ 1386 unsigned long id; 1387 /* 1388 * Arbitraty data associated with this power well. Platform and power 1389 * well specific. 1390 */ 1391 unsigned long data; 1392 const struct i915_power_well_ops *ops; 1393 }; 1394 1395 struct i915_power_domains { 1396 /* 1397 * Power wells needed for initialization at driver init and suspend 1398 * time are on. They are kept on until after the first modeset. 1399 */ 1400 bool init_power_on; 1401 bool initializing; 1402 int power_well_count; 1403 1404 struct mutex lock; 1405 int domain_use_count[POWER_DOMAIN_NUM]; 1406 struct i915_power_well *power_wells; 1407 }; 1408 1409 #define MAX_L3_SLICES 2 1410 struct intel_l3_parity { 1411 u32 *remap_info[MAX_L3_SLICES]; 1412 struct work_struct error_work; 1413 int which_slice; 1414 }; 1415 1416 struct i915_gem_mm { 1417 /** Memory allocator for GTT stolen memory */ 1418 struct drm_mm stolen; 1419 /** Protects the usage of the GTT stolen memory allocator. This is 1420 * always the inner lock when overlapping with struct_mutex. */ 1421 struct mutex stolen_lock; 1422 1423 /** List of all objects in gtt_space. Used to restore gtt 1424 * mappings on resume */ 1425 struct list_head bound_list; 1426 /** 1427 * List of objects which are not bound to the GTT (thus 1428 * are idle and not used by the GPU). These objects may or may 1429 * not actually have any pages attached. 1430 */ 1431 struct list_head unbound_list; 1432 1433 /** List of all objects in gtt_space, currently mmaped by userspace. 1434 * All objects within this list must also be on bound_list. 1435 */ 1436 struct list_head userfault_list; 1437 1438 /** 1439 * List of objects which are pending destruction. 1440 */ 1441 struct llist_head free_list; 1442 struct work_struct free_work; 1443 1444 /** Usable portion of the GTT for GEM */ 1445 dma_addr_t stolen_base; /* limited to low memory (32-bit) */ 1446 1447 /** PPGTT used for aliasing the PPGTT with the GTT */ 1448 struct i915_hw_ppgtt *aliasing_ppgtt; 1449 1450 struct notifier_block oom_notifier; 1451 struct notifier_block vmap_notifier; 1452 struct shrinker shrinker; 1453 1454 /** LRU list of objects with fence regs on them. */ 1455 struct list_head fence_list; 1456 1457 /** 1458 * Workqueue to fault in userptr pages, flushed by the execbuf 1459 * when required but otherwise left to userspace to try again 1460 * on EAGAIN. 1461 */ 1462 struct workqueue_struct *userptr_wq; 1463 1464 u64 unordered_timeline; 1465 1466 /* the indicator for dispatch video commands on two BSD rings */ 1467 atomic_t bsd_engine_dispatch_index; 1468 1469 /** Bit 6 swizzling required for X tiling */ 1470 uint32_t bit_6_swizzle_x; 1471 /** Bit 6 swizzling required for Y tiling */ 1472 uint32_t bit_6_swizzle_y; 1473 1474 /* accounting, useful for userland debugging */ 1475 spinlock_t object_stat_lock; 1476 u64 object_memory; 1477 u32 object_count; 1478 }; 1479 1480 struct drm_i915_error_state_buf { 1481 struct drm_i915_private *i915; 1482 unsigned bytes; 1483 unsigned size; 1484 int err; 1485 u8 *buf; 1486 loff_t start; 1487 loff_t pos; 1488 }; 1489 1490 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ 1491 #define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ 1492 1493 #define I915_ENGINE_DEAD_TIMEOUT (4 * HZ) /* Seqno, head and subunits dead */ 1494 #define I915_SEQNO_DEAD_TIMEOUT (12 * HZ) /* Seqno dead with active head */ 1495 1496 struct i915_gpu_error { 1497 /* For hangcheck timer */ 1498 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ 1499 #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) 1500 1501 struct delayed_work hangcheck_work; 1502 1503 /* For reset and error_state handling. */ 1504 spinlock_t lock; 1505 /* Protected by the above dev->gpu_error.lock. */ 1506 struct i915_gpu_state *first_error; 1507 1508 unsigned long missed_irq_rings; 1509 1510 /** 1511 * State variable controlling the reset flow and count 1512 * 1513 * This is a counter which gets incremented when reset is triggered, 1514 * 1515 * Before the reset commences, the I915_RESET_BACKOFF bit is set 1516 * meaning that any waiters holding onto the struct_mutex should 1517 * relinquish the lock immediately in order for the reset to start. 1518 * 1519 * If reset is not completed succesfully, the I915_WEDGE bit is 1520 * set meaning that hardware is terminally sour and there is no 1521 * recovery. All waiters on the reset_queue will be woken when 1522 * that happens. 1523 * 1524 * This counter is used by the wait_seqno code to notice that reset 1525 * event happened and it needs to restart the entire ioctl (since most 1526 * likely the seqno it waited for won't ever signal anytime soon). 1527 * 1528 * This is important for lock-free wait paths, where no contended lock 1529 * naturally enforces the correct ordering between the bail-out of the 1530 * waiter and the gpu reset work code. 1531 */ 1532 unsigned long reset_count; 1533 1534 /** 1535 * flags: Control various stages of the GPU reset 1536 * 1537 * #I915_RESET_BACKOFF - When we start a reset, we want to stop any 1538 * other users acquiring the struct_mutex. To do this we set the 1539 * #I915_RESET_BACKOFF bit in the error flags when we detect a reset 1540 * and then check for that bit before acquiring the struct_mutex (in 1541 * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a 1542 * secondary role in preventing two concurrent global reset attempts. 1543 * 1544 * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the 1545 * struct_mutex. We try to acquire the struct_mutex in the reset worker, 1546 * but it may be held by some long running waiter (that we cannot 1547 * interrupt without causing trouble). Once we are ready to do the GPU 1548 * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If 1549 * they already hold the struct_mutex and want to participate they can 1550 * inspect the bit and do the reset directly, otherwise the worker 1551 * waits for the struct_mutex. 1552 * 1553 * #I915_WEDGED - If reset fails and we can no longer use the GPU, 1554 * we set the #I915_WEDGED bit. Prior to command submission, e.g. 1555 * i915_gem_request_alloc(), this bit is checked and the sequence 1556 * aborted (with -EIO reported to userspace) if set. 1557 */ 1558 unsigned long flags; 1559 #define I915_RESET_BACKOFF 0 1560 #define I915_RESET_HANDOFF 1 1561 #define I915_WEDGED (BITS_PER_LONG - 1) 1562 1563 /** 1564 * Waitqueue to signal when a hang is detected. Used to for waiters 1565 * to release the struct_mutex for the reset to procede. 1566 */ 1567 wait_queue_head_t wait_queue; 1568 1569 /** 1570 * Waitqueue to signal when the reset has completed. Used by clients 1571 * that wait for dev_priv->mm.wedged to settle. 1572 */ 1573 wait_queue_head_t reset_queue; 1574 1575 /* For missed irq/seqno simulation. */ 1576 unsigned long test_irq_rings; 1577 }; 1578 1579 enum modeset_restore { 1580 MODESET_ON_LID_OPEN, 1581 MODESET_DONE, 1582 MODESET_SUSPENDED, 1583 }; 1584 1585 #define DP_AUX_A 0x40 1586 #define DP_AUX_B 0x10 1587 #define DP_AUX_C 0x20 1588 #define DP_AUX_D 0x30 1589 1590 #define DDC_PIN_B 0x05 1591 #define DDC_PIN_C 0x04 1592 #define DDC_PIN_D 0x06 1593 1594 struct ddi_vbt_port_info { 1595 /* 1596 * This is an index in the HDMI/DVI DDI buffer translation table. 1597 * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't 1598 * populate this field. 1599 */ 1600 #define HDMI_LEVEL_SHIFT_UNKNOWN 0xff 1601 uint8_t hdmi_level_shift; 1602 1603 uint8_t supports_dvi:1; 1604 uint8_t supports_hdmi:1; 1605 uint8_t supports_dp:1; 1606 uint8_t supports_edp:1; 1607 1608 uint8_t alternate_aux_channel; 1609 uint8_t alternate_ddc_pin; 1610 1611 uint8_t dp_boost_level; 1612 uint8_t hdmi_boost_level; 1613 }; 1614 1615 enum psr_lines_to_wait { 1616 PSR_0_LINES_TO_WAIT = 0, 1617 PSR_1_LINE_TO_WAIT, 1618 PSR_4_LINES_TO_WAIT, 1619 PSR_8_LINES_TO_WAIT 1620 }; 1621 1622 struct intel_vbt_data { 1623 struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ 1624 struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ 1625 1626 /* Feature bits */ 1627 unsigned int int_tv_support:1; 1628 unsigned int lvds_dither:1; 1629 unsigned int lvds_vbt:1; 1630 unsigned int int_crt_support:1; 1631 unsigned int lvds_use_ssc:1; 1632 unsigned int display_clock_mode:1; 1633 unsigned int fdi_rx_polarity_inverted:1; 1634 unsigned int panel_type:4; 1635 int lvds_ssc_freq; 1636 unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ 1637 1638 enum drrs_support_type drrs_type; 1639 1640 struct { 1641 int rate; 1642 int lanes; 1643 int preemphasis; 1644 int vswing; 1645 bool low_vswing; 1646 bool initialized; 1647 bool support; 1648 int bpp; 1649 struct edp_power_seq pps; 1650 } edp; 1651 1652 struct { 1653 bool full_link; 1654 bool require_aux_wakeup; 1655 int idle_frames; 1656 enum psr_lines_to_wait lines_to_wait; 1657 int tp1_wakeup_time; 1658 int tp2_tp3_wakeup_time; 1659 } psr; 1660 1661 struct { 1662 u16 pwm_freq_hz; 1663 bool present; 1664 bool active_low_pwm; 1665 u8 min_brightness; /* min_brightness/255 of max */ 1666 u8 controller; /* brightness controller number */ 1667 enum intel_backlight_type type; 1668 } backlight; 1669 1670 /* MIPI DSI */ 1671 struct { 1672 u16 panel_id; 1673 struct mipi_config *config; 1674 struct mipi_pps_data *pps; 1675 u8 seq_version; 1676 u32 size; 1677 u8 *data; 1678 const u8 *sequence[MIPI_SEQ_MAX]; 1679 } dsi; 1680 1681 int crt_ddc_pin; 1682 1683 int child_dev_num; 1684 union child_device_config *child_dev; 1685 1686 struct ddi_vbt_port_info ddi_port_info[I915_MAX_PORTS]; 1687 struct sdvo_device_mapping sdvo_mappings[2]; 1688 }; 1689 1690 enum intel_ddb_partitioning { 1691 INTEL_DDB_PART_1_2, 1692 INTEL_DDB_PART_5_6, /* IVB+ */ 1693 }; 1694 1695 struct intel_wm_level { 1696 bool enable; 1697 uint32_t pri_val; 1698 uint32_t spr_val; 1699 uint32_t cur_val; 1700 uint32_t fbc_val; 1701 }; 1702 1703 struct ilk_wm_values { 1704 uint32_t wm_pipe[3]; 1705 uint32_t wm_lp[3]; 1706 uint32_t wm_lp_spr[3]; 1707 uint32_t wm_linetime[3]; 1708 bool enable_fbc_wm; 1709 enum intel_ddb_partitioning partitioning; 1710 }; 1711 1712 struct g4x_pipe_wm { 1713 uint16_t plane[I915_MAX_PLANES]; 1714 uint16_t fbc; 1715 }; 1716 1717 struct g4x_sr_wm { 1718 uint16_t plane; 1719 uint16_t cursor; 1720 uint16_t fbc; 1721 }; 1722 1723 struct vlv_wm_ddl_values { 1724 uint8_t plane[I915_MAX_PLANES]; 1725 }; 1726 1727 struct vlv_wm_values { 1728 struct g4x_pipe_wm pipe[3]; 1729 struct g4x_sr_wm sr; 1730 struct vlv_wm_ddl_values ddl[3]; 1731 uint8_t level; 1732 bool cxsr; 1733 }; 1734 1735 struct g4x_wm_values { 1736 struct g4x_pipe_wm pipe[2]; 1737 struct g4x_sr_wm sr; 1738 struct g4x_sr_wm hpll; 1739 bool cxsr; 1740 bool hpll_en; 1741 bool fbc_en; 1742 }; 1743 1744 struct skl_ddb_entry { 1745 uint16_t start, end; /* in number of blocks, 'end' is exclusive */ 1746 }; 1747 1748 static inline uint16_t skl_ddb_entry_size(const struct skl_ddb_entry *entry) 1749 { 1750 return entry->end - entry->start; 1751 } 1752 1753 static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, 1754 const struct skl_ddb_entry *e2) 1755 { 1756 if (e1->start == e2->start && e1->end == e2->end) 1757 return true; 1758 1759 return false; 1760 } 1761 1762 struct skl_ddb_allocation { 1763 struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */ 1764 struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; 1765 }; 1766 1767 struct skl_wm_values { 1768 unsigned dirty_pipes; 1769 struct skl_ddb_allocation ddb; 1770 }; 1771 1772 struct skl_wm_level { 1773 bool plane_en; 1774 uint16_t plane_res_b; 1775 uint8_t plane_res_l; 1776 }; 1777 1778 /* 1779 * This struct helps tracking the state needed for runtime PM, which puts the 1780 * device in PCI D3 state. Notice that when this happens, nothing on the 1781 * graphics device works, even register access, so we don't get interrupts nor 1782 * anything else. 1783 * 1784 * Every piece of our code that needs to actually touch the hardware needs to 1785 * either call intel_runtime_pm_get or call intel_display_power_get with the 1786 * appropriate power domain. 1787 * 1788 * Our driver uses the autosuspend delay feature, which means we'll only really 1789 * suspend if we stay with zero refcount for a certain amount of time. The 1790 * default value is currently very conservative (see intel_runtime_pm_enable), but 1791 * it can be changed with the standard runtime PM files from sysfs. 1792 * 1793 * The irqs_disabled variable becomes true exactly after we disable the IRQs and 1794 * goes back to false exactly before we reenable the IRQs. We use this variable 1795 * to check if someone is trying to enable/disable IRQs while they're supposed 1796 * to be disabled. This shouldn't happen and we'll print some error messages in 1797 * case it happens. 1798 * 1799 * For more, read the Documentation/power/runtime_pm.txt. 1800 */ 1801 struct i915_runtime_pm { 1802 atomic_t wakeref_count; 1803 bool suspended; 1804 bool irqs_enabled; 1805 }; 1806 1807 enum intel_pipe_crc_source { 1808 INTEL_PIPE_CRC_SOURCE_NONE, 1809 INTEL_PIPE_CRC_SOURCE_PLANE1, 1810 INTEL_PIPE_CRC_SOURCE_PLANE2, 1811 INTEL_PIPE_CRC_SOURCE_PF, 1812 INTEL_PIPE_CRC_SOURCE_PIPE, 1813 /* TV/DP on pre-gen5/vlv can't use the pipe source. */ 1814 INTEL_PIPE_CRC_SOURCE_TV, 1815 INTEL_PIPE_CRC_SOURCE_DP_B, 1816 INTEL_PIPE_CRC_SOURCE_DP_C, 1817 INTEL_PIPE_CRC_SOURCE_DP_D, 1818 INTEL_PIPE_CRC_SOURCE_AUTO, 1819 INTEL_PIPE_CRC_SOURCE_MAX, 1820 }; 1821 1822 struct intel_pipe_crc_entry { 1823 uint32_t frame; 1824 uint32_t crc[5]; 1825 }; 1826 1827 #define INTEL_PIPE_CRC_ENTRIES_NR 128 1828 struct intel_pipe_crc { 1829 spinlock_t lock; 1830 bool opened; /* exclusive access to the result file */ 1831 struct intel_pipe_crc_entry *entries; 1832 enum intel_pipe_crc_source source; 1833 int head, tail; 1834 wait_queue_head_t wq; 1835 int skipped; 1836 }; 1837 1838 struct i915_frontbuffer_tracking { 1839 spinlock_t lock; 1840 1841 /* 1842 * Tracking bits for delayed frontbuffer flushing du to gpu activity or 1843 * scheduled flips. 1844 */ 1845 unsigned busy_bits; 1846 unsigned flip_bits; 1847 }; 1848 1849 struct i915_wa_reg { 1850 i915_reg_t addr; 1851 u32 value; 1852 /* bitmask representing WA bits */ 1853 u32 mask; 1854 }; 1855 1856 /* 1857 * RING_MAX_NONPRIV_SLOTS is per-engine but at this point we are only 1858 * allowing it for RCS as we don't foresee any requirement of having 1859 * a whitelist for other engines. When it is really required for 1860 * other engines then the limit need to be increased. 1861 */ 1862 #define I915_MAX_WA_REGS (16 + RING_MAX_NONPRIV_SLOTS) 1863 1864 struct i915_workarounds { 1865 struct i915_wa_reg reg[I915_MAX_WA_REGS]; 1866 u32 count; 1867 u32 hw_whitelist_count[I915_NUM_ENGINES]; 1868 }; 1869 1870 struct i915_virtual_gpu { 1871 bool active; 1872 }; 1873 1874 /* used in computing the new watermarks state */ 1875 struct intel_wm_config { 1876 unsigned int num_pipes_active; 1877 bool sprites_enabled; 1878 bool sprites_scaled; 1879 }; 1880 1881 struct i915_oa_format { 1882 u32 format; 1883 int size; 1884 }; 1885 1886 struct i915_oa_reg { 1887 i915_reg_t addr; 1888 u32 value; 1889 }; 1890 1891 struct i915_perf_stream; 1892 1893 /** 1894 * struct i915_perf_stream_ops - the OPs to support a specific stream type 1895 */ 1896 struct i915_perf_stream_ops { 1897 /** 1898 * @enable: Enables the collection of HW samples, either in response to 1899 * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened 1900 * without `I915_PERF_FLAG_DISABLED`. 1901 */ 1902 void (*enable)(struct i915_perf_stream *stream); 1903 1904 /** 1905 * @disable: Disables the collection of HW samples, either in response 1906 * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying 1907 * the stream. 1908 */ 1909 void (*disable)(struct i915_perf_stream *stream); 1910 1911 /** 1912 * @poll_wait: Call poll_wait, passing a wait queue that will be woken 1913 * once there is something ready to read() for the stream 1914 */ 1915 void (*poll_wait)(struct i915_perf_stream *stream, 1916 struct file *file, 1917 poll_table *wait); 1918 1919 /** 1920 * @wait_unlocked: For handling a blocking read, wait until there is 1921 * something to ready to read() for the stream. E.g. wait on the same 1922 * wait queue that would be passed to poll_wait(). 1923 */ 1924 int (*wait_unlocked)(struct i915_perf_stream *stream); 1925 1926 /** 1927 * @read: Copy buffered metrics as records to userspace 1928 * **buf**: the userspace, destination buffer 1929 * **count**: the number of bytes to copy, requested by userspace 1930 * **offset**: zero at the start of the read, updated as the read 1931 * proceeds, it represents how many bytes have been copied so far and 1932 * the buffer offset for copying the next record. 1933 * 1934 * Copy as many buffered i915 perf samples and records for this stream 1935 * to userspace as will fit in the given buffer. 1936 * 1937 * Only write complete records; returning -%ENOSPC if there isn't room 1938 * for a complete record. 1939 * 1940 * Return any error condition that results in a short read such as 1941 * -%ENOSPC or -%EFAULT, even though these may be squashed before 1942 * returning to userspace. 1943 */ 1944 int (*read)(struct i915_perf_stream *stream, 1945 char __user *buf, 1946 size_t count, 1947 size_t *offset); 1948 1949 /** 1950 * @destroy: Cleanup any stream specific resources. 1951 * 1952 * The stream will always be disabled before this is called. 1953 */ 1954 void (*destroy)(struct i915_perf_stream *stream); 1955 }; 1956 1957 /** 1958 * struct i915_perf_stream - state for a single open stream FD 1959 */ 1960 struct i915_perf_stream { 1961 /** 1962 * @dev_priv: i915 drm device 1963 */ 1964 struct drm_i915_private *dev_priv; 1965 1966 /** 1967 * @link: Links the stream into ``&drm_i915_private->streams`` 1968 */ 1969 struct list_head link; 1970 1971 /** 1972 * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*` 1973 * properties given when opening a stream, representing the contents 1974 * of a single sample as read() by userspace. 1975 */ 1976 u32 sample_flags; 1977 1978 /** 1979 * @sample_size: Considering the configured contents of a sample 1980 * combined with the required header size, this is the total size 1981 * of a single sample record. 1982 */ 1983 int sample_size; 1984 1985 /** 1986 * @ctx: %NULL if measuring system-wide across all contexts or a 1987 * specific context that is being monitored. 1988 */ 1989 struct i915_gem_context *ctx; 1990 1991 /** 1992 * @enabled: Whether the stream is currently enabled, considering 1993 * whether the stream was opened in a disabled state and based 1994 * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls. 1995 */ 1996 bool enabled; 1997 1998 /** 1999 * @ops: The callbacks providing the implementation of this specific 2000 * type of configured stream. 2001 */ 2002 const struct i915_perf_stream_ops *ops; 2003 }; 2004 2005 /** 2006 * struct i915_oa_ops - Gen specific implementation of an OA unit stream 2007 */ 2008 struct i915_oa_ops { 2009 /** 2010 * @init_oa_buffer: Resets the head and tail pointers of the 2011 * circular buffer for periodic OA reports. 2012 * 2013 * Called when first opening a stream for OA metrics, but also may be 2014 * called in response to an OA buffer overflow or other error 2015 * condition. 2016 * 2017 * Note it may be necessary to clear the full OA buffer here as part of 2018 * maintaining the invariable that new reports must be written to 2019 * zeroed memory for us to be able to reliable detect if an expected 2020 * report has not yet landed in memory. (At least on Haswell the OA 2021 * buffer tail pointer is not synchronized with reports being visible 2022 * to the CPU) 2023 */ 2024 void (*init_oa_buffer)(struct drm_i915_private *dev_priv); 2025 2026 /** 2027 * @select_metric_set: The auto generated code that checks whether a 2028 * requested OA config is applicable to the system and if so sets up 2029 * the mux, oa and flex eu register config pointers according to the 2030 * current dev_priv->perf.oa.metrics_set. 2031 */ 2032 int (*select_metric_set)(struct drm_i915_private *dev_priv); 2033 2034 /** 2035 * @enable_metric_set: Selects and applies any MUX configuration to set 2036 * up the Boolean and Custom (B/C) counters that are part of the 2037 * counter reports being sampled. May apply system constraints such as 2038 * disabling EU clock gating as required. 2039 */ 2040 int (*enable_metric_set)(struct drm_i915_private *dev_priv); 2041 2042 /** 2043 * @disable_metric_set: Remove system constraints associated with using 2044 * the OA unit. 2045 */ 2046 void (*disable_metric_set)(struct drm_i915_private *dev_priv); 2047 2048 /** 2049 * @oa_enable: Enable periodic sampling 2050 */ 2051 void (*oa_enable)(struct drm_i915_private *dev_priv); 2052 2053 /** 2054 * @oa_disable: Disable periodic sampling 2055 */ 2056 void (*oa_disable)(struct drm_i915_private *dev_priv); 2057 2058 /** 2059 * @read: Copy data from the circular OA buffer into a given userspace 2060 * buffer. 2061 */ 2062 int (*read)(struct i915_perf_stream *stream, 2063 char __user *buf, 2064 size_t count, 2065 size_t *offset); 2066 2067 /** 2068 * @oa_hw_tail_read: read the OA tail pointer register 2069 * 2070 * In particular this enables us to share all the fiddly code for 2071 * handling the OA unit tail pointer race that affects multiple 2072 * generations. 2073 */ 2074 u32 (*oa_hw_tail_read)(struct drm_i915_private *dev_priv); 2075 }; 2076 2077 struct intel_cdclk_state { 2078 unsigned int cdclk, vco, ref; 2079 }; 2080 2081 struct drm_i915_private { 2082 struct drm_device drm; 2083 2084 struct kmem_cache *objects; 2085 struct kmem_cache *vmas; 2086 struct kmem_cache *requests; 2087 struct kmem_cache *dependencies; 2088 struct kmem_cache *priorities; 2089 2090 const struct intel_device_info info; 2091 2092 void __iomem *regs; 2093 2094 struct intel_uncore uncore; 2095 2096 struct i915_virtual_gpu vgpu; 2097 2098 struct intel_gvt *gvt; 2099 2100 struct intel_huc huc; 2101 struct intel_guc guc; 2102 2103 struct intel_csr csr; 2104 2105 struct intel_gmbus gmbus[GMBUS_NUM_PINS]; 2106 2107 /** gmbus_mutex protects against concurrent usage of the single hw gmbus 2108 * controller on different i2c buses. */ 2109 struct mutex gmbus_mutex; 2110 2111 /** 2112 * Base address of the gmbus and gpio block. 2113 */ 2114 uint32_t gpio_mmio_base; 2115 2116 /* MMIO base address for MIPI regs */ 2117 uint32_t mipi_mmio_base; 2118 2119 uint32_t psr_mmio_base; 2120 2121 uint32_t pps_mmio_base; 2122 2123 wait_queue_head_t gmbus_wait_queue; 2124 2125 struct pci_dev *bridge_dev; 2126 struct i915_gem_context *kernel_context; 2127 struct intel_engine_cs *engine[I915_NUM_ENGINES]; 2128 struct i915_vma *semaphore; 2129 2130 struct drm_dma_handle *status_page_dmah; 2131 struct resource mch_res; 2132 2133 /* protects the irq masks */ 2134 spinlock_t irq_lock; 2135 2136 /* protects the mmio flip data */ 2137 spinlock_t mmio_flip_lock; 2138 2139 bool display_irqs_enabled; 2140 2141 /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */ 2142 struct pm_qos_request pm_qos; 2143 2144 /* Sideband mailbox protection */ 2145 struct mutex sb_lock; 2146 2147 /** Cached value of IMR to avoid reads in updating the bitfield */ 2148 union { 2149 u32 irq_mask; 2150 u32 de_irq_mask[I915_MAX_PIPES]; 2151 }; 2152 u32 gt_irq_mask; 2153 u32 pm_imr; 2154 u32 pm_ier; 2155 u32 pm_rps_events; 2156 u32 pm_guc_events; 2157 u32 pipestat_irq_mask[I915_MAX_PIPES]; 2158 2159 struct i915_hotplug hotplug; 2160 struct intel_fbc fbc; 2161 struct i915_drrs drrs; 2162 struct intel_opregion opregion; 2163 struct intel_vbt_data vbt; 2164 2165 bool preserve_bios_swizzle; 2166 2167 /* overlay */ 2168 struct intel_overlay *overlay; 2169 2170 /* backlight registers and fields in struct intel_panel */ 2171 struct mutex backlight_lock; 2172 2173 /* LVDS info */ 2174 bool no_aux_handshake; 2175 2176 /* protects panel power sequencer state */ 2177 struct mutex pps_mutex; 2178 2179 struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ 2180 int num_fence_regs; /* 8 on pre-965, 16 otherwise */ 2181 2182 unsigned int fsb_freq, mem_freq, is_ddr3; 2183 unsigned int skl_preferred_vco_freq; 2184 unsigned int max_cdclk_freq; 2185 2186 unsigned int max_dotclk_freq; 2187 unsigned int rawclk_freq; 2188 unsigned int hpll_freq; 2189 unsigned int czclk_freq; 2190 2191 struct { 2192 /* 2193 * The current logical cdclk state. 2194 * See intel_atomic_state.cdclk.logical 2195 * 2196 * For reading holding any crtc lock is sufficient, 2197 * for writing must hold all of them. 2198 */ 2199 struct intel_cdclk_state logical; 2200 /* 2201 * The current actual cdclk state. 2202 * See intel_atomic_state.cdclk.actual 2203 */ 2204 struct intel_cdclk_state actual; 2205 /* The current hardware cdclk state */ 2206 struct intel_cdclk_state hw; 2207 } cdclk; 2208 2209 /** 2210 * wq - Driver workqueue for GEM. 2211 * 2212 * NOTE: Work items scheduled here are not allowed to grab any modeset 2213 * locks, for otherwise the flushing done in the pageflip code will 2214 * result in deadlocks. 2215 */ 2216 struct workqueue_struct *wq; 2217 2218 /* Display functions */ 2219 struct drm_i915_display_funcs display; 2220 2221 /* PCH chipset type */ 2222 enum intel_pch pch_type; 2223 unsigned short pch_id; 2224 2225 unsigned long quirks; 2226 2227 enum modeset_restore modeset_restore; 2228 struct mutex modeset_restore_lock; 2229 struct drm_atomic_state *modeset_restore_state; 2230 struct drm_modeset_acquire_ctx reset_ctx; 2231 2232 struct list_head vm_list; /* Global list of all address spaces */ 2233 struct i915_ggtt ggtt; /* VM representing the global address space */ 2234 2235 struct i915_gem_mm mm; 2236 DECLARE_HASHTABLE(mm_structs, 7); 2237 struct mutex mm_lock; 2238 2239 /* The hw wants to have a stable context identifier for the lifetime 2240 * of the context (for OA, PASID, faults, etc). This is limited 2241 * in execlists to 21 bits. 2242 */ 2243 struct ida context_hw_ida; 2244 #define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */ 2245 2246 /* Kernel Modesetting */ 2247 2248 struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; 2249 struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; 2250 wait_queue_head_t pending_flip_queue; 2251 2252 #ifdef CONFIG_DEBUG_FS 2253 struct intel_pipe_crc pipe_crc[I915_MAX_PIPES]; 2254 #endif 2255 2256 /* dpll and cdclk state is protected by connection_mutex */ 2257 int num_shared_dpll; 2258 struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; 2259 const struct intel_dpll_mgr *dpll_mgr; 2260 2261 /* 2262 * dpll_lock serializes intel_{prepare,enable,disable}_shared_dpll. 2263 * Must be global rather than per dpll, because on some platforms 2264 * plls share registers. 2265 */ 2266 struct mutex dpll_lock; 2267 2268 unsigned int active_crtcs; 2269 unsigned int min_pixclk[I915_MAX_PIPES]; 2270 2271 int dpio_phy_iosf_port[I915_NUM_PHYS_VLV]; 2272 2273 struct i915_workarounds workarounds; 2274 2275 struct i915_frontbuffer_tracking fb_tracking; 2276 2277 struct intel_atomic_helper { 2278 struct llist_head free_list; 2279 struct work_struct free_work; 2280 } atomic_helper; 2281 2282 u16 orig_clock; 2283 2284 bool mchbar_need_disable; 2285 2286 struct intel_l3_parity l3_parity; 2287 2288 /* Cannot be determined by PCIID. You must always read a register. */ 2289 u32 edram_cap; 2290 2291 /* gen6+ rps state */ 2292 struct intel_gen6_power_mgmt rps; 2293 2294 /* ilk-only ips/rps state. Everything in here is protected by the global 2295 * mchdev_lock in intel_pm.c */ 2296 struct intel_ilk_power_mgmt ips; 2297 2298 struct i915_power_domains power_domains; 2299 2300 struct i915_psr psr; 2301 2302 struct i915_gpu_error gpu_error; 2303 2304 struct drm_i915_gem_object *vlv_pctx; 2305 2306 #ifdef CONFIG_DRM_FBDEV_EMULATION 2307 /* list of fbdev register on this device */ 2308 struct intel_fbdev *fbdev; 2309 struct work_struct fbdev_suspend_work; 2310 #endif 2311 2312 struct drm_property *broadcast_rgb_property; 2313 struct drm_property *force_audio_property; 2314 2315 /* hda/i915 audio component */ 2316 struct i915_audio_component *audio_component; 2317 bool audio_component_registered; 2318 /** 2319 * av_mutex - mutex for audio/video sync 2320 * 2321 */ 2322 struct mutex av_mutex; 2323 2324 struct list_head context_list; 2325 2326 u32 fdi_rx_config; 2327 2328 /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */ 2329 u32 chv_phy_control; 2330 /* 2331 * Shadows for CHV DPLL_MD regs to keep the state 2332 * checker somewhat working in the presence hardware 2333 * crappiness (can't read out DPLL_MD for pipes B & C). 2334 */ 2335 u32 chv_dpll_md[I915_MAX_PIPES]; 2336 u32 bxt_phy_grc; 2337 2338 u32 suspend_count; 2339 bool suspended_to_idle; 2340 struct i915_suspend_saved_registers regfile; 2341 struct vlv_s0ix_state vlv_s0ix_state; 2342 2343 enum { 2344 I915_SAGV_UNKNOWN = 0, 2345 I915_SAGV_DISABLED, 2346 I915_SAGV_ENABLED, 2347 I915_SAGV_NOT_CONTROLLED 2348 } sagv_status; 2349 2350 struct { 2351 /* 2352 * Raw watermark latency values: 2353 * in 0.1us units for WM0, 2354 * in 0.5us units for WM1+. 2355 */ 2356 /* primary */ 2357 uint16_t pri_latency[5]; 2358 /* sprite */ 2359 uint16_t spr_latency[5]; 2360 /* cursor */ 2361 uint16_t cur_latency[5]; 2362 /* 2363 * Raw watermark memory latency values 2364 * for SKL for all 8 levels 2365 * in 1us units. 2366 */ 2367 uint16_t skl_latency[8]; 2368 2369 /* current hardware state */ 2370 union { 2371 struct ilk_wm_values hw; 2372 struct skl_wm_values skl_hw; 2373 struct vlv_wm_values vlv; 2374 struct g4x_wm_values g4x; 2375 }; 2376 2377 uint8_t max_level; 2378 2379 /* 2380 * Should be held around atomic WM register writing; also 2381 * protects * intel_crtc->wm.active and 2382 * cstate->wm.need_postvbl_update. 2383 */ 2384 struct mutex wm_mutex; 2385 2386 /* 2387 * Set during HW readout of watermarks/DDB. Some platforms 2388 * need to know when we're still using BIOS-provided values 2389 * (which we don't fully trust). 2390 */ 2391 bool distrust_bios_wm; 2392 } wm; 2393 2394 struct i915_runtime_pm pm; 2395 2396 struct { 2397 bool initialized; 2398 2399 struct kobject *metrics_kobj; 2400 struct ctl_table_header *sysctl_header; 2401 2402 struct mutex lock; 2403 struct list_head streams; 2404 2405 struct { 2406 struct i915_perf_stream *exclusive_stream; 2407 2408 u32 specific_ctx_id; 2409 2410 struct hrtimer poll_check_timer; 2411 wait_queue_head_t poll_wq; 2412 bool pollin; 2413 2414 /** 2415 * For rate limiting any notifications of spurious 2416 * invalid OA reports 2417 */ 2418 struct ratelimit_state spurious_report_rs; 2419 2420 bool periodic; 2421 int period_exponent; 2422 int timestamp_frequency; 2423 2424 int metrics_set; 2425 2426 const struct i915_oa_reg *mux_regs[6]; 2427 int mux_regs_lens[6]; 2428 int n_mux_configs; 2429 2430 const struct i915_oa_reg *b_counter_regs; 2431 int b_counter_regs_len; 2432 const struct i915_oa_reg *flex_regs; 2433 int flex_regs_len; 2434 2435 struct { 2436 struct i915_vma *vma; 2437 u8 *vaddr; 2438 u32 last_ctx_id; 2439 int format; 2440 int format_size; 2441 2442 /** 2443 * Locks reads and writes to all head/tail state 2444 * 2445 * Consider: the head and tail pointer state 2446 * needs to be read consistently from a hrtimer 2447 * callback (atomic context) and read() fop 2448 * (user context) with tail pointer updates 2449 * happening in atomic context and head updates 2450 * in user context and the (unlikely) 2451 * possibility of read() errors needing to 2452 * reset all head/tail state. 2453 * 2454 * Note: Contention or performance aren't 2455 * currently a significant concern here 2456 * considering the relatively low frequency of 2457 * hrtimer callbacks (5ms period) and that 2458 * reads typically only happen in response to a 2459 * hrtimer event and likely complete before the 2460 * next callback. 2461 * 2462 * Note: This lock is not held *while* reading 2463 * and copying data to userspace so the value 2464 * of head observed in htrimer callbacks won't 2465 * represent any partial consumption of data. 2466 */ 2467 spinlock_t ptr_lock; 2468 2469 /** 2470 * One 'aging' tail pointer and one 'aged' 2471 * tail pointer ready to used for reading. 2472 * 2473 * Initial values of 0xffffffff are invalid 2474 * and imply that an update is required 2475 * (and should be ignored by an attempted 2476 * read) 2477 */ 2478 struct { 2479 u32 offset; 2480 } tails[2]; 2481 2482 /** 2483 * Index for the aged tail ready to read() 2484 * data up to. 2485 */ 2486 unsigned int aged_tail_idx; 2487 2488 /** 2489 * A monotonic timestamp for when the current 2490 * aging tail pointer was read; used to 2491 * determine when it is old enough to trust. 2492 */ 2493 u64 aging_timestamp; 2494 2495 /** 2496 * Although we can always read back the head 2497 * pointer register, we prefer to avoid 2498 * trusting the HW state, just to avoid any 2499 * risk that some hardware condition could 2500 * somehow bump the head pointer unpredictably 2501 * and cause us to forward the wrong OA buffer 2502 * data to userspace. 2503 */ 2504 u32 head; 2505 } oa_buffer; 2506 2507 u32 gen7_latched_oastatus1; 2508 u32 ctx_oactxctrl_offset; 2509 u32 ctx_flexeu0_offset; 2510 2511 /** 2512 * The RPT_ID/reason field for Gen8+ includes a bit 2513 * to determine if the CTX ID in the report is valid 2514 * but the specific bit differs between Gen 8 and 9 2515 */ 2516 u32 gen8_valid_ctx_bit; 2517 2518 struct i915_oa_ops ops; 2519 const struct i915_oa_format *oa_formats; 2520 int n_builtin_sets; 2521 } oa; 2522 } perf; 2523 2524 /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */ 2525 struct { 2526 void (*resume)(struct drm_i915_private *); 2527 void (*cleanup_engine)(struct intel_engine_cs *engine); 2528 2529 struct list_head timelines; 2530 struct i915_gem_timeline global_timeline; 2531 u32 active_requests; 2532 2533 /** 2534 * Is the GPU currently considered idle, or busy executing 2535 * userspace requests? Whilst idle, we allow runtime power 2536 * management to power down the hardware and display clocks. 2537 * In order to reduce the effect on performance, there 2538 * is a slight delay before we do so. 2539 */ 2540 bool awake; 2541 2542 /** 2543 * We leave the user IRQ off as much as possible, 2544 * but this means that requests will finish and never 2545 * be retired once the system goes idle. Set a timer to 2546 * fire periodically while the ring is running. When it 2547 * fires, go retire requests. 2548 */ 2549 struct delayed_work retire_work; 2550 2551 /** 2552 * When we detect an idle GPU, we want to turn on 2553 * powersaving features. So once we see that there 2554 * are no more requests outstanding and no more 2555 * arrive within a small period of time, we fire 2556 * off the idle_work. 2557 */ 2558 struct delayed_work idle_work; 2559 2560 ktime_t last_init_time; 2561 } gt; 2562 2563 /* perform PHY state sanity checks? */ 2564 bool chv_phy_assert[2]; 2565 2566 bool ipc_enabled; 2567 2568 /* Used to save the pipe-to-encoder mapping for audio */ 2569 struct intel_encoder *av_enc_map[I915_MAX_PIPES]; 2570 2571 /* necessary resource sharing with HDMI LPE audio driver. */ 2572 struct { 2573 struct platform_device *platdev; 2574 int irq; 2575 } lpe_audio; 2576 2577 /* 2578 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch 2579 * will be rejected. Instead look for a better place. 2580 */ 2581 }; 2582 2583 static inline struct drm_i915_private *to_i915(const struct drm_device *dev) 2584 { 2585 return container_of(dev, struct drm_i915_private, drm); 2586 } 2587 2588 static inline struct drm_i915_private *kdev_to_i915(struct device *kdev) 2589 { 2590 return to_i915(dev_get_drvdata(kdev)); 2591 } 2592 2593 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) 2594 { 2595 return container_of(guc, struct drm_i915_private, guc); 2596 } 2597 2598 static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc) 2599 { 2600 return container_of(huc, struct drm_i915_private, huc); 2601 } 2602 2603 /* Simple iterator over all initialised engines */ 2604 #define for_each_engine(engine__, dev_priv__, id__) \ 2605 for ((id__) = 0; \ 2606 (id__) < I915_NUM_ENGINES; \ 2607 (id__)++) \ 2608 for_each_if ((engine__) = (dev_priv__)->engine[(id__)]) 2609 2610 /* Iterator over subset of engines selected by mask */ 2611 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \ 2612 for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask; \ 2613 tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; ) 2614 2615 enum hdmi_force_audio { 2616 HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ 2617 HDMI_AUDIO_OFF, /* force turn off HDMI audio */ 2618 HDMI_AUDIO_AUTO, /* trust EDID */ 2619 HDMI_AUDIO_ON, /* force turn on HDMI audio */ 2620 }; 2621 2622 #define I915_GTT_OFFSET_NONE ((u32)-1) 2623 2624 /* 2625 * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is 2626 * considered to be the frontbuffer for the given plane interface-wise. This 2627 * doesn't mean that the hw necessarily already scans it out, but that any 2628 * rendering (by the cpu or gpu) will land in the frontbuffer eventually. 2629 * 2630 * We have one bit per pipe and per scanout plane type. 2631 */ 2632 #define INTEL_MAX_SPRITE_BITS_PER_PIPE 5 2633 #define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 2634 #define INTEL_FRONTBUFFER_PRIMARY(pipe) \ 2635 (1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) 2636 #define INTEL_FRONTBUFFER_CURSOR(pipe) \ 2637 (1 << (1 + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2638 #define INTEL_FRONTBUFFER_SPRITE(pipe, plane) \ 2639 (1 << (2 + plane + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2640 #define INTEL_FRONTBUFFER_OVERLAY(pipe) \ 2641 (1 << (2 + INTEL_MAX_SPRITE_BITS_PER_PIPE + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2642 #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ 2643 (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) 2644 2645 /* 2646 * Optimised SGL iterator for GEM objects 2647 */ 2648 static __always_inline struct sgt_iter { 2649 struct scatterlist *sgp; 2650 union { 2651 unsigned long pfn; 2652 dma_addr_t dma; 2653 }; 2654 unsigned int curr; 2655 unsigned int max; 2656 } __sgt_iter(struct scatterlist *sgl, bool dma) { 2657 struct sgt_iter s = { .sgp = sgl }; 2658 2659 if (s.sgp) { 2660 s.max = s.curr = s.sgp->offset; 2661 s.max += s.sgp->length; 2662 if (dma) 2663 s.dma = sg_dma_address(s.sgp); 2664 else 2665 s.pfn = page_to_pfn(sg_page(s.sgp)); 2666 } 2667 2668 return s; 2669 } 2670 2671 static inline struct scatterlist *____sg_next(struct scatterlist *sg) 2672 { 2673 ++sg; 2674 if (unlikely(sg_is_chain(sg))) 2675 sg = sg_chain_ptr(sg); 2676 return sg; 2677 } 2678 2679 /** 2680 * __sg_next - return the next scatterlist entry in a list 2681 * @sg: The current sg entry 2682 * 2683 * Description: 2684 * If the entry is the last, return NULL; otherwise, step to the next 2685 * element in the array (@sg@+1). If that's a chain pointer, follow it; 2686 * otherwise just return the pointer to the current element. 2687 **/ 2688 static inline struct scatterlist *__sg_next(struct scatterlist *sg) 2689 { 2690 #ifdef CONFIG_DEBUG_SG 2691 BUG_ON(sg->sg_magic != SG_MAGIC); 2692 #endif 2693 return sg_is_last(sg) ? NULL : ____sg_next(sg); 2694 } 2695 2696 /** 2697 * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table 2698 * @__dmap: DMA address (output) 2699 * @__iter: 'struct sgt_iter' (iterator state, internal) 2700 * @__sgt: sg_table to iterate over (input) 2701 */ 2702 #define for_each_sgt_dma(__dmap, __iter, __sgt) \ 2703 for ((__iter) = __sgt_iter((__sgt)->sgl, true); \ 2704 ((__dmap) = (__iter).dma + (__iter).curr); \ 2705 (((__iter).curr += PAGE_SIZE) < (__iter).max) || \ 2706 ((__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0)) 2707 2708 /** 2709 * for_each_sgt_page - iterate over the pages of the given sg_table 2710 * @__pp: page pointer (output) 2711 * @__iter: 'struct sgt_iter' (iterator state, internal) 2712 * @__sgt: sg_table to iterate over (input) 2713 */ 2714 #define for_each_sgt_page(__pp, __iter, __sgt) \ 2715 for ((__iter) = __sgt_iter((__sgt)->sgl, false); \ 2716 ((__pp) = (__iter).pfn == 0 ? NULL : \ 2717 pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \ 2718 (((__iter).curr += PAGE_SIZE) < (__iter).max) || \ 2719 ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0)) 2720 2721 static inline const struct intel_device_info * 2722 intel_info(const struct drm_i915_private *dev_priv) 2723 { 2724 return &dev_priv->info; 2725 } 2726 2727 #define INTEL_INFO(dev_priv) intel_info((dev_priv)) 2728 2729 #define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) 2730 #define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) 2731 2732 #define REVID_FOREVER 0xff 2733 #define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) 2734 2735 #define GEN_FOREVER (0) 2736 /* 2737 * Returns true if Gen is in inclusive range [Start, End]. 2738 * 2739 * Use GEN_FOREVER for unbound start and or end. 2740 */ 2741 #define IS_GEN(dev_priv, s, e) ({ \ 2742 unsigned int __s = (s), __e = (e); \ 2743 BUILD_BUG_ON(!__builtin_constant_p(s)); \ 2744 BUILD_BUG_ON(!__builtin_constant_p(e)); \ 2745 if ((__s) != GEN_FOREVER) \ 2746 __s = (s) - 1; \ 2747 if ((__e) == GEN_FOREVER) \ 2748 __e = BITS_PER_LONG - 1; \ 2749 else \ 2750 __e = (e) - 1; \ 2751 !!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \ 2752 }) 2753 2754 /* 2755 * Return true if revision is in range [since,until] inclusive. 2756 * 2757 * Use 0 for open-ended since, and REVID_FOREVER for open-ended until. 2758 */ 2759 #define IS_REVID(p, since, until) \ 2760 (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) 2761 2762 #define IS_I830(dev_priv) ((dev_priv)->info.platform == INTEL_I830) 2763 #define IS_I845G(dev_priv) ((dev_priv)->info.platform == INTEL_I845G) 2764 #define IS_I85X(dev_priv) ((dev_priv)->info.platform == INTEL_I85X) 2765 #define IS_I865G(dev_priv) ((dev_priv)->info.platform == INTEL_I865G) 2766 #define IS_I915G(dev_priv) ((dev_priv)->info.platform == INTEL_I915G) 2767 #define IS_I915GM(dev_priv) ((dev_priv)->info.platform == INTEL_I915GM) 2768 #define IS_I945G(dev_priv) ((dev_priv)->info.platform == INTEL_I945G) 2769 #define IS_I945GM(dev_priv) ((dev_priv)->info.platform == INTEL_I945GM) 2770 #define IS_I965G(dev_priv) ((dev_priv)->info.platform == INTEL_I965G) 2771 #define IS_I965GM(dev_priv) ((dev_priv)->info.platform == INTEL_I965GM) 2772 #define IS_G45(dev_priv) ((dev_priv)->info.platform == INTEL_G45) 2773 #define IS_GM45(dev_priv) ((dev_priv)->info.platform == INTEL_GM45) 2774 #define IS_G4X(dev_priv) (IS_G45(dev_priv) || IS_GM45(dev_priv)) 2775 #define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001) 2776 #define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011) 2777 #define IS_PINEVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_PINEVIEW) 2778 #define IS_G33(dev_priv) ((dev_priv)->info.platform == INTEL_G33) 2779 #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) 2780 #define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.platform == INTEL_IVYBRIDGE) 2781 #define IS_IVB_GT1(dev_priv) (INTEL_DEVID(dev_priv) == 0x0156 || \ 2782 INTEL_DEVID(dev_priv) == 0x0152 || \ 2783 INTEL_DEVID(dev_priv) == 0x015a) 2784 #define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_VALLEYVIEW) 2785 #define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_CHERRYVIEW) 2786 #define IS_HASWELL(dev_priv) ((dev_priv)->info.platform == INTEL_HASWELL) 2787 #define IS_BROADWELL(dev_priv) ((dev_priv)->info.platform == INTEL_BROADWELL) 2788 #define IS_SKYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_SKYLAKE) 2789 #define IS_BROXTON(dev_priv) ((dev_priv)->info.platform == INTEL_BROXTON) 2790 #define IS_KABYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_KABYLAKE) 2791 #define IS_GEMINILAKE(dev_priv) ((dev_priv)->info.platform == INTEL_GEMINILAKE) 2792 #define IS_COFFEELAKE(dev_priv) ((dev_priv)->info.platform == INTEL_COFFEELAKE) 2793 #define IS_CANNONLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_CANNONLAKE) 2794 #define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) 2795 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ 2796 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) 2797 #define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ 2798 ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 || \ 2799 (INTEL_DEVID(dev_priv) & 0xf) == 0xb || \ 2800 (INTEL_DEVID(dev_priv) & 0xf) == 0xe)) 2801 /* ULX machines are also considered ULT. */ 2802 #define IS_BDW_ULX(dev_priv) (IS_BROADWELL(dev_priv) && \ 2803 (INTEL_DEVID(dev_priv) & 0xf) == 0xe) 2804 #define IS_BDW_GT3(dev_priv) (IS_BROADWELL(dev_priv) && \ 2805 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) 2806 #define IS_HSW_ULT(dev_priv) (IS_HASWELL(dev_priv) && \ 2807 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00) 2808 #define IS_HSW_GT3(dev_priv) (IS_HASWELL(dev_priv) && \ 2809 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) 2810 /* ULX machines are also considered ULT. */ 2811 #define IS_HSW_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x0A0E || \ 2812 INTEL_DEVID(dev_priv) == 0x0A1E) 2813 #define IS_SKL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x1906 || \ 2814 INTEL_DEVID(dev_priv) == 0x1913 || \ 2815 INTEL_DEVID(dev_priv) == 0x1916 || \ 2816 INTEL_DEVID(dev_priv) == 0x1921 || \ 2817 INTEL_DEVID(dev_priv) == 0x1926) 2818 #define IS_SKL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x190E || \ 2819 INTEL_DEVID(dev_priv) == 0x1915 || \ 2820 INTEL_DEVID(dev_priv) == 0x191E) 2821 #define IS_KBL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x5906 || \ 2822 INTEL_DEVID(dev_priv) == 0x5913 || \ 2823 INTEL_DEVID(dev_priv) == 0x5916 || \ 2824 INTEL_DEVID(dev_priv) == 0x5921 || \ 2825 INTEL_DEVID(dev_priv) == 0x5926) 2826 #define IS_KBL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x590E || \ 2827 INTEL_DEVID(dev_priv) == 0x5915 || \ 2828 INTEL_DEVID(dev_priv) == 0x591E) 2829 #define IS_SKL_GT2(dev_priv) (IS_SKYLAKE(dev_priv) && \ 2830 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0010) 2831 #define IS_SKL_GT3(dev_priv) (IS_SKYLAKE(dev_priv) && \ 2832 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) 2833 #define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ 2834 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030) 2835 #define IS_KBL_GT2(dev_priv) (IS_KABYLAKE(dev_priv) && \ 2836 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0010) 2837 #define IS_KBL_GT3(dev_priv) (IS_KABYLAKE(dev_priv) && \ 2838 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) 2839 #define IS_CFL_ULT(dev_priv) (IS_COFFEELAKE(dev_priv) && \ 2840 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0) 2841 2842 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) 2843 2844 #define SKL_REVID_A0 0x0 2845 #define SKL_REVID_B0 0x1 2846 #define SKL_REVID_C0 0x2 2847 #define SKL_REVID_D0 0x3 2848 #define SKL_REVID_E0 0x4 2849 #define SKL_REVID_F0 0x5 2850 #define SKL_REVID_G0 0x6 2851 #define SKL_REVID_H0 0x7 2852 2853 #define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until)) 2854 2855 #define BXT_REVID_A0 0x0 2856 #define BXT_REVID_A1 0x1 2857 #define BXT_REVID_B0 0x3 2858 #define BXT_REVID_B_LAST 0x8 2859 #define BXT_REVID_C0 0x9 2860 2861 #define IS_BXT_REVID(dev_priv, since, until) \ 2862 (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) 2863 2864 #define KBL_REVID_A0 0x0 2865 #define KBL_REVID_B0 0x1 2866 #define KBL_REVID_C0 0x2 2867 #define KBL_REVID_D0 0x3 2868 #define KBL_REVID_E0 0x4 2869 2870 #define IS_KBL_REVID(dev_priv, since, until) \ 2871 (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until)) 2872 2873 #define GLK_REVID_A0 0x0 2874 #define GLK_REVID_A1 0x1 2875 2876 #define IS_GLK_REVID(dev_priv, since, until) \ 2877 (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until)) 2878 2879 #define CNL_REVID_A0 0x0 2880 #define CNL_REVID_B0 0x1 2881 2882 #define IS_CNL_REVID(p, since, until) \ 2883 (IS_CANNONLAKE(p) && IS_REVID(p, since, until)) 2884 2885 /* 2886 * The genX designation typically refers to the render engine, so render 2887 * capability related checks should use IS_GEN, while display and other checks 2888 * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular 2889 * chips, etc.). 2890 */ 2891 #define IS_GEN2(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(1))) 2892 #define IS_GEN3(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(2))) 2893 #define IS_GEN4(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(3))) 2894 #define IS_GEN5(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(4))) 2895 #define IS_GEN6(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(5))) 2896 #define IS_GEN7(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(6))) 2897 #define IS_GEN8(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(7))) 2898 #define IS_GEN9(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(8))) 2899 #define IS_GEN10(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(9))) 2900 2901 #define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) 2902 #define IS_GEN9_LP(dev_priv) (IS_GEN9(dev_priv) && IS_LP(dev_priv)) 2903 #define IS_GEN9_BC(dev_priv) (IS_GEN9(dev_priv) && !IS_LP(dev_priv)) 2904 2905 #define ENGINE_MASK(id) BIT(id) 2906 #define RENDER_RING ENGINE_MASK(RCS) 2907 #define BSD_RING ENGINE_MASK(VCS) 2908 #define BLT_RING ENGINE_MASK(BCS) 2909 #define VEBOX_RING ENGINE_MASK(VECS) 2910 #define BSD2_RING ENGINE_MASK(VCS2) 2911 #define ALL_ENGINES (~0) 2912 2913 #define HAS_ENGINE(dev_priv, id) \ 2914 (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id))) 2915 2916 #define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) 2917 #define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) 2918 #define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) 2919 #define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) 2920 2921 #define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc) 2922 #define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop) 2923 #define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED)) 2924 #define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ 2925 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) 2926 2927 #define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical) 2928 2929 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ 2930 ((dev_priv)->info.has_logical_ring_contexts) 2931 #define USES_PPGTT(dev_priv) (i915.enable_ppgtt) 2932 #define USES_FULL_PPGTT(dev_priv) (i915.enable_ppgtt >= 2) 2933 #define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3) 2934 2935 #define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay) 2936 #define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ 2937 ((dev_priv)->info.overlay_needs_physical) 2938 2939 /* Early gen2 have a totally busted CS tlb and require pinned batches. */ 2940 #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) 2941 2942 /* WaRsDisableCoarsePowerGating:skl,bxt */ 2943 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ 2944 (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) 2945 2946 /* 2947 * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts 2948 * even when in MSI mode. This results in spurious interrupt warnings if the 2949 * legacy irq no. is shared with another device. The kernel then disables that 2950 * interrupt source and so prevents the other device from working properly. 2951 */ 2952 #define HAS_AUX_IRQ(dev_priv) ((dev_priv)->info.gen >= 5) 2953 #define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq) 2954 2955 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte 2956 * rows, which changed the alignment requirements and fence programming. 2957 */ 2958 #define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ 2959 !(IS_I915G(dev_priv) || \ 2960 IS_I915GM(dev_priv))) 2961 #define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv) 2962 #define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug) 2963 2964 #define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) 2965 #define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr) 2966 #define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc) 2967 #define HAS_CUR_FBC(dev_priv) (!HAS_GMCH_DISPLAY(dev_priv) && INTEL_INFO(dev_priv)->gen >= 7) 2968 2969 #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) 2970 2971 #define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst) 2972 2973 #define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) 2974 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg) 2975 #define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr) 2976 #define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6) 2977 #define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p) 2978 2979 #define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr) 2980 2981 #define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) 2982 #define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) 2983 2984 /* 2985 * For now, anything with a GuC requires uCode loading, and then supports 2986 * command submission once loaded. But these are logically independent 2987 * properties, so we have separate macros to test them. 2988 */ 2989 #define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc) 2990 #define HAS_GUC_CT(dev_priv) ((dev_priv)->info.has_guc_ct) 2991 #define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) 2992 #define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) 2993 #define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) 2994 2995 #define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer) 2996 2997 #define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu) 2998 2999 #define INTEL_PCH_DEVICE_ID_MASK 0xff00 3000 #define INTEL_PCH_DEVICE_ID_MASK_EXT 0xff80 3001 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 3002 #define INTEL_PCH_CPT_DEVICE_ID_TYPE 0x1c00 3003 #define INTEL_PCH_PPT_DEVICE_ID_TYPE 0x1e00 3004 #define INTEL_PCH_LPT_DEVICE_ID_TYPE 0x8c00 3005 #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00 3006 #define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100 3007 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00 3008 #define INTEL_PCH_KBP_DEVICE_ID_TYPE 0xA200 3009 #define INTEL_PCH_CNP_DEVICE_ID_TYPE 0xA300 3010 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE 0x9D80 3011 #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 3012 #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 3013 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ 3014 3015 #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) 3016 #define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP) 3017 #define HAS_PCH_CNP_LP(dev_priv) \ 3018 ((dev_priv)->pch_id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) 3019 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP) 3020 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT) 3021 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT) 3022 #define HAS_PCH_LPT_LP(dev_priv) \ 3023 ((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) 3024 #define HAS_PCH_LPT_H(dev_priv) \ 3025 ((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) 3026 #define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT) 3027 #define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX) 3028 #define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP) 3029 #define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE) 3030 3031 #define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display) 3032 3033 #define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9) 3034 3035 /* DPF == dynamic parity feature */ 3036 #define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf) 3037 #define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \ 3038 2 : HAS_L3_DPF(dev_priv)) 3039 3040 #define GT_FREQUENCY_MULTIPLIER 50 3041 #define GEN9_FREQ_SCALER 3 3042 3043 #include "i915_trace.h" 3044 3045 static inline bool intel_vtd_active(void) 3046 { 3047 #ifdef CONFIG_INTEL_IOMMU 3048 if (intel_iommu_gfx_mapped) 3049 return true; 3050 #endif 3051 return false; 3052 } 3053 3054 static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) 3055 { 3056 return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active(); 3057 } 3058 3059 static inline bool 3060 intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) 3061 { 3062 return IS_BROXTON(dev_priv) && intel_vtd_active(); 3063 } 3064 3065 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, 3066 int enable_ppgtt); 3067 3068 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value); 3069 3070 /* i915_drv.c */ 3071 void __printf(3, 4) 3072 __i915_printk(struct drm_i915_private *dev_priv, const char *level, 3073 const char *fmt, ...); 3074 3075 #define i915_report_error(dev_priv, fmt, ...) \ 3076 __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__) 3077 3078 #ifdef CONFIG_COMPAT 3079 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, 3080 unsigned long arg); 3081 #else 3082 #define i915_compat_ioctl NULL 3083 #endif 3084 extern const struct dev_pm_ops i915_pm_ops; 3085 3086 extern int i915_driver_load(struct pci_dev *pdev, 3087 const struct pci_device_id *ent); 3088 extern void i915_driver_unload(struct drm_device *dev); 3089 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask); 3090 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); 3091 extern void i915_reset(struct drm_i915_private *dev_priv); 3092 extern int intel_guc_reset(struct drm_i915_private *dev_priv); 3093 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); 3094 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); 3095 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); 3096 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); 3097 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); 3098 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); 3099 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); 3100 3101 int intel_engines_init_mmio(struct drm_i915_private *dev_priv); 3102 int intel_engines_init(struct drm_i915_private *dev_priv); 3103 3104 /* intel_hotplug.c */ 3105 void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, 3106 u32 pin_mask, u32 long_mask); 3107 void intel_hpd_init(struct drm_i915_private *dev_priv); 3108 void intel_hpd_init_work(struct drm_i915_private *dev_priv); 3109 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); 3110 bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port); 3111 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin); 3112 void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin); 3113 3114 /* i915_irq.c */ 3115 static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv) 3116 { 3117 unsigned long delay; 3118 3119 if (unlikely(!i915.enable_hangcheck)) 3120 return; 3121 3122 /* Don't continually defer the hangcheck so that it is always run at 3123 * least once after work has been scheduled on any ring. Otherwise, 3124 * we will ignore a hung ring if a second ring is kept busy. 3125 */ 3126 3127 delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES); 3128 queue_delayed_work(system_long_wq, 3129 &dev_priv->gpu_error.hangcheck_work, delay); 3130 } 3131 3132 __printf(3, 4) 3133 void i915_handle_error(struct drm_i915_private *dev_priv, 3134 u32 engine_mask, 3135 const char *fmt, ...); 3136 3137 extern void intel_irq_init(struct drm_i915_private *dev_priv); 3138 extern void intel_irq_fini(struct drm_i915_private *dev_priv); 3139 int intel_irq_install(struct drm_i915_private *dev_priv); 3140 void intel_irq_uninstall(struct drm_i915_private *dev_priv); 3141 3142 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) 3143 { 3144 return dev_priv->gvt; 3145 } 3146 3147 static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) 3148 { 3149 return dev_priv->vgpu.active; 3150 } 3151 3152 void 3153 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 3154 u32 status_mask); 3155 3156 void 3157 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 3158 u32 status_mask); 3159 3160 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); 3161 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); 3162 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, 3163 uint32_t mask, 3164 uint32_t bits); 3165 void ilk_update_display_irq(struct drm_i915_private *dev_priv, 3166 uint32_t interrupt_mask, 3167 uint32_t enabled_irq_mask); 3168 static inline void 3169 ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) 3170 { 3171 ilk_update_display_irq(dev_priv, bits, bits); 3172 } 3173 static inline void 3174 ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) 3175 { 3176 ilk_update_display_irq(dev_priv, bits, 0); 3177 } 3178 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, 3179 enum pipe pipe, 3180 uint32_t interrupt_mask, 3181 uint32_t enabled_irq_mask); 3182 static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, 3183 enum pipe pipe, uint32_t bits) 3184 { 3185 bdw_update_pipe_irq(dev_priv, pipe, bits, bits); 3186 } 3187 static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, 3188 enum pipe pipe, uint32_t bits) 3189 { 3190 bdw_update_pipe_irq(dev_priv, pipe, bits, 0); 3191 } 3192 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, 3193 uint32_t interrupt_mask, 3194 uint32_t enabled_irq_mask); 3195 static inline void 3196 ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) 3197 { 3198 ibx_display_interrupt_update(dev_priv, bits, bits); 3199 } 3200 static inline void 3201 ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) 3202 { 3203 ibx_display_interrupt_update(dev_priv, bits, 0); 3204 } 3205 3206 /* i915_gem.c */ 3207 int i915_gem_create_ioctl(struct drm_device *dev, void *data, 3208 struct drm_file *file_priv); 3209 int i915_gem_pread_ioctl(struct drm_device *dev, void *data, 3210 struct drm_file *file_priv); 3211 int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 3212 struct drm_file *file_priv); 3213 int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 3214 struct drm_file *file_priv); 3215 int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 3216 struct drm_file *file_priv); 3217 int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 3218 struct drm_file *file_priv); 3219 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 3220 struct drm_file *file_priv); 3221 int i915_gem_execbuffer(struct drm_device *dev, void *data, 3222 struct drm_file *file_priv); 3223 int i915_gem_execbuffer2(struct drm_device *dev, void *data, 3224 struct drm_file *file_priv); 3225 int i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3226 struct drm_file *file_priv); 3227 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 3228 struct drm_file *file); 3229 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 3230 struct drm_file *file); 3231 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3232 struct drm_file *file_priv); 3233 int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 3234 struct drm_file *file_priv); 3235 int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, 3236 struct drm_file *file_priv); 3237 int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, 3238 struct drm_file *file_priv); 3239 int i915_gem_init_userptr(struct drm_i915_private *dev_priv); 3240 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); 3241 int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, 3242 struct drm_file *file); 3243 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 3244 struct drm_file *file_priv); 3245 int i915_gem_wait_ioctl(struct drm_device *dev, void *data, 3246 struct drm_file *file_priv); 3247 void i915_gem_sanitize(struct drm_i915_private *i915); 3248 int i915_gem_load_init(struct drm_i915_private *dev_priv); 3249 void i915_gem_load_cleanup(struct drm_i915_private *dev_priv); 3250 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); 3251 int i915_gem_freeze(struct drm_i915_private *dev_priv); 3252 int i915_gem_freeze_late(struct drm_i915_private *dev_priv); 3253 3254 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv); 3255 void i915_gem_object_free(struct drm_i915_gem_object *obj); 3256 void i915_gem_object_init(struct drm_i915_gem_object *obj, 3257 const struct drm_i915_gem_object_ops *ops); 3258 struct drm_i915_gem_object * 3259 i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size); 3260 struct drm_i915_gem_object * 3261 i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, 3262 const void *data, size_t size); 3263 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file); 3264 void i915_gem_free_object(struct drm_gem_object *obj); 3265 3266 static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915) 3267 { 3268 /* A single pass should suffice to release all the freed objects (along 3269 * most call paths) , but be a little more paranoid in that freeing 3270 * the objects does take a little amount of time, during which the rcu 3271 * callbacks could have added new objects into the freed list, and 3272 * armed the work again. 3273 */ 3274 do { 3275 rcu_barrier(); 3276 } while (flush_work(&i915->mm.free_work)); 3277 } 3278 3279 struct i915_vma * __must_check 3280 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 3281 const struct i915_ggtt_view *view, 3282 u64 size, 3283 u64 alignment, 3284 u64 flags); 3285 3286 int i915_gem_object_unbind(struct drm_i915_gem_object *obj); 3287 void i915_gem_release_mmap(struct drm_i915_gem_object *obj); 3288 3289 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); 3290 3291 static inline int __sg_page_count(const struct scatterlist *sg) 3292 { 3293 return sg->length >> PAGE_SHIFT; 3294 } 3295 3296 struct scatterlist * 3297 i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 3298 unsigned int n, unsigned int *offset); 3299 3300 struct page * 3301 i915_gem_object_get_page(struct drm_i915_gem_object *obj, 3302 unsigned int n); 3303 3304 struct page * 3305 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 3306 unsigned int n); 3307 3308 dma_addr_t 3309 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 3310 unsigned long n); 3311 3312 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 3313 struct sg_table *pages); 3314 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 3315 3316 static inline int __must_check 3317 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 3318 { 3319 might_lock(&obj->mm.lock); 3320 3321 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 3322 return 0; 3323 3324 return __i915_gem_object_get_pages(obj); 3325 } 3326 3327 static inline void 3328 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 3329 { 3330 GEM_BUG_ON(!obj->mm.pages); 3331 3332 atomic_inc(&obj->mm.pages_pin_count); 3333 } 3334 3335 static inline bool 3336 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 3337 { 3338 return atomic_read(&obj->mm.pages_pin_count); 3339 } 3340 3341 static inline void 3342 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 3343 { 3344 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 3345 GEM_BUG_ON(!obj->mm.pages); 3346 3347 atomic_dec(&obj->mm.pages_pin_count); 3348 } 3349 3350 static inline void 3351 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 3352 { 3353 __i915_gem_object_unpin_pages(obj); 3354 } 3355 3356 enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ 3357 I915_MM_NORMAL = 0, 3358 I915_MM_SHRINKER 3359 }; 3360 3361 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 3362 enum i915_mm_subclass subclass); 3363 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); 3364 3365 enum i915_map_type { 3366 I915_MAP_WB = 0, 3367 I915_MAP_WC, 3368 }; 3369 3370 /** 3371 * i915_gem_object_pin_map - return a contiguous mapping of the entire object 3372 * @obj: the object to map into kernel address space 3373 * @type: the type of mapping, used to select pgprot_t 3374 * 3375 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 3376 * pages and then returns a contiguous mapping of the backing storage into 3377 * the kernel address space. Based on the @type of mapping, the PTE will be 3378 * set to either WriteBack or WriteCombine (via pgprot_t). 3379 * 3380 * The caller is responsible for calling i915_gem_object_unpin_map() when the 3381 * mapping is no longer required. 3382 * 3383 * Returns the pointer through which to access the mapped object, or an 3384 * ERR_PTR() on error. 3385 */ 3386 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 3387 enum i915_map_type type); 3388 3389 /** 3390 * i915_gem_object_unpin_map - releases an earlier mapping 3391 * @obj: the object to unmap 3392 * 3393 * After pinning the object and mapping its pages, once you are finished 3394 * with your access, call i915_gem_object_unpin_map() to release the pin 3395 * upon the mapping. Once the pin count reaches zero, that mapping may be 3396 * removed. 3397 */ 3398 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 3399 { 3400 i915_gem_object_unpin_pages(obj); 3401 } 3402 3403 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 3404 unsigned int *needs_clflush); 3405 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, 3406 unsigned int *needs_clflush); 3407 #define CLFLUSH_BEFORE BIT(0) 3408 #define CLFLUSH_AFTER BIT(1) 3409 #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 3410 3411 static inline void 3412 i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj) 3413 { 3414 i915_gem_object_unpin_pages(obj); 3415 } 3416 3417 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); 3418 void i915_vma_move_to_active(struct i915_vma *vma, 3419 struct drm_i915_gem_request *req, 3420 unsigned int flags); 3421 int i915_gem_dumb_create(struct drm_file *file_priv, 3422 struct drm_device *dev, 3423 struct drm_mode_create_dumb *args); 3424 int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, 3425 uint32_t handle, uint64_t *offset); 3426 int i915_gem_mmap_gtt_version(void); 3427 3428 void i915_gem_track_fb(struct drm_i915_gem_object *old, 3429 struct drm_i915_gem_object *new, 3430 unsigned frontbuffer_bits); 3431 3432 int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); 3433 3434 struct drm_i915_gem_request * 3435 i915_gem_find_active_request(struct intel_engine_cs *engine); 3436 3437 void i915_gem_retire_requests(struct drm_i915_private *dev_priv); 3438 3439 static inline bool i915_reset_backoff(struct i915_gpu_error *error) 3440 { 3441 return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags)); 3442 } 3443 3444 static inline bool i915_reset_handoff(struct i915_gpu_error *error) 3445 { 3446 return unlikely(test_bit(I915_RESET_HANDOFF, &error->flags)); 3447 } 3448 3449 static inline bool i915_terminally_wedged(struct i915_gpu_error *error) 3450 { 3451 return unlikely(test_bit(I915_WEDGED, &error->flags)); 3452 } 3453 3454 static inline bool i915_reset_backoff_or_wedged(struct i915_gpu_error *error) 3455 { 3456 return i915_reset_backoff(error) | i915_terminally_wedged(error); 3457 } 3458 3459 static inline u32 i915_reset_count(struct i915_gpu_error *error) 3460 { 3461 return READ_ONCE(error->reset_count); 3462 } 3463 3464 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv); 3465 void i915_gem_reset(struct drm_i915_private *dev_priv); 3466 void i915_gem_reset_finish(struct drm_i915_private *dev_priv); 3467 void i915_gem_set_wedged(struct drm_i915_private *dev_priv); 3468 bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv); 3469 3470 void i915_gem_init_mmio(struct drm_i915_private *i915); 3471 int __must_check i915_gem_init(struct drm_i915_private *dev_priv); 3472 int __must_check i915_gem_init_hw(struct drm_i915_private *dev_priv); 3473 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv); 3474 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv); 3475 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, 3476 unsigned int flags); 3477 int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv); 3478 void i915_gem_resume(struct drm_i915_private *dev_priv); 3479 int i915_gem_fault(struct vm_fault *vmf); 3480 int i915_gem_object_wait(struct drm_i915_gem_object *obj, 3481 unsigned int flags, 3482 long timeout, 3483 struct intel_rps_client *rps); 3484 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 3485 unsigned int flags, 3486 int priority); 3487 #define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX 3488 3489 int __must_check 3490 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 3491 int __must_check 3492 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 3493 int __must_check 3494 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 3495 struct i915_vma * __must_check 3496 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3497 u32 alignment, 3498 const struct i915_ggtt_view *view); 3499 void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma); 3500 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, 3501 int align); 3502 int i915_gem_open(struct drm_device *dev, struct drm_file *file); 3503 void i915_gem_release(struct drm_device *dev, struct drm_file *file); 3504 3505 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3506 enum i915_cache_level cache_level); 3507 3508 struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, 3509 struct dma_buf *dma_buf); 3510 3511 struct dma_buf *i915_gem_prime_export(struct drm_device *dev, 3512 struct drm_gem_object *gem_obj, int flags); 3513 3514 static inline struct i915_hw_ppgtt * 3515 i915_vm_to_ppgtt(struct i915_address_space *vm) 3516 { 3517 return container_of(vm, struct i915_hw_ppgtt, base); 3518 } 3519 3520 /* i915_gem_fence_reg.c */ 3521 int __must_check i915_vma_get_fence(struct i915_vma *vma); 3522 int __must_check i915_vma_put_fence(struct i915_vma *vma); 3523 3524 void i915_gem_revoke_fences(struct drm_i915_private *dev_priv); 3525 void i915_gem_restore_fences(struct drm_i915_private *dev_priv); 3526 3527 void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv); 3528 void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, 3529 struct sg_table *pages); 3530 void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, 3531 struct sg_table *pages); 3532 3533 static inline struct i915_gem_context * 3534 i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) 3535 { 3536 struct i915_gem_context *ctx; 3537 3538 lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex); 3539 3540 ctx = idr_find(&file_priv->context_idr, id); 3541 if (!ctx) 3542 return ERR_PTR(-ENOENT); 3543 3544 return ctx; 3545 } 3546 3547 static inline struct i915_gem_context * 3548 i915_gem_context_get(struct i915_gem_context *ctx) 3549 { 3550 kref_get(&ctx->ref); 3551 return ctx; 3552 } 3553 3554 static inline void i915_gem_context_put(struct i915_gem_context *ctx) 3555 { 3556 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 3557 kref_put(&ctx->ref, i915_gem_context_free); 3558 } 3559 3560 static inline void i915_gem_context_put_unlocked(struct i915_gem_context *ctx) 3561 { 3562 struct mutex *lock = &ctx->i915->drm.struct_mutex; 3563 3564 if (kref_put_mutex(&ctx->ref, i915_gem_context_free, lock)) 3565 mutex_unlock(lock); 3566 } 3567 3568 static inline struct intel_timeline * 3569 i915_gem_context_lookup_timeline(struct i915_gem_context *ctx, 3570 struct intel_engine_cs *engine) 3571 { 3572 struct i915_address_space *vm; 3573 3574 vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base; 3575 return &vm->timeline.engine[engine->id]; 3576 } 3577 3578 int i915_perf_open_ioctl(struct drm_device *dev, void *data, 3579 struct drm_file *file); 3580 void i915_oa_init_reg_state(struct intel_engine_cs *engine, 3581 struct i915_gem_context *ctx, 3582 uint32_t *reg_state); 3583 3584 /* i915_gem_evict.c */ 3585 int __must_check i915_gem_evict_something(struct i915_address_space *vm, 3586 u64 min_size, u64 alignment, 3587 unsigned cache_level, 3588 u64 start, u64 end, 3589 unsigned flags); 3590 int __must_check i915_gem_evict_for_node(struct i915_address_space *vm, 3591 struct drm_mm_node *node, 3592 unsigned int flags); 3593 int i915_gem_evict_vm(struct i915_address_space *vm); 3594 3595 /* belongs in i915_gem_gtt.h */ 3596 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv) 3597 { 3598 wmb(); 3599 if (INTEL_GEN(dev_priv) < 6) 3600 intel_gtt_chipset_flush(); 3601 } 3602 3603 /* i915_gem_stolen.c */ 3604 int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, 3605 struct drm_mm_node *node, u64 size, 3606 unsigned alignment); 3607 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, 3608 struct drm_mm_node *node, u64 size, 3609 unsigned alignment, u64 start, 3610 u64 end); 3611 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, 3612 struct drm_mm_node *node); 3613 int i915_gem_init_stolen(struct drm_i915_private *dev_priv); 3614 void i915_gem_cleanup_stolen(struct drm_device *dev); 3615 struct drm_i915_gem_object * 3616 i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size); 3617 struct drm_i915_gem_object * 3618 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, 3619 u32 stolen_offset, 3620 u32 gtt_offset, 3621 u32 size); 3622 3623 /* i915_gem_internal.c */ 3624 struct drm_i915_gem_object * 3625 i915_gem_object_create_internal(struct drm_i915_private *dev_priv, 3626 phys_addr_t size); 3627 3628 /* i915_gem_shrinker.c */ 3629 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, 3630 unsigned long target, 3631 unsigned flags); 3632 #define I915_SHRINK_PURGEABLE 0x1 3633 #define I915_SHRINK_UNBOUND 0x2 3634 #define I915_SHRINK_BOUND 0x4 3635 #define I915_SHRINK_ACTIVE 0x8 3636 #define I915_SHRINK_VMAPS 0x10 3637 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); 3638 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv); 3639 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv); 3640 3641 3642 /* i915_gem_tiling.c */ 3643 static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 3644 { 3645 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 3646 3647 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 3648 i915_gem_object_is_tiled(obj); 3649 } 3650 3651 u32 i915_gem_fence_size(struct drm_i915_private *dev_priv, u32 size, 3652 unsigned int tiling, unsigned int stride); 3653 u32 i915_gem_fence_alignment(struct drm_i915_private *dev_priv, u32 size, 3654 unsigned int tiling, unsigned int stride); 3655 3656 /* i915_debugfs.c */ 3657 #ifdef CONFIG_DEBUG_FS 3658 int i915_debugfs_register(struct drm_i915_private *dev_priv); 3659 int i915_debugfs_connector_add(struct drm_connector *connector); 3660 void intel_display_crc_init(struct drm_i915_private *dev_priv); 3661 #else 3662 static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;} 3663 static inline int i915_debugfs_connector_add(struct drm_connector *connector) 3664 { return 0; } 3665 static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {} 3666 #endif 3667 3668 /* i915_gpu_error.c */ 3669 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) 3670 3671 __printf(2, 3) 3672 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); 3673 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, 3674 const struct i915_gpu_state *gpu); 3675 int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, 3676 struct drm_i915_private *i915, 3677 size_t count, loff_t pos); 3678 static inline void i915_error_state_buf_release( 3679 struct drm_i915_error_state_buf *eb) 3680 { 3681 kfree(eb->buf); 3682 } 3683 3684 struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); 3685 void i915_capture_error_state(struct drm_i915_private *dev_priv, 3686 u32 engine_mask, 3687 const char *error_msg); 3688 3689 static inline struct i915_gpu_state * 3690 i915_gpu_state_get(struct i915_gpu_state *gpu) 3691 { 3692 kref_get(&gpu->ref); 3693 return gpu; 3694 } 3695 3696 void __i915_gpu_state_free(struct kref *kref); 3697 static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) 3698 { 3699 if (gpu) 3700 kref_put(&gpu->ref, __i915_gpu_state_free); 3701 } 3702 3703 struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); 3704 void i915_reset_error_state(struct drm_i915_private *i915); 3705 3706 #else 3707 3708 static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, 3709 u32 engine_mask, 3710 const char *error_msg) 3711 { 3712 } 3713 3714 static inline struct i915_gpu_state * 3715 i915_first_error_state(struct drm_i915_private *i915) 3716 { 3717 return NULL; 3718 } 3719 3720 static inline void i915_reset_error_state(struct drm_i915_private *i915) 3721 { 3722 } 3723 3724 #endif 3725 3726 const char *i915_cache_level_str(struct drm_i915_private *i915, int type); 3727 3728 /* i915_cmd_parser.c */ 3729 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); 3730 void intel_engine_init_cmd_parser(struct intel_engine_cs *engine); 3731 void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); 3732 int intel_engine_cmd_parser(struct intel_engine_cs *engine, 3733 struct drm_i915_gem_object *batch_obj, 3734 struct drm_i915_gem_object *shadow_batch_obj, 3735 u32 batch_start_offset, 3736 u32 batch_len, 3737 bool is_master); 3738 3739 /* i915_perf.c */ 3740 extern void i915_perf_init(struct drm_i915_private *dev_priv); 3741 extern void i915_perf_fini(struct drm_i915_private *dev_priv); 3742 extern void i915_perf_register(struct drm_i915_private *dev_priv); 3743 extern void i915_perf_unregister(struct drm_i915_private *dev_priv); 3744 3745 /* i915_suspend.c */ 3746 extern int i915_save_state(struct drm_i915_private *dev_priv); 3747 extern int i915_restore_state(struct drm_i915_private *dev_priv); 3748 3749 /* i915_sysfs.c */ 3750 void i915_setup_sysfs(struct drm_i915_private *dev_priv); 3751 void i915_teardown_sysfs(struct drm_i915_private *dev_priv); 3752 3753 /* intel_lpe_audio.c */ 3754 int intel_lpe_audio_init(struct drm_i915_private *dev_priv); 3755 void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv); 3756 void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv); 3757 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv, 3758 enum pipe pipe, enum port port, 3759 const void *eld, int ls_clock, bool dp_output); 3760 3761 /* intel_i2c.c */ 3762 extern int intel_setup_gmbus(struct drm_i915_private *dev_priv); 3763 extern void intel_teardown_gmbus(struct drm_i915_private *dev_priv); 3764 extern bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv, 3765 unsigned int pin); 3766 3767 extern struct i2c_adapter * 3768 intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, unsigned int pin); 3769 extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed); 3770 extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit); 3771 static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) 3772 { 3773 return container_of(adapter, struct intel_gmbus, adapter)->force_bit; 3774 } 3775 extern void intel_i2c_reset(struct drm_i915_private *dev_priv); 3776 3777 /* intel_bios.c */ 3778 void intel_bios_init(struct drm_i915_private *dev_priv); 3779 bool intel_bios_is_valid_vbt(const void *buf, size_t size); 3780 bool intel_bios_is_tv_present(struct drm_i915_private *dev_priv); 3781 bool intel_bios_is_lvds_present(struct drm_i915_private *dev_priv, u8 *i2c_pin); 3782 bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port); 3783 bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port); 3784 bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port); 3785 bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port); 3786 bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, 3787 enum port port); 3788 bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, 3789 enum port port); 3790 3791 3792 /* intel_opregion.c */ 3793 #ifdef CONFIG_ACPI 3794 extern int intel_opregion_setup(struct drm_i915_private *dev_priv); 3795 extern void intel_opregion_register(struct drm_i915_private *dev_priv); 3796 extern void intel_opregion_unregister(struct drm_i915_private *dev_priv); 3797 extern void intel_opregion_asle_intr(struct drm_i915_private *dev_priv); 3798 extern int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, 3799 bool enable); 3800 extern int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv, 3801 pci_power_t state); 3802 extern int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv); 3803 #else 3804 static inline int intel_opregion_setup(struct drm_i915_private *dev) { return 0; } 3805 static inline void intel_opregion_register(struct drm_i915_private *dev_priv) { } 3806 static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { } 3807 static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) 3808 { 3809 } 3810 static inline int 3811 intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable) 3812 { 3813 return 0; 3814 } 3815 static inline int 3816 intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state) 3817 { 3818 return 0; 3819 } 3820 static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev) 3821 { 3822 return -ENODEV; 3823 } 3824 #endif 3825 3826 /* intel_acpi.c */ 3827 #ifdef CONFIG_ACPI 3828 extern void intel_register_dsm_handler(void); 3829 extern void intel_unregister_dsm_handler(void); 3830 #else 3831 static inline void intel_register_dsm_handler(void) { return; } 3832 static inline void intel_unregister_dsm_handler(void) { return; } 3833 #endif /* CONFIG_ACPI */ 3834 3835 /* intel_device_info.c */ 3836 static inline struct intel_device_info * 3837 mkwrite_device_info(struct drm_i915_private *dev_priv) 3838 { 3839 return (struct intel_device_info *)&dev_priv->info; 3840 } 3841 3842 const char *intel_platform_name(enum intel_platform platform); 3843 void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); 3844 void intel_device_info_dump(struct drm_i915_private *dev_priv); 3845 3846 /* modesetting */ 3847 extern void intel_modeset_init_hw(struct drm_device *dev); 3848 extern int intel_modeset_init(struct drm_device *dev); 3849 extern void intel_modeset_gem_init(struct drm_device *dev); 3850 extern void intel_modeset_cleanup(struct drm_device *dev); 3851 extern int intel_connector_register(struct drm_connector *); 3852 extern void intel_connector_unregister(struct drm_connector *); 3853 extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, 3854 bool state); 3855 extern void intel_display_resume(struct drm_device *dev); 3856 extern void i915_redisable_vga(struct drm_i915_private *dev_priv); 3857 extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv); 3858 extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); 3859 extern void intel_init_pch_refclk(struct drm_i915_private *dev_priv); 3860 extern int intel_set_rps(struct drm_i915_private *dev_priv, u8 val); 3861 extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, 3862 bool enable); 3863 3864 int i915_reg_read_ioctl(struct drm_device *dev, void *data, 3865 struct drm_file *file); 3866 3867 /* overlay */ 3868 extern struct intel_overlay_error_state * 3869 intel_overlay_capture_error_state(struct drm_i915_private *dev_priv); 3870 extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, 3871 struct intel_overlay_error_state *error); 3872 3873 extern struct intel_display_error_state * 3874 intel_display_capture_error_state(struct drm_i915_private *dev_priv); 3875 extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, 3876 struct intel_display_error_state *error); 3877 3878 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); 3879 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val); 3880 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, 3881 u32 reply_mask, u32 reply, int timeout_base_ms); 3882 3883 /* intel_sideband.c */ 3884 u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr); 3885 int vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val); 3886 u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr); 3887 u32 vlv_iosf_sb_read(struct drm_i915_private *dev_priv, u8 port, u32 reg); 3888 void vlv_iosf_sb_write(struct drm_i915_private *dev_priv, u8 port, u32 reg, u32 val); 3889 u32 vlv_cck_read(struct drm_i915_private *dev_priv, u32 reg); 3890 void vlv_cck_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 3891 u32 vlv_ccu_read(struct drm_i915_private *dev_priv, u32 reg); 3892 void vlv_ccu_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 3893 u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg); 3894 void vlv_bunit_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 3895 u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg); 3896 void vlv_dpio_write(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 val); 3897 u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg, 3898 enum intel_sbi_destination destination); 3899 void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, 3900 enum intel_sbi_destination destination); 3901 u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); 3902 void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 3903 3904 /* intel_dpio_phy.c */ 3905 void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, 3906 enum dpio_phy *phy, enum dpio_channel *ch); 3907 void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, 3908 enum port port, u32 margin, u32 scale, 3909 u32 enable, u32 deemphasis); 3910 void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); 3911 void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); 3912 bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, 3913 enum dpio_phy phy); 3914 bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, 3915 enum dpio_phy phy); 3916 uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, 3917 uint8_t lane_count); 3918 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, 3919 uint8_t lane_lat_optim_mask); 3920 uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); 3921 3922 void chv_set_phy_signal_level(struct intel_encoder *encoder, 3923 u32 deemph_reg_value, u32 margin_reg_value, 3924 bool uniq_trans_scale); 3925 void chv_data_lane_soft_reset(struct intel_encoder *encoder, 3926 bool reset); 3927 void chv_phy_pre_pll_enable(struct intel_encoder *encoder); 3928 void chv_phy_pre_encoder_enable(struct intel_encoder *encoder); 3929 void chv_phy_release_cl2_override(struct intel_encoder *encoder); 3930 void chv_phy_post_pll_disable(struct intel_encoder *encoder); 3931 3932 void vlv_set_phy_signal_level(struct intel_encoder *encoder, 3933 u32 demph_reg_value, u32 preemph_reg_value, 3934 u32 uniqtranscale_reg_value, u32 tx3_demph); 3935 void vlv_phy_pre_pll_enable(struct intel_encoder *encoder); 3936 void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder); 3937 void vlv_phy_reset_lanes(struct intel_encoder *encoder); 3938 3939 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); 3940 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); 3941 u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv, 3942 const i915_reg_t reg); 3943 3944 #define I915_READ8(reg) dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true) 3945 #define I915_WRITE8(reg, val) dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true) 3946 3947 #define I915_READ16(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), true) 3948 #define I915_WRITE16(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), true) 3949 #define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false) 3950 #define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false) 3951 3952 #define I915_READ(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), true) 3953 #define I915_WRITE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), true) 3954 #define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false) 3955 #define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false) 3956 3957 /* Be very careful with read/write 64-bit values. On 32-bit machines, they 3958 * will be implemented using 2 32-bit writes in an arbitrary order with 3959 * an arbitrary delay between them. This can cause the hardware to 3960 * act upon the intermediate value, possibly leading to corruption and 3961 * machine death. For this reason we do not support I915_WRITE64, or 3962 * dev_priv->uncore.funcs.mmio_writeq. 3963 * 3964 * When reading a 64-bit value as two 32-bit values, the delay may cause 3965 * the two reads to mismatch, e.g. a timestamp overflowing. Also note that 3966 * occasionally a 64-bit register does not actualy support a full readq 3967 * and must be read using two 32-bit reads. 3968 * 3969 * You have been warned. 3970 */ 3971 #define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true) 3972 3973 #define I915_READ64_2x32(lower_reg, upper_reg) ({ \ 3974 u32 upper, lower, old_upper, loop = 0; \ 3975 upper = I915_READ(upper_reg); \ 3976 do { \ 3977 old_upper = upper; \ 3978 lower = I915_READ(lower_reg); \ 3979 upper = I915_READ(upper_reg); \ 3980 } while (upper != old_upper && loop++ < 2); \ 3981 (u64)upper << 32 | lower; }) 3982 3983 #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) 3984 #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) 3985 3986 #define __raw_read(x, s) \ 3987 static inline uint##x##_t __raw_i915_read##x(const struct drm_i915_private *dev_priv, \ 3988 i915_reg_t reg) \ 3989 { \ 3990 return read##s(dev_priv->regs + i915_mmio_reg_offset(reg)); \ 3991 } 3992 3993 #define __raw_write(x, s) \ 3994 static inline void __raw_i915_write##x(const struct drm_i915_private *dev_priv, \ 3995 i915_reg_t reg, uint##x##_t val) \ 3996 { \ 3997 write##s(val, dev_priv->regs + i915_mmio_reg_offset(reg)); \ 3998 } 3999 __raw_read(8, b) 4000 __raw_read(16, w) 4001 __raw_read(32, l) 4002 __raw_read(64, q) 4003 4004 __raw_write(8, b) 4005 __raw_write(16, w) 4006 __raw_write(32, l) 4007 __raw_write(64, q) 4008 4009 #undef __raw_read 4010 #undef __raw_write 4011 4012 /* These are untraced mmio-accessors that are only valid to be used inside 4013 * critical sections, such as inside IRQ handlers, where forcewake is explicitly 4014 * controlled. 4015 * 4016 * Think twice, and think again, before using these. 4017 * 4018 * As an example, these accessors can possibly be used between: 4019 * 4020 * spin_lock_irq(&dev_priv->uncore.lock); 4021 * intel_uncore_forcewake_get__locked(); 4022 * 4023 * and 4024 * 4025 * intel_uncore_forcewake_put__locked(); 4026 * spin_unlock_irq(&dev_priv->uncore.lock); 4027 * 4028 * 4029 * Note: some registers may not need forcewake held, so 4030 * intel_uncore_forcewake_{get,put} can be omitted, see 4031 * intel_uncore_forcewake_for_reg(). 4032 * 4033 * Certain architectures will die if the same cacheline is concurrently accessed 4034 * by different clients (e.g. on Ivybridge). Access to registers should 4035 * therefore generally be serialised, by either the dev_priv->uncore.lock or 4036 * a more localised lock guarding all access to that bank of registers. 4037 */ 4038 #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) 4039 #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) 4040 #define I915_WRITE64_FW(reg__, val__) __raw_i915_write64(dev_priv, (reg__), (val__)) 4041 #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__) 4042 4043 /* "Broadcast RGB" property */ 4044 #define INTEL_BROADCAST_RGB_AUTO 0 4045 #define INTEL_BROADCAST_RGB_FULL 1 4046 #define INTEL_BROADCAST_RGB_LIMITED 2 4047 4048 static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv) 4049 { 4050 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 4051 return VLV_VGACNTRL; 4052 else if (INTEL_GEN(dev_priv) >= 5) 4053 return CPU_VGACNTRL; 4054 else 4055 return VGACNTRL; 4056 } 4057 4058 static inline unsigned long msecs_to_jiffies_timeout(const unsigned int m) 4059 { 4060 unsigned long j = msecs_to_jiffies(m); 4061 4062 return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); 4063 } 4064 4065 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 4066 { 4067 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 4068 } 4069 4070 static inline unsigned long 4071 timespec_to_jiffies_timeout(const struct timespec *value) 4072 { 4073 unsigned long j = timespec_to_jiffies(value); 4074 4075 return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); 4076 } 4077 4078 /* 4079 * If you need to wait X milliseconds between events A and B, but event B 4080 * doesn't happen exactly after event A, you record the timestamp (jiffies) of 4081 * when event A happened, then just before event B you call this function and 4082 * pass the timestamp as the first argument, and X as the second argument. 4083 */ 4084 static inline void 4085 wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) 4086 { 4087 unsigned long target_jiffies, tmp_jiffies, remaining_jiffies; 4088 4089 /* 4090 * Don't re-read the value of "jiffies" every time since it may change 4091 * behind our back and break the math. 4092 */ 4093 tmp_jiffies = jiffies; 4094 target_jiffies = timestamp_jiffies + 4095 msecs_to_jiffies_timeout(to_wait_ms); 4096 4097 if (time_after(target_jiffies, tmp_jiffies)) { 4098 remaining_jiffies = target_jiffies - tmp_jiffies; 4099 while (remaining_jiffies) 4100 remaining_jiffies = 4101 schedule_timeout_uninterruptible(remaining_jiffies); 4102 } 4103 } 4104 4105 static inline bool 4106 __i915_request_irq_complete(const struct drm_i915_gem_request *req) 4107 { 4108 struct intel_engine_cs *engine = req->engine; 4109 u32 seqno; 4110 4111 /* Note that the engine may have wrapped around the seqno, and 4112 * so our request->global_seqno will be ahead of the hardware, 4113 * even though it completed the request before wrapping. We catch 4114 * this by kicking all the waiters before resetting the seqno 4115 * in hardware, and also signal the fence. 4116 */ 4117 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &req->fence.flags)) 4118 return true; 4119 4120 /* The request was dequeued before we were awoken. We check after 4121 * inspecting the hw to confirm that this was the same request 4122 * that generated the HWS update. The memory barriers within 4123 * the request execution are sufficient to ensure that a check 4124 * after reading the value from hw matches this request. 4125 */ 4126 seqno = i915_gem_request_global_seqno(req); 4127 if (!seqno) 4128 return false; 4129 4130 /* Before we do the heavier coherent read of the seqno, 4131 * check the value (hopefully) in the CPU cacheline. 4132 */ 4133 if (__i915_gem_request_completed(req, seqno)) 4134 return true; 4135 4136 /* Ensure our read of the seqno is coherent so that we 4137 * do not "miss an interrupt" (i.e. if this is the last 4138 * request and the seqno write from the GPU is not visible 4139 * by the time the interrupt fires, we will see that the 4140 * request is incomplete and go back to sleep awaiting 4141 * another interrupt that will never come.) 4142 * 4143 * Strictly, we only need to do this once after an interrupt, 4144 * but it is easier and safer to do it every time the waiter 4145 * is woken. 4146 */ 4147 if (engine->irq_seqno_barrier && 4148 test_and_clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted)) { 4149 struct intel_breadcrumbs *b = &engine->breadcrumbs; 4150 4151 /* The ordering of irq_posted versus applying the barrier 4152 * is crucial. The clearing of the current irq_posted must 4153 * be visible before we perform the barrier operation, 4154 * such that if a subsequent interrupt arrives, irq_posted 4155 * is reasserted and our task rewoken (which causes us to 4156 * do another __i915_request_irq_complete() immediately 4157 * and reapply the barrier). Conversely, if the clear 4158 * occurs after the barrier, then an interrupt that arrived 4159 * whilst we waited on the barrier would not trigger a 4160 * barrier on the next pass, and the read may not see the 4161 * seqno update. 4162 */ 4163 engine->irq_seqno_barrier(engine); 4164 4165 /* If we consume the irq, but we are no longer the bottom-half, 4166 * the real bottom-half may not have serialised their own 4167 * seqno check with the irq-barrier (i.e. may have inspected 4168 * the seqno before we believe it coherent since they see 4169 * irq_posted == false but we are still running). 4170 */ 4171 spin_lock_irq(&b->irq_lock); 4172 if (b->irq_wait && b->irq_wait->tsk != current) 4173 /* Note that if the bottom-half is changed as we 4174 * are sending the wake-up, the new bottom-half will 4175 * be woken by whomever made the change. We only have 4176 * to worry about when we steal the irq-posted for 4177 * ourself. 4178 */ 4179 wake_up_process(b->irq_wait->tsk); 4180 spin_unlock_irq(&b->irq_lock); 4181 4182 if (__i915_gem_request_completed(req, seqno)) 4183 return true; 4184 } 4185 4186 return false; 4187 } 4188 4189 void i915_memcpy_init_early(struct drm_i915_private *dev_priv); 4190 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len); 4191 4192 /* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment, 4193 * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot 4194 * perform the operation. To check beforehand, pass in the parameters to 4195 * to i915_can_memcpy_from_wc() - since we only care about the low 4 bits, 4196 * you only need to pass in the minor offsets, page-aligned pointers are 4197 * always valid. 4198 * 4199 * For just checking for SSE4.1, in the foreknowledge that the future use 4200 * will be correctly aligned, just use i915_has_memcpy_from_wc(). 4201 */ 4202 #define i915_can_memcpy_from_wc(dst, src, len) \ 4203 i915_memcpy_from_wc((void *)((unsigned long)(dst) | (unsigned long)(src) | (len)), NULL, 0) 4204 4205 #define i915_has_memcpy_from_wc() \ 4206 i915_memcpy_from_wc(NULL, NULL, 0) 4207 4208 /* i915_mm.c */ 4209 int remap_io_mapping(struct vm_area_struct *vma, 4210 unsigned long addr, unsigned long pfn, unsigned long size, 4211 struct io_mapping *iomap); 4212 4213 static inline bool i915_gem_object_is_coherent(struct drm_i915_gem_object *obj) 4214 { 4215 return (obj->cache_level != I915_CACHE_NONE || 4216 HAS_LLC(to_i915(obj->base.dev))); 4217 } 4218 4219 #endif 4220