1d897a111SMichal Wajdeczko /* 2d897a111SMichal Wajdeczko * SPDX-License-Identifier: MIT 3d897a111SMichal Wajdeczko * 4d897a111SMichal Wajdeczko * Copyright � 2008-2018 Intel Corporation 5d897a111SMichal Wajdeczko */ 6d897a111SMichal Wajdeczko 7d897a111SMichal Wajdeczko #ifndef _I915_GPU_ERROR_H_ 8d897a111SMichal Wajdeczko #define _I915_GPU_ERROR_H_ 9d897a111SMichal Wajdeczko 10d897a111SMichal Wajdeczko #include <linux/kref.h> 11d897a111SMichal Wajdeczko #include <linux/ktime.h> 12d897a111SMichal Wajdeczko #include <linux/sched.h> 13d897a111SMichal Wajdeczko 14d897a111SMichal Wajdeczko #include <drm/drm_mm.h> 15d897a111SMichal Wajdeczko 16d897a111SMichal Wajdeczko #include "intel_device_info.h" 17d897a111SMichal Wajdeczko #include "intel_ringbuffer.h" 18d897a111SMichal Wajdeczko #include "intel_uc_fw.h" 19d897a111SMichal Wajdeczko 20d897a111SMichal Wajdeczko #include "i915_gem.h" 21d897a111SMichal Wajdeczko #include "i915_gem_gtt.h" 22d897a111SMichal Wajdeczko #include "i915_params.h" 23b7268c5eSChris Wilson #include "i915_scheduler.h" 24d897a111SMichal Wajdeczko 25d897a111SMichal Wajdeczko struct drm_i915_private; 26d897a111SMichal Wajdeczko struct intel_overlay_error_state; 27d897a111SMichal Wajdeczko struct intel_display_error_state; 28d897a111SMichal Wajdeczko 29d897a111SMichal Wajdeczko struct i915_gpu_state { 30d897a111SMichal Wajdeczko struct kref ref; 31d897a111SMichal Wajdeczko ktime_t time; 32d897a111SMichal Wajdeczko ktime_t boottime; 33d897a111SMichal Wajdeczko ktime_t uptime; 34043477b0SMika Kuoppala unsigned long capture; 35043477b0SMika Kuoppala unsigned long epoch; 36d897a111SMichal Wajdeczko 37d897a111SMichal Wajdeczko struct drm_i915_private *i915; 38d897a111SMichal Wajdeczko 39d897a111SMichal Wajdeczko char error_msg[128]; 40d897a111SMichal Wajdeczko bool simulated; 41d897a111SMichal Wajdeczko bool awake; 42d897a111SMichal Wajdeczko bool wakelock; 43d897a111SMichal Wajdeczko bool suspended; 44d897a111SMichal Wajdeczko int iommu; 45d897a111SMichal Wajdeczko u32 reset_count; 46d897a111SMichal Wajdeczko u32 suspend_count; 47d897a111SMichal Wajdeczko struct intel_device_info device_info; 48d897a111SMichal Wajdeczko struct intel_driver_caps driver_caps; 49d897a111SMichal Wajdeczko struct i915_params params; 50d897a111SMichal Wajdeczko 51d897a111SMichal Wajdeczko struct i915_error_uc { 52d897a111SMichal Wajdeczko struct intel_uc_fw guc_fw; 53d897a111SMichal Wajdeczko struct intel_uc_fw huc_fw; 54d897a111SMichal Wajdeczko struct drm_i915_error_object *guc_log; 55d897a111SMichal Wajdeczko } uc; 56d897a111SMichal Wajdeczko 57d897a111SMichal Wajdeczko /* Generic register state */ 58d897a111SMichal Wajdeczko u32 eir; 59d897a111SMichal Wajdeczko u32 pgtbl_er; 60d897a111SMichal Wajdeczko u32 ier; 616b7a6a7bSOscar Mateo u32 gtier[6], ngtier; 62d897a111SMichal Wajdeczko u32 ccid; 63d897a111SMichal Wajdeczko u32 derrmr; 64d897a111SMichal Wajdeczko u32 forcewake; 65d897a111SMichal Wajdeczko u32 error; /* gen6+ */ 66d897a111SMichal Wajdeczko u32 err_int; /* gen7 */ 67d897a111SMichal Wajdeczko u32 fault_data0; /* gen8, gen9 */ 68d897a111SMichal Wajdeczko u32 fault_data1; /* gen8, gen9 */ 69d897a111SMichal Wajdeczko u32 done_reg; 70d897a111SMichal Wajdeczko u32 gac_eco; 71d897a111SMichal Wajdeczko u32 gam_ecochk; 72d897a111SMichal Wajdeczko u32 gab_ctl; 73d897a111SMichal Wajdeczko u32 gfx_mode; 74d897a111SMichal Wajdeczko 75d897a111SMichal Wajdeczko u32 nfence; 76d897a111SMichal Wajdeczko u64 fence[I915_MAX_NUM_FENCES]; 77d897a111SMichal Wajdeczko struct intel_overlay_error_state *overlay; 78d897a111SMichal Wajdeczko struct intel_display_error_state *display; 79d897a111SMichal Wajdeczko 80d897a111SMichal Wajdeczko struct drm_i915_error_engine { 81d897a111SMichal Wajdeczko int engine_id; 82d897a111SMichal Wajdeczko /* Software tracked state */ 83d897a111SMichal Wajdeczko bool idle; 84d897a111SMichal Wajdeczko bool waiting; 85d897a111SMichal Wajdeczko int num_waiters; 86d897a111SMichal Wajdeczko unsigned long hangcheck_timestamp; 87d897a111SMichal Wajdeczko bool hangcheck_stalled; 88d897a111SMichal Wajdeczko enum intel_engine_hangcheck_action hangcheck_action; 89d897a111SMichal Wajdeczko struct i915_address_space *vm; 90d897a111SMichal Wajdeczko int num_requests; 91d897a111SMichal Wajdeczko u32 reset_count; 92d897a111SMichal Wajdeczko 93d897a111SMichal Wajdeczko /* position of active request inside the ring */ 94d897a111SMichal Wajdeczko u32 rq_head, rq_post, rq_tail; 95d897a111SMichal Wajdeczko 96d897a111SMichal Wajdeczko /* our own tracking of ring head and tail */ 97d897a111SMichal Wajdeczko u32 cpu_ring_head; 98d897a111SMichal Wajdeczko u32 cpu_ring_tail; 99d897a111SMichal Wajdeczko 100d897a111SMichal Wajdeczko u32 last_seqno; 101d897a111SMichal Wajdeczko 102d897a111SMichal Wajdeczko /* Register state */ 103d897a111SMichal Wajdeczko u32 start; 104d897a111SMichal Wajdeczko u32 tail; 105d897a111SMichal Wajdeczko u32 head; 106d897a111SMichal Wajdeczko u32 ctl; 107d897a111SMichal Wajdeczko u32 mode; 108d897a111SMichal Wajdeczko u32 hws; 109d897a111SMichal Wajdeczko u32 ipeir; 110d897a111SMichal Wajdeczko u32 ipehr; 111d897a111SMichal Wajdeczko u32 bbstate; 112d897a111SMichal Wajdeczko u32 instpm; 113d897a111SMichal Wajdeczko u32 instps; 114d897a111SMichal Wajdeczko u32 seqno; 115d897a111SMichal Wajdeczko u64 bbaddr; 116d897a111SMichal Wajdeczko u64 acthd; 117d897a111SMichal Wajdeczko u32 fault_reg; 118d897a111SMichal Wajdeczko u64 faddr; 119d897a111SMichal Wajdeczko u32 rc_psmi; /* sleep state */ 120d897a111SMichal Wajdeczko u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; 121d897a111SMichal Wajdeczko struct intel_instdone instdone; 122d897a111SMichal Wajdeczko 123d897a111SMichal Wajdeczko struct drm_i915_error_context { 124d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 125d897a111SMichal Wajdeczko pid_t pid; 126d897a111SMichal Wajdeczko u32 handle; 127d897a111SMichal Wajdeczko u32 hw_id; 128d897a111SMichal Wajdeczko int ban_score; 129d897a111SMichal Wajdeczko int active; 130d897a111SMichal Wajdeczko int guilty; 131d897a111SMichal Wajdeczko bool bannable; 132b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 133d897a111SMichal Wajdeczko } context; 134d897a111SMichal Wajdeczko 135d897a111SMichal Wajdeczko struct drm_i915_error_object { 136d897a111SMichal Wajdeczko u64 gtt_offset; 137d897a111SMichal Wajdeczko u64 gtt_size; 13883bc0f5bSChris Wilson int num_pages; 139d897a111SMichal Wajdeczko int page_count; 140d897a111SMichal Wajdeczko int unused; 141d897a111SMichal Wajdeczko u32 *pages[0]; 142d897a111SMichal Wajdeczko } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; 143d897a111SMichal Wajdeczko 144d897a111SMichal Wajdeczko struct drm_i915_error_object **user_bo; 145d897a111SMichal Wajdeczko long user_bo_count; 146d897a111SMichal Wajdeczko 147d897a111SMichal Wajdeczko struct drm_i915_error_object *wa_ctx; 148d897a111SMichal Wajdeczko struct drm_i915_error_object *default_state; 149d897a111SMichal Wajdeczko 150d897a111SMichal Wajdeczko struct drm_i915_error_request { 151d897a111SMichal Wajdeczko long jiffies; 152d897a111SMichal Wajdeczko pid_t pid; 153d897a111SMichal Wajdeczko u32 context; 154d897a111SMichal Wajdeczko int ban_score; 155d897a111SMichal Wajdeczko u32 seqno; 1563a068721SChris Wilson u32 start; 157d897a111SMichal Wajdeczko u32 head; 158d897a111SMichal Wajdeczko u32 tail; 159b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 160d897a111SMichal Wajdeczko } *requests, execlist[EXECLIST_MAX_PORTS]; 161d897a111SMichal Wajdeczko unsigned int num_ports; 162d897a111SMichal Wajdeczko 163d897a111SMichal Wajdeczko struct drm_i915_error_waiter { 164d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 165d897a111SMichal Wajdeczko pid_t pid; 166d897a111SMichal Wajdeczko u32 seqno; 167d897a111SMichal Wajdeczko } *waiters; 168d897a111SMichal Wajdeczko 169d897a111SMichal Wajdeczko struct { 170d897a111SMichal Wajdeczko u32 gfx_mode; 171d897a111SMichal Wajdeczko union { 172d897a111SMichal Wajdeczko u64 pdp[4]; 173d897a111SMichal Wajdeczko u32 pp_dir_base; 174d897a111SMichal Wajdeczko }; 175d897a111SMichal Wajdeczko } vm_info; 176d897a111SMichal Wajdeczko } engine[I915_NUM_ENGINES]; 177d897a111SMichal Wajdeczko 178d897a111SMichal Wajdeczko struct drm_i915_error_buffer { 179d897a111SMichal Wajdeczko u32 size; 180d897a111SMichal Wajdeczko u32 name; 1815c3f8c22SChris Wilson u32 wseqno; 182d897a111SMichal Wajdeczko u64 gtt_offset; 183d897a111SMichal Wajdeczko u32 read_domains; 184d897a111SMichal Wajdeczko u32 write_domain; 185d897a111SMichal Wajdeczko s32 fence_reg:I915_MAX_NUM_FENCE_BITS; 186d897a111SMichal Wajdeczko u32 tiling:2; 187d897a111SMichal Wajdeczko u32 dirty:1; 188d897a111SMichal Wajdeczko u32 purgeable:1; 189d897a111SMichal Wajdeczko u32 userptr:1; 190d897a111SMichal Wajdeczko s32 engine:4; 191d897a111SMichal Wajdeczko u32 cache_level:3; 192d897a111SMichal Wajdeczko } *active_bo[I915_NUM_ENGINES], *pinned_bo; 193d897a111SMichal Wajdeczko u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; 194d897a111SMichal Wajdeczko struct i915_address_space *active_vm[I915_NUM_ENGINES]; 1950e39037bSChris Wilson 1960e39037bSChris Wilson struct scatterlist *sgl, *fit; 197d897a111SMichal Wajdeczko }; 198d897a111SMichal Wajdeczko 199d897a111SMichal Wajdeczko struct i915_gpu_error { 200d897a111SMichal Wajdeczko /* For hangcheck timer */ 201d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ 202d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) 203d897a111SMichal Wajdeczko 204d897a111SMichal Wajdeczko struct delayed_work hangcheck_work; 205d897a111SMichal Wajdeczko 206d897a111SMichal Wajdeczko /* For reset and error_state handling. */ 207d897a111SMichal Wajdeczko spinlock_t lock; 208d897a111SMichal Wajdeczko /* Protected by the above dev->gpu_error.lock. */ 209d897a111SMichal Wajdeczko struct i915_gpu_state *first_error; 210d897a111SMichal Wajdeczko 211d897a111SMichal Wajdeczko atomic_t pending_fb_pin; 212d897a111SMichal Wajdeczko 213d897a111SMichal Wajdeczko unsigned long missed_irq_rings; 214d897a111SMichal Wajdeczko 215d897a111SMichal Wajdeczko /** 216d897a111SMichal Wajdeczko * State variable controlling the reset flow and count 217d897a111SMichal Wajdeczko * 218d897a111SMichal Wajdeczko * This is a counter which gets incremented when reset is triggered, 219d897a111SMichal Wajdeczko * 220d897a111SMichal Wajdeczko * Before the reset commences, the I915_RESET_BACKOFF bit is set 221d897a111SMichal Wajdeczko * meaning that any waiters holding onto the struct_mutex should 222d897a111SMichal Wajdeczko * relinquish the lock immediately in order for the reset to start. 223d897a111SMichal Wajdeczko * 224d897a111SMichal Wajdeczko * If reset is not completed successfully, the I915_WEDGE bit is 225d897a111SMichal Wajdeczko * set meaning that hardware is terminally sour and there is no 226d897a111SMichal Wajdeczko * recovery. All waiters on the reset_queue will be woken when 227d897a111SMichal Wajdeczko * that happens. 228d897a111SMichal Wajdeczko * 229d897a111SMichal Wajdeczko * This counter is used by the wait_seqno code to notice that reset 230d897a111SMichal Wajdeczko * event happened and it needs to restart the entire ioctl (since most 231d897a111SMichal Wajdeczko * likely the seqno it waited for won't ever signal anytime soon). 232d897a111SMichal Wajdeczko * 233d897a111SMichal Wajdeczko * This is important for lock-free wait paths, where no contended lock 234d897a111SMichal Wajdeczko * naturally enforces the correct ordering between the bail-out of the 235d897a111SMichal Wajdeczko * waiter and the gpu reset work code. 236d897a111SMichal Wajdeczko */ 237d897a111SMichal Wajdeczko unsigned long reset_count; 238d897a111SMichal Wajdeczko 239d897a111SMichal Wajdeczko /** 240d897a111SMichal Wajdeczko * flags: Control various stages of the GPU reset 241d897a111SMichal Wajdeczko * 242d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF - When we start a reset, we want to stop any 243d897a111SMichal Wajdeczko * other users acquiring the struct_mutex. To do this we set the 244d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF bit in the error flags when we detect a reset 245d897a111SMichal Wajdeczko * and then check for that bit before acquiring the struct_mutex (in 246d897a111SMichal Wajdeczko * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a 247d897a111SMichal Wajdeczko * secondary role in preventing two concurrent global reset attempts. 248d897a111SMichal Wajdeczko * 249d897a111SMichal Wajdeczko * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the 250d897a111SMichal Wajdeczko * struct_mutex. We try to acquire the struct_mutex in the reset worker, 251d897a111SMichal Wajdeczko * but it may be held by some long running waiter (that we cannot 252d897a111SMichal Wajdeczko * interrupt without causing trouble). Once we are ready to do the GPU 253d897a111SMichal Wajdeczko * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If 254d897a111SMichal Wajdeczko * they already hold the struct_mutex and want to participate they can 255d897a111SMichal Wajdeczko * inspect the bit and do the reset directly, otherwise the worker 256d897a111SMichal Wajdeczko * waits for the struct_mutex. 257d897a111SMichal Wajdeczko * 258d897a111SMichal Wajdeczko * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to 259d897a111SMichal Wajdeczko * acquire the struct_mutex to reset an engine, we need an explicit 260d897a111SMichal Wajdeczko * flag to prevent two concurrent reset attempts in the same engine. 261d897a111SMichal Wajdeczko * As the number of engines continues to grow, allocate the flags from 262d897a111SMichal Wajdeczko * the most significant bits. 263d897a111SMichal Wajdeczko * 264d897a111SMichal Wajdeczko * #I915_WEDGED - If reset fails and we can no longer use the GPU, 265d897a111SMichal Wajdeczko * we set the #I915_WEDGED bit. Prior to command submission, e.g. 266d897a111SMichal Wajdeczko * i915_request_alloc(), this bit is checked and the sequence 267d897a111SMichal Wajdeczko * aborted (with -EIO reported to userspace) if set. 268d897a111SMichal Wajdeczko */ 269d897a111SMichal Wajdeczko unsigned long flags; 270d897a111SMichal Wajdeczko #define I915_RESET_BACKOFF 0 271d897a111SMichal Wajdeczko #define I915_RESET_HANDOFF 1 272d897a111SMichal Wajdeczko #define I915_RESET_MODESET 2 273d897a111SMichal Wajdeczko #define I915_WEDGED (BITS_PER_LONG - 1) 274d897a111SMichal Wajdeczko #define I915_RESET_ENGINE (I915_WEDGED - I915_NUM_ENGINES) 275d897a111SMichal Wajdeczko 276d897a111SMichal Wajdeczko /** Number of times an engine has been reset */ 277d897a111SMichal Wajdeczko u32 reset_engine_count[I915_NUM_ENGINES]; 278d897a111SMichal Wajdeczko 279d0667e9cSChris Wilson /** Set of stalled engines with guilty requests, in the current reset */ 280d0667e9cSChris Wilson u32 stalled_mask; 281d0667e9cSChris Wilson 282ce800754SChris Wilson /** Reason for the current *global* reset */ 283ce800754SChris Wilson const char *reason; 284ce800754SChris Wilson 285d897a111SMichal Wajdeczko /** 286d897a111SMichal Wajdeczko * Waitqueue to signal when a hang is detected. Used to for waiters 287d897a111SMichal Wajdeczko * to release the struct_mutex for the reset to procede. 288d897a111SMichal Wajdeczko */ 289d897a111SMichal Wajdeczko wait_queue_head_t wait_queue; 290d897a111SMichal Wajdeczko 291d897a111SMichal Wajdeczko /** 292d897a111SMichal Wajdeczko * Waitqueue to signal when the reset has completed. Used by clients 293d897a111SMichal Wajdeczko * that wait for dev_priv->mm.wedged to settle. 294d897a111SMichal Wajdeczko */ 295d897a111SMichal Wajdeczko wait_queue_head_t reset_queue; 296d897a111SMichal Wajdeczko 297d897a111SMichal Wajdeczko /* For missed irq/seqno simulation. */ 298d897a111SMichal Wajdeczko unsigned long test_irq_rings; 299d897a111SMichal Wajdeczko }; 300d897a111SMichal Wajdeczko 301d897a111SMichal Wajdeczko struct drm_i915_error_state_buf { 302d897a111SMichal Wajdeczko struct drm_i915_private *i915; 3030e39037bSChris Wilson struct scatterlist *sgl, *cur, *end; 3040e39037bSChris Wilson 3050e39037bSChris Wilson char *buf; 3060e39037bSChris Wilson size_t bytes; 3070e39037bSChris Wilson size_t size; 3080e39037bSChris Wilson loff_t iter; 3090e39037bSChris Wilson 310d897a111SMichal Wajdeczko int err; 311d897a111SMichal Wajdeczko }; 312d897a111SMichal Wajdeczko 313d897a111SMichal Wajdeczko #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) 314d897a111SMichal Wajdeczko 315d897a111SMichal Wajdeczko __printf(2, 3) 316d897a111SMichal Wajdeczko void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); 317d897a111SMichal Wajdeczko 318d897a111SMichal Wajdeczko struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); 319d897a111SMichal Wajdeczko void i915_capture_error_state(struct drm_i915_private *dev_priv, 320d897a111SMichal Wajdeczko u32 engine_mask, 321d897a111SMichal Wajdeczko const char *error_msg); 322d897a111SMichal Wajdeczko 323d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 324d897a111SMichal Wajdeczko i915_gpu_state_get(struct i915_gpu_state *gpu) 325d897a111SMichal Wajdeczko { 326d897a111SMichal Wajdeczko kref_get(&gpu->ref); 327d897a111SMichal Wajdeczko return gpu; 328d897a111SMichal Wajdeczko } 329d897a111SMichal Wajdeczko 3300e39037bSChris Wilson ssize_t i915_gpu_state_copy_to_buffer(struct i915_gpu_state *error, 3310e39037bSChris Wilson char *buf, loff_t offset, size_t count); 3320e39037bSChris Wilson 333d897a111SMichal Wajdeczko void __i915_gpu_state_free(struct kref *kref); 334d897a111SMichal Wajdeczko static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) 335d897a111SMichal Wajdeczko { 336d897a111SMichal Wajdeczko if (gpu) 337d897a111SMichal Wajdeczko kref_put(&gpu->ref, __i915_gpu_state_free); 338d897a111SMichal Wajdeczko } 339d897a111SMichal Wajdeczko 340d897a111SMichal Wajdeczko struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); 341d897a111SMichal Wajdeczko void i915_reset_error_state(struct drm_i915_private *i915); 342fb6f0b64SChris Wilson void i915_disable_error_state(struct drm_i915_private *i915, int err); 343d897a111SMichal Wajdeczko 344d897a111SMichal Wajdeczko #else 345d897a111SMichal Wajdeczko 346d897a111SMichal Wajdeczko static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, 347d897a111SMichal Wajdeczko u32 engine_mask, 348d897a111SMichal Wajdeczko const char *error_msg) 349d897a111SMichal Wajdeczko { 350d897a111SMichal Wajdeczko } 351d897a111SMichal Wajdeczko 352d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 353d897a111SMichal Wajdeczko i915_first_error_state(struct drm_i915_private *i915) 354d897a111SMichal Wajdeczko { 355fb6f0b64SChris Wilson return ERR_PTR(-ENODEV); 356d897a111SMichal Wajdeczko } 357d897a111SMichal Wajdeczko 358d897a111SMichal Wajdeczko static inline void i915_reset_error_state(struct drm_i915_private *i915) 359d897a111SMichal Wajdeczko { 360d897a111SMichal Wajdeczko } 361d897a111SMichal Wajdeczko 362fb6f0b64SChris Wilson static inline void i915_disable_error_state(struct drm_i915_private *i915, 363fb6f0b64SChris Wilson int err) 364fb6f0b64SChris Wilson { 365fb6f0b64SChris Wilson } 366fb6f0b64SChris Wilson 367d897a111SMichal Wajdeczko #endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */ 368d897a111SMichal Wajdeczko 369d897a111SMichal Wajdeczko #endif /* _I915_GPU_ERROR_H_ */ 370