1d897a111SMichal Wajdeczko /* 2d897a111SMichal Wajdeczko * SPDX-License-Identifier: MIT 3d897a111SMichal Wajdeczko * 4d897a111SMichal Wajdeczko * Copyright � 2008-2018 Intel Corporation 5d897a111SMichal Wajdeczko */ 6d897a111SMichal Wajdeczko 7d897a111SMichal Wajdeczko #ifndef _I915_GPU_ERROR_H_ 8d897a111SMichal Wajdeczko #define _I915_GPU_ERROR_H_ 9d897a111SMichal Wajdeczko 10d897a111SMichal Wajdeczko #include <linux/kref.h> 11d897a111SMichal Wajdeczko #include <linux/ktime.h> 12d897a111SMichal Wajdeczko #include <linux/sched.h> 13d897a111SMichal Wajdeczko 14d897a111SMichal Wajdeczko #include <drm/drm_mm.h> 15d897a111SMichal Wajdeczko 16d897a111SMichal Wajdeczko #include "intel_device_info.h" 17d897a111SMichal Wajdeczko #include "intel_ringbuffer.h" 18d897a111SMichal Wajdeczko #include "intel_uc_fw.h" 19d897a111SMichal Wajdeczko 20d897a111SMichal Wajdeczko #include "i915_gem.h" 21d897a111SMichal Wajdeczko #include "i915_gem_gtt.h" 22d897a111SMichal Wajdeczko #include "i915_params.h" 23b7268c5eSChris Wilson #include "i915_scheduler.h" 24d897a111SMichal Wajdeczko 25d897a111SMichal Wajdeczko struct drm_i915_private; 26d897a111SMichal Wajdeczko struct intel_overlay_error_state; 27d897a111SMichal Wajdeczko struct intel_display_error_state; 28d897a111SMichal Wajdeczko 29d897a111SMichal Wajdeczko struct i915_gpu_state { 30d897a111SMichal Wajdeczko struct kref ref; 31d897a111SMichal Wajdeczko ktime_t time; 32d897a111SMichal Wajdeczko ktime_t boottime; 33d897a111SMichal Wajdeczko ktime_t uptime; 34043477b0SMika Kuoppala unsigned long capture; 35043477b0SMika Kuoppala unsigned long epoch; 36d897a111SMichal Wajdeczko 37d897a111SMichal Wajdeczko struct drm_i915_private *i915; 38d897a111SMichal Wajdeczko 39d897a111SMichal Wajdeczko char error_msg[128]; 40d897a111SMichal Wajdeczko bool simulated; 41d897a111SMichal Wajdeczko bool awake; 42d897a111SMichal Wajdeczko bool wakelock; 43d897a111SMichal Wajdeczko bool suspended; 44d897a111SMichal Wajdeczko int iommu; 45d897a111SMichal Wajdeczko u32 reset_count; 46d897a111SMichal Wajdeczko u32 suspend_count; 47d897a111SMichal Wajdeczko struct intel_device_info device_info; 48d897a111SMichal Wajdeczko struct intel_driver_caps driver_caps; 49d897a111SMichal Wajdeczko struct i915_params params; 50d897a111SMichal Wajdeczko 51d897a111SMichal Wajdeczko struct i915_error_uc { 52d897a111SMichal Wajdeczko struct intel_uc_fw guc_fw; 53d897a111SMichal Wajdeczko struct intel_uc_fw huc_fw; 54d897a111SMichal Wajdeczko struct drm_i915_error_object *guc_log; 55d897a111SMichal Wajdeczko } uc; 56d897a111SMichal Wajdeczko 57d897a111SMichal Wajdeczko /* Generic register state */ 58d897a111SMichal Wajdeczko u32 eir; 59d897a111SMichal Wajdeczko u32 pgtbl_er; 60d897a111SMichal Wajdeczko u32 ier; 616b7a6a7bSOscar Mateo u32 gtier[6], ngtier; 62d897a111SMichal Wajdeczko u32 ccid; 63d897a111SMichal Wajdeczko u32 derrmr; 64d897a111SMichal Wajdeczko u32 forcewake; 65d897a111SMichal Wajdeczko u32 error; /* gen6+ */ 66d897a111SMichal Wajdeczko u32 err_int; /* gen7 */ 67d897a111SMichal Wajdeczko u32 fault_data0; /* gen8, gen9 */ 68d897a111SMichal Wajdeczko u32 fault_data1; /* gen8, gen9 */ 69d897a111SMichal Wajdeczko u32 done_reg; 70d897a111SMichal Wajdeczko u32 gac_eco; 71d897a111SMichal Wajdeczko u32 gam_ecochk; 72d897a111SMichal Wajdeczko u32 gab_ctl; 73d897a111SMichal Wajdeczko u32 gfx_mode; 74d897a111SMichal Wajdeczko 75d897a111SMichal Wajdeczko u32 nfence; 76d897a111SMichal Wajdeczko u64 fence[I915_MAX_NUM_FENCES]; 77d897a111SMichal Wajdeczko struct intel_overlay_error_state *overlay; 78d897a111SMichal Wajdeczko struct intel_display_error_state *display; 79d897a111SMichal Wajdeczko 80d897a111SMichal Wajdeczko struct drm_i915_error_engine { 81d897a111SMichal Wajdeczko int engine_id; 82d897a111SMichal Wajdeczko /* Software tracked state */ 83d897a111SMichal Wajdeczko bool idle; 84d897a111SMichal Wajdeczko bool waiting; 85d897a111SMichal Wajdeczko int num_waiters; 86d897a111SMichal Wajdeczko unsigned long hangcheck_timestamp; 87d897a111SMichal Wajdeczko bool hangcheck_stalled; 88d897a111SMichal Wajdeczko enum intel_engine_hangcheck_action hangcheck_action; 89d897a111SMichal Wajdeczko struct i915_address_space *vm; 90d897a111SMichal Wajdeczko int num_requests; 91d897a111SMichal Wajdeczko u32 reset_count; 92d897a111SMichal Wajdeczko 93d897a111SMichal Wajdeczko /* position of active request inside the ring */ 94d897a111SMichal Wajdeczko u32 rq_head, rq_post, rq_tail; 95d897a111SMichal Wajdeczko 96d897a111SMichal Wajdeczko /* our own tracking of ring head and tail */ 97d897a111SMichal Wajdeczko u32 cpu_ring_head; 98d897a111SMichal Wajdeczko u32 cpu_ring_tail; 99d897a111SMichal Wajdeczko 100d897a111SMichal Wajdeczko u32 last_seqno; 101d897a111SMichal Wajdeczko 102d897a111SMichal Wajdeczko /* Register state */ 103d897a111SMichal Wajdeczko u32 start; 104d897a111SMichal Wajdeczko u32 tail; 105d897a111SMichal Wajdeczko u32 head; 106d897a111SMichal Wajdeczko u32 ctl; 107d897a111SMichal Wajdeczko u32 mode; 108d897a111SMichal Wajdeczko u32 hws; 109d897a111SMichal Wajdeczko u32 ipeir; 110d897a111SMichal Wajdeczko u32 ipehr; 111d897a111SMichal Wajdeczko u32 bbstate; 112d897a111SMichal Wajdeczko u32 instpm; 113d897a111SMichal Wajdeczko u32 instps; 114d897a111SMichal Wajdeczko u32 seqno; 115d897a111SMichal Wajdeczko u64 bbaddr; 116d897a111SMichal Wajdeczko u64 acthd; 117d897a111SMichal Wajdeczko u32 fault_reg; 118d897a111SMichal Wajdeczko u64 faddr; 119d897a111SMichal Wajdeczko u32 rc_psmi; /* sleep state */ 120d897a111SMichal Wajdeczko u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; 121d897a111SMichal Wajdeczko struct intel_instdone instdone; 122d897a111SMichal Wajdeczko 123d897a111SMichal Wajdeczko struct drm_i915_error_context { 124d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 125d897a111SMichal Wajdeczko pid_t pid; 126d897a111SMichal Wajdeczko u32 handle; 127d897a111SMichal Wajdeczko u32 hw_id; 128d897a111SMichal Wajdeczko int ban_score; 129d897a111SMichal Wajdeczko int active; 130d897a111SMichal Wajdeczko int guilty; 131d897a111SMichal Wajdeczko bool bannable; 132b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 133d897a111SMichal Wajdeczko } context; 134d897a111SMichal Wajdeczko 135d897a111SMichal Wajdeczko struct drm_i915_error_object { 136d897a111SMichal Wajdeczko u64 gtt_offset; 137d897a111SMichal Wajdeczko u64 gtt_size; 138d897a111SMichal Wajdeczko int page_count; 139d897a111SMichal Wajdeczko int unused; 140d897a111SMichal Wajdeczko u32 *pages[0]; 141d897a111SMichal Wajdeczko } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; 142d897a111SMichal Wajdeczko 143d897a111SMichal Wajdeczko struct drm_i915_error_object **user_bo; 144d897a111SMichal Wajdeczko long user_bo_count; 145d897a111SMichal Wajdeczko 146d897a111SMichal Wajdeczko struct drm_i915_error_object *wa_ctx; 147d897a111SMichal Wajdeczko struct drm_i915_error_object *default_state; 148d897a111SMichal Wajdeczko 149d897a111SMichal Wajdeczko struct drm_i915_error_request { 150d897a111SMichal Wajdeczko long jiffies; 151d897a111SMichal Wajdeczko pid_t pid; 152d897a111SMichal Wajdeczko u32 context; 153d897a111SMichal Wajdeczko int ban_score; 154d897a111SMichal Wajdeczko u32 seqno; 1553a068721SChris Wilson u32 start; 156d897a111SMichal Wajdeczko u32 head; 157d897a111SMichal Wajdeczko u32 tail; 158b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 159d897a111SMichal Wajdeczko } *requests, execlist[EXECLIST_MAX_PORTS]; 160d897a111SMichal Wajdeczko unsigned int num_ports; 161d897a111SMichal Wajdeczko 162d897a111SMichal Wajdeczko struct drm_i915_error_waiter { 163d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 164d897a111SMichal Wajdeczko pid_t pid; 165d897a111SMichal Wajdeczko u32 seqno; 166d897a111SMichal Wajdeczko } *waiters; 167d897a111SMichal Wajdeczko 168d897a111SMichal Wajdeczko struct { 169d897a111SMichal Wajdeczko u32 gfx_mode; 170d897a111SMichal Wajdeczko union { 171d897a111SMichal Wajdeczko u64 pdp[4]; 172d897a111SMichal Wajdeczko u32 pp_dir_base; 173d897a111SMichal Wajdeczko }; 174d897a111SMichal Wajdeczko } vm_info; 175d897a111SMichal Wajdeczko } engine[I915_NUM_ENGINES]; 176d897a111SMichal Wajdeczko 177d897a111SMichal Wajdeczko struct drm_i915_error_buffer { 178d897a111SMichal Wajdeczko u32 size; 179d897a111SMichal Wajdeczko u32 name; 1805c3f8c22SChris Wilson u32 wseqno; 181d897a111SMichal Wajdeczko u64 gtt_offset; 182d897a111SMichal Wajdeczko u32 read_domains; 183d897a111SMichal Wajdeczko u32 write_domain; 184d897a111SMichal Wajdeczko s32 fence_reg:I915_MAX_NUM_FENCE_BITS; 185d897a111SMichal Wajdeczko u32 tiling:2; 186d897a111SMichal Wajdeczko u32 dirty:1; 187d897a111SMichal Wajdeczko u32 purgeable:1; 188d897a111SMichal Wajdeczko u32 userptr:1; 189d897a111SMichal Wajdeczko s32 engine:4; 190d897a111SMichal Wajdeczko u32 cache_level:3; 191d897a111SMichal Wajdeczko } *active_bo[I915_NUM_ENGINES], *pinned_bo; 192d897a111SMichal Wajdeczko u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; 193d897a111SMichal Wajdeczko struct i915_address_space *active_vm[I915_NUM_ENGINES]; 194d897a111SMichal Wajdeczko }; 195d897a111SMichal Wajdeczko 196d897a111SMichal Wajdeczko struct i915_gpu_error { 197d897a111SMichal Wajdeczko /* For hangcheck timer */ 198d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ 199d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) 200d897a111SMichal Wajdeczko 201d897a111SMichal Wajdeczko struct delayed_work hangcheck_work; 202d897a111SMichal Wajdeczko 203d897a111SMichal Wajdeczko /* For reset and error_state handling. */ 204d897a111SMichal Wajdeczko spinlock_t lock; 205d897a111SMichal Wajdeczko /* Protected by the above dev->gpu_error.lock. */ 206d897a111SMichal Wajdeczko struct i915_gpu_state *first_error; 207d897a111SMichal Wajdeczko 208d897a111SMichal Wajdeczko atomic_t pending_fb_pin; 209d897a111SMichal Wajdeczko 210d897a111SMichal Wajdeczko unsigned long missed_irq_rings; 211d897a111SMichal Wajdeczko 212d897a111SMichal Wajdeczko /** 213d897a111SMichal Wajdeczko * State variable controlling the reset flow and count 214d897a111SMichal Wajdeczko * 215d897a111SMichal Wajdeczko * This is a counter which gets incremented when reset is triggered, 216d897a111SMichal Wajdeczko * 217d897a111SMichal Wajdeczko * Before the reset commences, the I915_RESET_BACKOFF bit is set 218d897a111SMichal Wajdeczko * meaning that any waiters holding onto the struct_mutex should 219d897a111SMichal Wajdeczko * relinquish the lock immediately in order for the reset to start. 220d897a111SMichal Wajdeczko * 221d897a111SMichal Wajdeczko * If reset is not completed successfully, the I915_WEDGE bit is 222d897a111SMichal Wajdeczko * set meaning that hardware is terminally sour and there is no 223d897a111SMichal Wajdeczko * recovery. All waiters on the reset_queue will be woken when 224d897a111SMichal Wajdeczko * that happens. 225d897a111SMichal Wajdeczko * 226d897a111SMichal Wajdeczko * This counter is used by the wait_seqno code to notice that reset 227d897a111SMichal Wajdeczko * event happened and it needs to restart the entire ioctl (since most 228d897a111SMichal Wajdeczko * likely the seqno it waited for won't ever signal anytime soon). 229d897a111SMichal Wajdeczko * 230d897a111SMichal Wajdeczko * This is important for lock-free wait paths, where no contended lock 231d897a111SMichal Wajdeczko * naturally enforces the correct ordering between the bail-out of the 232d897a111SMichal Wajdeczko * waiter and the gpu reset work code. 233d897a111SMichal Wajdeczko */ 234d897a111SMichal Wajdeczko unsigned long reset_count; 235d897a111SMichal Wajdeczko 236d897a111SMichal Wajdeczko /** 237d897a111SMichal Wajdeczko * flags: Control various stages of the GPU reset 238d897a111SMichal Wajdeczko * 239d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF - When we start a reset, we want to stop any 240d897a111SMichal Wajdeczko * other users acquiring the struct_mutex. To do this we set the 241d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF bit in the error flags when we detect a reset 242d897a111SMichal Wajdeczko * and then check for that bit before acquiring the struct_mutex (in 243d897a111SMichal Wajdeczko * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a 244d897a111SMichal Wajdeczko * secondary role in preventing two concurrent global reset attempts. 245d897a111SMichal Wajdeczko * 246d897a111SMichal Wajdeczko * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the 247d897a111SMichal Wajdeczko * struct_mutex. We try to acquire the struct_mutex in the reset worker, 248d897a111SMichal Wajdeczko * but it may be held by some long running waiter (that we cannot 249d897a111SMichal Wajdeczko * interrupt without causing trouble). Once we are ready to do the GPU 250d897a111SMichal Wajdeczko * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If 251d897a111SMichal Wajdeczko * they already hold the struct_mutex and want to participate they can 252d897a111SMichal Wajdeczko * inspect the bit and do the reset directly, otherwise the worker 253d897a111SMichal Wajdeczko * waits for the struct_mutex. 254d897a111SMichal Wajdeczko * 255d897a111SMichal Wajdeczko * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to 256d897a111SMichal Wajdeczko * acquire the struct_mutex to reset an engine, we need an explicit 257d897a111SMichal Wajdeczko * flag to prevent two concurrent reset attempts in the same engine. 258d897a111SMichal Wajdeczko * As the number of engines continues to grow, allocate the flags from 259d897a111SMichal Wajdeczko * the most significant bits. 260d897a111SMichal Wajdeczko * 261d897a111SMichal Wajdeczko * #I915_WEDGED - If reset fails and we can no longer use the GPU, 262d897a111SMichal Wajdeczko * we set the #I915_WEDGED bit. Prior to command submission, e.g. 263d897a111SMichal Wajdeczko * i915_request_alloc(), this bit is checked and the sequence 264d897a111SMichal Wajdeczko * aborted (with -EIO reported to userspace) if set. 265d897a111SMichal Wajdeczko */ 266d897a111SMichal Wajdeczko unsigned long flags; 267d897a111SMichal Wajdeczko #define I915_RESET_BACKOFF 0 268d897a111SMichal Wajdeczko #define I915_RESET_HANDOFF 1 269d897a111SMichal Wajdeczko #define I915_RESET_MODESET 2 270d897a111SMichal Wajdeczko #define I915_WEDGED (BITS_PER_LONG - 1) 271d897a111SMichal Wajdeczko #define I915_RESET_ENGINE (I915_WEDGED - I915_NUM_ENGINES) 272d897a111SMichal Wajdeczko 273d897a111SMichal Wajdeczko /** Number of times an engine has been reset */ 274d897a111SMichal Wajdeczko u32 reset_engine_count[I915_NUM_ENGINES]; 275d897a111SMichal Wajdeczko 276d0667e9cSChris Wilson /** Set of stalled engines with guilty requests, in the current reset */ 277d0667e9cSChris Wilson u32 stalled_mask; 278d0667e9cSChris Wilson 279ce800754SChris Wilson /** Reason for the current *global* reset */ 280ce800754SChris Wilson const char *reason; 281ce800754SChris Wilson 282d897a111SMichal Wajdeczko /** 283d897a111SMichal Wajdeczko * Waitqueue to signal when a hang is detected. Used to for waiters 284d897a111SMichal Wajdeczko * to release the struct_mutex for the reset to procede. 285d897a111SMichal Wajdeczko */ 286d897a111SMichal Wajdeczko wait_queue_head_t wait_queue; 287d897a111SMichal Wajdeczko 288d897a111SMichal Wajdeczko /** 289d897a111SMichal Wajdeczko * Waitqueue to signal when the reset has completed. Used by clients 290d897a111SMichal Wajdeczko * that wait for dev_priv->mm.wedged to settle. 291d897a111SMichal Wajdeczko */ 292d897a111SMichal Wajdeczko wait_queue_head_t reset_queue; 293d897a111SMichal Wajdeczko 294d897a111SMichal Wajdeczko /* For missed irq/seqno simulation. */ 295d897a111SMichal Wajdeczko unsigned long test_irq_rings; 296d897a111SMichal Wajdeczko }; 297d897a111SMichal Wajdeczko 298d897a111SMichal Wajdeczko struct drm_i915_error_state_buf { 299d897a111SMichal Wajdeczko struct drm_i915_private *i915; 300d897a111SMichal Wajdeczko unsigned int bytes; 301d897a111SMichal Wajdeczko unsigned int size; 302d897a111SMichal Wajdeczko int err; 303d897a111SMichal Wajdeczko u8 *buf; 304d897a111SMichal Wajdeczko loff_t start; 305d897a111SMichal Wajdeczko loff_t pos; 306d897a111SMichal Wajdeczko }; 307d897a111SMichal Wajdeczko 308d897a111SMichal Wajdeczko #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) 309d897a111SMichal Wajdeczko 310d897a111SMichal Wajdeczko __printf(2, 3) 311d897a111SMichal Wajdeczko void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); 312d897a111SMichal Wajdeczko int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, 313d897a111SMichal Wajdeczko const struct i915_gpu_state *gpu); 314d897a111SMichal Wajdeczko int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, 315d897a111SMichal Wajdeczko struct drm_i915_private *i915, 316d897a111SMichal Wajdeczko size_t count, loff_t pos); 317d897a111SMichal Wajdeczko 318d897a111SMichal Wajdeczko static inline void 319d897a111SMichal Wajdeczko i915_error_state_buf_release(struct drm_i915_error_state_buf *eb) 320d897a111SMichal Wajdeczko { 321d897a111SMichal Wajdeczko kfree(eb->buf); 322d897a111SMichal Wajdeczko } 323d897a111SMichal Wajdeczko 324d897a111SMichal Wajdeczko struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); 325d897a111SMichal Wajdeczko void i915_capture_error_state(struct drm_i915_private *dev_priv, 326d897a111SMichal Wajdeczko u32 engine_mask, 327d897a111SMichal Wajdeczko const char *error_msg); 328d897a111SMichal Wajdeczko 329d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 330d897a111SMichal Wajdeczko i915_gpu_state_get(struct i915_gpu_state *gpu) 331d897a111SMichal Wajdeczko { 332d897a111SMichal Wajdeczko kref_get(&gpu->ref); 333d897a111SMichal Wajdeczko return gpu; 334d897a111SMichal Wajdeczko } 335d897a111SMichal Wajdeczko 336d897a111SMichal Wajdeczko void __i915_gpu_state_free(struct kref *kref); 337d897a111SMichal Wajdeczko static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) 338d897a111SMichal Wajdeczko { 339d897a111SMichal Wajdeczko if (gpu) 340d897a111SMichal Wajdeczko kref_put(&gpu->ref, __i915_gpu_state_free); 341d897a111SMichal Wajdeczko } 342d897a111SMichal Wajdeczko 343d897a111SMichal Wajdeczko struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); 344d897a111SMichal Wajdeczko void i915_reset_error_state(struct drm_i915_private *i915); 345d897a111SMichal Wajdeczko 346d897a111SMichal Wajdeczko #else 347d897a111SMichal Wajdeczko 348d897a111SMichal Wajdeczko static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, 349d897a111SMichal Wajdeczko u32 engine_mask, 350d897a111SMichal Wajdeczko const char *error_msg) 351d897a111SMichal Wajdeczko { 352d897a111SMichal Wajdeczko } 353d897a111SMichal Wajdeczko 354d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 355d897a111SMichal Wajdeczko i915_first_error_state(struct drm_i915_private *i915) 356d897a111SMichal Wajdeczko { 357d897a111SMichal Wajdeczko return NULL; 358d897a111SMichal Wajdeczko } 359d897a111SMichal Wajdeczko 360d897a111SMichal Wajdeczko static inline void i915_reset_error_state(struct drm_i915_private *i915) 361d897a111SMichal Wajdeczko { 362d897a111SMichal Wajdeczko } 363d897a111SMichal Wajdeczko 364d897a111SMichal Wajdeczko #endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */ 365d897a111SMichal Wajdeczko 366d897a111SMichal Wajdeczko #endif /* _I915_GPU_ERROR_H_ */ 367