1d897a111SMichal Wajdeczko /* 2d897a111SMichal Wajdeczko * SPDX-License-Identifier: MIT 3d897a111SMichal Wajdeczko * 4d897a111SMichal Wajdeczko * Copyright � 2008-2018 Intel Corporation 5d897a111SMichal Wajdeczko */ 6d897a111SMichal Wajdeczko 7d897a111SMichal Wajdeczko #ifndef _I915_GPU_ERROR_H_ 8d897a111SMichal Wajdeczko #define _I915_GPU_ERROR_H_ 9d897a111SMichal Wajdeczko 10d897a111SMichal Wajdeczko #include <linux/kref.h> 11d897a111SMichal Wajdeczko #include <linux/ktime.h> 12d897a111SMichal Wajdeczko #include <linux/sched.h> 13d897a111SMichal Wajdeczko 14d897a111SMichal Wajdeczko #include <drm/drm_mm.h> 15d897a111SMichal Wajdeczko 16d897a111SMichal Wajdeczko #include "intel_device_info.h" 17d897a111SMichal Wajdeczko #include "intel_ringbuffer.h" 18d897a111SMichal Wajdeczko #include "intel_uc_fw.h" 19d897a111SMichal Wajdeczko 20d897a111SMichal Wajdeczko #include "i915_gem.h" 21d897a111SMichal Wajdeczko #include "i915_gem_gtt.h" 22d897a111SMichal Wajdeczko #include "i915_params.h" 23b7268c5eSChris Wilson #include "i915_scheduler.h" 24d897a111SMichal Wajdeczko 25d897a111SMichal Wajdeczko struct drm_i915_private; 26d897a111SMichal Wajdeczko struct intel_overlay_error_state; 27d897a111SMichal Wajdeczko struct intel_display_error_state; 28d897a111SMichal Wajdeczko 29d897a111SMichal Wajdeczko struct i915_gpu_state { 30d897a111SMichal Wajdeczko struct kref ref; 31d897a111SMichal Wajdeczko ktime_t time; 32d897a111SMichal Wajdeczko ktime_t boottime; 33d897a111SMichal Wajdeczko ktime_t uptime; 34043477b0SMika Kuoppala unsigned long capture; 35043477b0SMika Kuoppala unsigned long epoch; 36d897a111SMichal Wajdeczko 37d897a111SMichal Wajdeczko struct drm_i915_private *i915; 38d897a111SMichal Wajdeczko 39d897a111SMichal Wajdeczko char error_msg[128]; 40d897a111SMichal Wajdeczko bool simulated; 41d897a111SMichal Wajdeczko bool awake; 42d897a111SMichal Wajdeczko bool wakelock; 43d897a111SMichal Wajdeczko bool suspended; 44d897a111SMichal Wajdeczko int iommu; 45d897a111SMichal Wajdeczko u32 reset_count; 46d897a111SMichal Wajdeczko u32 suspend_count; 47d897a111SMichal Wajdeczko struct intel_device_info device_info; 48d897a111SMichal Wajdeczko struct intel_driver_caps driver_caps; 49d897a111SMichal Wajdeczko struct i915_params params; 50d897a111SMichal Wajdeczko 51d897a111SMichal Wajdeczko struct i915_error_uc { 52d897a111SMichal Wajdeczko struct intel_uc_fw guc_fw; 53d897a111SMichal Wajdeczko struct intel_uc_fw huc_fw; 54d897a111SMichal Wajdeczko struct drm_i915_error_object *guc_log; 55d897a111SMichal Wajdeczko } uc; 56d897a111SMichal Wajdeczko 57d897a111SMichal Wajdeczko /* Generic register state */ 58d897a111SMichal Wajdeczko u32 eir; 59d897a111SMichal Wajdeczko u32 pgtbl_er; 60d897a111SMichal Wajdeczko u32 ier; 61d897a111SMichal Wajdeczko u32 gtier[4], ngtier; 62d897a111SMichal Wajdeczko u32 ccid; 63d897a111SMichal Wajdeczko u32 derrmr; 64d897a111SMichal Wajdeczko u32 forcewake; 65d897a111SMichal Wajdeczko u32 error; /* gen6+ */ 66d897a111SMichal Wajdeczko u32 err_int; /* gen7 */ 67d897a111SMichal Wajdeczko u32 fault_data0; /* gen8, gen9 */ 68d897a111SMichal Wajdeczko u32 fault_data1; /* gen8, gen9 */ 69d897a111SMichal Wajdeczko u32 done_reg; 70d897a111SMichal Wajdeczko u32 gac_eco; 71d897a111SMichal Wajdeczko u32 gam_ecochk; 72d897a111SMichal Wajdeczko u32 gab_ctl; 73d897a111SMichal Wajdeczko u32 gfx_mode; 74d897a111SMichal Wajdeczko 75d897a111SMichal Wajdeczko u32 nfence; 76d897a111SMichal Wajdeczko u64 fence[I915_MAX_NUM_FENCES]; 77d897a111SMichal Wajdeczko struct intel_overlay_error_state *overlay; 78d897a111SMichal Wajdeczko struct intel_display_error_state *display; 79d897a111SMichal Wajdeczko 80d897a111SMichal Wajdeczko struct drm_i915_error_engine { 81d897a111SMichal Wajdeczko int engine_id; 82d897a111SMichal Wajdeczko /* Software tracked state */ 83d897a111SMichal Wajdeczko bool idle; 84d897a111SMichal Wajdeczko bool waiting; 85d897a111SMichal Wajdeczko int num_waiters; 86d897a111SMichal Wajdeczko unsigned long hangcheck_timestamp; 87d897a111SMichal Wajdeczko bool hangcheck_stalled; 88d897a111SMichal Wajdeczko enum intel_engine_hangcheck_action hangcheck_action; 89d897a111SMichal Wajdeczko struct i915_address_space *vm; 90d897a111SMichal Wajdeczko int num_requests; 91d897a111SMichal Wajdeczko u32 reset_count; 92d897a111SMichal Wajdeczko 93d897a111SMichal Wajdeczko /* position of active request inside the ring */ 94d897a111SMichal Wajdeczko u32 rq_head, rq_post, rq_tail; 95d897a111SMichal Wajdeczko 96d897a111SMichal Wajdeczko /* our own tracking of ring head and tail */ 97d897a111SMichal Wajdeczko u32 cpu_ring_head; 98d897a111SMichal Wajdeczko u32 cpu_ring_tail; 99d897a111SMichal Wajdeczko 100d897a111SMichal Wajdeczko u32 last_seqno; 101d897a111SMichal Wajdeczko 102d897a111SMichal Wajdeczko /* Register state */ 103d897a111SMichal Wajdeczko u32 start; 104d897a111SMichal Wajdeczko u32 tail; 105d897a111SMichal Wajdeczko u32 head; 106d897a111SMichal Wajdeczko u32 ctl; 107d897a111SMichal Wajdeczko u32 mode; 108d897a111SMichal Wajdeczko u32 hws; 109d897a111SMichal Wajdeczko u32 ipeir; 110d897a111SMichal Wajdeczko u32 ipehr; 111d897a111SMichal Wajdeczko u32 bbstate; 112d897a111SMichal Wajdeczko u32 instpm; 113d897a111SMichal Wajdeczko u32 instps; 114d897a111SMichal Wajdeczko u32 seqno; 115d897a111SMichal Wajdeczko u64 bbaddr; 116d897a111SMichal Wajdeczko u64 acthd; 117d897a111SMichal Wajdeczko u32 fault_reg; 118d897a111SMichal Wajdeczko u64 faddr; 119d897a111SMichal Wajdeczko u32 rc_psmi; /* sleep state */ 120d897a111SMichal Wajdeczko u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; 121d897a111SMichal Wajdeczko struct intel_instdone instdone; 122d897a111SMichal Wajdeczko 123d897a111SMichal Wajdeczko struct drm_i915_error_context { 124d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 125d897a111SMichal Wajdeczko pid_t pid; 126d897a111SMichal Wajdeczko u32 handle; 127d897a111SMichal Wajdeczko u32 hw_id; 128d897a111SMichal Wajdeczko int ban_score; 129d897a111SMichal Wajdeczko int active; 130d897a111SMichal Wajdeczko int guilty; 131d897a111SMichal Wajdeczko bool bannable; 132b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 133d897a111SMichal Wajdeczko } context; 134d897a111SMichal Wajdeczko 135d897a111SMichal Wajdeczko struct drm_i915_error_object { 136d897a111SMichal Wajdeczko u64 gtt_offset; 137d897a111SMichal Wajdeczko u64 gtt_size; 138d897a111SMichal Wajdeczko int page_count; 139d897a111SMichal Wajdeczko int unused; 140d897a111SMichal Wajdeczko u32 *pages[0]; 141d897a111SMichal Wajdeczko } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; 142d897a111SMichal Wajdeczko 143d897a111SMichal Wajdeczko struct drm_i915_error_object **user_bo; 144d897a111SMichal Wajdeczko long user_bo_count; 145d897a111SMichal Wajdeczko 146d897a111SMichal Wajdeczko struct drm_i915_error_object *wa_ctx; 147d897a111SMichal Wajdeczko struct drm_i915_error_object *default_state; 148d897a111SMichal Wajdeczko 149d897a111SMichal Wajdeczko struct drm_i915_error_request { 150d897a111SMichal Wajdeczko long jiffies; 151d897a111SMichal Wajdeczko pid_t pid; 152d897a111SMichal Wajdeczko u32 context; 153d897a111SMichal Wajdeczko int ban_score; 154d897a111SMichal Wajdeczko u32 seqno; 155d897a111SMichal Wajdeczko u32 head; 156d897a111SMichal Wajdeczko u32 tail; 157b7268c5eSChris Wilson struct i915_sched_attr sched_attr; 158d897a111SMichal Wajdeczko } *requests, execlist[EXECLIST_MAX_PORTS]; 159d897a111SMichal Wajdeczko unsigned int num_ports; 160d897a111SMichal Wajdeczko 161d897a111SMichal Wajdeczko struct drm_i915_error_waiter { 162d897a111SMichal Wajdeczko char comm[TASK_COMM_LEN]; 163d897a111SMichal Wajdeczko pid_t pid; 164d897a111SMichal Wajdeczko u32 seqno; 165d897a111SMichal Wajdeczko } *waiters; 166d897a111SMichal Wajdeczko 167d897a111SMichal Wajdeczko struct { 168d897a111SMichal Wajdeczko u32 gfx_mode; 169d897a111SMichal Wajdeczko union { 170d897a111SMichal Wajdeczko u64 pdp[4]; 171d897a111SMichal Wajdeczko u32 pp_dir_base; 172d897a111SMichal Wajdeczko }; 173d897a111SMichal Wajdeczko } vm_info; 174d897a111SMichal Wajdeczko } engine[I915_NUM_ENGINES]; 175d897a111SMichal Wajdeczko 176d897a111SMichal Wajdeczko struct drm_i915_error_buffer { 177d897a111SMichal Wajdeczko u32 size; 178d897a111SMichal Wajdeczko u32 name; 179d897a111SMichal Wajdeczko u32 rseqno[I915_NUM_ENGINES], wseqno; 180d897a111SMichal Wajdeczko u64 gtt_offset; 181d897a111SMichal Wajdeczko u32 read_domains; 182d897a111SMichal Wajdeczko u32 write_domain; 183d897a111SMichal Wajdeczko s32 fence_reg:I915_MAX_NUM_FENCE_BITS; 184d897a111SMichal Wajdeczko u32 tiling:2; 185d897a111SMichal Wajdeczko u32 dirty:1; 186d897a111SMichal Wajdeczko u32 purgeable:1; 187d897a111SMichal Wajdeczko u32 userptr:1; 188d897a111SMichal Wajdeczko s32 engine:4; 189d897a111SMichal Wajdeczko u32 cache_level:3; 190d897a111SMichal Wajdeczko } *active_bo[I915_NUM_ENGINES], *pinned_bo; 191d897a111SMichal Wajdeczko u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; 192d897a111SMichal Wajdeczko struct i915_address_space *active_vm[I915_NUM_ENGINES]; 193d897a111SMichal Wajdeczko }; 194d897a111SMichal Wajdeczko 195d897a111SMichal Wajdeczko struct i915_gpu_error { 196d897a111SMichal Wajdeczko /* For hangcheck timer */ 197d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ 198d897a111SMichal Wajdeczko #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) 199d897a111SMichal Wajdeczko 200d897a111SMichal Wajdeczko struct delayed_work hangcheck_work; 201d897a111SMichal Wajdeczko 202d897a111SMichal Wajdeczko /* For reset and error_state handling. */ 203d897a111SMichal Wajdeczko spinlock_t lock; 204d897a111SMichal Wajdeczko /* Protected by the above dev->gpu_error.lock. */ 205d897a111SMichal Wajdeczko struct i915_gpu_state *first_error; 206d897a111SMichal Wajdeczko 207d897a111SMichal Wajdeczko atomic_t pending_fb_pin; 208d897a111SMichal Wajdeczko 209d897a111SMichal Wajdeczko unsigned long missed_irq_rings; 210d897a111SMichal Wajdeczko 211d897a111SMichal Wajdeczko /** 212d897a111SMichal Wajdeczko * State variable controlling the reset flow and count 213d897a111SMichal Wajdeczko * 214d897a111SMichal Wajdeczko * This is a counter which gets incremented when reset is triggered, 215d897a111SMichal Wajdeczko * 216d897a111SMichal Wajdeczko * Before the reset commences, the I915_RESET_BACKOFF bit is set 217d897a111SMichal Wajdeczko * meaning that any waiters holding onto the struct_mutex should 218d897a111SMichal Wajdeczko * relinquish the lock immediately in order for the reset to start. 219d897a111SMichal Wajdeczko * 220d897a111SMichal Wajdeczko * If reset is not completed successfully, the I915_WEDGE bit is 221d897a111SMichal Wajdeczko * set meaning that hardware is terminally sour and there is no 222d897a111SMichal Wajdeczko * recovery. All waiters on the reset_queue will be woken when 223d897a111SMichal Wajdeczko * that happens. 224d897a111SMichal Wajdeczko * 225d897a111SMichal Wajdeczko * This counter is used by the wait_seqno code to notice that reset 226d897a111SMichal Wajdeczko * event happened and it needs to restart the entire ioctl (since most 227d897a111SMichal Wajdeczko * likely the seqno it waited for won't ever signal anytime soon). 228d897a111SMichal Wajdeczko * 229d897a111SMichal Wajdeczko * This is important for lock-free wait paths, where no contended lock 230d897a111SMichal Wajdeczko * naturally enforces the correct ordering between the bail-out of the 231d897a111SMichal Wajdeczko * waiter and the gpu reset work code. 232d897a111SMichal Wajdeczko */ 233d897a111SMichal Wajdeczko unsigned long reset_count; 234d897a111SMichal Wajdeczko 235d897a111SMichal Wajdeczko /** 236d897a111SMichal Wajdeczko * flags: Control various stages of the GPU reset 237d897a111SMichal Wajdeczko * 238d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF - When we start a reset, we want to stop any 239d897a111SMichal Wajdeczko * other users acquiring the struct_mutex. To do this we set the 240d897a111SMichal Wajdeczko * #I915_RESET_BACKOFF bit in the error flags when we detect a reset 241d897a111SMichal Wajdeczko * and then check for that bit before acquiring the struct_mutex (in 242d897a111SMichal Wajdeczko * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a 243d897a111SMichal Wajdeczko * secondary role in preventing two concurrent global reset attempts. 244d897a111SMichal Wajdeczko * 245d897a111SMichal Wajdeczko * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the 246d897a111SMichal Wajdeczko * struct_mutex. We try to acquire the struct_mutex in the reset worker, 247d897a111SMichal Wajdeczko * but it may be held by some long running waiter (that we cannot 248d897a111SMichal Wajdeczko * interrupt without causing trouble). Once we are ready to do the GPU 249d897a111SMichal Wajdeczko * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If 250d897a111SMichal Wajdeczko * they already hold the struct_mutex and want to participate they can 251d897a111SMichal Wajdeczko * inspect the bit and do the reset directly, otherwise the worker 252d897a111SMichal Wajdeczko * waits for the struct_mutex. 253d897a111SMichal Wajdeczko * 254d897a111SMichal Wajdeczko * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to 255d897a111SMichal Wajdeczko * acquire the struct_mutex to reset an engine, we need an explicit 256d897a111SMichal Wajdeczko * flag to prevent two concurrent reset attempts in the same engine. 257d897a111SMichal Wajdeczko * As the number of engines continues to grow, allocate the flags from 258d897a111SMichal Wajdeczko * the most significant bits. 259d897a111SMichal Wajdeczko * 260d897a111SMichal Wajdeczko * #I915_WEDGED - If reset fails and we can no longer use the GPU, 261d897a111SMichal Wajdeczko * we set the #I915_WEDGED bit. Prior to command submission, e.g. 262d897a111SMichal Wajdeczko * i915_request_alloc(), this bit is checked and the sequence 263d897a111SMichal Wajdeczko * aborted (with -EIO reported to userspace) if set. 264d897a111SMichal Wajdeczko */ 265d897a111SMichal Wajdeczko unsigned long flags; 266d897a111SMichal Wajdeczko #define I915_RESET_BACKOFF 0 267d897a111SMichal Wajdeczko #define I915_RESET_HANDOFF 1 268d897a111SMichal Wajdeczko #define I915_RESET_MODESET 2 269d897a111SMichal Wajdeczko #define I915_WEDGED (BITS_PER_LONG - 1) 270d897a111SMichal Wajdeczko #define I915_RESET_ENGINE (I915_WEDGED - I915_NUM_ENGINES) 271d897a111SMichal Wajdeczko 272d897a111SMichal Wajdeczko /** Number of times an engine has been reset */ 273d897a111SMichal Wajdeczko u32 reset_engine_count[I915_NUM_ENGINES]; 274d897a111SMichal Wajdeczko 275d0667e9cSChris Wilson /** Set of stalled engines with guilty requests, in the current reset */ 276d0667e9cSChris Wilson u32 stalled_mask; 277d0667e9cSChris Wilson 278ce800754SChris Wilson /** Reason for the current *global* reset */ 279ce800754SChris Wilson const char *reason; 280ce800754SChris Wilson 281d897a111SMichal Wajdeczko /** 282d897a111SMichal Wajdeczko * Waitqueue to signal when a hang is detected. Used to for waiters 283d897a111SMichal Wajdeczko * to release the struct_mutex for the reset to procede. 284d897a111SMichal Wajdeczko */ 285d897a111SMichal Wajdeczko wait_queue_head_t wait_queue; 286d897a111SMichal Wajdeczko 287d897a111SMichal Wajdeczko /** 288d897a111SMichal Wajdeczko * Waitqueue to signal when the reset has completed. Used by clients 289d897a111SMichal Wajdeczko * that wait for dev_priv->mm.wedged to settle. 290d897a111SMichal Wajdeczko */ 291d897a111SMichal Wajdeczko wait_queue_head_t reset_queue; 292d897a111SMichal Wajdeczko 293d897a111SMichal Wajdeczko /* For missed irq/seqno simulation. */ 294d897a111SMichal Wajdeczko unsigned long test_irq_rings; 295d897a111SMichal Wajdeczko }; 296d897a111SMichal Wajdeczko 297d897a111SMichal Wajdeczko struct drm_i915_error_state_buf { 298d897a111SMichal Wajdeczko struct drm_i915_private *i915; 299d897a111SMichal Wajdeczko unsigned int bytes; 300d897a111SMichal Wajdeczko unsigned int size; 301d897a111SMichal Wajdeczko int err; 302d897a111SMichal Wajdeczko u8 *buf; 303d897a111SMichal Wajdeczko loff_t start; 304d897a111SMichal Wajdeczko loff_t pos; 305d897a111SMichal Wajdeczko }; 306d897a111SMichal Wajdeczko 307d897a111SMichal Wajdeczko #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) 308d897a111SMichal Wajdeczko 309d897a111SMichal Wajdeczko __printf(2, 3) 310d897a111SMichal Wajdeczko void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); 311d897a111SMichal Wajdeczko int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, 312d897a111SMichal Wajdeczko const struct i915_gpu_state *gpu); 313d897a111SMichal Wajdeczko int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, 314d897a111SMichal Wajdeczko struct drm_i915_private *i915, 315d897a111SMichal Wajdeczko size_t count, loff_t pos); 316d897a111SMichal Wajdeczko 317d897a111SMichal Wajdeczko static inline void 318d897a111SMichal Wajdeczko i915_error_state_buf_release(struct drm_i915_error_state_buf *eb) 319d897a111SMichal Wajdeczko { 320d897a111SMichal Wajdeczko kfree(eb->buf); 321d897a111SMichal Wajdeczko } 322d897a111SMichal Wajdeczko 323d897a111SMichal Wajdeczko struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); 324d897a111SMichal Wajdeczko void i915_capture_error_state(struct drm_i915_private *dev_priv, 325d897a111SMichal Wajdeczko u32 engine_mask, 326d897a111SMichal Wajdeczko const char *error_msg); 327d897a111SMichal Wajdeczko 328d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 329d897a111SMichal Wajdeczko i915_gpu_state_get(struct i915_gpu_state *gpu) 330d897a111SMichal Wajdeczko { 331d897a111SMichal Wajdeczko kref_get(&gpu->ref); 332d897a111SMichal Wajdeczko return gpu; 333d897a111SMichal Wajdeczko } 334d897a111SMichal Wajdeczko 335d897a111SMichal Wajdeczko void __i915_gpu_state_free(struct kref *kref); 336d897a111SMichal Wajdeczko static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) 337d897a111SMichal Wajdeczko { 338d897a111SMichal Wajdeczko if (gpu) 339d897a111SMichal Wajdeczko kref_put(&gpu->ref, __i915_gpu_state_free); 340d897a111SMichal Wajdeczko } 341d897a111SMichal Wajdeczko 342d897a111SMichal Wajdeczko struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); 343d897a111SMichal Wajdeczko void i915_reset_error_state(struct drm_i915_private *i915); 344d897a111SMichal Wajdeczko 345d897a111SMichal Wajdeczko #else 346d897a111SMichal Wajdeczko 347d897a111SMichal Wajdeczko static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, 348d897a111SMichal Wajdeczko u32 engine_mask, 349d897a111SMichal Wajdeczko const char *error_msg) 350d897a111SMichal Wajdeczko { 351d897a111SMichal Wajdeczko } 352d897a111SMichal Wajdeczko 353d897a111SMichal Wajdeczko static inline struct i915_gpu_state * 354d897a111SMichal Wajdeczko i915_first_error_state(struct drm_i915_private *i915) 355d897a111SMichal Wajdeczko { 356d897a111SMichal Wajdeczko return NULL; 357d897a111SMichal Wajdeczko } 358d897a111SMichal Wajdeczko 359d897a111SMichal Wajdeczko static inline void i915_reset_error_state(struct drm_i915_private *i915) 360d897a111SMichal Wajdeczko { 361d897a111SMichal Wajdeczko } 362d897a111SMichal Wajdeczko 363d897a111SMichal Wajdeczko #endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */ 364d897a111SMichal Wajdeczko 365d897a111SMichal Wajdeczko #endif /* _I915_GPU_ERROR_H_ */ 366