124f90d66SChris Wilson // SPDX-License-Identifier: MIT 2112ed2d3SChris Wilson /* 3112ed2d3SChris Wilson * Copyright © 2018 Intel Corporation 4112ed2d3SChris Wilson */ 5112ed2d3SChris Wilson 610be98a7SChris Wilson #include "gem/i915_gem_pm.h" 7750e76b4SChris Wilson #include "gt/intel_engine_user.h" 8baea429dSTvrtko Ursulin #include "gt/intel_gt.h" 9112ed2d3SChris Wilson #include "i915_selftest.h" 10112ed2d3SChris Wilson #include "intel_reset.h" 11112ed2d3SChris Wilson 12112ed2d3SChris Wilson #include "selftests/igt_flush_test.h" 13112ed2d3SChris Wilson #include "selftests/igt_reset.h" 14112ed2d3SChris Wilson #include "selftests/igt_spinner.h" 15112ed2d3SChris Wilson #include "selftests/mock_drm.h" 16112ed2d3SChris Wilson 1710be98a7SChris Wilson #include "gem/selftests/igt_gem_utils.h" 1810be98a7SChris Wilson #include "gem/selftests/mock_context.h" 1910be98a7SChris Wilson 20112ed2d3SChris Wilson static const struct wo_register { 21112ed2d3SChris Wilson enum intel_platform platform; 22112ed2d3SChris Wilson u32 reg; 23112ed2d3SChris Wilson } wo_registers[] = { 24112ed2d3SChris Wilson { INTEL_GEMINILAKE, 0x731c } 25112ed2d3SChris Wilson }; 26112ed2d3SChris Wilson 27112ed2d3SChris Wilson struct wa_lists { 28112ed2d3SChris Wilson struct i915_wa_list gt_wa_list; 29112ed2d3SChris Wilson struct { 30112ed2d3SChris Wilson struct i915_wa_list wa_list; 31fde93886STvrtko Ursulin struct i915_wa_list ctx_wa_list; 32112ed2d3SChris Wilson } engine[I915_NUM_ENGINES]; 33112ed2d3SChris Wilson }; 34112ed2d3SChris Wilson 3541f0bc49SChris Wilson static int request_add_sync(struct i915_request *rq, int err) 3641f0bc49SChris Wilson { 3741f0bc49SChris Wilson i915_request_get(rq); 3841f0bc49SChris Wilson i915_request_add(rq); 3941f0bc49SChris Wilson if (i915_request_wait(rq, 0, HZ / 5) < 0) 4041f0bc49SChris Wilson err = -EIO; 4141f0bc49SChris Wilson i915_request_put(rq); 4241f0bc49SChris Wilson 4341f0bc49SChris Wilson return err; 4441f0bc49SChris Wilson } 4541f0bc49SChris Wilson 4641f0bc49SChris Wilson static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 4741f0bc49SChris Wilson { 4841f0bc49SChris Wilson int err = 0; 4941f0bc49SChris Wilson 5041f0bc49SChris Wilson i915_request_get(rq); 5141f0bc49SChris Wilson i915_request_add(rq); 5241f0bc49SChris Wilson if (spin && !igt_wait_for_spinner(spin, rq)) 5341f0bc49SChris Wilson err = -ETIMEDOUT; 5441f0bc49SChris Wilson i915_request_put(rq); 5541f0bc49SChris Wilson 5641f0bc49SChris Wilson return err; 5741f0bc49SChris Wilson } 5841f0bc49SChris Wilson 59112ed2d3SChris Wilson static void 60bb3d4c9dSChris Wilson reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 61112ed2d3SChris Wilson { 62112ed2d3SChris Wilson struct intel_engine_cs *engine; 63112ed2d3SChris Wilson enum intel_engine_id id; 64112ed2d3SChris Wilson 65112ed2d3SChris Wilson memset(lists, 0, sizeof(*lists)); 66112ed2d3SChris Wilson 673e1f0a51SJohn Harrison wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 68bb3d4c9dSChris Wilson gt_init_workarounds(gt->i915, &lists->gt_wa_list); 69112ed2d3SChris Wilson wa_init_finish(&lists->gt_wa_list); 70112ed2d3SChris Wilson 715d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 72112ed2d3SChris Wilson struct i915_wa_list *wal = &lists->engine[id].wa_list; 73112ed2d3SChris Wilson 743e1f0a51SJohn Harrison wa_init_start(wal, "REF", engine->name); 75112ed2d3SChris Wilson engine_init_workarounds(engine, wal); 76112ed2d3SChris Wilson wa_init_finish(wal); 77fde93886STvrtko Ursulin 78fde93886STvrtko Ursulin __intel_engine_init_ctx_wa(engine, 79fde93886STvrtko Ursulin &lists->engine[id].ctx_wa_list, 803e1f0a51SJohn Harrison "CTX_REF"); 81112ed2d3SChris Wilson } 82112ed2d3SChris Wilson } 83112ed2d3SChris Wilson 84112ed2d3SChris Wilson static void 85bb3d4c9dSChris Wilson reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 86112ed2d3SChris Wilson { 87112ed2d3SChris Wilson struct intel_engine_cs *engine; 88112ed2d3SChris Wilson enum intel_engine_id id; 89112ed2d3SChris Wilson 905d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 91112ed2d3SChris Wilson intel_wa_list_free(&lists->engine[id].wa_list); 92112ed2d3SChris Wilson 93112ed2d3SChris Wilson intel_wa_list_free(&lists->gt_wa_list); 94112ed2d3SChris Wilson } 95112ed2d3SChris Wilson 96112ed2d3SChris Wilson static struct drm_i915_gem_object * 9704adaba8SChris Wilson read_nonprivs(struct intel_context *ce) 98112ed2d3SChris Wilson { 9904adaba8SChris Wilson struct intel_engine_cs *engine = ce->engine; 100112ed2d3SChris Wilson const u32 base = engine->mmio_base; 101112ed2d3SChris Wilson struct drm_i915_gem_object *result; 102112ed2d3SChris Wilson struct i915_request *rq; 103112ed2d3SChris Wilson struct i915_vma *vma; 104112ed2d3SChris Wilson u32 srm, *cs; 105112ed2d3SChris Wilson int err; 106112ed2d3SChris Wilson int i; 107112ed2d3SChris Wilson 108112ed2d3SChris Wilson result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 109112ed2d3SChris Wilson if (IS_ERR(result)) 110112ed2d3SChris Wilson return result; 111112ed2d3SChris Wilson 112112ed2d3SChris Wilson i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 113112ed2d3SChris Wilson 11474827b53SMaarten Lankhorst cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 115112ed2d3SChris Wilson if (IS_ERR(cs)) { 116112ed2d3SChris Wilson err = PTR_ERR(cs); 117112ed2d3SChris Wilson goto err_obj; 118112ed2d3SChris Wilson } 119112ed2d3SChris Wilson memset(cs, 0xc5, PAGE_SIZE); 120112ed2d3SChris Wilson i915_gem_object_flush_map(result); 121112ed2d3SChris Wilson i915_gem_object_unpin_map(result); 122112ed2d3SChris Wilson 123ba4134a4STvrtko Ursulin vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 124112ed2d3SChris Wilson if (IS_ERR(vma)) { 125112ed2d3SChris Wilson err = PTR_ERR(vma); 126112ed2d3SChris Wilson goto err_obj; 127112ed2d3SChris Wilson } 128112ed2d3SChris Wilson 129112ed2d3SChris Wilson err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 130112ed2d3SChris Wilson if (err) 131112ed2d3SChris Wilson goto err_obj; 132112ed2d3SChris Wilson 13304adaba8SChris Wilson rq = intel_context_create_request(ce); 134112ed2d3SChris Wilson if (IS_ERR(rq)) { 135112ed2d3SChris Wilson err = PTR_ERR(rq); 136112ed2d3SChris Wilson goto err_pin; 137112ed2d3SChris Wilson } 138112ed2d3SChris Wilson 1396951e589SChris Wilson i915_vma_lock(vma); 14070d6894dSChris Wilson err = i915_request_await_object(rq, vma->obj, true); 14170d6894dSChris Wilson if (err == 0) 142112ed2d3SChris Wilson err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 1436951e589SChris Wilson i915_vma_unlock(vma); 144112ed2d3SChris Wilson if (err) 145112ed2d3SChris Wilson goto err_req; 146112ed2d3SChris Wilson 147112ed2d3SChris Wilson srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 148*c816723bSLucas De Marchi if (GRAPHICS_VER(engine->i915) >= 8) 149112ed2d3SChris Wilson srm++; 150112ed2d3SChris Wilson 151112ed2d3SChris Wilson cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 152112ed2d3SChris Wilson if (IS_ERR(cs)) { 153112ed2d3SChris Wilson err = PTR_ERR(cs); 154112ed2d3SChris Wilson goto err_req; 155112ed2d3SChris Wilson } 156112ed2d3SChris Wilson 157112ed2d3SChris Wilson for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 158112ed2d3SChris Wilson *cs++ = srm; 159112ed2d3SChris Wilson *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 160112ed2d3SChris Wilson *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 161112ed2d3SChris Wilson *cs++ = 0; 162112ed2d3SChris Wilson } 163112ed2d3SChris Wilson intel_ring_advance(rq, cs); 164112ed2d3SChris Wilson 165112ed2d3SChris Wilson i915_request_add(rq); 166112ed2d3SChris Wilson i915_vma_unpin(vma); 167112ed2d3SChris Wilson 168112ed2d3SChris Wilson return result; 169112ed2d3SChris Wilson 170112ed2d3SChris Wilson err_req: 171112ed2d3SChris Wilson i915_request_add(rq); 172112ed2d3SChris Wilson err_pin: 173112ed2d3SChris Wilson i915_vma_unpin(vma); 174112ed2d3SChris Wilson err_obj: 175112ed2d3SChris Wilson i915_gem_object_put(result); 176112ed2d3SChris Wilson return ERR_PTR(err); 177112ed2d3SChris Wilson } 178112ed2d3SChris Wilson 179112ed2d3SChris Wilson static u32 180112ed2d3SChris Wilson get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 181112ed2d3SChris Wilson { 182112ed2d3SChris Wilson i915_reg_t reg = i < engine->whitelist.count ? 183112ed2d3SChris Wilson engine->whitelist.list[i].reg : 184112ed2d3SChris Wilson RING_NOPID(engine->mmio_base); 185112ed2d3SChris Wilson 186112ed2d3SChris Wilson return i915_mmio_reg_offset(reg); 187112ed2d3SChris Wilson } 188112ed2d3SChris Wilson 189112ed2d3SChris Wilson static void 190112ed2d3SChris Wilson print_results(const struct intel_engine_cs *engine, const u32 *results) 191112ed2d3SChris Wilson { 192112ed2d3SChris Wilson unsigned int i; 193112ed2d3SChris Wilson 194112ed2d3SChris Wilson for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 195112ed2d3SChris Wilson u32 expected = get_whitelist_reg(engine, i); 196112ed2d3SChris Wilson u32 actual = results[i]; 197112ed2d3SChris Wilson 198112ed2d3SChris Wilson pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 199112ed2d3SChris Wilson i, expected, actual); 200112ed2d3SChris Wilson } 201112ed2d3SChris Wilson } 202112ed2d3SChris Wilson 20304adaba8SChris Wilson static int check_whitelist(struct intel_context *ce) 204112ed2d3SChris Wilson { 20504adaba8SChris Wilson struct intel_engine_cs *engine = ce->engine; 206112ed2d3SChris Wilson struct drm_i915_gem_object *results; 207cb823ed9SChris Wilson struct intel_wedge_me wedge; 208112ed2d3SChris Wilson u32 *vaddr; 209112ed2d3SChris Wilson int err; 210112ed2d3SChris Wilson int i; 211112ed2d3SChris Wilson 21204adaba8SChris Wilson results = read_nonprivs(ce); 213112ed2d3SChris Wilson if (IS_ERR(results)) 214112ed2d3SChris Wilson return PTR_ERR(results); 215112ed2d3SChris Wilson 216112ed2d3SChris Wilson err = 0; 21780f0b679SMaarten Lankhorst i915_gem_object_lock(results, NULL); 218bb3d4c9dSChris Wilson intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 219112ed2d3SChris Wilson err = i915_gem_object_set_to_cpu_domain(results, false); 22074827b53SMaarten Lankhorst 221bb3d4c9dSChris Wilson if (intel_gt_is_wedged(engine->gt)) 222112ed2d3SChris Wilson err = -EIO; 223112ed2d3SChris Wilson if (err) 224112ed2d3SChris Wilson goto out_put; 225112ed2d3SChris Wilson 226112ed2d3SChris Wilson vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 227112ed2d3SChris Wilson if (IS_ERR(vaddr)) { 228112ed2d3SChris Wilson err = PTR_ERR(vaddr); 229112ed2d3SChris Wilson goto out_put; 230112ed2d3SChris Wilson } 231112ed2d3SChris Wilson 232112ed2d3SChris Wilson for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 233112ed2d3SChris Wilson u32 expected = get_whitelist_reg(engine, i); 234112ed2d3SChris Wilson u32 actual = vaddr[i]; 235112ed2d3SChris Wilson 236112ed2d3SChris Wilson if (expected != actual) { 237112ed2d3SChris Wilson print_results(engine, vaddr); 238112ed2d3SChris Wilson pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 239112ed2d3SChris Wilson i, expected, actual); 240112ed2d3SChris Wilson 241112ed2d3SChris Wilson err = -EINVAL; 242112ed2d3SChris Wilson break; 243112ed2d3SChris Wilson } 244112ed2d3SChris Wilson } 245112ed2d3SChris Wilson 246112ed2d3SChris Wilson i915_gem_object_unpin_map(results); 247112ed2d3SChris Wilson out_put: 24874827b53SMaarten Lankhorst i915_gem_object_unlock(results); 249112ed2d3SChris Wilson i915_gem_object_put(results); 250112ed2d3SChris Wilson return err; 251112ed2d3SChris Wilson } 252112ed2d3SChris Wilson 253112ed2d3SChris Wilson static int do_device_reset(struct intel_engine_cs *engine) 254112ed2d3SChris Wilson { 255cb823ed9SChris Wilson intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 256112ed2d3SChris Wilson return 0; 257112ed2d3SChris Wilson } 258112ed2d3SChris Wilson 259112ed2d3SChris Wilson static int do_engine_reset(struct intel_engine_cs *engine) 260112ed2d3SChris Wilson { 261cb823ed9SChris Wilson return intel_engine_reset(engine, "live_workarounds"); 262112ed2d3SChris Wilson } 263112ed2d3SChris Wilson 264112ed2d3SChris Wilson static int 265112ed2d3SChris Wilson switch_to_scratch_context(struct intel_engine_cs *engine, 266112ed2d3SChris Wilson struct igt_spinner *spin) 267112ed2d3SChris Wilson { 268f277bc0cSChris Wilson struct intel_context *ce; 269112ed2d3SChris Wilson struct i915_request *rq; 270112ed2d3SChris Wilson int err = 0; 271112ed2d3SChris Wilson 272e6ba7648SChris Wilson ce = intel_context_create(engine); 273e6ba7648SChris Wilson if (IS_ERR(ce)) 274e6ba7648SChris Wilson return PTR_ERR(ce); 275f277bc0cSChris Wilson 276f277bc0cSChris Wilson rq = igt_spinner_create_request(spin, ce, MI_NOOP); 277f277bc0cSChris Wilson intel_context_put(ce); 278112ed2d3SChris Wilson 279112ed2d3SChris Wilson if (IS_ERR(rq)) { 280112ed2d3SChris Wilson spin = NULL; 281112ed2d3SChris Wilson err = PTR_ERR(rq); 282112ed2d3SChris Wilson goto err; 283112ed2d3SChris Wilson } 284112ed2d3SChris Wilson 28541f0bc49SChris Wilson err = request_add_spin(rq, spin); 286112ed2d3SChris Wilson err: 287112ed2d3SChris Wilson if (err && spin) 288112ed2d3SChris Wilson igt_spinner_end(spin); 289112ed2d3SChris Wilson 290112ed2d3SChris Wilson return err; 291112ed2d3SChris Wilson } 292112ed2d3SChris Wilson 293112ed2d3SChris Wilson static int check_whitelist_across_reset(struct intel_engine_cs *engine, 294112ed2d3SChris Wilson int (*reset)(struct intel_engine_cs *), 295112ed2d3SChris Wilson const char *name) 296112ed2d3SChris Wilson { 29704adaba8SChris Wilson struct intel_context *ce, *tmp; 298112ed2d3SChris Wilson struct igt_spinner spin; 299112ed2d3SChris Wilson intel_wakeref_t wakeref; 300112ed2d3SChris Wilson int err; 301112ed2d3SChris Wilson 3023e1f0a51SJohn Harrison pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 3033e1f0a51SJohn Harrison engine->whitelist.count, engine->name, name); 304112ed2d3SChris Wilson 30504adaba8SChris Wilson ce = intel_context_create(engine); 30604adaba8SChris Wilson if (IS_ERR(ce)) 30704adaba8SChris Wilson return PTR_ERR(ce); 308112ed2d3SChris Wilson 309f277bc0cSChris Wilson err = igt_spinner_init(&spin, engine->gt); 310cf3bd1a0SChris Wilson if (err) 311cf3bd1a0SChris Wilson goto out_ctx; 312cf3bd1a0SChris Wilson 31304adaba8SChris Wilson err = check_whitelist(ce); 314112ed2d3SChris Wilson if (err) { 315112ed2d3SChris Wilson pr_err("Invalid whitelist *before* %s reset!\n", name); 316cf3bd1a0SChris Wilson goto out_spin; 317112ed2d3SChris Wilson } 318112ed2d3SChris Wilson 319112ed2d3SChris Wilson err = switch_to_scratch_context(engine, &spin); 320112ed2d3SChris Wilson if (err) 321cf3bd1a0SChris Wilson goto out_spin; 322112ed2d3SChris Wilson 323cd6a8513SChris Wilson with_intel_runtime_pm(engine->uncore->rpm, wakeref) 324112ed2d3SChris Wilson err = reset(engine); 325112ed2d3SChris Wilson 326112ed2d3SChris Wilson igt_spinner_end(&spin); 327112ed2d3SChris Wilson 328112ed2d3SChris Wilson if (err) { 329112ed2d3SChris Wilson pr_err("%s reset failed\n", name); 330cf3bd1a0SChris Wilson goto out_spin; 331112ed2d3SChris Wilson } 332112ed2d3SChris Wilson 33304adaba8SChris Wilson err = check_whitelist(ce); 334112ed2d3SChris Wilson if (err) { 335112ed2d3SChris Wilson pr_err("Whitelist not preserved in context across %s reset!\n", 336112ed2d3SChris Wilson name); 337cf3bd1a0SChris Wilson goto out_spin; 338112ed2d3SChris Wilson } 339112ed2d3SChris Wilson 34004adaba8SChris Wilson tmp = intel_context_create(engine); 341cf3bd1a0SChris Wilson if (IS_ERR(tmp)) { 342cf3bd1a0SChris Wilson err = PTR_ERR(tmp); 343cf3bd1a0SChris Wilson goto out_spin; 344cf3bd1a0SChris Wilson } 34504adaba8SChris Wilson intel_context_put(ce); 34604adaba8SChris Wilson ce = tmp; 347112ed2d3SChris Wilson 34804adaba8SChris Wilson err = check_whitelist(ce); 349112ed2d3SChris Wilson if (err) { 350112ed2d3SChris Wilson pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 351112ed2d3SChris Wilson name); 352cf3bd1a0SChris Wilson goto out_spin; 353112ed2d3SChris Wilson } 354112ed2d3SChris Wilson 355cf3bd1a0SChris Wilson out_spin: 356cf3bd1a0SChris Wilson igt_spinner_fini(&spin); 357cf3bd1a0SChris Wilson out_ctx: 35804adaba8SChris Wilson intel_context_put(ce); 359112ed2d3SChris Wilson return err; 360112ed2d3SChris Wilson } 361112ed2d3SChris Wilson 362e6ba7648SChris Wilson static struct i915_vma *create_batch(struct i915_address_space *vm) 363112ed2d3SChris Wilson { 364112ed2d3SChris Wilson struct drm_i915_gem_object *obj; 365112ed2d3SChris Wilson struct i915_vma *vma; 366112ed2d3SChris Wilson int err; 367112ed2d3SChris Wilson 368e6ba7648SChris Wilson obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 369112ed2d3SChris Wilson if (IS_ERR(obj)) 370112ed2d3SChris Wilson return ERR_CAST(obj); 371112ed2d3SChris Wilson 372a4e7ccdaSChris Wilson vma = i915_vma_instance(obj, vm, NULL); 373112ed2d3SChris Wilson if (IS_ERR(vma)) { 374112ed2d3SChris Wilson err = PTR_ERR(vma); 375112ed2d3SChris Wilson goto err_obj; 376112ed2d3SChris Wilson } 377112ed2d3SChris Wilson 378112ed2d3SChris Wilson err = i915_vma_pin(vma, 0, 0, PIN_USER); 379112ed2d3SChris Wilson if (err) 380112ed2d3SChris Wilson goto err_obj; 381112ed2d3SChris Wilson 382112ed2d3SChris Wilson return vma; 383112ed2d3SChris Wilson 384112ed2d3SChris Wilson err_obj: 385112ed2d3SChris Wilson i915_gem_object_put(obj); 386112ed2d3SChris Wilson return ERR_PTR(err); 387112ed2d3SChris Wilson } 388112ed2d3SChris Wilson 389112ed2d3SChris Wilson static u32 reg_write(u32 old, u32 new, u32 rsvd) 390112ed2d3SChris Wilson { 391112ed2d3SChris Wilson if (rsvd == 0x0000ffff) { 392112ed2d3SChris Wilson old &= ~(new >> 16); 393112ed2d3SChris Wilson old |= new & (new >> 16); 394112ed2d3SChris Wilson } else { 395112ed2d3SChris Wilson old &= ~rsvd; 396112ed2d3SChris Wilson old |= new & rsvd; 397112ed2d3SChris Wilson } 398112ed2d3SChris Wilson 399112ed2d3SChris Wilson return old; 400112ed2d3SChris Wilson } 401112ed2d3SChris Wilson 402112ed2d3SChris Wilson static bool wo_register(struct intel_engine_cs *engine, u32 reg) 403112ed2d3SChris Wilson { 404112ed2d3SChris Wilson enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 405112ed2d3SChris Wilson int i; 406112ed2d3SChris Wilson 4071e2b7f49SJohn Harrison if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 4081e2b7f49SJohn Harrison RING_FORCE_TO_NONPRIV_ACCESS_WR) 4091e2b7f49SJohn Harrison return true; 4101e2b7f49SJohn Harrison 411112ed2d3SChris Wilson for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 412112ed2d3SChris Wilson if (wo_registers[i].platform == platform && 413112ed2d3SChris Wilson wo_registers[i].reg == reg) 414112ed2d3SChris Wilson return true; 415112ed2d3SChris Wilson } 416112ed2d3SChris Wilson 417112ed2d3SChris Wilson return false; 418112ed2d3SChris Wilson } 419112ed2d3SChris Wilson 420c95ebab1SChris Wilson static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 421c95ebab1SChris Wilson { 422c95ebab1SChris Wilson reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 423c95ebab1SChris Wilson switch (reg) { 424c95ebab1SChris Wilson case 0x358: 425c95ebab1SChris Wilson case 0x35c: 426c95ebab1SChris Wilson case 0x3a8: 427c95ebab1SChris Wilson return true; 428c95ebab1SChris Wilson 429c95ebab1SChris Wilson default: 430c95ebab1SChris Wilson return false; 431c95ebab1SChris Wilson } 432c95ebab1SChris Wilson } 433c95ebab1SChris Wilson 434767662bcSRobert M. Fosha static bool ro_register(u32 reg) 435767662bcSRobert M. Fosha { 4361e2b7f49SJohn Harrison if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 4371e2b7f49SJohn Harrison RING_FORCE_TO_NONPRIV_ACCESS_RD) 438767662bcSRobert M. Fosha return true; 439767662bcSRobert M. Fosha 440767662bcSRobert M. Fosha return false; 441767662bcSRobert M. Fosha } 442767662bcSRobert M. Fosha 443767662bcSRobert M. Fosha static int whitelist_writable_count(struct intel_engine_cs *engine) 444767662bcSRobert M. Fosha { 445767662bcSRobert M. Fosha int count = engine->whitelist.count; 446767662bcSRobert M. Fosha int i; 447767662bcSRobert M. Fosha 448767662bcSRobert M. Fosha for (i = 0; i < engine->whitelist.count; i++) { 449767662bcSRobert M. Fosha u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 450767662bcSRobert M. Fosha 451767662bcSRobert M. Fosha if (ro_register(reg)) 452767662bcSRobert M. Fosha count--; 453767662bcSRobert M. Fosha } 454767662bcSRobert M. Fosha 455767662bcSRobert M. Fosha return count; 456767662bcSRobert M. Fosha } 457767662bcSRobert M. Fosha 458e6ba7648SChris Wilson static int check_dirty_whitelist(struct intel_context *ce) 459112ed2d3SChris Wilson { 460112ed2d3SChris Wilson const u32 values[] = { 461112ed2d3SChris Wilson 0x00000000, 462112ed2d3SChris Wilson 0x01010101, 463112ed2d3SChris Wilson 0x10100101, 464112ed2d3SChris Wilson 0x03030303, 465112ed2d3SChris Wilson 0x30300303, 466112ed2d3SChris Wilson 0x05050505, 467112ed2d3SChris Wilson 0x50500505, 468112ed2d3SChris Wilson 0x0f0f0f0f, 469112ed2d3SChris Wilson 0xf00ff00f, 470112ed2d3SChris Wilson 0x10101010, 471112ed2d3SChris Wilson 0xf0f01010, 472112ed2d3SChris Wilson 0x30303030, 473112ed2d3SChris Wilson 0xa0a03030, 474112ed2d3SChris Wilson 0x50505050, 475112ed2d3SChris Wilson 0xc0c05050, 476112ed2d3SChris Wilson 0xf0f0f0f0, 477112ed2d3SChris Wilson 0x11111111, 478112ed2d3SChris Wilson 0x33333333, 479112ed2d3SChris Wilson 0x55555555, 480112ed2d3SChris Wilson 0x0000ffff, 481112ed2d3SChris Wilson 0x00ff00ff, 482112ed2d3SChris Wilson 0xff0000ff, 483112ed2d3SChris Wilson 0xffff00ff, 484112ed2d3SChris Wilson 0xffffffff, 485112ed2d3SChris Wilson }; 486e6ba7648SChris Wilson struct intel_engine_cs *engine = ce->engine; 487112ed2d3SChris Wilson struct i915_vma *scratch; 488112ed2d3SChris Wilson struct i915_vma *batch; 489a4d86249SChris Wilson int err = 0, i, v, sz; 490112ed2d3SChris Wilson u32 *cs, *results; 491112ed2d3SChris Wilson 492a4d86249SChris Wilson sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 4932a665968SMaarten Lankhorst scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 494112ed2d3SChris Wilson if (IS_ERR(scratch)) 495112ed2d3SChris Wilson return PTR_ERR(scratch); 496112ed2d3SChris Wilson 497e6ba7648SChris Wilson batch = create_batch(ce->vm); 498112ed2d3SChris Wilson if (IS_ERR(batch)) { 499112ed2d3SChris Wilson err = PTR_ERR(batch); 500112ed2d3SChris Wilson goto out_scratch; 501112ed2d3SChris Wilson } 502112ed2d3SChris Wilson 503112ed2d3SChris Wilson for (i = 0; i < engine->whitelist.count; i++) { 504112ed2d3SChris Wilson u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 50574827b53SMaarten Lankhorst struct i915_gem_ww_ctx ww; 506112ed2d3SChris Wilson u64 addr = scratch->node.start; 507112ed2d3SChris Wilson struct i915_request *rq; 508112ed2d3SChris Wilson u32 srm, lrm, rsvd; 509112ed2d3SChris Wilson u32 expect; 510112ed2d3SChris Wilson int idx; 511aee20aaeSJohn Harrison bool ro_reg; 512112ed2d3SChris Wilson 513112ed2d3SChris Wilson if (wo_register(engine, reg)) 514112ed2d3SChris Wilson continue; 515112ed2d3SChris Wilson 516c95ebab1SChris Wilson if (timestamp(engine, reg)) 517c95ebab1SChris Wilson continue; /* timestamps are expected to autoincrement */ 518c95ebab1SChris Wilson 519aee20aaeSJohn Harrison ro_reg = ro_register(reg); 520767662bcSRobert M. Fosha 52174827b53SMaarten Lankhorst i915_gem_ww_ctx_init(&ww, false); 52274827b53SMaarten Lankhorst retry: 52374827b53SMaarten Lankhorst cs = NULL; 52474827b53SMaarten Lankhorst err = i915_gem_object_lock(scratch->obj, &ww); 52574827b53SMaarten Lankhorst if (!err) 52674827b53SMaarten Lankhorst err = i915_gem_object_lock(batch->obj, &ww); 52774827b53SMaarten Lankhorst if (!err) 52874827b53SMaarten Lankhorst err = intel_context_pin_ww(ce, &ww); 52974827b53SMaarten Lankhorst if (err) 53074827b53SMaarten Lankhorst goto out; 53174827b53SMaarten Lankhorst 53274827b53SMaarten Lankhorst cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 53374827b53SMaarten Lankhorst if (IS_ERR(cs)) { 53474827b53SMaarten Lankhorst err = PTR_ERR(cs); 53574827b53SMaarten Lankhorst goto out_ctx; 53674827b53SMaarten Lankhorst } 53774827b53SMaarten Lankhorst 53874827b53SMaarten Lankhorst results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 53974827b53SMaarten Lankhorst if (IS_ERR(results)) { 54074827b53SMaarten Lankhorst err = PTR_ERR(results); 54174827b53SMaarten Lankhorst goto out_unmap_batch; 54274827b53SMaarten Lankhorst } 54374827b53SMaarten Lankhorst 5446b441c62SMika Kuoppala /* Clear non priv flags */ 5456b441c62SMika Kuoppala reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 5466b441c62SMika Kuoppala 547112ed2d3SChris Wilson srm = MI_STORE_REGISTER_MEM; 548112ed2d3SChris Wilson lrm = MI_LOAD_REGISTER_MEM; 549*c816723bSLucas De Marchi if (GRAPHICS_VER(engine->i915) >= 8) 550112ed2d3SChris Wilson lrm++, srm++; 551112ed2d3SChris Wilson 552112ed2d3SChris Wilson pr_debug("%s: Writing garbage to %x\n", 553112ed2d3SChris Wilson engine->name, reg); 554112ed2d3SChris Wilson 555112ed2d3SChris Wilson /* SRM original */ 556112ed2d3SChris Wilson *cs++ = srm; 557112ed2d3SChris Wilson *cs++ = reg; 558112ed2d3SChris Wilson *cs++ = lower_32_bits(addr); 559112ed2d3SChris Wilson *cs++ = upper_32_bits(addr); 560112ed2d3SChris Wilson 561112ed2d3SChris Wilson idx = 1; 562112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 563112ed2d3SChris Wilson /* LRI garbage */ 564112ed2d3SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(1); 565112ed2d3SChris Wilson *cs++ = reg; 566112ed2d3SChris Wilson *cs++ = values[v]; 567112ed2d3SChris Wilson 568112ed2d3SChris Wilson /* SRM result */ 569112ed2d3SChris Wilson *cs++ = srm; 570112ed2d3SChris Wilson *cs++ = reg; 571112ed2d3SChris Wilson *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 572112ed2d3SChris Wilson *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 573112ed2d3SChris Wilson idx++; 574112ed2d3SChris Wilson } 575112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 576112ed2d3SChris Wilson /* LRI garbage */ 577112ed2d3SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(1); 578112ed2d3SChris Wilson *cs++ = reg; 579112ed2d3SChris Wilson *cs++ = ~values[v]; 580112ed2d3SChris Wilson 581112ed2d3SChris Wilson /* SRM result */ 582112ed2d3SChris Wilson *cs++ = srm; 583112ed2d3SChris Wilson *cs++ = reg; 584112ed2d3SChris Wilson *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 585112ed2d3SChris Wilson *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 586112ed2d3SChris Wilson idx++; 587112ed2d3SChris Wilson } 588112ed2d3SChris Wilson GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 589112ed2d3SChris Wilson 590112ed2d3SChris Wilson /* LRM original -- don't leave garbage in the context! */ 591112ed2d3SChris Wilson *cs++ = lrm; 592112ed2d3SChris Wilson *cs++ = reg; 593112ed2d3SChris Wilson *cs++ = lower_32_bits(addr); 594112ed2d3SChris Wilson *cs++ = upper_32_bits(addr); 595112ed2d3SChris Wilson 596112ed2d3SChris Wilson *cs++ = MI_BATCH_BUFFER_END; 597112ed2d3SChris Wilson 598112ed2d3SChris Wilson i915_gem_object_flush_map(batch->obj); 599112ed2d3SChris Wilson i915_gem_object_unpin_map(batch->obj); 600baea429dSTvrtko Ursulin intel_gt_chipset_flush(engine->gt); 60174827b53SMaarten Lankhorst cs = NULL; 602112ed2d3SChris Wilson 60374827b53SMaarten Lankhorst rq = i915_request_create(ce); 604112ed2d3SChris Wilson if (IS_ERR(rq)) { 605112ed2d3SChris Wilson err = PTR_ERR(rq); 60674827b53SMaarten Lankhorst goto out_unmap_scratch; 607112ed2d3SChris Wilson } 608112ed2d3SChris Wilson 609112ed2d3SChris Wilson if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 610112ed2d3SChris Wilson err = engine->emit_init_breadcrumb(rq); 611112ed2d3SChris Wilson if (err) 612112ed2d3SChris Wilson goto err_request; 613112ed2d3SChris Wilson } 614112ed2d3SChris Wilson 6151d5b7773SChris Wilson err = i915_request_await_object(rq, batch->obj, false); 6161d5b7773SChris Wilson if (err == 0) 6171d5b7773SChris Wilson err = i915_vma_move_to_active(batch, rq, 0); 6181d5b7773SChris Wilson if (err) 6191d5b7773SChris Wilson goto err_request; 6201d5b7773SChris Wilson 621bd46aa22SChris Wilson err = i915_request_await_object(rq, scratch->obj, true); 622bd46aa22SChris Wilson if (err == 0) 623bd46aa22SChris Wilson err = i915_vma_move_to_active(scratch, rq, 624bd46aa22SChris Wilson EXEC_OBJECT_WRITE); 625bd46aa22SChris Wilson if (err) 626bd46aa22SChris Wilson goto err_request; 627bd46aa22SChris Wilson 628112ed2d3SChris Wilson err = engine->emit_bb_start(rq, 629112ed2d3SChris Wilson batch->node.start, PAGE_SIZE, 630112ed2d3SChris Wilson 0); 631112ed2d3SChris Wilson if (err) 632112ed2d3SChris Wilson goto err_request; 633112ed2d3SChris Wilson 634112ed2d3SChris Wilson err_request: 63541f0bc49SChris Wilson err = request_add_sync(rq, err); 63641f0bc49SChris Wilson if (err) { 637112ed2d3SChris Wilson pr_err("%s: Futzing %x timedout; cancelling test\n", 638112ed2d3SChris Wilson engine->name, reg); 639bb3d4c9dSChris Wilson intel_gt_set_wedged(engine->gt); 64074827b53SMaarten Lankhorst goto out_unmap_scratch; 641112ed2d3SChris Wilson } 642112ed2d3SChris Wilson 643112ed2d3SChris Wilson GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 644aee20aaeSJohn Harrison if (!ro_reg) { 645aee20aaeSJohn Harrison /* detect write masking */ 646aee20aaeSJohn Harrison rsvd = results[ARRAY_SIZE(values)]; 647112ed2d3SChris Wilson if (!rsvd) { 648112ed2d3SChris Wilson pr_err("%s: Unable to write to whitelisted register %x\n", 649112ed2d3SChris Wilson engine->name, reg); 650112ed2d3SChris Wilson err = -EINVAL; 65174827b53SMaarten Lankhorst goto out_unmap_scratch; 652112ed2d3SChris Wilson } 653cc649a9eSArnd Bergmann } else { 654cc649a9eSArnd Bergmann rsvd = 0; 655aee20aaeSJohn Harrison } 656112ed2d3SChris Wilson 657112ed2d3SChris Wilson expect = results[0]; 658112ed2d3SChris Wilson idx = 1; 659112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 660aee20aaeSJohn Harrison if (ro_reg) 661aee20aaeSJohn Harrison expect = results[0]; 662aee20aaeSJohn Harrison else 663112ed2d3SChris Wilson expect = reg_write(expect, values[v], rsvd); 664aee20aaeSJohn Harrison 665112ed2d3SChris Wilson if (results[idx] != expect) 666112ed2d3SChris Wilson err++; 667112ed2d3SChris Wilson idx++; 668112ed2d3SChris Wilson } 669112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 670aee20aaeSJohn Harrison if (ro_reg) 671aee20aaeSJohn Harrison expect = results[0]; 672aee20aaeSJohn Harrison else 673112ed2d3SChris Wilson expect = reg_write(expect, ~values[v], rsvd); 674aee20aaeSJohn Harrison 675112ed2d3SChris Wilson if (results[idx] != expect) 676112ed2d3SChris Wilson err++; 677112ed2d3SChris Wilson idx++; 678112ed2d3SChris Wilson } 679112ed2d3SChris Wilson if (err) { 680112ed2d3SChris Wilson pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 681112ed2d3SChris Wilson engine->name, err, reg); 682112ed2d3SChris Wilson 683aee20aaeSJohn Harrison if (ro_reg) 684aee20aaeSJohn Harrison pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 685aee20aaeSJohn Harrison engine->name, reg, results[0]); 686aee20aaeSJohn Harrison else 687112ed2d3SChris Wilson pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 688112ed2d3SChris Wilson engine->name, reg, results[0], rsvd); 689112ed2d3SChris Wilson 690112ed2d3SChris Wilson expect = results[0]; 691112ed2d3SChris Wilson idx = 1; 692112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 693112ed2d3SChris Wilson u32 w = values[v]; 694112ed2d3SChris Wilson 695aee20aaeSJohn Harrison if (ro_reg) 696aee20aaeSJohn Harrison expect = results[0]; 697aee20aaeSJohn Harrison else 698112ed2d3SChris Wilson expect = reg_write(expect, w, rsvd); 699112ed2d3SChris Wilson pr_info("Wrote %08x, read %08x, expect %08x\n", 700112ed2d3SChris Wilson w, results[idx], expect); 701112ed2d3SChris Wilson idx++; 702112ed2d3SChris Wilson } 703112ed2d3SChris Wilson for (v = 0; v < ARRAY_SIZE(values); v++) { 704112ed2d3SChris Wilson u32 w = ~values[v]; 705112ed2d3SChris Wilson 706aee20aaeSJohn Harrison if (ro_reg) 707aee20aaeSJohn Harrison expect = results[0]; 708aee20aaeSJohn Harrison else 709112ed2d3SChris Wilson expect = reg_write(expect, w, rsvd); 710112ed2d3SChris Wilson pr_info("Wrote %08x, read %08x, expect %08x\n", 711112ed2d3SChris Wilson w, results[idx], expect); 712112ed2d3SChris Wilson idx++; 713112ed2d3SChris Wilson } 714112ed2d3SChris Wilson 715112ed2d3SChris Wilson err = -EINVAL; 716112ed2d3SChris Wilson } 71774827b53SMaarten Lankhorst out_unmap_scratch: 718112ed2d3SChris Wilson i915_gem_object_unpin_map(scratch->obj); 71974827b53SMaarten Lankhorst out_unmap_batch: 72074827b53SMaarten Lankhorst if (cs) 72174827b53SMaarten Lankhorst i915_gem_object_unpin_map(batch->obj); 72274827b53SMaarten Lankhorst out_ctx: 72374827b53SMaarten Lankhorst intel_context_unpin(ce); 72474827b53SMaarten Lankhorst out: 72574827b53SMaarten Lankhorst if (err == -EDEADLK) { 72674827b53SMaarten Lankhorst err = i915_gem_ww_ctx_backoff(&ww); 72774827b53SMaarten Lankhorst if (!err) 72874827b53SMaarten Lankhorst goto retry; 72974827b53SMaarten Lankhorst } 73074827b53SMaarten Lankhorst i915_gem_ww_ctx_fini(&ww); 731112ed2d3SChris Wilson if (err) 732112ed2d3SChris Wilson break; 733112ed2d3SChris Wilson } 734112ed2d3SChris Wilson 735e6ba7648SChris Wilson if (igt_flush_test(engine->i915)) 736112ed2d3SChris Wilson err = -EIO; 73774827b53SMaarten Lankhorst 738112ed2d3SChris Wilson i915_vma_unpin_and_release(&batch, 0); 739112ed2d3SChris Wilson out_scratch: 740112ed2d3SChris Wilson i915_vma_unpin_and_release(&scratch, 0); 741112ed2d3SChris Wilson return err; 742112ed2d3SChris Wilson } 743112ed2d3SChris Wilson 744112ed2d3SChris Wilson static int live_dirty_whitelist(void *arg) 745112ed2d3SChris Wilson { 746bb3d4c9dSChris Wilson struct intel_gt *gt = arg; 747112ed2d3SChris Wilson struct intel_engine_cs *engine; 748112ed2d3SChris Wilson enum intel_engine_id id; 749112ed2d3SChris Wilson 750112ed2d3SChris Wilson /* Can the user write to the whitelisted registers? */ 751112ed2d3SChris Wilson 752*c816723bSLucas De Marchi if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 753112ed2d3SChris Wilson return 0; 754112ed2d3SChris Wilson 7555d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 756e6ba7648SChris Wilson struct intel_context *ce; 757e6ba7648SChris Wilson int err; 758e6ba7648SChris Wilson 759112ed2d3SChris Wilson if (engine->whitelist.count == 0) 760112ed2d3SChris Wilson continue; 761112ed2d3SChris Wilson 762e6ba7648SChris Wilson ce = intel_context_create(engine); 763e6ba7648SChris Wilson if (IS_ERR(ce)) 764e6ba7648SChris Wilson return PTR_ERR(ce); 765e6ba7648SChris Wilson 766e6ba7648SChris Wilson err = check_dirty_whitelist(ce); 767e6ba7648SChris Wilson intel_context_put(ce); 768112ed2d3SChris Wilson if (err) 769e6ba7648SChris Wilson return err; 770112ed2d3SChris Wilson } 771112ed2d3SChris Wilson 772e6ba7648SChris Wilson return 0; 773112ed2d3SChris Wilson } 774112ed2d3SChris Wilson 775112ed2d3SChris Wilson static int live_reset_whitelist(void *arg) 776112ed2d3SChris Wilson { 777bb3d4c9dSChris Wilson struct intel_gt *gt = arg; 778bb3d4c9dSChris Wilson struct intel_engine_cs *engine; 779bb3d4c9dSChris Wilson enum intel_engine_id id; 780112ed2d3SChris Wilson int err = 0; 781112ed2d3SChris Wilson 782112ed2d3SChris Wilson /* If we reset the gpu, we should not lose the RING_NONPRIV */ 783bb3d4c9dSChris Wilson igt_global_reset_lock(gt); 784112ed2d3SChris Wilson 7855d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 786bb3d4c9dSChris Wilson if (engine->whitelist.count == 0) 787bb3d4c9dSChris Wilson continue; 788112ed2d3SChris Wilson 789bb3d4c9dSChris Wilson if (intel_has_reset_engine(gt)) { 790112ed2d3SChris Wilson err = check_whitelist_across_reset(engine, 791112ed2d3SChris Wilson do_engine_reset, 792112ed2d3SChris Wilson "engine"); 793112ed2d3SChris Wilson if (err) 794112ed2d3SChris Wilson goto out; 795112ed2d3SChris Wilson } 796112ed2d3SChris Wilson 797bb3d4c9dSChris Wilson if (intel_has_gpu_reset(gt)) { 798112ed2d3SChris Wilson err = check_whitelist_across_reset(engine, 799112ed2d3SChris Wilson do_device_reset, 800112ed2d3SChris Wilson "device"); 801112ed2d3SChris Wilson if (err) 802112ed2d3SChris Wilson goto out; 803112ed2d3SChris Wilson } 804bb3d4c9dSChris Wilson } 805112ed2d3SChris Wilson 806112ed2d3SChris Wilson out: 807bb3d4c9dSChris Wilson igt_global_reset_unlock(gt); 808112ed2d3SChris Wilson return err; 809112ed2d3SChris Wilson } 810112ed2d3SChris Wilson 81104adaba8SChris Wilson static int read_whitelisted_registers(struct intel_context *ce, 812112ed2d3SChris Wilson struct i915_vma *results) 813112ed2d3SChris Wilson { 81404adaba8SChris Wilson struct intel_engine_cs *engine = ce->engine; 815112ed2d3SChris Wilson struct i915_request *rq; 816112ed2d3SChris Wilson int i, err = 0; 817112ed2d3SChris Wilson u32 srm, *cs; 818112ed2d3SChris Wilson 81904adaba8SChris Wilson rq = intel_context_create_request(ce); 820112ed2d3SChris Wilson if (IS_ERR(rq)) 821112ed2d3SChris Wilson return PTR_ERR(rq); 822112ed2d3SChris Wilson 823cd9ba7b6SChris Wilson i915_vma_lock(results); 824cd9ba7b6SChris Wilson err = i915_request_await_object(rq, results->obj, true); 825cd9ba7b6SChris Wilson if (err == 0) 826cd9ba7b6SChris Wilson err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 827cd9ba7b6SChris Wilson i915_vma_unlock(results); 828cd9ba7b6SChris Wilson if (err) 829cd9ba7b6SChris Wilson goto err_req; 830cd9ba7b6SChris Wilson 831112ed2d3SChris Wilson srm = MI_STORE_REGISTER_MEM; 832*c816723bSLucas De Marchi if (GRAPHICS_VER(engine->i915) >= 8) 833112ed2d3SChris Wilson srm++; 834112ed2d3SChris Wilson 835112ed2d3SChris Wilson cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 836112ed2d3SChris Wilson if (IS_ERR(cs)) { 837112ed2d3SChris Wilson err = PTR_ERR(cs); 838112ed2d3SChris Wilson goto err_req; 839112ed2d3SChris Wilson } 840112ed2d3SChris Wilson 841112ed2d3SChris Wilson for (i = 0; i < engine->whitelist.count; i++) { 842112ed2d3SChris Wilson u64 offset = results->node.start + sizeof(u32) * i; 843767662bcSRobert M. Fosha u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 844767662bcSRobert M. Fosha 8456b441c62SMika Kuoppala /* Clear non priv flags */ 8466b441c62SMika Kuoppala reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 847112ed2d3SChris Wilson 848112ed2d3SChris Wilson *cs++ = srm; 849767662bcSRobert M. Fosha *cs++ = reg; 850112ed2d3SChris Wilson *cs++ = lower_32_bits(offset); 851112ed2d3SChris Wilson *cs++ = upper_32_bits(offset); 852112ed2d3SChris Wilson } 853112ed2d3SChris Wilson intel_ring_advance(rq, cs); 854112ed2d3SChris Wilson 855112ed2d3SChris Wilson err_req: 85641f0bc49SChris Wilson return request_add_sync(rq, err); 857112ed2d3SChris Wilson } 858112ed2d3SChris Wilson 85904adaba8SChris Wilson static int scrub_whitelisted_registers(struct intel_context *ce) 860112ed2d3SChris Wilson { 86104adaba8SChris Wilson struct intel_engine_cs *engine = ce->engine; 862112ed2d3SChris Wilson struct i915_request *rq; 863112ed2d3SChris Wilson struct i915_vma *batch; 864112ed2d3SChris Wilson int i, err = 0; 865112ed2d3SChris Wilson u32 *cs; 866112ed2d3SChris Wilson 86704adaba8SChris Wilson batch = create_batch(ce->vm); 868112ed2d3SChris Wilson if (IS_ERR(batch)) 869112ed2d3SChris Wilson return PTR_ERR(batch); 870112ed2d3SChris Wilson 87174827b53SMaarten Lankhorst cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 872112ed2d3SChris Wilson if (IS_ERR(cs)) { 873112ed2d3SChris Wilson err = PTR_ERR(cs); 874112ed2d3SChris Wilson goto err_batch; 875112ed2d3SChris Wilson } 876112ed2d3SChris Wilson 877767662bcSRobert M. Fosha *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 878112ed2d3SChris Wilson for (i = 0; i < engine->whitelist.count; i++) { 879767662bcSRobert M. Fosha u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 880767662bcSRobert M. Fosha 881767662bcSRobert M. Fosha if (ro_register(reg)) 882767662bcSRobert M. Fosha continue; 883767662bcSRobert M. Fosha 8846b441c62SMika Kuoppala /* Clear non priv flags */ 8856b441c62SMika Kuoppala reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 8866b441c62SMika Kuoppala 887767662bcSRobert M. Fosha *cs++ = reg; 888112ed2d3SChris Wilson *cs++ = 0xffffffff; 889112ed2d3SChris Wilson } 890112ed2d3SChris Wilson *cs++ = MI_BATCH_BUFFER_END; 891112ed2d3SChris Wilson 892112ed2d3SChris Wilson i915_gem_object_flush_map(batch->obj); 893baea429dSTvrtko Ursulin intel_gt_chipset_flush(engine->gt); 894112ed2d3SChris Wilson 89504adaba8SChris Wilson rq = intel_context_create_request(ce); 896112ed2d3SChris Wilson if (IS_ERR(rq)) { 897112ed2d3SChris Wilson err = PTR_ERR(rq); 898112ed2d3SChris Wilson goto err_unpin; 899112ed2d3SChris Wilson } 900112ed2d3SChris Wilson 901112ed2d3SChris Wilson if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 902112ed2d3SChris Wilson err = engine->emit_init_breadcrumb(rq); 903112ed2d3SChris Wilson if (err) 904112ed2d3SChris Wilson goto err_request; 905112ed2d3SChris Wilson } 906112ed2d3SChris Wilson 9071d5b7773SChris Wilson i915_vma_lock(batch); 9081d5b7773SChris Wilson err = i915_request_await_object(rq, batch->obj, false); 9091d5b7773SChris Wilson if (err == 0) 9101d5b7773SChris Wilson err = i915_vma_move_to_active(batch, rq, 0); 9111d5b7773SChris Wilson i915_vma_unlock(batch); 9121d5b7773SChris Wilson if (err) 9131d5b7773SChris Wilson goto err_request; 9141d5b7773SChris Wilson 915112ed2d3SChris Wilson /* Perform the writes from an unprivileged "user" batch */ 916112ed2d3SChris Wilson err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 917112ed2d3SChris Wilson 918112ed2d3SChris Wilson err_request: 91941f0bc49SChris Wilson err = request_add_sync(rq, err); 920112ed2d3SChris Wilson 921112ed2d3SChris Wilson err_unpin: 922112ed2d3SChris Wilson i915_gem_object_unpin_map(batch->obj); 923112ed2d3SChris Wilson err_batch: 924112ed2d3SChris Wilson i915_vma_unpin_and_release(&batch, 0); 925112ed2d3SChris Wilson return err; 926112ed2d3SChris Wilson } 927112ed2d3SChris Wilson 928112ed2d3SChris Wilson struct regmask { 929112ed2d3SChris Wilson i915_reg_t reg; 9303e6e4c21SLucas De Marchi u8 graphics_ver; 931112ed2d3SChris Wilson }; 932112ed2d3SChris Wilson 933112ed2d3SChris Wilson static bool find_reg(struct drm_i915_private *i915, 934112ed2d3SChris Wilson i915_reg_t reg, 935112ed2d3SChris Wilson const struct regmask *tbl, 936112ed2d3SChris Wilson unsigned long count) 937112ed2d3SChris Wilson { 938112ed2d3SChris Wilson u32 offset = i915_mmio_reg_offset(reg); 939112ed2d3SChris Wilson 940112ed2d3SChris Wilson while (count--) { 9413e6e4c21SLucas De Marchi if (GRAPHICS_VER(i915) == tbl->graphics_ver && 942112ed2d3SChris Wilson i915_mmio_reg_offset(tbl->reg) == offset) 943112ed2d3SChris Wilson return true; 944112ed2d3SChris Wilson tbl++; 945112ed2d3SChris Wilson } 946112ed2d3SChris Wilson 947112ed2d3SChris Wilson return false; 948112ed2d3SChris Wilson } 949112ed2d3SChris Wilson 950112ed2d3SChris Wilson static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 951112ed2d3SChris Wilson { 952112ed2d3SChris Wilson /* Alas, we must pardon some whitelists. Mistakes already made */ 953112ed2d3SChris Wilson static const struct regmask pardon[] = { 9543e6e4c21SLucas De Marchi { GEN9_CTX_PREEMPT_REG, 9 }, 9553e6e4c21SLucas De Marchi { GEN8_L3SQCREG4, 9 }, 956112ed2d3SChris Wilson }; 957112ed2d3SChris Wilson 958112ed2d3SChris Wilson return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 959112ed2d3SChris Wilson } 960112ed2d3SChris Wilson 961112ed2d3SChris Wilson static bool result_eq(struct intel_engine_cs *engine, 962112ed2d3SChris Wilson u32 a, u32 b, i915_reg_t reg) 963112ed2d3SChris Wilson { 964112ed2d3SChris Wilson if (a != b && !pardon_reg(engine->i915, reg)) { 965112ed2d3SChris Wilson pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 966112ed2d3SChris Wilson i915_mmio_reg_offset(reg), a, b); 967112ed2d3SChris Wilson return false; 968112ed2d3SChris Wilson } 969112ed2d3SChris Wilson 970112ed2d3SChris Wilson return true; 971112ed2d3SChris Wilson } 972112ed2d3SChris Wilson 973112ed2d3SChris Wilson static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 974112ed2d3SChris Wilson { 975112ed2d3SChris Wilson /* Some registers do not seem to behave and our writes unreadable */ 976112ed2d3SChris Wilson static const struct regmask wo[] = { 9773e6e4c21SLucas De Marchi { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 978112ed2d3SChris Wilson }; 979112ed2d3SChris Wilson 980112ed2d3SChris Wilson return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 981112ed2d3SChris Wilson } 982112ed2d3SChris Wilson 983112ed2d3SChris Wilson static bool result_neq(struct intel_engine_cs *engine, 984112ed2d3SChris Wilson u32 a, u32 b, i915_reg_t reg) 985112ed2d3SChris Wilson { 986112ed2d3SChris Wilson if (a == b && !writeonly_reg(engine->i915, reg)) { 987112ed2d3SChris Wilson pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 988112ed2d3SChris Wilson i915_mmio_reg_offset(reg), a); 989112ed2d3SChris Wilson return false; 990112ed2d3SChris Wilson } 991112ed2d3SChris Wilson 992112ed2d3SChris Wilson return true; 993112ed2d3SChris Wilson } 994112ed2d3SChris Wilson 995112ed2d3SChris Wilson static int 996112ed2d3SChris Wilson check_whitelisted_registers(struct intel_engine_cs *engine, 997112ed2d3SChris Wilson struct i915_vma *A, 998112ed2d3SChris Wilson struct i915_vma *B, 999112ed2d3SChris Wilson bool (*fn)(struct intel_engine_cs *engine, 1000112ed2d3SChris Wilson u32 a, u32 b, 1001112ed2d3SChris Wilson i915_reg_t reg)) 1002112ed2d3SChris Wilson { 1003112ed2d3SChris Wilson u32 *a, *b; 1004112ed2d3SChris Wilson int i, err; 1005112ed2d3SChris Wilson 100674827b53SMaarten Lankhorst a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1007112ed2d3SChris Wilson if (IS_ERR(a)) 1008112ed2d3SChris Wilson return PTR_ERR(a); 1009112ed2d3SChris Wilson 101074827b53SMaarten Lankhorst b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1011112ed2d3SChris Wilson if (IS_ERR(b)) { 1012112ed2d3SChris Wilson err = PTR_ERR(b); 1013112ed2d3SChris Wilson goto err_a; 1014112ed2d3SChris Wilson } 1015112ed2d3SChris Wilson 1016112ed2d3SChris Wilson err = 0; 1017112ed2d3SChris Wilson for (i = 0; i < engine->whitelist.count; i++) { 1018361b6905SLionel Landwerlin const struct i915_wa *wa = &engine->whitelist.list[i]; 1019361b6905SLionel Landwerlin 10201e2b7f49SJohn Harrison if (i915_mmio_reg_offset(wa->reg) & 10211e2b7f49SJohn Harrison RING_FORCE_TO_NONPRIV_ACCESS_RD) 1022361b6905SLionel Landwerlin continue; 1023361b6905SLionel Landwerlin 1024361b6905SLionel Landwerlin if (!fn(engine, a[i], b[i], wa->reg)) 1025112ed2d3SChris Wilson err = -EINVAL; 1026112ed2d3SChris Wilson } 1027112ed2d3SChris Wilson 1028112ed2d3SChris Wilson i915_gem_object_unpin_map(B->obj); 1029112ed2d3SChris Wilson err_a: 1030112ed2d3SChris Wilson i915_gem_object_unpin_map(A->obj); 1031112ed2d3SChris Wilson return err; 1032112ed2d3SChris Wilson } 1033112ed2d3SChris Wilson 1034112ed2d3SChris Wilson static int live_isolated_whitelist(void *arg) 1035112ed2d3SChris Wilson { 1036bb3d4c9dSChris Wilson struct intel_gt *gt = arg; 1037112ed2d3SChris Wilson struct { 1038112ed2d3SChris Wilson struct i915_vma *scratch[2]; 1039112ed2d3SChris Wilson } client[2] = {}; 1040112ed2d3SChris Wilson struct intel_engine_cs *engine; 1041112ed2d3SChris Wilson enum intel_engine_id id; 1042112ed2d3SChris Wilson int i, err = 0; 1043112ed2d3SChris Wilson 1044112ed2d3SChris Wilson /* 1045112ed2d3SChris Wilson * Check that a write into a whitelist register works, but 1046112ed2d3SChris Wilson * invisible to a second context. 1047112ed2d3SChris Wilson */ 1048112ed2d3SChris Wilson 1049bb3d4c9dSChris Wilson if (!intel_engines_has_context_isolation(gt->i915)) 1050112ed2d3SChris Wilson return 0; 1051112ed2d3SChris Wilson 1052112ed2d3SChris Wilson for (i = 0; i < ARRAY_SIZE(client); i++) { 1053a4d86249SChris Wilson client[i].scratch[0] = 10542a665968SMaarten Lankhorst __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1055112ed2d3SChris Wilson if (IS_ERR(client[i].scratch[0])) { 1056112ed2d3SChris Wilson err = PTR_ERR(client[i].scratch[0]); 1057112ed2d3SChris Wilson goto err; 1058112ed2d3SChris Wilson } 1059112ed2d3SChris Wilson 1060a4d86249SChris Wilson client[i].scratch[1] = 10612a665968SMaarten Lankhorst __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1062112ed2d3SChris Wilson if (IS_ERR(client[i].scratch[1])) { 1063112ed2d3SChris Wilson err = PTR_ERR(client[i].scratch[1]); 1064112ed2d3SChris Wilson i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1065112ed2d3SChris Wilson goto err; 1066112ed2d3SChris Wilson } 1067112ed2d3SChris Wilson } 1068112ed2d3SChris Wilson 10695d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 107004adaba8SChris Wilson struct intel_context *ce[2]; 107104adaba8SChris Wilson 1072bb3d4c9dSChris Wilson if (!engine->kernel_context->vm) 1073bb3d4c9dSChris Wilson continue; 1074bb3d4c9dSChris Wilson 1075767662bcSRobert M. Fosha if (!whitelist_writable_count(engine)) 1076112ed2d3SChris Wilson continue; 1077112ed2d3SChris Wilson 107804adaba8SChris Wilson ce[0] = intel_context_create(engine); 107904adaba8SChris Wilson if (IS_ERR(ce[0])) { 108004adaba8SChris Wilson err = PTR_ERR(ce[0]); 108104adaba8SChris Wilson break; 108204adaba8SChris Wilson } 108304adaba8SChris Wilson ce[1] = intel_context_create(engine); 108404adaba8SChris Wilson if (IS_ERR(ce[1])) { 108504adaba8SChris Wilson err = PTR_ERR(ce[1]); 108604adaba8SChris Wilson intel_context_put(ce[0]); 108704adaba8SChris Wilson break; 108804adaba8SChris Wilson } 108904adaba8SChris Wilson 1090112ed2d3SChris Wilson /* Read default values */ 109104adaba8SChris Wilson err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1092112ed2d3SChris Wilson if (err) 109304adaba8SChris Wilson goto err_ce; 1094112ed2d3SChris Wilson 1095112ed2d3SChris Wilson /* Try to overwrite registers (should only affect ctx0) */ 109604adaba8SChris Wilson err = scrub_whitelisted_registers(ce[0]); 1097112ed2d3SChris Wilson if (err) 109804adaba8SChris Wilson goto err_ce; 1099112ed2d3SChris Wilson 1100112ed2d3SChris Wilson /* Read values from ctx1, we expect these to be defaults */ 110104adaba8SChris Wilson err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1102112ed2d3SChris Wilson if (err) 110304adaba8SChris Wilson goto err_ce; 1104112ed2d3SChris Wilson 1105112ed2d3SChris Wilson /* Verify that both reads return the same default values */ 1106112ed2d3SChris Wilson err = check_whitelisted_registers(engine, 1107112ed2d3SChris Wilson client[0].scratch[0], 1108112ed2d3SChris Wilson client[1].scratch[0], 1109112ed2d3SChris Wilson result_eq); 1110112ed2d3SChris Wilson if (err) 111104adaba8SChris Wilson goto err_ce; 1112112ed2d3SChris Wilson 1113112ed2d3SChris Wilson /* Read back the updated values in ctx0 */ 111404adaba8SChris Wilson err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1115112ed2d3SChris Wilson if (err) 111604adaba8SChris Wilson goto err_ce; 1117112ed2d3SChris Wilson 1118112ed2d3SChris Wilson /* User should be granted privilege to overwhite regs */ 1119112ed2d3SChris Wilson err = check_whitelisted_registers(engine, 1120112ed2d3SChris Wilson client[0].scratch[0], 1121112ed2d3SChris Wilson client[0].scratch[1], 1122112ed2d3SChris Wilson result_neq); 112304adaba8SChris Wilson err_ce: 112404adaba8SChris Wilson intel_context_put(ce[1]); 112504adaba8SChris Wilson intel_context_put(ce[0]); 1126112ed2d3SChris Wilson if (err) 112704adaba8SChris Wilson break; 1128112ed2d3SChris Wilson } 1129112ed2d3SChris Wilson 1130112ed2d3SChris Wilson err: 1131112ed2d3SChris Wilson for (i = 0; i < ARRAY_SIZE(client); i++) { 1132112ed2d3SChris Wilson i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1133112ed2d3SChris Wilson i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1134112ed2d3SChris Wilson } 1135112ed2d3SChris Wilson 1136bb3d4c9dSChris Wilson if (igt_flush_test(gt->i915)) 1137112ed2d3SChris Wilson err = -EIO; 1138112ed2d3SChris Wilson 1139112ed2d3SChris Wilson return err; 1140112ed2d3SChris Wilson } 1141112ed2d3SChris Wilson 1142fde93886STvrtko Ursulin static bool 114304adaba8SChris Wilson verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1144fde93886STvrtko Ursulin const char *str) 1145112ed2d3SChris Wilson { 114604adaba8SChris Wilson struct intel_engine_cs *engine; 114704adaba8SChris Wilson enum intel_engine_id id; 1148112ed2d3SChris Wilson bool ok = true; 1149112ed2d3SChris Wilson 115004adaba8SChris Wilson ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str); 1151112ed2d3SChris Wilson 115204adaba8SChris Wilson for_each_engine(engine, gt, id) { 115304adaba8SChris Wilson struct intel_context *ce; 115404adaba8SChris Wilson 115504adaba8SChris Wilson ce = intel_context_create(engine); 115604adaba8SChris Wilson if (IS_ERR(ce)) 115704adaba8SChris Wilson return false; 1158fde93886STvrtko Ursulin 1159fde93886STvrtko Ursulin ok &= engine_wa_list_verify(ce, 1160112ed2d3SChris Wilson &lists->engine[id].wa_list, 1161112ed2d3SChris Wilson str) == 0; 1162fde93886STvrtko Ursulin 1163fde93886STvrtko Ursulin ok &= engine_wa_list_verify(ce, 1164fde93886STvrtko Ursulin &lists->engine[id].ctx_wa_list, 1165fde93886STvrtko Ursulin str) == 0; 116604adaba8SChris Wilson 116704adaba8SChris Wilson intel_context_put(ce); 1168112ed2d3SChris Wilson } 1169112ed2d3SChris Wilson 1170112ed2d3SChris Wilson return ok; 1171112ed2d3SChris Wilson } 1172112ed2d3SChris Wilson 1173112ed2d3SChris Wilson static int 1174fde93886STvrtko Ursulin live_gpu_reset_workarounds(void *arg) 1175112ed2d3SChris Wilson { 1176bb3d4c9dSChris Wilson struct intel_gt *gt = arg; 1177112ed2d3SChris Wilson intel_wakeref_t wakeref; 1178112ed2d3SChris Wilson struct wa_lists lists; 1179112ed2d3SChris Wilson bool ok; 1180112ed2d3SChris Wilson 1181bb3d4c9dSChris Wilson if (!intel_has_gpu_reset(gt)) 1182112ed2d3SChris Wilson return 0; 1183112ed2d3SChris Wilson 1184112ed2d3SChris Wilson pr_info("Verifying after GPU reset...\n"); 1185112ed2d3SChris Wilson 1186bb3d4c9dSChris Wilson igt_global_reset_lock(gt); 1187bb3d4c9dSChris Wilson wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1188112ed2d3SChris Wilson 1189bb3d4c9dSChris Wilson reference_lists_init(gt, &lists); 1190112ed2d3SChris Wilson 119104adaba8SChris Wilson ok = verify_wa_lists(gt, &lists, "before reset"); 1192112ed2d3SChris Wilson if (!ok) 1193112ed2d3SChris Wilson goto out; 1194112ed2d3SChris Wilson 1195bb3d4c9dSChris Wilson intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1196112ed2d3SChris Wilson 119704adaba8SChris Wilson ok = verify_wa_lists(gt, &lists, "after reset"); 1198112ed2d3SChris Wilson 1199112ed2d3SChris Wilson out: 1200bb3d4c9dSChris Wilson reference_lists_fini(gt, &lists); 1201bb3d4c9dSChris Wilson intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1202bb3d4c9dSChris Wilson igt_global_reset_unlock(gt); 1203112ed2d3SChris Wilson 1204112ed2d3SChris Wilson return ok ? 0 : -ESRCH; 1205112ed2d3SChris Wilson } 1206112ed2d3SChris Wilson 1207112ed2d3SChris Wilson static int 1208fde93886STvrtko Ursulin live_engine_reset_workarounds(void *arg) 1209112ed2d3SChris Wilson { 1210bb3d4c9dSChris Wilson struct intel_gt *gt = arg; 121104adaba8SChris Wilson struct intel_engine_cs *engine; 121204adaba8SChris Wilson enum intel_engine_id id; 1213f277bc0cSChris Wilson struct intel_context *ce; 1214112ed2d3SChris Wilson struct igt_spinner spin; 1215112ed2d3SChris Wilson struct i915_request *rq; 1216112ed2d3SChris Wilson intel_wakeref_t wakeref; 1217112ed2d3SChris Wilson struct wa_lists lists; 1218112ed2d3SChris Wilson int ret = 0; 1219112ed2d3SChris Wilson 1220bb3d4c9dSChris Wilson if (!intel_has_reset_engine(gt)) 1221112ed2d3SChris Wilson return 0; 1222112ed2d3SChris Wilson 1223bb3d4c9dSChris Wilson igt_global_reset_lock(gt); 1224bb3d4c9dSChris Wilson wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1225112ed2d3SChris Wilson 1226bb3d4c9dSChris Wilson reference_lists_init(gt, &lists); 1227112ed2d3SChris Wilson 122804adaba8SChris Wilson for_each_engine(engine, gt, id) { 1229112ed2d3SChris Wilson bool ok; 1230112ed2d3SChris Wilson 1231112ed2d3SChris Wilson pr_info("Verifying after %s reset...\n", engine->name); 123204adaba8SChris Wilson ce = intel_context_create(engine); 123304adaba8SChris Wilson if (IS_ERR(ce)) { 123404adaba8SChris Wilson ret = PTR_ERR(ce); 123504adaba8SChris Wilson break; 123604adaba8SChris Wilson } 1237112ed2d3SChris Wilson 123804adaba8SChris Wilson ok = verify_wa_lists(gt, &lists, "before reset"); 1239112ed2d3SChris Wilson if (!ok) { 1240112ed2d3SChris Wilson ret = -ESRCH; 1241112ed2d3SChris Wilson goto err; 1242112ed2d3SChris Wilson } 1243112ed2d3SChris Wilson 124431052811SChris Wilson ret = intel_engine_reset(engine, "live_workarounds:idle"); 124531052811SChris Wilson if (ret) { 124631052811SChris Wilson pr_err("%s: Reset failed while idle\n", engine->name); 124731052811SChris Wilson goto err; 124831052811SChris Wilson } 1249112ed2d3SChris Wilson 125004adaba8SChris Wilson ok = verify_wa_lists(gt, &lists, "after idle reset"); 1251112ed2d3SChris Wilson if (!ok) { 1252112ed2d3SChris Wilson ret = -ESRCH; 1253112ed2d3SChris Wilson goto err; 1254112ed2d3SChris Wilson } 1255112ed2d3SChris Wilson 1256f277bc0cSChris Wilson ret = igt_spinner_init(&spin, engine->gt); 1257112ed2d3SChris Wilson if (ret) 1258112ed2d3SChris Wilson goto err; 1259112ed2d3SChris Wilson 1260f277bc0cSChris Wilson rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1261112ed2d3SChris Wilson if (IS_ERR(rq)) { 1262112ed2d3SChris Wilson ret = PTR_ERR(rq); 1263112ed2d3SChris Wilson igt_spinner_fini(&spin); 1264112ed2d3SChris Wilson goto err; 1265112ed2d3SChris Wilson } 1266112ed2d3SChris Wilson 126741f0bc49SChris Wilson ret = request_add_spin(rq, &spin); 126841f0bc49SChris Wilson if (ret) { 126931052811SChris Wilson pr_err("%s: Spinner failed to start\n", engine->name); 1270112ed2d3SChris Wilson igt_spinner_fini(&spin); 1271112ed2d3SChris Wilson goto err; 1272112ed2d3SChris Wilson } 1273112ed2d3SChris Wilson 127431052811SChris Wilson ret = intel_engine_reset(engine, "live_workarounds:active"); 127531052811SChris Wilson if (ret) { 127631052811SChris Wilson pr_err("%s: Reset failed on an active spinner\n", 127731052811SChris Wilson engine->name); 127831052811SChris Wilson igt_spinner_fini(&spin); 127931052811SChris Wilson goto err; 128031052811SChris Wilson } 1281112ed2d3SChris Wilson 1282112ed2d3SChris Wilson igt_spinner_end(&spin); 1283112ed2d3SChris Wilson igt_spinner_fini(&spin); 1284112ed2d3SChris Wilson 128504adaba8SChris Wilson ok = verify_wa_lists(gt, &lists, "after busy reset"); 1286112ed2d3SChris Wilson if (!ok) { 1287112ed2d3SChris Wilson ret = -ESRCH; 1288112ed2d3SChris Wilson goto err; 1289112ed2d3SChris Wilson } 129004adaba8SChris Wilson 1291112ed2d3SChris Wilson err: 129204adaba8SChris Wilson intel_context_put(ce); 129304adaba8SChris Wilson if (ret) 129404adaba8SChris Wilson break; 129504adaba8SChris Wilson } 129604adaba8SChris Wilson 1297bb3d4c9dSChris Wilson reference_lists_fini(gt, &lists); 1298bb3d4c9dSChris Wilson intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1299bb3d4c9dSChris Wilson igt_global_reset_unlock(gt); 1300112ed2d3SChris Wilson 1301bb3d4c9dSChris Wilson igt_flush_test(gt->i915); 1302112ed2d3SChris Wilson 1303112ed2d3SChris Wilson return ret; 1304112ed2d3SChris Wilson } 1305112ed2d3SChris Wilson 1306112ed2d3SChris Wilson int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1307112ed2d3SChris Wilson { 1308112ed2d3SChris Wilson static const struct i915_subtest tests[] = { 1309112ed2d3SChris Wilson SUBTEST(live_dirty_whitelist), 1310112ed2d3SChris Wilson SUBTEST(live_reset_whitelist), 1311112ed2d3SChris Wilson SUBTEST(live_isolated_whitelist), 1312fde93886STvrtko Ursulin SUBTEST(live_gpu_reset_workarounds), 1313fde93886STvrtko Ursulin SUBTEST(live_engine_reset_workarounds), 1314112ed2d3SChris Wilson }; 1315112ed2d3SChris Wilson 1316cb823ed9SChris Wilson if (intel_gt_is_wedged(&i915->gt)) 1317112ed2d3SChris Wilson return 0; 1318112ed2d3SChris Wilson 1319bb3d4c9dSChris Wilson return intel_gt_live_subtests(tests, &i915->gt); 1320112ed2d3SChris Wilson } 1321