124f90d66SChris Wilson // SPDX-License-Identifier: MIT 2f0c02c1bSTvrtko Ursulin /* 3f0c02c1bSTvrtko Ursulin * Copyright © 2017-2018 Intel Corporation 4f0c02c1bSTvrtko Ursulin */ 5f0c02c1bSTvrtko Ursulin 6f0c02c1bSTvrtko Ursulin #include <linux/prime_numbers.h> 7f0c02c1bSTvrtko Ursulin 8bb5e4397SChris Wilson #include "intel_context.h" 9bb5e4397SChris Wilson #include "intel_engine_heartbeat.h" 107e805762SChris Wilson #include "intel_engine_pm.h" 1145233ab2SChris Wilson #include "intel_gpu_commands.h" 12cb823ed9SChris Wilson #include "intel_gt.h" 1366101975SChris Wilson #include "intel_gt_requests.h" 142871ea85SChris Wilson #include "intel_ring.h" 151b90e4a4SChris Wilson #include "selftest_engine_heartbeat.h" 16f0c02c1bSTvrtko Ursulin 17f0c02c1bSTvrtko Ursulin #include "../selftests/i915_random.h" 18f0c02c1bSTvrtko Ursulin #include "../i915_selftest.h" 19f0c02c1bSTvrtko Ursulin 206e7a21e7SChris Wilson #include "selftests/igt_flush_test.h" 216e7a21e7SChris Wilson #include "selftests/lib_sw_fence.h" 226e7a21e7SChris Wilson #include "selftests/mock_gem_device.h" 23f0c02c1bSTvrtko Ursulin #include "selftests/mock_timeline.h" 24f0c02c1bSTvrtko Ursulin 25f0c02c1bSTvrtko Ursulin static struct page *hwsp_page(struct intel_timeline *tl) 26f0c02c1bSTvrtko Ursulin { 27f0c02c1bSTvrtko Ursulin struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; 28f0c02c1bSTvrtko Ursulin 29f0c02c1bSTvrtko Ursulin GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 30f0c02c1bSTvrtko Ursulin return sg_page(obj->mm.pages->sgl); 31f0c02c1bSTvrtko Ursulin } 32f0c02c1bSTvrtko Ursulin 33f0c02c1bSTvrtko Ursulin static unsigned long hwsp_cacheline(struct intel_timeline *tl) 34f0c02c1bSTvrtko Ursulin { 35f0c02c1bSTvrtko Ursulin unsigned long address = (unsigned long)page_address(hwsp_page(tl)); 36f0c02c1bSTvrtko Ursulin 3712ca695dSMaarten Lankhorst return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES; 38f0c02c1bSTvrtko Ursulin } 39f0c02c1bSTvrtko Ursulin 401060974cSMaarten Lankhorst static int selftest_tl_pin(struct intel_timeline *tl) 411060974cSMaarten Lankhorst { 421060974cSMaarten Lankhorst struct i915_gem_ww_ctx ww; 431060974cSMaarten Lankhorst int err; 441060974cSMaarten Lankhorst 451060974cSMaarten Lankhorst i915_gem_ww_ctx_init(&ww, false); 461060974cSMaarten Lankhorst retry: 471060974cSMaarten Lankhorst err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww); 481060974cSMaarten Lankhorst if (!err) 491060974cSMaarten Lankhorst err = intel_timeline_pin(tl, &ww); 501060974cSMaarten Lankhorst 511060974cSMaarten Lankhorst if (err == -EDEADLK) { 521060974cSMaarten Lankhorst err = i915_gem_ww_ctx_backoff(&ww); 531060974cSMaarten Lankhorst if (!err) 541060974cSMaarten Lankhorst goto retry; 551060974cSMaarten Lankhorst } 561060974cSMaarten Lankhorst i915_gem_ww_ctx_fini(&ww); 571060974cSMaarten Lankhorst return err; 581060974cSMaarten Lankhorst } 591060974cSMaarten Lankhorst 6012ca695dSMaarten Lankhorst /* Only half of seqno's are usable, see __intel_timeline_get_seqno() */ 6112ca695dSMaarten Lankhorst #define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2) 62f0c02c1bSTvrtko Ursulin 63f0c02c1bSTvrtko Ursulin struct mock_hwsp_freelist { 645f65d5a6SChris Wilson struct intel_gt *gt; 65f0c02c1bSTvrtko Ursulin struct radix_tree_root cachelines; 66f0c02c1bSTvrtko Ursulin struct intel_timeline **history; 67f0c02c1bSTvrtko Ursulin unsigned long count, max; 68f0c02c1bSTvrtko Ursulin struct rnd_state prng; 69f0c02c1bSTvrtko Ursulin }; 70f0c02c1bSTvrtko Ursulin 71f0c02c1bSTvrtko Ursulin enum { 72f0c02c1bSTvrtko Ursulin SHUFFLE = BIT(0), 73f0c02c1bSTvrtko Ursulin }; 74f0c02c1bSTvrtko Ursulin 75f0c02c1bSTvrtko Ursulin static void __mock_hwsp_record(struct mock_hwsp_freelist *state, 76f0c02c1bSTvrtko Ursulin unsigned int idx, 77f0c02c1bSTvrtko Ursulin struct intel_timeline *tl) 78f0c02c1bSTvrtko Ursulin { 79f0c02c1bSTvrtko Ursulin tl = xchg(&state->history[idx], tl); 80f0c02c1bSTvrtko Ursulin if (tl) { 81f0c02c1bSTvrtko Ursulin radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); 822c8ab333SMaarten Lankhorst intel_timeline_unpin(tl); 83f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 84f0c02c1bSTvrtko Ursulin } 85f0c02c1bSTvrtko Ursulin } 86f0c02c1bSTvrtko Ursulin 87f0c02c1bSTvrtko Ursulin static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, 88f0c02c1bSTvrtko Ursulin unsigned int count, 89f0c02c1bSTvrtko Ursulin unsigned int flags) 90f0c02c1bSTvrtko Ursulin { 91f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 92f0c02c1bSTvrtko Ursulin unsigned int idx; 93f0c02c1bSTvrtko Ursulin 94f0c02c1bSTvrtko Ursulin while (count--) { 95f0c02c1bSTvrtko Ursulin unsigned long cacheline; 96f0c02c1bSTvrtko Ursulin int err; 97f0c02c1bSTvrtko Ursulin 98d1bf5dd8SChris Wilson tl = intel_timeline_create(state->gt); 99f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) 100f0c02c1bSTvrtko Ursulin return PTR_ERR(tl); 101f0c02c1bSTvrtko Ursulin 1021060974cSMaarten Lankhorst err = selftest_tl_pin(tl); 1032c8ab333SMaarten Lankhorst if (err) { 1042c8ab333SMaarten Lankhorst intel_timeline_put(tl); 1052c8ab333SMaarten Lankhorst return err; 1062c8ab333SMaarten Lankhorst } 1072c8ab333SMaarten Lankhorst 108f0c02c1bSTvrtko Ursulin cacheline = hwsp_cacheline(tl); 109f0c02c1bSTvrtko Ursulin err = radix_tree_insert(&state->cachelines, cacheline, tl); 110f0c02c1bSTvrtko Ursulin if (err) { 111f0c02c1bSTvrtko Ursulin if (err == -EEXIST) { 112f0c02c1bSTvrtko Ursulin pr_err("HWSP cacheline %lu already used; duplicate allocation!\n", 113f0c02c1bSTvrtko Ursulin cacheline); 114f0c02c1bSTvrtko Ursulin } 1152c8ab333SMaarten Lankhorst intel_timeline_unpin(tl); 116f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 117f0c02c1bSTvrtko Ursulin return err; 118f0c02c1bSTvrtko Ursulin } 119f0c02c1bSTvrtko Ursulin 120f0c02c1bSTvrtko Ursulin idx = state->count++ % state->max; 121f0c02c1bSTvrtko Ursulin __mock_hwsp_record(state, idx, tl); 122f0c02c1bSTvrtko Ursulin } 123f0c02c1bSTvrtko Ursulin 124f0c02c1bSTvrtko Ursulin if (flags & SHUFFLE) 125f0c02c1bSTvrtko Ursulin i915_prandom_shuffle(state->history, 126f0c02c1bSTvrtko Ursulin sizeof(*state->history), 127f0c02c1bSTvrtko Ursulin min(state->count, state->max), 128f0c02c1bSTvrtko Ursulin &state->prng); 129f0c02c1bSTvrtko Ursulin 130f0c02c1bSTvrtko Ursulin count = i915_prandom_u32_max_state(min(state->count, state->max), 131f0c02c1bSTvrtko Ursulin &state->prng); 132f0c02c1bSTvrtko Ursulin while (count--) { 133f0c02c1bSTvrtko Ursulin idx = --state->count % state->max; 134f0c02c1bSTvrtko Ursulin __mock_hwsp_record(state, idx, NULL); 135f0c02c1bSTvrtko Ursulin } 136f0c02c1bSTvrtko Ursulin 137f0c02c1bSTvrtko Ursulin return 0; 138f0c02c1bSTvrtko Ursulin } 139f0c02c1bSTvrtko Ursulin 140f0c02c1bSTvrtko Ursulin static int mock_hwsp_freelist(void *arg) 141f0c02c1bSTvrtko Ursulin { 142f0c02c1bSTvrtko Ursulin struct mock_hwsp_freelist state; 1435f65d5a6SChris Wilson struct drm_i915_private *i915; 144f0c02c1bSTvrtko Ursulin const struct { 145f0c02c1bSTvrtko Ursulin const char *name; 146f0c02c1bSTvrtko Ursulin unsigned int flags; 147f0c02c1bSTvrtko Ursulin } phases[] = { 148f0c02c1bSTvrtko Ursulin { "linear", 0 }, 149f0c02c1bSTvrtko Ursulin { "shuffled", SHUFFLE }, 150f0c02c1bSTvrtko Ursulin { }, 151f0c02c1bSTvrtko Ursulin }, *p; 152f0c02c1bSTvrtko Ursulin unsigned int na; 153f0c02c1bSTvrtko Ursulin int err = 0; 154f0c02c1bSTvrtko Ursulin 1555f65d5a6SChris Wilson i915 = mock_gem_device(); 1565f65d5a6SChris Wilson if (!i915) 1575f65d5a6SChris Wilson return -ENOMEM; 1585f65d5a6SChris Wilson 159f0c02c1bSTvrtko Ursulin INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL); 160f0c02c1bSTvrtko Ursulin state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed); 161f0c02c1bSTvrtko Ursulin 162*c14adcbdSMichał Winiarski state.gt = to_gt(i915); 163f0c02c1bSTvrtko Ursulin 164f0c02c1bSTvrtko Ursulin /* 165f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check that their HWSP do not overlap. 166f0c02c1bSTvrtko Ursulin * Free some, and try again. 167f0c02c1bSTvrtko Ursulin */ 168f0c02c1bSTvrtko Ursulin 169f0c02c1bSTvrtko Ursulin state.max = PAGE_SIZE / sizeof(*state.history); 170f0c02c1bSTvrtko Ursulin state.count = 0; 171f0c02c1bSTvrtko Ursulin state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL); 172f0c02c1bSTvrtko Ursulin if (!state.history) { 173f0c02c1bSTvrtko Ursulin err = -ENOMEM; 174f0c02c1bSTvrtko Ursulin goto err_put; 175f0c02c1bSTvrtko Ursulin } 176f0c02c1bSTvrtko Ursulin 177f0c02c1bSTvrtko Ursulin for (p = phases; p->name; p++) { 178f0c02c1bSTvrtko Ursulin pr_debug("%s(%s)\n", __func__, p->name); 179f0c02c1bSTvrtko Ursulin for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) { 180f0c02c1bSTvrtko Ursulin err = __mock_hwsp_timeline(&state, na, p->flags); 181f0c02c1bSTvrtko Ursulin if (err) 182f0c02c1bSTvrtko Ursulin goto out; 183f0c02c1bSTvrtko Ursulin } 184f0c02c1bSTvrtko Ursulin } 185f0c02c1bSTvrtko Ursulin 186f0c02c1bSTvrtko Ursulin out: 187f0c02c1bSTvrtko Ursulin for (na = 0; na < state.max; na++) 188f0c02c1bSTvrtko Ursulin __mock_hwsp_record(&state, na, NULL); 189f0c02c1bSTvrtko Ursulin kfree(state.history); 190f0c02c1bSTvrtko Ursulin err_put: 19182be0d75SDaniel Vetter mock_destroy_device(i915); 192f0c02c1bSTvrtko Ursulin return err; 193f0c02c1bSTvrtko Ursulin } 194f0c02c1bSTvrtko Ursulin 195f0c02c1bSTvrtko Ursulin struct __igt_sync { 196f0c02c1bSTvrtko Ursulin const char *name; 197f0c02c1bSTvrtko Ursulin u32 seqno; 198f0c02c1bSTvrtko Ursulin bool expected; 199f0c02c1bSTvrtko Ursulin bool set; 200f0c02c1bSTvrtko Ursulin }; 201f0c02c1bSTvrtko Ursulin 202f0c02c1bSTvrtko Ursulin static int __igt_sync(struct intel_timeline *tl, 203f0c02c1bSTvrtko Ursulin u64 ctx, 204f0c02c1bSTvrtko Ursulin const struct __igt_sync *p, 205f0c02c1bSTvrtko Ursulin const char *name) 206f0c02c1bSTvrtko Ursulin { 207f0c02c1bSTvrtko Ursulin int ret; 208f0c02c1bSTvrtko Ursulin 209f0c02c1bSTvrtko Ursulin if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) { 210f0c02c1bSTvrtko Ursulin pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n", 211f0c02c1bSTvrtko Ursulin name, p->name, ctx, p->seqno, yesno(p->expected)); 212f0c02c1bSTvrtko Ursulin return -EINVAL; 213f0c02c1bSTvrtko Ursulin } 214f0c02c1bSTvrtko Ursulin 215f0c02c1bSTvrtko Ursulin if (p->set) { 216f0c02c1bSTvrtko Ursulin ret = __intel_timeline_sync_set(tl, ctx, p->seqno); 217f0c02c1bSTvrtko Ursulin if (ret) 218f0c02c1bSTvrtko Ursulin return ret; 219f0c02c1bSTvrtko Ursulin } 220f0c02c1bSTvrtko Ursulin 221f0c02c1bSTvrtko Ursulin return 0; 222f0c02c1bSTvrtko Ursulin } 223f0c02c1bSTvrtko Ursulin 224f0c02c1bSTvrtko Ursulin static int igt_sync(void *arg) 225f0c02c1bSTvrtko Ursulin { 226f0c02c1bSTvrtko Ursulin const struct __igt_sync pass[] = { 227f0c02c1bSTvrtko Ursulin { "unset", 0, false, false }, 228f0c02c1bSTvrtko Ursulin { "new", 0, false, true }, 229f0c02c1bSTvrtko Ursulin { "0a", 0, true, true }, 230f0c02c1bSTvrtko Ursulin { "1a", 1, false, true }, 231f0c02c1bSTvrtko Ursulin { "1b", 1, true, true }, 232f0c02c1bSTvrtko Ursulin { "0b", 0, true, false }, 233f0c02c1bSTvrtko Ursulin { "2a", 2, false, true }, 234f0c02c1bSTvrtko Ursulin { "4", 4, false, true }, 235f0c02c1bSTvrtko Ursulin { "INT_MAX", INT_MAX, false, true }, 236f0c02c1bSTvrtko Ursulin { "INT_MAX-1", INT_MAX-1, true, false }, 237f0c02c1bSTvrtko Ursulin { "INT_MAX+1", (u32)INT_MAX+1, false, true }, 238f0c02c1bSTvrtko Ursulin { "INT_MAX", INT_MAX, true, false }, 239f0c02c1bSTvrtko Ursulin { "UINT_MAX", UINT_MAX, false, true }, 240f0c02c1bSTvrtko Ursulin { "wrap", 0, false, true }, 241f0c02c1bSTvrtko Ursulin { "unwrap", UINT_MAX, true, false }, 242f0c02c1bSTvrtko Ursulin {}, 243f0c02c1bSTvrtko Ursulin }, *p; 244f0c02c1bSTvrtko Ursulin struct intel_timeline tl; 245f0c02c1bSTvrtko Ursulin int order, offset; 246f0c02c1bSTvrtko Ursulin int ret = -ENODEV; 247f0c02c1bSTvrtko Ursulin 248f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 249f0c02c1bSTvrtko Ursulin for (p = pass; p->name; p++) { 250f0c02c1bSTvrtko Ursulin for (order = 1; order < 64; order++) { 251f0c02c1bSTvrtko Ursulin for (offset = -1; offset <= (order > 1); offset++) { 252f0c02c1bSTvrtko Ursulin u64 ctx = BIT_ULL(order) + offset; 253f0c02c1bSTvrtko Ursulin 254f0c02c1bSTvrtko Ursulin ret = __igt_sync(&tl, ctx, p, "1"); 255f0c02c1bSTvrtko Ursulin if (ret) 256f0c02c1bSTvrtko Ursulin goto out; 257f0c02c1bSTvrtko Ursulin } 258f0c02c1bSTvrtko Ursulin } 259f0c02c1bSTvrtko Ursulin } 260f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 261f0c02c1bSTvrtko Ursulin 262f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 263f0c02c1bSTvrtko Ursulin for (order = 1; order < 64; order++) { 264f0c02c1bSTvrtko Ursulin for (offset = -1; offset <= (order > 1); offset++) { 265f0c02c1bSTvrtko Ursulin u64 ctx = BIT_ULL(order) + offset; 266f0c02c1bSTvrtko Ursulin 267f0c02c1bSTvrtko Ursulin for (p = pass; p->name; p++) { 268f0c02c1bSTvrtko Ursulin ret = __igt_sync(&tl, ctx, p, "2"); 269f0c02c1bSTvrtko Ursulin if (ret) 270f0c02c1bSTvrtko Ursulin goto out; 271f0c02c1bSTvrtko Ursulin } 272f0c02c1bSTvrtko Ursulin } 273f0c02c1bSTvrtko Ursulin } 274f0c02c1bSTvrtko Ursulin 275f0c02c1bSTvrtko Ursulin out: 276f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 277f0c02c1bSTvrtko Ursulin return ret; 278f0c02c1bSTvrtko Ursulin } 279f0c02c1bSTvrtko Ursulin 280f0c02c1bSTvrtko Ursulin static unsigned int random_engine(struct rnd_state *rnd) 281f0c02c1bSTvrtko Ursulin { 282f0c02c1bSTvrtko Ursulin return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd); 283f0c02c1bSTvrtko Ursulin } 284f0c02c1bSTvrtko Ursulin 285f0c02c1bSTvrtko Ursulin static int bench_sync(void *arg) 286f0c02c1bSTvrtko Ursulin { 287f0c02c1bSTvrtko Ursulin struct rnd_state prng; 288f0c02c1bSTvrtko Ursulin struct intel_timeline tl; 289f0c02c1bSTvrtko Ursulin unsigned long end_time, count; 290f0c02c1bSTvrtko Ursulin u64 prng32_1M; 291f0c02c1bSTvrtko Ursulin ktime_t kt; 292f0c02c1bSTvrtko Ursulin int order, last_order; 293f0c02c1bSTvrtko Ursulin 294f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 295f0c02c1bSTvrtko Ursulin 296f0c02c1bSTvrtko Ursulin /* Lookups from cache are very fast and so the random number generation 297f0c02c1bSTvrtko Ursulin * and the loop itself becomes a significant factor in the per-iteration 298f0c02c1bSTvrtko Ursulin * timings. We try to compensate the results by measuring the overhead 299f0c02c1bSTvrtko Ursulin * of the prng and subtract it from the reported results. 300f0c02c1bSTvrtko Ursulin */ 301f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 302f0c02c1bSTvrtko Ursulin count = 0; 303f0c02c1bSTvrtko Ursulin kt = ktime_get(); 304f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 305f0c02c1bSTvrtko Ursulin do { 306f0c02c1bSTvrtko Ursulin u32 x; 307f0c02c1bSTvrtko Ursulin 308f0c02c1bSTvrtko Ursulin /* Make sure the compiler doesn't optimise away the prng call */ 309f0c02c1bSTvrtko Ursulin WRITE_ONCE(x, prandom_u32_state(&prng)); 310f0c02c1bSTvrtko Ursulin 311f0c02c1bSTvrtko Ursulin count++; 312f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 313f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 314f0c02c1bSTvrtko Ursulin pr_debug("%s: %lu random evaluations, %lluns/prng\n", 315f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 316f0c02c1bSTvrtko Ursulin prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count); 317f0c02c1bSTvrtko Ursulin 318f0c02c1bSTvrtko Ursulin /* Benchmark (only) setting random context ids */ 319f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 320f0c02c1bSTvrtko Ursulin count = 0; 321f0c02c1bSTvrtko Ursulin kt = ktime_get(); 322f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 323f0c02c1bSTvrtko Ursulin do { 324f0c02c1bSTvrtko Ursulin u64 id = i915_prandom_u64_state(&prng); 325f0c02c1bSTvrtko Ursulin 326f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, 0); 327f0c02c1bSTvrtko Ursulin count++; 328f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 329f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 330f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 331f0c02c1bSTvrtko Ursulin pr_info("%s: %lu random insertions, %lluns/insert\n", 332f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 333f0c02c1bSTvrtko Ursulin 334f0c02c1bSTvrtko Ursulin /* Benchmark looking up the exact same context ids as we just set */ 335f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 336f0c02c1bSTvrtko Ursulin end_time = count; 337f0c02c1bSTvrtko Ursulin kt = ktime_get(); 338f0c02c1bSTvrtko Ursulin while (end_time--) { 339f0c02c1bSTvrtko Ursulin u64 id = i915_prandom_u64_state(&prng); 340f0c02c1bSTvrtko Ursulin 341f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, id, 0)) { 342f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 343f0c02c1bSTvrtko Ursulin pr_err("Lookup of %llu failed\n", id); 344f0c02c1bSTvrtko Ursulin return -EINVAL; 345f0c02c1bSTvrtko Ursulin } 346f0c02c1bSTvrtko Ursulin } 347f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 348f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 349f0c02c1bSTvrtko Ursulin pr_info("%s: %lu random lookups, %lluns/lookup\n", 350f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 351f0c02c1bSTvrtko Ursulin 352f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 353f0c02c1bSTvrtko Ursulin cond_resched(); 354f0c02c1bSTvrtko Ursulin 355f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 356f0c02c1bSTvrtko Ursulin 357f0c02c1bSTvrtko Ursulin /* Benchmark setting the first N (in order) contexts */ 358f0c02c1bSTvrtko Ursulin count = 0; 359f0c02c1bSTvrtko Ursulin kt = ktime_get(); 360f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 361f0c02c1bSTvrtko Ursulin do { 362f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, count++, 0); 363f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 364f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 365f0c02c1bSTvrtko Ursulin pr_info("%s: %lu in-order insertions, %lluns/insert\n", 366f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 367f0c02c1bSTvrtko Ursulin 368f0c02c1bSTvrtko Ursulin /* Benchmark looking up the exact same context ids as we just set */ 369f0c02c1bSTvrtko Ursulin end_time = count; 370f0c02c1bSTvrtko Ursulin kt = ktime_get(); 371f0c02c1bSTvrtko Ursulin while (end_time--) { 372f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) { 373f0c02c1bSTvrtko Ursulin pr_err("Lookup of %lu failed\n", end_time); 374f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 375f0c02c1bSTvrtko Ursulin return -EINVAL; 376f0c02c1bSTvrtko Ursulin } 377f0c02c1bSTvrtko Ursulin } 378f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 379f0c02c1bSTvrtko Ursulin pr_info("%s: %lu in-order lookups, %lluns/lookup\n", 380f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 381f0c02c1bSTvrtko Ursulin 382f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 383f0c02c1bSTvrtko Ursulin cond_resched(); 384f0c02c1bSTvrtko Ursulin 385f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 386f0c02c1bSTvrtko Ursulin 387f0c02c1bSTvrtko Ursulin /* Benchmark searching for a random context id and maybe changing it */ 388f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 389f0c02c1bSTvrtko Ursulin count = 0; 390f0c02c1bSTvrtko Ursulin kt = ktime_get(); 391f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 392f0c02c1bSTvrtko Ursulin do { 393f0c02c1bSTvrtko Ursulin u32 id = random_engine(&prng); 394f0c02c1bSTvrtko Ursulin u32 seqno = prandom_u32_state(&prng); 395f0c02c1bSTvrtko Ursulin 396f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, id, seqno)) 397f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, seqno); 398f0c02c1bSTvrtko Ursulin 399f0c02c1bSTvrtko Ursulin count++; 400f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 401f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 402f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 403f0c02c1bSTvrtko Ursulin pr_info("%s: %lu repeated insert/lookups, %lluns/op\n", 404f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 405f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 406f0c02c1bSTvrtko Ursulin cond_resched(); 407f0c02c1bSTvrtko Ursulin 408f0c02c1bSTvrtko Ursulin /* Benchmark searching for a known context id and changing the seqno */ 409f0c02c1bSTvrtko Ursulin for (last_order = 1, order = 1; order < 32; 410f0c02c1bSTvrtko Ursulin ({ int tmp = last_order; last_order = order; order += tmp; })) { 411f0c02c1bSTvrtko Ursulin unsigned int mask = BIT(order) - 1; 412f0c02c1bSTvrtko Ursulin 413f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 414f0c02c1bSTvrtko Ursulin 415f0c02c1bSTvrtko Ursulin count = 0; 416f0c02c1bSTvrtko Ursulin kt = ktime_get(); 417f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 418f0c02c1bSTvrtko Ursulin do { 419f0c02c1bSTvrtko Ursulin /* Without assuming too many details of the underlying 420f0c02c1bSTvrtko Ursulin * implementation, try to identify its phase-changes 421f0c02c1bSTvrtko Ursulin * (if any)! 422f0c02c1bSTvrtko Ursulin */ 423f0c02c1bSTvrtko Ursulin u64 id = (u64)(count & mask) << order; 424f0c02c1bSTvrtko Ursulin 425f0c02c1bSTvrtko Ursulin __intel_timeline_sync_is_later(&tl, id, 0); 426f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, 0); 427f0c02c1bSTvrtko Ursulin 428f0c02c1bSTvrtko Ursulin count++; 429f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 430f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 431f0c02c1bSTvrtko Ursulin pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n", 432f0c02c1bSTvrtko Ursulin __func__, count, order, 433f0c02c1bSTvrtko Ursulin (long long)div64_ul(ktime_to_ns(kt), count)); 434f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 435f0c02c1bSTvrtko Ursulin cond_resched(); 436f0c02c1bSTvrtko Ursulin } 437f0c02c1bSTvrtko Ursulin 438f0c02c1bSTvrtko Ursulin return 0; 439f0c02c1bSTvrtko Ursulin } 440f0c02c1bSTvrtko Ursulin 441f0c02c1bSTvrtko Ursulin int intel_timeline_mock_selftests(void) 442f0c02c1bSTvrtko Ursulin { 443f0c02c1bSTvrtko Ursulin static const struct i915_subtest tests[] = { 444f0c02c1bSTvrtko Ursulin SUBTEST(mock_hwsp_freelist), 445f0c02c1bSTvrtko Ursulin SUBTEST(igt_sync), 446f0c02c1bSTvrtko Ursulin SUBTEST(bench_sync), 447f0c02c1bSTvrtko Ursulin }; 448f0c02c1bSTvrtko Ursulin 449f0c02c1bSTvrtko Ursulin return i915_subtests(tests, NULL); 450f0c02c1bSTvrtko Ursulin } 451f0c02c1bSTvrtko Ursulin 452f0c02c1bSTvrtko Ursulin static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) 453f0c02c1bSTvrtko Ursulin { 454f0c02c1bSTvrtko Ursulin u32 *cs; 455f0c02c1bSTvrtko Ursulin 456f0c02c1bSTvrtko Ursulin cs = intel_ring_begin(rq, 4); 457f0c02c1bSTvrtko Ursulin if (IS_ERR(cs)) 458f0c02c1bSTvrtko Ursulin return PTR_ERR(cs); 459f0c02c1bSTvrtko Ursulin 460c816723bSLucas De Marchi if (GRAPHICS_VER(rq->engine->i915) >= 8) { 461f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 462f0c02c1bSTvrtko Ursulin *cs++ = addr; 463f0c02c1bSTvrtko Ursulin *cs++ = 0; 464f0c02c1bSTvrtko Ursulin *cs++ = value; 465c816723bSLucas De Marchi } else if (GRAPHICS_VER(rq->engine->i915) >= 4) { 466f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 467f0c02c1bSTvrtko Ursulin *cs++ = 0; 468f0c02c1bSTvrtko Ursulin *cs++ = addr; 469f0c02c1bSTvrtko Ursulin *cs++ = value; 470f0c02c1bSTvrtko Ursulin } else { 471f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 472f0c02c1bSTvrtko Ursulin *cs++ = addr; 473f0c02c1bSTvrtko Ursulin *cs++ = value; 474f0c02c1bSTvrtko Ursulin *cs++ = MI_NOOP; 475f0c02c1bSTvrtko Ursulin } 476f0c02c1bSTvrtko Ursulin 477f0c02c1bSTvrtko Ursulin intel_ring_advance(rq, cs); 478f0c02c1bSTvrtko Ursulin 479f0c02c1bSTvrtko Ursulin return 0; 480f0c02c1bSTvrtko Ursulin } 481f0c02c1bSTvrtko Ursulin 482f0c02c1bSTvrtko Ursulin static struct i915_request * 4832c8ab333SMaarten Lankhorst checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value) 484f0c02c1bSTvrtko Ursulin { 485f0c02c1bSTvrtko Ursulin struct i915_request *rq; 486f0c02c1bSTvrtko Ursulin int err; 487f0c02c1bSTvrtko Ursulin 4881060974cSMaarten Lankhorst err = selftest_tl_pin(tl); 489f0c02c1bSTvrtko Ursulin if (err) { 490f0c02c1bSTvrtko Ursulin rq = ERR_PTR(err); 491f0c02c1bSTvrtko Ursulin goto out; 492f0c02c1bSTvrtko Ursulin } 493f0c02c1bSTvrtko Ursulin 4942c8ab333SMaarten Lankhorst if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) { 4952c8ab333SMaarten Lankhorst pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", 4962c8ab333SMaarten Lankhorst *tl->hwsp_seqno, tl->seqno); 4972c8ab333SMaarten Lankhorst intel_timeline_unpin(tl); 4982c8ab333SMaarten Lankhorst return ERR_PTR(-EINVAL); 4992c8ab333SMaarten Lankhorst } 5002c8ab333SMaarten Lankhorst 501de5825beSChris Wilson rq = intel_engine_create_kernel_request(engine); 502f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) 503f0c02c1bSTvrtko Ursulin goto out_unpin; 504f0c02c1bSTvrtko Ursulin 5057e805762SChris Wilson i915_request_get(rq); 5067e805762SChris Wilson 507f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); 508f0c02c1bSTvrtko Ursulin i915_request_add(rq); 5097e805762SChris Wilson if (err) { 5107e805762SChris Wilson i915_request_put(rq); 511f0c02c1bSTvrtko Ursulin rq = ERR_PTR(err); 5127e805762SChris Wilson } 513f0c02c1bSTvrtko Ursulin 514f0c02c1bSTvrtko Ursulin out_unpin: 515f0c02c1bSTvrtko Ursulin intel_timeline_unpin(tl); 516f0c02c1bSTvrtko Ursulin out: 517f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) 518f0c02c1bSTvrtko Ursulin pr_err("Failed to write to timeline!\n"); 519f0c02c1bSTvrtko Ursulin return rq; 520f0c02c1bSTvrtko Ursulin } 521f0c02c1bSTvrtko Ursulin 522f0c02c1bSTvrtko Ursulin static int live_hwsp_engine(void *arg) 523f0c02c1bSTvrtko Ursulin { 524f0c02c1bSTvrtko Ursulin #define NUM_TIMELINES 4096 5255f65d5a6SChris Wilson struct intel_gt *gt = arg; 526f0c02c1bSTvrtko Ursulin struct intel_timeline **timelines; 527f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 528f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 529f0c02c1bSTvrtko Ursulin unsigned long count, n; 530f0c02c1bSTvrtko Ursulin int err = 0; 531f0c02c1bSTvrtko Ursulin 532f0c02c1bSTvrtko Ursulin /* 533f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check we can write 534f0c02c1bSTvrtko Ursulin * independently to each of their breadcrumb slots. 535f0c02c1bSTvrtko Ursulin */ 536f0c02c1bSTvrtko Ursulin 537f0c02c1bSTvrtko Ursulin timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, 538f0c02c1bSTvrtko Ursulin sizeof(*timelines), 539f0c02c1bSTvrtko Ursulin GFP_KERNEL); 540f0c02c1bSTvrtko Ursulin if (!timelines) 541f0c02c1bSTvrtko Ursulin return -ENOMEM; 542f0c02c1bSTvrtko Ursulin 543f0c02c1bSTvrtko Ursulin count = 0; 5445d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 545f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 546f0c02c1bSTvrtko Ursulin continue; 547f0c02c1bSTvrtko Ursulin 5487e805762SChris Wilson intel_engine_pm_get(engine); 5497e805762SChris Wilson 550f0c02c1bSTvrtko Ursulin for (n = 0; n < NUM_TIMELINES; n++) { 551f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 552f0c02c1bSTvrtko Ursulin struct i915_request *rq; 553f0c02c1bSTvrtko Ursulin 5542c8ab333SMaarten Lankhorst tl = intel_timeline_create(gt); 555f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 556f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 5577e805762SChris Wilson break; 558f0c02c1bSTvrtko Ursulin } 559f0c02c1bSTvrtko Ursulin 5602c8ab333SMaarten Lankhorst rq = checked_tl_write(tl, engine, count); 561f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 562f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 563f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 5647e805762SChris Wilson break; 565f0c02c1bSTvrtko Ursulin } 566f0c02c1bSTvrtko Ursulin 567f0c02c1bSTvrtko Ursulin timelines[count++] = tl; 5687e805762SChris Wilson i915_request_put(rq); 569f0c02c1bSTvrtko Ursulin } 570f0c02c1bSTvrtko Ursulin 5717e805762SChris Wilson intel_engine_pm_put(engine); 5727e805762SChris Wilson if (err) 5737e805762SChris Wilson break; 5747e805762SChris Wilson } 5757e805762SChris Wilson 5765f65d5a6SChris Wilson if (igt_flush_test(gt->i915)) 577f0c02c1bSTvrtko Ursulin err = -EIO; 578f0c02c1bSTvrtko Ursulin 579f0c02c1bSTvrtko Ursulin for (n = 0; n < count; n++) { 580f0c02c1bSTvrtko Ursulin struct intel_timeline *tl = timelines[n]; 581f0c02c1bSTvrtko Ursulin 582e310b435SChris Wilson if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { 583e310b435SChris Wilson GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", 584e310b435SChris Wilson n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); 585d45171acSChris Wilson GEM_TRACE_DUMP(); 586f0c02c1bSTvrtko Ursulin err = -EINVAL; 587f0c02c1bSTvrtko Ursulin } 588f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 589f0c02c1bSTvrtko Ursulin } 590f0c02c1bSTvrtko Ursulin 591f0c02c1bSTvrtko Ursulin kvfree(timelines); 592f0c02c1bSTvrtko Ursulin return err; 593f0c02c1bSTvrtko Ursulin #undef NUM_TIMELINES 594f0c02c1bSTvrtko Ursulin } 595f0c02c1bSTvrtko Ursulin 596f0c02c1bSTvrtko Ursulin static int live_hwsp_alternate(void *arg) 597f0c02c1bSTvrtko Ursulin { 598f0c02c1bSTvrtko Ursulin #define NUM_TIMELINES 4096 5995f65d5a6SChris Wilson struct intel_gt *gt = arg; 600f0c02c1bSTvrtko Ursulin struct intel_timeline **timelines; 601f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 602f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 603f0c02c1bSTvrtko Ursulin unsigned long count, n; 604f0c02c1bSTvrtko Ursulin int err = 0; 605f0c02c1bSTvrtko Ursulin 606f0c02c1bSTvrtko Ursulin /* 607f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check we can write 608f0c02c1bSTvrtko Ursulin * independently to each of their breadcrumb slots with adjacent 609f0c02c1bSTvrtko Ursulin * engines. 610f0c02c1bSTvrtko Ursulin */ 611f0c02c1bSTvrtko Ursulin 612f0c02c1bSTvrtko Ursulin timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, 613f0c02c1bSTvrtko Ursulin sizeof(*timelines), 614f0c02c1bSTvrtko Ursulin GFP_KERNEL); 615f0c02c1bSTvrtko Ursulin if (!timelines) 616f0c02c1bSTvrtko Ursulin return -ENOMEM; 617f0c02c1bSTvrtko Ursulin 618f0c02c1bSTvrtko Ursulin count = 0; 619f0c02c1bSTvrtko Ursulin for (n = 0; n < NUM_TIMELINES; n++) { 6205d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 621f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 622f0c02c1bSTvrtko Ursulin struct i915_request *rq; 623f0c02c1bSTvrtko Ursulin 624f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 625f0c02c1bSTvrtko Ursulin continue; 626f0c02c1bSTvrtko Ursulin 6272c8ab333SMaarten Lankhorst tl = intel_timeline_create(gt); 628f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 629f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 630f0c02c1bSTvrtko Ursulin goto out; 631f0c02c1bSTvrtko Ursulin } 632f0c02c1bSTvrtko Ursulin 6337e805762SChris Wilson intel_engine_pm_get(engine); 6342c8ab333SMaarten Lankhorst rq = checked_tl_write(tl, engine, count); 6357e805762SChris Wilson intel_engine_pm_put(engine); 636f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 637f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 638f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 639f0c02c1bSTvrtko Ursulin goto out; 640f0c02c1bSTvrtko Ursulin } 641f0c02c1bSTvrtko Ursulin 642f0c02c1bSTvrtko Ursulin timelines[count++] = tl; 6437e805762SChris Wilson i915_request_put(rq); 644f0c02c1bSTvrtko Ursulin } 645f0c02c1bSTvrtko Ursulin } 646f0c02c1bSTvrtko Ursulin 647f0c02c1bSTvrtko Ursulin out: 6485f65d5a6SChris Wilson if (igt_flush_test(gt->i915)) 649f0c02c1bSTvrtko Ursulin err = -EIO; 650f0c02c1bSTvrtko Ursulin 651f0c02c1bSTvrtko Ursulin for (n = 0; n < count; n++) { 652f0c02c1bSTvrtko Ursulin struct intel_timeline *tl = timelines[n]; 653f0c02c1bSTvrtko Ursulin 654e310b435SChris Wilson if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { 655e310b435SChris Wilson GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", 656e310b435SChris Wilson n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); 657d45171acSChris Wilson GEM_TRACE_DUMP(); 658f0c02c1bSTvrtko Ursulin err = -EINVAL; 659f0c02c1bSTvrtko Ursulin } 660f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 661f0c02c1bSTvrtko Ursulin } 662f0c02c1bSTvrtko Ursulin 663f0c02c1bSTvrtko Ursulin kvfree(timelines); 664f0c02c1bSTvrtko Ursulin return err; 665f0c02c1bSTvrtko Ursulin #undef NUM_TIMELINES 666f0c02c1bSTvrtko Ursulin } 667f0c02c1bSTvrtko Ursulin 668f0c02c1bSTvrtko Ursulin static int live_hwsp_wrap(void *arg) 669f0c02c1bSTvrtko Ursulin { 6705f65d5a6SChris Wilson struct intel_gt *gt = arg; 671f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 672f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 673f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 674f0c02c1bSTvrtko Ursulin int err = 0; 675f0c02c1bSTvrtko Ursulin 676f0c02c1bSTvrtko Ursulin /* 677f0c02c1bSTvrtko Ursulin * Across a seqno wrap, we need to keep the old cacheline alive for 678f0c02c1bSTvrtko Ursulin * foreign GPU references. 679f0c02c1bSTvrtko Ursulin */ 680f0c02c1bSTvrtko Ursulin 681d1bf5dd8SChris Wilson tl = intel_timeline_create(gt); 6827e805762SChris Wilson if (IS_ERR(tl)) 6837e805762SChris Wilson return PTR_ERR(tl); 6847e805762SChris Wilson 68512ca695dSMaarten Lankhorst if (!tl->has_initial_breadcrumb) 686f0c02c1bSTvrtko Ursulin goto out_free; 687f0c02c1bSTvrtko Ursulin 6881060974cSMaarten Lankhorst err = selftest_tl_pin(tl); 689f0c02c1bSTvrtko Ursulin if (err) 690f0c02c1bSTvrtko Ursulin goto out_free; 691f0c02c1bSTvrtko Ursulin 6925d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 693f0c02c1bSTvrtko Ursulin const u32 *hwsp_seqno[2]; 694f0c02c1bSTvrtko Ursulin struct i915_request *rq; 695f0c02c1bSTvrtko Ursulin u32 seqno[2]; 696f0c02c1bSTvrtko Ursulin 697f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 698f0c02c1bSTvrtko Ursulin continue; 699f0c02c1bSTvrtko Ursulin 700de5825beSChris Wilson rq = intel_engine_create_kernel_request(engine); 701f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 702f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 703f0c02c1bSTvrtko Ursulin goto out; 704f0c02c1bSTvrtko Ursulin } 705f0c02c1bSTvrtko Ursulin 706f0c02c1bSTvrtko Ursulin tl->seqno = -4u; 707f0c02c1bSTvrtko Ursulin 70825ffd4b1SChris Wilson mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); 709f0c02c1bSTvrtko Ursulin err = intel_timeline_get_seqno(tl, rq, &seqno[0]); 71025ffd4b1SChris Wilson mutex_unlock(&tl->mutex); 711f0c02c1bSTvrtko Ursulin if (err) { 712f0c02c1bSTvrtko Ursulin i915_request_add(rq); 713f0c02c1bSTvrtko Ursulin goto out; 714f0c02c1bSTvrtko Ursulin } 715f0c02c1bSTvrtko Ursulin pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n", 716f0c02c1bSTvrtko Ursulin seqno[0], tl->hwsp_offset); 717f0c02c1bSTvrtko Ursulin 718f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]); 719f0c02c1bSTvrtko Ursulin if (err) { 720f0c02c1bSTvrtko Ursulin i915_request_add(rq); 721f0c02c1bSTvrtko Ursulin goto out; 722f0c02c1bSTvrtko Ursulin } 723f0c02c1bSTvrtko Ursulin hwsp_seqno[0] = tl->hwsp_seqno; 724f0c02c1bSTvrtko Ursulin 72525ffd4b1SChris Wilson mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); 726f0c02c1bSTvrtko Ursulin err = intel_timeline_get_seqno(tl, rq, &seqno[1]); 72725ffd4b1SChris Wilson mutex_unlock(&tl->mutex); 728f0c02c1bSTvrtko Ursulin if (err) { 729f0c02c1bSTvrtko Ursulin i915_request_add(rq); 730f0c02c1bSTvrtko Ursulin goto out; 731f0c02c1bSTvrtko Ursulin } 732f0c02c1bSTvrtko Ursulin pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n", 733f0c02c1bSTvrtko Ursulin seqno[1], tl->hwsp_offset); 734f0c02c1bSTvrtko Ursulin 735f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]); 736f0c02c1bSTvrtko Ursulin if (err) { 737f0c02c1bSTvrtko Ursulin i915_request_add(rq); 738f0c02c1bSTvrtko Ursulin goto out; 739f0c02c1bSTvrtko Ursulin } 740f0c02c1bSTvrtko Ursulin hwsp_seqno[1] = tl->hwsp_seqno; 741f0c02c1bSTvrtko Ursulin 742f0c02c1bSTvrtko Ursulin /* With wrap should come a new hwsp */ 743f0c02c1bSTvrtko Ursulin GEM_BUG_ON(seqno[1] >= seqno[0]); 744f0c02c1bSTvrtko Ursulin GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]); 745f0c02c1bSTvrtko Ursulin 746f0c02c1bSTvrtko Ursulin i915_request_add(rq); 747f0c02c1bSTvrtko Ursulin 748f0c02c1bSTvrtko Ursulin if (i915_request_wait(rq, 0, HZ / 5) < 0) { 749f0c02c1bSTvrtko Ursulin pr_err("Wait for timeline writes timed out!\n"); 750f0c02c1bSTvrtko Ursulin err = -EIO; 751f0c02c1bSTvrtko Ursulin goto out; 752f0c02c1bSTvrtko Ursulin } 753f0c02c1bSTvrtko Ursulin 754e310b435SChris Wilson if (READ_ONCE(*hwsp_seqno[0]) != seqno[0] || 755e310b435SChris Wilson READ_ONCE(*hwsp_seqno[1]) != seqno[1]) { 756f0c02c1bSTvrtko Ursulin pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n", 757f0c02c1bSTvrtko Ursulin *hwsp_seqno[0], *hwsp_seqno[1], 758f0c02c1bSTvrtko Ursulin seqno[0], seqno[1]); 759f0c02c1bSTvrtko Ursulin err = -EINVAL; 760f0c02c1bSTvrtko Ursulin goto out; 761f0c02c1bSTvrtko Ursulin } 762f0c02c1bSTvrtko Ursulin 76366101975SChris Wilson intel_gt_retire_requests(gt); /* recycle HWSP */ 764f0c02c1bSTvrtko Ursulin } 765f0c02c1bSTvrtko Ursulin 766f0c02c1bSTvrtko Ursulin out: 7675f65d5a6SChris Wilson if (igt_flush_test(gt->i915)) 768f0c02c1bSTvrtko Ursulin err = -EIO; 769f0c02c1bSTvrtko Ursulin 770f0c02c1bSTvrtko Ursulin intel_timeline_unpin(tl); 771f0c02c1bSTvrtko Ursulin out_free: 772f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 773f0c02c1bSTvrtko Ursulin return err; 774f0c02c1bSTvrtko Ursulin } 775f0c02c1bSTvrtko Ursulin 7766e7a21e7SChris Wilson static int emit_read_hwsp(struct i915_request *rq, 7776e7a21e7SChris Wilson u32 seqno, u32 hwsp, 7786e7a21e7SChris Wilson u32 *addr) 7796e7a21e7SChris Wilson { 7806e7a21e7SChris Wilson const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0)); 7816e7a21e7SChris Wilson u32 *cs; 7826e7a21e7SChris Wilson 7836e7a21e7SChris Wilson cs = intel_ring_begin(rq, 12); 7846e7a21e7SChris Wilson if (IS_ERR(cs)) 7856e7a21e7SChris Wilson return PTR_ERR(cs); 7866e7a21e7SChris Wilson 7876e7a21e7SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 7886e7a21e7SChris Wilson *cs++ = *addr; 7896e7a21e7SChris Wilson *cs++ = 0; 7906e7a21e7SChris Wilson *cs++ = seqno; 7916e7a21e7SChris Wilson *addr += 4; 7926e7a21e7SChris Wilson 7936e7a21e7SChris Wilson *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT; 7946e7a21e7SChris Wilson *cs++ = gpr; 7956e7a21e7SChris Wilson *cs++ = hwsp; 7966e7a21e7SChris Wilson *cs++ = 0; 7976e7a21e7SChris Wilson 7986e7a21e7SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 7996e7a21e7SChris Wilson *cs++ = gpr; 8006e7a21e7SChris Wilson *cs++ = *addr; 8016e7a21e7SChris Wilson *cs++ = 0; 8026e7a21e7SChris Wilson *addr += 4; 8036e7a21e7SChris Wilson 8046e7a21e7SChris Wilson intel_ring_advance(rq, cs); 8056e7a21e7SChris Wilson 8066e7a21e7SChris Wilson return 0; 8076e7a21e7SChris Wilson } 8086e7a21e7SChris Wilson 8096e7a21e7SChris Wilson struct hwsp_watcher { 8106e7a21e7SChris Wilson struct i915_vma *vma; 8116e7a21e7SChris Wilson struct i915_request *rq; 8126e7a21e7SChris Wilson u32 addr; 8136e7a21e7SChris Wilson u32 *map; 8146e7a21e7SChris Wilson }; 8156e7a21e7SChris Wilson 8166e7a21e7SChris Wilson static bool cmp_lt(u32 a, u32 b) 8176e7a21e7SChris Wilson { 8186e7a21e7SChris Wilson return a < b; 8196e7a21e7SChris Wilson } 8206e7a21e7SChris Wilson 8216e7a21e7SChris Wilson static bool cmp_gte(u32 a, u32 b) 8226e7a21e7SChris Wilson { 8236e7a21e7SChris Wilson return a >= b; 8246e7a21e7SChris Wilson } 8256e7a21e7SChris Wilson 8266e7a21e7SChris Wilson static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt) 8276e7a21e7SChris Wilson { 8286e7a21e7SChris Wilson struct drm_i915_gem_object *obj; 8296e7a21e7SChris Wilson struct i915_vma *vma; 8306e7a21e7SChris Wilson 8316e7a21e7SChris Wilson obj = i915_gem_object_create_internal(gt->i915, SZ_2M); 8326e7a21e7SChris Wilson if (IS_ERR(obj)) 8336e7a21e7SChris Wilson return PTR_ERR(obj); 8346e7a21e7SChris Wilson 8351060974cSMaarten Lankhorst w->map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 8366e7a21e7SChris Wilson if (IS_ERR(w->map)) { 8376e7a21e7SChris Wilson i915_gem_object_put(obj); 8386e7a21e7SChris Wilson return PTR_ERR(w->map); 8396e7a21e7SChris Wilson } 8406e7a21e7SChris Wilson 8411060974cSMaarten Lankhorst vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); 8426e7a21e7SChris Wilson if (IS_ERR(vma)) { 8436e7a21e7SChris Wilson i915_gem_object_put(obj); 8446e7a21e7SChris Wilson return PTR_ERR(vma); 8456e7a21e7SChris Wilson } 8466e7a21e7SChris Wilson 8476e7a21e7SChris Wilson w->vma = vma; 8486e7a21e7SChris Wilson w->addr = i915_ggtt_offset(vma); 8496e7a21e7SChris Wilson return 0; 8506e7a21e7SChris Wilson } 8516e7a21e7SChris Wilson 85212ca695dSMaarten Lankhorst static void switch_tl_lock(struct i915_request *from, struct i915_request *to) 85312ca695dSMaarten Lankhorst { 85412ca695dSMaarten Lankhorst /* some light mutex juggling required; think co-routines */ 85512ca695dSMaarten Lankhorst 85612ca695dSMaarten Lankhorst if (from) { 85712ca695dSMaarten Lankhorst lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie); 85812ca695dSMaarten Lankhorst mutex_unlock(&from->context->timeline->mutex); 85912ca695dSMaarten Lankhorst } 86012ca695dSMaarten Lankhorst 86112ca695dSMaarten Lankhorst if (to) { 86212ca695dSMaarten Lankhorst mutex_lock(&to->context->timeline->mutex); 86312ca695dSMaarten Lankhorst to->cookie = lockdep_pin_lock(&to->context->timeline->mutex); 86412ca695dSMaarten Lankhorst } 86512ca695dSMaarten Lankhorst } 86612ca695dSMaarten Lankhorst 8676e7a21e7SChris Wilson static int create_watcher(struct hwsp_watcher *w, 8686e7a21e7SChris Wilson struct intel_engine_cs *engine, 8696e7a21e7SChris Wilson int ringsz) 8706e7a21e7SChris Wilson { 8716e7a21e7SChris Wilson struct intel_context *ce; 8726e7a21e7SChris Wilson 8736e7a21e7SChris Wilson ce = intel_context_create(engine); 8746e7a21e7SChris Wilson if (IS_ERR(ce)) 8756e7a21e7SChris Wilson return PTR_ERR(ce); 8766e7a21e7SChris Wilson 87774e4b909SJason Ekstrand ce->ring_size = ringsz; 8786e7a21e7SChris Wilson w->rq = intel_context_create_request(ce); 8796e7a21e7SChris Wilson intel_context_put(ce); 8806e7a21e7SChris Wilson if (IS_ERR(w->rq)) 8816e7a21e7SChris Wilson return PTR_ERR(w->rq); 8826e7a21e7SChris Wilson 8836e7a21e7SChris Wilson w->addr = i915_ggtt_offset(w->vma); 8846e7a21e7SChris Wilson 88512ca695dSMaarten Lankhorst switch_tl_lock(w->rq, NULL); 8866e7a21e7SChris Wilson 8876e7a21e7SChris Wilson return 0; 8886e7a21e7SChris Wilson } 8896e7a21e7SChris Wilson 8906e7a21e7SChris Wilson static int check_watcher(struct hwsp_watcher *w, const char *name, 8916e7a21e7SChris Wilson bool (*op)(u32 hwsp, u32 seqno)) 8926e7a21e7SChris Wilson { 8936e7a21e7SChris Wilson struct i915_request *rq = fetch_and_zero(&w->rq); 8946e7a21e7SChris Wilson u32 offset, end; 8956e7a21e7SChris Wilson int err; 8966e7a21e7SChris Wilson 8976e7a21e7SChris Wilson GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size); 8986e7a21e7SChris Wilson 8996e7a21e7SChris Wilson i915_request_get(rq); 90012ca695dSMaarten Lankhorst switch_tl_lock(NULL, rq); 9016e7a21e7SChris Wilson i915_request_add(rq); 9026e7a21e7SChris Wilson 9036e7a21e7SChris Wilson if (i915_request_wait(rq, 0, HZ) < 0) { 9046e7a21e7SChris Wilson err = -ETIME; 9056e7a21e7SChris Wilson goto out; 9066e7a21e7SChris Wilson } 9076e7a21e7SChris Wilson 9086e7a21e7SChris Wilson err = 0; 9096e7a21e7SChris Wilson offset = 0; 9106e7a21e7SChris Wilson end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map); 9116e7a21e7SChris Wilson while (offset < end) { 9126e7a21e7SChris Wilson if (!op(w->map[offset + 1], w->map[offset])) { 9136e7a21e7SChris Wilson pr_err("Watcher '%s' found HWSP value %x for seqno %x\n", 9146e7a21e7SChris Wilson name, w->map[offset + 1], w->map[offset]); 9156e7a21e7SChris Wilson err = -EINVAL; 9166e7a21e7SChris Wilson } 9176e7a21e7SChris Wilson 9186e7a21e7SChris Wilson offset += 2; 9196e7a21e7SChris Wilson } 9206e7a21e7SChris Wilson 9216e7a21e7SChris Wilson out: 9226e7a21e7SChris Wilson i915_request_put(rq); 9236e7a21e7SChris Wilson return err; 9246e7a21e7SChris Wilson } 9256e7a21e7SChris Wilson 9266e7a21e7SChris Wilson static void cleanup_watcher(struct hwsp_watcher *w) 9276e7a21e7SChris Wilson { 9286e7a21e7SChris Wilson if (w->rq) { 92912ca695dSMaarten Lankhorst switch_tl_lock(NULL, w->rq); 9306e7a21e7SChris Wilson 9316e7a21e7SChris Wilson i915_request_add(w->rq); 9326e7a21e7SChris Wilson } 9336e7a21e7SChris Wilson 9346e7a21e7SChris Wilson i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP); 9356e7a21e7SChris Wilson } 9366e7a21e7SChris Wilson 9376e7a21e7SChris Wilson static bool retire_requests(struct intel_timeline *tl) 9386e7a21e7SChris Wilson { 9396e7a21e7SChris Wilson struct i915_request *rq, *rn; 9406e7a21e7SChris Wilson 9416e7a21e7SChris Wilson mutex_lock(&tl->mutex); 9426e7a21e7SChris Wilson list_for_each_entry_safe(rq, rn, &tl->requests, link) 9436e7a21e7SChris Wilson if (!i915_request_retire(rq)) 9446e7a21e7SChris Wilson break; 9456e7a21e7SChris Wilson mutex_unlock(&tl->mutex); 9466e7a21e7SChris Wilson 9476e7a21e7SChris Wilson return !i915_active_fence_isset(&tl->last_request); 9486e7a21e7SChris Wilson } 9496e7a21e7SChris Wilson 9506e7a21e7SChris Wilson static struct i915_request *wrap_timeline(struct i915_request *rq) 9516e7a21e7SChris Wilson { 9526e7a21e7SChris Wilson struct intel_context *ce = rq->context; 9536e7a21e7SChris Wilson struct intel_timeline *tl = ce->timeline; 9546e7a21e7SChris Wilson u32 seqno = rq->fence.seqno; 9556e7a21e7SChris Wilson 9566e7a21e7SChris Wilson while (tl->seqno >= seqno) { /* Cause a wrap */ 9576e7a21e7SChris Wilson i915_request_put(rq); 9586e7a21e7SChris Wilson rq = intel_context_create_request(ce); 9596e7a21e7SChris Wilson if (IS_ERR(rq)) 9606e7a21e7SChris Wilson return rq; 9616e7a21e7SChris Wilson 9626e7a21e7SChris Wilson i915_request_get(rq); 9636e7a21e7SChris Wilson i915_request_add(rq); 9646e7a21e7SChris Wilson } 9656e7a21e7SChris Wilson 9666e7a21e7SChris Wilson i915_request_put(rq); 96712ca695dSMaarten Lankhorst rq = i915_request_create(ce); 9686e7a21e7SChris Wilson if (IS_ERR(rq)) 9696e7a21e7SChris Wilson return rq; 9706e7a21e7SChris Wilson 9716e7a21e7SChris Wilson i915_request_get(rq); 9726e7a21e7SChris Wilson i915_request_add(rq); 9736e7a21e7SChris Wilson 9746e7a21e7SChris Wilson return rq; 9756e7a21e7SChris Wilson } 9766e7a21e7SChris Wilson 9776e7a21e7SChris Wilson static int live_hwsp_read(void *arg) 9786e7a21e7SChris Wilson { 9796e7a21e7SChris Wilson struct intel_gt *gt = arg; 9806e7a21e7SChris Wilson struct hwsp_watcher watcher[2] = {}; 9816e7a21e7SChris Wilson struct intel_engine_cs *engine; 9826e7a21e7SChris Wilson struct intel_timeline *tl; 9836e7a21e7SChris Wilson enum intel_engine_id id; 9846e7a21e7SChris Wilson int err = 0; 9856e7a21e7SChris Wilson int i; 9866e7a21e7SChris Wilson 9876e7a21e7SChris Wilson /* 9886e7a21e7SChris Wilson * If we take a reference to the HWSP for reading on the GPU, that 9896e7a21e7SChris Wilson * read may be arbitrarily delayed (either by foreign fence or 9906e7a21e7SChris Wilson * priority saturation) and a wrap can happen within 30 minutes. 9916e7a21e7SChris Wilson * When the GPU read is finally submitted it should be correct, 9926e7a21e7SChris Wilson * even across multiple wraps. 9936e7a21e7SChris Wilson */ 9946e7a21e7SChris Wilson 995c816723bSLucas De Marchi if (GRAPHICS_VER(gt->i915) < 8) /* CS convenience [SRM/LRM] */ 9966e7a21e7SChris Wilson return 0; 9976e7a21e7SChris Wilson 9986e7a21e7SChris Wilson tl = intel_timeline_create(gt); 9996e7a21e7SChris Wilson if (IS_ERR(tl)) 10006e7a21e7SChris Wilson return PTR_ERR(tl); 10016e7a21e7SChris Wilson 100212ca695dSMaarten Lankhorst if (!tl->has_initial_breadcrumb) 10036e7a21e7SChris Wilson goto out_free; 10046e7a21e7SChris Wilson 10056e7a21e7SChris Wilson for (i = 0; i < ARRAY_SIZE(watcher); i++) { 10066e7a21e7SChris Wilson err = setup_watcher(&watcher[i], gt); 10076e7a21e7SChris Wilson if (err) 10086e7a21e7SChris Wilson goto out; 10096e7a21e7SChris Wilson } 10106e7a21e7SChris Wilson 10116e7a21e7SChris Wilson for_each_engine(engine, gt, id) { 10126e7a21e7SChris Wilson struct intel_context *ce; 10136e7a21e7SChris Wilson unsigned long count = 0; 10146e7a21e7SChris Wilson IGT_TIMEOUT(end_time); 10156e7a21e7SChris Wilson 10166e7a21e7SChris Wilson /* Create a request we can use for remote reading of the HWSP */ 10176e7a21e7SChris Wilson err = create_watcher(&watcher[1], engine, SZ_512K); 10186e7a21e7SChris Wilson if (err) 10196e7a21e7SChris Wilson goto out; 10206e7a21e7SChris Wilson 10216e7a21e7SChris Wilson do { 10226e7a21e7SChris Wilson struct i915_sw_fence *submit; 10236e7a21e7SChris Wilson struct i915_request *rq; 102412ca695dSMaarten Lankhorst u32 hwsp, dummy; 10256e7a21e7SChris Wilson 10266e7a21e7SChris Wilson submit = heap_fence_create(GFP_KERNEL); 10276e7a21e7SChris Wilson if (!submit) { 10286e7a21e7SChris Wilson err = -ENOMEM; 10296e7a21e7SChris Wilson goto out; 10306e7a21e7SChris Wilson } 10316e7a21e7SChris Wilson 10326e7a21e7SChris Wilson err = create_watcher(&watcher[0], engine, SZ_4K); 10336e7a21e7SChris Wilson if (err) 10346e7a21e7SChris Wilson goto out; 10356e7a21e7SChris Wilson 10366e7a21e7SChris Wilson ce = intel_context_create(engine); 10376e7a21e7SChris Wilson if (IS_ERR(ce)) { 10386e7a21e7SChris Wilson err = PTR_ERR(ce); 10396e7a21e7SChris Wilson goto out; 10406e7a21e7SChris Wilson } 10416e7a21e7SChris Wilson 10426e7a21e7SChris Wilson ce->timeline = intel_timeline_get(tl); 10436e7a21e7SChris Wilson 104412ca695dSMaarten Lankhorst /* Ensure timeline is mapped, done during first pin */ 104512ca695dSMaarten Lankhorst err = intel_context_pin(ce); 104612ca695dSMaarten Lankhorst if (err) { 104712ca695dSMaarten Lankhorst intel_context_put(ce); 104812ca695dSMaarten Lankhorst goto out; 104912ca695dSMaarten Lankhorst } 105012ca695dSMaarten Lankhorst 105112ca695dSMaarten Lankhorst /* 105212ca695dSMaarten Lankhorst * Start at a new wrap, and set seqno right before another wrap, 105312ca695dSMaarten Lankhorst * saving 30 minutes of nops 105412ca695dSMaarten Lankhorst */ 105512ca695dSMaarten Lankhorst tl->seqno = -12u + 2 * (count & 3); 105612ca695dSMaarten Lankhorst __intel_timeline_get_seqno(tl, &dummy); 105712ca695dSMaarten Lankhorst 105812ca695dSMaarten Lankhorst rq = i915_request_create(ce); 10596e7a21e7SChris Wilson if (IS_ERR(rq)) { 10606e7a21e7SChris Wilson err = PTR_ERR(rq); 106112ca695dSMaarten Lankhorst intel_context_unpin(ce); 10626e7a21e7SChris Wilson intel_context_put(ce); 10636e7a21e7SChris Wilson goto out; 10646e7a21e7SChris Wilson } 10656e7a21e7SChris Wilson 10666e7a21e7SChris Wilson err = i915_sw_fence_await_dma_fence(&rq->submit, 10676e7a21e7SChris Wilson &watcher[0].rq->fence, 0, 10686e7a21e7SChris Wilson GFP_KERNEL); 10696e7a21e7SChris Wilson if (err < 0) { 10706e7a21e7SChris Wilson i915_request_add(rq); 107112ca695dSMaarten Lankhorst intel_context_unpin(ce); 10726e7a21e7SChris Wilson intel_context_put(ce); 10736e7a21e7SChris Wilson goto out; 10746e7a21e7SChris Wilson } 10756e7a21e7SChris Wilson 107612ca695dSMaarten Lankhorst switch_tl_lock(rq, watcher[0].rq); 10776e7a21e7SChris Wilson err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp); 10786e7a21e7SChris Wilson if (err == 0) 10796e7a21e7SChris Wilson err = emit_read_hwsp(watcher[0].rq, /* before */ 10806e7a21e7SChris Wilson rq->fence.seqno, hwsp, 10816e7a21e7SChris Wilson &watcher[0].addr); 108212ca695dSMaarten Lankhorst switch_tl_lock(watcher[0].rq, rq); 10836e7a21e7SChris Wilson if (err) { 10846e7a21e7SChris Wilson i915_request_add(rq); 108512ca695dSMaarten Lankhorst intel_context_unpin(ce); 10866e7a21e7SChris Wilson intel_context_put(ce); 10876e7a21e7SChris Wilson goto out; 10886e7a21e7SChris Wilson } 10896e7a21e7SChris Wilson 109012ca695dSMaarten Lankhorst switch_tl_lock(rq, watcher[1].rq); 10916e7a21e7SChris Wilson err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp); 10926e7a21e7SChris Wilson if (err == 0) 10936e7a21e7SChris Wilson err = emit_read_hwsp(watcher[1].rq, /* after */ 10946e7a21e7SChris Wilson rq->fence.seqno, hwsp, 10956e7a21e7SChris Wilson &watcher[1].addr); 109612ca695dSMaarten Lankhorst switch_tl_lock(watcher[1].rq, rq); 10976e7a21e7SChris Wilson if (err) { 10986e7a21e7SChris Wilson i915_request_add(rq); 109912ca695dSMaarten Lankhorst intel_context_unpin(ce); 11006e7a21e7SChris Wilson intel_context_put(ce); 11016e7a21e7SChris Wilson goto out; 11026e7a21e7SChris Wilson } 11036e7a21e7SChris Wilson 11046e7a21e7SChris Wilson i915_request_get(rq); 11056e7a21e7SChris Wilson i915_request_add(rq); 11066e7a21e7SChris Wilson 11076e7a21e7SChris Wilson rq = wrap_timeline(rq); 110812ca695dSMaarten Lankhorst intel_context_unpin(ce); 11096e7a21e7SChris Wilson intel_context_put(ce); 11106e7a21e7SChris Wilson if (IS_ERR(rq)) { 11116e7a21e7SChris Wilson err = PTR_ERR(rq); 11126e7a21e7SChris Wilson goto out; 11136e7a21e7SChris Wilson } 11146e7a21e7SChris Wilson 11156e7a21e7SChris Wilson err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit, 11166e7a21e7SChris Wilson &rq->fence, 0, 11176e7a21e7SChris Wilson GFP_KERNEL); 11186e7a21e7SChris Wilson if (err < 0) { 11196e7a21e7SChris Wilson i915_request_put(rq); 11206e7a21e7SChris Wilson goto out; 11216e7a21e7SChris Wilson } 11226e7a21e7SChris Wilson 11236e7a21e7SChris Wilson err = check_watcher(&watcher[0], "before", cmp_lt); 11246e7a21e7SChris Wilson i915_sw_fence_commit(submit); 11256e7a21e7SChris Wilson heap_fence_put(submit); 11266e7a21e7SChris Wilson if (err) { 11276e7a21e7SChris Wilson i915_request_put(rq); 11286e7a21e7SChris Wilson goto out; 11296e7a21e7SChris Wilson } 11306e7a21e7SChris Wilson count++; 11316e7a21e7SChris Wilson 11326e7a21e7SChris Wilson /* Flush the timeline before manually wrapping again */ 11336e7a21e7SChris Wilson if (i915_request_wait(rq, 11346e7a21e7SChris Wilson I915_WAIT_INTERRUPTIBLE, 11356e7a21e7SChris Wilson HZ) < 0) { 11366e7a21e7SChris Wilson err = -ETIME; 11376e7a21e7SChris Wilson i915_request_put(rq); 11386e7a21e7SChris Wilson goto out; 11396e7a21e7SChris Wilson } 11406e7a21e7SChris Wilson retire_requests(tl); 11416e7a21e7SChris Wilson i915_request_put(rq); 114216cfcb0fSChris Wilson 114316cfcb0fSChris Wilson /* Single requests are limited to half a ring at most */ 114416cfcb0fSChris Wilson if (8 * watcher[1].rq->ring->emit > 114516cfcb0fSChris Wilson 3 * watcher[1].rq->ring->size) 114616cfcb0fSChris Wilson break; 114716cfcb0fSChris Wilson 114812ca695dSMaarten Lankhorst } while (!__igt_timeout(end_time, NULL) && 114912ca695dSMaarten Lankhorst count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2); 11506e7a21e7SChris Wilson 11516e7a21e7SChris Wilson pr_info("%s: simulated %lu wraps\n", engine->name, count); 11526e7a21e7SChris Wilson err = check_watcher(&watcher[1], "after", cmp_gte); 11536e7a21e7SChris Wilson if (err) 11546e7a21e7SChris Wilson goto out; 11556e7a21e7SChris Wilson } 11566e7a21e7SChris Wilson 11576e7a21e7SChris Wilson out: 11586e7a21e7SChris Wilson for (i = 0; i < ARRAY_SIZE(watcher); i++) 11596e7a21e7SChris Wilson cleanup_watcher(&watcher[i]); 11606e7a21e7SChris Wilson 11616e7a21e7SChris Wilson if (igt_flush_test(gt->i915)) 11626e7a21e7SChris Wilson err = -EIO; 11636e7a21e7SChris Wilson 11646e7a21e7SChris Wilson out_free: 11656e7a21e7SChris Wilson intel_timeline_put(tl); 11666e7a21e7SChris Wilson return err; 11676e7a21e7SChris Wilson } 11686e7a21e7SChris Wilson 1169bb5e4397SChris Wilson static int live_hwsp_rollover_kernel(void *arg) 1170bb5e4397SChris Wilson { 1171bb5e4397SChris Wilson struct intel_gt *gt = arg; 1172bb5e4397SChris Wilson struct intel_engine_cs *engine; 1173bb5e4397SChris Wilson enum intel_engine_id id; 1174bb5e4397SChris Wilson int err = 0; 1175bb5e4397SChris Wilson 1176bb5e4397SChris Wilson /* 1177bb5e4397SChris Wilson * Run the host for long enough, and even the kernel context will 1178bb5e4397SChris Wilson * see a seqno rollover. 1179bb5e4397SChris Wilson */ 1180bb5e4397SChris Wilson 1181bb5e4397SChris Wilson for_each_engine(engine, gt, id) { 1182bb5e4397SChris Wilson struct intel_context *ce = engine->kernel_context; 1183bb5e4397SChris Wilson struct intel_timeline *tl = ce->timeline; 1184bb5e4397SChris Wilson struct i915_request *rq[3] = {}; 1185bb5e4397SChris Wilson int i; 1186bb5e4397SChris Wilson 11871b90e4a4SChris Wilson st_engine_heartbeat_disable(engine); 1188bb5e4397SChris Wilson if (intel_gt_wait_for_idle(gt, HZ / 2)) { 1189bb5e4397SChris Wilson err = -EIO; 1190bb5e4397SChris Wilson goto out; 1191bb5e4397SChris Wilson } 1192bb5e4397SChris Wilson 1193bb5e4397SChris Wilson GEM_BUG_ON(i915_active_fence_isset(&tl->last_request)); 119412ca695dSMaarten Lankhorst tl->seqno = -2u; 1195bb5e4397SChris Wilson WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); 1196bb5e4397SChris Wilson 1197bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) { 1198bb5e4397SChris Wilson struct i915_request *this; 1199bb5e4397SChris Wilson 1200bb5e4397SChris Wilson this = i915_request_create(ce); 1201bb5e4397SChris Wilson if (IS_ERR(this)) { 1202bb5e4397SChris Wilson err = PTR_ERR(this); 1203bb5e4397SChris Wilson goto out; 1204bb5e4397SChris Wilson } 1205bb5e4397SChris Wilson 1206bb5e4397SChris Wilson pr_debug("%s: create fence.seqnp:%d\n", 1207bb5e4397SChris Wilson engine->name, 1208bb5e4397SChris Wilson lower_32_bits(this->fence.seqno)); 1209bb5e4397SChris Wilson 1210bb5e4397SChris Wilson GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); 1211bb5e4397SChris Wilson 1212bb5e4397SChris Wilson rq[i] = i915_request_get(this); 1213bb5e4397SChris Wilson i915_request_add(this); 1214bb5e4397SChris Wilson } 1215bb5e4397SChris Wilson 1216bb5e4397SChris Wilson /* We expected a wrap! */ 1217bb5e4397SChris Wilson GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); 1218bb5e4397SChris Wilson 1219bb5e4397SChris Wilson if (i915_request_wait(rq[2], 0, HZ / 5) < 0) { 1220bb5e4397SChris Wilson pr_err("Wait for timeline wrap timed out!\n"); 1221bb5e4397SChris Wilson err = -EIO; 1222bb5e4397SChris Wilson goto out; 1223bb5e4397SChris Wilson } 1224bb5e4397SChris Wilson 1225bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) { 1226bb5e4397SChris Wilson if (!i915_request_completed(rq[i])) { 1227bb5e4397SChris Wilson pr_err("Pre-wrap request not completed!\n"); 1228bb5e4397SChris Wilson err = -EINVAL; 1229bb5e4397SChris Wilson goto out; 1230bb5e4397SChris Wilson } 1231bb5e4397SChris Wilson } 1232bb5e4397SChris Wilson 1233bb5e4397SChris Wilson out: 1234bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) 1235bb5e4397SChris Wilson i915_request_put(rq[i]); 12361b90e4a4SChris Wilson st_engine_heartbeat_enable(engine); 1237bb5e4397SChris Wilson if (err) 1238bb5e4397SChris Wilson break; 1239bb5e4397SChris Wilson } 1240bb5e4397SChris Wilson 1241bb5e4397SChris Wilson if (igt_flush_test(gt->i915)) 1242bb5e4397SChris Wilson err = -EIO; 1243bb5e4397SChris Wilson 1244bb5e4397SChris Wilson return err; 1245bb5e4397SChris Wilson } 1246bb5e4397SChris Wilson 1247bb5e4397SChris Wilson static int live_hwsp_rollover_user(void *arg) 1248bb5e4397SChris Wilson { 1249bb5e4397SChris Wilson struct intel_gt *gt = arg; 1250bb5e4397SChris Wilson struct intel_engine_cs *engine; 1251bb5e4397SChris Wilson enum intel_engine_id id; 1252bb5e4397SChris Wilson int err = 0; 1253bb5e4397SChris Wilson 1254bb5e4397SChris Wilson /* 1255bb5e4397SChris Wilson * Simulate a long running user context, and force the seqno wrap 1256bb5e4397SChris Wilson * on the user's timeline. 1257bb5e4397SChris Wilson */ 1258bb5e4397SChris Wilson 1259bb5e4397SChris Wilson for_each_engine(engine, gt, id) { 1260bb5e4397SChris Wilson struct i915_request *rq[3] = {}; 1261bb5e4397SChris Wilson struct intel_timeline *tl; 1262bb5e4397SChris Wilson struct intel_context *ce; 1263bb5e4397SChris Wilson int i; 1264bb5e4397SChris Wilson 1265bb5e4397SChris Wilson ce = intel_context_create(engine); 1266bb5e4397SChris Wilson if (IS_ERR(ce)) 1267bb5e4397SChris Wilson return PTR_ERR(ce); 1268bb5e4397SChris Wilson 1269bb5e4397SChris Wilson err = intel_context_alloc_state(ce); 1270bb5e4397SChris Wilson if (err) 1271bb5e4397SChris Wilson goto out; 1272bb5e4397SChris Wilson 1273bb5e4397SChris Wilson tl = ce->timeline; 127412ca695dSMaarten Lankhorst if (!tl->has_initial_breadcrumb) 1275bb5e4397SChris Wilson goto out; 1276bb5e4397SChris Wilson 12772c8ab333SMaarten Lankhorst err = intel_context_pin(ce); 12782c8ab333SMaarten Lankhorst if (err) 12792c8ab333SMaarten Lankhorst goto out; 12802c8ab333SMaarten Lankhorst 128112ca695dSMaarten Lankhorst tl->seqno = -4u; 1282bb5e4397SChris Wilson WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); 1283bb5e4397SChris Wilson 1284bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) { 1285bb5e4397SChris Wilson struct i915_request *this; 1286bb5e4397SChris Wilson 1287bb5e4397SChris Wilson this = intel_context_create_request(ce); 1288bb5e4397SChris Wilson if (IS_ERR(this)) { 1289bb5e4397SChris Wilson err = PTR_ERR(this); 12902c8ab333SMaarten Lankhorst goto out_unpin; 1291bb5e4397SChris Wilson } 1292bb5e4397SChris Wilson 1293bb5e4397SChris Wilson pr_debug("%s: create fence.seqnp:%d\n", 1294bb5e4397SChris Wilson engine->name, 1295bb5e4397SChris Wilson lower_32_bits(this->fence.seqno)); 1296bb5e4397SChris Wilson 1297bb5e4397SChris Wilson GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); 1298bb5e4397SChris Wilson 1299bb5e4397SChris Wilson rq[i] = i915_request_get(this); 1300bb5e4397SChris Wilson i915_request_add(this); 1301bb5e4397SChris Wilson } 1302bb5e4397SChris Wilson 1303bb5e4397SChris Wilson /* We expected a wrap! */ 1304bb5e4397SChris Wilson GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); 1305bb5e4397SChris Wilson 1306bb5e4397SChris Wilson if (i915_request_wait(rq[2], 0, HZ / 5) < 0) { 1307bb5e4397SChris Wilson pr_err("Wait for timeline wrap timed out!\n"); 1308bb5e4397SChris Wilson err = -EIO; 13092c8ab333SMaarten Lankhorst goto out_unpin; 1310bb5e4397SChris Wilson } 1311bb5e4397SChris Wilson 1312bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) { 1313bb5e4397SChris Wilson if (!i915_request_completed(rq[i])) { 1314bb5e4397SChris Wilson pr_err("Pre-wrap request not completed!\n"); 1315bb5e4397SChris Wilson err = -EINVAL; 13162c8ab333SMaarten Lankhorst goto out_unpin; 1317bb5e4397SChris Wilson } 1318bb5e4397SChris Wilson } 13192c8ab333SMaarten Lankhorst out_unpin: 13202c8ab333SMaarten Lankhorst intel_context_unpin(ce); 1321bb5e4397SChris Wilson out: 1322bb5e4397SChris Wilson for (i = 0; i < ARRAY_SIZE(rq); i++) 1323bb5e4397SChris Wilson i915_request_put(rq[i]); 1324bb5e4397SChris Wilson intel_context_put(ce); 1325bb5e4397SChris Wilson if (err) 1326bb5e4397SChris Wilson break; 1327bb5e4397SChris Wilson } 1328bb5e4397SChris Wilson 1329bb5e4397SChris Wilson if (igt_flush_test(gt->i915)) 1330bb5e4397SChris Wilson err = -EIO; 1331bb5e4397SChris Wilson 1332bb5e4397SChris Wilson return err; 1333bb5e4397SChris Wilson } 1334bb5e4397SChris Wilson 1335f0c02c1bSTvrtko Ursulin static int live_hwsp_recycle(void *arg) 1336f0c02c1bSTvrtko Ursulin { 13375f65d5a6SChris Wilson struct intel_gt *gt = arg; 1338f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 1339f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 1340f0c02c1bSTvrtko Ursulin unsigned long count; 1341f0c02c1bSTvrtko Ursulin int err = 0; 1342f0c02c1bSTvrtko Ursulin 1343f0c02c1bSTvrtko Ursulin /* 1344f0c02c1bSTvrtko Ursulin * Check seqno writes into one timeline at a time. We expect to 1345f0c02c1bSTvrtko Ursulin * recycle the breadcrumb slot between iterations and neither 1346f0c02c1bSTvrtko Ursulin * want to confuse ourselves or the GPU. 1347f0c02c1bSTvrtko Ursulin */ 1348f0c02c1bSTvrtko Ursulin 1349f0c02c1bSTvrtko Ursulin count = 0; 13505d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 1351f0c02c1bSTvrtko Ursulin IGT_TIMEOUT(end_time); 1352f0c02c1bSTvrtko Ursulin 1353f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 1354f0c02c1bSTvrtko Ursulin continue; 1355f0c02c1bSTvrtko Ursulin 13567e805762SChris Wilson intel_engine_pm_get(engine); 13577e805762SChris Wilson 1358f0c02c1bSTvrtko Ursulin do { 1359f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 1360f0c02c1bSTvrtko Ursulin struct i915_request *rq; 1361f0c02c1bSTvrtko Ursulin 13622c8ab333SMaarten Lankhorst tl = intel_timeline_create(gt); 1363f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 1364f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 13657e805762SChris Wilson break; 1366f0c02c1bSTvrtko Ursulin } 1367f0c02c1bSTvrtko Ursulin 13682c8ab333SMaarten Lankhorst rq = checked_tl_write(tl, engine, count); 1369f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 1370f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 1371f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 13727e805762SChris Wilson break; 1373f0c02c1bSTvrtko Ursulin } 1374f0c02c1bSTvrtko Ursulin 1375f0c02c1bSTvrtko Ursulin if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1376f0c02c1bSTvrtko Ursulin pr_err("Wait for timeline writes timed out!\n"); 13777e805762SChris Wilson i915_request_put(rq); 1378f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 1379f0c02c1bSTvrtko Ursulin err = -EIO; 13807e805762SChris Wilson break; 1381f0c02c1bSTvrtko Ursulin } 1382f0c02c1bSTvrtko Ursulin 1383e310b435SChris Wilson if (READ_ONCE(*tl->hwsp_seqno) != count) { 1384e310b435SChris Wilson GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n", 1385e310b435SChris Wilson count, tl->fence_context, 1386e310b435SChris Wilson tl->hwsp_offset, *tl->hwsp_seqno); 1387d45171acSChris Wilson GEM_TRACE_DUMP(); 1388f0c02c1bSTvrtko Ursulin err = -EINVAL; 1389f0c02c1bSTvrtko Ursulin } 1390f0c02c1bSTvrtko Ursulin 13917e805762SChris Wilson i915_request_put(rq); 1392f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 1393f0c02c1bSTvrtko Ursulin count++; 1394f0c02c1bSTvrtko Ursulin 1395f0c02c1bSTvrtko Ursulin if (err) 13967e805762SChris Wilson break; 1397f0c02c1bSTvrtko Ursulin } while (!__igt_timeout(end_time, NULL)); 1398f0c02c1bSTvrtko Ursulin 13997e805762SChris Wilson intel_engine_pm_put(engine); 14007e805762SChris Wilson if (err) 14017e805762SChris Wilson break; 14027e805762SChris Wilson } 1403f0c02c1bSTvrtko Ursulin 1404f0c02c1bSTvrtko Ursulin return err; 1405f0c02c1bSTvrtko Ursulin } 1406f0c02c1bSTvrtko Ursulin 1407f0c02c1bSTvrtko Ursulin int intel_timeline_live_selftests(struct drm_i915_private *i915) 1408f0c02c1bSTvrtko Ursulin { 1409f0c02c1bSTvrtko Ursulin static const struct i915_subtest tests[] = { 1410f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_recycle), 1411f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_engine), 1412f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_alternate), 1413f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_wrap), 14146e7a21e7SChris Wilson SUBTEST(live_hwsp_read), 1415bb5e4397SChris Wilson SUBTEST(live_hwsp_rollover_kernel), 1416bb5e4397SChris Wilson SUBTEST(live_hwsp_rollover_user), 1417f0c02c1bSTvrtko Ursulin }; 1418f0c02c1bSTvrtko Ursulin 1419*c14adcbdSMichał Winiarski if (intel_gt_is_wedged(to_gt(i915))) 1420f0c02c1bSTvrtko Ursulin return 0; 1421f0c02c1bSTvrtko Ursulin 1422*c14adcbdSMichał Winiarski return intel_gt_live_subtests(tests, to_gt(i915)); 1423f0c02c1bSTvrtko Ursulin } 1424