1f0c02c1bSTvrtko Ursulin /* 2f0c02c1bSTvrtko Ursulin * SPDX-License-Identifier: MIT 3f0c02c1bSTvrtko Ursulin * 4f0c02c1bSTvrtko Ursulin * Copyright © 2017-2018 Intel Corporation 5f0c02c1bSTvrtko Ursulin */ 6f0c02c1bSTvrtko Ursulin 7f0c02c1bSTvrtko Ursulin #include <linux/prime_numbers.h> 8f0c02c1bSTvrtko Ursulin 97e805762SChris Wilson #include "intel_engine_pm.h" 10cb823ed9SChris Wilson #include "intel_gt.h" 11f0c02c1bSTvrtko Ursulin 12f0c02c1bSTvrtko Ursulin #include "../selftests/i915_random.h" 13f0c02c1bSTvrtko Ursulin #include "../i915_selftest.h" 14f0c02c1bSTvrtko Ursulin 15f0c02c1bSTvrtko Ursulin #include "../selftests/igt_flush_test.h" 16f0c02c1bSTvrtko Ursulin #include "../selftests/mock_gem_device.h" 17f0c02c1bSTvrtko Ursulin #include "selftests/mock_timeline.h" 18f0c02c1bSTvrtko Ursulin 19f0c02c1bSTvrtko Ursulin static struct page *hwsp_page(struct intel_timeline *tl) 20f0c02c1bSTvrtko Ursulin { 21f0c02c1bSTvrtko Ursulin struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; 22f0c02c1bSTvrtko Ursulin 23f0c02c1bSTvrtko Ursulin GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 24f0c02c1bSTvrtko Ursulin return sg_page(obj->mm.pages->sgl); 25f0c02c1bSTvrtko Ursulin } 26f0c02c1bSTvrtko Ursulin 27f0c02c1bSTvrtko Ursulin static unsigned long hwsp_cacheline(struct intel_timeline *tl) 28f0c02c1bSTvrtko Ursulin { 29f0c02c1bSTvrtko Ursulin unsigned long address = (unsigned long)page_address(hwsp_page(tl)); 30f0c02c1bSTvrtko Ursulin 31f0c02c1bSTvrtko Ursulin return (address + tl->hwsp_offset) / CACHELINE_BYTES; 32f0c02c1bSTvrtko Ursulin } 33f0c02c1bSTvrtko Ursulin 34f0c02c1bSTvrtko Ursulin #define CACHELINES_PER_PAGE (PAGE_SIZE / CACHELINE_BYTES) 35f0c02c1bSTvrtko Ursulin 36f0c02c1bSTvrtko Ursulin struct mock_hwsp_freelist { 37f0c02c1bSTvrtko Ursulin struct drm_i915_private *i915; 38f0c02c1bSTvrtko Ursulin struct radix_tree_root cachelines; 39f0c02c1bSTvrtko Ursulin struct intel_timeline **history; 40f0c02c1bSTvrtko Ursulin unsigned long count, max; 41f0c02c1bSTvrtko Ursulin struct rnd_state prng; 42f0c02c1bSTvrtko Ursulin }; 43f0c02c1bSTvrtko Ursulin 44f0c02c1bSTvrtko Ursulin enum { 45f0c02c1bSTvrtko Ursulin SHUFFLE = BIT(0), 46f0c02c1bSTvrtko Ursulin }; 47f0c02c1bSTvrtko Ursulin 48f0c02c1bSTvrtko Ursulin static void __mock_hwsp_record(struct mock_hwsp_freelist *state, 49f0c02c1bSTvrtko Ursulin unsigned int idx, 50f0c02c1bSTvrtko Ursulin struct intel_timeline *tl) 51f0c02c1bSTvrtko Ursulin { 52f0c02c1bSTvrtko Ursulin tl = xchg(&state->history[idx], tl); 53f0c02c1bSTvrtko Ursulin if (tl) { 54f0c02c1bSTvrtko Ursulin radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); 55f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 56f0c02c1bSTvrtko Ursulin } 57f0c02c1bSTvrtko Ursulin } 58f0c02c1bSTvrtko Ursulin 59f0c02c1bSTvrtko Ursulin static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, 60f0c02c1bSTvrtko Ursulin unsigned int count, 61f0c02c1bSTvrtko Ursulin unsigned int flags) 62f0c02c1bSTvrtko Ursulin { 63f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 64f0c02c1bSTvrtko Ursulin unsigned int idx; 65f0c02c1bSTvrtko Ursulin 66f0c02c1bSTvrtko Ursulin while (count--) { 67f0c02c1bSTvrtko Ursulin unsigned long cacheline; 68f0c02c1bSTvrtko Ursulin int err; 69f0c02c1bSTvrtko Ursulin 70f0c02c1bSTvrtko Ursulin tl = intel_timeline_create(&state->i915->gt, NULL); 71f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) 72f0c02c1bSTvrtko Ursulin return PTR_ERR(tl); 73f0c02c1bSTvrtko Ursulin 74f0c02c1bSTvrtko Ursulin cacheline = hwsp_cacheline(tl); 75f0c02c1bSTvrtko Ursulin err = radix_tree_insert(&state->cachelines, cacheline, tl); 76f0c02c1bSTvrtko Ursulin if (err) { 77f0c02c1bSTvrtko Ursulin if (err == -EEXIST) { 78f0c02c1bSTvrtko Ursulin pr_err("HWSP cacheline %lu already used; duplicate allocation!\n", 79f0c02c1bSTvrtko Ursulin cacheline); 80f0c02c1bSTvrtko Ursulin } 81f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 82f0c02c1bSTvrtko Ursulin return err; 83f0c02c1bSTvrtko Ursulin } 84f0c02c1bSTvrtko Ursulin 85f0c02c1bSTvrtko Ursulin idx = state->count++ % state->max; 86f0c02c1bSTvrtko Ursulin __mock_hwsp_record(state, idx, tl); 87f0c02c1bSTvrtko Ursulin } 88f0c02c1bSTvrtko Ursulin 89f0c02c1bSTvrtko Ursulin if (flags & SHUFFLE) 90f0c02c1bSTvrtko Ursulin i915_prandom_shuffle(state->history, 91f0c02c1bSTvrtko Ursulin sizeof(*state->history), 92f0c02c1bSTvrtko Ursulin min(state->count, state->max), 93f0c02c1bSTvrtko Ursulin &state->prng); 94f0c02c1bSTvrtko Ursulin 95f0c02c1bSTvrtko Ursulin count = i915_prandom_u32_max_state(min(state->count, state->max), 96f0c02c1bSTvrtko Ursulin &state->prng); 97f0c02c1bSTvrtko Ursulin while (count--) { 98f0c02c1bSTvrtko Ursulin idx = --state->count % state->max; 99f0c02c1bSTvrtko Ursulin __mock_hwsp_record(state, idx, NULL); 100f0c02c1bSTvrtko Ursulin } 101f0c02c1bSTvrtko Ursulin 102f0c02c1bSTvrtko Ursulin return 0; 103f0c02c1bSTvrtko Ursulin } 104f0c02c1bSTvrtko Ursulin 105f0c02c1bSTvrtko Ursulin static int mock_hwsp_freelist(void *arg) 106f0c02c1bSTvrtko Ursulin { 107f0c02c1bSTvrtko Ursulin struct mock_hwsp_freelist state; 108f0c02c1bSTvrtko Ursulin const struct { 109f0c02c1bSTvrtko Ursulin const char *name; 110f0c02c1bSTvrtko Ursulin unsigned int flags; 111f0c02c1bSTvrtko Ursulin } phases[] = { 112f0c02c1bSTvrtko Ursulin { "linear", 0 }, 113f0c02c1bSTvrtko Ursulin { "shuffled", SHUFFLE }, 114f0c02c1bSTvrtko Ursulin { }, 115f0c02c1bSTvrtko Ursulin }, *p; 116f0c02c1bSTvrtko Ursulin unsigned int na; 117f0c02c1bSTvrtko Ursulin int err = 0; 118f0c02c1bSTvrtko Ursulin 119f0c02c1bSTvrtko Ursulin INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL); 120f0c02c1bSTvrtko Ursulin state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed); 121f0c02c1bSTvrtko Ursulin 122f0c02c1bSTvrtko Ursulin state.i915 = mock_gem_device(); 123f0c02c1bSTvrtko Ursulin if (!state.i915) 124f0c02c1bSTvrtko Ursulin return -ENOMEM; 125f0c02c1bSTvrtko Ursulin 126f0c02c1bSTvrtko Ursulin /* 127f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check that their HWSP do not overlap. 128f0c02c1bSTvrtko Ursulin * Free some, and try again. 129f0c02c1bSTvrtko Ursulin */ 130f0c02c1bSTvrtko Ursulin 131f0c02c1bSTvrtko Ursulin state.max = PAGE_SIZE / sizeof(*state.history); 132f0c02c1bSTvrtko Ursulin state.count = 0; 133f0c02c1bSTvrtko Ursulin state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL); 134f0c02c1bSTvrtko Ursulin if (!state.history) { 135f0c02c1bSTvrtko Ursulin err = -ENOMEM; 136f0c02c1bSTvrtko Ursulin goto err_put; 137f0c02c1bSTvrtko Ursulin } 138f0c02c1bSTvrtko Ursulin 139f0c02c1bSTvrtko Ursulin for (p = phases; p->name; p++) { 140f0c02c1bSTvrtko Ursulin pr_debug("%s(%s)\n", __func__, p->name); 141f0c02c1bSTvrtko Ursulin for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) { 142f0c02c1bSTvrtko Ursulin err = __mock_hwsp_timeline(&state, na, p->flags); 143f0c02c1bSTvrtko Ursulin if (err) 144f0c02c1bSTvrtko Ursulin goto out; 145f0c02c1bSTvrtko Ursulin } 146f0c02c1bSTvrtko Ursulin } 147f0c02c1bSTvrtko Ursulin 148f0c02c1bSTvrtko Ursulin out: 149f0c02c1bSTvrtko Ursulin for (na = 0; na < state.max; na++) 150f0c02c1bSTvrtko Ursulin __mock_hwsp_record(&state, na, NULL); 151f0c02c1bSTvrtko Ursulin kfree(state.history); 152f0c02c1bSTvrtko Ursulin err_put: 153f0c02c1bSTvrtko Ursulin drm_dev_put(&state.i915->drm); 154f0c02c1bSTvrtko Ursulin return err; 155f0c02c1bSTvrtko Ursulin } 156f0c02c1bSTvrtko Ursulin 157f0c02c1bSTvrtko Ursulin struct __igt_sync { 158f0c02c1bSTvrtko Ursulin const char *name; 159f0c02c1bSTvrtko Ursulin u32 seqno; 160f0c02c1bSTvrtko Ursulin bool expected; 161f0c02c1bSTvrtko Ursulin bool set; 162f0c02c1bSTvrtko Ursulin }; 163f0c02c1bSTvrtko Ursulin 164f0c02c1bSTvrtko Ursulin static int __igt_sync(struct intel_timeline *tl, 165f0c02c1bSTvrtko Ursulin u64 ctx, 166f0c02c1bSTvrtko Ursulin const struct __igt_sync *p, 167f0c02c1bSTvrtko Ursulin const char *name) 168f0c02c1bSTvrtko Ursulin { 169f0c02c1bSTvrtko Ursulin int ret; 170f0c02c1bSTvrtko Ursulin 171f0c02c1bSTvrtko Ursulin if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) { 172f0c02c1bSTvrtko Ursulin pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n", 173f0c02c1bSTvrtko Ursulin name, p->name, ctx, p->seqno, yesno(p->expected)); 174f0c02c1bSTvrtko Ursulin return -EINVAL; 175f0c02c1bSTvrtko Ursulin } 176f0c02c1bSTvrtko Ursulin 177f0c02c1bSTvrtko Ursulin if (p->set) { 178f0c02c1bSTvrtko Ursulin ret = __intel_timeline_sync_set(tl, ctx, p->seqno); 179f0c02c1bSTvrtko Ursulin if (ret) 180f0c02c1bSTvrtko Ursulin return ret; 181f0c02c1bSTvrtko Ursulin } 182f0c02c1bSTvrtko Ursulin 183f0c02c1bSTvrtko Ursulin return 0; 184f0c02c1bSTvrtko Ursulin } 185f0c02c1bSTvrtko Ursulin 186f0c02c1bSTvrtko Ursulin static int igt_sync(void *arg) 187f0c02c1bSTvrtko Ursulin { 188f0c02c1bSTvrtko Ursulin const struct __igt_sync pass[] = { 189f0c02c1bSTvrtko Ursulin { "unset", 0, false, false }, 190f0c02c1bSTvrtko Ursulin { "new", 0, false, true }, 191f0c02c1bSTvrtko Ursulin { "0a", 0, true, true }, 192f0c02c1bSTvrtko Ursulin { "1a", 1, false, true }, 193f0c02c1bSTvrtko Ursulin { "1b", 1, true, true }, 194f0c02c1bSTvrtko Ursulin { "0b", 0, true, false }, 195f0c02c1bSTvrtko Ursulin { "2a", 2, false, true }, 196f0c02c1bSTvrtko Ursulin { "4", 4, false, true }, 197f0c02c1bSTvrtko Ursulin { "INT_MAX", INT_MAX, false, true }, 198f0c02c1bSTvrtko Ursulin { "INT_MAX-1", INT_MAX-1, true, false }, 199f0c02c1bSTvrtko Ursulin { "INT_MAX+1", (u32)INT_MAX+1, false, true }, 200f0c02c1bSTvrtko Ursulin { "INT_MAX", INT_MAX, true, false }, 201f0c02c1bSTvrtko Ursulin { "UINT_MAX", UINT_MAX, false, true }, 202f0c02c1bSTvrtko Ursulin { "wrap", 0, false, true }, 203f0c02c1bSTvrtko Ursulin { "unwrap", UINT_MAX, true, false }, 204f0c02c1bSTvrtko Ursulin {}, 205f0c02c1bSTvrtko Ursulin }, *p; 206f0c02c1bSTvrtko Ursulin struct intel_timeline tl; 207f0c02c1bSTvrtko Ursulin int order, offset; 208f0c02c1bSTvrtko Ursulin int ret = -ENODEV; 209f0c02c1bSTvrtko Ursulin 210f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 211f0c02c1bSTvrtko Ursulin for (p = pass; p->name; p++) { 212f0c02c1bSTvrtko Ursulin for (order = 1; order < 64; order++) { 213f0c02c1bSTvrtko Ursulin for (offset = -1; offset <= (order > 1); offset++) { 214f0c02c1bSTvrtko Ursulin u64 ctx = BIT_ULL(order) + offset; 215f0c02c1bSTvrtko Ursulin 216f0c02c1bSTvrtko Ursulin ret = __igt_sync(&tl, ctx, p, "1"); 217f0c02c1bSTvrtko Ursulin if (ret) 218f0c02c1bSTvrtko Ursulin goto out; 219f0c02c1bSTvrtko Ursulin } 220f0c02c1bSTvrtko Ursulin } 221f0c02c1bSTvrtko Ursulin } 222f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 223f0c02c1bSTvrtko Ursulin 224f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 225f0c02c1bSTvrtko Ursulin for (order = 1; order < 64; order++) { 226f0c02c1bSTvrtko Ursulin for (offset = -1; offset <= (order > 1); offset++) { 227f0c02c1bSTvrtko Ursulin u64 ctx = BIT_ULL(order) + offset; 228f0c02c1bSTvrtko Ursulin 229f0c02c1bSTvrtko Ursulin for (p = pass; p->name; p++) { 230f0c02c1bSTvrtko Ursulin ret = __igt_sync(&tl, ctx, p, "2"); 231f0c02c1bSTvrtko Ursulin if (ret) 232f0c02c1bSTvrtko Ursulin goto out; 233f0c02c1bSTvrtko Ursulin } 234f0c02c1bSTvrtko Ursulin } 235f0c02c1bSTvrtko Ursulin } 236f0c02c1bSTvrtko Ursulin 237f0c02c1bSTvrtko Ursulin out: 238f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 239f0c02c1bSTvrtko Ursulin return ret; 240f0c02c1bSTvrtko Ursulin } 241f0c02c1bSTvrtko Ursulin 242f0c02c1bSTvrtko Ursulin static unsigned int random_engine(struct rnd_state *rnd) 243f0c02c1bSTvrtko Ursulin { 244f0c02c1bSTvrtko Ursulin return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd); 245f0c02c1bSTvrtko Ursulin } 246f0c02c1bSTvrtko Ursulin 247f0c02c1bSTvrtko Ursulin static int bench_sync(void *arg) 248f0c02c1bSTvrtko Ursulin { 249f0c02c1bSTvrtko Ursulin struct rnd_state prng; 250f0c02c1bSTvrtko Ursulin struct intel_timeline tl; 251f0c02c1bSTvrtko Ursulin unsigned long end_time, count; 252f0c02c1bSTvrtko Ursulin u64 prng32_1M; 253f0c02c1bSTvrtko Ursulin ktime_t kt; 254f0c02c1bSTvrtko Ursulin int order, last_order; 255f0c02c1bSTvrtko Ursulin 256f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 257f0c02c1bSTvrtko Ursulin 258f0c02c1bSTvrtko Ursulin /* Lookups from cache are very fast and so the random number generation 259f0c02c1bSTvrtko Ursulin * and the loop itself becomes a significant factor in the per-iteration 260f0c02c1bSTvrtko Ursulin * timings. We try to compensate the results by measuring the overhead 261f0c02c1bSTvrtko Ursulin * of the prng and subtract it from the reported results. 262f0c02c1bSTvrtko Ursulin */ 263f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 264f0c02c1bSTvrtko Ursulin count = 0; 265f0c02c1bSTvrtko Ursulin kt = ktime_get(); 266f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 267f0c02c1bSTvrtko Ursulin do { 268f0c02c1bSTvrtko Ursulin u32 x; 269f0c02c1bSTvrtko Ursulin 270f0c02c1bSTvrtko Ursulin /* Make sure the compiler doesn't optimise away the prng call */ 271f0c02c1bSTvrtko Ursulin WRITE_ONCE(x, prandom_u32_state(&prng)); 272f0c02c1bSTvrtko Ursulin 273f0c02c1bSTvrtko Ursulin count++; 274f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 275f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 276f0c02c1bSTvrtko Ursulin pr_debug("%s: %lu random evaluations, %lluns/prng\n", 277f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 278f0c02c1bSTvrtko Ursulin prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count); 279f0c02c1bSTvrtko Ursulin 280f0c02c1bSTvrtko Ursulin /* Benchmark (only) setting random context ids */ 281f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 282f0c02c1bSTvrtko Ursulin count = 0; 283f0c02c1bSTvrtko Ursulin kt = ktime_get(); 284f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 285f0c02c1bSTvrtko Ursulin do { 286f0c02c1bSTvrtko Ursulin u64 id = i915_prandom_u64_state(&prng); 287f0c02c1bSTvrtko Ursulin 288f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, 0); 289f0c02c1bSTvrtko Ursulin count++; 290f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 291f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 292f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 293f0c02c1bSTvrtko Ursulin pr_info("%s: %lu random insertions, %lluns/insert\n", 294f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 295f0c02c1bSTvrtko Ursulin 296f0c02c1bSTvrtko Ursulin /* Benchmark looking up the exact same context ids as we just set */ 297f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 298f0c02c1bSTvrtko Ursulin end_time = count; 299f0c02c1bSTvrtko Ursulin kt = ktime_get(); 300f0c02c1bSTvrtko Ursulin while (end_time--) { 301f0c02c1bSTvrtko Ursulin u64 id = i915_prandom_u64_state(&prng); 302f0c02c1bSTvrtko Ursulin 303f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, id, 0)) { 304f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 305f0c02c1bSTvrtko Ursulin pr_err("Lookup of %llu failed\n", id); 306f0c02c1bSTvrtko Ursulin return -EINVAL; 307f0c02c1bSTvrtko Ursulin } 308f0c02c1bSTvrtko Ursulin } 309f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 310f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 311f0c02c1bSTvrtko Ursulin pr_info("%s: %lu random lookups, %lluns/lookup\n", 312f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 313f0c02c1bSTvrtko Ursulin 314f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 315f0c02c1bSTvrtko Ursulin cond_resched(); 316f0c02c1bSTvrtko Ursulin 317f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 318f0c02c1bSTvrtko Ursulin 319f0c02c1bSTvrtko Ursulin /* Benchmark setting the first N (in order) contexts */ 320f0c02c1bSTvrtko Ursulin count = 0; 321f0c02c1bSTvrtko Ursulin kt = ktime_get(); 322f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 323f0c02c1bSTvrtko Ursulin do { 324f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, count++, 0); 325f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 326f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 327f0c02c1bSTvrtko Ursulin pr_info("%s: %lu in-order insertions, %lluns/insert\n", 328f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 329f0c02c1bSTvrtko Ursulin 330f0c02c1bSTvrtko Ursulin /* Benchmark looking up the exact same context ids as we just set */ 331f0c02c1bSTvrtko Ursulin end_time = count; 332f0c02c1bSTvrtko Ursulin kt = ktime_get(); 333f0c02c1bSTvrtko Ursulin while (end_time--) { 334f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) { 335f0c02c1bSTvrtko Ursulin pr_err("Lookup of %lu failed\n", end_time); 336f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 337f0c02c1bSTvrtko Ursulin return -EINVAL; 338f0c02c1bSTvrtko Ursulin } 339f0c02c1bSTvrtko Ursulin } 340f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 341f0c02c1bSTvrtko Ursulin pr_info("%s: %lu in-order lookups, %lluns/lookup\n", 342f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 343f0c02c1bSTvrtko Ursulin 344f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 345f0c02c1bSTvrtko Ursulin cond_resched(); 346f0c02c1bSTvrtko Ursulin 347f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 348f0c02c1bSTvrtko Ursulin 349f0c02c1bSTvrtko Ursulin /* Benchmark searching for a random context id and maybe changing it */ 350f0c02c1bSTvrtko Ursulin prandom_seed_state(&prng, i915_selftest.random_seed); 351f0c02c1bSTvrtko Ursulin count = 0; 352f0c02c1bSTvrtko Ursulin kt = ktime_get(); 353f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 354f0c02c1bSTvrtko Ursulin do { 355f0c02c1bSTvrtko Ursulin u32 id = random_engine(&prng); 356f0c02c1bSTvrtko Ursulin u32 seqno = prandom_u32_state(&prng); 357f0c02c1bSTvrtko Ursulin 358f0c02c1bSTvrtko Ursulin if (!__intel_timeline_sync_is_later(&tl, id, seqno)) 359f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, seqno); 360f0c02c1bSTvrtko Ursulin 361f0c02c1bSTvrtko Ursulin count++; 362f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 363f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 364f0c02c1bSTvrtko Ursulin kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); 365f0c02c1bSTvrtko Ursulin pr_info("%s: %lu repeated insert/lookups, %lluns/op\n", 366f0c02c1bSTvrtko Ursulin __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); 367f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 368f0c02c1bSTvrtko Ursulin cond_resched(); 369f0c02c1bSTvrtko Ursulin 370f0c02c1bSTvrtko Ursulin /* Benchmark searching for a known context id and changing the seqno */ 371f0c02c1bSTvrtko Ursulin for (last_order = 1, order = 1; order < 32; 372f0c02c1bSTvrtko Ursulin ({ int tmp = last_order; last_order = order; order += tmp; })) { 373f0c02c1bSTvrtko Ursulin unsigned int mask = BIT(order) - 1; 374f0c02c1bSTvrtko Ursulin 375f0c02c1bSTvrtko Ursulin mock_timeline_init(&tl, 0); 376f0c02c1bSTvrtko Ursulin 377f0c02c1bSTvrtko Ursulin count = 0; 378f0c02c1bSTvrtko Ursulin kt = ktime_get(); 379f0c02c1bSTvrtko Ursulin end_time = jiffies + HZ/10; 380f0c02c1bSTvrtko Ursulin do { 381f0c02c1bSTvrtko Ursulin /* Without assuming too many details of the underlying 382f0c02c1bSTvrtko Ursulin * implementation, try to identify its phase-changes 383f0c02c1bSTvrtko Ursulin * (if any)! 384f0c02c1bSTvrtko Ursulin */ 385f0c02c1bSTvrtko Ursulin u64 id = (u64)(count & mask) << order; 386f0c02c1bSTvrtko Ursulin 387f0c02c1bSTvrtko Ursulin __intel_timeline_sync_is_later(&tl, id, 0); 388f0c02c1bSTvrtko Ursulin __intel_timeline_sync_set(&tl, id, 0); 389f0c02c1bSTvrtko Ursulin 390f0c02c1bSTvrtko Ursulin count++; 391f0c02c1bSTvrtko Ursulin } while (!time_after(jiffies, end_time)); 392f0c02c1bSTvrtko Ursulin kt = ktime_sub(ktime_get(), kt); 393f0c02c1bSTvrtko Ursulin pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n", 394f0c02c1bSTvrtko Ursulin __func__, count, order, 395f0c02c1bSTvrtko Ursulin (long long)div64_ul(ktime_to_ns(kt), count)); 396f0c02c1bSTvrtko Ursulin mock_timeline_fini(&tl); 397f0c02c1bSTvrtko Ursulin cond_resched(); 398f0c02c1bSTvrtko Ursulin } 399f0c02c1bSTvrtko Ursulin 400f0c02c1bSTvrtko Ursulin return 0; 401f0c02c1bSTvrtko Ursulin } 402f0c02c1bSTvrtko Ursulin 403f0c02c1bSTvrtko Ursulin int intel_timeline_mock_selftests(void) 404f0c02c1bSTvrtko Ursulin { 405f0c02c1bSTvrtko Ursulin static const struct i915_subtest tests[] = { 406f0c02c1bSTvrtko Ursulin SUBTEST(mock_hwsp_freelist), 407f0c02c1bSTvrtko Ursulin SUBTEST(igt_sync), 408f0c02c1bSTvrtko Ursulin SUBTEST(bench_sync), 409f0c02c1bSTvrtko Ursulin }; 410f0c02c1bSTvrtko Ursulin 411f0c02c1bSTvrtko Ursulin return i915_subtests(tests, NULL); 412f0c02c1bSTvrtko Ursulin } 413f0c02c1bSTvrtko Ursulin 414f0c02c1bSTvrtko Ursulin static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) 415f0c02c1bSTvrtko Ursulin { 416f0c02c1bSTvrtko Ursulin u32 *cs; 417f0c02c1bSTvrtko Ursulin 418f0c02c1bSTvrtko Ursulin cs = intel_ring_begin(rq, 4); 419f0c02c1bSTvrtko Ursulin if (IS_ERR(cs)) 420f0c02c1bSTvrtko Ursulin return PTR_ERR(cs); 421f0c02c1bSTvrtko Ursulin 422f0c02c1bSTvrtko Ursulin if (INTEL_GEN(rq->i915) >= 8) { 423f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 424f0c02c1bSTvrtko Ursulin *cs++ = addr; 425f0c02c1bSTvrtko Ursulin *cs++ = 0; 426f0c02c1bSTvrtko Ursulin *cs++ = value; 427f0c02c1bSTvrtko Ursulin } else if (INTEL_GEN(rq->i915) >= 4) { 428f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 429f0c02c1bSTvrtko Ursulin *cs++ = 0; 430f0c02c1bSTvrtko Ursulin *cs++ = addr; 431f0c02c1bSTvrtko Ursulin *cs++ = value; 432f0c02c1bSTvrtko Ursulin } else { 433f0c02c1bSTvrtko Ursulin *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 434f0c02c1bSTvrtko Ursulin *cs++ = addr; 435f0c02c1bSTvrtko Ursulin *cs++ = value; 436f0c02c1bSTvrtko Ursulin *cs++ = MI_NOOP; 437f0c02c1bSTvrtko Ursulin } 438f0c02c1bSTvrtko Ursulin 439f0c02c1bSTvrtko Ursulin intel_ring_advance(rq, cs); 440f0c02c1bSTvrtko Ursulin 441f0c02c1bSTvrtko Ursulin return 0; 442f0c02c1bSTvrtko Ursulin } 443f0c02c1bSTvrtko Ursulin 444f0c02c1bSTvrtko Ursulin static struct i915_request * 445f0c02c1bSTvrtko Ursulin tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value) 446f0c02c1bSTvrtko Ursulin { 447f0c02c1bSTvrtko Ursulin struct i915_request *rq; 448f0c02c1bSTvrtko Ursulin int err; 449f0c02c1bSTvrtko Ursulin 450f0c02c1bSTvrtko Ursulin err = intel_timeline_pin(tl); 451f0c02c1bSTvrtko Ursulin if (err) { 452f0c02c1bSTvrtko Ursulin rq = ERR_PTR(err); 453f0c02c1bSTvrtko Ursulin goto out; 454f0c02c1bSTvrtko Ursulin } 455f0c02c1bSTvrtko Ursulin 456f0c02c1bSTvrtko Ursulin rq = i915_request_create(engine->kernel_context); 457f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) 458f0c02c1bSTvrtko Ursulin goto out_unpin; 459f0c02c1bSTvrtko Ursulin 4607e805762SChris Wilson i915_request_get(rq); 4617e805762SChris Wilson 462f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); 463f0c02c1bSTvrtko Ursulin i915_request_add(rq); 4647e805762SChris Wilson if (err) { 4657e805762SChris Wilson i915_request_put(rq); 466f0c02c1bSTvrtko Ursulin rq = ERR_PTR(err); 4677e805762SChris Wilson } 468f0c02c1bSTvrtko Ursulin 469f0c02c1bSTvrtko Ursulin out_unpin: 470f0c02c1bSTvrtko Ursulin intel_timeline_unpin(tl); 471f0c02c1bSTvrtko Ursulin out: 472f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) 473f0c02c1bSTvrtko Ursulin pr_err("Failed to write to timeline!\n"); 474f0c02c1bSTvrtko Ursulin return rq; 475f0c02c1bSTvrtko Ursulin } 476f0c02c1bSTvrtko Ursulin 477f0c02c1bSTvrtko Ursulin static struct intel_timeline * 478f0c02c1bSTvrtko Ursulin checked_intel_timeline_create(struct drm_i915_private *i915) 479f0c02c1bSTvrtko Ursulin { 480f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 481f0c02c1bSTvrtko Ursulin 482f0c02c1bSTvrtko Ursulin tl = intel_timeline_create(&i915->gt, NULL); 483f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) 484f0c02c1bSTvrtko Ursulin return tl; 485f0c02c1bSTvrtko Ursulin 486f0c02c1bSTvrtko Ursulin if (*tl->hwsp_seqno != tl->seqno) { 487f0c02c1bSTvrtko Ursulin pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", 488f0c02c1bSTvrtko Ursulin *tl->hwsp_seqno, tl->seqno); 489f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 490f0c02c1bSTvrtko Ursulin return ERR_PTR(-EINVAL); 491f0c02c1bSTvrtko Ursulin } 492f0c02c1bSTvrtko Ursulin 493f0c02c1bSTvrtko Ursulin return tl; 494f0c02c1bSTvrtko Ursulin } 495f0c02c1bSTvrtko Ursulin 496f0c02c1bSTvrtko Ursulin static int live_hwsp_engine(void *arg) 497f0c02c1bSTvrtko Ursulin { 498f0c02c1bSTvrtko Ursulin #define NUM_TIMELINES 4096 499f0c02c1bSTvrtko Ursulin struct drm_i915_private *i915 = arg; 500f0c02c1bSTvrtko Ursulin struct intel_timeline **timelines; 501f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 502f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 503f0c02c1bSTvrtko Ursulin unsigned long count, n; 504f0c02c1bSTvrtko Ursulin int err = 0; 505f0c02c1bSTvrtko Ursulin 506f0c02c1bSTvrtko Ursulin /* 507f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check we can write 508f0c02c1bSTvrtko Ursulin * independently to each of their breadcrumb slots. 509f0c02c1bSTvrtko Ursulin */ 510f0c02c1bSTvrtko Ursulin 511f0c02c1bSTvrtko Ursulin timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, 512f0c02c1bSTvrtko Ursulin sizeof(*timelines), 513f0c02c1bSTvrtko Ursulin GFP_KERNEL); 514f0c02c1bSTvrtko Ursulin if (!timelines) 515f0c02c1bSTvrtko Ursulin return -ENOMEM; 516f0c02c1bSTvrtko Ursulin 517f0c02c1bSTvrtko Ursulin count = 0; 518f0c02c1bSTvrtko Ursulin for_each_engine(engine, i915, id) { 519f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 520f0c02c1bSTvrtko Ursulin continue; 521f0c02c1bSTvrtko Ursulin 5227e805762SChris Wilson intel_engine_pm_get(engine); 5237e805762SChris Wilson 524f0c02c1bSTvrtko Ursulin for (n = 0; n < NUM_TIMELINES; n++) { 525f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 526f0c02c1bSTvrtko Ursulin struct i915_request *rq; 527f0c02c1bSTvrtko Ursulin 528f0c02c1bSTvrtko Ursulin tl = checked_intel_timeline_create(i915); 529f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 530f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 5317e805762SChris Wilson break; 532f0c02c1bSTvrtko Ursulin } 533f0c02c1bSTvrtko Ursulin 534f0c02c1bSTvrtko Ursulin rq = tl_write(tl, engine, count); 535f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 536f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 537f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 5387e805762SChris Wilson break; 539f0c02c1bSTvrtko Ursulin } 540f0c02c1bSTvrtko Ursulin 541f0c02c1bSTvrtko Ursulin timelines[count++] = tl; 5427e805762SChris Wilson i915_request_put(rq); 543f0c02c1bSTvrtko Ursulin } 544f0c02c1bSTvrtko Ursulin 5457e805762SChris Wilson intel_engine_pm_put(engine); 5467e805762SChris Wilson if (err) 5477e805762SChris Wilson break; 5487e805762SChris Wilson } 5497e805762SChris Wilson 5507e805762SChris Wilson if (igt_flush_test(i915)) 551f0c02c1bSTvrtko Ursulin err = -EIO; 552f0c02c1bSTvrtko Ursulin 553f0c02c1bSTvrtko Ursulin for (n = 0; n < count; n++) { 554f0c02c1bSTvrtko Ursulin struct intel_timeline *tl = timelines[n]; 555f0c02c1bSTvrtko Ursulin 556f0c02c1bSTvrtko Ursulin if (!err && *tl->hwsp_seqno != n) { 557f0c02c1bSTvrtko Ursulin pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", 558f0c02c1bSTvrtko Ursulin n, *tl->hwsp_seqno); 559f0c02c1bSTvrtko Ursulin err = -EINVAL; 560f0c02c1bSTvrtko Ursulin } 561f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 562f0c02c1bSTvrtko Ursulin } 563f0c02c1bSTvrtko Ursulin 564f0c02c1bSTvrtko Ursulin kvfree(timelines); 565f0c02c1bSTvrtko Ursulin return err; 566f0c02c1bSTvrtko Ursulin #undef NUM_TIMELINES 567f0c02c1bSTvrtko Ursulin } 568f0c02c1bSTvrtko Ursulin 569f0c02c1bSTvrtko Ursulin static int live_hwsp_alternate(void *arg) 570f0c02c1bSTvrtko Ursulin { 571f0c02c1bSTvrtko Ursulin #define NUM_TIMELINES 4096 572f0c02c1bSTvrtko Ursulin struct drm_i915_private *i915 = arg; 573f0c02c1bSTvrtko Ursulin struct intel_timeline **timelines; 574f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 575f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 576f0c02c1bSTvrtko Ursulin unsigned long count, n; 577f0c02c1bSTvrtko Ursulin int err = 0; 578f0c02c1bSTvrtko Ursulin 579f0c02c1bSTvrtko Ursulin /* 580f0c02c1bSTvrtko Ursulin * Create a bunch of timelines and check we can write 581f0c02c1bSTvrtko Ursulin * independently to each of their breadcrumb slots with adjacent 582f0c02c1bSTvrtko Ursulin * engines. 583f0c02c1bSTvrtko Ursulin */ 584f0c02c1bSTvrtko Ursulin 585f0c02c1bSTvrtko Ursulin timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, 586f0c02c1bSTvrtko Ursulin sizeof(*timelines), 587f0c02c1bSTvrtko Ursulin GFP_KERNEL); 588f0c02c1bSTvrtko Ursulin if (!timelines) 589f0c02c1bSTvrtko Ursulin return -ENOMEM; 590f0c02c1bSTvrtko Ursulin 591f0c02c1bSTvrtko Ursulin count = 0; 592f0c02c1bSTvrtko Ursulin for (n = 0; n < NUM_TIMELINES; n++) { 593f0c02c1bSTvrtko Ursulin for_each_engine(engine, i915, id) { 594f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 595f0c02c1bSTvrtko Ursulin struct i915_request *rq; 596f0c02c1bSTvrtko Ursulin 597f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 598f0c02c1bSTvrtko Ursulin continue; 599f0c02c1bSTvrtko Ursulin 600f0c02c1bSTvrtko Ursulin tl = checked_intel_timeline_create(i915); 601f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 6027e805762SChris Wilson intel_engine_pm_put(engine); 603f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 604f0c02c1bSTvrtko Ursulin goto out; 605f0c02c1bSTvrtko Ursulin } 606f0c02c1bSTvrtko Ursulin 6077e805762SChris Wilson intel_engine_pm_get(engine); 608f0c02c1bSTvrtko Ursulin rq = tl_write(tl, engine, count); 6097e805762SChris Wilson intel_engine_pm_put(engine); 610f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 611f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 612f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 613f0c02c1bSTvrtko Ursulin goto out; 614f0c02c1bSTvrtko Ursulin } 615f0c02c1bSTvrtko Ursulin 616f0c02c1bSTvrtko Ursulin timelines[count++] = tl; 6177e805762SChris Wilson i915_request_put(rq); 618f0c02c1bSTvrtko Ursulin } 619f0c02c1bSTvrtko Ursulin } 620f0c02c1bSTvrtko Ursulin 621f0c02c1bSTvrtko Ursulin out: 6227e805762SChris Wilson if (igt_flush_test(i915)) 623f0c02c1bSTvrtko Ursulin err = -EIO; 624f0c02c1bSTvrtko Ursulin 625f0c02c1bSTvrtko Ursulin for (n = 0; n < count; n++) { 626f0c02c1bSTvrtko Ursulin struct intel_timeline *tl = timelines[n]; 627f0c02c1bSTvrtko Ursulin 628f0c02c1bSTvrtko Ursulin if (!err && *tl->hwsp_seqno != n) { 629f0c02c1bSTvrtko Ursulin pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", 630f0c02c1bSTvrtko Ursulin n, *tl->hwsp_seqno); 631f0c02c1bSTvrtko Ursulin err = -EINVAL; 632f0c02c1bSTvrtko Ursulin } 633f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 634f0c02c1bSTvrtko Ursulin } 635f0c02c1bSTvrtko Ursulin 636f0c02c1bSTvrtko Ursulin kvfree(timelines); 637f0c02c1bSTvrtko Ursulin return err; 638f0c02c1bSTvrtko Ursulin #undef NUM_TIMELINES 639f0c02c1bSTvrtko Ursulin } 640f0c02c1bSTvrtko Ursulin 641f0c02c1bSTvrtko Ursulin static int live_hwsp_wrap(void *arg) 642f0c02c1bSTvrtko Ursulin { 643f0c02c1bSTvrtko Ursulin struct drm_i915_private *i915 = arg; 644f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 645f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 646f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 647f0c02c1bSTvrtko Ursulin int err = 0; 648f0c02c1bSTvrtko Ursulin 649f0c02c1bSTvrtko Ursulin /* 650f0c02c1bSTvrtko Ursulin * Across a seqno wrap, we need to keep the old cacheline alive for 651f0c02c1bSTvrtko Ursulin * foreign GPU references. 652f0c02c1bSTvrtko Ursulin */ 653f0c02c1bSTvrtko Ursulin 654f0c02c1bSTvrtko Ursulin tl = intel_timeline_create(&i915->gt, NULL); 6557e805762SChris Wilson if (IS_ERR(tl)) 6567e805762SChris Wilson return PTR_ERR(tl); 6577e805762SChris Wilson 658f0c02c1bSTvrtko Ursulin if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline) 659f0c02c1bSTvrtko Ursulin goto out_free; 660f0c02c1bSTvrtko Ursulin 661f0c02c1bSTvrtko Ursulin err = intel_timeline_pin(tl); 662f0c02c1bSTvrtko Ursulin if (err) 663f0c02c1bSTvrtko Ursulin goto out_free; 664f0c02c1bSTvrtko Ursulin 665f0c02c1bSTvrtko Ursulin for_each_engine(engine, i915, id) { 666f0c02c1bSTvrtko Ursulin const u32 *hwsp_seqno[2]; 667f0c02c1bSTvrtko Ursulin struct i915_request *rq; 668f0c02c1bSTvrtko Ursulin u32 seqno[2]; 669f0c02c1bSTvrtko Ursulin 670f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 671f0c02c1bSTvrtko Ursulin continue; 672f0c02c1bSTvrtko Ursulin 6737e805762SChris Wilson intel_engine_pm_get(engine); 674f0c02c1bSTvrtko Ursulin rq = i915_request_create(engine->kernel_context); 6757e805762SChris Wilson intel_engine_pm_put(engine); 676f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 677f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 678f0c02c1bSTvrtko Ursulin goto out; 679f0c02c1bSTvrtko Ursulin } 680f0c02c1bSTvrtko Ursulin 681f0c02c1bSTvrtko Ursulin tl->seqno = -4u; 682f0c02c1bSTvrtko Ursulin 68325ffd4b1SChris Wilson mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); 684f0c02c1bSTvrtko Ursulin err = intel_timeline_get_seqno(tl, rq, &seqno[0]); 68525ffd4b1SChris Wilson mutex_unlock(&tl->mutex); 686f0c02c1bSTvrtko Ursulin if (err) { 687f0c02c1bSTvrtko Ursulin i915_request_add(rq); 688f0c02c1bSTvrtko Ursulin goto out; 689f0c02c1bSTvrtko Ursulin } 690f0c02c1bSTvrtko Ursulin pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n", 691f0c02c1bSTvrtko Ursulin seqno[0], tl->hwsp_offset); 692f0c02c1bSTvrtko Ursulin 693f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]); 694f0c02c1bSTvrtko Ursulin if (err) { 695f0c02c1bSTvrtko Ursulin i915_request_add(rq); 696f0c02c1bSTvrtko Ursulin goto out; 697f0c02c1bSTvrtko Ursulin } 698f0c02c1bSTvrtko Ursulin hwsp_seqno[0] = tl->hwsp_seqno; 699f0c02c1bSTvrtko Ursulin 70025ffd4b1SChris Wilson mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); 701f0c02c1bSTvrtko Ursulin err = intel_timeline_get_seqno(tl, rq, &seqno[1]); 70225ffd4b1SChris Wilson mutex_unlock(&tl->mutex); 703f0c02c1bSTvrtko Ursulin if (err) { 704f0c02c1bSTvrtko Ursulin i915_request_add(rq); 705f0c02c1bSTvrtko Ursulin goto out; 706f0c02c1bSTvrtko Ursulin } 707f0c02c1bSTvrtko Ursulin pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n", 708f0c02c1bSTvrtko Ursulin seqno[1], tl->hwsp_offset); 709f0c02c1bSTvrtko Ursulin 710f0c02c1bSTvrtko Ursulin err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]); 711f0c02c1bSTvrtko Ursulin if (err) { 712f0c02c1bSTvrtko Ursulin i915_request_add(rq); 713f0c02c1bSTvrtko Ursulin goto out; 714f0c02c1bSTvrtko Ursulin } 715f0c02c1bSTvrtko Ursulin hwsp_seqno[1] = tl->hwsp_seqno; 716f0c02c1bSTvrtko Ursulin 717f0c02c1bSTvrtko Ursulin /* With wrap should come a new hwsp */ 718f0c02c1bSTvrtko Ursulin GEM_BUG_ON(seqno[1] >= seqno[0]); 719f0c02c1bSTvrtko Ursulin GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]); 720f0c02c1bSTvrtko Ursulin 721f0c02c1bSTvrtko Ursulin i915_request_add(rq); 722f0c02c1bSTvrtko Ursulin 723f0c02c1bSTvrtko Ursulin if (i915_request_wait(rq, 0, HZ / 5) < 0) { 724f0c02c1bSTvrtko Ursulin pr_err("Wait for timeline writes timed out!\n"); 725f0c02c1bSTvrtko Ursulin err = -EIO; 726f0c02c1bSTvrtko Ursulin goto out; 727f0c02c1bSTvrtko Ursulin } 728f0c02c1bSTvrtko Ursulin 729f0c02c1bSTvrtko Ursulin if (*hwsp_seqno[0] != seqno[0] || *hwsp_seqno[1] != seqno[1]) { 730f0c02c1bSTvrtko Ursulin pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n", 731f0c02c1bSTvrtko Ursulin *hwsp_seqno[0], *hwsp_seqno[1], 732f0c02c1bSTvrtko Ursulin seqno[0], seqno[1]); 733f0c02c1bSTvrtko Ursulin err = -EINVAL; 734f0c02c1bSTvrtko Ursulin goto out; 735f0c02c1bSTvrtko Ursulin } 736f0c02c1bSTvrtko Ursulin 737f0c02c1bSTvrtko Ursulin i915_retire_requests(i915); /* recycle HWSP */ 738f0c02c1bSTvrtko Ursulin } 739f0c02c1bSTvrtko Ursulin 740f0c02c1bSTvrtko Ursulin out: 7417e805762SChris Wilson if (igt_flush_test(i915)) 742f0c02c1bSTvrtko Ursulin err = -EIO; 743f0c02c1bSTvrtko Ursulin 744f0c02c1bSTvrtko Ursulin intel_timeline_unpin(tl); 745f0c02c1bSTvrtko Ursulin out_free: 746f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 747f0c02c1bSTvrtko Ursulin return err; 748f0c02c1bSTvrtko Ursulin } 749f0c02c1bSTvrtko Ursulin 750f0c02c1bSTvrtko Ursulin static int live_hwsp_recycle(void *arg) 751f0c02c1bSTvrtko Ursulin { 752f0c02c1bSTvrtko Ursulin struct drm_i915_private *i915 = arg; 753f0c02c1bSTvrtko Ursulin struct intel_engine_cs *engine; 754f0c02c1bSTvrtko Ursulin enum intel_engine_id id; 755f0c02c1bSTvrtko Ursulin unsigned long count; 756f0c02c1bSTvrtko Ursulin int err = 0; 757f0c02c1bSTvrtko Ursulin 758f0c02c1bSTvrtko Ursulin /* 759f0c02c1bSTvrtko Ursulin * Check seqno writes into one timeline at a time. We expect to 760f0c02c1bSTvrtko Ursulin * recycle the breadcrumb slot between iterations and neither 761f0c02c1bSTvrtko Ursulin * want to confuse ourselves or the GPU. 762f0c02c1bSTvrtko Ursulin */ 763f0c02c1bSTvrtko Ursulin 764f0c02c1bSTvrtko Ursulin count = 0; 765f0c02c1bSTvrtko Ursulin for_each_engine(engine, i915, id) { 766f0c02c1bSTvrtko Ursulin IGT_TIMEOUT(end_time); 767f0c02c1bSTvrtko Ursulin 768f0c02c1bSTvrtko Ursulin if (!intel_engine_can_store_dword(engine)) 769f0c02c1bSTvrtko Ursulin continue; 770f0c02c1bSTvrtko Ursulin 7717e805762SChris Wilson intel_engine_pm_get(engine); 7727e805762SChris Wilson 773f0c02c1bSTvrtko Ursulin do { 774f0c02c1bSTvrtko Ursulin struct intel_timeline *tl; 775f0c02c1bSTvrtko Ursulin struct i915_request *rq; 776f0c02c1bSTvrtko Ursulin 777f0c02c1bSTvrtko Ursulin tl = checked_intel_timeline_create(i915); 778f0c02c1bSTvrtko Ursulin if (IS_ERR(tl)) { 779f0c02c1bSTvrtko Ursulin err = PTR_ERR(tl); 7807e805762SChris Wilson break; 781f0c02c1bSTvrtko Ursulin } 782f0c02c1bSTvrtko Ursulin 783f0c02c1bSTvrtko Ursulin rq = tl_write(tl, engine, count); 784f0c02c1bSTvrtko Ursulin if (IS_ERR(rq)) { 785f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 786f0c02c1bSTvrtko Ursulin err = PTR_ERR(rq); 7877e805762SChris Wilson break; 788f0c02c1bSTvrtko Ursulin } 789f0c02c1bSTvrtko Ursulin 790f0c02c1bSTvrtko Ursulin if (i915_request_wait(rq, 0, HZ / 5) < 0) { 791f0c02c1bSTvrtko Ursulin pr_err("Wait for timeline writes timed out!\n"); 7927e805762SChris Wilson i915_request_put(rq); 793f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 794f0c02c1bSTvrtko Ursulin err = -EIO; 7957e805762SChris Wilson break; 796f0c02c1bSTvrtko Ursulin } 797f0c02c1bSTvrtko Ursulin 798f0c02c1bSTvrtko Ursulin if (*tl->hwsp_seqno != count) { 799f0c02c1bSTvrtko Ursulin pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", 800f0c02c1bSTvrtko Ursulin count, *tl->hwsp_seqno); 801f0c02c1bSTvrtko Ursulin err = -EINVAL; 802f0c02c1bSTvrtko Ursulin } 803f0c02c1bSTvrtko Ursulin 8047e805762SChris Wilson i915_request_put(rq); 805f0c02c1bSTvrtko Ursulin intel_timeline_put(tl); 806f0c02c1bSTvrtko Ursulin count++; 807f0c02c1bSTvrtko Ursulin 808f0c02c1bSTvrtko Ursulin if (err) 8097e805762SChris Wilson break; 810f0c02c1bSTvrtko Ursulin } while (!__igt_timeout(end_time, NULL)); 811f0c02c1bSTvrtko Ursulin 8127e805762SChris Wilson intel_engine_pm_put(engine); 8137e805762SChris Wilson if (err) 8147e805762SChris Wilson break; 8157e805762SChris Wilson } 816f0c02c1bSTvrtko Ursulin 817f0c02c1bSTvrtko Ursulin return err; 818f0c02c1bSTvrtko Ursulin } 819f0c02c1bSTvrtko Ursulin 820f0c02c1bSTvrtko Ursulin int intel_timeline_live_selftests(struct drm_i915_private *i915) 821f0c02c1bSTvrtko Ursulin { 822f0c02c1bSTvrtko Ursulin static const struct i915_subtest tests[] = { 823f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_recycle), 824f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_engine), 825f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_alternate), 826f0c02c1bSTvrtko Ursulin SUBTEST(live_hwsp_wrap), 827f0c02c1bSTvrtko Ursulin }; 828f0c02c1bSTvrtko Ursulin 829cb823ed9SChris Wilson if (intel_gt_is_wedged(&i915->gt)) 830f0c02c1bSTvrtko Ursulin return 0; 831f0c02c1bSTvrtko Ursulin 83263251685SChris Wilson return i915_live_subtests(tests, i915); 833f0c02c1bSTvrtko Ursulin } 834