Lines Matching +full:cs +full:- +full:out
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
75 request = mock_request(rcs0(i915)->kernel_context, HZ / 10); in igt_add_request()
77 return -ENOMEM; in igt_add_request()
89 int err = -EINVAL; in igt_wait_request()
93 request = mock_request(rcs0(i915)->kernel_context, T); in igt_wait_request()
95 return -ENOMEM; in igt_wait_request()
99 if (i915_request_wait(request, 0, 0) != -ETIME) { in igt_wait_request()
104 if (i915_request_wait(request, 0, T) != -ETIME) { in igt_wait_request()
116 if (i915_request_wait(request, 0, 0) != -ETIME) { in igt_wait_request()
126 if (i915_request_wait(request, 0, T / 2) != -ETIME) { in igt_wait_request()
131 if (i915_request_wait(request, 0, T) == -ETIME) { in igt_wait_request()
132 pr_err("request wait timed out!\n"); in igt_wait_request()
141 if (i915_request_wait(request, 0, T) == -ETIME) { in igt_wait_request()
142 pr_err("request wait timed out when already complete!\n"); in igt_wait_request()
158 int err = -EINVAL; in igt_fence_wait()
162 request = mock_request(rcs0(i915)->kernel_context, T); in igt_fence_wait()
164 return -ENOMEM; in igt_fence_wait()
166 if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) { in igt_fence_wait()
168 goto out; in igt_fence_wait()
173 if (dma_fence_is_signaled(&request->fence)) { in igt_fence_wait()
175 goto out; in igt_fence_wait()
178 if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) { in igt_fence_wait()
180 goto out; in igt_fence_wait()
183 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { in igt_fence_wait()
184 pr_err("fence wait timed out (expected success)!\n"); in igt_fence_wait()
185 goto out; in igt_fence_wait()
188 if (!dma_fence_is_signaled(&request->fence)) { in igt_fence_wait()
190 goto out; in igt_fence_wait()
193 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { in igt_fence_wait()
194 pr_err("fence wait timed out when complete (expected success)!\n"); in igt_fence_wait()
195 goto out; in igt_fence_wait()
199 out: in igt_fence_wait()
210 int err = -EINVAL; in igt_request_rewind()
214 err = -ENOMEM; in igt_request_rewind()
223 err = -ENOMEM; in igt_request_rewind()
232 err = -ENOMEM; in igt_request_rewind()
241 err = -ENOMEM; in igt_request_rewind()
254 request->engine->submit_request(request); in igt_request_rewind()
258 if (i915_request_wait(vip, 0, HZ) == -ETIME) { in igt_request_rewind()
259 pr_err("timed out waiting for high priority request\n"); in igt_request_rewind()
313 struct smoketest *t = thread->t; in __igt_breadcrumbs_smoketest()
314 const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1; in __igt_breadcrumbs_smoketest()
315 const unsigned int total = 4 * t->ncontexts + 1; in __igt_breadcrumbs_smoketest()
333 thread->result = -ENOMEM; in __igt_breadcrumbs_smoketest()
339 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
343 while (!READ_ONCE(thread->stop)) { in __igt_breadcrumbs_smoketest()
349 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
357 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
366 t->contexts[order[n] % t->ncontexts]; in __igt_breadcrumbs_smoketest()
370 ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx); in __igt_breadcrumbs_smoketest()
372 rq = t->request_alloc(ce); in __igt_breadcrumbs_smoketest()
380 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, in __igt_breadcrumbs_smoketest()
389 &rq->fence, in __igt_breadcrumbs_smoketest()
403 if (!wait_event_timeout(wait->wait, in __igt_breadcrumbs_smoketest()
406 struct i915_request *rq = requests[count - 1]; in __igt_breadcrumbs_smoketest()
408 pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n", in __igt_breadcrumbs_smoketest()
409 atomic_read(&wait->pending), count, in __igt_breadcrumbs_smoketest()
410 rq->fence.context, rq->fence.seqno, in __igt_breadcrumbs_smoketest()
411 t->engine->name); in __igt_breadcrumbs_smoketest()
414 intel_gt_set_wedged(t->engine->gt); in __igt_breadcrumbs_smoketest()
417 err = -EIO; in __igt_breadcrumbs_smoketest()
424 &rq->fence.flags)) { in __igt_breadcrumbs_smoketest()
426 rq->fence.context, rq->fence.seqno); in __igt_breadcrumbs_smoketest()
427 err = -EINVAL; in __igt_breadcrumbs_smoketest()
445 atomic_long_add(num_fences, &t->num_fences); in __igt_breadcrumbs_smoketest()
446 atomic_long_add(num_waits, &t->num_waits); in __igt_breadcrumbs_smoketest()
451 thread->result = err; in __igt_breadcrumbs_smoketest()
476 return -ENOMEM; in mock_breadcrumbs_smoketest()
480 ret = -ENOMEM; in mock_breadcrumbs_smoketest()
485 t.contexts[n] = mock_context(t.engine->i915, "mock"); in mock_breadcrumbs_smoketest()
487 ret = -ENOMEM; in mock_breadcrumbs_smoketest()
557 return -ENOMEM; in i915_request_mock_selftests()
559 with_intel_runtime_pm(&i915->runtime_pm, wakeref) in i915_request_mock_selftests()
572 int err = -ENODEV; in live_nop_request()
585 err = igt_live_test_begin(&t, i915, __func__, engine->name); in live_nop_request()
597 request = i915_request_create(engine->kernel_context); in live_nop_request()
607 * and submission of our breadcrumbs - in live_nop_request()
635 engine->name, in live_nop_request()
650 if (igt_spinner_init(&spin, engine->gt)) in __cancel_inactive()
651 return -ENOMEM; in __cancel_inactive()
665 pr_debug("%s: Cancelling inactive request\n", engine->name); in __cancel_inactive()
666 i915_request_cancel(rq, -EINTR); in __cancel_inactive()
671 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_inactive()
673 pr_err("%s: Failed to cancel inactive request\n", engine->name); in __cancel_inactive()
674 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_inactive()
675 err = -ETIME; in __cancel_inactive()
679 if (rq->fence.error != -EINTR) { in __cancel_inactive()
681 engine->name, rq->fence.error); in __cancel_inactive()
682 err = -EINVAL; in __cancel_inactive()
692 pr_err("%s: %s error %d\n", __func__, engine->name, err); in __cancel_inactive()
703 if (igt_spinner_init(&spin, engine->gt)) in __cancel_active()
704 return -ENOMEM; in __cancel_active()
718 pr_debug("%s: Cancelling active request\n", engine->name); in __cancel_active()
722 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_active()
724 pr_err("Failed to start spinner on %s\n", engine->name); in __cancel_active()
725 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_active()
726 err = -ETIME; in __cancel_active()
729 i915_request_cancel(rq, -EINTR); in __cancel_active()
732 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_active()
734 pr_err("%s: Failed to cancel active request\n", engine->name); in __cancel_active()
735 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_active()
736 err = -ETIME; in __cancel_active()
740 if (rq->fence.error != -EINTR) { in __cancel_active()
742 engine->name, rq->fence.error); in __cancel_active()
743 err = -EINVAL; in __cancel_active()
753 pr_err("%s: %s error %d\n", __func__, engine->name, err); in __cancel_active()
764 if (igt_spinner_init(&spin, engine->gt)) in __cancel_completed()
765 return -ENOMEM; in __cancel_completed()
783 err = -ETIME; in __cancel_completed()
787 pr_debug("%s: Cancelling completed request\n", engine->name); in __cancel_completed()
788 i915_request_cancel(rq, -EINTR); in __cancel_completed()
789 if (rq->fence.error) { in __cancel_completed()
791 engine->name, rq->fence.error); in __cancel_completed()
792 err = -EINVAL; in __cancel_completed()
802 pr_err("%s: %s error %d\n", __func__, engine->name, err); in __cancel_completed()
807 * Test to prove a non-preemptable request can be cancelled and a subsequent
810 * Testing methodology is to create a non-preemptible request and submit it,
827 !intel_has_reset_engine(engine->gt)) in __cancel_reset()
830 preempt_timeout_ms = engine->props.preempt_timeout_ms; in __cancel_reset()
831 engine->props.preempt_timeout_ms = 100; in __cancel_reset()
833 if (igt_spinner_init(&spin, engine->gt)) in __cancel_reset()
848 pr_debug("%s: Cancelling active non-preemptable request\n", in __cancel_reset()
849 engine->name); in __cancel_reset()
853 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_reset()
855 pr_err("Failed to start spinner on %s\n", engine->name); in __cancel_reset()
856 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_reset()
857 err = -ETIME; in __cancel_reset()
867 i915_request_cancel(rq, -EINTR); in __cancel_reset()
870 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_reset()
872 pr_err("%s: Failed to cancel hung request\n", engine->name); in __cancel_reset()
873 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_reset()
874 err = -ETIME; in __cancel_reset()
878 if (rq->fence.error != -EINTR) { in __cancel_reset()
880 engine->name, rq->fence.error); in __cancel_reset()
881 err = -EINVAL; in __cancel_reset()
886 struct drm_printer p = drm_info_printer(engine->i915->drm.dev); in __cancel_reset()
888 pr_err("%s: Failed to complete nop request\n", engine->name); in __cancel_reset()
889 intel_engine_dump(engine, &p, "%s\n", engine->name); in __cancel_reset()
890 err = -ETIME; in __cancel_reset()
894 if (nop->fence.error != 0) { in __cancel_reset()
896 engine->name, nop->fence.error); in __cancel_reset()
897 err = -EINVAL; in __cancel_reset()
909 engine->props.preempt_timeout_ms = preempt_timeout_ms; in __cancel_reset()
911 pr_err("%s: %s error %d\n", __func__, engine->name, err); in __cancel_reset()
932 err = igt_live_test_begin(&t, i915, __func__, engine->name); in live_cancel_request()
954 return -EIO; in live_cancel_request()
967 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); in empty_batch()
984 vma = i915_vma_instance(obj, gt->vm, NULL); in empty_batch()
1010 return rq->engine->emit_bb_start(rq, in emit_bb_start()
1023 request = i915_request_create(engine->kernel_context); in empty_request()
1057 batch = empty_batch(engine->gt); in live_empty_request()
1061 err = igt_live_test_begin(&t, i915, __func__, engine->name); in live_empty_request()
1105 engine->name, in live_empty_request()
1121 const int ver = GRAPHICS_VER(gt->i915); in recursive_batch()
1126 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); in recursive_batch()
1130 vma = i915_vma_instance(obj, gt->vm, NULL); in recursive_batch()
1175 cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); in recursive_batch_resolve()
1181 __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd)); in recursive_batch_resolve()
1182 i915_gem_object_unpin_map(batch->obj); in recursive_batch_resolve()
1184 intel_gt_chipset_flush(batch->vm->gt); in recursive_batch_resolve()
1201 * send a recursive batch to each engine - checking that we don't in live_all_engines()
1207 return -ENOMEM; in live_all_engines()
1217 batch = recursive_batch(engine->gt); in live_all_engines()
1233 GEM_BUG_ON(request[idx]->context->vm != batch->vm); in live_all_engines()
1240 request[idx]->batch = batch; in live_all_engines()
1255 __func__, engine->name); in live_all_engines()
1256 err = -EINVAL; in live_all_engines()
1264 err = recursive_batch_resolve(request[idx]->batch); in live_all_engines()
1283 __func__, engine->name, err); in live_all_engines()
1288 i915_vma_unpin(rq->batch); in live_all_engines()
1289 i915_vma_put(rq->batch); in live_all_engines()
1305 if (rq->batch) { in live_all_engines()
1306 i915_vma_unpin(rq->batch); in live_all_engines()
1307 i915_vma_put(rq->batch); in live_all_engines()
1331 * tests that we don't execute requests out of order, even though in live_sequential_engines()
1337 return -ENOMEM; in live_sequential_engines()
1347 batch = recursive_batch(engine->gt); in live_sequential_engines()
1351 __func__, engine->name, err); in live_sequential_engines()
1360 __func__, engine->name, err); in live_sequential_engines()
1363 GEM_BUG_ON(request[idx]->context->vm != batch->vm); in live_sequential_engines()
1367 &prev->fence); in live_sequential_engines()
1371 __func__, engine->name, err); in live_sequential_engines()
1381 request[idx]->batch = batch; in live_sequential_engines()
1401 __func__, engine->name); in live_sequential_engines()
1402 err = -EINVAL; in live_sequential_engines()
1406 err = recursive_batch_resolve(request[idx]->batch); in live_sequential_engines()
1418 __func__, engine->name, err); in live_sequential_engines()
1436 cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj, in live_sequential_engines()
1441 __i915_gem_object_flush_map(request[idx]->batch->obj, in live_sequential_engines()
1443 i915_gem_object_unpin_map(request[idx]->batch->obj); in live_sequential_engines()
1445 intel_gt_chipset_flush(engine->gt); in live_sequential_engines()
1448 i915_vma_put(request[idx]->batch); in live_sequential_engines()
1468 struct intel_engine_cs *engine = thread->engine; in __live_parallel_engine1()
1478 rq = i915_request_create(engine->kernel_context); in __live_parallel_engine1()
1489 err = -ETIME; in __live_parallel_engine1()
1498 pr_info("%s: %lu request + sync\n", engine->name, count); in __live_parallel_engine1()
1499 thread->result = err; in __live_parallel_engine1()
1506 struct intel_engine_cs *engine = thread->engine; in __live_parallel_engineN()
1516 rq = i915_request_create(engine->kernel_context); in __live_parallel_engineN()
1527 pr_info("%s: %lu requests\n", engine->name, count); in __live_parallel_engineN()
1528 thread->result = err; in __live_parallel_engineN()
1533 if (atomic_dec_and_test(&i915->selftest.counter)) { in wake_all()
1534 wake_up_var(&i915->selftest.counter); in wake_all()
1546 if (wait_var_event_timeout(&i915->selftest.counter, in wait_for_all()
1547 !atomic_read(&i915->selftest.counter), in wait_for_all()
1551 return -ETIME; in wait_for_all()
1558 struct intel_engine_cs *engine = thread->engine; in __live_parallel_spin()
1569 if (igt_spinner_init(&spin, engine->gt)) { in __live_parallel_spin()
1570 wake_all(engine->i915); in __live_parallel_spin()
1571 thread->result = -ENOMEM; in __live_parallel_spin()
1577 engine->kernel_context, in __live_parallel_spin()
1582 if (err == -ENODEV) in __live_parallel_spin()
1584 wake_all(engine->i915); in __live_parallel_spin()
1592 err = wait_for_all(engine->i915); in __live_parallel_spin()
1594 pr_err("Failed to start spinner on %s\n", engine->name); in __live_parallel_spin()
1595 err = -EINVAL; in __live_parallel_spin()
1600 err = -EIO; in __live_parallel_spin()
1605 thread->result = err; in __live_parallel_spin()
1630 return -ENOMEM; in live_parallel_engines()
1642 atomic_set(&i915->selftest.counter, nengines); in live_parallel_engines()
1649 engine->name); in live_parallel_engines()
1680 err = -EIO; in live_parallel_engines()
1702 if (HAS_EXECLISTS(ctx->i915)) in max_batches()
1711 ret = rq->ring->size - rq->reserved_space; in max_batches()
1714 sz = rq->ring->emit - rq->head; in max_batches()
1716 sz += rq->ring->size; in max_batches()
1748 wakeref = intel_runtime_pm_get(&i915->runtime_pm); in live_breadcrumbs_smoketest()
1758 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1764 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1774 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1803 smoke[idx].max_batch, engine->name); in live_breadcrumbs_smoketest()
1866 intel_runtime_pm_put(&i915->runtime_pm, wakeref); in live_breadcrumbs_smoketest()
1894 rq = intel_engine_create_kernel_request(ce->engine); in switch_to_kernel_sync()
1898 fence = i915_active_fence_get(&ce->timeline->last_request); in switch_to_kernel_sync()
1907 err = -ETIME; in switch_to_kernel_sync()
1910 while (!err && !intel_engine_is_idle(ce->engine)) in switch_to_kernel_sync()
1911 intel_engine_flush_submission(ce->engine); in switch_to_kernel_sync()
1934 return *a - *b; in cmp_u32()
1955 u64 ns = intel_gt_clock_interval_to_ns(engine->gt, cycles); in cycles_to_ns()
1960 static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset) in emit_timestamp_store() argument
1962 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in emit_timestamp_store()
1963 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base))); in emit_timestamp_store()
1964 *cs++ = offset; in emit_timestamp_store()
1965 *cs++ = 0; in emit_timestamp_store()
1967 return cs; in emit_timestamp_store()
1970 static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value) in emit_store_dw() argument
1972 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_store_dw()
1973 *cs++ = offset; in emit_store_dw()
1974 *cs++ = 0; in emit_store_dw()
1975 *cs++ = value; in emit_store_dw()
1977 return cs; in emit_store_dw()
1980 static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset) in emit_semaphore_poll() argument
1982 *cs++ = MI_SEMAPHORE_WAIT | in emit_semaphore_poll()
1986 *cs++ = value; in emit_semaphore_poll()
1987 *cs++ = offset; in emit_semaphore_poll()
1988 *cs++ = 0; in emit_semaphore_poll()
1990 return cs; in emit_semaphore_poll()
1993 static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value) in emit_semaphore_poll_until() argument
1995 return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset); in emit_semaphore_poll_until()
2006 return memset32(ce->engine->status_page.addr + 1000, 0, 21); in hwsp_scratch()
2011 return (i915_ggtt_offset(ce->engine->status_page.vma) + in hwsp_offset()
2021 u32 *cs; in measure_semaphore_response() local
2033 * Semaphore latency: B - A in measure_semaphore_response()
2036 semaphore_set(sema, -1); in measure_semaphore_response()
2042 cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed)); in measure_semaphore_response()
2043 if (IS_ERR(cs)) { in measure_semaphore_response()
2045 err = PTR_ERR(cs); in measure_semaphore_response()
2049 cs = emit_store_dw(cs, offset, 0); in measure_semaphore_response()
2051 cs = emit_semaphore_poll_until(cs, offset, i); in measure_semaphore_response()
2052 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_semaphore_response()
2053 cs = emit_store_dw(cs, offset, 0); in measure_semaphore_response()
2056 intel_ring_advance(rq, cs); in measure_semaphore_response()
2060 err = -EIO; in measure_semaphore_response()
2066 cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_semaphore_response()
2071 err = -EIO; in measure_semaphore_response()
2075 elapsed[i - 1] = sema[i] - cycles; in measure_semaphore_response()
2080 ce->engine->name, cycles >> TF_BIAS, in measure_semaphore_response()
2081 cycles_to_ns(ce->engine, cycles)); in measure_semaphore_response()
2083 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_semaphore_response()
2086 intel_gt_set_wedged(ce->engine->gt); in measure_semaphore_response()
2095 u32 *cs; in measure_idle_dispatch() local
2107 * Submission latency: B - A in measure_idle_dispatch()
2113 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_idle_dispatch()
2123 cs = intel_ring_begin(rq, 4); in measure_idle_dispatch()
2124 if (IS_ERR(cs)) { in measure_idle_dispatch()
2126 err = PTR_ERR(cs); in measure_idle_dispatch()
2130 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_idle_dispatch()
2132 intel_ring_advance(rq, cs); in measure_idle_dispatch()
2136 elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_idle_dispatch()
2142 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_idle_dispatch()
2147 elapsed[i] = sema[i] - elapsed[i]; in measure_idle_dispatch()
2151 ce->engine->name, cycles >> TF_BIAS, in measure_idle_dispatch()
2152 cycles_to_ns(ce->engine, cycles)); in measure_idle_dispatch()
2154 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_idle_dispatch()
2157 intel_gt_set_wedged(ce->engine->gt); in measure_idle_dispatch()
2166 u32 *cs; in measure_busy_dispatch() local
2179 * Submission latency: B - A in measure_busy_dispatch()
2191 cs = intel_ring_begin(rq, 12); in measure_busy_dispatch()
2192 if (IS_ERR(cs)) { in measure_busy_dispatch()
2194 err = PTR_ERR(cs); in measure_busy_dispatch()
2198 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); in measure_busy_dispatch()
2199 cs = emit_semaphore_poll_until(cs, offset, i); in measure_busy_dispatch()
2200 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_busy_dispatch()
2202 intel_ring_advance(rq, cs); in measure_busy_dispatch()
2204 if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) { in measure_busy_dispatch()
2205 err = -EIO; in measure_busy_dispatch()
2211 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_busy_dispatch()
2214 semaphore_set(sema, i - 1); in measure_busy_dispatch()
2218 wait_for(READ_ONCE(sema[i - 1]), 500); in measure_busy_dispatch()
2219 semaphore_set(sema, i - 1); in measure_busy_dispatch()
2222 GEM_BUG_ON(sema[i] == -1); in measure_busy_dispatch()
2223 elapsed[i - 1] = sema[i] - elapsed[i]; in measure_busy_dispatch()
2228 ce->engine->name, cycles >> TF_BIAS, in measure_busy_dispatch()
2229 cycles_to_ns(ce->engine, cycles)); in measure_busy_dispatch()
2231 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_busy_dispatch()
2234 intel_gt_set_wedged(ce->engine->gt); in measure_busy_dispatch()
2241 i915_ggtt_offset(engine->status_page.vma) + in plug()
2244 u32 *cs; in plug() local
2246 rq = i915_request_create(engine->kernel_context); in plug()
2250 cs = intel_ring_begin(rq, 4); in plug()
2251 if (IS_ERR(cs)) { in plug()
2253 return PTR_ERR(cs); in plug()
2256 cs = emit_semaphore_poll(cs, mode, value, offset); in plug()
2258 intel_ring_advance(rq, cs); in plug()
2283 * Request latency: B - A in measure_inter_request()
2286 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); in measure_inter_request()
2293 return -ENOMEM; in measure_inter_request()
2296 intel_engine_flush_submission(ce->engine); in measure_inter_request()
2299 u32 *cs; in measure_inter_request() local
2307 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, in measure_inter_request()
2315 cs = intel_ring_begin(rq, 4); in measure_inter_request()
2316 if (IS_ERR(cs)) { in measure_inter_request()
2318 err = PTR_ERR(cs); in measure_inter_request()
2322 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_inter_request()
2324 intel_ring_advance(rq, cs); in measure_inter_request()
2328 intel_engine_flush_submission(ce->engine); in measure_inter_request()
2332 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_inter_request()
2337 elapsed[i - 1] = sema[i + 1] - sema[i]; in measure_inter_request()
2340 pr_info("%s: inter-request latency %d cycles, %lluns\n", in measure_inter_request()
2341 ce->engine->name, cycles >> TF_BIAS, in measure_inter_request()
2342 cycles_to_ns(ce->engine, cycles)); in measure_inter_request()
2344 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_inter_request()
2351 intel_gt_set_wedged(ce->engine->gt); in measure_inter_request()
2362 u32 *cs; in measure_context_switch() local
2368 * the inter-context setup we require. in measure_context_switch()
2374 * Context switch latency: B - A in measure_context_switch()
2377 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); in measure_context_switch()
2383 ce, ce->engine->kernel_context in measure_context_switch()
2398 &fence->fence); in measure_context_switch()
2405 cs = intel_ring_begin(rq, 4); in measure_context_switch()
2406 if (IS_ERR(cs)) { in measure_context_switch()
2408 err = PTR_ERR(cs); in measure_context_switch()
2412 cs = emit_timestamp_store(cs, ce, addr); in measure_context_switch()
2415 intel_ring_advance(rq, cs); in measure_context_switch()
2424 intel_engine_flush_submission(ce->engine); in measure_context_switch()
2427 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_context_switch()
2432 elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1]; in measure_context_switch()
2436 ce->engine->name, cycles >> TF_BIAS, in measure_context_switch()
2437 cycles_to_ns(ce->engine, cycles)); in measure_context_switch()
2439 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_context_switch()
2445 intel_gt_set_wedged(ce->engine->gt); in measure_context_switch()
2454 u32 *cs; in measure_preemption() local
2470 * Preemption dispatch latency: B - A in measure_preemption()
2471 * Preemption switch latency: C - B in measure_preemption()
2474 if (!intel_engine_has_preemption(ce->engine)) in measure_preemption()
2487 cs = intel_ring_begin(rq, 12); in measure_preemption()
2488 if (IS_ERR(cs)) { in measure_preemption()
2490 err = PTR_ERR(cs); in measure_preemption()
2494 cs = emit_store_dw(cs, addr, -1); in measure_preemption()
2495 cs = emit_semaphore_poll_until(cs, offset, i); in measure_preemption()
2496 cs = emit_timestamp_store(cs, ce, addr + sizeof(u32)); in measure_preemption()
2498 intel_ring_advance(rq, cs); in measure_preemption()
2501 if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) { in measure_preemption()
2502 err = -EIO; in measure_preemption()
2506 rq = i915_request_create(ce->engine->kernel_context); in measure_preemption()
2512 cs = intel_ring_begin(rq, 8); in measure_preemption()
2513 if (IS_ERR(cs)) { in measure_preemption()
2515 err = PTR_ERR(cs); in measure_preemption()
2519 cs = emit_timestamp_store(cs, ce, addr); in measure_preemption()
2520 cs = emit_store_dw(cs, offset, i); in measure_preemption()
2522 intel_ring_advance(rq, cs); in measure_preemption()
2523 rq->sched.attr.priority = I915_PRIORITY_BARRIER; in measure_preemption()
2525 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_preemption()
2529 if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) { in measure_preemption()
2530 err = -EIO; in measure_preemption()
2535 elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1]; in measure_preemption()
2539 ce->engine->name, cycles >> TF_BIAS, in measure_preemption()
2540 cycles_to_ns(ce->engine, cycles)); in measure_preemption()
2543 elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0]; in measure_preemption()
2547 ce->engine->name, cycles >> TF_BIAS, in measure_preemption()
2548 cycles_to_ns(ce->engine, cycles)); in measure_preemption()
2550 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_preemption()
2553 intel_gt_set_wedged(ce->engine->gt); in measure_preemption()
2566 smp_store_mb(s->seen, true); /* be safe, be strong */ in signal_cb()
2574 u32 *cs; in measure_completion() local
2586 * Completion latency: B - A in measure_completion()
2599 cs = intel_ring_begin(rq, 12); in measure_completion()
2600 if (IS_ERR(cs)) { in measure_completion()
2602 err = PTR_ERR(cs); in measure_completion()
2606 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); in measure_completion()
2607 cs = emit_semaphore_poll_until(cs, offset, i); in measure_completion()
2608 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_completion()
2610 intel_ring_advance(rq, cs); in measure_completion()
2612 dma_fence_add_callback(&rq->fence, &cb.base, signal_cb); in measure_completion()
2615 intel_engine_flush_submission(ce->engine); in measure_completion()
2616 if (wait_for(READ_ONCE(sema[i]) == -1, 50)) { in measure_completion()
2617 err = -EIO; in measure_completion()
2626 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_completion()
2630 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_completion()
2635 GEM_BUG_ON(sema[i + 1] == -1); in measure_completion()
2636 elapsed[i] = elapsed[i] - sema[i + 1]; in measure_completion()
2641 ce->engine->name, cycles >> TF_BIAS, in measure_completion()
2642 cycles_to_ns(ce->engine, cycles)); in measure_completion()
2644 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_completion()
2647 intel_gt_set_wedged(ce->engine->gt); in measure_completion()
2654 atomic_inc(>->rps.num_waiters); in rps_pin()
2655 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in rps_pin()
2657 mutex_lock(>->rps.lock); in rps_pin()
2658 intel_rps_set(>->rps, gt->rps.max_freq); in rps_pin()
2659 mutex_unlock(>->rps.lock); in rps_pin()
2664 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in rps_unpin()
2665 atomic_dec(>->rps.num_waiters); in rps_unpin()
2675 if (GRAPHICS_VER(i915) < 8) /* per-engine CS timestamp, semaphores */ in perf_request_latency()
2686 goto out; in perf_request_latency()
2692 goto out; in perf_request_latency()
2696 rps_pin(engine->gt); in perf_request_latency()
2713 rps_unpin(engine->gt); in perf_request_latency()
2719 goto out; in perf_request_latency()
2722 out: in perf_request_latency()
2724 err = -EIO; in perf_request_latency()
2737 GEM_BUG_ON(!ps->nengines); in s_sync0()
2741 rq = i915_request_create(ps->ce[idx]); in s_sync0()
2751 err = -ETIME; in s_sync0()
2756 if (++idx == ps->nengines) in s_sync0()
2771 GEM_BUG_ON(!ps->nengines); in s_sync1()
2775 rq = i915_request_create(ps->ce[idx]); in s_sync1()
2785 err = -ETIME; in s_sync1()
2791 if (++idx == ps->nengines) in s_sync1()
2805 GEM_BUG_ON(!ps->nengines); in s_many()
2809 rq = i915_request_create(ps->ce[idx]); in s_many()
2815 if (++idx == ps->nengines) in s_many()
2842 return -ENOMEM; in perf_series_engines()
2847 return -ENOMEM; in perf_series_engines()
2852 ps->i915 = i915; in perf_series_engines()
2853 ps->nengines = nengines; in perf_series_engines()
2862 goto out; in perf_series_engines()
2868 goto out; in perf_series_engines()
2871 ps->ce[idx++] = ce; in perf_series_engines()
2873 GEM_BUG_ON(idx != ps->nengines); in perf_series_engines()
2887 struct intel_context *ce = ps->ce[idx]; in perf_series_engines()
2889 p->engine = ps->ce[idx]->engine; in perf_series_engines()
2890 intel_engine_pm_get(p->engine); in perf_series_engines()
2892 if (intel_engine_supports_stats(p->engine)) in perf_series_engines()
2893 p->busy = intel_engine_get_busy_time(p->engine, in perf_series_engines()
2894 &p->time) + 1; in perf_series_engines()
2896 p->time = ktime_get(); in perf_series_engines()
2897 p->runtime = -intel_context_get_total_runtime_ns(ce); in perf_series_engines()
2902 err = -EIO; in perf_series_engines()
2906 struct intel_context *ce = ps->ce[idx]; in perf_series_engines()
2910 if (p->busy) in perf_series_engines()
2911 p->busy = ktime_sub(intel_engine_get_busy_time(p->engine, in perf_series_engines()
2913 p->busy - 1); in perf_series_engines()
2916 p->time = ktime_sub(now, p->time); in perf_series_engines()
2919 p->runtime += intel_context_get_total_runtime_ns(ce); in perf_series_engines()
2920 intel_engine_pm_put(p->engine); in perf_series_engines()
2922 busy = 100 * ktime_to_ns(p->busy); in perf_series_engines()
2923 dt = ktime_to_ns(p->time); in perf_series_engines()
2926 busy -= integer * dt; in perf_series_engines()
2934 name, p->engine->name, ce->timeline->seqno, in perf_series_engines()
2936 div_u64(p->runtime, 1000 * 1000), in perf_series_engines()
2937 div_u64(ktime_to_ns(p->time), 1000 * 1000)); in perf_series_engines()
2941 out: in perf_series_engines()
2943 if (IS_ERR_OR_NULL(ps->ce[idx])) in perf_series_engines()
2946 intel_context_unpin(ps->ce[idx]); in perf_series_engines()
2947 intel_context_put(ps->ce[idx]); in perf_series_engines()
2967 struct perf_stats *p = &thread->p; in p_sync0()
2968 struct intel_engine_cs *engine = p->engine; in p_sync0()
2977 thread->result = PTR_ERR(ce); in p_sync0()
2984 thread->result = err; in p_sync0()
2989 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_sync0()
2992 p->time = ktime_get(); in p_sync0()
3011 err = -ETIME; in p_sync0()
3022 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_sync0()
3023 p->busy); in p_sync0()
3024 p->time = ktime_sub(now, p->time); in p_sync0()
3026 p->time = ktime_sub(ktime_get(), p->time); in p_sync0()
3030 p->runtime = intel_context_get_total_runtime_ns(ce); in p_sync0()
3031 p->count = count; in p_sync0()
3035 thread->result = err; in p_sync0()
3041 struct perf_stats *p = &thread->p; in p_sync1()
3042 struct intel_engine_cs *engine = p->engine; in p_sync1()
3052 thread->result = PTR_ERR(ce); in p_sync1()
3059 thread->result = err; in p_sync1()
3064 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_sync1()
3067 p->time = ktime_get(); in p_sync1()
3086 err = -ETIME; in p_sync1()
3099 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_sync1()
3100 p->busy); in p_sync1()
3101 p->time = ktime_sub(now, p->time); in p_sync1()
3103 p->time = ktime_sub(ktime_get(), p->time); in p_sync1()
3107 p->runtime = intel_context_get_total_runtime_ns(ce); in p_sync1()
3108 p->count = count; in p_sync1()
3112 thread->result = err; in p_sync1()
3118 struct perf_stats *p = &thread->p; in p_many()
3119 struct intel_engine_cs *engine = p->engine; in p_many()
3128 thread->result = PTR_ERR(ce); in p_many()
3135 thread->result = err; in p_many()
3140 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_many()
3143 p->time = ktime_get(); in p_many()
3164 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_many()
3165 p->busy); in p_many()
3166 p->time = ktime_sub(now, p->time); in p_many()
3168 p->time = ktime_sub(ktime_get(), p->time); in p_many()
3172 p->runtime = intel_context_get_total_runtime_ns(ce); in p_many()
3173 p->count = count; in p_many()
3177 thread->result = err; in p_many()
3198 return -ENOMEM; in perf_parallel_engines()
3212 atomic_set(&i915->selftest.counter, nengines); in perf_parallel_engines()
3223 engine->name); in perf_parallel_engines()
3258 err = -EIO; in perf_parallel_engines()
3265 u64 busy = 100 * ktime_to_ns(p->busy); in perf_parallel_engines()
3266 u64 dt = ktime_to_ns(p->time); in perf_parallel_engines()
3271 busy -= integer * dt; in perf_parallel_engines()
3278 GEM_BUG_ON(engine != p->engine); in perf_parallel_engines()
3280 name, engine->name, p->count, integer, decimal, in perf_parallel_engines()
3281 div_u64(p->runtime, 1000 * 1000), in perf_parallel_engines()
3282 div_u64(ktime_to_ns(p->time), 1000 * 1000)); in perf_parallel_engines()