Lines Matching refs:stream
430 struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream) in __oa_regs() argument
432 return &stream->engine->oa_group->regs; in __oa_regs()
435 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream) in gen12_oa_hw_tail_read() argument
437 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
439 return intel_uncore_read(uncore, __oa_regs(stream)->oa_tail_ptr) & in gen12_oa_hw_tail_read()
443 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream) in gen8_oa_hw_tail_read() argument
445 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
450 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) in gen7_oa_hw_tail_read() argument
452 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
461 static u64 oa_report_id(struct i915_perf_stream *stream, void *report) in oa_report_id() argument
463 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; in oa_report_id()
466 static u64 oa_report_reason(struct i915_perf_stream *stream, void *report) in oa_report_reason() argument
468 return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) & in oa_report_reason()
469 (GRAPHICS_VER(stream->perf->i915) == 12 ? in oa_report_reason()
474 static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report) in oa_report_id_clear() argument
476 if (oa_report_header_64bit(stream)) in oa_report_id_clear()
482 static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report) in oa_report_ctx_invalid() argument
484 return !(oa_report_id(stream, report) & in oa_report_ctx_invalid()
485 stream->perf->gen8_valid_ctx_bit); in oa_report_ctx_invalid()
488 static u64 oa_timestamp(struct i915_perf_stream *stream, void *report) in oa_timestamp() argument
490 return oa_report_header_64bit(stream) ? in oa_timestamp()
495 static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report) in oa_timestamp_clear() argument
497 if (oa_report_header_64bit(stream)) in oa_timestamp_clear()
503 static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report) in oa_context_id() argument
505 u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2]; in oa_context_id()
507 return ctx_id & stream->specific_ctx_id_mask; in oa_context_id()
510 static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report) in oa_context_id_squash() argument
512 if (oa_report_header_64bit(stream)) in oa_context_id_squash()
541 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) in oa_buffer_check_unlocked() argument
543 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
544 int report_size = stream->oa_buffer.format->size; in oa_buffer_check_unlocked()
555 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
557 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
563 partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail); in oa_buffer_check_unlocked()
573 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
574 read_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
590 void *report = stream->oa_buffer.vaddr + tail; in oa_buffer_check_unlocked()
592 if (oa_report_id(stream, report) || in oa_buffer_check_unlocked()
593 oa_timestamp(stream, report)) in oa_buffer_check_unlocked()
600 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
601 drm_notice(&stream->uncore->i915->drm, in oa_buffer_check_unlocked()
605 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
607 pollin = OA_TAKEN(stream->oa_buffer.tail, in oa_buffer_check_unlocked()
608 stream->oa_buffer.head) >= report_size; in oa_buffer_check_unlocked()
610 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
630 static int append_oa_status(struct i915_perf_stream *stream, in append_oa_status() argument
666 static int append_oa_sample(struct i915_perf_stream *stream, in append_oa_sample() argument
672 int report_size = stream->oa_buffer.format->size; in append_oa_sample()
679 header.size = stream->sample_size; in append_oa_sample()
689 oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE; in append_oa_sample()
697 if (copy_to_user(buf, stream->oa_buffer.vaddr, in append_oa_sample()
730 static int gen8_append_oa_reports(struct i915_perf_stream *stream, in gen8_append_oa_reports() argument
735 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
736 int report_size = stream->oa_buffer.format->size; in gen8_append_oa_reports()
737 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
738 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
745 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
748 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
750 head = stream->oa_buffer.head; in gen8_append_oa_reports()
751 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
753 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
789 reason = oa_report_reason(stream, report); in gen8_append_oa_reports()
790 ctx_id = oa_context_id(stream, report32); in gen8_append_oa_reports()
832 if (oa_report_ctx_invalid(stream, report) && in gen8_append_oa_reports()
833 GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 50)) { in gen8_append_oa_reports()
835 oa_context_id_squash(stream, report32); in gen8_append_oa_reports()
869 if (!stream->ctx || in gen8_append_oa_reports()
870 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
871 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
878 if (stream->ctx && in gen8_append_oa_reports()
879 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
880 oa_context_id_squash(stream, report32); in gen8_append_oa_reports()
883 ret = append_oa_sample(stream, buf, count, offset, in gen8_append_oa_reports()
888 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
896 oa_report_id_clear(stream, report32); in gen8_append_oa_reports()
897 oa_timestamp_clear(stream, report32); in gen8_append_oa_reports()
899 u8 *oa_buf_end = stream->oa_buffer.vaddr + in gen8_append_oa_reports()
916 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
917 __oa_regs(stream)->oa_head_ptr : in gen8_append_oa_reports()
920 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
929 stream->oa_buffer.head = head; in gen8_append_oa_reports()
931 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
957 static int gen8_oa_read(struct i915_perf_stream *stream, in gen8_oa_read() argument
962 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
967 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
970 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
971 __oa_regs(stream)->oa_status : in gen8_oa_read()
991 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
996 drm_dbg(&stream->perf->i915->drm, in gen8_oa_read()
998 stream->period_exponent); in gen8_oa_read()
1000 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
1001 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
1011 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
1024 return gen8_append_oa_reports(stream, buf, count, offset); in gen8_oa_read()
1048 static int gen7_append_oa_reports(struct i915_perf_stream *stream, in gen7_append_oa_reports() argument
1053 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
1054 int report_size = stream->oa_buffer.format->size; in gen7_append_oa_reports()
1055 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
1056 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
1063 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
1066 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1068 head = stream->oa_buffer.head; in gen7_append_oa_reports()
1069 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
1071 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1121 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1127 ret = append_oa_sample(stream, buf, count, offset, report); in gen7_append_oa_reports()
1139 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1149 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1151 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1173 static int gen7_oa_read(struct i915_perf_stream *stream, in gen7_oa_read() argument
1178 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1182 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1192 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1215 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1220 drm_dbg(&stream->perf->i915->drm, in gen7_oa_read()
1222 stream->period_exponent); in gen7_oa_read()
1224 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1225 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1231 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1235 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1239 return gen7_append_oa_reports(stream, buf, count, offset); in gen7_oa_read()
1256 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) in i915_oa_wait_unlocked() argument
1259 if (!stream->periodic) in i915_oa_wait_unlocked()
1262 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1263 oa_buffer_check_unlocked(stream)); in i915_oa_wait_unlocked()
1276 static void i915_oa_poll_wait(struct i915_perf_stream *stream, in i915_oa_poll_wait() argument
1280 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1295 static int i915_oa_read(struct i915_perf_stream *stream, in i915_oa_read() argument
1300 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1303 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) in oa_pin_context() argument
1306 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1312 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1340 stream->pinned_ctx = ce; in oa_pin_context()
1341 return stream->pinned_ctx; in oa_pin_context()
1435 static int gen12_get_render_context_id(struct i915_perf_stream *stream) in gen12_get_render_context_id() argument
1440 if (intel_engine_uses_guc(stream->engine)) { in gen12_get_render_context_id()
1441 ret = gen12_guc_sw_ctx_id(stream->pinned_ctx, &ctx_id); in gen12_get_render_context_id()
1447 } else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) { in gen12_get_render_context_id()
1460 stream->specific_ctx_id = ctx_id & mask; in gen12_get_render_context_id()
1461 stream->specific_ctx_id_mask = mask; in gen12_get_render_context_id()
1547 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) in oa_get_render_ctx_id() argument
1552 ce = oa_pin_context(stream); in oa_get_render_ctx_id()
1556 if (engine_supports_mi_query(stream->engine) && in oa_get_render_ctx_id()
1557 HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) { in oa_get_render_ctx_id()
1565 drm_err(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1567 stream->engine->name); in oa_get_render_ctx_id()
1578 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1579 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1596 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1602 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1605 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1607 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1613 ret = gen12_get_render_context_id(stream); in oa_get_render_ctx_id()
1620 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1622 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1624 stream->specific_ctx_id, in oa_get_render_ctx_id()
1625 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1637 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) in oa_put_render_ctx_id() argument
1641 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1647 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1648 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1652 free_oa_buffer(struct i915_perf_stream *stream) in free_oa_buffer() argument
1654 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1657 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1661 free_oa_configs(struct i915_perf_stream *stream) in free_oa_configs() argument
1665 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1666 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1671 free_noa_wait(struct i915_perf_stream *stream) in free_noa_wait() argument
1673 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1686 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) in i915_oa_stream_destroy() argument
1688 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1689 struct intel_gt *gt = stream->engine->gt; in i915_oa_stream_destroy()
1690 struct i915_perf_group *g = stream->engine->oa_group; in i915_oa_stream_destroy()
1692 if (WARN_ON(stream != g->exclusive_stream)) in i915_oa_stream_destroy()
1702 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1704 free_oa_buffer(stream); in i915_oa_stream_destroy()
1709 if (stream->override_gucrc) in i915_oa_stream_destroy()
1713 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1714 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1716 if (stream->ctx) in i915_oa_stream_destroy()
1717 oa_put_render_ctx_id(stream); in i915_oa_stream_destroy()
1719 free_oa_configs(stream); in i915_oa_stream_destroy()
1720 free_noa_wait(stream); in i915_oa_stream_destroy()
1729 static void gen7_init_oa_buffer(struct i915_perf_stream *stream) in gen7_init_oa_buffer() argument
1731 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1732 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1735 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1742 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1750 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1752 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1758 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1771 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1774 static void gen8_init_oa_buffer(struct i915_perf_stream *stream) in gen8_init_oa_buffer() argument
1776 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1777 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1780 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1784 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1801 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1808 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1810 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1824 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1827 static void gen12_init_oa_buffer(struct i915_perf_stream *stream) in gen12_init_oa_buffer() argument
1829 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1830 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1833 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1835 intel_uncore_write(uncore, __oa_regs(stream)->oa_status, 0); in gen12_init_oa_buffer()
1836 intel_uncore_write(uncore, __oa_regs(stream)->oa_head_ptr, in gen12_init_oa_buffer()
1838 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1848 intel_uncore_write(uncore, __oa_regs(stream)->oa_buffer, gtt_offset | in gen12_init_oa_buffer()
1850 intel_uncore_write(uncore, __oa_regs(stream)->oa_tail_ptr, in gen12_init_oa_buffer()
1854 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1861 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1863 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1877 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1878 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1881 static int alloc_oa_buffer(struct i915_perf_stream *stream) in alloc_oa_buffer() argument
1883 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1884 struct intel_gt *gt = stream->engine->gt; in alloc_oa_buffer()
1889 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1895 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1920 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1922 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1924 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1925 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1937 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1938 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1943 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, in save_restore_register() argument
1952 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1958 *cs++ = i915_ggtt_offset(stream->noa_wait) + offset + 4 * d; in save_restore_register()
1965 static int alloc_noa_wait(struct i915_perf_stream *stream) in alloc_noa_wait() argument
1967 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1968 struct intel_gt *gt = stream->engine->gt; in alloc_noa_wait()
1972 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), in alloc_noa_wait()
1973 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1974 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
2031 stream->noa_wait = vma; in alloc_noa_wait()
2039 stream, cs, true /* save */, CS_GPR(i), in alloc_noa_wait()
2042 stream, cs, true /* save */, mi_predicate_result, in alloc_noa_wait()
2158 stream, cs, false /* restore */, CS_GPR(i), in alloc_noa_wait()
2161 stream, cs, false /* restore */, mi_predicate_result, in alloc_noa_wait()
2222 alloc_oa_config_buffer(struct i915_perf_stream *stream, in alloc_oa_config_buffer() argument
2242 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
2271 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
2274 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
2281 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
2289 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
2310 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config) in get_oa_vma() argument
2318 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
2326 oa_bo = alloc_oa_config_buffer(stream, oa_config); in get_oa_vma()
2335 emit_oa_config(struct i915_perf_stream *stream, in emit_oa_config() argument
2345 vma = get_oa_vma(stream, oa_config); in emit_oa_config()
2405 static struct intel_context *oa_context(struct i915_perf_stream *stream) in oa_context() argument
2407 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2411 hsw_enable_metric_set(struct i915_perf_stream *stream, in hsw_enable_metric_set() argument
2414 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2431 return emit_oa_config(stream, in hsw_enable_metric_set()
2432 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2436 static void hsw_disable_metric_set(struct i915_perf_stream *stream) in hsw_disable_metric_set() argument
2438 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2478 const struct i915_perf_stream *stream) in gen8_update_reg_state_unlocked() argument
2480 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2481 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2496 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2497 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2502 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2609 static int gen8_configure_context(struct i915_perf_stream *stream, in gen8_configure_context() argument
2639 static int gen12_configure_oar_context(struct i915_perf_stream *stream, in gen12_configure_oar_context() argument
2643 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2644 u32 format = stream->oa_buffer.format->format; in gen12_configure_oar_context()
2645 u32 offset = stream->perf->ctx_oactxctrl_offset; in gen12_configure_oar_context()
2715 oa_configure_all_contexts(struct i915_perf_stream *stream, in oa_configure_all_contexts() argument
2720 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2722 struct intel_gt *gt = stream->engine->gt; in oa_configure_all_contexts()
2751 err = gen8_configure_context(stream, ctx, regs, num_regs); in oa_configure_all_contexts()
2785 lrc_configure_all_contexts(struct i915_perf_stream *stream, in lrc_configure_all_contexts() argument
2789 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in lrc_configure_all_contexts()
2791 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2814 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2815 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2821 return oa_configure_all_contexts(stream, in lrc_configure_all_contexts()
2827 gen8_enable_metric_set(struct i915_perf_stream *stream, in gen8_enable_metric_set() argument
2830 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2831 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2857 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2868 ret = lrc_configure_all_contexts(stream, oa_config, active); in gen8_enable_metric_set()
2872 return emit_oa_config(stream, in gen8_enable_metric_set()
2873 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2877 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) in oag_report_ctx_switches() argument
2880 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2885 gen12_enable_metric_set(struct i915_perf_stream *stream, in gen12_enable_metric_set() argument
2888 struct drm_i915_private *i915 = stream->perf->i915; in gen12_enable_metric_set()
2889 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2890 bool periodic = stream->periodic; in gen12_enable_metric_set()
2891 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2907 intel_uncore_write(uncore, __oa_regs(stream)->oa_debug, in gen12_enable_metric_set()
2915 oag_report_ctx_switches(stream)); in gen12_enable_metric_set()
2917 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctx_ctrl, periodic ? in gen12_enable_metric_set()
2938 if (stream->ctx) { in gen12_enable_metric_set()
2939 ret = gen12_configure_oar_context(stream, active); in gen12_enable_metric_set()
2944 return emit_oa_config(stream, in gen12_enable_metric_set()
2945 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2949 static void gen8_disable_metric_set(struct i915_perf_stream *stream) in gen8_disable_metric_set() argument
2951 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2954 lrc_configure_all_contexts(stream, NULL, NULL); in gen8_disable_metric_set()
2959 static void gen11_disable_metric_set(struct i915_perf_stream *stream) in gen11_disable_metric_set() argument
2961 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2964 lrc_configure_all_contexts(stream, NULL, NULL); in gen11_disable_metric_set()
2970 static void gen12_disable_metric_set(struct i915_perf_stream *stream) in gen12_disable_metric_set() argument
2972 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2973 struct drm_i915_private *i915 = stream->perf->i915; in gen12_disable_metric_set()
2988 if (stream->ctx) in gen12_disable_metric_set()
2989 gen12_configure_oar_context(stream, NULL); in gen12_disable_metric_set()
3001 static void gen7_oa_enable(struct i915_perf_stream *stream) in gen7_oa_enable() argument
3003 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
3004 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
3005 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
3006 bool periodic = stream->periodic; in gen7_oa_enable()
3007 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
3008 u32 report_format = stream->oa_buffer.format->format; in gen7_oa_enable()
3019 gen7_init_oa_buffer(stream); in gen7_oa_enable()
3031 static void gen8_oa_enable(struct i915_perf_stream *stream) in gen8_oa_enable() argument
3033 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
3034 u32 report_format = stream->oa_buffer.format->format; in gen8_oa_enable()
3045 gen8_init_oa_buffer(stream); in gen8_oa_enable()
3057 static void gen12_oa_enable(struct i915_perf_stream *stream) in gen12_oa_enable() argument
3066 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
3069 gen12_init_oa_buffer(stream); in gen12_oa_enable()
3071 regs = __oa_regs(stream); in gen12_oa_enable()
3072 val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) | in gen12_oa_enable()
3075 intel_uncore_write(stream->uncore, regs->oa_ctrl, val); in gen12_oa_enable()
3087 static void i915_oa_stream_enable(struct i915_perf_stream *stream) in i915_oa_stream_enable() argument
3089 stream->pollin = false; in i915_oa_stream_enable()
3091 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
3093 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
3094 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
3095 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
3099 static void gen7_oa_disable(struct i915_perf_stream *stream) in gen7_oa_disable() argument
3101 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
3107 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
3111 static void gen8_oa_disable(struct i915_perf_stream *stream) in gen8_oa_disable() argument
3113 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
3119 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
3123 static void gen12_oa_disable(struct i915_perf_stream *stream) in gen12_oa_disable() argument
3125 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
3127 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctrl, 0); in gen12_oa_disable()
3129 __oa_regs(stream)->oa_ctrl, in gen12_oa_disable()
3132 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3140 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3152 static void i915_oa_stream_disable(struct i915_perf_stream *stream) in i915_oa_stream_disable() argument
3154 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
3156 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
3157 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
3169 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) in i915_perf_stream_enable_sync() argument
3178 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
3262 static int i915_oa_stream_init(struct i915_perf_stream *stream, in i915_oa_stream_init() argument
3266 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
3267 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
3273 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3286 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3292 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
3293 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3299 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3310 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3316 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3321 stream->engine = props->engine; in i915_oa_stream_init()
3322 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
3324 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
3326 stream->oa_buffer.format = &perf->oa_formats[props->oa_format]; in i915_oa_stream_init()
3327 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0)) in i915_oa_stream_init()
3330 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
3331 stream->sample_size += stream->oa_buffer.format->size; in i915_oa_stream_init()
3333 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
3335 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
3336 if (stream->periodic) in i915_oa_stream_init()
3337 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
3339 if (stream->ctx) { in i915_oa_stream_init()
3340 ret = oa_get_render_ctx_id(stream); in i915_oa_stream_init()
3342 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3348 ret = alloc_noa_wait(stream); in i915_oa_stream_init()
3350 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3355 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
3356 if (!stream->oa_config) { in i915_oa_stream_init()
3357 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3375 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
3376 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3389 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3394 stream->override_gucrc = true; in i915_oa_stream_init()
3397 ret = alloc_oa_buffer(stream); in i915_oa_stream_init()
3401 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
3403 stream->engine->gt->perf.sseu = props->sseu; in i915_oa_stream_init()
3404 WRITE_ONCE(g->exclusive_stream, stream); in i915_oa_stream_init()
3406 ret = i915_perf_stream_enable_sync(stream); in i915_oa_stream_init()
3408 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3413 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3415 stream->oa_config->uuid); in i915_oa_stream_init()
3417 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
3419 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
3420 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
3421 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
3422 mutex_init(&stream->lock); in i915_oa_stream_init()
3428 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
3430 free_oa_buffer(stream); in i915_oa_stream_init()
3433 if (stream->override_gucrc) in i915_oa_stream_init()
3437 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3438 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
3440 free_oa_configs(stream); in i915_oa_stream_init()
3443 free_noa_wait(stream); in i915_oa_stream_init()
3446 if (stream->ctx) in i915_oa_stream_init()
3447 oa_put_render_ctx_id(stream); in i915_oa_stream_init()
3455 struct i915_perf_stream *stream; in i915_oa_init_reg_state() local
3461 stream = READ_ONCE(engine->oa_group->exclusive_stream); in i915_oa_init_reg_state()
3462 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3463 gen8_update_reg_state_unlocked(ce, stream); in i915_oa_init_reg_state()
3489 struct i915_perf_stream *stream = file->private_data; in i915_perf_read() local
3497 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3509 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3513 mutex_lock(&stream->lock); in i915_perf_read()
3514 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3515 mutex_unlock(&stream->lock); in i915_perf_read()
3518 mutex_lock(&stream->lock); in i915_perf_read()
3519 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3520 mutex_unlock(&stream->lock); in i915_perf_read()
3535 stream->pollin = false; in i915_perf_read()
3543 struct i915_perf_stream *stream = in oa_poll_check_timer_cb() local
3544 container_of(hrtimer, typeof(*stream), poll_check_timer); in oa_poll_check_timer_cb()
3546 if (oa_buffer_check_unlocked(stream)) { in oa_poll_check_timer_cb()
3547 stream->pollin = true; in oa_poll_check_timer_cb()
3548 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3552 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3569 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream, in i915_perf_poll_locked() argument
3575 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3583 if (stream->pollin) in i915_perf_poll_locked()
3604 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll() local
3607 mutex_lock(&stream->lock); in i915_perf_poll()
3608 ret = i915_perf_poll_locked(stream, file, wait); in i915_perf_poll()
3609 mutex_unlock(&stream->lock); in i915_perf_poll()
3624 static void i915_perf_enable_locked(struct i915_perf_stream *stream) in i915_perf_enable_locked() argument
3626 if (stream->enabled) in i915_perf_enable_locked()
3630 stream->enabled = true; in i915_perf_enable_locked()
3632 if (stream->ops->enable) in i915_perf_enable_locked()
3633 stream->ops->enable(stream); in i915_perf_enable_locked()
3635 if (stream->hold_preemption) in i915_perf_enable_locked()
3636 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3653 static void i915_perf_disable_locked(struct i915_perf_stream *stream) in i915_perf_disable_locked() argument
3655 if (!stream->enabled) in i915_perf_disable_locked()
3659 stream->enabled = false; in i915_perf_disable_locked()
3661 if (stream->hold_preemption) in i915_perf_disable_locked()
3662 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3664 if (stream->ops->disable) in i915_perf_disable_locked()
3665 stream->ops->disable(stream); in i915_perf_disable_locked()
3668 static long i915_perf_config_locked(struct i915_perf_stream *stream, in i915_perf_config_locked() argument
3672 long ret = stream->oa_config->id; in i915_perf_config_locked()
3674 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3678 if (config != stream->oa_config) { in i915_perf_config_locked()
3690 err = emit_oa_config(stream, config, oa_context(stream), NULL); in i915_perf_config_locked()
3692 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3711 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, in i915_perf_ioctl_locked() argument
3717 i915_perf_enable_locked(stream); in i915_perf_ioctl_locked()
3720 i915_perf_disable_locked(stream); in i915_perf_ioctl_locked()
3723 return i915_perf_config_locked(stream, arg); in i915_perf_ioctl_locked()
3744 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl() local
3747 mutex_lock(&stream->lock); in i915_perf_ioctl()
3748 ret = i915_perf_ioctl_locked(stream, cmd, arg); in i915_perf_ioctl()
3749 mutex_unlock(&stream->lock); in i915_perf_ioctl()
3764 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) in i915_perf_destroy_locked() argument
3766 if (stream->enabled) in i915_perf_destroy_locked()
3767 i915_perf_disable_locked(stream); in i915_perf_destroy_locked()
3769 if (stream->ops->destroy) in i915_perf_destroy_locked()
3770 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3772 if (stream->ctx) in i915_perf_destroy_locked()
3773 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3775 kfree(stream); in i915_perf_destroy_locked()
3791 struct i915_perf_stream *stream = file->private_data; in i915_perf_release() local
3792 struct i915_perf *perf = stream->perf; in i915_perf_release()
3793 struct intel_gt *gt = stream->engine->gt; in i915_perf_release()
3801 i915_perf_destroy_locked(stream); in i915_perf_release()
3856 struct i915_perf_stream *stream = NULL; in i915_perf_open_ioctl_locked() local
3932 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in i915_perf_open_ioctl_locked()
3933 if (!stream) { in i915_perf_open_ioctl_locked()
3938 stream->perf = perf; in i915_perf_open_ioctl_locked()
3939 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3940 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3942 ret = i915_oa_stream_init(stream, param, props); in i915_perf_open_ioctl_locked()
3950 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3960 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); in i915_perf_open_ioctl_locked()
3967 i915_perf_enable_locked(stream); in i915_perf_open_ioctl_locked()
3977 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3978 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3980 kfree(stream); in i915_perf_open_ioctl_locked()