Lines Matching +full:cpu +full:- +full:centric
2 * Copyright © 2015-2016 Intel Corporation
40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
44 * without special privileges. Access to system-wide metrics requires root
58 * might sample sets of tightly-coupled counters, depending on the
70 * interleaved with event-type specific members.
74 * with the CPU, using HW specific packing formats for counter sets. Sometimes
76 * would be acceptable to expose them to unprivileged applications - to hide
88 * into perf's currently cpu centric design.
96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
102 * For posterity, in case we might re-visit trying to adapt core perf to be
106 * - The perf based OA PMU driver broke some significant design assumptions:
108 * Existing perf pmus are used for profiling work on a cpu and we were
110 * implications, the need to fake cpu-related data (such as user/kernel
112 * as a way to forward device-specific status records.
115 * involvement from the CPU, making our PMU driver the first of a kind.
117 * Given the way we were periodically forward data from the GPU-mapped, OA
123 * explicitly initiated from the cpu (say in response to a userspace read())
125 * trigger a report from the cpu on demand.
130 * opened, there's no clear precedent for being able to provide group-wide
139 * for combining with the side-band raw reports it captures using
142 * - As a side note on perf's grouping feature; there was also some concern
158 * one time. The OA unit is not designed to allow re-configuration while in
178 * - It felt like our perf based PMU was making some technical compromises
182 * cpu core, while our device pmu related to neither. Events opened with a
184 * that process - so not appropriate for us. When an event is related to a
185 * cpu id, perf ensures pmu methods will be invoked via an inter process
187 * perf events for a specific cpu. This was workable but it meant the
228 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
236 * CPU).
239 * by checking for a zeroed report-id field in tail reports, we want to account
259 * non-periodic reports (such as on context switch) or the OA unit may be
314 * code assumes all reports have a power-of-two size and ~(size - 1) can
342 * struct perf_open_properties - for validated properties given to open a stream
357 * @poll_oa_period: The period in nanoseconds at which the CPU will check for OA
401 kfree(oa_config->flex_regs); in i915_oa_config_release()
402 kfree(oa_config->b_counter_regs); in i915_oa_config_release()
403 kfree(oa_config->mux_regs); in i915_oa_config_release()
414 oa_config = idr_find(&perf->metrics_idr, metrics_set); in i915_perf_get_oa_config()
424 i915_oa_config_put(oa_bo->oa_config); in free_oa_config_bo()
425 i915_vma_put(oa_bo->vma); in free_oa_config_bo()
432 return &stream->engine->oa_group->regs; in __oa_regs()
437 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
439 return intel_uncore_read(uncore, __oa_regs(stream)->oa_tail_ptr) & in gen12_oa_hw_tail_read()
445 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
452 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
459 ((__s)->oa_buffer.format->header == HDR_64_BIT)
469 (GRAPHICS_VER(stream->perf->i915) == 12 ? in oa_report_reason()
485 stream->perf->gen8_valid_ctx_bit); in oa_report_ctx_invalid()
507 return ctx_id & stream->specific_ctx_id_mask; in oa_context_id()
519 * oa_buffer_check_unlocked - check for data and update tail ptr state
527 * pointer having a race with respect to what data is visible to the CPU.
543 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
544 int report_size = stream->oa_buffer.format->size; in oa_buffer_check_unlocked()
555 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
557 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
563 partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail); in oa_buffer_check_unlocked()
571 * anywhere between this head and stream->oa_buffer.tail. in oa_buffer_check_unlocked()
573 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
574 read_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
590 void *report = stream->oa_buffer.vaddr + tail; in oa_buffer_check_unlocked()
596 tail = (tail - report_size) & (OA_BUFFER_SIZE - 1); in oa_buffer_check_unlocked()
600 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
601 drm_notice(&stream->uncore->i915->drm, in oa_buffer_check_unlocked()
605 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
607 pollin = OA_TAKEN(stream->oa_buffer.tail, in oa_buffer_check_unlocked()
608 stream->oa_buffer.head) >= report_size; in oa_buffer_check_unlocked()
610 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
616 * append_oa_status - Appends a status record to a userspace read() buffer.
617 * @stream: An i915-perf stream opened for OA metrics
638 if ((count - *offset) < header.size) in append_oa_status()
639 return -ENOSPC; in append_oa_status()
642 return -EFAULT; in append_oa_status()
650 * append_oa_sample - Copies single OA report into userspace read() buffer.
651 * @stream: An i915-perf stream opened for OA metrics
658 * properties when opening a stream, tracked as `stream->sample_flags`. This
672 int report_size = stream->oa_buffer.format->size; in append_oa_sample()
679 header.size = stream->sample_size; in append_oa_sample()
681 if ((count - *offset) < header.size) in append_oa_sample()
682 return -ENOSPC; in append_oa_sample()
686 return -EFAULT; in append_oa_sample()
689 oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE; in append_oa_sample()
690 report_size_partial = oa_buf_end - report; in append_oa_sample()
694 return -EFAULT; in append_oa_sample()
697 if (copy_to_user(buf, stream->oa_buffer.vaddr, in append_oa_sample()
698 report_size - report_size_partial)) in append_oa_sample()
699 return -EFAULT; in append_oa_sample()
701 return -EFAULT; in append_oa_sample()
710 * gen8_append_oa_reports - Copies all buffered OA reports into
712 * @stream: An i915-perf stream opened for OA metrics
717 * Notably any error condition resulting in a short read (-%ENOSPC or
718 * -%EFAULT) will be returned even though one or more records may
725 * and back-to-front you're not alone, but this follows the
735 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
736 int report_size = stream->oa_buffer.format->size; in gen8_append_oa_reports()
737 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
738 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
739 u32 mask = (OA_BUFFER_SIZE - 1); in gen8_append_oa_reports()
745 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
746 return -EIO; in gen8_append_oa_reports()
748 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
750 head = stream->oa_buffer.head; in gen8_append_oa_reports()
751 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
753 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
759 head -= gtt_offset; in gen8_append_oa_reports()
760 tail -= gtt_offset; in gen8_append_oa_reports()
768 if (drm_WARN_ONCE(&uncore->i915->drm, in gen8_append_oa_reports()
773 return -EIO; in gen8_append_oa_reports()
794 * invalid to be sure we avoid false-positive, single-context in gen8_append_oa_reports()
806 * context-switch-report: This is a report with the reason type in gen8_append_oa_reports()
807 * being context-switch. It is generated when a context switches in gen8_append_oa_reports()
810 * context-valid-bit: A bit that is set in the report ID field in gen8_append_oa_reports()
813 * gpu-idle: A condition characterized by a in gen8_append_oa_reports()
814 * context-switch-report with context-valid-bit set to 0. in gen8_append_oa_reports()
816 * On prior platforms, context-id-valid bit is set to 0 only in gen8_append_oa_reports()
819 * On XEHP platforms, context-valid-bit is set to 1 in a context in gen8_append_oa_reports()
825 * context ID field and the context-valid-bit is 0. The logic in gen8_append_oa_reports()
833 GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 50)) { in gen8_append_oa_reports()
841 * stop the counters from updating as system-wide / global in gen8_append_oa_reports()
845 * filtered on the cpu but it's not worth trying to in gen8_append_oa_reports()
849 * provide a side-band view of the real values. in gen8_append_oa_reports()
853 * needs be forwarded bookend context-switch reports so that it in gen8_append_oa_reports()
866 * switches since it's not-uncommon for periodic samples to in gen8_append_oa_reports()
869 if (!stream->ctx || in gen8_append_oa_reports()
870 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
871 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
878 if (stream->ctx && in gen8_append_oa_reports()
879 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
888 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
899 u8 *oa_buf_end = stream->oa_buffer.vaddr + in gen8_append_oa_reports()
901 u32 part = oa_buf_end - (u8 *)report32; in gen8_append_oa_reports()
908 memset(oa_buf_base, 0, report_size - part); in gen8_append_oa_reports()
916 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
917 __oa_regs(stream)->oa_head_ptr : in gen8_append_oa_reports()
920 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
929 stream->oa_buffer.head = head; in gen8_append_oa_reports()
931 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
938 * gen8_oa_read - copy status records then buffered OA reports
939 * @stream: An i915-perf stream opened for OA metrics
962 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
967 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
968 return -EIO; in gen8_oa_read()
970 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
971 __oa_regs(stream)->oa_status : in gen8_oa_read()
996 drm_dbg(&stream->perf->i915->drm, in gen8_oa_read()
998 stream->period_exponent); in gen8_oa_read()
1000 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
1001 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
1004 * Note: .oa_enable() is expected to re-init the oabuffer and in gen8_oa_read()
1019 IS_GRAPHICS_VER(uncore->i915, 8, 11) ? in gen8_oa_read()
1028 * gen7_append_oa_reports - Copies all buffered OA reports into
1030 * @stream: An i915-perf stream opened for OA metrics
1035 * Notably any error condition resulting in a short read (-%ENOSPC or
1036 * -%EFAULT) will be returned even though one or more records may
1043 * and back-to-front you're not alone, but this follows the
1053 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
1054 int report_size = stream->oa_buffer.format->size; in gen7_append_oa_reports()
1055 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
1056 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
1057 u32 mask = (OA_BUFFER_SIZE - 1); in gen7_append_oa_reports()
1063 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
1064 return -EIO; in gen7_append_oa_reports()
1066 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1068 head = stream->oa_buffer.head; in gen7_append_oa_reports()
1069 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
1071 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1076 head -= gtt_offset; in gen7_append_oa_reports()
1077 tail -= gtt_offset; in gen7_append_oa_reports()
1085 if (drm_WARN_ONCE(&uncore->i915->drm, in gen7_append_oa_reports()
1090 return -EIO; in gen7_append_oa_reports()
1107 if (drm_WARN_ON(&uncore->i915->drm, in gen7_append_oa_reports()
1108 (OA_BUFFER_SIZE - head) < report_size)) { in gen7_append_oa_reports()
1109 drm_err(&uncore->i915->drm, in gen7_append_oa_reports()
1110 "Spurious OA head ptr: non-integral report offset\n"); in gen7_append_oa_reports()
1114 /* The report-ID field for periodic samples includes in gen7_append_oa_reports()
1121 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1122 drm_notice(&uncore->i915->drm, in gen7_append_oa_reports()
1139 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1149 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1151 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1158 * gen7_oa_read - copy status records then buffered OA reports
1159 * @stream: An i915-perf stream opened for OA metrics
1178 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1182 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1183 return -EIO; in gen7_oa_read()
1192 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1196 * - The status can be interpreted to mean that the buffer is in gen7_oa_read()
1198 * which will start to report a near-empty buffer after an in gen7_oa_read()
1203 * - Since it also implies the HW has started overwriting old in gen7_oa_read()
1208 * - In the future we may want to introduce a flight recorder in gen7_oa_read()
1220 drm_dbg(&stream->perf->i915->drm, in gen7_oa_read()
1222 stream->period_exponent); in gen7_oa_read()
1224 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1225 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1235 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1243 * i915_oa_wait_unlocked - handles blocking IO until OA data available
1244 * @stream: An i915-perf stream opened for OA metrics
1247 * for OA metrics. It waits until the hrtimer callback finds a non-empty
1251 * since any subsequent read handling will return -EAGAIN if there isn't
1259 if (!stream->periodic) in i915_oa_wait_unlocked()
1260 return -EIO; in i915_oa_wait_unlocked()
1262 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1267 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1268 * @stream: An i915-perf stream opened for OA metrics
1280 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1284 * i915_oa_read - just calls through to &i915_oa_ops->read
1285 * @stream: An i915-perf stream opened for OA metrics
1300 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1306 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1309 int err = -ENODEV; in oa_pin_context()
1312 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1330 if (err == -EDEADLK) { in oa_pin_context()
1340 stream->pinned_ctx = ce; in oa_pin_context()
1341 return stream->pinned_ctx; in oa_pin_context()
1350 if (GRAPHICS_VER(rq->i915) >= 8) in __store_reg_to_mem()
1383 err = -ETIME; in __read_reg()
1397 scratch = __vm_create_scratch_for_read_pinned(&ce->engine->gt->ggtt->vm, 4); in gen12_guc_sw_ctx_id()
1405 err = __read_reg(ce, RING_EXECLIST_STATUS_HI(ce->engine->mmio_base), in gen12_guc_sw_ctx_id()
1410 val = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); in gen12_guc_sw_ctx_id()
1417 i915_gem_object_unpin_map(scratch->obj); in gen12_guc_sw_ctx_id()
1426 * 0 - (NUM_CONTEXT_TAG -1) are used by other contexts
1440 if (intel_engine_uses_guc(stream->engine)) { in gen12_get_render_context_id()
1441 ret = gen12_guc_sw_ctx_id(stream->pinned_ctx, &ctx_id); in gen12_get_render_context_id()
1445 mask = ((1U << GEN12_GUC_SW_CTX_ID_WIDTH) - 1) << in gen12_get_render_context_id()
1446 (GEN12_GUC_SW_CTX_ID_SHIFT - 32); in gen12_get_render_context_id()
1447 } else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) { in gen12_get_render_context_id()
1448 ctx_id = (XEHP_MAX_CONTEXT_HW_ID - 1) << in gen12_get_render_context_id()
1449 (XEHP_SW_CTX_ID_SHIFT - 32); in gen12_get_render_context_id()
1451 mask = ((1U << XEHP_SW_CTX_ID_WIDTH) - 1) << in gen12_get_render_context_id()
1452 (XEHP_SW_CTX_ID_SHIFT - 32); in gen12_get_render_context_id()
1454 ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << in gen12_get_render_context_id()
1455 (GEN11_SW_CTX_ID_SHIFT - 32); in gen12_get_render_context_id()
1457 mask = ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << in gen12_get_render_context_id()
1458 (GEN11_SW_CTX_ID_SHIFT - 32); in gen12_get_render_context_id()
1460 stream->specific_ctx_id = ctx_id & mask; in gen12_get_render_context_id()
1461 stream->specific_ctx_id_mask = mask; in gen12_get_render_context_id()
1486 u32 offset, len = (ce->engine->context_size - PAGE_SIZE) / 4; in oa_context_image_offset()
1487 u32 *state = ce->lrc_reg_state; in oa_context_image_offset()
1489 if (drm_WARN_ON(&ce->engine->i915->drm, !state)) in oa_context_image_offset()
1495 * We expect reg-value pairs in MI_LRI command, so in oa_context_image_offset()
1498 drm_WARN_ON(&ce->engine->i915->drm, in oa_context_image_offset()
1513 i915_reg_t reg = GEN12_OACTXCONTROL(ce->engine->mmio_base); in set_oa_ctx_ctrl_offset()
1514 struct i915_perf *perf = &ce->engine->i915->perf; in set_oa_ctx_ctrl_offset()
1515 u32 offset = perf->ctx_oactxctrl_offset; in set_oa_ctx_ctrl_offset()
1522 perf->ctx_oactxctrl_offset = offset; in set_oa_ctx_ctrl_offset()
1524 drm_dbg(&ce->engine->i915->drm, in set_oa_ctx_ctrl_offset()
1526 ce->engine->name, offset); in set_oa_ctx_ctrl_offset()
1529 return offset && offset != U32_MAX ? 0 : -ENODEV; in set_oa_ctx_ctrl_offset()
1534 return engine->class == RENDER_CLASS; in engine_supports_mi_query()
1538 * oa_get_render_ctx_id - determine and hold ctx hw id
1539 * @stream: An i915-perf stream opened for OA metrics
1556 if (engine_supports_mi_query(stream->engine) && in oa_get_render_ctx_id()
1557 HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) { in oa_get_render_ctx_id()
1565 drm_err(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1567 stream->engine->name); in oa_get_render_ctx_id()
1572 switch (GRAPHICS_VER(ce->engine->i915)) { in oa_get_render_ctx_id()
1578 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1579 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1585 if (intel_engine_uses_guc(ce->engine)) { in oa_get_render_ctx_id()
1596 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1602 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1603 (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; in oa_get_render_ctx_id()
1605 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1606 (1U << GEN8_CTX_ID_WIDTH) - 1; in oa_get_render_ctx_id()
1607 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1617 MISSING_CASE(GRAPHICS_VER(ce->engine->i915)); in oa_get_render_ctx_id()
1620 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1622 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1624 stream->specific_ctx_id, in oa_get_render_ctx_id()
1625 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1631 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1632 * @stream: An i915-perf stream opened for OA metrics
1641 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1643 ce->tag = 0; /* recomputed on next submission after parking */ in oa_put_render_ctx_id()
1647 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1648 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1654 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1657 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1665 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1666 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1673 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1678 return engine->oa_group; in engine_supports_oa()
1683 return engine->oa_group && engine->oa_group->type == type; in engine_supports_oa_format()
1688 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1689 struct intel_gt *gt = stream->engine->gt; in i915_oa_stream_destroy()
1690 struct i915_perf_group *g = stream->engine->oa_group; in i915_oa_stream_destroy()
1692 if (WARN_ON(stream != g->exclusive_stream)) in i915_oa_stream_destroy()
1701 WRITE_ONCE(g->exclusive_stream, NULL); in i915_oa_stream_destroy()
1702 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1709 if (stream->override_gucrc) in i915_oa_stream_destroy()
1710 drm_WARN_ON(>->i915->drm, in i915_oa_stream_destroy()
1711 intel_guc_slpc_unset_gucrc_mode(>->uc.guc.slpc)); in i915_oa_stream_destroy()
1713 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1714 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1716 if (stream->ctx) in i915_oa_stream_destroy()
1722 if (perf->spurious_report_rs.missed) { in i915_oa_stream_destroy()
1723 drm_notice(>->i915->drm, in i915_oa_stream_destroy()
1725 perf->spurious_report_rs.missed); in i915_oa_stream_destroy()
1731 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1732 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1735 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1737 /* Pre-DevBDW: OABUFFER must be set with counters off, in gen7_init_oa_buffer()
1742 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1750 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1752 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1758 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1762 * first allocating), we may re-init the OA buffer, either in gen7_init_oa_buffer()
1763 * when re-enabling a stream or in error/reset paths. in gen7_init_oa_buffer()
1765 * The reason we clear the buffer for each re-init is for the in gen7_init_oa_buffer()
1767 * report-id field to make sure it's non-zero which relies on in gen7_init_oa_buffer()
1771 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1776 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1777 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1780 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1784 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1801 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1808 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1810 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1815 * first allocating), we may re-init the OA buffer, either in gen8_init_oa_buffer()
1816 * when re-enabling a stream or in error/reset paths. in gen8_init_oa_buffer()
1818 * The reason we clear the buffer for each re-init is for the in gen8_init_oa_buffer()
1820 * reason field to make sure it's non-zero which relies on in gen8_init_oa_buffer()
1824 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1829 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1830 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1833 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1835 intel_uncore_write(uncore, __oa_regs(stream)->oa_status, 0); in gen12_init_oa_buffer()
1836 intel_uncore_write(uncore, __oa_regs(stream)->oa_head_ptr, in gen12_init_oa_buffer()
1838 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1848 intel_uncore_write(uncore, __oa_regs(stream)->oa_buffer, gtt_offset | in gen12_init_oa_buffer()
1850 intel_uncore_write(uncore, __oa_regs(stream)->oa_tail_ptr, in gen12_init_oa_buffer()
1854 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1861 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1863 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1868 * first allocating), we may re-init the OA buffer, either in gen12_init_oa_buffer()
1869 * when re-enabling a stream or in error/reset paths. in gen12_init_oa_buffer()
1871 * The reason we clear the buffer for each re-init is for the in gen12_init_oa_buffer()
1873 * reason field to make sure it's non-zero which relies on in gen12_init_oa_buffer()
1877 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1878 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1883 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1884 struct intel_gt *gt = stream->engine->gt; in alloc_oa_buffer()
1889 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1890 return -ENODEV; in alloc_oa_buffer()
1895 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1897 drm_err(&i915->drm, "Failed to allocate OA buffer\n"); in alloc_oa_buffer()
1904 vma = i915_vma_instance(bo, >->ggtt->vm, NULL); in alloc_oa_buffer()
1916 drm_err(>->i915->drm, "Failed to pin OA buffer %d\n", ret); in alloc_oa_buffer()
1920 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1922 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1924 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1925 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1937 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1938 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1952 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1958 *cs++ = i915_ggtt_offset(stream->noa_wait) + offset + 4 * d; in save_restore_register()
1967 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1968 struct intel_gt *gt = stream->engine->gt; in alloc_noa_wait()
1971 const u64 delay_ticks = 0xffffffffffffffff - in alloc_noa_wait()
1972 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), in alloc_noa_wait()
1973 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1974 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1992 * gt->scratch was being used to save/restore the GPR registers, but on in alloc_noa_wait()
1999 drm_err(&i915->drm, in alloc_noa_wait()
2015 vma = i915_vma_instance(bo, >->ggtt->vm, NULL); in alloc_noa_wait()
2031 stream->noa_wait = vma; in alloc_noa_wait()
2056 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
2074 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
2094 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
2106 *cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4; in alloc_noa_wait()
2114 * (((1 * << 64) - 1) - delay_ns) in alloc_noa_wait()
2137 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
2149 *cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4; in alloc_noa_wait()
2167 GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch)); in alloc_noa_wait()
2177 if (ret == -EDEADLK) { in alloc_noa_wait()
2197 n_regs - i, in write_cs_mi_lri()
2234 return ERR_PTR(-ENOMEM); in alloc_oa_config_buffer()
2236 config_length += num_lri_dwords(oa_config->mux_regs_len); in alloc_oa_config_buffer()
2237 config_length += num_lri_dwords(oa_config->b_counter_regs_len); in alloc_oa_config_buffer()
2238 config_length += num_lri_dwords(oa_config->flex_regs_len); in alloc_oa_config_buffer()
2242 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
2261 oa_config->mux_regs, in alloc_oa_config_buffer()
2262 oa_config->mux_regs_len); in alloc_oa_config_buffer()
2264 oa_config->b_counter_regs, in alloc_oa_config_buffer()
2265 oa_config->b_counter_regs_len); in alloc_oa_config_buffer()
2267 oa_config->flex_regs, in alloc_oa_config_buffer()
2268 oa_config->flex_regs_len); in alloc_oa_config_buffer()
2271 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
2274 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
2280 oa_bo->vma = i915_vma_instance(obj, in alloc_oa_config_buffer()
2281 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
2283 if (IS_ERR(oa_bo->vma)) { in alloc_oa_config_buffer()
2284 err = PTR_ERR(oa_bo->vma); in alloc_oa_config_buffer()
2288 oa_bo->oa_config = i915_oa_config_get(oa_config); in alloc_oa_config_buffer()
2289 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
2292 if (err == -EDEADLK) { in alloc_oa_config_buffer()
2318 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
2319 if (oa_bo->oa_config == oa_config && in get_oa_vma()
2320 memcmp(oa_bo->oa_config->uuid, in get_oa_vma()
2321 oa_config->uuid, in get_oa_vma()
2322 sizeof(oa_config->uuid)) == 0) in get_oa_vma()
2331 return i915_vma_get(oa_bo->vma); in get_oa_vma()
2351 err = i915_gem_object_lock(vma->obj, &ww); in emit_oa_config()
2359 intel_engine_pm_get(ce->engine); in emit_oa_config()
2361 intel_engine_pm_put(ce->engine); in emit_oa_config()
2383 err = rq->engine->emit_bb_start(rq, in emit_oa_config()
2394 if (err == -EDEADLK) { in emit_oa_config()
2407 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2414 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2421 * unable to count the events from non-render clock domain. in hsw_enable_metric_set()
2423 * count the events from non-render domain. Unit level clock in hsw_enable_metric_set()
2432 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2438 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2462 for (i = 0; i < oa_config->flex_regs_len; i++) { in oa_config_flex_reg()
2463 if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio) in oa_config_flex_reg()
2464 return oa_config->flex_regs[i].value; in oa_config_flex_reg()
2473 * It's fine to put out-of-date values into these per-context registers
2480 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2481 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2492 u32 *reg_state = ce->lrc_reg_state; in gen8_update_reg_state_unlocked()
2496 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2497 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2502 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2523 offset = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET; in gen8_store_flex()
2526 *cs++ = offset + flex->offset * sizeof(u32); in gen8_store_flex()
2528 *cs++ = flex->value; in gen8_store_flex()
2529 } while (flex++, --count); in gen8_store_flex()
2551 *cs++ = i915_mmio_reg_offset(flex->reg); in gen8_load_flex()
2552 *cs++ = flex->value; in gen8_load_flex()
2553 } while (flex++, --count); in gen8_load_flex()
2567 rq = intel_engine_create_kernel_request(ce->engine); in gen8_modify_context()
2588 intel_engine_pm_get(ce->engine); in gen8_modify_self()
2590 intel_engine_pm_put(ce->engine); in gen8_modify_self()
2618 GEM_BUG_ON(ce == ce->engine->kernel_context); in gen8_configure_context()
2620 if (ce->engine->class != RENDER_CLASS) in gen8_configure_context()
2627 flex->value = intel_sseu_make_rpcs(ce->engine->gt, &ce->sseu); in gen8_configure_context()
2643 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2644 u32 format = stream->oa_buffer.format->format; in gen12_configure_oar_context()
2645 u32 offset = stream->perf->ctx_oactxctrl_offset; in gen12_configure_oar_context()
2665 RING_CONTEXT_CONTROL(ce->engine->mmio_base), in gen12_configure_oar_context()
2690 * Manages updating the per-context aspects of the OA stream
2700 * won't automatically reload an out-of-date timer exponent even
2704 * - Ensure the currently running context's per-context OA state is
2706 * - Ensure that all existing contexts will have the correct per-context
2708 * - Ensure any new contexts will be initialized with the correct
2709 * per-context OA state.
2720 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2722 struct intel_gt *gt = stream->engine->gt; in oa_configure_all_contexts()
2726 lockdep_assert_held(>->perf.lock); in oa_configure_all_contexts()
2731 * lite-restore). This means we can't safely update a context's image, in oa_configure_all_contexts()
2744 spin_lock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2745 list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) { in oa_configure_all_contexts()
2746 if (!kref_get_unless_zero(&ctx->ref)) in oa_configure_all_contexts()
2749 spin_unlock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2757 spin_lock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2761 spin_unlock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2769 struct intel_context *ce = engine->kernel_context; in oa_configure_all_contexts()
2771 if (engine->class != RENDER_CLASS) in oa_configure_all_contexts()
2774 regs[0].value = intel_sseu_make_rpcs(engine->gt, &ce->sseu); in oa_configure_all_contexts()
2789 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in lrc_configure_all_contexts()
2791 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2814 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2815 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2830 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2831 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2846 * Currently none of the high-level metrics we have depend on knowing in gen8_enable_metric_set()
2857 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2873 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2880 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2888 struct drm_i915_private *i915 = stream->perf->i915; in gen12_enable_metric_set()
2889 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2890 bool periodic = stream->periodic; in gen12_enable_metric_set()
2891 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2901 intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN, in gen12_enable_metric_set()
2907 intel_uncore_write(uncore, __oa_regs(stream)->oa_debug, in gen12_enable_metric_set()
2917 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctx_ctrl, periodic ? in gen12_enable_metric_set()
2938 if (stream->ctx) { in gen12_enable_metric_set()
2945 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2951 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2961 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2972 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2973 struct drm_i915_private *i915 = stream->perf->i915; in gen12_disable_metric_set()
2981 intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN, in gen12_disable_metric_set()
2988 if (stream->ctx) in gen12_disable_metric_set()
3003 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
3004 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
3005 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
3006 bool periodic = stream->periodic; in gen7_oa_enable()
3007 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
3008 u32 report_format = stream->oa_buffer.format->format; in gen7_oa_enable()
3033 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
3034 u32 report_format = stream->oa_buffer.format->format; in gen8_oa_enable()
3049 * filtering and instead filter on the cpu based on the context-id in gen8_oa_enable()
3066 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
3072 val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) | in gen12_oa_enable()
3075 intel_uncore_write(stream->uncore, regs->oa_ctrl, val); in gen12_oa_enable()
3079 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
3089 stream->pollin = false; in i915_oa_stream_enable()
3091 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
3093 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
3094 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
3095 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
3101 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
3107 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
3113 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
3119 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
3125 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
3127 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctrl, 0); in gen12_oa_disable()
3129 __oa_regs(stream)->oa_ctrl, in gen12_oa_disable()
3132 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3140 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3145 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
3154 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
3156 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
3157 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
3176 return -ENOMEM; in i915_perf_stream_enable_sync()
3178 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
3190 const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu; in get_default_sseu_config()
3194 if (GRAPHICS_VER(engine->i915) == 11) { in get_default_sseu_config()
3197 * we select - just turn off low bits in the amount of half of in get_default_sseu_config()
3200 out_sseu->subslice_mask = in get_default_sseu_config()
3201 ~(~0 << (hweight8(out_sseu->subslice_mask) / 2)); in get_default_sseu_config()
3202 out_sseu->slice_mask = 0x1; in get_default_sseu_config()
3211 if (drm_sseu->engine.engine_class != engine->uabi_class || in get_sseu_config()
3212 drm_sseu->engine.engine_instance != engine->uabi_instance) in get_sseu_config()
3213 return -EINVAL; in get_sseu_config()
3215 return i915_gem_user_to_context_sseu(engine->gt, drm_sseu, out_sseu); in get_sseu_config()
3232 with_intel_runtime_pm(to_gt(i915)->uncore->rpm, wakeref) in i915_perf_oa_timestamp_frequency()
3233 reg = intel_uncore_read(to_gt(i915)->uncore, RPM_CONFIG0); in i915_perf_oa_timestamp_frequency()
3238 return to_gt(i915)->clock_frequency << (3 - shift); in i915_perf_oa_timestamp_frequency()
3241 return to_gt(i915)->clock_frequency; in i915_perf_oa_timestamp_frequency()
3245 * i915_oa_stream_init - validate combined props for OA stream and init
3266 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
3267 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
3272 if (!props->engine) { in i915_oa_stream_init()
3273 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3275 return -EINVAL; in i915_oa_stream_init()
3277 gt = props->engine->gt; in i915_oa_stream_init()
3278 g = props->engine->oa_group; in i915_oa_stream_init()
3285 if (!perf->metrics_kobj) { in i915_oa_stream_init()
3286 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3288 return -EINVAL; in i915_oa_stream_init()
3291 if (!(props->sample_flags & SAMPLE_OA_REPORT) && in i915_oa_stream_init()
3292 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
3293 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3295 return -EINVAL; in i915_oa_stream_init()
3298 if (!perf->ops.enable_metric_set) { in i915_oa_stream_init()
3299 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3301 return -ENODEV; in i915_oa_stream_init()
3309 if (g->exclusive_stream) { in i915_oa_stream_init()
3310 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3312 return -EBUSY; in i915_oa_stream_init()
3315 if (!props->oa_format) { in i915_oa_stream_init()
3316 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3318 return -EINVAL; in i915_oa_stream_init()
3321 stream->engine = props->engine; in i915_oa_stream_init()
3322 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
3324 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
3326 stream->oa_buffer.format = &perf->oa_formats[props->oa_format]; in i915_oa_stream_init()
3327 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0)) in i915_oa_stream_init()
3328 return -EINVAL; in i915_oa_stream_init()
3330 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
3331 stream->sample_size += stream->oa_buffer.format->size; in i915_oa_stream_init()
3333 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
3335 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
3336 if (stream->periodic) in i915_oa_stream_init()
3337 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
3339 if (stream->ctx) { in i915_oa_stream_init()
3342 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3350 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3355 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
3356 if (!stream->oa_config) { in i915_oa_stream_init()
3357 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3358 "Invalid OA config id=%i\n", props->metrics_set); in i915_oa_stream_init()
3359 ret = -EINVAL; in i915_oa_stream_init()
3363 /* PRM - observability performance counters: in i915_oa_stream_init()
3375 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
3376 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3383 if (intel_uc_uses_guc_rc(>->uc) && in i915_oa_stream_init()
3384 (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_C0) || in i915_oa_stream_init()
3385 IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0))) { in i915_oa_stream_init()
3386 ret = intel_guc_slpc_override_gucrc_mode(>->uc.guc.slpc, in i915_oa_stream_init()
3389 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3394 stream->override_gucrc = true; in i915_oa_stream_init()
3401 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
3403 stream->engine->gt->perf.sseu = props->sseu; in i915_oa_stream_init()
3404 WRITE_ONCE(g->exclusive_stream, stream); in i915_oa_stream_init()
3408 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3413 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3415 stream->oa_config->uuid); in i915_oa_stream_init()
3417 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
3419 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
3420 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
3421 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
3422 mutex_init(&stream->lock); in i915_oa_stream_init()
3427 WRITE_ONCE(g->exclusive_stream, NULL); in i915_oa_stream_init()
3428 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
3433 if (stream->override_gucrc) in i915_oa_stream_init()
3434 intel_guc_slpc_unset_gucrc_mode(>->uc.guc.slpc); in i915_oa_stream_init()
3437 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3438 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
3446 if (stream->ctx) in i915_oa_stream_init()
3457 if (engine->class != RENDER_CLASS) in i915_oa_init_reg_state()
3461 stream = READ_ONCE(engine->oa_group->exclusive_stream); in i915_oa_init_reg_state()
3462 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3467 * i915_perf_read - handles read() FOP for i915 perf stream FDs
3475 * &i915_perf_stream_ops->read but to save having stream implementations (of
3489 struct i915_perf_stream *stream = file->private_data; in i915_perf_read()
3497 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3498 return -EIO; in i915_perf_read()
3500 if (!(file->f_flags & O_NONBLOCK)) { in i915_perf_read()
3502 * stream->ops->wait_unlocked. in i915_perf_read()
3509 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3513 mutex_lock(&stream->lock); in i915_perf_read()
3514 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3515 mutex_unlock(&stream->lock); in i915_perf_read()
3518 mutex_lock(&stream->lock); in i915_perf_read()
3519 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3520 mutex_unlock(&stream->lock); in i915_perf_read()
3527 * and read() returning -EAGAIN. Clearing the oa.pollin state here in i915_perf_read()
3530 * The exception to this is if ops->read() returned -ENOSPC which means in i915_perf_read()
3534 if (ret != -ENOSPC) in i915_perf_read()
3535 stream->pollin = false; in i915_perf_read()
3537 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */ in i915_perf_read()
3538 return offset ?: (ret ?: -EAGAIN); in i915_perf_read()
3547 stream->pollin = true; in oa_poll_check_timer_cb()
3548 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3552 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3558 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3564 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
3575 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3583 if (stream->pollin) in i915_perf_poll_locked()
3590 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3604 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll()
3607 mutex_lock(&stream->lock); in i915_perf_poll()
3609 mutex_unlock(&stream->lock); in i915_perf_poll()
3615 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
3626 if (stream->enabled) in i915_perf_enable_locked()
3629 /* Allow stream->ops->enable() to refer to this */ in i915_perf_enable_locked()
3630 stream->enabled = true; in i915_perf_enable_locked()
3632 if (stream->ops->enable) in i915_perf_enable_locked()
3633 stream->ops->enable(stream); in i915_perf_enable_locked()
3635 if (stream->hold_preemption) in i915_perf_enable_locked()
3636 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3640 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
3645 * The intention is that disabling an re-enabling a stream will ideally be
3646 * cheaper than destroying and re-opening a stream with the same configuration,
3648 * must be retained between disabling and re-enabling a stream.
3651 * to attempt to read from the stream (-EIO).
3655 if (!stream->enabled) in i915_perf_disable_locked()
3658 /* Allow stream->ops->disable() to refer to this */ in i915_perf_disable_locked()
3659 stream->enabled = false; in i915_perf_disable_locked()
3661 if (stream->hold_preemption) in i915_perf_disable_locked()
3662 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3664 if (stream->ops->disable) in i915_perf_disable_locked()
3665 stream->ops->disable(stream); in i915_perf_disable_locked()
3672 long ret = stream->oa_config->id; in i915_perf_config_locked()
3674 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3676 return -EINVAL; in i915_perf_config_locked()
3678 if (config != stream->oa_config) { in i915_perf_config_locked()
3692 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3703 * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3708 * Returns: zero on success or a negative error code. Returns -EINVAL for
3726 return -EINVAL; in i915_perf_ioctl_locked()
3730 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3737 * Returns: zero on success or a negative error code. Returns -EINVAL for
3744 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl()
3747 mutex_lock(&stream->lock); in i915_perf_ioctl()
3749 mutex_unlock(&stream->lock); in i915_perf_ioctl()
3755 * i915_perf_destroy_locked - destroy an i915 perf stream
3761 * Note: The >->perf.lock mutex has been taken to serialize
3762 * with any non-file-operation driver hooks.
3766 if (stream->enabled) in i915_perf_destroy_locked()
3769 if (stream->ops->destroy) in i915_perf_destroy_locked()
3770 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3772 if (stream->ctx) in i915_perf_destroy_locked()
3773 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3779 * i915_perf_release - handles userspace close() of a stream file
3791 struct i915_perf_stream *stream = file->private_data; in i915_perf_release()
3792 struct i915_perf *perf = stream->perf; in i915_perf_release()
3793 struct intel_gt *gt = stream->engine->gt; in i915_perf_release()
3797 * other user of stream->lock. Use the perf lock to destroy the stream in i915_perf_release()
3800 mutex_lock(>->perf.lock); in i915_perf_release()
3802 mutex_unlock(>->perf.lock); in i915_perf_release()
3805 drm_dev_put(&perf->i915->drm); in i915_perf_release()
3826 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3835 * behalf of i915_perf_open_ioctl() with the >->perf.lock mutex
3836 * taken to serialize with any non-file-operation driver hooks.
3862 if (props->single_context) { in i915_perf_open_ioctl_locked()
3863 u32 ctx_handle = props->ctx_handle; in i915_perf_open_ioctl_locked()
3864 struct drm_i915_file_private *file_priv = file->driver_priv; in i915_perf_open_ioctl_locked()
3868 drm_dbg(&perf->i915->drm, in i915_perf_open_ioctl_locked()
3880 * non-privileged client. in i915_perf_open_ioctl_locked()
3882 * For Gen8->11 the OA unit no longer supports clock gating off for a in i915_perf_open_ioctl_locked()
3884 * from updating as system-wide / global values. Even though we can in i915_perf_open_ioctl_locked()
3895 if (IS_HASWELL(perf->i915) && specific_ctx) in i915_perf_open_ioctl_locked()
3897 else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx && in i915_perf_open_ioctl_locked()
3898 (props->sample_flags & SAMPLE_OA_REPORT) == 0) in i915_perf_open_ioctl_locked()
3901 if (props->hold_preemption) { in i915_perf_open_ioctl_locked()
3902 if (!props->single_context) { in i915_perf_open_ioctl_locked()
3903 drm_dbg(&perf->i915->drm, in i915_perf_open_ioctl_locked()
3905 ret = -EINVAL; in i915_perf_open_ioctl_locked()
3914 if (props->has_sseu) in i915_perf_open_ioctl_locked()
3917 get_default_sseu_config(&props->sseu, props->engine); in i915_perf_open_ioctl_locked()
3926 drm_dbg(&perf->i915->drm, in i915_perf_open_ioctl_locked()
3928 ret = -EACCES; in i915_perf_open_ioctl_locked()
3934 ret = -ENOMEM; in i915_perf_open_ioctl_locked()
3938 stream->perf = perf; in i915_perf_open_ioctl_locked()
3939 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3940 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3946 /* we avoid simply assigning stream->sample_flags = props->sample_flags in i915_perf_open_ioctl_locked()
3950 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3951 ret = -ENODEV; in i915_perf_open_ioctl_locked()
3955 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) in i915_perf_open_ioctl_locked()
3957 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) in i915_perf_open_ioctl_locked()
3966 if (!(param->flags & I915_PERF_FLAG_DISABLED)) in i915_perf_open_ioctl_locked()
3972 drm_dev_get(&perf->i915->drm); in i915_perf_open_ioctl_locked()
3977 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3978 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3991 u32 den = i915_perf_oa_timestamp_frequency(perf->i915); in oa_exponent_to_ns()
3993 return div_u64(nom + den - 1, den); in oa_exponent_to_ns()
3999 return test_bit(format, perf->format_mask); in oa_format_valid()
4005 __set_bit(format, perf->format_mask); in oa_format_add()
4009 * read_properties_unlocked - validate + copy userspace stream open properties
4039 props->poll_oa_period = DEFAULT_POLL_PERIOD_NS; in read_properties_unlocked()
4048 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4050 return -EINVAL; in read_properties_unlocked()
4070 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4072 return -EINVAL; in read_properties_unlocked()
4077 props->single_context = 1; in read_properties_unlocked()
4078 props->ctx_handle = value; in read_properties_unlocked()
4082 props->sample_flags |= SAMPLE_OA_REPORT; in read_properties_unlocked()
4086 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4088 return -EINVAL; in read_properties_unlocked()
4090 props->metrics_set = value; in read_properties_unlocked()
4094 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4095 "Out-of-range OA report format %llu\n", in read_properties_unlocked()
4097 return -EINVAL; in read_properties_unlocked()
4100 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4103 return -EINVAL; in read_properties_unlocked()
4105 props->oa_format = value; in read_properties_unlocked()
4109 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4112 return -EINVAL; in read_properties_unlocked()
4138 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4141 return -EACCES; in read_properties_unlocked()
4144 props->oa_periodic = true; in read_properties_unlocked()
4145 props->oa_period_exponent = value; in read_properties_unlocked()
4148 props->hold_preemption = !!value; in read_properties_unlocked()
4151 if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 50)) { in read_properties_unlocked()
4152 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4154 GRAPHICS_VER_FULL(perf->i915)); in read_properties_unlocked()
4155 return -ENODEV; in read_properties_unlocked()
4161 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4163 return -EFAULT; in read_properties_unlocked()
4170 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4173 return -EINVAL; in read_properties_unlocked()
4175 props->poll_oa_period = value; in read_properties_unlocked()
4187 return -EINVAL; in read_properties_unlocked()
4195 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4196 "OA engine-class and engine-instance parameters must be passed together\n"); in read_properties_unlocked()
4197 return -EINVAL; in read_properties_unlocked()
4200 props->engine = intel_engine_lookup_user(perf->i915, class, instance); in read_properties_unlocked()
4201 if (!props->engine) { in read_properties_unlocked()
4202 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4205 return -EINVAL; in read_properties_unlocked()
4208 if (!engine_supports_oa(props->engine)) { in read_properties_unlocked()
4209 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4212 return -EINVAL; in read_properties_unlocked()
4220 if (IS_MTL_MEDIA_STEP(props->engine->i915, STEP_A0, STEP_C0) && in read_properties_unlocked()
4221 props->engine->oa_group->type == TYPE_OAM && in read_properties_unlocked()
4222 intel_check_bios_c6_setup(&props->engine->gt->rc6)) { in read_properties_unlocked()
4223 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4225 return -EINVAL; in read_properties_unlocked()
4228 i = array_index_nospec(props->oa_format, I915_OA_FORMAT_MAX); in read_properties_unlocked()
4229 f = &perf->oa_formats[i]; in read_properties_unlocked()
4230 if (!engine_supports_oa_format(props->engine, f->type)) { in read_properties_unlocked()
4231 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4233 f->type, props->engine->class); in read_properties_unlocked()
4234 return -EINVAL; in read_properties_unlocked()
4238 ret = get_sseu_config(&props->sseu, props->engine, &user_sseu); in read_properties_unlocked()
4240 drm_dbg(&perf->i915->drm, in read_properties_unlocked()
4244 props->has_sseu = true; in read_properties_unlocked()
4251 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
4261 * i915-perf stream is expected to be a suitable interface for other forms of
4268 * i915_perf_open_ioctl_locked() after taking the >->perf.lock
4269 * mutex for serializing with any non-file-operation driver hooks.
4277 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_open_ioctl()
4284 if (!perf->i915) in i915_perf_open_ioctl()
4285 return -ENOTSUPP; in i915_perf_open_ioctl()
4290 if (param->flags & ~known_open_flags) { in i915_perf_open_ioctl()
4291 drm_dbg(&perf->i915->drm, in i915_perf_open_ioctl()
4293 return -EINVAL; in i915_perf_open_ioctl()
4297 u64_to_user_ptr(param->properties_ptr), in i915_perf_open_ioctl()
4298 param->num_properties, in i915_perf_open_ioctl()
4303 gt = props.engine->gt; in i915_perf_open_ioctl()
4305 mutex_lock(>->perf.lock); in i915_perf_open_ioctl()
4307 mutex_unlock(>->perf.lock); in i915_perf_open_ioctl()
4313 * i915_perf_register - exposes i915-perf to userspace
4318 * used to open an i915-perf stream.
4322 struct i915_perf *perf = &i915->perf; in i915_perf_register()
4325 if (!perf->i915) in i915_perf_register()
4332 mutex_lock(>->perf.lock); in i915_perf_register()
4334 perf->metrics_kobj = in i915_perf_register()
4336 &i915->drm.primary->kdev->kobj); in i915_perf_register()
4338 mutex_unlock(>->perf.lock); in i915_perf_register()
4342 * i915_perf_unregister - hide i915-perf from userspace
4345 * i915-perf state cleanup is split up into an 'unregister' and
4352 struct i915_perf *perf = &i915->perf; in i915_perf_unregister()
4354 if (!perf->metrics_kobj) in i915_perf_unregister()
4357 kobject_put(perf->metrics_kobj); in i915_perf_unregister()
4358 perf->metrics_kobj = NULL; in i915_perf_unregister()
4383 while (table->start || table->end) { in reg_in_range_table()
4384 if (addr >= table->start && addr <= table->end) in reg_in_range_table()
4397 { .start = 0x2710, .end = 0x272c }, /* OASTARTTRIG[1-8] */
4398 { .start = 0x2740, .end = 0x275c }, /* OAREPORTTRIG[1-8] */
4399 { .start = 0x2770, .end = 0x27ac }, /* OACEC[0-7][0-1] */
4405 { .start = 0xd900, .end = 0xd91c }, /* GEN12_OAG_OASTARTTRIG[1-8] */
4406 { .start = 0xd920, .end = 0xd93c }, /* GEN12_OAG_OAREPORTTRIG1[1-8] */
4407 { .start = 0xd940, .end = 0xd97c }, /* GEN12_OAG_CEC[0-7][0-1] */
4408 { .start = 0xdc00, .end = 0xdc3c }, /* GEN12_OAG_SCEC[0-7][0-1] */
4415 { .start = 0x393000, .end = 0x39301c }, /* GEN12_OAM_STARTTRIG1[1-8] */
4416 { .start = 0x393020, .end = 0x39303c }, /* GEN12_OAM_REPORTTRIG1[1-8] */
4417 { .start = 0x393040, .end = 0x39307c }, /* GEN12_OAM_CEC[0-7][0-1] */
4418 { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */
4424 { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */
4429 { .start = 0x91b8, .end = 0x91cc }, /* OA_PERFCNT[1-2], OA_PERFMATRIX */
4430 { .start = 0x9800, .end = 0x9888 }, /* MICRO_BP0_0 - NOA_WRITE */
4436 { .start = 0x09e80, .end = 0x09ea4 }, /* HSW_MBVID2_NOA[0-9] */
4448 { .start = 0x0d00, .end = 0x0d2c }, /* RPM_CONFIG[0-1], NOA_CONFIG[0-8] */
4454 { .start = 0x91c8, .end = 0x91dc }, /* OA_PERFCNT[3-4] */
4459 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4460 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4472 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4473 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4517 if (HAS_OAM(perf->i915) && in mtl_is_valid_oam_b_counter_addr()
4518 GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70)) in mtl_is_valid_oam_b_counter_addr()
4533 if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70)) in gen12_is_valid_mux_addr()
4573 return ERR_PTR(-EINVAL); in alloc_oa_regs()
4577 return ERR_PTR(-ENOMEM); in alloc_oa_regs()
4587 drm_dbg(&perf->i915->drm, in alloc_oa_regs()
4589 err = -EINVAL; in alloc_oa_regs()
4617 return sprintf(buf, "%d\n", oa_config->id); in show_dynamic_id()
4623 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); in create_dynamic_oa_sysfs_entry()
4624 oa_config->sysfs_metric_id.attr.name = "id"; in create_dynamic_oa_sysfs_entry()
4625 oa_config->sysfs_metric_id.attr.mode = S_IRUGO; in create_dynamic_oa_sysfs_entry()
4626 oa_config->sysfs_metric_id.show = show_dynamic_id; in create_dynamic_oa_sysfs_entry()
4627 oa_config->sysfs_metric_id.store = NULL; in create_dynamic_oa_sysfs_entry()
4629 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; in create_dynamic_oa_sysfs_entry()
4630 oa_config->attrs[1] = NULL; in create_dynamic_oa_sysfs_entry()
4632 oa_config->sysfs_metric.name = oa_config->uuid; in create_dynamic_oa_sysfs_entry()
4633 oa_config->sysfs_metric.attrs = oa_config->attrs; in create_dynamic_oa_sysfs_entry()
4635 return sysfs_create_group(perf->metrics_kobj, in create_dynamic_oa_sysfs_entry()
4636 &oa_config->sysfs_metric); in create_dynamic_oa_sysfs_entry()
4640 * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
4655 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_add_config_ioctl()
4661 if (!perf->i915) in i915_perf_add_config_ioctl()
4662 return -ENOTSUPP; in i915_perf_add_config_ioctl()
4664 if (!perf->metrics_kobj) { in i915_perf_add_config_ioctl()
4665 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4667 return -EINVAL; in i915_perf_add_config_ioctl()
4671 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4673 return -EACCES; in i915_perf_add_config_ioctl()
4676 if ((!args->mux_regs_ptr || !args->n_mux_regs) && in i915_perf_add_config_ioctl()
4677 (!args->boolean_regs_ptr || !args->n_boolean_regs) && in i915_perf_add_config_ioctl()
4678 (!args->flex_regs_ptr || !args->n_flex_regs)) { in i915_perf_add_config_ioctl()
4679 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4681 return -EINVAL; in i915_perf_add_config_ioctl()
4686 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4688 return -ENOMEM; in i915_perf_add_config_ioctl()
4691 oa_config->perf = perf; in i915_perf_add_config_ioctl()
4692 kref_init(&oa_config->ref); in i915_perf_add_config_ioctl()
4694 if (!uuid_is_valid(args->uuid)) { in i915_perf_add_config_ioctl()
4695 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4697 err = -EINVAL; in i915_perf_add_config_ioctl()
4701 /* Last character in oa_config->uuid will be 0 because oa_config is in i915_perf_add_config_ioctl()
4704 memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid)); in i915_perf_add_config_ioctl()
4706 oa_config->mux_regs_len = args->n_mux_regs; in i915_perf_add_config_ioctl()
4708 perf->ops.is_valid_mux_reg, in i915_perf_add_config_ioctl()
4709 u64_to_user_ptr(args->mux_regs_ptr), in i915_perf_add_config_ioctl()
4710 args->n_mux_regs); in i915_perf_add_config_ioctl()
4713 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4718 oa_config->mux_regs = regs; in i915_perf_add_config_ioctl()
4720 oa_config->b_counter_regs_len = args->n_boolean_regs; in i915_perf_add_config_ioctl()
4722 perf->ops.is_valid_b_counter_reg, in i915_perf_add_config_ioctl()
4723 u64_to_user_ptr(args->boolean_regs_ptr), in i915_perf_add_config_ioctl()
4724 args->n_boolean_regs); in i915_perf_add_config_ioctl()
4727 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4732 oa_config->b_counter_regs = regs; in i915_perf_add_config_ioctl()
4734 if (GRAPHICS_VER(perf->i915) < 8) { in i915_perf_add_config_ioctl()
4735 if (args->n_flex_regs != 0) { in i915_perf_add_config_ioctl()
4736 err = -EINVAL; in i915_perf_add_config_ioctl()
4740 oa_config->flex_regs_len = args->n_flex_regs; in i915_perf_add_config_ioctl()
4742 perf->ops.is_valid_flex_reg, in i915_perf_add_config_ioctl()
4743 u64_to_user_ptr(args->flex_regs_ptr), in i915_perf_add_config_ioctl()
4744 args->n_flex_regs); in i915_perf_add_config_ioctl()
4747 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4752 oa_config->flex_regs = regs; in i915_perf_add_config_ioctl()
4755 err = mutex_lock_interruptible(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4762 idr_for_each_entry(&perf->metrics_idr, tmp, id) { in i915_perf_add_config_ioctl()
4763 if (!strcmp(tmp->uuid, oa_config->uuid)) { in i915_perf_add_config_ioctl()
4764 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4766 err = -EADDRINUSE; in i915_perf_add_config_ioctl()
4773 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4779 oa_config->id = idr_alloc(&perf->metrics_idr, in i915_perf_add_config_ioctl()
4782 if (oa_config->id < 0) { in i915_perf_add_config_ioctl()
4783 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4785 err = oa_config->id; in i915_perf_add_config_ioctl()
4788 id = oa_config->id; in i915_perf_add_config_ioctl()
4790 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4791 "Added config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_add_config_ioctl()
4792 mutex_unlock(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4797 mutex_unlock(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4800 drm_dbg(&perf->i915->drm, in i915_perf_add_config_ioctl()
4806 * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
4819 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_remove_config_ioctl()
4824 if (!perf->i915) in i915_perf_remove_config_ioctl()
4825 return -ENOTSUPP; in i915_perf_remove_config_ioctl()
4828 drm_dbg(&perf->i915->drm, in i915_perf_remove_config_ioctl()
4830 return -EACCES; in i915_perf_remove_config_ioctl()
4833 ret = mutex_lock_interruptible(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4837 oa_config = idr_find(&perf->metrics_idr, *arg); in i915_perf_remove_config_ioctl()
4839 drm_dbg(&perf->i915->drm, in i915_perf_remove_config_ioctl()
4841 ret = -ENOENT; in i915_perf_remove_config_ioctl()
4845 GEM_BUG_ON(*arg != oa_config->id); in i915_perf_remove_config_ioctl()
4847 sysfs_remove_group(perf->metrics_kobj, &oa_config->sysfs_metric); in i915_perf_remove_config_ioctl()
4849 idr_remove(&perf->metrics_idr, *arg); in i915_perf_remove_config_ioctl()
4851 mutex_unlock(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4853 drm_dbg(&perf->i915->drm, in i915_perf_remove_config_ioctl()
4854 "Removed config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_remove_config_ioctl()
4861 mutex_unlock(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4894 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) { in __oam_engine_group()
4899 drm_WARN_ON(&engine->i915->drm, in __oam_engine_group()
4900 engine->gt->type != GT_MEDIA); in __oam_engine_group()
4910 switch (engine->class) { in __oa_engine_group()
4955 int i, num_groups = gt->perf.num_perf_groups; in oa_init_groups()
4958 struct i915_perf_group *g = >->perf.group[i]; in oa_init_groups()
4961 if (g->num_engines == 0) in oa_init_groups()
4964 if (i == PERF_GROUP_OAG && gt->type != GT_MEDIA) { in oa_init_groups()
4965 g->regs = __oag_regs(); in oa_init_groups()
4966 g->type = TYPE_OAG; in oa_init_groups()
4967 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { in oa_init_groups()
4968 g->regs = __oam_regs(mtl_oa_base[i]); in oa_init_groups()
4969 g->type = TYPE_OAM; in oa_init_groups()
4983 return -ENOMEM; in oa_init_gt()
4988 engine->oa_group = NULL; in oa_init_gt()
4991 engine->oa_group = &g[index]; in oa_init_gt()
4995 gt->perf.num_perf_groups = num_groups; in oa_init_gt()
4996 gt->perf.group = g; in oa_init_gt()
5008 for_each_gt(gt, perf->i915, i) { in oa_init_engine_groups()
5019 struct drm_i915_private *i915 = perf->i915; in oa_init_supported_formats()
5020 enum intel_platform platform = INTEL_INFO(i915)->platform; in oa_init_supported_formats()
5075 struct i915_perf *perf = &i915->perf; in i915_perf_init_info()
5079 perf->ctx_oactxctrl_offset = 0x120; in i915_perf_init_info()
5080 perf->ctx_flexeu0_offset = 0x2ce; in i915_perf_init_info()
5081 perf->gen8_valid_ctx_bit = BIT(25); in i915_perf_init_info()
5084 perf->ctx_oactxctrl_offset = 0x128; in i915_perf_init_info()
5085 perf->ctx_flexeu0_offset = 0x3de; in i915_perf_init_info()
5086 perf->gen8_valid_ctx_bit = BIT(16); in i915_perf_init_info()
5089 perf->ctx_oactxctrl_offset = 0x124; in i915_perf_init_info()
5090 perf->ctx_flexeu0_offset = 0x78e; in i915_perf_init_info()
5091 perf->gen8_valid_ctx_bit = BIT(16); in i915_perf_init_info()
5094 perf->gen8_valid_ctx_bit = BIT(16); in i915_perf_init_info()
5097 * cache the value in perf->ctx_oactxctrl_offset. in i915_perf_init_info()
5106 * i915_perf_init - initialize i915-perf state on module bind
5109 * Initializes i915-perf state without exposing anything to userspace.
5111 * Note: i915-perf initialization is split into an 'init' and 'register'
5116 struct i915_perf *perf = &i915->perf; in i915_perf_init()
5118 perf->oa_formats = oa_formats; in i915_perf_init()
5120 perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; in i915_perf_init()
5121 perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr; in i915_perf_init()
5122 perf->ops.is_valid_flex_reg = NULL; in i915_perf_init()
5123 perf->ops.enable_metric_set = hsw_enable_metric_set; in i915_perf_init()
5124 perf->ops.disable_metric_set = hsw_disable_metric_set; in i915_perf_init()
5125 perf->ops.oa_enable = gen7_oa_enable; in i915_perf_init()
5126 perf->ops.oa_disable = gen7_oa_disable; in i915_perf_init()
5127 perf->ops.read = gen7_oa_read; in i915_perf_init()
5128 perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read; in i915_perf_init()
5136 perf->ops.read = gen8_oa_read; in i915_perf_init()
5140 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
5142 perf->ops.is_valid_mux_reg = in i915_perf_init()
5144 perf->ops.is_valid_flex_reg = in i915_perf_init()
5148 perf->ops.is_valid_mux_reg = in i915_perf_init()
5152 perf->ops.oa_enable = gen8_oa_enable; in i915_perf_init()
5153 perf->ops.oa_disable = gen8_oa_disable; in i915_perf_init()
5154 perf->ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
5155 perf->ops.disable_metric_set = gen8_disable_metric_set; in i915_perf_init()
5156 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
5158 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
5160 perf->ops.is_valid_mux_reg = in i915_perf_init()
5162 perf->ops.is_valid_flex_reg = in i915_perf_init()
5165 perf->ops.oa_enable = gen8_oa_enable; in i915_perf_init()
5166 perf->ops.oa_disable = gen8_oa_disable; in i915_perf_init()
5167 perf->ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
5168 perf->ops.disable_metric_set = gen11_disable_metric_set; in i915_perf_init()
5169 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
5171 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
5175 perf->ops.is_valid_mux_reg = in i915_perf_init()
5177 perf->ops.is_valid_flex_reg = in i915_perf_init()
5180 perf->ops.oa_enable = gen12_oa_enable; in i915_perf_init()
5181 perf->ops.oa_disable = gen12_oa_disable; in i915_perf_init()
5182 perf->ops.enable_metric_set = gen12_enable_metric_set; in i915_perf_init()
5183 perf->ops.disable_metric_set = gen12_disable_metric_set; in i915_perf_init()
5184 perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read; in i915_perf_init()
5188 if (perf->ops.enable_metric_set) { in i915_perf_init()
5193 mutex_init(>->perf.lock); in i915_perf_init()
5196 oa_sample_rate_hard_limit = to_gt(i915)->clock_frequency / 2; in i915_perf_init()
5198 mutex_init(&perf->metrics_lock); in i915_perf_init()
5199 idr_init_base(&perf->metrics_idr, 1); in i915_perf_init()
5211 ratelimit_state_init(&perf->spurious_report_rs, 5 * HZ, 10); in i915_perf_init()
5216 ratelimit_set_flags(&perf->spurious_report_rs, in i915_perf_init()
5219 ratelimit_state_init(&perf->tail_pointer_race, in i915_perf_init()
5221 ratelimit_set_flags(&perf->tail_pointer_race, in i915_perf_init()
5224 atomic64_set(&perf->noa_programming_delay, in i915_perf_init()
5227 perf->i915 = i915; in i915_perf_init()
5231 drm_err(&i915->drm, in i915_perf_init()
5260 * i915_perf_fini - Counter part to i915_perf_init()
5265 struct i915_perf *perf = &i915->perf; in i915_perf_fini()
5269 if (!perf->i915) in i915_perf_fini()
5272 for_each_gt(gt, perf->i915, i) in i915_perf_fini()
5273 kfree(gt->perf.group); in i915_perf_fini()
5275 idr_for_each(&perf->metrics_idr, destroy_config, perf); in i915_perf_fini()
5276 idr_destroy(&perf->metrics_idr); in i915_perf_fini()
5278 memset(&perf->ops, 0, sizeof(perf->ops)); in i915_perf_fini()
5279 perf->i915 = NULL; in i915_perf_fini()
5283 * i915_perf_ioctl_version - Version of the i915-perf subsystem
5326 if (gt->type == GT_MEDIA && in i915_perf_ioctl_version()
5327 intel_check_bios_c6_setup(>->rc6)) in i915_perf_ioctl_version()