213 static u64 precise_datala_hsw(struct perf_event *event, u64 status)  in precise_datala_hsw()  argument
219 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)  in precise_datala_hsw()
221 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)  in precise_datala_hsw()
232 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {  in precise_datala_hsw()
259 static u64 __adl_latency_data_small(struct perf_event *event, u64 status,  in __adl_latency_data_small()  argument
264 	WARN_ON_ONCE(hybrid_pmu(event->pmu)->cpu_type == hybrid_big);  in __adl_latency_data_small()
267 	val = hybrid_var(event->pmu, pebs_data_source)[dse];  in __adl_latency_data_small()
279 u64 adl_latency_data_small(struct perf_event *event, u64 status)  in adl_latency_data_small()  argument
285 	return __adl_latency_data_small(event, status, dse.ld_dse,  in adl_latency_data_small()
291 u64 mtl_latency_data_small(struct perf_event *event, u64 status)  in mtl_latency_data_small()  argument
297 	return __adl_latency_data_small(event, status, dse.mtl_dse,  in mtl_latency_data_small()
302 static u64 load_latency_data(struct perf_event *event, u64 status)  in load_latency_data()  argument
312 	val = hybrid_var(event->pmu, pebs_data_source)[dse.ld_dse];  in load_latency_data()
351 static u64 store_latency_data(struct perf_event *event, u64 status)  in store_latency_data()  argument
362 	val = hybrid_var(event->pmu, pebs_data_source)[dse.st_lat_dse];  in store_latency_data()
764 	struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in intel_pmu_drain_bts_buffer()  local
772 	if (!event)  in intel_pmu_drain_bts_buffer()
788 	perf_sample_data_init(&data, 0, event->hw.last_period);  in intel_pmu_drain_bts_buffer()
806 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
817 	perf_prepare_sample(&data, event, &regs);  in intel_pmu_drain_bts_buffer()
818 	perf_prepare_header(&header, &data, event, &regs);  in intel_pmu_drain_bts_buffer()
820 	if (perf_output_begin(&handle, &data, event,  in intel_pmu_drain_bts_buffer()
826 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
833 		perf_output_sample(&handle, &header, &data, event);  in intel_pmu_drain_bts_buffer()
839 	event->hw.interrupts++;  in intel_pmu_drain_bts_buffer()
840 	event->pending_kill = POLL_IN;  in intel_pmu_drain_bts_buffer()
1088 struct event_constraint *intel_pebs_constraints(struct perf_event *event)  in intel_pebs_constraints()  argument
1090 	struct event_constraint *pebs_constraints = hybrid(event->pmu, pebs_constraints);  in intel_pebs_constraints()
1093 	if (!event->attr.precise_ip)  in intel_pebs_constraints()
1098 			if (constraint_match(c, event->hw.config)) {  in intel_pebs_constraints()
1099 				event->hw.flags |= c->flags;  in intel_pebs_constraints()
1186 static u64 pebs_update_adaptive_cfg(struct perf_event *event)  in pebs_update_adaptive_cfg()  argument
1188 	struct perf_event_attr *attr = &event->attr;  in pebs_update_adaptive_cfg()
1234 		  struct perf_event *event, bool add)  in pebs_update_state()  argument
1236 	struct pmu *pmu = event->pmu;  in pebs_update_state()
1262 		pebs_data_cfg = pebs_update_adaptive_cfg(event);  in pebs_update_state()
1271 void intel_pmu_pebs_add(struct perf_event *event)  in intel_pmu_pebs_add()  argument
1274 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_add()
1283 	pebs_update_state(needed_cb, cpuc, event, true);  in intel_pmu_pebs_add()
1286 static void intel_pmu_pebs_via_pt_disable(struct perf_event *event)  in intel_pmu_pebs_via_pt_disable()  argument
1290 	if (!is_pebs_pt(event))  in intel_pmu_pebs_via_pt_disable()
1297 static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)  in intel_pmu_pebs_via_pt_enable()  argument
1300 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_via_pt_enable()
1306 	if (!is_pebs_pt(event))  in intel_pmu_pebs_via_pt_enable()
1309 	if (!(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))  in intel_pmu_pebs_via_pt_enable()
1332 void intel_pmu_pebs_enable(struct perf_event *event)  in intel_pmu_pebs_enable()  argument
1336 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_enable()
1344 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))  in intel_pmu_pebs_enable()
1346 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_enable()
1386 	intel_pmu_pebs_via_pt_enable(event);  in intel_pmu_pebs_enable()
1389 void intel_pmu_pebs_del(struct perf_event *event)  in intel_pmu_pebs_del()  argument
1392 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_del()
1401 	pebs_update_state(needed_cb, cpuc, event, false);  in intel_pmu_pebs_del()
1404 void intel_pmu_pebs_disable(struct perf_event *event)  in intel_pmu_pebs_disable()  argument
1407 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_disable()
1413 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&  in intel_pmu_pebs_disable()
1416 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_disable()
1419 	intel_pmu_pebs_via_pt_disable(event);  in intel_pmu_pebs_disable()
1567 static u64 get_data_src(struct perf_event *event, u64 aux)  in get_data_src()  argument
1570 	int fl = event->hw.flags;  in get_data_src()
1574 		val = load_latency_data(event, aux);  in get_data_src()
1576 		val = store_latency_data(event, aux);  in get_data_src()
1578 		val = x86_pmu.pebs_latency_data(event, aux);  in get_data_src()
1580 		val = precise_datala_hsw(event, aux);  in get_data_src()
1586 static void setup_pebs_time(struct perf_event *event,  in setup_pebs_time()  argument
1591 	if (event->attr.use_clockid != 0)  in setup_pebs_time()
1611 static void setup_pebs_fixed_sample_data(struct perf_event *event,  in setup_pebs_fixed_sample_data()  argument
1628 	sample_type = event->attr.sample_type;  in setup_pebs_fixed_sample_data()
1629 	fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;  in setup_pebs_fixed_sample_data()
1631 	perf_sample_data_init(data, 0, event->hw.last_period);  in setup_pebs_fixed_sample_data()
1633 	data->period = event->hw.last_period;  in setup_pebs_fixed_sample_data()
1647 		data->data_src.val = get_data_src(event, pebs->dse);  in setup_pebs_fixed_sample_data()
1658 		perf_sample_save_callchain(data, event, iregs);  in setup_pebs_fixed_sample_data()
1699 	if (event->attr.precise_ip > 1) {  in setup_pebs_fixed_sample_data()
1755 		setup_pebs_time(event, data, pebs->tsc);  in setup_pebs_fixed_sample_data()
1757 	if (has_branch_stack(event))  in setup_pebs_fixed_sample_data()
1758 		perf_sample_save_brstack(data, event, &cpuc->lbr_stack);  in setup_pebs_fixed_sample_data()
1792 static void setup_pebs_adaptive_sample_data(struct perf_event *event,  in setup_pebs_adaptive_sample_data()  argument
1812 	sample_type = event->attr.sample_type;  in setup_pebs_adaptive_sample_data()
1814 	perf_sample_data_init(data, 0, event->hw.last_period);  in setup_pebs_adaptive_sample_data()
1815 	data->period = event->hw.last_period;  in setup_pebs_adaptive_sample_data()
1817 	setup_pebs_time(event, data, basic->tsc);  in setup_pebs_adaptive_sample_data()
1826 		perf_sample_save_callchain(data, event, iregs);  in setup_pebs_adaptive_sample_data()
1854 		if (event->attr.precise_ip < 2) {  in setup_pebs_adaptive_sample_data()
1888 			data->data_src.val = get_data_src(event, meminfo->aux);  in setup_pebs_adaptive_sample_data()
1917 		if (has_branch_stack(event)) {  in setup_pebs_adaptive_sample_data()
1919 			perf_sample_save_brstack(data, event, &cpuc->lbr_stack);  in setup_pebs_adaptive_sample_data()
1968 void intel_pmu_auto_reload_read(struct perf_event *event)  in intel_pmu_auto_reload_read()  argument
1970 	WARN_ON(!(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD));  in intel_pmu_auto_reload_read()
1972 	perf_pmu_disable(event->pmu);  in intel_pmu_auto_reload_read()
1974 	perf_pmu_enable(event->pmu);  in intel_pmu_auto_reload_read()
1981 intel_pmu_save_and_restart_reload(struct perf_event *event, int count)  in intel_pmu_save_and_restart_reload()  argument
1983 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_save_and_restart_reload()
2029 	local64_add(new - old + count * period, &event->count);  in intel_pmu_save_and_restart_reload()
2033 	perf_event_update_userpage(event);  in intel_pmu_save_and_restart_reload()
2039 __intel_pmu_pebs_event(struct perf_event *event,  in __intel_pmu_pebs_event()  argument
2051 	struct hw_perf_event *hwc = &event->hw;  in __intel_pmu_pebs_event()
2064 		intel_pmu_save_and_restart_reload(event, count);  in __intel_pmu_pebs_event()
2065 	} else if (!intel_pmu_save_and_restart(event))  in __intel_pmu_pebs_event()
2072 		setup_sample(event, iregs, at, data, regs);  in __intel_pmu_pebs_event()
2073 		perf_event_output(event, data, regs);  in __intel_pmu_pebs_event()
2079 	setup_sample(event, iregs, at, data, regs);  in __intel_pmu_pebs_event()
2087 		perf_event_output(event, data, regs);  in __intel_pmu_pebs_event()
2093 		if (perf_event_overflow(event, data, regs))  in __intel_pmu_pebs_event()
2094 			x86_pmu_stop(event, 0);  in __intel_pmu_pebs_event()
2102 	struct perf_event *event = cpuc->events[0]; /* PMC0 only */  in intel_pmu_drain_pebs_core()  local
2120 	WARN_ON_ONCE(!event);  in intel_pmu_drain_pebs_core()
2122 	if (!event->attr.precise_ip)  in intel_pmu_drain_pebs_core()
2127 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)  in intel_pmu_drain_pebs_core()
2128 			intel_pmu_save_and_restart_reload(event, 0);  in intel_pmu_drain_pebs_core()
2132 	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,  in intel_pmu_drain_pebs_core()
2138 	struct perf_event *event;  in intel_pmu_pebs_event_update_no_drain()  local
2149 		event = cpuc->events[bit];  in intel_pmu_pebs_event_update_no_drain()
2150 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)  in intel_pmu_pebs_event_update_no_drain()
2151 			intel_pmu_save_and_restart_reload(event, 0);  in intel_pmu_pebs_event_update_no_drain()
2159 	struct perf_event *event;  in intel_pmu_drain_pebs_nhm()  local
2246 		event = cpuc->events[bit];  in intel_pmu_drain_pebs_nhm()
2247 		if (WARN_ON_ONCE(!event))  in intel_pmu_drain_pebs_nhm()
2250 		if (WARN_ON_ONCE(!event->attr.precise_ip))  in intel_pmu_drain_pebs_nhm()
2255 			perf_log_lost_samples(event, error[bit]);  in intel_pmu_drain_pebs_nhm()
2257 			if (iregs && perf_event_account_interrupt(event))  in intel_pmu_drain_pebs_nhm()
2258 				x86_pmu_stop(event, 0);  in intel_pmu_drain_pebs_nhm()
2262 			__intel_pmu_pebs_event(event, iregs, data, base,  in intel_pmu_drain_pebs_nhm()
2276 	struct perf_event *event;  in intel_pmu_drain_pebs_icl()  local
2312 		event = cpuc->events[bit];  in intel_pmu_drain_pebs_icl()
2313 		if (WARN_ON_ONCE(!event))  in intel_pmu_drain_pebs_icl()
2316 		if (WARN_ON_ONCE(!event->attr.precise_ip))  in intel_pmu_drain_pebs_icl()
2319 		__intel_pmu_pebs_event(event, iregs, data, base,  in intel_pmu_drain_pebs_icl()