1 /* SPDX-License-Identifier: MIT */
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #ifndef _I915_PERF_TYPES_H_
7 #define _I915_PERF_TYPES_H_
8 
9 #include <linux/atomic.h>
10 #include <linux/device.h>
11 #include <linux/hrtimer.h>
12 #include <linux/llist.h>
13 #include <linux/poll.h>
14 #include <linux/sysfs.h>
15 #include <linux/types.h>
16 #include <linux/uuid.h>
17 #include <linux/wait.h>
18 #include <uapi/drm/i915_drm.h>
19 
20 #include "gt/intel_sseu.h"
21 #include "i915_reg_defs.h"
22 #include "intel_wakeref.h"
23 
24 struct drm_i915_private;
25 struct file;
26 struct i915_active;
27 struct i915_gem_context;
28 struct i915_perf;
29 struct i915_vma;
30 struct intel_context;
31 struct intel_engine_cs;
32 
33 struct i915_oa_format {
34 	u32 format;
35 	int size;
36 };
37 
38 struct i915_oa_reg {
39 	i915_reg_t addr;
40 	u32 value;
41 };
42 
43 struct i915_oa_config {
44 	struct i915_perf *perf;
45 
46 	char uuid[UUID_STRING_LEN + 1];
47 	int id;
48 
49 	const struct i915_oa_reg *mux_regs;
50 	u32 mux_regs_len;
51 	const struct i915_oa_reg *b_counter_regs;
52 	u32 b_counter_regs_len;
53 	const struct i915_oa_reg *flex_regs;
54 	u32 flex_regs_len;
55 
56 	struct attribute_group sysfs_metric;
57 	struct attribute *attrs[2];
58 	struct kobj_attribute sysfs_metric_id;
59 
60 	struct kref ref;
61 	struct rcu_head rcu;
62 };
63 
64 struct i915_perf_stream;
65 
66 /**
67  * struct i915_perf_stream_ops - the OPs to support a specific stream type
68  */
69 struct i915_perf_stream_ops {
70 	/**
71 	 * @enable: Enables the collection of HW samples, either in response to
72 	 * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened
73 	 * without `I915_PERF_FLAG_DISABLED`.
74 	 */
75 	void (*enable)(struct i915_perf_stream *stream);
76 
77 	/**
78 	 * @disable: Disables the collection of HW samples, either in response
79 	 * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying
80 	 * the stream.
81 	 */
82 	void (*disable)(struct i915_perf_stream *stream);
83 
84 	/**
85 	 * @poll_wait: Call poll_wait, passing a wait queue that will be woken
86 	 * once there is something ready to read() for the stream
87 	 */
88 	void (*poll_wait)(struct i915_perf_stream *stream,
89 			  struct file *file,
90 			  poll_table *wait);
91 
92 	/**
93 	 * @wait_unlocked: For handling a blocking read, wait until there is
94 	 * something to ready to read() for the stream. E.g. wait on the same
95 	 * wait queue that would be passed to poll_wait().
96 	 */
97 	int (*wait_unlocked)(struct i915_perf_stream *stream);
98 
99 	/**
100 	 * @read: Copy buffered metrics as records to userspace
101 	 * **buf**: the userspace, destination buffer
102 	 * **count**: the number of bytes to copy, requested by userspace
103 	 * **offset**: zero at the start of the read, updated as the read
104 	 * proceeds, it represents how many bytes have been copied so far and
105 	 * the buffer offset for copying the next record.
106 	 *
107 	 * Copy as many buffered i915 perf samples and records for this stream
108 	 * to userspace as will fit in the given buffer.
109 	 *
110 	 * Only write complete records; returning -%ENOSPC if there isn't room
111 	 * for a complete record.
112 	 *
113 	 * Return any error condition that results in a short read such as
114 	 * -%ENOSPC or -%EFAULT, even though these may be squashed before
115 	 * returning to userspace.
116 	 */
117 	int (*read)(struct i915_perf_stream *stream,
118 		    char __user *buf,
119 		    size_t count,
120 		    size_t *offset);
121 
122 	/**
123 	 * @destroy: Cleanup any stream specific resources.
124 	 *
125 	 * The stream will always be disabled before this is called.
126 	 */
127 	void (*destroy)(struct i915_perf_stream *stream);
128 };
129 
130 /**
131  * struct i915_perf_stream - state for a single open stream FD
132  */
133 struct i915_perf_stream {
134 	/**
135 	 * @perf: i915_perf backpointer
136 	 */
137 	struct i915_perf *perf;
138 
139 	/**
140 	 * @uncore: mmio access path
141 	 */
142 	struct intel_uncore *uncore;
143 
144 	/**
145 	 * @engine: Engine associated with this performance stream.
146 	 */
147 	struct intel_engine_cs *engine;
148 
149 	/**
150 	 * @lock: Lock associated with operations on stream
151 	 */
152 	struct mutex lock;
153 
154 	/**
155 	 * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*`
156 	 * properties given when opening a stream, representing the contents
157 	 * of a single sample as read() by userspace.
158 	 */
159 	u32 sample_flags;
160 
161 	/**
162 	 * @sample_size: Considering the configured contents of a sample
163 	 * combined with the required header size, this is the total size
164 	 * of a single sample record.
165 	 */
166 	int sample_size;
167 
168 	/**
169 	 * @ctx: %NULL if measuring system-wide across all contexts or a
170 	 * specific context that is being monitored.
171 	 */
172 	struct i915_gem_context *ctx;
173 
174 	/**
175 	 * @enabled: Whether the stream is currently enabled, considering
176 	 * whether the stream was opened in a disabled state and based
177 	 * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls.
178 	 */
179 	bool enabled;
180 
181 	/**
182 	 * @hold_preemption: Whether preemption is put on hold for command
183 	 * submissions done on the @ctx. This is useful for some drivers that
184 	 * cannot easily post process the OA buffer context to subtract delta
185 	 * of performance counters not associated with @ctx.
186 	 */
187 	bool hold_preemption;
188 
189 	/**
190 	 * @ops: The callbacks providing the implementation of this specific
191 	 * type of configured stream.
192 	 */
193 	const struct i915_perf_stream_ops *ops;
194 
195 	/**
196 	 * @oa_config: The OA configuration used by the stream.
197 	 */
198 	struct i915_oa_config *oa_config;
199 
200 	/**
201 	 * @oa_config_bos: A list of struct i915_oa_config_bo allocated lazily
202 	 * each time @oa_config changes.
203 	 */
204 	struct llist_head oa_config_bos;
205 
206 	/**
207 	 * @pinned_ctx: The OA context specific information.
208 	 */
209 	struct intel_context *pinned_ctx;
210 
211 	/**
212 	 * @specific_ctx_id: The id of the specific context.
213 	 */
214 	u32 specific_ctx_id;
215 
216 	/**
217 	 * @specific_ctx_id_mask: The mask used to masking specific_ctx_id bits.
218 	 */
219 	u32 specific_ctx_id_mask;
220 
221 	/**
222 	 * @poll_check_timer: High resolution timer that will periodically
223 	 * check for data in the circular OA buffer for notifying userspace
224 	 * (e.g. during a read() or poll()).
225 	 */
226 	struct hrtimer poll_check_timer;
227 
228 	/**
229 	 * @poll_wq: The wait queue that hrtimer callback wakes when it
230 	 * sees data ready to read in the circular OA buffer.
231 	 */
232 	wait_queue_head_t poll_wq;
233 
234 	/**
235 	 * @pollin: Whether there is data available to read.
236 	 */
237 	bool pollin;
238 
239 	/**
240 	 * @periodic: Whether periodic sampling is currently enabled.
241 	 */
242 	bool periodic;
243 
244 	/**
245 	 * @period_exponent: The OA unit sampling frequency is derived from this.
246 	 */
247 	int period_exponent;
248 
249 	/**
250 	 * @oa_buffer: State of the OA buffer.
251 	 */
252 	struct {
253 		const struct i915_oa_format *format;
254 		struct i915_vma *vma;
255 		u8 *vaddr;
256 		u32 last_ctx_id;
257 		int size_exponent;
258 
259 		/**
260 		 * @ptr_lock: Locks reads and writes to all head/tail state
261 		 *
262 		 * Consider: the head and tail pointer state needs to be read
263 		 * consistently from a hrtimer callback (atomic context) and
264 		 * read() fop (user context) with tail pointer updates happening
265 		 * in atomic context and head updates in user context and the
266 		 * (unlikely) possibility of read() errors needing to reset all
267 		 * head/tail state.
268 		 *
269 		 * Note: Contention/performance aren't currently a significant
270 		 * concern here considering the relatively low frequency of
271 		 * hrtimer callbacks (5ms period) and that reads typically only
272 		 * happen in response to a hrtimer event and likely complete
273 		 * before the next callback.
274 		 *
275 		 * Note: This lock is not held *while* reading and copying data
276 		 * to userspace so the value of head observed in htrimer
277 		 * callbacks won't represent any partial consumption of data.
278 		 */
279 		spinlock_t ptr_lock;
280 
281 		/**
282 		 * @aging_tail: The last HW tail reported by HW. The data
283 		 * might not have made it to memory yet though.
284 		 */
285 		u32 aging_tail;
286 
287 		/**
288 		 * @aging_timestamp: A monotonic timestamp for when the current aging tail pointer
289 		 * was read; used to determine when it is old enough to trust.
290 		 */
291 		u64 aging_timestamp;
292 
293 		/**
294 		 * @head: Although we can always read back the head pointer register,
295 		 * we prefer to avoid trusting the HW state, just to avoid any
296 		 * risk that some hardware condition could * somehow bump the
297 		 * head pointer unpredictably and cause us to forward the wrong
298 		 * OA buffer data to userspace.
299 		 */
300 		u32 head;
301 
302 		/**
303 		 * @tail: The last verified tail that can be read by userspace.
304 		 */
305 		u32 tail;
306 	} oa_buffer;
307 
308 	/**
309 	 * @noa_wait: A batch buffer doing a wait on the GPU for the NOA logic to be
310 	 * reprogrammed.
311 	 */
312 	struct i915_vma *noa_wait;
313 
314 	/**
315 	 * @poll_oa_period: The period in nanoseconds at which the OA
316 	 * buffer should be checked for available data.
317 	 */
318 	u64 poll_oa_period;
319 };
320 
321 /**
322  * struct i915_oa_ops - Gen specific implementation of an OA unit stream
323  */
324 struct i915_oa_ops {
325 	/**
326 	 * @is_valid_b_counter_reg: Validates register's address for
327 	 * programming boolean counters for a particular platform.
328 	 */
329 	bool (*is_valid_b_counter_reg)(struct i915_perf *perf, u32 addr);
330 
331 	/**
332 	 * @is_valid_mux_reg: Validates register's address for programming mux
333 	 * for a particular platform.
334 	 */
335 	bool (*is_valid_mux_reg)(struct i915_perf *perf, u32 addr);
336 
337 	/**
338 	 * @is_valid_flex_reg: Validates register's address for programming
339 	 * flex EU filtering for a particular platform.
340 	 */
341 	bool (*is_valid_flex_reg)(struct i915_perf *perf, u32 addr);
342 
343 	/**
344 	 * @enable_metric_set: Selects and applies any MUX configuration to set
345 	 * up the Boolean and Custom (B/C) counters that are part of the
346 	 * counter reports being sampled. May apply system constraints such as
347 	 * disabling EU clock gating as required.
348 	 */
349 	int (*enable_metric_set)(struct i915_perf_stream *stream,
350 				 struct i915_active *active);
351 
352 	/**
353 	 * @disable_metric_set: Remove system constraints associated with using
354 	 * the OA unit.
355 	 */
356 	void (*disable_metric_set)(struct i915_perf_stream *stream);
357 
358 	/**
359 	 * @oa_enable: Enable periodic sampling
360 	 */
361 	void (*oa_enable)(struct i915_perf_stream *stream);
362 
363 	/**
364 	 * @oa_disable: Disable periodic sampling
365 	 */
366 	void (*oa_disable)(struct i915_perf_stream *stream);
367 
368 	/**
369 	 * @read: Copy data from the circular OA buffer into a given userspace
370 	 * buffer.
371 	 */
372 	int (*read)(struct i915_perf_stream *stream,
373 		    char __user *buf,
374 		    size_t count,
375 		    size_t *offset);
376 
377 	/**
378 	 * @oa_hw_tail_read: read the OA tail pointer register
379 	 *
380 	 * In particular this enables us to share all the fiddly code for
381 	 * handling the OA unit tail pointer race that affects multiple
382 	 * generations.
383 	 */
384 	u32 (*oa_hw_tail_read)(struct i915_perf_stream *stream);
385 };
386 
387 struct i915_perf_gt {
388 	/*
389 	 * Lock associated with anything below within this structure.
390 	 */
391 	struct mutex lock;
392 
393 	/**
394 	 * @sseu: sseu configuration selected to run while perf is active,
395 	 * applies to all contexts.
396 	 */
397 	struct intel_sseu sseu;
398 
399 	/*
400 	 * @exclusive_stream: The stream currently using the OA unit. This is
401 	 * sometimes accessed outside a syscall associated to its file
402 	 * descriptor.
403 	 */
404 	struct i915_perf_stream *exclusive_stream;
405 };
406 
407 struct i915_perf {
408 	struct drm_i915_private *i915;
409 
410 	struct kobject *metrics_kobj;
411 
412 	/*
413 	 * Lock associated with adding/modifying/removing OA configs
414 	 * in perf->metrics_idr.
415 	 */
416 	struct mutex metrics_lock;
417 
418 	/*
419 	 * List of dynamic configurations (struct i915_oa_config), you
420 	 * need to hold perf->metrics_lock to access it.
421 	 */
422 	struct idr metrics_idr;
423 
424 	/**
425 	 * For rate limiting any notifications of spurious
426 	 * invalid OA reports
427 	 */
428 	struct ratelimit_state spurious_report_rs;
429 
430 	/**
431 	 * For rate limiting any notifications of tail pointer
432 	 * race.
433 	 */
434 	struct ratelimit_state tail_pointer_race;
435 
436 	u32 gen7_latched_oastatus1;
437 	u32 ctx_oactxctrl_offset;
438 	u32 ctx_flexeu0_offset;
439 
440 	/**
441 	 * The RPT_ID/reason field for Gen8+ includes a bit
442 	 * to determine if the CTX ID in the report is valid
443 	 * but the specific bit differs between Gen 8 and 9
444 	 */
445 	u32 gen8_valid_ctx_bit;
446 
447 	struct i915_oa_ops ops;
448 	const struct i915_oa_format *oa_formats;
449 
450 	/**
451 	 * Use a format mask to store the supported formats
452 	 * for a platform.
453 	 */
454 #define FORMAT_MASK_SIZE DIV_ROUND_UP(I915_OA_FORMAT_MAX - 1, BITS_PER_LONG)
455 	unsigned long format_mask[FORMAT_MASK_SIZE];
456 
457 	atomic64_t noa_programming_delay;
458 };
459 
460 #endif /* _I915_PERF_TYPES_H_ */
461