xref: /openbmc/linux/drivers/gpu/drm/i915/i915_perf.c (revision a36954f5)
1 /*
2  * Copyright © 2015-2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *   Robert Bragg <robert@sixbynine.org>
25  */
26 
27 
28 /**
29  * DOC: i915 Perf Overview
30  *
31  * Gen graphics supports a large number of performance counters that can help
32  * driver and application developers understand and optimize their use of the
33  * GPU.
34  *
35  * This i915 perf interface enables userspace to configure and open a file
36  * descriptor representing a stream of GPU metrics which can then be read() as
37  * a stream of sample records.
38  *
39  * The interface is particularly suited to exposing buffered metrics that are
40  * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
41  *
42  * Streams representing a single context are accessible to applications with a
43  * corresponding drm file descriptor, such that OpenGL can use the interface
44  * without special privileges. Access to system-wide metrics requires root
45  * privileges by default, unless changed via the dev.i915.perf_event_paranoid
46  * sysctl option.
47  *
48  */
49 
50 /**
51  * DOC: i915 Perf History and Comparison with Core Perf
52  *
53  * The interface was initially inspired by the core Perf infrastructure but
54  * some notable differences are:
55  *
56  * i915 perf file descriptors represent a "stream" instead of an "event"; where
57  * a perf event primarily corresponds to a single 64bit value, while a stream
58  * might sample sets of tightly-coupled counters, depending on the
59  * configuration.  For example the Gen OA unit isn't designed to support
60  * orthogonal configurations of individual counters; it's configured for a set
61  * of related counters. Samples for an i915 perf stream capturing OA metrics
62  * will include a set of counter values packed in a compact HW specific format.
63  * The OA unit supports a number of different packing formats which can be
64  * selected by the user opening the stream. Perf has support for grouping
65  * events, but each event in the group is configured, validated and
66  * authenticated individually with separate system calls.
67  *
68  * i915 perf stream configurations are provided as an array of u64 (key,value)
69  * pairs, instead of a fixed struct with multiple miscellaneous config members,
70  * interleaved with event-type specific members.
71  *
72  * i915 perf doesn't support exposing metrics via an mmap'd circular buffer.
73  * The supported metrics are being written to memory by the GPU unsynchronized
74  * with the CPU, using HW specific packing formats for counter sets. Sometimes
75  * the constraints on HW configuration require reports to be filtered before it
76  * would be acceptable to expose them to unprivileged applications - to hide
77  * the metrics of other processes/contexts. For these use cases a read() based
78  * interface is a good fit, and provides an opportunity to filter data as it
79  * gets copied from the GPU mapped buffers to userspace buffers.
80  *
81  *
82  * Issues hit with first prototype based on Core Perf
83  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
84  *
85  * The first prototype of this driver was based on the core perf
86  * infrastructure, and while we did make that mostly work, with some changes to
87  * perf, we found we were breaking or working around too many assumptions baked
88  * into perf's currently cpu centric design.
89  *
90  * In the end we didn't see a clear benefit to making perf's implementation and
91  * interface more complex by changing design assumptions while we knew we still
92  * wouldn't be able to use any existing perf based userspace tools.
93  *
94  * Also considering the Gen specific nature of the Observability hardware and
95  * how userspace will sometimes need to combine i915 perf OA metrics with
96  * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
97  * expecting the interface to be used by a platform specific userspace such as
98  * OpenGL or tools. This is to say; we aren't inherently missing out on having
99  * a standard vendor/architecture agnostic interface by not using perf.
100  *
101  *
102  * For posterity, in case we might re-visit trying to adapt core perf to be
103  * better suited to exposing i915 metrics these were the main pain points we
104  * hit:
105  *
106  * - The perf based OA PMU driver broke some significant design assumptions:
107  *
108  *   Existing perf pmus are used for profiling work on a cpu and we were
109  *   introducing the idea of _IS_DEVICE pmus with different security
110  *   implications, the need to fake cpu-related data (such as user/kernel
111  *   registers) to fit with perf's current design, and adding _DEVICE records
112  *   as a way to forward device-specific status records.
113  *
114  *   The OA unit writes reports of counters into a circular buffer, without
115  *   involvement from the CPU, making our PMU driver the first of a kind.
116  *
117  *   Given the way we were periodically forward data from the GPU-mapped, OA
118  *   buffer to perf's buffer, those bursts of sample writes looked to perf like
119  *   we were sampling too fast and so we had to subvert its throttling checks.
120  *
121  *   Perf supports groups of counters and allows those to be read via
122  *   transactions internally but transactions currently seem designed to be
123  *   explicitly initiated from the cpu (say in response to a userspace read())
124  *   and while we could pull a report out of the OA buffer we can't
125  *   trigger a report from the cpu on demand.
126  *
127  *   Related to being report based; the OA counters are configured in HW as a
128  *   set while perf generally expects counter configurations to be orthogonal.
129  *   Although counters can be associated with a group leader as they are
130  *   opened, there's no clear precedent for being able to provide group-wide
131  *   configuration attributes (for example we want to let userspace choose the
132  *   OA unit report format used to capture all counters in a set, or specify a
133  *   GPU context to filter metrics on). We avoided using perf's grouping
134  *   feature and forwarded OA reports to userspace via perf's 'raw' sample
135  *   field. This suited our userspace well considering how coupled the counters
136  *   are when dealing with normalizing. It would be inconvenient to split
137  *   counters up into separate events, only to require userspace to recombine
138  *   them. For Mesa it's also convenient to be forwarded raw, periodic reports
139  *   for combining with the side-band raw reports it captures using
140  *   MI_REPORT_PERF_COUNT commands.
141  *
142  *   - As a side note on perf's grouping feature; there was also some concern
143  *     that using PERF_FORMAT_GROUP as a way to pack together counter values
144  *     would quite drastically inflate our sample sizes, which would likely
145  *     lower the effective sampling resolutions we could use when the available
146  *     memory bandwidth is limited.
147  *
148  *     With the OA unit's report formats, counters are packed together as 32
149  *     or 40bit values, with the largest report size being 256 bytes.
150  *
151  *     PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a
152  *     documented ordering to the values, implying PERF_FORMAT_ID must also be
153  *     used to add a 64bit ID before each value; giving 16 bytes per counter.
154  *
155  *   Related to counter orthogonality; we can't time share the OA unit, while
156  *   event scheduling is a central design idea within perf for allowing
157  *   userspace to open + enable more events than can be configured in HW at any
158  *   one time.  The OA unit is not designed to allow re-configuration while in
159  *   use. We can't reconfigure the OA unit without losing internal OA unit
160  *   state which we can't access explicitly to save and restore. Reconfiguring
161  *   the OA unit is also relatively slow, involving ~100 register writes. From
162  *   userspace Mesa also depends on a stable OA configuration when emitting
163  *   MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be
164  *   disabled while there are outstanding MI_RPC commands lest we hang the
165  *   command streamer.
166  *
167  *   The contents of sample records aren't extensible by device drivers (i.e.
168  *   the sample_type bits). As an example; Sourab Gupta had been looking to
169  *   attach GPU timestamps to our OA samples. We were shoehorning OA reports
170  *   into sample records by using the 'raw' field, but it's tricky to pack more
171  *   than one thing into this field because events/core.c currently only lets a
172  *   pmu give a single raw data pointer plus len which will be copied into the
173  *   ring buffer. To include more than the OA report we'd have to copy the
174  *   report into an intermediate larger buffer. I'd been considering allowing a
175  *   vector of data+len values to be specified for copying the raw data, but
176  *   it felt like a kludge to being using the raw field for this purpose.
177  *
178  * - It felt like our perf based PMU was making some technical compromises
179  *   just for the sake of using perf:
180  *
181  *   perf_event_open() requires events to either relate to a pid or a specific
182  *   cpu core, while our device pmu related to neither.  Events opened with a
183  *   pid will be automatically enabled/disabled according to the scheduling of
184  *   that process - so not appropriate for us. When an event is related to a
185  *   cpu id, perf ensures pmu methods will be invoked via an inter process
186  *   interrupt on that core. To avoid invasive changes our userspace opened OA
187  *   perf events for a specific cpu. This was workable but it meant the
188  *   majority of the OA driver ran in atomic context, including all OA report
189  *   forwarding, which wasn't really necessary in our case and seems to make
190  *   our locking requirements somewhat complex as we handled the interaction
191  *   with the rest of the i915 driver.
192  */
193 
194 #include <linux/anon_inodes.h>
195 #include <linux/sizes.h>
196 
197 #include "i915_drv.h"
198 #include "i915_oa_hsw.h"
199 
200 /* HW requires this to be a power of two, between 128k and 16M, though driver
201  * is currently generally designed assuming the largest 16M size is used such
202  * that the overflow cases are unlikely in normal operation.
203  */
204 #define OA_BUFFER_SIZE		SZ_16M
205 
206 #define OA_TAKEN(tail, head)	((tail - head) & (OA_BUFFER_SIZE - 1))
207 
208 /* There's a HW race condition between OA unit tail pointer register updates and
209  * writes to memory whereby the tail pointer can sometimes get ahead of what's
210  * been written out to the OA buffer so far.
211  *
212  * Although this can be observed explicitly by checking for a zeroed report-id
213  * field in tail reports, it seems preferable to account for this earlier e.g.
214  * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles
215  * in this situation.
216  *
217  * To give time for the most recent reports to land before they may be copied to
218  * userspace, the driver operates as if the tail pointer effectively lags behind
219  * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated
220  * based on this constant in nanoseconds, the current OA sampling exponent
221  * and current report size.
222  *
223  * There is also a fallback check while reading to simply skip over reports with
224  * a zeroed report-id.
225  */
226 #define OA_TAIL_MARGIN_NSEC	100000ULL
227 
228 /* frequency for checking whether the OA unit has written new reports to the
229  * circular OA buffer...
230  */
231 #define POLL_FREQUENCY 200
232 #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY)
233 
234 /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */
235 static int zero;
236 static int one = 1;
237 static u32 i915_perf_stream_paranoid = true;
238 
239 /* The maximum exponent the hardware accepts is 63 (essentially it selects one
240  * of the 64bit timestamp bits to trigger reports from) but there's currently
241  * no known use case for sampling as infrequently as once per 47 thousand years.
242  *
243  * Since the timestamps included in OA reports are only 32bits it seems
244  * reasonable to limit the OA exponent where it's still possible to account for
245  * overflow in OA report timestamps.
246  */
247 #define OA_EXPONENT_MAX 31
248 
249 #define INVALID_CTX_ID 0xffffffff
250 
251 
252 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
253  *
254  * 160ns is the smallest sampling period we can theoretically program the OA
255  * unit with on Haswell, corresponding to 6.25MHz.
256  */
257 static int oa_sample_rate_hard_limit = 6250000;
258 
259 /* Theoretically we can program the OA unit to sample every 160ns but don't
260  * allow that by default unless root...
261  *
262  * The default threshold of 100000Hz is based on perf's similar
263  * kernel.perf_event_max_sample_rate sysctl parameter.
264  */
265 static u32 i915_oa_max_sample_rate = 100000;
266 
267 /* XXX: beware if future OA HW adds new report formats that the current
268  * code assumes all reports have a power-of-two size and ~(size - 1) can
269  * be used as a mask to align the OA tail pointer.
270  */
271 static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = {
272 	[I915_OA_FORMAT_A13]	    = { 0, 64 },
273 	[I915_OA_FORMAT_A29]	    = { 1, 128 },
274 	[I915_OA_FORMAT_A13_B8_C8]  = { 2, 128 },
275 	/* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */
276 	[I915_OA_FORMAT_B4_C8]	    = { 4, 64 },
277 	[I915_OA_FORMAT_A45_B8_C8]  = { 5, 256 },
278 	[I915_OA_FORMAT_B4_C8_A16]  = { 6, 128 },
279 	[I915_OA_FORMAT_C4_B8]	    = { 7, 64 },
280 };
281 
282 #define SAMPLE_OA_REPORT      (1<<0)
283 
284 /**
285  * struct perf_open_properties - for validated properties given to open a stream
286  * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags
287  * @single_context: Whether a single or all gpu contexts should be monitored
288  * @ctx_handle: A gem ctx handle for use with @single_context
289  * @metrics_set: An ID for an OA unit metric set advertised via sysfs
290  * @oa_format: An OA unit HW report format
291  * @oa_periodic: Whether to enable periodic OA unit sampling
292  * @oa_period_exponent: The OA unit sampling period is derived from this
293  *
294  * As read_properties_unlocked() enumerates and validates the properties given
295  * to open a stream of metrics the configuration is built up in the structure
296  * which starts out zero initialized.
297  */
298 struct perf_open_properties {
299 	u32 sample_flags;
300 
301 	u64 single_context:1;
302 	u64 ctx_handle;
303 
304 	/* OA sampling state */
305 	int metrics_set;
306 	int oa_format;
307 	bool oa_periodic;
308 	int oa_period_exponent;
309 };
310 
311 /* NB: This is either called via fops or the poll check hrtimer (atomic ctx)
312  *
313  * It's safe to read OA config state here unlocked, assuming that this is only
314  * called while the stream is enabled, while the global OA configuration can't
315  * be modified.
316  *
317  * Note: we don't lock around the head/tail reads even though there's the slim
318  * possibility of read() fop errors forcing a re-init of the OA buffer
319  * pointers.  A race here could result in a false positive !empty status which
320  * is acceptable.
321  */
322 static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv)
323 {
324 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
325 	u32 oastatus2 = I915_READ(GEN7_OASTATUS2);
326 	u32 oastatus1 = I915_READ(GEN7_OASTATUS1);
327 	u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
328 	u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
329 
330 	return OA_TAKEN(tail, head) <
331 		dev_priv->perf.oa.tail_margin + report_size;
332 }
333 
334 /**
335  * append_oa_status - Appends a status record to a userspace read() buffer.
336  * @stream: An i915-perf stream opened for OA metrics
337  * @buf: destination buffer given by userspace
338  * @count: the number of bytes userspace wants to read
339  * @offset: (inout): the current position for writing into @buf
340  * @type: The kind of status to report to userspace
341  *
342  * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`)
343  * into the userspace read() buffer.
344  *
345  * The @buf @offset will only be updated on success.
346  *
347  * Returns: 0 on success, negative error code on failure.
348  */
349 static int append_oa_status(struct i915_perf_stream *stream,
350 			    char __user *buf,
351 			    size_t count,
352 			    size_t *offset,
353 			    enum drm_i915_perf_record_type type)
354 {
355 	struct drm_i915_perf_record_header header = { type, 0, sizeof(header) };
356 
357 	if ((count - *offset) < header.size)
358 		return -ENOSPC;
359 
360 	if (copy_to_user(buf + *offset, &header, sizeof(header)))
361 		return -EFAULT;
362 
363 	(*offset) += header.size;
364 
365 	return 0;
366 }
367 
368 /**
369  * append_oa_sample - Copies single OA report into userspace read() buffer.
370  * @stream: An i915-perf stream opened for OA metrics
371  * @buf: destination buffer given by userspace
372  * @count: the number of bytes userspace wants to read
373  * @offset: (inout): the current position for writing into @buf
374  * @report: A single OA report to (optionally) include as part of the sample
375  *
376  * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*`
377  * properties when opening a stream, tracked as `stream->sample_flags`. This
378  * function copies the requested components of a single sample to the given
379  * read() @buf.
380  *
381  * The @buf @offset will only be updated on success.
382  *
383  * Returns: 0 on success, negative error code on failure.
384  */
385 static int append_oa_sample(struct i915_perf_stream *stream,
386 			    char __user *buf,
387 			    size_t count,
388 			    size_t *offset,
389 			    const u8 *report)
390 {
391 	struct drm_i915_private *dev_priv = stream->dev_priv;
392 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
393 	struct drm_i915_perf_record_header header;
394 	u32 sample_flags = stream->sample_flags;
395 
396 	header.type = DRM_I915_PERF_RECORD_SAMPLE;
397 	header.pad = 0;
398 	header.size = stream->sample_size;
399 
400 	if ((count - *offset) < header.size)
401 		return -ENOSPC;
402 
403 	buf += *offset;
404 	if (copy_to_user(buf, &header, sizeof(header)))
405 		return -EFAULT;
406 	buf += sizeof(header);
407 
408 	if (sample_flags & SAMPLE_OA_REPORT) {
409 		if (copy_to_user(buf, report, report_size))
410 			return -EFAULT;
411 	}
412 
413 	(*offset) += header.size;
414 
415 	return 0;
416 }
417 
418 /**
419  * Copies all buffered OA reports into userspace read() buffer.
420  * @stream: An i915-perf stream opened for OA metrics
421  * @buf: destination buffer given by userspace
422  * @count: the number of bytes userspace wants to read
423  * @offset: (inout): the current position for writing into @buf
424  * @head_ptr: (inout): the current oa buffer cpu read position
425  * @tail: the current oa buffer gpu write position
426  *
427  * Notably any error condition resulting in a short read (-%ENOSPC or
428  * -%EFAULT) will be returned even though one or more records may
429  * have been successfully copied. In this case it's up to the caller
430  * to decide if the error should be squashed before returning to
431  * userspace.
432  *
433  * Note: reports are consumed from the head, and appended to the
434  * tail, so the head chases the tail?... If you think that's mad
435  * and back-to-front you're not alone, but this follows the
436  * Gen PRM naming convention.
437  *
438  * Returns: 0 on success, negative error code on failure.
439  */
440 static int gen7_append_oa_reports(struct i915_perf_stream *stream,
441 				  char __user *buf,
442 				  size_t count,
443 				  size_t *offset,
444 				  u32 *head_ptr,
445 				  u32 tail)
446 {
447 	struct drm_i915_private *dev_priv = stream->dev_priv;
448 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
449 	u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr;
450 	int tail_margin = dev_priv->perf.oa.tail_margin;
451 	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
452 	u32 mask = (OA_BUFFER_SIZE - 1);
453 	u32 head;
454 	u32 taken;
455 	int ret = 0;
456 
457 	if (WARN_ON(!stream->enabled))
458 		return -EIO;
459 
460 	head = *head_ptr - gtt_offset;
461 	tail -= gtt_offset;
462 
463 	/* The OA unit is expected to wrap the tail pointer according to the OA
464 	 * buffer size and since we should never write a misaligned head
465 	 * pointer we don't expect to read one back either...
466 	 */
467 	if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE ||
468 	    head % report_size) {
469 		DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n",
470 			  head, tail);
471 		dev_priv->perf.oa.ops.oa_disable(dev_priv);
472 		dev_priv->perf.oa.ops.oa_enable(dev_priv);
473 		*head_ptr = I915_READ(GEN7_OASTATUS2) &
474 			GEN7_OASTATUS2_HEAD_MASK;
475 		return -EIO;
476 	}
477 
478 
479 	/* The tail pointer increases in 64 byte increments, not in report_size
480 	 * steps...
481 	 */
482 	tail &= ~(report_size - 1);
483 
484 	/* Move the tail pointer back by the current tail_margin to account for
485 	 * the possibility that the latest reports may not have really landed
486 	 * in memory yet...
487 	 */
488 
489 	if (OA_TAKEN(tail, head) < report_size + tail_margin)
490 		return -EAGAIN;
491 
492 	tail -= tail_margin;
493 	tail &= mask;
494 
495 	for (/* none */;
496 	     (taken = OA_TAKEN(tail, head));
497 	     head = (head + report_size) & mask) {
498 		u8 *report = oa_buf_base + head;
499 		u32 *report32 = (void *)report;
500 
501 		/* All the report sizes factor neatly into the buffer
502 		 * size so we never expect to see a report split
503 		 * between the beginning and end of the buffer.
504 		 *
505 		 * Given the initial alignment check a misalignment
506 		 * here would imply a driver bug that would result
507 		 * in an overrun.
508 		 */
509 		if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) {
510 			DRM_ERROR("Spurious OA head ptr: non-integral report offset\n");
511 			break;
512 		}
513 
514 		/* The report-ID field for periodic samples includes
515 		 * some undocumented flags related to what triggered
516 		 * the report and is never expected to be zero so we
517 		 * can check that the report isn't invalid before
518 		 * copying it to userspace...
519 		 */
520 		if (report32[0] == 0) {
521 			DRM_NOTE("Skipping spurious, invalid OA report\n");
522 			continue;
523 		}
524 
525 		ret = append_oa_sample(stream, buf, count, offset, report);
526 		if (ret)
527 			break;
528 
529 		/* The above report-id field sanity check is based on
530 		 * the assumption that the OA buffer is initially
531 		 * zeroed and we reset the field after copying so the
532 		 * check is still meaningful once old reports start
533 		 * being overwritten.
534 		 */
535 		report32[0] = 0;
536 	}
537 
538 	*head_ptr = gtt_offset + head;
539 
540 	return ret;
541 }
542 
543 /**
544  * gen7_oa_read - copy status records then buffered OA reports
545  * @stream: An i915-perf stream opened for OA metrics
546  * @buf: destination buffer given by userspace
547  * @count: the number of bytes userspace wants to read
548  * @offset: (inout): the current position for writing into @buf
549  *
550  * Checks Gen 7 specific OA unit status registers and if necessary appends
551  * corresponding status records for userspace (such as for a buffer full
552  * condition) and then initiate appending any buffered OA reports.
553  *
554  * Updates @offset according to the number of bytes successfully copied into
555  * the userspace buffer.
556  *
557  * Returns: zero on success or a negative error code
558  */
559 static int gen7_oa_read(struct i915_perf_stream *stream,
560 			char __user *buf,
561 			size_t count,
562 			size_t *offset)
563 {
564 	struct drm_i915_private *dev_priv = stream->dev_priv;
565 	int report_size = dev_priv->perf.oa.oa_buffer.format_size;
566 	u32 oastatus2;
567 	u32 oastatus1;
568 	u32 head;
569 	u32 tail;
570 	int ret;
571 
572 	if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr))
573 		return -EIO;
574 
575 	oastatus2 = I915_READ(GEN7_OASTATUS2);
576 	oastatus1 = I915_READ(GEN7_OASTATUS1);
577 
578 	head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
579 	tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
580 
581 	/* XXX: On Haswell we don't have a safe way to clear oastatus1
582 	 * bits while the OA unit is enabled (while the tail pointer
583 	 * may be updated asynchronously) so we ignore status bits
584 	 * that have already been reported to userspace.
585 	 */
586 	oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1;
587 
588 	/* We treat OABUFFER_OVERFLOW as a significant error:
589 	 *
590 	 * - The status can be interpreted to mean that the buffer is
591 	 *   currently full (with a higher precedence than OA_TAKEN()
592 	 *   which will start to report a near-empty buffer after an
593 	 *   overflow) but it's awkward that we can't clear the status
594 	 *   on Haswell, so without a reset we won't be able to catch
595 	 *   the state again.
596 	 *
597 	 * - Since it also implies the HW has started overwriting old
598 	 *   reports it may also affect our sanity checks for invalid
599 	 *   reports when copying to userspace that assume new reports
600 	 *   are being written to cleared memory.
601 	 *
602 	 * - In the future we may want to introduce a flight recorder
603 	 *   mode where the driver will automatically maintain a safe
604 	 *   guard band between head/tail, avoiding this overflow
605 	 *   condition, but we avoid the added driver complexity for
606 	 *   now.
607 	 */
608 	if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) {
609 		ret = append_oa_status(stream, buf, count, offset,
610 				       DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
611 		if (ret)
612 			return ret;
613 
614 		DRM_DEBUG("OA buffer overflow: force restart\n");
615 
616 		dev_priv->perf.oa.ops.oa_disable(dev_priv);
617 		dev_priv->perf.oa.ops.oa_enable(dev_priv);
618 
619 		oastatus2 = I915_READ(GEN7_OASTATUS2);
620 		oastatus1 = I915_READ(GEN7_OASTATUS1);
621 
622 		head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK;
623 		tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
624 	}
625 
626 	if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) {
627 		ret = append_oa_status(stream, buf, count, offset,
628 				       DRM_I915_PERF_RECORD_OA_REPORT_LOST);
629 		if (ret)
630 			return ret;
631 		dev_priv->perf.oa.gen7_latched_oastatus1 |=
632 			GEN7_OASTATUS1_REPORT_LOST;
633 	}
634 
635 	ret = gen7_append_oa_reports(stream, buf, count, offset,
636 				     &head, tail);
637 
638 	/* All the report sizes are a power of two and the
639 	 * head should always be incremented by some multiple
640 	 * of the report size.
641 	 *
642 	 * A warning here, but notably if we later read back a
643 	 * misaligned pointer we will treat that as a bug since
644 	 * it could lead to a buffer overrun.
645 	 */
646 	WARN_ONCE(head & (report_size - 1),
647 		  "i915: Writing misaligned OA head pointer");
648 
649 	/* Note: we update the head pointer here even if an error
650 	 * was returned since the error may represent a short read
651 	 * where some some reports were successfully copied.
652 	 */
653 	I915_WRITE(GEN7_OASTATUS2,
654 		   ((head & GEN7_OASTATUS2_HEAD_MASK) |
655 		    OA_MEM_SELECT_GGTT));
656 
657 	return ret;
658 }
659 
660 /**
661  * i915_oa_wait_unlocked - handles blocking IO until OA data available
662  * @stream: An i915-perf stream opened for OA metrics
663  *
664  * Called when userspace tries to read() from a blocking stream FD opened
665  * for OA metrics. It waits until the hrtimer callback finds a non-empty
666  * OA buffer and wakes us.
667  *
668  * Note: it's acceptable to have this return with some false positives
669  * since any subsequent read handling will return -EAGAIN if there isn't
670  * really data ready for userspace yet.
671  *
672  * Returns: zero on success or a negative error code
673  */
674 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
675 {
676 	struct drm_i915_private *dev_priv = stream->dev_priv;
677 
678 	/* We would wait indefinitely if periodic sampling is not enabled */
679 	if (!dev_priv->perf.oa.periodic)
680 		return -EIO;
681 
682 	/* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it
683 	 * just performs mmio reads of the OA buffer head + tail pointers and
684 	 * it's assumed we're handling some operation that implies the stream
685 	 * can't be destroyed until completion (such as a read()) that ensures
686 	 * the device + OA buffer can't disappear
687 	 */
688 	return wait_event_interruptible(dev_priv->perf.oa.poll_wq,
689 					!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv));
690 }
691 
692 /**
693  * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
694  * @stream: An i915-perf stream opened for OA metrics
695  * @file: An i915 perf stream file
696  * @wait: poll() state table
697  *
698  * For handling userspace polling on an i915 perf stream opened for OA metrics,
699  * this starts a poll_wait with the wait queue that our hrtimer callback wakes
700  * when it sees data ready to read in the circular OA buffer.
701  */
702 static void i915_oa_poll_wait(struct i915_perf_stream *stream,
703 			      struct file *file,
704 			      poll_table *wait)
705 {
706 	struct drm_i915_private *dev_priv = stream->dev_priv;
707 
708 	poll_wait(file, &dev_priv->perf.oa.poll_wq, wait);
709 }
710 
711 /**
712  * i915_oa_read - just calls through to &i915_oa_ops->read
713  * @stream: An i915-perf stream opened for OA metrics
714  * @buf: destination buffer given by userspace
715  * @count: the number of bytes userspace wants to read
716  * @offset: (inout): the current position for writing into @buf
717  *
718  * Updates @offset according to the number of bytes successfully copied into
719  * the userspace buffer.
720  *
721  * Returns: zero on success or a negative error code
722  */
723 static int i915_oa_read(struct i915_perf_stream *stream,
724 			char __user *buf,
725 			size_t count,
726 			size_t *offset)
727 {
728 	struct drm_i915_private *dev_priv = stream->dev_priv;
729 
730 	return dev_priv->perf.oa.ops.read(stream, buf, count, offset);
731 }
732 
733 /**
734  * oa_get_render_ctx_id - determine and hold ctx hw id
735  * @stream: An i915-perf stream opened for OA metrics
736  *
737  * Determine the render context hw id, and ensure it remains fixed for the
738  * lifetime of the stream. This ensures that we don't have to worry about
739  * updating the context ID in OACONTROL on the fly.
740  *
741  * Returns: zero on success or a negative error code
742  */
743 static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
744 {
745 	struct drm_i915_private *dev_priv = stream->dev_priv;
746 	struct intel_engine_cs *engine = dev_priv->engine[RCS];
747 	int ret;
748 
749 	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
750 	if (ret)
751 		return ret;
752 
753 	/* As the ID is the gtt offset of the context's vma we pin
754 	 * the vma to ensure the ID remains fixed.
755 	 *
756 	 * NB: implied RCS engine...
757 	 */
758 	ret = engine->context_pin(engine, stream->ctx);
759 	if (ret)
760 		goto unlock;
761 
762 	/* Explicitly track the ID (instead of calling i915_ggtt_offset()
763 	 * on the fly) considering the difference with gen8+ and
764 	 * execlists
765 	 */
766 	dev_priv->perf.oa.specific_ctx_id =
767 		i915_ggtt_offset(stream->ctx->engine[engine->id].state);
768 
769 unlock:
770 	mutex_unlock(&dev_priv->drm.struct_mutex);
771 
772 	return ret;
773 }
774 
775 /**
776  * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
777  * @stream: An i915-perf stream opened for OA metrics
778  *
779  * In case anything needed doing to ensure the context HW ID would remain valid
780  * for the lifetime of the stream, then that can be undone here.
781  */
782 static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
783 {
784 	struct drm_i915_private *dev_priv = stream->dev_priv;
785 	struct intel_engine_cs *engine = dev_priv->engine[RCS];
786 
787 	mutex_lock(&dev_priv->drm.struct_mutex);
788 
789 	dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
790 	engine->context_unpin(engine, stream->ctx);
791 
792 	mutex_unlock(&dev_priv->drm.struct_mutex);
793 }
794 
795 static void
796 free_oa_buffer(struct drm_i915_private *i915)
797 {
798 	mutex_lock(&i915->drm.struct_mutex);
799 
800 	i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj);
801 	i915_vma_unpin(i915->perf.oa.oa_buffer.vma);
802 	i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj);
803 
804 	i915->perf.oa.oa_buffer.vma = NULL;
805 	i915->perf.oa.oa_buffer.vaddr = NULL;
806 
807 	mutex_unlock(&i915->drm.struct_mutex);
808 }
809 
810 static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
811 {
812 	struct drm_i915_private *dev_priv = stream->dev_priv;
813 
814 	BUG_ON(stream != dev_priv->perf.oa.exclusive_stream);
815 
816 	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
817 
818 	free_oa_buffer(dev_priv);
819 
820 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
821 	intel_runtime_pm_put(dev_priv);
822 
823 	if (stream->ctx)
824 		oa_put_render_ctx_id(stream);
825 
826 	dev_priv->perf.oa.exclusive_stream = NULL;
827 }
828 
829 static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv)
830 {
831 	u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma);
832 
833 	/* Pre-DevBDW: OABUFFER must be set with counters off,
834 	 * before OASTATUS1, but after OASTATUS2
835 	 */
836 	I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */
837 	I915_WRITE(GEN7_OABUFFER, gtt_offset);
838 	I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */
839 
840 	/* On Haswell we have to track which OASTATUS1 flags we've
841 	 * already seen since they can't be cleared while periodic
842 	 * sampling is enabled.
843 	 */
844 	dev_priv->perf.oa.gen7_latched_oastatus1 = 0;
845 
846 	/* NB: although the OA buffer will initially be allocated
847 	 * zeroed via shmfs (and so this memset is redundant when
848 	 * first allocating), we may re-init the OA buffer, either
849 	 * when re-enabling a stream or in error/reset paths.
850 	 *
851 	 * The reason we clear the buffer for each re-init is for the
852 	 * sanity check in gen7_append_oa_reports() that looks at the
853 	 * report-id field to make sure it's non-zero which relies on
854 	 * the assumption that new reports are being written to zeroed
855 	 * memory...
856 	 */
857 	memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
858 
859 	/* Maybe make ->pollin per-stream state if we support multiple
860 	 * concurrent streams in the future.
861 	 */
862 	dev_priv->perf.oa.pollin = false;
863 }
864 
865 static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
866 {
867 	struct drm_i915_gem_object *bo;
868 	struct i915_vma *vma;
869 	int ret;
870 
871 	if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma))
872 		return -ENODEV;
873 
874 	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
875 	if (ret)
876 		return ret;
877 
878 	BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
879 	BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
880 
881 	bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE);
882 	if (IS_ERR(bo)) {
883 		DRM_ERROR("Failed to allocate OA buffer\n");
884 		ret = PTR_ERR(bo);
885 		goto unlock;
886 	}
887 
888 	ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC);
889 	if (ret)
890 		goto err_unref;
891 
892 	/* PreHSW required 512K alignment, HSW requires 16M */
893 	vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0);
894 	if (IS_ERR(vma)) {
895 		ret = PTR_ERR(vma);
896 		goto err_unref;
897 	}
898 	dev_priv->perf.oa.oa_buffer.vma = vma;
899 
900 	dev_priv->perf.oa.oa_buffer.vaddr =
901 		i915_gem_object_pin_map(bo, I915_MAP_WB);
902 	if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) {
903 		ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr);
904 		goto err_unpin;
905 	}
906 
907 	dev_priv->perf.oa.ops.init_oa_buffer(dev_priv);
908 
909 	DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n",
910 			 i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma),
911 			 dev_priv->perf.oa.oa_buffer.vaddr);
912 
913 	goto unlock;
914 
915 err_unpin:
916 	__i915_vma_unpin(vma);
917 
918 err_unref:
919 	i915_gem_object_put(bo);
920 
921 	dev_priv->perf.oa.oa_buffer.vaddr = NULL;
922 	dev_priv->perf.oa.oa_buffer.vma = NULL;
923 
924 unlock:
925 	mutex_unlock(&dev_priv->drm.struct_mutex);
926 	return ret;
927 }
928 
929 static void config_oa_regs(struct drm_i915_private *dev_priv,
930 			   const struct i915_oa_reg *regs,
931 			   int n_regs)
932 {
933 	int i;
934 
935 	for (i = 0; i < n_regs; i++) {
936 		const struct i915_oa_reg *reg = regs + i;
937 
938 		I915_WRITE(reg->addr, reg->value);
939 	}
940 }
941 
942 static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
943 {
944 	int ret = i915_oa_select_metric_set_hsw(dev_priv);
945 
946 	if (ret)
947 		return ret;
948 
949 	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) |
950 				      GT_NOA_ENABLE));
951 
952 	/* PRM:
953 	 *
954 	 * OA unit is using “crclk” for its functionality. When trunk
955 	 * level clock gating takes place, OA clock would be gated,
956 	 * unable to count the events from non-render clock domain.
957 	 * Render clock gating must be disabled when OA is enabled to
958 	 * count the events from non-render domain. Unit level clock
959 	 * gating for RCS should also be disabled.
960 	 */
961 	I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
962 				    ~GEN7_DOP_CLOCK_GATE_ENABLE));
963 	I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) |
964 				  GEN6_CSUNIT_CLOCK_GATE_DISABLE));
965 
966 	config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs,
967 		       dev_priv->perf.oa.mux_regs_len);
968 
969 	/* It apparently takes a fairly long time for a new MUX
970 	 * configuration to be be applied after these register writes.
971 	 * This delay duration was derived empirically based on the
972 	 * render_basic config but hopefully it covers the maximum
973 	 * configuration latency.
974 	 *
975 	 * As a fallback, the checks in _append_oa_reports() to skip
976 	 * invalid OA reports do also seem to work to discard reports
977 	 * generated before this config has completed - albeit not
978 	 * silently.
979 	 *
980 	 * Unfortunately this is essentially a magic number, since we
981 	 * don't currently know of a reliable mechanism for predicting
982 	 * how long the MUX config will take to apply and besides
983 	 * seeing invalid reports we don't know of a reliable way to
984 	 * explicitly check that the MUX config has landed.
985 	 *
986 	 * It's even possible we've miss characterized the underlying
987 	 * problem - it just seems like the simplest explanation why
988 	 * a delay at this location would mitigate any invalid reports.
989 	 */
990 	usleep_range(15000, 20000);
991 
992 	config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs,
993 		       dev_priv->perf.oa.b_counter_regs_len);
994 
995 	return 0;
996 }
997 
998 static void hsw_disable_metric_set(struct drm_i915_private *dev_priv)
999 {
1000 	I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) &
1001 				  ~GEN6_CSUNIT_CLOCK_GATE_DISABLE));
1002 	I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) |
1003 				    GEN7_DOP_CLOCK_GATE_ENABLE));
1004 
1005 	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
1006 				      ~GT_NOA_ENABLE));
1007 }
1008 
1009 static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv)
1010 {
1011 	lockdep_assert_held(&dev_priv->perf.hook_lock);
1012 
1013 	if (dev_priv->perf.oa.exclusive_stream->enabled) {
1014 		struct i915_gem_context *ctx =
1015 			dev_priv->perf.oa.exclusive_stream->ctx;
1016 		u32 ctx_id = dev_priv->perf.oa.specific_ctx_id;
1017 
1018 		bool periodic = dev_priv->perf.oa.periodic;
1019 		u32 period_exponent = dev_priv->perf.oa.period_exponent;
1020 		u32 report_format = dev_priv->perf.oa.oa_buffer.format;
1021 
1022 		I915_WRITE(GEN7_OACONTROL,
1023 			   (ctx_id & GEN7_OACONTROL_CTX_MASK) |
1024 			   (period_exponent <<
1025 			    GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
1026 			   (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
1027 			   (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
1028 			   (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
1029 			   GEN7_OACONTROL_ENABLE);
1030 	} else
1031 		I915_WRITE(GEN7_OACONTROL, 0);
1032 }
1033 
1034 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
1035 {
1036 	unsigned long flags;
1037 
1038 	/* Reset buf pointers so we don't forward reports from before now.
1039 	 *
1040 	 * Think carefully if considering trying to avoid this, since it
1041 	 * also ensures status flags and the buffer itself are cleared
1042 	 * in error paths, and we have checks for invalid reports based
1043 	 * on the assumption that certain fields are written to zeroed
1044 	 * memory which this helps maintains.
1045 	 */
1046 	gen7_init_oa_buffer(dev_priv);
1047 
1048 	spin_lock_irqsave(&dev_priv->perf.hook_lock, flags);
1049 	gen7_update_oacontrol_locked(dev_priv);
1050 	spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags);
1051 }
1052 
1053 /**
1054  * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
1055  * @stream: An i915 perf stream opened for OA metrics
1056  *
1057  * [Re]enables hardware periodic sampling according to the period configured
1058  * when opening the stream. This also starts a hrtimer that will periodically
1059  * check for data in the circular OA buffer for notifying userspace (e.g.
1060  * during a read() or poll()).
1061  */
1062 static void i915_oa_stream_enable(struct i915_perf_stream *stream)
1063 {
1064 	struct drm_i915_private *dev_priv = stream->dev_priv;
1065 
1066 	dev_priv->perf.oa.ops.oa_enable(dev_priv);
1067 
1068 	if (dev_priv->perf.oa.periodic)
1069 		hrtimer_start(&dev_priv->perf.oa.poll_check_timer,
1070 			      ns_to_ktime(POLL_PERIOD),
1071 			      HRTIMER_MODE_REL_PINNED);
1072 }
1073 
1074 static void gen7_oa_disable(struct drm_i915_private *dev_priv)
1075 {
1076 	I915_WRITE(GEN7_OACONTROL, 0);
1077 }
1078 
1079 /**
1080  * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
1081  * @stream: An i915 perf stream opened for OA metrics
1082  *
1083  * Stops the OA unit from periodically writing counter reports into the
1084  * circular OA buffer. This also stops the hrtimer that periodically checks for
1085  * data in the circular OA buffer, for notifying userspace.
1086  */
1087 static void i915_oa_stream_disable(struct i915_perf_stream *stream)
1088 {
1089 	struct drm_i915_private *dev_priv = stream->dev_priv;
1090 
1091 	dev_priv->perf.oa.ops.oa_disable(dev_priv);
1092 
1093 	if (dev_priv->perf.oa.periodic)
1094 		hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer);
1095 }
1096 
1097 static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent)
1098 {
1099 	return div_u64(1000000000ULL * (2ULL << exponent),
1100 		       dev_priv->perf.oa.timestamp_frequency);
1101 }
1102 
1103 static const struct i915_perf_stream_ops i915_oa_stream_ops = {
1104 	.destroy = i915_oa_stream_destroy,
1105 	.enable = i915_oa_stream_enable,
1106 	.disable = i915_oa_stream_disable,
1107 	.wait_unlocked = i915_oa_wait_unlocked,
1108 	.poll_wait = i915_oa_poll_wait,
1109 	.read = i915_oa_read,
1110 };
1111 
1112 /**
1113  * i915_oa_stream_init - validate combined props for OA stream and init
1114  * @stream: An i915 perf stream
1115  * @param: The open parameters passed to `DRM_I915_PERF_OPEN`
1116  * @props: The property state that configures stream (individually validated)
1117  *
1118  * While read_properties_unlocked() validates properties in isolation it
1119  * doesn't ensure that the combination necessarily makes sense.
1120  *
1121  * At this point it has been determined that userspace wants a stream of
1122  * OA metrics, but still we need to further validate the combined
1123  * properties are OK.
1124  *
1125  * If the configuration makes sense then we can allocate memory for
1126  * a circular OA buffer and apply the requested metric set configuration.
1127  *
1128  * Returns: zero on success or a negative error code.
1129  */
1130 static int i915_oa_stream_init(struct i915_perf_stream *stream,
1131 			       struct drm_i915_perf_open_param *param,
1132 			       struct perf_open_properties *props)
1133 {
1134 	struct drm_i915_private *dev_priv = stream->dev_priv;
1135 	int format_size;
1136 	int ret;
1137 
1138 	/* If the sysfs metrics/ directory wasn't registered for some
1139 	 * reason then don't let userspace try their luck with config
1140 	 * IDs
1141 	 */
1142 	if (!dev_priv->perf.metrics_kobj) {
1143 		DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
1144 		return -EINVAL;
1145 	}
1146 
1147 	if (!(props->sample_flags & SAMPLE_OA_REPORT)) {
1148 		DRM_DEBUG("Only OA report sampling supported\n");
1149 		return -EINVAL;
1150 	}
1151 
1152 	if (!dev_priv->perf.oa.ops.init_oa_buffer) {
1153 		DRM_DEBUG("OA unit not supported\n");
1154 		return -ENODEV;
1155 	}
1156 
1157 	/* To avoid the complexity of having to accurately filter
1158 	 * counter reports and marshal to the appropriate client
1159 	 * we currently only allow exclusive access
1160 	 */
1161 	if (dev_priv->perf.oa.exclusive_stream) {
1162 		DRM_DEBUG("OA unit already in use\n");
1163 		return -EBUSY;
1164 	}
1165 
1166 	if (!props->metrics_set) {
1167 		DRM_DEBUG("OA metric set not specified\n");
1168 		return -EINVAL;
1169 	}
1170 
1171 	if (!props->oa_format) {
1172 		DRM_DEBUG("OA report format not specified\n");
1173 		return -EINVAL;
1174 	}
1175 
1176 	stream->sample_size = sizeof(struct drm_i915_perf_record_header);
1177 
1178 	format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size;
1179 
1180 	stream->sample_flags |= SAMPLE_OA_REPORT;
1181 	stream->sample_size += format_size;
1182 
1183 	dev_priv->perf.oa.oa_buffer.format_size = format_size;
1184 	if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0))
1185 		return -EINVAL;
1186 
1187 	dev_priv->perf.oa.oa_buffer.format =
1188 		dev_priv->perf.oa.oa_formats[props->oa_format].format;
1189 
1190 	dev_priv->perf.oa.metrics_set = props->metrics_set;
1191 
1192 	dev_priv->perf.oa.periodic = props->oa_periodic;
1193 	if (dev_priv->perf.oa.periodic) {
1194 		u32 tail;
1195 
1196 		dev_priv->perf.oa.period_exponent = props->oa_period_exponent;
1197 
1198 		/* See comment for OA_TAIL_MARGIN_NSEC for details
1199 		 * about this tail_margin...
1200 		 */
1201 		tail = div64_u64(OA_TAIL_MARGIN_NSEC,
1202 				 oa_exponent_to_ns(dev_priv,
1203 						   props->oa_period_exponent));
1204 		dev_priv->perf.oa.tail_margin = (tail + 1) * format_size;
1205 	}
1206 
1207 	if (stream->ctx) {
1208 		ret = oa_get_render_ctx_id(stream);
1209 		if (ret)
1210 			return ret;
1211 	}
1212 
1213 	ret = alloc_oa_buffer(dev_priv);
1214 	if (ret)
1215 		goto err_oa_buf_alloc;
1216 
1217 	/* PRM - observability performance counters:
1218 	 *
1219 	 *   OACONTROL, performance counter enable, note:
1220 	 *
1221 	 *   "When this bit is set, in order to have coherent counts,
1222 	 *   RC6 power state and trunk clock gating must be disabled.
1223 	 *   This can be achieved by programming MMIO registers as
1224 	 *   0xA094=0 and 0xA090[31]=1"
1225 	 *
1226 	 *   In our case we are expecting that taking pm + FORCEWAKE
1227 	 *   references will effectively disable RC6.
1228 	 */
1229 	intel_runtime_pm_get(dev_priv);
1230 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1231 
1232 	ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv);
1233 	if (ret)
1234 		goto err_enable;
1235 
1236 	stream->ops = &i915_oa_stream_ops;
1237 
1238 	dev_priv->perf.oa.exclusive_stream = stream;
1239 
1240 	return 0;
1241 
1242 err_enable:
1243 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1244 	intel_runtime_pm_put(dev_priv);
1245 	free_oa_buffer(dev_priv);
1246 
1247 err_oa_buf_alloc:
1248 	if (stream->ctx)
1249 		oa_put_render_ctx_id(stream);
1250 
1251 	return ret;
1252 }
1253 
1254 /**
1255  * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation
1256  * @stream: An i915 perf stream
1257  * @file: An i915 perf stream file
1258  * @buf: destination buffer given by userspace
1259  * @count: the number of bytes userspace wants to read
1260  * @ppos: (inout) file seek position (unused)
1261  *
1262  * Besides wrapping &i915_perf_stream_ops->read this provides a common place to
1263  * ensure that if we've successfully copied any data then reporting that takes
1264  * precedence over any internal error status, so the data isn't lost.
1265  *
1266  * For example ret will be -ENOSPC whenever there is more buffered data than
1267  * can be copied to userspace, but that's only interesting if we weren't able
1268  * to copy some data because it implies the userspace buffer is too small to
1269  * receive a single record (and we never split records).
1270  *
1271  * Another case with ret == -EFAULT is more of a grey area since it would seem
1272  * like bad form for userspace to ask us to overrun its buffer, but the user
1273  * knows best:
1274  *
1275  *   http://yarchive.net/comp/linux/partial_reads_writes.html
1276  *
1277  * Returns: The number of bytes copied or a negative error code on failure.
1278  */
1279 static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream,
1280 				     struct file *file,
1281 				     char __user *buf,
1282 				     size_t count,
1283 				     loff_t *ppos)
1284 {
1285 	/* Note we keep the offset (aka bytes read) separate from any
1286 	 * error status so that the final check for whether we return
1287 	 * the bytes read with a higher precedence than any error (see
1288 	 * comment below) doesn't need to be handled/duplicated in
1289 	 * stream->ops->read() implementations.
1290 	 */
1291 	size_t offset = 0;
1292 	int ret = stream->ops->read(stream, buf, count, &offset);
1293 
1294 	return offset ?: (ret ?: -EAGAIN);
1295 }
1296 
1297 /**
1298  * i915_perf_read - handles read() FOP for i915 perf stream FDs
1299  * @file: An i915 perf stream file
1300  * @buf: destination buffer given by userspace
1301  * @count: the number of bytes userspace wants to read
1302  * @ppos: (inout) file seek position (unused)
1303  *
1304  * The entry point for handling a read() on a stream file descriptor from
1305  * userspace. Most of the work is left to the i915_perf_read_locked() and
1306  * &i915_perf_stream_ops->read but to save having stream implementations (of
1307  * which we might have multiple later) we handle blocking read here.
1308  *
1309  * We can also consistently treat trying to read from a disabled stream
1310  * as an IO error so implementations can assume the stream is enabled
1311  * while reading.
1312  *
1313  * Returns: The number of bytes copied or a negative error code on failure.
1314  */
1315 static ssize_t i915_perf_read(struct file *file,
1316 			      char __user *buf,
1317 			      size_t count,
1318 			      loff_t *ppos)
1319 {
1320 	struct i915_perf_stream *stream = file->private_data;
1321 	struct drm_i915_private *dev_priv = stream->dev_priv;
1322 	ssize_t ret;
1323 
1324 	/* To ensure it's handled consistently we simply treat all reads of a
1325 	 * disabled stream as an error. In particular it might otherwise lead
1326 	 * to a deadlock for blocking file descriptors...
1327 	 */
1328 	if (!stream->enabled)
1329 		return -EIO;
1330 
1331 	if (!(file->f_flags & O_NONBLOCK)) {
1332 		/* There's the small chance of false positives from
1333 		 * stream->ops->wait_unlocked.
1334 		 *
1335 		 * E.g. with single context filtering since we only wait until
1336 		 * oabuffer has >= 1 report we don't immediately know whether
1337 		 * any reports really belong to the current context
1338 		 */
1339 		do {
1340 			ret = stream->ops->wait_unlocked(stream);
1341 			if (ret)
1342 				return ret;
1343 
1344 			mutex_lock(&dev_priv->perf.lock);
1345 			ret = i915_perf_read_locked(stream, file,
1346 						    buf, count, ppos);
1347 			mutex_unlock(&dev_priv->perf.lock);
1348 		} while (ret == -EAGAIN);
1349 	} else {
1350 		mutex_lock(&dev_priv->perf.lock);
1351 		ret = i915_perf_read_locked(stream, file, buf, count, ppos);
1352 		mutex_unlock(&dev_priv->perf.lock);
1353 	}
1354 
1355 	if (ret >= 0) {
1356 		/* Maybe make ->pollin per-stream state if we support multiple
1357 		 * concurrent streams in the future.
1358 		 */
1359 		dev_priv->perf.oa.pollin = false;
1360 	}
1361 
1362 	return ret;
1363 }
1364 
1365 static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
1366 {
1367 	struct drm_i915_private *dev_priv =
1368 		container_of(hrtimer, typeof(*dev_priv),
1369 			     perf.oa.poll_check_timer);
1370 
1371 	if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) {
1372 		dev_priv->perf.oa.pollin = true;
1373 		wake_up(&dev_priv->perf.oa.poll_wq);
1374 	}
1375 
1376 	hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD));
1377 
1378 	return HRTIMER_RESTART;
1379 }
1380 
1381 /**
1382  * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
1383  * @dev_priv: i915 device instance
1384  * @stream: An i915 perf stream
1385  * @file: An i915 perf stream file
1386  * @wait: poll() state table
1387  *
1388  * For handling userspace polling on an i915 perf stream, this calls through to
1389  * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
1390  * will be woken for new stream data.
1391  *
1392  * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
1393  * with any non-file-operation driver hooks.
1394  *
1395  * Returns: any poll events that are ready without sleeping
1396  */
1397 static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv,
1398 					  struct i915_perf_stream *stream,
1399 					  struct file *file,
1400 					  poll_table *wait)
1401 {
1402 	unsigned int events = 0;
1403 
1404 	stream->ops->poll_wait(stream, file, wait);
1405 
1406 	/* Note: we don't explicitly check whether there's something to read
1407 	 * here since this path may be very hot depending on what else
1408 	 * userspace is polling, or on the timeout in use. We rely solely on
1409 	 * the hrtimer/oa_poll_check_timer_cb to notify us when there are
1410 	 * samples to read.
1411 	 */
1412 	if (dev_priv->perf.oa.pollin)
1413 		events |= POLLIN;
1414 
1415 	return events;
1416 }
1417 
1418 /**
1419  * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
1420  * @file: An i915 perf stream file
1421  * @wait: poll() state table
1422  *
1423  * For handling userspace polling on an i915 perf stream, this ensures
1424  * poll_wait() gets called with a wait queue that will be woken for new stream
1425  * data.
1426  *
1427  * Note: Implementation deferred to i915_perf_poll_locked()
1428  *
1429  * Returns: any poll events that are ready without sleeping
1430  */
1431 static unsigned int i915_perf_poll(struct file *file, poll_table *wait)
1432 {
1433 	struct i915_perf_stream *stream = file->private_data;
1434 	struct drm_i915_private *dev_priv = stream->dev_priv;
1435 	int ret;
1436 
1437 	mutex_lock(&dev_priv->perf.lock);
1438 	ret = i915_perf_poll_locked(dev_priv, stream, file, wait);
1439 	mutex_unlock(&dev_priv->perf.lock);
1440 
1441 	return ret;
1442 }
1443 
1444 /**
1445  * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
1446  * @stream: A disabled i915 perf stream
1447  *
1448  * [Re]enables the associated capture of data for this stream.
1449  *
1450  * If a stream was previously enabled then there's currently no intention
1451  * to provide userspace any guarantee about the preservation of previously
1452  * buffered data.
1453  */
1454 static void i915_perf_enable_locked(struct i915_perf_stream *stream)
1455 {
1456 	if (stream->enabled)
1457 		return;
1458 
1459 	/* Allow stream->ops->enable() to refer to this */
1460 	stream->enabled = true;
1461 
1462 	if (stream->ops->enable)
1463 		stream->ops->enable(stream);
1464 }
1465 
1466 /**
1467  * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
1468  * @stream: An enabled i915 perf stream
1469  *
1470  * Disables the associated capture of data for this stream.
1471  *
1472  * The intention is that disabling an re-enabling a stream will ideally be
1473  * cheaper than destroying and re-opening a stream with the same configuration,
1474  * though there are no formal guarantees about what state or buffered data
1475  * must be retained between disabling and re-enabling a stream.
1476  *
1477  * Note: while a stream is disabled it's considered an error for userspace
1478  * to attempt to read from the stream (-EIO).
1479  */
1480 static void i915_perf_disable_locked(struct i915_perf_stream *stream)
1481 {
1482 	if (!stream->enabled)
1483 		return;
1484 
1485 	/* Allow stream->ops->disable() to refer to this */
1486 	stream->enabled = false;
1487 
1488 	if (stream->ops->disable)
1489 		stream->ops->disable(stream);
1490 }
1491 
1492 /**
1493  * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
1494  * @stream: An i915 perf stream
1495  * @cmd: the ioctl request
1496  * @arg: the ioctl data
1497  *
1498  * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
1499  * with any non-file-operation driver hooks.
1500  *
1501  * Returns: zero on success or a negative error code. Returns -EINVAL for
1502  * an unknown ioctl request.
1503  */
1504 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
1505 				   unsigned int cmd,
1506 				   unsigned long arg)
1507 {
1508 	switch (cmd) {
1509 	case I915_PERF_IOCTL_ENABLE:
1510 		i915_perf_enable_locked(stream);
1511 		return 0;
1512 	case I915_PERF_IOCTL_DISABLE:
1513 		i915_perf_disable_locked(stream);
1514 		return 0;
1515 	}
1516 
1517 	return -EINVAL;
1518 }
1519 
1520 /**
1521  * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
1522  * @file: An i915 perf stream file
1523  * @cmd: the ioctl request
1524  * @arg: the ioctl data
1525  *
1526  * Implementation deferred to i915_perf_ioctl_locked().
1527  *
1528  * Returns: zero on success or a negative error code. Returns -EINVAL for
1529  * an unknown ioctl request.
1530  */
1531 static long i915_perf_ioctl(struct file *file,
1532 			    unsigned int cmd,
1533 			    unsigned long arg)
1534 {
1535 	struct i915_perf_stream *stream = file->private_data;
1536 	struct drm_i915_private *dev_priv = stream->dev_priv;
1537 	long ret;
1538 
1539 	mutex_lock(&dev_priv->perf.lock);
1540 	ret = i915_perf_ioctl_locked(stream, cmd, arg);
1541 	mutex_unlock(&dev_priv->perf.lock);
1542 
1543 	return ret;
1544 }
1545 
1546 /**
1547  * i915_perf_destroy_locked - destroy an i915 perf stream
1548  * @stream: An i915 perf stream
1549  *
1550  * Frees all resources associated with the given i915 perf @stream, disabling
1551  * any associated data capture in the process.
1552  *
1553  * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
1554  * with any non-file-operation driver hooks.
1555  */
1556 static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
1557 {
1558 	if (stream->enabled)
1559 		i915_perf_disable_locked(stream);
1560 
1561 	if (stream->ops->destroy)
1562 		stream->ops->destroy(stream);
1563 
1564 	list_del(&stream->link);
1565 
1566 	if (stream->ctx)
1567 		i915_gem_context_put_unlocked(stream->ctx);
1568 
1569 	kfree(stream);
1570 }
1571 
1572 /**
1573  * i915_perf_release - handles userspace close() of a stream file
1574  * @inode: anonymous inode associated with file
1575  * @file: An i915 perf stream file
1576  *
1577  * Cleans up any resources associated with an open i915 perf stream file.
1578  *
1579  * NB: close() can't really fail from the userspace point of view.
1580  *
1581  * Returns: zero on success or a negative error code.
1582  */
1583 static int i915_perf_release(struct inode *inode, struct file *file)
1584 {
1585 	struct i915_perf_stream *stream = file->private_data;
1586 	struct drm_i915_private *dev_priv = stream->dev_priv;
1587 
1588 	mutex_lock(&dev_priv->perf.lock);
1589 	i915_perf_destroy_locked(stream);
1590 	mutex_unlock(&dev_priv->perf.lock);
1591 
1592 	return 0;
1593 }
1594 
1595 
1596 static const struct file_operations fops = {
1597 	.owner		= THIS_MODULE,
1598 	.llseek		= no_llseek,
1599 	.release	= i915_perf_release,
1600 	.poll		= i915_perf_poll,
1601 	.read		= i915_perf_read,
1602 	.unlocked_ioctl	= i915_perf_ioctl,
1603 };
1604 
1605 
1606 static struct i915_gem_context *
1607 lookup_context(struct drm_i915_private *dev_priv,
1608 	       struct drm_i915_file_private *file_priv,
1609 	       u32 ctx_user_handle)
1610 {
1611 	struct i915_gem_context *ctx;
1612 	int ret;
1613 
1614 	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
1615 	if (ret)
1616 		return ERR_PTR(ret);
1617 
1618 	ctx = i915_gem_context_lookup(file_priv, ctx_user_handle);
1619 	if (!IS_ERR(ctx))
1620 		i915_gem_context_get(ctx);
1621 
1622 	mutex_unlock(&dev_priv->drm.struct_mutex);
1623 
1624 	return ctx;
1625 }
1626 
1627 /**
1628  * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
1629  * @dev_priv: i915 device instance
1630  * @param: The open parameters passed to 'DRM_I915_PERF_OPEN`
1631  * @props: individually validated u64 property value pairs
1632  * @file: drm file
1633  *
1634  * See i915_perf_ioctl_open() for interface details.
1635  *
1636  * Implements further stream config validation and stream initialization on
1637  * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex
1638  * taken to serialize with any non-file-operation driver hooks.
1639  *
1640  * Note: at this point the @props have only been validated in isolation and
1641  * it's still necessary to validate that the combination of properties makes
1642  * sense.
1643  *
1644  * In the case where userspace is interested in OA unit metrics then further
1645  * config validation and stream initialization details will be handled by
1646  * i915_oa_stream_init(). The code here should only validate config state that
1647  * will be relevant to all stream types / backends.
1648  *
1649  * Returns: zero on success or a negative error code.
1650  */
1651 static int
1652 i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv,
1653 			    struct drm_i915_perf_open_param *param,
1654 			    struct perf_open_properties *props,
1655 			    struct drm_file *file)
1656 {
1657 	struct i915_gem_context *specific_ctx = NULL;
1658 	struct i915_perf_stream *stream = NULL;
1659 	unsigned long f_flags = 0;
1660 	int stream_fd;
1661 	int ret;
1662 
1663 	if (props->single_context) {
1664 		u32 ctx_handle = props->ctx_handle;
1665 		struct drm_i915_file_private *file_priv = file->driver_priv;
1666 
1667 		specific_ctx = lookup_context(dev_priv, file_priv, ctx_handle);
1668 		if (IS_ERR(specific_ctx)) {
1669 			ret = PTR_ERR(specific_ctx);
1670 			if (ret != -EINTR)
1671 				DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
1672 					  ctx_handle);
1673 			goto err;
1674 		}
1675 	}
1676 
1677 	/* Similar to perf's kernel.perf_paranoid_cpu sysctl option
1678 	 * we check a dev.i915.perf_stream_paranoid sysctl option
1679 	 * to determine if it's ok to access system wide OA counters
1680 	 * without CAP_SYS_ADMIN privileges.
1681 	 */
1682 	if (!specific_ctx &&
1683 	    i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) {
1684 		DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n");
1685 		ret = -EACCES;
1686 		goto err_ctx;
1687 	}
1688 
1689 	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
1690 	if (!stream) {
1691 		ret = -ENOMEM;
1692 		goto err_ctx;
1693 	}
1694 
1695 	stream->dev_priv = dev_priv;
1696 	stream->ctx = specific_ctx;
1697 
1698 	ret = i915_oa_stream_init(stream, param, props);
1699 	if (ret)
1700 		goto err_alloc;
1701 
1702 	/* we avoid simply assigning stream->sample_flags = props->sample_flags
1703 	 * to have _stream_init check the combination of sample flags more
1704 	 * thoroughly, but still this is the expected result at this point.
1705 	 */
1706 	if (WARN_ON(stream->sample_flags != props->sample_flags)) {
1707 		ret = -ENODEV;
1708 		goto err_flags;
1709 	}
1710 
1711 	list_add(&stream->link, &dev_priv->perf.streams);
1712 
1713 	if (param->flags & I915_PERF_FLAG_FD_CLOEXEC)
1714 		f_flags |= O_CLOEXEC;
1715 	if (param->flags & I915_PERF_FLAG_FD_NONBLOCK)
1716 		f_flags |= O_NONBLOCK;
1717 
1718 	stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags);
1719 	if (stream_fd < 0) {
1720 		ret = stream_fd;
1721 		goto err_open;
1722 	}
1723 
1724 	if (!(param->flags & I915_PERF_FLAG_DISABLED))
1725 		i915_perf_enable_locked(stream);
1726 
1727 	return stream_fd;
1728 
1729 err_open:
1730 	list_del(&stream->link);
1731 err_flags:
1732 	if (stream->ops->destroy)
1733 		stream->ops->destroy(stream);
1734 err_alloc:
1735 	kfree(stream);
1736 err_ctx:
1737 	if (specific_ctx)
1738 		i915_gem_context_put_unlocked(specific_ctx);
1739 err:
1740 	return ret;
1741 }
1742 
1743 /**
1744  * read_properties_unlocked - validate + copy userspace stream open properties
1745  * @dev_priv: i915 device instance
1746  * @uprops: The array of u64 key value pairs given by userspace
1747  * @n_props: The number of key value pairs expected in @uprops
1748  * @props: The stream configuration built up while validating properties
1749  *
1750  * Note this function only validates properties in isolation it doesn't
1751  * validate that the combination of properties makes sense or that all
1752  * properties necessary for a particular kind of stream have been set.
1753  *
1754  * Note that there currently aren't any ordering requirements for properties so
1755  * we shouldn't validate or assume anything about ordering here. This doesn't
1756  * rule out defining new properties with ordering requirements in the future.
1757  */
1758 static int read_properties_unlocked(struct drm_i915_private *dev_priv,
1759 				    u64 __user *uprops,
1760 				    u32 n_props,
1761 				    struct perf_open_properties *props)
1762 {
1763 	u64 __user *uprop = uprops;
1764 	int i;
1765 
1766 	memset(props, 0, sizeof(struct perf_open_properties));
1767 
1768 	if (!n_props) {
1769 		DRM_DEBUG("No i915 perf properties given\n");
1770 		return -EINVAL;
1771 	}
1772 
1773 	/* Considering that ID = 0 is reserved and assuming that we don't
1774 	 * (currently) expect any configurations to ever specify duplicate
1775 	 * values for a particular property ID then the last _PROP_MAX value is
1776 	 * one greater than the maximum number of properties we expect to get
1777 	 * from userspace.
1778 	 */
1779 	if (n_props >= DRM_I915_PERF_PROP_MAX) {
1780 		DRM_DEBUG("More i915 perf properties specified than exist\n");
1781 		return -EINVAL;
1782 	}
1783 
1784 	for (i = 0; i < n_props; i++) {
1785 		u64 oa_period, oa_freq_hz;
1786 		u64 id, value;
1787 		int ret;
1788 
1789 		ret = get_user(id, uprop);
1790 		if (ret)
1791 			return ret;
1792 
1793 		ret = get_user(value, uprop + 1);
1794 		if (ret)
1795 			return ret;
1796 
1797 		if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) {
1798 			DRM_DEBUG("Unknown i915 perf property ID\n");
1799 			return -EINVAL;
1800 		}
1801 
1802 		switch ((enum drm_i915_perf_property_id)id) {
1803 		case DRM_I915_PERF_PROP_CTX_HANDLE:
1804 			props->single_context = 1;
1805 			props->ctx_handle = value;
1806 			break;
1807 		case DRM_I915_PERF_PROP_SAMPLE_OA:
1808 			props->sample_flags |= SAMPLE_OA_REPORT;
1809 			break;
1810 		case DRM_I915_PERF_PROP_OA_METRICS_SET:
1811 			if (value == 0 ||
1812 			    value > dev_priv->perf.oa.n_builtin_sets) {
1813 				DRM_DEBUG("Unknown OA metric set ID\n");
1814 				return -EINVAL;
1815 			}
1816 			props->metrics_set = value;
1817 			break;
1818 		case DRM_I915_PERF_PROP_OA_FORMAT:
1819 			if (value == 0 || value >= I915_OA_FORMAT_MAX) {
1820 				DRM_DEBUG("Invalid OA report format\n");
1821 				return -EINVAL;
1822 			}
1823 			if (!dev_priv->perf.oa.oa_formats[value].size) {
1824 				DRM_DEBUG("Invalid OA report format\n");
1825 				return -EINVAL;
1826 			}
1827 			props->oa_format = value;
1828 			break;
1829 		case DRM_I915_PERF_PROP_OA_EXPONENT:
1830 			if (value > OA_EXPONENT_MAX) {
1831 				DRM_DEBUG("OA timer exponent too high (> %u)\n",
1832 					 OA_EXPONENT_MAX);
1833 				return -EINVAL;
1834 			}
1835 
1836 			/* Theoretically we can program the OA unit to sample
1837 			 * every 160ns but don't allow that by default unless
1838 			 * root.
1839 			 *
1840 			 * On Haswell the period is derived from the exponent
1841 			 * as:
1842 			 *
1843 			 *   period = 80ns * 2^(exponent + 1)
1844 			 */
1845 			BUILD_BUG_ON(sizeof(oa_period) != 8);
1846 			oa_period = 80ull * (2ull << value);
1847 
1848 			/* This check is primarily to ensure that oa_period <=
1849 			 * UINT32_MAX (before passing to do_div which only
1850 			 * accepts a u32 denominator), but we can also skip
1851 			 * checking anything < 1Hz which implicitly can't be
1852 			 * limited via an integer oa_max_sample_rate.
1853 			 */
1854 			if (oa_period <= NSEC_PER_SEC) {
1855 				u64 tmp = NSEC_PER_SEC;
1856 				do_div(tmp, oa_period);
1857 				oa_freq_hz = tmp;
1858 			} else
1859 				oa_freq_hz = 0;
1860 
1861 			if (oa_freq_hz > i915_oa_max_sample_rate &&
1862 			    !capable(CAP_SYS_ADMIN)) {
1863 				DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n",
1864 					  i915_oa_max_sample_rate);
1865 				return -EACCES;
1866 			}
1867 
1868 			props->oa_periodic = true;
1869 			props->oa_period_exponent = value;
1870 			break;
1871 		case DRM_I915_PERF_PROP_MAX:
1872 			MISSING_CASE(id);
1873 			return -EINVAL;
1874 		}
1875 
1876 		uprop += 2;
1877 	}
1878 
1879 	return 0;
1880 }
1881 
1882 /**
1883  * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
1884  * @dev: drm device
1885  * @data: ioctl data copied from userspace (unvalidated)
1886  * @file: drm file
1887  *
1888  * Validates the stream open parameters given by userspace including flags
1889  * and an array of u64 key, value pair properties.
1890  *
1891  * Very little is assumed up front about the nature of the stream being
1892  * opened (for instance we don't assume it's for periodic OA unit metrics). An
1893  * i915-perf stream is expected to be a suitable interface for other forms of
1894  * buffered data written by the GPU besides periodic OA metrics.
1895  *
1896  * Note we copy the properties from userspace outside of the i915 perf
1897  * mutex to avoid an awkward lockdep with mmap_sem.
1898  *
1899  * Most of the implementation details are handled by
1900  * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock
1901  * mutex for serializing with any non-file-operation driver hooks.
1902  *
1903  * Return: A newly opened i915 Perf stream file descriptor or negative
1904  * error code on failure.
1905  */
1906 int i915_perf_open_ioctl(struct drm_device *dev, void *data,
1907 			 struct drm_file *file)
1908 {
1909 	struct drm_i915_private *dev_priv = dev->dev_private;
1910 	struct drm_i915_perf_open_param *param = data;
1911 	struct perf_open_properties props;
1912 	u32 known_open_flags;
1913 	int ret;
1914 
1915 	if (!dev_priv->perf.initialized) {
1916 		DRM_DEBUG("i915 perf interface not available for this system\n");
1917 		return -ENOTSUPP;
1918 	}
1919 
1920 	known_open_flags = I915_PERF_FLAG_FD_CLOEXEC |
1921 			   I915_PERF_FLAG_FD_NONBLOCK |
1922 			   I915_PERF_FLAG_DISABLED;
1923 	if (param->flags & ~known_open_flags) {
1924 		DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n");
1925 		return -EINVAL;
1926 	}
1927 
1928 	ret = read_properties_unlocked(dev_priv,
1929 				       u64_to_user_ptr(param->properties_ptr),
1930 				       param->num_properties,
1931 				       &props);
1932 	if (ret)
1933 		return ret;
1934 
1935 	mutex_lock(&dev_priv->perf.lock);
1936 	ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file);
1937 	mutex_unlock(&dev_priv->perf.lock);
1938 
1939 	return ret;
1940 }
1941 
1942 /**
1943  * i915_perf_register - exposes i915-perf to userspace
1944  * @dev_priv: i915 device instance
1945  *
1946  * In particular OA metric sets are advertised under a sysfs metrics/
1947  * directory allowing userspace to enumerate valid IDs that can be
1948  * used to open an i915-perf stream.
1949  */
1950 void i915_perf_register(struct drm_i915_private *dev_priv)
1951 {
1952 	if (!IS_HASWELL(dev_priv))
1953 		return;
1954 
1955 	if (!dev_priv->perf.initialized)
1956 		return;
1957 
1958 	/* To be sure we're synchronized with an attempted
1959 	 * i915_perf_open_ioctl(); considering that we register after
1960 	 * being exposed to userspace.
1961 	 */
1962 	mutex_lock(&dev_priv->perf.lock);
1963 
1964 	dev_priv->perf.metrics_kobj =
1965 		kobject_create_and_add("metrics",
1966 				       &dev_priv->drm.primary->kdev->kobj);
1967 	if (!dev_priv->perf.metrics_kobj)
1968 		goto exit;
1969 
1970 	if (i915_perf_register_sysfs_hsw(dev_priv)) {
1971 		kobject_put(dev_priv->perf.metrics_kobj);
1972 		dev_priv->perf.metrics_kobj = NULL;
1973 	}
1974 
1975 exit:
1976 	mutex_unlock(&dev_priv->perf.lock);
1977 }
1978 
1979 /**
1980  * i915_perf_unregister - hide i915-perf from userspace
1981  * @dev_priv: i915 device instance
1982  *
1983  * i915-perf state cleanup is split up into an 'unregister' and
1984  * 'deinit' phase where the interface is first hidden from
1985  * userspace by i915_perf_unregister() before cleaning up
1986  * remaining state in i915_perf_fini().
1987  */
1988 void i915_perf_unregister(struct drm_i915_private *dev_priv)
1989 {
1990 	if (!IS_HASWELL(dev_priv))
1991 		return;
1992 
1993 	if (!dev_priv->perf.metrics_kobj)
1994 		return;
1995 
1996 	i915_perf_unregister_sysfs_hsw(dev_priv);
1997 
1998 	kobject_put(dev_priv->perf.metrics_kobj);
1999 	dev_priv->perf.metrics_kobj = NULL;
2000 }
2001 
2002 static struct ctl_table oa_table[] = {
2003 	{
2004 	 .procname = "perf_stream_paranoid",
2005 	 .data = &i915_perf_stream_paranoid,
2006 	 .maxlen = sizeof(i915_perf_stream_paranoid),
2007 	 .mode = 0644,
2008 	 .proc_handler = proc_dointvec_minmax,
2009 	 .extra1 = &zero,
2010 	 .extra2 = &one,
2011 	 },
2012 	{
2013 	 .procname = "oa_max_sample_rate",
2014 	 .data = &i915_oa_max_sample_rate,
2015 	 .maxlen = sizeof(i915_oa_max_sample_rate),
2016 	 .mode = 0644,
2017 	 .proc_handler = proc_dointvec_minmax,
2018 	 .extra1 = &zero,
2019 	 .extra2 = &oa_sample_rate_hard_limit,
2020 	 },
2021 	{}
2022 };
2023 
2024 static struct ctl_table i915_root[] = {
2025 	{
2026 	 .procname = "i915",
2027 	 .maxlen = 0,
2028 	 .mode = 0555,
2029 	 .child = oa_table,
2030 	 },
2031 	{}
2032 };
2033 
2034 static struct ctl_table dev_root[] = {
2035 	{
2036 	 .procname = "dev",
2037 	 .maxlen = 0,
2038 	 .mode = 0555,
2039 	 .child = i915_root,
2040 	 },
2041 	{}
2042 };
2043 
2044 /**
2045  * i915_perf_init - initialize i915-perf state on module load
2046  * @dev_priv: i915 device instance
2047  *
2048  * Initializes i915-perf state without exposing anything to userspace.
2049  *
2050  * Note: i915-perf initialization is split into an 'init' and 'register'
2051  * phase with the i915_perf_register() exposing state to userspace.
2052  */
2053 void i915_perf_init(struct drm_i915_private *dev_priv)
2054 {
2055 	if (!IS_HASWELL(dev_priv))
2056 		return;
2057 
2058 	hrtimer_init(&dev_priv->perf.oa.poll_check_timer,
2059 		     CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2060 	dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb;
2061 	init_waitqueue_head(&dev_priv->perf.oa.poll_wq);
2062 
2063 	INIT_LIST_HEAD(&dev_priv->perf.streams);
2064 	mutex_init(&dev_priv->perf.lock);
2065 	spin_lock_init(&dev_priv->perf.hook_lock);
2066 
2067 	dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer;
2068 	dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set;
2069 	dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set;
2070 	dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable;
2071 	dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable;
2072 	dev_priv->perf.oa.ops.read = gen7_oa_read;
2073 	dev_priv->perf.oa.ops.oa_buffer_is_empty =
2074 		gen7_oa_buffer_is_empty_fop_unlocked;
2075 
2076 	dev_priv->perf.oa.timestamp_frequency = 12500000;
2077 
2078 	dev_priv->perf.oa.oa_formats = hsw_oa_formats;
2079 
2080 	dev_priv->perf.oa.n_builtin_sets =
2081 		i915_oa_n_builtin_metric_sets_hsw;
2082 
2083 	dev_priv->perf.sysctl_header = register_sysctl_table(dev_root);
2084 
2085 	dev_priv->perf.initialized = true;
2086 }
2087 
2088 /**
2089  * i915_perf_fini - Counter part to i915_perf_init()
2090  * @dev_priv: i915 device instance
2091  */
2092 void i915_perf_fini(struct drm_i915_private *dev_priv)
2093 {
2094 	if (!dev_priv->perf.initialized)
2095 		return;
2096 
2097 	unregister_sysctl_table(dev_priv->perf.sysctl_header);
2098 
2099 	memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops));
2100 	dev_priv->perf.initialized = false;
2101 }
2102