1 /* 2 * Copyright © 2015-2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Robert Bragg <robert@sixbynine.org> 25 */ 26 27 28 /** 29 * DOC: i915 Perf Overview 30 * 31 * Gen graphics supports a large number of performance counters that can help 32 * driver and application developers understand and optimize their use of the 33 * GPU. 34 * 35 * This i915 perf interface enables userspace to configure and open a file 36 * descriptor representing a stream of GPU metrics which can then be read() as 37 * a stream of sample records. 38 * 39 * The interface is particularly suited to exposing buffered metrics that are 40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU. 41 * 42 * Streams representing a single context are accessible to applications with a 43 * corresponding drm file descriptor, such that OpenGL can use the interface 44 * without special privileges. Access to system-wide metrics requires root 45 * privileges by default, unless changed via the dev.i915.perf_event_paranoid 46 * sysctl option. 47 * 48 */ 49 50 /** 51 * DOC: i915 Perf History and Comparison with Core Perf 52 * 53 * The interface was initially inspired by the core Perf infrastructure but 54 * some notable differences are: 55 * 56 * i915 perf file descriptors represent a "stream" instead of an "event"; where 57 * a perf event primarily corresponds to a single 64bit value, while a stream 58 * might sample sets of tightly-coupled counters, depending on the 59 * configuration. For example the Gen OA unit isn't designed to support 60 * orthogonal configurations of individual counters; it's configured for a set 61 * of related counters. Samples for an i915 perf stream capturing OA metrics 62 * will include a set of counter values packed in a compact HW specific format. 63 * The OA unit supports a number of different packing formats which can be 64 * selected by the user opening the stream. Perf has support for grouping 65 * events, but each event in the group is configured, validated and 66 * authenticated individually with separate system calls. 67 * 68 * i915 perf stream configurations are provided as an array of u64 (key,value) 69 * pairs, instead of a fixed struct with multiple miscellaneous config members, 70 * interleaved with event-type specific members. 71 * 72 * i915 perf doesn't support exposing metrics via an mmap'd circular buffer. 73 * The supported metrics are being written to memory by the GPU unsynchronized 74 * with the CPU, using HW specific packing formats for counter sets. Sometimes 75 * the constraints on HW configuration require reports to be filtered before it 76 * would be acceptable to expose them to unprivileged applications - to hide 77 * the metrics of other processes/contexts. For these use cases a read() based 78 * interface is a good fit, and provides an opportunity to filter data as it 79 * gets copied from the GPU mapped buffers to userspace buffers. 80 * 81 * 82 * Issues hit with first prototype based on Core Perf 83 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 84 * 85 * The first prototype of this driver was based on the core perf 86 * infrastructure, and while we did make that mostly work, with some changes to 87 * perf, we found we were breaking or working around too many assumptions baked 88 * into perf's currently cpu centric design. 89 * 90 * In the end we didn't see a clear benefit to making perf's implementation and 91 * interface more complex by changing design assumptions while we knew we still 92 * wouldn't be able to use any existing perf based userspace tools. 93 * 94 * Also considering the Gen specific nature of the Observability hardware and 95 * how userspace will sometimes need to combine i915 perf OA metrics with 96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're 97 * expecting the interface to be used by a platform specific userspace such as 98 * OpenGL or tools. This is to say; we aren't inherently missing out on having 99 * a standard vendor/architecture agnostic interface by not using perf. 100 * 101 * 102 * For posterity, in case we might re-visit trying to adapt core perf to be 103 * better suited to exposing i915 metrics these were the main pain points we 104 * hit: 105 * 106 * - The perf based OA PMU driver broke some significant design assumptions: 107 * 108 * Existing perf pmus are used for profiling work on a cpu and we were 109 * introducing the idea of _IS_DEVICE pmus with different security 110 * implications, the need to fake cpu-related data (such as user/kernel 111 * registers) to fit with perf's current design, and adding _DEVICE records 112 * as a way to forward device-specific status records. 113 * 114 * The OA unit writes reports of counters into a circular buffer, without 115 * involvement from the CPU, making our PMU driver the first of a kind. 116 * 117 * Given the way we were periodically forward data from the GPU-mapped, OA 118 * buffer to perf's buffer, those bursts of sample writes looked to perf like 119 * we were sampling too fast and so we had to subvert its throttling checks. 120 * 121 * Perf supports groups of counters and allows those to be read via 122 * transactions internally but transactions currently seem designed to be 123 * explicitly initiated from the cpu (say in response to a userspace read()) 124 * and while we could pull a report out of the OA buffer we can't 125 * trigger a report from the cpu on demand. 126 * 127 * Related to being report based; the OA counters are configured in HW as a 128 * set while perf generally expects counter configurations to be orthogonal. 129 * Although counters can be associated with a group leader as they are 130 * opened, there's no clear precedent for being able to provide group-wide 131 * configuration attributes (for example we want to let userspace choose the 132 * OA unit report format used to capture all counters in a set, or specify a 133 * GPU context to filter metrics on). We avoided using perf's grouping 134 * feature and forwarded OA reports to userspace via perf's 'raw' sample 135 * field. This suited our userspace well considering how coupled the counters 136 * are when dealing with normalizing. It would be inconvenient to split 137 * counters up into separate events, only to require userspace to recombine 138 * them. For Mesa it's also convenient to be forwarded raw, periodic reports 139 * for combining with the side-band raw reports it captures using 140 * MI_REPORT_PERF_COUNT commands. 141 * 142 * - As a side note on perf's grouping feature; there was also some concern 143 * that using PERF_FORMAT_GROUP as a way to pack together counter values 144 * would quite drastically inflate our sample sizes, which would likely 145 * lower the effective sampling resolutions we could use when the available 146 * memory bandwidth is limited. 147 * 148 * With the OA unit's report formats, counters are packed together as 32 149 * or 40bit values, with the largest report size being 256 bytes. 150 * 151 * PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a 152 * documented ordering to the values, implying PERF_FORMAT_ID must also be 153 * used to add a 64bit ID before each value; giving 16 bytes per counter. 154 * 155 * Related to counter orthogonality; we can't time share the OA unit, while 156 * event scheduling is a central design idea within perf for allowing 157 * userspace to open + enable more events than can be configured in HW at any 158 * one time. The OA unit is not designed to allow re-configuration while in 159 * use. We can't reconfigure the OA unit without losing internal OA unit 160 * state which we can't access explicitly to save and restore. Reconfiguring 161 * the OA unit is also relatively slow, involving ~100 register writes. From 162 * userspace Mesa also depends on a stable OA configuration when emitting 163 * MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be 164 * disabled while there are outstanding MI_RPC commands lest we hang the 165 * command streamer. 166 * 167 * The contents of sample records aren't extensible by device drivers (i.e. 168 * the sample_type bits). As an example; Sourab Gupta had been looking to 169 * attach GPU timestamps to our OA samples. We were shoehorning OA reports 170 * into sample records by using the 'raw' field, but it's tricky to pack more 171 * than one thing into this field because events/core.c currently only lets a 172 * pmu give a single raw data pointer plus len which will be copied into the 173 * ring buffer. To include more than the OA report we'd have to copy the 174 * report into an intermediate larger buffer. I'd been considering allowing a 175 * vector of data+len values to be specified for copying the raw data, but 176 * it felt like a kludge to being using the raw field for this purpose. 177 * 178 * - It felt like our perf based PMU was making some technical compromises 179 * just for the sake of using perf: 180 * 181 * perf_event_open() requires events to either relate to a pid or a specific 182 * cpu core, while our device pmu related to neither. Events opened with a 183 * pid will be automatically enabled/disabled according to the scheduling of 184 * that process - so not appropriate for us. When an event is related to a 185 * cpu id, perf ensures pmu methods will be invoked via an inter process 186 * interrupt on that core. To avoid invasive changes our userspace opened OA 187 * perf events for a specific cpu. This was workable but it meant the 188 * majority of the OA driver ran in atomic context, including all OA report 189 * forwarding, which wasn't really necessary in our case and seems to make 190 * our locking requirements somewhat complex as we handled the interaction 191 * with the rest of the i915 driver. 192 */ 193 194 #include <linux/anon_inodes.h> 195 #include <linux/sizes.h> 196 #include <linux/uuid.h> 197 198 #include "i915_drv.h" 199 #include "i915_oa_hsw.h" 200 #include "i915_oa_bdw.h" 201 #include "i915_oa_chv.h" 202 #include "i915_oa_sklgt2.h" 203 #include "i915_oa_sklgt3.h" 204 #include "i915_oa_sklgt4.h" 205 #include "i915_oa_bxt.h" 206 #include "i915_oa_kblgt2.h" 207 #include "i915_oa_kblgt3.h" 208 #include "i915_oa_glk.h" 209 #include "i915_oa_cflgt2.h" 210 #include "i915_oa_cflgt3.h" 211 #include "i915_oa_cnl.h" 212 #include "i915_oa_icl.h" 213 214 /* HW requires this to be a power of two, between 128k and 16M, though driver 215 * is currently generally designed assuming the largest 16M size is used such 216 * that the overflow cases are unlikely in normal operation. 217 */ 218 #define OA_BUFFER_SIZE SZ_16M 219 220 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) 221 222 /** 223 * DOC: OA Tail Pointer Race 224 * 225 * There's a HW race condition between OA unit tail pointer register updates and 226 * writes to memory whereby the tail pointer can sometimes get ahead of what's 227 * been written out to the OA buffer so far (in terms of what's visible to the 228 * CPU). 229 * 230 * Although this can be observed explicitly while copying reports to userspace 231 * by checking for a zeroed report-id field in tail reports, we want to account 232 * for this earlier, as part of the oa_buffer_check to avoid lots of redundant 233 * read() attempts. 234 * 235 * In effect we define a tail pointer for reading that lags the real tail 236 * pointer by at least %OA_TAIL_MARGIN_NSEC nanoseconds, which gives enough 237 * time for the corresponding reports to become visible to the CPU. 238 * 239 * To manage this we actually track two tail pointers: 240 * 1) An 'aging' tail with an associated timestamp that is tracked until we 241 * can trust the corresponding data is visible to the CPU; at which point 242 * it is considered 'aged'. 243 * 2) An 'aged' tail that can be used for read()ing. 244 * 245 * The two separate pointers let us decouple read()s from tail pointer aging. 246 * 247 * The tail pointers are checked and updated at a limited rate within a hrtimer 248 * callback (the same callback that is used for delivering EPOLLIN events) 249 * 250 * Initially the tails are marked invalid with %INVALID_TAIL_PTR which 251 * indicates that an updated tail pointer is needed. 252 * 253 * Most of the implementation details for this workaround are in 254 * oa_buffer_check_unlocked() and _append_oa_reports() 255 * 256 * Note for posterity: previously the driver used to define an effective tail 257 * pointer that lagged the real pointer by a 'tail margin' measured in bytes 258 * derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency. 259 * This was flawed considering that the OA unit may also automatically generate 260 * non-periodic reports (such as on context switch) or the OA unit may be 261 * enabled without any periodic sampling. 262 */ 263 #define OA_TAIL_MARGIN_NSEC 100000ULL 264 #define INVALID_TAIL_PTR 0xffffffff 265 266 /* frequency for checking whether the OA unit has written new reports to the 267 * circular OA buffer... 268 */ 269 #define POLL_FREQUENCY 200 270 #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) 271 272 /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */ 273 static int zero; 274 static int one = 1; 275 static u32 i915_perf_stream_paranoid = true; 276 277 /* The maximum exponent the hardware accepts is 63 (essentially it selects one 278 * of the 64bit timestamp bits to trigger reports from) but there's currently 279 * no known use case for sampling as infrequently as once per 47 thousand years. 280 * 281 * Since the timestamps included in OA reports are only 32bits it seems 282 * reasonable to limit the OA exponent where it's still possible to account for 283 * overflow in OA report timestamps. 284 */ 285 #define OA_EXPONENT_MAX 31 286 287 #define INVALID_CTX_ID 0xffffffff 288 289 /* On Gen8+ automatically triggered OA reports include a 'reason' field... */ 290 #define OAREPORT_REASON_MASK 0x3f 291 #define OAREPORT_REASON_SHIFT 19 292 #define OAREPORT_REASON_TIMER (1<<0) 293 #define OAREPORT_REASON_CTX_SWITCH (1<<3) 294 #define OAREPORT_REASON_CLK_RATIO (1<<5) 295 296 297 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate 298 * 299 * The highest sampling frequency we can theoretically program the OA unit 300 * with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell. 301 * 302 * Initialized just before we register the sysctl parameter. 303 */ 304 static int oa_sample_rate_hard_limit; 305 306 /* Theoretically we can program the OA unit to sample every 160ns but don't 307 * allow that by default unless root... 308 * 309 * The default threshold of 100000Hz is based on perf's similar 310 * kernel.perf_event_max_sample_rate sysctl parameter. 311 */ 312 static u32 i915_oa_max_sample_rate = 100000; 313 314 /* XXX: beware if future OA HW adds new report formats that the current 315 * code assumes all reports have a power-of-two size and ~(size - 1) can 316 * be used as a mask to align the OA tail pointer. 317 */ 318 static const struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { 319 [I915_OA_FORMAT_A13] = { 0, 64 }, 320 [I915_OA_FORMAT_A29] = { 1, 128 }, 321 [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, 322 /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ 323 [I915_OA_FORMAT_B4_C8] = { 4, 64 }, 324 [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, 325 [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, 326 [I915_OA_FORMAT_C4_B8] = { 7, 64 }, 327 }; 328 329 static const struct i915_oa_format gen8_plus_oa_formats[I915_OA_FORMAT_MAX] = { 330 [I915_OA_FORMAT_A12] = { 0, 64 }, 331 [I915_OA_FORMAT_A12_B8_C8] = { 2, 128 }, 332 [I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 }, 333 [I915_OA_FORMAT_C4_B8] = { 7, 64 }, 334 }; 335 336 #define SAMPLE_OA_REPORT (1<<0) 337 338 /** 339 * struct perf_open_properties - for validated properties given to open a stream 340 * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags 341 * @single_context: Whether a single or all gpu contexts should be monitored 342 * @ctx_handle: A gem ctx handle for use with @single_context 343 * @metrics_set: An ID for an OA unit metric set advertised via sysfs 344 * @oa_format: An OA unit HW report format 345 * @oa_periodic: Whether to enable periodic OA unit sampling 346 * @oa_period_exponent: The OA unit sampling period is derived from this 347 * 348 * As read_properties_unlocked() enumerates and validates the properties given 349 * to open a stream of metrics the configuration is built up in the structure 350 * which starts out zero initialized. 351 */ 352 struct perf_open_properties { 353 u32 sample_flags; 354 355 u64 single_context:1; 356 u64 ctx_handle; 357 358 /* OA sampling state */ 359 int metrics_set; 360 int oa_format; 361 bool oa_periodic; 362 int oa_period_exponent; 363 }; 364 365 static void free_oa_config(struct drm_i915_private *dev_priv, 366 struct i915_oa_config *oa_config) 367 { 368 if (!PTR_ERR(oa_config->flex_regs)) 369 kfree(oa_config->flex_regs); 370 if (!PTR_ERR(oa_config->b_counter_regs)) 371 kfree(oa_config->b_counter_regs); 372 if (!PTR_ERR(oa_config->mux_regs)) 373 kfree(oa_config->mux_regs); 374 kfree(oa_config); 375 } 376 377 static void put_oa_config(struct drm_i915_private *dev_priv, 378 struct i915_oa_config *oa_config) 379 { 380 if (!atomic_dec_and_test(&oa_config->ref_count)) 381 return; 382 383 free_oa_config(dev_priv, oa_config); 384 } 385 386 static int get_oa_config(struct drm_i915_private *dev_priv, 387 int metrics_set, 388 struct i915_oa_config **out_config) 389 { 390 int ret; 391 392 if (metrics_set == 1) { 393 *out_config = &dev_priv->perf.oa.test_config; 394 atomic_inc(&dev_priv->perf.oa.test_config.ref_count); 395 return 0; 396 } 397 398 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); 399 if (ret) 400 return ret; 401 402 *out_config = idr_find(&dev_priv->perf.metrics_idr, metrics_set); 403 if (!*out_config) 404 ret = -EINVAL; 405 else 406 atomic_inc(&(*out_config)->ref_count); 407 408 mutex_unlock(&dev_priv->perf.metrics_lock); 409 410 return ret; 411 } 412 413 static u32 gen8_oa_hw_tail_read(struct drm_i915_private *dev_priv) 414 { 415 return I915_READ(GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK; 416 } 417 418 static u32 gen7_oa_hw_tail_read(struct drm_i915_private *dev_priv) 419 { 420 u32 oastatus1 = I915_READ(GEN7_OASTATUS1); 421 422 return oastatus1 & GEN7_OASTATUS1_TAIL_MASK; 423 } 424 425 /** 426 * oa_buffer_check_unlocked - check for data and update tail ptr state 427 * @dev_priv: i915 device instance 428 * 429 * This is either called via fops (for blocking reads in user ctx) or the poll 430 * check hrtimer (atomic ctx) to check the OA buffer tail pointer and check 431 * if there is data available for userspace to read. 432 * 433 * This function is central to providing a workaround for the OA unit tail 434 * pointer having a race with respect to what data is visible to the CPU. 435 * It is responsible for reading tail pointers from the hardware and giving 436 * the pointers time to 'age' before they are made available for reading. 437 * (See description of OA_TAIL_MARGIN_NSEC above for further details.) 438 * 439 * Besides returning true when there is data available to read() this function 440 * also has the side effect of updating the oa_buffer.tails[], .aging_timestamp 441 * and .aged_tail_idx state used for reading. 442 * 443 * Note: It's safe to read OA config state here unlocked, assuming that this is 444 * only called while the stream is enabled, while the global OA configuration 445 * can't be modified. 446 * 447 * Returns: %true if the OA buffer contains data, else %false 448 */ 449 static bool oa_buffer_check_unlocked(struct drm_i915_private *dev_priv) 450 { 451 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 452 unsigned long flags; 453 unsigned int aged_idx; 454 u32 head, hw_tail, aged_tail, aging_tail; 455 u64 now; 456 457 /* We have to consider the (unlikely) possibility that read() errors 458 * could result in an OA buffer reset which might reset the head, 459 * tails[] and aged_tail state. 460 */ 461 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 462 463 /* NB: The head we observe here might effectively be a little out of 464 * date (between head and tails[aged_idx].offset if there is currently 465 * a read() in progress. 466 */ 467 head = dev_priv->perf.oa.oa_buffer.head; 468 469 aged_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; 470 aged_tail = dev_priv->perf.oa.oa_buffer.tails[aged_idx].offset; 471 aging_tail = dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset; 472 473 hw_tail = dev_priv->perf.oa.ops.oa_hw_tail_read(dev_priv); 474 475 /* The tail pointer increases in 64 byte increments, 476 * not in report_size steps... 477 */ 478 hw_tail &= ~(report_size - 1); 479 480 now = ktime_get_mono_fast_ns(); 481 482 /* Update the aged tail 483 * 484 * Flip the tail pointer available for read()s once the aging tail is 485 * old enough to trust that the corresponding data will be visible to 486 * the CPU... 487 * 488 * Do this before updating the aging pointer in case we may be able to 489 * immediately start aging a new pointer too (if new data has become 490 * available) without needing to wait for a later hrtimer callback. 491 */ 492 if (aging_tail != INVALID_TAIL_PTR && 493 ((now - dev_priv->perf.oa.oa_buffer.aging_timestamp) > 494 OA_TAIL_MARGIN_NSEC)) { 495 496 aged_idx ^= 1; 497 dev_priv->perf.oa.oa_buffer.aged_tail_idx = aged_idx; 498 499 aged_tail = aging_tail; 500 501 /* Mark that we need a new pointer to start aging... */ 502 dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR; 503 aging_tail = INVALID_TAIL_PTR; 504 } 505 506 /* Update the aging tail 507 * 508 * We throttle aging tail updates until we have a new tail that 509 * represents >= one report more data than is already available for 510 * reading. This ensures there will be enough data for a successful 511 * read once this new pointer has aged and ensures we will give the new 512 * pointer time to age. 513 */ 514 if (aging_tail == INVALID_TAIL_PTR && 515 (aged_tail == INVALID_TAIL_PTR || 516 OA_TAKEN(hw_tail, aged_tail) >= report_size)) { 517 struct i915_vma *vma = dev_priv->perf.oa.oa_buffer.vma; 518 u32 gtt_offset = i915_ggtt_offset(vma); 519 520 /* Be paranoid and do a bounds check on the pointer read back 521 * from hardware, just in case some spurious hardware condition 522 * could put the tail out of bounds... 523 */ 524 if (hw_tail >= gtt_offset && 525 hw_tail < (gtt_offset + OA_BUFFER_SIZE)) { 526 dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = 527 aging_tail = hw_tail; 528 dev_priv->perf.oa.oa_buffer.aging_timestamp = now; 529 } else { 530 DRM_ERROR("Ignoring spurious out of range OA buffer tail pointer = %u\n", 531 hw_tail); 532 } 533 } 534 535 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 536 537 return aged_tail == INVALID_TAIL_PTR ? 538 false : OA_TAKEN(aged_tail, head) >= report_size; 539 } 540 541 /** 542 * append_oa_status - Appends a status record to a userspace read() buffer. 543 * @stream: An i915-perf stream opened for OA metrics 544 * @buf: destination buffer given by userspace 545 * @count: the number of bytes userspace wants to read 546 * @offset: (inout): the current position for writing into @buf 547 * @type: The kind of status to report to userspace 548 * 549 * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`) 550 * into the userspace read() buffer. 551 * 552 * The @buf @offset will only be updated on success. 553 * 554 * Returns: 0 on success, negative error code on failure. 555 */ 556 static int append_oa_status(struct i915_perf_stream *stream, 557 char __user *buf, 558 size_t count, 559 size_t *offset, 560 enum drm_i915_perf_record_type type) 561 { 562 struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; 563 564 if ((count - *offset) < header.size) 565 return -ENOSPC; 566 567 if (copy_to_user(buf + *offset, &header, sizeof(header))) 568 return -EFAULT; 569 570 (*offset) += header.size; 571 572 return 0; 573 } 574 575 /** 576 * append_oa_sample - Copies single OA report into userspace read() buffer. 577 * @stream: An i915-perf stream opened for OA metrics 578 * @buf: destination buffer given by userspace 579 * @count: the number of bytes userspace wants to read 580 * @offset: (inout): the current position for writing into @buf 581 * @report: A single OA report to (optionally) include as part of the sample 582 * 583 * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*` 584 * properties when opening a stream, tracked as `stream->sample_flags`. This 585 * function copies the requested components of a single sample to the given 586 * read() @buf. 587 * 588 * The @buf @offset will only be updated on success. 589 * 590 * Returns: 0 on success, negative error code on failure. 591 */ 592 static int append_oa_sample(struct i915_perf_stream *stream, 593 char __user *buf, 594 size_t count, 595 size_t *offset, 596 const u8 *report) 597 { 598 struct drm_i915_private *dev_priv = stream->dev_priv; 599 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 600 struct drm_i915_perf_record_header header; 601 u32 sample_flags = stream->sample_flags; 602 603 header.type = DRM_I915_PERF_RECORD_SAMPLE; 604 header.pad = 0; 605 header.size = stream->sample_size; 606 607 if ((count - *offset) < header.size) 608 return -ENOSPC; 609 610 buf += *offset; 611 if (copy_to_user(buf, &header, sizeof(header))) 612 return -EFAULT; 613 buf += sizeof(header); 614 615 if (sample_flags & SAMPLE_OA_REPORT) { 616 if (copy_to_user(buf, report, report_size)) 617 return -EFAULT; 618 } 619 620 (*offset) += header.size; 621 622 return 0; 623 } 624 625 /** 626 * Copies all buffered OA reports into userspace read() buffer. 627 * @stream: An i915-perf stream opened for OA metrics 628 * @buf: destination buffer given by userspace 629 * @count: the number of bytes userspace wants to read 630 * @offset: (inout): the current position for writing into @buf 631 * 632 * Notably any error condition resulting in a short read (-%ENOSPC or 633 * -%EFAULT) will be returned even though one or more records may 634 * have been successfully copied. In this case it's up to the caller 635 * to decide if the error should be squashed before returning to 636 * userspace. 637 * 638 * Note: reports are consumed from the head, and appended to the 639 * tail, so the tail chases the head?... If you think that's mad 640 * and back-to-front you're not alone, but this follows the 641 * Gen PRM naming convention. 642 * 643 * Returns: 0 on success, negative error code on failure. 644 */ 645 static int gen8_append_oa_reports(struct i915_perf_stream *stream, 646 char __user *buf, 647 size_t count, 648 size_t *offset) 649 { 650 struct drm_i915_private *dev_priv = stream->dev_priv; 651 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 652 u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; 653 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 654 u32 mask = (OA_BUFFER_SIZE - 1); 655 size_t start_offset = *offset; 656 unsigned long flags; 657 unsigned int aged_tail_idx; 658 u32 head, tail; 659 u32 taken; 660 int ret = 0; 661 662 if (WARN_ON(!stream->enabled)) 663 return -EIO; 664 665 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 666 667 head = dev_priv->perf.oa.oa_buffer.head; 668 aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; 669 tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; 670 671 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 672 673 /* 674 * An invalid tail pointer here means we're still waiting for the poll 675 * hrtimer callback to give us a pointer 676 */ 677 if (tail == INVALID_TAIL_PTR) 678 return -EAGAIN; 679 680 /* 681 * NB: oa_buffer.head/tail include the gtt_offset which we don't want 682 * while indexing relative to oa_buf_base. 683 */ 684 head -= gtt_offset; 685 tail -= gtt_offset; 686 687 /* 688 * An out of bounds or misaligned head or tail pointer implies a driver 689 * bug since we validate + align the tail pointers we read from the 690 * hardware and we are in full control of the head pointer which should 691 * only be incremented by multiples of the report size (notably also 692 * all a power of two). 693 */ 694 if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size || 695 tail > OA_BUFFER_SIZE || tail % report_size, 696 "Inconsistent OA buffer pointers: head = %u, tail = %u\n", 697 head, tail)) 698 return -EIO; 699 700 701 for (/* none */; 702 (taken = OA_TAKEN(tail, head)); 703 head = (head + report_size) & mask) { 704 u8 *report = oa_buf_base + head; 705 u32 *report32 = (void *)report; 706 u32 ctx_id; 707 u32 reason; 708 709 /* 710 * All the report sizes factor neatly into the buffer 711 * size so we never expect to see a report split 712 * between the beginning and end of the buffer. 713 * 714 * Given the initial alignment check a misalignment 715 * here would imply a driver bug that would result 716 * in an overrun. 717 */ 718 if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { 719 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); 720 break; 721 } 722 723 /* 724 * The reason field includes flags identifying what 725 * triggered this specific report (mostly timer 726 * triggered or e.g. due to a context switch). 727 * 728 * This field is never expected to be zero so we can 729 * check that the report isn't invalid before copying 730 * it to userspace... 731 */ 732 reason = ((report32[0] >> OAREPORT_REASON_SHIFT) & 733 OAREPORT_REASON_MASK); 734 if (reason == 0) { 735 if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) 736 DRM_NOTE("Skipping spurious, invalid OA report\n"); 737 continue; 738 } 739 740 ctx_id = report32[2] & dev_priv->perf.oa.specific_ctx_id_mask; 741 742 /* 743 * Squash whatever is in the CTX_ID field if it's marked as 744 * invalid to be sure we avoid false-positive, single-context 745 * filtering below... 746 * 747 * Note: that we don't clear the valid_ctx_bit so userspace can 748 * understand that the ID has been squashed by the kernel. 749 */ 750 if (!(report32[0] & dev_priv->perf.oa.gen8_valid_ctx_bit)) 751 ctx_id = report32[2] = INVALID_CTX_ID; 752 753 /* 754 * NB: For Gen 8 the OA unit no longer supports clock gating 755 * off for a specific context and the kernel can't securely 756 * stop the counters from updating as system-wide / global 757 * values. 758 * 759 * Automatic reports now include a context ID so reports can be 760 * filtered on the cpu but it's not worth trying to 761 * automatically subtract/hide counter progress for other 762 * contexts while filtering since we can't stop userspace 763 * issuing MI_REPORT_PERF_COUNT commands which would still 764 * provide a side-band view of the real values. 765 * 766 * To allow userspace (such as Mesa/GL_INTEL_performance_query) 767 * to normalize counters for a single filtered context then it 768 * needs be forwarded bookend context-switch reports so that it 769 * can track switches in between MI_REPORT_PERF_COUNT commands 770 * and can itself subtract/ignore the progress of counters 771 * associated with other contexts. Note that the hardware 772 * automatically triggers reports when switching to a new 773 * context which are tagged with the ID of the newly active 774 * context. To avoid the complexity (and likely fragility) of 775 * reading ahead while parsing reports to try and minimize 776 * forwarding redundant context switch reports (i.e. between 777 * other, unrelated contexts) we simply elect to forward them 778 * all. 779 * 780 * We don't rely solely on the reason field to identify context 781 * switches since it's not-uncommon for periodic samples to 782 * identify a switch before any 'context switch' report. 783 */ 784 if (!dev_priv->perf.oa.exclusive_stream->ctx || 785 dev_priv->perf.oa.specific_ctx_id == ctx_id || 786 (dev_priv->perf.oa.oa_buffer.last_ctx_id == 787 dev_priv->perf.oa.specific_ctx_id) || 788 reason & OAREPORT_REASON_CTX_SWITCH) { 789 790 /* 791 * While filtering for a single context we avoid 792 * leaking the IDs of other contexts. 793 */ 794 if (dev_priv->perf.oa.exclusive_stream->ctx && 795 dev_priv->perf.oa.specific_ctx_id != ctx_id) { 796 report32[2] = INVALID_CTX_ID; 797 } 798 799 ret = append_oa_sample(stream, buf, count, offset, 800 report); 801 if (ret) 802 break; 803 804 dev_priv->perf.oa.oa_buffer.last_ctx_id = ctx_id; 805 } 806 807 /* 808 * The above reason field sanity check is based on 809 * the assumption that the OA buffer is initially 810 * zeroed and we reset the field after copying so the 811 * check is still meaningful once old reports start 812 * being overwritten. 813 */ 814 report32[0] = 0; 815 } 816 817 if (start_offset != *offset) { 818 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 819 820 /* 821 * We removed the gtt_offset for the copy loop above, indexing 822 * relative to oa_buf_base so put back here... 823 */ 824 head += gtt_offset; 825 826 I915_WRITE(GEN8_OAHEADPTR, head & GEN8_OAHEADPTR_MASK); 827 dev_priv->perf.oa.oa_buffer.head = head; 828 829 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 830 } 831 832 return ret; 833 } 834 835 /** 836 * gen8_oa_read - copy status records then buffered OA reports 837 * @stream: An i915-perf stream opened for OA metrics 838 * @buf: destination buffer given by userspace 839 * @count: the number of bytes userspace wants to read 840 * @offset: (inout): the current position for writing into @buf 841 * 842 * Checks OA unit status registers and if necessary appends corresponding 843 * status records for userspace (such as for a buffer full condition) and then 844 * initiate appending any buffered OA reports. 845 * 846 * Updates @offset according to the number of bytes successfully copied into 847 * the userspace buffer. 848 * 849 * NB: some data may be successfully copied to the userspace buffer 850 * even if an error is returned, and this is reflected in the 851 * updated @offset. 852 * 853 * Returns: zero on success or a negative error code 854 */ 855 static int gen8_oa_read(struct i915_perf_stream *stream, 856 char __user *buf, 857 size_t count, 858 size_t *offset) 859 { 860 struct drm_i915_private *dev_priv = stream->dev_priv; 861 u32 oastatus; 862 int ret; 863 864 if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) 865 return -EIO; 866 867 oastatus = I915_READ(GEN8_OASTATUS); 868 869 /* 870 * We treat OABUFFER_OVERFLOW as a significant error: 871 * 872 * Although theoretically we could handle this more gracefully 873 * sometimes, some Gens don't correctly suppress certain 874 * automatically triggered reports in this condition and so we 875 * have to assume that old reports are now being trampled 876 * over. 877 * 878 * Considering how we don't currently give userspace control 879 * over the OA buffer size and always configure a large 16MB 880 * buffer, then a buffer overflow does anyway likely indicate 881 * that something has gone quite badly wrong. 882 */ 883 if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) { 884 ret = append_oa_status(stream, buf, count, offset, 885 DRM_I915_PERF_RECORD_OA_BUFFER_LOST); 886 if (ret) 887 return ret; 888 889 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", 890 dev_priv->perf.oa.period_exponent); 891 892 dev_priv->perf.oa.ops.oa_disable(dev_priv); 893 dev_priv->perf.oa.ops.oa_enable(dev_priv); 894 895 /* 896 * Note: .oa_enable() is expected to re-init the oabuffer and 897 * reset GEN8_OASTATUS for us 898 */ 899 oastatus = I915_READ(GEN8_OASTATUS); 900 } 901 902 if (oastatus & GEN8_OASTATUS_REPORT_LOST) { 903 ret = append_oa_status(stream, buf, count, offset, 904 DRM_I915_PERF_RECORD_OA_REPORT_LOST); 905 if (ret) 906 return ret; 907 I915_WRITE(GEN8_OASTATUS, 908 oastatus & ~GEN8_OASTATUS_REPORT_LOST); 909 } 910 911 return gen8_append_oa_reports(stream, buf, count, offset); 912 } 913 914 /** 915 * Copies all buffered OA reports into userspace read() buffer. 916 * @stream: An i915-perf stream opened for OA metrics 917 * @buf: destination buffer given by userspace 918 * @count: the number of bytes userspace wants to read 919 * @offset: (inout): the current position for writing into @buf 920 * 921 * Notably any error condition resulting in a short read (-%ENOSPC or 922 * -%EFAULT) will be returned even though one or more records may 923 * have been successfully copied. In this case it's up to the caller 924 * to decide if the error should be squashed before returning to 925 * userspace. 926 * 927 * Note: reports are consumed from the head, and appended to the 928 * tail, so the tail chases the head?... If you think that's mad 929 * and back-to-front you're not alone, but this follows the 930 * Gen PRM naming convention. 931 * 932 * Returns: 0 on success, negative error code on failure. 933 */ 934 static int gen7_append_oa_reports(struct i915_perf_stream *stream, 935 char __user *buf, 936 size_t count, 937 size_t *offset) 938 { 939 struct drm_i915_private *dev_priv = stream->dev_priv; 940 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 941 u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; 942 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 943 u32 mask = (OA_BUFFER_SIZE - 1); 944 size_t start_offset = *offset; 945 unsigned long flags; 946 unsigned int aged_tail_idx; 947 u32 head, tail; 948 u32 taken; 949 int ret = 0; 950 951 if (WARN_ON(!stream->enabled)) 952 return -EIO; 953 954 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 955 956 head = dev_priv->perf.oa.oa_buffer.head; 957 aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; 958 tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; 959 960 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 961 962 /* An invalid tail pointer here means we're still waiting for the poll 963 * hrtimer callback to give us a pointer 964 */ 965 if (tail == INVALID_TAIL_PTR) 966 return -EAGAIN; 967 968 /* NB: oa_buffer.head/tail include the gtt_offset which we don't want 969 * while indexing relative to oa_buf_base. 970 */ 971 head -= gtt_offset; 972 tail -= gtt_offset; 973 974 /* An out of bounds or misaligned head or tail pointer implies a driver 975 * bug since we validate + align the tail pointers we read from the 976 * hardware and we are in full control of the head pointer which should 977 * only be incremented by multiples of the report size (notably also 978 * all a power of two). 979 */ 980 if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size || 981 tail > OA_BUFFER_SIZE || tail % report_size, 982 "Inconsistent OA buffer pointers: head = %u, tail = %u\n", 983 head, tail)) 984 return -EIO; 985 986 987 for (/* none */; 988 (taken = OA_TAKEN(tail, head)); 989 head = (head + report_size) & mask) { 990 u8 *report = oa_buf_base + head; 991 u32 *report32 = (void *)report; 992 993 /* All the report sizes factor neatly into the buffer 994 * size so we never expect to see a report split 995 * between the beginning and end of the buffer. 996 * 997 * Given the initial alignment check a misalignment 998 * here would imply a driver bug that would result 999 * in an overrun. 1000 */ 1001 if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { 1002 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); 1003 break; 1004 } 1005 1006 /* The report-ID field for periodic samples includes 1007 * some undocumented flags related to what triggered 1008 * the report and is never expected to be zero so we 1009 * can check that the report isn't invalid before 1010 * copying it to userspace... 1011 */ 1012 if (report32[0] == 0) { 1013 if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) 1014 DRM_NOTE("Skipping spurious, invalid OA report\n"); 1015 continue; 1016 } 1017 1018 ret = append_oa_sample(stream, buf, count, offset, report); 1019 if (ret) 1020 break; 1021 1022 /* The above report-id field sanity check is based on 1023 * the assumption that the OA buffer is initially 1024 * zeroed and we reset the field after copying so the 1025 * check is still meaningful once old reports start 1026 * being overwritten. 1027 */ 1028 report32[0] = 0; 1029 } 1030 1031 if (start_offset != *offset) { 1032 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1033 1034 /* We removed the gtt_offset for the copy loop above, indexing 1035 * relative to oa_buf_base so put back here... 1036 */ 1037 head += gtt_offset; 1038 1039 I915_WRITE(GEN7_OASTATUS2, 1040 ((head & GEN7_OASTATUS2_HEAD_MASK) | 1041 GEN7_OASTATUS2_MEM_SELECT_GGTT)); 1042 dev_priv->perf.oa.oa_buffer.head = head; 1043 1044 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1045 } 1046 1047 return ret; 1048 } 1049 1050 /** 1051 * gen7_oa_read - copy status records then buffered OA reports 1052 * @stream: An i915-perf stream opened for OA metrics 1053 * @buf: destination buffer given by userspace 1054 * @count: the number of bytes userspace wants to read 1055 * @offset: (inout): the current position for writing into @buf 1056 * 1057 * Checks Gen 7 specific OA unit status registers and if necessary appends 1058 * corresponding status records for userspace (such as for a buffer full 1059 * condition) and then initiate appending any buffered OA reports. 1060 * 1061 * Updates @offset according to the number of bytes successfully copied into 1062 * the userspace buffer. 1063 * 1064 * Returns: zero on success or a negative error code 1065 */ 1066 static int gen7_oa_read(struct i915_perf_stream *stream, 1067 char __user *buf, 1068 size_t count, 1069 size_t *offset) 1070 { 1071 struct drm_i915_private *dev_priv = stream->dev_priv; 1072 u32 oastatus1; 1073 int ret; 1074 1075 if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) 1076 return -EIO; 1077 1078 oastatus1 = I915_READ(GEN7_OASTATUS1); 1079 1080 /* XXX: On Haswell we don't have a safe way to clear oastatus1 1081 * bits while the OA unit is enabled (while the tail pointer 1082 * may be updated asynchronously) so we ignore status bits 1083 * that have already been reported to userspace. 1084 */ 1085 oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; 1086 1087 /* We treat OABUFFER_OVERFLOW as a significant error: 1088 * 1089 * - The status can be interpreted to mean that the buffer is 1090 * currently full (with a higher precedence than OA_TAKEN() 1091 * which will start to report a near-empty buffer after an 1092 * overflow) but it's awkward that we can't clear the status 1093 * on Haswell, so without a reset we won't be able to catch 1094 * the state again. 1095 * 1096 * - Since it also implies the HW has started overwriting old 1097 * reports it may also affect our sanity checks for invalid 1098 * reports when copying to userspace that assume new reports 1099 * are being written to cleared memory. 1100 * 1101 * - In the future we may want to introduce a flight recorder 1102 * mode where the driver will automatically maintain a safe 1103 * guard band between head/tail, avoiding this overflow 1104 * condition, but we avoid the added driver complexity for 1105 * now. 1106 */ 1107 if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { 1108 ret = append_oa_status(stream, buf, count, offset, 1109 DRM_I915_PERF_RECORD_OA_BUFFER_LOST); 1110 if (ret) 1111 return ret; 1112 1113 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", 1114 dev_priv->perf.oa.period_exponent); 1115 1116 dev_priv->perf.oa.ops.oa_disable(dev_priv); 1117 dev_priv->perf.oa.ops.oa_enable(dev_priv); 1118 1119 oastatus1 = I915_READ(GEN7_OASTATUS1); 1120 } 1121 1122 if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { 1123 ret = append_oa_status(stream, buf, count, offset, 1124 DRM_I915_PERF_RECORD_OA_REPORT_LOST); 1125 if (ret) 1126 return ret; 1127 dev_priv->perf.oa.gen7_latched_oastatus1 |= 1128 GEN7_OASTATUS1_REPORT_LOST; 1129 } 1130 1131 return gen7_append_oa_reports(stream, buf, count, offset); 1132 } 1133 1134 /** 1135 * i915_oa_wait_unlocked - handles blocking IO until OA data available 1136 * @stream: An i915-perf stream opened for OA metrics 1137 * 1138 * Called when userspace tries to read() from a blocking stream FD opened 1139 * for OA metrics. It waits until the hrtimer callback finds a non-empty 1140 * OA buffer and wakes us. 1141 * 1142 * Note: it's acceptable to have this return with some false positives 1143 * since any subsequent read handling will return -EAGAIN if there isn't 1144 * really data ready for userspace yet. 1145 * 1146 * Returns: zero on success or a negative error code 1147 */ 1148 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) 1149 { 1150 struct drm_i915_private *dev_priv = stream->dev_priv; 1151 1152 /* We would wait indefinitely if periodic sampling is not enabled */ 1153 if (!dev_priv->perf.oa.periodic) 1154 return -EIO; 1155 1156 return wait_event_interruptible(dev_priv->perf.oa.poll_wq, 1157 oa_buffer_check_unlocked(dev_priv)); 1158 } 1159 1160 /** 1161 * i915_oa_poll_wait - call poll_wait() for an OA stream poll() 1162 * @stream: An i915-perf stream opened for OA metrics 1163 * @file: An i915 perf stream file 1164 * @wait: poll() state table 1165 * 1166 * For handling userspace polling on an i915 perf stream opened for OA metrics, 1167 * this starts a poll_wait with the wait queue that our hrtimer callback wakes 1168 * when it sees data ready to read in the circular OA buffer. 1169 */ 1170 static void i915_oa_poll_wait(struct i915_perf_stream *stream, 1171 struct file *file, 1172 poll_table *wait) 1173 { 1174 struct drm_i915_private *dev_priv = stream->dev_priv; 1175 1176 poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); 1177 } 1178 1179 /** 1180 * i915_oa_read - just calls through to &i915_oa_ops->read 1181 * @stream: An i915-perf stream opened for OA metrics 1182 * @buf: destination buffer given by userspace 1183 * @count: the number of bytes userspace wants to read 1184 * @offset: (inout): the current position for writing into @buf 1185 * 1186 * Updates @offset according to the number of bytes successfully copied into 1187 * the userspace buffer. 1188 * 1189 * Returns: zero on success or a negative error code 1190 */ 1191 static int i915_oa_read(struct i915_perf_stream *stream, 1192 char __user *buf, 1193 size_t count, 1194 size_t *offset) 1195 { 1196 struct drm_i915_private *dev_priv = stream->dev_priv; 1197 1198 return dev_priv->perf.oa.ops.read(stream, buf, count, offset); 1199 } 1200 1201 static struct intel_context *oa_pin_context(struct drm_i915_private *i915, 1202 struct i915_gem_context *ctx) 1203 { 1204 struct intel_engine_cs *engine = i915->engine[RCS]; 1205 struct intel_context *ce; 1206 int ret; 1207 1208 ret = i915_mutex_lock_interruptible(&i915->drm); 1209 if (ret) 1210 return ERR_PTR(ret); 1211 1212 /* 1213 * As the ID is the gtt offset of the context's vma we 1214 * pin the vma to ensure the ID remains fixed. 1215 * 1216 * NB: implied RCS engine... 1217 */ 1218 ce = intel_context_pin(ctx, engine); 1219 mutex_unlock(&i915->drm.struct_mutex); 1220 if (IS_ERR(ce)) 1221 return ce; 1222 1223 i915->perf.oa.pinned_ctx = ce; 1224 1225 return ce; 1226 } 1227 1228 /** 1229 * oa_get_render_ctx_id - determine and hold ctx hw id 1230 * @stream: An i915-perf stream opened for OA metrics 1231 * 1232 * Determine the render context hw id, and ensure it remains fixed for the 1233 * lifetime of the stream. This ensures that we don't have to worry about 1234 * updating the context ID in OACONTROL on the fly. 1235 * 1236 * Returns: zero on success or a negative error code 1237 */ 1238 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) 1239 { 1240 struct drm_i915_private *i915 = stream->dev_priv; 1241 struct intel_context *ce; 1242 1243 ce = oa_pin_context(i915, stream->ctx); 1244 if (IS_ERR(ce)) 1245 return PTR_ERR(ce); 1246 1247 switch (INTEL_GEN(i915)) { 1248 case 7: { 1249 /* 1250 * On Haswell we don't do any post processing of the reports 1251 * and don't need to use the mask. 1252 */ 1253 i915->perf.oa.specific_ctx_id = i915_ggtt_offset(ce->state); 1254 i915->perf.oa.specific_ctx_id_mask = 0; 1255 break; 1256 } 1257 1258 case 8: 1259 case 9: 1260 case 10: 1261 if (USES_GUC_SUBMISSION(i915)) { 1262 /* 1263 * When using GuC, the context descriptor we write in 1264 * i915 is read by GuC and rewritten before it's 1265 * actually written into the hardware. The LRCA is 1266 * what is put into the context id field of the 1267 * context descriptor by GuC. Because it's aligned to 1268 * a page, the lower 12bits are always at 0 and 1269 * dropped by GuC. They won't be part of the context 1270 * ID in the OA reports, so squash those lower bits. 1271 */ 1272 i915->perf.oa.specific_ctx_id = 1273 lower_32_bits(ce->lrc_desc) >> 12; 1274 1275 /* 1276 * GuC uses the top bit to signal proxy submission, so 1277 * ignore that bit. 1278 */ 1279 i915->perf.oa.specific_ctx_id_mask = 1280 (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; 1281 } else { 1282 i915->perf.oa.specific_ctx_id_mask = 1283 (1U << GEN8_CTX_ID_WIDTH) - 1; 1284 i915->perf.oa.specific_ctx_id = 1285 upper_32_bits(ce->lrc_desc); 1286 i915->perf.oa.specific_ctx_id &= 1287 i915->perf.oa.specific_ctx_id_mask; 1288 } 1289 break; 1290 1291 case 11: { 1292 i915->perf.oa.specific_ctx_id_mask = 1293 ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32) | 1294 ((1U << GEN11_ENGINE_INSTANCE_WIDTH) - 1) << (GEN11_ENGINE_INSTANCE_SHIFT - 32) | 1295 ((1 << GEN11_ENGINE_CLASS_WIDTH) - 1) << (GEN11_ENGINE_CLASS_SHIFT - 32); 1296 i915->perf.oa.specific_ctx_id = upper_32_bits(ce->lrc_desc); 1297 i915->perf.oa.specific_ctx_id &= 1298 i915->perf.oa.specific_ctx_id_mask; 1299 break; 1300 } 1301 1302 default: 1303 MISSING_CASE(INTEL_GEN(i915)); 1304 } 1305 1306 DRM_DEBUG_DRIVER("filtering on ctx_id=0x%x ctx_id_mask=0x%x\n", 1307 i915->perf.oa.specific_ctx_id, 1308 i915->perf.oa.specific_ctx_id_mask); 1309 1310 return 0; 1311 } 1312 1313 /** 1314 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold 1315 * @stream: An i915-perf stream opened for OA metrics 1316 * 1317 * In case anything needed doing to ensure the context HW ID would remain valid 1318 * for the lifetime of the stream, then that can be undone here. 1319 */ 1320 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) 1321 { 1322 struct drm_i915_private *dev_priv = stream->dev_priv; 1323 struct intel_context *ce; 1324 1325 dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; 1326 dev_priv->perf.oa.specific_ctx_id_mask = 0; 1327 1328 ce = fetch_and_zero(&dev_priv->perf.oa.pinned_ctx); 1329 if (ce) { 1330 mutex_lock(&dev_priv->drm.struct_mutex); 1331 intel_context_unpin(ce); 1332 mutex_unlock(&dev_priv->drm.struct_mutex); 1333 } 1334 } 1335 1336 static void 1337 free_oa_buffer(struct drm_i915_private *i915) 1338 { 1339 mutex_lock(&i915->drm.struct_mutex); 1340 1341 i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); 1342 i915_vma_unpin(i915->perf.oa.oa_buffer.vma); 1343 i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); 1344 1345 i915->perf.oa.oa_buffer.vma = NULL; 1346 i915->perf.oa.oa_buffer.vaddr = NULL; 1347 1348 mutex_unlock(&i915->drm.struct_mutex); 1349 } 1350 1351 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) 1352 { 1353 struct drm_i915_private *dev_priv = stream->dev_priv; 1354 1355 BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); 1356 1357 /* 1358 * Unset exclusive_stream first, it will be checked while disabling 1359 * the metric set on gen8+. 1360 */ 1361 mutex_lock(&dev_priv->drm.struct_mutex); 1362 dev_priv->perf.oa.exclusive_stream = NULL; 1363 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); 1364 mutex_unlock(&dev_priv->drm.struct_mutex); 1365 1366 free_oa_buffer(dev_priv); 1367 1368 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1369 intel_runtime_pm_put(dev_priv); 1370 1371 if (stream->ctx) 1372 oa_put_render_ctx_id(stream); 1373 1374 put_oa_config(dev_priv, stream->oa_config); 1375 1376 if (dev_priv->perf.oa.spurious_report_rs.missed) { 1377 DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n", 1378 dev_priv->perf.oa.spurious_report_rs.missed); 1379 } 1380 } 1381 1382 static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) 1383 { 1384 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 1385 unsigned long flags; 1386 1387 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1388 1389 /* Pre-DevBDW: OABUFFER must be set with counters off, 1390 * before OASTATUS1, but after OASTATUS2 1391 */ 1392 I915_WRITE(GEN7_OASTATUS2, 1393 gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT); /* head */ 1394 dev_priv->perf.oa.oa_buffer.head = gtt_offset; 1395 1396 I915_WRITE(GEN7_OABUFFER, gtt_offset); 1397 1398 I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ 1399 1400 /* Mark that we need updated tail pointers to read from... */ 1401 dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; 1402 dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; 1403 1404 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1405 1406 /* On Haswell we have to track which OASTATUS1 flags we've 1407 * already seen since they can't be cleared while periodic 1408 * sampling is enabled. 1409 */ 1410 dev_priv->perf.oa.gen7_latched_oastatus1 = 0; 1411 1412 /* NB: although the OA buffer will initially be allocated 1413 * zeroed via shmfs (and so this memset is redundant when 1414 * first allocating), we may re-init the OA buffer, either 1415 * when re-enabling a stream or in error/reset paths. 1416 * 1417 * The reason we clear the buffer for each re-init is for the 1418 * sanity check in gen7_append_oa_reports() that looks at the 1419 * report-id field to make sure it's non-zero which relies on 1420 * the assumption that new reports are being written to zeroed 1421 * memory... 1422 */ 1423 memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); 1424 1425 /* Maybe make ->pollin per-stream state if we support multiple 1426 * concurrent streams in the future. 1427 */ 1428 dev_priv->perf.oa.pollin = false; 1429 } 1430 1431 static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv) 1432 { 1433 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 1434 unsigned long flags; 1435 1436 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1437 1438 I915_WRITE(GEN8_OASTATUS, 0); 1439 I915_WRITE(GEN8_OAHEADPTR, gtt_offset); 1440 dev_priv->perf.oa.oa_buffer.head = gtt_offset; 1441 1442 I915_WRITE(GEN8_OABUFFER_UDW, 0); 1443 1444 /* 1445 * PRM says: 1446 * 1447 * "This MMIO must be set before the OATAILPTR 1448 * register and after the OAHEADPTR register. This is 1449 * to enable proper functionality of the overflow 1450 * bit." 1451 */ 1452 I915_WRITE(GEN8_OABUFFER, gtt_offset | 1453 OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT); 1454 I915_WRITE(GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK); 1455 1456 /* Mark that we need updated tail pointers to read from... */ 1457 dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; 1458 dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; 1459 1460 /* 1461 * Reset state used to recognise context switches, affecting which 1462 * reports we will forward to userspace while filtering for a single 1463 * context. 1464 */ 1465 dev_priv->perf.oa.oa_buffer.last_ctx_id = INVALID_CTX_ID; 1466 1467 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); 1468 1469 /* 1470 * NB: although the OA buffer will initially be allocated 1471 * zeroed via shmfs (and so this memset is redundant when 1472 * first allocating), we may re-init the OA buffer, either 1473 * when re-enabling a stream or in error/reset paths. 1474 * 1475 * The reason we clear the buffer for each re-init is for the 1476 * sanity check in gen8_append_oa_reports() that looks at the 1477 * reason field to make sure it's non-zero which relies on 1478 * the assumption that new reports are being written to zeroed 1479 * memory... 1480 */ 1481 memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); 1482 1483 /* 1484 * Maybe make ->pollin per-stream state if we support multiple 1485 * concurrent streams in the future. 1486 */ 1487 dev_priv->perf.oa.pollin = false; 1488 } 1489 1490 static int alloc_oa_buffer(struct drm_i915_private *dev_priv) 1491 { 1492 struct drm_i915_gem_object *bo; 1493 struct i915_vma *vma; 1494 int ret; 1495 1496 if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) 1497 return -ENODEV; 1498 1499 ret = i915_mutex_lock_interruptible(&dev_priv->drm); 1500 if (ret) 1501 return ret; 1502 1503 BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); 1504 BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); 1505 1506 bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE); 1507 if (IS_ERR(bo)) { 1508 DRM_ERROR("Failed to allocate OA buffer\n"); 1509 ret = PTR_ERR(bo); 1510 goto unlock; 1511 } 1512 1513 ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); 1514 if (ret) 1515 goto err_unref; 1516 1517 /* PreHSW required 512K alignment, HSW requires 16M */ 1518 vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); 1519 if (IS_ERR(vma)) { 1520 ret = PTR_ERR(vma); 1521 goto err_unref; 1522 } 1523 dev_priv->perf.oa.oa_buffer.vma = vma; 1524 1525 dev_priv->perf.oa.oa_buffer.vaddr = 1526 i915_gem_object_pin_map(bo, I915_MAP_WB); 1527 if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { 1528 ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); 1529 goto err_unpin; 1530 } 1531 1532 dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); 1533 1534 DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", 1535 i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), 1536 dev_priv->perf.oa.oa_buffer.vaddr); 1537 1538 goto unlock; 1539 1540 err_unpin: 1541 __i915_vma_unpin(vma); 1542 1543 err_unref: 1544 i915_gem_object_put(bo); 1545 1546 dev_priv->perf.oa.oa_buffer.vaddr = NULL; 1547 dev_priv->perf.oa.oa_buffer.vma = NULL; 1548 1549 unlock: 1550 mutex_unlock(&dev_priv->drm.struct_mutex); 1551 return ret; 1552 } 1553 1554 static void config_oa_regs(struct drm_i915_private *dev_priv, 1555 const struct i915_oa_reg *regs, 1556 u32 n_regs) 1557 { 1558 u32 i; 1559 1560 for (i = 0; i < n_regs; i++) { 1561 const struct i915_oa_reg *reg = regs + i; 1562 1563 I915_WRITE(reg->addr, reg->value); 1564 } 1565 } 1566 1567 static int hsw_enable_metric_set(struct drm_i915_private *dev_priv, 1568 const struct i915_oa_config *oa_config) 1569 { 1570 /* PRM: 1571 * 1572 * OA unit is using “crclk” for its functionality. When trunk 1573 * level clock gating takes place, OA clock would be gated, 1574 * unable to count the events from non-render clock domain. 1575 * Render clock gating must be disabled when OA is enabled to 1576 * count the events from non-render domain. Unit level clock 1577 * gating for RCS should also be disabled. 1578 */ 1579 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & 1580 ~GEN7_DOP_CLOCK_GATE_ENABLE)); 1581 I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | 1582 GEN6_CSUNIT_CLOCK_GATE_DISABLE)); 1583 1584 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); 1585 1586 /* It apparently takes a fairly long time for a new MUX 1587 * configuration to be be applied after these register writes. 1588 * This delay duration was derived empirically based on the 1589 * render_basic config but hopefully it covers the maximum 1590 * configuration latency. 1591 * 1592 * As a fallback, the checks in _append_oa_reports() to skip 1593 * invalid OA reports do also seem to work to discard reports 1594 * generated before this config has completed - albeit not 1595 * silently. 1596 * 1597 * Unfortunately this is essentially a magic number, since we 1598 * don't currently know of a reliable mechanism for predicting 1599 * how long the MUX config will take to apply and besides 1600 * seeing invalid reports we don't know of a reliable way to 1601 * explicitly check that the MUX config has landed. 1602 * 1603 * It's even possible we've miss characterized the underlying 1604 * problem - it just seems like the simplest explanation why 1605 * a delay at this location would mitigate any invalid reports. 1606 */ 1607 usleep_range(15000, 20000); 1608 1609 config_oa_regs(dev_priv, oa_config->b_counter_regs, 1610 oa_config->b_counter_regs_len); 1611 1612 return 0; 1613 } 1614 1615 static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) 1616 { 1617 I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & 1618 ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); 1619 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | 1620 GEN7_DOP_CLOCK_GATE_ENABLE)); 1621 1622 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & 1623 ~GT_NOA_ENABLE)); 1624 } 1625 1626 /* 1627 * NB: It must always remain pointer safe to run this even if the OA unit 1628 * has been disabled. 1629 * 1630 * It's fine to put out-of-date values into these per-context registers 1631 * in the case that the OA unit has been disabled. 1632 */ 1633 static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx, 1634 u32 *reg_state, 1635 const struct i915_oa_config *oa_config) 1636 { 1637 struct drm_i915_private *dev_priv = ctx->i915; 1638 u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset; 1639 u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset; 1640 /* The MMIO offsets for Flex EU registers aren't contiguous */ 1641 u32 flex_mmio[] = { 1642 i915_mmio_reg_offset(EU_PERF_CNTL0), 1643 i915_mmio_reg_offset(EU_PERF_CNTL1), 1644 i915_mmio_reg_offset(EU_PERF_CNTL2), 1645 i915_mmio_reg_offset(EU_PERF_CNTL3), 1646 i915_mmio_reg_offset(EU_PERF_CNTL4), 1647 i915_mmio_reg_offset(EU_PERF_CNTL5), 1648 i915_mmio_reg_offset(EU_PERF_CNTL6), 1649 }; 1650 int i; 1651 1652 reg_state[ctx_oactxctrl] = i915_mmio_reg_offset(GEN8_OACTXCONTROL); 1653 reg_state[ctx_oactxctrl+1] = (dev_priv->perf.oa.period_exponent << 1654 GEN8_OA_TIMER_PERIOD_SHIFT) | 1655 (dev_priv->perf.oa.periodic ? 1656 GEN8_OA_TIMER_ENABLE : 0) | 1657 GEN8_OA_COUNTER_RESUME; 1658 1659 for (i = 0; i < ARRAY_SIZE(flex_mmio); i++) { 1660 u32 state_offset = ctx_flexeu0 + i * 2; 1661 u32 mmio = flex_mmio[i]; 1662 1663 /* 1664 * This arbitrary default will select the 'EU FPU0 Pipeline 1665 * Active' event. In the future it's anticipated that there 1666 * will be an explicit 'No Event' we can select, but not yet... 1667 */ 1668 u32 value = 0; 1669 1670 if (oa_config) { 1671 u32 j; 1672 1673 for (j = 0; j < oa_config->flex_regs_len; j++) { 1674 if (i915_mmio_reg_offset(oa_config->flex_regs[j].addr) == mmio) { 1675 value = oa_config->flex_regs[j].value; 1676 break; 1677 } 1678 } 1679 } 1680 1681 reg_state[state_offset] = mmio; 1682 reg_state[state_offset+1] = value; 1683 } 1684 } 1685 1686 /* 1687 * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This 1688 * is only used by the kernel context. 1689 */ 1690 static int gen8_emit_oa_config(struct i915_request *rq, 1691 const struct i915_oa_config *oa_config) 1692 { 1693 struct drm_i915_private *dev_priv = rq->i915; 1694 /* The MMIO offsets for Flex EU registers aren't contiguous */ 1695 u32 flex_mmio[] = { 1696 i915_mmio_reg_offset(EU_PERF_CNTL0), 1697 i915_mmio_reg_offset(EU_PERF_CNTL1), 1698 i915_mmio_reg_offset(EU_PERF_CNTL2), 1699 i915_mmio_reg_offset(EU_PERF_CNTL3), 1700 i915_mmio_reg_offset(EU_PERF_CNTL4), 1701 i915_mmio_reg_offset(EU_PERF_CNTL5), 1702 i915_mmio_reg_offset(EU_PERF_CNTL6), 1703 }; 1704 u32 *cs; 1705 int i; 1706 1707 cs = intel_ring_begin(rq, ARRAY_SIZE(flex_mmio) * 2 + 4); 1708 if (IS_ERR(cs)) 1709 return PTR_ERR(cs); 1710 1711 *cs++ = MI_LOAD_REGISTER_IMM(ARRAY_SIZE(flex_mmio) + 1); 1712 1713 *cs++ = i915_mmio_reg_offset(GEN8_OACTXCONTROL); 1714 *cs++ = (dev_priv->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | 1715 (dev_priv->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) | 1716 GEN8_OA_COUNTER_RESUME; 1717 1718 for (i = 0; i < ARRAY_SIZE(flex_mmio); i++) { 1719 u32 mmio = flex_mmio[i]; 1720 1721 /* 1722 * This arbitrary default will select the 'EU FPU0 Pipeline 1723 * Active' event. In the future it's anticipated that there 1724 * will be an explicit 'No Event' we can select, but not 1725 * yet... 1726 */ 1727 u32 value = 0; 1728 1729 if (oa_config) { 1730 u32 j; 1731 1732 for (j = 0; j < oa_config->flex_regs_len; j++) { 1733 if (i915_mmio_reg_offset(oa_config->flex_regs[j].addr) == mmio) { 1734 value = oa_config->flex_regs[j].value; 1735 break; 1736 } 1737 } 1738 } 1739 1740 *cs++ = mmio; 1741 *cs++ = value; 1742 } 1743 1744 *cs++ = MI_NOOP; 1745 intel_ring_advance(rq, cs); 1746 1747 return 0; 1748 } 1749 1750 static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_priv, 1751 const struct i915_oa_config *oa_config) 1752 { 1753 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 1754 struct i915_timeline *timeline; 1755 struct i915_request *rq; 1756 int ret; 1757 1758 lockdep_assert_held(&dev_priv->drm.struct_mutex); 1759 1760 i915_retire_requests(dev_priv); 1761 1762 rq = i915_request_alloc(engine, dev_priv->kernel_context); 1763 if (IS_ERR(rq)) 1764 return PTR_ERR(rq); 1765 1766 ret = gen8_emit_oa_config(rq, oa_config); 1767 if (ret) { 1768 i915_request_add(rq); 1769 return ret; 1770 } 1771 1772 /* Queue this switch after all other activity */ 1773 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { 1774 struct i915_request *prev; 1775 1776 prev = i915_gem_active_raw(&timeline->last_request, 1777 &dev_priv->drm.struct_mutex); 1778 if (prev) 1779 i915_request_await_dma_fence(rq, &prev->fence); 1780 } 1781 1782 i915_request_add(rq); 1783 1784 return 0; 1785 } 1786 1787 /* 1788 * Manages updating the per-context aspects of the OA stream 1789 * configuration across all contexts. 1790 * 1791 * The awkward consideration here is that OACTXCONTROL controls the 1792 * exponent for periodic sampling which is primarily used for system 1793 * wide profiling where we'd like a consistent sampling period even in 1794 * the face of context switches. 1795 * 1796 * Our approach of updating the register state context (as opposed to 1797 * say using a workaround batch buffer) ensures that the hardware 1798 * won't automatically reload an out-of-date timer exponent even 1799 * transiently before a WA BB could be parsed. 1800 * 1801 * This function needs to: 1802 * - Ensure the currently running context's per-context OA state is 1803 * updated 1804 * - Ensure that all existing contexts will have the correct per-context 1805 * OA state if they are scheduled for use. 1806 * - Ensure any new contexts will be initialized with the correct 1807 * per-context OA state. 1808 * 1809 * Note: it's only the RCS/Render context that has any OA state. 1810 */ 1811 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv, 1812 const struct i915_oa_config *oa_config) 1813 { 1814 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 1815 struct i915_gem_context *ctx; 1816 int ret; 1817 unsigned int wait_flags = I915_WAIT_LOCKED; 1818 1819 lockdep_assert_held(&dev_priv->drm.struct_mutex); 1820 1821 /* Switch away from any user context. */ 1822 ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config); 1823 if (ret) 1824 goto out; 1825 1826 /* 1827 * The OA register config is setup through the context image. This image 1828 * might be written to by the GPU on context switch (in particular on 1829 * lite-restore). This means we can't safely update a context's image, 1830 * if this context is scheduled/submitted to run on the GPU. 1831 * 1832 * We could emit the OA register config through the batch buffer but 1833 * this might leave small interval of time where the OA unit is 1834 * configured at an invalid sampling period. 1835 * 1836 * So far the best way to work around this issue seems to be draining 1837 * the GPU from any submitted work. 1838 */ 1839 ret = i915_gem_wait_for_idle(dev_priv, wait_flags); 1840 if (ret) 1841 goto out; 1842 1843 /* Update all contexts now that we've stalled the submission. */ 1844 list_for_each_entry(ctx, &dev_priv->contexts.list, link) { 1845 struct intel_context *ce = to_intel_context(ctx, engine); 1846 u32 *regs; 1847 1848 /* OA settings will be set upon first use */ 1849 if (!ce->state) 1850 continue; 1851 1852 regs = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB); 1853 if (IS_ERR(regs)) { 1854 ret = PTR_ERR(regs); 1855 goto out; 1856 } 1857 1858 ce->state->obj->mm.dirty = true; 1859 regs += LRC_STATE_PN * PAGE_SIZE / sizeof(*regs); 1860 1861 gen8_update_reg_state_unlocked(ctx, regs, oa_config); 1862 1863 i915_gem_object_unpin_map(ce->state->obj); 1864 } 1865 1866 out: 1867 return ret; 1868 } 1869 1870 static int gen8_enable_metric_set(struct drm_i915_private *dev_priv, 1871 const struct i915_oa_config *oa_config) 1872 { 1873 int ret; 1874 1875 /* 1876 * We disable slice/unslice clock ratio change reports on SKL since 1877 * they are too noisy. The HW generates a lot of redundant reports 1878 * where the ratio hasn't really changed causing a lot of redundant 1879 * work to processes and increasing the chances we'll hit buffer 1880 * overruns. 1881 * 1882 * Although we don't currently use the 'disable overrun' OABUFFER 1883 * feature it's worth noting that clock ratio reports have to be 1884 * disabled before considering to use that feature since the HW doesn't 1885 * correctly block these reports. 1886 * 1887 * Currently none of the high-level metrics we have depend on knowing 1888 * this ratio to normalize. 1889 * 1890 * Note: This register is not power context saved and restored, but 1891 * that's OK considering that we disable RC6 while the OA unit is 1892 * enabled. 1893 * 1894 * The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to 1895 * be read back from automatically triggered reports, as part of the 1896 * RPT_ID field. 1897 */ 1898 if (IS_GEN(dev_priv, 9, 11)) { 1899 I915_WRITE(GEN8_OA_DEBUG, 1900 _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | 1901 GEN9_OA_DEBUG_INCLUDE_CLK_RATIO)); 1902 } 1903 1904 /* 1905 * Update all contexts prior writing the mux configurations as we need 1906 * to make sure all slices/subslices are ON before writing to NOA 1907 * registers. 1908 */ 1909 ret = gen8_configure_all_contexts(dev_priv, oa_config); 1910 if (ret) 1911 return ret; 1912 1913 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); 1914 1915 config_oa_regs(dev_priv, oa_config->b_counter_regs, 1916 oa_config->b_counter_regs_len); 1917 1918 return 0; 1919 } 1920 1921 static void gen8_disable_metric_set(struct drm_i915_private *dev_priv) 1922 { 1923 /* Reset all contexts' slices/subslices configurations. */ 1924 gen8_configure_all_contexts(dev_priv, NULL); 1925 1926 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & 1927 ~GT_NOA_ENABLE)); 1928 } 1929 1930 static void gen10_disable_metric_set(struct drm_i915_private *dev_priv) 1931 { 1932 /* Reset all contexts' slices/subslices configurations. */ 1933 gen8_configure_all_contexts(dev_priv, NULL); 1934 1935 /* Make sure we disable noa to save power. */ 1936 I915_WRITE(RPM_CONFIG1, 1937 I915_READ(RPM_CONFIG1) & ~GEN10_GT_NOA_ENABLE); 1938 } 1939 1940 static void gen7_oa_enable(struct drm_i915_private *dev_priv) 1941 { 1942 struct i915_gem_context *ctx = 1943 dev_priv->perf.oa.exclusive_stream->ctx; 1944 u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; 1945 bool periodic = dev_priv->perf.oa.periodic; 1946 u32 period_exponent = dev_priv->perf.oa.period_exponent; 1947 u32 report_format = dev_priv->perf.oa.oa_buffer.format; 1948 1949 /* 1950 * Reset buf pointers so we don't forward reports from before now. 1951 * 1952 * Think carefully if considering trying to avoid this, since it 1953 * also ensures status flags and the buffer itself are cleared 1954 * in error paths, and we have checks for invalid reports based 1955 * on the assumption that certain fields are written to zeroed 1956 * memory which this helps maintains. 1957 */ 1958 gen7_init_oa_buffer(dev_priv); 1959 1960 I915_WRITE(GEN7_OACONTROL, 1961 (ctx_id & GEN7_OACONTROL_CTX_MASK) | 1962 (period_exponent << 1963 GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | 1964 (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | 1965 (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | 1966 (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | 1967 GEN7_OACONTROL_ENABLE); 1968 } 1969 1970 static void gen8_oa_enable(struct drm_i915_private *dev_priv) 1971 { 1972 u32 report_format = dev_priv->perf.oa.oa_buffer.format; 1973 1974 /* 1975 * Reset buf pointers so we don't forward reports from before now. 1976 * 1977 * Think carefully if considering trying to avoid this, since it 1978 * also ensures status flags and the buffer itself are cleared 1979 * in error paths, and we have checks for invalid reports based 1980 * on the assumption that certain fields are written to zeroed 1981 * memory which this helps maintains. 1982 */ 1983 gen8_init_oa_buffer(dev_priv); 1984 1985 /* 1986 * Note: we don't rely on the hardware to perform single context 1987 * filtering and instead filter on the cpu based on the context-id 1988 * field of reports 1989 */ 1990 I915_WRITE(GEN8_OACONTROL, (report_format << 1991 GEN8_OA_REPORT_FORMAT_SHIFT) | 1992 GEN8_OA_COUNTER_ENABLE); 1993 } 1994 1995 /** 1996 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream 1997 * @stream: An i915 perf stream opened for OA metrics 1998 * 1999 * [Re]enables hardware periodic sampling according to the period configured 2000 * when opening the stream. This also starts a hrtimer that will periodically 2001 * check for data in the circular OA buffer for notifying userspace (e.g. 2002 * during a read() or poll()). 2003 */ 2004 static void i915_oa_stream_enable(struct i915_perf_stream *stream) 2005 { 2006 struct drm_i915_private *dev_priv = stream->dev_priv; 2007 2008 dev_priv->perf.oa.ops.oa_enable(dev_priv); 2009 2010 if (dev_priv->perf.oa.periodic) 2011 hrtimer_start(&dev_priv->perf.oa.poll_check_timer, 2012 ns_to_ktime(POLL_PERIOD), 2013 HRTIMER_MODE_REL_PINNED); 2014 } 2015 2016 static void gen7_oa_disable(struct drm_i915_private *dev_priv) 2017 { 2018 I915_WRITE(GEN7_OACONTROL, 0); 2019 if (intel_wait_for_register(dev_priv, 2020 GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0, 2021 50)) 2022 DRM_ERROR("wait for OA to be disabled timed out\n"); 2023 } 2024 2025 static void gen8_oa_disable(struct drm_i915_private *dev_priv) 2026 { 2027 I915_WRITE(GEN8_OACONTROL, 0); 2028 if (intel_wait_for_register(dev_priv, 2029 GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0, 2030 50)) 2031 DRM_ERROR("wait for OA to be disabled timed out\n"); 2032 } 2033 2034 /** 2035 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream 2036 * @stream: An i915 perf stream opened for OA metrics 2037 * 2038 * Stops the OA unit from periodically writing counter reports into the 2039 * circular OA buffer. This also stops the hrtimer that periodically checks for 2040 * data in the circular OA buffer, for notifying userspace. 2041 */ 2042 static void i915_oa_stream_disable(struct i915_perf_stream *stream) 2043 { 2044 struct drm_i915_private *dev_priv = stream->dev_priv; 2045 2046 dev_priv->perf.oa.ops.oa_disable(dev_priv); 2047 2048 if (dev_priv->perf.oa.periodic) 2049 hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); 2050 } 2051 2052 static const struct i915_perf_stream_ops i915_oa_stream_ops = { 2053 .destroy = i915_oa_stream_destroy, 2054 .enable = i915_oa_stream_enable, 2055 .disable = i915_oa_stream_disable, 2056 .wait_unlocked = i915_oa_wait_unlocked, 2057 .poll_wait = i915_oa_poll_wait, 2058 .read = i915_oa_read, 2059 }; 2060 2061 /** 2062 * i915_oa_stream_init - validate combined props for OA stream and init 2063 * @stream: An i915 perf stream 2064 * @param: The open parameters passed to `DRM_I915_PERF_OPEN` 2065 * @props: The property state that configures stream (individually validated) 2066 * 2067 * While read_properties_unlocked() validates properties in isolation it 2068 * doesn't ensure that the combination necessarily makes sense. 2069 * 2070 * At this point it has been determined that userspace wants a stream of 2071 * OA metrics, but still we need to further validate the combined 2072 * properties are OK. 2073 * 2074 * If the configuration makes sense then we can allocate memory for 2075 * a circular OA buffer and apply the requested metric set configuration. 2076 * 2077 * Returns: zero on success or a negative error code. 2078 */ 2079 static int i915_oa_stream_init(struct i915_perf_stream *stream, 2080 struct drm_i915_perf_open_param *param, 2081 struct perf_open_properties *props) 2082 { 2083 struct drm_i915_private *dev_priv = stream->dev_priv; 2084 int format_size; 2085 int ret; 2086 2087 /* If the sysfs metrics/ directory wasn't registered for some 2088 * reason then don't let userspace try their luck with config 2089 * IDs 2090 */ 2091 if (!dev_priv->perf.metrics_kobj) { 2092 DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); 2093 return -EINVAL; 2094 } 2095 2096 if (!(props->sample_flags & SAMPLE_OA_REPORT)) { 2097 DRM_DEBUG("Only OA report sampling supported\n"); 2098 return -EINVAL; 2099 } 2100 2101 if (!dev_priv->perf.oa.ops.init_oa_buffer) { 2102 DRM_DEBUG("OA unit not supported\n"); 2103 return -ENODEV; 2104 } 2105 2106 /* To avoid the complexity of having to accurately filter 2107 * counter reports and marshal to the appropriate client 2108 * we currently only allow exclusive access 2109 */ 2110 if (dev_priv->perf.oa.exclusive_stream) { 2111 DRM_DEBUG("OA unit already in use\n"); 2112 return -EBUSY; 2113 } 2114 2115 if (!props->oa_format) { 2116 DRM_DEBUG("OA report format not specified\n"); 2117 return -EINVAL; 2118 } 2119 2120 /* We set up some ratelimit state to potentially throttle any _NOTES 2121 * about spurious, invalid OA reports which we don't forward to 2122 * userspace. 2123 * 2124 * The initialization is associated with opening the stream (not driver 2125 * init) considering we print a _NOTE about any throttling when closing 2126 * the stream instead of waiting until driver _fini which no one would 2127 * ever see. 2128 * 2129 * Using the same limiting factors as printk_ratelimit() 2130 */ 2131 ratelimit_state_init(&dev_priv->perf.oa.spurious_report_rs, 2132 5 * HZ, 10); 2133 /* Since we use a DRM_NOTE for spurious reports it would be 2134 * inconsistent to let __ratelimit() automatically print a warning for 2135 * throttling. 2136 */ 2137 ratelimit_set_flags(&dev_priv->perf.oa.spurious_report_rs, 2138 RATELIMIT_MSG_ON_RELEASE); 2139 2140 stream->sample_size = sizeof(struct drm_i915_perf_record_header); 2141 2142 format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; 2143 2144 stream->sample_flags |= SAMPLE_OA_REPORT; 2145 stream->sample_size += format_size; 2146 2147 dev_priv->perf.oa.oa_buffer.format_size = format_size; 2148 if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) 2149 return -EINVAL; 2150 2151 dev_priv->perf.oa.oa_buffer.format = 2152 dev_priv->perf.oa.oa_formats[props->oa_format].format; 2153 2154 dev_priv->perf.oa.periodic = props->oa_periodic; 2155 if (dev_priv->perf.oa.periodic) 2156 dev_priv->perf.oa.period_exponent = props->oa_period_exponent; 2157 2158 if (stream->ctx) { 2159 ret = oa_get_render_ctx_id(stream); 2160 if (ret) { 2161 DRM_DEBUG("Invalid context id to filter with\n"); 2162 return ret; 2163 } 2164 } 2165 2166 ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config); 2167 if (ret) { 2168 DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set); 2169 goto err_config; 2170 } 2171 2172 /* PRM - observability performance counters: 2173 * 2174 * OACONTROL, performance counter enable, note: 2175 * 2176 * "When this bit is set, in order to have coherent counts, 2177 * RC6 power state and trunk clock gating must be disabled. 2178 * This can be achieved by programming MMIO registers as 2179 * 0xA094=0 and 0xA090[31]=1" 2180 * 2181 * In our case we are expecting that taking pm + FORCEWAKE 2182 * references will effectively disable RC6. 2183 */ 2184 intel_runtime_pm_get(dev_priv); 2185 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2186 2187 ret = alloc_oa_buffer(dev_priv); 2188 if (ret) 2189 goto err_oa_buf_alloc; 2190 2191 ret = i915_mutex_lock_interruptible(&dev_priv->drm); 2192 if (ret) 2193 goto err_lock; 2194 2195 ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv, 2196 stream->oa_config); 2197 if (ret) { 2198 DRM_DEBUG("Unable to enable metric set\n"); 2199 goto err_enable; 2200 } 2201 2202 stream->ops = &i915_oa_stream_ops; 2203 2204 dev_priv->perf.oa.exclusive_stream = stream; 2205 2206 mutex_unlock(&dev_priv->drm.struct_mutex); 2207 2208 return 0; 2209 2210 err_enable: 2211 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); 2212 mutex_unlock(&dev_priv->drm.struct_mutex); 2213 2214 err_lock: 2215 free_oa_buffer(dev_priv); 2216 2217 err_oa_buf_alloc: 2218 put_oa_config(dev_priv, stream->oa_config); 2219 2220 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2221 intel_runtime_pm_put(dev_priv); 2222 2223 err_config: 2224 if (stream->ctx) 2225 oa_put_render_ctx_id(stream); 2226 2227 return ret; 2228 } 2229 2230 void i915_oa_init_reg_state(struct intel_engine_cs *engine, 2231 struct i915_gem_context *ctx, 2232 u32 *reg_state) 2233 { 2234 struct i915_perf_stream *stream; 2235 2236 if (engine->id != RCS) 2237 return; 2238 2239 stream = engine->i915->perf.oa.exclusive_stream; 2240 if (stream) 2241 gen8_update_reg_state_unlocked(ctx, reg_state, stream->oa_config); 2242 } 2243 2244 /** 2245 * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation 2246 * @stream: An i915 perf stream 2247 * @file: An i915 perf stream file 2248 * @buf: destination buffer given by userspace 2249 * @count: the number of bytes userspace wants to read 2250 * @ppos: (inout) file seek position (unused) 2251 * 2252 * Besides wrapping &i915_perf_stream_ops->read this provides a common place to 2253 * ensure that if we've successfully copied any data then reporting that takes 2254 * precedence over any internal error status, so the data isn't lost. 2255 * 2256 * For example ret will be -ENOSPC whenever there is more buffered data than 2257 * can be copied to userspace, but that's only interesting if we weren't able 2258 * to copy some data because it implies the userspace buffer is too small to 2259 * receive a single record (and we never split records). 2260 * 2261 * Another case with ret == -EFAULT is more of a grey area since it would seem 2262 * like bad form for userspace to ask us to overrun its buffer, but the user 2263 * knows best: 2264 * 2265 * http://yarchive.net/comp/linux/partial_reads_writes.html 2266 * 2267 * Returns: The number of bytes copied or a negative error code on failure. 2268 */ 2269 static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, 2270 struct file *file, 2271 char __user *buf, 2272 size_t count, 2273 loff_t *ppos) 2274 { 2275 /* Note we keep the offset (aka bytes read) separate from any 2276 * error status so that the final check for whether we return 2277 * the bytes read with a higher precedence than any error (see 2278 * comment below) doesn't need to be handled/duplicated in 2279 * stream->ops->read() implementations. 2280 */ 2281 size_t offset = 0; 2282 int ret = stream->ops->read(stream, buf, count, &offset); 2283 2284 return offset ?: (ret ?: -EAGAIN); 2285 } 2286 2287 /** 2288 * i915_perf_read - handles read() FOP for i915 perf stream FDs 2289 * @file: An i915 perf stream file 2290 * @buf: destination buffer given by userspace 2291 * @count: the number of bytes userspace wants to read 2292 * @ppos: (inout) file seek position (unused) 2293 * 2294 * The entry point for handling a read() on a stream file descriptor from 2295 * userspace. Most of the work is left to the i915_perf_read_locked() and 2296 * &i915_perf_stream_ops->read but to save having stream implementations (of 2297 * which we might have multiple later) we handle blocking read here. 2298 * 2299 * We can also consistently treat trying to read from a disabled stream 2300 * as an IO error so implementations can assume the stream is enabled 2301 * while reading. 2302 * 2303 * Returns: The number of bytes copied or a negative error code on failure. 2304 */ 2305 static ssize_t i915_perf_read(struct file *file, 2306 char __user *buf, 2307 size_t count, 2308 loff_t *ppos) 2309 { 2310 struct i915_perf_stream *stream = file->private_data; 2311 struct drm_i915_private *dev_priv = stream->dev_priv; 2312 ssize_t ret; 2313 2314 /* To ensure it's handled consistently we simply treat all reads of a 2315 * disabled stream as an error. In particular it might otherwise lead 2316 * to a deadlock for blocking file descriptors... 2317 */ 2318 if (!stream->enabled) 2319 return -EIO; 2320 2321 if (!(file->f_flags & O_NONBLOCK)) { 2322 /* There's the small chance of false positives from 2323 * stream->ops->wait_unlocked. 2324 * 2325 * E.g. with single context filtering since we only wait until 2326 * oabuffer has >= 1 report we don't immediately know whether 2327 * any reports really belong to the current context 2328 */ 2329 do { 2330 ret = stream->ops->wait_unlocked(stream); 2331 if (ret) 2332 return ret; 2333 2334 mutex_lock(&dev_priv->perf.lock); 2335 ret = i915_perf_read_locked(stream, file, 2336 buf, count, ppos); 2337 mutex_unlock(&dev_priv->perf.lock); 2338 } while (ret == -EAGAIN); 2339 } else { 2340 mutex_lock(&dev_priv->perf.lock); 2341 ret = i915_perf_read_locked(stream, file, buf, count, ppos); 2342 mutex_unlock(&dev_priv->perf.lock); 2343 } 2344 2345 /* We allow the poll checking to sometimes report false positive EPOLLIN 2346 * events where we might actually report EAGAIN on read() if there's 2347 * not really any data available. In this situation though we don't 2348 * want to enter a busy loop between poll() reporting a EPOLLIN event 2349 * and read() returning -EAGAIN. Clearing the oa.pollin state here 2350 * effectively ensures we back off until the next hrtimer callback 2351 * before reporting another EPOLLIN event. 2352 */ 2353 if (ret >= 0 || ret == -EAGAIN) { 2354 /* Maybe make ->pollin per-stream state if we support multiple 2355 * concurrent streams in the future. 2356 */ 2357 dev_priv->perf.oa.pollin = false; 2358 } 2359 2360 return ret; 2361 } 2362 2363 static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) 2364 { 2365 struct drm_i915_private *dev_priv = 2366 container_of(hrtimer, typeof(*dev_priv), 2367 perf.oa.poll_check_timer); 2368 2369 if (oa_buffer_check_unlocked(dev_priv)) { 2370 dev_priv->perf.oa.pollin = true; 2371 wake_up(&dev_priv->perf.oa.poll_wq); 2372 } 2373 2374 hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); 2375 2376 return HRTIMER_RESTART; 2377 } 2378 2379 /** 2380 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream 2381 * @dev_priv: i915 device instance 2382 * @stream: An i915 perf stream 2383 * @file: An i915 perf stream file 2384 * @wait: poll() state table 2385 * 2386 * For handling userspace polling on an i915 perf stream, this calls through to 2387 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that 2388 * will be woken for new stream data. 2389 * 2390 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 2391 * with any non-file-operation driver hooks. 2392 * 2393 * Returns: any poll events that are ready without sleeping 2394 */ 2395 static __poll_t i915_perf_poll_locked(struct drm_i915_private *dev_priv, 2396 struct i915_perf_stream *stream, 2397 struct file *file, 2398 poll_table *wait) 2399 { 2400 __poll_t events = 0; 2401 2402 stream->ops->poll_wait(stream, file, wait); 2403 2404 /* Note: we don't explicitly check whether there's something to read 2405 * here since this path may be very hot depending on what else 2406 * userspace is polling, or on the timeout in use. We rely solely on 2407 * the hrtimer/oa_poll_check_timer_cb to notify us when there are 2408 * samples to read. 2409 */ 2410 if (dev_priv->perf.oa.pollin) 2411 events |= EPOLLIN; 2412 2413 return events; 2414 } 2415 2416 /** 2417 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream 2418 * @file: An i915 perf stream file 2419 * @wait: poll() state table 2420 * 2421 * For handling userspace polling on an i915 perf stream, this ensures 2422 * poll_wait() gets called with a wait queue that will be woken for new stream 2423 * data. 2424 * 2425 * Note: Implementation deferred to i915_perf_poll_locked() 2426 * 2427 * Returns: any poll events that are ready without sleeping 2428 */ 2429 static __poll_t i915_perf_poll(struct file *file, poll_table *wait) 2430 { 2431 struct i915_perf_stream *stream = file->private_data; 2432 struct drm_i915_private *dev_priv = stream->dev_priv; 2433 __poll_t ret; 2434 2435 mutex_lock(&dev_priv->perf.lock); 2436 ret = i915_perf_poll_locked(dev_priv, stream, file, wait); 2437 mutex_unlock(&dev_priv->perf.lock); 2438 2439 return ret; 2440 } 2441 2442 /** 2443 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl 2444 * @stream: A disabled i915 perf stream 2445 * 2446 * [Re]enables the associated capture of data for this stream. 2447 * 2448 * If a stream was previously enabled then there's currently no intention 2449 * to provide userspace any guarantee about the preservation of previously 2450 * buffered data. 2451 */ 2452 static void i915_perf_enable_locked(struct i915_perf_stream *stream) 2453 { 2454 if (stream->enabled) 2455 return; 2456 2457 /* Allow stream->ops->enable() to refer to this */ 2458 stream->enabled = true; 2459 2460 if (stream->ops->enable) 2461 stream->ops->enable(stream); 2462 } 2463 2464 /** 2465 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl 2466 * @stream: An enabled i915 perf stream 2467 * 2468 * Disables the associated capture of data for this stream. 2469 * 2470 * The intention is that disabling an re-enabling a stream will ideally be 2471 * cheaper than destroying and re-opening a stream with the same configuration, 2472 * though there are no formal guarantees about what state or buffered data 2473 * must be retained between disabling and re-enabling a stream. 2474 * 2475 * Note: while a stream is disabled it's considered an error for userspace 2476 * to attempt to read from the stream (-EIO). 2477 */ 2478 static void i915_perf_disable_locked(struct i915_perf_stream *stream) 2479 { 2480 if (!stream->enabled) 2481 return; 2482 2483 /* Allow stream->ops->disable() to refer to this */ 2484 stream->enabled = false; 2485 2486 if (stream->ops->disable) 2487 stream->ops->disable(stream); 2488 } 2489 2490 /** 2491 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs 2492 * @stream: An i915 perf stream 2493 * @cmd: the ioctl request 2494 * @arg: the ioctl data 2495 * 2496 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 2497 * with any non-file-operation driver hooks. 2498 * 2499 * Returns: zero on success or a negative error code. Returns -EINVAL for 2500 * an unknown ioctl request. 2501 */ 2502 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, 2503 unsigned int cmd, 2504 unsigned long arg) 2505 { 2506 switch (cmd) { 2507 case I915_PERF_IOCTL_ENABLE: 2508 i915_perf_enable_locked(stream); 2509 return 0; 2510 case I915_PERF_IOCTL_DISABLE: 2511 i915_perf_disable_locked(stream); 2512 return 0; 2513 } 2514 2515 return -EINVAL; 2516 } 2517 2518 /** 2519 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs 2520 * @file: An i915 perf stream file 2521 * @cmd: the ioctl request 2522 * @arg: the ioctl data 2523 * 2524 * Implementation deferred to i915_perf_ioctl_locked(). 2525 * 2526 * Returns: zero on success or a negative error code. Returns -EINVAL for 2527 * an unknown ioctl request. 2528 */ 2529 static long i915_perf_ioctl(struct file *file, 2530 unsigned int cmd, 2531 unsigned long arg) 2532 { 2533 struct i915_perf_stream *stream = file->private_data; 2534 struct drm_i915_private *dev_priv = stream->dev_priv; 2535 long ret; 2536 2537 mutex_lock(&dev_priv->perf.lock); 2538 ret = i915_perf_ioctl_locked(stream, cmd, arg); 2539 mutex_unlock(&dev_priv->perf.lock); 2540 2541 return ret; 2542 } 2543 2544 /** 2545 * i915_perf_destroy_locked - destroy an i915 perf stream 2546 * @stream: An i915 perf stream 2547 * 2548 * Frees all resources associated with the given i915 perf @stream, disabling 2549 * any associated data capture in the process. 2550 * 2551 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 2552 * with any non-file-operation driver hooks. 2553 */ 2554 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) 2555 { 2556 if (stream->enabled) 2557 i915_perf_disable_locked(stream); 2558 2559 if (stream->ops->destroy) 2560 stream->ops->destroy(stream); 2561 2562 list_del(&stream->link); 2563 2564 if (stream->ctx) 2565 i915_gem_context_put(stream->ctx); 2566 2567 kfree(stream); 2568 } 2569 2570 /** 2571 * i915_perf_release - handles userspace close() of a stream file 2572 * @inode: anonymous inode associated with file 2573 * @file: An i915 perf stream file 2574 * 2575 * Cleans up any resources associated with an open i915 perf stream file. 2576 * 2577 * NB: close() can't really fail from the userspace point of view. 2578 * 2579 * Returns: zero on success or a negative error code. 2580 */ 2581 static int i915_perf_release(struct inode *inode, struct file *file) 2582 { 2583 struct i915_perf_stream *stream = file->private_data; 2584 struct drm_i915_private *dev_priv = stream->dev_priv; 2585 2586 mutex_lock(&dev_priv->perf.lock); 2587 i915_perf_destroy_locked(stream); 2588 mutex_unlock(&dev_priv->perf.lock); 2589 2590 return 0; 2591 } 2592 2593 2594 static const struct file_operations fops = { 2595 .owner = THIS_MODULE, 2596 .llseek = no_llseek, 2597 .release = i915_perf_release, 2598 .poll = i915_perf_poll, 2599 .read = i915_perf_read, 2600 .unlocked_ioctl = i915_perf_ioctl, 2601 /* Our ioctl have no arguments, so it's safe to use the same function 2602 * to handle 32bits compatibility. 2603 */ 2604 .compat_ioctl = i915_perf_ioctl, 2605 }; 2606 2607 2608 /** 2609 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD 2610 * @dev_priv: i915 device instance 2611 * @param: The open parameters passed to 'DRM_I915_PERF_OPEN` 2612 * @props: individually validated u64 property value pairs 2613 * @file: drm file 2614 * 2615 * See i915_perf_ioctl_open() for interface details. 2616 * 2617 * Implements further stream config validation and stream initialization on 2618 * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex 2619 * taken to serialize with any non-file-operation driver hooks. 2620 * 2621 * Note: at this point the @props have only been validated in isolation and 2622 * it's still necessary to validate that the combination of properties makes 2623 * sense. 2624 * 2625 * In the case where userspace is interested in OA unit metrics then further 2626 * config validation and stream initialization details will be handled by 2627 * i915_oa_stream_init(). The code here should only validate config state that 2628 * will be relevant to all stream types / backends. 2629 * 2630 * Returns: zero on success or a negative error code. 2631 */ 2632 static int 2633 i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, 2634 struct drm_i915_perf_open_param *param, 2635 struct perf_open_properties *props, 2636 struct drm_file *file) 2637 { 2638 struct i915_gem_context *specific_ctx = NULL; 2639 struct i915_perf_stream *stream = NULL; 2640 unsigned long f_flags = 0; 2641 bool privileged_op = true; 2642 int stream_fd; 2643 int ret; 2644 2645 if (props->single_context) { 2646 u32 ctx_handle = props->ctx_handle; 2647 struct drm_i915_file_private *file_priv = file->driver_priv; 2648 2649 specific_ctx = i915_gem_context_lookup(file_priv, ctx_handle); 2650 if (!specific_ctx) { 2651 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", 2652 ctx_handle); 2653 ret = -ENOENT; 2654 goto err; 2655 } 2656 } 2657 2658 /* 2659 * On Haswell the OA unit supports clock gating off for a specific 2660 * context and in this mode there's no visibility of metrics for the 2661 * rest of the system, which we consider acceptable for a 2662 * non-privileged client. 2663 * 2664 * For Gen8+ the OA unit no longer supports clock gating off for a 2665 * specific context and the kernel can't securely stop the counters 2666 * from updating as system-wide / global values. Even though we can 2667 * filter reports based on the included context ID we can't block 2668 * clients from seeing the raw / global counter values via 2669 * MI_REPORT_PERF_COUNT commands and so consider it a privileged op to 2670 * enable the OA unit by default. 2671 */ 2672 if (IS_HASWELL(dev_priv) && specific_ctx) 2673 privileged_op = false; 2674 2675 /* Similar to perf's kernel.perf_paranoid_cpu sysctl option 2676 * we check a dev.i915.perf_stream_paranoid sysctl option 2677 * to determine if it's ok to access system wide OA counters 2678 * without CAP_SYS_ADMIN privileges. 2679 */ 2680 if (privileged_op && 2681 i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { 2682 DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n"); 2683 ret = -EACCES; 2684 goto err_ctx; 2685 } 2686 2687 stream = kzalloc(sizeof(*stream), GFP_KERNEL); 2688 if (!stream) { 2689 ret = -ENOMEM; 2690 goto err_ctx; 2691 } 2692 2693 stream->dev_priv = dev_priv; 2694 stream->ctx = specific_ctx; 2695 2696 ret = i915_oa_stream_init(stream, param, props); 2697 if (ret) 2698 goto err_alloc; 2699 2700 /* we avoid simply assigning stream->sample_flags = props->sample_flags 2701 * to have _stream_init check the combination of sample flags more 2702 * thoroughly, but still this is the expected result at this point. 2703 */ 2704 if (WARN_ON(stream->sample_flags != props->sample_flags)) { 2705 ret = -ENODEV; 2706 goto err_flags; 2707 } 2708 2709 list_add(&stream->link, &dev_priv->perf.streams); 2710 2711 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) 2712 f_flags |= O_CLOEXEC; 2713 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) 2714 f_flags |= O_NONBLOCK; 2715 2716 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); 2717 if (stream_fd < 0) { 2718 ret = stream_fd; 2719 goto err_open; 2720 } 2721 2722 if (!(param->flags & I915_PERF_FLAG_DISABLED)) 2723 i915_perf_enable_locked(stream); 2724 2725 return stream_fd; 2726 2727 err_open: 2728 list_del(&stream->link); 2729 err_flags: 2730 if (stream->ops->destroy) 2731 stream->ops->destroy(stream); 2732 err_alloc: 2733 kfree(stream); 2734 err_ctx: 2735 if (specific_ctx) 2736 i915_gem_context_put(specific_ctx); 2737 err: 2738 return ret; 2739 } 2740 2741 static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) 2742 { 2743 return div64_u64(1000000000ULL * (2ULL << exponent), 2744 1000ULL * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz); 2745 } 2746 2747 /** 2748 * read_properties_unlocked - validate + copy userspace stream open properties 2749 * @dev_priv: i915 device instance 2750 * @uprops: The array of u64 key value pairs given by userspace 2751 * @n_props: The number of key value pairs expected in @uprops 2752 * @props: The stream configuration built up while validating properties 2753 * 2754 * Note this function only validates properties in isolation it doesn't 2755 * validate that the combination of properties makes sense or that all 2756 * properties necessary for a particular kind of stream have been set. 2757 * 2758 * Note that there currently aren't any ordering requirements for properties so 2759 * we shouldn't validate or assume anything about ordering here. This doesn't 2760 * rule out defining new properties with ordering requirements in the future. 2761 */ 2762 static int read_properties_unlocked(struct drm_i915_private *dev_priv, 2763 u64 __user *uprops, 2764 u32 n_props, 2765 struct perf_open_properties *props) 2766 { 2767 u64 __user *uprop = uprops; 2768 u32 i; 2769 2770 memset(props, 0, sizeof(struct perf_open_properties)); 2771 2772 if (!n_props) { 2773 DRM_DEBUG("No i915 perf properties given\n"); 2774 return -EINVAL; 2775 } 2776 2777 /* Considering that ID = 0 is reserved and assuming that we don't 2778 * (currently) expect any configurations to ever specify duplicate 2779 * values for a particular property ID then the last _PROP_MAX value is 2780 * one greater than the maximum number of properties we expect to get 2781 * from userspace. 2782 */ 2783 if (n_props >= DRM_I915_PERF_PROP_MAX) { 2784 DRM_DEBUG("More i915 perf properties specified than exist\n"); 2785 return -EINVAL; 2786 } 2787 2788 for (i = 0; i < n_props; i++) { 2789 u64 oa_period, oa_freq_hz; 2790 u64 id, value; 2791 int ret; 2792 2793 ret = get_user(id, uprop); 2794 if (ret) 2795 return ret; 2796 2797 ret = get_user(value, uprop + 1); 2798 if (ret) 2799 return ret; 2800 2801 if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) { 2802 DRM_DEBUG("Unknown i915 perf property ID\n"); 2803 return -EINVAL; 2804 } 2805 2806 switch ((enum drm_i915_perf_property_id)id) { 2807 case DRM_I915_PERF_PROP_CTX_HANDLE: 2808 props->single_context = 1; 2809 props->ctx_handle = value; 2810 break; 2811 case DRM_I915_PERF_PROP_SAMPLE_OA: 2812 if (value) 2813 props->sample_flags |= SAMPLE_OA_REPORT; 2814 break; 2815 case DRM_I915_PERF_PROP_OA_METRICS_SET: 2816 if (value == 0) { 2817 DRM_DEBUG("Unknown OA metric set ID\n"); 2818 return -EINVAL; 2819 } 2820 props->metrics_set = value; 2821 break; 2822 case DRM_I915_PERF_PROP_OA_FORMAT: 2823 if (value == 0 || value >= I915_OA_FORMAT_MAX) { 2824 DRM_DEBUG("Out-of-range OA report format %llu\n", 2825 value); 2826 return -EINVAL; 2827 } 2828 if (!dev_priv->perf.oa.oa_formats[value].size) { 2829 DRM_DEBUG("Unsupported OA report format %llu\n", 2830 value); 2831 return -EINVAL; 2832 } 2833 props->oa_format = value; 2834 break; 2835 case DRM_I915_PERF_PROP_OA_EXPONENT: 2836 if (value > OA_EXPONENT_MAX) { 2837 DRM_DEBUG("OA timer exponent too high (> %u)\n", 2838 OA_EXPONENT_MAX); 2839 return -EINVAL; 2840 } 2841 2842 /* Theoretically we can program the OA unit to sample 2843 * e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns 2844 * for BXT. We don't allow such high sampling 2845 * frequencies by default unless root. 2846 */ 2847 2848 BUILD_BUG_ON(sizeof(oa_period) != 8); 2849 oa_period = oa_exponent_to_ns(dev_priv, value); 2850 2851 /* This check is primarily to ensure that oa_period <= 2852 * UINT32_MAX (before passing to do_div which only 2853 * accepts a u32 denominator), but we can also skip 2854 * checking anything < 1Hz which implicitly can't be 2855 * limited via an integer oa_max_sample_rate. 2856 */ 2857 if (oa_period <= NSEC_PER_SEC) { 2858 u64 tmp = NSEC_PER_SEC; 2859 do_div(tmp, oa_period); 2860 oa_freq_hz = tmp; 2861 } else 2862 oa_freq_hz = 0; 2863 2864 if (oa_freq_hz > i915_oa_max_sample_rate && 2865 !capable(CAP_SYS_ADMIN)) { 2866 DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n", 2867 i915_oa_max_sample_rate); 2868 return -EACCES; 2869 } 2870 2871 props->oa_periodic = true; 2872 props->oa_period_exponent = value; 2873 break; 2874 case DRM_I915_PERF_PROP_MAX: 2875 MISSING_CASE(id); 2876 return -EINVAL; 2877 } 2878 2879 uprop += 2; 2880 } 2881 2882 return 0; 2883 } 2884 2885 /** 2886 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD 2887 * @dev: drm device 2888 * @data: ioctl data copied from userspace (unvalidated) 2889 * @file: drm file 2890 * 2891 * Validates the stream open parameters given by userspace including flags 2892 * and an array of u64 key, value pair properties. 2893 * 2894 * Very little is assumed up front about the nature of the stream being 2895 * opened (for instance we don't assume it's for periodic OA unit metrics). An 2896 * i915-perf stream is expected to be a suitable interface for other forms of 2897 * buffered data written by the GPU besides periodic OA metrics. 2898 * 2899 * Note we copy the properties from userspace outside of the i915 perf 2900 * mutex to avoid an awkward lockdep with mmap_sem. 2901 * 2902 * Most of the implementation details are handled by 2903 * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock 2904 * mutex for serializing with any non-file-operation driver hooks. 2905 * 2906 * Return: A newly opened i915 Perf stream file descriptor or negative 2907 * error code on failure. 2908 */ 2909 int i915_perf_open_ioctl(struct drm_device *dev, void *data, 2910 struct drm_file *file) 2911 { 2912 struct drm_i915_private *dev_priv = dev->dev_private; 2913 struct drm_i915_perf_open_param *param = data; 2914 struct perf_open_properties props; 2915 u32 known_open_flags; 2916 int ret; 2917 2918 if (!dev_priv->perf.initialized) { 2919 DRM_DEBUG("i915 perf interface not available for this system\n"); 2920 return -ENOTSUPP; 2921 } 2922 2923 known_open_flags = I915_PERF_FLAG_FD_CLOEXEC | 2924 I915_PERF_FLAG_FD_NONBLOCK | 2925 I915_PERF_FLAG_DISABLED; 2926 if (param->flags & ~known_open_flags) { 2927 DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n"); 2928 return -EINVAL; 2929 } 2930 2931 ret = read_properties_unlocked(dev_priv, 2932 u64_to_user_ptr(param->properties_ptr), 2933 param->num_properties, 2934 &props); 2935 if (ret) 2936 return ret; 2937 2938 mutex_lock(&dev_priv->perf.lock); 2939 ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); 2940 mutex_unlock(&dev_priv->perf.lock); 2941 2942 return ret; 2943 } 2944 2945 /** 2946 * i915_perf_register - exposes i915-perf to userspace 2947 * @dev_priv: i915 device instance 2948 * 2949 * In particular OA metric sets are advertised under a sysfs metrics/ 2950 * directory allowing userspace to enumerate valid IDs that can be 2951 * used to open an i915-perf stream. 2952 */ 2953 void i915_perf_register(struct drm_i915_private *dev_priv) 2954 { 2955 int ret; 2956 2957 if (!dev_priv->perf.initialized) 2958 return; 2959 2960 /* To be sure we're synchronized with an attempted 2961 * i915_perf_open_ioctl(); considering that we register after 2962 * being exposed to userspace. 2963 */ 2964 mutex_lock(&dev_priv->perf.lock); 2965 2966 dev_priv->perf.metrics_kobj = 2967 kobject_create_and_add("metrics", 2968 &dev_priv->drm.primary->kdev->kobj); 2969 if (!dev_priv->perf.metrics_kobj) 2970 goto exit; 2971 2972 sysfs_attr_init(&dev_priv->perf.oa.test_config.sysfs_metric_id.attr); 2973 2974 if (IS_HASWELL(dev_priv)) { 2975 i915_perf_load_test_config_hsw(dev_priv); 2976 } else if (IS_BROADWELL(dev_priv)) { 2977 i915_perf_load_test_config_bdw(dev_priv); 2978 } else if (IS_CHERRYVIEW(dev_priv)) { 2979 i915_perf_load_test_config_chv(dev_priv); 2980 } else if (IS_SKYLAKE(dev_priv)) { 2981 if (IS_SKL_GT2(dev_priv)) 2982 i915_perf_load_test_config_sklgt2(dev_priv); 2983 else if (IS_SKL_GT3(dev_priv)) 2984 i915_perf_load_test_config_sklgt3(dev_priv); 2985 else if (IS_SKL_GT4(dev_priv)) 2986 i915_perf_load_test_config_sklgt4(dev_priv); 2987 } else if (IS_BROXTON(dev_priv)) { 2988 i915_perf_load_test_config_bxt(dev_priv); 2989 } else if (IS_KABYLAKE(dev_priv)) { 2990 if (IS_KBL_GT2(dev_priv)) 2991 i915_perf_load_test_config_kblgt2(dev_priv); 2992 else if (IS_KBL_GT3(dev_priv)) 2993 i915_perf_load_test_config_kblgt3(dev_priv); 2994 } else if (IS_GEMINILAKE(dev_priv)) { 2995 i915_perf_load_test_config_glk(dev_priv); 2996 } else if (IS_COFFEELAKE(dev_priv)) { 2997 if (IS_CFL_GT2(dev_priv)) 2998 i915_perf_load_test_config_cflgt2(dev_priv); 2999 if (IS_CFL_GT3(dev_priv)) 3000 i915_perf_load_test_config_cflgt3(dev_priv); 3001 } else if (IS_CANNONLAKE(dev_priv)) { 3002 i915_perf_load_test_config_cnl(dev_priv); 3003 } else if (IS_ICELAKE(dev_priv)) { 3004 i915_perf_load_test_config_icl(dev_priv); 3005 } 3006 3007 if (dev_priv->perf.oa.test_config.id == 0) 3008 goto sysfs_error; 3009 3010 ret = sysfs_create_group(dev_priv->perf.metrics_kobj, 3011 &dev_priv->perf.oa.test_config.sysfs_metric); 3012 if (ret) 3013 goto sysfs_error; 3014 3015 atomic_set(&dev_priv->perf.oa.test_config.ref_count, 1); 3016 3017 goto exit; 3018 3019 sysfs_error: 3020 kobject_put(dev_priv->perf.metrics_kobj); 3021 dev_priv->perf.metrics_kobj = NULL; 3022 3023 exit: 3024 mutex_unlock(&dev_priv->perf.lock); 3025 } 3026 3027 /** 3028 * i915_perf_unregister - hide i915-perf from userspace 3029 * @dev_priv: i915 device instance 3030 * 3031 * i915-perf state cleanup is split up into an 'unregister' and 3032 * 'deinit' phase where the interface is first hidden from 3033 * userspace by i915_perf_unregister() before cleaning up 3034 * remaining state in i915_perf_fini(). 3035 */ 3036 void i915_perf_unregister(struct drm_i915_private *dev_priv) 3037 { 3038 if (!dev_priv->perf.metrics_kobj) 3039 return; 3040 3041 sysfs_remove_group(dev_priv->perf.metrics_kobj, 3042 &dev_priv->perf.oa.test_config.sysfs_metric); 3043 3044 kobject_put(dev_priv->perf.metrics_kobj); 3045 dev_priv->perf.metrics_kobj = NULL; 3046 } 3047 3048 static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr) 3049 { 3050 static const i915_reg_t flex_eu_regs[] = { 3051 EU_PERF_CNTL0, 3052 EU_PERF_CNTL1, 3053 EU_PERF_CNTL2, 3054 EU_PERF_CNTL3, 3055 EU_PERF_CNTL4, 3056 EU_PERF_CNTL5, 3057 EU_PERF_CNTL6, 3058 }; 3059 int i; 3060 3061 for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) { 3062 if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr) 3063 return true; 3064 } 3065 return false; 3066 } 3067 3068 static bool gen7_is_valid_b_counter_addr(struct drm_i915_private *dev_priv, u32 addr) 3069 { 3070 return (addr >= i915_mmio_reg_offset(OASTARTTRIG1) && 3071 addr <= i915_mmio_reg_offset(OASTARTTRIG8)) || 3072 (addr >= i915_mmio_reg_offset(OAREPORTTRIG1) && 3073 addr <= i915_mmio_reg_offset(OAREPORTTRIG8)) || 3074 (addr >= i915_mmio_reg_offset(OACEC0_0) && 3075 addr <= i915_mmio_reg_offset(OACEC7_1)); 3076 } 3077 3078 static bool gen7_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) 3079 { 3080 return addr == i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) || 3081 (addr >= i915_mmio_reg_offset(MICRO_BP0_0) && 3082 addr <= i915_mmio_reg_offset(NOA_WRITE)) || 3083 (addr >= i915_mmio_reg_offset(OA_PERFCNT1_LO) && 3084 addr <= i915_mmio_reg_offset(OA_PERFCNT2_HI)) || 3085 (addr >= i915_mmio_reg_offset(OA_PERFMATRIX_LO) && 3086 addr <= i915_mmio_reg_offset(OA_PERFMATRIX_HI)); 3087 } 3088 3089 static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) 3090 { 3091 return gen7_is_valid_mux_addr(dev_priv, addr) || 3092 addr == i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) || 3093 (addr >= i915_mmio_reg_offset(RPM_CONFIG0) && 3094 addr <= i915_mmio_reg_offset(NOA_CONFIG(8))); 3095 } 3096 3097 static bool gen10_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) 3098 { 3099 return gen8_is_valid_mux_addr(dev_priv, addr) || 3100 (addr >= i915_mmio_reg_offset(OA_PERFCNT3_LO) && 3101 addr <= i915_mmio_reg_offset(OA_PERFCNT4_HI)); 3102 } 3103 3104 static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) 3105 { 3106 return gen7_is_valid_mux_addr(dev_priv, addr) || 3107 (addr >= 0x25100 && addr <= 0x2FF90) || 3108 (addr >= i915_mmio_reg_offset(HSW_MBVID2_NOA0) && 3109 addr <= i915_mmio_reg_offset(HSW_MBVID2_NOA9)) || 3110 addr == i915_mmio_reg_offset(HSW_MBVID2_MISR0); 3111 } 3112 3113 static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) 3114 { 3115 return gen7_is_valid_mux_addr(dev_priv, addr) || 3116 (addr >= 0x182300 && addr <= 0x1823A4); 3117 } 3118 3119 static uint32_t mask_reg_value(u32 reg, u32 val) 3120 { 3121 /* HALF_SLICE_CHICKEN2 is programmed with a the 3122 * WaDisableSTUnitPowerOptimization workaround. Make sure the value 3123 * programmed by userspace doesn't change this. 3124 */ 3125 if (i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) == reg) 3126 val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE); 3127 3128 /* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function 3129 * indicated by its name and a bunch of selection fields used by OA 3130 * configs. 3131 */ 3132 if (i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) == reg) 3133 val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE); 3134 3135 return val; 3136 } 3137 3138 static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv, 3139 bool (*is_valid)(struct drm_i915_private *dev_priv, u32 addr), 3140 u32 __user *regs, 3141 u32 n_regs) 3142 { 3143 struct i915_oa_reg *oa_regs; 3144 int err; 3145 u32 i; 3146 3147 if (!n_regs) 3148 return NULL; 3149 3150 if (!access_ok(VERIFY_READ, regs, n_regs * sizeof(u32) * 2)) 3151 return ERR_PTR(-EFAULT); 3152 3153 /* No is_valid function means we're not allowing any register to be programmed. */ 3154 GEM_BUG_ON(!is_valid); 3155 if (!is_valid) 3156 return ERR_PTR(-EINVAL); 3157 3158 oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL); 3159 if (!oa_regs) 3160 return ERR_PTR(-ENOMEM); 3161 3162 for (i = 0; i < n_regs; i++) { 3163 u32 addr, value; 3164 3165 err = get_user(addr, regs); 3166 if (err) 3167 goto addr_err; 3168 3169 if (!is_valid(dev_priv, addr)) { 3170 DRM_DEBUG("Invalid oa_reg address: %X\n", addr); 3171 err = -EINVAL; 3172 goto addr_err; 3173 } 3174 3175 err = get_user(value, regs + 1); 3176 if (err) 3177 goto addr_err; 3178 3179 oa_regs[i].addr = _MMIO(addr); 3180 oa_regs[i].value = mask_reg_value(addr, value); 3181 3182 regs += 2; 3183 } 3184 3185 return oa_regs; 3186 3187 addr_err: 3188 kfree(oa_regs); 3189 return ERR_PTR(err); 3190 } 3191 3192 static ssize_t show_dynamic_id(struct device *dev, 3193 struct device_attribute *attr, 3194 char *buf) 3195 { 3196 struct i915_oa_config *oa_config = 3197 container_of(attr, typeof(*oa_config), sysfs_metric_id); 3198 3199 return sprintf(buf, "%d\n", oa_config->id); 3200 } 3201 3202 static int create_dynamic_oa_sysfs_entry(struct drm_i915_private *dev_priv, 3203 struct i915_oa_config *oa_config) 3204 { 3205 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); 3206 oa_config->sysfs_metric_id.attr.name = "id"; 3207 oa_config->sysfs_metric_id.attr.mode = S_IRUGO; 3208 oa_config->sysfs_metric_id.show = show_dynamic_id; 3209 oa_config->sysfs_metric_id.store = NULL; 3210 3211 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; 3212 oa_config->attrs[1] = NULL; 3213 3214 oa_config->sysfs_metric.name = oa_config->uuid; 3215 oa_config->sysfs_metric.attrs = oa_config->attrs; 3216 3217 return sysfs_create_group(dev_priv->perf.metrics_kobj, 3218 &oa_config->sysfs_metric); 3219 } 3220 3221 /** 3222 * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config 3223 * @dev: drm device 3224 * @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from 3225 * userspace (unvalidated) 3226 * @file: drm file 3227 * 3228 * Validates the submitted OA register to be saved into a new OA config that 3229 * can then be used for programming the OA unit and its NOA network. 3230 * 3231 * Returns: A new allocated config number to be used with the perf open ioctl 3232 * or a negative error code on failure. 3233 */ 3234 int i915_perf_add_config_ioctl(struct drm_device *dev, void *data, 3235 struct drm_file *file) 3236 { 3237 struct drm_i915_private *dev_priv = dev->dev_private; 3238 struct drm_i915_perf_oa_config *args = data; 3239 struct i915_oa_config *oa_config, *tmp; 3240 int err, id; 3241 3242 if (!dev_priv->perf.initialized) { 3243 DRM_DEBUG("i915 perf interface not available for this system\n"); 3244 return -ENOTSUPP; 3245 } 3246 3247 if (!dev_priv->perf.metrics_kobj) { 3248 DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); 3249 return -EINVAL; 3250 } 3251 3252 if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { 3253 DRM_DEBUG("Insufficient privileges to add i915 OA config\n"); 3254 return -EACCES; 3255 } 3256 3257 if ((!args->mux_regs_ptr || !args->n_mux_regs) && 3258 (!args->boolean_regs_ptr || !args->n_boolean_regs) && 3259 (!args->flex_regs_ptr || !args->n_flex_regs)) { 3260 DRM_DEBUG("No OA registers given\n"); 3261 return -EINVAL; 3262 } 3263 3264 oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL); 3265 if (!oa_config) { 3266 DRM_DEBUG("Failed to allocate memory for the OA config\n"); 3267 return -ENOMEM; 3268 } 3269 3270 atomic_set(&oa_config->ref_count, 1); 3271 3272 if (!uuid_is_valid(args->uuid)) { 3273 DRM_DEBUG("Invalid uuid format for OA config\n"); 3274 err = -EINVAL; 3275 goto reg_err; 3276 } 3277 3278 /* Last character in oa_config->uuid will be 0 because oa_config is 3279 * kzalloc. 3280 */ 3281 memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid)); 3282 3283 oa_config->mux_regs_len = args->n_mux_regs; 3284 oa_config->mux_regs = 3285 alloc_oa_regs(dev_priv, 3286 dev_priv->perf.oa.ops.is_valid_mux_reg, 3287 u64_to_user_ptr(args->mux_regs_ptr), 3288 args->n_mux_regs); 3289 3290 if (IS_ERR(oa_config->mux_regs)) { 3291 DRM_DEBUG("Failed to create OA config for mux_regs\n"); 3292 err = PTR_ERR(oa_config->mux_regs); 3293 goto reg_err; 3294 } 3295 3296 oa_config->b_counter_regs_len = args->n_boolean_regs; 3297 oa_config->b_counter_regs = 3298 alloc_oa_regs(dev_priv, 3299 dev_priv->perf.oa.ops.is_valid_b_counter_reg, 3300 u64_to_user_ptr(args->boolean_regs_ptr), 3301 args->n_boolean_regs); 3302 3303 if (IS_ERR(oa_config->b_counter_regs)) { 3304 DRM_DEBUG("Failed to create OA config for b_counter_regs\n"); 3305 err = PTR_ERR(oa_config->b_counter_regs); 3306 goto reg_err; 3307 } 3308 3309 if (INTEL_GEN(dev_priv) < 8) { 3310 if (args->n_flex_regs != 0) { 3311 err = -EINVAL; 3312 goto reg_err; 3313 } 3314 } else { 3315 oa_config->flex_regs_len = args->n_flex_regs; 3316 oa_config->flex_regs = 3317 alloc_oa_regs(dev_priv, 3318 dev_priv->perf.oa.ops.is_valid_flex_reg, 3319 u64_to_user_ptr(args->flex_regs_ptr), 3320 args->n_flex_regs); 3321 3322 if (IS_ERR(oa_config->flex_regs)) { 3323 DRM_DEBUG("Failed to create OA config for flex_regs\n"); 3324 err = PTR_ERR(oa_config->flex_regs); 3325 goto reg_err; 3326 } 3327 } 3328 3329 err = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); 3330 if (err) 3331 goto reg_err; 3332 3333 /* We shouldn't have too many configs, so this iteration shouldn't be 3334 * too costly. 3335 */ 3336 idr_for_each_entry(&dev_priv->perf.metrics_idr, tmp, id) { 3337 if (!strcmp(tmp->uuid, oa_config->uuid)) { 3338 DRM_DEBUG("OA config already exists with this uuid\n"); 3339 err = -EADDRINUSE; 3340 goto sysfs_err; 3341 } 3342 } 3343 3344 err = create_dynamic_oa_sysfs_entry(dev_priv, oa_config); 3345 if (err) { 3346 DRM_DEBUG("Failed to create sysfs entry for OA config\n"); 3347 goto sysfs_err; 3348 } 3349 3350 /* Config id 0 is invalid, id 1 for kernel stored test config. */ 3351 oa_config->id = idr_alloc(&dev_priv->perf.metrics_idr, 3352 oa_config, 2, 3353 0, GFP_KERNEL); 3354 if (oa_config->id < 0) { 3355 DRM_DEBUG("Failed to create sysfs entry for OA config\n"); 3356 err = oa_config->id; 3357 goto sysfs_err; 3358 } 3359 3360 mutex_unlock(&dev_priv->perf.metrics_lock); 3361 3362 DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id); 3363 3364 return oa_config->id; 3365 3366 sysfs_err: 3367 mutex_unlock(&dev_priv->perf.metrics_lock); 3368 reg_err: 3369 put_oa_config(dev_priv, oa_config); 3370 DRM_DEBUG("Failed to add new OA config\n"); 3371 return err; 3372 } 3373 3374 /** 3375 * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config 3376 * @dev: drm device 3377 * @data: ioctl data (pointer to u64 integer) copied from userspace 3378 * @file: drm file 3379 * 3380 * Configs can be removed while being used, the will stop appearing in sysfs 3381 * and their content will be freed when the stream using the config is closed. 3382 * 3383 * Returns: 0 on success or a negative error code on failure. 3384 */ 3385 int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data, 3386 struct drm_file *file) 3387 { 3388 struct drm_i915_private *dev_priv = dev->dev_private; 3389 u64 *arg = data; 3390 struct i915_oa_config *oa_config; 3391 int ret; 3392 3393 if (!dev_priv->perf.initialized) { 3394 DRM_DEBUG("i915 perf interface not available for this system\n"); 3395 return -ENOTSUPP; 3396 } 3397 3398 if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { 3399 DRM_DEBUG("Insufficient privileges to remove i915 OA config\n"); 3400 return -EACCES; 3401 } 3402 3403 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); 3404 if (ret) 3405 goto lock_err; 3406 3407 oa_config = idr_find(&dev_priv->perf.metrics_idr, *arg); 3408 if (!oa_config) { 3409 DRM_DEBUG("Failed to remove unknown OA config\n"); 3410 ret = -ENOENT; 3411 goto config_err; 3412 } 3413 3414 GEM_BUG_ON(*arg != oa_config->id); 3415 3416 sysfs_remove_group(dev_priv->perf.metrics_kobj, 3417 &oa_config->sysfs_metric); 3418 3419 idr_remove(&dev_priv->perf.metrics_idr, *arg); 3420 3421 DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id); 3422 3423 put_oa_config(dev_priv, oa_config); 3424 3425 config_err: 3426 mutex_unlock(&dev_priv->perf.metrics_lock); 3427 lock_err: 3428 return ret; 3429 } 3430 3431 static struct ctl_table oa_table[] = { 3432 { 3433 .procname = "perf_stream_paranoid", 3434 .data = &i915_perf_stream_paranoid, 3435 .maxlen = sizeof(i915_perf_stream_paranoid), 3436 .mode = 0644, 3437 .proc_handler = proc_dointvec_minmax, 3438 .extra1 = &zero, 3439 .extra2 = &one, 3440 }, 3441 { 3442 .procname = "oa_max_sample_rate", 3443 .data = &i915_oa_max_sample_rate, 3444 .maxlen = sizeof(i915_oa_max_sample_rate), 3445 .mode = 0644, 3446 .proc_handler = proc_dointvec_minmax, 3447 .extra1 = &zero, 3448 .extra2 = &oa_sample_rate_hard_limit, 3449 }, 3450 {} 3451 }; 3452 3453 static struct ctl_table i915_root[] = { 3454 { 3455 .procname = "i915", 3456 .maxlen = 0, 3457 .mode = 0555, 3458 .child = oa_table, 3459 }, 3460 {} 3461 }; 3462 3463 static struct ctl_table dev_root[] = { 3464 { 3465 .procname = "dev", 3466 .maxlen = 0, 3467 .mode = 0555, 3468 .child = i915_root, 3469 }, 3470 {} 3471 }; 3472 3473 /** 3474 * i915_perf_init - initialize i915-perf state on module load 3475 * @dev_priv: i915 device instance 3476 * 3477 * Initializes i915-perf state without exposing anything to userspace. 3478 * 3479 * Note: i915-perf initialization is split into an 'init' and 'register' 3480 * phase with the i915_perf_register() exposing state to userspace. 3481 */ 3482 void i915_perf_init(struct drm_i915_private *dev_priv) 3483 { 3484 if (IS_HASWELL(dev_priv)) { 3485 dev_priv->perf.oa.ops.is_valid_b_counter_reg = 3486 gen7_is_valid_b_counter_addr; 3487 dev_priv->perf.oa.ops.is_valid_mux_reg = 3488 hsw_is_valid_mux_addr; 3489 dev_priv->perf.oa.ops.is_valid_flex_reg = NULL; 3490 dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; 3491 dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; 3492 dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; 3493 dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; 3494 dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; 3495 dev_priv->perf.oa.ops.read = gen7_oa_read; 3496 dev_priv->perf.oa.ops.oa_hw_tail_read = 3497 gen7_oa_hw_tail_read; 3498 3499 dev_priv->perf.oa.oa_formats = hsw_oa_formats; 3500 } else if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { 3501 /* Note: that although we could theoretically also support the 3502 * legacy ringbuffer mode on BDW (and earlier iterations of 3503 * this driver, before upstreaming did this) it didn't seem 3504 * worth the complexity to maintain now that BDW+ enable 3505 * execlist mode by default. 3506 */ 3507 dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats; 3508 3509 dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer; 3510 dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable; 3511 dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable; 3512 dev_priv->perf.oa.ops.read = gen8_oa_read; 3513 dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read; 3514 3515 if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) { 3516 dev_priv->perf.oa.ops.is_valid_b_counter_reg = 3517 gen7_is_valid_b_counter_addr; 3518 dev_priv->perf.oa.ops.is_valid_mux_reg = 3519 gen8_is_valid_mux_addr; 3520 dev_priv->perf.oa.ops.is_valid_flex_reg = 3521 gen8_is_valid_flex_addr; 3522 3523 if (IS_CHERRYVIEW(dev_priv)) { 3524 dev_priv->perf.oa.ops.is_valid_mux_reg = 3525 chv_is_valid_mux_addr; 3526 } 3527 3528 dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; 3529 dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set; 3530 3531 if (IS_GEN8(dev_priv)) { 3532 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120; 3533 dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce; 3534 3535 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25); 3536 } else { 3537 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; 3538 dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; 3539 3540 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); 3541 } 3542 } else if (IS_GEN(dev_priv, 10, 11)) { 3543 dev_priv->perf.oa.ops.is_valid_b_counter_reg = 3544 gen7_is_valid_b_counter_addr; 3545 dev_priv->perf.oa.ops.is_valid_mux_reg = 3546 gen10_is_valid_mux_addr; 3547 dev_priv->perf.oa.ops.is_valid_flex_reg = 3548 gen8_is_valid_flex_addr; 3549 3550 dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; 3551 dev_priv->perf.oa.ops.disable_metric_set = gen10_disable_metric_set; 3552 3553 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; 3554 dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; 3555 3556 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); 3557 } 3558 } 3559 3560 if (dev_priv->perf.oa.ops.enable_metric_set) { 3561 hrtimer_init(&dev_priv->perf.oa.poll_check_timer, 3562 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3563 dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; 3564 init_waitqueue_head(&dev_priv->perf.oa.poll_wq); 3565 3566 INIT_LIST_HEAD(&dev_priv->perf.streams); 3567 mutex_init(&dev_priv->perf.lock); 3568 spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock); 3569 3570 oa_sample_rate_hard_limit = 1000 * 3571 (INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); 3572 dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); 3573 3574 mutex_init(&dev_priv->perf.metrics_lock); 3575 idr_init(&dev_priv->perf.metrics_idr); 3576 3577 dev_priv->perf.initialized = true; 3578 } 3579 } 3580 3581 static int destroy_config(int id, void *p, void *data) 3582 { 3583 struct drm_i915_private *dev_priv = data; 3584 struct i915_oa_config *oa_config = p; 3585 3586 put_oa_config(dev_priv, oa_config); 3587 3588 return 0; 3589 } 3590 3591 /** 3592 * i915_perf_fini - Counter part to i915_perf_init() 3593 * @dev_priv: i915 device instance 3594 */ 3595 void i915_perf_fini(struct drm_i915_private *dev_priv) 3596 { 3597 if (!dev_priv->perf.initialized) 3598 return; 3599 3600 idr_for_each(&dev_priv->perf.metrics_idr, destroy_config, dev_priv); 3601 idr_destroy(&dev_priv->perf.metrics_idr); 3602 3603 unregister_sysctl_table(dev_priv->perf.sysctl_header); 3604 3605 memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); 3606 3607 dev_priv->perf.initialized = false; 3608 } 3609