1 /* 2 * Performance event support for the System z CPU-measurement Sampling Facility 3 * 4 * Copyright IBM Corp. 2013 5 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License (version 2 only) 9 * as published by the Free Software Foundation. 10 */ 11 #define KMSG_COMPONENT "cpum_sf" 12 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 13 14 #include <linux/kernel.h> 15 #include <linux/kernel_stat.h> 16 #include <linux/perf_event.h> 17 #include <linux/percpu.h> 18 #include <linux/pid.h> 19 #include <linux/notifier.h> 20 #include <linux/export.h> 21 #include <linux/slab.h> 22 #include <linux/mm.h> 23 #include <linux/moduleparam.h> 24 #include <asm/cpu_mf.h> 25 #include <asm/irq.h> 26 #include <asm/debug.h> 27 #include <asm/timex.h> 28 29 /* Minimum number of sample-data-block-tables: 30 * At least one table is required for the sampling buffer structure. 31 * A single table contains up to 511 pointers to sample-data-blocks. 32 */ 33 #define CPUM_SF_MIN_SDBT 1 34 35 /* Number of sample-data-blocks per sample-data-block-table (SDBT): 36 * A table contains SDB pointers (8 bytes) and one table-link entry 37 * that points to the origin of the next SDBT. 38 */ 39 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8) 40 41 /* Maximum page offset for an SDBT table-link entry: 42 * If this page offset is reached, a table-link entry to the next SDBT 43 * must be added. 44 */ 45 #define CPUM_SF_SDBT_TL_OFFSET (CPUM_SF_SDB_PER_TABLE * 8) 46 static inline int require_table_link(const void *sdbt) 47 { 48 return ((unsigned long) sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET; 49 } 50 51 /* Minimum and maximum sampling buffer sizes: 52 * 53 * This number represents the maximum size of the sampling buffer taking 54 * the number of sample-data-block-tables into account. Note that these 55 * numbers apply to the basic-sampling function only. 56 * The maximum number of SDBs is increased by CPUM_SF_SDB_DIAG_FACTOR if 57 * the diagnostic-sampling function is active. 58 * 59 * Sampling buffer size Buffer characteristics 60 * --------------------------------------------------- 61 * 64KB == 16 pages (4KB per page) 62 * 1 page for SDB-tables 63 * 15 pages for SDBs 64 * 65 * 32MB == 8192 pages (4KB per page) 66 * 16 pages for SDB-tables 67 * 8176 pages for SDBs 68 */ 69 static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15; 70 static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176; 71 static unsigned long __read_mostly CPUM_SF_SDB_DIAG_FACTOR = 1; 72 73 struct sf_buffer { 74 unsigned long *sdbt; /* Sample-data-block-table origin */ 75 /* buffer characteristics (required for buffer increments) */ 76 unsigned long num_sdb; /* Number of sample-data-blocks */ 77 unsigned long num_sdbt; /* Number of sample-data-block-tables */ 78 unsigned long *tail; /* last sample-data-block-table */ 79 }; 80 81 struct aux_buffer { 82 struct sf_buffer sfb; 83 unsigned long head; /* index of SDB of buffer head */ 84 unsigned long alert_mark; /* index of SDB of alert request position */ 85 unsigned long empty_mark; /* mark of SDB not marked full */ 86 unsigned long *sdb_index; /* SDB address for fast lookup */ 87 unsigned long *sdbt_index; /* SDBT address for fast lookup */ 88 }; 89 90 struct cpu_hw_sf { 91 /* CPU-measurement sampling information block */ 92 struct hws_qsi_info_block qsi; 93 /* CPU-measurement sampling control block */ 94 struct hws_lsctl_request_block lsctl; 95 struct sf_buffer sfb; /* Sampling buffer */ 96 unsigned int flags; /* Status flags */ 97 struct perf_event *event; /* Scheduled perf event */ 98 struct perf_output_handle handle; /* AUX buffer output handle */ 99 }; 100 static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf); 101 102 /* Debug feature */ 103 static debug_info_t *sfdbg; 104 105 /* 106 * sf_disable() - Switch off sampling facility 107 */ 108 static int sf_disable(void) 109 { 110 struct hws_lsctl_request_block sreq; 111 112 memset(&sreq, 0, sizeof(sreq)); 113 return lsctl(&sreq); 114 } 115 116 /* 117 * sf_buffer_available() - Check for an allocated sampling buffer 118 */ 119 static int sf_buffer_available(struct cpu_hw_sf *cpuhw) 120 { 121 return !!cpuhw->sfb.sdbt; 122 } 123 124 /* 125 * deallocate sampling facility buffer 126 */ 127 static void free_sampling_buffer(struct sf_buffer *sfb) 128 { 129 unsigned long *sdbt, *curr; 130 131 if (!sfb->sdbt) 132 return; 133 134 sdbt = sfb->sdbt; 135 curr = sdbt; 136 137 /* Free the SDBT after all SDBs are processed... */ 138 while (1) { 139 if (!*curr || !sdbt) 140 break; 141 142 /* Process table-link entries */ 143 if (is_link_entry(curr)) { 144 curr = get_next_sdbt(curr); 145 if (sdbt) 146 free_page((unsigned long) sdbt); 147 148 /* If the origin is reached, sampling buffer is freed */ 149 if (curr == sfb->sdbt) 150 break; 151 else 152 sdbt = curr; 153 } else { 154 /* Process SDB pointer */ 155 if (*curr) { 156 free_page(*curr); 157 curr++; 158 } 159 } 160 } 161 162 debug_sprintf_event(sfdbg, 5, 163 "free_sampling_buffer: freed sdbt=%p\n", sfb->sdbt); 164 memset(sfb, 0, sizeof(*sfb)); 165 } 166 167 static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags) 168 { 169 unsigned long sdb, *trailer; 170 171 /* Allocate and initialize sample-data-block */ 172 sdb = get_zeroed_page(gfp_flags); 173 if (!sdb) 174 return -ENOMEM; 175 trailer = trailer_entry_ptr(sdb); 176 *trailer = SDB_TE_ALERT_REQ_MASK; 177 178 /* Link SDB into the sample-data-block-table */ 179 *sdbt = sdb; 180 181 return 0; 182 } 183 184 /* 185 * realloc_sampling_buffer() - extend sampler memory 186 * 187 * Allocates new sample-data-blocks and adds them to the specified sampling 188 * buffer memory. 189 * 190 * Important: This modifies the sampling buffer and must be called when the 191 * sampling facility is disabled. 192 * 193 * Returns zero on success, non-zero otherwise. 194 */ 195 static int realloc_sampling_buffer(struct sf_buffer *sfb, 196 unsigned long num_sdb, gfp_t gfp_flags) 197 { 198 int i, rc; 199 unsigned long *new, *tail; 200 201 if (!sfb->sdbt || !sfb->tail) 202 return -EINVAL; 203 204 if (!is_link_entry(sfb->tail)) 205 return -EINVAL; 206 207 /* Append to the existing sampling buffer, overwriting the table-link 208 * register. 209 * The tail variables always points to the "tail" (last and table-link) 210 * entry in an SDB-table. 211 */ 212 tail = sfb->tail; 213 214 /* Do a sanity check whether the table-link entry points to 215 * the sampling buffer origin. 216 */ 217 if (sfb->sdbt != get_next_sdbt(tail)) { 218 debug_sprintf_event(sfdbg, 3, "realloc_sampling_buffer: " 219 "sampling buffer is not linked: origin=%p" 220 "tail=%p\n", 221 (void *) sfb->sdbt, (void *) tail); 222 return -EINVAL; 223 } 224 225 /* Allocate remaining SDBs */ 226 rc = 0; 227 for (i = 0; i < num_sdb; i++) { 228 /* Allocate a new SDB-table if it is full. */ 229 if (require_table_link(tail)) { 230 new = (unsigned long *) get_zeroed_page(gfp_flags); 231 if (!new) { 232 rc = -ENOMEM; 233 break; 234 } 235 sfb->num_sdbt++; 236 /* Link current page to tail of chain */ 237 *tail = (unsigned long)(void *) new + 1; 238 tail = new; 239 } 240 241 /* Allocate a new sample-data-block. 242 * If there is not enough memory, stop the realloc process 243 * and simply use what was allocated. If this is a temporary 244 * issue, a new realloc call (if required) might succeed. 245 */ 246 rc = alloc_sample_data_block(tail, gfp_flags); 247 if (rc) 248 break; 249 sfb->num_sdb++; 250 tail++; 251 } 252 253 /* Link sampling buffer to its origin */ 254 *tail = (unsigned long) sfb->sdbt + 1; 255 sfb->tail = tail; 256 257 debug_sprintf_event(sfdbg, 4, "realloc_sampling_buffer: new buffer" 258 " settings: sdbt=%lu sdb=%lu\n", 259 sfb->num_sdbt, sfb->num_sdb); 260 return rc; 261 } 262 263 /* 264 * allocate_sampling_buffer() - allocate sampler memory 265 * 266 * Allocates and initializes a sampling buffer structure using the 267 * specified number of sample-data-blocks (SDB). For each allocation, 268 * a 4K page is used. The number of sample-data-block-tables (SDBT) 269 * are calculated from SDBs. 270 * Also set the ALERT_REQ mask in each SDBs trailer. 271 * 272 * Returns zero on success, non-zero otherwise. 273 */ 274 static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb) 275 { 276 int rc; 277 278 if (sfb->sdbt) 279 return -EINVAL; 280 281 /* Allocate the sample-data-block-table origin */ 282 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); 283 if (!sfb->sdbt) 284 return -ENOMEM; 285 sfb->num_sdb = 0; 286 sfb->num_sdbt = 1; 287 288 /* Link the table origin to point to itself to prepare for 289 * realloc_sampling_buffer() invocation. 290 */ 291 sfb->tail = sfb->sdbt; 292 *sfb->tail = (unsigned long)(void *) sfb->sdbt + 1; 293 294 /* Allocate requested number of sample-data-blocks */ 295 rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL); 296 if (rc) { 297 free_sampling_buffer(sfb); 298 debug_sprintf_event(sfdbg, 4, "alloc_sampling_buffer: " 299 "realloc_sampling_buffer failed with rc=%i\n", rc); 300 } else 301 debug_sprintf_event(sfdbg, 4, 302 "alloc_sampling_buffer: tear=%p dear=%p\n", 303 sfb->sdbt, (void *) *sfb->sdbt); 304 return rc; 305 } 306 307 static void sfb_set_limits(unsigned long min, unsigned long max) 308 { 309 struct hws_qsi_info_block si; 310 311 CPUM_SF_MIN_SDB = min; 312 CPUM_SF_MAX_SDB = max; 313 314 memset(&si, 0, sizeof(si)); 315 if (!qsi(&si)) 316 CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes); 317 } 318 319 static unsigned long sfb_max_limit(struct hw_perf_event *hwc) 320 { 321 return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR 322 : CPUM_SF_MAX_SDB; 323 } 324 325 static unsigned long sfb_pending_allocs(struct sf_buffer *sfb, 326 struct hw_perf_event *hwc) 327 { 328 if (!sfb->sdbt) 329 return SFB_ALLOC_REG(hwc); 330 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) 331 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; 332 return 0; 333 } 334 335 static int sfb_has_pending_allocs(struct sf_buffer *sfb, 336 struct hw_perf_event *hwc) 337 { 338 return sfb_pending_allocs(sfb, hwc) > 0; 339 } 340 341 static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc) 342 { 343 /* Limit the number of SDBs to not exceed the maximum */ 344 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); 345 if (num) 346 SFB_ALLOC_REG(hwc) += num; 347 } 348 349 static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc) 350 { 351 SFB_ALLOC_REG(hwc) = 0; 352 sfb_account_allocs(num, hwc); 353 } 354 355 static void deallocate_buffers(struct cpu_hw_sf *cpuhw) 356 { 357 if (cpuhw->sfb.sdbt) 358 free_sampling_buffer(&cpuhw->sfb); 359 } 360 361 static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc) 362 { 363 unsigned long n_sdb, freq, factor; 364 size_t sample_size; 365 366 /* Calculate sampling buffers using 4K pages 367 * 368 * 1. Determine the sample data size which depends on the used 369 * sampling functions, for example, basic-sampling or 370 * basic-sampling with diagnostic-sampling. 371 * 372 * 2. Use the sampling frequency as input. The sampling buffer is 373 * designed for almost one second. This can be adjusted through 374 * the "factor" variable. 375 * In any case, alloc_sampling_buffer() sets the Alert Request 376 * Control indicator to trigger a measurement-alert to harvest 377 * sample-data-blocks (sdb). 378 * 379 * 3. Compute the number of sample-data-blocks and ensure a minimum 380 * of CPUM_SF_MIN_SDB. Also ensure the upper limit does not 381 * exceed a "calculated" maximum. The symbolic maximum is 382 * designed for basic-sampling only and needs to be increased if 383 * diagnostic-sampling is active. 384 * See also the remarks for these symbolic constants. 385 * 386 * 4. Compute the number of sample-data-block-tables (SDBT) and 387 * ensure a minimum of CPUM_SF_MIN_SDBT (one table can manage up 388 * to 511 SDBs). 389 */ 390 sample_size = sizeof(struct hws_basic_entry); 391 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); 392 factor = 1; 393 n_sdb = DIV_ROUND_UP(freq, factor * ((PAGE_SIZE-64) / sample_size)); 394 if (n_sdb < CPUM_SF_MIN_SDB) 395 n_sdb = CPUM_SF_MIN_SDB; 396 397 /* If there is already a sampling buffer allocated, it is very likely 398 * that the sampling facility is enabled too. If the event to be 399 * initialized requires a greater sampling buffer, the allocation must 400 * be postponed. Changing the sampling buffer requires the sampling 401 * facility to be in the disabled state. So, account the number of 402 * required SDBs and let cpumsf_pmu_enable() resize the buffer just 403 * before the event is started. 404 */ 405 sfb_init_allocs(n_sdb, hwc); 406 if (sf_buffer_available(cpuhw)) 407 return 0; 408 409 debug_sprintf_event(sfdbg, 3, 410 "allocate_buffers: rate=%lu f=%lu sdb=%lu/%lu" 411 " sample_size=%lu cpuhw=%p\n", 412 SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc), 413 sample_size, cpuhw); 414 415 return alloc_sampling_buffer(&cpuhw->sfb, 416 sfb_pending_allocs(&cpuhw->sfb, hwc)); 417 } 418 419 static unsigned long min_percent(unsigned int percent, unsigned long base, 420 unsigned long min) 421 { 422 return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100)); 423 } 424 425 static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base) 426 { 427 /* Use a percentage-based approach to extend the sampling facility 428 * buffer. Accept up to 5% sample data loss. 429 * Vary the extents between 1% to 5% of the current number of 430 * sample-data-blocks. 431 */ 432 if (ratio <= 5) 433 return 0; 434 if (ratio <= 25) 435 return min_percent(1, base, 1); 436 if (ratio <= 50) 437 return min_percent(1, base, 1); 438 if (ratio <= 75) 439 return min_percent(2, base, 2); 440 if (ratio <= 100) 441 return min_percent(3, base, 3); 442 if (ratio <= 250) 443 return min_percent(4, base, 4); 444 445 return min_percent(5, base, 8); 446 } 447 448 static void sfb_account_overflows(struct cpu_hw_sf *cpuhw, 449 struct hw_perf_event *hwc) 450 { 451 unsigned long ratio, num; 452 453 if (!OVERFLOW_REG(hwc)) 454 return; 455 456 /* The sample_overflow contains the average number of sample data 457 * that has been lost because sample-data-blocks were full. 458 * 459 * Calculate the total number of sample data entries that has been 460 * discarded. Then calculate the ratio of lost samples to total samples 461 * per second in percent. 462 */ 463 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, 464 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); 465 466 /* Compute number of sample-data-blocks */ 467 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); 468 if (num) 469 sfb_account_allocs(num, hwc); 470 471 debug_sprintf_event(sfdbg, 5, "sfb: overflow: overflow=%llu ratio=%lu" 472 " num=%lu\n", OVERFLOW_REG(hwc), ratio, num); 473 OVERFLOW_REG(hwc) = 0; 474 } 475 476 /* extend_sampling_buffer() - Extend sampling buffer 477 * @sfb: Sampling buffer structure (for local CPU) 478 * @hwc: Perf event hardware structure 479 * 480 * Use this function to extend the sampling buffer based on the overflow counter 481 * and postponed allocation extents stored in the specified Perf event hardware. 482 * 483 * Important: This function disables the sampling facility in order to safely 484 * change the sampling buffer structure. Do not call this function 485 * when the PMU is active. 486 */ 487 static void extend_sampling_buffer(struct sf_buffer *sfb, 488 struct hw_perf_event *hwc) 489 { 490 unsigned long num, num_old; 491 int rc; 492 493 num = sfb_pending_allocs(sfb, hwc); 494 if (!num) 495 return; 496 num_old = sfb->num_sdb; 497 498 /* Disable the sampling facility to reset any states and also 499 * clear pending measurement alerts. 500 */ 501 sf_disable(); 502 503 /* Extend the sampling buffer. 504 * This memory allocation typically happens in an atomic context when 505 * called by perf. Because this is a reallocation, it is fine if the 506 * new SDB-request cannot be satisfied immediately. 507 */ 508 rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC); 509 if (rc) 510 debug_sprintf_event(sfdbg, 5, "sfb: extend: realloc " 511 "failed with rc=%i\n", rc); 512 513 if (sfb_has_pending_allocs(sfb, hwc)) 514 debug_sprintf_event(sfdbg, 5, "sfb: extend: " 515 "req=%lu alloc=%lu remaining=%lu\n", 516 num, sfb->num_sdb - num_old, 517 sfb_pending_allocs(sfb, hwc)); 518 } 519 520 521 /* Number of perf events counting hardware events */ 522 static atomic_t num_events; 523 /* Used to avoid races in calling reserve/release_cpumf_hardware */ 524 static DEFINE_MUTEX(pmc_reserve_mutex); 525 526 #define PMC_INIT 0 527 #define PMC_RELEASE 1 528 #define PMC_FAILURE 2 529 static void setup_pmc_cpu(void *flags) 530 { 531 int err; 532 struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf); 533 534 err = 0; 535 switch (*((int *) flags)) { 536 case PMC_INIT: 537 memset(cpusf, 0, sizeof(*cpusf)); 538 err = qsi(&cpusf->qsi); 539 if (err) 540 break; 541 cpusf->flags |= PMU_F_RESERVED; 542 err = sf_disable(); 543 if (err) 544 pr_err("Switching off the sampling facility failed " 545 "with rc=%i\n", err); 546 debug_sprintf_event(sfdbg, 5, 547 "setup_pmc_cpu: initialized: cpuhw=%p\n", cpusf); 548 break; 549 case PMC_RELEASE: 550 cpusf->flags &= ~PMU_F_RESERVED; 551 err = sf_disable(); 552 if (err) { 553 pr_err("Switching off the sampling facility failed " 554 "with rc=%i\n", err); 555 } else 556 deallocate_buffers(cpusf); 557 debug_sprintf_event(sfdbg, 5, 558 "setup_pmc_cpu: released: cpuhw=%p\n", cpusf); 559 break; 560 } 561 if (err) 562 *((int *) flags) |= PMC_FAILURE; 563 } 564 565 static void release_pmc_hardware(void) 566 { 567 int flags = PMC_RELEASE; 568 569 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); 570 on_each_cpu(setup_pmc_cpu, &flags, 1); 571 } 572 573 static int reserve_pmc_hardware(void) 574 { 575 int flags = PMC_INIT; 576 577 on_each_cpu(setup_pmc_cpu, &flags, 1); 578 if (flags & PMC_FAILURE) { 579 release_pmc_hardware(); 580 return -ENODEV; 581 } 582 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); 583 584 return 0; 585 } 586 587 static void hw_perf_event_destroy(struct perf_event *event) 588 { 589 /* Release PMC if this is the last perf event */ 590 if (!atomic_add_unless(&num_events, -1, 1)) { 591 mutex_lock(&pmc_reserve_mutex); 592 if (atomic_dec_return(&num_events) == 0) 593 release_pmc_hardware(); 594 mutex_unlock(&pmc_reserve_mutex); 595 } 596 } 597 598 static void hw_init_period(struct hw_perf_event *hwc, u64 period) 599 { 600 hwc->sample_period = period; 601 hwc->last_period = hwc->sample_period; 602 local64_set(&hwc->period_left, hwc->sample_period); 603 } 604 605 static void hw_reset_registers(struct hw_perf_event *hwc, 606 unsigned long *sdbt_origin) 607 { 608 /* (Re)set to first sample-data-block-table */ 609 TEAR_REG(hwc) = (unsigned long) sdbt_origin; 610 } 611 612 static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si, 613 unsigned long rate) 614 { 615 return clamp_t(unsigned long, rate, 616 si->min_sampl_rate, si->max_sampl_rate); 617 } 618 619 static u32 cpumsf_pid_type(struct perf_event *event, 620 u32 pid, enum pid_type type) 621 { 622 struct task_struct *tsk; 623 624 /* Idle process */ 625 if (!pid) 626 goto out; 627 628 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 629 pid = -1; 630 if (tsk) { 631 /* 632 * Only top level events contain the pid namespace in which 633 * they are created. 634 */ 635 if (event->parent) 636 event = event->parent; 637 pid = __task_pid_nr_ns(tsk, type, event->ns); 638 /* 639 * See also 1d953111b648 640 * "perf/core: Don't report zero PIDs for exiting tasks". 641 */ 642 if (!pid && !pid_alive(tsk)) 643 pid = -1; 644 } 645 out: 646 return pid; 647 } 648 649 static void cpumsf_output_event_pid(struct perf_event *event, 650 struct perf_sample_data *data, 651 struct pt_regs *regs) 652 { 653 u32 pid; 654 struct perf_event_header header; 655 struct perf_output_handle handle; 656 657 /* 658 * Obtain the PID from the basic-sampling data entry and 659 * correct the data->tid_entry.pid value. 660 */ 661 pid = data->tid_entry.pid; 662 663 /* Protect callchain buffers, tasks */ 664 rcu_read_lock(); 665 666 perf_prepare_sample(&header, data, event, regs); 667 if (perf_output_begin(&handle, event, header.size)) 668 goto out; 669 670 /* Update the process ID (see also kernel/events/core.c) */ 671 data->tid_entry.pid = cpumsf_pid_type(event, pid, __PIDTYPE_TGID); 672 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); 673 674 perf_output_sample(&handle, &header, data, event); 675 perf_output_end(&handle); 676 out: 677 rcu_read_unlock(); 678 } 679 680 static int __hw_perf_event_init(struct perf_event *event) 681 { 682 struct cpu_hw_sf *cpuhw; 683 struct hws_qsi_info_block si; 684 struct perf_event_attr *attr = &event->attr; 685 struct hw_perf_event *hwc = &event->hw; 686 unsigned long rate; 687 int cpu, err; 688 689 /* Reserve CPU-measurement sampling facility */ 690 err = 0; 691 if (!atomic_inc_not_zero(&num_events)) { 692 mutex_lock(&pmc_reserve_mutex); 693 if (atomic_read(&num_events) == 0 && reserve_pmc_hardware()) 694 err = -EBUSY; 695 else 696 atomic_inc(&num_events); 697 mutex_unlock(&pmc_reserve_mutex); 698 } 699 event->destroy = hw_perf_event_destroy; 700 701 if (err) 702 goto out; 703 704 /* Access per-CPU sampling information (query sampling info) */ 705 /* 706 * The event->cpu value can be -1 to count on every CPU, for example, 707 * when attaching to a task. If this is specified, use the query 708 * sampling info from the current CPU, otherwise use event->cpu to 709 * retrieve the per-CPU information. 710 * Later, cpuhw indicates whether to allocate sampling buffers for a 711 * particular CPU (cpuhw!=NULL) or each online CPU (cpuw==NULL). 712 */ 713 memset(&si, 0, sizeof(si)); 714 cpuhw = NULL; 715 if (event->cpu == -1) 716 qsi(&si); 717 else { 718 /* Event is pinned to a particular CPU, retrieve the per-CPU 719 * sampling structure for accessing the CPU-specific QSI. 720 */ 721 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); 722 si = cpuhw->qsi; 723 } 724 725 /* Check sampling facility authorization and, if not authorized, 726 * fall back to other PMUs. It is safe to check any CPU because 727 * the authorization is identical for all configured CPUs. 728 */ 729 if (!si.as) { 730 err = -ENOENT; 731 goto out; 732 } 733 734 /* Always enable basic sampling */ 735 SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE; 736 737 /* Check if diagnostic sampling is requested. Deny if the required 738 * sampling authorization is missing. 739 */ 740 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { 741 if (!si.ad) { 742 err = -EPERM; 743 goto out; 744 } 745 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE; 746 } 747 748 /* Check and set other sampling flags */ 749 if (attr->config1 & PERF_CPUM_SF_FULL_BLOCKS) 750 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FULL_BLOCKS; 751 752 /* The sampling information (si) contains information about the 753 * min/max sampling intervals and the CPU speed. So calculate the 754 * correct sampling interval and avoid the whole period adjust 755 * feedback loop. 756 */ 757 rate = 0; 758 if (attr->freq) { 759 rate = freq_to_sample_rate(&si, attr->sample_freq); 760 rate = hw_limit_rate(&si, rate); 761 attr->freq = 0; 762 attr->sample_period = rate; 763 } else { 764 /* The min/max sampling rates specifies the valid range 765 * of sample periods. If the specified sample period is 766 * out of range, limit the period to the range boundary. 767 */ 768 rate = hw_limit_rate(&si, hwc->sample_period); 769 770 /* The perf core maintains a maximum sample rate that is 771 * configurable through the sysctl interface. Ensure the 772 * sampling rate does not exceed this value. This also helps 773 * to avoid throttling when pushing samples with 774 * perf_event_overflow(). 775 */ 776 if (sample_rate_to_freq(&si, rate) > 777 sysctl_perf_event_sample_rate) { 778 err = -EINVAL; 779 debug_sprintf_event(sfdbg, 1, "Sampling rate exceeds maximum perf sample rate\n"); 780 goto out; 781 } 782 } 783 SAMPL_RATE(hwc) = rate; 784 hw_init_period(hwc, SAMPL_RATE(hwc)); 785 786 /* Initialize sample data overflow accounting */ 787 hwc->extra_reg.reg = REG_OVERFLOW; 788 OVERFLOW_REG(hwc) = 0; 789 790 /* Use AUX buffer. No need to allocate it by ourself */ 791 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) 792 return 0; 793 794 /* Allocate the per-CPU sampling buffer using the CPU information 795 * from the event. If the event is not pinned to a particular 796 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling 797 * buffers for each online CPU. 798 */ 799 if (cpuhw) 800 /* Event is pinned to a particular CPU */ 801 err = allocate_buffers(cpuhw, hwc); 802 else { 803 /* Event is not pinned, allocate sampling buffer on 804 * each online CPU 805 */ 806 for_each_online_cpu(cpu) { 807 cpuhw = &per_cpu(cpu_hw_sf, cpu); 808 err = allocate_buffers(cpuhw, hwc); 809 if (err) 810 break; 811 } 812 } 813 814 /* If PID/TID sampling is active, replace the default overflow 815 * handler to extract and resolve the PIDs from the basic-sampling 816 * data entries. 817 */ 818 if (event->attr.sample_type & PERF_SAMPLE_TID) 819 if (is_default_overflow_handler(event)) 820 event->overflow_handler = cpumsf_output_event_pid; 821 out: 822 return err; 823 } 824 825 static int cpumsf_pmu_event_init(struct perf_event *event) 826 { 827 int err; 828 829 /* No support for taken branch sampling */ 830 if (has_branch_stack(event)) 831 return -EOPNOTSUPP; 832 833 switch (event->attr.type) { 834 case PERF_TYPE_RAW: 835 if ((event->attr.config != PERF_EVENT_CPUM_SF) && 836 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) 837 return -ENOENT; 838 break; 839 case PERF_TYPE_HARDWARE: 840 /* Support sampling of CPU cycles in addition to the 841 * counter facility. However, the counter facility 842 * is more precise and, hence, restrict this PMU to 843 * sampling events only. 844 */ 845 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) 846 return -ENOENT; 847 if (!is_sampling_event(event)) 848 return -ENOENT; 849 break; 850 default: 851 return -ENOENT; 852 } 853 854 /* Check online status of the CPU to which the event is pinned */ 855 if (event->cpu >= 0 && !cpu_online(event->cpu)) 856 return -ENODEV; 857 858 /* Force reset of idle/hv excludes regardless of what the 859 * user requested. 860 */ 861 if (event->attr.exclude_hv) 862 event->attr.exclude_hv = 0; 863 if (event->attr.exclude_idle) 864 event->attr.exclude_idle = 0; 865 866 err = __hw_perf_event_init(event); 867 if (unlikely(err)) 868 if (event->destroy) 869 event->destroy(event); 870 return err; 871 } 872 873 static void cpumsf_pmu_enable(struct pmu *pmu) 874 { 875 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 876 struct hw_perf_event *hwc; 877 int err; 878 879 if (cpuhw->flags & PMU_F_ENABLED) 880 return; 881 882 if (cpuhw->flags & PMU_F_ERR_MASK) 883 return; 884 885 /* Check whether to extent the sampling buffer. 886 * 887 * Two conditions trigger an increase of the sampling buffer for a 888 * perf event: 889 * 1. Postponed buffer allocations from the event initialization. 890 * 2. Sampling overflows that contribute to pending allocations. 891 * 892 * Note that the extend_sampling_buffer() function disables the sampling 893 * facility, but it can be fully re-enabled using sampling controls that 894 * have been saved in cpumsf_pmu_disable(). 895 */ 896 if (cpuhw->event) { 897 hwc = &cpuhw->event->hw; 898 if (!(SAMPL_DIAG_MODE(hwc))) { 899 /* 900 * Account number of overflow-designated 901 * buffer extents 902 */ 903 sfb_account_overflows(cpuhw, hwc); 904 if (sfb_has_pending_allocs(&cpuhw->sfb, hwc)) 905 extend_sampling_buffer(&cpuhw->sfb, hwc); 906 } 907 } 908 909 /* (Re)enable the PMU and sampling facility */ 910 cpuhw->flags |= PMU_F_ENABLED; 911 barrier(); 912 913 err = lsctl(&cpuhw->lsctl); 914 if (err) { 915 cpuhw->flags &= ~PMU_F_ENABLED; 916 pr_err("Loading sampling controls failed: op=%i err=%i\n", 917 1, err); 918 return; 919 } 920 921 /* Load current program parameter */ 922 lpp(&S390_lowcore.lpp); 923 924 debug_sprintf_event(sfdbg, 6, "pmu_enable: es=%i cs=%i ed=%i cd=%i " 925 "tear=%p dear=%p\n", cpuhw->lsctl.es, cpuhw->lsctl.cs, 926 cpuhw->lsctl.ed, cpuhw->lsctl.cd, 927 (void *) cpuhw->lsctl.tear, (void *) cpuhw->lsctl.dear); 928 } 929 930 static void cpumsf_pmu_disable(struct pmu *pmu) 931 { 932 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 933 struct hws_lsctl_request_block inactive; 934 struct hws_qsi_info_block si; 935 int err; 936 937 if (!(cpuhw->flags & PMU_F_ENABLED)) 938 return; 939 940 if (cpuhw->flags & PMU_F_ERR_MASK) 941 return; 942 943 /* Switch off sampling activation control */ 944 inactive = cpuhw->lsctl; 945 inactive.cs = 0; 946 inactive.cd = 0; 947 948 err = lsctl(&inactive); 949 if (err) { 950 pr_err("Loading sampling controls failed: op=%i err=%i\n", 951 2, err); 952 return; 953 } 954 955 /* Save state of TEAR and DEAR register contents */ 956 if (!qsi(&si)) { 957 /* TEAR/DEAR values are valid only if the sampling facility is 958 * enabled. Note that cpumsf_pmu_disable() might be called even 959 * for a disabled sampling facility because cpumsf_pmu_enable() 960 * controls the enable/disable state. 961 */ 962 if (si.es) { 963 cpuhw->lsctl.tear = si.tear; 964 cpuhw->lsctl.dear = si.dear; 965 } 966 } else 967 debug_sprintf_event(sfdbg, 3, "cpumsf_pmu_disable: " 968 "qsi() failed with err=%i\n", err); 969 970 cpuhw->flags &= ~PMU_F_ENABLED; 971 } 972 973 /* perf_exclude_event() - Filter event 974 * @event: The perf event 975 * @regs: pt_regs structure 976 * @sde_regs: Sample-data-entry (sde) regs structure 977 * 978 * Filter perf events according to their exclude specification. 979 * 980 * Return non-zero if the event shall be excluded. 981 */ 982 static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs, 983 struct perf_sf_sde_regs *sde_regs) 984 { 985 if (event->attr.exclude_user && user_mode(regs)) 986 return 1; 987 if (event->attr.exclude_kernel && !user_mode(regs)) 988 return 1; 989 if (event->attr.exclude_guest && sde_regs->in_guest) 990 return 1; 991 if (event->attr.exclude_host && !sde_regs->in_guest) 992 return 1; 993 return 0; 994 } 995 996 /* perf_push_sample() - Push samples to perf 997 * @event: The perf event 998 * @sample: Hardware sample data 999 * 1000 * Use the hardware sample data to create perf event sample. The sample 1001 * is the pushed to the event subsystem and the function checks for 1002 * possible event overflows. If an event overflow occurs, the PMU is 1003 * stopped. 1004 * 1005 * Return non-zero if an event overflow occurred. 1006 */ 1007 static int perf_push_sample(struct perf_event *event, 1008 struct hws_basic_entry *basic) 1009 { 1010 int overflow; 1011 struct pt_regs regs; 1012 struct perf_sf_sde_regs *sde_regs; 1013 struct perf_sample_data data; 1014 1015 /* Setup perf sample */ 1016 perf_sample_data_init(&data, 0, event->hw.last_period); 1017 1018 /* Setup pt_regs to look like an CPU-measurement external interrupt 1019 * using the Program Request Alert code. The regs.int_parm_long 1020 * field which is unused contains additional sample-data-entry related 1021 * indicators. 1022 */ 1023 memset(®s, 0, sizeof(regs)); 1024 regs.int_code = 0x1407; 1025 regs.int_parm = CPU_MF_INT_SF_PRA; 1026 sde_regs = (struct perf_sf_sde_regs *) ®s.int_parm_long; 1027 1028 psw_bits(regs.psw).ia = basic->ia; 1029 psw_bits(regs.psw).dat = basic->T; 1030 psw_bits(regs.psw).wait = basic->W; 1031 psw_bits(regs.psw).pstate = basic->P; 1032 psw_bits(regs.psw).as = basic->AS; 1033 1034 /* 1035 * Use the hardware provided configuration level to decide if the 1036 * sample belongs to a guest or host. If that is not available, 1037 * fall back to the following heuristics: 1038 * A non-zero guest program parameter always indicates a guest 1039 * sample. Some early samples or samples from guests without 1040 * lpp usage would be misaccounted to the host. We use the asn 1041 * value as an addon heuristic to detect most of these guest samples. 1042 * If the value differs from 0xffff (the host value), we assume to 1043 * be a KVM guest. 1044 */ 1045 switch (basic->CL) { 1046 case 1: /* logical partition */ 1047 sde_regs->in_guest = 0; 1048 break; 1049 case 2: /* virtual machine */ 1050 sde_regs->in_guest = 1; 1051 break; 1052 default: /* old machine, use heuristics */ 1053 if (basic->gpp || basic->prim_asn != 0xffff) 1054 sde_regs->in_guest = 1; 1055 break; 1056 } 1057 1058 /* 1059 * Store the PID value from the sample-data-entry to be 1060 * processed and resolved by cpumsf_output_event_pid(). 1061 */ 1062 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; 1063 1064 overflow = 0; 1065 if (perf_exclude_event(event, ®s, sde_regs)) 1066 goto out; 1067 if (perf_event_overflow(event, &data, ®s)) { 1068 overflow = 1; 1069 event->pmu->stop(event, 0); 1070 } 1071 perf_event_update_userpage(event); 1072 out: 1073 return overflow; 1074 } 1075 1076 static void perf_event_count_update(struct perf_event *event, u64 count) 1077 { 1078 local64_add(count, &event->count); 1079 } 1080 1081 static void debug_sample_entry(struct hws_basic_entry *sample, 1082 struct hws_trailer_entry *te) 1083 { 1084 debug_sprintf_event(sfdbg, 4, "hw_collect_samples: Found unknown " 1085 "sampling data entry: te->f=%i basic.def=%04x (%p)\n", 1086 te->f, sample->def, sample); 1087 } 1088 1089 /* hw_collect_samples() - Walk through a sample-data-block and collect samples 1090 * @event: The perf event 1091 * @sdbt: Sample-data-block table 1092 * @overflow: Event overflow counter 1093 * 1094 * Walks through a sample-data-block and collects sampling data entries that are 1095 * then pushed to the perf event subsystem. Depending on the sampling function, 1096 * there can be either basic-sampling or combined-sampling data entries. A 1097 * combined-sampling data entry consists of a basic- and a diagnostic-sampling 1098 * data entry. The sampling function is determined by the flags in the perf 1099 * event hardware structure. The function always works with a combined-sampling 1100 * data entry but ignores the the diagnostic portion if it is not available. 1101 * 1102 * Note that the implementation focuses on basic-sampling data entries and, if 1103 * such an entry is not valid, the entire combined-sampling data entry is 1104 * ignored. 1105 * 1106 * The overflow variables counts the number of samples that has been discarded 1107 * due to a perf event overflow. 1108 */ 1109 static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, 1110 unsigned long long *overflow) 1111 { 1112 struct hws_trailer_entry *te; 1113 struct hws_basic_entry *sample; 1114 1115 te = (struct hws_trailer_entry *) trailer_entry_ptr(*sdbt); 1116 sample = (struct hws_basic_entry *) *sdbt; 1117 while ((unsigned long *) sample < (unsigned long *) te) { 1118 /* Check for an empty sample */ 1119 if (!sample->def) 1120 break; 1121 1122 /* Update perf event period */ 1123 perf_event_count_update(event, SAMPL_RATE(&event->hw)); 1124 1125 /* Check whether sample is valid */ 1126 if (sample->def == 0x0001) { 1127 /* If an event overflow occurred, the PMU is stopped to 1128 * throttle event delivery. Remaining sample data is 1129 * discarded. 1130 */ 1131 if (!*overflow) { 1132 /* Check whether sample is consistent */ 1133 if (sample->I == 0 && sample->W == 0) { 1134 /* Deliver sample data to perf */ 1135 *overflow = perf_push_sample(event, 1136 sample); 1137 } 1138 } else 1139 /* Count discarded samples */ 1140 *overflow += 1; 1141 } else { 1142 debug_sample_entry(sample, te); 1143 /* Sample slot is not yet written or other record. 1144 * 1145 * This condition can occur if the buffer was reused 1146 * from a combined basic- and diagnostic-sampling. 1147 * If only basic-sampling is then active, entries are 1148 * written into the larger diagnostic entries. 1149 * This is typically the case for sample-data-blocks 1150 * that are not full. Stop processing if the first 1151 * invalid format was detected. 1152 */ 1153 if (!te->f) 1154 break; 1155 } 1156 1157 /* Reset sample slot and advance to next sample */ 1158 sample->def = 0; 1159 sample++; 1160 } 1161 } 1162 1163 /* hw_perf_event_update() - Process sampling buffer 1164 * @event: The perf event 1165 * @flush_all: Flag to also flush partially filled sample-data-blocks 1166 * 1167 * Processes the sampling buffer and create perf event samples. 1168 * The sampling buffer position are retrieved and saved in the TEAR_REG 1169 * register of the specified perf event. 1170 * 1171 * Only full sample-data-blocks are processed. Specify the flash_all flag 1172 * to also walk through partially filled sample-data-blocks. It is ignored 1173 * if PERF_CPUM_SF_FULL_BLOCKS is set. The PERF_CPUM_SF_FULL_BLOCKS flag 1174 * enforces the processing of full sample-data-blocks only (trailer entries 1175 * with the block-full-indicator bit set). 1176 */ 1177 static void hw_perf_event_update(struct perf_event *event, int flush_all) 1178 { 1179 struct hw_perf_event *hwc = &event->hw; 1180 struct hws_trailer_entry *te; 1181 unsigned long *sdbt; 1182 unsigned long long event_overflow, sampl_overflow, num_sdb, te_flags; 1183 int done; 1184 1185 /* 1186 * AUX buffer is used when in diagnostic sampling mode. 1187 * No perf events/samples are created. 1188 */ 1189 if (SAMPL_DIAG_MODE(&event->hw)) 1190 return; 1191 1192 if (flush_all && SDB_FULL_BLOCKS(hwc)) 1193 flush_all = 0; 1194 1195 sdbt = (unsigned long *) TEAR_REG(hwc); 1196 done = event_overflow = sampl_overflow = num_sdb = 0; 1197 while (!done) { 1198 /* Get the trailer entry of the sample-data-block */ 1199 te = (struct hws_trailer_entry *) trailer_entry_ptr(*sdbt); 1200 1201 /* Leave loop if no more work to do (block full indicator) */ 1202 if (!te->f) { 1203 done = 1; 1204 if (!flush_all) 1205 break; 1206 } 1207 1208 /* Check the sample overflow count */ 1209 if (te->overflow) 1210 /* Account sample overflows and, if a particular limit 1211 * is reached, extend the sampling buffer. 1212 * For details, see sfb_account_overflows(). 1213 */ 1214 sampl_overflow += te->overflow; 1215 1216 /* Timestamps are valid for full sample-data-blocks only */ 1217 debug_sprintf_event(sfdbg, 6, "hw_perf_event_update: sdbt=%p " 1218 "overflow=%llu timestamp=0x%llx\n", 1219 sdbt, te->overflow, 1220 (te->f) ? trailer_timestamp(te) : 0ULL); 1221 1222 /* Collect all samples from a single sample-data-block and 1223 * flag if an (perf) event overflow happened. If so, the PMU 1224 * is stopped and remaining samples will be discarded. 1225 */ 1226 hw_collect_samples(event, sdbt, &event_overflow); 1227 num_sdb++; 1228 1229 /* Reset trailer (using compare-double-and-swap) */ 1230 do { 1231 te_flags = te->flags & ~SDB_TE_BUFFER_FULL_MASK; 1232 te_flags |= SDB_TE_ALERT_REQ_MASK; 1233 } while (!cmpxchg_double(&te->flags, &te->overflow, 1234 te->flags, te->overflow, 1235 te_flags, 0ULL)); 1236 1237 /* Advance to next sample-data-block */ 1238 sdbt++; 1239 if (is_link_entry(sdbt)) 1240 sdbt = get_next_sdbt(sdbt); 1241 1242 /* Update event hardware registers */ 1243 TEAR_REG(hwc) = (unsigned long) sdbt; 1244 1245 /* Stop processing sample-data if all samples of the current 1246 * sample-data-block were flushed even if it was not full. 1247 */ 1248 if (flush_all && done) 1249 break; 1250 1251 /* If an event overflow happened, discard samples by 1252 * processing any remaining sample-data-blocks. 1253 */ 1254 if (event_overflow) 1255 flush_all = 1; 1256 } 1257 1258 /* Account sample overflows in the event hardware structure */ 1259 if (sampl_overflow) 1260 OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) + 1261 sampl_overflow, 1 + num_sdb); 1262 if (sampl_overflow || event_overflow) 1263 debug_sprintf_event(sfdbg, 4, "hw_perf_event_update: " 1264 "overflow stats: sample=%llu event=%llu\n", 1265 sampl_overflow, event_overflow); 1266 } 1267 1268 #define AUX_SDB_INDEX(aux, i) ((i) % aux->sfb.num_sdb) 1269 #define AUX_SDB_NUM(aux, start, end) (end >= start ? end - start + 1 : 0) 1270 #define AUX_SDB_NUM_ALERT(aux) AUX_SDB_NUM(aux, aux->head, aux->alert_mark) 1271 #define AUX_SDB_NUM_EMPTY(aux) AUX_SDB_NUM(aux, aux->head, aux->empty_mark) 1272 1273 /* 1274 * Get trailer entry by index of SDB. 1275 */ 1276 static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux, 1277 unsigned long index) 1278 { 1279 unsigned long sdb; 1280 1281 index = AUX_SDB_INDEX(aux, index); 1282 sdb = aux->sdb_index[index]; 1283 return (struct hws_trailer_entry *)trailer_entry_ptr(sdb); 1284 } 1285 1286 /* 1287 * Finish sampling on the cpu. Called by cpumsf_pmu_del() with pmu 1288 * disabled. Collect the full SDBs in AUX buffer which have not reached 1289 * the point of alert indicator. And ignore the SDBs which are not 1290 * full. 1291 * 1292 * 1. Scan SDBs to see how much data is there and consume them. 1293 * 2. Remove alert indicator in the buffer. 1294 */ 1295 static void aux_output_end(struct perf_output_handle *handle) 1296 { 1297 unsigned long i, range_scan, idx; 1298 struct aux_buffer *aux; 1299 struct hws_trailer_entry *te; 1300 1301 aux = perf_get_aux(handle); 1302 if (!aux) 1303 return; 1304 1305 range_scan = AUX_SDB_NUM_ALERT(aux); 1306 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { 1307 te = aux_sdb_trailer(aux, idx); 1308 if (!(te->flags & SDB_TE_BUFFER_FULL_MASK)) 1309 break; 1310 } 1311 /* i is num of SDBs which are full */ 1312 perf_aux_output_end(handle, i << PAGE_SHIFT); 1313 1314 /* Remove alert indicators in the buffer */ 1315 te = aux_sdb_trailer(aux, aux->alert_mark); 1316 te->flags &= ~SDB_TE_ALERT_REQ_MASK; 1317 1318 debug_sprintf_event(sfdbg, 6, "aux_output_end: collect %lx SDBs\n", i); 1319 } 1320 1321 /* 1322 * Start sampling on the CPU. Called by cpumsf_pmu_add() when an event 1323 * is first added to the CPU or rescheduled again to the CPU. It is called 1324 * with pmu disabled. 1325 * 1326 * 1. Reset the trailer of SDBs to get ready for new data. 1327 * 2. Tell the hardware where to put the data by reset the SDBs buffer 1328 * head(tear/dear). 1329 */ 1330 static int aux_output_begin(struct perf_output_handle *handle, 1331 struct aux_buffer *aux, 1332 struct cpu_hw_sf *cpuhw) 1333 { 1334 unsigned long range; 1335 unsigned long i, range_scan, idx; 1336 unsigned long head, base, offset; 1337 struct hws_trailer_entry *te; 1338 1339 if (WARN_ON_ONCE(handle->head & ~PAGE_MASK)) 1340 return -EINVAL; 1341 1342 aux->head = handle->head >> PAGE_SHIFT; 1343 range = (handle->size + 1) >> PAGE_SHIFT; 1344 if (range <= 1) 1345 return -ENOMEM; 1346 1347 /* 1348 * SDBs between aux->head and aux->empty_mark are already ready 1349 * for new data. range_scan is num of SDBs not within them. 1350 */ 1351 if (range > AUX_SDB_NUM_EMPTY(aux)) { 1352 range_scan = range - AUX_SDB_NUM_EMPTY(aux); 1353 idx = aux->empty_mark + 1; 1354 for (i = 0; i < range_scan; i++, idx++) { 1355 te = aux_sdb_trailer(aux, idx); 1356 te->flags = te->flags & ~SDB_TE_BUFFER_FULL_MASK; 1357 te->flags = te->flags & ~SDB_TE_ALERT_REQ_MASK; 1358 te->overflow = 0; 1359 } 1360 /* Save the position of empty SDBs */ 1361 aux->empty_mark = aux->head + range - 1; 1362 } 1363 1364 /* Set alert indicator */ 1365 aux->alert_mark = aux->head + range/2 - 1; 1366 te = aux_sdb_trailer(aux, aux->alert_mark); 1367 te->flags = te->flags | SDB_TE_ALERT_REQ_MASK; 1368 1369 /* Reset hardware buffer head */ 1370 head = AUX_SDB_INDEX(aux, aux->head); 1371 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; 1372 offset = head % CPUM_SF_SDB_PER_TABLE; 1373 cpuhw->lsctl.tear = base + offset * sizeof(unsigned long); 1374 cpuhw->lsctl.dear = aux->sdb_index[head]; 1375 1376 debug_sprintf_event(sfdbg, 6, "aux_output_begin: " 1377 "head->alert_mark->empty_mark (num_alert, range)" 1378 "[%lx -> %lx -> %lx] (%lx, %lx) " 1379 "tear index %lx, tear %lx dear %lx\n", 1380 aux->head, aux->alert_mark, aux->empty_mark, 1381 AUX_SDB_NUM_ALERT(aux), range, 1382 head / CPUM_SF_SDB_PER_TABLE, 1383 cpuhw->lsctl.tear, 1384 cpuhw->lsctl.dear); 1385 1386 return 0; 1387 } 1388 1389 /* 1390 * Set alert indicator on SDB at index @alert_index while sampler is running. 1391 * 1392 * Return true if successfully. 1393 * Return false if full indicator is already set by hardware sampler. 1394 */ 1395 static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index, 1396 unsigned long long *overflow) 1397 { 1398 unsigned long long orig_overflow, orig_flags, new_flags; 1399 struct hws_trailer_entry *te; 1400 1401 te = aux_sdb_trailer(aux, alert_index); 1402 do { 1403 orig_flags = te->flags; 1404 orig_overflow = te->overflow; 1405 *overflow = orig_overflow; 1406 if (orig_flags & SDB_TE_BUFFER_FULL_MASK) { 1407 /* 1408 * SDB is already set by hardware. 1409 * Abort and try to set somewhere 1410 * behind. 1411 */ 1412 return false; 1413 } 1414 new_flags = orig_flags | SDB_TE_ALERT_REQ_MASK; 1415 } while (!cmpxchg_double(&te->flags, &te->overflow, 1416 orig_flags, orig_overflow, 1417 new_flags, 0ULL)); 1418 return true; 1419 } 1420 1421 /* 1422 * aux_reset_buffer() - Scan and setup SDBs for new samples 1423 * @aux: The AUX buffer to set 1424 * @range: The range of SDBs to scan started from aux->head 1425 * @overflow: Set to overflow count 1426 * 1427 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is 1428 * marked as empty, check if it is already set full by the hardware sampler. 1429 * If yes, that means new data is already there before we can set an alert 1430 * indicator. Caller should try to set alert indicator to some position behind. 1431 * 1432 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used 1433 * previously and have already been consumed by user space. Reset these SDBs 1434 * (clear full indicator and alert indicator) for new data. 1435 * If aux->alert_mark fall in this area, just set it. Overflow count is 1436 * recorded while scanning. 1437 * 1438 * SDBs between aux->head and aux->empty_mark are already reset at last time. 1439 * and ready for new samples. So scanning on this area could be skipped. 1440 * 1441 * Return true if alert indicator is set successfully and false if not. 1442 */ 1443 static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, 1444 unsigned long long *overflow) 1445 { 1446 unsigned long long orig_overflow, orig_flags, new_flags; 1447 unsigned long i, range_scan, idx; 1448 struct hws_trailer_entry *te; 1449 1450 if (range <= AUX_SDB_NUM_EMPTY(aux)) 1451 /* 1452 * No need to scan. All SDBs in range are marked as empty. 1453 * Just set alert indicator. Should check race with hardware 1454 * sampler. 1455 */ 1456 return aux_set_alert(aux, aux->alert_mark, overflow); 1457 1458 if (aux->alert_mark <= aux->empty_mark) 1459 /* 1460 * Set alert indicator on empty SDB. Should check race 1461 * with hardware sampler. 1462 */ 1463 if (!aux_set_alert(aux, aux->alert_mark, overflow)) 1464 return false; 1465 1466 /* 1467 * Scan the SDBs to clear full and alert indicator used previously. 1468 * Start scanning from one SDB behind empty_mark. If the new alert 1469 * indicator fall into this range, set it. 1470 */ 1471 range_scan = range - AUX_SDB_NUM_EMPTY(aux); 1472 idx = aux->empty_mark + 1; 1473 for (i = 0; i < range_scan; i++, idx++) { 1474 te = aux_sdb_trailer(aux, idx); 1475 do { 1476 orig_flags = te->flags; 1477 orig_overflow = te->overflow; 1478 new_flags = orig_flags & ~SDB_TE_BUFFER_FULL_MASK; 1479 if (idx == aux->alert_mark) 1480 new_flags |= SDB_TE_ALERT_REQ_MASK; 1481 else 1482 new_flags &= ~SDB_TE_ALERT_REQ_MASK; 1483 } while (!cmpxchg_double(&te->flags, &te->overflow, 1484 orig_flags, orig_overflow, 1485 new_flags, 0ULL)); 1486 *overflow += orig_overflow; 1487 } 1488 1489 /* Update empty_mark to new position */ 1490 aux->empty_mark = aux->head + range - 1; 1491 1492 return true; 1493 } 1494 1495 /* 1496 * Measurement alert handler for diagnostic mode sampling. 1497 */ 1498 static void hw_collect_aux(struct cpu_hw_sf *cpuhw) 1499 { 1500 struct aux_buffer *aux; 1501 int done = 0; 1502 unsigned long range = 0, size; 1503 unsigned long long overflow = 0; 1504 struct perf_output_handle *handle = &cpuhw->handle; 1505 unsigned long num_sdb; 1506 1507 aux = perf_get_aux(handle); 1508 if (WARN_ON_ONCE(!aux)) 1509 return; 1510 1511 /* Inform user space new data arrived */ 1512 size = AUX_SDB_NUM_ALERT(aux) << PAGE_SHIFT; 1513 perf_aux_output_end(handle, size); 1514 num_sdb = aux->sfb.num_sdb; 1515 1516 while (!done) { 1517 /* Get an output handle */ 1518 aux = perf_aux_output_begin(handle, cpuhw->event); 1519 if (handle->size == 0) { 1520 pr_err("The AUX buffer with %lu pages for the " 1521 "diagnostic-sampling mode is full\n", 1522 num_sdb); 1523 debug_sprintf_event(sfdbg, 1, "AUX buffer used up\n"); 1524 break; 1525 } 1526 if (WARN_ON_ONCE(!aux)) 1527 return; 1528 1529 /* Update head and alert_mark to new position */ 1530 aux->head = handle->head >> PAGE_SHIFT; 1531 range = (handle->size + 1) >> PAGE_SHIFT; 1532 if (range == 1) 1533 aux->alert_mark = aux->head; 1534 else 1535 aux->alert_mark = aux->head + range/2 - 1; 1536 1537 if (aux_reset_buffer(aux, range, &overflow)) { 1538 if (!overflow) { 1539 done = 1; 1540 break; 1541 } 1542 size = range << PAGE_SHIFT; 1543 perf_aux_output_end(&cpuhw->handle, size); 1544 pr_err("Sample data caused the AUX buffer with %lu " 1545 "pages to overflow\n", num_sdb); 1546 debug_sprintf_event(sfdbg, 1, "head %lx range %lx " 1547 "overflow %llx\n", 1548 aux->head, range, overflow); 1549 } else { 1550 size = AUX_SDB_NUM_ALERT(aux) << PAGE_SHIFT; 1551 perf_aux_output_end(&cpuhw->handle, size); 1552 debug_sprintf_event(sfdbg, 6, "head %lx alert %lx " 1553 "already full, try another\n", 1554 aux->head, aux->alert_mark); 1555 } 1556 } 1557 1558 if (done) 1559 debug_sprintf_event(sfdbg, 6, "aux_reset_buffer: " 1560 "[%lx -> %lx -> %lx] (%lx, %lx)\n", 1561 aux->head, aux->alert_mark, aux->empty_mark, 1562 AUX_SDB_NUM_ALERT(aux), range); 1563 } 1564 1565 /* 1566 * Callback when freeing AUX buffers. 1567 */ 1568 static void aux_buffer_free(void *data) 1569 { 1570 struct aux_buffer *aux = data; 1571 unsigned long i, num_sdbt; 1572 1573 if (!aux) 1574 return; 1575 1576 /* Free SDBT. SDB is freed by the caller */ 1577 num_sdbt = aux->sfb.num_sdbt; 1578 for (i = 0; i < num_sdbt; i++) 1579 free_page(aux->sdbt_index[i]); 1580 1581 kfree(aux->sdbt_index); 1582 kfree(aux->sdb_index); 1583 kfree(aux); 1584 1585 debug_sprintf_event(sfdbg, 4, "aux_buffer_free: free " 1586 "%lu SDBTs\n", num_sdbt); 1587 } 1588 1589 /* 1590 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling 1591 * @cpu: On which to allocate, -1 means current 1592 * @pages: Array of pointers to buffer pages passed from perf core 1593 * @nr_pages: Total pages 1594 * @snapshot: Flag for snapshot mode 1595 * 1596 * This is the callback when setup an event using AUX buffer. Perf tool can 1597 * trigger this by an additional mmap() call on the event. Unlike the buffer 1598 * for basic samples, AUX buffer belongs to the event. It is scheduled with 1599 * the task among online cpus when it is a per-thread event. 1600 * 1601 * Return the private AUX buffer structure if success or NULL if fails. 1602 */ 1603 static void *aux_buffer_setup(int cpu, void **pages, int nr_pages, 1604 bool snapshot) 1605 { 1606 struct sf_buffer *sfb; 1607 struct aux_buffer *aux; 1608 unsigned long *new, *tail; 1609 int i, n_sdbt; 1610 1611 if (!nr_pages || !pages) 1612 return NULL; 1613 1614 if (nr_pages > CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR) { 1615 pr_err("AUX buffer size (%i pages) is larger than the " 1616 "maximum sampling buffer limit\n", 1617 nr_pages); 1618 return NULL; 1619 } else if (nr_pages < CPUM_SF_MIN_SDB * CPUM_SF_SDB_DIAG_FACTOR) { 1620 pr_err("AUX buffer size (%i pages) is less than the " 1621 "minimum sampling buffer limit\n", 1622 nr_pages); 1623 return NULL; 1624 } 1625 1626 /* Allocate aux_buffer struct for the event */ 1627 aux = kmalloc(sizeof(struct aux_buffer), GFP_KERNEL); 1628 if (!aux) 1629 goto no_aux; 1630 sfb = &aux->sfb; 1631 1632 /* Allocate sdbt_index for fast reference */ 1633 n_sdbt = (nr_pages + CPUM_SF_SDB_PER_TABLE - 1) / CPUM_SF_SDB_PER_TABLE; 1634 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); 1635 if (!aux->sdbt_index) 1636 goto no_sdbt_index; 1637 1638 /* Allocate sdb_index for fast reference */ 1639 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); 1640 if (!aux->sdb_index) 1641 goto no_sdb_index; 1642 1643 /* Allocate the first SDBT */ 1644 sfb->num_sdbt = 0; 1645 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); 1646 if (!sfb->sdbt) 1647 goto no_sdbt; 1648 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; 1649 tail = sfb->tail = sfb->sdbt; 1650 1651 /* 1652 * Link the provided pages of AUX buffer to SDBT. 1653 * Allocate SDBT if needed. 1654 */ 1655 for (i = 0; i < nr_pages; i++, tail++) { 1656 if (require_table_link(tail)) { 1657 new = (unsigned long *) get_zeroed_page(GFP_KERNEL); 1658 if (!new) 1659 goto no_sdbt; 1660 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; 1661 /* Link current page to tail of chain */ 1662 *tail = (unsigned long)(void *) new + 1; 1663 tail = new; 1664 } 1665 /* Tail is the entry in a SDBT */ 1666 *tail = (unsigned long)pages[i]; 1667 aux->sdb_index[i] = (unsigned long)pages[i]; 1668 } 1669 sfb->num_sdb = nr_pages; 1670 1671 /* Link the last entry in the SDBT to the first SDBT */ 1672 *tail = (unsigned long) sfb->sdbt + 1; 1673 sfb->tail = tail; 1674 1675 /* 1676 * Initial all SDBs are zeroed. Mark it as empty. 1677 * So there is no need to clear the full indicator 1678 * when this event is first added. 1679 */ 1680 aux->empty_mark = sfb->num_sdb - 1; 1681 1682 debug_sprintf_event(sfdbg, 4, "aux_buffer_setup: setup %lu SDBTs" 1683 " and %lu SDBs\n", 1684 sfb->num_sdbt, sfb->num_sdb); 1685 1686 return aux; 1687 1688 no_sdbt: 1689 /* SDBs (AUX buffer pages) are freed by caller */ 1690 for (i = 0; i < sfb->num_sdbt; i++) 1691 free_page(aux->sdbt_index[i]); 1692 kfree(aux->sdb_index); 1693 no_sdb_index: 1694 kfree(aux->sdbt_index); 1695 no_sdbt_index: 1696 kfree(aux); 1697 no_aux: 1698 return NULL; 1699 } 1700 1701 static void cpumsf_pmu_read(struct perf_event *event) 1702 { 1703 /* Nothing to do ... updates are interrupt-driven */ 1704 } 1705 1706 /* Activate sampling control. 1707 * Next call of pmu_enable() starts sampling. 1708 */ 1709 static void cpumsf_pmu_start(struct perf_event *event, int flags) 1710 { 1711 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 1712 1713 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1714 return; 1715 1716 if (flags & PERF_EF_RELOAD) 1717 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1718 1719 perf_pmu_disable(event->pmu); 1720 event->hw.state = 0; 1721 cpuhw->lsctl.cs = 1; 1722 if (SAMPL_DIAG_MODE(&event->hw)) 1723 cpuhw->lsctl.cd = 1; 1724 perf_pmu_enable(event->pmu); 1725 } 1726 1727 /* Deactivate sampling control. 1728 * Next call of pmu_enable() stops sampling. 1729 */ 1730 static void cpumsf_pmu_stop(struct perf_event *event, int flags) 1731 { 1732 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 1733 1734 if (event->hw.state & PERF_HES_STOPPED) 1735 return; 1736 1737 perf_pmu_disable(event->pmu); 1738 cpuhw->lsctl.cs = 0; 1739 cpuhw->lsctl.cd = 0; 1740 event->hw.state |= PERF_HES_STOPPED; 1741 1742 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { 1743 hw_perf_event_update(event, 1); 1744 event->hw.state |= PERF_HES_UPTODATE; 1745 } 1746 perf_pmu_enable(event->pmu); 1747 } 1748 1749 static int cpumsf_pmu_add(struct perf_event *event, int flags) 1750 { 1751 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 1752 struct aux_buffer *aux; 1753 int err; 1754 1755 if (cpuhw->flags & PMU_F_IN_USE) 1756 return -EAGAIN; 1757 1758 if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt) 1759 return -EINVAL; 1760 1761 err = 0; 1762 perf_pmu_disable(event->pmu); 1763 1764 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1765 1766 /* Set up sampling controls. Always program the sampling register 1767 * using the SDB-table start. Reset TEAR_REG event hardware register 1768 * that is used by hw_perf_event_update() to store the sampling buffer 1769 * position after samples have been flushed. 1770 */ 1771 cpuhw->lsctl.s = 0; 1772 cpuhw->lsctl.h = 1; 1773 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); 1774 if (!SAMPL_DIAG_MODE(&event->hw)) { 1775 cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt; 1776 cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt; 1777 hw_reset_registers(&event->hw, cpuhw->sfb.sdbt); 1778 } 1779 1780 /* Ensure sampling functions are in the disabled state. If disabled, 1781 * switch on sampling enable control. */ 1782 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { 1783 err = -EAGAIN; 1784 goto out; 1785 } 1786 if (SAMPL_DIAG_MODE(&event->hw)) { 1787 aux = perf_aux_output_begin(&cpuhw->handle, event); 1788 if (!aux) { 1789 err = -EINVAL; 1790 goto out; 1791 } 1792 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); 1793 if (err) 1794 goto out; 1795 cpuhw->lsctl.ed = 1; 1796 } 1797 cpuhw->lsctl.es = 1; 1798 1799 /* Set in_use flag and store event */ 1800 cpuhw->event = event; 1801 cpuhw->flags |= PMU_F_IN_USE; 1802 1803 if (flags & PERF_EF_START) 1804 cpumsf_pmu_start(event, PERF_EF_RELOAD); 1805 out: 1806 perf_event_update_userpage(event); 1807 perf_pmu_enable(event->pmu); 1808 return err; 1809 } 1810 1811 static void cpumsf_pmu_del(struct perf_event *event, int flags) 1812 { 1813 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); 1814 1815 perf_pmu_disable(event->pmu); 1816 cpumsf_pmu_stop(event, PERF_EF_UPDATE); 1817 1818 cpuhw->lsctl.es = 0; 1819 cpuhw->lsctl.ed = 0; 1820 cpuhw->flags &= ~PMU_F_IN_USE; 1821 cpuhw->event = NULL; 1822 1823 if (SAMPL_DIAG_MODE(&event->hw)) 1824 aux_output_end(&cpuhw->handle); 1825 perf_event_update_userpage(event); 1826 perf_pmu_enable(event->pmu); 1827 } 1828 1829 CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC, PERF_EVENT_CPUM_SF); 1830 CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC_DIAG, PERF_EVENT_CPUM_SF_DIAG); 1831 1832 static struct attribute *cpumsf_pmu_events_attr[] = { 1833 CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC), 1834 NULL, 1835 NULL, 1836 }; 1837 1838 PMU_FORMAT_ATTR(event, "config:0-63"); 1839 1840 static struct attribute *cpumsf_pmu_format_attr[] = { 1841 &format_attr_event.attr, 1842 NULL, 1843 }; 1844 1845 static struct attribute_group cpumsf_pmu_events_group = { 1846 .name = "events", 1847 .attrs = cpumsf_pmu_events_attr, 1848 }; 1849 static struct attribute_group cpumsf_pmu_format_group = { 1850 .name = "format", 1851 .attrs = cpumsf_pmu_format_attr, 1852 }; 1853 static const struct attribute_group *cpumsf_pmu_attr_groups[] = { 1854 &cpumsf_pmu_events_group, 1855 &cpumsf_pmu_format_group, 1856 NULL, 1857 }; 1858 1859 static struct pmu cpumf_sampling = { 1860 .pmu_enable = cpumsf_pmu_enable, 1861 .pmu_disable = cpumsf_pmu_disable, 1862 1863 .event_init = cpumsf_pmu_event_init, 1864 .add = cpumsf_pmu_add, 1865 .del = cpumsf_pmu_del, 1866 1867 .start = cpumsf_pmu_start, 1868 .stop = cpumsf_pmu_stop, 1869 .read = cpumsf_pmu_read, 1870 1871 .attr_groups = cpumsf_pmu_attr_groups, 1872 1873 .setup_aux = aux_buffer_setup, 1874 .free_aux = aux_buffer_free, 1875 }; 1876 1877 static void cpumf_measurement_alert(struct ext_code ext_code, 1878 unsigned int alert, unsigned long unused) 1879 { 1880 struct cpu_hw_sf *cpuhw; 1881 1882 if (!(alert & CPU_MF_INT_SF_MASK)) 1883 return; 1884 inc_irq_stat(IRQEXT_CMS); 1885 cpuhw = this_cpu_ptr(&cpu_hw_sf); 1886 1887 /* Measurement alerts are shared and might happen when the PMU 1888 * is not reserved. Ignore these alerts in this case. */ 1889 if (!(cpuhw->flags & PMU_F_RESERVED)) 1890 return; 1891 1892 /* The processing below must take care of multiple alert events that 1893 * might be indicated concurrently. */ 1894 1895 /* Program alert request */ 1896 if (alert & CPU_MF_INT_SF_PRA) { 1897 if (cpuhw->flags & PMU_F_IN_USE) 1898 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) 1899 hw_collect_aux(cpuhw); 1900 else 1901 hw_perf_event_update(cpuhw->event, 0); 1902 else 1903 WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE)); 1904 } 1905 1906 /* Report measurement alerts only for non-PRA codes */ 1907 if (alert != CPU_MF_INT_SF_PRA) 1908 debug_sprintf_event(sfdbg, 6, "measurement alert: 0x%x\n", alert); 1909 1910 /* Sampling authorization change request */ 1911 if (alert & CPU_MF_INT_SF_SACA) 1912 qsi(&cpuhw->qsi); 1913 1914 /* Loss of sample data due to high-priority machine activities */ 1915 if (alert & CPU_MF_INT_SF_LSDA) { 1916 pr_err("Sample data was lost\n"); 1917 cpuhw->flags |= PMU_F_ERR_LSDA; 1918 sf_disable(); 1919 } 1920 1921 /* Invalid sampling buffer entry */ 1922 if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) { 1923 pr_err("A sampling buffer entry is incorrect (alert=0x%x)\n", 1924 alert); 1925 cpuhw->flags |= PMU_F_ERR_IBE; 1926 sf_disable(); 1927 } 1928 } 1929 static int cpusf_pmu_setup(unsigned int cpu, int flags) 1930 { 1931 /* Ignore the notification if no events are scheduled on the PMU. 1932 * This might be racy... 1933 */ 1934 if (!atomic_read(&num_events)) 1935 return 0; 1936 1937 local_irq_disable(); 1938 setup_pmc_cpu(&flags); 1939 local_irq_enable(); 1940 return 0; 1941 } 1942 1943 static int s390_pmu_sf_online_cpu(unsigned int cpu) 1944 { 1945 return cpusf_pmu_setup(cpu, PMC_INIT); 1946 } 1947 1948 static int s390_pmu_sf_offline_cpu(unsigned int cpu) 1949 { 1950 return cpusf_pmu_setup(cpu, PMC_RELEASE); 1951 } 1952 1953 static int param_get_sfb_size(char *buffer, const struct kernel_param *kp) 1954 { 1955 if (!cpum_sf_avail()) 1956 return -ENODEV; 1957 return sprintf(buffer, "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB); 1958 } 1959 1960 static int param_set_sfb_size(const char *val, const struct kernel_param *kp) 1961 { 1962 int rc; 1963 unsigned long min, max; 1964 1965 if (!cpum_sf_avail()) 1966 return -ENODEV; 1967 if (!val || !strlen(val)) 1968 return -EINVAL; 1969 1970 /* Valid parameter values: "min,max" or "max" */ 1971 min = CPUM_SF_MIN_SDB; 1972 max = CPUM_SF_MAX_SDB; 1973 if (strchr(val, ',')) 1974 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; 1975 else 1976 rc = kstrtoul(val, 10, &max); 1977 1978 if (min < 2 || min >= max || max > get_num_physpages()) 1979 rc = -EINVAL; 1980 if (rc) 1981 return rc; 1982 1983 sfb_set_limits(min, max); 1984 pr_info("The sampling buffer limits have changed to: " 1985 "min=%lu max=%lu (diag=x%lu)\n", 1986 CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB, CPUM_SF_SDB_DIAG_FACTOR); 1987 return 0; 1988 } 1989 1990 #define param_check_sfb_size(name, p) __param_check(name, p, void) 1991 static const struct kernel_param_ops param_ops_sfb_size = { 1992 .set = param_set_sfb_size, 1993 .get = param_get_sfb_size, 1994 }; 1995 1996 #define RS_INIT_FAILURE_QSI 0x0001 1997 #define RS_INIT_FAILURE_BSDES 0x0002 1998 #define RS_INIT_FAILURE_ALRT 0x0003 1999 #define RS_INIT_FAILURE_PERF 0x0004 2000 static void __init pr_cpumsf_err(unsigned int reason) 2001 { 2002 pr_err("Sampling facility support for perf is not available: " 2003 "reason=%04x\n", reason); 2004 } 2005 2006 static int __init init_cpum_sampling_pmu(void) 2007 { 2008 struct hws_qsi_info_block si; 2009 int err; 2010 2011 if (!cpum_sf_avail()) 2012 return -ENODEV; 2013 2014 memset(&si, 0, sizeof(si)); 2015 if (qsi(&si)) { 2016 pr_cpumsf_err(RS_INIT_FAILURE_QSI); 2017 return -ENODEV; 2018 } 2019 2020 if (!si.as && !si.ad) 2021 return -ENODEV; 2022 2023 if (si.bsdes != sizeof(struct hws_basic_entry)) { 2024 pr_cpumsf_err(RS_INIT_FAILURE_BSDES); 2025 return -EINVAL; 2026 } 2027 2028 if (si.ad) { 2029 sfb_set_limits(CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB); 2030 cpumsf_pmu_events_attr[1] = 2031 CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC_DIAG); 2032 } 2033 2034 sfdbg = debug_register(KMSG_COMPONENT, 2, 1, 80); 2035 if (!sfdbg) 2036 pr_err("Registering for s390dbf failed\n"); 2037 debug_register_view(sfdbg, &debug_sprintf_view); 2038 2039 err = register_external_irq(EXT_IRQ_MEASURE_ALERT, 2040 cpumf_measurement_alert); 2041 if (err) { 2042 pr_cpumsf_err(RS_INIT_FAILURE_ALRT); 2043 goto out; 2044 } 2045 2046 err = perf_pmu_register(&cpumf_sampling, "cpum_sf", PERF_TYPE_RAW); 2047 if (err) { 2048 pr_cpumsf_err(RS_INIT_FAILURE_PERF); 2049 unregister_external_irq(EXT_IRQ_MEASURE_ALERT, 2050 cpumf_measurement_alert); 2051 goto out; 2052 } 2053 2054 cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "perf/s390/sf:online", 2055 s390_pmu_sf_online_cpu, s390_pmu_sf_offline_cpu); 2056 out: 2057 return err; 2058 } 2059 arch_initcall(init_cpum_sampling_pmu); 2060 core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0640); 2061