Lines Matching +full:10 +full:base +full:- +full:te
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for the System z CPU-measurement Sampling Facility
27 /* Minimum number of sample-data-block-tables:
29 * A single table contains up to 511 pointers to sample-data-blocks.
33 /* Number of sample-data-blocks per sample-data-block-table (SDBT):
34 * A table contains SDB pointers (8 bytes) and one table-link entry
37 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
39 /* Maximum page offset for an SDBT table-link entry:
40 * If this page offset is reached, a table-link entry to the next SDBT
52 * the number of sample-data-block-tables into account. Note that these
53 * numbers apply to the basic-sampling function only.
55 * the diagnostic-sampling function is active.
58 * ---------------------------------------------------
60 * 1 page for SDB-tables
64 * 16 pages for SDB-tables
72 unsigned long *sdbt; /* Sample-data-block-table origin */
74 unsigned long num_sdb; /* Number of sample-data-blocks */
75 unsigned long num_sdbt; /* Number of sample-data-block-tables */
76 unsigned long *tail; /* last sample-data-block-table */
89 /* CPU-measurement sampling information block */
91 /* CPU-measurement sampling control block */
107 return (USEC_PER_SEC / freq) * qsi->cpu_speed; in freq_to_sample_rate()
113 return USEC_PER_SEC * qsi->cpu_speed / rate; in sample_rate_to_freq()
117 static inline unsigned long long trailer_timestamp(struct hws_trailer_entry *te) in trailer_timestamp() argument
120 if (te->header.t) in trailer_timestamp()
121 return *((unsigned long long *)&te->timestamp[1]); in trailer_timestamp()
124 return *((unsigned long long *)&te->timestamp[0]); in trailer_timestamp()
134 ret -= sizeof(struct hws_trailer_entry); in trailer_entry_ptr()
155 * sf_disable() - Switch off sampling facility
166 * sf_buffer_available() - Check for an allocated sampling buffer
170 return !!cpuhw->sfb.sdbt; in sf_buffer_available()
180 if (!sfb->sdbt) in free_sampling_buffer()
183 sdbt = sfb->sdbt; in free_sampling_buffer()
191 /* Process table-link entries */ in free_sampling_buffer()
198 if (curr == sfb->sdbt) in free_sampling_buffer()
212 (unsigned long)sfb->sdbt); in free_sampling_buffer()
218 struct hws_trailer_entry *te; in alloc_sample_data_block() local
221 /* Allocate and initialize sample-data-block */ in alloc_sample_data_block()
224 return -ENOMEM; in alloc_sample_data_block()
225 te = trailer_entry_ptr(sdb); in alloc_sample_data_block()
226 te->header.a = 1; in alloc_sample_data_block()
228 /* Link SDB into the sample-data-block-table */ in alloc_sample_data_block()
235 * realloc_sampling_buffer() - extend sampler memory
237 * Allocates new sample-data-blocks and adds them to the specified sampling
243 * Returns zero on success, non-zero otherwise.
251 if (!sfb->sdbt || !sfb->tail) in realloc_sampling_buffer()
252 return -EINVAL; in realloc_sampling_buffer()
254 if (!is_link_entry(sfb->tail)) in realloc_sampling_buffer()
255 return -EINVAL; in realloc_sampling_buffer()
257 /* Append to the existing sampling buffer, overwriting the table-link in realloc_sampling_buffer()
259 * The tail variables always points to the "tail" (last and table-link) in realloc_sampling_buffer()
260 * entry in an SDB-table. in realloc_sampling_buffer()
262 tail = sfb->tail; in realloc_sampling_buffer()
264 /* Do a sanity check whether the table-link entry points to in realloc_sampling_buffer()
267 if (sfb->sdbt != get_next_sdbt(tail)) { in realloc_sampling_buffer()
271 (unsigned long)sfb->sdbt, in realloc_sampling_buffer()
273 return -EINVAL; in realloc_sampling_buffer()
279 /* Allocate a new SDB-table if it is full. */ in realloc_sampling_buffer()
283 rc = -ENOMEM; in realloc_sampling_buffer()
286 sfb->num_sdbt++; in realloc_sampling_buffer()
293 /* Allocate a new sample-data-block. in realloc_sampling_buffer()
306 sfb->num_sdbt--; in realloc_sampling_buffer()
312 sfb->num_sdb++; in realloc_sampling_buffer()
318 *tail = virt_to_phys(sfb->sdbt) + 1; in realloc_sampling_buffer()
319 sfb->tail = tail; in realloc_sampling_buffer()
323 sfb->num_sdbt, sfb->num_sdb); in realloc_sampling_buffer()
328 * allocate_sampling_buffer() - allocate sampler memory
331 * specified number of sample-data-blocks (SDB). For each allocation,
332 * a 4K page is used. The number of sample-data-block-tables (SDBT)
336 * Returns zero on success, non-zero otherwise.
342 if (sfb->sdbt) in alloc_sampling_buffer()
343 return -EINVAL; in alloc_sampling_buffer()
345 /* Allocate the sample-data-block-table origin */ in alloc_sampling_buffer()
346 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in alloc_sampling_buffer()
347 if (!sfb->sdbt) in alloc_sampling_buffer()
348 return -ENOMEM; in alloc_sampling_buffer()
349 sfb->num_sdb = 0; in alloc_sampling_buffer()
350 sfb->num_sdbt = 1; in alloc_sampling_buffer()
355 sfb->tail = sfb->sdbt; in alloc_sampling_buffer()
356 *sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1; in alloc_sampling_buffer()
358 /* Allocate requested number of sample-data-blocks */ in alloc_sampling_buffer()
368 (unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt); in alloc_sampling_buffer()
393 if (!sfb->sdbt) in sfb_pending_allocs()
395 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) in sfb_pending_allocs()
396 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; in sfb_pending_allocs()
409 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); in sfb_account_allocs()
422 if (cpuhw->sfb.sdbt) in deallocate_buffers()
423 free_sampling_buffer(&cpuhw->sfb); in deallocate_buffers()
440 * Control indicator to trigger a measurement-alert to harvest in allocate_buffers()
441 * sample-data-blocks (SDB). This is done per SDB. This in allocate_buffers()
446 * n-th page. This is counterproductive as one IRQ triggers in allocate_buffers()
456 * 4. Compute the number of sample-data-block-tables (SDBT) and in allocate_buffers()
461 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); in allocate_buffers()
482 return alloc_sampling_buffer(&cpuhw->sfb, in allocate_buffers()
483 sfb_pending_allocs(&cpuhw->sfb, hwc)); in allocate_buffers()
486 static unsigned long min_percent(unsigned int percent, unsigned long base, in min_percent() argument
489 return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100)); in min_percent()
492 static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base) in compute_sfb_extent() argument
494 /* Use a percentage-based approach to extend the sampling facility in compute_sfb_extent()
497 * sample-data-blocks. in compute_sfb_extent()
502 return min_percent(1, base, 1); in compute_sfb_extent()
504 return min_percent(1, base, 1); in compute_sfb_extent()
506 return min_percent(2, base, 2); in compute_sfb_extent()
508 return min_percent(3, base, 3); in compute_sfb_extent()
510 return min_percent(4, base, 4); in compute_sfb_extent()
512 return min_percent(5, base, 8); in compute_sfb_extent()
524 * that has been lost because sample-data-blocks were full. in sfb_account_overflows()
530 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, in sfb_account_overflows()
531 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); in sfb_account_overflows()
533 /* Compute number of sample-data-blocks */ in sfb_account_overflows()
534 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); in sfb_account_overflows()
543 /* extend_sampling_buffer() - Extend sampling buffer
563 num_old = sfb->num_sdb; in extend_sampling_buffer()
573 * new SDB-request cannot be satisfied immediately. in extend_sampling_buffer()
583 __func__, num, sfb->num_sdb - num_old, in extend_sampling_buffer()
603 err = qsi(&cpusf->qsi); in setup_pmc_cpu()
606 cpusf->flags |= PMU_F_RESERVED; in setup_pmc_cpu()
610 cpusf->flags &= ~PMU_F_RESERVED; in setup_pmc_cpu()
637 return -ENODEV; in reserve_pmc_hardware()
647 if (!atomic_add_unless(&num_events, -1, 1)) { in hw_perf_event_destroy()
657 hwc->sample_period = period; in hw_init_period()
658 hwc->last_period = hwc->sample_period; in hw_init_period()
659 local64_set(&hwc->period_left, hwc->sample_period); in hw_init_period()
666 si->min_sampl_rate, si->max_sampl_rate); in hw_limit_rate()
679 pid = -1; in cpumsf_pid_type()
685 if (event->parent) in cpumsf_pid_type()
686 event = event->parent; in cpumsf_pid_type()
687 pid = __task_pid_nr_ns(tsk, type, event->ns); in cpumsf_pid_type()
693 pid = -1; in cpumsf_pid_type()
708 * Obtain the PID from the basic-sampling data entry and in cpumsf_output_event_pid()
709 * correct the data->tid_entry.pid value. in cpumsf_output_event_pid()
711 pid = data->tid_entry.pid; in cpumsf_output_event_pid()
722 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID); in cpumsf_output_event_pid()
723 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); in cpumsf_output_event_pid()
783 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init_rate()
784 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init_rate()
787 if (attr->freq) { in __hw_perf_event_init_rate()
788 if (!attr->sample_freq) in __hw_perf_event_init_rate()
789 return -EINVAL; in __hw_perf_event_init_rate()
790 rate = getrate(attr->freq, attr->sample_freq, si); in __hw_perf_event_init_rate()
791 attr->freq = 0; /* Don't call perf_adjust_period() */ in __hw_perf_event_init_rate()
794 rate = getrate(attr->freq, attr->sample_period, si); in __hw_perf_event_init_rate()
796 return -EINVAL; in __hw_perf_event_init_rate()
798 attr->sample_period = rate; in __hw_perf_event_init_rate()
802 __func__, event->cpu, event->attr.sample_period, in __hw_perf_event_init_rate()
803 event->attr.freq, SAMPLE_FREQ_MODE(hwc)); in __hw_perf_event_init_rate()
811 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
812 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
815 /* Reserve CPU-measurement sampling facility */ in __hw_perf_event_init()
820 err = -EBUSY; in __hw_perf_event_init()
825 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
830 /* Access per-CPU sampling information (query sampling info) */ in __hw_perf_event_init()
832 * The event->cpu value can be -1 to count on every CPU, for example, in __hw_perf_event_init()
834 * sampling info from the current CPU, otherwise use event->cpu to in __hw_perf_event_init()
835 * retrieve the per-CPU information. in __hw_perf_event_init()
841 if (event->cpu == -1) in __hw_perf_event_init()
844 /* Event is pinned to a particular CPU, retrieve the per-CPU in __hw_perf_event_init()
845 * sampling structure for accessing the CPU-specific QSI. in __hw_perf_event_init()
847 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in __hw_perf_event_init()
848 si = cpuhw->qsi; in __hw_perf_event_init()
856 err = -ENOENT; in __hw_perf_event_init()
862 err = -EBUSY; in __hw_perf_event_init()
872 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { in __hw_perf_event_init()
874 err = -EPERM; in __hw_perf_event_init()
885 hwc->extra_reg.reg = REG_OVERFLOW; in __hw_perf_event_init()
889 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) in __hw_perf_event_init()
892 /* Allocate the per-CPU sampling buffer using the CPU information in __hw_perf_event_init()
894 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling in __hw_perf_event_init()
913 * handler to extract and resolve the PIDs from the basic-sampling in __hw_perf_event_init()
916 if (event->attr.sample_type & PERF_SAMPLE_TID) in __hw_perf_event_init()
918 event->overflow_handler = cpumsf_output_event_pid; in __hw_perf_event_init()
925 u64 sample_type = event->attr.sample_type; in is_callchain_event()
938 return -EOPNOTSUPP; in cpumsf_pmu_event_init()
940 switch (event->attr.type) { in cpumsf_pmu_event_init()
942 if ((event->attr.config != PERF_EVENT_CPUM_SF) && in cpumsf_pmu_event_init()
943 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) in cpumsf_pmu_event_init()
944 return -ENOENT; in cpumsf_pmu_event_init()
952 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) in cpumsf_pmu_event_init()
953 return -ENOENT; in cpumsf_pmu_event_init()
955 return -ENOENT; in cpumsf_pmu_event_init()
958 return -ENOENT; in cpumsf_pmu_event_init()
964 if (event->attr.exclude_hv) in cpumsf_pmu_event_init()
965 event->attr.exclude_hv = 0; in cpumsf_pmu_event_init()
966 if (event->attr.exclude_idle) in cpumsf_pmu_event_init()
967 event->attr.exclude_idle = 0; in cpumsf_pmu_event_init()
971 if (event->destroy) in cpumsf_pmu_event_init()
972 event->destroy(event); in cpumsf_pmu_event_init()
982 if (cpuhw->flags & PMU_F_ENABLED) in cpumsf_pmu_enable()
985 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_enable()
996 * facility, but it can be fully re-enabled using sampling controls that in cpumsf_pmu_enable()
999 if (cpuhw->event) { in cpumsf_pmu_enable()
1000 hwc = &cpuhw->event->hw; in cpumsf_pmu_enable()
1003 * Account number of overflow-designated in cpumsf_pmu_enable()
1007 extend_sampling_buffer(&cpuhw->sfb, hwc); in cpumsf_pmu_enable()
1010 cpuhw->lsctl.interval = SAMPL_RATE(&cpuhw->event->hw); in cpumsf_pmu_enable()
1014 cpuhw->flags |= PMU_F_ENABLED; in cpumsf_pmu_enable()
1017 err = lsctl(&cpuhw->lsctl); in cpumsf_pmu_enable()
1019 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_enable()
1029 cpuhw->lsctl.es, cpuhw->lsctl.cs, cpuhw->lsctl.ed, in cpumsf_pmu_enable()
1030 cpuhw->lsctl.cd, cpuhw->lsctl.interval, in cpumsf_pmu_enable()
1031 cpuhw->lsctl.tear, cpuhw->lsctl.dear); in cpumsf_pmu_enable()
1041 if (!(cpuhw->flags & PMU_F_ENABLED)) in cpumsf_pmu_disable()
1044 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_disable()
1048 inactive = cpuhw->lsctl; in cpumsf_pmu_disable()
1067 cpuhw->lsctl.tear = si.tear; in cpumsf_pmu_disable()
1068 cpuhw->lsctl.dear = si.dear; in cpumsf_pmu_disable()
1074 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_disable()
1077 /* perf_exclude_event() - Filter event
1080 * @sde_regs: Sample-data-entry (sde) regs structure
1084 * Return non-zero if the event shall be excluded.
1089 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
1091 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
1093 if (event->attr.exclude_guest && sde_regs->in_guest) in perf_exclude_event()
1095 if (event->attr.exclude_host && !sde_regs->in_guest) in perf_exclude_event()
1100 /* perf_push_sample() - Push samples to perf
1109 * Return non-zero if an event overflow occurred.
1120 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_push_sample()
1122 /* Setup pt_regs to look like an CPU-measurement external interrupt in perf_push_sample()
1124 * field which is unused contains additional sample-data-entry related in perf_push_sample()
1132 psw_bits(regs.psw).ia = basic->ia; in perf_push_sample()
1133 psw_bits(regs.psw).dat = basic->T; in perf_push_sample()
1134 psw_bits(regs.psw).wait = basic->W; in perf_push_sample()
1135 psw_bits(regs.psw).pstate = basic->P; in perf_push_sample()
1136 psw_bits(regs.psw).as = basic->AS; in perf_push_sample()
1142 * A non-zero guest program parameter always indicates a guest in perf_push_sample()
1149 switch (basic->CL) { in perf_push_sample()
1151 sde_regs->in_guest = 0; in perf_push_sample()
1154 sde_regs->in_guest = 1; in perf_push_sample()
1157 if (basic->gpp || basic->prim_asn != 0xffff) in perf_push_sample()
1158 sde_regs->in_guest = 1; in perf_push_sample()
1163 * Store the PID value from the sample-data-entry to be in perf_push_sample()
1166 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; in perf_push_sample()
1173 event->pmu->stop(event, 0); in perf_push_sample()
1182 local64_add(count, &event->count); in perf_event_count_update()
1185 /* hw_collect_samples() - Walk through a sample-data-block and collect samples
1187 * @sdbt: Sample-data-block table
1190 * Walks through a sample-data-block and collects sampling data entries that are
1192 * there can be either basic-sampling or combined-sampling data entries. A
1193 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1195 * event hardware structure. The function always works with a combined-sampling
1198 * Note that the implementation focuses on basic-sampling data entries and, if
1199 * such an entry is not valid, the entire combined-sampling data entry is
1208 struct hws_trailer_entry *te; in hw_collect_samples() local
1211 te = trailer_entry_ptr((unsigned long)sdbt); in hw_collect_samples()
1213 while ((unsigned long *)sample < (unsigned long *)te) { in hw_collect_samples()
1215 if (!sample->def || sample->LS) in hw_collect_samples()
1219 perf_event_count_update(event, SAMPL_RATE(&event->hw)); in hw_collect_samples()
1222 if (sample->def == 0x0001) { in hw_collect_samples()
1229 if (sample->I == 0 && sample->W == 0) { in hw_collect_samples()
1240 " sampling data entry: te->f %i" in hw_collect_samples()
1242 te->header.f, sample->def, sample); in hw_collect_samples()
1246 * from a combined basic- and diagnostic-sampling. in hw_collect_samples()
1247 * If only basic-sampling is then active, entries are in hw_collect_samples()
1249 * This is typically the case for sample-data-blocks in hw_collect_samples()
1253 if (!te->header.f) in hw_collect_samples()
1258 sample->def = 0; in hw_collect_samples()
1263 /* hw_perf_event_update() - Process sampling buffer
1265 * @flush_all: Flag to also flush partially filled sample-data-blocks
1271 * Only full sample-data-blocks are processed. Specify the flush_all flag
1272 * to also walk through partially filled sample-data-blocks.
1278 struct hw_perf_event *hwc = &event->hw; in hw_perf_event_update()
1279 struct hws_trailer_entry *te; in hw_perf_event_update() local
1287 if (SAMPL_DIAG_MODE(&event->hw)) in hw_perf_event_update()
1293 /* Get the trailer entry of the sample-data-block */ in hw_perf_event_update()
1295 te = trailer_entry_ptr(sdb); in hw_perf_event_update()
1298 if (!te->header.f) { in hw_perf_event_update()
1305 if (te->header.overflow) in hw_perf_event_update()
1310 sampl_overflow += te->header.overflow; in hw_perf_event_update()
1312 /* Timestamps are valid for full sample-data-blocks only */ in hw_perf_event_update()
1316 te->header.overflow, in hw_perf_event_update()
1317 (te->header.f) ? trailer_timestamp(te) : 0ULL); in hw_perf_event_update()
1319 /* Collect all samples from a single sample-data-block and in hw_perf_event_update()
1326 /* Reset trailer (using compare-double-and-swap) */ in hw_perf_event_update()
1327 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in hw_perf_event_update()
1334 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in hw_perf_event_update()
1337 /* Advance to next sample-data-block */ in hw_perf_event_update()
1345 /* Stop processing sample-data if all samples of the current in hw_perf_event_update()
1346 * sample-data-block were flushed even if it was not full. in hw_perf_event_update()
1366 SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); in hw_perf_event_update()
1369 DIV_ROUND_UP(SAMPL_RATE(hwc), 10)); in hw_perf_event_update()
1383 return i % aux->sfb.num_sdb; in aux_sdb_index()
1388 return end >= start ? end - start + 1 : 0; in aux_sdb_num()
1393 return aux_sdb_num(aux->head, aux->alert_mark); in aux_sdb_num_alert()
1398 return aux_sdb_num(aux->head, aux->empty_mark); in aux_sdb_num_empty()
1410 sdb = aux->sdb_index[index]; in aux_sdb_trailer()
1427 struct hws_trailer_entry *te; in aux_output_end() local
1434 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { in aux_output_end()
1435 te = aux_sdb_trailer(aux, idx); in aux_output_end()
1436 if (!te->header.f) in aux_output_end()
1443 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_end()
1444 te->header.a = 0; in aux_output_end()
1447 __func__, i, range_scan, aux->head); in aux_output_end()
1463 unsigned long range, i, range_scan, idx, head, base, offset; in aux_output_begin() local
1464 struct hws_trailer_entry *te; in aux_output_begin() local
1466 if (handle->head & ~PAGE_MASK) in aux_output_begin()
1467 return -EINVAL; in aux_output_begin()
1469 aux->head = handle->head >> PAGE_SHIFT; in aux_output_begin()
1470 range = (handle->size + 1) >> PAGE_SHIFT; in aux_output_begin()
1472 return -ENOMEM; in aux_output_begin()
1475 * SDBs between aux->head and aux->empty_mark are already ready in aux_output_begin()
1480 __func__, range, aux->head, aux->alert_mark, in aux_output_begin()
1481 aux->empty_mark); in aux_output_begin()
1483 range_scan = range - aux_sdb_num_empty(aux); in aux_output_begin()
1484 idx = aux->empty_mark + 1; in aux_output_begin()
1486 te = aux_sdb_trailer(aux, idx); in aux_output_begin()
1487 te->header.f = 0; in aux_output_begin()
1488 te->header.a = 0; in aux_output_begin()
1489 te->header.overflow = 0; in aux_output_begin()
1492 aux->empty_mark = aux->head + range - 1; in aux_output_begin()
1496 aux->alert_mark = aux->head + range/2 - 1; in aux_output_begin()
1497 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_begin()
1498 te->header.a = 1; in aux_output_begin()
1501 head = aux_sdb_index(aux, aux->head); in aux_output_begin()
1502 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; in aux_output_begin()
1504 cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long); in aux_output_begin()
1505 cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]); in aux_output_begin()
1509 aux->head, aux->alert_mark, aux->empty_mark, in aux_output_begin()
1511 cpuhw->lsctl.tear, cpuhw->lsctl.dear); in aux_output_begin()
1526 struct hws_trailer_entry *te; in aux_set_alert() local
1528 te = aux_sdb_trailer(aux, alert_index); in aux_set_alert()
1529 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_set_alert()
1544 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in aux_set_alert()
1550 * aux_reset_buffer() - Scan and setup SDBs for new samples
1552 * @range: The range of SDBs to scan started from aux->head
1555 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1560 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1563 * If aux->alert_mark fall in this area, just set it. Overflow count is
1566 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1577 struct hws_trailer_entry *te; in aux_reset_buffer() local
1580 "empty %ld\n", __func__, range, aux->head, in aux_reset_buffer()
1581 aux->alert_mark, aux->empty_mark); in aux_reset_buffer()
1588 return aux_set_alert(aux, aux->alert_mark, overflow); in aux_reset_buffer()
1590 if (aux->alert_mark <= aux->empty_mark) in aux_reset_buffer()
1595 if (!aux_set_alert(aux, aux->alert_mark, overflow)) in aux_reset_buffer()
1603 range_scan = range - aux_sdb_num_empty(aux); in aux_reset_buffer()
1604 idx_old = idx = aux->empty_mark + 1; in aux_reset_buffer()
1606 te = aux_sdb_trailer(aux, idx); in aux_reset_buffer()
1607 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_reset_buffer()
1614 if (idx == aux->alert_mark) in aux_reset_buffer()
1618 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in aux_reset_buffer()
1624 aux->empty_mark = aux->head + range - 1; in aux_reset_buffer()
1628 idx - 1, aux->empty_mark); in aux_reset_buffer()
1641 struct perf_output_handle *handle = &cpuhw->handle; in hw_collect_aux()
1654 num_sdb = aux->sfb.num_sdb; in hw_collect_aux()
1657 aux = perf_aux_output_begin(handle, cpuhw->event); in hw_collect_aux()
1658 if (handle->size == 0) { in hw_collect_aux()
1660 "diagnostic-sampling mode is full\n", in hw_collect_aux()
1668 aux->head = handle->head >> PAGE_SHIFT; in hw_collect_aux()
1669 range = (handle->size + 1) >> PAGE_SHIFT; in hw_collect_aux()
1671 aux->alert_mark = aux->head; in hw_collect_aux()
1673 aux->alert_mark = aux->head + range/2 - 1; in hw_collect_aux()
1681 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1683 "pages to overflow\n", aux->sfb.num_sdb); in hw_collect_aux()
1686 aux->head, range, overflow); in hw_collect_aux()
1689 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1693 aux->head, aux->alert_mark); in hw_collect_aux()
1699 "empty %ld\n", __func__, aux->head, in hw_collect_aux()
1700 aux->alert_mark, aux->empty_mark); in hw_collect_aux()
1715 num_sdbt = aux->sfb.num_sdbt; in aux_buffer_free()
1717 free_page(aux->sdbt_index[i]); in aux_buffer_free()
1719 kfree(aux->sdbt_index); in aux_buffer_free()
1720 kfree(aux->sdb_index); in aux_buffer_free()
1728 struct hws_trailer_entry *te; in aux_sdb_init() local
1730 te = trailer_entry_ptr(sdb); in aux_sdb_init()
1732 /* Save clock base */ in aux_sdb_init()
1733 te->clock_base = 1; in aux_sdb_init()
1734 te->progusage2 = tod_clock_base.tod; in aux_sdb_init()
1738 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1739 * @event: Event the buffer is setup for, event->cpu == -1 means current
1747 * the task among online cpus when it is a per-thread event.
1778 sfb = &aux->sfb; in aux_buffer_setup()
1782 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1783 if (!aux->sdbt_index) in aux_buffer_setup()
1787 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1788 if (!aux->sdb_index) in aux_buffer_setup()
1792 sfb->num_sdbt = 0; in aux_buffer_setup()
1793 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in aux_buffer_setup()
1794 if (!sfb->sdbt) in aux_buffer_setup()
1796 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; in aux_buffer_setup()
1797 tail = sfb->tail = sfb->sdbt; in aux_buffer_setup()
1808 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; in aux_buffer_setup()
1815 aux->sdb_index[i] = (unsigned long)pages[i]; in aux_buffer_setup()
1818 sfb->num_sdb = nr_pages; in aux_buffer_setup()
1821 *tail = virt_to_phys(sfb->sdbt) + 1; in aux_buffer_setup()
1822 sfb->tail = tail; in aux_buffer_setup()
1829 aux->empty_mark = sfb->num_sdb - 1; in aux_buffer_setup()
1832 sfb->num_sdbt, sfb->num_sdb); in aux_buffer_setup()
1838 for (i = 0; i < sfb->num_sdbt; i++) in aux_buffer_setup()
1839 free_page(aux->sdbt_index[i]); in aux_buffer_setup()
1840 kfree(aux->sdb_index); in aux_buffer_setup()
1842 kfree(aux->sdbt_index); in aux_buffer_setup()
1851 /* Nothing to do ... updates are interrupt-driven */ in cpumsf_pmu_read()
1856 * Return non-zero on error and zero on passed checks.
1865 if (event->cpu == -1) { in cpumsf_pmu_check_period()
1867 return -ENODEV; in cpumsf_pmu_check_period()
1869 /* Event is pinned to a particular CPU, retrieve the per-CPU in cpumsf_pmu_check_period()
1870 * sampling structure for accessing the CPU-specific QSI. in cpumsf_pmu_check_period()
1872 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in cpumsf_pmu_check_period()
1874 si = cpuhw->qsi; in cpumsf_pmu_check_period()
1877 do_freq = !!SAMPLE_FREQ_MODE(&event->hw); in cpumsf_pmu_check_period()
1880 return -EINVAL; in cpumsf_pmu_check_period()
1882 event->attr.sample_period = rate; in cpumsf_pmu_check_period()
1883 SAMPL_RATE(&event->hw) = rate; in cpumsf_pmu_check_period()
1884 hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); in cpumsf_pmu_check_period()
1887 __func__, event->cpu, value, in cpumsf_pmu_check_period()
1888 event->attr.sample_period, do_freq); in cpumsf_pmu_check_period()
1899 if (!(event->hw.state & PERF_HES_STOPPED)) in cpumsf_pmu_start()
1901 perf_pmu_disable(event->pmu); in cpumsf_pmu_start()
1902 event->hw.state = 0; in cpumsf_pmu_start()
1903 cpuhw->lsctl.cs = 1; in cpumsf_pmu_start()
1904 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_start()
1905 cpuhw->lsctl.cd = 1; in cpumsf_pmu_start()
1906 perf_pmu_enable(event->pmu); in cpumsf_pmu_start()
1916 if (event->hw.state & PERF_HES_STOPPED) in cpumsf_pmu_stop()
1919 perf_pmu_disable(event->pmu); in cpumsf_pmu_stop()
1920 cpuhw->lsctl.cs = 0; in cpumsf_pmu_stop()
1921 cpuhw->lsctl.cd = 0; in cpumsf_pmu_stop()
1922 event->hw.state |= PERF_HES_STOPPED; in cpumsf_pmu_stop()
1924 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { in cpumsf_pmu_stop()
1926 if (cpuhw->flags & PMU_F_RESERVED) in cpumsf_pmu_stop()
1928 event->hw.state |= PERF_HES_UPTODATE; in cpumsf_pmu_stop()
1930 perf_pmu_enable(event->pmu); in cpumsf_pmu_stop()
1939 if (cpuhw->flags & PMU_F_IN_USE) in cpumsf_pmu_add()
1940 return -EAGAIN; in cpumsf_pmu_add()
1942 if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt) in cpumsf_pmu_add()
1943 return -EINVAL; in cpumsf_pmu_add()
1946 perf_pmu_disable(event->pmu); in cpumsf_pmu_add()
1948 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumsf_pmu_add()
1951 * using the SDB-table start. Reset TEAR_REG event hardware register in cpumsf_pmu_add()
1955 cpuhw->lsctl.s = 0; in cpumsf_pmu_add()
1956 cpuhw->lsctl.h = 1; in cpumsf_pmu_add()
1957 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); in cpumsf_pmu_add()
1958 if (!SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1959 cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt); in cpumsf_pmu_add()
1960 cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1961 TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1966 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { in cpumsf_pmu_add()
1967 err = -EAGAIN; in cpumsf_pmu_add()
1970 if (SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1971 aux = perf_aux_output_begin(&cpuhw->handle, event); in cpumsf_pmu_add()
1973 err = -EINVAL; in cpumsf_pmu_add()
1976 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); in cpumsf_pmu_add()
1979 cpuhw->lsctl.ed = 1; in cpumsf_pmu_add()
1981 cpuhw->lsctl.es = 1; in cpumsf_pmu_add()
1984 cpuhw->event = event; in cpumsf_pmu_add()
1985 cpuhw->flags |= PMU_F_IN_USE; in cpumsf_pmu_add()
1991 perf_pmu_enable(event->pmu); in cpumsf_pmu_add()
1999 perf_pmu_disable(event->pmu); in cpumsf_pmu_del()
2002 cpuhw->lsctl.es = 0; in cpumsf_pmu_del()
2003 cpuhw->lsctl.ed = 0; in cpumsf_pmu_del()
2004 cpuhw->flags &= ~PMU_F_IN_USE; in cpumsf_pmu_del()
2005 cpuhw->event = NULL; in cpumsf_pmu_del()
2007 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_del()
2008 aux_output_end(&cpuhw->handle); in cpumsf_pmu_del()
2010 perf_pmu_enable(event->pmu); in cpumsf_pmu_del()
2042 PMU_FORMAT_ATTR(event, "config:0-63");
2097 if (!(cpuhw->flags & PMU_F_RESERVED)) in cpumf_measurement_alert()
2105 if (cpuhw->flags & PMU_F_IN_USE) in cpumf_measurement_alert()
2106 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) in cpumf_measurement_alert()
2109 hw_perf_event_update(cpuhw->event, 0); in cpumf_measurement_alert()
2111 WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE)); in cpumf_measurement_alert()
2114 /* Report measurement alerts only for non-PRA codes */ in cpumf_measurement_alert()
2121 qsi(&cpuhw->qsi); in cpumf_measurement_alert()
2123 /* Loss of sample data due to high-priority machine activities */ in cpumf_measurement_alert()
2126 cpuhw->flags |= PMU_F_ERR_LSDA; in cpumf_measurement_alert()
2134 cpuhw->flags |= PMU_F_ERR_IBE; in cpumf_measurement_alert()
2166 return -ENODEV; in param_get_sfb_size()
2176 return -ENODEV; in param_set_sfb_size()
2178 return -EINVAL; in param_set_sfb_size()
2184 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; in param_set_sfb_size()
2186 rc = kstrtoul(val, 10, &max); in param_set_sfb_size()
2189 rc = -EINVAL; in param_set_sfb_size()
2222 return -ENODEV; in init_cpum_sampling_pmu()
2227 return -ENODEV; in init_cpum_sampling_pmu()
2231 return -ENODEV; in init_cpum_sampling_pmu()
2235 return -EINVAL; in init_cpum_sampling_pmu()
2250 return -ENOMEM; in init_cpum_sampling_pmu()