1 /* 2 * intel-bts.c: Intel Processor Trace support 3 * Copyright (c) 2013-2015, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #include <errno.h> 17 #include <linux/kernel.h> 18 #include <linux/types.h> 19 #include <linux/bitops.h> 20 #include <linux/log2.h> 21 22 #include "../../util/cpumap.h" 23 #include "../../util/evsel.h" 24 #include "../../util/evlist.h" 25 #include "../../util/session.h" 26 #include "../../util/util.h" 27 #include "../../util/pmu.h" 28 #include "../../util/debug.h" 29 #include "../../util/tsc.h" 30 #include "../../util/auxtrace.h" 31 #include "../../util/intel-bts.h" 32 33 #define KiB(x) ((x) * 1024) 34 #define MiB(x) ((x) * 1024 * 1024) 35 #define KiB_MASK(x) (KiB(x) - 1) 36 #define MiB_MASK(x) (MiB(x) - 1) 37 38 #define INTEL_BTS_DFLT_SAMPLE_SIZE KiB(4) 39 40 #define INTEL_BTS_MAX_SAMPLE_SIZE KiB(60) 41 42 struct intel_bts_snapshot_ref { 43 void *ref_buf; 44 size_t ref_offset; 45 bool wrapped; 46 }; 47 48 struct intel_bts_recording { 49 struct auxtrace_record itr; 50 struct perf_pmu *intel_bts_pmu; 51 struct perf_evlist *evlist; 52 bool snapshot_mode; 53 size_t snapshot_size; 54 int snapshot_ref_cnt; 55 struct intel_bts_snapshot_ref *snapshot_refs; 56 }; 57 58 struct branch { 59 u64 from; 60 u64 to; 61 u64 misc; 62 }; 63 64 static size_t 65 intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused, 66 struct perf_evlist *evlist __maybe_unused) 67 { 68 return INTEL_BTS_AUXTRACE_PRIV_SIZE; 69 } 70 71 static int intel_bts_info_fill(struct auxtrace_record *itr, 72 struct perf_session *session, 73 struct auxtrace_info_event *auxtrace_info, 74 size_t priv_size) 75 { 76 struct intel_bts_recording *btsr = 77 container_of(itr, struct intel_bts_recording, itr); 78 struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; 79 struct perf_event_mmap_page *pc; 80 struct perf_tsc_conversion tc = { .time_mult = 0, }; 81 bool cap_user_time_zero = false; 82 int err; 83 84 if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE) 85 return -EINVAL; 86 87 if (!session->evlist->nr_mmaps) 88 return -EINVAL; 89 90 pc = session->evlist->mmap[0].base; 91 if (pc) { 92 err = perf_read_tsc_conversion(pc, &tc); 93 if (err) { 94 if (err != -EOPNOTSUPP) 95 return err; 96 } else { 97 cap_user_time_zero = tc.time_mult != 0; 98 } 99 if (!cap_user_time_zero) 100 ui__warning("Intel BTS: TSC not available\n"); 101 } 102 103 auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS; 104 auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type; 105 auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift; 106 auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult; 107 auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero; 108 auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero; 109 auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode; 110 111 return 0; 112 } 113 114 static int intel_bts_recording_options(struct auxtrace_record *itr, 115 struct perf_evlist *evlist, 116 struct record_opts *opts) 117 { 118 struct intel_bts_recording *btsr = 119 container_of(itr, struct intel_bts_recording, itr); 120 struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; 121 struct perf_evsel *evsel, *intel_bts_evsel = NULL; 122 const struct cpu_map *cpus = evlist->cpus; 123 bool privileged = geteuid() == 0 || perf_event_paranoid() < 0; 124 125 btsr->evlist = evlist; 126 btsr->snapshot_mode = opts->auxtrace_snapshot_mode; 127 128 evlist__for_each_entry(evlist, evsel) { 129 if (evsel->attr.type == intel_bts_pmu->type) { 130 if (intel_bts_evsel) { 131 pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n"); 132 return -EINVAL; 133 } 134 evsel->attr.freq = 0; 135 evsel->attr.sample_period = 1; 136 intel_bts_evsel = evsel; 137 opts->full_auxtrace = true; 138 } 139 } 140 141 if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) { 142 pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n"); 143 return -EINVAL; 144 } 145 146 if (!opts->full_auxtrace) 147 return 0; 148 149 if (opts->full_auxtrace && !cpu_map__empty(cpus)) { 150 pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n"); 151 return -EINVAL; 152 } 153 154 /* Set default sizes for snapshot mode */ 155 if (opts->auxtrace_snapshot_mode) { 156 if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) { 157 if (privileged) { 158 opts->auxtrace_mmap_pages = MiB(4) / page_size; 159 } else { 160 opts->auxtrace_mmap_pages = KiB(128) / page_size; 161 if (opts->mmap_pages == UINT_MAX) 162 opts->mmap_pages = KiB(256) / page_size; 163 } 164 } else if (!opts->auxtrace_mmap_pages && !privileged && 165 opts->mmap_pages == UINT_MAX) { 166 opts->mmap_pages = KiB(256) / page_size; 167 } 168 if (!opts->auxtrace_snapshot_size) 169 opts->auxtrace_snapshot_size = 170 opts->auxtrace_mmap_pages * (size_t)page_size; 171 if (!opts->auxtrace_mmap_pages) { 172 size_t sz = opts->auxtrace_snapshot_size; 173 174 sz = round_up(sz, page_size) / page_size; 175 opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); 176 } 177 if (opts->auxtrace_snapshot_size > 178 opts->auxtrace_mmap_pages * (size_t)page_size) { 179 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n", 180 opts->auxtrace_snapshot_size, 181 opts->auxtrace_mmap_pages * (size_t)page_size); 182 return -EINVAL; 183 } 184 if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) { 185 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n"); 186 return -EINVAL; 187 } 188 pr_debug2("Intel BTS snapshot size: %zu\n", 189 opts->auxtrace_snapshot_size); 190 } 191 192 /* Set default sizes for full trace mode */ 193 if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { 194 if (privileged) { 195 opts->auxtrace_mmap_pages = MiB(4) / page_size; 196 } else { 197 opts->auxtrace_mmap_pages = KiB(128) / page_size; 198 if (opts->mmap_pages == UINT_MAX) 199 opts->mmap_pages = KiB(256) / page_size; 200 } 201 } 202 203 /* Validate auxtrace_mmap_pages */ 204 if (opts->auxtrace_mmap_pages) { 205 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; 206 size_t min_sz; 207 208 if (opts->auxtrace_snapshot_mode) 209 min_sz = KiB(4); 210 else 211 min_sz = KiB(8); 212 213 if (sz < min_sz || !is_power_of_2(sz)) { 214 pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n", 215 min_sz / 1024); 216 return -EINVAL; 217 } 218 } 219 220 if (intel_bts_evsel) { 221 /* 222 * To obtain the auxtrace buffer file descriptor, the auxtrace event 223 * must come first. 224 */ 225 perf_evlist__to_front(evlist, intel_bts_evsel); 226 /* 227 * In the case of per-cpu mmaps, we need the CPU on the 228 * AUX event. 229 */ 230 if (!cpu_map__empty(cpus)) 231 perf_evsel__set_sample_bit(intel_bts_evsel, CPU); 232 } 233 234 /* Add dummy event to keep tracking */ 235 if (opts->full_auxtrace) { 236 struct perf_evsel *tracking_evsel; 237 int err; 238 239 err = parse_events(evlist, "dummy:u", NULL); 240 if (err) 241 return err; 242 243 tracking_evsel = perf_evlist__last(evlist); 244 245 perf_evlist__set_tracking_event(evlist, tracking_evsel); 246 247 tracking_evsel->attr.freq = 0; 248 tracking_evsel->attr.sample_period = 1; 249 } 250 251 return 0; 252 } 253 254 static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr, 255 struct record_opts *opts, 256 const char *str) 257 { 258 struct intel_bts_recording *btsr = 259 container_of(itr, struct intel_bts_recording, itr); 260 unsigned long long snapshot_size = 0; 261 char *endptr; 262 263 if (str) { 264 snapshot_size = strtoull(str, &endptr, 0); 265 if (*endptr || snapshot_size > SIZE_MAX) 266 return -1; 267 } 268 269 opts->auxtrace_snapshot_mode = true; 270 opts->auxtrace_snapshot_size = snapshot_size; 271 272 btsr->snapshot_size = snapshot_size; 273 274 return 0; 275 } 276 277 static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused) 278 { 279 return rdtsc(); 280 } 281 282 static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr, 283 int idx) 284 { 285 const size_t sz = sizeof(struct intel_bts_snapshot_ref); 286 int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2; 287 struct intel_bts_snapshot_ref *refs; 288 289 if (!new_cnt) 290 new_cnt = 16; 291 292 while (new_cnt <= idx) 293 new_cnt *= 2; 294 295 refs = calloc(new_cnt, sz); 296 if (!refs) 297 return -ENOMEM; 298 299 memcpy(refs, btsr->snapshot_refs, cnt * sz); 300 301 btsr->snapshot_refs = refs; 302 btsr->snapshot_ref_cnt = new_cnt; 303 304 return 0; 305 } 306 307 static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr) 308 { 309 int i; 310 311 for (i = 0; i < btsr->snapshot_ref_cnt; i++) 312 zfree(&btsr->snapshot_refs[i].ref_buf); 313 zfree(&btsr->snapshot_refs); 314 } 315 316 static void intel_bts_recording_free(struct auxtrace_record *itr) 317 { 318 struct intel_bts_recording *btsr = 319 container_of(itr, struct intel_bts_recording, itr); 320 321 intel_bts_free_snapshot_refs(btsr); 322 free(btsr); 323 } 324 325 static int intel_bts_snapshot_start(struct auxtrace_record *itr) 326 { 327 struct intel_bts_recording *btsr = 328 container_of(itr, struct intel_bts_recording, itr); 329 struct perf_evsel *evsel; 330 331 evlist__for_each_entry(btsr->evlist, evsel) { 332 if (evsel->attr.type == btsr->intel_bts_pmu->type) 333 return perf_evsel__disable(evsel); 334 } 335 return -EINVAL; 336 } 337 338 static int intel_bts_snapshot_finish(struct auxtrace_record *itr) 339 { 340 struct intel_bts_recording *btsr = 341 container_of(itr, struct intel_bts_recording, itr); 342 struct perf_evsel *evsel; 343 344 evlist__for_each_entry(btsr->evlist, evsel) { 345 if (evsel->attr.type == btsr->intel_bts_pmu->type) 346 return perf_evsel__enable(evsel); 347 } 348 return -EINVAL; 349 } 350 351 static bool intel_bts_first_wrap(u64 *data, size_t buf_size) 352 { 353 int i, a, b; 354 355 b = buf_size >> 3; 356 a = b - 512; 357 if (a < 0) 358 a = 0; 359 360 for (i = a; i < b; i++) { 361 if (data[i]) 362 return true; 363 } 364 365 return false; 366 } 367 368 static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx, 369 struct auxtrace_mmap *mm, unsigned char *data, 370 u64 *head, u64 *old) 371 { 372 struct intel_bts_recording *btsr = 373 container_of(itr, struct intel_bts_recording, itr); 374 bool wrapped; 375 int err; 376 377 pr_debug3("%s: mmap index %d old head %zu new head %zu\n", 378 __func__, idx, (size_t)*old, (size_t)*head); 379 380 if (idx >= btsr->snapshot_ref_cnt) { 381 err = intel_bts_alloc_snapshot_refs(btsr, idx); 382 if (err) 383 goto out_err; 384 } 385 386 wrapped = btsr->snapshot_refs[idx].wrapped; 387 if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) { 388 btsr->snapshot_refs[idx].wrapped = true; 389 wrapped = true; 390 } 391 392 /* 393 * In full trace mode 'head' continually increases. However in snapshot 394 * mode 'head' is an offset within the buffer. Here 'old' and 'head' 395 * are adjusted to match the full trace case which expects that 'old' is 396 * always less than 'head'. 397 */ 398 if (wrapped) { 399 *old = *head; 400 *head += mm->len; 401 } else { 402 if (mm->mask) 403 *old &= mm->mask; 404 else 405 *old %= mm->len; 406 if (*old > *head) 407 *head += mm->len; 408 } 409 410 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n", 411 __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head); 412 413 return 0; 414 415 out_err: 416 pr_err("%s: failed, error %d\n", __func__, err); 417 return err; 418 } 419 420 static int intel_bts_read_finish(struct auxtrace_record *itr, int idx) 421 { 422 struct intel_bts_recording *btsr = 423 container_of(itr, struct intel_bts_recording, itr); 424 struct perf_evsel *evsel; 425 426 evlist__for_each_entry(btsr->evlist, evsel) { 427 if (evsel->attr.type == btsr->intel_bts_pmu->type) 428 return perf_evlist__enable_event_idx(btsr->evlist, 429 evsel, idx); 430 } 431 return -EINVAL; 432 } 433 434 struct auxtrace_record *intel_bts_recording_init(int *err) 435 { 436 struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME); 437 struct intel_bts_recording *btsr; 438 439 if (!intel_bts_pmu) 440 return NULL; 441 442 if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) { 443 *err = -errno; 444 return NULL; 445 } 446 447 btsr = zalloc(sizeof(struct intel_bts_recording)); 448 if (!btsr) { 449 *err = -ENOMEM; 450 return NULL; 451 } 452 453 btsr->intel_bts_pmu = intel_bts_pmu; 454 btsr->itr.recording_options = intel_bts_recording_options; 455 btsr->itr.info_priv_size = intel_bts_info_priv_size; 456 btsr->itr.info_fill = intel_bts_info_fill; 457 btsr->itr.free = intel_bts_recording_free; 458 btsr->itr.snapshot_start = intel_bts_snapshot_start; 459 btsr->itr.snapshot_finish = intel_bts_snapshot_finish; 460 btsr->itr.find_snapshot = intel_bts_find_snapshot; 461 btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options; 462 btsr->itr.reference = intel_bts_reference; 463 btsr->itr.read_finish = intel_bts_read_finish; 464 btsr->itr.alignment = sizeof(struct branch); 465 return &btsr->itr; 466 } 467