1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2015-2018 Linaro Limited. 4 * 5 * Author: Tor Jeremiassen <tor@ti.com> 6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/coresight-pmu.h> 11 #include <linux/err.h> 12 #include <linux/kernel.h> 13 #include <linux/log2.h> 14 #include <linux/types.h> 15 #include <linux/zalloc.h> 16 17 #include <opencsd/ocsd_if_types.h> 18 #include <stdlib.h> 19 20 #include "auxtrace.h" 21 #include "color.h" 22 #include "cs-etm.h" 23 #include "cs-etm-decoder/cs-etm-decoder.h" 24 #include "debug.h" 25 #include "dso.h" 26 #include "evlist.h" 27 #include "intlist.h" 28 #include "machine.h" 29 #include "map.h" 30 #include "perf.h" 31 #include "session.h" 32 #include "map_symbol.h" 33 #include "branch.h" 34 #include "symbol.h" 35 #include "tool.h" 36 #include "thread.h" 37 #include "thread-stack.h" 38 #include <tools/libc_compat.h> 39 #include "util/synthetic-events.h" 40 41 struct cs_etm_auxtrace { 42 struct auxtrace auxtrace; 43 struct auxtrace_queues queues; 44 struct auxtrace_heap heap; 45 struct itrace_synth_opts synth_opts; 46 struct perf_session *session; 47 struct machine *machine; 48 struct thread *unknown_thread; 49 50 u8 timeless_decoding; 51 u8 snapshot_mode; 52 u8 data_queued; 53 u8 sample_branches; 54 u8 sample_instructions; 55 56 int num_cpu; 57 u64 latest_kernel_timestamp; 58 u32 auxtrace_type; 59 u64 branches_sample_type; 60 u64 branches_id; 61 u64 instructions_sample_type; 62 u64 instructions_sample_period; 63 u64 instructions_id; 64 u64 **metadata; 65 u64 kernel_start; 66 unsigned int pmu_type; 67 }; 68 69 struct cs_etm_traceid_queue { 70 u8 trace_chan_id; 71 pid_t pid, tid; 72 u64 period_instructions; 73 size_t last_branch_pos; 74 union perf_event *event_buf; 75 struct thread *thread; 76 struct branch_stack *last_branch; 77 struct branch_stack *last_branch_rb; 78 struct cs_etm_packet *prev_packet; 79 struct cs_etm_packet *packet; 80 struct cs_etm_packet_queue packet_queue; 81 }; 82 83 struct cs_etm_queue { 84 struct cs_etm_auxtrace *etm; 85 struct cs_etm_decoder *decoder; 86 struct auxtrace_buffer *buffer; 87 unsigned int queue_nr; 88 u8 pending_timestamp_chan_id; 89 u64 offset; 90 const unsigned char *buf; 91 size_t buf_len, buf_used; 92 /* Conversion between traceID and index in traceid_queues array */ 93 struct intlist *traceid_queues_list; 94 struct cs_etm_traceid_queue **traceid_queues; 95 }; 96 97 /* RB tree for quick conversion between traceID and metadata pointers */ 98 static struct intlist *traceid_list; 99 100 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm); 101 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm); 102 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 103 pid_t tid); 104 static int cs_etm__get_data_block(struct cs_etm_queue *etmq); 105 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq); 106 107 /* PTMs ETMIDR [11:8] set to b0011 */ 108 #define ETMIDR_PTM_VERSION 0x00000300 109 110 /* 111 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to 112 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply 113 * encode the etm queue number as the upper 16 bit and the channel as 114 * the lower 16 bit. 115 */ 116 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \ 117 (queue_nr << 16 | trace_chan_id) 118 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16) 119 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff) 120 121 static u32 cs_etm__get_v7_protocol_version(u32 etmidr) 122 { 123 etmidr &= ETMIDR_PTM_VERSION; 124 125 if (etmidr == ETMIDR_PTM_VERSION) 126 return CS_ETM_PROTO_PTM; 127 128 return CS_ETM_PROTO_ETMV3; 129 } 130 131 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic) 132 { 133 struct int_node *inode; 134 u64 *metadata; 135 136 inode = intlist__find(traceid_list, trace_chan_id); 137 if (!inode) 138 return -EINVAL; 139 140 metadata = inode->priv; 141 *magic = metadata[CS_ETM_MAGIC]; 142 return 0; 143 } 144 145 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu) 146 { 147 struct int_node *inode; 148 u64 *metadata; 149 150 inode = intlist__find(traceid_list, trace_chan_id); 151 if (!inode) 152 return -EINVAL; 153 154 metadata = inode->priv; 155 *cpu = (int)metadata[CS_ETM_CPU]; 156 return 0; 157 } 158 159 /* 160 * The returned PID format is presented by two bits: 161 * 162 * Bit ETM_OPT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced; 163 * Bit ETM_OPT_CTXTID2: CONTEXTIDR_EL2 is traced. 164 * 165 * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2 166 * are enabled at the same time when the session runs on an EL2 kernel. 167 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be 168 * recorded in the trace data, the tool will selectively use 169 * CONTEXTIDR_EL2 as PID. 170 */ 171 int cs_etm__get_pid_fmt(u8 trace_chan_id, u64 *pid_fmt) 172 { 173 struct int_node *inode; 174 u64 *metadata, val; 175 176 inode = intlist__find(traceid_list, trace_chan_id); 177 if (!inode) 178 return -EINVAL; 179 180 metadata = inode->priv; 181 182 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) { 183 val = metadata[CS_ETM_ETMCR]; 184 /* CONTEXTIDR is traced */ 185 if (val & BIT(ETM_OPT_CTXTID)) 186 *pid_fmt = BIT(ETM_OPT_CTXTID); 187 } else { 188 val = metadata[CS_ETMV4_TRCCONFIGR]; 189 /* CONTEXTIDR_EL2 is traced */ 190 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT))) 191 *pid_fmt = BIT(ETM_OPT_CTXTID2); 192 /* CONTEXTIDR_EL1 is traced */ 193 else if (val & BIT(ETM4_CFG_BIT_CTXTID)) 194 *pid_fmt = BIT(ETM_OPT_CTXTID); 195 } 196 197 return 0; 198 } 199 200 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, 201 u8 trace_chan_id) 202 { 203 /* 204 * When a timestamp packet is encountered the backend code 205 * is stopped so that the front end has time to process packets 206 * that were accumulated in the traceID queue. Since there can 207 * be more than one channel per cs_etm_queue, we need to specify 208 * what traceID queue needs servicing. 209 */ 210 etmq->pending_timestamp_chan_id = trace_chan_id; 211 } 212 213 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, 214 u8 *trace_chan_id) 215 { 216 struct cs_etm_packet_queue *packet_queue; 217 218 if (!etmq->pending_timestamp_chan_id) 219 return 0; 220 221 if (trace_chan_id) 222 *trace_chan_id = etmq->pending_timestamp_chan_id; 223 224 packet_queue = cs_etm__etmq_get_packet_queue(etmq, 225 etmq->pending_timestamp_chan_id); 226 if (!packet_queue) 227 return 0; 228 229 /* Acknowledge pending status */ 230 etmq->pending_timestamp_chan_id = 0; 231 232 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ 233 return packet_queue->cs_timestamp; 234 } 235 236 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) 237 { 238 int i; 239 240 queue->head = 0; 241 queue->tail = 0; 242 queue->packet_count = 0; 243 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) { 244 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN; 245 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR; 246 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR; 247 queue->packet_buffer[i].instr_count = 0; 248 queue->packet_buffer[i].last_instr_taken_branch = false; 249 queue->packet_buffer[i].last_instr_size = 0; 250 queue->packet_buffer[i].last_instr_type = 0; 251 queue->packet_buffer[i].last_instr_subtype = 0; 252 queue->packet_buffer[i].last_instr_cond = 0; 253 queue->packet_buffer[i].flags = 0; 254 queue->packet_buffer[i].exception_number = UINT32_MAX; 255 queue->packet_buffer[i].trace_chan_id = UINT8_MAX; 256 queue->packet_buffer[i].cpu = INT_MIN; 257 } 258 } 259 260 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq) 261 { 262 int idx; 263 struct int_node *inode; 264 struct cs_etm_traceid_queue *tidq; 265 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 266 267 intlist__for_each_entry(inode, traceid_queues_list) { 268 idx = (int)(intptr_t)inode->priv; 269 tidq = etmq->traceid_queues[idx]; 270 cs_etm__clear_packet_queue(&tidq->packet_queue); 271 } 272 } 273 274 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, 275 struct cs_etm_traceid_queue *tidq, 276 u8 trace_chan_id) 277 { 278 int rc = -ENOMEM; 279 struct auxtrace_queue *queue; 280 struct cs_etm_auxtrace *etm = etmq->etm; 281 282 cs_etm__clear_packet_queue(&tidq->packet_queue); 283 284 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 285 tidq->tid = queue->tid; 286 tidq->pid = -1; 287 tidq->trace_chan_id = trace_chan_id; 288 289 tidq->packet = zalloc(sizeof(struct cs_etm_packet)); 290 if (!tidq->packet) 291 goto out; 292 293 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet)); 294 if (!tidq->prev_packet) 295 goto out_free; 296 297 if (etm->synth_opts.last_branch) { 298 size_t sz = sizeof(struct branch_stack); 299 300 sz += etm->synth_opts.last_branch_sz * 301 sizeof(struct branch_entry); 302 tidq->last_branch = zalloc(sz); 303 if (!tidq->last_branch) 304 goto out_free; 305 tidq->last_branch_rb = zalloc(sz); 306 if (!tidq->last_branch_rb) 307 goto out_free; 308 } 309 310 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); 311 if (!tidq->event_buf) 312 goto out_free; 313 314 return 0; 315 316 out_free: 317 zfree(&tidq->last_branch_rb); 318 zfree(&tidq->last_branch); 319 zfree(&tidq->prev_packet); 320 zfree(&tidq->packet); 321 out: 322 return rc; 323 } 324 325 static struct cs_etm_traceid_queue 326 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 327 { 328 int idx; 329 struct int_node *inode; 330 struct intlist *traceid_queues_list; 331 struct cs_etm_traceid_queue *tidq, **traceid_queues; 332 struct cs_etm_auxtrace *etm = etmq->etm; 333 334 if (etm->timeless_decoding) 335 trace_chan_id = CS_ETM_PER_THREAD_TRACEID; 336 337 traceid_queues_list = etmq->traceid_queues_list; 338 339 /* 340 * Check if the traceid_queue exist for this traceID by looking 341 * in the queue list. 342 */ 343 inode = intlist__find(traceid_queues_list, trace_chan_id); 344 if (inode) { 345 idx = (int)(intptr_t)inode->priv; 346 return etmq->traceid_queues[idx]; 347 } 348 349 /* We couldn't find a traceid_queue for this traceID, allocate one */ 350 tidq = malloc(sizeof(*tidq)); 351 if (!tidq) 352 return NULL; 353 354 memset(tidq, 0, sizeof(*tidq)); 355 356 /* Get a valid index for the new traceid_queue */ 357 idx = intlist__nr_entries(traceid_queues_list); 358 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */ 359 inode = intlist__findnew(traceid_queues_list, trace_chan_id); 360 if (!inode) 361 goto out_free; 362 363 /* Associate this traceID with this index */ 364 inode->priv = (void *)(intptr_t)idx; 365 366 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id)) 367 goto out_free; 368 369 /* Grow the traceid_queues array by one unit */ 370 traceid_queues = etmq->traceid_queues; 371 traceid_queues = reallocarray(traceid_queues, 372 idx + 1, 373 sizeof(*traceid_queues)); 374 375 /* 376 * On failure reallocarray() returns NULL and the original block of 377 * memory is left untouched. 378 */ 379 if (!traceid_queues) 380 goto out_free; 381 382 traceid_queues[idx] = tidq; 383 etmq->traceid_queues = traceid_queues; 384 385 return etmq->traceid_queues[idx]; 386 387 out_free: 388 /* 389 * Function intlist__remove() removes the inode from the list 390 * and delete the memory associated to it. 391 */ 392 intlist__remove(traceid_queues_list, inode); 393 free(tidq); 394 395 return NULL; 396 } 397 398 struct cs_etm_packet_queue 399 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 400 { 401 struct cs_etm_traceid_queue *tidq; 402 403 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 404 if (tidq) 405 return &tidq->packet_queue; 406 407 return NULL; 408 } 409 410 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, 411 struct cs_etm_traceid_queue *tidq) 412 { 413 struct cs_etm_packet *tmp; 414 415 if (etm->sample_branches || etm->synth_opts.last_branch || 416 etm->sample_instructions) { 417 /* 418 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for 419 * the next incoming packet. 420 */ 421 tmp = tidq->packet; 422 tidq->packet = tidq->prev_packet; 423 tidq->prev_packet = tmp; 424 } 425 } 426 427 static void cs_etm__packet_dump(const char *pkt_string) 428 { 429 const char *color = PERF_COLOR_BLUE; 430 int len = strlen(pkt_string); 431 432 if (len && (pkt_string[len-1] == '\n')) 433 color_fprintf(stdout, color, " %s", pkt_string); 434 else 435 color_fprintf(stdout, color, " %s\n", pkt_string); 436 437 fflush(stdout); 438 } 439 440 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, 441 struct cs_etm_auxtrace *etm, int idx, 442 u32 etmidr) 443 { 444 u64 **metadata = etm->metadata; 445 446 t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr); 447 t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR]; 448 t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR]; 449 } 450 451 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, 452 struct cs_etm_auxtrace *etm, int idx) 453 { 454 u64 **metadata = etm->metadata; 455 456 t_params[idx].protocol = CS_ETM_PROTO_ETMV4i; 457 t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0]; 458 t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1]; 459 t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2]; 460 t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8]; 461 t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR]; 462 t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR]; 463 } 464 465 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, 466 struct cs_etm_auxtrace *etm) 467 { 468 int i; 469 u32 etmidr; 470 u64 architecture; 471 472 for (i = 0; i < etm->num_cpu; i++) { 473 architecture = etm->metadata[i][CS_ETM_MAGIC]; 474 475 switch (architecture) { 476 case __perf_cs_etmv3_magic: 477 etmidr = etm->metadata[i][CS_ETM_ETMIDR]; 478 cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr); 479 break; 480 case __perf_cs_etmv4_magic: 481 cs_etm__set_trace_param_etmv4(t_params, etm, i); 482 break; 483 default: 484 return -EINVAL; 485 } 486 } 487 488 return 0; 489 } 490 491 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params, 492 struct cs_etm_queue *etmq, 493 enum cs_etm_decoder_operation mode) 494 { 495 int ret = -EINVAL; 496 497 if (!(mode < CS_ETM_OPERATION_MAX)) 498 goto out; 499 500 d_params->packet_printer = cs_etm__packet_dump; 501 d_params->operation = mode; 502 d_params->data = etmq; 503 d_params->formatted = true; 504 d_params->fsyncs = false; 505 d_params->hsyncs = false; 506 d_params->frame_aligned = true; 507 508 ret = 0; 509 out: 510 return ret; 511 } 512 513 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm, 514 struct auxtrace_buffer *buffer) 515 { 516 int ret; 517 const char *color = PERF_COLOR_BLUE; 518 struct cs_etm_decoder_params d_params; 519 struct cs_etm_trace_params *t_params; 520 struct cs_etm_decoder *decoder; 521 size_t buffer_used = 0; 522 523 fprintf(stdout, "\n"); 524 color_fprintf(stdout, color, 525 ". ... CoreSight ETM Trace data: size %zu bytes\n", 526 buffer->size); 527 528 /* Use metadata to fill in trace parameters for trace decoder */ 529 t_params = zalloc(sizeof(*t_params) * etm->num_cpu); 530 531 if (!t_params) 532 return; 533 534 if (cs_etm__init_trace_params(t_params, etm)) 535 goto out_free; 536 537 /* Set decoder parameters to simply print the trace packets */ 538 if (cs_etm__init_decoder_params(&d_params, NULL, 539 CS_ETM_OPERATION_PRINT)) 540 goto out_free; 541 542 decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params); 543 544 if (!decoder) 545 goto out_free; 546 do { 547 size_t consumed; 548 549 ret = cs_etm_decoder__process_data_block( 550 decoder, buffer->offset, 551 &((u8 *)buffer->data)[buffer_used], 552 buffer->size - buffer_used, &consumed); 553 if (ret) 554 break; 555 556 buffer_used += consumed; 557 } while (buffer_used < buffer->size); 558 559 cs_etm_decoder__free(decoder); 560 561 out_free: 562 zfree(&t_params); 563 } 564 565 static int cs_etm__flush_events(struct perf_session *session, 566 struct perf_tool *tool) 567 { 568 int ret; 569 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 570 struct cs_etm_auxtrace, 571 auxtrace); 572 if (dump_trace) 573 return 0; 574 575 if (!tool->ordered_events) 576 return -EINVAL; 577 578 ret = cs_etm__update_queues(etm); 579 580 if (ret < 0) 581 return ret; 582 583 if (etm->timeless_decoding) 584 return cs_etm__process_timeless_queues(etm, -1); 585 586 return cs_etm__process_queues(etm); 587 } 588 589 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) 590 { 591 int idx; 592 uintptr_t priv; 593 struct int_node *inode, *tmp; 594 struct cs_etm_traceid_queue *tidq; 595 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 596 597 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) { 598 priv = (uintptr_t)inode->priv; 599 idx = priv; 600 601 /* Free this traceid_queue from the array */ 602 tidq = etmq->traceid_queues[idx]; 603 thread__zput(tidq->thread); 604 zfree(&tidq->event_buf); 605 zfree(&tidq->last_branch); 606 zfree(&tidq->last_branch_rb); 607 zfree(&tidq->prev_packet); 608 zfree(&tidq->packet); 609 zfree(&tidq); 610 611 /* 612 * Function intlist__remove() removes the inode from the list 613 * and delete the memory associated to it. 614 */ 615 intlist__remove(traceid_queues_list, inode); 616 } 617 618 /* Then the RB tree itself */ 619 intlist__delete(traceid_queues_list); 620 etmq->traceid_queues_list = NULL; 621 622 /* finally free the traceid_queues array */ 623 zfree(&etmq->traceid_queues); 624 } 625 626 static void cs_etm__free_queue(void *priv) 627 { 628 struct cs_etm_queue *etmq = priv; 629 630 if (!etmq) 631 return; 632 633 cs_etm_decoder__free(etmq->decoder); 634 cs_etm__free_traceid_queues(etmq); 635 free(etmq); 636 } 637 638 static void cs_etm__free_events(struct perf_session *session) 639 { 640 unsigned int i; 641 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 642 struct cs_etm_auxtrace, 643 auxtrace); 644 struct auxtrace_queues *queues = &aux->queues; 645 646 for (i = 0; i < queues->nr_queues; i++) { 647 cs_etm__free_queue(queues->queue_array[i].priv); 648 queues->queue_array[i].priv = NULL; 649 } 650 651 auxtrace_queues__free(queues); 652 } 653 654 static void cs_etm__free(struct perf_session *session) 655 { 656 int i; 657 struct int_node *inode, *tmp; 658 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 659 struct cs_etm_auxtrace, 660 auxtrace); 661 cs_etm__free_events(session); 662 session->auxtrace = NULL; 663 664 /* First remove all traceID/metadata nodes for the RB tree */ 665 intlist__for_each_entry_safe(inode, tmp, traceid_list) 666 intlist__remove(traceid_list, inode); 667 /* Then the RB tree itself */ 668 intlist__delete(traceid_list); 669 670 for (i = 0; i < aux->num_cpu; i++) 671 zfree(&aux->metadata[i]); 672 673 thread__zput(aux->unknown_thread); 674 zfree(&aux->metadata); 675 zfree(&aux); 676 } 677 678 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session, 679 struct evsel *evsel) 680 { 681 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 682 struct cs_etm_auxtrace, 683 auxtrace); 684 685 return evsel->core.attr.type == aux->pmu_type; 686 } 687 688 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address) 689 { 690 struct machine *machine; 691 692 machine = etmq->etm->machine; 693 694 if (address >= etmq->etm->kernel_start) { 695 if (machine__is_host(machine)) 696 return PERF_RECORD_MISC_KERNEL; 697 else 698 return PERF_RECORD_MISC_GUEST_KERNEL; 699 } else { 700 if (machine__is_host(machine)) 701 return PERF_RECORD_MISC_USER; 702 else if (perf_guest) 703 return PERF_RECORD_MISC_GUEST_USER; 704 else 705 return PERF_RECORD_MISC_HYPERVISOR; 706 } 707 } 708 709 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, 710 u64 address, size_t size, u8 *buffer) 711 { 712 u8 cpumode; 713 u64 offset; 714 int len; 715 struct thread *thread; 716 struct machine *machine; 717 struct addr_location al; 718 struct cs_etm_traceid_queue *tidq; 719 720 if (!etmq) 721 return 0; 722 723 machine = etmq->etm->machine; 724 cpumode = cs_etm__cpu_mode(etmq, address); 725 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 726 if (!tidq) 727 return 0; 728 729 thread = tidq->thread; 730 if (!thread) { 731 if (cpumode != PERF_RECORD_MISC_KERNEL) 732 return 0; 733 thread = etmq->etm->unknown_thread; 734 } 735 736 if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso) 737 return 0; 738 739 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR && 740 dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE)) 741 return 0; 742 743 offset = al.map->map_ip(al.map, address); 744 745 map__load(al.map); 746 747 len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size); 748 749 if (len <= 0) 750 return 0; 751 752 return len; 753 } 754 755 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm) 756 { 757 struct cs_etm_decoder_params d_params; 758 struct cs_etm_trace_params *t_params = NULL; 759 struct cs_etm_queue *etmq; 760 761 etmq = zalloc(sizeof(*etmq)); 762 if (!etmq) 763 return NULL; 764 765 etmq->traceid_queues_list = intlist__new(NULL); 766 if (!etmq->traceid_queues_list) 767 goto out_free; 768 769 /* Use metadata to fill in trace parameters for trace decoder */ 770 t_params = zalloc(sizeof(*t_params) * etm->num_cpu); 771 772 if (!t_params) 773 goto out_free; 774 775 if (cs_etm__init_trace_params(t_params, etm)) 776 goto out_free; 777 778 /* Set decoder parameters to decode trace packets */ 779 if (cs_etm__init_decoder_params(&d_params, etmq, 780 CS_ETM_OPERATION_DECODE)) 781 goto out_free; 782 783 etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params); 784 785 if (!etmq->decoder) 786 goto out_free; 787 788 /* 789 * Register a function to handle all memory accesses required by 790 * the trace decoder library. 791 */ 792 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 793 0x0L, ((u64) -1L), 794 cs_etm__mem_access)) 795 goto out_free_decoder; 796 797 zfree(&t_params); 798 return etmq; 799 800 out_free_decoder: 801 cs_etm_decoder__free(etmq->decoder); 802 out_free: 803 intlist__delete(etmq->traceid_queues_list); 804 free(etmq); 805 806 return NULL; 807 } 808 809 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, 810 struct auxtrace_queue *queue, 811 unsigned int queue_nr) 812 { 813 int ret = 0; 814 unsigned int cs_queue_nr; 815 u8 trace_chan_id; 816 u64 cs_timestamp; 817 struct cs_etm_queue *etmq = queue->priv; 818 819 if (list_empty(&queue->head) || etmq) 820 goto out; 821 822 etmq = cs_etm__alloc_queue(etm); 823 824 if (!etmq) { 825 ret = -ENOMEM; 826 goto out; 827 } 828 829 queue->priv = etmq; 830 etmq->etm = etm; 831 etmq->queue_nr = queue_nr; 832 etmq->offset = 0; 833 834 if (etm->timeless_decoding) 835 goto out; 836 837 /* 838 * We are under a CPU-wide trace scenario. As such we need to know 839 * when the code that generated the traces started to execute so that 840 * it can be correlated with execution on other CPUs. So we get a 841 * handle on the beginning of traces and decode until we find a 842 * timestamp. The timestamp is then added to the auxtrace min heap 843 * in order to know what nibble (of all the etmqs) to decode first. 844 */ 845 while (1) { 846 /* 847 * Fetch an aux_buffer from this etmq. Bail if no more 848 * blocks or an error has been encountered. 849 */ 850 ret = cs_etm__get_data_block(etmq); 851 if (ret <= 0) 852 goto out; 853 854 /* 855 * Run decoder on the trace block. The decoder will stop when 856 * encountering a CS timestamp, a full packet queue or the end of 857 * trace for that block. 858 */ 859 ret = cs_etm__decode_data_block(etmq); 860 if (ret) 861 goto out; 862 863 /* 864 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all 865 * the timestamp calculation for us. 866 */ 867 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 868 869 /* We found a timestamp, no need to continue. */ 870 if (cs_timestamp) 871 break; 872 873 /* 874 * We didn't find a timestamp so empty all the traceid packet 875 * queues before looking for another timestamp packet, either 876 * in the current data block or a new one. Packets that were 877 * just decoded are useless since no timestamp has been 878 * associated with them. As such simply discard them. 879 */ 880 cs_etm__clear_all_packet_queues(etmq); 881 } 882 883 /* 884 * We have a timestamp. Add it to the min heap to reflect when 885 * instructions conveyed by the range packets of this traceID queue 886 * started to execute. Once the same has been done for all the traceID 887 * queues of each etmq, redenring and decoding can start in 888 * chronological order. 889 * 890 * Note that packets decoded above are still in the traceID's packet 891 * queue and will be processed in cs_etm__process_queues(). 892 */ 893 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 894 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); 895 out: 896 return ret; 897 } 898 899 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm) 900 { 901 unsigned int i; 902 int ret; 903 904 if (!etm->kernel_start) 905 etm->kernel_start = machine__kernel_start(etm->machine); 906 907 for (i = 0; i < etm->queues.nr_queues; i++) { 908 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i); 909 if (ret) 910 return ret; 911 } 912 913 return 0; 914 } 915 916 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm) 917 { 918 if (etm->queues.new_data) { 919 etm->queues.new_data = false; 920 return cs_etm__setup_queues(etm); 921 } 922 923 return 0; 924 } 925 926 static inline 927 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq, 928 struct cs_etm_traceid_queue *tidq) 929 { 930 struct branch_stack *bs_src = tidq->last_branch_rb; 931 struct branch_stack *bs_dst = tidq->last_branch; 932 size_t nr = 0; 933 934 /* 935 * Set the number of records before early exit: ->nr is used to 936 * determine how many branches to copy from ->entries. 937 */ 938 bs_dst->nr = bs_src->nr; 939 940 /* 941 * Early exit when there is nothing to copy. 942 */ 943 if (!bs_src->nr) 944 return; 945 946 /* 947 * As bs_src->entries is a circular buffer, we need to copy from it in 948 * two steps. First, copy the branches from the most recently inserted 949 * branch ->last_branch_pos until the end of bs_src->entries buffer. 950 */ 951 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos; 952 memcpy(&bs_dst->entries[0], 953 &bs_src->entries[tidq->last_branch_pos], 954 sizeof(struct branch_entry) * nr); 955 956 /* 957 * If we wrapped around at least once, the branches from the beginning 958 * of the bs_src->entries buffer and until the ->last_branch_pos element 959 * are older valid branches: copy them over. The total number of 960 * branches copied over will be equal to the number of branches asked by 961 * the user in last_branch_sz. 962 */ 963 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) { 964 memcpy(&bs_dst->entries[nr], 965 &bs_src->entries[0], 966 sizeof(struct branch_entry) * tidq->last_branch_pos); 967 } 968 } 969 970 static inline 971 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) 972 { 973 tidq->last_branch_pos = 0; 974 tidq->last_branch_rb->nr = 0; 975 } 976 977 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, 978 u8 trace_chan_id, u64 addr) 979 { 980 u8 instrBytes[2]; 981 982 cs_etm__mem_access(etmq, trace_chan_id, addr, 983 ARRAY_SIZE(instrBytes), instrBytes); 984 /* 985 * T32 instruction size is indicated by bits[15:11] of the first 986 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 987 * denote a 32-bit instruction. 988 */ 989 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; 990 } 991 992 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) 993 { 994 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ 995 if (packet->sample_type == CS_ETM_DISCONTINUITY) 996 return 0; 997 998 return packet->start_addr; 999 } 1000 1001 static inline 1002 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) 1003 { 1004 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ 1005 if (packet->sample_type == CS_ETM_DISCONTINUITY) 1006 return 0; 1007 1008 return packet->end_addr - packet->last_instr_size; 1009 } 1010 1011 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, 1012 u64 trace_chan_id, 1013 const struct cs_etm_packet *packet, 1014 u64 offset) 1015 { 1016 if (packet->isa == CS_ETM_ISA_T32) { 1017 u64 addr = packet->start_addr; 1018 1019 while (offset) { 1020 addr += cs_etm__t32_instr_size(etmq, 1021 trace_chan_id, addr); 1022 offset--; 1023 } 1024 return addr; 1025 } 1026 1027 /* Assume a 4 byte instruction size (A32/A64) */ 1028 return packet->start_addr + offset * 4; 1029 } 1030 1031 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, 1032 struct cs_etm_traceid_queue *tidq) 1033 { 1034 struct branch_stack *bs = tidq->last_branch_rb; 1035 struct branch_entry *be; 1036 1037 /* 1038 * The branches are recorded in a circular buffer in reverse 1039 * chronological order: we start recording from the last element of the 1040 * buffer down. After writing the first element of the stack, move the 1041 * insert position back to the end of the buffer. 1042 */ 1043 if (!tidq->last_branch_pos) 1044 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz; 1045 1046 tidq->last_branch_pos -= 1; 1047 1048 be = &bs->entries[tidq->last_branch_pos]; 1049 be->from = cs_etm__last_executed_instr(tidq->prev_packet); 1050 be->to = cs_etm__first_executed_instr(tidq->packet); 1051 /* No support for mispredict */ 1052 be->flags.mispred = 0; 1053 be->flags.predicted = 1; 1054 1055 /* 1056 * Increment bs->nr until reaching the number of last branches asked by 1057 * the user on the command line. 1058 */ 1059 if (bs->nr < etmq->etm->synth_opts.last_branch_sz) 1060 bs->nr += 1; 1061 } 1062 1063 static int cs_etm__inject_event(union perf_event *event, 1064 struct perf_sample *sample, u64 type) 1065 { 1066 event->header.size = perf_event__sample_event_size(sample, type, 0); 1067 return perf_event__synthesize_sample(event, type, 0, sample); 1068 } 1069 1070 1071 static int 1072 cs_etm__get_trace(struct cs_etm_queue *etmq) 1073 { 1074 struct auxtrace_buffer *aux_buffer = etmq->buffer; 1075 struct auxtrace_buffer *old_buffer = aux_buffer; 1076 struct auxtrace_queue *queue; 1077 1078 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 1079 1080 aux_buffer = auxtrace_buffer__next(queue, aux_buffer); 1081 1082 /* If no more data, drop the previous auxtrace_buffer and return */ 1083 if (!aux_buffer) { 1084 if (old_buffer) 1085 auxtrace_buffer__drop_data(old_buffer); 1086 etmq->buf_len = 0; 1087 return 0; 1088 } 1089 1090 etmq->buffer = aux_buffer; 1091 1092 /* If the aux_buffer doesn't have data associated, try to load it */ 1093 if (!aux_buffer->data) { 1094 /* get the file desc associated with the perf data file */ 1095 int fd = perf_data__fd(etmq->etm->session->data); 1096 1097 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); 1098 if (!aux_buffer->data) 1099 return -ENOMEM; 1100 } 1101 1102 /* If valid, drop the previous buffer */ 1103 if (old_buffer) 1104 auxtrace_buffer__drop_data(old_buffer); 1105 1106 etmq->buf_used = 0; 1107 etmq->buf_len = aux_buffer->size; 1108 etmq->buf = aux_buffer->data; 1109 1110 return etmq->buf_len; 1111 } 1112 1113 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm, 1114 struct cs_etm_traceid_queue *tidq) 1115 { 1116 if ((!tidq->thread) && (tidq->tid != -1)) 1117 tidq->thread = machine__find_thread(etm->machine, -1, 1118 tidq->tid); 1119 1120 if (tidq->thread) 1121 tidq->pid = tidq->thread->pid_; 1122 } 1123 1124 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq, 1125 pid_t tid, u8 trace_chan_id) 1126 { 1127 int cpu, err = -EINVAL; 1128 struct cs_etm_auxtrace *etm = etmq->etm; 1129 struct cs_etm_traceid_queue *tidq; 1130 1131 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 1132 if (!tidq) 1133 return err; 1134 1135 if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0) 1136 return err; 1137 1138 err = machine__set_current_tid(etm->machine, cpu, tid, tid); 1139 if (err) 1140 return err; 1141 1142 tidq->tid = tid; 1143 thread__zput(tidq->thread); 1144 1145 cs_etm__set_pid_tid_cpu(etm, tidq); 1146 return 0; 1147 } 1148 1149 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) 1150 { 1151 return !!etmq->etm->timeless_decoding; 1152 } 1153 1154 static void cs_etm__copy_insn(struct cs_etm_queue *etmq, 1155 u64 trace_chan_id, 1156 const struct cs_etm_packet *packet, 1157 struct perf_sample *sample) 1158 { 1159 /* 1160 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY 1161 * packet, so directly bail out with 'insn_len' = 0. 1162 */ 1163 if (packet->sample_type == CS_ETM_DISCONTINUITY) { 1164 sample->insn_len = 0; 1165 return; 1166 } 1167 1168 /* 1169 * T32 instruction size might be 32-bit or 16-bit, decide by calling 1170 * cs_etm__t32_instr_size(). 1171 */ 1172 if (packet->isa == CS_ETM_ISA_T32) 1173 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id, 1174 sample->ip); 1175 /* Otherwise, A64 and A32 instruction size are always 32-bit. */ 1176 else 1177 sample->insn_len = 4; 1178 1179 cs_etm__mem_access(etmq, trace_chan_id, sample->ip, 1180 sample->insn_len, (void *)sample->insn); 1181 } 1182 1183 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, 1184 struct cs_etm_traceid_queue *tidq, 1185 u64 addr, u64 period) 1186 { 1187 int ret = 0; 1188 struct cs_etm_auxtrace *etm = etmq->etm; 1189 union perf_event *event = tidq->event_buf; 1190 struct perf_sample sample = {.ip = 0,}; 1191 1192 event->sample.header.type = PERF_RECORD_SAMPLE; 1193 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr); 1194 event->sample.header.size = sizeof(struct perf_event_header); 1195 1196 if (!etm->timeless_decoding) 1197 sample.time = etm->latest_kernel_timestamp; 1198 sample.ip = addr; 1199 sample.pid = tidq->pid; 1200 sample.tid = tidq->tid; 1201 sample.id = etmq->etm->instructions_id; 1202 sample.stream_id = etmq->etm->instructions_id; 1203 sample.period = period; 1204 sample.cpu = tidq->packet->cpu; 1205 sample.flags = tidq->prev_packet->flags; 1206 sample.cpumode = event->sample.header.misc; 1207 1208 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample); 1209 1210 if (etm->synth_opts.last_branch) 1211 sample.branch_stack = tidq->last_branch; 1212 1213 if (etm->synth_opts.inject) { 1214 ret = cs_etm__inject_event(event, &sample, 1215 etm->instructions_sample_type); 1216 if (ret) 1217 return ret; 1218 } 1219 1220 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1221 1222 if (ret) 1223 pr_err( 1224 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1225 ret); 1226 1227 return ret; 1228 } 1229 1230 /* 1231 * The cs etm packet encodes an instruction range between a branch target 1232 * and the next taken branch. Generate sample accordingly. 1233 */ 1234 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, 1235 struct cs_etm_traceid_queue *tidq) 1236 { 1237 int ret = 0; 1238 struct cs_etm_auxtrace *etm = etmq->etm; 1239 struct perf_sample sample = {.ip = 0,}; 1240 union perf_event *event = tidq->event_buf; 1241 struct dummy_branch_stack { 1242 u64 nr; 1243 u64 hw_idx; 1244 struct branch_entry entries; 1245 } dummy_bs; 1246 u64 ip; 1247 1248 ip = cs_etm__last_executed_instr(tidq->prev_packet); 1249 1250 event->sample.header.type = PERF_RECORD_SAMPLE; 1251 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip); 1252 event->sample.header.size = sizeof(struct perf_event_header); 1253 1254 if (!etm->timeless_decoding) 1255 sample.time = etm->latest_kernel_timestamp; 1256 sample.ip = ip; 1257 sample.pid = tidq->pid; 1258 sample.tid = tidq->tid; 1259 sample.addr = cs_etm__first_executed_instr(tidq->packet); 1260 sample.id = etmq->etm->branches_id; 1261 sample.stream_id = etmq->etm->branches_id; 1262 sample.period = 1; 1263 sample.cpu = tidq->packet->cpu; 1264 sample.flags = tidq->prev_packet->flags; 1265 sample.cpumode = event->sample.header.misc; 1266 1267 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet, 1268 &sample); 1269 1270 /* 1271 * perf report cannot handle events without a branch stack 1272 */ 1273 if (etm->synth_opts.last_branch) { 1274 dummy_bs = (struct dummy_branch_stack){ 1275 .nr = 1, 1276 .hw_idx = -1ULL, 1277 .entries = { 1278 .from = sample.ip, 1279 .to = sample.addr, 1280 }, 1281 }; 1282 sample.branch_stack = (struct branch_stack *)&dummy_bs; 1283 } 1284 1285 if (etm->synth_opts.inject) { 1286 ret = cs_etm__inject_event(event, &sample, 1287 etm->branches_sample_type); 1288 if (ret) 1289 return ret; 1290 } 1291 1292 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1293 1294 if (ret) 1295 pr_err( 1296 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1297 ret); 1298 1299 return ret; 1300 } 1301 1302 struct cs_etm_synth { 1303 struct perf_tool dummy_tool; 1304 struct perf_session *session; 1305 }; 1306 1307 static int cs_etm__event_synth(struct perf_tool *tool, 1308 union perf_event *event, 1309 struct perf_sample *sample __maybe_unused, 1310 struct machine *machine __maybe_unused) 1311 { 1312 struct cs_etm_synth *cs_etm_synth = 1313 container_of(tool, struct cs_etm_synth, dummy_tool); 1314 1315 return perf_session__deliver_synth_event(cs_etm_synth->session, 1316 event, NULL); 1317 } 1318 1319 static int cs_etm__synth_event(struct perf_session *session, 1320 struct perf_event_attr *attr, u64 id) 1321 { 1322 struct cs_etm_synth cs_etm_synth; 1323 1324 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth)); 1325 cs_etm_synth.session = session; 1326 1327 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1, 1328 &id, cs_etm__event_synth); 1329 } 1330 1331 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, 1332 struct perf_session *session) 1333 { 1334 struct evlist *evlist = session->evlist; 1335 struct evsel *evsel; 1336 struct perf_event_attr attr; 1337 bool found = false; 1338 u64 id; 1339 int err; 1340 1341 evlist__for_each_entry(evlist, evsel) { 1342 if (evsel->core.attr.type == etm->pmu_type) { 1343 found = true; 1344 break; 1345 } 1346 } 1347 1348 if (!found) { 1349 pr_debug("No selected events with CoreSight Trace data\n"); 1350 return 0; 1351 } 1352 1353 memset(&attr, 0, sizeof(struct perf_event_attr)); 1354 attr.size = sizeof(struct perf_event_attr); 1355 attr.type = PERF_TYPE_HARDWARE; 1356 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; 1357 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | 1358 PERF_SAMPLE_PERIOD; 1359 if (etm->timeless_decoding) 1360 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; 1361 else 1362 attr.sample_type |= PERF_SAMPLE_TIME; 1363 1364 attr.exclude_user = evsel->core.attr.exclude_user; 1365 attr.exclude_kernel = evsel->core.attr.exclude_kernel; 1366 attr.exclude_hv = evsel->core.attr.exclude_hv; 1367 attr.exclude_host = evsel->core.attr.exclude_host; 1368 attr.exclude_guest = evsel->core.attr.exclude_guest; 1369 attr.sample_id_all = evsel->core.attr.sample_id_all; 1370 attr.read_format = evsel->core.attr.read_format; 1371 1372 /* create new id val to be a fixed offset from evsel id */ 1373 id = evsel->core.id[0] + 1000000000; 1374 1375 if (!id) 1376 id = 1; 1377 1378 if (etm->synth_opts.branches) { 1379 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; 1380 attr.sample_period = 1; 1381 attr.sample_type |= PERF_SAMPLE_ADDR; 1382 err = cs_etm__synth_event(session, &attr, id); 1383 if (err) 1384 return err; 1385 etm->sample_branches = true; 1386 etm->branches_sample_type = attr.sample_type; 1387 etm->branches_id = id; 1388 id += 1; 1389 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; 1390 } 1391 1392 if (etm->synth_opts.last_branch) { 1393 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 1394 /* 1395 * We don't use the hardware index, but the sample generation 1396 * code uses the new format branch_stack with this field, 1397 * so the event attributes must indicate that it's present. 1398 */ 1399 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; 1400 } 1401 1402 if (etm->synth_opts.instructions) { 1403 attr.config = PERF_COUNT_HW_INSTRUCTIONS; 1404 attr.sample_period = etm->synth_opts.period; 1405 etm->instructions_sample_period = attr.sample_period; 1406 err = cs_etm__synth_event(session, &attr, id); 1407 if (err) 1408 return err; 1409 etm->sample_instructions = true; 1410 etm->instructions_sample_type = attr.sample_type; 1411 etm->instructions_id = id; 1412 id += 1; 1413 } 1414 1415 return 0; 1416 } 1417 1418 static int cs_etm__sample(struct cs_etm_queue *etmq, 1419 struct cs_etm_traceid_queue *tidq) 1420 { 1421 struct cs_etm_auxtrace *etm = etmq->etm; 1422 int ret; 1423 u8 trace_chan_id = tidq->trace_chan_id; 1424 u64 instrs_prev; 1425 1426 /* Get instructions remainder from previous packet */ 1427 instrs_prev = tidq->period_instructions; 1428 1429 tidq->period_instructions += tidq->packet->instr_count; 1430 1431 /* 1432 * Record a branch when the last instruction in 1433 * PREV_PACKET is a branch. 1434 */ 1435 if (etm->synth_opts.last_branch && 1436 tidq->prev_packet->sample_type == CS_ETM_RANGE && 1437 tidq->prev_packet->last_instr_taken_branch) 1438 cs_etm__update_last_branch_rb(etmq, tidq); 1439 1440 if (etm->sample_instructions && 1441 tidq->period_instructions >= etm->instructions_sample_period) { 1442 /* 1443 * Emit instruction sample periodically 1444 * TODO: allow period to be defined in cycles and clock time 1445 */ 1446 1447 /* 1448 * Below diagram demonstrates the instruction samples 1449 * generation flows: 1450 * 1451 * Instrs Instrs Instrs Instrs 1452 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3) 1453 * | | | | 1454 * V V V V 1455 * -------------------------------------------------- 1456 * ^ ^ 1457 * | | 1458 * Period Period 1459 * instructions(Pi) instructions(Pi') 1460 * 1461 * | | 1462 * \---------------- -----------------/ 1463 * V 1464 * tidq->packet->instr_count 1465 * 1466 * Instrs Sample(n...) are the synthesised samples occurring 1467 * every etm->instructions_sample_period instructions - as 1468 * defined on the perf command line. Sample(n) is being the 1469 * last sample before the current etm packet, n+1 to n+3 1470 * samples are generated from the current etm packet. 1471 * 1472 * tidq->packet->instr_count represents the number of 1473 * instructions in the current etm packet. 1474 * 1475 * Period instructions (Pi) contains the the number of 1476 * instructions executed after the sample point(n) from the 1477 * previous etm packet. This will always be less than 1478 * etm->instructions_sample_period. 1479 * 1480 * When generate new samples, it combines with two parts 1481 * instructions, one is the tail of the old packet and another 1482 * is the head of the new coming packet, to generate 1483 * sample(n+1); sample(n+2) and sample(n+3) consume the 1484 * instructions with sample period. After sample(n+3), the rest 1485 * instructions will be used by later packet and it is assigned 1486 * to tidq->period_instructions for next round calculation. 1487 */ 1488 1489 /* 1490 * Get the initial offset into the current packet instructions; 1491 * entry conditions ensure that instrs_prev is less than 1492 * etm->instructions_sample_period. 1493 */ 1494 u64 offset = etm->instructions_sample_period - instrs_prev; 1495 u64 addr; 1496 1497 /* Prepare last branches for instruction sample */ 1498 if (etm->synth_opts.last_branch) 1499 cs_etm__copy_last_branch_rb(etmq, tidq); 1500 1501 while (tidq->period_instructions >= 1502 etm->instructions_sample_period) { 1503 /* 1504 * Calculate the address of the sampled instruction (-1 1505 * as sample is reported as though instruction has just 1506 * been executed, but PC has not advanced to next 1507 * instruction) 1508 */ 1509 addr = cs_etm__instr_addr(etmq, trace_chan_id, 1510 tidq->packet, offset - 1); 1511 ret = cs_etm__synth_instruction_sample( 1512 etmq, tidq, addr, 1513 etm->instructions_sample_period); 1514 if (ret) 1515 return ret; 1516 1517 offset += etm->instructions_sample_period; 1518 tidq->period_instructions -= 1519 etm->instructions_sample_period; 1520 } 1521 } 1522 1523 if (etm->sample_branches) { 1524 bool generate_sample = false; 1525 1526 /* Generate sample for tracing on packet */ 1527 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY) 1528 generate_sample = true; 1529 1530 /* Generate sample for branch taken packet */ 1531 if (tidq->prev_packet->sample_type == CS_ETM_RANGE && 1532 tidq->prev_packet->last_instr_taken_branch) 1533 generate_sample = true; 1534 1535 if (generate_sample) { 1536 ret = cs_etm__synth_branch_sample(etmq, tidq); 1537 if (ret) 1538 return ret; 1539 } 1540 } 1541 1542 cs_etm__packet_swap(etm, tidq); 1543 1544 return 0; 1545 } 1546 1547 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) 1548 { 1549 /* 1550 * When the exception packet is inserted, whether the last instruction 1551 * in previous range packet is taken branch or not, we need to force 1552 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures 1553 * to generate branch sample for the instruction range before the 1554 * exception is trapped to kernel or before the exception returning. 1555 * 1556 * The exception packet includes the dummy address values, so don't 1557 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful 1558 * for generating instruction and branch samples. 1559 */ 1560 if (tidq->prev_packet->sample_type == CS_ETM_RANGE) 1561 tidq->prev_packet->last_instr_taken_branch = true; 1562 1563 return 0; 1564 } 1565 1566 static int cs_etm__flush(struct cs_etm_queue *etmq, 1567 struct cs_etm_traceid_queue *tidq) 1568 { 1569 int err = 0; 1570 struct cs_etm_auxtrace *etm = etmq->etm; 1571 1572 /* Handle start tracing packet */ 1573 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY) 1574 goto swap_packet; 1575 1576 if (etmq->etm->synth_opts.last_branch && 1577 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1578 u64 addr; 1579 1580 /* Prepare last branches for instruction sample */ 1581 cs_etm__copy_last_branch_rb(etmq, tidq); 1582 1583 /* 1584 * Generate a last branch event for the branches left in the 1585 * circular buffer at the end of the trace. 1586 * 1587 * Use the address of the end of the last reported execution 1588 * range 1589 */ 1590 addr = cs_etm__last_executed_instr(tidq->prev_packet); 1591 1592 err = cs_etm__synth_instruction_sample( 1593 etmq, tidq, addr, 1594 tidq->period_instructions); 1595 if (err) 1596 return err; 1597 1598 tidq->period_instructions = 0; 1599 1600 } 1601 1602 if (etm->sample_branches && 1603 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1604 err = cs_etm__synth_branch_sample(etmq, tidq); 1605 if (err) 1606 return err; 1607 } 1608 1609 swap_packet: 1610 cs_etm__packet_swap(etm, tidq); 1611 1612 /* Reset last branches after flush the trace */ 1613 if (etm->synth_opts.last_branch) 1614 cs_etm__reset_last_branch_rb(tidq); 1615 1616 return err; 1617 } 1618 1619 static int cs_etm__end_block(struct cs_etm_queue *etmq, 1620 struct cs_etm_traceid_queue *tidq) 1621 { 1622 int err; 1623 1624 /* 1625 * It has no new packet coming and 'etmq->packet' contains the stale 1626 * packet which was set at the previous time with packets swapping; 1627 * so skip to generate branch sample to avoid stale packet. 1628 * 1629 * For this case only flush branch stack and generate a last branch 1630 * event for the branches left in the circular buffer at the end of 1631 * the trace. 1632 */ 1633 if (etmq->etm->synth_opts.last_branch && 1634 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1635 u64 addr; 1636 1637 /* Prepare last branches for instruction sample */ 1638 cs_etm__copy_last_branch_rb(etmq, tidq); 1639 1640 /* 1641 * Use the address of the end of the last reported execution 1642 * range. 1643 */ 1644 addr = cs_etm__last_executed_instr(tidq->prev_packet); 1645 1646 err = cs_etm__synth_instruction_sample( 1647 etmq, tidq, addr, 1648 tidq->period_instructions); 1649 if (err) 1650 return err; 1651 1652 tidq->period_instructions = 0; 1653 } 1654 1655 return 0; 1656 } 1657 /* 1658 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue 1659 * if need be. 1660 * Returns: < 0 if error 1661 * = 0 if no more auxtrace_buffer to read 1662 * > 0 if the current buffer isn't empty yet 1663 */ 1664 static int cs_etm__get_data_block(struct cs_etm_queue *etmq) 1665 { 1666 int ret; 1667 1668 if (!etmq->buf_len) { 1669 ret = cs_etm__get_trace(etmq); 1670 if (ret <= 0) 1671 return ret; 1672 /* 1673 * We cannot assume consecutive blocks in the data file 1674 * are contiguous, reset the decoder to force re-sync. 1675 */ 1676 ret = cs_etm_decoder__reset(etmq->decoder); 1677 if (ret) 1678 return ret; 1679 } 1680 1681 return etmq->buf_len; 1682 } 1683 1684 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, 1685 struct cs_etm_packet *packet, 1686 u64 end_addr) 1687 { 1688 /* Initialise to keep compiler happy */ 1689 u16 instr16 = 0; 1690 u32 instr32 = 0; 1691 u64 addr; 1692 1693 switch (packet->isa) { 1694 case CS_ETM_ISA_T32: 1695 /* 1696 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247: 1697 * 1698 * b'15 b'8 1699 * +-----------------+--------+ 1700 * | 1 1 0 1 1 1 1 1 | imm8 | 1701 * +-----------------+--------+ 1702 * 1703 * According to the specification, it only defines SVC for T32 1704 * with 16 bits instruction and has no definition for 32bits; 1705 * so below only read 2 bytes as instruction size for T32. 1706 */ 1707 addr = end_addr - 2; 1708 cs_etm__mem_access(etmq, trace_chan_id, addr, 1709 sizeof(instr16), (u8 *)&instr16); 1710 if ((instr16 & 0xFF00) == 0xDF00) 1711 return true; 1712 1713 break; 1714 case CS_ETM_ISA_A32: 1715 /* 1716 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247: 1717 * 1718 * b'31 b'28 b'27 b'24 1719 * +---------+---------+-------------------------+ 1720 * | !1111 | 1 1 1 1 | imm24 | 1721 * +---------+---------+-------------------------+ 1722 */ 1723 addr = end_addr - 4; 1724 cs_etm__mem_access(etmq, trace_chan_id, addr, 1725 sizeof(instr32), (u8 *)&instr32); 1726 if ((instr32 & 0x0F000000) == 0x0F000000 && 1727 (instr32 & 0xF0000000) != 0xF0000000) 1728 return true; 1729 1730 break; 1731 case CS_ETM_ISA_A64: 1732 /* 1733 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294: 1734 * 1735 * b'31 b'21 b'4 b'0 1736 * +-----------------------+---------+-----------+ 1737 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 | 1738 * +-----------------------+---------+-----------+ 1739 */ 1740 addr = end_addr - 4; 1741 cs_etm__mem_access(etmq, trace_chan_id, addr, 1742 sizeof(instr32), (u8 *)&instr32); 1743 if ((instr32 & 0xFFE0001F) == 0xd4000001) 1744 return true; 1745 1746 break; 1747 case CS_ETM_ISA_UNKNOWN: 1748 default: 1749 break; 1750 } 1751 1752 return false; 1753 } 1754 1755 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, 1756 struct cs_etm_traceid_queue *tidq, u64 magic) 1757 { 1758 u8 trace_chan_id = tidq->trace_chan_id; 1759 struct cs_etm_packet *packet = tidq->packet; 1760 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1761 1762 if (magic == __perf_cs_etmv3_magic) 1763 if (packet->exception_number == CS_ETMV3_EXC_SVC) 1764 return true; 1765 1766 /* 1767 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and 1768 * HVC cases; need to check if it's SVC instruction based on 1769 * packet address. 1770 */ 1771 if (magic == __perf_cs_etmv4_magic) { 1772 if (packet->exception_number == CS_ETMV4_EXC_CALL && 1773 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, 1774 prev_packet->end_addr)) 1775 return true; 1776 } 1777 1778 return false; 1779 } 1780 1781 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, 1782 u64 magic) 1783 { 1784 struct cs_etm_packet *packet = tidq->packet; 1785 1786 if (magic == __perf_cs_etmv3_magic) 1787 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT || 1788 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT || 1789 packet->exception_number == CS_ETMV3_EXC_PE_RESET || 1790 packet->exception_number == CS_ETMV3_EXC_IRQ || 1791 packet->exception_number == CS_ETMV3_EXC_FIQ) 1792 return true; 1793 1794 if (magic == __perf_cs_etmv4_magic) 1795 if (packet->exception_number == CS_ETMV4_EXC_RESET || 1796 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || 1797 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || 1798 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG || 1799 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG || 1800 packet->exception_number == CS_ETMV4_EXC_IRQ || 1801 packet->exception_number == CS_ETMV4_EXC_FIQ) 1802 return true; 1803 1804 return false; 1805 } 1806 1807 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, 1808 struct cs_etm_traceid_queue *tidq, 1809 u64 magic) 1810 { 1811 u8 trace_chan_id = tidq->trace_chan_id; 1812 struct cs_etm_packet *packet = tidq->packet; 1813 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1814 1815 if (magic == __perf_cs_etmv3_magic) 1816 if (packet->exception_number == CS_ETMV3_EXC_SMC || 1817 packet->exception_number == CS_ETMV3_EXC_HYP || 1818 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE || 1819 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR || 1820 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT || 1821 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT || 1822 packet->exception_number == CS_ETMV3_EXC_GENERIC) 1823 return true; 1824 1825 if (magic == __perf_cs_etmv4_magic) { 1826 if (packet->exception_number == CS_ETMV4_EXC_TRAP || 1827 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || 1828 packet->exception_number == CS_ETMV4_EXC_INST_FAULT || 1829 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT) 1830 return true; 1831 1832 /* 1833 * For CS_ETMV4_EXC_CALL, except SVC other instructions 1834 * (SMC, HVC) are taken as sync exceptions. 1835 */ 1836 if (packet->exception_number == CS_ETMV4_EXC_CALL && 1837 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, 1838 prev_packet->end_addr)) 1839 return true; 1840 1841 /* 1842 * ETMv4 has 5 bits for exception number; if the numbers 1843 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ] 1844 * they are implementation defined exceptions. 1845 * 1846 * For this case, simply take it as sync exception. 1847 */ 1848 if (packet->exception_number > CS_ETMV4_EXC_FIQ && 1849 packet->exception_number <= CS_ETMV4_EXC_END) 1850 return true; 1851 } 1852 1853 return false; 1854 } 1855 1856 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, 1857 struct cs_etm_traceid_queue *tidq) 1858 { 1859 struct cs_etm_packet *packet = tidq->packet; 1860 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1861 u8 trace_chan_id = tidq->trace_chan_id; 1862 u64 magic; 1863 int ret; 1864 1865 switch (packet->sample_type) { 1866 case CS_ETM_RANGE: 1867 /* 1868 * Immediate branch instruction without neither link nor 1869 * return flag, it's normal branch instruction within 1870 * the function. 1871 */ 1872 if (packet->last_instr_type == OCSD_INSTR_BR && 1873 packet->last_instr_subtype == OCSD_S_INSTR_NONE) { 1874 packet->flags = PERF_IP_FLAG_BRANCH; 1875 1876 if (packet->last_instr_cond) 1877 packet->flags |= PERF_IP_FLAG_CONDITIONAL; 1878 } 1879 1880 /* 1881 * Immediate branch instruction with link (e.g. BL), this is 1882 * branch instruction for function call. 1883 */ 1884 if (packet->last_instr_type == OCSD_INSTR_BR && 1885 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 1886 packet->flags = PERF_IP_FLAG_BRANCH | 1887 PERF_IP_FLAG_CALL; 1888 1889 /* 1890 * Indirect branch instruction with link (e.g. BLR), this is 1891 * branch instruction for function call. 1892 */ 1893 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1894 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 1895 packet->flags = PERF_IP_FLAG_BRANCH | 1896 PERF_IP_FLAG_CALL; 1897 1898 /* 1899 * Indirect branch instruction with subtype of 1900 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for 1901 * function return for A32/T32. 1902 */ 1903 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1904 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET) 1905 packet->flags = PERF_IP_FLAG_BRANCH | 1906 PERF_IP_FLAG_RETURN; 1907 1908 /* 1909 * Indirect branch instruction without link (e.g. BR), usually 1910 * this is used for function return, especially for functions 1911 * within dynamic link lib. 1912 */ 1913 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1914 packet->last_instr_subtype == OCSD_S_INSTR_NONE) 1915 packet->flags = PERF_IP_FLAG_BRANCH | 1916 PERF_IP_FLAG_RETURN; 1917 1918 /* Return instruction for function return. */ 1919 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1920 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET) 1921 packet->flags = PERF_IP_FLAG_BRANCH | 1922 PERF_IP_FLAG_RETURN; 1923 1924 /* 1925 * Decoder might insert a discontinuity in the middle of 1926 * instruction packets, fixup prev_packet with flag 1927 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace. 1928 */ 1929 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY) 1930 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 1931 PERF_IP_FLAG_TRACE_BEGIN; 1932 1933 /* 1934 * If the previous packet is an exception return packet 1935 * and the return address just follows SVC instruction, 1936 * it needs to calibrate the previous packet sample flags 1937 * as PERF_IP_FLAG_SYSCALLRET. 1938 */ 1939 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | 1940 PERF_IP_FLAG_RETURN | 1941 PERF_IP_FLAG_INTERRUPT) && 1942 cs_etm__is_svc_instr(etmq, trace_chan_id, 1943 packet, packet->start_addr)) 1944 prev_packet->flags = PERF_IP_FLAG_BRANCH | 1945 PERF_IP_FLAG_RETURN | 1946 PERF_IP_FLAG_SYSCALLRET; 1947 break; 1948 case CS_ETM_DISCONTINUITY: 1949 /* 1950 * The trace is discontinuous, if the previous packet is 1951 * instruction packet, set flag PERF_IP_FLAG_TRACE_END 1952 * for previous packet. 1953 */ 1954 if (prev_packet->sample_type == CS_ETM_RANGE) 1955 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 1956 PERF_IP_FLAG_TRACE_END; 1957 break; 1958 case CS_ETM_EXCEPTION: 1959 ret = cs_etm__get_magic(packet->trace_chan_id, &magic); 1960 if (ret) 1961 return ret; 1962 1963 /* The exception is for system call. */ 1964 if (cs_etm__is_syscall(etmq, tidq, magic)) 1965 packet->flags = PERF_IP_FLAG_BRANCH | 1966 PERF_IP_FLAG_CALL | 1967 PERF_IP_FLAG_SYSCALLRET; 1968 /* 1969 * The exceptions are triggered by external signals from bus, 1970 * interrupt controller, debug module, PE reset or halt. 1971 */ 1972 else if (cs_etm__is_async_exception(tidq, magic)) 1973 packet->flags = PERF_IP_FLAG_BRANCH | 1974 PERF_IP_FLAG_CALL | 1975 PERF_IP_FLAG_ASYNC | 1976 PERF_IP_FLAG_INTERRUPT; 1977 /* 1978 * Otherwise, exception is caused by trap, instruction & 1979 * data fault, or alignment errors. 1980 */ 1981 else if (cs_etm__is_sync_exception(etmq, tidq, magic)) 1982 packet->flags = PERF_IP_FLAG_BRANCH | 1983 PERF_IP_FLAG_CALL | 1984 PERF_IP_FLAG_INTERRUPT; 1985 1986 /* 1987 * When the exception packet is inserted, since exception 1988 * packet is not used standalone for generating samples 1989 * and it's affiliation to the previous instruction range 1990 * packet; so set previous range packet flags to tell perf 1991 * it is an exception taken branch. 1992 */ 1993 if (prev_packet->sample_type == CS_ETM_RANGE) 1994 prev_packet->flags = packet->flags; 1995 break; 1996 case CS_ETM_EXCEPTION_RET: 1997 /* 1998 * When the exception return packet is inserted, since 1999 * exception return packet is not used standalone for 2000 * generating samples and it's affiliation to the previous 2001 * instruction range packet; so set previous range packet 2002 * flags to tell perf it is an exception return branch. 2003 * 2004 * The exception return can be for either system call or 2005 * other exception types; unfortunately the packet doesn't 2006 * contain exception type related info so we cannot decide 2007 * the exception type purely based on exception return packet. 2008 * If we record the exception number from exception packet and 2009 * reuse it for exception return packet, this is not reliable 2010 * due the trace can be discontinuity or the interrupt can 2011 * be nested, thus the recorded exception number cannot be 2012 * used for exception return packet for these two cases. 2013 * 2014 * For exception return packet, we only need to distinguish the 2015 * packet is for system call or for other types. Thus the 2016 * decision can be deferred when receive the next packet which 2017 * contains the return address, based on the return address we 2018 * can read out the previous instruction and check if it's a 2019 * system call instruction and then calibrate the sample flag 2020 * as needed. 2021 */ 2022 if (prev_packet->sample_type == CS_ETM_RANGE) 2023 prev_packet->flags = PERF_IP_FLAG_BRANCH | 2024 PERF_IP_FLAG_RETURN | 2025 PERF_IP_FLAG_INTERRUPT; 2026 break; 2027 case CS_ETM_EMPTY: 2028 default: 2029 break; 2030 } 2031 2032 return 0; 2033 } 2034 2035 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq) 2036 { 2037 int ret = 0; 2038 size_t processed = 0; 2039 2040 /* 2041 * Packets are decoded and added to the decoder's packet queue 2042 * until the decoder packet processing callback has requested that 2043 * processing stops or there is nothing left in the buffer. Normal 2044 * operations that stop processing are a timestamp packet or a full 2045 * decoder buffer queue. 2046 */ 2047 ret = cs_etm_decoder__process_data_block(etmq->decoder, 2048 etmq->offset, 2049 &etmq->buf[etmq->buf_used], 2050 etmq->buf_len, 2051 &processed); 2052 if (ret) 2053 goto out; 2054 2055 etmq->offset += processed; 2056 etmq->buf_used += processed; 2057 etmq->buf_len -= processed; 2058 2059 out: 2060 return ret; 2061 } 2062 2063 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, 2064 struct cs_etm_traceid_queue *tidq) 2065 { 2066 int ret; 2067 struct cs_etm_packet_queue *packet_queue; 2068 2069 packet_queue = &tidq->packet_queue; 2070 2071 /* Process each packet in this chunk */ 2072 while (1) { 2073 ret = cs_etm_decoder__get_packet(packet_queue, 2074 tidq->packet); 2075 if (ret <= 0) 2076 /* 2077 * Stop processing this chunk on 2078 * end of data or error 2079 */ 2080 break; 2081 2082 /* 2083 * Since packet addresses are swapped in packet 2084 * handling within below switch() statements, 2085 * thus setting sample flags must be called 2086 * prior to switch() statement to use address 2087 * information before packets swapping. 2088 */ 2089 ret = cs_etm__set_sample_flags(etmq, tidq); 2090 if (ret < 0) 2091 break; 2092 2093 switch (tidq->packet->sample_type) { 2094 case CS_ETM_RANGE: 2095 /* 2096 * If the packet contains an instruction 2097 * range, generate instruction sequence 2098 * events. 2099 */ 2100 cs_etm__sample(etmq, tidq); 2101 break; 2102 case CS_ETM_EXCEPTION: 2103 case CS_ETM_EXCEPTION_RET: 2104 /* 2105 * If the exception packet is coming, 2106 * make sure the previous instruction 2107 * range packet to be handled properly. 2108 */ 2109 cs_etm__exception(tidq); 2110 break; 2111 case CS_ETM_DISCONTINUITY: 2112 /* 2113 * Discontinuity in trace, flush 2114 * previous branch stack 2115 */ 2116 cs_etm__flush(etmq, tidq); 2117 break; 2118 case CS_ETM_EMPTY: 2119 /* 2120 * Should not receive empty packet, 2121 * report error. 2122 */ 2123 pr_err("CS ETM Trace: empty packet\n"); 2124 return -EINVAL; 2125 default: 2126 break; 2127 } 2128 } 2129 2130 return ret; 2131 } 2132 2133 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) 2134 { 2135 int idx; 2136 struct int_node *inode; 2137 struct cs_etm_traceid_queue *tidq; 2138 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 2139 2140 intlist__for_each_entry(inode, traceid_queues_list) { 2141 idx = (int)(intptr_t)inode->priv; 2142 tidq = etmq->traceid_queues[idx]; 2143 2144 /* Ignore return value */ 2145 cs_etm__process_traceid_queue(etmq, tidq); 2146 2147 /* 2148 * Generate an instruction sample with the remaining 2149 * branchstack entries. 2150 */ 2151 cs_etm__flush(etmq, tidq); 2152 } 2153 } 2154 2155 static int cs_etm__run_decoder(struct cs_etm_queue *etmq) 2156 { 2157 int err = 0; 2158 struct cs_etm_traceid_queue *tidq; 2159 2160 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID); 2161 if (!tidq) 2162 return -EINVAL; 2163 2164 /* Go through each buffer in the queue and decode them one by one */ 2165 while (1) { 2166 err = cs_etm__get_data_block(etmq); 2167 if (err <= 0) 2168 return err; 2169 2170 /* Run trace decoder until buffer consumed or end of trace */ 2171 do { 2172 err = cs_etm__decode_data_block(etmq); 2173 if (err) 2174 return err; 2175 2176 /* 2177 * Process each packet in this chunk, nothing to do if 2178 * an error occurs other than hoping the next one will 2179 * be better. 2180 */ 2181 err = cs_etm__process_traceid_queue(etmq, tidq); 2182 2183 } while (etmq->buf_len); 2184 2185 if (err == 0) 2186 /* Flush any remaining branch stack entries */ 2187 err = cs_etm__end_block(etmq, tidq); 2188 } 2189 2190 return err; 2191 } 2192 2193 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 2194 pid_t tid) 2195 { 2196 unsigned int i; 2197 struct auxtrace_queues *queues = &etm->queues; 2198 2199 for (i = 0; i < queues->nr_queues; i++) { 2200 struct auxtrace_queue *queue = &etm->queues.queue_array[i]; 2201 struct cs_etm_queue *etmq = queue->priv; 2202 struct cs_etm_traceid_queue *tidq; 2203 2204 if (!etmq) 2205 continue; 2206 2207 tidq = cs_etm__etmq_get_traceid_queue(etmq, 2208 CS_ETM_PER_THREAD_TRACEID); 2209 2210 if (!tidq) 2211 continue; 2212 2213 if ((tid == -1) || (tidq->tid == tid)) { 2214 cs_etm__set_pid_tid_cpu(etm, tidq); 2215 cs_etm__run_decoder(etmq); 2216 } 2217 } 2218 2219 return 0; 2220 } 2221 2222 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm) 2223 { 2224 int ret = 0; 2225 unsigned int cs_queue_nr, queue_nr; 2226 u8 trace_chan_id; 2227 u64 cs_timestamp; 2228 struct auxtrace_queue *queue; 2229 struct cs_etm_queue *etmq; 2230 struct cs_etm_traceid_queue *tidq; 2231 2232 while (1) { 2233 if (!etm->heap.heap_cnt) 2234 goto out; 2235 2236 /* Take the entry at the top of the min heap */ 2237 cs_queue_nr = etm->heap.heap_array[0].queue_nr; 2238 queue_nr = TO_QUEUE_NR(cs_queue_nr); 2239 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr); 2240 queue = &etm->queues.queue_array[queue_nr]; 2241 etmq = queue->priv; 2242 2243 /* 2244 * Remove the top entry from the heap since we are about 2245 * to process it. 2246 */ 2247 auxtrace_heap__pop(&etm->heap); 2248 2249 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 2250 if (!tidq) { 2251 /* 2252 * No traceID queue has been allocated for this traceID, 2253 * which means something somewhere went very wrong. No 2254 * other choice than simply exit. 2255 */ 2256 ret = -EINVAL; 2257 goto out; 2258 } 2259 2260 /* 2261 * Packets associated with this timestamp are already in 2262 * the etmq's traceID queue, so process them. 2263 */ 2264 ret = cs_etm__process_traceid_queue(etmq, tidq); 2265 if (ret < 0) 2266 goto out; 2267 2268 /* 2269 * Packets for this timestamp have been processed, time to 2270 * move on to the next timestamp, fetching a new auxtrace_buffer 2271 * if need be. 2272 */ 2273 refetch: 2274 ret = cs_etm__get_data_block(etmq); 2275 if (ret < 0) 2276 goto out; 2277 2278 /* 2279 * No more auxtrace_buffers to process in this etmq, simply 2280 * move on to another entry in the auxtrace_heap. 2281 */ 2282 if (!ret) 2283 continue; 2284 2285 ret = cs_etm__decode_data_block(etmq); 2286 if (ret) 2287 goto out; 2288 2289 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 2290 2291 if (!cs_timestamp) { 2292 /* 2293 * Function cs_etm__decode_data_block() returns when 2294 * there is no more traces to decode in the current 2295 * auxtrace_buffer OR when a timestamp has been 2296 * encountered on any of the traceID queues. Since we 2297 * did not get a timestamp, there is no more traces to 2298 * process in this auxtrace_buffer. As such empty and 2299 * flush all traceID queues. 2300 */ 2301 cs_etm__clear_all_traceid_queues(etmq); 2302 2303 /* Fetch another auxtrace_buffer for this etmq */ 2304 goto refetch; 2305 } 2306 2307 /* 2308 * Add to the min heap the timestamp for packets that have 2309 * just been decoded. They will be processed and synthesized 2310 * during the next call to cs_etm__process_traceid_queue() for 2311 * this queue/traceID. 2312 */ 2313 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 2314 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); 2315 } 2316 2317 out: 2318 return ret; 2319 } 2320 2321 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm, 2322 union perf_event *event) 2323 { 2324 struct thread *th; 2325 2326 if (etm->timeless_decoding) 2327 return 0; 2328 2329 /* 2330 * Add the tid/pid to the log so that we can get a match when 2331 * we get a contextID from the decoder. 2332 */ 2333 th = machine__findnew_thread(etm->machine, 2334 event->itrace_start.pid, 2335 event->itrace_start.tid); 2336 if (!th) 2337 return -ENOMEM; 2338 2339 thread__put(th); 2340 2341 return 0; 2342 } 2343 2344 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm, 2345 union perf_event *event) 2346 { 2347 struct thread *th; 2348 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; 2349 2350 /* 2351 * Context switch in per-thread mode are irrelevant since perf 2352 * will start/stop tracing as the process is scheduled. 2353 */ 2354 if (etm->timeless_decoding) 2355 return 0; 2356 2357 /* 2358 * SWITCH_IN events carry the next process to be switched out while 2359 * SWITCH_OUT events carry the process to be switched in. As such 2360 * we don't care about IN events. 2361 */ 2362 if (!out) 2363 return 0; 2364 2365 /* 2366 * Add the tid/pid to the log so that we can get a match when 2367 * we get a contextID from the decoder. 2368 */ 2369 th = machine__findnew_thread(etm->machine, 2370 event->context_switch.next_prev_pid, 2371 event->context_switch.next_prev_tid); 2372 if (!th) 2373 return -ENOMEM; 2374 2375 thread__put(th); 2376 2377 return 0; 2378 } 2379 2380 static int cs_etm__process_event(struct perf_session *session, 2381 union perf_event *event, 2382 struct perf_sample *sample, 2383 struct perf_tool *tool) 2384 { 2385 int err = 0; 2386 u64 sample_kernel_timestamp; 2387 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2388 struct cs_etm_auxtrace, 2389 auxtrace); 2390 2391 if (dump_trace) 2392 return 0; 2393 2394 if (!tool->ordered_events) { 2395 pr_err("CoreSight ETM Trace requires ordered events\n"); 2396 return -EINVAL; 2397 } 2398 2399 if (sample->time && (sample->time != (u64) -1)) 2400 sample_kernel_timestamp = sample->time; 2401 else 2402 sample_kernel_timestamp = 0; 2403 2404 if (sample_kernel_timestamp || etm->timeless_decoding) { 2405 err = cs_etm__update_queues(etm); 2406 if (err) 2407 return err; 2408 } 2409 2410 /* 2411 * Don't wait for cs_etm__flush_events() in per-thread/timeless mode to start the decode. We 2412 * need the tid of the PERF_RECORD_EXIT event to assign to the synthesised samples because 2413 * ETM_OPT_CTXTID is not enabled. 2414 */ 2415 if (etm->timeless_decoding && 2416 event->header.type == PERF_RECORD_EXIT) 2417 return cs_etm__process_timeless_queues(etm, 2418 event->fork.tid); 2419 2420 if (event->header.type == PERF_RECORD_ITRACE_START) 2421 return cs_etm__process_itrace_start(etm, event); 2422 else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) 2423 return cs_etm__process_switch_cpu_wide(etm, event); 2424 2425 if (!etm->timeless_decoding && event->header.type == PERF_RECORD_AUX) { 2426 /* 2427 * Record the latest kernel timestamp available in the header 2428 * for samples so that synthesised samples occur from this point 2429 * onwards. 2430 */ 2431 etm->latest_kernel_timestamp = sample_kernel_timestamp; 2432 } 2433 2434 return 0; 2435 } 2436 2437 static void dump_queued_data(struct cs_etm_auxtrace *etm, 2438 struct perf_record_auxtrace *event) 2439 { 2440 struct auxtrace_buffer *buf; 2441 unsigned int i; 2442 /* 2443 * Find all buffers with same reference in the queues and dump them. 2444 * This is because the queues can contain multiple entries of the same 2445 * buffer that were split on aux records. 2446 */ 2447 for (i = 0; i < etm->queues.nr_queues; ++i) 2448 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list) 2449 if (buf->reference == event->reference) 2450 cs_etm__dump_event(etm, buf); 2451 } 2452 2453 static int cs_etm__process_auxtrace_event(struct perf_session *session, 2454 union perf_event *event, 2455 struct perf_tool *tool __maybe_unused) 2456 { 2457 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2458 struct cs_etm_auxtrace, 2459 auxtrace); 2460 if (!etm->data_queued) { 2461 struct auxtrace_buffer *buffer; 2462 off_t data_offset; 2463 int fd = perf_data__fd(session->data); 2464 bool is_pipe = perf_data__is_pipe(session->data); 2465 int err; 2466 2467 if (is_pipe) 2468 data_offset = 0; 2469 else { 2470 data_offset = lseek(fd, 0, SEEK_CUR); 2471 if (data_offset == -1) 2472 return -errno; 2473 } 2474 2475 err = auxtrace_queues__add_event(&etm->queues, session, 2476 event, data_offset, &buffer); 2477 if (err) 2478 return err; 2479 2480 if (dump_trace) 2481 if (auxtrace_buffer__get_data(buffer, fd)) { 2482 cs_etm__dump_event(etm, buffer); 2483 auxtrace_buffer__put_data(buffer); 2484 } 2485 } else if (dump_trace) 2486 dump_queued_data(etm, &event->auxtrace); 2487 2488 return 0; 2489 } 2490 2491 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm) 2492 { 2493 struct evsel *evsel; 2494 struct evlist *evlist = etm->session->evlist; 2495 bool timeless_decoding = true; 2496 2497 /* Override timeless mode with user input from --itrace=Z */ 2498 if (etm->synth_opts.timeless_decoding) 2499 return true; 2500 2501 /* 2502 * Circle through the list of event and complain if we find one 2503 * with the time bit set. 2504 */ 2505 evlist__for_each_entry(evlist, evsel) { 2506 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) 2507 timeless_decoding = false; 2508 } 2509 2510 return timeless_decoding; 2511 } 2512 2513 static const char * const cs_etm_global_header_fmts[] = { 2514 [CS_HEADER_VERSION] = " Header version %llx\n", 2515 [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n", 2516 [CS_ETM_SNAPSHOT] = " Snapshot %llx\n", 2517 }; 2518 2519 static const char * const cs_etm_priv_fmts[] = { 2520 [CS_ETM_MAGIC] = " Magic number %llx\n", 2521 [CS_ETM_CPU] = " CPU %lld\n", 2522 [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", 2523 [CS_ETM_ETMCR] = " ETMCR %llx\n", 2524 [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n", 2525 [CS_ETM_ETMCCER] = " ETMCCER %llx\n", 2526 [CS_ETM_ETMIDR] = " ETMIDR %llx\n", 2527 }; 2528 2529 static const char * const cs_etmv4_priv_fmts[] = { 2530 [CS_ETM_MAGIC] = " Magic number %llx\n", 2531 [CS_ETM_CPU] = " CPU %lld\n", 2532 [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", 2533 [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n", 2534 [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", 2535 [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n", 2536 [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n", 2537 [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n", 2538 [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n", 2539 [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", 2540 }; 2541 2542 static const char * const param_unk_fmt = 2543 " Unknown parameter [%d] %llx\n"; 2544 static const char * const magic_unk_fmt = 2545 " Magic number Unknown %llx\n"; 2546 2547 static int cs_etm__print_cpu_metadata_v0(__u64 *val, int *offset) 2548 { 2549 int i = *offset, j, nr_params = 0, fmt_offset; 2550 __u64 magic; 2551 2552 /* check magic value */ 2553 magic = val[i + CS_ETM_MAGIC]; 2554 if ((magic != __perf_cs_etmv3_magic) && 2555 (magic != __perf_cs_etmv4_magic)) { 2556 /* failure - note bad magic value */ 2557 fprintf(stdout, magic_unk_fmt, magic); 2558 return -EINVAL; 2559 } 2560 2561 /* print common header block */ 2562 fprintf(stdout, cs_etm_priv_fmts[CS_ETM_MAGIC], val[i++]); 2563 fprintf(stdout, cs_etm_priv_fmts[CS_ETM_CPU], val[i++]); 2564 2565 if (magic == __perf_cs_etmv3_magic) { 2566 nr_params = CS_ETM_NR_TRC_PARAMS_V0; 2567 fmt_offset = CS_ETM_ETMCR; 2568 /* after common block, offset format index past NR_PARAMS */ 2569 for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) 2570 fprintf(stdout, cs_etm_priv_fmts[j], val[i]); 2571 } else if (magic == __perf_cs_etmv4_magic) { 2572 nr_params = CS_ETMV4_NR_TRC_PARAMS_V0; 2573 fmt_offset = CS_ETMV4_TRCCONFIGR; 2574 /* after common block, offset format index past NR_PARAMS */ 2575 for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) 2576 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); 2577 } 2578 *offset = i; 2579 return 0; 2580 } 2581 2582 static int cs_etm__print_cpu_metadata_v1(__u64 *val, int *offset) 2583 { 2584 int i = *offset, j, total_params = 0; 2585 __u64 magic; 2586 2587 magic = val[i + CS_ETM_MAGIC]; 2588 /* total params to print is NR_PARAMS + common block size for v1 */ 2589 total_params = val[i + CS_ETM_NR_TRC_PARAMS] + CS_ETM_COMMON_BLK_MAX_V1; 2590 2591 if (magic == __perf_cs_etmv3_magic) { 2592 for (j = 0; j < total_params; j++, i++) { 2593 /* if newer record - could be excess params */ 2594 if (j >= CS_ETM_PRIV_MAX) 2595 fprintf(stdout, param_unk_fmt, j, val[i]); 2596 else 2597 fprintf(stdout, cs_etm_priv_fmts[j], val[i]); 2598 } 2599 } else if (magic == __perf_cs_etmv4_magic) { 2600 for (j = 0; j < total_params; j++, i++) { 2601 /* if newer record - could be excess params */ 2602 if (j >= CS_ETMV4_PRIV_MAX) 2603 fprintf(stdout, param_unk_fmt, j, val[i]); 2604 else 2605 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); 2606 } 2607 } else { 2608 /* failure - note bad magic value and error out */ 2609 fprintf(stdout, magic_unk_fmt, magic); 2610 return -EINVAL; 2611 } 2612 *offset = i; 2613 return 0; 2614 } 2615 2616 static void cs_etm__print_auxtrace_info(__u64 *val, int num) 2617 { 2618 int i, cpu = 0, version, err; 2619 2620 /* bail out early on bad header version */ 2621 version = val[0]; 2622 if (version > CS_HEADER_CURRENT_VERSION) { 2623 /* failure.. return */ 2624 fprintf(stdout, " Unknown Header Version = %x, ", version); 2625 fprintf(stdout, "Version supported <= %x\n", CS_HEADER_CURRENT_VERSION); 2626 return; 2627 } 2628 2629 for (i = 0; i < CS_HEADER_VERSION_MAX; i++) 2630 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]); 2631 2632 for (i = CS_HEADER_VERSION_MAX; cpu < num; cpu++) { 2633 if (version == 0) 2634 err = cs_etm__print_cpu_metadata_v0(val, &i); 2635 else if (version == 1) 2636 err = cs_etm__print_cpu_metadata_v1(val, &i); 2637 if (err) 2638 return; 2639 } 2640 } 2641 2642 /* 2643 * Read a single cpu parameter block from the auxtrace_info priv block. 2644 * 2645 * For version 1 there is a per cpu nr_params entry. If we are handling 2646 * version 1 file, then there may be less, the same, or more params 2647 * indicated by this value than the compile time number we understand. 2648 * 2649 * For a version 0 info block, there are a fixed number, and we need to 2650 * fill out the nr_param value in the metadata we create. 2651 */ 2652 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset, 2653 int out_blk_size, int nr_params_v0) 2654 { 2655 u64 *metadata = NULL; 2656 int hdr_version; 2657 int nr_in_params, nr_out_params, nr_cmn_params; 2658 int i, k; 2659 2660 metadata = zalloc(sizeof(*metadata) * out_blk_size); 2661 if (!metadata) 2662 return NULL; 2663 2664 /* read block current index & version */ 2665 i = *buff_in_offset; 2666 hdr_version = buff_in[CS_HEADER_VERSION]; 2667 2668 if (!hdr_version) { 2669 /* read version 0 info block into a version 1 metadata block */ 2670 nr_in_params = nr_params_v0; 2671 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC]; 2672 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU]; 2673 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params; 2674 /* remaining block params at offset +1 from source */ 2675 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++) 2676 metadata[k + 1] = buff_in[i + k]; 2677 /* version 0 has 2 common params */ 2678 nr_cmn_params = 2; 2679 } else { 2680 /* read version 1 info block - input and output nr_params may differ */ 2681 /* version 1 has 3 common params */ 2682 nr_cmn_params = 3; 2683 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS]; 2684 2685 /* if input has more params than output - skip excess */ 2686 nr_out_params = nr_in_params + nr_cmn_params; 2687 if (nr_out_params > out_blk_size) 2688 nr_out_params = out_blk_size; 2689 2690 for (k = CS_ETM_MAGIC; k < nr_out_params; k++) 2691 metadata[k] = buff_in[i + k]; 2692 2693 /* record the actual nr params we copied */ 2694 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params; 2695 } 2696 2697 /* adjust in offset by number of in params used */ 2698 i += nr_in_params + nr_cmn_params; 2699 *buff_in_offset = i; 2700 return metadata; 2701 } 2702 2703 /** 2704 * Puts a fragment of an auxtrace buffer into the auxtrace queues based 2705 * on the bounds of aux_event, if it matches with the buffer that's at 2706 * file_offset. 2707 * 2708 * Normally, whole auxtrace buffers would be added to the queue. But we 2709 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder 2710 * is reset across each buffer, so splitting the buffers up in advance has 2711 * the same effect. 2712 */ 2713 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, 2714 struct perf_record_aux *aux_event, struct perf_sample *sample) 2715 { 2716 int err; 2717 char buf[PERF_SAMPLE_MAX_SIZE]; 2718 union perf_event *auxtrace_event_union; 2719 struct perf_record_auxtrace *auxtrace_event; 2720 union perf_event auxtrace_fragment; 2721 __u64 aux_offset, aux_size; 2722 2723 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2724 struct cs_etm_auxtrace, 2725 auxtrace); 2726 2727 /* 2728 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got 2729 * from looping through the auxtrace index. 2730 */ 2731 err = perf_session__peek_event(session, file_offset, buf, 2732 PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); 2733 if (err) 2734 return err; 2735 auxtrace_event = &auxtrace_event_union->auxtrace; 2736 if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) 2737 return -EINVAL; 2738 2739 if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || 2740 auxtrace_event->header.size != sz) { 2741 return -EINVAL; 2742 } 2743 2744 /* 2745 * In per-thread mode, CPU is set to -1, but TID will be set instead. See 2746 * auxtrace_mmap_params__set_idx(). Return 'not found' if neither CPU nor TID match. 2747 */ 2748 if ((auxtrace_event->cpu == (__u32) -1 && auxtrace_event->tid != sample->tid) || 2749 auxtrace_event->cpu != sample->cpu) 2750 return 1; 2751 2752 if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { 2753 /* 2754 * Clamp size in snapshot mode. The buffer size is clamped in 2755 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect 2756 * the buffer size. 2757 */ 2758 aux_size = min(aux_event->aux_size, auxtrace_event->size); 2759 2760 /* 2761 * In this mode, the head also points to the end of the buffer so aux_offset 2762 * needs to have the size subtracted so it points to the beginning as in normal mode 2763 */ 2764 aux_offset = aux_event->aux_offset - aux_size; 2765 } else { 2766 aux_size = aux_event->aux_size; 2767 aux_offset = aux_event->aux_offset; 2768 } 2769 2770 if (aux_offset >= auxtrace_event->offset && 2771 aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { 2772 /* 2773 * If this AUX event was inside this buffer somewhere, create a new auxtrace event 2774 * based on the sizes of the aux event, and queue that fragment. 2775 */ 2776 auxtrace_fragment.auxtrace = *auxtrace_event; 2777 auxtrace_fragment.auxtrace.size = aux_size; 2778 auxtrace_fragment.auxtrace.offset = aux_offset; 2779 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; 2780 2781 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 2782 " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); 2783 return auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment, 2784 file_offset, NULL); 2785 } 2786 2787 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ 2788 return 1; 2789 } 2790 2791 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event, 2792 u64 offset __maybe_unused, void *data __maybe_unused) 2793 { 2794 struct perf_sample sample; 2795 int ret; 2796 struct auxtrace_index_entry *ent; 2797 struct auxtrace_index *auxtrace_index; 2798 struct evsel *evsel; 2799 size_t i; 2800 2801 /* Don't care about any other events, we're only queuing buffers for AUX events */ 2802 if (event->header.type != PERF_RECORD_AUX) 2803 return 0; 2804 2805 if (event->header.size < sizeof(struct perf_record_aux)) 2806 return -EINVAL; 2807 2808 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ 2809 if (!event->aux.aux_size) 2810 return 0; 2811 2812 /* 2813 * Parse the sample, we need the sample_id_all data that comes after the event so that the 2814 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. 2815 */ 2816 evsel = evlist__event2evsel(session->evlist, event); 2817 if (!evsel) 2818 return -EINVAL; 2819 ret = evsel__parse_sample(evsel, event, &sample); 2820 if (ret) 2821 return ret; 2822 2823 /* 2824 * Loop through the auxtrace index to find the buffer that matches up with this aux event. 2825 */ 2826 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { 2827 for (i = 0; i < auxtrace_index->nr; i++) { 2828 ent = &auxtrace_index->entries[i]; 2829 ret = cs_etm__queue_aux_fragment(session, ent->file_offset, 2830 ent->sz, &event->aux, &sample); 2831 /* 2832 * Stop search on error or successful values. Continue search on 2833 * 1 ('not found') 2834 */ 2835 if (ret != 1) 2836 return ret; 2837 } 2838 } 2839 2840 /* 2841 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but 2842 * don't exit with an error because it will still be possible to decode other aux records. 2843 */ 2844 pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 2845 " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); 2846 return 0; 2847 } 2848 2849 static int cs_etm__queue_aux_records(struct perf_session *session) 2850 { 2851 struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, 2852 struct auxtrace_index, list); 2853 if (index && index->nr > 0) 2854 return perf_session__peek_events(session, session->header.data_offset, 2855 session->header.data_size, 2856 cs_etm__queue_aux_records_cb, NULL); 2857 2858 /* 2859 * We would get here if there are no entries in the index (either no auxtrace 2860 * buffers or no index at all). Fail silently as there is the possibility of 2861 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still 2862 * false. 2863 * 2864 * In that scenario, buffers will not be split by AUX records. 2865 */ 2866 return 0; 2867 } 2868 2869 int cs_etm__process_auxtrace_info(union perf_event *event, 2870 struct perf_session *session) 2871 { 2872 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; 2873 struct cs_etm_auxtrace *etm = NULL; 2874 struct int_node *inode; 2875 unsigned int pmu_type; 2876 int event_header_size = sizeof(struct perf_event_header); 2877 int info_header_size; 2878 int total_size = auxtrace_info->header.size; 2879 int priv_size = 0; 2880 int num_cpu, trcidr_idx; 2881 int err = 0; 2882 int i, j; 2883 u64 *ptr, *hdr = NULL; 2884 u64 **metadata = NULL; 2885 u64 hdr_version; 2886 2887 /* 2888 * sizeof(auxtrace_info_event::type) + 2889 * sizeof(auxtrace_info_event::reserved) == 8 2890 */ 2891 info_header_size = 8; 2892 2893 if (total_size < (event_header_size + info_header_size)) 2894 return -EINVAL; 2895 2896 priv_size = total_size - event_header_size - info_header_size; 2897 2898 /* First the global part */ 2899 ptr = (u64 *) auxtrace_info->priv; 2900 2901 /* Look for version of the header */ 2902 hdr_version = ptr[0]; 2903 if (hdr_version > CS_HEADER_CURRENT_VERSION) { 2904 /* print routine will print an error on bad version */ 2905 if (dump_trace) 2906 cs_etm__print_auxtrace_info(auxtrace_info->priv, 0); 2907 return -EINVAL; 2908 } 2909 2910 hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_MAX); 2911 if (!hdr) 2912 return -ENOMEM; 2913 2914 /* Extract header information - see cs-etm.h for format */ 2915 for (i = 0; i < CS_HEADER_VERSION_MAX; i++) 2916 hdr[i] = ptr[i]; 2917 num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff; 2918 pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) & 2919 0xffffffff); 2920 2921 /* 2922 * Create an RB tree for traceID-metadata tuple. Since the conversion 2923 * has to be made for each packet that gets decoded, optimizing access 2924 * in anything other than a sequential array is worth doing. 2925 */ 2926 traceid_list = intlist__new(NULL); 2927 if (!traceid_list) { 2928 err = -ENOMEM; 2929 goto err_free_hdr; 2930 } 2931 2932 metadata = zalloc(sizeof(*metadata) * num_cpu); 2933 if (!metadata) { 2934 err = -ENOMEM; 2935 goto err_free_traceid_list; 2936 } 2937 2938 /* 2939 * The metadata is stored in the auxtrace_info section and encodes 2940 * the configuration of the ARM embedded trace macrocell which is 2941 * required by the trace decoder to properly decode the trace due 2942 * to its highly compressed nature. 2943 */ 2944 for (j = 0; j < num_cpu; j++) { 2945 if (ptr[i] == __perf_cs_etmv3_magic) { 2946 metadata[j] = 2947 cs_etm__create_meta_blk(ptr, &i, 2948 CS_ETM_PRIV_MAX, 2949 CS_ETM_NR_TRC_PARAMS_V0); 2950 2951 /* The traceID is our handle */ 2952 trcidr_idx = CS_ETM_ETMTRACEIDR; 2953 2954 } else if (ptr[i] == __perf_cs_etmv4_magic) { 2955 metadata[j] = 2956 cs_etm__create_meta_blk(ptr, &i, 2957 CS_ETMV4_PRIV_MAX, 2958 CS_ETMV4_NR_TRC_PARAMS_V0); 2959 2960 /* The traceID is our handle */ 2961 trcidr_idx = CS_ETMV4_TRCTRACEIDR; 2962 } 2963 2964 if (!metadata[j]) { 2965 err = -ENOMEM; 2966 goto err_free_metadata; 2967 } 2968 2969 /* Get an RB node for this CPU */ 2970 inode = intlist__findnew(traceid_list, metadata[j][trcidr_idx]); 2971 2972 /* Something went wrong, no need to continue */ 2973 if (!inode) { 2974 err = -ENOMEM; 2975 goto err_free_metadata; 2976 } 2977 2978 /* 2979 * The node for that CPU should not be taken. 2980 * Back out if that's the case. 2981 */ 2982 if (inode->priv) { 2983 err = -EINVAL; 2984 goto err_free_metadata; 2985 } 2986 /* All good, associate the traceID with the metadata pointer */ 2987 inode->priv = metadata[j]; 2988 } 2989 2990 /* 2991 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and 2992 * CS_ETMV4_PRIV_MAX mark how many double words are in the 2993 * global metadata, and each cpu's metadata respectively. 2994 * The following tests if the correct number of double words was 2995 * present in the auxtrace info section. 2996 */ 2997 if (i * 8 != priv_size) { 2998 err = -EINVAL; 2999 goto err_free_metadata; 3000 } 3001 3002 etm = zalloc(sizeof(*etm)); 3003 3004 if (!etm) { 3005 err = -ENOMEM; 3006 goto err_free_metadata; 3007 } 3008 3009 err = auxtrace_queues__init(&etm->queues); 3010 if (err) 3011 goto err_free_etm; 3012 3013 if (session->itrace_synth_opts->set) { 3014 etm->synth_opts = *session->itrace_synth_opts; 3015 } else { 3016 itrace_synth_opts__set_default(&etm->synth_opts, 3017 session->itrace_synth_opts->default_no_sample); 3018 etm->synth_opts.callchain = false; 3019 } 3020 3021 etm->session = session; 3022 etm->machine = &session->machines.host; 3023 3024 etm->num_cpu = num_cpu; 3025 etm->pmu_type = pmu_type; 3026 etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0); 3027 etm->metadata = metadata; 3028 etm->auxtrace_type = auxtrace_info->type; 3029 etm->timeless_decoding = cs_etm__is_timeless_decoding(etm); 3030 3031 etm->auxtrace.process_event = cs_etm__process_event; 3032 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event; 3033 etm->auxtrace.flush_events = cs_etm__flush_events; 3034 etm->auxtrace.free_events = cs_etm__free_events; 3035 etm->auxtrace.free = cs_etm__free; 3036 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace; 3037 session->auxtrace = &etm->auxtrace; 3038 3039 etm->unknown_thread = thread__new(999999999, 999999999); 3040 if (!etm->unknown_thread) { 3041 err = -ENOMEM; 3042 goto err_free_queues; 3043 } 3044 3045 /* 3046 * Initialize list node so that at thread__zput() we can avoid 3047 * segmentation fault at list_del_init(). 3048 */ 3049 INIT_LIST_HEAD(&etm->unknown_thread->node); 3050 3051 err = thread__set_comm(etm->unknown_thread, "unknown", 0); 3052 if (err) 3053 goto err_delete_thread; 3054 3055 if (thread__init_maps(etm->unknown_thread, etm->machine)) { 3056 err = -ENOMEM; 3057 goto err_delete_thread; 3058 } 3059 3060 if (dump_trace) { 3061 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu); 3062 } 3063 3064 err = cs_etm__synth_events(etm, session); 3065 if (err) 3066 goto err_delete_thread; 3067 3068 err = cs_etm__queue_aux_records(session); 3069 if (err) 3070 goto err_delete_thread; 3071 3072 etm->data_queued = etm->queues.populated; 3073 3074 return 0; 3075 3076 err_delete_thread: 3077 thread__zput(etm->unknown_thread); 3078 err_free_queues: 3079 auxtrace_queues__free(&etm->queues); 3080 session->auxtrace = NULL; 3081 err_free_etm: 3082 zfree(&etm); 3083 err_free_metadata: 3084 /* No need to check @metadata[j], free(NULL) is supported */ 3085 for (j = 0; j < num_cpu; j++) 3086 zfree(&metadata[j]); 3087 zfree(&metadata); 3088 err_free_traceid_list: 3089 intlist__delete(traceid_list); 3090 err_free_hdr: 3091 zfree(&hdr); 3092 /* 3093 * At this point, as a minimum we have valid header. Dump the rest of 3094 * the info section - the print routines will error out on structural 3095 * issues. 3096 */ 3097 if (dump_trace) 3098 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu); 3099 return err; 3100 } 3101