454 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_packet_queues()  local
459 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_packet_queues()
460 		cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__clear_all_packet_queues()
465 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__init_traceid_queue()  argument
472 	cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__init_traceid_queue()
475 	tidq->trace_chan_id = trace_chan_id;  in cs_etm__init_traceid_queue()
476 	tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;  in cs_etm__init_traceid_queue()
477 	tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,  in cs_etm__init_traceid_queue()
479 	tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);  in cs_etm__init_traceid_queue()
481 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
482 	if (!tidq->packet)  in cs_etm__init_traceid_queue()
485 	tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
486 	if (!tidq->prev_packet)  in cs_etm__init_traceid_queue()
494 		tidq->last_branch = zalloc(sz);  in cs_etm__init_traceid_queue()
495 		if (!tidq->last_branch)  in cs_etm__init_traceid_queue()
497 		tidq->last_branch_rb = zalloc(sz);  in cs_etm__init_traceid_queue()
498 		if (!tidq->last_branch_rb)  in cs_etm__init_traceid_queue()
502 	tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in cs_etm__init_traceid_queue()
503 	if (!tidq->event_buf)  in cs_etm__init_traceid_queue()
509 	zfree(&tidq->last_branch_rb);  in cs_etm__init_traceid_queue()
510 	zfree(&tidq->last_branch);  in cs_etm__init_traceid_queue()
511 	zfree(&tidq->prev_packet);  in cs_etm__init_traceid_queue()
512 	zfree(&tidq->packet);  in cs_etm__init_traceid_queue()
523 	struct cs_etm_traceid_queue *tidq, **traceid_queues;  in cs_etm__etmq_get_traceid_queue()  local
542 	tidq = malloc(sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
543 	if (!tidq)  in cs_etm__etmq_get_traceid_queue()
546 	memset(tidq, 0, sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
558 	if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))  in cs_etm__etmq_get_traceid_queue()
574 	traceid_queues[idx] = tidq;  in cs_etm__etmq_get_traceid_queue()
585 	free(tidq);  in cs_etm__etmq_get_traceid_queue()
593 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_get_packet_queue()  local
595 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_get_packet_queue()
596 	if (tidq)  in cs_etm__etmq_get_packet_queue()
597 		return &tidq->packet_queue;  in cs_etm__etmq_get_packet_queue()
603 				struct cs_etm_traceid_queue *tidq)  in cs_etm__packet_swap()  argument
621 		tmp = tidq->packet;  in cs_etm__packet_swap()
622 		tidq->packet = tidq->prev_packet;  in cs_etm__packet_swap()
623 		tidq->prev_packet = tmp;  in cs_etm__packet_swap()
624 		tidq->prev_packet_el = tidq->el;  in cs_etm__packet_swap()
625 		thread__put(tidq->prev_packet_thread);  in cs_etm__packet_swap()
626 		tidq->prev_packet_thread = thread__get(tidq->thread);  in cs_etm__packet_swap()
792 	struct cs_etm_traceid_queue *tidq;  in cs_etm__free_traceid_queues()  local
800 		tidq = etmq->traceid_queues[idx];  in cs_etm__free_traceid_queues()
801 		thread__zput(tidq->thread);  in cs_etm__free_traceid_queues()
802 		thread__zput(tidq->prev_packet_thread);  in cs_etm__free_traceid_queues()
803 		zfree(&tidq->event_buf);  in cs_etm__free_traceid_queues()
804 		zfree(&tidq->last_branch);  in cs_etm__free_traceid_queues()
805 		zfree(&tidq->last_branch_rb);  in cs_etm__free_traceid_queues()
806 		zfree(&tidq->prev_packet);  in cs_etm__free_traceid_queues()
807 		zfree(&tidq->packet);  in cs_etm__free_traceid_queues()
808 		zfree(&tidq);  in cs_etm__free_traceid_queues()
952 	struct cs_etm_traceid_queue *tidq;  in cs_etm__mem_access()  local
959 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__mem_access()
960 	if (!tidq)  in cs_etm__mem_access()
974 			assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);  in cs_etm__mem_access()
976 			assert(tidq->el == ocsd_EL2);  in cs_etm__mem_access()
978 			assert(tidq->el == ocsd_EL3);  in cs_etm__mem_access()
981 	cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);  in cs_etm__mem_access()
983 	if (!thread__find_map(tidq->thread, cpumode, address, &al))  in cs_etm__mem_access()
998 	len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),  in cs_etm__mem_access()
1177 				 struct cs_etm_traceid_queue *tidq)  in cs_etm__copy_last_branch_rb()  argument
1179 	struct branch_stack *bs_src = tidq->last_branch_rb;  in cs_etm__copy_last_branch_rb()
1180 	struct branch_stack *bs_dst = tidq->last_branch;  in cs_etm__copy_last_branch_rb()
1200 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
1202 	       &bs_src->entries[tidq->last_branch_pos],  in cs_etm__copy_last_branch_rb()
1215 		       sizeof(struct branch_entry) * tidq->last_branch_pos);  in cs_etm__copy_last_branch_rb()
1220 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)  in cs_etm__reset_last_branch_rb()  argument
1222 	tidq->last_branch_pos = 0;  in cs_etm__reset_last_branch_rb()
1223 	tidq->last_branch_rb->nr = 0;  in cs_etm__reset_last_branch_rb()
1281 					  struct cs_etm_traceid_queue *tidq)  in cs_etm__update_last_branch_rb()  argument
1283 	struct branch_stack *bs = tidq->last_branch_rb;  in cs_etm__update_last_branch_rb()
1292 	if (!tidq->last_branch_pos)  in cs_etm__update_last_branch_rb()
1293 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1295 	tidq->last_branch_pos -= 1;  in cs_etm__update_last_branch_rb()
1297 	be       = &bs->entries[tidq->last_branch_pos];  in cs_etm__update_last_branch_rb()
1298 	be->from = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__update_last_branch_rb()
1299 	be->to	 = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__update_last_branch_rb()
1363 			       struct cs_etm_traceid_queue *tidq, pid_t tid,  in cs_etm__set_thread()  argument
1369 		thread__zput(tidq->thread);  in cs_etm__set_thread()
1370 		tidq->thread = machine__find_thread(machine, -1, tid);  in cs_etm__set_thread()
1374 	if (!tidq->thread)  in cs_etm__set_thread()
1375 		tidq->thread = machine__idle_thread(machine);  in cs_etm__set_thread()
1377 	tidq->el = el;  in cs_etm__set_thread()
1383 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_set_tid_el()  local
1385 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_set_tid_el()
1386 	if (!tidq)  in cs_etm__etmq_set_tid_el()
1389 	cs_etm__set_thread(etmq, tidq, tid, el);  in cs_etm__etmq_set_tid_el()
1438 					       struct cs_etm_traceid_queue *tidq)  in cs_etm__resolve_sample_time()  argument
1441 	struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;  in cs_etm__resolve_sample_time()
1450 					    struct cs_etm_traceid_queue *tidq,  in cs_etm__synth_instruction_sample()  argument
1455 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_instruction_sample()
1459 	event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);  in cs_etm__synth_instruction_sample()
1463 	sample.time = cs_etm__resolve_sample_time(etmq, tidq);  in cs_etm__synth_instruction_sample()
1466 	sample.pid = thread__pid(tidq->thread);  in cs_etm__synth_instruction_sample()
1467 	sample.tid = thread__tid(tidq->thread);  in cs_etm__synth_instruction_sample()
1471 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_instruction_sample()
1472 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_instruction_sample()
1475 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);  in cs_etm__synth_instruction_sample()
1478 		sample.branch_stack = tidq->last_branch;  in cs_etm__synth_instruction_sample()
1502 				       struct cs_etm_traceid_queue *tidq)  in cs_etm__synth_branch_sample()  argument
1507 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_branch_sample()
1515 	ip = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__synth_branch_sample()
1519 						     tidq->prev_packet_el);  in cs_etm__synth_branch_sample()
1523 	sample.time = cs_etm__resolve_sample_time(etmq, tidq);  in cs_etm__synth_branch_sample()
1526 	sample.pid = thread__pid(tidq->prev_packet_thread);  in cs_etm__synth_branch_sample()
1527 	sample.tid = thread__tid(tidq->prev_packet_thread);  in cs_etm__synth_branch_sample()
1528 	sample.addr = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__synth_branch_sample()
1532 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_branch_sample()
1533 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_branch_sample()
1536 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,  in cs_etm__synth_branch_sample()
1686 			  struct cs_etm_traceid_queue *tidq)  in cs_etm__sample()  argument
1690 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__sample()
1694 	instrs_prev = tidq->period_instructions;  in cs_etm__sample()
1696 	tidq->period_instructions += tidq->packet->instr_count;  in cs_etm__sample()
1703 	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1704 	    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1705 		cs_etm__update_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1708 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1766 			cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1768 		while (tidq->period_instructions >=  in cs_etm__sample()
1777 						  tidq->packet, offset - 1);  in cs_etm__sample()
1779 				etmq, tidq, addr,  in cs_etm__sample()
1785 			tidq->period_instructions -=  in cs_etm__sample()
1794 		if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)  in cs_etm__sample()
1798 		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1799 		    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1803 			ret = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__sample()
1809 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1814 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)  in cs_etm__exception()  argument
1827 	if (tidq->prev_packet->sample_type == CS_ETM_RANGE)  in cs_etm__exception()
1828 		tidq->prev_packet->last_instr_taken_branch = true;  in cs_etm__exception()
1834 			 struct cs_etm_traceid_queue *tidq)  in cs_etm__flush()  argument
1840 	if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)  in cs_etm__flush()
1845 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1849 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__flush()
1858 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__flush()
1861 			etmq, tidq, addr,  in cs_etm__flush()
1862 			tidq->period_instructions);  in cs_etm__flush()
1866 		tidq->period_instructions = 0;  in cs_etm__flush()
1871 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1872 		err = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__flush()
1878 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1882 		cs_etm__reset_last_branch_rb(tidq);  in cs_etm__flush()
1888 			     struct cs_etm_traceid_queue *tidq)  in cs_etm__end_block()  argument
1903 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__end_block()
1907 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__end_block()
1913 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__end_block()
1916 			etmq, tidq, addr,  in cs_etm__end_block()
1917 			tidq->period_instructions);  in cs_etm__end_block()
1921 		tidq->period_instructions = 0;  in cs_etm__end_block()
2025 			       struct cs_etm_traceid_queue *tidq, u64 magic)  in cs_etm__is_syscall()  argument
2027 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_syscall()
2028 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_syscall()
2029 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_syscall()
2050 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,  in cs_etm__is_async_exception()  argument
2053 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_async_exception()
2077 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__is_sync_exception()  argument
2080 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_sync_exception()
2081 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_sync_exception()
2082 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_sync_exception()
2126 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_sample_flags()  argument
2128 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__set_sample_flags()
2129 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__set_sample_flags()
2130 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__set_sample_flags()
2233 		if (cs_etm__is_syscall(etmq, tidq, magic))  in cs_etm__set_sample_flags()
2241 		else if (cs_etm__is_async_exception(tidq, magic))  in cs_etm__set_sample_flags()
2250 		else if (cs_etm__is_sync_exception(etmq, tidq, magic))  in cs_etm__set_sample_flags()
2333 					 struct cs_etm_traceid_queue *tidq)  in cs_etm__process_traceid_queue()  argument
2338 	packet_queue = &tidq->packet_queue;  in cs_etm__process_traceid_queue()
2343 						 tidq->packet);  in cs_etm__process_traceid_queue()
2358 		ret = cs_etm__set_sample_flags(etmq, tidq);  in cs_etm__process_traceid_queue()
2362 		switch (tidq->packet->sample_type) {  in cs_etm__process_traceid_queue()
2369 			cs_etm__sample(etmq, tidq);  in cs_etm__process_traceid_queue()
2378 			cs_etm__exception(tidq);  in cs_etm__process_traceid_queue()
2385 			cs_etm__flush(etmq, tidq);  in cs_etm__process_traceid_queue()
2406 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_traceid_queues()  local
2411 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_traceid_queues()
2414 		cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2420 		cs_etm__flush(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2427 	struct cs_etm_traceid_queue *tidq;  in cs_etm__run_per_thread_timeless_decoder()  local
2429 	tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);  in cs_etm__run_per_thread_timeless_decoder()
2430 	if (!tidq)  in cs_etm__run_per_thread_timeless_decoder()
2450 			err = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__run_per_thread_timeless_decoder()
2456 			err = cs_etm__end_block(etmq, tidq);  in cs_etm__run_per_thread_timeless_decoder()
2465 	struct cs_etm_traceid_queue *tidq;  in cs_etm__run_per_cpu_timeless_decoder()  local
2489 				tidq = etmq->traceid_queues[idx];  in cs_etm__run_per_cpu_timeless_decoder()
2490 				cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__run_per_cpu_timeless_decoder()
2496 			tidq = etmq->traceid_queues[idx];  in cs_etm__run_per_cpu_timeless_decoder()
2498 			err = cs_etm__end_block(etmq, tidq);  in cs_etm__run_per_cpu_timeless_decoder()
2516 		struct cs_etm_traceid_queue *tidq;  in cs_etm__process_timeless_queues()  local
2522 			tidq = cs_etm__etmq_get_traceid_queue(  in cs_etm__process_timeless_queues()
2525 			if (!tidq)  in cs_etm__process_timeless_queues()
2528 			if (tid == -1 || thread__tid(tidq->thread) == tid)  in cs_etm__process_timeless_queues()
2545 	struct cs_etm_traceid_queue *tidq;  in cs_etm__process_timestamped_queues()  local
2578 		tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__process_timestamped_queues()
2579 		if (!tidq) {  in cs_etm__process_timestamped_queues()
2593 		ret = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__process_timestamped_queues()