xref: /openbmc/linux/tools/perf/arch/arm/util/cs-etm.c (revision cd6d421e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2015 Linaro Limited. All rights reserved.
4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5  */
6 
7 #include <api/fs/fs.h>
8 #include <linux/bits.h>
9 #include <linux/bitops.h>
10 #include <linux/compiler.h>
11 #include <linux/coresight-pmu.h>
12 #include <linux/kernel.h>
13 #include <linux/log2.h>
14 #include <linux/string.h>
15 #include <linux/types.h>
16 #include <linux/zalloc.h>
17 
18 #include "cs-etm.h"
19 #include "../../util/debug.h"
20 #include "../../util/record.h"
21 #include "../../util/auxtrace.h"
22 #include "../../util/cpumap.h"
23 #include "../../util/event.h"
24 #include "../../util/evlist.h"
25 #include "../../util/evsel.h"
26 #include "../../util/perf_api_probe.h"
27 #include "../../util/evsel_config.h"
28 #include "../../util/pmu.h"
29 #include "../../util/cs-etm.h"
30 #include <internal/lib.h> // page_size
31 #include "../../util/session.h"
32 
33 #include <errno.h>
34 #include <stdlib.h>
35 #include <sys/stat.h>
36 
37 struct cs_etm_recording {
38 	struct auxtrace_record	itr;
39 	struct perf_pmu		*cs_etm_pmu;
40 	struct evlist		*evlist;
41 	int			wrapped_cnt;
42 	bool			*wrapped;
43 	bool			snapshot_mode;
44 	size_t			snapshot_size;
45 };
46 
47 static const char *metadata_etmv3_ro[CS_ETM_PRIV_MAX] = {
48 	[CS_ETM_ETMCCER]	= "mgmt/etmccer",
49 	[CS_ETM_ETMIDR]		= "mgmt/etmidr",
50 };
51 
52 static const char *metadata_etmv4_ro[CS_ETMV4_PRIV_MAX] = {
53 	[CS_ETMV4_TRCIDR0]		= "trcidr/trcidr0",
54 	[CS_ETMV4_TRCIDR1]		= "trcidr/trcidr1",
55 	[CS_ETMV4_TRCIDR2]		= "trcidr/trcidr2",
56 	[CS_ETMV4_TRCIDR8]		= "trcidr/trcidr8",
57 	[CS_ETMV4_TRCAUTHSTATUS]	= "mgmt/trcauthstatus",
58 };
59 
60 static bool cs_etm_is_etmv4(struct auxtrace_record *itr, int cpu);
61 
62 static int cs_etm_set_context_id(struct auxtrace_record *itr,
63 				 struct evsel *evsel, int cpu)
64 {
65 	struct cs_etm_recording *ptr;
66 	struct perf_pmu *cs_etm_pmu;
67 	char path[PATH_MAX];
68 	int err = -EINVAL;
69 	u32 val;
70 
71 	ptr = container_of(itr, struct cs_etm_recording, itr);
72 	cs_etm_pmu = ptr->cs_etm_pmu;
73 
74 	if (!cs_etm_is_etmv4(itr, cpu))
75 		goto out;
76 
77 	/* Get a handle on TRCIRD2 */
78 	snprintf(path, PATH_MAX, "cpu%d/%s",
79 		 cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR2]);
80 	err = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
81 
82 	/* There was a problem reading the file, bailing out */
83 	if (err != 1) {
84 		pr_err("%s: can't read file %s\n",
85 		       CORESIGHT_ETM_PMU_NAME, path);
86 		goto out;
87 	}
88 
89 	/*
90 	 * TRCIDR2.CIDSIZE, bit [9-5], indicates whether contextID tracing
91 	 * is supported:
92 	 *  0b00000 Context ID tracing is not supported.
93 	 *  0b00100 Maximum of 32-bit Context ID size.
94 	 *  All other values are reserved.
95 	 */
96 	val = BMVAL(val, 5, 9);
97 	if (!val || val != 0x4) {
98 		err = -EINVAL;
99 		goto out;
100 	}
101 
102 	/* All good, let the kernel know */
103 	evsel->core.attr.config |= (1 << ETM_OPT_CTXTID);
104 	err = 0;
105 
106 out:
107 
108 	return err;
109 }
110 
111 static int cs_etm_set_timestamp(struct auxtrace_record *itr,
112 				struct evsel *evsel, int cpu)
113 {
114 	struct cs_etm_recording *ptr;
115 	struct perf_pmu *cs_etm_pmu;
116 	char path[PATH_MAX];
117 	int err = -EINVAL;
118 	u32 val;
119 
120 	ptr = container_of(itr, struct cs_etm_recording, itr);
121 	cs_etm_pmu = ptr->cs_etm_pmu;
122 
123 	if (!cs_etm_is_etmv4(itr, cpu))
124 		goto out;
125 
126 	/* Get a handle on TRCIRD0 */
127 	snprintf(path, PATH_MAX, "cpu%d/%s",
128 		 cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
129 	err = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
130 
131 	/* There was a problem reading the file, bailing out */
132 	if (err != 1) {
133 		pr_err("%s: can't read file %s\n",
134 		       CORESIGHT_ETM_PMU_NAME, path);
135 		goto out;
136 	}
137 
138 	/*
139 	 * TRCIDR0.TSSIZE, bit [28-24], indicates whether global timestamping
140 	 * is supported:
141 	 *  0b00000 Global timestamping is not implemented
142 	 *  0b00110 Implementation supports a maximum timestamp of 48bits.
143 	 *  0b01000 Implementation supports a maximum timestamp of 64bits.
144 	 */
145 	val &= GENMASK(28, 24);
146 	if (!val) {
147 		err = -EINVAL;
148 		goto out;
149 	}
150 
151 	/* All good, let the kernel know */
152 	evsel->core.attr.config |= (1 << ETM_OPT_TS);
153 	err = 0;
154 
155 out:
156 	return err;
157 }
158 
159 #define ETM_SET_OPT_CTXTID	(1 << 0)
160 #define ETM_SET_OPT_TS		(1 << 1)
161 #define ETM_SET_OPT_MASK	(ETM_SET_OPT_CTXTID | ETM_SET_OPT_TS)
162 
163 static int cs_etm_set_option(struct auxtrace_record *itr,
164 			     struct evsel *evsel, u32 option)
165 {
166 	int i, err = -EINVAL;
167 	struct perf_cpu_map *event_cpus = evsel->evlist->core.cpus;
168 	struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
169 
170 	/* Set option of each CPU we have */
171 	for (i = 0; i < cpu__max_cpu(); i++) {
172 		if (!cpu_map__has(event_cpus, i) ||
173 		    !cpu_map__has(online_cpus, i))
174 			continue;
175 
176 		if (option & ETM_SET_OPT_CTXTID) {
177 			err = cs_etm_set_context_id(itr, evsel, i);
178 			if (err)
179 				goto out;
180 		}
181 		if (option & ETM_SET_OPT_TS) {
182 			err = cs_etm_set_timestamp(itr, evsel, i);
183 			if (err)
184 				goto out;
185 		}
186 		if (option & ~(ETM_SET_OPT_MASK))
187 			/* Nothing else is currently supported */
188 			goto out;
189 	}
190 
191 	err = 0;
192 out:
193 	perf_cpu_map__put(online_cpus);
194 	return err;
195 }
196 
197 static int cs_etm_parse_snapshot_options(struct auxtrace_record *itr,
198 					 struct record_opts *opts,
199 					 const char *str)
200 {
201 	struct cs_etm_recording *ptr =
202 				container_of(itr, struct cs_etm_recording, itr);
203 	unsigned long long snapshot_size = 0;
204 	char *endptr;
205 
206 	if (str) {
207 		snapshot_size = strtoull(str, &endptr, 0);
208 		if (*endptr || snapshot_size > SIZE_MAX)
209 			return -1;
210 	}
211 
212 	opts->auxtrace_snapshot_mode = true;
213 	opts->auxtrace_snapshot_size = snapshot_size;
214 	ptr->snapshot_size = snapshot_size;
215 
216 	return 0;
217 }
218 
219 static int cs_etm_set_sink_attr(struct perf_pmu *pmu,
220 				struct evsel *evsel)
221 {
222 	char msg[BUFSIZ], path[PATH_MAX], *sink;
223 	struct evsel_config_term *term;
224 	int ret = -EINVAL;
225 	u32 hash;
226 
227 	if (evsel->core.attr.config2 & GENMASK(31, 0))
228 		return 0;
229 
230 	list_for_each_entry(term, &evsel->config_terms, list) {
231 		if (term->type != EVSEL__CONFIG_TERM_DRV_CFG)
232 			continue;
233 
234 		sink = term->val.str;
235 		snprintf(path, PATH_MAX, "sinks/%s", sink);
236 
237 		ret = perf_pmu__scan_file(pmu, path, "%x", &hash);
238 		if (ret != 1) {
239 			pr_err("failed to set sink \"%s\" on event %s with %d (%s)\n",
240 			       sink, evsel__name(evsel), errno,
241 			       str_error_r(errno, msg, sizeof(msg)));
242 			return ret;
243 		}
244 
245 		evsel->core.attr.config2 |= hash;
246 		return 0;
247 	}
248 
249 	/*
250 	 * No sink was provided on the command line - allow the CoreSight
251 	 * system to look for a default
252 	 */
253 	return 0;
254 }
255 
256 static int cs_etm_recording_options(struct auxtrace_record *itr,
257 				    struct evlist *evlist,
258 				    struct record_opts *opts)
259 {
260 	int ret;
261 	struct cs_etm_recording *ptr =
262 				container_of(itr, struct cs_etm_recording, itr);
263 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
264 	struct evsel *evsel, *cs_etm_evsel = NULL;
265 	struct perf_cpu_map *cpus = evlist->core.cpus;
266 	bool privileged = perf_event_paranoid_check(-1);
267 	int err = 0;
268 
269 	ptr->evlist = evlist;
270 	ptr->snapshot_mode = opts->auxtrace_snapshot_mode;
271 
272 	if (!record_opts__no_switch_events(opts) &&
273 	    perf_can_record_switch_events())
274 		opts->record_switch_events = true;
275 
276 	evlist__for_each_entry(evlist, evsel) {
277 		if (evsel->core.attr.type == cs_etm_pmu->type) {
278 			if (cs_etm_evsel) {
279 				pr_err("There may be only one %s event\n",
280 				       CORESIGHT_ETM_PMU_NAME);
281 				return -EINVAL;
282 			}
283 			evsel->core.attr.freq = 0;
284 			evsel->core.attr.sample_period = 1;
285 			cs_etm_evsel = evsel;
286 			opts->full_auxtrace = true;
287 		}
288 	}
289 
290 	/* no need to continue if at least one event of interest was found */
291 	if (!cs_etm_evsel)
292 		return 0;
293 
294 	ret = cs_etm_set_sink_attr(cs_etm_pmu, cs_etm_evsel);
295 	if (ret)
296 		return ret;
297 
298 	if (opts->use_clockid) {
299 		pr_err("Cannot use clockid (-k option) with %s\n",
300 		       CORESIGHT_ETM_PMU_NAME);
301 		return -EINVAL;
302 	}
303 
304 	/* we are in snapshot mode */
305 	if (opts->auxtrace_snapshot_mode) {
306 		/*
307 		 * No size were given to '-S' or '-m,', so go with
308 		 * the default
309 		 */
310 		if (!opts->auxtrace_snapshot_size &&
311 		    !opts->auxtrace_mmap_pages) {
312 			if (privileged) {
313 				opts->auxtrace_mmap_pages = MiB(4) / page_size;
314 			} else {
315 				opts->auxtrace_mmap_pages =
316 							KiB(128) / page_size;
317 				if (opts->mmap_pages == UINT_MAX)
318 					opts->mmap_pages = KiB(256) / page_size;
319 			}
320 		} else if (!opts->auxtrace_mmap_pages && !privileged &&
321 						opts->mmap_pages == UINT_MAX) {
322 			opts->mmap_pages = KiB(256) / page_size;
323 		}
324 
325 		/*
326 		 * '-m,xyz' was specified but no snapshot size, so make the
327 		 * snapshot size as big as the auxtrace mmap area.
328 		 */
329 		if (!opts->auxtrace_snapshot_size) {
330 			opts->auxtrace_snapshot_size =
331 				opts->auxtrace_mmap_pages * (size_t)page_size;
332 		}
333 
334 		/*
335 		 * -Sxyz was specified but no auxtrace mmap area, so make the
336 		 * auxtrace mmap area big enough to fit the requested snapshot
337 		 * size.
338 		 */
339 		if (!opts->auxtrace_mmap_pages) {
340 			size_t sz = opts->auxtrace_snapshot_size;
341 
342 			sz = round_up(sz, page_size) / page_size;
343 			opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
344 		}
345 
346 		/* Snapshost size can't be bigger than the auxtrace area */
347 		if (opts->auxtrace_snapshot_size >
348 				opts->auxtrace_mmap_pages * (size_t)page_size) {
349 			pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
350 			       opts->auxtrace_snapshot_size,
351 			       opts->auxtrace_mmap_pages * (size_t)page_size);
352 			return -EINVAL;
353 		}
354 
355 		/* Something went wrong somewhere - this shouldn't happen */
356 		if (!opts->auxtrace_snapshot_size ||
357 		    !opts->auxtrace_mmap_pages) {
358 			pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
359 			return -EINVAL;
360 		}
361 	}
362 
363 	/* We are in full trace mode but '-m,xyz' wasn't specified */
364 	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
365 		if (privileged) {
366 			opts->auxtrace_mmap_pages = MiB(4) / page_size;
367 		} else {
368 			opts->auxtrace_mmap_pages = KiB(128) / page_size;
369 			if (opts->mmap_pages == UINT_MAX)
370 				opts->mmap_pages = KiB(256) / page_size;
371 		}
372 
373 	}
374 
375 	/* Validate auxtrace_mmap_pages provided by user */
376 	if (opts->auxtrace_mmap_pages) {
377 		unsigned int max_page = (KiB(128) / page_size);
378 		size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
379 
380 		if (!privileged &&
381 		    opts->auxtrace_mmap_pages > max_page) {
382 			opts->auxtrace_mmap_pages = max_page;
383 			pr_err("auxtrace too big, truncating to %d\n",
384 			       max_page);
385 		}
386 
387 		if (!is_power_of_2(sz)) {
388 			pr_err("Invalid mmap size for %s: must be a power of 2\n",
389 			       CORESIGHT_ETM_PMU_NAME);
390 			return -EINVAL;
391 		}
392 	}
393 
394 	if (opts->auxtrace_snapshot_mode)
395 		pr_debug2("%s snapshot size: %zu\n", CORESIGHT_ETM_PMU_NAME,
396 			  opts->auxtrace_snapshot_size);
397 
398 	/*
399 	 * To obtain the auxtrace buffer file descriptor, the auxtrace
400 	 * event must come first.
401 	 */
402 	evlist__to_front(evlist, cs_etm_evsel);
403 
404 	/*
405 	 * In the case of per-cpu mmaps, we need the CPU on the
406 	 * AUX event.  We also need the contextID in order to be notified
407 	 * when a context switch happened.
408 	 */
409 	if (!perf_cpu_map__empty(cpus)) {
410 		evsel__set_sample_bit(cs_etm_evsel, CPU);
411 
412 		err = cs_etm_set_option(itr, cs_etm_evsel,
413 					ETM_SET_OPT_CTXTID | ETM_SET_OPT_TS);
414 		if (err)
415 			goto out;
416 	}
417 
418 	/* Add dummy event to keep tracking */
419 	if (opts->full_auxtrace) {
420 		struct evsel *tracking_evsel;
421 
422 		err = parse_events(evlist, "dummy:u", NULL);
423 		if (err)
424 			goto out;
425 
426 		tracking_evsel = evlist__last(evlist);
427 		evlist__set_tracking_event(evlist, tracking_evsel);
428 
429 		tracking_evsel->core.attr.freq = 0;
430 		tracking_evsel->core.attr.sample_period = 1;
431 
432 		/* In per-cpu case, always need the time of mmap events etc */
433 		if (!perf_cpu_map__empty(cpus))
434 			evsel__set_sample_bit(tracking_evsel, TIME);
435 	}
436 
437 out:
438 	return err;
439 }
440 
441 static u64 cs_etm_get_config(struct auxtrace_record *itr)
442 {
443 	u64 config = 0;
444 	struct cs_etm_recording *ptr =
445 			container_of(itr, struct cs_etm_recording, itr);
446 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
447 	struct evlist *evlist = ptr->evlist;
448 	struct evsel *evsel;
449 
450 	evlist__for_each_entry(evlist, evsel) {
451 		if (evsel->core.attr.type == cs_etm_pmu->type) {
452 			/*
453 			 * Variable perf_event_attr::config is assigned to
454 			 * ETMv3/PTM.  The bit fields have been made to match
455 			 * the ETMv3.5 ETRMCR register specification.  See the
456 			 * PMU_FORMAT_ATTR() declarations in
457 			 * drivers/hwtracing/coresight/coresight-perf.c for
458 			 * details.
459 			 */
460 			config = evsel->core.attr.config;
461 			break;
462 		}
463 	}
464 
465 	return config;
466 }
467 
468 #ifndef BIT
469 #define BIT(N) (1UL << (N))
470 #endif
471 
472 static u64 cs_etmv4_get_config(struct auxtrace_record *itr)
473 {
474 	u64 config = 0;
475 	u64 config_opts = 0;
476 
477 	/*
478 	 * The perf event variable config bits represent both
479 	 * the command line options and register programming
480 	 * bits in ETMv3/PTM. For ETMv4 we must remap options
481 	 * to real bits
482 	 */
483 	config_opts = cs_etm_get_config(itr);
484 	if (config_opts & BIT(ETM_OPT_CYCACC))
485 		config |= BIT(ETM4_CFG_BIT_CYCACC);
486 	if (config_opts & BIT(ETM_OPT_CTXTID))
487 		config |= BIT(ETM4_CFG_BIT_CTXTID);
488 	if (config_opts & BIT(ETM_OPT_TS))
489 		config |= BIT(ETM4_CFG_BIT_TS);
490 	if (config_opts & BIT(ETM_OPT_RETSTK))
491 		config |= BIT(ETM4_CFG_BIT_RETSTK);
492 
493 	return config;
494 }
495 
496 static size_t
497 cs_etm_info_priv_size(struct auxtrace_record *itr __maybe_unused,
498 		      struct evlist *evlist __maybe_unused)
499 {
500 	int i;
501 	int etmv3 = 0, etmv4 = 0;
502 	struct perf_cpu_map *event_cpus = evlist->core.cpus;
503 	struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
504 
505 	/* cpu map is not empty, we have specific CPUs to work with */
506 	if (!perf_cpu_map__empty(event_cpus)) {
507 		for (i = 0; i < cpu__max_cpu(); i++) {
508 			if (!cpu_map__has(event_cpus, i) ||
509 			    !cpu_map__has(online_cpus, i))
510 				continue;
511 
512 			if (cs_etm_is_etmv4(itr, i))
513 				etmv4++;
514 			else
515 				etmv3++;
516 		}
517 	} else {
518 		/* get configuration for all CPUs in the system */
519 		for (i = 0; i < cpu__max_cpu(); i++) {
520 			if (!cpu_map__has(online_cpus, i))
521 				continue;
522 
523 			if (cs_etm_is_etmv4(itr, i))
524 				etmv4++;
525 			else
526 				etmv3++;
527 		}
528 	}
529 
530 	perf_cpu_map__put(online_cpus);
531 
532 	return (CS_ETM_HEADER_SIZE +
533 	       (etmv4 * CS_ETMV4_PRIV_SIZE) +
534 	       (etmv3 * CS_ETMV3_PRIV_SIZE));
535 }
536 
537 static bool cs_etm_is_etmv4(struct auxtrace_record *itr, int cpu)
538 {
539 	bool ret = false;
540 	char path[PATH_MAX];
541 	int scan;
542 	unsigned int val;
543 	struct cs_etm_recording *ptr =
544 			container_of(itr, struct cs_etm_recording, itr);
545 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
546 
547 	/* Take any of the RO files for ETMv4 and see if it present */
548 	snprintf(path, PATH_MAX, "cpu%d/%s",
549 		 cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
550 	scan = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
551 
552 	/* The file was read successfully, we have a winner */
553 	if (scan == 1)
554 		ret = true;
555 
556 	return ret;
557 }
558 
559 static int cs_etm_get_ro(struct perf_pmu *pmu, int cpu, const char *path)
560 {
561 	char pmu_path[PATH_MAX];
562 	int scan;
563 	unsigned int val = 0;
564 
565 	/* Get RO metadata from sysfs */
566 	snprintf(pmu_path, PATH_MAX, "cpu%d/%s", cpu, path);
567 
568 	scan = perf_pmu__scan_file(pmu, pmu_path, "%x", &val);
569 	if (scan != 1)
570 		pr_err("%s: error reading: %s\n", __func__, pmu_path);
571 
572 	return val;
573 }
574 
575 static void cs_etm_get_metadata(int cpu, u32 *offset,
576 				struct auxtrace_record *itr,
577 				struct perf_record_auxtrace_info *info)
578 {
579 	u32 increment;
580 	u64 magic;
581 	struct cs_etm_recording *ptr =
582 			container_of(itr, struct cs_etm_recording, itr);
583 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
584 
585 	/* first see what kind of tracer this cpu is affined to */
586 	if (cs_etm_is_etmv4(itr, cpu)) {
587 		magic = __perf_cs_etmv4_magic;
588 		/* Get trace configuration register */
589 		info->priv[*offset + CS_ETMV4_TRCCONFIGR] =
590 						cs_etmv4_get_config(itr);
591 		/* Get traceID from the framework */
592 		info->priv[*offset + CS_ETMV4_TRCTRACEIDR] =
593 						coresight_get_trace_id(cpu);
594 		/* Get read-only information from sysFS */
595 		info->priv[*offset + CS_ETMV4_TRCIDR0] =
596 			cs_etm_get_ro(cs_etm_pmu, cpu,
597 				      metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
598 		info->priv[*offset + CS_ETMV4_TRCIDR1] =
599 			cs_etm_get_ro(cs_etm_pmu, cpu,
600 				      metadata_etmv4_ro[CS_ETMV4_TRCIDR1]);
601 		info->priv[*offset + CS_ETMV4_TRCIDR2] =
602 			cs_etm_get_ro(cs_etm_pmu, cpu,
603 				      metadata_etmv4_ro[CS_ETMV4_TRCIDR2]);
604 		info->priv[*offset + CS_ETMV4_TRCIDR8] =
605 			cs_etm_get_ro(cs_etm_pmu, cpu,
606 				      metadata_etmv4_ro[CS_ETMV4_TRCIDR8]);
607 		info->priv[*offset + CS_ETMV4_TRCAUTHSTATUS] =
608 			cs_etm_get_ro(cs_etm_pmu, cpu,
609 				      metadata_etmv4_ro
610 				      [CS_ETMV4_TRCAUTHSTATUS]);
611 
612 		/* How much space was used */
613 		increment = CS_ETMV4_PRIV_MAX;
614 	} else {
615 		magic = __perf_cs_etmv3_magic;
616 		/* Get configuration register */
617 		info->priv[*offset + CS_ETM_ETMCR] = cs_etm_get_config(itr);
618 		/* Get traceID from the framework */
619 		info->priv[*offset + CS_ETM_ETMTRACEIDR] =
620 						coresight_get_trace_id(cpu);
621 		/* Get read-only information from sysFS */
622 		info->priv[*offset + CS_ETM_ETMCCER] =
623 			cs_etm_get_ro(cs_etm_pmu, cpu,
624 				      metadata_etmv3_ro[CS_ETM_ETMCCER]);
625 		info->priv[*offset + CS_ETM_ETMIDR] =
626 			cs_etm_get_ro(cs_etm_pmu, cpu,
627 				      metadata_etmv3_ro[CS_ETM_ETMIDR]);
628 
629 		/* How much space was used */
630 		increment = CS_ETM_PRIV_MAX;
631 	}
632 
633 	/* Build generic header portion */
634 	info->priv[*offset + CS_ETM_MAGIC] = magic;
635 	info->priv[*offset + CS_ETM_CPU] = cpu;
636 	/* Where the next CPU entry should start from */
637 	*offset += increment;
638 }
639 
640 static int cs_etm_info_fill(struct auxtrace_record *itr,
641 			    struct perf_session *session,
642 			    struct perf_record_auxtrace_info *info,
643 			    size_t priv_size)
644 {
645 	int i;
646 	u32 offset;
647 	u64 nr_cpu, type;
648 	struct perf_cpu_map *cpu_map;
649 	struct perf_cpu_map *event_cpus = session->evlist->core.cpus;
650 	struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
651 	struct cs_etm_recording *ptr =
652 			container_of(itr, struct cs_etm_recording, itr);
653 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
654 
655 	if (priv_size != cs_etm_info_priv_size(itr, session->evlist))
656 		return -EINVAL;
657 
658 	if (!session->evlist->core.nr_mmaps)
659 		return -EINVAL;
660 
661 	/* If the cpu_map is empty all online CPUs are involved */
662 	if (perf_cpu_map__empty(event_cpus)) {
663 		cpu_map = online_cpus;
664 	} else {
665 		/* Make sure all specified CPUs are online */
666 		for (i = 0; i < perf_cpu_map__nr(event_cpus); i++) {
667 			if (cpu_map__has(event_cpus, i) &&
668 			    !cpu_map__has(online_cpus, i))
669 				return -EINVAL;
670 		}
671 
672 		cpu_map = event_cpus;
673 	}
674 
675 	nr_cpu = perf_cpu_map__nr(cpu_map);
676 	/* Get PMU type as dynamically assigned by the core */
677 	type = cs_etm_pmu->type;
678 
679 	/* First fill out the session header */
680 	info->type = PERF_AUXTRACE_CS_ETM;
681 	info->priv[CS_HEADER_VERSION_0] = 0;
682 	info->priv[CS_PMU_TYPE_CPUS] = type << 32;
683 	info->priv[CS_PMU_TYPE_CPUS] |= nr_cpu;
684 	info->priv[CS_ETM_SNAPSHOT] = ptr->snapshot_mode;
685 
686 	offset = CS_ETM_SNAPSHOT + 1;
687 
688 	for (i = 0; i < cpu__max_cpu() && offset < priv_size; i++)
689 		if (cpu_map__has(cpu_map, i))
690 			cs_etm_get_metadata(i, &offset, itr, info);
691 
692 	perf_cpu_map__put(online_cpus);
693 
694 	return 0;
695 }
696 
697 static int cs_etm_alloc_wrapped_array(struct cs_etm_recording *ptr, int idx)
698 {
699 	bool *wrapped;
700 	int cnt = ptr->wrapped_cnt;
701 
702 	/* Make @ptr->wrapped as big as @idx */
703 	while (cnt <= idx)
704 		cnt++;
705 
706 	/*
707 	 * Free'ed in cs_etm_recording_free().  Using realloc() to avoid
708 	 * cross compilation problems where the host's system supports
709 	 * reallocarray() but not the target.
710 	 */
711 	wrapped = realloc(ptr->wrapped, cnt * sizeof(bool));
712 	if (!wrapped)
713 		return -ENOMEM;
714 
715 	wrapped[cnt - 1] = false;
716 	ptr->wrapped_cnt = cnt;
717 	ptr->wrapped = wrapped;
718 
719 	return 0;
720 }
721 
722 static bool cs_etm_buffer_has_wrapped(unsigned char *buffer,
723 				      size_t buffer_size, u64 head)
724 {
725 	u64 i, watermark;
726 	u64 *buf = (u64 *)buffer;
727 	size_t buf_size = buffer_size;
728 
729 	/*
730 	 * We want to look the very last 512 byte (chosen arbitrarily) in
731 	 * the ring buffer.
732 	 */
733 	watermark = buf_size - 512;
734 
735 	/*
736 	 * @head is continuously increasing - if its value is equal or greater
737 	 * than the size of the ring buffer, it has wrapped around.
738 	 */
739 	if (head >= buffer_size)
740 		return true;
741 
742 	/*
743 	 * The value of @head is somewhere within the size of the ring buffer.
744 	 * This can be that there hasn't been enough data to fill the ring
745 	 * buffer yet or the trace time was so long that @head has numerically
746 	 * wrapped around.  To find we need to check if we have data at the very
747 	 * end of the ring buffer.  We can reliably do this because mmap'ed
748 	 * pages are zeroed out and there is a fresh mapping with every new
749 	 * session.
750 	 */
751 
752 	/* @head is less than 512 byte from the end of the ring buffer */
753 	if (head > watermark)
754 		watermark = head;
755 
756 	/*
757 	 * Speed things up by using 64 bit transactions (see "u64 *buf" above)
758 	 */
759 	watermark >>= 3;
760 	buf_size >>= 3;
761 
762 	/*
763 	 * If we find trace data at the end of the ring buffer, @head has
764 	 * been there and has numerically wrapped around at least once.
765 	 */
766 	for (i = watermark; i < buf_size; i++)
767 		if (buf[i])
768 			return true;
769 
770 	return false;
771 }
772 
773 static int cs_etm_find_snapshot(struct auxtrace_record *itr,
774 				int idx, struct auxtrace_mmap *mm,
775 				unsigned char *data,
776 				u64 *head, u64 *old)
777 {
778 	int err;
779 	bool wrapped;
780 	struct cs_etm_recording *ptr =
781 			container_of(itr, struct cs_etm_recording, itr);
782 
783 	/*
784 	 * Allocate memory to keep track of wrapping if this is the first
785 	 * time we deal with this *mm.
786 	 */
787 	if (idx >= ptr->wrapped_cnt) {
788 		err = cs_etm_alloc_wrapped_array(ptr, idx);
789 		if (err)
790 			return err;
791 	}
792 
793 	/*
794 	 * Check to see if *head has wrapped around.  If it hasn't only the
795 	 * amount of data between *head and *old is snapshot'ed to avoid
796 	 * bloating the perf.data file with zeros.  But as soon as *head has
797 	 * wrapped around the entire size of the AUX ring buffer it taken.
798 	 */
799 	wrapped = ptr->wrapped[idx];
800 	if (!wrapped && cs_etm_buffer_has_wrapped(data, mm->len, *head)) {
801 		wrapped = true;
802 		ptr->wrapped[idx] = true;
803 	}
804 
805 	pr_debug3("%s: mmap index %d old head %zu new head %zu size %zu\n",
806 		  __func__, idx, (size_t)*old, (size_t)*head, mm->len);
807 
808 	/* No wrap has occurred, we can just use *head and *old. */
809 	if (!wrapped)
810 		return 0;
811 
812 	/*
813 	 * *head has wrapped around - adjust *head and *old to pickup the
814 	 * entire content of the AUX buffer.
815 	 */
816 	if (*head >= mm->len) {
817 		*old = *head - mm->len;
818 	} else {
819 		*head += mm->len;
820 		*old = *head - mm->len;
821 	}
822 
823 	return 0;
824 }
825 
826 static int cs_etm_snapshot_start(struct auxtrace_record *itr)
827 {
828 	struct cs_etm_recording *ptr =
829 			container_of(itr, struct cs_etm_recording, itr);
830 	struct evsel *evsel;
831 
832 	evlist__for_each_entry(ptr->evlist, evsel) {
833 		if (evsel->core.attr.type == ptr->cs_etm_pmu->type)
834 			return evsel__disable(evsel);
835 	}
836 	return -EINVAL;
837 }
838 
839 static int cs_etm_snapshot_finish(struct auxtrace_record *itr)
840 {
841 	struct cs_etm_recording *ptr =
842 			container_of(itr, struct cs_etm_recording, itr);
843 	struct evsel *evsel;
844 
845 	evlist__for_each_entry(ptr->evlist, evsel) {
846 		if (evsel->core.attr.type == ptr->cs_etm_pmu->type)
847 			return evsel__enable(evsel);
848 	}
849 	return -EINVAL;
850 }
851 
852 static u64 cs_etm_reference(struct auxtrace_record *itr __maybe_unused)
853 {
854 	return (((u64) rand() <<  0) & 0x00000000FFFFFFFFull) |
855 		(((u64) rand() << 32) & 0xFFFFFFFF00000000ull);
856 }
857 
858 static void cs_etm_recording_free(struct auxtrace_record *itr)
859 {
860 	struct cs_etm_recording *ptr =
861 			container_of(itr, struct cs_etm_recording, itr);
862 
863 	zfree(&ptr->wrapped);
864 	free(ptr);
865 }
866 
867 struct auxtrace_record *cs_etm_record_init(int *err)
868 {
869 	struct perf_pmu *cs_etm_pmu;
870 	struct cs_etm_recording *ptr;
871 
872 	cs_etm_pmu = perf_pmu__find(CORESIGHT_ETM_PMU_NAME);
873 
874 	if (!cs_etm_pmu) {
875 		*err = -EINVAL;
876 		goto out;
877 	}
878 
879 	ptr = zalloc(sizeof(struct cs_etm_recording));
880 	if (!ptr) {
881 		*err = -ENOMEM;
882 		goto out;
883 	}
884 
885 	ptr->cs_etm_pmu			= cs_etm_pmu;
886 	ptr->itr.pmu			= cs_etm_pmu;
887 	ptr->itr.parse_snapshot_options	= cs_etm_parse_snapshot_options;
888 	ptr->itr.recording_options	= cs_etm_recording_options;
889 	ptr->itr.info_priv_size		= cs_etm_info_priv_size;
890 	ptr->itr.info_fill		= cs_etm_info_fill;
891 	ptr->itr.find_snapshot		= cs_etm_find_snapshot;
892 	ptr->itr.snapshot_start		= cs_etm_snapshot_start;
893 	ptr->itr.snapshot_finish	= cs_etm_snapshot_finish;
894 	ptr->itr.reference		= cs_etm_reference;
895 	ptr->itr.free			= cs_etm_recording_free;
896 	ptr->itr.read_finish		= auxtrace_record__read_finish;
897 
898 	*err = 0;
899 	return &ptr->itr;
900 out:
901 	return NULL;
902 }
903