1 /* 2 * Copyright(C) 2015 Linaro Limited. All rights reserved. 3 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published by 7 * the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/coresight.h> 19 #include <linux/coresight-pmu.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/list.h> 23 #include <linux/mm.h> 24 #include <linux/init.h> 25 #include <linux/perf_event.h> 26 #include <linux/slab.h> 27 #include <linux/types.h> 28 #include <linux/workqueue.h> 29 30 #include "coresight-etm-perf.h" 31 #include "coresight-priv.h" 32 33 static struct pmu etm_pmu; 34 static bool etm_perf_up; 35 36 /** 37 * struct etm_event_data - Coresight specifics associated to an event 38 * @work: Handle to free allocated memory outside IRQ context. 39 * @mask: Hold the CPU(s) this event was set for. 40 * @snk_config: The sink configuration. 41 * @path: An array of path, each slot for one CPU. 42 */ 43 struct etm_event_data { 44 struct work_struct work; 45 cpumask_t mask; 46 void *snk_config; 47 struct list_head **path; 48 }; 49 50 static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); 51 static DEFINE_PER_CPU(struct coresight_device *, csdev_src); 52 53 /* ETMv3.5/PTM's ETMCR is 'config' */ 54 PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); 55 PMU_FORMAT_ATTR(timestamp, "config:" __stringify(ETM_OPT_TS)); 56 57 static struct attribute *etm_config_formats_attr[] = { 58 &format_attr_cycacc.attr, 59 &format_attr_timestamp.attr, 60 NULL, 61 }; 62 63 static struct attribute_group etm_pmu_format_group = { 64 .name = "format", 65 .attrs = etm_config_formats_attr, 66 }; 67 68 static const struct attribute_group *etm_pmu_attr_groups[] = { 69 &etm_pmu_format_group, 70 NULL, 71 }; 72 73 static void etm_event_read(struct perf_event *event) {} 74 75 static int etm_addr_filters_alloc(struct perf_event *event) 76 { 77 struct etm_filters *filters; 78 int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu); 79 80 filters = kzalloc_node(sizeof(struct etm_filters), GFP_KERNEL, node); 81 if (!filters) 82 return -ENOMEM; 83 84 if (event->parent) 85 memcpy(filters, event->parent->hw.addr_filters, 86 sizeof(*filters)); 87 88 event->hw.addr_filters = filters; 89 90 return 0; 91 } 92 93 static void etm_event_destroy(struct perf_event *event) 94 { 95 kfree(event->hw.addr_filters); 96 event->hw.addr_filters = NULL; 97 } 98 99 static int etm_event_init(struct perf_event *event) 100 { 101 int ret = 0; 102 103 if (event->attr.type != etm_pmu.type) { 104 ret = -ENOENT; 105 goto out; 106 } 107 108 ret = etm_addr_filters_alloc(event); 109 if (ret) 110 goto out; 111 112 event->destroy = etm_event_destroy; 113 out: 114 return ret; 115 } 116 117 static void free_event_data(struct work_struct *work) 118 { 119 int cpu; 120 cpumask_t *mask; 121 struct etm_event_data *event_data; 122 struct coresight_device *sink; 123 124 event_data = container_of(work, struct etm_event_data, work); 125 mask = &event_data->mask; 126 /* 127 * First deal with the sink configuration. See comment in 128 * etm_setup_aux() about why we take the first available path. 129 */ 130 if (event_data->snk_config) { 131 cpu = cpumask_first(mask); 132 sink = coresight_get_sink(event_data->path[cpu]); 133 if (sink_ops(sink)->free_buffer) 134 sink_ops(sink)->free_buffer(event_data->snk_config); 135 } 136 137 for_each_cpu(cpu, mask) { 138 if (!(IS_ERR_OR_NULL(event_data->path[cpu]))) 139 coresight_release_path(event_data->path[cpu]); 140 } 141 142 kfree(event_data->path); 143 kfree(event_data); 144 } 145 146 static void *alloc_event_data(int cpu) 147 { 148 int size; 149 cpumask_t *mask; 150 struct etm_event_data *event_data; 151 152 /* First get memory for the session's data */ 153 event_data = kzalloc(sizeof(struct etm_event_data), GFP_KERNEL); 154 if (!event_data) 155 return NULL; 156 157 /* Make sure nothing disappears under us */ 158 get_online_cpus(); 159 size = num_online_cpus(); 160 161 mask = &event_data->mask; 162 if (cpu != -1) 163 cpumask_set_cpu(cpu, mask); 164 else 165 cpumask_copy(mask, cpu_online_mask); 166 put_online_cpus(); 167 168 /* 169 * Each CPU has a single path between source and destination. As such 170 * allocate an array using CPU numbers as indexes. That way a path 171 * for any CPU can easily be accessed at any given time. We proceed 172 * the same way for sessions involving a single CPU. The cost of 173 * unused memory when dealing with single CPU trace scenarios is small 174 * compared to the cost of searching through an optimized array. 175 */ 176 event_data->path = kcalloc(size, 177 sizeof(struct list_head *), GFP_KERNEL); 178 if (!event_data->path) { 179 kfree(event_data); 180 return NULL; 181 } 182 183 return event_data; 184 } 185 186 static void etm_free_aux(void *data) 187 { 188 struct etm_event_data *event_data = data; 189 190 schedule_work(&event_data->work); 191 } 192 193 static void *etm_setup_aux(int event_cpu, void **pages, 194 int nr_pages, bool overwrite) 195 { 196 int cpu; 197 cpumask_t *mask; 198 struct coresight_device *sink; 199 struct etm_event_data *event_data = NULL; 200 201 event_data = alloc_event_data(event_cpu); 202 if (!event_data) 203 return NULL; 204 INIT_WORK(&event_data->work, free_event_data); 205 206 /* 207 * In theory nothing prevent tracers in a trace session from being 208 * associated with different sinks, nor having a sink per tracer. But 209 * until we have HW with this kind of topology we need to assume tracers 210 * in a trace session are using the same sink. Therefore go through 211 * the coresight bus and pick the first enabled sink. 212 * 213 * When operated from sysFS users are responsible to enable the sink 214 * while from perf, the perf tools will do it based on the choice made 215 * on the cmd line. As such the "enable_sink" flag in sysFS is reset. 216 */ 217 sink = coresight_get_enabled_sink(true); 218 if (!sink) 219 goto err; 220 221 mask = &event_data->mask; 222 223 /* Setup the path for each CPU in a trace session */ 224 for_each_cpu(cpu, mask) { 225 struct coresight_device *csdev; 226 227 csdev = per_cpu(csdev_src, cpu); 228 if (!csdev) 229 goto err; 230 231 /* 232 * Building a path doesn't enable it, it simply builds a 233 * list of devices from source to sink that can be 234 * referenced later when the path is actually needed. 235 */ 236 event_data->path[cpu] = coresight_build_path(csdev, sink); 237 if (IS_ERR(event_data->path[cpu])) 238 goto err; 239 } 240 241 if (!sink_ops(sink)->alloc_buffer) 242 goto err; 243 244 cpu = cpumask_first(mask); 245 /* Get the AUX specific data from the sink buffer */ 246 event_data->snk_config = 247 sink_ops(sink)->alloc_buffer(sink, cpu, pages, 248 nr_pages, overwrite); 249 if (!event_data->snk_config) 250 goto err; 251 252 out: 253 return event_data; 254 255 err: 256 etm_free_aux(event_data); 257 event_data = NULL; 258 goto out; 259 } 260 261 static void etm_event_start(struct perf_event *event, int flags) 262 { 263 int cpu = smp_processor_id(); 264 struct etm_event_data *event_data; 265 struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); 266 struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); 267 268 if (!csdev) 269 goto fail; 270 271 /* 272 * Deal with the ring buffer API and get a handle on the 273 * session's information. 274 */ 275 event_data = perf_aux_output_begin(handle, event); 276 if (!event_data) 277 goto fail; 278 279 /* We need a sink, no need to continue without one */ 280 sink = coresight_get_sink(event_data->path[cpu]); 281 if (WARN_ON_ONCE(!sink || !sink_ops(sink)->set_buffer)) 282 goto fail_end_stop; 283 284 /* Configure the sink */ 285 if (sink_ops(sink)->set_buffer(sink, handle, 286 event_data->snk_config)) 287 goto fail_end_stop; 288 289 /* Nothing will happen without a path */ 290 if (coresight_enable_path(event_data->path[cpu], CS_MODE_PERF)) 291 goto fail_end_stop; 292 293 /* Tell the perf core the event is alive */ 294 event->hw.state = 0; 295 296 /* Finally enable the tracer */ 297 if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF)) 298 goto fail_end_stop; 299 300 out: 301 return; 302 303 fail_end_stop: 304 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 305 perf_aux_output_end(handle, 0); 306 fail: 307 event->hw.state = PERF_HES_STOPPED; 308 goto out; 309 } 310 311 static void etm_event_stop(struct perf_event *event, int mode) 312 { 313 int cpu = smp_processor_id(); 314 unsigned long size; 315 struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); 316 struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); 317 struct etm_event_data *event_data = perf_get_aux(handle); 318 319 if (event->hw.state == PERF_HES_STOPPED) 320 return; 321 322 if (!csdev) 323 return; 324 325 sink = coresight_get_sink(event_data->path[cpu]); 326 if (!sink) 327 return; 328 329 /* stop tracer */ 330 source_ops(csdev)->disable(csdev, event); 331 332 /* tell the core */ 333 event->hw.state = PERF_HES_STOPPED; 334 335 if (mode & PERF_EF_UPDATE) { 336 if (WARN_ON_ONCE(handle->event != event)) 337 return; 338 339 /* update trace information */ 340 if (!sink_ops(sink)->update_buffer) 341 return; 342 343 sink_ops(sink)->update_buffer(sink, handle, 344 event_data->snk_config); 345 346 if (!sink_ops(sink)->reset_buffer) 347 return; 348 349 size = sink_ops(sink)->reset_buffer(sink, handle, 350 event_data->snk_config); 351 352 perf_aux_output_end(handle, size); 353 } 354 355 /* Disabling the path make its elements available to other sessions */ 356 coresight_disable_path(event_data->path[cpu]); 357 } 358 359 static int etm_event_add(struct perf_event *event, int mode) 360 { 361 int ret = 0; 362 struct hw_perf_event *hwc = &event->hw; 363 364 if (mode & PERF_EF_START) { 365 etm_event_start(event, 0); 366 if (hwc->state & PERF_HES_STOPPED) 367 ret = -EINVAL; 368 } else { 369 hwc->state = PERF_HES_STOPPED; 370 } 371 372 return ret; 373 } 374 375 static void etm_event_del(struct perf_event *event, int mode) 376 { 377 etm_event_stop(event, PERF_EF_UPDATE); 378 } 379 380 static int etm_addr_filters_validate(struct list_head *filters) 381 { 382 bool range = false, address = false; 383 int index = 0; 384 struct perf_addr_filter *filter; 385 386 list_for_each_entry(filter, filters, entry) { 387 /* 388 * No need to go further if there's no more 389 * room for filters. 390 */ 391 if (++index > ETM_ADDR_CMP_MAX) 392 return -EOPNOTSUPP; 393 394 /* 395 * As taken from the struct perf_addr_filter documentation: 396 * @range: 1: range, 0: address 397 * 398 * At this time we don't allow range and start/stop filtering 399 * to cohabitate, they have to be mutually exclusive. 400 */ 401 if ((filter->range == 1) && address) 402 return -EOPNOTSUPP; 403 404 if ((filter->range == 0) && range) 405 return -EOPNOTSUPP; 406 407 /* 408 * For range filtering, the second address in the address 409 * range comparator needs to be higher than the first. 410 * Invalid otherwise. 411 */ 412 if (filter->range && filter->size == 0) 413 return -EINVAL; 414 415 /* 416 * Everything checks out with this filter, record what we've 417 * received before moving on to the next one. 418 */ 419 if (filter->range) 420 range = true; 421 else 422 address = true; 423 } 424 425 return 0; 426 } 427 428 static void etm_addr_filters_sync(struct perf_event *event) 429 { 430 struct perf_addr_filters_head *head = perf_event_addr_filters(event); 431 unsigned long start, stop, *offs = event->addr_filters_offs; 432 struct etm_filters *filters = event->hw.addr_filters; 433 struct etm_filter *etm_filter; 434 struct perf_addr_filter *filter; 435 int i = 0; 436 437 list_for_each_entry(filter, &head->list, entry) { 438 start = filter->offset + offs[i]; 439 stop = start + filter->size; 440 etm_filter = &filters->etm_filter[i]; 441 442 if (filter->range == 1) { 443 etm_filter->start_addr = start; 444 etm_filter->stop_addr = stop; 445 etm_filter->type = ETM_ADDR_TYPE_RANGE; 446 } else { 447 if (filter->filter == 1) { 448 etm_filter->start_addr = start; 449 etm_filter->type = ETM_ADDR_TYPE_START; 450 } else { 451 etm_filter->stop_addr = stop; 452 etm_filter->type = ETM_ADDR_TYPE_STOP; 453 } 454 } 455 i++; 456 } 457 458 filters->nr_filters = i; 459 } 460 461 int etm_perf_symlink(struct coresight_device *csdev, bool link) 462 { 463 char entry[sizeof("cpu9999999")]; 464 int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev); 465 struct device *pmu_dev = etm_pmu.dev; 466 struct device *cs_dev = &csdev->dev; 467 468 sprintf(entry, "cpu%d", cpu); 469 470 if (!etm_perf_up) 471 return -EPROBE_DEFER; 472 473 if (link) { 474 ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry); 475 if (ret) 476 return ret; 477 per_cpu(csdev_src, cpu) = csdev; 478 } else { 479 sysfs_remove_link(&pmu_dev->kobj, entry); 480 per_cpu(csdev_src, cpu) = NULL; 481 } 482 483 return 0; 484 } 485 486 static int __init etm_perf_init(void) 487 { 488 int ret; 489 490 etm_pmu.capabilities = PERF_PMU_CAP_EXCLUSIVE; 491 492 etm_pmu.attr_groups = etm_pmu_attr_groups; 493 etm_pmu.task_ctx_nr = perf_sw_context; 494 etm_pmu.read = etm_event_read; 495 etm_pmu.event_init = etm_event_init; 496 etm_pmu.setup_aux = etm_setup_aux; 497 etm_pmu.free_aux = etm_free_aux; 498 etm_pmu.start = etm_event_start; 499 etm_pmu.stop = etm_event_stop; 500 etm_pmu.add = etm_event_add; 501 etm_pmu.del = etm_event_del; 502 etm_pmu.addr_filters_sync = etm_addr_filters_sync; 503 etm_pmu.addr_filters_validate = etm_addr_filters_validate; 504 etm_pmu.nr_addr_filters = ETM_ADDR_CMP_MAX; 505 506 ret = perf_pmu_register(&etm_pmu, CORESIGHT_ETM_PMU_NAME, -1); 507 if (ret == 0) 508 etm_perf_up = true; 509 510 return ret; 511 } 512 device_initcall(etm_perf_init); 513