1 /* 2 * Copyright(C) 2015 Linaro Limited. All rights reserved. 3 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published by 7 * the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/coresight.h> 19 #include <linux/coresight-pmu.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/list.h> 23 #include <linux/mm.h> 24 #include <linux/init.h> 25 #include <linux/perf_event.h> 26 #include <linux/slab.h> 27 #include <linux/types.h> 28 #include <linux/workqueue.h> 29 30 #include "coresight-etm-perf.h" 31 #include "coresight-priv.h" 32 33 static struct pmu etm_pmu; 34 static bool etm_perf_up; 35 36 /** 37 * struct etm_event_data - Coresight specifics associated to an event 38 * @work: Handle to free allocated memory outside IRQ context. 39 * @mask: Hold the CPU(s) this event was set for. 40 * @snk_config: The sink configuration. 41 * @path: An array of path, each slot for one CPU. 42 */ 43 struct etm_event_data { 44 struct work_struct work; 45 cpumask_t mask; 46 void *snk_config; 47 struct list_head **path; 48 }; 49 50 static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); 51 static DEFINE_PER_CPU(struct coresight_device *, csdev_src); 52 53 /* ETMv3.5/PTM's ETMCR is 'config' */ 54 PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); 55 PMU_FORMAT_ATTR(timestamp, "config:" __stringify(ETM_OPT_TS)); 56 57 static struct attribute *etm_config_formats_attr[] = { 58 &format_attr_cycacc.attr, 59 &format_attr_timestamp.attr, 60 NULL, 61 }; 62 63 static struct attribute_group etm_pmu_format_group = { 64 .name = "format", 65 .attrs = etm_config_formats_attr, 66 }; 67 68 static const struct attribute_group *etm_pmu_attr_groups[] = { 69 &etm_pmu_format_group, 70 NULL, 71 }; 72 73 static void etm_event_read(struct perf_event *event) {} 74 75 static int etm_addr_filters_alloc(struct perf_event *event) 76 { 77 struct etm_filters *filters; 78 int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu); 79 80 filters = kzalloc_node(sizeof(struct etm_filters), GFP_KERNEL, node); 81 if (!filters) 82 return -ENOMEM; 83 84 if (event->parent) 85 memcpy(filters, event->parent->hw.addr_filters, 86 sizeof(*filters)); 87 88 event->hw.addr_filters = filters; 89 90 return 0; 91 } 92 93 static void etm_event_destroy(struct perf_event *event) 94 { 95 kfree(event->hw.addr_filters); 96 event->hw.addr_filters = NULL; 97 } 98 99 static int etm_event_init(struct perf_event *event) 100 { 101 int ret = 0; 102 103 if (event->attr.type != etm_pmu.type) { 104 ret = -ENOENT; 105 goto out; 106 } 107 108 ret = etm_addr_filters_alloc(event); 109 if (ret) 110 goto out; 111 112 event->destroy = etm_event_destroy; 113 out: 114 return ret; 115 } 116 117 static void free_event_data(struct work_struct *work) 118 { 119 int cpu; 120 cpumask_t *mask; 121 struct etm_event_data *event_data; 122 struct coresight_device *sink; 123 124 event_data = container_of(work, struct etm_event_data, work); 125 mask = &event_data->mask; 126 /* 127 * First deal with the sink configuration. See comment in 128 * etm_setup_aux() about why we take the first available path. 129 */ 130 if (event_data->snk_config) { 131 cpu = cpumask_first(mask); 132 sink = coresight_get_sink(event_data->path[cpu]); 133 if (sink_ops(sink)->free_buffer) 134 sink_ops(sink)->free_buffer(event_data->snk_config); 135 } 136 137 for_each_cpu(cpu, mask) { 138 if (!(IS_ERR_OR_NULL(event_data->path[cpu]))) 139 coresight_release_path(event_data->path[cpu]); 140 } 141 142 kfree(event_data->path); 143 kfree(event_data); 144 } 145 146 static void *alloc_event_data(int cpu) 147 { 148 int size; 149 cpumask_t *mask; 150 struct etm_event_data *event_data; 151 152 /* First get memory for the session's data */ 153 event_data = kzalloc(sizeof(struct etm_event_data), GFP_KERNEL); 154 if (!event_data) 155 return NULL; 156 157 /* Make sure nothing disappears under us */ 158 get_online_cpus(); 159 size = num_online_cpus(); 160 161 mask = &event_data->mask; 162 if (cpu != -1) 163 cpumask_set_cpu(cpu, mask); 164 else 165 cpumask_copy(mask, cpu_online_mask); 166 put_online_cpus(); 167 168 /* 169 * Each CPU has a single path between source and destination. As such 170 * allocate an array using CPU numbers as indexes. That way a path 171 * for any CPU can easily be accessed at any given time. We proceed 172 * the same way for sessions involving a single CPU. The cost of 173 * unused memory when dealing with single CPU trace scenarios is small 174 * compared to the cost of searching through an optimized array. 175 */ 176 event_data->path = kcalloc(size, 177 sizeof(struct list_head *), GFP_KERNEL); 178 if (!event_data->path) { 179 kfree(event_data); 180 return NULL; 181 } 182 183 return event_data; 184 } 185 186 static void etm_free_aux(void *data) 187 { 188 struct etm_event_data *event_data = data; 189 190 schedule_work(&event_data->work); 191 } 192 193 static void *etm_setup_aux(int event_cpu, void **pages, 194 int nr_pages, bool overwrite) 195 { 196 int cpu; 197 cpumask_t *mask; 198 struct coresight_device *sink; 199 struct etm_event_data *event_data = NULL; 200 201 event_data = alloc_event_data(event_cpu); 202 if (!event_data) 203 return NULL; 204 205 /* 206 * In theory nothing prevent tracers in a trace session from being 207 * associated with different sinks, nor having a sink per tracer. But 208 * until we have HW with this kind of topology we need to assume tracers 209 * in a trace session are using the same sink. Therefore go through 210 * the coresight bus and pick the first enabled sink. 211 * 212 * When operated from sysFS users are responsible to enable the sink 213 * while from perf, the perf tools will do it based on the choice made 214 * on the cmd line. As such the "enable_sink" flag in sysFS is reset. 215 */ 216 sink = coresight_get_enabled_sink(true); 217 if (!sink) 218 goto err; 219 220 INIT_WORK(&event_data->work, free_event_data); 221 222 mask = &event_data->mask; 223 224 /* Setup the path for each CPU in a trace session */ 225 for_each_cpu(cpu, mask) { 226 struct coresight_device *csdev; 227 228 csdev = per_cpu(csdev_src, cpu); 229 if (!csdev) 230 goto err; 231 232 /* 233 * Building a path doesn't enable it, it simply builds a 234 * list of devices from source to sink that can be 235 * referenced later when the path is actually needed. 236 */ 237 event_data->path[cpu] = coresight_build_path(csdev, sink); 238 if (IS_ERR(event_data->path[cpu])) 239 goto err; 240 } 241 242 if (!sink_ops(sink)->alloc_buffer) 243 goto err; 244 245 cpu = cpumask_first(mask); 246 /* Get the AUX specific data from the sink buffer */ 247 event_data->snk_config = 248 sink_ops(sink)->alloc_buffer(sink, cpu, pages, 249 nr_pages, overwrite); 250 if (!event_data->snk_config) 251 goto err; 252 253 out: 254 return event_data; 255 256 err: 257 etm_free_aux(event_data); 258 event_data = NULL; 259 goto out; 260 } 261 262 static void etm_event_start(struct perf_event *event, int flags) 263 { 264 int cpu = smp_processor_id(); 265 struct etm_event_data *event_data; 266 struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); 267 struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); 268 269 if (!csdev) 270 goto fail; 271 272 /* 273 * Deal with the ring buffer API and get a handle on the 274 * session's information. 275 */ 276 event_data = perf_aux_output_begin(handle, event); 277 if (!event_data) 278 goto fail; 279 280 /* We need a sink, no need to continue without one */ 281 sink = coresight_get_sink(event_data->path[cpu]); 282 if (WARN_ON_ONCE(!sink || !sink_ops(sink)->set_buffer)) 283 goto fail_end_stop; 284 285 /* Configure the sink */ 286 if (sink_ops(sink)->set_buffer(sink, handle, 287 event_data->snk_config)) 288 goto fail_end_stop; 289 290 /* Nothing will happen without a path */ 291 if (coresight_enable_path(event_data->path[cpu], CS_MODE_PERF)) 292 goto fail_end_stop; 293 294 /* Tell the perf core the event is alive */ 295 event->hw.state = 0; 296 297 /* Finally enable the tracer */ 298 if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF)) 299 goto fail_end_stop; 300 301 out: 302 return; 303 304 fail_end_stop: 305 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 306 perf_aux_output_end(handle, 0); 307 fail: 308 event->hw.state = PERF_HES_STOPPED; 309 goto out; 310 } 311 312 static void etm_event_stop(struct perf_event *event, int mode) 313 { 314 int cpu = smp_processor_id(); 315 unsigned long size; 316 struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); 317 struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); 318 struct etm_event_data *event_data = perf_get_aux(handle); 319 320 if (event->hw.state == PERF_HES_STOPPED) 321 return; 322 323 if (!csdev) 324 return; 325 326 sink = coresight_get_sink(event_data->path[cpu]); 327 if (!sink) 328 return; 329 330 /* stop tracer */ 331 source_ops(csdev)->disable(csdev, event); 332 333 /* tell the core */ 334 event->hw.state = PERF_HES_STOPPED; 335 336 if (mode & PERF_EF_UPDATE) { 337 if (WARN_ON_ONCE(handle->event != event)) 338 return; 339 340 /* update trace information */ 341 if (!sink_ops(sink)->update_buffer) 342 return; 343 344 sink_ops(sink)->update_buffer(sink, handle, 345 event_data->snk_config); 346 347 if (!sink_ops(sink)->reset_buffer) 348 return; 349 350 size = sink_ops(sink)->reset_buffer(sink, handle, 351 event_data->snk_config); 352 353 perf_aux_output_end(handle, size); 354 } 355 356 /* Disabling the path make its elements available to other sessions */ 357 coresight_disable_path(event_data->path[cpu]); 358 } 359 360 static int etm_event_add(struct perf_event *event, int mode) 361 { 362 int ret = 0; 363 struct hw_perf_event *hwc = &event->hw; 364 365 if (mode & PERF_EF_START) { 366 etm_event_start(event, 0); 367 if (hwc->state & PERF_HES_STOPPED) 368 ret = -EINVAL; 369 } else { 370 hwc->state = PERF_HES_STOPPED; 371 } 372 373 return ret; 374 } 375 376 static void etm_event_del(struct perf_event *event, int mode) 377 { 378 etm_event_stop(event, PERF_EF_UPDATE); 379 } 380 381 static int etm_addr_filters_validate(struct list_head *filters) 382 { 383 bool range = false, address = false; 384 int index = 0; 385 struct perf_addr_filter *filter; 386 387 list_for_each_entry(filter, filters, entry) { 388 /* 389 * No need to go further if there's no more 390 * room for filters. 391 */ 392 if (++index > ETM_ADDR_CMP_MAX) 393 return -EOPNOTSUPP; 394 395 /* 396 * As taken from the struct perf_addr_filter documentation: 397 * @range: 1: range, 0: address 398 * 399 * At this time we don't allow range and start/stop filtering 400 * to cohabitate, they have to be mutually exclusive. 401 */ 402 if ((filter->range == 1) && address) 403 return -EOPNOTSUPP; 404 405 if ((filter->range == 0) && range) 406 return -EOPNOTSUPP; 407 408 /* 409 * For range filtering, the second address in the address 410 * range comparator needs to be higher than the first. 411 * Invalid otherwise. 412 */ 413 if (filter->range && filter->size == 0) 414 return -EINVAL; 415 416 /* 417 * Everything checks out with this filter, record what we've 418 * received before moving on to the next one. 419 */ 420 if (filter->range) 421 range = true; 422 else 423 address = true; 424 } 425 426 return 0; 427 } 428 429 static void etm_addr_filters_sync(struct perf_event *event) 430 { 431 struct perf_addr_filters_head *head = perf_event_addr_filters(event); 432 unsigned long start, stop, *offs = event->addr_filters_offs; 433 struct etm_filters *filters = event->hw.addr_filters; 434 struct etm_filter *etm_filter; 435 struct perf_addr_filter *filter; 436 int i = 0; 437 438 list_for_each_entry(filter, &head->list, entry) { 439 start = filter->offset + offs[i]; 440 stop = start + filter->size; 441 etm_filter = &filters->etm_filter[i]; 442 443 if (filter->range == 1) { 444 etm_filter->start_addr = start; 445 etm_filter->stop_addr = stop; 446 etm_filter->type = ETM_ADDR_TYPE_RANGE; 447 } else { 448 if (filter->filter == 1) { 449 etm_filter->start_addr = start; 450 etm_filter->type = ETM_ADDR_TYPE_START; 451 } else { 452 etm_filter->stop_addr = stop; 453 etm_filter->type = ETM_ADDR_TYPE_STOP; 454 } 455 } 456 i++; 457 } 458 459 filters->nr_filters = i; 460 } 461 462 int etm_perf_symlink(struct coresight_device *csdev, bool link) 463 { 464 char entry[sizeof("cpu9999999")]; 465 int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev); 466 struct device *pmu_dev = etm_pmu.dev; 467 struct device *cs_dev = &csdev->dev; 468 469 sprintf(entry, "cpu%d", cpu); 470 471 if (!etm_perf_up) 472 return -EPROBE_DEFER; 473 474 if (link) { 475 ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry); 476 if (ret) 477 return ret; 478 per_cpu(csdev_src, cpu) = csdev; 479 } else { 480 sysfs_remove_link(&pmu_dev->kobj, entry); 481 per_cpu(csdev_src, cpu) = NULL; 482 } 483 484 return 0; 485 } 486 487 static int __init etm_perf_init(void) 488 { 489 int ret; 490 491 etm_pmu.capabilities = PERF_PMU_CAP_EXCLUSIVE; 492 493 etm_pmu.attr_groups = etm_pmu_attr_groups; 494 etm_pmu.task_ctx_nr = perf_sw_context; 495 etm_pmu.read = etm_event_read; 496 etm_pmu.event_init = etm_event_init; 497 etm_pmu.setup_aux = etm_setup_aux; 498 etm_pmu.free_aux = etm_free_aux; 499 etm_pmu.start = etm_event_start; 500 etm_pmu.stop = etm_event_stop; 501 etm_pmu.add = etm_event_add; 502 etm_pmu.del = etm_event_del; 503 etm_pmu.addr_filters_sync = etm_addr_filters_sync; 504 etm_pmu.addr_filters_validate = etm_addr_filters_validate; 505 etm_pmu.nr_addr_filters = ETM_ADDR_CMP_MAX; 506 507 ret = perf_pmu_register(&etm_pmu, CORESIGHT_ETM_PMU_NAME, -1); 508 if (ret == 0) 509 etm_perf_up = true; 510 511 return ret; 512 } 513 device_initcall(etm_perf_init); 514