1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2016 Linaro Limited. All rights reserved. 4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 5 */ 6 7 #include <linux/atomic.h> 8 #include <linux/circ_buf.h> 9 #include <linux/coresight.h> 10 #include <linux/perf_event.h> 11 #include <linux/slab.h> 12 #include "coresight-priv.h" 13 #include "coresight-tmc.h" 14 #include "coresight-etm-perf.h" 15 16 static int tmc_set_etf_buffer(struct coresight_device *csdev, 17 struct perf_output_handle *handle); 18 19 static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) 20 { 21 CS_UNLOCK(drvdata->base); 22 23 /* Wait for TMCSReady bit to be set */ 24 tmc_wait_for_tmcready(drvdata); 25 26 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); 27 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | 28 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | 29 TMC_FFCR_TRIGON_TRIGIN, 30 drvdata->base + TMC_FFCR); 31 32 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); 33 tmc_enable_hw(drvdata); 34 35 CS_LOCK(drvdata->base); 36 } 37 38 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) 39 { 40 int rc = coresight_claim_device(drvdata->base); 41 42 if (rc) 43 return rc; 44 45 __tmc_etb_enable_hw(drvdata); 46 return 0; 47 } 48 49 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) 50 { 51 char *bufp; 52 u32 read_data, lost; 53 54 /* Check if the buffer wrapped around. */ 55 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL; 56 bufp = drvdata->buf; 57 drvdata->len = 0; 58 while (1) { 59 read_data = readl_relaxed(drvdata->base + TMC_RRD); 60 if (read_data == 0xFFFFFFFF) 61 break; 62 memcpy(bufp, &read_data, 4); 63 bufp += 4; 64 drvdata->len += 4; 65 } 66 67 if (lost) 68 coresight_insert_barrier_packet(drvdata->buf); 69 return; 70 } 71 72 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) 73 { 74 CS_UNLOCK(drvdata->base); 75 76 tmc_flush_and_stop(drvdata); 77 /* 78 * When operating in sysFS mode the content of the buffer needs to be 79 * read before the TMC is disabled. 80 */ 81 if (drvdata->mode == CS_MODE_SYSFS) 82 tmc_etb_dump_hw(drvdata); 83 tmc_disable_hw(drvdata); 84 85 CS_LOCK(drvdata->base); 86 } 87 88 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) 89 { 90 __tmc_etb_disable_hw(drvdata); 91 coresight_disclaim_device(drvdata->base); 92 } 93 94 static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) 95 { 96 CS_UNLOCK(drvdata->base); 97 98 /* Wait for TMCSReady bit to be set */ 99 tmc_wait_for_tmcready(drvdata); 100 101 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE); 102 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI, 103 drvdata->base + TMC_FFCR); 104 writel_relaxed(0x0, drvdata->base + TMC_BUFWM); 105 tmc_enable_hw(drvdata); 106 107 CS_LOCK(drvdata->base); 108 } 109 110 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) 111 { 112 int rc = coresight_claim_device(drvdata->base); 113 114 if (rc) 115 return rc; 116 117 __tmc_etf_enable_hw(drvdata); 118 return 0; 119 } 120 121 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) 122 { 123 CS_UNLOCK(drvdata->base); 124 125 tmc_flush_and_stop(drvdata); 126 tmc_disable_hw(drvdata); 127 coresight_disclaim_device_unlocked(drvdata->base); 128 CS_LOCK(drvdata->base); 129 } 130 131 /* 132 * Return the available trace data in the buffer from @pos, with 133 * a maximum limit of @len, updating the @bufpp on where to 134 * find it. 135 */ 136 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata, 137 loff_t pos, size_t len, char **bufpp) 138 { 139 ssize_t actual = len; 140 141 /* Adjust the len to available size @pos */ 142 if (pos + actual > drvdata->len) 143 actual = drvdata->len - pos; 144 if (actual > 0) 145 *bufpp = drvdata->buf + pos; 146 return actual; 147 } 148 149 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) 150 { 151 int ret = 0; 152 bool used = false; 153 char *buf = NULL; 154 unsigned long flags; 155 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 156 157 /* 158 * If we don't have a buffer release the lock and allocate memory. 159 * Otherwise keep the lock and move along. 160 */ 161 spin_lock_irqsave(&drvdata->spinlock, flags); 162 if (!drvdata->buf) { 163 spin_unlock_irqrestore(&drvdata->spinlock, flags); 164 165 /* Allocating the memory here while outside of the spinlock */ 166 buf = kzalloc(drvdata->size, GFP_KERNEL); 167 if (!buf) 168 return -ENOMEM; 169 170 /* Let's try again */ 171 spin_lock_irqsave(&drvdata->spinlock, flags); 172 } 173 174 if (drvdata->reading) { 175 ret = -EBUSY; 176 goto out; 177 } 178 179 /* 180 * In sysFS mode we can have multiple writers per sink. Since this 181 * sink is already enabled no memory is needed and the HW need not be 182 * touched. 183 */ 184 if (drvdata->mode == CS_MODE_SYSFS) { 185 atomic_inc(csdev->refcnt); 186 goto out; 187 } 188 189 /* 190 * If drvdata::buf isn't NULL, memory was allocated for a previous 191 * trace run but wasn't read. If so simply zero-out the memory. 192 * Otherwise use the memory allocated above. 193 * 194 * The memory is freed when users read the buffer using the 195 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for 196 * details. 197 */ 198 if (drvdata->buf) { 199 memset(drvdata->buf, 0, drvdata->size); 200 } else { 201 used = true; 202 drvdata->buf = buf; 203 } 204 205 ret = tmc_etb_enable_hw(drvdata); 206 if (!ret) { 207 drvdata->mode = CS_MODE_SYSFS; 208 atomic_inc(csdev->refcnt); 209 } else { 210 /* Free up the buffer if we failed to enable */ 211 used = false; 212 } 213 out: 214 spin_unlock_irqrestore(&drvdata->spinlock, flags); 215 216 /* Free memory outside the spinlock if need be */ 217 if (!used) 218 kfree(buf); 219 220 return ret; 221 } 222 223 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) 224 { 225 int ret = 0; 226 pid_t pid; 227 unsigned long flags; 228 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 229 struct perf_output_handle *handle = data; 230 231 spin_lock_irqsave(&drvdata->spinlock, flags); 232 do { 233 ret = -EINVAL; 234 if (drvdata->reading) 235 break; 236 /* 237 * No need to continue if the ETB/ETF is already operated 238 * from sysFS. 239 */ 240 if (drvdata->mode == CS_MODE_SYSFS) { 241 ret = -EBUSY; 242 break; 243 } 244 245 /* Get a handle on the pid of the process to monitor */ 246 pid = task_pid_nr(handle->event->owner); 247 248 if (drvdata->pid != -1 && drvdata->pid != pid) { 249 ret = -EBUSY; 250 break; 251 } 252 253 ret = tmc_set_etf_buffer(csdev, handle); 254 if (ret) 255 break; 256 257 /* 258 * No HW configuration is needed if the sink is already in 259 * use for this session. 260 */ 261 if (drvdata->pid == pid) { 262 atomic_inc(csdev->refcnt); 263 break; 264 } 265 266 ret = tmc_etb_enable_hw(drvdata); 267 if (!ret) { 268 /* Associate with monitored process. */ 269 drvdata->pid = pid; 270 drvdata->mode = CS_MODE_PERF; 271 atomic_inc(csdev->refcnt); 272 } 273 } while (0); 274 spin_unlock_irqrestore(&drvdata->spinlock, flags); 275 276 return ret; 277 } 278 279 static int tmc_enable_etf_sink(struct coresight_device *csdev, 280 u32 mode, void *data) 281 { 282 int ret; 283 284 switch (mode) { 285 case CS_MODE_SYSFS: 286 ret = tmc_enable_etf_sink_sysfs(csdev); 287 break; 288 case CS_MODE_PERF: 289 ret = tmc_enable_etf_sink_perf(csdev, data); 290 break; 291 /* We shouldn't be here */ 292 default: 293 ret = -EINVAL; 294 break; 295 } 296 297 if (ret) 298 return ret; 299 300 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n"); 301 return 0; 302 } 303 304 static int tmc_disable_etf_sink(struct coresight_device *csdev) 305 { 306 unsigned long flags; 307 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 308 309 spin_lock_irqsave(&drvdata->spinlock, flags); 310 311 if (drvdata->reading) { 312 spin_unlock_irqrestore(&drvdata->spinlock, flags); 313 return -EBUSY; 314 } 315 316 if (atomic_dec_return(csdev->refcnt)) { 317 spin_unlock_irqrestore(&drvdata->spinlock, flags); 318 return -EBUSY; 319 } 320 321 /* Complain if we (somehow) got out of sync */ 322 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); 323 tmc_etb_disable_hw(drvdata); 324 /* Dissociate from monitored process. */ 325 drvdata->pid = -1; 326 drvdata->mode = CS_MODE_DISABLED; 327 328 spin_unlock_irqrestore(&drvdata->spinlock, flags); 329 330 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n"); 331 return 0; 332 } 333 334 static int tmc_enable_etf_link(struct coresight_device *csdev, 335 int inport, int outport) 336 { 337 int ret; 338 unsigned long flags; 339 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 340 341 spin_lock_irqsave(&drvdata->spinlock, flags); 342 if (drvdata->reading) { 343 spin_unlock_irqrestore(&drvdata->spinlock, flags); 344 return -EBUSY; 345 } 346 347 ret = tmc_etf_enable_hw(drvdata); 348 if (!ret) 349 drvdata->mode = CS_MODE_SYSFS; 350 spin_unlock_irqrestore(&drvdata->spinlock, flags); 351 352 if (!ret) 353 dev_dbg(&csdev->dev, "TMC-ETF enabled\n"); 354 return ret; 355 } 356 357 static void tmc_disable_etf_link(struct coresight_device *csdev, 358 int inport, int outport) 359 { 360 unsigned long flags; 361 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 362 363 spin_lock_irqsave(&drvdata->spinlock, flags); 364 if (drvdata->reading) { 365 spin_unlock_irqrestore(&drvdata->spinlock, flags); 366 return; 367 } 368 369 tmc_etf_disable_hw(drvdata); 370 drvdata->mode = CS_MODE_DISABLED; 371 spin_unlock_irqrestore(&drvdata->spinlock, flags); 372 373 dev_dbg(&csdev->dev, "TMC-ETF disabled\n"); 374 } 375 376 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, 377 struct perf_event *event, void **pages, 378 int nr_pages, bool overwrite) 379 { 380 int node, cpu = event->cpu; 381 struct cs_buffers *buf; 382 383 if (cpu == -1) 384 cpu = smp_processor_id(); 385 node = cpu_to_node(cpu); 386 387 /* Allocate memory structure for interaction with Perf */ 388 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node); 389 if (!buf) 390 return NULL; 391 392 buf->snapshot = overwrite; 393 buf->nr_pages = nr_pages; 394 buf->data_pages = pages; 395 396 return buf; 397 } 398 399 static void tmc_free_etf_buffer(void *config) 400 { 401 struct cs_buffers *buf = config; 402 403 kfree(buf); 404 } 405 406 static int tmc_set_etf_buffer(struct coresight_device *csdev, 407 struct perf_output_handle *handle) 408 { 409 int ret = 0; 410 unsigned long head; 411 struct cs_buffers *buf = etm_perf_sink_config(handle); 412 413 if (!buf) 414 return -EINVAL; 415 416 /* wrap head around to the amount of space we have */ 417 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); 418 419 /* find the page to write to */ 420 buf->cur = head / PAGE_SIZE; 421 422 /* and offset within that page */ 423 buf->offset = head % PAGE_SIZE; 424 425 local_set(&buf->data_size, 0); 426 427 return ret; 428 } 429 430 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, 431 struct perf_output_handle *handle, 432 void *sink_config) 433 { 434 bool lost = false; 435 int i, cur; 436 const u32 *barrier; 437 u32 *buf_ptr; 438 u64 read_ptr, write_ptr; 439 u32 status; 440 unsigned long offset, to_read = 0, flags; 441 struct cs_buffers *buf = sink_config; 442 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 443 444 if (!buf) 445 return 0; 446 447 /* This shouldn't happen */ 448 if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF)) 449 return 0; 450 451 spin_lock_irqsave(&drvdata->spinlock, flags); 452 453 /* Don't do anything if another tracer is using this sink */ 454 if (atomic_read(csdev->refcnt) != 1) 455 goto out; 456 457 CS_UNLOCK(drvdata->base); 458 459 tmc_flush_and_stop(drvdata); 460 461 read_ptr = tmc_read_rrp(drvdata); 462 write_ptr = tmc_read_rwp(drvdata); 463 464 /* 465 * Get a hold of the status register and see if a wrap around 466 * has occurred. If so adjust things accordingly. 467 */ 468 status = readl_relaxed(drvdata->base + TMC_STS); 469 if (status & TMC_STS_FULL) { 470 lost = true; 471 to_read = drvdata->size; 472 } else { 473 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size); 474 } 475 476 /* 477 * The TMC RAM buffer may be bigger than the space available in the 478 * perf ring buffer (handle->size). If so advance the RRP so that we 479 * get the latest trace data. In snapshot mode none of that matters 480 * since we are expected to clobber stale data in favour of the latest 481 * traces. 482 */ 483 if (!buf->snapshot && to_read > handle->size) { 484 u32 mask = 0; 485 486 /* 487 * The value written to RRP must be byte-address aligned to 488 * the width of the trace memory databus _and_ to a frame 489 * boundary (16 byte), whichever is the biggest. For example, 490 * for 32-bit, 64-bit and 128-bit wide trace memory, the four 491 * LSBs must be 0s. For 256-bit wide trace memory, the five 492 * LSBs must be 0s. 493 */ 494 switch (drvdata->memwidth) { 495 case TMC_MEM_INTF_WIDTH_32BITS: 496 case TMC_MEM_INTF_WIDTH_64BITS: 497 case TMC_MEM_INTF_WIDTH_128BITS: 498 mask = GENMASK(31, 4); 499 break; 500 case TMC_MEM_INTF_WIDTH_256BITS: 501 mask = GENMASK(31, 5); 502 break; 503 } 504 505 /* 506 * Make sure the new size is aligned in accordance with the 507 * requirement explained above. 508 */ 509 to_read = handle->size & mask; 510 /* Move the RAM read pointer up */ 511 read_ptr = (write_ptr + drvdata->size) - to_read; 512 /* Make sure we are still within our limits */ 513 if (read_ptr > (drvdata->size - 1)) 514 read_ptr -= drvdata->size; 515 /* Tell the HW */ 516 tmc_write_rrp(drvdata, read_ptr); 517 lost = true; 518 } 519 520 /* 521 * Don't set the TRUNCATED flag in snapshot mode because 1) the 522 * captured buffer is expected to be truncated and 2) a full buffer 523 * prevents the event from being re-enabled by the perf core, 524 * resulting in stale data being send to user space. 525 */ 526 if (!buf->snapshot && lost) 527 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 528 529 cur = buf->cur; 530 offset = buf->offset; 531 barrier = barrier_pkt; 532 533 /* for every byte to read */ 534 for (i = 0; i < to_read; i += 4) { 535 buf_ptr = buf->data_pages[cur] + offset; 536 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD); 537 538 if (lost && *barrier) { 539 *buf_ptr = *barrier; 540 barrier++; 541 } 542 543 offset += 4; 544 if (offset >= PAGE_SIZE) { 545 offset = 0; 546 cur++; 547 /* wrap around at the end of the buffer */ 548 cur &= buf->nr_pages - 1; 549 } 550 } 551 552 /* 553 * In snapshot mode we simply increment the head by the number of byte 554 * that were written. User space function cs_etm_find_snapshot() will 555 * figure out how many bytes to get from the AUX buffer based on the 556 * position of the head. 557 */ 558 if (buf->snapshot) 559 handle->head += to_read; 560 561 CS_LOCK(drvdata->base); 562 out: 563 spin_unlock_irqrestore(&drvdata->spinlock, flags); 564 565 return to_read; 566 } 567 568 static const struct coresight_ops_sink tmc_etf_sink_ops = { 569 .enable = tmc_enable_etf_sink, 570 .disable = tmc_disable_etf_sink, 571 .alloc_buffer = tmc_alloc_etf_buffer, 572 .free_buffer = tmc_free_etf_buffer, 573 .update_buffer = tmc_update_etf_buffer, 574 }; 575 576 static const struct coresight_ops_link tmc_etf_link_ops = { 577 .enable = tmc_enable_etf_link, 578 .disable = tmc_disable_etf_link, 579 }; 580 581 const struct coresight_ops tmc_etb_cs_ops = { 582 .sink_ops = &tmc_etf_sink_ops, 583 }; 584 585 const struct coresight_ops tmc_etf_cs_ops = { 586 .sink_ops = &tmc_etf_sink_ops, 587 .link_ops = &tmc_etf_link_ops, 588 }; 589 590 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) 591 { 592 enum tmc_mode mode; 593 int ret = 0; 594 unsigned long flags; 595 596 /* config types are set a boot time and never change */ 597 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && 598 drvdata->config_type != TMC_CONFIG_TYPE_ETF)) 599 return -EINVAL; 600 601 spin_lock_irqsave(&drvdata->spinlock, flags); 602 603 if (drvdata->reading) { 604 ret = -EBUSY; 605 goto out; 606 } 607 608 /* There is no point in reading a TMC in HW FIFO mode */ 609 mode = readl_relaxed(drvdata->base + TMC_MODE); 610 if (mode != TMC_MODE_CIRCULAR_BUFFER) { 611 ret = -EINVAL; 612 goto out; 613 } 614 615 /* Don't interfere if operated from Perf */ 616 if (drvdata->mode == CS_MODE_PERF) { 617 ret = -EINVAL; 618 goto out; 619 } 620 621 /* If drvdata::buf is NULL the trace data has been read already */ 622 if (drvdata->buf == NULL) { 623 ret = -EINVAL; 624 goto out; 625 } 626 627 /* Disable the TMC if need be */ 628 if (drvdata->mode == CS_MODE_SYSFS) 629 __tmc_etb_disable_hw(drvdata); 630 631 drvdata->reading = true; 632 out: 633 spin_unlock_irqrestore(&drvdata->spinlock, flags); 634 635 return ret; 636 } 637 638 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) 639 { 640 char *buf = NULL; 641 enum tmc_mode mode; 642 unsigned long flags; 643 644 /* config types are set a boot time and never change */ 645 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && 646 drvdata->config_type != TMC_CONFIG_TYPE_ETF)) 647 return -EINVAL; 648 649 spin_lock_irqsave(&drvdata->spinlock, flags); 650 651 /* There is no point in reading a TMC in HW FIFO mode */ 652 mode = readl_relaxed(drvdata->base + TMC_MODE); 653 if (mode != TMC_MODE_CIRCULAR_BUFFER) { 654 spin_unlock_irqrestore(&drvdata->spinlock, flags); 655 return -EINVAL; 656 } 657 658 /* Re-enable the TMC if need be */ 659 if (drvdata->mode == CS_MODE_SYSFS) { 660 /* 661 * The trace run will continue with the same allocated trace 662 * buffer. As such zero-out the buffer so that we don't end 663 * up with stale data. 664 * 665 * Since the tracer is still enabled drvdata::buf 666 * can't be NULL. 667 */ 668 memset(drvdata->buf, 0, drvdata->size); 669 __tmc_etb_enable_hw(drvdata); 670 } else { 671 /* 672 * The ETB/ETF is not tracing and the buffer was just read. 673 * As such prepare to free the trace buffer. 674 */ 675 buf = drvdata->buf; 676 drvdata->buf = NULL; 677 } 678 679 drvdata->reading = false; 680 spin_unlock_irqrestore(&drvdata->spinlock, flags); 681 682 /* 683 * Free allocated memory outside of the spinlock. There is no need 684 * to assert the validity of 'buf' since calling kfree(NULL) is safe. 685 */ 686 kfree(buf); 687 688 return 0; 689 } 690