1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2016 Linaro Limited. All rights reserved. 4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 5 */ 6 7 #include <linux/atomic.h> 8 #include <linux/coresight.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/iommu.h> 11 #include <linux/idr.h> 12 #include <linux/mutex.h> 13 #include <linux/refcount.h> 14 #include <linux/slab.h> 15 #include <linux/types.h> 16 #include <linux/vmalloc.h> 17 #include "coresight-catu.h" 18 #include "coresight-etm-perf.h" 19 #include "coresight-priv.h" 20 #include "coresight-tmc.h" 21 22 struct etr_flat_buf { 23 struct device *dev; 24 dma_addr_t daddr; 25 void *vaddr; 26 size_t size; 27 }; 28 29 /* 30 * etr_perf_buffer - Perf buffer used for ETR 31 * @drvdata - The ETR drvdaga this buffer has been allocated for. 32 * @etr_buf - Actual buffer used by the ETR 33 * @pid - The PID this etr_perf_buffer belongs to. 34 * @snaphost - Perf session mode 35 * @head - handle->head at the beginning of the session. 36 * @nr_pages - Number of pages in the ring buffer. 37 * @pages - Array of Pages in the ring buffer. 38 */ 39 struct etr_perf_buffer { 40 struct tmc_drvdata *drvdata; 41 struct etr_buf *etr_buf; 42 pid_t pid; 43 bool snapshot; 44 unsigned long head; 45 int nr_pages; 46 void **pages; 47 }; 48 49 /* Convert the perf index to an offset within the ETR buffer */ 50 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) 51 52 /* Lower limit for ETR hardware buffer */ 53 #define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M 54 55 /* 56 * The TMC ETR SG has a page size of 4K. The SG table contains pointers 57 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from 58 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could 59 * contain more than one SG buffer and tables. 60 * 61 * A table entry has the following format: 62 * 63 * ---Bit31------------Bit4-------Bit1-----Bit0-- 64 * | Address[39:12] | SBZ | Entry Type | 65 * ---------------------------------------------- 66 * 67 * Address: Bits [39:12] of a physical page address. Bits [11:0] are 68 * always zero. 69 * 70 * Entry type: 71 * b00 - Reserved. 72 * b01 - Last entry in the tables, points to 4K page buffer. 73 * b10 - Normal entry, points to 4K page buffer. 74 * b11 - Link. The address points to the base of next table. 75 */ 76 77 typedef u32 sgte_t; 78 79 #define ETR_SG_PAGE_SHIFT 12 80 #define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT) 81 #define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE) 82 #define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t)) 83 #define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t)) 84 85 #define ETR_SG_ET_MASK 0x3 86 #define ETR_SG_ET_LAST 0x1 87 #define ETR_SG_ET_NORMAL 0x2 88 #define ETR_SG_ET_LINK 0x3 89 90 #define ETR_SG_ADDR_SHIFT 4 91 92 #define ETR_SG_ENTRY(addr, type) \ 93 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \ 94 (type & ETR_SG_ET_MASK)) 95 96 #define ETR_SG_ADDR(entry) \ 97 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT) 98 #define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK) 99 100 /* 101 * struct etr_sg_table : ETR SG Table 102 * @sg_table: Generic SG Table holding the data/table pages. 103 * @hwaddr: hwaddress used by the TMC, which is the base 104 * address of the table. 105 */ 106 struct etr_sg_table { 107 struct tmc_sg_table *sg_table; 108 dma_addr_t hwaddr; 109 }; 110 111 /* 112 * tmc_etr_sg_table_entries: Total number of table entries required to map 113 * @nr_pages system pages. 114 * 115 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages. 116 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers, 117 * with the last entry pointing to another page of table entries. 118 * If we spill over to a new page for mapping 1 entry, we could as 119 * well replace the link entry of the previous page with the last entry. 120 */ 121 static inline unsigned long __attribute_const__ 122 tmc_etr_sg_table_entries(int nr_pages) 123 { 124 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE; 125 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1); 126 /* 127 * If we spill over to a new page for 1 entry, we could as well 128 * make it the LAST entry in the previous page, skipping the Link 129 * address. 130 */ 131 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2)) 132 nr_sglinks--; 133 return nr_sgpages + nr_sglinks; 134 } 135 136 /* 137 * tmc_pages_get_offset: Go through all the pages in the tmc_pages 138 * and map the device address @addr to an offset within the virtual 139 * contiguous buffer. 140 */ 141 static long 142 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr) 143 { 144 int i; 145 dma_addr_t page_start; 146 147 for (i = 0; i < tmc_pages->nr_pages; i++) { 148 page_start = tmc_pages->daddrs[i]; 149 if (addr >= page_start && addr < (page_start + PAGE_SIZE)) 150 return i * PAGE_SIZE + (addr - page_start); 151 } 152 153 return -EINVAL; 154 } 155 156 /* 157 * tmc_pages_free : Unmap and free the pages used by tmc_pages. 158 * If the pages were not allocated in tmc_pages_alloc(), we would 159 * simply drop the refcount. 160 */ 161 static void tmc_pages_free(struct tmc_pages *tmc_pages, 162 struct device *dev, enum dma_data_direction dir) 163 { 164 int i; 165 struct device *real_dev = dev->parent; 166 167 for (i = 0; i < tmc_pages->nr_pages; i++) { 168 if (tmc_pages->daddrs && tmc_pages->daddrs[i]) 169 dma_unmap_page(real_dev, tmc_pages->daddrs[i], 170 PAGE_SIZE, dir); 171 if (tmc_pages->pages && tmc_pages->pages[i]) 172 __free_page(tmc_pages->pages[i]); 173 } 174 175 kfree(tmc_pages->pages); 176 kfree(tmc_pages->daddrs); 177 tmc_pages->pages = NULL; 178 tmc_pages->daddrs = NULL; 179 tmc_pages->nr_pages = 0; 180 } 181 182 /* 183 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages. 184 * If @pages is not NULL, the list of page virtual addresses are 185 * used as the data pages. The pages are then dma_map'ed for @dev 186 * with dma_direction @dir. 187 * 188 * Returns 0 upon success, else the error number. 189 */ 190 static int tmc_pages_alloc(struct tmc_pages *tmc_pages, 191 struct device *dev, int node, 192 enum dma_data_direction dir, void **pages) 193 { 194 int i, nr_pages; 195 dma_addr_t paddr; 196 struct page *page; 197 struct device *real_dev = dev->parent; 198 199 nr_pages = tmc_pages->nr_pages; 200 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs), 201 GFP_KERNEL); 202 if (!tmc_pages->daddrs) 203 return -ENOMEM; 204 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages), 205 GFP_KERNEL); 206 if (!tmc_pages->pages) { 207 kfree(tmc_pages->daddrs); 208 tmc_pages->daddrs = NULL; 209 return -ENOMEM; 210 } 211 212 for (i = 0; i < nr_pages; i++) { 213 if (pages && pages[i]) { 214 page = virt_to_page(pages[i]); 215 /* Hold a refcount on the page */ 216 get_page(page); 217 } else { 218 page = alloc_pages_node(node, 219 GFP_KERNEL | __GFP_ZERO, 0); 220 } 221 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir); 222 if (dma_mapping_error(real_dev, paddr)) 223 goto err; 224 tmc_pages->daddrs[i] = paddr; 225 tmc_pages->pages[i] = page; 226 } 227 return 0; 228 err: 229 tmc_pages_free(tmc_pages, dev, dir); 230 return -ENOMEM; 231 } 232 233 static inline long 234 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr) 235 { 236 return tmc_pages_get_offset(&sg_table->data_pages, addr); 237 } 238 239 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table) 240 { 241 if (sg_table->table_vaddr) 242 vunmap(sg_table->table_vaddr); 243 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE); 244 } 245 246 static void tmc_free_data_pages(struct tmc_sg_table *sg_table) 247 { 248 if (sg_table->data_vaddr) 249 vunmap(sg_table->data_vaddr); 250 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE); 251 } 252 253 void tmc_free_sg_table(struct tmc_sg_table *sg_table) 254 { 255 tmc_free_table_pages(sg_table); 256 tmc_free_data_pages(sg_table); 257 } 258 259 /* 260 * Alloc pages for the table. Since this will be used by the device, 261 * allocate the pages closer to the device (i.e, dev_to_node(dev) 262 * rather than the CPU node). 263 */ 264 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table) 265 { 266 int rc; 267 struct tmc_pages *table_pages = &sg_table->table_pages; 268 269 rc = tmc_pages_alloc(table_pages, sg_table->dev, 270 dev_to_node(sg_table->dev), 271 DMA_TO_DEVICE, NULL); 272 if (rc) 273 return rc; 274 sg_table->table_vaddr = vmap(table_pages->pages, 275 table_pages->nr_pages, 276 VM_MAP, 277 PAGE_KERNEL); 278 if (!sg_table->table_vaddr) 279 rc = -ENOMEM; 280 else 281 sg_table->table_daddr = table_pages->daddrs[0]; 282 return rc; 283 } 284 285 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages) 286 { 287 int rc; 288 289 /* Allocate data pages on the node requested by the caller */ 290 rc = tmc_pages_alloc(&sg_table->data_pages, 291 sg_table->dev, sg_table->node, 292 DMA_FROM_DEVICE, pages); 293 if (!rc) { 294 sg_table->data_vaddr = vmap(sg_table->data_pages.pages, 295 sg_table->data_pages.nr_pages, 296 VM_MAP, 297 PAGE_KERNEL); 298 if (!sg_table->data_vaddr) 299 rc = -ENOMEM; 300 } 301 return rc; 302 } 303 304 /* 305 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table 306 * and data buffers. TMC writes to the data buffers and reads from the SG 307 * Table pages. 308 * 309 * @dev - Coresight device to which page should be DMA mapped. 310 * @node - Numa node for mem allocations 311 * @nr_tpages - Number of pages for the table entries. 312 * @nr_dpages - Number of pages for Data buffer. 313 * @pages - Optional list of virtual address of pages. 314 */ 315 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev, 316 int node, 317 int nr_tpages, 318 int nr_dpages, 319 void **pages) 320 { 321 long rc; 322 struct tmc_sg_table *sg_table; 323 324 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL); 325 if (!sg_table) 326 return ERR_PTR(-ENOMEM); 327 sg_table->data_pages.nr_pages = nr_dpages; 328 sg_table->table_pages.nr_pages = nr_tpages; 329 sg_table->node = node; 330 sg_table->dev = dev; 331 332 rc = tmc_alloc_data_pages(sg_table, pages); 333 if (!rc) 334 rc = tmc_alloc_table_pages(sg_table); 335 if (rc) { 336 tmc_free_sg_table(sg_table); 337 kfree(sg_table); 338 return ERR_PTR(rc); 339 } 340 341 return sg_table; 342 } 343 344 /* 345 * tmc_sg_table_sync_data_range: Sync the data buffer written 346 * by the device from @offset upto a @size bytes. 347 */ 348 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table, 349 u64 offset, u64 size) 350 { 351 int i, index, start; 352 int npages = DIV_ROUND_UP(size, PAGE_SIZE); 353 struct device *real_dev = table->dev->parent; 354 struct tmc_pages *data = &table->data_pages; 355 356 start = offset >> PAGE_SHIFT; 357 for (i = start; i < (start + npages); i++) { 358 index = i % data->nr_pages; 359 dma_sync_single_for_cpu(real_dev, data->daddrs[index], 360 PAGE_SIZE, DMA_FROM_DEVICE); 361 } 362 } 363 364 /* tmc_sg_sync_table: Sync the page table */ 365 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table) 366 { 367 int i; 368 struct device *real_dev = sg_table->dev->parent; 369 struct tmc_pages *table_pages = &sg_table->table_pages; 370 371 for (i = 0; i < table_pages->nr_pages; i++) 372 dma_sync_single_for_device(real_dev, table_pages->daddrs[i], 373 PAGE_SIZE, DMA_TO_DEVICE); 374 } 375 376 /* 377 * tmc_sg_table_get_data: Get the buffer pointer for data @offset 378 * in the SG buffer. The @bufpp is updated to point to the buffer. 379 * Returns : 380 * the length of linear data available at @offset. 381 * or 382 * <= 0 if no data is available. 383 */ 384 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table, 385 u64 offset, size_t len, char **bufpp) 386 { 387 size_t size; 388 int pg_idx = offset >> PAGE_SHIFT; 389 int pg_offset = offset & (PAGE_SIZE - 1); 390 struct tmc_pages *data_pages = &sg_table->data_pages; 391 392 size = tmc_sg_table_buf_size(sg_table); 393 if (offset >= size) 394 return -EINVAL; 395 396 /* Make sure we don't go beyond the end */ 397 len = (len < (size - offset)) ? len : size - offset; 398 /* Respect the page boundaries */ 399 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset); 400 if (len > 0) 401 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset; 402 return len; 403 } 404 405 #ifdef ETR_SG_DEBUG 406 /* Map a dma address to virtual address */ 407 static unsigned long 408 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table, 409 dma_addr_t addr, bool table) 410 { 411 long offset; 412 unsigned long base; 413 struct tmc_pages *tmc_pages; 414 415 if (table) { 416 tmc_pages = &sg_table->table_pages; 417 base = (unsigned long)sg_table->table_vaddr; 418 } else { 419 tmc_pages = &sg_table->data_pages; 420 base = (unsigned long)sg_table->data_vaddr; 421 } 422 423 offset = tmc_pages_get_offset(tmc_pages, addr); 424 if (offset < 0) 425 return 0; 426 return base + offset; 427 } 428 429 /* Dump the given sg_table */ 430 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) 431 { 432 sgte_t *ptr; 433 int i = 0; 434 dma_addr_t addr; 435 struct tmc_sg_table *sg_table = etr_table->sg_table; 436 437 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table, 438 etr_table->hwaddr, true); 439 while (ptr) { 440 addr = ETR_SG_ADDR(*ptr); 441 switch (ETR_SG_ET(*ptr)) { 442 case ETR_SG_ET_NORMAL: 443 dev_dbg(sg_table->dev, 444 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr); 445 ptr++; 446 break; 447 case ETR_SG_ET_LINK: 448 dev_dbg(sg_table->dev, 449 "%05d: *** %p\t:{L} 0x%llx ***\n", 450 i, ptr, addr); 451 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table, 452 addr, true); 453 break; 454 case ETR_SG_ET_LAST: 455 dev_dbg(sg_table->dev, 456 "%05d: ### %p\t:[L] 0x%llx ###\n", 457 i, ptr, addr); 458 return; 459 default: 460 dev_dbg(sg_table->dev, 461 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n", 462 i, ptr, addr); 463 return; 464 } 465 i++; 466 } 467 dev_dbg(sg_table->dev, "******* End of Table *****\n"); 468 } 469 #else 470 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {} 471 #endif 472 473 /* 474 * Populate the SG Table page table entries from table/data 475 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages. 476 * So does a Table page. So we keep track of indices of the tables 477 * in each system page and move the pointers accordingly. 478 */ 479 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size)) 480 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table) 481 { 482 dma_addr_t paddr; 483 int i, type, nr_entries; 484 int tpidx = 0; /* index to the current system table_page */ 485 int sgtidx = 0; /* index to the sg_table within the current syspage */ 486 int sgtentry = 0; /* the entry within the sg_table */ 487 int dpidx = 0; /* index to the current system data_page */ 488 int spidx = 0; /* index to the SG page within the current data page */ 489 sgte_t *ptr; /* pointer to the table entry to fill */ 490 struct tmc_sg_table *sg_table = etr_table->sg_table; 491 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs; 492 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs; 493 494 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages); 495 /* 496 * Use the contiguous virtual address of the table to update entries. 497 */ 498 ptr = sg_table->table_vaddr; 499 /* 500 * Fill all the entries, except the last entry to avoid special 501 * checks within the loop. 502 */ 503 for (i = 0; i < nr_entries - 1; i++) { 504 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) { 505 /* 506 * Last entry in a sg_table page is a link address to 507 * the next table page. If this sg_table is the last 508 * one in the system page, it links to the first 509 * sg_table in the next system page. Otherwise, it 510 * links to the next sg_table page within the system 511 * page. 512 */ 513 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) { 514 paddr = table_daddrs[tpidx + 1]; 515 } else { 516 paddr = table_daddrs[tpidx] + 517 (ETR_SG_PAGE_SIZE * (sgtidx + 1)); 518 } 519 type = ETR_SG_ET_LINK; 520 } else { 521 /* 522 * Update the indices to the data_pages to point to the 523 * next sg_page in the data buffer. 524 */ 525 type = ETR_SG_ET_NORMAL; 526 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE; 527 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE)) 528 dpidx++; 529 } 530 *ptr++ = ETR_SG_ENTRY(paddr, type); 531 /* 532 * Move to the next table pointer, moving the table page index 533 * if necessary 534 */ 535 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) { 536 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE)) 537 tpidx++; 538 } 539 } 540 541 /* Set up the last entry, which is always a data pointer */ 542 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE; 543 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST); 544 } 545 546 /* 547 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and 548 * populate the table. 549 * 550 * @dev - Device pointer for the TMC 551 * @node - NUMA node where the memory should be allocated 552 * @size - Total size of the data buffer 553 * @pages - Optional list of page virtual address 554 */ 555 static struct etr_sg_table * 556 tmc_init_etr_sg_table(struct device *dev, int node, 557 unsigned long size, void **pages) 558 { 559 int nr_entries, nr_tpages; 560 int nr_dpages = size >> PAGE_SHIFT; 561 struct tmc_sg_table *sg_table; 562 struct etr_sg_table *etr_table; 563 564 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL); 565 if (!etr_table) 566 return ERR_PTR(-ENOMEM); 567 nr_entries = tmc_etr_sg_table_entries(nr_dpages); 568 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE); 569 570 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages); 571 if (IS_ERR(sg_table)) { 572 kfree(etr_table); 573 return ERR_CAST(sg_table); 574 } 575 576 etr_table->sg_table = sg_table; 577 /* TMC should use table base address for DBA */ 578 etr_table->hwaddr = sg_table->table_daddr; 579 tmc_etr_sg_table_populate(etr_table); 580 /* Sync the table pages for the HW */ 581 tmc_sg_table_sync_table(sg_table); 582 tmc_etr_sg_table_dump(etr_table); 583 584 return etr_table; 585 } 586 587 /* 588 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer. 589 */ 590 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata, 591 struct etr_buf *etr_buf, int node, 592 void **pages) 593 { 594 struct etr_flat_buf *flat_buf; 595 struct device *real_dev = drvdata->csdev->dev.parent; 596 597 /* We cannot reuse existing pages for flat buf */ 598 if (pages) 599 return -EINVAL; 600 601 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL); 602 if (!flat_buf) 603 return -ENOMEM; 604 605 flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size, 606 &flat_buf->daddr, GFP_KERNEL); 607 if (!flat_buf->vaddr) { 608 kfree(flat_buf); 609 return -ENOMEM; 610 } 611 612 flat_buf->size = etr_buf->size; 613 flat_buf->dev = &drvdata->csdev->dev; 614 etr_buf->hwaddr = flat_buf->daddr; 615 etr_buf->mode = ETR_MODE_FLAT; 616 etr_buf->private = flat_buf; 617 return 0; 618 } 619 620 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf) 621 { 622 struct etr_flat_buf *flat_buf = etr_buf->private; 623 624 if (flat_buf && flat_buf->daddr) { 625 struct device *real_dev = flat_buf->dev->parent; 626 627 dma_free_coherent(real_dev, flat_buf->size, 628 flat_buf->vaddr, flat_buf->daddr); 629 } 630 kfree(flat_buf); 631 } 632 633 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) 634 { 635 /* 636 * Adjust the buffer to point to the beginning of the trace data 637 * and update the available trace data. 638 */ 639 etr_buf->offset = rrp - etr_buf->hwaddr; 640 if (etr_buf->full) 641 etr_buf->len = etr_buf->size; 642 else 643 etr_buf->len = rwp - rrp; 644 } 645 646 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf, 647 u64 offset, size_t len, char **bufpp) 648 { 649 struct etr_flat_buf *flat_buf = etr_buf->private; 650 651 *bufpp = (char *)flat_buf->vaddr + offset; 652 /* 653 * tmc_etr_buf_get_data already adjusts the length to handle 654 * buffer wrapping around. 655 */ 656 return len; 657 } 658 659 static const struct etr_buf_operations etr_flat_buf_ops = { 660 .alloc = tmc_etr_alloc_flat_buf, 661 .free = tmc_etr_free_flat_buf, 662 .sync = tmc_etr_sync_flat_buf, 663 .get_data = tmc_etr_get_data_flat_buf, 664 }; 665 666 /* 667 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters 668 * appropriately. 669 */ 670 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata, 671 struct etr_buf *etr_buf, int node, 672 void **pages) 673 { 674 struct etr_sg_table *etr_table; 675 struct device *dev = &drvdata->csdev->dev; 676 677 etr_table = tmc_init_etr_sg_table(dev, node, 678 etr_buf->size, pages); 679 if (IS_ERR(etr_table)) 680 return -ENOMEM; 681 etr_buf->hwaddr = etr_table->hwaddr; 682 etr_buf->mode = ETR_MODE_ETR_SG; 683 etr_buf->private = etr_table; 684 return 0; 685 } 686 687 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf) 688 { 689 struct etr_sg_table *etr_table = etr_buf->private; 690 691 if (etr_table) { 692 tmc_free_sg_table(etr_table->sg_table); 693 kfree(etr_table); 694 } 695 } 696 697 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset, 698 size_t len, char **bufpp) 699 { 700 struct etr_sg_table *etr_table = etr_buf->private; 701 702 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp); 703 } 704 705 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) 706 { 707 long r_offset, w_offset; 708 struct etr_sg_table *etr_table = etr_buf->private; 709 struct tmc_sg_table *table = etr_table->sg_table; 710 711 /* Convert hw address to offset in the buffer */ 712 r_offset = tmc_sg_get_data_page_offset(table, rrp); 713 if (r_offset < 0) { 714 dev_warn(table->dev, 715 "Unable to map RRP %llx to offset\n", rrp); 716 etr_buf->len = 0; 717 return; 718 } 719 720 w_offset = tmc_sg_get_data_page_offset(table, rwp); 721 if (w_offset < 0) { 722 dev_warn(table->dev, 723 "Unable to map RWP %llx to offset\n", rwp); 724 etr_buf->len = 0; 725 return; 726 } 727 728 etr_buf->offset = r_offset; 729 if (etr_buf->full) 730 etr_buf->len = etr_buf->size; 731 else 732 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) + 733 w_offset - r_offset; 734 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len); 735 } 736 737 static const struct etr_buf_operations etr_sg_buf_ops = { 738 .alloc = tmc_etr_alloc_sg_buf, 739 .free = tmc_etr_free_sg_buf, 740 .sync = tmc_etr_sync_sg_buf, 741 .get_data = tmc_etr_get_data_sg_buf, 742 }; 743 744 /* 745 * TMC ETR could be connected to a CATU device, which can provide address 746 * translation service. This is represented by the Output port of the TMC 747 * (ETR) connected to the input port of the CATU. 748 * 749 * Returns : coresight_device ptr for the CATU device if a CATU is found. 750 * : NULL otherwise. 751 */ 752 struct coresight_device * 753 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata) 754 { 755 int i; 756 struct coresight_device *tmp, *etr = drvdata->csdev; 757 758 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU)) 759 return NULL; 760 761 for (i = 0; i < etr->pdata->nr_outport; i++) { 762 tmp = etr->pdata->conns[i].child_dev; 763 if (tmp && coresight_is_catu_device(tmp)) 764 return tmp; 765 } 766 767 return NULL; 768 } 769 770 static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata, 771 struct etr_buf *etr_buf) 772 { 773 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata); 774 775 if (catu && helper_ops(catu)->enable) 776 return helper_ops(catu)->enable(catu, etr_buf); 777 return 0; 778 } 779 780 static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata) 781 { 782 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata); 783 784 if (catu && helper_ops(catu)->disable) 785 helper_ops(catu)->disable(catu, drvdata->etr_buf); 786 } 787 788 static const struct etr_buf_operations *etr_buf_ops[] = { 789 [ETR_MODE_FLAT] = &etr_flat_buf_ops, 790 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops, 791 [ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU) 792 ? &etr_catu_buf_ops : NULL, 793 }; 794 795 static inline int tmc_etr_mode_alloc_buf(int mode, 796 struct tmc_drvdata *drvdata, 797 struct etr_buf *etr_buf, int node, 798 void **pages) 799 { 800 int rc = -EINVAL; 801 802 switch (mode) { 803 case ETR_MODE_FLAT: 804 case ETR_MODE_ETR_SG: 805 case ETR_MODE_CATU: 806 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc) 807 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf, 808 node, pages); 809 if (!rc) 810 etr_buf->ops = etr_buf_ops[mode]; 811 return rc; 812 default: 813 return -EINVAL; 814 } 815 } 816 817 /* 818 * tmc_alloc_etr_buf: Allocate a buffer use by ETR. 819 * @drvdata : ETR device details. 820 * @size : size of the requested buffer. 821 * @flags : Required properties for the buffer. 822 * @node : Node for memory allocations. 823 * @pages : An optional list of pages. 824 */ 825 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata, 826 ssize_t size, int flags, 827 int node, void **pages) 828 { 829 int rc = -ENOMEM; 830 bool has_etr_sg, has_iommu; 831 bool has_sg, has_catu; 832 struct etr_buf *etr_buf; 833 struct device *dev = &drvdata->csdev->dev; 834 835 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG); 836 has_iommu = iommu_get_domain_for_dev(dev->parent); 837 has_catu = !!tmc_etr_get_catu_device(drvdata); 838 839 has_sg = has_catu || has_etr_sg; 840 841 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL); 842 if (!etr_buf) 843 return ERR_PTR(-ENOMEM); 844 845 etr_buf->size = size; 846 847 /* 848 * If we have to use an existing list of pages, we cannot reliably 849 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise, 850 * we use the contiguous DMA memory if at least one of the following 851 * conditions is true: 852 * a) The ETR cannot use Scatter-Gather. 853 * b) we have a backing IOMMU 854 * c) The requested memory size is smaller (< 1M). 855 * 856 * Fallback to available mechanisms. 857 * 858 */ 859 if (!pages && 860 (!has_sg || has_iommu || size < SZ_1M)) 861 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata, 862 etr_buf, node, pages); 863 if (rc && has_etr_sg) 864 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata, 865 etr_buf, node, pages); 866 if (rc && has_catu) 867 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata, 868 etr_buf, node, pages); 869 if (rc) { 870 kfree(etr_buf); 871 return ERR_PTR(rc); 872 } 873 874 refcount_set(&etr_buf->refcount, 1); 875 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n", 876 (unsigned long)size >> 10, etr_buf->mode); 877 return etr_buf; 878 } 879 880 static void tmc_free_etr_buf(struct etr_buf *etr_buf) 881 { 882 WARN_ON(!etr_buf->ops || !etr_buf->ops->free); 883 etr_buf->ops->free(etr_buf); 884 kfree(etr_buf); 885 } 886 887 /* 888 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset 889 * with a maximum of @len bytes. 890 * Returns: The size of the linear data available @pos, with *bufpp 891 * updated to point to the buffer. 892 */ 893 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf, 894 u64 offset, size_t len, char **bufpp) 895 { 896 /* Adjust the length to limit this transaction to end of buffer */ 897 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset; 898 899 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp); 900 } 901 902 static inline s64 903 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset) 904 { 905 ssize_t len; 906 char *bufp; 907 908 len = tmc_etr_buf_get_data(etr_buf, offset, 909 CORESIGHT_BARRIER_PKT_SIZE, &bufp); 910 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE)) 911 return -EINVAL; 912 coresight_insert_barrier_packet(bufp); 913 return offset + CORESIGHT_BARRIER_PKT_SIZE; 914 } 915 916 /* 917 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata. 918 * Makes sure the trace data is synced to the memory for consumption. 919 * @etr_buf->offset will hold the offset to the beginning of the trace data 920 * within the buffer, with @etr_buf->len bytes to consume. 921 */ 922 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata) 923 { 924 struct etr_buf *etr_buf = drvdata->etr_buf; 925 u64 rrp, rwp; 926 u32 status; 927 928 rrp = tmc_read_rrp(drvdata); 929 rwp = tmc_read_rwp(drvdata); 930 status = readl_relaxed(drvdata->base + TMC_STS); 931 932 /* 933 * If there were memory errors in the session, truncate the 934 * buffer. 935 */ 936 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) { 937 dev_dbg(&drvdata->csdev->dev, 938 "tmc memory error detected, truncating buffer\n"); 939 etr_buf->len = 0; 940 etr_buf->full = 0; 941 return; 942 } 943 944 etr_buf->full = status & TMC_STS_FULL; 945 946 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync); 947 948 etr_buf->ops->sync(etr_buf, rrp, rwp); 949 } 950 951 static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata) 952 { 953 u32 axictl, sts; 954 struct etr_buf *etr_buf = drvdata->etr_buf; 955 956 CS_UNLOCK(drvdata->base); 957 958 /* Wait for TMCSReady bit to be set */ 959 tmc_wait_for_tmcready(drvdata); 960 961 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ); 962 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); 963 964 axictl = readl_relaxed(drvdata->base + TMC_AXICTL); 965 axictl &= ~TMC_AXICTL_CLEAR_MASK; 966 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16); 967 axictl |= TMC_AXICTL_AXCACHE_OS; 968 969 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) { 970 axictl &= ~TMC_AXICTL_ARCACHE_MASK; 971 axictl |= TMC_AXICTL_ARCACHE_OS; 972 } 973 974 if (etr_buf->mode == ETR_MODE_ETR_SG) 975 axictl |= TMC_AXICTL_SCT_GAT_MODE; 976 977 writel_relaxed(axictl, drvdata->base + TMC_AXICTL); 978 tmc_write_dba(drvdata, etr_buf->hwaddr); 979 /* 980 * If the TMC pointers must be programmed before the session, 981 * we have to set it properly (i.e, RRP/RWP to base address and 982 * STS to "not full"). 983 */ 984 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) { 985 tmc_write_rrp(drvdata, etr_buf->hwaddr); 986 tmc_write_rwp(drvdata, etr_buf->hwaddr); 987 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL; 988 writel_relaxed(sts, drvdata->base + TMC_STS); 989 } 990 991 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | 992 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | 993 TMC_FFCR_TRIGON_TRIGIN, 994 drvdata->base + TMC_FFCR); 995 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); 996 tmc_enable_hw(drvdata); 997 998 CS_LOCK(drvdata->base); 999 } 1000 1001 static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, 1002 struct etr_buf *etr_buf) 1003 { 1004 int rc; 1005 1006 /* Callers should provide an appropriate buffer for use */ 1007 if (WARN_ON(!etr_buf)) 1008 return -EINVAL; 1009 1010 if ((etr_buf->mode == ETR_MODE_ETR_SG) && 1011 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG))) 1012 return -EINVAL; 1013 1014 if (WARN_ON(drvdata->etr_buf)) 1015 return -EBUSY; 1016 1017 /* 1018 * If this ETR is connected to a CATU, enable it before we turn 1019 * this on. 1020 */ 1021 rc = tmc_etr_enable_catu(drvdata, etr_buf); 1022 if (rc) 1023 return rc; 1024 rc = coresight_claim_device(drvdata->base); 1025 if (!rc) { 1026 drvdata->etr_buf = etr_buf; 1027 __tmc_etr_enable_hw(drvdata); 1028 } 1029 1030 return rc; 1031 } 1032 1033 /* 1034 * Return the available trace data in the buffer (starts at etr_buf->offset, 1035 * limited by etr_buf->len) from @pos, with a maximum limit of @len, 1036 * also updating the @bufpp on where to find it. Since the trace data 1037 * starts at anywhere in the buffer, depending on the RRP, we adjust the 1038 * @len returned to handle buffer wrapping around. 1039 * 1040 * We are protected here by drvdata->reading != 0, which ensures the 1041 * sysfs_buf stays alive. 1042 */ 1043 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, 1044 loff_t pos, size_t len, char **bufpp) 1045 { 1046 s64 offset; 1047 ssize_t actual = len; 1048 struct etr_buf *etr_buf = drvdata->sysfs_buf; 1049 1050 if (pos + actual > etr_buf->len) 1051 actual = etr_buf->len - pos; 1052 if (actual <= 0) 1053 return actual; 1054 1055 /* Compute the offset from which we read the data */ 1056 offset = etr_buf->offset + pos; 1057 if (offset >= etr_buf->size) 1058 offset -= etr_buf->size; 1059 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp); 1060 } 1061 1062 static struct etr_buf * 1063 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata) 1064 { 1065 return tmc_alloc_etr_buf(drvdata, drvdata->size, 1066 0, cpu_to_node(0), NULL); 1067 } 1068 1069 static void 1070 tmc_etr_free_sysfs_buf(struct etr_buf *buf) 1071 { 1072 if (buf) 1073 tmc_free_etr_buf(buf); 1074 } 1075 1076 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata) 1077 { 1078 struct etr_buf *etr_buf = drvdata->etr_buf; 1079 1080 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) { 1081 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf); 1082 drvdata->sysfs_buf = NULL; 1083 } else { 1084 tmc_sync_etr_buf(drvdata); 1085 /* 1086 * Insert barrier packets at the beginning, if there was 1087 * an overflow. 1088 */ 1089 if (etr_buf->full) 1090 tmc_etr_buf_insert_barrier_packet(etr_buf, 1091 etr_buf->offset); 1092 } 1093 } 1094 1095 static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) 1096 { 1097 CS_UNLOCK(drvdata->base); 1098 1099 tmc_flush_and_stop(drvdata); 1100 /* 1101 * When operating in sysFS mode the content of the buffer needs to be 1102 * read before the TMC is disabled. 1103 */ 1104 if (drvdata->mode == CS_MODE_SYSFS) 1105 tmc_etr_sync_sysfs_buf(drvdata); 1106 1107 tmc_disable_hw(drvdata); 1108 1109 CS_LOCK(drvdata->base); 1110 1111 } 1112 1113 static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) 1114 { 1115 __tmc_etr_disable_hw(drvdata); 1116 /* Disable CATU device if this ETR is connected to one */ 1117 tmc_etr_disable_catu(drvdata); 1118 coresight_disclaim_device(drvdata->base); 1119 /* Reset the ETR buf used by hardware */ 1120 drvdata->etr_buf = NULL; 1121 } 1122 1123 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) 1124 { 1125 int ret = 0; 1126 unsigned long flags; 1127 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 1128 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL; 1129 1130 /* 1131 * If we are enabling the ETR from disabled state, we need to make 1132 * sure we have a buffer with the right size. The etr_buf is not reset 1133 * immediately after we stop the tracing in SYSFS mode as we wait for 1134 * the user to collect the data. We may be able to reuse the existing 1135 * buffer, provided the size matches. Any allocation has to be done 1136 * with the lock released. 1137 */ 1138 spin_lock_irqsave(&drvdata->spinlock, flags); 1139 sysfs_buf = READ_ONCE(drvdata->sysfs_buf); 1140 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) { 1141 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1142 1143 /* Allocate memory with the locks released */ 1144 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata); 1145 if (IS_ERR(new_buf)) 1146 return PTR_ERR(new_buf); 1147 1148 /* Let's try again */ 1149 spin_lock_irqsave(&drvdata->spinlock, flags); 1150 } 1151 1152 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) { 1153 ret = -EBUSY; 1154 goto out; 1155 } 1156 1157 /* 1158 * In sysFS mode we can have multiple writers per sink. Since this 1159 * sink is already enabled no memory is needed and the HW need not be 1160 * touched, even if the buffer size has changed. 1161 */ 1162 if (drvdata->mode == CS_MODE_SYSFS) { 1163 atomic_inc(csdev->refcnt); 1164 goto out; 1165 } 1166 1167 /* 1168 * If we don't have a buffer or it doesn't match the requested size, 1169 * use the buffer allocated above. Otherwise reuse the existing buffer. 1170 */ 1171 sysfs_buf = READ_ONCE(drvdata->sysfs_buf); 1172 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) { 1173 free_buf = sysfs_buf; 1174 drvdata->sysfs_buf = new_buf; 1175 } 1176 1177 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf); 1178 if (!ret) { 1179 drvdata->mode = CS_MODE_SYSFS; 1180 atomic_inc(csdev->refcnt); 1181 } 1182 out: 1183 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1184 1185 /* Free memory outside the spinlock if need be */ 1186 if (free_buf) 1187 tmc_etr_free_sysfs_buf(free_buf); 1188 1189 if (!ret) 1190 dev_dbg(&csdev->dev, "TMC-ETR enabled\n"); 1191 1192 return ret; 1193 } 1194 1195 /* 1196 * alloc_etr_buf: Allocate ETR buffer for use by perf. 1197 * The size of the hardware buffer is dependent on the size configured 1198 * via sysfs and the perf ring buffer size. We prefer to allocate the 1199 * largest possible size, scaling down the size by half until it 1200 * reaches a minimum limit (1M), beyond which we give up. 1201 */ 1202 static struct etr_buf * 1203 alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, 1204 int nr_pages, void **pages, bool snapshot) 1205 { 1206 int node; 1207 struct etr_buf *etr_buf; 1208 unsigned long size; 1209 1210 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); 1211 /* 1212 * Try to match the perf ring buffer size if it is larger 1213 * than the size requested via sysfs. 1214 */ 1215 if ((nr_pages << PAGE_SHIFT) > drvdata->size) { 1216 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT), 1217 0, node, NULL); 1218 if (!IS_ERR(etr_buf)) 1219 goto done; 1220 } 1221 1222 /* 1223 * Else switch to configured size for this ETR 1224 * and scale down until we hit the minimum limit. 1225 */ 1226 size = drvdata->size; 1227 do { 1228 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL); 1229 if (!IS_ERR(etr_buf)) 1230 goto done; 1231 size /= 2; 1232 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE); 1233 1234 return ERR_PTR(-ENOMEM); 1235 1236 done: 1237 return etr_buf; 1238 } 1239 1240 static struct etr_buf * 1241 get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata, 1242 struct perf_event *event, int nr_pages, 1243 void **pages, bool snapshot) 1244 { 1245 int ret; 1246 pid_t pid = task_pid_nr(event->owner); 1247 struct etr_buf *etr_buf; 1248 1249 retry: 1250 /* 1251 * An etr_perf_buffer is associated with an event and holds a reference 1252 * to the AUX ring buffer that was created for that event. In CPU-wide 1253 * N:1 mode multiple events (one per CPU), each with its own AUX ring 1254 * buffer, share a sink. As such an etr_perf_buffer is created for each 1255 * event but a single etr_buf associated with the ETR is shared between 1256 * them. The last event in a trace session will copy the content of the 1257 * etr_buf to its AUX ring buffer. Ring buffer associated to other 1258 * events are simply not used an freed as events are destoyed. We still 1259 * need to allocate a ring buffer for each event since we don't know 1260 * which event will be last. 1261 */ 1262 1263 /* 1264 * The first thing to do here is check if an etr_buf has already been 1265 * allocated for this session. If so it is shared with this event, 1266 * otherwise it is created. 1267 */ 1268 mutex_lock(&drvdata->idr_mutex); 1269 etr_buf = idr_find(&drvdata->idr, pid); 1270 if (etr_buf) { 1271 refcount_inc(&etr_buf->refcount); 1272 mutex_unlock(&drvdata->idr_mutex); 1273 return etr_buf; 1274 } 1275 1276 /* If we made it here no buffer has been allocated, do so now. */ 1277 mutex_unlock(&drvdata->idr_mutex); 1278 1279 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); 1280 if (IS_ERR(etr_buf)) 1281 return etr_buf; 1282 1283 /* Now that we have a buffer, add it to the IDR. */ 1284 mutex_lock(&drvdata->idr_mutex); 1285 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL); 1286 mutex_unlock(&drvdata->idr_mutex); 1287 1288 /* Another event with this session ID has allocated this buffer. */ 1289 if (ret == -ENOSPC) { 1290 tmc_free_etr_buf(etr_buf); 1291 goto retry; 1292 } 1293 1294 /* The IDR can't allocate room for a new session, abandon ship. */ 1295 if (ret == -ENOMEM) { 1296 tmc_free_etr_buf(etr_buf); 1297 return ERR_PTR(ret); 1298 } 1299 1300 1301 return etr_buf; 1302 } 1303 1304 static struct etr_buf * 1305 get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata, 1306 struct perf_event *event, int nr_pages, 1307 void **pages, bool snapshot) 1308 { 1309 /* 1310 * In per-thread mode the etr_buf isn't shared, so just go ahead 1311 * with memory allocation. 1312 */ 1313 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); 1314 } 1315 1316 static struct etr_buf * 1317 get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, 1318 int nr_pages, void **pages, bool snapshot) 1319 { 1320 if (event->cpu == -1) 1321 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages, 1322 pages, snapshot); 1323 1324 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages, 1325 pages, snapshot); 1326 } 1327 1328 static struct etr_perf_buffer * 1329 tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event, 1330 int nr_pages, void **pages, bool snapshot) 1331 { 1332 int node; 1333 struct etr_buf *etr_buf; 1334 struct etr_perf_buffer *etr_perf; 1335 1336 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); 1337 1338 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node); 1339 if (!etr_perf) 1340 return ERR_PTR(-ENOMEM); 1341 1342 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot); 1343 if (!IS_ERR(etr_buf)) 1344 goto done; 1345 1346 kfree(etr_perf); 1347 return ERR_PTR(-ENOMEM); 1348 1349 done: 1350 /* 1351 * Keep a reference to the ETR this buffer has been allocated for 1352 * in order to have access to the IDR in tmc_free_etr_buffer(). 1353 */ 1354 etr_perf->drvdata = drvdata; 1355 etr_perf->etr_buf = etr_buf; 1356 1357 return etr_perf; 1358 } 1359 1360 1361 static void *tmc_alloc_etr_buffer(struct coresight_device *csdev, 1362 struct perf_event *event, void **pages, 1363 int nr_pages, bool snapshot) 1364 { 1365 struct etr_perf_buffer *etr_perf; 1366 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 1367 1368 etr_perf = tmc_etr_setup_perf_buf(drvdata, event, 1369 nr_pages, pages, snapshot); 1370 if (IS_ERR(etr_perf)) { 1371 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n"); 1372 return NULL; 1373 } 1374 1375 etr_perf->pid = task_pid_nr(event->owner); 1376 etr_perf->snapshot = snapshot; 1377 etr_perf->nr_pages = nr_pages; 1378 etr_perf->pages = pages; 1379 1380 return etr_perf; 1381 } 1382 1383 static void tmc_free_etr_buffer(void *config) 1384 { 1385 struct etr_perf_buffer *etr_perf = config; 1386 struct tmc_drvdata *drvdata = etr_perf->drvdata; 1387 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf; 1388 1389 if (!etr_buf) 1390 goto free_etr_perf_buffer; 1391 1392 mutex_lock(&drvdata->idr_mutex); 1393 /* If we are not the last one to use the buffer, don't touch it. */ 1394 if (!refcount_dec_and_test(&etr_buf->refcount)) { 1395 mutex_unlock(&drvdata->idr_mutex); 1396 goto free_etr_perf_buffer; 1397 } 1398 1399 /* We are the last one, remove from the IDR and free the buffer. */ 1400 buf = idr_remove(&drvdata->idr, etr_perf->pid); 1401 mutex_unlock(&drvdata->idr_mutex); 1402 1403 /* 1404 * Something went very wrong if the buffer associated with this ID 1405 * is not the same in the IDR. Leak to avoid use after free. 1406 */ 1407 if (buf && WARN_ON(buf != etr_buf)) 1408 goto free_etr_perf_buffer; 1409 1410 tmc_free_etr_buf(etr_perf->etr_buf); 1411 1412 free_etr_perf_buffer: 1413 kfree(etr_perf); 1414 } 1415 1416 /* 1417 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware 1418 * buffer to the perf ring buffer. 1419 */ 1420 static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf, 1421 unsigned long src_offset, 1422 unsigned long to_copy) 1423 { 1424 long bytes; 1425 long pg_idx, pg_offset; 1426 unsigned long head = etr_perf->head; 1427 char **dst_pages, *src_buf; 1428 struct etr_buf *etr_buf = etr_perf->etr_buf; 1429 1430 head = etr_perf->head; 1431 pg_idx = head >> PAGE_SHIFT; 1432 pg_offset = head & (PAGE_SIZE - 1); 1433 dst_pages = (char **)etr_perf->pages; 1434 1435 while (to_copy > 0) { 1436 /* 1437 * In one iteration, we can copy minimum of : 1438 * 1) what is available in the source buffer, 1439 * 2) what is available in the source buffer, before it 1440 * wraps around. 1441 * 3) what is available in the destination page. 1442 * in one iteration. 1443 */ 1444 if (src_offset >= etr_buf->size) 1445 src_offset -= etr_buf->size; 1446 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy, 1447 &src_buf); 1448 if (WARN_ON_ONCE(bytes <= 0)) 1449 break; 1450 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset)); 1451 1452 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes); 1453 1454 to_copy -= bytes; 1455 1456 /* Move destination pointers */ 1457 pg_offset += bytes; 1458 if (pg_offset == PAGE_SIZE) { 1459 pg_offset = 0; 1460 if (++pg_idx == etr_perf->nr_pages) 1461 pg_idx = 0; 1462 } 1463 1464 /* Move source pointers */ 1465 src_offset += bytes; 1466 } 1467 } 1468 1469 /* 1470 * tmc_update_etr_buffer : Update the perf ring buffer with the 1471 * available trace data. We use software double buffering at the moment. 1472 * 1473 * TODO: Add support for reusing the perf ring buffer. 1474 */ 1475 static unsigned long 1476 tmc_update_etr_buffer(struct coresight_device *csdev, 1477 struct perf_output_handle *handle, 1478 void *config) 1479 { 1480 bool lost = false; 1481 unsigned long flags, offset, size = 0; 1482 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 1483 struct etr_perf_buffer *etr_perf = config; 1484 struct etr_buf *etr_buf = etr_perf->etr_buf; 1485 1486 spin_lock_irqsave(&drvdata->spinlock, flags); 1487 1488 /* Don't do anything if another tracer is using this sink */ 1489 if (atomic_read(csdev->refcnt) != 1) { 1490 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1491 goto out; 1492 } 1493 1494 if (WARN_ON(drvdata->perf_buf != etr_buf)) { 1495 lost = true; 1496 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1497 goto out; 1498 } 1499 1500 CS_UNLOCK(drvdata->base); 1501 1502 tmc_flush_and_stop(drvdata); 1503 tmc_sync_etr_buf(drvdata); 1504 1505 CS_LOCK(drvdata->base); 1506 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1507 1508 lost = etr_buf->full; 1509 offset = etr_buf->offset; 1510 size = etr_buf->len; 1511 1512 /* 1513 * The ETR buffer may be bigger than the space available in the 1514 * perf ring buffer (handle->size). If so advance the offset so that we 1515 * get the latest trace data. In snapshot mode none of that matters 1516 * since we are expected to clobber stale data in favour of the latest 1517 * traces. 1518 */ 1519 if (!etr_perf->snapshot && size > handle->size) { 1520 u32 mask = tmc_get_memwidth_mask(drvdata); 1521 1522 /* 1523 * Make sure the new size is aligned in accordance with the 1524 * requirement explained in function tmc_get_memwidth_mask(). 1525 */ 1526 size = handle->size & mask; 1527 offset = etr_buf->offset + etr_buf->len - size; 1528 1529 if (offset >= etr_buf->size) 1530 offset -= etr_buf->size; 1531 lost = true; 1532 } 1533 1534 /* Insert barrier packets at the beginning, if there was an overflow */ 1535 if (lost) 1536 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset); 1537 tmc_etr_sync_perf_buffer(etr_perf, offset, size); 1538 1539 /* 1540 * In snapshot mode we simply increment the head by the number of byte 1541 * that were written. User space function cs_etm_find_snapshot() will 1542 * figure out how many bytes to get from the AUX buffer based on the 1543 * position of the head. 1544 */ 1545 if (etr_perf->snapshot) 1546 handle->head += size; 1547 out: 1548 /* 1549 * Don't set the TRUNCATED flag in snapshot mode because 1) the 1550 * captured buffer is expected to be truncated and 2) a full buffer 1551 * prevents the event from being re-enabled by the perf core, 1552 * resulting in stale data being send to user space. 1553 */ 1554 if (!etr_perf->snapshot && lost) 1555 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 1556 return size; 1557 } 1558 1559 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) 1560 { 1561 int rc = 0; 1562 pid_t pid; 1563 unsigned long flags; 1564 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 1565 struct perf_output_handle *handle = data; 1566 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle); 1567 1568 spin_lock_irqsave(&drvdata->spinlock, flags); 1569 /* Don't use this sink if it is already claimed by sysFS */ 1570 if (drvdata->mode == CS_MODE_SYSFS) { 1571 rc = -EBUSY; 1572 goto unlock_out; 1573 } 1574 1575 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) { 1576 rc = -EINVAL; 1577 goto unlock_out; 1578 } 1579 1580 /* Get a handle on the pid of the process to monitor */ 1581 pid = etr_perf->pid; 1582 1583 /* Do not proceed if this device is associated with another session */ 1584 if (drvdata->pid != -1 && drvdata->pid != pid) { 1585 rc = -EBUSY; 1586 goto unlock_out; 1587 } 1588 1589 etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf); 1590 1591 /* 1592 * No HW configuration is needed if the sink is already in 1593 * use for this session. 1594 */ 1595 if (drvdata->pid == pid) { 1596 atomic_inc(csdev->refcnt); 1597 goto unlock_out; 1598 } 1599 1600 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf); 1601 if (!rc) { 1602 /* Associate with monitored process. */ 1603 drvdata->pid = pid; 1604 drvdata->mode = CS_MODE_PERF; 1605 drvdata->perf_buf = etr_perf->etr_buf; 1606 atomic_inc(csdev->refcnt); 1607 } 1608 1609 unlock_out: 1610 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1611 return rc; 1612 } 1613 1614 static int tmc_enable_etr_sink(struct coresight_device *csdev, 1615 u32 mode, void *data) 1616 { 1617 switch (mode) { 1618 case CS_MODE_SYSFS: 1619 return tmc_enable_etr_sink_sysfs(csdev); 1620 case CS_MODE_PERF: 1621 return tmc_enable_etr_sink_perf(csdev, data); 1622 } 1623 1624 /* We shouldn't be here */ 1625 return -EINVAL; 1626 } 1627 1628 static int tmc_disable_etr_sink(struct coresight_device *csdev) 1629 { 1630 unsigned long flags; 1631 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 1632 1633 spin_lock_irqsave(&drvdata->spinlock, flags); 1634 1635 if (drvdata->reading) { 1636 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1637 return -EBUSY; 1638 } 1639 1640 if (atomic_dec_return(csdev->refcnt)) { 1641 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1642 return -EBUSY; 1643 } 1644 1645 /* Complain if we (somehow) got out of sync */ 1646 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); 1647 tmc_etr_disable_hw(drvdata); 1648 /* Dissociate from monitored process. */ 1649 drvdata->pid = -1; 1650 drvdata->mode = CS_MODE_DISABLED; 1651 /* Reset perf specific data */ 1652 drvdata->perf_buf = NULL; 1653 1654 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1655 1656 dev_dbg(&csdev->dev, "TMC-ETR disabled\n"); 1657 return 0; 1658 } 1659 1660 static const struct coresight_ops_sink tmc_etr_sink_ops = { 1661 .enable = tmc_enable_etr_sink, 1662 .disable = tmc_disable_etr_sink, 1663 .alloc_buffer = tmc_alloc_etr_buffer, 1664 .update_buffer = tmc_update_etr_buffer, 1665 .free_buffer = tmc_free_etr_buffer, 1666 }; 1667 1668 const struct coresight_ops tmc_etr_cs_ops = { 1669 .sink_ops = &tmc_etr_sink_ops, 1670 }; 1671 1672 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) 1673 { 1674 int ret = 0; 1675 unsigned long flags; 1676 1677 /* config types are set a boot time and never change */ 1678 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) 1679 return -EINVAL; 1680 1681 spin_lock_irqsave(&drvdata->spinlock, flags); 1682 if (drvdata->reading) { 1683 ret = -EBUSY; 1684 goto out; 1685 } 1686 1687 /* 1688 * We can safely allow reads even if the ETR is operating in PERF mode, 1689 * since the sysfs session is captured in mode specific data. 1690 * If drvdata::sysfs_data is NULL the trace data has been read already. 1691 */ 1692 if (!drvdata->sysfs_buf) { 1693 ret = -EINVAL; 1694 goto out; 1695 } 1696 1697 /* Disable the TMC if we are trying to read from a running session. */ 1698 if (drvdata->mode == CS_MODE_SYSFS) 1699 __tmc_etr_disable_hw(drvdata); 1700 1701 drvdata->reading = true; 1702 out: 1703 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1704 1705 return ret; 1706 } 1707 1708 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) 1709 { 1710 unsigned long flags; 1711 struct etr_buf *sysfs_buf = NULL; 1712 1713 /* config types are set a boot time and never change */ 1714 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) 1715 return -EINVAL; 1716 1717 spin_lock_irqsave(&drvdata->spinlock, flags); 1718 1719 /* RE-enable the TMC if need be */ 1720 if (drvdata->mode == CS_MODE_SYSFS) { 1721 /* 1722 * The trace run will continue with the same allocated trace 1723 * buffer. Since the tracer is still enabled drvdata::buf can't 1724 * be NULL. 1725 */ 1726 __tmc_etr_enable_hw(drvdata); 1727 } else { 1728 /* 1729 * The ETR is not tracing and the buffer was just read. 1730 * As such prepare to free the trace buffer. 1731 */ 1732 sysfs_buf = drvdata->sysfs_buf; 1733 drvdata->sysfs_buf = NULL; 1734 } 1735 1736 drvdata->reading = false; 1737 spin_unlock_irqrestore(&drvdata->spinlock, flags); 1738 1739 /* Free allocated memory out side of the spinlock */ 1740 if (sysfs_buf) 1741 tmc_etr_free_sysfs_buf(sysfs_buf); 1742 1743 return 0; 1744 } 1745