1 /* 2 * Performance events ring-buffer code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/perf_event.h> 13 #include <linux/vmalloc.h> 14 #include <linux/slab.h> 15 #include <linux/circ_buf.h> 16 #include <linux/poll.h> 17 18 #include "internal.h" 19 20 static void perf_output_wakeup(struct perf_output_handle *handle) 21 { 22 atomic_set(&handle->rb->poll, POLLIN); 23 24 handle->event->pending_wakeup = 1; 25 irq_work_queue(&handle->event->pending); 26 } 27 28 /* 29 * We need to ensure a later event_id doesn't publish a head when a former 30 * event isn't done writing. However since we need to deal with NMIs we 31 * cannot fully serialize things. 32 * 33 * We only publish the head (and generate a wakeup) when the outer-most 34 * event completes. 35 */ 36 static void perf_output_get_handle(struct perf_output_handle *handle) 37 { 38 struct ring_buffer *rb = handle->rb; 39 40 preempt_disable(); 41 local_inc(&rb->nest); 42 handle->wakeup = local_read(&rb->wakeup); 43 } 44 45 static void perf_output_put_handle(struct perf_output_handle *handle) 46 { 47 struct ring_buffer *rb = handle->rb; 48 unsigned long head; 49 50 again: 51 head = local_read(&rb->head); 52 53 /* 54 * IRQ/NMI can happen here, which means we can miss a head update. 55 */ 56 57 if (!local_dec_and_test(&rb->nest)) 58 goto out; 59 60 /* 61 * Since the mmap() consumer (userspace) can run on a different CPU: 62 * 63 * kernel user 64 * 65 * if (LOAD ->data_tail) { LOAD ->data_head 66 * (A) smp_rmb() (C) 67 * STORE $data LOAD $data 68 * smp_wmb() (B) smp_mb() (D) 69 * STORE ->data_head STORE ->data_tail 70 * } 71 * 72 * Where A pairs with D, and B pairs with C. 73 * 74 * In our case (A) is a control dependency that separates the load of 75 * the ->data_tail and the stores of $data. In case ->data_tail 76 * indicates there is no room in the buffer to store $data we do not. 77 * 78 * D needs to be a full barrier since it separates the data READ 79 * from the tail WRITE. 80 * 81 * For B a WMB is sufficient since it separates two WRITEs, and for C 82 * an RMB is sufficient since it separates two READs. 83 * 84 * See perf_output_begin(). 85 */ 86 smp_wmb(); /* B, matches C */ 87 rb->user_page->data_head = head; 88 89 /* 90 * Now check if we missed an update -- rely on previous implied 91 * compiler barriers to force a re-read. 92 */ 93 if (unlikely(head != local_read(&rb->head))) { 94 local_inc(&rb->nest); 95 goto again; 96 } 97 98 if (handle->wakeup != local_read(&rb->wakeup)) 99 perf_output_wakeup(handle); 100 101 out: 102 preempt_enable(); 103 } 104 105 int perf_output_begin(struct perf_output_handle *handle, 106 struct perf_event *event, unsigned int size) 107 { 108 struct ring_buffer *rb; 109 unsigned long tail, offset, head; 110 int have_lost, page_shift; 111 struct { 112 struct perf_event_header header; 113 u64 id; 114 u64 lost; 115 } lost_event; 116 117 rcu_read_lock(); 118 /* 119 * For inherited events we send all the output towards the parent. 120 */ 121 if (event->parent) 122 event = event->parent; 123 124 rb = rcu_dereference(event->rb); 125 if (unlikely(!rb)) 126 goto out; 127 128 if (unlikely(!rb->nr_pages)) 129 goto out; 130 131 handle->rb = rb; 132 handle->event = event; 133 134 have_lost = local_read(&rb->lost); 135 if (unlikely(have_lost)) { 136 size += sizeof(lost_event); 137 if (event->attr.sample_id_all) 138 size += event->id_header_size; 139 } 140 141 perf_output_get_handle(handle); 142 143 do { 144 tail = READ_ONCE(rb->user_page->data_tail); 145 offset = head = local_read(&rb->head); 146 if (!rb->overwrite && 147 unlikely(CIRC_SPACE(head, tail, perf_data_size(rb)) < size)) 148 goto fail; 149 150 /* 151 * The above forms a control dependency barrier separating the 152 * @tail load above from the data stores below. Since the @tail 153 * load is required to compute the branch to fail below. 154 * 155 * A, matches D; the full memory barrier userspace SHOULD issue 156 * after reading the data and before storing the new tail 157 * position. 158 * 159 * See perf_output_put_handle(). 160 */ 161 162 head += size; 163 } while (local_cmpxchg(&rb->head, offset, head) != offset); 164 165 /* 166 * We rely on the implied barrier() by local_cmpxchg() to ensure 167 * none of the data stores below can be lifted up by the compiler. 168 */ 169 170 if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) 171 local_add(rb->watermark, &rb->wakeup); 172 173 page_shift = PAGE_SHIFT + page_order(rb); 174 175 handle->page = (offset >> page_shift) & (rb->nr_pages - 1); 176 offset &= (1UL << page_shift) - 1; 177 handle->addr = rb->data_pages[handle->page] + offset; 178 handle->size = (1UL << page_shift) - offset; 179 180 if (unlikely(have_lost)) { 181 struct perf_sample_data sample_data; 182 183 lost_event.header.size = sizeof(lost_event); 184 lost_event.header.type = PERF_RECORD_LOST; 185 lost_event.header.misc = 0; 186 lost_event.id = event->id; 187 lost_event.lost = local_xchg(&rb->lost, 0); 188 189 perf_event_header__init_id(&lost_event.header, 190 &sample_data, event); 191 perf_output_put(handle, lost_event); 192 perf_event__output_id_sample(event, handle, &sample_data); 193 } 194 195 return 0; 196 197 fail: 198 local_inc(&rb->lost); 199 perf_output_put_handle(handle); 200 out: 201 rcu_read_unlock(); 202 203 return -ENOSPC; 204 } 205 206 unsigned int perf_output_copy(struct perf_output_handle *handle, 207 const void *buf, unsigned int len) 208 { 209 return __output_copy(handle, buf, len); 210 } 211 212 unsigned int perf_output_skip(struct perf_output_handle *handle, 213 unsigned int len) 214 { 215 return __output_skip(handle, NULL, len); 216 } 217 218 void perf_output_end(struct perf_output_handle *handle) 219 { 220 perf_output_put_handle(handle); 221 rcu_read_unlock(); 222 } 223 224 static void rb_irq_work(struct irq_work *work); 225 226 static void 227 ring_buffer_init(struct ring_buffer *rb, long watermark, int flags) 228 { 229 long max_size = perf_data_size(rb); 230 231 if (watermark) 232 rb->watermark = min(max_size, watermark); 233 234 if (!rb->watermark) 235 rb->watermark = max_size / 2; 236 237 if (flags & RING_BUFFER_WRITABLE) 238 rb->overwrite = 0; 239 else 240 rb->overwrite = 1; 241 242 atomic_set(&rb->refcount, 1); 243 244 INIT_LIST_HEAD(&rb->event_list); 245 spin_lock_init(&rb->event_lock); 246 init_irq_work(&rb->irq_work, rb_irq_work); 247 } 248 249 static void ring_buffer_put_async(struct ring_buffer *rb) 250 { 251 if (!atomic_dec_and_test(&rb->refcount)) 252 return; 253 254 rb->rcu_head.next = (void *)rb; 255 irq_work_queue(&rb->irq_work); 256 } 257 258 /* 259 * This is called before hardware starts writing to the AUX area to 260 * obtain an output handle and make sure there's room in the buffer. 261 * When the capture completes, call perf_aux_output_end() to commit 262 * the recorded data to the buffer. 263 * 264 * The ordering is similar to that of perf_output_{begin,end}, with 265 * the exception of (B), which should be taken care of by the pmu 266 * driver, since ordering rules will differ depending on hardware. 267 */ 268 void *perf_aux_output_begin(struct perf_output_handle *handle, 269 struct perf_event *event) 270 { 271 struct perf_event *output_event = event; 272 unsigned long aux_head, aux_tail; 273 struct ring_buffer *rb; 274 275 if (output_event->parent) 276 output_event = output_event->parent; 277 278 /* 279 * Since this will typically be open across pmu::add/pmu::del, we 280 * grab ring_buffer's refcount instead of holding rcu read lock 281 * to make sure it doesn't disappear under us. 282 */ 283 rb = ring_buffer_get(output_event); 284 if (!rb) 285 return NULL; 286 287 if (!rb_has_aux(rb) || !atomic_inc_not_zero(&rb->aux_refcount)) 288 goto err; 289 290 /* 291 * Nesting is not supported for AUX area, make sure nested 292 * writers are caught early 293 */ 294 if (WARN_ON_ONCE(local_xchg(&rb->aux_nest, 1))) 295 goto err_put; 296 297 aux_head = local_read(&rb->aux_head); 298 299 handle->rb = rb; 300 handle->event = event; 301 handle->head = aux_head; 302 handle->size = 0; 303 304 /* 305 * In overwrite mode, AUX data stores do not depend on aux_tail, 306 * therefore (A) control dependency barrier does not exist. The 307 * (B) <-> (C) ordering is still observed by the pmu driver. 308 */ 309 if (!rb->aux_overwrite) { 310 aux_tail = ACCESS_ONCE(rb->user_page->aux_tail); 311 handle->wakeup = local_read(&rb->aux_wakeup) + rb->aux_watermark; 312 if (aux_head - aux_tail < perf_aux_size(rb)) 313 handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb)); 314 315 /* 316 * handle->size computation depends on aux_tail load; this forms a 317 * control dependency barrier separating aux_tail load from aux data 318 * store that will be enabled on successful return 319 */ 320 if (!handle->size) { /* A, matches D */ 321 event->pending_disable = 1; 322 perf_output_wakeup(handle); 323 local_set(&rb->aux_nest, 0); 324 goto err_put; 325 } 326 } 327 328 return handle->rb->aux_priv; 329 330 err_put: 331 rb_free_aux(rb); 332 333 err: 334 ring_buffer_put_async(rb); 335 handle->event = NULL; 336 337 return NULL; 338 } 339 340 /* 341 * Commit the data written by hardware into the ring buffer by adjusting 342 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the 343 * pmu driver's responsibility to observe ordering rules of the hardware, 344 * so that all the data is externally visible before this is called. 345 */ 346 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size, 347 bool truncated) 348 { 349 struct ring_buffer *rb = handle->rb; 350 unsigned long aux_head; 351 u64 flags = 0; 352 353 if (truncated) 354 flags |= PERF_AUX_FLAG_TRUNCATED; 355 356 /* in overwrite mode, driver provides aux_head via handle */ 357 if (rb->aux_overwrite) { 358 flags |= PERF_AUX_FLAG_OVERWRITE; 359 360 aux_head = handle->head; 361 local_set(&rb->aux_head, aux_head); 362 } else { 363 aux_head = local_read(&rb->aux_head); 364 local_add(size, &rb->aux_head); 365 } 366 367 if (size || flags) { 368 /* 369 * Only send RECORD_AUX if we have something useful to communicate 370 */ 371 372 perf_event_aux_event(handle->event, aux_head, size, flags); 373 } 374 375 aux_head = rb->user_page->aux_head = local_read(&rb->aux_head); 376 377 if (aux_head - local_read(&rb->aux_wakeup) >= rb->aux_watermark) { 378 perf_output_wakeup(handle); 379 local_add(rb->aux_watermark, &rb->aux_wakeup); 380 } 381 handle->event = NULL; 382 383 local_set(&rb->aux_nest, 0); 384 rb_free_aux(rb); 385 ring_buffer_put_async(rb); 386 } 387 388 /* 389 * Skip over a given number of bytes in the AUX buffer, due to, for example, 390 * hardware's alignment constraints. 391 */ 392 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size) 393 { 394 struct ring_buffer *rb = handle->rb; 395 unsigned long aux_head; 396 397 if (size > handle->size) 398 return -ENOSPC; 399 400 local_add(size, &rb->aux_head); 401 402 aux_head = rb->user_page->aux_head = local_read(&rb->aux_head); 403 if (aux_head - local_read(&rb->aux_wakeup) >= rb->aux_watermark) { 404 perf_output_wakeup(handle); 405 local_add(rb->aux_watermark, &rb->aux_wakeup); 406 handle->wakeup = local_read(&rb->aux_wakeup) + 407 rb->aux_watermark; 408 } 409 410 handle->head = aux_head; 411 handle->size -= size; 412 413 return 0; 414 } 415 416 void *perf_get_aux(struct perf_output_handle *handle) 417 { 418 /* this is only valid between perf_aux_output_begin and *_end */ 419 if (!handle->event) 420 return NULL; 421 422 return handle->rb->aux_priv; 423 } 424 425 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY) 426 427 static struct page *rb_alloc_aux_page(int node, int order) 428 { 429 struct page *page; 430 431 if (order > MAX_ORDER) 432 order = MAX_ORDER; 433 434 do { 435 page = alloc_pages_node(node, PERF_AUX_GFP, order); 436 } while (!page && order--); 437 438 if (page && order) { 439 /* 440 * Communicate the allocation size to the driver: 441 * if we managed to secure a high-order allocation, 442 * set its first page's private to this order; 443 * !PagePrivate(page) means it's just a normal page. 444 */ 445 split_page(page, order); 446 SetPagePrivate(page); 447 set_page_private(page, order); 448 } 449 450 return page; 451 } 452 453 static void rb_free_aux_page(struct ring_buffer *rb, int idx) 454 { 455 struct page *page = virt_to_page(rb->aux_pages[idx]); 456 457 ClearPagePrivate(page); 458 page->mapping = NULL; 459 __free_page(page); 460 } 461 462 int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event, 463 pgoff_t pgoff, int nr_pages, long watermark, int flags) 464 { 465 bool overwrite = !(flags & RING_BUFFER_WRITABLE); 466 int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); 467 int ret = -ENOMEM, max_order = 0; 468 469 if (!has_aux(event)) 470 return -ENOTSUPP; 471 472 if (event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) { 473 /* 474 * We need to start with the max_order that fits in nr_pages, 475 * not the other way around, hence ilog2() and not get_order. 476 */ 477 max_order = ilog2(nr_pages); 478 479 /* 480 * PMU requests more than one contiguous chunks of memory 481 * for SW double buffering 482 */ 483 if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_SW_DOUBLEBUF) && 484 !overwrite) { 485 if (!max_order) 486 return -EINVAL; 487 488 max_order--; 489 } 490 } 491 492 rb->aux_pages = kzalloc_node(nr_pages * sizeof(void *), GFP_KERNEL, node); 493 if (!rb->aux_pages) 494 return -ENOMEM; 495 496 rb->free_aux = event->pmu->free_aux; 497 for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) { 498 struct page *page; 499 int last, order; 500 501 order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages)); 502 page = rb_alloc_aux_page(node, order); 503 if (!page) 504 goto out; 505 506 for (last = rb->aux_nr_pages + (1 << page_private(page)); 507 last > rb->aux_nr_pages; rb->aux_nr_pages++) 508 rb->aux_pages[rb->aux_nr_pages] = page_address(page++); 509 } 510 511 /* 512 * In overwrite mode, PMUs that don't support SG may not handle more 513 * than one contiguous allocation, since they rely on PMI to do double 514 * buffering. In this case, the entire buffer has to be one contiguous 515 * chunk. 516 */ 517 if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) && 518 overwrite) { 519 struct page *page = virt_to_page(rb->aux_pages[0]); 520 521 if (page_private(page) != max_order) 522 goto out; 523 } 524 525 rb->aux_priv = event->pmu->setup_aux(event->cpu, rb->aux_pages, nr_pages, 526 overwrite); 527 if (!rb->aux_priv) 528 goto out; 529 530 ret = 0; 531 532 /* 533 * aux_pages (and pmu driver's private data, aux_priv) will be 534 * referenced in both producer's and consumer's contexts, thus 535 * we keep a refcount here to make sure either of the two can 536 * reference them safely. 537 */ 538 atomic_set(&rb->aux_refcount, 1); 539 540 rb->aux_overwrite = overwrite; 541 rb->aux_watermark = watermark; 542 543 if (!rb->aux_watermark && !rb->aux_overwrite) 544 rb->aux_watermark = nr_pages << (PAGE_SHIFT - 1); 545 546 out: 547 if (!ret) 548 rb->aux_pgoff = pgoff; 549 else 550 rb_free_aux(rb); 551 552 return ret; 553 } 554 555 static void __rb_free_aux(struct ring_buffer *rb) 556 { 557 int pg; 558 559 if (rb->aux_priv) { 560 rb->free_aux(rb->aux_priv); 561 rb->free_aux = NULL; 562 rb->aux_priv = NULL; 563 } 564 565 if (rb->aux_nr_pages) { 566 for (pg = 0; pg < rb->aux_nr_pages; pg++) 567 rb_free_aux_page(rb, pg); 568 569 kfree(rb->aux_pages); 570 rb->aux_nr_pages = 0; 571 } 572 } 573 574 void rb_free_aux(struct ring_buffer *rb) 575 { 576 if (atomic_dec_and_test(&rb->aux_refcount)) 577 irq_work_queue(&rb->irq_work); 578 } 579 580 static void rb_irq_work(struct irq_work *work) 581 { 582 struct ring_buffer *rb = container_of(work, struct ring_buffer, irq_work); 583 584 if (!atomic_read(&rb->aux_refcount)) 585 __rb_free_aux(rb); 586 587 if (rb->rcu_head.next == (void *)rb) 588 call_rcu(&rb->rcu_head, rb_free_rcu); 589 } 590 591 #ifndef CONFIG_PERF_USE_VMALLOC 592 593 /* 594 * Back perf_mmap() with regular GFP_KERNEL-0 pages. 595 */ 596 597 static struct page * 598 __perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff) 599 { 600 if (pgoff > rb->nr_pages) 601 return NULL; 602 603 if (pgoff == 0) 604 return virt_to_page(rb->user_page); 605 606 return virt_to_page(rb->data_pages[pgoff - 1]); 607 } 608 609 static void *perf_mmap_alloc_page(int cpu) 610 { 611 struct page *page; 612 int node; 613 614 node = (cpu == -1) ? cpu : cpu_to_node(cpu); 615 page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); 616 if (!page) 617 return NULL; 618 619 return page_address(page); 620 } 621 622 struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) 623 { 624 struct ring_buffer *rb; 625 unsigned long size; 626 int i; 627 628 size = sizeof(struct ring_buffer); 629 size += nr_pages * sizeof(void *); 630 631 rb = kzalloc(size, GFP_KERNEL); 632 if (!rb) 633 goto fail; 634 635 rb->user_page = perf_mmap_alloc_page(cpu); 636 if (!rb->user_page) 637 goto fail_user_page; 638 639 for (i = 0; i < nr_pages; i++) { 640 rb->data_pages[i] = perf_mmap_alloc_page(cpu); 641 if (!rb->data_pages[i]) 642 goto fail_data_pages; 643 } 644 645 rb->nr_pages = nr_pages; 646 647 ring_buffer_init(rb, watermark, flags); 648 649 return rb; 650 651 fail_data_pages: 652 for (i--; i >= 0; i--) 653 free_page((unsigned long)rb->data_pages[i]); 654 655 free_page((unsigned long)rb->user_page); 656 657 fail_user_page: 658 kfree(rb); 659 660 fail: 661 return NULL; 662 } 663 664 static void perf_mmap_free_page(unsigned long addr) 665 { 666 struct page *page = virt_to_page((void *)addr); 667 668 page->mapping = NULL; 669 __free_page(page); 670 } 671 672 void rb_free(struct ring_buffer *rb) 673 { 674 int i; 675 676 perf_mmap_free_page((unsigned long)rb->user_page); 677 for (i = 0; i < rb->nr_pages; i++) 678 perf_mmap_free_page((unsigned long)rb->data_pages[i]); 679 kfree(rb); 680 } 681 682 #else 683 static int data_page_nr(struct ring_buffer *rb) 684 { 685 return rb->nr_pages << page_order(rb); 686 } 687 688 static struct page * 689 __perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff) 690 { 691 /* The '>' counts in the user page. */ 692 if (pgoff > data_page_nr(rb)) 693 return NULL; 694 695 return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE); 696 } 697 698 static void perf_mmap_unmark_page(void *addr) 699 { 700 struct page *page = vmalloc_to_page(addr); 701 702 page->mapping = NULL; 703 } 704 705 static void rb_free_work(struct work_struct *work) 706 { 707 struct ring_buffer *rb; 708 void *base; 709 int i, nr; 710 711 rb = container_of(work, struct ring_buffer, work); 712 nr = data_page_nr(rb); 713 714 base = rb->user_page; 715 /* The '<=' counts in the user page. */ 716 for (i = 0; i <= nr; i++) 717 perf_mmap_unmark_page(base + (i * PAGE_SIZE)); 718 719 vfree(base); 720 kfree(rb); 721 } 722 723 void rb_free(struct ring_buffer *rb) 724 { 725 schedule_work(&rb->work); 726 } 727 728 struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) 729 { 730 struct ring_buffer *rb; 731 unsigned long size; 732 void *all_buf; 733 734 size = sizeof(struct ring_buffer); 735 size += sizeof(void *); 736 737 rb = kzalloc(size, GFP_KERNEL); 738 if (!rb) 739 goto fail; 740 741 INIT_WORK(&rb->work, rb_free_work); 742 743 all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE); 744 if (!all_buf) 745 goto fail_all_buf; 746 747 rb->user_page = all_buf; 748 rb->data_pages[0] = all_buf + PAGE_SIZE; 749 rb->page_order = ilog2(nr_pages); 750 rb->nr_pages = !!nr_pages; 751 752 ring_buffer_init(rb, watermark, flags); 753 754 return rb; 755 756 fail_all_buf: 757 kfree(rb); 758 759 fail: 760 return NULL; 761 } 762 763 #endif 764 765 struct page * 766 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff) 767 { 768 if (rb->aux_nr_pages) { 769 /* above AUX space */ 770 if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) 771 return NULL; 772 773 /* AUX space */ 774 if (pgoff >= rb->aux_pgoff) 775 return virt_to_page(rb->aux_pages[pgoff - rb->aux_pgoff]); 776 } 777 778 return __perf_mmap_to_page(rb, pgoff); 779 } 780