18e86e015SThomas Gleixner // SPDX-License-Identifier: GPL-2.0
276369139SFrederic Weisbecker /*
376369139SFrederic Weisbecker * Performance events ring-buffer code:
476369139SFrederic Weisbecker *
576369139SFrederic Weisbecker * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
676369139SFrederic Weisbecker * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
790eec103SPeter Zijlstra * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8d36b6910SAl Viro * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
976369139SFrederic Weisbecker */
1076369139SFrederic Weisbecker
1176369139SFrederic Weisbecker #include <linux/perf_event.h>
1276369139SFrederic Weisbecker #include <linux/vmalloc.h>
1376369139SFrederic Weisbecker #include <linux/slab.h>
1426c86da8SPeter Zijlstra #include <linux/circ_buf.h>
157c60fc0eSJiri Olsa #include <linux/poll.h>
164411ec1dSPeter Zijlstra #include <linux/nospec.h>
1776369139SFrederic Weisbecker
1876369139SFrederic Weisbecker #include "internal.h"
1976369139SFrederic Weisbecker
perf_output_wakeup(struct perf_output_handle * handle)2076369139SFrederic Weisbecker static void perf_output_wakeup(struct perf_output_handle *handle)
2176369139SFrederic Weisbecker {
22a9a08845SLinus Torvalds atomic_set(&handle->rb->poll, EPOLLIN);
2376369139SFrederic Weisbecker
2476369139SFrederic Weisbecker handle->event->pending_wakeup = 1;
25ca6c2132SPeter Zijlstra irq_work_queue(&handle->event->pending_irq);
2676369139SFrederic Weisbecker }
2776369139SFrederic Weisbecker
2876369139SFrederic Weisbecker /*
2976369139SFrederic Weisbecker * We need to ensure a later event_id doesn't publish a head when a former
3076369139SFrederic Weisbecker * event isn't done writing. However since we need to deal with NMIs we
3176369139SFrederic Weisbecker * cannot fully serialize things.
3276369139SFrederic Weisbecker *
3376369139SFrederic Weisbecker * We only publish the head (and generate a wakeup) when the outer-most
3476369139SFrederic Weisbecker * event completes.
3576369139SFrederic Weisbecker */
perf_output_get_handle(struct perf_output_handle * handle)3676369139SFrederic Weisbecker static void perf_output_get_handle(struct perf_output_handle *handle)
3776369139SFrederic Weisbecker {
3856de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb = handle->rb;
3976369139SFrederic Weisbecker
4076369139SFrederic Weisbecker preempt_disable();
415322ea58SPeter Zijlstra
425322ea58SPeter Zijlstra /*
435322ea58SPeter Zijlstra * Avoid an explicit LOAD/STORE such that architectures with memops
445322ea58SPeter Zijlstra * can use them.
455322ea58SPeter Zijlstra */
465322ea58SPeter Zijlstra (*(volatile unsigned int *)&rb->nest)++;
4776369139SFrederic Weisbecker handle->wakeup = local_read(&rb->wakeup);
4876369139SFrederic Weisbecker }
4976369139SFrederic Weisbecker
perf_output_put_handle(struct perf_output_handle * handle)5076369139SFrederic Weisbecker static void perf_output_put_handle(struct perf_output_handle *handle)
5176369139SFrederic Weisbecker {
5256de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb = handle->rb;
5376369139SFrederic Weisbecker unsigned long head;
545322ea58SPeter Zijlstra unsigned int nest;
555322ea58SPeter Zijlstra
565322ea58SPeter Zijlstra /*
575322ea58SPeter Zijlstra * If this isn't the outermost nesting, we don't have to update
585322ea58SPeter Zijlstra * @rb->user_page->data_head.
595322ea58SPeter Zijlstra */
605322ea58SPeter Zijlstra nest = READ_ONCE(rb->nest);
615322ea58SPeter Zijlstra if (nest > 1) {
625322ea58SPeter Zijlstra WRITE_ONCE(rb->nest, nest - 1);
635322ea58SPeter Zijlstra goto out;
645322ea58SPeter Zijlstra }
6576369139SFrederic Weisbecker
6676369139SFrederic Weisbecker again:
673f9fbe9bSPeter Zijlstra /*
683f9fbe9bSPeter Zijlstra * In order to avoid publishing a head value that goes backwards,
693f9fbe9bSPeter Zijlstra * we must ensure the load of @rb->head happens after we've
703f9fbe9bSPeter Zijlstra * incremented @rb->nest.
713f9fbe9bSPeter Zijlstra *
723f9fbe9bSPeter Zijlstra * Otherwise we can observe a @rb->head value before one published
733f9fbe9bSPeter Zijlstra * by an IRQ/NMI happening between the load and the increment.
743f9fbe9bSPeter Zijlstra */
753f9fbe9bSPeter Zijlstra barrier();
7676369139SFrederic Weisbecker head = local_read(&rb->head);
7776369139SFrederic Weisbecker
7876369139SFrederic Weisbecker /*
791b038c6eSYabin Cui * IRQ/NMI can happen here and advance @rb->head, causing our
801b038c6eSYabin Cui * load above to be stale.
8176369139SFrederic Weisbecker */
8276369139SFrederic Weisbecker
831b038c6eSYabin Cui /*
84bf378d34SPeter Zijlstra * Since the mmap() consumer (userspace) can run on a different CPU:
85bf378d34SPeter Zijlstra *
86bf378d34SPeter Zijlstra * kernel user
87bf378d34SPeter Zijlstra *
88c7f2e3cdSPeter Zijlstra * if (LOAD ->data_tail) { LOAD ->data_head
89c7f2e3cdSPeter Zijlstra * (A) smp_rmb() (C)
90c7f2e3cdSPeter Zijlstra * STORE $data LOAD $data
91bf378d34SPeter Zijlstra * smp_wmb() (B) smp_mb() (D)
92c7f2e3cdSPeter Zijlstra * STORE ->data_head STORE ->data_tail
93c7f2e3cdSPeter Zijlstra * }
94bf378d34SPeter Zijlstra *
95bf378d34SPeter Zijlstra * Where A pairs with D, and B pairs with C.
96bf378d34SPeter Zijlstra *
97c7f2e3cdSPeter Zijlstra * In our case (A) is a control dependency that separates the load of
98c7f2e3cdSPeter Zijlstra * the ->data_tail and the stores of $data. In case ->data_tail
99c7f2e3cdSPeter Zijlstra * indicates there is no room in the buffer to store $data we do not.
100bf378d34SPeter Zijlstra *
101c7f2e3cdSPeter Zijlstra * D needs to be a full barrier since it separates the data READ
102bf378d34SPeter Zijlstra * from the tail WRITE.
103bf378d34SPeter Zijlstra *
104bf378d34SPeter Zijlstra * For B a WMB is sufficient since it separates two WRITEs, and for C
105bf378d34SPeter Zijlstra * an RMB is sufficient since it separates two READs.
106bf378d34SPeter Zijlstra *
107bf378d34SPeter Zijlstra * See perf_output_begin().
10876369139SFrederic Weisbecker */
109c7f2e3cdSPeter Zijlstra smp_wmb(); /* B, matches C */
1104d839dd9SPeter Zijlstra WRITE_ONCE(rb->user_page->data_head, head);
11176369139SFrederic Weisbecker
11276369139SFrederic Weisbecker /*
1131b038c6eSYabin Cui * We must publish the head before decrementing the nest count,
1141b038c6eSYabin Cui * otherwise an IRQ/NMI can publish a more recent head value and our
1151b038c6eSYabin Cui * write will (temporarily) publish a stale value.
11676369139SFrederic Weisbecker */
1171b038c6eSYabin Cui barrier();
1185322ea58SPeter Zijlstra WRITE_ONCE(rb->nest, 0);
1191b038c6eSYabin Cui
1201b038c6eSYabin Cui /*
1211b038c6eSYabin Cui * Ensure we decrement @rb->nest before we validate the @rb->head.
1221b038c6eSYabin Cui * Otherwise we cannot be sure we caught the 'last' nested update.
1231b038c6eSYabin Cui */
1241b038c6eSYabin Cui barrier();
12576369139SFrederic Weisbecker if (unlikely(head != local_read(&rb->head))) {
1265322ea58SPeter Zijlstra WRITE_ONCE(rb->nest, 1);
12776369139SFrederic Weisbecker goto again;
12876369139SFrederic Weisbecker }
12976369139SFrederic Weisbecker
13076369139SFrederic Weisbecker if (handle->wakeup != local_read(&rb->wakeup))
13176369139SFrederic Weisbecker perf_output_wakeup(handle);
13276369139SFrederic Weisbecker
13376369139SFrederic Weisbecker out:
13476369139SFrederic Weisbecker preempt_enable();
13576369139SFrederic Weisbecker }
13676369139SFrederic Weisbecker
13757d6a793SMathieu Malaterre static __always_inline bool
ring_buffer_has_space(unsigned long head,unsigned long tail,unsigned long data_size,unsigned int size,bool backward)138d1b26c70SWang Nan ring_buffer_has_space(unsigned long head, unsigned long tail,
139d1b26c70SWang Nan unsigned long data_size, unsigned int size,
140d1b26c70SWang Nan bool backward)
141d1b26c70SWang Nan {
142d1b26c70SWang Nan if (!backward)
143d1b26c70SWang Nan return CIRC_SPACE(head, tail, data_size) >= size;
144d1b26c70SWang Nan else
145d1b26c70SWang Nan return CIRC_SPACE(tail, head, data_size) >= size;
146d1b26c70SWang Nan }
147d1b26c70SWang Nan
14857d6a793SMathieu Malaterre static __always_inline int
__perf_output_begin(struct perf_output_handle * handle,struct perf_sample_data * data,struct perf_event * event,unsigned int size,bool backward)149d1b26c70SWang Nan __perf_output_begin(struct perf_output_handle *handle,
150267fb273SPeter Zijlstra struct perf_sample_data *data,
151d1b26c70SWang Nan struct perf_event *event, unsigned int size,
152d1b26c70SWang Nan bool backward)
15376369139SFrederic Weisbecker {
15456de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb;
15576369139SFrederic Weisbecker unsigned long tail, offset, head;
156524feca5SPeter Zijlstra int have_lost, page_shift;
15776369139SFrederic Weisbecker struct {
15876369139SFrederic Weisbecker struct perf_event_header header;
15976369139SFrederic Weisbecker u64 id;
16076369139SFrederic Weisbecker u64 lost;
16176369139SFrederic Weisbecker } lost_event;
16276369139SFrederic Weisbecker
16376369139SFrederic Weisbecker rcu_read_lock();
16476369139SFrederic Weisbecker /*
16576369139SFrederic Weisbecker * For inherited events we send all the output towards the parent.
16676369139SFrederic Weisbecker */
16776369139SFrederic Weisbecker if (event->parent)
16876369139SFrederic Weisbecker event = event->parent;
16976369139SFrederic Weisbecker
17076369139SFrederic Weisbecker rb = rcu_dereference(event->rb);
171c72b42a3SPeter Zijlstra if (unlikely(!rb))
172c72b42a3SPeter Zijlstra goto out;
173c72b42a3SPeter Zijlstra
17486e7972fSWang Nan if (unlikely(rb->paused)) {
175119a784cSNamhyung Kim if (rb->nr_pages) {
17686e7972fSWang Nan local_inc(&rb->lost);
177119a784cSNamhyung Kim atomic64_inc(&event->lost_samples);
178119a784cSNamhyung Kim }
17976369139SFrederic Weisbecker goto out;
18086e7972fSWang Nan }
18176369139SFrederic Weisbecker
18276369139SFrederic Weisbecker handle->rb = rb;
18376369139SFrederic Weisbecker handle->event = event;
18476369139SFrederic Weisbecker
18576369139SFrederic Weisbecker have_lost = local_read(&rb->lost);
186c72b42a3SPeter Zijlstra if (unlikely(have_lost)) {
187d20a973fSPeter Zijlstra size += sizeof(lost_event);
188d20a973fSPeter Zijlstra if (event->attr.sample_id_all)
189d20a973fSPeter Zijlstra size += event->id_header_size;
19076369139SFrederic Weisbecker }
19176369139SFrederic Weisbecker
19276369139SFrederic Weisbecker perf_output_get_handle(handle);
19376369139SFrederic Weisbecker
1941af61adbSUros Bizjak offset = local_read(&rb->head);
19576369139SFrederic Weisbecker do {
1961af61adbSUros Bizjak head = offset;
197105ff3cbSLinus Torvalds tail = READ_ONCE(rb->user_page->data_tail);
198d1b26c70SWang Nan if (!rb->overwrite) {
199d1b26c70SWang Nan if (unlikely(!ring_buffer_has_space(head, tail,
200d1b26c70SWang Nan perf_data_size(rb),
201d1b26c70SWang Nan size, backward)))
20276369139SFrederic Weisbecker goto fail;
203d1b26c70SWang Nan }
204c7f2e3cdSPeter Zijlstra
205c7f2e3cdSPeter Zijlstra /*
206c7f2e3cdSPeter Zijlstra * The above forms a control dependency barrier separating the
207c7f2e3cdSPeter Zijlstra * @tail load above from the data stores below. Since the @tail
208c7f2e3cdSPeter Zijlstra * load is required to compute the branch to fail below.
209c7f2e3cdSPeter Zijlstra *
210c7f2e3cdSPeter Zijlstra * A, matches D; the full memory barrier userspace SHOULD issue
211c7f2e3cdSPeter Zijlstra * after reading the data and before storing the new tail
212c7f2e3cdSPeter Zijlstra * position.
213c7f2e3cdSPeter Zijlstra *
214c7f2e3cdSPeter Zijlstra * See perf_output_put_handle().
215c7f2e3cdSPeter Zijlstra */
216c7f2e3cdSPeter Zijlstra
217d1b26c70SWang Nan if (!backward)
21826c86da8SPeter Zijlstra head += size;
219d1b26c70SWang Nan else
220d1b26c70SWang Nan head -= size;
2211af61adbSUros Bizjak } while (!local_try_cmpxchg(&rb->head, &offset, head));
22276369139SFrederic Weisbecker
223d1b26c70SWang Nan if (backward) {
224d1b26c70SWang Nan offset = head;
225d1b26c70SWang Nan head = (u64)(-head);
226d1b26c70SWang Nan }
227d1b26c70SWang Nan
22885f59edfSPeter Zijlstra /*
229c7f2e3cdSPeter Zijlstra * We rely on the implied barrier() by local_cmpxchg() to ensure
230c7f2e3cdSPeter Zijlstra * none of the data stores below can be lifted up by the compiler.
23185f59edfSPeter Zijlstra */
23285f59edfSPeter Zijlstra
233c72b42a3SPeter Zijlstra if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))
23476369139SFrederic Weisbecker local_add(rb->watermark, &rb->wakeup);
23576369139SFrederic Weisbecker
236524feca5SPeter Zijlstra page_shift = PAGE_SHIFT + page_order(rb);
237524feca5SPeter Zijlstra
238524feca5SPeter Zijlstra handle->page = (offset >> page_shift) & (rb->nr_pages - 1);
239524feca5SPeter Zijlstra offset &= (1UL << page_shift) - 1;
240524feca5SPeter Zijlstra handle->addr = rb->data_pages[handle->page] + offset;
241524feca5SPeter Zijlstra handle->size = (1UL << page_shift) - offset;
24276369139SFrederic Weisbecker
243c72b42a3SPeter Zijlstra if (unlikely(have_lost)) {
244d20a973fSPeter Zijlstra lost_event.header.size = sizeof(lost_event);
24576369139SFrederic Weisbecker lost_event.header.type = PERF_RECORD_LOST;
24676369139SFrederic Weisbecker lost_event.header.misc = 0;
24776369139SFrederic Weisbecker lost_event.id = event->id;
24876369139SFrederic Weisbecker lost_event.lost = local_xchg(&rb->lost, 0);
24976369139SFrederic Weisbecker
250267fb273SPeter Zijlstra /* XXX mostly redundant; @data is already fully initializes */
251267fb273SPeter Zijlstra perf_event_header__init_id(&lost_event.header, data, event);
25276369139SFrederic Weisbecker perf_output_put(handle, lost_event);
253267fb273SPeter Zijlstra perf_event__output_id_sample(event, handle, data);
25476369139SFrederic Weisbecker }
25576369139SFrederic Weisbecker
25676369139SFrederic Weisbecker return 0;
25776369139SFrederic Weisbecker
25876369139SFrederic Weisbecker fail:
25976369139SFrederic Weisbecker local_inc(&rb->lost);
260119a784cSNamhyung Kim atomic64_inc(&event->lost_samples);
26176369139SFrederic Weisbecker perf_output_put_handle(handle);
26276369139SFrederic Weisbecker out:
26376369139SFrederic Weisbecker rcu_read_unlock();
26476369139SFrederic Weisbecker
26576369139SFrederic Weisbecker return -ENOSPC;
26676369139SFrederic Weisbecker }
26776369139SFrederic Weisbecker
perf_output_begin_forward(struct perf_output_handle * handle,struct perf_sample_data * data,struct perf_event * event,unsigned int size)2689ecda41aSWang Nan int perf_output_begin_forward(struct perf_output_handle *handle,
269267fb273SPeter Zijlstra struct perf_sample_data *data,
270d1b26c70SWang Nan struct perf_event *event, unsigned int size)
271d1b26c70SWang Nan {
272267fb273SPeter Zijlstra return __perf_output_begin(handle, data, event, size, false);
273d1b26c70SWang Nan }
274d1b26c70SWang Nan
perf_output_begin_backward(struct perf_output_handle * handle,struct perf_sample_data * data,struct perf_event * event,unsigned int size)2759ecda41aSWang Nan int perf_output_begin_backward(struct perf_output_handle *handle,
276267fb273SPeter Zijlstra struct perf_sample_data *data,
2779ecda41aSWang Nan struct perf_event *event, unsigned int size)
2789ecda41aSWang Nan {
279267fb273SPeter Zijlstra return __perf_output_begin(handle, data, event, size, true);
2809ecda41aSWang Nan }
2819ecda41aSWang Nan
perf_output_begin(struct perf_output_handle * handle,struct perf_sample_data * data,struct perf_event * event,unsigned int size)2829ecda41aSWang Nan int perf_output_begin(struct perf_output_handle *handle,
283267fb273SPeter Zijlstra struct perf_sample_data *data,
2849ecda41aSWang Nan struct perf_event *event, unsigned int size)
2859ecda41aSWang Nan {
2869ecda41aSWang Nan
287267fb273SPeter Zijlstra return __perf_output_begin(handle, data, event, size,
2889ecda41aSWang Nan unlikely(is_write_backward(event)));
2899ecda41aSWang Nan }
2909ecda41aSWang Nan
perf_output_copy(struct perf_output_handle * handle,const void * buf,unsigned int len)29191d7753aSFrederic Weisbecker unsigned int perf_output_copy(struct perf_output_handle *handle,
29276369139SFrederic Weisbecker const void *buf, unsigned int len)
29376369139SFrederic Weisbecker {
29491d7753aSFrederic Weisbecker return __output_copy(handle, buf, len);
29576369139SFrederic Weisbecker }
29676369139SFrederic Weisbecker
perf_output_skip(struct perf_output_handle * handle,unsigned int len)2975685e0ffSJiri Olsa unsigned int perf_output_skip(struct perf_output_handle *handle,
2985685e0ffSJiri Olsa unsigned int len)
2995685e0ffSJiri Olsa {
3005685e0ffSJiri Olsa return __output_skip(handle, NULL, len);
3015685e0ffSJiri Olsa }
3025685e0ffSJiri Olsa
perf_output_end(struct perf_output_handle * handle)30376369139SFrederic Weisbecker void perf_output_end(struct perf_output_handle *handle)
30476369139SFrederic Weisbecker {
30576369139SFrederic Weisbecker perf_output_put_handle(handle);
30676369139SFrederic Weisbecker rcu_read_unlock();
30776369139SFrederic Weisbecker }
30876369139SFrederic Weisbecker
30976369139SFrederic Weisbecker static void
ring_buffer_init(struct perf_buffer * rb,long watermark,int flags)31056de4e8fSSteven Rostedt (VMware) ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)
31176369139SFrederic Weisbecker {
31276369139SFrederic Weisbecker long max_size = perf_data_size(rb);
31376369139SFrederic Weisbecker
31476369139SFrederic Weisbecker if (watermark)
31576369139SFrederic Weisbecker rb->watermark = min(max_size, watermark);
31676369139SFrederic Weisbecker
31776369139SFrederic Weisbecker if (!rb->watermark)
31876369139SFrederic Weisbecker rb->watermark = max_size / 2;
31976369139SFrederic Weisbecker
32076369139SFrederic Weisbecker if (flags & RING_BUFFER_WRITABLE)
321dd9c086dSStephane Eranian rb->overwrite = 0;
322dd9c086dSStephane Eranian else
323dd9c086dSStephane Eranian rb->overwrite = 1;
32476369139SFrederic Weisbecker
325fecb8ed2SElena Reshetova refcount_set(&rb->refcount, 1);
32610c6db11SPeter Zijlstra
32710c6db11SPeter Zijlstra INIT_LIST_HEAD(&rb->event_list);
32810c6db11SPeter Zijlstra spin_lock_init(&rb->event_lock);
32986e7972fSWang Nan
33086e7972fSWang Nan /*
33186e7972fSWang Nan * perf_output_begin() only checks rb->paused, therefore
33286e7972fSWang Nan * rb->paused must be true if we have no pages for output.
33386e7972fSWang Nan */
33486e7972fSWang Nan if (!rb->nr_pages)
33586e7972fSWang Nan rb->paused = 1;
336*c4b69beeSPeter Zijlstra
337*c4b69beeSPeter Zijlstra mutex_init(&rb->aux_mutex);
33876369139SFrederic Weisbecker }
33976369139SFrederic Weisbecker
perf_aux_output_flag(struct perf_output_handle * handle,u64 flags)340f4c0b0aaSWill Deacon void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)
341f4c0b0aaSWill Deacon {
342f4c0b0aaSWill Deacon /*
343f4c0b0aaSWill Deacon * OVERWRITE is determined by perf_aux_output_end() and can't
344f4c0b0aaSWill Deacon * be passed in directly.
345f4c0b0aaSWill Deacon */
346f4c0b0aaSWill Deacon if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE))
347f4c0b0aaSWill Deacon return;
348f4c0b0aaSWill Deacon
349f4c0b0aaSWill Deacon handle->aux_flags |= flags;
350f4c0b0aaSWill Deacon }
351f4c0b0aaSWill Deacon EXPORT_SYMBOL_GPL(perf_aux_output_flag);
352f4c0b0aaSWill Deacon
353fdc26706SAlexander Shishkin /*
354fdc26706SAlexander Shishkin * This is called before hardware starts writing to the AUX area to
355fdc26706SAlexander Shishkin * obtain an output handle and make sure there's room in the buffer.
356fdc26706SAlexander Shishkin * When the capture completes, call perf_aux_output_end() to commit
357fdc26706SAlexander Shishkin * the recorded data to the buffer.
358fdc26706SAlexander Shishkin *
359fdc26706SAlexander Shishkin * The ordering is similar to that of perf_output_{begin,end}, with
360fdc26706SAlexander Shishkin * the exception of (B), which should be taken care of by the pmu
361fdc26706SAlexander Shishkin * driver, since ordering rules will differ depending on hardware.
362af5bb4edSAlexander Shishkin *
363af5bb4edSAlexander Shishkin * Call this from pmu::start(); see the comment in perf_aux_output_end()
364af5bb4edSAlexander Shishkin * about its use in pmu callbacks. Both can also be called from the PMI
365af5bb4edSAlexander Shishkin * handler if needed.
366fdc26706SAlexander Shishkin */
perf_aux_output_begin(struct perf_output_handle * handle,struct perf_event * event)367fdc26706SAlexander Shishkin void *perf_aux_output_begin(struct perf_output_handle *handle,
368fdc26706SAlexander Shishkin struct perf_event *event)
369fdc26706SAlexander Shishkin {
370fdc26706SAlexander Shishkin struct perf_event *output_event = event;
371fdc26706SAlexander Shishkin unsigned long aux_head, aux_tail;
37256de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb;
3735322ea58SPeter Zijlstra unsigned int nest;
374fdc26706SAlexander Shishkin
375fdc26706SAlexander Shishkin if (output_event->parent)
376fdc26706SAlexander Shishkin output_event = output_event->parent;
377fdc26706SAlexander Shishkin
378fdc26706SAlexander Shishkin /*
379fdc26706SAlexander Shishkin * Since this will typically be open across pmu::add/pmu::del, we
380fdc26706SAlexander Shishkin * grab ring_buffer's refcount instead of holding rcu read lock
381fdc26706SAlexander Shishkin * to make sure it doesn't disappear under us.
382fdc26706SAlexander Shishkin */
383fdc26706SAlexander Shishkin rb = ring_buffer_get(output_event);
384fdc26706SAlexander Shishkin if (!rb)
385fdc26706SAlexander Shishkin return NULL;
386fdc26706SAlexander Shishkin
387b79ccaddSAlexander Shishkin if (!rb_has_aux(rb))
388fdc26706SAlexander Shishkin goto err;
389fdc26706SAlexander Shishkin
390fdc26706SAlexander Shishkin /*
391b79ccaddSAlexander Shishkin * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
392b79ccaddSAlexander Shishkin * about to get freed, so we leave immediately.
393b79ccaddSAlexander Shishkin *
394b79ccaddSAlexander Shishkin * Checking rb::aux_mmap_count and rb::refcount has to be done in
395b79ccaddSAlexander Shishkin * the same order, see perf_mmap_close. Otherwise we end up freeing
396b79ccaddSAlexander Shishkin * aux pages in this path, which is a bug, because in_atomic().
397dcb10a96SAlexander Shishkin */
398dcb10a96SAlexander Shishkin if (!atomic_read(&rb->aux_mmap_count))
399b79ccaddSAlexander Shishkin goto err;
400b79ccaddSAlexander Shishkin
401ca3bb3d0SElena Reshetova if (!refcount_inc_not_zero(&rb->aux_refcount))
402b79ccaddSAlexander Shishkin goto err;
403dcb10a96SAlexander Shishkin
4045322ea58SPeter Zijlstra nest = READ_ONCE(rb->aux_nest);
405dcb10a96SAlexander Shishkin /*
406fdc26706SAlexander Shishkin * Nesting is not supported for AUX area, make sure nested
407fdc26706SAlexander Shishkin * writers are caught early
408fdc26706SAlexander Shishkin */
4095322ea58SPeter Zijlstra if (WARN_ON_ONCE(nest))
410fdc26706SAlexander Shishkin goto err_put;
411fdc26706SAlexander Shishkin
4125322ea58SPeter Zijlstra WRITE_ONCE(rb->aux_nest, nest + 1);
4135322ea58SPeter Zijlstra
4142ab346cfSWill Deacon aux_head = rb->aux_head;
415fdc26706SAlexander Shishkin
416fdc26706SAlexander Shishkin handle->rb = rb;
417fdc26706SAlexander Shishkin handle->event = event;
418fdc26706SAlexander Shishkin handle->head = aux_head;
4192023a0d2SAlexander Shishkin handle->size = 0;
420f4c0b0aaSWill Deacon handle->aux_flags = 0;
4212023a0d2SAlexander Shishkin
4222023a0d2SAlexander Shishkin /*
4232023a0d2SAlexander Shishkin * In overwrite mode, AUX data stores do not depend on aux_tail,
4242023a0d2SAlexander Shishkin * therefore (A) control dependency barrier does not exist. The
4252023a0d2SAlexander Shishkin * (B) <-> (C) ordering is still observed by the pmu driver.
4262023a0d2SAlexander Shishkin */
4272023a0d2SAlexander Shishkin if (!rb->aux_overwrite) {
4286aa7de05SMark Rutland aux_tail = READ_ONCE(rb->user_page->aux_tail);
4292ab346cfSWill Deacon handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
430fdc26706SAlexander Shishkin if (aux_head - aux_tail < perf_aux_size(rb))
431fdc26706SAlexander Shishkin handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
432fdc26706SAlexander Shishkin
433fdc26706SAlexander Shishkin /*
434fdc26706SAlexander Shishkin * handle->size computation depends on aux_tail load; this forms a
435fdc26706SAlexander Shishkin * control dependency barrier separating aux_tail load from aux data
436fdc26706SAlexander Shishkin * store that will be enabled on successful return
437fdc26706SAlexander Shishkin */
438fdc26706SAlexander Shishkin if (!handle->size) { /* A, matches D */
4391d54ad94SPeter Zijlstra event->pending_disable = smp_processor_id();
440fdc26706SAlexander Shishkin perf_output_wakeup(handle);
4415322ea58SPeter Zijlstra WRITE_ONCE(rb->aux_nest, 0);
442fdc26706SAlexander Shishkin goto err_put;
443fdc26706SAlexander Shishkin }
4442023a0d2SAlexander Shishkin }
445fdc26706SAlexander Shishkin
446fdc26706SAlexander Shishkin return handle->rb->aux_priv;
447fdc26706SAlexander Shishkin
448fdc26706SAlexander Shishkin err_put:
449af5bb4edSAlexander Shishkin /* can't be last */
450fdc26706SAlexander Shishkin rb_free_aux(rb);
451fdc26706SAlexander Shishkin
452fdc26706SAlexander Shishkin err:
45395ff4ca2SAlexander Shishkin ring_buffer_put(rb);
454fdc26706SAlexander Shishkin handle->event = NULL;
455fdc26706SAlexander Shishkin
456fdc26706SAlexander Shishkin return NULL;
457fdc26706SAlexander Shishkin }
458bc1d2020SWill Deacon EXPORT_SYMBOL_GPL(perf_aux_output_begin);
459fdc26706SAlexander Shishkin
rb_need_aux_wakeup(struct perf_buffer * rb)46056de4e8fSSteven Rostedt (VMware) static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb)
461441430ebSAlexander Shishkin {
462441430ebSAlexander Shishkin if (rb->aux_overwrite)
463441430ebSAlexander Shishkin return false;
464441430ebSAlexander Shishkin
465441430ebSAlexander Shishkin if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {
466441430ebSAlexander Shishkin rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);
467441430ebSAlexander Shishkin return true;
468441430ebSAlexander Shishkin }
469441430ebSAlexander Shishkin
470441430ebSAlexander Shishkin return false;
471441430ebSAlexander Shishkin }
472441430ebSAlexander Shishkin
473fdc26706SAlexander Shishkin /*
474fdc26706SAlexander Shishkin * Commit the data written by hardware into the ring buffer by adjusting
475fdc26706SAlexander Shishkin * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
476fdc26706SAlexander Shishkin * pmu driver's responsibility to observe ordering rules of the hardware,
477fdc26706SAlexander Shishkin * so that all the data is externally visible before this is called.
478af5bb4edSAlexander Shishkin *
479af5bb4edSAlexander Shishkin * Note: this has to be called from pmu::stop() callback, as the assumption
480af5bb4edSAlexander Shishkin * of the AUX buffer management code is that after pmu::stop(), the AUX
481af5bb4edSAlexander Shishkin * transaction must be stopped and therefore drop the AUX reference count.
482fdc26706SAlexander Shishkin */
perf_aux_output_end(struct perf_output_handle * handle,unsigned long size)483f4c0b0aaSWill Deacon void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
484fdc26706SAlexander Shishkin {
485ae0c2d99SAlexander Shishkin bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);
48656de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb = handle->rb;
4872023a0d2SAlexander Shishkin unsigned long aux_head;
488fdc26706SAlexander Shishkin
4892023a0d2SAlexander Shishkin /* in overwrite mode, driver provides aux_head via handle */
4902023a0d2SAlexander Shishkin if (rb->aux_overwrite) {
491f4c0b0aaSWill Deacon handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;
4922023a0d2SAlexander Shishkin
4932023a0d2SAlexander Shishkin aux_head = handle->head;
4942ab346cfSWill Deacon rb->aux_head = aux_head;
4952023a0d2SAlexander Shishkin } else {
496f4c0b0aaSWill Deacon handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;
497f4c0b0aaSWill Deacon
4982ab346cfSWill Deacon aux_head = rb->aux_head;
4992ab346cfSWill Deacon rb->aux_head += size;
5002023a0d2SAlexander Shishkin }
501fdc26706SAlexander Shishkin
502fdc26706SAlexander Shishkin /*
503fdc26706SAlexander Shishkin * Only send RECORD_AUX if we have something useful to communicate
5041627314fSAlexander Shishkin *
5051627314fSAlexander Shishkin * Note: the OVERWRITE records by themselves are not considered
5061627314fSAlexander Shishkin * useful, as they don't communicate any *new* information,
5071627314fSAlexander Shishkin * aside from the short-lived offset, that becomes history at
5081627314fSAlexander Shishkin * the next event sched-in and therefore isn't useful.
5091627314fSAlexander Shishkin * The userspace that needs to copy out AUX data in overwrite
5101627314fSAlexander Shishkin * mode should know to use user_page::aux_head for the actual
5111627314fSAlexander Shishkin * offset. So, from now on we don't output AUX records that
5121627314fSAlexander Shishkin * have *only* OVERWRITE flag set.
513fdc26706SAlexander Shishkin */
514339bc418SAlexander Shishkin if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))
515f4c0b0aaSWill Deacon perf_event_aux_event(handle->event, aux_head, size,
516f4c0b0aaSWill Deacon handle->aux_flags);
517fdc26706SAlexander Shishkin
5184d839dd9SPeter Zijlstra WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
519441430ebSAlexander Shishkin if (rb_need_aux_wakeup(rb))
5203f56e687SAlexander Shishkin wakeup = true;
5213f56e687SAlexander Shishkin
5223f56e687SAlexander Shishkin if (wakeup) {
523f4c0b0aaSWill Deacon if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)
5241d54ad94SPeter Zijlstra handle->event->pending_disable = smp_processor_id();
5253f56e687SAlexander Shishkin perf_output_wakeup(handle);
5263f56e687SAlexander Shishkin }
5273f56e687SAlexander Shishkin
528fdc26706SAlexander Shishkin handle->event = NULL;
529fdc26706SAlexander Shishkin
5305322ea58SPeter Zijlstra WRITE_ONCE(rb->aux_nest, 0);
531af5bb4edSAlexander Shishkin /* can't be last */
532fdc26706SAlexander Shishkin rb_free_aux(rb);
53395ff4ca2SAlexander Shishkin ring_buffer_put(rb);
534fdc26706SAlexander Shishkin }
535bc1d2020SWill Deacon EXPORT_SYMBOL_GPL(perf_aux_output_end);
536fdc26706SAlexander Shishkin
537fdc26706SAlexander Shishkin /*
538fdc26706SAlexander Shishkin * Skip over a given number of bytes in the AUX buffer, due to, for example,
539fdc26706SAlexander Shishkin * hardware's alignment constraints.
540fdc26706SAlexander Shishkin */
perf_aux_output_skip(struct perf_output_handle * handle,unsigned long size)541fdc26706SAlexander Shishkin int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)
542fdc26706SAlexander Shishkin {
54356de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb = handle->rb;
544fdc26706SAlexander Shishkin
545fdc26706SAlexander Shishkin if (size > handle->size)
546fdc26706SAlexander Shishkin return -ENOSPC;
547fdc26706SAlexander Shishkin
5482ab346cfSWill Deacon rb->aux_head += size;
549fdc26706SAlexander Shishkin
5504d839dd9SPeter Zijlstra WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
551441430ebSAlexander Shishkin if (rb_need_aux_wakeup(rb)) {
5521a594131SAlexander Shishkin perf_output_wakeup(handle);
5532ab346cfSWill Deacon handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
5541a594131SAlexander Shishkin }
5551a594131SAlexander Shishkin
5562ab346cfSWill Deacon handle->head = rb->aux_head;
557fdc26706SAlexander Shishkin handle->size -= size;
558fdc26706SAlexander Shishkin
559fdc26706SAlexander Shishkin return 0;
560fdc26706SAlexander Shishkin }
561bc1d2020SWill Deacon EXPORT_SYMBOL_GPL(perf_aux_output_skip);
562fdc26706SAlexander Shishkin
perf_get_aux(struct perf_output_handle * handle)563fdc26706SAlexander Shishkin void *perf_get_aux(struct perf_output_handle *handle)
564fdc26706SAlexander Shishkin {
565fdc26706SAlexander Shishkin /* this is only valid between perf_aux_output_begin and *_end */
566fdc26706SAlexander Shishkin if (!handle->event)
567fdc26706SAlexander Shishkin return NULL;
568fdc26706SAlexander Shishkin
569fdc26706SAlexander Shishkin return handle->rb->aux_priv;
570fdc26706SAlexander Shishkin }
571bc1d2020SWill Deacon EXPORT_SYMBOL_GPL(perf_get_aux);
572fdc26706SAlexander Shishkin
573a4faf00dSAlexander Shishkin /*
574a4faf00dSAlexander Shishkin * Copy out AUX data from an AUX handle.
575a4faf00dSAlexander Shishkin */
perf_output_copy_aux(struct perf_output_handle * aux_handle,struct perf_output_handle * handle,unsigned long from,unsigned long to)576a4faf00dSAlexander Shishkin long perf_output_copy_aux(struct perf_output_handle *aux_handle,
577a4faf00dSAlexander Shishkin struct perf_output_handle *handle,
578a4faf00dSAlexander Shishkin unsigned long from, unsigned long to)
579a4faf00dSAlexander Shishkin {
58056de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb = aux_handle->rb;
581a4faf00dSAlexander Shishkin unsigned long tocopy, remainder, len = 0;
582a4faf00dSAlexander Shishkin void *addr;
583a4faf00dSAlexander Shishkin
584a4faf00dSAlexander Shishkin from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
585a4faf00dSAlexander Shishkin to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
586a4faf00dSAlexander Shishkin
587a4faf00dSAlexander Shishkin do {
588a4faf00dSAlexander Shishkin tocopy = PAGE_SIZE - offset_in_page(from);
589a4faf00dSAlexander Shishkin if (to > from)
590a4faf00dSAlexander Shishkin tocopy = min(tocopy, to - from);
591a4faf00dSAlexander Shishkin if (!tocopy)
592a4faf00dSAlexander Shishkin break;
593a4faf00dSAlexander Shishkin
594a4faf00dSAlexander Shishkin addr = rb->aux_pages[from >> PAGE_SHIFT];
595a4faf00dSAlexander Shishkin addr += offset_in_page(from);
596a4faf00dSAlexander Shishkin
597a4faf00dSAlexander Shishkin remainder = perf_output_copy(handle, addr, tocopy);
598a4faf00dSAlexander Shishkin if (remainder)
599a4faf00dSAlexander Shishkin return -EFAULT;
600a4faf00dSAlexander Shishkin
601a4faf00dSAlexander Shishkin len += tocopy;
602a4faf00dSAlexander Shishkin from += tocopy;
603a4faf00dSAlexander Shishkin from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
604a4faf00dSAlexander Shishkin } while (to != from);
605a4faf00dSAlexander Shishkin
606a4faf00dSAlexander Shishkin return len;
607a4faf00dSAlexander Shishkin }
608a4faf00dSAlexander Shishkin
6090a4e38e6SAlexander Shishkin #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
6100a4e38e6SAlexander Shishkin
rb_alloc_aux_page(int node,int order)6110a4e38e6SAlexander Shishkin static struct page *rb_alloc_aux_page(int node, int order)
6120a4e38e6SAlexander Shishkin {
6130a4e38e6SAlexander Shishkin struct page *page;
6140a4e38e6SAlexander Shishkin
61523baf831SKirill A. Shutemov if (order > MAX_ORDER)
61623baf831SKirill A. Shutemov order = MAX_ORDER;
6170a4e38e6SAlexander Shishkin
6180a4e38e6SAlexander Shishkin do {
6190a4e38e6SAlexander Shishkin page = alloc_pages_node(node, PERF_AUX_GFP, order);
6200a4e38e6SAlexander Shishkin } while (!page && order--);
6210a4e38e6SAlexander Shishkin
6220a4e38e6SAlexander Shishkin if (page && order) {
6230a4e38e6SAlexander Shishkin /*
624c2ad6b51SAlexander Shishkin * Communicate the allocation size to the driver:
625c2ad6b51SAlexander Shishkin * if we managed to secure a high-order allocation,
626c2ad6b51SAlexander Shishkin * set its first page's private to this order;
627c2ad6b51SAlexander Shishkin * !PagePrivate(page) means it's just a normal page.
6280a4e38e6SAlexander Shishkin */
6290a4e38e6SAlexander Shishkin split_page(page, order);
6300a4e38e6SAlexander Shishkin SetPagePrivate(page);
6310a4e38e6SAlexander Shishkin set_page_private(page, order);
6320a4e38e6SAlexander Shishkin }
6330a4e38e6SAlexander Shishkin
6340a4e38e6SAlexander Shishkin return page;
6350a4e38e6SAlexander Shishkin }
6360a4e38e6SAlexander Shishkin
rb_free_aux_page(struct perf_buffer * rb,int idx)63756de4e8fSSteven Rostedt (VMware) static void rb_free_aux_page(struct perf_buffer *rb, int idx)
6380a4e38e6SAlexander Shishkin {
6390a4e38e6SAlexander Shishkin struct page *page = virt_to_page(rb->aux_pages[idx]);
6400a4e38e6SAlexander Shishkin
6410a4e38e6SAlexander Shishkin ClearPagePrivate(page);
6420a4e38e6SAlexander Shishkin page->mapping = NULL;
6430a4e38e6SAlexander Shishkin __free_page(page);
6440a4e38e6SAlexander Shishkin }
6450a4e38e6SAlexander Shishkin
__rb_free_aux(struct perf_buffer * rb)64656de4e8fSSteven Rostedt (VMware) static void __rb_free_aux(struct perf_buffer *rb)
64745c815f0SAlexander Shishkin {
64845c815f0SAlexander Shishkin int pg;
64945c815f0SAlexander Shishkin
65095ff4ca2SAlexander Shishkin /*
65195ff4ca2SAlexander Shishkin * Should never happen, the last reference should be dropped from
65295ff4ca2SAlexander Shishkin * perf_mmap_close() path, which first stops aux transactions (which
65395ff4ca2SAlexander Shishkin * in turn are the atomic holders of aux_refcount) and then does the
65495ff4ca2SAlexander Shishkin * last rb_free_aux().
65595ff4ca2SAlexander Shishkin */
65695ff4ca2SAlexander Shishkin WARN_ON_ONCE(in_atomic());
65795ff4ca2SAlexander Shishkin
65845c815f0SAlexander Shishkin if (rb->aux_priv) {
65945c815f0SAlexander Shishkin rb->free_aux(rb->aux_priv);
66045c815f0SAlexander Shishkin rb->free_aux = NULL;
66145c815f0SAlexander Shishkin rb->aux_priv = NULL;
66245c815f0SAlexander Shishkin }
66345c815f0SAlexander Shishkin
66445c815f0SAlexander Shishkin if (rb->aux_nr_pages) {
66545c815f0SAlexander Shishkin for (pg = 0; pg < rb->aux_nr_pages; pg++)
66645c815f0SAlexander Shishkin rb_free_aux_page(rb, pg);
66745c815f0SAlexander Shishkin
66845c815f0SAlexander Shishkin kfree(rb->aux_pages);
66945c815f0SAlexander Shishkin rb->aux_nr_pages = 0;
67045c815f0SAlexander Shishkin }
67145c815f0SAlexander Shishkin }
67245c815f0SAlexander Shishkin
rb_alloc_aux(struct perf_buffer * rb,struct perf_event * event,pgoff_t pgoff,int nr_pages,long watermark,int flags)67356de4e8fSSteven Rostedt (VMware) int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
6741a594131SAlexander Shishkin pgoff_t pgoff, int nr_pages, long watermark, int flags)
67545bfb2e5SPeter Zijlstra {
67645bfb2e5SPeter Zijlstra bool overwrite = !(flags & RING_BUFFER_WRITABLE);
67745bfb2e5SPeter Zijlstra int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
6785768402fSAlexander Shishkin int ret = -ENOMEM, max_order;
67945bfb2e5SPeter Zijlstra
68045bfb2e5SPeter Zijlstra if (!has_aux(event))
6818a1898dbSHendrik Brueckner return -EOPNOTSUPP;
68245bfb2e5SPeter Zijlstra
683d68e6799SAlexander Shishkin if (!overwrite) {
684d68e6799SAlexander Shishkin /*
685d68e6799SAlexander Shishkin * Watermark defaults to half the buffer, and so does the
686d68e6799SAlexander Shishkin * max_order, to aid PMU drivers in double buffering.
687d68e6799SAlexander Shishkin */
688d68e6799SAlexander Shishkin if (!watermark)
6890c8a2ef1SAdrian Hunter watermark = min_t(unsigned long,
6900c8a2ef1SAdrian Hunter U32_MAX,
6910c8a2ef1SAdrian Hunter (unsigned long)nr_pages << (PAGE_SHIFT - 1));
692d68e6799SAlexander Shishkin
693d68e6799SAlexander Shishkin /*
694d68e6799SAlexander Shishkin * Use aux_watermark as the basis for chunking to
695d68e6799SAlexander Shishkin * help PMU drivers honor the watermark.
696d68e6799SAlexander Shishkin */
697d68e6799SAlexander Shishkin max_order = get_order(watermark);
698d68e6799SAlexander Shishkin } else {
6990a4e38e6SAlexander Shishkin /*
7000a4e38e6SAlexander Shishkin * We need to start with the max_order that fits in nr_pages,
7010a4e38e6SAlexander Shishkin * not the other way around, hence ilog2() and not get_order.
7020a4e38e6SAlexander Shishkin */
7030a4e38e6SAlexander Shishkin max_order = ilog2(nr_pages);
704d68e6799SAlexander Shishkin watermark = 0;
7056a279230SAlexander Shishkin }
7066a279230SAlexander Shishkin
7072e905e60SShuai Xue /*
7082e905e60SShuai Xue * kcalloc_node() is unable to allocate buffer if the size is larger
7092e905e60SShuai Xue * than: PAGE_SIZE << MAX_ORDER; directly bail out in this case.
7102e905e60SShuai Xue */
7112e905e60SShuai Xue if (get_order((unsigned long)nr_pages * sizeof(void *)) > MAX_ORDER)
7122e905e60SShuai Xue return -ENOMEM;
713590b5b7dSKees Cook rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,
714590b5b7dSKees Cook node);
71545bfb2e5SPeter Zijlstra if (!rb->aux_pages)
71645bfb2e5SPeter Zijlstra return -ENOMEM;
71745bfb2e5SPeter Zijlstra
71845bfb2e5SPeter Zijlstra rb->free_aux = event->pmu->free_aux;
7190a4e38e6SAlexander Shishkin for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) {
72045bfb2e5SPeter Zijlstra struct page *page;
7210a4e38e6SAlexander Shishkin int last, order;
72245bfb2e5SPeter Zijlstra
7230a4e38e6SAlexander Shishkin order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages));
7240a4e38e6SAlexander Shishkin page = rb_alloc_aux_page(node, order);
72545bfb2e5SPeter Zijlstra if (!page)
72645bfb2e5SPeter Zijlstra goto out;
72745bfb2e5SPeter Zijlstra
7280a4e38e6SAlexander Shishkin for (last = rb->aux_nr_pages + (1 << page_private(page));
7290a4e38e6SAlexander Shishkin last > rb->aux_nr_pages; rb->aux_nr_pages++)
7300a4e38e6SAlexander Shishkin rb->aux_pages[rb->aux_nr_pages] = page_address(page++);
73145bfb2e5SPeter Zijlstra }
73245bfb2e5SPeter Zijlstra
733aa319bcdSAlexander Shishkin /*
734aa319bcdSAlexander Shishkin * In overwrite mode, PMUs that don't support SG may not handle more
735aa319bcdSAlexander Shishkin * than one contiguous allocation, since they rely on PMI to do double
736aa319bcdSAlexander Shishkin * buffering. In this case, the entire buffer has to be one contiguous
737aa319bcdSAlexander Shishkin * chunk.
738aa319bcdSAlexander Shishkin */
739aa319bcdSAlexander Shishkin if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) &&
740aa319bcdSAlexander Shishkin overwrite) {
741aa319bcdSAlexander Shishkin struct page *page = virt_to_page(rb->aux_pages[0]);
742aa319bcdSAlexander Shishkin
743aa319bcdSAlexander Shishkin if (page_private(page) != max_order)
744aa319bcdSAlexander Shishkin goto out;
745aa319bcdSAlexander Shishkin }
746aa319bcdSAlexander Shishkin
74784001866SMathieu Poirier rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages,
74845bfb2e5SPeter Zijlstra overwrite);
74945bfb2e5SPeter Zijlstra if (!rb->aux_priv)
75045bfb2e5SPeter Zijlstra goto out;
75145bfb2e5SPeter Zijlstra
75245bfb2e5SPeter Zijlstra ret = 0;
75345bfb2e5SPeter Zijlstra
75445bfb2e5SPeter Zijlstra /*
75545bfb2e5SPeter Zijlstra * aux_pages (and pmu driver's private data, aux_priv) will be
75645bfb2e5SPeter Zijlstra * referenced in both producer's and consumer's contexts, thus
75745bfb2e5SPeter Zijlstra * we keep a refcount here to make sure either of the two can
75845bfb2e5SPeter Zijlstra * reference them safely.
75945bfb2e5SPeter Zijlstra */
760ca3bb3d0SElena Reshetova refcount_set(&rb->aux_refcount, 1);
76145bfb2e5SPeter Zijlstra
7622023a0d2SAlexander Shishkin rb->aux_overwrite = overwrite;
7631a594131SAlexander Shishkin rb->aux_watermark = watermark;
7641a594131SAlexander Shishkin
76545bfb2e5SPeter Zijlstra out:
76645bfb2e5SPeter Zijlstra if (!ret)
76745bfb2e5SPeter Zijlstra rb->aux_pgoff = pgoff;
76845bfb2e5SPeter Zijlstra else
76945c815f0SAlexander Shishkin __rb_free_aux(rb);
77045bfb2e5SPeter Zijlstra
77145bfb2e5SPeter Zijlstra return ret;
77245bfb2e5SPeter Zijlstra }
77345bfb2e5SPeter Zijlstra
rb_free_aux(struct perf_buffer * rb)77456de4e8fSSteven Rostedt (VMware) void rb_free_aux(struct perf_buffer *rb)
77545bfb2e5SPeter Zijlstra {
776ca3bb3d0SElena Reshetova if (refcount_dec_and_test(&rb->aux_refcount))
77745bfb2e5SPeter Zijlstra __rb_free_aux(rb);
77845bfb2e5SPeter Zijlstra }
77945bfb2e5SPeter Zijlstra
78076369139SFrederic Weisbecker #ifndef CONFIG_PERF_USE_VMALLOC
78176369139SFrederic Weisbecker
78276369139SFrederic Weisbecker /*
78376369139SFrederic Weisbecker * Back perf_mmap() with regular GFP_KERNEL-0 pages.
78476369139SFrederic Weisbecker */
78576369139SFrederic Weisbecker
78645bfb2e5SPeter Zijlstra static struct page *
__perf_mmap_to_page(struct perf_buffer * rb,unsigned long pgoff)78756de4e8fSSteven Rostedt (VMware) __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
78876369139SFrederic Weisbecker {
78976369139SFrederic Weisbecker if (pgoff > rb->nr_pages)
79076369139SFrederic Weisbecker return NULL;
79176369139SFrederic Weisbecker
79276369139SFrederic Weisbecker if (pgoff == 0)
79376369139SFrederic Weisbecker return virt_to_page(rb->user_page);
79476369139SFrederic Weisbecker
79576369139SFrederic Weisbecker return virt_to_page(rb->data_pages[pgoff - 1]);
79676369139SFrederic Weisbecker }
79776369139SFrederic Weisbecker
perf_mmap_alloc_page(int cpu)79876369139SFrederic Weisbecker static void *perf_mmap_alloc_page(int cpu)
79976369139SFrederic Weisbecker {
80076369139SFrederic Weisbecker struct page *page;
80176369139SFrederic Weisbecker int node;
80276369139SFrederic Weisbecker
80376369139SFrederic Weisbecker node = (cpu == -1) ? cpu : cpu_to_node(cpu);
80476369139SFrederic Weisbecker page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
80576369139SFrederic Weisbecker if (!page)
80676369139SFrederic Weisbecker return NULL;
80776369139SFrederic Weisbecker
80876369139SFrederic Weisbecker return page_address(page);
80976369139SFrederic Weisbecker }
81076369139SFrederic Weisbecker
perf_mmap_free_page(void * addr)811d7e78706SYunfeng Ye static void perf_mmap_free_page(void *addr)
812d7e78706SYunfeng Ye {
813d7e78706SYunfeng Ye struct page *page = virt_to_page(addr);
814d7e78706SYunfeng Ye
815d7e78706SYunfeng Ye page->mapping = NULL;
816d7e78706SYunfeng Ye __free_page(page);
817d7e78706SYunfeng Ye }
818d7e78706SYunfeng Ye
rb_alloc(int nr_pages,long watermark,int cpu,int flags)81956de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
82076369139SFrederic Weisbecker {
82156de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb;
82276369139SFrederic Weisbecker unsigned long size;
8239483409aSNamhyung Kim int i, node;
82476369139SFrederic Weisbecker
82556de4e8fSSteven Rostedt (VMware) size = sizeof(struct perf_buffer);
82676369139SFrederic Weisbecker size += nr_pages * sizeof(void *);
82776369139SFrederic Weisbecker
82823baf831SKirill A. Shutemov if (order_base_2(size) > PAGE_SHIFT+MAX_ORDER)
8299dff0aa9SMark Rutland goto fail;
8309dff0aa9SMark Rutland
8319483409aSNamhyung Kim node = (cpu == -1) ? cpu : cpu_to_node(cpu);
8329483409aSNamhyung Kim rb = kzalloc_node(size, GFP_KERNEL, node);
83376369139SFrederic Weisbecker if (!rb)
83476369139SFrederic Weisbecker goto fail;
83576369139SFrederic Weisbecker
83676369139SFrederic Weisbecker rb->user_page = perf_mmap_alloc_page(cpu);
83776369139SFrederic Weisbecker if (!rb->user_page)
83876369139SFrederic Weisbecker goto fail_user_page;
83976369139SFrederic Weisbecker
84076369139SFrederic Weisbecker for (i = 0; i < nr_pages; i++) {
84176369139SFrederic Weisbecker rb->data_pages[i] = perf_mmap_alloc_page(cpu);
84276369139SFrederic Weisbecker if (!rb->data_pages[i])
84376369139SFrederic Weisbecker goto fail_data_pages;
84476369139SFrederic Weisbecker }
84576369139SFrederic Weisbecker
84676369139SFrederic Weisbecker rb->nr_pages = nr_pages;
84776369139SFrederic Weisbecker
84876369139SFrederic Weisbecker ring_buffer_init(rb, watermark, flags);
84976369139SFrederic Weisbecker
85076369139SFrederic Weisbecker return rb;
85176369139SFrederic Weisbecker
85276369139SFrederic Weisbecker fail_data_pages:
85376369139SFrederic Weisbecker for (i--; i >= 0; i--)
854d7e78706SYunfeng Ye perf_mmap_free_page(rb->data_pages[i]);
85576369139SFrederic Weisbecker
856d7e78706SYunfeng Ye perf_mmap_free_page(rb->user_page);
85776369139SFrederic Weisbecker
85876369139SFrederic Weisbecker fail_user_page:
85976369139SFrederic Weisbecker kfree(rb);
86076369139SFrederic Weisbecker
86176369139SFrederic Weisbecker fail:
86276369139SFrederic Weisbecker return NULL;
86376369139SFrederic Weisbecker }
86476369139SFrederic Weisbecker
rb_free(struct perf_buffer * rb)86556de4e8fSSteven Rostedt (VMware) void rb_free(struct perf_buffer *rb)
86676369139SFrederic Weisbecker {
86776369139SFrederic Weisbecker int i;
86876369139SFrederic Weisbecker
8698a9f91c5SYunfeng Ye perf_mmap_free_page(rb->user_page);
87076369139SFrederic Weisbecker for (i = 0; i < rb->nr_pages; i++)
8718a9f91c5SYunfeng Ye perf_mmap_free_page(rb->data_pages[i]);
87276369139SFrederic Weisbecker kfree(rb);
87376369139SFrederic Weisbecker }
87476369139SFrederic Weisbecker
87576369139SFrederic Weisbecker #else
87645bfb2e5SPeter Zijlstra static struct page *
__perf_mmap_to_page(struct perf_buffer * rb,unsigned long pgoff)87756de4e8fSSteven Rostedt (VMware) __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
87876369139SFrederic Weisbecker {
8795919b309SJiri Olsa /* The '>' counts in the user page. */
8805919b309SJiri Olsa if (pgoff > data_page_nr(rb))
88176369139SFrederic Weisbecker return NULL;
88276369139SFrederic Weisbecker
88376369139SFrederic Weisbecker return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
88476369139SFrederic Weisbecker }
88576369139SFrederic Weisbecker
perf_mmap_unmark_page(void * addr)88676369139SFrederic Weisbecker static void perf_mmap_unmark_page(void *addr)
88776369139SFrederic Weisbecker {
88876369139SFrederic Weisbecker struct page *page = vmalloc_to_page(addr);
88976369139SFrederic Weisbecker
89076369139SFrederic Weisbecker page->mapping = NULL;
89176369139SFrederic Weisbecker }
89276369139SFrederic Weisbecker
rb_free_work(struct work_struct * work)89376369139SFrederic Weisbecker static void rb_free_work(struct work_struct *work)
89476369139SFrederic Weisbecker {
89556de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb;
89676369139SFrederic Weisbecker void *base;
89776369139SFrederic Weisbecker int i, nr;
89876369139SFrederic Weisbecker
89956de4e8fSSteven Rostedt (VMware) rb = container_of(work, struct perf_buffer, work);
9005919b309SJiri Olsa nr = data_page_nr(rb);
90176369139SFrederic Weisbecker
90276369139SFrederic Weisbecker base = rb->user_page;
9035919b309SJiri Olsa /* The '<=' counts in the user page. */
9045919b309SJiri Olsa for (i = 0; i <= nr; i++)
90576369139SFrederic Weisbecker perf_mmap_unmark_page(base + (i * PAGE_SIZE));
90676369139SFrederic Weisbecker
90776369139SFrederic Weisbecker vfree(base);
90876369139SFrederic Weisbecker kfree(rb);
90976369139SFrederic Weisbecker }
91076369139SFrederic Weisbecker
rb_free(struct perf_buffer * rb)91156de4e8fSSteven Rostedt (VMware) void rb_free(struct perf_buffer *rb)
91276369139SFrederic Weisbecker {
91376369139SFrederic Weisbecker schedule_work(&rb->work);
91476369139SFrederic Weisbecker }
91576369139SFrederic Weisbecker
rb_alloc(int nr_pages,long watermark,int cpu,int flags)91656de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
91776369139SFrederic Weisbecker {
91856de4e8fSSteven Rostedt (VMware) struct perf_buffer *rb;
91976369139SFrederic Weisbecker unsigned long size;
92076369139SFrederic Weisbecker void *all_buf;
9219483409aSNamhyung Kim int node;
92276369139SFrederic Weisbecker
92356de4e8fSSteven Rostedt (VMware) size = sizeof(struct perf_buffer);
92476369139SFrederic Weisbecker size += sizeof(void *);
92576369139SFrederic Weisbecker
9269483409aSNamhyung Kim node = (cpu == -1) ? cpu : cpu_to_node(cpu);
9279483409aSNamhyung Kim rb = kzalloc_node(size, GFP_KERNEL, node);
92876369139SFrederic Weisbecker if (!rb)
92976369139SFrederic Weisbecker goto fail;
93076369139SFrederic Weisbecker
93176369139SFrederic Weisbecker INIT_WORK(&rb->work, rb_free_work);
93276369139SFrederic Weisbecker
93376369139SFrederic Weisbecker all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
93476369139SFrederic Weisbecker if (!all_buf)
93576369139SFrederic Weisbecker goto fail_all_buf;
93676369139SFrederic Weisbecker
93776369139SFrederic Weisbecker rb->user_page = all_buf;
93876369139SFrederic Weisbecker rb->data_pages[0] = all_buf + PAGE_SIZE;
9398184059eSPeter Zijlstra if (nr_pages) {
9408184059eSPeter Zijlstra rb->nr_pages = 1;
94176369139SFrederic Weisbecker rb->page_order = ilog2(nr_pages);
9428184059eSPeter Zijlstra }
94376369139SFrederic Weisbecker
94476369139SFrederic Weisbecker ring_buffer_init(rb, watermark, flags);
94576369139SFrederic Weisbecker
94676369139SFrederic Weisbecker return rb;
94776369139SFrederic Weisbecker
94876369139SFrederic Weisbecker fail_all_buf:
94976369139SFrederic Weisbecker kfree(rb);
95076369139SFrederic Weisbecker
95176369139SFrederic Weisbecker fail:
95276369139SFrederic Weisbecker return NULL;
95376369139SFrederic Weisbecker }
95476369139SFrederic Weisbecker
95576369139SFrederic Weisbecker #endif
95645bfb2e5SPeter Zijlstra
95745bfb2e5SPeter Zijlstra struct page *
perf_mmap_to_page(struct perf_buffer * rb,unsigned long pgoff)95856de4e8fSSteven Rostedt (VMware) perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
95945bfb2e5SPeter Zijlstra {
96045bfb2e5SPeter Zijlstra if (rb->aux_nr_pages) {
96145bfb2e5SPeter Zijlstra /* above AUX space */
96245bfb2e5SPeter Zijlstra if (pgoff > rb->aux_pgoff + rb->aux_nr_pages)
96345bfb2e5SPeter Zijlstra return NULL;
96445bfb2e5SPeter Zijlstra
96545bfb2e5SPeter Zijlstra /* AUX space */
9664411ec1dSPeter Zijlstra if (pgoff >= rb->aux_pgoff) {
9674411ec1dSPeter Zijlstra int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages);
9684411ec1dSPeter Zijlstra return virt_to_page(rb->aux_pages[aux_pgoff]);
9694411ec1dSPeter Zijlstra }
97045bfb2e5SPeter Zijlstra }
97145bfb2e5SPeter Zijlstra
97245bfb2e5SPeter Zijlstra return __perf_mmap_to_page(rb, pgoff);
97345bfb2e5SPeter Zijlstra }
974