ring_buffer.c (8843e06f67b14f71c044bf6267b2387784c7e198) ring_buffer.c (2455f0e124d317dd08d337a7550a78a224d4ba41)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic ring buffer
4 *
5 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 */
7#include <linux/trace_recursion.h>
8#include <linux/trace_events.h>

--- 1567 unchanged lines hidden (view full) ---

1576
1577 if (RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK))
1578 return 1;
1579
1580 return 0;
1581}
1582
1583/**
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic ring buffer
4 *
5 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 */
7#include <linux/trace_recursion.h>
8#include <linux/trace_events.h>

--- 1567 unchanged lines hidden (view full) ---

1576
1577 if (RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK))
1578 return 1;
1579
1580 return 0;
1581}
1582
1583/**
1584 * rb_check_list - make sure a pointer to a list has the last bits zero
1585 */
1586static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
1587 struct list_head *list)
1588{
1589 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev))
1590 return 1;
1591 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next))
1592 return 1;
1593 return 0;
1594}
1595
1596/**
1584 * rb_check_pages - integrity check of buffer pages
1585 * @cpu_buffer: CPU buffer with pages to test
1586 *
1587 * As a safety measure we check to make sure the data pages have not
1588 * been corrupted.
1589 */
1590static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
1591{
1597 * rb_check_pages - integrity check of buffer pages
1598 * @cpu_buffer: CPU buffer with pages to test
1599 *
1600 * As a safety measure we check to make sure the data pages have not
1601 * been corrupted.
1602 */
1603static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
1604{
1592 struct list_head *head = rb_list_head(cpu_buffer->pages);
1593 struct list_head *tmp;
1605 struct list_head *head = cpu_buffer->pages;
1606 struct buffer_page *bpage, *tmp;
1594
1607
1595 if (RB_WARN_ON(cpu_buffer,
1596 rb_list_head(rb_list_head(head->next)->prev) != head))
1608 /* Reset the head page if it exists */
1609 if (cpu_buffer->head_page)
1610 rb_set_head_page(cpu_buffer);
1611
1612 rb_head_page_deactivate(cpu_buffer);
1613
1614 if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
1597 return -1;
1615 return -1;
1616 if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
1617 return -1;
1598
1618
1599 if (RB_WARN_ON(cpu_buffer,
1600 rb_list_head(rb_list_head(head->prev)->next) != head))
1619 if (rb_check_list(cpu_buffer, head))
1601 return -1;
1602
1620 return -1;
1621
1603 for (tmp = rb_list_head(head->next); tmp != head; tmp = rb_list_head(tmp->next)) {
1622 list_for_each_entry_safe(bpage, tmp, head, list) {
1604 if (RB_WARN_ON(cpu_buffer,
1623 if (RB_WARN_ON(cpu_buffer,
1605 rb_list_head(rb_list_head(tmp->next)->prev) != tmp))
1624 bpage->list.next->prev != &bpage->list))
1606 return -1;
1625 return -1;
1607
1608 if (RB_WARN_ON(cpu_buffer,
1626 if (RB_WARN_ON(cpu_buffer,
1609 rb_list_head(rb_list_head(tmp->prev)->next) != tmp))
1627 bpage->list.prev->next != &bpage->list))
1610 return -1;
1628 return -1;
1629 if (rb_check_list(cpu_buffer, &bpage->list))
1630 return -1;
1611 }
1612
1631 }
1632
1633 rb_head_page_activate(cpu_buffer);
1634
1613 return 0;
1614}
1615
1616static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1617 long nr_pages, struct list_head *pages)
1618{
1619 struct buffer_page *bpage, *tmp;
1620 bool user_thread = current->mm != NULL;

--- 1238 unchanged lines hidden (view full) ---

2859 (unsigned long long)info->delta,
2860 (unsigned long long)info->ts,
2861 (unsigned long long)info->before,
2862 (unsigned long long)info->after,
2863 (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),
2864 sched_clock_stable() ? "" :
2865 "If you just came from a suspend/resume,\n"
2866 "please switch to the trace global clock:\n"
1635 return 0;
1636}
1637
1638static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1639 long nr_pages, struct list_head *pages)
1640{
1641 struct buffer_page *bpage, *tmp;
1642 bool user_thread = current->mm != NULL;

--- 1238 unchanged lines hidden (view full) ---

2881 (unsigned long long)info->delta,
2882 (unsigned long long)info->ts,
2883 (unsigned long long)info->before,
2884 (unsigned long long)info->after,
2885 (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),
2886 sched_clock_stable() ? "" :
2887 "If you just came from a suspend/resume,\n"
2888 "please switch to the trace global clock:\n"
2867 " echo global > /sys/kernel/debug/tracing/trace_clock\n"
2889 " echo global > /sys/kernel/tracing/trace_clock\n"
2868 "or add trace_clock=global to the kernel command line\n");
2869}
2870
2871static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
2872 struct ring_buffer_event **event,
2873 struct rb_event_info *info,
2874 u64 *delta,
2875 unsigned int *length)

--- 2723 unchanged lines hidden (view full) ---

5599 * @buffer: the buffer the page was allocate for
5600 * @cpu: the cpu buffer the page came from
5601 * @data: the page to free
5602 *
5603 * Free a page allocated from ring_buffer_alloc_read_page.
5604 */
5605void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)
5606{
2890 "or add trace_clock=global to the kernel command line\n");
2891}
2892
2893static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
2894 struct ring_buffer_event **event,
2895 struct rb_event_info *info,
2896 u64 *delta,
2897 unsigned int *length)

--- 2723 unchanged lines hidden (view full) ---

5621 * @buffer: the buffer the page was allocate for
5622 * @cpu: the cpu buffer the page came from
5623 * @data: the page to free
5624 *
5625 * Free a page allocated from ring_buffer_alloc_read_page.
5626 */
5627void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)
5628{
5607 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
5629 struct ring_buffer_per_cpu *cpu_buffer;
5608 struct buffer_data_page *bpage = data;
5609 struct page *page = virt_to_page(bpage);
5610 unsigned long flags;
5611
5630 struct buffer_data_page *bpage = data;
5631 struct page *page = virt_to_page(bpage);
5632 unsigned long flags;
5633
5634 if (!buffer || !buffer->buffers || !buffer->buffers[cpu])
5635 return;
5636
5637 cpu_buffer = buffer->buffers[cpu];
5638
5612 /* If the page is still in use someplace else, we can't reuse it */
5613 if (page_ref_count(page) > 1)
5614 goto out;
5615
5616 local_irq_save(flags);
5617 arch_spin_lock(&cpu_buffer->lock);
5618
5619 if (!cpu_buffer->free_page) {

--- 578 unchanged lines hidden --- 		5639 /* If the page is still in use someplace else, we can't reuse it */
5640 if (page_ref_count(page) > 1)
5641 goto out;
5642
5643 local_irq_save(flags);
5644 arch_spin_lock(&cpu_buffer->lock);
5645
5646 if (!cpu_buffer->free_page) {

--- 578 unchanged lines hidden ---