1 /* SPDX-License-Identifier: GPL-2.0 */
2 
3 #ifndef _KERNEL_PRINTK_RINGBUFFER_H
4 #define _KERNEL_PRINTK_RINGBUFFER_H
5 
6 #include <linux/atomic.h>
7 #include <linux/dev_printk.h>
8 
9 /*
10  * Meta information about each stored message.
11  *
12  * All fields are set by the printk code except for @seq, which is
13  * set by the ringbuffer code.
14  */
15 struct printk_info {
16 	u64	seq;		/* sequence number */
17 	u64	ts_nsec;	/* timestamp in nanoseconds */
18 	u16	text_len;	/* length of text message */
19 	u8	facility;	/* syslog facility */
20 	u8	flags:5;	/* internal record flags */
21 	u8	level:3;	/* syslog level */
22 	u32	caller_id;	/* thread id or processor id */
23 
24 	struct dev_printk_info	dev_info;
25 };
26 
27 /*
28  * A structure providing the buffers, used by writers and readers.
29  *
30  * Writers:
31  * Using prb_rec_init_wr(), a writer sets @text_buf_size before calling
32  * prb_reserve(). On success, prb_reserve() sets @info and @text_buf to
33  * buffers reserved for that writer.
34  *
35  * Readers:
36  * Using prb_rec_init_rd(), a reader sets all fields before calling
37  * prb_read_valid(). Note that the reader provides the @info and @text_buf,
38  * buffers. On success, the struct pointed to by @info will be filled and
39  * the char array pointed to by @text_buf will be filled with text data.
40  */
41 struct printk_record {
42 	struct printk_info	*info;
43 	char			*text_buf;
44 	unsigned int		text_buf_size;
45 };
46 
47 /* Specifies the logical position and span of a data block. */
48 struct prb_data_blk_lpos {
49 	unsigned long	begin;
50 	unsigned long	next;
51 };
52 
53 /*
54  * A descriptor: the complete meta-data for a record.
55  *
56  * @state_var: A bitwise combination of descriptor ID and descriptor state.
57  */
58 struct prb_desc {
59 	atomic_long_t			state_var;
60 	struct prb_data_blk_lpos	text_blk_lpos;
61 };
62 
63 /* A ringbuffer of "ID + data" elements. */
64 struct prb_data_ring {
65 	unsigned int	size_bits;
66 	char		*data;
67 	atomic_long_t	head_lpos;
68 	atomic_long_t	tail_lpos;
69 };
70 
71 /* A ringbuffer of "struct prb_desc" elements. */
72 struct prb_desc_ring {
73 	unsigned int		count_bits;
74 	struct prb_desc		*descs;
75 	struct printk_info	*infos;
76 	atomic_long_t		head_id;
77 	atomic_long_t		tail_id;
78 	atomic_long_t		last_finalized_id;
79 };
80 
81 /*
82  * The high level structure representing the printk ringbuffer.
83  *
84  * @fail: Count of failed prb_reserve() calls where not even a data-less
85  *        record was created.
86  */
87 struct printk_ringbuffer {
88 	struct prb_desc_ring	desc_ring;
89 	struct prb_data_ring	text_data_ring;
90 	atomic_long_t		fail;
91 };
92 
93 /*
94  * Used by writers as a reserve/commit handle.
95  *
96  * @rb:         Ringbuffer where the entry is reserved.
97  * @irqflags:   Saved irq flags to restore on entry commit.
98  * @id:         ID of the reserved descriptor.
99  * @text_space: Total occupied buffer space in the text data ring, including
100  *              ID, alignment padding, and wrapping data blocks.
101  *
102  * This structure is an opaque handle for writers. Its contents are only
103  * to be used by the ringbuffer implementation.
104  */
105 struct prb_reserved_entry {
106 	struct printk_ringbuffer	*rb;
107 	unsigned long			irqflags;
108 	unsigned long			id;
109 	unsigned int			text_space;
110 };
111 
112 /* The possible responses of a descriptor state-query. */
113 enum desc_state {
114 	desc_miss	=  -1,	/* ID mismatch (pseudo state) */
115 	desc_reserved	= 0x0,	/* reserved, in use by writer */
116 	desc_committed	= 0x1,	/* committed by writer, could get reopened */
117 	desc_finalized	= 0x2,	/* committed, no further modification allowed */
118 	desc_reusable	= 0x3,	/* free, not yet used by any writer */
119 };
120 
121 #define _DATA_SIZE(sz_bits)	(1UL << (sz_bits))
122 #define _DESCS_COUNT(ct_bits)	(1U << (ct_bits))
123 #define DESC_SV_BITS		(sizeof(unsigned long) * 8)
124 #define DESC_FLAGS_SHIFT	(DESC_SV_BITS - 2)
125 #define DESC_FLAGS_MASK		(3UL << DESC_FLAGS_SHIFT)
126 #define DESC_STATE(sv)		(3UL & (sv >> DESC_FLAGS_SHIFT))
127 #define DESC_SV(id, state)	(((unsigned long)state << DESC_FLAGS_SHIFT) | id)
128 #define DESC_ID_MASK		(~DESC_FLAGS_MASK)
129 #define DESC_ID(sv)		((sv) & DESC_ID_MASK)
130 #define FAILED_LPOS		0x1
131 #define NO_LPOS			0x3
132 
133 #define FAILED_BLK_LPOS	\
134 {				\
135 	.begin	= FAILED_LPOS,	\
136 	.next	= FAILED_LPOS,	\
137 }
138 
139 /*
140  * Descriptor Bootstrap
141  *
142  * The descriptor array is minimally initialized to allow immediate usage
143  * by readers and writers. The requirements that the descriptor array
144  * initialization must satisfy:
145  *
146  *   Req1
147  *     The tail must point to an existing (committed or reusable) descriptor.
148  *     This is required by the implementation of prb_first_seq().
149  *
150  *   Req2
151  *     Readers must see that the ringbuffer is initially empty.
152  *
153  *   Req3
154  *     The first record reserved by a writer is assigned sequence number 0.
155  *
156  * To satisfy Req1, the tail initially points to a descriptor that is
157  * minimally initialized (having no data block, i.e. data-less with the
158  * data block's lpos @begin and @next values set to FAILED_LPOS).
159  *
160  * To satisfy Req2, the initial tail descriptor is initialized to the
161  * reusable state. Readers recognize reusable descriptors as existing
162  * records, but skip over them.
163  *
164  * To satisfy Req3, the last descriptor in the array is used as the initial
165  * head (and tail) descriptor. This allows the first record reserved by a
166  * writer (head + 1) to be the first descriptor in the array. (Only the first
167  * descriptor in the array could have a valid sequence number of 0.)
168  *
169  * The first time a descriptor is reserved, it is assigned a sequence number
170  * with the value of the array index. A "first time reserved" descriptor can
171  * be recognized because it has a sequence number of 0 but does not have an
172  * index of 0. (Only the first descriptor in the array could have a valid
173  * sequence number of 0.) After the first reservation, all future reservations
174  * (recycling) simply involve incrementing the sequence number by the array
175  * count.
176  *
177  *   Hack #1
178  *     Only the first descriptor in the array is allowed to have the sequence
179  *     number 0. In this case it is not possible to recognize if it is being
180  *     reserved the first time (set to index value) or has been reserved
181  *     previously (increment by the array count). This is handled by _always_
182  *     incrementing the sequence number by the array count when reserving the
183  *     first descriptor in the array. In order to satisfy Req3, the sequence
184  *     number of the first descriptor in the array is initialized to minus
185  *     the array count. Then, upon the first reservation, it is incremented
186  *     to 0, thus satisfying Req3.
187  *
188  *   Hack #2
189  *     prb_first_seq() can be called at any time by readers to retrieve the
190  *     sequence number of the tail descriptor. However, due to Req2 and Req3,
191  *     initially there are no records to report the sequence number of
192  *     (sequence numbers are u64 and there is nothing less than 0). To handle
193  *     this, the sequence number of the initial tail descriptor is initialized
194  *     to 0. Technically this is incorrect, because there is no record with
195  *     sequence number 0 (yet) and the tail descriptor is not the first
196  *     descriptor in the array. But it allows prb_read_valid() to correctly
197  *     report the existence of a record for _any_ given sequence number at all
198  *     times. Bootstrapping is complete when the tail is pushed the first
199  *     time, thus finally pointing to the first descriptor reserved by a
200  *     writer, which has the assigned sequence number 0.
201  */
202 
203 /*
204  * Initiating Logical Value Overflows
205  *
206  * Both logical position (lpos) and ID values can be mapped to array indexes
207  * but may experience overflows during the lifetime of the system. To ensure
208  * that printk_ringbuffer can handle the overflows for these types, initial
209  * values are chosen that map to the correct initial array indexes, but will
210  * result in overflows soon.
211  *
212  *   BLK0_LPOS
213  *     The initial @head_lpos and @tail_lpos for data rings. It is at index
214  *     0 and the lpos value is such that it will overflow on the first wrap.
215  *
216  *   DESC0_ID
217  *     The initial @head_id and @tail_id for the desc ring. It is at the last
218  *     index of the descriptor array (see Req3 above) and the ID value is such
219  *     that it will overflow on the second wrap.
220  */
221 #define BLK0_LPOS(sz_bits)	(-(_DATA_SIZE(sz_bits)))
222 #define DESC0_ID(ct_bits)	DESC_ID(-(_DESCS_COUNT(ct_bits) + 1))
223 #define DESC0_SV(ct_bits)	DESC_SV(DESC0_ID(ct_bits), desc_reusable)
224 
225 /*
226  * Define a ringbuffer with an external text data buffer. The same as
227  * DEFINE_PRINTKRB() but requires specifying an external buffer for the
228  * text data.
229  *
230  * Note: The specified external buffer must be of the size:
231  *       2 ^ (descbits + avgtextbits)
232  */
233 #define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf)			\
234 static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = {				\
235 	/* the initial head and tail */								\
236 	[_DESCS_COUNT(descbits) - 1] = {							\
237 		/* reusable */									\
238 		.state_var	= ATOMIC_INIT(DESC0_SV(descbits)),				\
239 		/* no associated data block */							\
240 		.text_blk_lpos	= FAILED_BLK_LPOS,						\
241 	},											\
242 };												\
243 static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = {				\
244 	/* this will be the first record reserved by a writer */				\
245 	[0] = {											\
246 		/* will be incremented to 0 on the first reservation */				\
247 		.seq = -(u64)_DESCS_COUNT(descbits),						\
248 	},											\
249 	/* the initial head and tail */								\
250 	[_DESCS_COUNT(descbits) - 1] = {							\
251 		/* reports the first seq value during the bootstrap phase */			\
252 		.seq = 0,									\
253 	},											\
254 };												\
255 static struct printk_ringbuffer name = {							\
256 	.desc_ring = {										\
257 		.count_bits	= descbits,							\
258 		.descs		= &_##name##_descs[0],						\
259 		.infos		= &_##name##_infos[0],						\
260 		.head_id	= ATOMIC_INIT(DESC0_ID(descbits)),				\
261 		.tail_id	= ATOMIC_INIT(DESC0_ID(descbits)),				\
262 		.last_finalized_id = ATOMIC_INIT(DESC0_ID(descbits)),				\
263 	},											\
264 	.text_data_ring = {									\
265 		.size_bits	= (avgtextbits) + (descbits),					\
266 		.data		= text_buf,							\
267 		.head_lpos	= ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))),	\
268 		.tail_lpos	= ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))),	\
269 	},											\
270 	.fail			= ATOMIC_LONG_INIT(0),						\
271 }
272 
273 /**
274  * DEFINE_PRINTKRB() - Define a ringbuffer.
275  *
276  * @name:        The name of the ringbuffer variable.
277  * @descbits:    The number of descriptors as a power-of-2 value.
278  * @avgtextbits: The average text data size per record as a power-of-2 value.
279  *
280  * This is a macro for defining a ringbuffer and all internal structures
281  * such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a
282  * variant where the text data buffer can be specified externally.
283  */
284 #define DEFINE_PRINTKRB(name, descbits, avgtextbits)				\
285 static char _##name##_text[1U << ((avgtextbits) + (descbits))]			\
286 			__aligned(__alignof__(unsigned long));			\
287 _DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0])
288 
289 /* Writer Interface */
290 
291 /**
292  * prb_rec_init_wr() - Initialize a buffer for writing records.
293  *
294  * @r:             The record to initialize.
295  * @text_buf_size: The needed text buffer size.
296  */
297 static inline void prb_rec_init_wr(struct printk_record *r,
298 				   unsigned int text_buf_size)
299 {
300 	r->info = NULL;
301 	r->text_buf = NULL;
302 	r->text_buf_size = text_buf_size;
303 }
304 
305 bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
306 		 struct printk_record *r);
307 bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
308 			 struct printk_record *r, u32 caller_id, unsigned int max_size);
309 void prb_commit(struct prb_reserved_entry *e);
310 void prb_final_commit(struct prb_reserved_entry *e);
311 
312 void prb_init(struct printk_ringbuffer *rb,
313 	      char *text_buf, unsigned int text_buf_size,
314 	      struct prb_desc *descs, unsigned int descs_count_bits,
315 	      struct printk_info *infos);
316 unsigned int prb_record_text_space(struct prb_reserved_entry *e);
317 
318 /* Reader Interface */
319 
320 /**
321  * prb_rec_init_rd() - Initialize a buffer for reading records.
322  *
323  * @r:             The record to initialize.
324  * @info:          A buffer to store record meta-data.
325  * @text_buf:      A buffer to store text data.
326  * @text_buf_size: The size of @text_buf.
327  *
328  * Initialize all the fields that a reader is interested in. All arguments
329  * (except @r) are optional. Only record data for arguments that are
330  * non-NULL or non-zero will be read.
331  */
332 static inline void prb_rec_init_rd(struct printk_record *r,
333 				   struct printk_info *info,
334 				   char *text_buf, unsigned int text_buf_size)
335 {
336 	r->info = info;
337 	r->text_buf = text_buf;
338 	r->text_buf_size = text_buf_size;
339 }
340 
341 /**
342  * prb_for_each_record() - Iterate over the records of a ringbuffer.
343  *
344  * @from: The sequence number to begin with.
345  * @rb:   The ringbuffer to iterate over.
346  * @s:    A u64 to store the sequence number on each iteration.
347  * @r:    A printk_record to store the record on each iteration.
348  *
349  * This is a macro for conveniently iterating over a ringbuffer.
350  * Note that @s may not be the sequence number of the record on each
351  * iteration. For the sequence number, @r->info->seq should be checked.
352  *
353  * Context: Any context.
354  */
355 #define prb_for_each_record(from, rb, s, r) \
356 for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1)
357 
358 /**
359  * prb_for_each_info() - Iterate over the meta data of a ringbuffer.
360  *
361  * @from: The sequence number to begin with.
362  * @rb:   The ringbuffer to iterate over.
363  * @s:    A u64 to store the sequence number on each iteration.
364  * @i:    A printk_info to store the record meta data on each iteration.
365  * @lc:   An unsigned int to store the text line count of each record.
366  *
367  * This is a macro for conveniently iterating over a ringbuffer.
368  * Note that @s may not be the sequence number of the record on each
369  * iteration. For the sequence number, @r->info->seq should be checked.
370  *
371  * Context: Any context.
372  */
373 #define prb_for_each_info(from, rb, s, i, lc) \
374 for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1)
375 
376 bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
377 		    struct printk_record *r);
378 bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
379 			 struct printk_info *info, unsigned int *line_count);
380 
381 u64 prb_first_valid_seq(struct printk_ringbuffer *rb);
382 u64 prb_next_seq(struct printk_ringbuffer *rb);
383 
384 #endif /* _KERNEL_PRINTK_RINGBUFFER_H */
385