1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Stage 1 of the trace events. 4 * 5 * Override the macros in the event tracepoint header <trace/events/XXX.h> 6 * to include the following: 7 * 8 * struct trace_event_raw_<call> { 9 * struct trace_entry ent; 10 * <type> <item>; 11 * <type2> <item2>[<len>]; 12 * [...] 13 * }; 14 * 15 * The <type> <item> is created by the __field(type, item) macro or 16 * the __array(type2, item2, len) macro. 17 * We simply do "type item;", and that will create the fields 18 * in the structure. 19 */ 20 21 #include <linux/trace_events.h> 22 23 #ifndef TRACE_SYSTEM_VAR 24 #define TRACE_SYSTEM_VAR TRACE_SYSTEM 25 #endif 26 27 #include "stages/init.h" 28 29 /* 30 * DECLARE_EVENT_CLASS can be used to add a generic function 31 * handlers for events. That is, if all events have the same 32 * parameters and just have distinct trace points. 33 * Each tracepoint can be defined with DEFINE_EVENT and that 34 * will map the DECLARE_EVENT_CLASS to the tracepoint. 35 * 36 * TRACE_EVENT is a one to one mapping between tracepoint and template. 37 */ 38 #undef TRACE_EVENT 39 #define TRACE_EVENT(name, proto, args, tstruct, assign, print) \ 40 DECLARE_EVENT_CLASS(name, \ 41 PARAMS(proto), \ 42 PARAMS(args), \ 43 PARAMS(tstruct), \ 44 PARAMS(assign), \ 45 PARAMS(print)); \ 46 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); 47 48 #include "stages/stage1_struct_define.h" 49 50 #undef DECLARE_EVENT_CLASS 51 #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \ 52 struct trace_event_raw_##name { \ 53 struct trace_entry ent; \ 54 tstruct \ 55 char __data[]; \ 56 }; \ 57 \ 58 static struct trace_event_class event_class_##name; 59 60 #undef DEFINE_EVENT 61 #define DEFINE_EVENT(template, name, proto, args) \ 62 static struct trace_event_call __used \ 63 __attribute__((__aligned__(4))) event_##name 64 65 #undef DEFINE_EVENT_FN 66 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \ 67 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 68 69 #undef DEFINE_EVENT_PRINT 70 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ 71 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 72 73 /* Callbacks are meaningless to ftrace. */ 74 #undef TRACE_EVENT_FN 75 #define TRACE_EVENT_FN(name, proto, args, tstruct, \ 76 assign, print, reg, unreg) \ 77 TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \ 78 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 79 80 #undef TRACE_EVENT_FN_COND 81 #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \ 82 assign, print, reg, unreg) \ 83 TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond), \ 84 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 85 86 #undef TRACE_EVENT_FLAGS 87 #define TRACE_EVENT_FLAGS(name, value) \ 88 __TRACE_EVENT_FLAGS(name, value) 89 90 #undef TRACE_EVENT_PERF_PERM 91 #define TRACE_EVENT_PERF_PERM(name, expr...) \ 92 __TRACE_EVENT_PERF_PERM(name, expr) 93 94 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 95 96 /* 97 * Stage 2 of the trace events. 98 * 99 * Include the following: 100 * 101 * struct trace_event_data_offsets_<call> { 102 * u32 <item1>; 103 * u32 <item2>; 104 * [...] 105 * }; 106 * 107 * The __dynamic_array() macro will create each u32 <item>, this is 108 * to keep the offset of each array from the beginning of the event. 109 * The size of an array is also encoded, in the higher 16 bits of <item>. 110 */ 111 112 #include "stages/stage2_data_offsets.h" 113 114 #undef DECLARE_EVENT_CLASS 115 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 116 struct trace_event_data_offsets_##call { \ 117 tstruct; \ 118 }; 119 120 #undef DEFINE_EVENT 121 #define DEFINE_EVENT(template, name, proto, args) 122 123 #undef DEFINE_EVENT_PRINT 124 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 125 126 #undef TRACE_EVENT_FLAGS 127 #define TRACE_EVENT_FLAGS(event, flag) 128 129 #undef TRACE_EVENT_PERF_PERM 130 #define TRACE_EVENT_PERF_PERM(event, expr...) 131 132 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 133 134 /* 135 * Stage 3 of the trace events. 136 * 137 * Override the macros in the event tracepoint header <trace/events/XXX.h> 138 * to include the following: 139 * 140 * enum print_line_t 141 * trace_raw_output_<call>(struct trace_iterator *iter, int flags) 142 * { 143 * struct trace_seq *s = &iter->seq; 144 * struct trace_event_raw_<call> *field; <-- defined in stage 1 145 * struct trace_seq *p = &iter->tmp_seq; 146 * 147 * -------(for event)------- 148 * 149 * struct trace_entry *entry; 150 * 151 * entry = iter->ent; 152 * 153 * if (entry->type != event_<call>->event.type) { 154 * WARN_ON_ONCE(1); 155 * return TRACE_TYPE_UNHANDLED; 156 * } 157 * 158 * field = (typeof(field))entry; 159 * 160 * trace_seq_init(p); 161 * return trace_output_call(iter, <call>, <TP_printk> "\n"); 162 * 163 * ------(or, for event class)------ 164 * 165 * int ret; 166 * 167 * field = (typeof(field))iter->ent; 168 * 169 * ret = trace_raw_output_prep(iter, trace_event); 170 * if (ret != TRACE_TYPE_HANDLED) 171 * return ret; 172 * 173 * trace_event_printf(iter, <TP_printk> "\n"); 174 * 175 * return trace_handle_return(s); 176 * ------- 177 * } 178 * 179 * This is the method used to print the raw event to the trace 180 * output format. Note, this is not needed if the data is read 181 * in binary. 182 */ 183 184 #include "stages/stage3_trace_output.h" 185 186 #undef DECLARE_EVENT_CLASS 187 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 188 static notrace enum print_line_t \ 189 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 190 struct trace_event *trace_event) \ 191 { \ 192 struct trace_seq *s = &iter->seq; \ 193 struct trace_seq __maybe_unused *p = &iter->tmp_seq; \ 194 struct trace_event_raw_##call *field; \ 195 int ret; \ 196 \ 197 field = (typeof(field))iter->ent; \ 198 \ 199 ret = trace_raw_output_prep(iter, trace_event); \ 200 if (ret != TRACE_TYPE_HANDLED) \ 201 return ret; \ 202 \ 203 trace_event_printf(iter, print); \ 204 \ 205 return trace_handle_return(s); \ 206 } \ 207 static struct trace_event_functions trace_event_type_funcs_##call = { \ 208 .trace = trace_raw_output_##call, \ 209 }; 210 211 #undef DEFINE_EVENT_PRINT 212 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 213 static notrace enum print_line_t \ 214 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 215 struct trace_event *event) \ 216 { \ 217 struct trace_event_raw_##template *field; \ 218 struct trace_entry *entry; \ 219 struct trace_seq *p = &iter->tmp_seq; \ 220 \ 221 entry = iter->ent; \ 222 \ 223 if (entry->type != event_##call.event.type) { \ 224 WARN_ON_ONCE(1); \ 225 return TRACE_TYPE_UNHANDLED; \ 226 } \ 227 \ 228 field = (typeof(field))entry; \ 229 \ 230 trace_seq_init(p); \ 231 return trace_output_call(iter, #call, print); \ 232 } \ 233 static struct trace_event_functions trace_event_type_funcs_##call = { \ 234 .trace = trace_raw_output_##call, \ 235 }; 236 237 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 238 239 #include "stages/stage4_event_fields.h" 240 241 #undef DECLARE_EVENT_CLASS 242 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ 243 static struct trace_event_fields trace_event_fields_##call[] = { \ 244 tstruct \ 245 {} }; 246 247 #undef DECLARE_EVENT_SYSCALL_CLASS 248 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 249 250 #undef DEFINE_EVENT_PRINT 251 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 252 253 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 254 255 #include "stages/stage5_get_offsets.h" 256 257 #undef DECLARE_EVENT_CLASS 258 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 259 static inline notrace int trace_event_get_offsets_##call( \ 260 struct trace_event_data_offsets_##call *__data_offsets, proto) \ 261 { \ 262 int __data_size = 0; \ 263 int __maybe_unused __item_length; \ 264 struct trace_event_raw_##call __maybe_unused *entry; \ 265 \ 266 tstruct; \ 267 \ 268 return __data_size; \ 269 } 270 271 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 272 273 /* 274 * Stage 4 of the trace events. 275 * 276 * Override the macros in the event tracepoint header <trace/events/XXX.h> 277 * to include the following: 278 * 279 * For those macros defined with TRACE_EVENT: 280 * 281 * static struct trace_event_call event_<call>; 282 * 283 * static void trace_event_raw_event_<call>(void *__data, proto) 284 * { 285 * struct trace_event_file *trace_file = __data; 286 * struct trace_event_call *event_call = trace_file->event_call; 287 * struct trace_event_data_offsets_<call> __maybe_unused __data_offsets; 288 * unsigned long eflags = trace_file->flags; 289 * enum event_trigger_type __tt = ETT_NONE; 290 * struct ring_buffer_event *event; 291 * struct trace_event_raw_<call> *entry; <-- defined in stage 1 292 * struct trace_buffer *buffer; 293 * unsigned long irq_flags; 294 * int __data_size; 295 * int pc; 296 * 297 * if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) { 298 * if (eflags & EVENT_FILE_FL_TRIGGER_MODE) 299 * event_triggers_call(trace_file, NULL); 300 * if (eflags & EVENT_FILE_FL_SOFT_DISABLED) 301 * return; 302 * } 303 * 304 * local_save_flags(irq_flags); 305 * pc = preempt_count(); 306 * 307 * __data_size = trace_event_get_offsets_<call>(&__data_offsets, args); 308 * 309 * event = trace_event_buffer_lock_reserve(&buffer, trace_file, 310 * event_<call>->event.type, 311 * sizeof(*entry) + __data_size, 312 * irq_flags, pc); 313 * if (!event) 314 * return; 315 * entry = ring_buffer_event_data(event); 316 * 317 * { <assign>; } <-- Here we assign the entries by the __field and 318 * __array macros. 319 * 320 * if (eflags & EVENT_FILE_FL_TRIGGER_COND) 321 * __tt = event_triggers_call(trace_file, entry); 322 * 323 * if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, 324 * &trace_file->flags)) 325 * ring_buffer_discard_commit(buffer, event); 326 * else if (!filter_check_discard(trace_file, entry, buffer, event)) 327 * trace_buffer_unlock_commit(buffer, event, irq_flags, pc); 328 * 329 * if (__tt) 330 * event_triggers_post_call(trace_file, __tt); 331 * } 332 * 333 * static struct trace_event ftrace_event_type_<call> = { 334 * .trace = trace_raw_output_<call>, <-- stage 2 335 * }; 336 * 337 * static char print_fmt_<call>[] = <TP_printk>; 338 * 339 * static struct trace_event_class __used event_class_<template> = { 340 * .system = "<system>", 341 * .fields_array = trace_event_fields_<call>, 342 * .fields = LIST_HEAD_INIT(event_class_##call.fields), 343 * .raw_init = trace_event_raw_init, 344 * .probe = trace_event_raw_event_##call, 345 * .reg = trace_event_reg, 346 * }; 347 * 348 * static struct trace_event_call event_<call> = { 349 * .class = event_class_<template>, 350 * { 351 * .tp = &__tracepoint_<call>, 352 * }, 353 * .event = &ftrace_event_type_<call>, 354 * .print_fmt = print_fmt_<call>, 355 * .flags = TRACE_EVENT_FL_TRACEPOINT, 356 * }; 357 * // its only safe to use pointers when doing linker tricks to 358 * // create an array. 359 * static struct trace_event_call __used 360 * __section("_ftrace_events") *__event_<call> = &event_<call>; 361 * 362 */ 363 364 #ifdef CONFIG_PERF_EVENTS 365 366 #define _TRACE_PERF_PROTO(call, proto) \ 367 static notrace void \ 368 perf_trace_##call(void *__data, proto); 369 370 #define _TRACE_PERF_INIT(call) \ 371 .perf_probe = perf_trace_##call, 372 373 #else 374 #define _TRACE_PERF_PROTO(call, proto) 375 #define _TRACE_PERF_INIT(call) 376 #endif /* CONFIG_PERF_EVENTS */ 377 378 #include "stages/stage6_event_callback.h" 379 380 381 #undef __DECLARE_EVENT_CLASS 382 #define __DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 383 static notrace void \ 384 do_trace_event_raw_event_##call(void *__data, proto) \ 385 { \ 386 struct trace_event_file *trace_file = __data; \ 387 struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\ 388 struct trace_event_buffer fbuffer; \ 389 struct trace_event_raw_##call *entry; \ 390 int __data_size; \ 391 \ 392 if (trace_trigger_soft_disabled(trace_file)) \ 393 return; \ 394 \ 395 __data_size = trace_event_get_offsets_##call(&__data_offsets, args); \ 396 \ 397 entry = trace_event_buffer_reserve(&fbuffer, trace_file, \ 398 sizeof(*entry) + __data_size); \ 399 \ 400 if (!entry) \ 401 return; \ 402 \ 403 tstruct \ 404 \ 405 { assign; } \ 406 \ 407 trace_event_buffer_commit(&fbuffer); \ 408 } 409 410 #undef DECLARE_EVENT_CLASS 411 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 412 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 413 PARAMS(assign), PARAMS(print)) \ 414 static notrace void \ 415 trace_event_raw_event_##call(void *__data, proto) \ 416 { \ 417 do_trace_event_raw_event_##call(__data, args); \ 418 } 419 420 #undef DECLARE_EVENT_SYSCALL_CLASS 421 #define DECLARE_EVENT_SYSCALL_CLASS(call, proto, args, tstruct, assign, print) \ 422 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 423 PARAMS(assign), PARAMS(print)) \ 424 static notrace void \ 425 trace_event_raw_event_##call(void *__data, proto) \ 426 { \ 427 preempt_disable_notrace(); \ 428 do_trace_event_raw_event_##call(__data, args); \ 429 preempt_enable_notrace(); \ 430 } 431 432 /* 433 * The ftrace_test_probe is compiled out, it is only here as a build time check 434 * to make sure that if the tracepoint handling changes, the ftrace probe will 435 * fail to compile unless it too is updated. 436 */ 437 438 #undef DEFINE_EVENT 439 #define DEFINE_EVENT(template, call, proto, args) \ 440 static inline void ftrace_test_probe_##call(void) \ 441 { \ 442 check_trace_callback_type_##call(trace_event_raw_event_##template); \ 443 } 444 445 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 446 447 #undef __DECLARE_EVENT_CLASS 448 449 #include "stages/stage7_class_define.h" 450 451 #undef DECLARE_EVENT_CLASS 452 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 453 _TRACE_PERF_PROTO(call, PARAMS(proto)); \ 454 static char print_fmt_##call[] = print; \ 455 static struct trace_event_class __used __refdata event_class_##call = { \ 456 .system = TRACE_SYSTEM_STRING, \ 457 .fields_array = trace_event_fields_##call, \ 458 .fields = LIST_HEAD_INIT(event_class_##call.fields),\ 459 .raw_init = trace_event_raw_init, \ 460 .probe = trace_event_raw_event_##call, \ 461 .reg = trace_event_reg, \ 462 _TRACE_PERF_INIT(call) \ 463 }; 464 465 #undef DEFINE_EVENT 466 #define DEFINE_EVENT(template, call, proto, args) \ 467 \ 468 static struct trace_event_call __used event_##call = { \ 469 .class = &event_class_##template, \ 470 { \ 471 .tp = &__tracepoint_##call, \ 472 }, \ 473 .event.funcs = &trace_event_type_funcs_##template, \ 474 .print_fmt = print_fmt_##template, \ 475 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 476 }; \ 477 static struct trace_event_call __used \ 478 __section("_ftrace_events") *__event_##call = &event_##call 479 480 #undef DEFINE_EVENT_PRINT 481 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 482 \ 483 static char print_fmt_##call[] = print; \ 484 \ 485 static struct trace_event_call __used event_##call = { \ 486 .class = &event_class_##template, \ 487 { \ 488 .tp = &__tracepoint_##call, \ 489 }, \ 490 .event.funcs = &trace_event_type_funcs_##call, \ 491 .print_fmt = print_fmt_##call, \ 492 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 493 }; \ 494 static struct trace_event_call __used \ 495 __section("_ftrace_events") *__event_##call = &event_##call 496 497 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 498