xref: /openbmc/linux/rust/kernel/init/macros.rs (revision 4af84c6a)
1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2 
3 //! This module provides the macros that actually implement the proc-macros `pin_data` and
4 //! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
5 //! macros.
6 //!
7 //! These macros should never be called directly, since they expect their input to be
8 //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9 //! safe code! Use the public facing macros instead.
10 //!
11 //! This architecture has been chosen because the kernel does not yet have access to `syn` which
12 //! would make matters a lot easier for implementing these as proc-macros.
13 //!
14 //! # Macro expansion example
15 //!
16 //! This section is intended for readers trying to understand the macros in this module and the
17 //! `pin_init!` macros from `init.rs`.
18 //!
19 //! We will look at the following example:
20 //!
21 //! ```rust,ignore
22 //! # use kernel::init::*;
23 //! # use core::pin::Pin;
24 //! #[pin_data]
25 //! #[repr(C)]
26 //! struct Bar<T> {
27 //!     #[pin]
28 //!     t: T,
29 //!     pub x: usize,
30 //! }
31 //!
32 //! impl<T> Bar<T> {
33 //!     fn new(t: T) -> impl PinInit<Self> {
34 //!         pin_init!(Self { t, x: 0 })
35 //!     }
36 //! }
37 //!
38 //! #[pin_data(PinnedDrop)]
39 //! struct Foo {
40 //!     a: usize,
41 //!     #[pin]
42 //!     b: Bar<u32>,
43 //! }
44 //!
45 //! #[pinned_drop]
46 //! impl PinnedDrop for Foo {
47 //!     fn drop(self: Pin<&mut Self>) {
48 //!         pr_info!("{self:p} is getting dropped.");
49 //!     }
50 //! }
51 //!
52 //! let a = 42;
53 //! let initializer = pin_init!(Foo {
54 //!     a,
55 //!     b <- Bar::new(36),
56 //! });
57 //! ```
58 //!
59 //! This example includes the most common and important features of the pin-init API.
60 //!
61 //! Below you can find individual section about the different macro invocations. Here are some
62 //! general things we need to take into account when designing macros:
63 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
64 //!   this ensures that the correct item is used, since users could define their own `mod core {}`
65 //!   and then their own `panic!` inside to execute arbitrary code inside of our macro.
66 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
67 //!   expressions inside of an `unsafe` block in the macro, because this would allow users to do
68 //!   `unsafe` operations without an associated `unsafe` block.
69 //!
70 //! ## `#[pin_data]` on `Bar`
71 //!
72 //! This macro is used to specify which fields are structurally pinned and which fields are not. It
73 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
74 //!
75 //! Here is the definition of `Bar` from our example:
76 //!
77 //! ```rust,ignore
78 //! # use kernel::init::*;
79 //! #[pin_data]
80 //! #[repr(C)]
81 //! struct Bar<T> {
82 //!     #[pin]
83 //!     t: T,
84 //!     pub x: usize,
85 //! }
86 //! ```
87 //!
88 //! This expands to the following code:
89 //!
90 //! ```rust,ignore
91 //! // Firstly the normal definition of the struct, attributes are preserved:
92 //! #[repr(C)]
93 //! struct Bar<T> {
94 //!     t: T,
95 //!     pub x: usize,
96 //! }
97 //! // Then an anonymous constant is defined, this is because we do not want any code to access the
98 //! // types that we define inside:
99 //! const _: () = {
100 //!     // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101 //!     // since we need to implement access functions for each field and thus need to know its
102 //!     // type.
103 //!     struct __ThePinData<T> {
104 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105 //!     }
106 //!     // We implement `Copy` for the pin-data struct, since all functions it defines will take
107 //!     // `self` by value.
108 //!     impl<T> ::core::clone::Clone for __ThePinData<T> {
109 //!         fn clone(&self) -> Self {
110 //!             *self
111 //!         }
112 //!     }
113 //!     impl<T> ::core::marker::Copy for __ThePinData<T> {}
114 //!     // For every field of `Bar`, the pin-data struct will define a function with the same name
115 //!     // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116 //!     // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117 //!     // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118 //!     #[allow(dead_code)]
119 //!     impl<T> __ThePinData<T> {
120 //!         unsafe fn t<E>(
121 //!             self,
122 //!             slot: *mut T,
123 //!             // Since `t` is `#[pin]`, this is `PinInit`.
124 //!             init: impl ::kernel::init::PinInit<T, E>,
125 //!         ) -> ::core::result::Result<(), E> {
126 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
127 //!         }
128 //!         pub unsafe fn x<E>(
129 //!             self,
130 //!             slot: *mut usize,
131 //!             // Since `x` is not `#[pin]`, this is `Init`.
132 //!             init: impl ::kernel::init::Init<usize, E>,
133 //!         ) -> ::core::result::Result<(), E> {
134 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
135 //!         }
136 //!     }
137 //!     // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138 //!     // that we constructed above.
139 //!     unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
140 //!         type PinData = __ThePinData<T>;
141 //!         unsafe fn __pin_data() -> Self::PinData {
142 //!             __ThePinData {
143 //!                 __phantom: ::core::marker::PhantomData,
144 //!             }
145 //!         }
146 //!     }
147 //!     // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148 //!     // struct. This is important to ensure that no user can implement a rouge `__pin_data`
149 //!     // function without using `unsafe`.
150 //!     unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
151 //!         type Datee = Bar<T>;
152 //!     }
153 //!     // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154 //!     // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155 //!     // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156 //!     // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157 //!     // for two reasons:
158 //!     // - `__phantom`: every generic must be used, since we cannot really know which generics
159 //!     //   are used, we declere all and then use everything here once.
160 //!     // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161 //!     //   over it. The lifetime is needed to work around the limitation that trait bounds must
162 //!     //   not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163 //!     //   unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164 //!     //   into accepting these bounds regardless.
165 //!     #[allow(dead_code)]
166 //!     struct __Unpin<'__pin, T> {
167 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169 //!         // Our only `#[pin]` field is `t`.
170 //!         t: T,
171 //!     }
172 //!     #[doc(hidden)]
173 //!     impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174 //!     where
175 //!         __Unpin<'__pin, T>: ::core::marker::Unpin,
176 //!     {}
177 //!     // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178 //!     // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179 //!     // UB with only safe code, so we disallow this by giving a trait implementation error using
180 //!     // a direct impl and a blanket implementation.
181 //!     trait MustNotImplDrop {}
182 //!     // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183 //!     // (normally people want to know if a type has any kind of drop glue at all, here we want
184 //!     // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185 //!     #[allow(drop_bounds)]
186 //!     impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187 //!     impl<T> MustNotImplDrop for Bar<T> {}
188 //!     // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189 //!     // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190 //!     // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191 //!     #[allow(non_camel_case_types)]
192 //!     trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193 //!     impl<
194 //!         T: ::kernel::init::PinnedDrop,
195 //!     > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196 //!     impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197 //! };
198 //! ```
199 //!
200 //! ## `pin_init!` in `impl Bar`
201 //!
202 //! This macro creates an pin-initializer for the given struct. It requires that the struct is
203 //! annotated by `#[pin_data]`.
204 //!
205 //! Here is the impl on `Bar` defining the new function:
206 //!
207 //! ```rust,ignore
208 //! impl<T> Bar<T> {
209 //!     fn new(t: T) -> impl PinInit<Self> {
210 //!         pin_init!(Self { t, x: 0 })
211 //!     }
212 //! }
213 //! ```
214 //!
215 //! This expands to the following code:
216 //!
217 //! ```rust,ignore
218 //! impl<T> Bar<T> {
219 //!     fn new(t: T) -> impl PinInit<Self> {
220 //!         {
221 //!             // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222 //!             // return type and shadow it later when we insert the arbitrary user code. That way
223 //!             // there will be no possibility of returning without `unsafe`.
224 //!             struct __InitOk;
225 //!             // Get the data about fields from the supplied type.
226 //!             // - the function is unsafe, hence the unsafe block
227 //!             // - we `use` the `HasPinData` trait in the block, it is only available in that
228 //!             //   scope.
229 //!             let data = unsafe {
230 //!                 use ::kernel::init::__internal::HasPinData;
231 //!                 Self::__pin_data()
232 //!             };
233 //!             // Ensure that `data` really is of type `PinData` and help with type inference:
234 //!             let init = ::kernel::init::__internal::PinData::make_closure::<
235 //!                 _,
236 //!                 __InitOk,
237 //!                 ::core::convert::Infallible,
238 //!             >(data, move |slot| {
239 //!                 {
240 //!                     // Shadow the structure so it cannot be used to return early. If a user
241 //!                     // tries to write `return Ok(__InitOk)`, then they get a type error,
242 //!                     // since that will refer to this struct instead of the one defined
243 //!                     // above.
244 //!                     struct __InitOk;
245 //!                     // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246 //!                     {
247 //!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248 //!                     }
249 //!                     // Since initialization could fail later (not in this case, since the
250 //!                     // error type is `Infallible`) we will need to drop this field if there
251 //!                     // is an error later. This `DropGuard` will drop the field when it gets
252 //!                     // dropped and has not yet been forgotten.
253 //!                     let t = unsafe {
254 //!                         ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255 //!                     };
256 //!                     // Expansion of `x: 0,`:
257 //!                     // Since this can be an arbitrary expression we cannot place it inside
258 //!                     // of the `unsafe` block, so we bind it here.
259 //!                     {
260 //!                         let x = 0;
261 //!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262 //!                     }
263 //!                     // We again create a `DropGuard`.
264 //!                     let x = unsafe {
265 //!                         ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266 //!                     };
267 //!                     // Since initialization has successfully completed, we can now forget
268 //!                     // the guards. This is not `mem::forget`, since we only have
269 //!                     // `&DropGuard`.
270 //!                     ::core::mem::forget(x);
271 //!                     ::core::mem::forget(t);
272 //!                     // Here we use the type checker to ensure that every field has been
273 //!                     // initialized exactly once, since this is `if false` it will never get
274 //!                     // executed, but still type-checked.
275 //!                     // Additionally we abuse `slot` to automatically infer the correct type
276 //!                     // for the struct. This is also another check that every field is
277 //!                     // accessible from this scope.
278 //!                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
279 //!                     let _ = || {
280 //!                         unsafe {
281 //!                             ::core::ptr::write(
282 //!                                 slot,
283 //!                                 Self {
284 //!                                     // We only care about typecheck finding every field
285 //!                                     // here, the expression does not matter, just conjure
286 //!                                     // one using `panic!()`:
287 //!                                     t: ::core::panic!(),
288 //!                                     x: ::core::panic!(),
289 //!                                 },
290 //!                             );
291 //!                         };
292 //!                     };
293 //!                 }
294 //!                 // We leave the scope above and gain access to the previously shadowed
295 //!                 // `__InitOk` that we need to return.
296 //!                 Ok(__InitOk)
297 //!             });
298 //!             // Change the return type from `__InitOk` to `()`.
299 //!             let init = move |
300 //!                 slot,
301 //!             | -> ::core::result::Result<(), ::core::convert::Infallible> {
302 //!                 init(slot).map(|__InitOk| ())
303 //!             };
304 //!             // Construct the initializer.
305 //!             let init = unsafe {
306 //!                 ::kernel::init::pin_init_from_closure::<
307 //!                     _,
308 //!                     ::core::convert::Infallible,
309 //!                 >(init)
310 //!             };
311 //!             init
312 //!         }
313 //!     }
314 //! }
315 //! ```
316 //!
317 //! ## `#[pin_data]` on `Foo`
318 //!
319 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320 //! differences/new things in the expansion of the `Foo` definition:
321 //!
322 //! ```rust,ignore
323 //! #[pin_data(PinnedDrop)]
324 //! struct Foo {
325 //!     a: usize,
326 //!     #[pin]
327 //!     b: Bar<u32>,
328 //! }
329 //! ```
330 //!
331 //! This expands to the following code:
332 //!
333 //! ```rust,ignore
334 //! struct Foo {
335 //!     a: usize,
336 //!     b: Bar<u32>,
337 //! }
338 //! const _: () = {
339 //!     struct __ThePinData {
340 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341 //!     }
342 //!     impl ::core::clone::Clone for __ThePinData {
343 //!         fn clone(&self) -> Self {
344 //!             *self
345 //!         }
346 //!     }
347 //!     impl ::core::marker::Copy for __ThePinData {}
348 //!     #[allow(dead_code)]
349 //!     impl __ThePinData {
350 //!         unsafe fn b<E>(
351 //!             self,
352 //!             slot: *mut Bar<u32>,
353 //!             init: impl ::kernel::init::PinInit<Bar<u32>, E>,
354 //!         ) -> ::core::result::Result<(), E> {
355 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
356 //!         }
357 //!         unsafe fn a<E>(
358 //!             self,
359 //!             slot: *mut usize,
360 //!             init: impl ::kernel::init::Init<usize, E>,
361 //!         ) -> ::core::result::Result<(), E> {
362 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
363 //!         }
364 //!     }
365 //!     unsafe impl ::kernel::init::__internal::HasPinData for Foo {
366 //!         type PinData = __ThePinData;
367 //!         unsafe fn __pin_data() -> Self::PinData {
368 //!             __ThePinData {
369 //!                 __phantom: ::core::marker::PhantomData,
370 //!             }
371 //!         }
372 //!     }
373 //!     unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
374 //!         type Datee = Foo;
375 //!     }
376 //!     #[allow(dead_code)]
377 //!     struct __Unpin<'__pin> {
378 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380 //!         b: Bar<u32>,
381 //!     }
382 //!     #[doc(hidden)]
383 //!     impl<'__pin> ::core::marker::Unpin for Foo
384 //!     where
385 //!         __Unpin<'__pin>: ::core::marker::Unpin,
386 //!     {}
387 //!     // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388 //!     // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389 //!     // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390 //!     impl ::core::ops::Drop for Foo {
391 //!         fn drop(&mut self) {
392 //!             // Since we are getting dropped, no one else has a reference to `self` and thus we
393 //!             // can assume that we never move.
394 //!             let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395 //!             // Create the unsafe token that proves that we are inside of a destructor, this
396 //!             // type is only allowed to be created in a destructor.
397 //!             let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
398 //!             ::kernel::init::PinnedDrop::drop(pinned, token);
399 //!         }
400 //!     }
401 //! };
402 //! ```
403 //!
404 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405 //!
406 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407 //! extra parameter that should not be used at all. The macro hides that parameter.
408 //!
409 //! Here is the `PinnedDrop` impl for `Foo`:
410 //!
411 //! ```rust,ignore
412 //! #[pinned_drop]
413 //! impl PinnedDrop for Foo {
414 //!     fn drop(self: Pin<&mut Self>) {
415 //!         pr_info!("{self:p} is getting dropped.");
416 //!     }
417 //! }
418 //! ```
419 //!
420 //! This expands to the following code:
421 //!
422 //! ```rust,ignore
423 //! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424 //! unsafe impl ::kernel::init::PinnedDrop for Foo {
425 //!     fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
426 //!         pr_info!("{self:p} is getting dropped.");
427 //!     }
428 //! }
429 //! ```
430 //!
431 //! ## `pin_init!` on `Foo`
432 //!
433 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434 //! of `pin_init!` on `Foo`:
435 //!
436 //! ```rust,ignore
437 //! let a = 42;
438 //! let initializer = pin_init!(Foo {
439 //!     a,
440 //!     b <- Bar::new(36),
441 //! });
442 //! ```
443 //!
444 //! This expands to the following code:
445 //!
446 //! ```rust,ignore
447 //! let a = 42;
448 //! let initializer = {
449 //!     struct __InitOk;
450 //!     let data = unsafe {
451 //!         use ::kernel::init::__internal::HasPinData;
452 //!         Foo::__pin_data()
453 //!     };
454 //!     let init = ::kernel::init::__internal::PinData::make_closure::<
455 //!         _,
456 //!         __InitOk,
457 //!         ::core::convert::Infallible,
458 //!     >(data, move |slot| {
459 //!         {
460 //!             struct __InitOk;
461 //!             {
462 //!                 unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463 //!             }
464 //!             let a = unsafe {
465 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466 //!             };
467 //!             let init = Bar::new(36);
468 //!             unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469 //!             let b = unsafe {
470 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471 //!             };
472 //!             ::core::mem::forget(b);
473 //!             ::core::mem::forget(a);
474 //!             #[allow(unreachable_code, clippy::diverging_sub_expression)]
475 //!             let _ = || {
476 //!                 unsafe {
477 //!                     ::core::ptr::write(
478 //!                         slot,
479 //!                         Foo {
480 //!                             a: ::core::panic!(),
481 //!                             b: ::core::panic!(),
482 //!                         },
483 //!                     );
484 //!                 };
485 //!             };
486 //!         }
487 //!         Ok(__InitOk)
488 //!     });
489 //!     let init = move |
490 //!         slot,
491 //!     | -> ::core::result::Result<(), ::core::convert::Infallible> {
492 //!         init(slot).map(|__InitOk| ())
493 //!     };
494 //!     let init = unsafe {
495 //!         ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496 //!     };
497 //!     init
498 //! };
499 //! ```
500 
501 /// Creates a `unsafe impl<...> PinnedDrop for $type` block.
502 ///
503 /// See [`PinnedDrop`] for more information.
504 #[doc(hidden)]
505 #[macro_export]
506 macro_rules! __pinned_drop {
507     (
508         @impl_sig($($impl_sig:tt)*),
509         @impl_body(
510             $(#[$($attr:tt)*])*
511             fn drop($($sig:tt)*) {
512                 $($inner:tt)*
513             }
514         ),
515     ) => {
516         unsafe $($impl_sig)* {
517             // Inherit all attributes and the type/ident tokens for the signature.
518             $(#[$($attr)*])*
519             fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
520                 $($inner)*
521             }
522         }
523     }
524 }
525 
526 /// This macro first parses the struct definition such that it separates pinned and not pinned
527 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
528 #[doc(hidden)]
529 #[macro_export]
530 macro_rules! __pin_data {
531     // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
532     (parse_input:
533         @args($($pinned_drop:ident)?),
534         @sig(
535             $(#[$($struct_attr:tt)*])*
536             $vis:vis struct $name:ident
537             $(where $($whr:tt)*)?
538         ),
539         @impl_generics($($impl_generics:tt)*),
540         @ty_generics($($ty_generics:tt)*),
541         @body({ $($fields:tt)* }),
542     ) => {
543         // We now use token munching to iterate through all of the fields. While doing this we
544         // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
545         // wants these to be structurally pinned. The rest of the fields are the
546         // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
547         // order to declare the struct.
548         //
549         // In this call we also put some explaining comments for the parameters.
550         $crate::__pin_data!(find_pinned_fields:
551             // Attributes on the struct itself, these will just be propagated to be put onto the
552             // struct definition.
553             @struct_attrs($(#[$($struct_attr)*])*),
554             // The visibility of the struct.
555             @vis($vis),
556             // The name of the struct.
557             @name($name),
558             // The 'impl generics', the generics that will need to be specified on the struct inside
559             // of an `impl<$ty_generics>` block.
560             @impl_generics($($impl_generics)*),
561             // The 'ty generics', the generics that will need to be specified on the impl blocks.
562             @ty_generics($($ty_generics)*),
563             // The where clause of any impl block and the declaration.
564             @where($($($whr)*)?),
565             // The remaining fields tokens that need to be processed.
566             // We add a `,` at the end to ensure correct parsing.
567             @fields_munch($($fields)* ,),
568             // The pinned fields.
569             @pinned(),
570             // The not pinned fields.
571             @not_pinned(),
572             // All fields.
573             @fields(),
574             // The accumulator containing all attributes already parsed.
575             @accum(),
576             // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
577             @is_pinned(),
578             // The proc-macro argument, this should be `PinnedDrop` or ``.
579             @pinned_drop($($pinned_drop)?),
580         );
581     };
582     (find_pinned_fields:
583         @struct_attrs($($struct_attrs:tt)*),
584         @vis($vis:vis),
585         @name($name:ident),
586         @impl_generics($($impl_generics:tt)*),
587         @ty_generics($($ty_generics:tt)*),
588         @where($($whr:tt)*),
589         // We found a PhantomPinned field, this should generally be pinned!
590         @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
591         @pinned($($pinned:tt)*),
592         @not_pinned($($not_pinned:tt)*),
593         @fields($($fields:tt)*),
594         @accum($($accum:tt)*),
595         // This field is not pinned.
596         @is_pinned(),
597         @pinned_drop($($pinned_drop:ident)?),
598     ) => {
599         ::core::compile_error!(concat!(
600             "The field `",
601             stringify!($field),
602             "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
603         ));
604         $crate::__pin_data!(find_pinned_fields:
605             @struct_attrs($($struct_attrs)*),
606             @vis($vis),
607             @name($name),
608             @impl_generics($($impl_generics)*),
609             @ty_generics($($ty_generics)*),
610             @where($($whr)*),
611             @fields_munch($($rest)*),
612             @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
613             @not_pinned($($not_pinned)*),
614             @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
615             @accum(),
616             @is_pinned(),
617             @pinned_drop($($pinned_drop)?),
618         );
619     };
620     (find_pinned_fields:
621         @struct_attrs($($struct_attrs:tt)*),
622         @vis($vis:vis),
623         @name($name:ident),
624         @impl_generics($($impl_generics:tt)*),
625         @ty_generics($($ty_generics:tt)*),
626         @where($($whr:tt)*),
627         // We reached the field declaration.
628         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
629         @pinned($($pinned:tt)*),
630         @not_pinned($($not_pinned:tt)*),
631         @fields($($fields:tt)*),
632         @accum($($accum:tt)*),
633         // This field is pinned.
634         @is_pinned(yes),
635         @pinned_drop($($pinned_drop:ident)?),
636     ) => {
637         $crate::__pin_data!(find_pinned_fields:
638             @struct_attrs($($struct_attrs)*),
639             @vis($vis),
640             @name($name),
641             @impl_generics($($impl_generics)*),
642             @ty_generics($($ty_generics)*),
643             @where($($whr)*),
644             @fields_munch($($rest)*),
645             @pinned($($pinned)* $($accum)* $field: $type,),
646             @not_pinned($($not_pinned)*),
647             @fields($($fields)* $($accum)* $field: $type,),
648             @accum(),
649             @is_pinned(),
650             @pinned_drop($($pinned_drop)?),
651         );
652     };
653     (find_pinned_fields:
654         @struct_attrs($($struct_attrs:tt)*),
655         @vis($vis:vis),
656         @name($name:ident),
657         @impl_generics($($impl_generics:tt)*),
658         @ty_generics($($ty_generics:tt)*),
659         @where($($whr:tt)*),
660         // We reached the field declaration.
661         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
662         @pinned($($pinned:tt)*),
663         @not_pinned($($not_pinned:tt)*),
664         @fields($($fields:tt)*),
665         @accum($($accum:tt)*),
666         // This field is not pinned.
667         @is_pinned(),
668         @pinned_drop($($pinned_drop:ident)?),
669     ) => {
670         $crate::__pin_data!(find_pinned_fields:
671             @struct_attrs($($struct_attrs)*),
672             @vis($vis),
673             @name($name),
674             @impl_generics($($impl_generics)*),
675             @ty_generics($($ty_generics)*),
676             @where($($whr)*),
677             @fields_munch($($rest)*),
678             @pinned($($pinned)*),
679             @not_pinned($($not_pinned)* $($accum)* $field: $type,),
680             @fields($($fields)* $($accum)* $field: $type,),
681             @accum(),
682             @is_pinned(),
683             @pinned_drop($($pinned_drop)?),
684         );
685     };
686     (find_pinned_fields:
687         @struct_attrs($($struct_attrs:tt)*),
688         @vis($vis:vis),
689         @name($name:ident),
690         @impl_generics($($impl_generics:tt)*),
691         @ty_generics($($ty_generics:tt)*),
692         @where($($whr:tt)*),
693         // We found the `#[pin]` attr.
694         @fields_munch(#[pin] $($rest:tt)*),
695         @pinned($($pinned:tt)*),
696         @not_pinned($($not_pinned:tt)*),
697         @fields($($fields:tt)*),
698         @accum($($accum:tt)*),
699         @is_pinned($($is_pinned:ident)?),
700         @pinned_drop($($pinned_drop:ident)?),
701     ) => {
702         $crate::__pin_data!(find_pinned_fields:
703             @struct_attrs($($struct_attrs)*),
704             @vis($vis),
705             @name($name),
706             @impl_generics($($impl_generics)*),
707             @ty_generics($($ty_generics)*),
708             @where($($whr)*),
709             @fields_munch($($rest)*),
710             // We do not include `#[pin]` in the list of attributes, since it is not actually an
711             // attribute that is defined somewhere.
712             @pinned($($pinned)*),
713             @not_pinned($($not_pinned)*),
714             @fields($($fields)*),
715             @accum($($accum)*),
716             // Set this to `yes`.
717             @is_pinned(yes),
718             @pinned_drop($($pinned_drop)?),
719         );
720     };
721     (find_pinned_fields:
722         @struct_attrs($($struct_attrs:tt)*),
723         @vis($vis:vis),
724         @name($name:ident),
725         @impl_generics($($impl_generics:tt)*),
726         @ty_generics($($ty_generics:tt)*),
727         @where($($whr:tt)*),
728         // We reached the field declaration with visibility, for simplicity we only munch the
729         // visibility and put it into `$accum`.
730         @fields_munch($fvis:vis $field:ident $($rest:tt)*),
731         @pinned($($pinned:tt)*),
732         @not_pinned($($not_pinned:tt)*),
733         @fields($($fields:tt)*),
734         @accum($($accum:tt)*),
735         @is_pinned($($is_pinned:ident)?),
736         @pinned_drop($($pinned_drop:ident)?),
737     ) => {
738         $crate::__pin_data!(find_pinned_fields:
739             @struct_attrs($($struct_attrs)*),
740             @vis($vis),
741             @name($name),
742             @impl_generics($($impl_generics)*),
743             @ty_generics($($ty_generics)*),
744             @where($($whr)*),
745             @fields_munch($field $($rest)*),
746             @pinned($($pinned)*),
747             @not_pinned($($not_pinned)*),
748             @fields($($fields)*),
749             @accum($($accum)* $fvis),
750             @is_pinned($($is_pinned)?),
751             @pinned_drop($($pinned_drop)?),
752         );
753     };
754     (find_pinned_fields:
755         @struct_attrs($($struct_attrs:tt)*),
756         @vis($vis:vis),
757         @name($name:ident),
758         @impl_generics($($impl_generics:tt)*),
759         @ty_generics($($ty_generics:tt)*),
760         @where($($whr:tt)*),
761         // Some other attribute, just put it into `$accum`.
762         @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
763         @pinned($($pinned:tt)*),
764         @not_pinned($($not_pinned:tt)*),
765         @fields($($fields:tt)*),
766         @accum($($accum:tt)*),
767         @is_pinned($($is_pinned:ident)?),
768         @pinned_drop($($pinned_drop:ident)?),
769     ) => {
770         $crate::__pin_data!(find_pinned_fields:
771             @struct_attrs($($struct_attrs)*),
772             @vis($vis),
773             @name($name),
774             @impl_generics($($impl_generics)*),
775             @ty_generics($($ty_generics)*),
776             @where($($whr)*),
777             @fields_munch($($rest)*),
778             @pinned($($pinned)*),
779             @not_pinned($($not_pinned)*),
780             @fields($($fields)*),
781             @accum($($accum)* #[$($attr)*]),
782             @is_pinned($($is_pinned)?),
783             @pinned_drop($($pinned_drop)?),
784         );
785     };
786     (find_pinned_fields:
787         @struct_attrs($($struct_attrs:tt)*),
788         @vis($vis:vis),
789         @name($name:ident),
790         @impl_generics($($impl_generics:tt)*),
791         @ty_generics($($ty_generics:tt)*),
792         @where($($whr:tt)*),
793         // We reached the end of the fields, plus an optional additional comma, since we added one
794         // before and the user is also allowed to put a trailing comma.
795         @fields_munch($(,)?),
796         @pinned($($pinned:tt)*),
797         @not_pinned($($not_pinned:tt)*),
798         @fields($($fields:tt)*),
799         @accum(),
800         @is_pinned(),
801         @pinned_drop($($pinned_drop:ident)?),
802     ) => {
803         // Declare the struct with all fields in the correct order.
804         $($struct_attrs)*
805         $vis struct $name <$($impl_generics)*>
806         where $($whr)*
807         {
808             $($fields)*
809         }
810 
811         // We put the rest into this const item, because it then will not be accessible to anything
812         // outside.
813         const _: () = {
814             // We declare this struct which will host all of the projection function for our type.
815             // it will be invariant over all generic parameters which are inherited from the
816             // struct.
817             $vis struct __ThePinData<$($impl_generics)*>
818             where $($whr)*
819             {
820                 __phantom: ::core::marker::PhantomData<
821                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
822                 >,
823             }
824 
825             impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
826             where $($whr)*
827             {
828                 fn clone(&self) -> Self { *self }
829             }
830 
831             impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
832             where $($whr)*
833             {}
834 
835             // Make all projection functions.
836             $crate::__pin_data!(make_pin_data:
837                 @pin_data(__ThePinData),
838                 @impl_generics($($impl_generics)*),
839                 @ty_generics($($ty_generics)*),
840                 @where($($whr)*),
841                 @pinned($($pinned)*),
842                 @not_pinned($($not_pinned)*),
843             );
844 
845             // SAFETY: We have added the correct projection functions above to `__ThePinData` and
846             // we also use the least restrictive generics possible.
847             unsafe impl<$($impl_generics)*>
848                 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
849             where $($whr)*
850             {
851                 type PinData = __ThePinData<$($ty_generics)*>;
852 
853                 unsafe fn __pin_data() -> Self::PinData {
854                     __ThePinData { __phantom: ::core::marker::PhantomData }
855                 }
856             }
857 
858             unsafe impl<$($impl_generics)*>
859                 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
860             where $($whr)*
861             {
862                 type Datee = $name<$($ty_generics)*>;
863             }
864 
865             // This struct will be used for the unpin analysis. Since only structurally pinned
866             // fields are relevant whether the struct should implement `Unpin`.
867             #[allow(dead_code)]
868             struct __Unpin <'__pin, $($impl_generics)*>
869             where $($whr)*
870             {
871                 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
872                 __phantom: ::core::marker::PhantomData<
873                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
874                 >,
875                 // Only the pinned fields.
876                 $($pinned)*
877             }
878 
879             #[doc(hidden)]
880             impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
881             where
882                 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
883                 $($whr)*
884             {}
885 
886             // We need to disallow normal `Drop` implementation, the exact behavior depends on
887             // whether `PinnedDrop` was specified as the parameter.
888             $crate::__pin_data!(drop_prevention:
889                 @name($name),
890                 @impl_generics($($impl_generics)*),
891                 @ty_generics($($ty_generics)*),
892                 @where($($whr)*),
893                 @pinned_drop($($pinned_drop)?),
894             );
895         };
896     };
897     // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
898     (drop_prevention:
899         @name($name:ident),
900         @impl_generics($($impl_generics:tt)*),
901         @ty_generics($($ty_generics:tt)*),
902         @where($($whr:tt)*),
903         @pinned_drop(),
904     ) => {
905         // We prevent this by creating a trait that will be implemented for all types implementing
906         // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
907         // if it also implements `Drop`
908         trait MustNotImplDrop {}
909         #[allow(drop_bounds)]
910         impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
911         impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
912         where $($whr)* {}
913         // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
914         // They might implement `PinnedDrop` correctly for the struct, but forget to give
915         // `PinnedDrop` as the parameter to `#[pin_data]`.
916         #[allow(non_camel_case_types)]
917         trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
918         impl<T: $crate::init::PinnedDrop>
919             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
920         impl<$($impl_generics)*>
921             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
922         where $($whr)* {}
923     };
924     // When `PinnedDrop` was specified we just implement `Drop` and delegate.
925     (drop_prevention:
926         @name($name:ident),
927         @impl_generics($($impl_generics:tt)*),
928         @ty_generics($($ty_generics:tt)*),
929         @where($($whr:tt)*),
930         @pinned_drop(PinnedDrop),
931     ) => {
932         impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
933         where $($whr)*
934         {
935             fn drop(&mut self) {
936                 // SAFETY: Since this is a destructor, `self` will not move after this function
937                 // terminates, since it is inaccessible.
938                 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
939                 // SAFETY: Since this is a drop function, we can create this token to call the
940                 // pinned destructor of this type.
941                 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
942                 $crate::init::PinnedDrop::drop(pinned, token);
943             }
944         }
945     };
946     // If some other parameter was specified, we emit a readable error.
947     (drop_prevention:
948         @name($name:ident),
949         @impl_generics($($impl_generics:tt)*),
950         @ty_generics($($ty_generics:tt)*),
951         @where($($whr:tt)*),
952         @pinned_drop($($rest:tt)*),
953     ) => {
954         compile_error!(
955             "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
956             stringify!($($rest)*),
957         );
958     };
959     (make_pin_data:
960         @pin_data($pin_data:ident),
961         @impl_generics($($impl_generics:tt)*),
962         @ty_generics($($ty_generics:tt)*),
963         @where($($whr:tt)*),
964         @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
965         @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
966     ) => {
967         // For every field, we create a projection function according to its projection type. If a
968         // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
969         // structurally pinned, then it can be initialized via `Init`.
970         //
971         // The functions are `unsafe` to prevent accidentally calling them.
972         #[allow(dead_code)]
973         impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
974         where $($whr)*
975         {
976             $(
977                 $(#[$($p_attr)*])*
978                 $pvis unsafe fn $p_field<E>(
979                     self,
980                     slot: *mut $p_type,
981                     init: impl $crate::init::PinInit<$p_type, E>,
982                 ) -> ::core::result::Result<(), E> {
983                     unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
984                 }
985             )*
986             $(
987                 $(#[$($attr)*])*
988                 $fvis unsafe fn $field<E>(
989                     self,
990                     slot: *mut $type,
991                     init: impl $crate::init::Init<$type, E>,
992                 ) -> ::core::result::Result<(), E> {
993                     unsafe { $crate::init::Init::__init(init, slot) }
994                 }
995             )*
996         }
997     };
998 }
999 
1000 /// The internal init macro. Do not call manually!
1001 ///
1002 /// This is called by the `{try_}{pin_}init!` macros with various inputs.
1003 ///
1004 /// This macro has multiple internal call configurations, these are always the very first ident:
1005 /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1006 /// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1007 /// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1008 /// - `make_initializer`: recursively create the struct initializer that guarantees that every
1009 ///   field has been initialized exactly once.
1010 #[doc(hidden)]
1011 #[macro_export]
1012 macro_rules! __init_internal {
1013     (
1014         @this($($this:ident)?),
1015         @typ($t:path),
1016         @fields($($fields:tt)*),
1017         @error($err:ty),
1018         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1019         // case.
1020         @data($data:ident, $($use_data:ident)?),
1021         // `HasPinData` or `HasInitData`.
1022         @has_data($has_data:ident, $get_data:ident),
1023         // `pin_init_from_closure` or `init_from_closure`.
1024         @construct_closure($construct_closure:ident),
1025         @munch_fields(),
1026     ) => {
1027         $crate::__init_internal!(with_update_parsed:
1028             @this($($this)?),
1029             @typ($t),
1030             @fields($($fields)*),
1031             @error($err),
1032             @data($data, $($use_data)?),
1033             @has_data($has_data, $get_data),
1034             @construct_closure($construct_closure),
1035             @zeroed(), // Nothing means default behavior.
1036         )
1037     };
1038     (
1039         @this($($this:ident)?),
1040         @typ($t:path),
1041         @fields($($fields:tt)*),
1042         @error($err:ty),
1043         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1044         // case.
1045         @data($data:ident, $($use_data:ident)?),
1046         // `HasPinData` or `HasInitData`.
1047         @has_data($has_data:ident, $get_data:ident),
1048         // `pin_init_from_closure` or `init_from_closure`.
1049         @construct_closure($construct_closure:ident),
1050         @munch_fields(..Zeroable::zeroed()),
1051     ) => {
1052         $crate::__init_internal!(with_update_parsed:
1053             @this($($this)?),
1054             @typ($t),
1055             @fields($($fields)*),
1056             @error($err),
1057             @data($data, $($use_data)?),
1058             @has_data($has_data, $get_data),
1059             @construct_closure($construct_closure),
1060             @zeroed(()), // `()` means zero all fields not mentioned.
1061         )
1062     };
1063     (
1064         @this($($this:ident)?),
1065         @typ($t:path),
1066         @fields($($fields:tt)*),
1067         @error($err:ty),
1068         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1069         // case.
1070         @data($data:ident, $($use_data:ident)?),
1071         // `HasPinData` or `HasInitData`.
1072         @has_data($has_data:ident, $get_data:ident),
1073         // `pin_init_from_closure` or `init_from_closure`.
1074         @construct_closure($construct_closure:ident),
1075         @munch_fields($ignore:tt $($rest:tt)*),
1076     ) => {
1077         $crate::__init_internal!(
1078             @this($($this)?),
1079             @typ($t),
1080             @fields($($fields)*),
1081             @error($err),
1082             @data($data, $($use_data)?),
1083             @has_data($has_data, $get_data),
1084             @construct_closure($construct_closure),
1085             @munch_fields($($rest)*),
1086         )
1087     };
1088     (with_update_parsed:
1089         @this($($this:ident)?),
1090         @typ($t:path),
1091         @fields($($fields:tt)*),
1092         @error($err:ty),
1093         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1094         // case.
1095         @data($data:ident, $($use_data:ident)?),
1096         // `HasPinData` or `HasInitData`.
1097         @has_data($has_data:ident, $get_data:ident),
1098         // `pin_init_from_closure` or `init_from_closure`.
1099         @construct_closure($construct_closure:ident),
1100         @zeroed($($init_zeroed:expr)?),
1101     ) => {{
1102         // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1103         // type and shadow it later when we insert the arbitrary user code. That way there will be
1104         // no possibility of returning without `unsafe`.
1105         struct __InitOk;
1106         // Get the data about fields from the supplied type.
1107         let data = unsafe {
1108             use $crate::init::__internal::$has_data;
1109             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1110             // information that is associated to already parsed fragments, so a path fragment
1111             // cannot be used in this position. Doing the retokenization results in valid rust
1112             // code.
1113             ::kernel::macros::paste!($t::$get_data())
1114         };
1115         // Ensure that `data` really is of type `$data` and help with type inference:
1116         let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1117             data,
1118             move |slot| {
1119                 {
1120                     // Shadow the structure so it cannot be used to return early.
1121                     struct __InitOk;
1122                     // If `$init_zeroed` is present we should zero the slot now and not emit an
1123                     // error when fields are missing (since they will be zeroed). We also have to
1124                     // check that the type actually implements `Zeroable`.
1125                     $({
1126                         fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1127                         // Ensure that the struct is indeed `Zeroable`.
1128                         assert_zeroable(slot);
1129                         // SAFETY: The type implements `Zeroable` by the check above.
1130                         unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1131                         $init_zeroed // This will be `()` if set.
1132                     })?
1133                     // Create the `this` so it can be referenced by the user inside of the
1134                     // expressions creating the individual fields.
1135                     $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1136                     // Initialize every field.
1137                     $crate::__init_internal!(init_slot($($use_data)?):
1138                         @data(data),
1139                         @slot(slot),
1140                         @guards(),
1141                         @munch_fields($($fields)*,),
1142                     );
1143                     // We use unreachable code to ensure that all fields have been mentioned exactly
1144                     // once, this struct initializer will still be type-checked and complain with a
1145                     // very natural error message if a field is forgotten/mentioned more than once.
1146                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
1147                     let _ = || {
1148                         $crate::__init_internal!(make_initializer:
1149                             @slot(slot),
1150                             @type_name($t),
1151                             @munch_fields($($fields)*,),
1152                             @acc(),
1153                         );
1154                     };
1155                 }
1156                 Ok(__InitOk)
1157             }
1158         );
1159         let init = move |slot| -> ::core::result::Result<(), $err> {
1160             init(slot).map(|__InitOk| ())
1161         };
1162         let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1163         init
1164     }};
1165     (init_slot($($use_data:ident)?):
1166         @data($data:ident),
1167         @slot($slot:ident),
1168         @guards($($guards:ident,)*),
1169         @munch_fields($(..Zeroable::zeroed())? $(,)?),
1170     ) => {
1171         // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1172         // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1173         $(::core::mem::forget($guards);)*
1174     };
1175     (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1176         @data($data:ident),
1177         @slot($slot:ident),
1178         @guards($($guards:ident,)*),
1179         // In-place initialization syntax.
1180         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1181     ) => {
1182         let init = $val;
1183         // Call the initializer.
1184         //
1185         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1186         // return when an error/panic occurs.
1187         // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1188         unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1189         // Create the drop guard:
1190         //
1191         // We rely on macro hygiene to make it impossible for users to access this local variable.
1192         // We use `paste!` to create new hygiene for `$field`.
1193         ::kernel::macros::paste! {
1194             // SAFETY: We forget the guard later when initialization has succeeded.
1195             let [<$field>] = unsafe {
1196                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1197             };
1198 
1199             $crate::__init_internal!(init_slot($use_data):
1200                 @data($data),
1201                 @slot($slot),
1202                 @guards([<$field>], $($guards,)*),
1203                 @munch_fields($($rest)*),
1204             );
1205         }
1206     };
1207     (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1208         @data($data:ident),
1209         @slot($slot:ident),
1210         @guards($($guards:ident,)*),
1211         // In-place initialization syntax.
1212         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1213     ) => {
1214         let init = $val;
1215         // Call the initializer.
1216         //
1217         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1218         // return when an error/panic occurs.
1219         unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1220         // Create the drop guard:
1221         //
1222         // We rely on macro hygiene to make it impossible for users to access this local variable.
1223         // We use `paste!` to create new hygiene for `$field`.
1224         ::kernel::macros::paste! {
1225             // SAFETY: We forget the guard later when initialization has succeeded.
1226             let [<$field>] = unsafe {
1227                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1228             };
1229 
1230             $crate::__init_internal!(init_slot():
1231                 @data($data),
1232                 @slot($slot),
1233                 @guards([<$field>], $($guards,)*),
1234                 @munch_fields($($rest)*),
1235             );
1236         }
1237     };
1238     (init_slot($($use_data:ident)?):
1239         @data($data:ident),
1240         @slot($slot:ident),
1241         @guards($($guards:ident,)*),
1242         // Init by-value.
1243         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1244     ) => {
1245         {
1246             $(let $field = $val;)?
1247             // Initialize the field.
1248             //
1249             // SAFETY: The memory at `slot` is uninitialized.
1250             unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1251         }
1252         // Create the drop guard:
1253         //
1254         // We rely on macro hygiene to make it impossible for users to access this local variable.
1255         // We use `paste!` to create new hygiene for `$field`.
1256         ::kernel::macros::paste! {
1257             // SAFETY: We forget the guard later when initialization has succeeded.
1258             let [<$field>] = unsafe {
1259                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1260             };
1261 
1262             $crate::__init_internal!(init_slot($($use_data)?):
1263                 @data($data),
1264                 @slot($slot),
1265                 @guards([<$field>], $($guards,)*),
1266                 @munch_fields($($rest)*),
1267             );
1268         }
1269     };
1270     (make_initializer:
1271         @slot($slot:ident),
1272         @type_name($t:path),
1273         @munch_fields(..Zeroable::zeroed() $(,)?),
1274         @acc($($acc:tt)*),
1275     ) => {
1276         // Endpoint, nothing more to munch, create the initializer. Since the users specified
1277         // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1278         // not been overwritten are thus zero and initialized. We still check that all fields are
1279         // actually accessible by using the struct update syntax ourselves.
1280         // We are inside of a closure that is never executed and thus we can abuse `slot` to
1281         // get the correct type inference here:
1282         #[allow(unused_assignments)]
1283         unsafe {
1284             let mut zeroed = ::core::mem::zeroed();
1285             // We have to use type inference here to make zeroed have the correct type. This does
1286             // not get executed, so it has no effect.
1287             ::core::ptr::write($slot, zeroed);
1288             zeroed = ::core::mem::zeroed();
1289             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1290             // information that is associated to already parsed fragments, so a path fragment
1291             // cannot be used in this position. Doing the retokenization results in valid rust
1292             // code.
1293             ::kernel::macros::paste!(
1294                 ::core::ptr::write($slot, $t {
1295                     $($acc)*
1296                     ..zeroed
1297                 });
1298             );
1299         }
1300     };
1301     (make_initializer:
1302         @slot($slot:ident),
1303         @type_name($t:path),
1304         @munch_fields($(,)?),
1305         @acc($($acc:tt)*),
1306     ) => {
1307         // Endpoint, nothing more to munch, create the initializer.
1308         // Since we are in the closure that is never called, this will never get executed.
1309         // We abuse `slot` to get the correct type inference here:
1310         unsafe {
1311             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1312             // information that is associated to already parsed fragments, so a path fragment
1313             // cannot be used in this position. Doing the retokenization results in valid rust
1314             // code.
1315             ::kernel::macros::paste!(
1316                 ::core::ptr::write($slot, $t {
1317                     $($acc)*
1318                 });
1319             );
1320         }
1321     };
1322     (make_initializer:
1323         @slot($slot:ident),
1324         @type_name($t:path),
1325         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1326         @acc($($acc:tt)*),
1327     ) => {
1328         $crate::__init_internal!(make_initializer:
1329             @slot($slot),
1330             @type_name($t),
1331             @munch_fields($($rest)*),
1332             @acc($($acc)* $field: ::core::panic!(),),
1333         );
1334     };
1335     (make_initializer:
1336         @slot($slot:ident),
1337         @type_name($t:path),
1338         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1339         @acc($($acc:tt)*),
1340     ) => {
1341         $crate::__init_internal!(make_initializer:
1342             @slot($slot),
1343             @type_name($t),
1344             @munch_fields($($rest)*),
1345             @acc($($acc)* $field: ::core::panic!(),),
1346         );
1347     };
1348 }
1349 
1350 #[doc(hidden)]
1351 #[macro_export]
1352 macro_rules! __derive_zeroable {
1353     (parse_input:
1354         @sig(
1355             $(#[$($struct_attr:tt)*])*
1356             $vis:vis struct $name:ident
1357             $(where $($whr:tt)*)?
1358         ),
1359         @impl_generics($($impl_generics:tt)*),
1360         @ty_generics($($ty_generics:tt)*),
1361         @body({
1362             $(
1363                 $(#[$($field_attr:tt)*])*
1364                 $field:ident : $field_ty:ty
1365             ),* $(,)?
1366         }),
1367     ) => {
1368         // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1369         #[automatically_derived]
1370         unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1371         where
1372             $($($whr)*)?
1373         {}
1374         const _: () = {
1375             fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1376             fn ensure_zeroable<$($impl_generics)*>()
1377                 where $($($whr)*)?
1378             {
1379                 $(assert_zeroable::<$field_ty>();)*
1380             }
1381         };
1382     };
1383 }
1384