xref: /openbmc/linux/rust/kernel/init.rs (revision 92c4a1e7)
1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2 
3 //! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4 //!
5 //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6 //! overflow.
7 //!
8 //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9 //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10 //!
11 //! # Overview
12 //!
13 //! To initialize a `struct` with an in-place constructor you will need two things:
14 //! - an in-place constructor,
15 //! - a memory location that can hold your `struct`.
16 //!
17 //! To get an in-place constructor there are generally three options:
18 //! - directly creating an in-place constructor using the [`pin_init!`] macro,
19 //! - a custom function/macro returning an in-place constructor provided by someone else,
20 //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
21 //!
22 //! Aside from pinned initialization, this API also supports in-place construction without pinning,
23 //! the macros/types/functions are generally named like the pinned variants without the `pin`
24 //! prefix.
25 //!
26 //! # Examples
27 //!
28 //! ## Using the [`pin_init!`] macro
29 //!
30 //! If you want to use [`PinInit`], then you will have to annotate your `struct` with
31 //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
32 //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
33 //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
34 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
35 //!
36 //! ```rust
37 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
38 //! use kernel::{prelude::*, sync::Mutex, new_mutex};
39 //! # use core::pin::Pin;
40 //! #[pin_data]
41 //! struct Foo {
42 //!     #[pin]
43 //!     a: Mutex<usize>,
44 //!     b: u32,
45 //! }
46 //!
47 //! let foo = pin_init!(Foo {
48 //!     a <- new_mutex!(42, "Foo::a"),
49 //!     b: 24,
50 //! });
51 //! ```
52 //!
53 //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
54 //! (or just the stack) to actually initialize a `Foo`:
55 //!
56 //! ```rust
57 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
58 //! # use kernel::{prelude::*, sync::Mutex, new_mutex};
59 //! # use core::pin::Pin;
60 //! # #[pin_data]
61 //! # struct Foo {
62 //! #     #[pin]
63 //! #     a: Mutex<usize>,
64 //! #     b: u32,
65 //! # }
66 //! # let foo = pin_init!(Foo {
67 //! #     a <- new_mutex!(42, "Foo::a"),
68 //! #     b: 24,
69 //! # });
70 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo);
71 //! ```
72 //!
73 //! For more information see the [`pin_init!`] macro.
74 //!
75 //! ## Using a custom function/macro that returns an initializer
76 //!
77 //! Many types from the kernel supply a function/macro that returns an initializer, because the
78 //! above method only works for types where you can access the fields.
79 //!
80 //! ```rust
81 //! # use kernel::{new_mutex, sync::{Arc, Mutex}};
82 //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx"));
83 //! ```
84 //!
85 //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
86 //!
87 //! ```rust
88 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
89 //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init};
90 //! #[pin_data]
91 //! struct DriverData {
92 //!     #[pin]
93 //!     status: Mutex<i32>,
94 //!     buffer: Box<[u8; 1_000_000]>,
95 //! }
96 //!
97 //! impl DriverData {
98 //!     fn new() -> impl PinInit<Self, Error> {
99 //!         try_pin_init!(Self {
100 //!             status <- new_mutex!(0, "DriverData::status"),
101 //!             buffer: Box::init(kernel::init::zeroed())?,
102 //!         })
103 //!     }
104 //! }
105 //! ```
106 //!
107 //! [`sync`]: kernel::sync
108 //! [pinning]: https://doc.rust-lang.org/std/pin/index.html
109 //! [structurally pinned fields]:
110 //!     https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
111 //! [`Arc<T>`]: crate::sync::Arc
112 //! [`impl PinInit<Foo>`]: PinInit
113 //! [`impl PinInit<T, E>`]: PinInit
114 //! [`impl Init<T, E>`]: Init
115 //! [`Opaque`]: kernel::types::Opaque
116 //! [`pin_data`]: ::macros::pin_data
117 
118 use crate::{
119     error::{self, Error},
120     sync::UniqueArc,
121 };
122 use alloc::boxed::Box;
123 use core::{
124     alloc::AllocError, cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit,
125     pin::Pin, ptr,
126 };
127 
128 #[doc(hidden)]
129 pub mod __internal;
130 #[doc(hidden)]
131 pub mod macros;
132 
133 /// Construct an in-place, pinned initializer for `struct`s.
134 ///
135 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
136 /// [`try_pin_init!`].
137 ///
138 /// The syntax is almost identical to that of a normal `struct` initializer:
139 ///
140 /// ```rust
141 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
142 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
143 /// # use core::pin::Pin;
144 /// #[pin_data]
145 /// struct Foo {
146 ///     a: usize,
147 ///     b: Bar,
148 /// }
149 ///
150 /// #[pin_data]
151 /// struct Bar {
152 ///     x: u32,
153 /// }
154 ///
155 /// # fn demo() -> impl PinInit<Foo> {
156 /// let a = 42;
157 ///
158 /// let initializer = pin_init!(Foo {
159 ///     a,
160 ///     b: Bar {
161 ///         x: 64,
162 ///     },
163 /// });
164 /// # initializer }
165 /// # Box::pin_init(demo()).unwrap();
166 /// ```
167 ///
168 /// Arbitrary Rust expressions can be used to set the value of a variable.
169 ///
170 /// The fields are initialized in the order that they appear in the initializer. So it is possible
171 /// to read already initialized fields using raw pointers.
172 ///
173 /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
174 /// initializer.
175 ///
176 /// # Init-functions
177 ///
178 /// When working with this API it is often desired to let others construct your types without
179 /// giving access to all fields. This is where you would normally write a plain function `new`
180 /// that would return a new instance of your type. With this API that is also possible.
181 /// However, there are a few extra things to keep in mind.
182 ///
183 /// To create an initializer function, simply declare it like this:
184 ///
185 /// ```rust
186 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
187 /// # use kernel::{init, pin_init, prelude::*, init::*};
188 /// # use core::pin::Pin;
189 /// # #[pin_data]
190 /// # struct Foo {
191 /// #     a: usize,
192 /// #     b: Bar,
193 /// # }
194 /// # #[pin_data]
195 /// # struct Bar {
196 /// #     x: u32,
197 /// # }
198 /// impl Foo {
199 ///     fn new() -> impl PinInit<Self> {
200 ///         pin_init!(Self {
201 ///             a: 42,
202 ///             b: Bar {
203 ///                 x: 64,
204 ///             },
205 ///         })
206 ///     }
207 /// }
208 /// ```
209 ///
210 /// Users of `Foo` can now create it like this:
211 ///
212 /// ```rust
213 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
214 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
215 /// # use core::pin::Pin;
216 /// # #[pin_data]
217 /// # struct Foo {
218 /// #     a: usize,
219 /// #     b: Bar,
220 /// # }
221 /// # #[pin_data]
222 /// # struct Bar {
223 /// #     x: u32,
224 /// # }
225 /// # impl Foo {
226 /// #     fn new() -> impl PinInit<Self> {
227 /// #         pin_init!(Self {
228 /// #             a: 42,
229 /// #             b: Bar {
230 /// #                 x: 64,
231 /// #             },
232 /// #         })
233 /// #     }
234 /// # }
235 /// let foo = Box::pin_init(Foo::new());
236 /// ```
237 ///
238 /// They can also easily embed it into their own `struct`s:
239 ///
240 /// ```rust
241 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
242 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
243 /// # use core::pin::Pin;
244 /// # #[pin_data]
245 /// # struct Foo {
246 /// #     a: usize,
247 /// #     b: Bar,
248 /// # }
249 /// # #[pin_data]
250 /// # struct Bar {
251 /// #     x: u32,
252 /// # }
253 /// # impl Foo {
254 /// #     fn new() -> impl PinInit<Self> {
255 /// #         pin_init!(Self {
256 /// #             a: 42,
257 /// #             b: Bar {
258 /// #                 x: 64,
259 /// #             },
260 /// #         })
261 /// #     }
262 /// # }
263 /// #[pin_data]
264 /// struct FooContainer {
265 ///     #[pin]
266 ///     foo1: Foo,
267 ///     #[pin]
268 ///     foo2: Foo,
269 ///     other: u32,
270 /// }
271 ///
272 /// impl FooContainer {
273 ///     fn new(other: u32) -> impl PinInit<Self> {
274 ///         pin_init!(Self {
275 ///             foo1 <- Foo::new(),
276 ///             foo2 <- Foo::new(),
277 ///             other,
278 ///         })
279 ///     }
280 /// }
281 /// ```
282 ///
283 /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
284 /// This signifies that the given field is initialized in-place. As with `struct` initializers, just
285 /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
286 ///
287 /// # Syntax
288 ///
289 /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
290 /// the following modifications is expected:
291 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
292 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
293 ///   pointer named `this` inside of the initializer.
294 ///
295 /// For instance:
296 ///
297 /// ```rust
298 /// # use kernel::pin_init;
299 /// # use macros::pin_data;
300 /// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
301 /// #[pin_data]
302 /// struct Buf {
303 ///     // `ptr` points into `buf`.
304 ///     ptr: *mut u8,
305 ///     buf: [u8; 64],
306 ///     #[pin]
307 ///     pin: PhantomPinned,
308 /// }
309 /// pin_init!(&this in Buf {
310 ///     buf: [0; 64],
311 ///     ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
312 ///     pin: PhantomPinned,
313 /// });
314 /// ```
315 ///
316 /// [`try_pin_init!`]: kernel::try_pin_init
317 /// [`NonNull<Self>`]: core::ptr::NonNull
318 // For a detailed example of how this macro works, see the module documentation of the hidden
319 // module `__internal` inside of `init/__internal.rs`.
320 #[macro_export]
321 macro_rules! pin_init {
322     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
323         $($fields:tt)*
324     }) => {
325         $crate::try_pin_init!(
326             @this($($this)?),
327             @typ($t $(::<$($generics),*>)?),
328             @fields($($fields)*),
329             @error(::core::convert::Infallible),
330         )
331     };
332 }
333 
334 /// Construct an in-place, fallible pinned initializer for `struct`s.
335 ///
336 /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
337 ///
338 /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
339 /// initialization and return the error.
340 ///
341 /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
342 /// initialization fails, the memory can be safely deallocated without any further modifications.
343 ///
344 /// This macro defaults the error to [`Error`].
345 ///
346 /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
347 /// after the `struct` initializer to specify the error type you want to use.
348 ///
349 /// # Examples
350 ///
351 /// ```rust
352 /// # #![feature(new_uninit)]
353 /// use kernel::{init::{self, PinInit}, error::Error};
354 /// #[pin_data]
355 /// struct BigBuf {
356 ///     big: Box<[u8; 1024 * 1024 * 1024]>,
357 ///     small: [u8; 1024 * 1024],
358 ///     ptr: *mut u8,
359 /// }
360 ///
361 /// impl BigBuf {
362 ///     fn new() -> impl PinInit<Self, Error> {
363 ///         try_pin_init!(Self {
364 ///             big: Box::init(init::zeroed())?,
365 ///             small: [0; 1024 * 1024],
366 ///             ptr: core::ptr::null_mut(),
367 ///         }? Error)
368 ///     }
369 /// }
370 /// ```
371 // For a detailed example of how this macro works, see the module documentation of the hidden
372 // module `__internal` inside of `init/__internal.rs`.
373 #[macro_export]
374 macro_rules! try_pin_init {
375     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
376         $($fields:tt)*
377     }) => {
378         $crate::try_pin_init!(
379             @this($($this)?),
380             @typ($t $(::<$($generics),*>)? ),
381             @fields($($fields)*),
382             @error($crate::error::Error),
383         )
384     };
385     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
386         $($fields:tt)*
387     }? $err:ty) => {
388         $crate::try_pin_init!(
389             @this($($this)?),
390             @typ($t $(::<$($generics),*>)? ),
391             @fields($($fields)*),
392             @error($err),
393         )
394     };
395     (
396         @this($($this:ident)?),
397         @typ($t:ident $(::<$($generics:ty),*>)?),
398         @fields($($fields:tt)*),
399         @error($err:ty),
400     ) => {{
401         // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
402         // type and shadow it later when we insert the arbitrary user code. That way there will be
403         // no possibility of returning without `unsafe`.
404         struct __InitOk;
405         // Get the pin data from the supplied type.
406         let data = unsafe {
407             use $crate::init::__internal::HasPinData;
408             $t$(::<$($generics),*>)?::__pin_data()
409         };
410         // Ensure that `data` really is of type `PinData` and help with type inference:
411         let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>(
412             data,
413             move |slot| {
414                 {
415                     // Shadow the structure so it cannot be used to return early.
416                     struct __InitOk;
417                     // Create the `this` so it can be referenced by the user inside of the
418                     // expressions creating the individual fields.
419                     $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
420                     // Initialize every field.
421                     $crate::try_pin_init!(init_slot:
422                         @data(data),
423                         @slot(slot),
424                         @munch_fields($($fields)*,),
425                     );
426                     // We use unreachable code to ensure that all fields have been mentioned exactly
427                     // once, this struct initializer will still be type-checked and complain with a
428                     // very natural error message if a field is forgotten/mentioned more than once.
429                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
430                     if false {
431                         $crate::try_pin_init!(make_initializer:
432                             @slot(slot),
433                             @type_name($t),
434                             @munch_fields($($fields)*,),
435                             @acc(),
436                         );
437                     }
438                     // Forget all guards, since initialization was a success.
439                     $crate::try_pin_init!(forget_guards:
440                         @munch_fields($($fields)*,),
441                     );
442                 }
443                 Ok(__InitOk)
444             }
445         );
446         let init = move |slot| -> ::core::result::Result<(), $err> {
447             init(slot).map(|__InitOk| ())
448         };
449         let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) };
450         init
451     }};
452     (init_slot:
453         @data($data:ident),
454         @slot($slot:ident),
455         @munch_fields($(,)?),
456     ) => {
457         // Endpoint of munching, no fields are left.
458     };
459     (init_slot:
460         @data($data:ident),
461         @slot($slot:ident),
462         // In-place initialization syntax.
463         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
464     ) => {
465         let $field = $val;
466         // Call the initializer.
467         //
468         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
469         // return when an error/panic occurs.
470         // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
471         unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? };
472         // Create the drop guard.
473         //
474         // We only give access to `&DropGuard`, so it cannot be forgotten via safe code.
475         //
476         // SAFETY: We forget the guard later when initialization has succeeded.
477         let $field = &unsafe {
478             $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
479         };
480 
481         $crate::try_pin_init!(init_slot:
482             @data($data),
483             @slot($slot),
484             @munch_fields($($rest)*),
485         );
486     };
487     (init_slot:
488         @data($data:ident),
489         @slot($slot:ident),
490         // Direct value init, this is safe for every field.
491         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
492     ) => {
493         $(let $field = $val;)?
494         // Initialize the field.
495         //
496         // SAFETY: The memory at `slot` is uninitialized.
497         unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
498         // Create the drop guard:
499         //
500         // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
501         //
502         // SAFETY: We forget the guard later when initialization has succeeded.
503         let $field = &unsafe {
504             $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
505         };
506 
507         $crate::try_pin_init!(init_slot:
508             @data($data),
509             @slot($slot),
510             @munch_fields($($rest)*),
511         );
512     };
513     (make_initializer:
514         @slot($slot:ident),
515         @type_name($t:ident),
516         @munch_fields($(,)?),
517         @acc($($acc:tt)*),
518     ) => {
519         // Endpoint, nothing more to munch, create the initializer.
520         // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
521         // get the correct type inference here:
522         unsafe {
523             ::core::ptr::write($slot, $t {
524                 $($acc)*
525             });
526         }
527     };
528     (make_initializer:
529         @slot($slot:ident),
530         @type_name($t:ident),
531         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
532         @acc($($acc:tt)*),
533     ) => {
534         $crate::try_pin_init!(make_initializer:
535             @slot($slot),
536             @type_name($t),
537             @munch_fields($($rest)*),
538             @acc($($acc)* $field: ::core::panic!(),),
539         );
540     };
541     (make_initializer:
542         @slot($slot:ident),
543         @type_name($t:ident),
544         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
545         @acc($($acc:tt)*),
546     ) => {
547         $crate::try_pin_init!(make_initializer:
548             @slot($slot),
549             @type_name($t),
550             @munch_fields($($rest)*),
551             @acc($($acc)* $field: ::core::panic!(),),
552         );
553     };
554     (forget_guards:
555         @munch_fields($(,)?),
556     ) => {
557         // Munching finished.
558     };
559     (forget_guards:
560         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
561     ) => {
562         unsafe { $crate::init::__internal::DropGuard::forget($field) };
563 
564         $crate::try_pin_init!(forget_guards:
565             @munch_fields($($rest)*),
566         );
567     };
568     (forget_guards:
569         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
570     ) => {
571         unsafe { $crate::init::__internal::DropGuard::forget($field) };
572 
573         $crate::try_pin_init!(forget_guards:
574             @munch_fields($($rest)*),
575         );
576     };
577 }
578 
579 /// Construct an in-place initializer for `struct`s.
580 ///
581 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
582 /// [`try_init!`].
583 ///
584 /// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
585 /// - `unsafe` code must guarantee either full initialization or return an error and allow
586 ///   deallocation of the memory.
587 /// - the fields are initialized in the order given in the initializer.
588 /// - no references to fields are allowed to be created inside of the initializer.
589 ///
590 /// This initializer is for initializing data in-place that might later be moved. If you want to
591 /// pin-initialize, use [`pin_init!`].
592 // For a detailed example of how this macro works, see the module documentation of the hidden
593 // module `__internal` inside of `init/__internal.rs`.
594 #[macro_export]
595 macro_rules! init {
596     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
597         $($fields:tt)*
598     }) => {
599         $crate::try_init!(
600             @this($($this)?),
601             @typ($t $(::<$($generics),*>)?),
602             @fields($($fields)*),
603             @error(::core::convert::Infallible),
604         )
605     }
606 }
607 
608 /// Construct an in-place fallible initializer for `struct`s.
609 ///
610 /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
611 /// [`init!`].
612 ///
613 /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
614 /// append `? $type` after the `struct` initializer.
615 /// The safety caveats from [`try_pin_init!`] also apply:
616 /// - `unsafe` code must guarantee either full initialization or return an error and allow
617 ///   deallocation of the memory.
618 /// - the fields are initialized in the order given in the initializer.
619 /// - no references to fields are allowed to be created inside of the initializer.
620 ///
621 /// # Examples
622 ///
623 /// ```rust
624 /// use kernel::{init::PinInit, error::Error, InPlaceInit};
625 /// struct BigBuf {
626 ///     big: Box<[u8; 1024 * 1024 * 1024]>,
627 ///     small: [u8; 1024 * 1024],
628 /// }
629 ///
630 /// impl BigBuf {
631 ///     fn new() -> impl Init<Self, Error> {
632 ///         try_init!(Self {
633 ///             big: Box::init(zeroed())?,
634 ///             small: [0; 1024 * 1024],
635 ///         }? Error)
636 ///     }
637 /// }
638 /// ```
639 // For a detailed example of how this macro works, see the module documentation of the hidden
640 // module `__internal` inside of `init/__internal.rs`.
641 #[macro_export]
642 macro_rules! try_init {
643     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
644         $($fields:tt)*
645     }) => {
646         $crate::try_init!(
647             @this($($this)?),
648             @typ($t $(::<$($generics),*>)?),
649             @fields($($fields)*),
650             @error($crate::error::Error),
651         )
652     };
653     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
654         $($fields:tt)*
655     }? $err:ty) => {
656         $crate::try_init!(
657             @this($($this)?),
658             @typ($t $(::<$($generics),*>)?),
659             @fields($($fields)*),
660             @error($err),
661         )
662     };
663     (
664         @this($($this:ident)?),
665         @typ($t:ident $(::<$($generics:ty),*>)?),
666         @fields($($fields:tt)*),
667         @error($err:ty),
668     ) => {{
669         // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
670         // type and shadow it later when we insert the arbitrary user code. That way there will be
671         // no possibility of returning without `unsafe`.
672         struct __InitOk;
673         // Get the init data from the supplied type.
674         let data = unsafe {
675             use $crate::init::__internal::HasInitData;
676             $t$(::<$($generics),*>)?::__init_data()
677         };
678         // Ensure that `data` really is of type `InitData` and help with type inference:
679         let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>(
680             data,
681             move |slot| {
682                 {
683                     // Shadow the structure so it cannot be used to return early.
684                     struct __InitOk;
685                     // Create the `this` so it can be referenced by the user inside of the
686                     // expressions creating the individual fields.
687                     $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
688                     // Initialize every field.
689                     $crate::try_init!(init_slot:
690                         @slot(slot),
691                         @munch_fields($($fields)*,),
692                     );
693                     // We use unreachable code to ensure that all fields have been mentioned exactly
694                     // once, this struct initializer will still be type-checked and complain with a
695                     // very natural error message if a field is forgotten/mentioned more than once.
696                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
697                     if false {
698                         $crate::try_init!(make_initializer:
699                             @slot(slot),
700                             @type_name($t),
701                             @munch_fields($($fields)*,),
702                             @acc(),
703                         );
704                     }
705                     // Forget all guards, since initialization was a success.
706                     $crate::try_init!(forget_guards:
707                         @munch_fields($($fields)*,),
708                     );
709                 }
710                 Ok(__InitOk)
711             }
712         );
713         let init = move |slot| -> ::core::result::Result<(), $err> {
714             init(slot).map(|__InitOk| ())
715         };
716         let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) };
717         init
718     }};
719     (init_slot:
720         @slot($slot:ident),
721         @munch_fields( $(,)?),
722     ) => {
723         // Endpoint of munching, no fields are left.
724     };
725     (init_slot:
726         @slot($slot:ident),
727         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
728     ) => {
729         let $field = $val;
730         // Call the initializer.
731         //
732         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
733         // return when an error/panic occurs.
734         unsafe {
735             $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?;
736         }
737         // Create the drop guard.
738         //
739         // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
740         //
741         // SAFETY: We forget the guard later when initialization has succeeded.
742         let $field = &unsafe {
743             $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
744         };
745 
746         $crate::try_init!(init_slot:
747             @slot($slot),
748             @munch_fields($($rest)*),
749         );
750     };
751     (init_slot:
752         @slot($slot:ident),
753         // Direct value init.
754         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
755     ) => {
756         $(let $field = $val;)?
757         // Call the initializer.
758         //
759         // SAFETY: The memory at `slot` is uninitialized.
760         unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
761         // Create the drop guard.
762         //
763         // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
764         //
765         // SAFETY: We forget the guard later when initialization has succeeded.
766         let $field = &unsafe {
767             $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
768         };
769 
770         $crate::try_init!(init_slot:
771             @slot($slot),
772             @munch_fields($($rest)*),
773         );
774     };
775     (make_initializer:
776         @slot($slot:ident),
777         @type_name($t:ident),
778         @munch_fields( $(,)?),
779         @acc($($acc:tt)*),
780     ) => {
781         // Endpoint, nothing more to munch, create the initializer.
782         // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
783         // get the correct type inference here:
784         unsafe {
785             ::core::ptr::write($slot, $t {
786                 $($acc)*
787             });
788         }
789     };
790     (make_initializer:
791         @slot($slot:ident),
792         @type_name($t:ident),
793         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
794         @acc($($acc:tt)*),
795     ) => {
796         $crate::try_init!(make_initializer:
797             @slot($slot),
798             @type_name($t),
799             @munch_fields($($rest)*),
800             @acc($($acc)*$field: ::core::panic!(),),
801         );
802     };
803     (make_initializer:
804         @slot($slot:ident),
805         @type_name($t:ident),
806         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
807         @acc($($acc:tt)*),
808     ) => {
809         $crate::try_init!(make_initializer:
810             @slot($slot),
811             @type_name($t),
812             @munch_fields($($rest)*),
813             @acc($($acc)*$field: ::core::panic!(),),
814         );
815     };
816     (forget_guards:
817         @munch_fields($(,)?),
818     ) => {
819         // Munching finished.
820     };
821     (forget_guards:
822         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
823     ) => {
824         unsafe { $crate::init::__internal::DropGuard::forget($field) };
825 
826         $crate::try_init!(forget_guards:
827             @munch_fields($($rest)*),
828         );
829     };
830     (forget_guards:
831         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
832     ) => {
833         unsafe { $crate::init::__internal::DropGuard::forget($field) };
834 
835         $crate::try_init!(forget_guards:
836             @munch_fields($($rest)*),
837         );
838     };
839 }
840 
841 /// A pin-initializer for the type `T`.
842 ///
843 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
844 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::pin_init`] function of a
845 /// smart pointer like [`Arc<T>`] on this.
846 ///
847 /// Also see the [module description](self).
848 ///
849 /// # Safety
850 ///
851 /// When implementing this type you will need to take great care. Also there are probably very few
852 /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
853 ///
854 /// The [`PinInit::__pinned_init`] function
855 /// - returns `Ok(())` if it initialized every field of `slot`,
856 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
857 ///     - `slot` can be deallocated without UB occurring,
858 ///     - `slot` does not need to be dropped,
859 ///     - `slot` is not partially initialized.
860 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
861 ///
862 /// [`Arc<T>`]: crate::sync::Arc
863 /// [`Arc::pin_init`]: crate::sync::Arc::pin_init
864 #[must_use = "An initializer must be used in order to create its value."]
865 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
866     /// Initializes `slot`.
867     ///
868     /// # Safety
869     ///
870     /// - `slot` is a valid pointer to uninitialized memory.
871     /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
872     ///   deallocate.
873     /// - `slot` will not move until it is dropped, i.e. it will be pinned.
874     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
875 }
876 
877 /// An initializer for `T`.
878 ///
879 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
880 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::init`] function of a smart
881 /// pointer like [`Arc<T>`] on this. Because [`PinInit<T, E>`] is a super trait, you can
882 /// use every function that takes it as well.
883 ///
884 /// Also see the [module description](self).
885 ///
886 /// # Safety
887 ///
888 /// When implementing this type you will need to take great care. Also there are probably very few
889 /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
890 ///
891 /// The [`Init::__init`] function
892 /// - returns `Ok(())` if it initialized every field of `slot`,
893 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
894 ///     - `slot` can be deallocated without UB occurring,
895 ///     - `slot` does not need to be dropped,
896 ///     - `slot` is not partially initialized.
897 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
898 ///
899 /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
900 /// code as `__init`.
901 ///
902 /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
903 /// move the pointee after initialization.
904 ///
905 /// [`Arc<T>`]: crate::sync::Arc
906 #[must_use = "An initializer must be used in order to create its value."]
907 pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized {
908     /// Initializes `slot`.
909     ///
910     /// # Safety
911     ///
912     /// - `slot` is a valid pointer to uninitialized memory.
913     /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
914     ///   deallocate.
915     unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
916 }
917 
918 // SAFETY: Every in-place initializer can also be used as a pin-initializer.
919 unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I
920 where
921     I: Init<T, E>,
922 {
923     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
924         // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not
925         // require `slot` to not move after init.
926         unsafe { self.__init(slot) }
927     }
928 }
929 
930 /// Creates a new [`PinInit<T, E>`] from the given closure.
931 ///
932 /// # Safety
933 ///
934 /// The closure:
935 /// - returns `Ok(())` if it initialized every field of `slot`,
936 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
937 ///     - `slot` can be deallocated without UB occurring,
938 ///     - `slot` does not need to be dropped,
939 ///     - `slot` is not partially initialized.
940 /// - may assume that the `slot` does not move if `T: !Unpin`,
941 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
942 #[inline]
943 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
944     f: impl FnOnce(*mut T) -> Result<(), E>,
945 ) -> impl PinInit<T, E> {
946     __internal::InitClosure(f, PhantomData)
947 }
948 
949 /// Creates a new [`Init<T, E>`] from the given closure.
950 ///
951 /// # Safety
952 ///
953 /// The closure:
954 /// - returns `Ok(())` if it initialized every field of `slot`,
955 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
956 ///     - `slot` can be deallocated without UB occurring,
957 ///     - `slot` does not need to be dropped,
958 ///     - `slot` is not partially initialized.
959 /// - the `slot` may move after initialization.
960 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
961 #[inline]
962 pub const unsafe fn init_from_closure<T: ?Sized, E>(
963     f: impl FnOnce(*mut T) -> Result<(), E>,
964 ) -> impl Init<T, E> {
965     __internal::InitClosure(f, PhantomData)
966 }
967 
968 /// An initializer that leaves the memory uninitialized.
969 ///
970 /// The initializer is a no-op. The `slot` memory is not changed.
971 #[inline]
972 pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
973     // SAFETY: The memory is allowed to be uninitialized.
974     unsafe { init_from_closure(|_| Ok(())) }
975 }
976 
977 // SAFETY: Every type can be initialized by-value.
978 unsafe impl<T> Init<T> for T {
979     unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
980         unsafe { slot.write(self) };
981         Ok(())
982     }
983 }
984 
985 /// Smart pointer that can initialize memory in-place.
986 pub trait InPlaceInit<T>: Sized {
987     /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
988     /// type.
989     ///
990     /// If `T: !Unpin` it will not be able to move afterwards.
991     fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
992     where
993         E: From<AllocError>;
994 
995     /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
996     /// type.
997     ///
998     /// If `T: !Unpin` it will not be able to move afterwards.
999     fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>>
1000     where
1001         Error: From<E>,
1002     {
1003         // SAFETY: We delegate to `init` and only change the error type.
1004         let init = unsafe {
1005             pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1006         };
1007         Self::try_pin_init(init)
1008     }
1009 
1010     /// Use the given initializer to in-place initialize a `T`.
1011     fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1012     where
1013         E: From<AllocError>;
1014 
1015     /// Use the given initializer to in-place initialize a `T`.
1016     fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
1017     where
1018         Error: From<E>,
1019     {
1020         // SAFETY: We delegate to `init` and only change the error type.
1021         let init = unsafe {
1022             init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1023         };
1024         Self::try_init(init)
1025     }
1026 }
1027 
1028 impl<T> InPlaceInit<T> for Box<T> {
1029     #[inline]
1030     fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1031     where
1032         E: From<AllocError>,
1033     {
1034         let mut this = Box::try_new_uninit()?;
1035         let slot = this.as_mut_ptr();
1036         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1037         // slot is valid and will not be moved, because we pin it later.
1038         unsafe { init.__pinned_init(slot)? };
1039         // SAFETY: All fields have been initialized.
1040         Ok(unsafe { this.assume_init() }.into())
1041     }
1042 
1043     #[inline]
1044     fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1045     where
1046         E: From<AllocError>,
1047     {
1048         let mut this = Box::try_new_uninit()?;
1049         let slot = this.as_mut_ptr();
1050         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1051         // slot is valid.
1052         unsafe { init.__init(slot)? };
1053         // SAFETY: All fields have been initialized.
1054         Ok(unsafe { this.assume_init() })
1055     }
1056 }
1057 
1058 impl<T> InPlaceInit<T> for UniqueArc<T> {
1059     #[inline]
1060     fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1061     where
1062         E: From<AllocError>,
1063     {
1064         let mut this = UniqueArc::try_new_uninit()?;
1065         let slot = this.as_mut_ptr();
1066         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1067         // slot is valid and will not be moved, because we pin it later.
1068         unsafe { init.__pinned_init(slot)? };
1069         // SAFETY: All fields have been initialized.
1070         Ok(unsafe { this.assume_init() }.into())
1071     }
1072 
1073     #[inline]
1074     fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1075     where
1076         E: From<AllocError>,
1077     {
1078         let mut this = UniqueArc::try_new_uninit()?;
1079         let slot = this.as_mut_ptr();
1080         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1081         // slot is valid.
1082         unsafe { init.__init(slot)? };
1083         // SAFETY: All fields have been initialized.
1084         Ok(unsafe { this.assume_init() })
1085     }
1086 }
1087