1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. 4 //! 5 //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack 6 //! overflow. 7 //! 8 //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential 9 //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. 10 //! 11 //! # Overview 12 //! 13 //! To initialize a `struct` with an in-place constructor you will need two things: 14 //! - an in-place constructor, 15 //! - a memory location that can hold your `struct`. 16 //! 17 //! To get an in-place constructor there are generally three options: 18 //! - directly creating an in-place constructor using the [`pin_init!`] macro, 19 //! - a custom function/macro returning an in-place constructor provided by someone else, 20 //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. 21 //! 22 //! Aside from pinned initialization, this API also supports in-place construction without pinning, 23 //! the macros/types/functions are generally named like the pinned variants without the `pin` 24 //! prefix. 25 //! 26 //! # Examples 27 //! 28 //! ## Using the [`pin_init!`] macro 29 //! 30 //! If you want to use [`PinInit`], then you will have to annotate your `struct` with 31 //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for 32 //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via 33 //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is 34 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. 35 //! 36 //! ```rust 37 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 38 //! use kernel::{prelude::*, sync::Mutex, new_mutex}; 39 //! # use core::pin::Pin; 40 //! #[pin_data] 41 //! struct Foo { 42 //! #[pin] 43 //! a: Mutex<usize>, 44 //! b: u32, 45 //! } 46 //! 47 //! let foo = pin_init!(Foo { 48 //! a <- new_mutex!(42, "Foo::a"), 49 //! b: 24, 50 //! }); 51 //! ``` 52 //! 53 //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like 54 //! (or just the stack) to actually initialize a `Foo`: 55 //! 56 //! ```rust 57 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 58 //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; 59 //! # use core::pin::Pin; 60 //! # #[pin_data] 61 //! # struct Foo { 62 //! # #[pin] 63 //! # a: Mutex<usize>, 64 //! # b: u32, 65 //! # } 66 //! # let foo = pin_init!(Foo { 67 //! # a <- new_mutex!(42, "Foo::a"), 68 //! # b: 24, 69 //! # }); 70 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); 71 //! ``` 72 //! 73 //! For more information see the [`pin_init!`] macro. 74 //! 75 //! ## Using a custom function/macro that returns an initializer 76 //! 77 //! Many types from the kernel supply a function/macro that returns an initializer, because the 78 //! above method only works for types where you can access the fields. 79 //! 80 //! ```rust 81 //! # use kernel::{new_mutex, sync::{Arc, Mutex}}; 82 //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); 83 //! ``` 84 //! 85 //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: 86 //! 87 //! ```rust 88 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 89 //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; 90 //! #[pin_data] 91 //! struct DriverData { 92 //! #[pin] 93 //! status: Mutex<i32>, 94 //! buffer: Box<[u8; 1_000_000]>, 95 //! } 96 //! 97 //! impl DriverData { 98 //! fn new() -> impl PinInit<Self, Error> { 99 //! try_pin_init!(Self { 100 //! status <- new_mutex!(0, "DriverData::status"), 101 //! buffer: Box::init(kernel::init::zeroed())?, 102 //! }) 103 //! } 104 //! } 105 //! ``` 106 //! 107 //! [`sync`]: kernel::sync 108 //! [pinning]: https://doc.rust-lang.org/std/pin/index.html 109 //! [structurally pinned fields]: 110 //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field 111 //! [`Arc<T>`]: crate::sync::Arc 112 //! [`impl PinInit<Foo>`]: PinInit 113 //! [`impl PinInit<T, E>`]: PinInit 114 //! [`impl Init<T, E>`]: Init 115 //! [`Opaque`]: kernel::types::Opaque 116 //! [`pin_data`]: ::macros::pin_data 117 118 use crate::{ 119 error::{self, Error}, 120 sync::UniqueArc, 121 }; 122 use alloc::boxed::Box; 123 use core::{ 124 alloc::AllocError, cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, 125 pin::Pin, ptr, 126 }; 127 128 #[doc(hidden)] 129 pub mod __internal; 130 #[doc(hidden)] 131 pub mod macros; 132 133 /// Construct an in-place, pinned initializer for `struct`s. 134 /// 135 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use 136 /// [`try_pin_init!`]. 137 /// 138 /// The syntax is almost identical to that of a normal `struct` initializer: 139 /// 140 /// ```rust 141 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 142 /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 143 /// # use core::pin::Pin; 144 /// #[pin_data] 145 /// struct Foo { 146 /// a: usize, 147 /// b: Bar, 148 /// } 149 /// 150 /// #[pin_data] 151 /// struct Bar { 152 /// x: u32, 153 /// } 154 /// 155 /// # fn demo() -> impl PinInit<Foo> { 156 /// let a = 42; 157 /// 158 /// let initializer = pin_init!(Foo { 159 /// a, 160 /// b: Bar { 161 /// x: 64, 162 /// }, 163 /// }); 164 /// # initializer } 165 /// # Box::pin_init(demo()).unwrap(); 166 /// ``` 167 /// 168 /// Arbitrary Rust expressions can be used to set the value of a variable. 169 /// 170 /// The fields are initialized in the order that they appear in the initializer. So it is possible 171 /// to read already initialized fields using raw pointers. 172 /// 173 /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the 174 /// initializer. 175 /// 176 /// # Init-functions 177 /// 178 /// When working with this API it is often desired to let others construct your types without 179 /// giving access to all fields. This is where you would normally write a plain function `new` 180 /// that would return a new instance of your type. With this API that is also possible. 181 /// However, there are a few extra things to keep in mind. 182 /// 183 /// To create an initializer function, simply declare it like this: 184 /// 185 /// ```rust 186 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 187 /// # use kernel::{init, pin_init, prelude::*, init::*}; 188 /// # use core::pin::Pin; 189 /// # #[pin_data] 190 /// # struct Foo { 191 /// # a: usize, 192 /// # b: Bar, 193 /// # } 194 /// # #[pin_data] 195 /// # struct Bar { 196 /// # x: u32, 197 /// # } 198 /// impl Foo { 199 /// fn new() -> impl PinInit<Self> { 200 /// pin_init!(Self { 201 /// a: 42, 202 /// b: Bar { 203 /// x: 64, 204 /// }, 205 /// }) 206 /// } 207 /// } 208 /// ``` 209 /// 210 /// Users of `Foo` can now create it like this: 211 /// 212 /// ```rust 213 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 214 /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 215 /// # use core::pin::Pin; 216 /// # #[pin_data] 217 /// # struct Foo { 218 /// # a: usize, 219 /// # b: Bar, 220 /// # } 221 /// # #[pin_data] 222 /// # struct Bar { 223 /// # x: u32, 224 /// # } 225 /// # impl Foo { 226 /// # fn new() -> impl PinInit<Self> { 227 /// # pin_init!(Self { 228 /// # a: 42, 229 /// # b: Bar { 230 /// # x: 64, 231 /// # }, 232 /// # }) 233 /// # } 234 /// # } 235 /// let foo = Box::pin_init(Foo::new()); 236 /// ``` 237 /// 238 /// They can also easily embed it into their own `struct`s: 239 /// 240 /// ```rust 241 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 242 /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 243 /// # use core::pin::Pin; 244 /// # #[pin_data] 245 /// # struct Foo { 246 /// # a: usize, 247 /// # b: Bar, 248 /// # } 249 /// # #[pin_data] 250 /// # struct Bar { 251 /// # x: u32, 252 /// # } 253 /// # impl Foo { 254 /// # fn new() -> impl PinInit<Self> { 255 /// # pin_init!(Self { 256 /// # a: 42, 257 /// # b: Bar { 258 /// # x: 64, 259 /// # }, 260 /// # }) 261 /// # } 262 /// # } 263 /// #[pin_data] 264 /// struct FooContainer { 265 /// #[pin] 266 /// foo1: Foo, 267 /// #[pin] 268 /// foo2: Foo, 269 /// other: u32, 270 /// } 271 /// 272 /// impl FooContainer { 273 /// fn new(other: u32) -> impl PinInit<Self> { 274 /// pin_init!(Self { 275 /// foo1 <- Foo::new(), 276 /// foo2 <- Foo::new(), 277 /// other, 278 /// }) 279 /// } 280 /// } 281 /// ``` 282 /// 283 /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. 284 /// This signifies that the given field is initialized in-place. As with `struct` initializers, just 285 /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. 286 /// 287 /// # Syntax 288 /// 289 /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with 290 /// the following modifications is expected: 291 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. 292 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] 293 /// pointer named `this` inside of the initializer. 294 /// 295 /// For instance: 296 /// 297 /// ```rust 298 /// # use kernel::pin_init; 299 /// # use macros::pin_data; 300 /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; 301 /// #[pin_data] 302 /// struct Buf { 303 /// // `ptr` points into `buf`. 304 /// ptr: *mut u8, 305 /// buf: [u8; 64], 306 /// #[pin] 307 /// pin: PhantomPinned, 308 /// } 309 /// pin_init!(&this in Buf { 310 /// buf: [0; 64], 311 /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, 312 /// pin: PhantomPinned, 313 /// }); 314 /// ``` 315 /// 316 /// [`try_pin_init!`]: kernel::try_pin_init 317 /// [`NonNull<Self>`]: core::ptr::NonNull 318 // For a detailed example of how this macro works, see the module documentation of the hidden 319 // module `__internal` inside of `init/__internal.rs`. 320 #[macro_export] 321 macro_rules! pin_init { 322 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 323 $($fields:tt)* 324 }) => { 325 $crate::try_pin_init!( 326 @this($($this)?), 327 @typ($t $(::<$($generics),*>)?), 328 @fields($($fields)*), 329 @error(::core::convert::Infallible), 330 ) 331 }; 332 } 333 334 /// Construct an in-place, fallible pinned initializer for `struct`s. 335 /// 336 /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. 337 /// 338 /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop 339 /// initialization and return the error. 340 /// 341 /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when 342 /// initialization fails, the memory can be safely deallocated without any further modifications. 343 /// 344 /// This macro defaults the error to [`Error`]. 345 /// 346 /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` 347 /// after the `struct` initializer to specify the error type you want to use. 348 /// 349 /// # Examples 350 /// 351 /// ```rust 352 /// # #![feature(new_uninit)] 353 /// use kernel::{init::{self, PinInit}, error::Error}; 354 /// #[pin_data] 355 /// struct BigBuf { 356 /// big: Box<[u8; 1024 * 1024 * 1024]>, 357 /// small: [u8; 1024 * 1024], 358 /// ptr: *mut u8, 359 /// } 360 /// 361 /// impl BigBuf { 362 /// fn new() -> impl PinInit<Self, Error> { 363 /// try_pin_init!(Self { 364 /// big: Box::init(init::zeroed())?, 365 /// small: [0; 1024 * 1024], 366 /// ptr: core::ptr::null_mut(), 367 /// }? Error) 368 /// } 369 /// } 370 /// ``` 371 // For a detailed example of how this macro works, see the module documentation of the hidden 372 // module `__internal` inside of `init/__internal.rs`. 373 #[macro_export] 374 macro_rules! try_pin_init { 375 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 376 $($fields:tt)* 377 }) => { 378 $crate::try_pin_init!( 379 @this($($this)?), 380 @typ($t $(::<$($generics),*>)? ), 381 @fields($($fields)*), 382 @error($crate::error::Error), 383 ) 384 }; 385 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 386 $($fields:tt)* 387 }? $err:ty) => { 388 $crate::try_pin_init!( 389 @this($($this)?), 390 @typ($t $(::<$($generics),*>)? ), 391 @fields($($fields)*), 392 @error($err), 393 ) 394 }; 395 ( 396 @this($($this:ident)?), 397 @typ($t:ident $(::<$($generics:ty),*>)?), 398 @fields($($fields:tt)*), 399 @error($err:ty), 400 ) => {{ 401 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 402 // type and shadow it later when we insert the arbitrary user code. That way there will be 403 // no possibility of returning without `unsafe`. 404 struct __InitOk; 405 // Get the pin data from the supplied type. 406 let data = unsafe { 407 use $crate::init::__internal::HasPinData; 408 $t$(::<$($generics),*>)?::__pin_data() 409 }; 410 // Ensure that `data` really is of type `PinData` and help with type inference: 411 let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( 412 data, 413 move |slot| { 414 { 415 // Shadow the structure so it cannot be used to return early. 416 struct __InitOk; 417 // Create the `this` so it can be referenced by the user inside of the 418 // expressions creating the individual fields. 419 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 420 // Initialize every field. 421 $crate::try_pin_init!(init_slot: 422 @data(data), 423 @slot(slot), 424 @munch_fields($($fields)*,), 425 ); 426 // We use unreachable code to ensure that all fields have been mentioned exactly 427 // once, this struct initializer will still be type-checked and complain with a 428 // very natural error message if a field is forgotten/mentioned more than once. 429 #[allow(unreachable_code, clippy::diverging_sub_expression)] 430 if false { 431 $crate::try_pin_init!(make_initializer: 432 @slot(slot), 433 @type_name($t), 434 @munch_fields($($fields)*,), 435 @acc(), 436 ); 437 } 438 // Forget all guards, since initialization was a success. 439 $crate::try_pin_init!(forget_guards: 440 @munch_fields($($fields)*,), 441 ); 442 } 443 Ok(__InitOk) 444 } 445 ); 446 let init = move |slot| -> ::core::result::Result<(), $err> { 447 init(slot).map(|__InitOk| ()) 448 }; 449 let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; 450 init 451 }}; 452 (init_slot: 453 @data($data:ident), 454 @slot($slot:ident), 455 @munch_fields($(,)?), 456 ) => { 457 // Endpoint of munching, no fields are left. 458 }; 459 (init_slot: 460 @data($data:ident), 461 @slot($slot:ident), 462 // In-place initialization syntax. 463 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 464 ) => { 465 let $field = $val; 466 // Call the initializer. 467 // 468 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 469 // return when an error/panic occurs. 470 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. 471 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; 472 // Create the drop guard. 473 // 474 // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. 475 // 476 // SAFETY: We forget the guard later when initialization has succeeded. 477 let $field = &unsafe { 478 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 479 }; 480 481 $crate::try_pin_init!(init_slot: 482 @data($data), 483 @slot($slot), 484 @munch_fields($($rest)*), 485 ); 486 }; 487 (init_slot: 488 @data($data:ident), 489 @slot($slot:ident), 490 // Direct value init, this is safe for every field. 491 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 492 ) => { 493 $(let $field = $val;)? 494 // Initialize the field. 495 // 496 // SAFETY: The memory at `slot` is uninitialized. 497 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 498 // Create the drop guard: 499 // 500 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 501 // 502 // SAFETY: We forget the guard later when initialization has succeeded. 503 let $field = &unsafe { 504 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 505 }; 506 507 $crate::try_pin_init!(init_slot: 508 @data($data), 509 @slot($slot), 510 @munch_fields($($rest)*), 511 ); 512 }; 513 (make_initializer: 514 @slot($slot:ident), 515 @type_name($t:ident), 516 @munch_fields($(,)?), 517 @acc($($acc:tt)*), 518 ) => { 519 // Endpoint, nothing more to munch, create the initializer. 520 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to 521 // get the correct type inference here: 522 unsafe { 523 ::core::ptr::write($slot, $t { 524 $($acc)* 525 }); 526 } 527 }; 528 (make_initializer: 529 @slot($slot:ident), 530 @type_name($t:ident), 531 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 532 @acc($($acc:tt)*), 533 ) => { 534 $crate::try_pin_init!(make_initializer: 535 @slot($slot), 536 @type_name($t), 537 @munch_fields($($rest)*), 538 @acc($($acc)* $field: ::core::panic!(),), 539 ); 540 }; 541 (make_initializer: 542 @slot($slot:ident), 543 @type_name($t:ident), 544 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 545 @acc($($acc:tt)*), 546 ) => { 547 $crate::try_pin_init!(make_initializer: 548 @slot($slot), 549 @type_name($t), 550 @munch_fields($($rest)*), 551 @acc($($acc)* $field: ::core::panic!(),), 552 ); 553 }; 554 (forget_guards: 555 @munch_fields($(,)?), 556 ) => { 557 // Munching finished. 558 }; 559 (forget_guards: 560 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 561 ) => { 562 unsafe { $crate::init::__internal::DropGuard::forget($field) }; 563 564 $crate::try_pin_init!(forget_guards: 565 @munch_fields($($rest)*), 566 ); 567 }; 568 (forget_guards: 569 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 570 ) => { 571 unsafe { $crate::init::__internal::DropGuard::forget($field) }; 572 573 $crate::try_pin_init!(forget_guards: 574 @munch_fields($($rest)*), 575 ); 576 }; 577 } 578 579 /// Construct an in-place initializer for `struct`s. 580 /// 581 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use 582 /// [`try_init!`]. 583 /// 584 /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: 585 /// - `unsafe` code must guarantee either full initialization or return an error and allow 586 /// deallocation of the memory. 587 /// - the fields are initialized in the order given in the initializer. 588 /// - no references to fields are allowed to be created inside of the initializer. 589 /// 590 /// This initializer is for initializing data in-place that might later be moved. If you want to 591 /// pin-initialize, use [`pin_init!`]. 592 // For a detailed example of how this macro works, see the module documentation of the hidden 593 // module `__internal` inside of `init/__internal.rs`. 594 #[macro_export] 595 macro_rules! init { 596 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 597 $($fields:tt)* 598 }) => { 599 $crate::try_init!( 600 @this($($this)?), 601 @typ($t $(::<$($generics),*>)?), 602 @fields($($fields)*), 603 @error(::core::convert::Infallible), 604 ) 605 } 606 } 607 608 /// Construct an in-place fallible initializer for `struct`s. 609 /// 610 /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use 611 /// [`init!`]. 612 /// 613 /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, 614 /// append `? $type` after the `struct` initializer. 615 /// The safety caveats from [`try_pin_init!`] also apply: 616 /// - `unsafe` code must guarantee either full initialization or return an error and allow 617 /// deallocation of the memory. 618 /// - the fields are initialized in the order given in the initializer. 619 /// - no references to fields are allowed to be created inside of the initializer. 620 /// 621 /// # Examples 622 /// 623 /// ```rust 624 /// use kernel::{init::PinInit, error::Error, InPlaceInit}; 625 /// struct BigBuf { 626 /// big: Box<[u8; 1024 * 1024 * 1024]>, 627 /// small: [u8; 1024 * 1024], 628 /// } 629 /// 630 /// impl BigBuf { 631 /// fn new() -> impl Init<Self, Error> { 632 /// try_init!(Self { 633 /// big: Box::init(zeroed())?, 634 /// small: [0; 1024 * 1024], 635 /// }? Error) 636 /// } 637 /// } 638 /// ``` 639 // For a detailed example of how this macro works, see the module documentation of the hidden 640 // module `__internal` inside of `init/__internal.rs`. 641 #[macro_export] 642 macro_rules! try_init { 643 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 644 $($fields:tt)* 645 }) => { 646 $crate::try_init!( 647 @this($($this)?), 648 @typ($t $(::<$($generics),*>)?), 649 @fields($($fields)*), 650 @error($crate::error::Error), 651 ) 652 }; 653 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 654 $($fields:tt)* 655 }? $err:ty) => { 656 $crate::try_init!( 657 @this($($this)?), 658 @typ($t $(::<$($generics),*>)?), 659 @fields($($fields)*), 660 @error($err), 661 ) 662 }; 663 ( 664 @this($($this:ident)?), 665 @typ($t:ident $(::<$($generics:ty),*>)?), 666 @fields($($fields:tt)*), 667 @error($err:ty), 668 ) => {{ 669 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 670 // type and shadow it later when we insert the arbitrary user code. That way there will be 671 // no possibility of returning without `unsafe`. 672 struct __InitOk; 673 // Get the init data from the supplied type. 674 let data = unsafe { 675 use $crate::init::__internal::HasInitData; 676 $t$(::<$($generics),*>)?::__init_data() 677 }; 678 // Ensure that `data` really is of type `InitData` and help with type inference: 679 let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( 680 data, 681 move |slot| { 682 { 683 // Shadow the structure so it cannot be used to return early. 684 struct __InitOk; 685 // Create the `this` so it can be referenced by the user inside of the 686 // expressions creating the individual fields. 687 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 688 // Initialize every field. 689 $crate::try_init!(init_slot: 690 @slot(slot), 691 @munch_fields($($fields)*,), 692 ); 693 // We use unreachable code to ensure that all fields have been mentioned exactly 694 // once, this struct initializer will still be type-checked and complain with a 695 // very natural error message if a field is forgotten/mentioned more than once. 696 #[allow(unreachable_code, clippy::diverging_sub_expression)] 697 if false { 698 $crate::try_init!(make_initializer: 699 @slot(slot), 700 @type_name($t), 701 @munch_fields($($fields)*,), 702 @acc(), 703 ); 704 } 705 // Forget all guards, since initialization was a success. 706 $crate::try_init!(forget_guards: 707 @munch_fields($($fields)*,), 708 ); 709 } 710 Ok(__InitOk) 711 } 712 ); 713 let init = move |slot| -> ::core::result::Result<(), $err> { 714 init(slot).map(|__InitOk| ()) 715 }; 716 let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; 717 init 718 }}; 719 (init_slot: 720 @slot($slot:ident), 721 @munch_fields( $(,)?), 722 ) => { 723 // Endpoint of munching, no fields are left. 724 }; 725 (init_slot: 726 @slot($slot:ident), 727 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 728 ) => { 729 let $field = $val; 730 // Call the initializer. 731 // 732 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 733 // return when an error/panic occurs. 734 unsafe { 735 $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; 736 } 737 // Create the drop guard. 738 // 739 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 740 // 741 // SAFETY: We forget the guard later when initialization has succeeded. 742 let $field = &unsafe { 743 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 744 }; 745 746 $crate::try_init!(init_slot: 747 @slot($slot), 748 @munch_fields($($rest)*), 749 ); 750 }; 751 (init_slot: 752 @slot($slot:ident), 753 // Direct value init. 754 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 755 ) => { 756 $(let $field = $val;)? 757 // Call the initializer. 758 // 759 // SAFETY: The memory at `slot` is uninitialized. 760 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 761 // Create the drop guard. 762 // 763 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 764 // 765 // SAFETY: We forget the guard later when initialization has succeeded. 766 let $field = &unsafe { 767 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 768 }; 769 770 $crate::try_init!(init_slot: 771 @slot($slot), 772 @munch_fields($($rest)*), 773 ); 774 }; 775 (make_initializer: 776 @slot($slot:ident), 777 @type_name($t:ident), 778 @munch_fields( $(,)?), 779 @acc($($acc:tt)*), 780 ) => { 781 // Endpoint, nothing more to munch, create the initializer. 782 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to 783 // get the correct type inference here: 784 unsafe { 785 ::core::ptr::write($slot, $t { 786 $($acc)* 787 }); 788 } 789 }; 790 (make_initializer: 791 @slot($slot:ident), 792 @type_name($t:ident), 793 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 794 @acc($($acc:tt)*), 795 ) => { 796 $crate::try_init!(make_initializer: 797 @slot($slot), 798 @type_name($t), 799 @munch_fields($($rest)*), 800 @acc($($acc)*$field: ::core::panic!(),), 801 ); 802 }; 803 (make_initializer: 804 @slot($slot:ident), 805 @type_name($t:ident), 806 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 807 @acc($($acc:tt)*), 808 ) => { 809 $crate::try_init!(make_initializer: 810 @slot($slot), 811 @type_name($t), 812 @munch_fields($($rest)*), 813 @acc($($acc)*$field: ::core::panic!(),), 814 ); 815 }; 816 (forget_guards: 817 @munch_fields($(,)?), 818 ) => { 819 // Munching finished. 820 }; 821 (forget_guards: 822 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 823 ) => { 824 unsafe { $crate::init::__internal::DropGuard::forget($field) }; 825 826 $crate::try_init!(forget_guards: 827 @munch_fields($($rest)*), 828 ); 829 }; 830 (forget_guards: 831 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 832 ) => { 833 unsafe { $crate::init::__internal::DropGuard::forget($field) }; 834 835 $crate::try_init!(forget_guards: 836 @munch_fields($($rest)*), 837 ); 838 }; 839 } 840 841 /// A pin-initializer for the type `T`. 842 /// 843 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can 844 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::pin_init`] function of a 845 /// smart pointer like [`Arc<T>`] on this. 846 /// 847 /// Also see the [module description](self). 848 /// 849 /// # Safety 850 /// 851 /// When implementing this type you will need to take great care. Also there are probably very few 852 /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. 853 /// 854 /// The [`PinInit::__pinned_init`] function 855 /// - returns `Ok(())` if it initialized every field of `slot`, 856 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 857 /// - `slot` can be deallocated without UB occurring, 858 /// - `slot` does not need to be dropped, 859 /// - `slot` is not partially initialized. 860 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 861 /// 862 /// [`Arc<T>`]: crate::sync::Arc 863 /// [`Arc::pin_init`]: crate::sync::Arc::pin_init 864 #[must_use = "An initializer must be used in order to create its value."] 865 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { 866 /// Initializes `slot`. 867 /// 868 /// # Safety 869 /// 870 /// - `slot` is a valid pointer to uninitialized memory. 871 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to 872 /// deallocate. 873 /// - `slot` will not move until it is dropped, i.e. it will be pinned. 874 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; 875 } 876 877 /// An initializer for `T`. 878 /// 879 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can 880 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::init`] function of a smart 881 /// pointer like [`Arc<T>`] on this. Because [`PinInit<T, E>`] is a super trait, you can 882 /// use every function that takes it as well. 883 /// 884 /// Also see the [module description](self). 885 /// 886 /// # Safety 887 /// 888 /// When implementing this type you will need to take great care. Also there are probably very few 889 /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. 890 /// 891 /// The [`Init::__init`] function 892 /// - returns `Ok(())` if it initialized every field of `slot`, 893 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 894 /// - `slot` can be deallocated without UB occurring, 895 /// - `slot` does not need to be dropped, 896 /// - `slot` is not partially initialized. 897 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 898 /// 899 /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same 900 /// code as `__init`. 901 /// 902 /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to 903 /// move the pointee after initialization. 904 /// 905 /// [`Arc<T>`]: crate::sync::Arc 906 #[must_use = "An initializer must be used in order to create its value."] 907 pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { 908 /// Initializes `slot`. 909 /// 910 /// # Safety 911 /// 912 /// - `slot` is a valid pointer to uninitialized memory. 913 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to 914 /// deallocate. 915 unsafe fn __init(self, slot: *mut T) -> Result<(), E>; 916 } 917 918 // SAFETY: Every in-place initializer can also be used as a pin-initializer. 919 unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I 920 where 921 I: Init<T, E>, 922 { 923 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { 924 // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not 925 // require `slot` to not move after init. 926 unsafe { self.__init(slot) } 927 } 928 } 929 930 /// Creates a new [`PinInit<T, E>`] from the given closure. 931 /// 932 /// # Safety 933 /// 934 /// The closure: 935 /// - returns `Ok(())` if it initialized every field of `slot`, 936 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 937 /// - `slot` can be deallocated without UB occurring, 938 /// - `slot` does not need to be dropped, 939 /// - `slot` is not partially initialized. 940 /// - may assume that the `slot` does not move if `T: !Unpin`, 941 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 942 #[inline] 943 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( 944 f: impl FnOnce(*mut T) -> Result<(), E>, 945 ) -> impl PinInit<T, E> { 946 __internal::InitClosure(f, PhantomData) 947 } 948 949 /// Creates a new [`Init<T, E>`] from the given closure. 950 /// 951 /// # Safety 952 /// 953 /// The closure: 954 /// - returns `Ok(())` if it initialized every field of `slot`, 955 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 956 /// - `slot` can be deallocated without UB occurring, 957 /// - `slot` does not need to be dropped, 958 /// - `slot` is not partially initialized. 959 /// - the `slot` may move after initialization. 960 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 961 #[inline] 962 pub const unsafe fn init_from_closure<T: ?Sized, E>( 963 f: impl FnOnce(*mut T) -> Result<(), E>, 964 ) -> impl Init<T, E> { 965 __internal::InitClosure(f, PhantomData) 966 } 967 968 /// An initializer that leaves the memory uninitialized. 969 /// 970 /// The initializer is a no-op. The `slot` memory is not changed. 971 #[inline] 972 pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { 973 // SAFETY: The memory is allowed to be uninitialized. 974 unsafe { init_from_closure(|_| Ok(())) } 975 } 976 977 // SAFETY: Every type can be initialized by-value. 978 unsafe impl<T> Init<T> for T { 979 unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { 980 unsafe { slot.write(self) }; 981 Ok(()) 982 } 983 } 984 985 /// Smart pointer that can initialize memory in-place. 986 pub trait InPlaceInit<T>: Sized { 987 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this 988 /// type. 989 /// 990 /// If `T: !Unpin` it will not be able to move afterwards. 991 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 992 where 993 E: From<AllocError>; 994 995 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this 996 /// type. 997 /// 998 /// If `T: !Unpin` it will not be able to move afterwards. 999 fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> 1000 where 1001 Error: From<E>, 1002 { 1003 // SAFETY: We delegate to `init` and only change the error type. 1004 let init = unsafe { 1005 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) 1006 }; 1007 Self::try_pin_init(init) 1008 } 1009 1010 /// Use the given initializer to in-place initialize a `T`. 1011 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1012 where 1013 E: From<AllocError>; 1014 1015 /// Use the given initializer to in-place initialize a `T`. 1016 fn init<E>(init: impl Init<T, E>) -> error::Result<Self> 1017 where 1018 Error: From<E>, 1019 { 1020 // SAFETY: We delegate to `init` and only change the error type. 1021 let init = unsafe { 1022 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) 1023 }; 1024 Self::try_init(init) 1025 } 1026 } 1027 1028 impl<T> InPlaceInit<T> for Box<T> { 1029 #[inline] 1030 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 1031 where 1032 E: From<AllocError>, 1033 { 1034 let mut this = Box::try_new_uninit()?; 1035 let slot = this.as_mut_ptr(); 1036 // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1037 // slot is valid and will not be moved, because we pin it later. 1038 unsafe { init.__pinned_init(slot)? }; 1039 // SAFETY: All fields have been initialized. 1040 Ok(unsafe { this.assume_init() }.into()) 1041 } 1042 1043 #[inline] 1044 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1045 where 1046 E: From<AllocError>, 1047 { 1048 let mut this = Box::try_new_uninit()?; 1049 let slot = this.as_mut_ptr(); 1050 // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1051 // slot is valid. 1052 unsafe { init.__init(slot)? }; 1053 // SAFETY: All fields have been initialized. 1054 Ok(unsafe { this.assume_init() }) 1055 } 1056 } 1057 1058 impl<T> InPlaceInit<T> for UniqueArc<T> { 1059 #[inline] 1060 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 1061 where 1062 E: From<AllocError>, 1063 { 1064 let mut this = UniqueArc::try_new_uninit()?; 1065 let slot = this.as_mut_ptr(); 1066 // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1067 // slot is valid and will not be moved, because we pin it later. 1068 unsafe { init.__pinned_init(slot)? }; 1069 // SAFETY: All fields have been initialized. 1070 Ok(unsafe { this.assume_init() }.into()) 1071 } 1072 1073 #[inline] 1074 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1075 where 1076 E: From<AllocError>, 1077 { 1078 let mut this = UniqueArc::try_new_uninit()?; 1079 let slot = this.as_mut_ptr(); 1080 // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1081 // slot is valid. 1082 unsafe { init.__init(slot)? }; 1083 // SAFETY: All fields have been initialized. 1084 Ok(unsafe { this.assume_init() }) 1085 } 1086 } 1087