Loading...
Note: File does not exist in v6.2.
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4//!
5//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6//! overflow.
7//!
8//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10//!
11//! # Overview
12//!
13//! To initialize a `struct` with an in-place constructor you will need two things:
14//! - an in-place constructor,
15//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17//!
18//! To get an in-place constructor there are generally three options:
19//! - directly creating an in-place constructor using the [`pin_init!`] macro,
20//! - a custom function/macro returning an in-place constructor provided by someone else,
21//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22//!
23//! Aside from pinned initialization, this API also supports in-place construction without pinning,
24//! the macros/types/functions are generally named like the pinned variants without the `pin`
25//! prefix.
26//!
27//! # Examples
28//!
29//! ## Using the [`pin_init!`] macro
30//!
31//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36//!
37//! ```rust
38//! # #![expect(clippy::disallowed_names)]
39//! use kernel::sync::{new_mutex, Mutex};
40//! # use core::pin::Pin;
41//! #[pin_data]
42//! struct Foo {
43//! #[pin]
44//! a: Mutex<usize>,
45//! b: u32,
46//! }
47//!
48//! let foo = pin_init!(Foo {
49//! a <- new_mutex!(42, "Foo::a"),
50//! b: 24,
51//! });
52//! ```
53//!
54//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55//! (or just the stack) to actually initialize a `Foo`:
56//!
57//! ```rust
58//! # #![expect(clippy::disallowed_names)]
59//! # use kernel::sync::{new_mutex, Mutex};
60//! # use core::pin::Pin;
61//! # #[pin_data]
62//! # struct Foo {
63//! # #[pin]
64//! # a: Mutex<usize>,
65//! # b: u32,
66//! # }
67//! # let foo = pin_init!(Foo {
68//! # a <- new_mutex!(42, "Foo::a"),
69//! # b: 24,
70//! # });
71//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL);
72//! ```
73//!
74//! For more information see the [`pin_init!`] macro.
75//!
76//! ## Using a custom function/macro that returns an initializer
77//!
78//! Many types from the kernel supply a function/macro that returns an initializer, because the
79//! above method only works for types where you can access the fields.
80//!
81//! ```rust
82//! # use kernel::sync::{new_mutex, Arc, Mutex};
83//! let mtx: Result<Arc<Mutex<usize>>> =
84//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
85//! ```
86//!
87//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
88//!
89//! ```rust
90//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
91//! #[pin_data]
92//! struct DriverData {
93//! #[pin]
94//! status: Mutex<i32>,
95//! buffer: KBox<[u8; 1_000_000]>,
96//! }
97//!
98//! impl DriverData {
99//! fn new() -> impl PinInit<Self, Error> {
100//! try_pin_init!(Self {
101//! status <- new_mutex!(0, "DriverData::status"),
102//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?,
103//! })
104//! }
105//! }
106//! ```
107//!
108//! ## Manual creation of an initializer
109//!
110//! Often when working with primitives the previous approaches are not sufficient. That is where
111//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
112//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
113//! actually does the initialization in the correct way. Here are the things to look out for
114//! (we are calling the parameter to the closure `slot`):
115//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
116//! `slot` now contains a valid bit pattern for the type `T`,
117//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
118//! you need to take care to clean up anything if your initialization fails mid-way,
119//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
120//! `slot` gets called.
121//!
122//! ```rust
123//! # #![expect(unreachable_pub, clippy::disallowed_names)]
124//! use kernel::{init, types::Opaque};
125//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
126//! # mod bindings {
127//! # #![expect(non_camel_case_types)]
128//! # #![expect(clippy::missing_safety_doc)]
129//! # pub struct foo;
130//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
131//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
132//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
133//! # }
134//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
135//! # trait FromErrno {
136//! # fn from_errno(errno: kernel::ffi::c_int) -> Error {
137//! # // Dummy error that can be constructed outside the `kernel` crate.
138//! # Error::from(core::fmt::Error)
139//! # }
140//! # }
141//! # impl FromErrno for Error {}
142//! /// # Invariants
143//! ///
144//! /// `foo` is always initialized
145//! #[pin_data(PinnedDrop)]
146//! pub struct RawFoo {
147//! #[pin]
148//! foo: Opaque<bindings::foo>,
149//! #[pin]
150//! _p: PhantomPinned,
151//! }
152//!
153//! impl RawFoo {
154//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
155//! // SAFETY:
156//! // - when the closure returns `Ok(())`, then it has successfully initialized and
157//! // enabled `foo`,
158//! // - when it returns `Err(e)`, then it has cleaned up before
159//! unsafe {
160//! init::pin_init_from_closure(move |slot: *mut Self| {
161//! // `slot` contains uninit memory, avoid creating a reference.
162//! let foo = addr_of_mut!((*slot).foo);
163//!
164//! // Initialize the `foo`
165//! bindings::init_foo(Opaque::raw_get(foo));
166//!
167//! // Try to enable it.
168//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
169//! if err != 0 {
170//! // Enabling has failed, first clean up the foo and then return the error.
171//! bindings::destroy_foo(Opaque::raw_get(foo));
172//! return Err(Error::from_errno(err));
173//! }
174//!
175//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
176//! Ok(())
177//! })
178//! }
179//! }
180//! }
181//!
182//! #[pinned_drop]
183//! impl PinnedDrop for RawFoo {
184//! fn drop(self: Pin<&mut Self>) {
185//! // SAFETY: Since `foo` is initialized, destroying is safe.
186//! unsafe { bindings::destroy_foo(self.foo.get()) };
187//! }
188//! }
189//! ```
190//!
191//! For the special case where initializing a field is a single FFI-function call that cannot fail,
192//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
193//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
194//! with [`pin_init!`].
195//!
196//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
197//! the `kernel` crate. The [`sync`] module is a good starting point.
198//!
199//! [`sync`]: kernel::sync
200//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
201//! [structurally pinned fields]:
202//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
203//! [stack]: crate::stack_pin_init
204//! [`Arc<T>`]: crate::sync::Arc
205//! [`impl PinInit<Foo>`]: PinInit
206//! [`impl PinInit<T, E>`]: PinInit
207//! [`impl Init<T, E>`]: Init
208//! [`Opaque`]: kernel::types::Opaque
209//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
210//! [`pin_data`]: ::macros::pin_data
211//! [`pin_init!`]: crate::pin_init!
212
213use crate::{
214 alloc::{AllocError, Flags, KBox},
215 error::{self, Error},
216 sync::Arc,
217 sync::UniqueArc,
218 types::{Opaque, ScopeGuard},
219};
220use core::{
221 cell::UnsafeCell,
222 convert::Infallible,
223 marker::PhantomData,
224 mem::MaybeUninit,
225 num::*,
226 pin::Pin,
227 ptr::{self, NonNull},
228};
229
230#[doc(hidden)]
231pub mod __internal;
232#[doc(hidden)]
233pub mod macros;
234
235/// Initialize and pin a type directly on the stack.
236///
237/// # Examples
238///
239/// ```rust
240/// # #![expect(clippy::disallowed_names)]
241/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
242/// # use core::pin::Pin;
243/// #[pin_data]
244/// struct Foo {
245/// #[pin]
246/// a: Mutex<usize>,
247/// b: Bar,
248/// }
249///
250/// #[pin_data]
251/// struct Bar {
252/// x: u32,
253/// }
254///
255/// stack_pin_init!(let foo = pin_init!(Foo {
256/// a <- new_mutex!(42),
257/// b: Bar {
258/// x: 64,
259/// },
260/// }));
261/// let foo: Pin<&mut Foo> = foo;
262/// pr_info!("a: {}", &*foo.a.lock());
263/// ```
264///
265/// # Syntax
266///
267/// A normal `let` binding with optional type annotation. The expression is expected to implement
268/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
269/// type, then use [`stack_try_pin_init!`].
270///
271/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
272#[macro_export]
273macro_rules! stack_pin_init {
274 (let $var:ident $(: $t:ty)? = $val:expr) => {
275 let val = $val;
276 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
277 let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
278 Ok(res) => res,
279 Err(x) => {
280 let x: ::core::convert::Infallible = x;
281 match x {}
282 }
283 };
284 };
285}
286
287/// Initialize and pin a type directly on the stack.
288///
289/// # Examples
290///
291/// ```rust,ignore
292/// # #![expect(clippy::disallowed_names)]
293/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
294/// # use macros::pin_data;
295/// # use core::{alloc::AllocError, pin::Pin};
296/// #[pin_data]
297/// struct Foo {
298/// #[pin]
299/// a: Mutex<usize>,
300/// b: KBox<Bar>,
301/// }
302///
303/// struct Bar {
304/// x: u32,
305/// }
306///
307/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
308/// a <- new_mutex!(42),
309/// b: KBox::new(Bar {
310/// x: 64,
311/// }, GFP_KERNEL)?,
312/// }));
313/// let foo = foo.unwrap();
314/// pr_info!("a: {}", &*foo.a.lock());
315/// ```
316///
317/// ```rust,ignore
318/// # #![expect(clippy::disallowed_names)]
319/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
320/// # use macros::pin_data;
321/// # use core::{alloc::AllocError, pin::Pin};
322/// #[pin_data]
323/// struct Foo {
324/// #[pin]
325/// a: Mutex<usize>,
326/// b: KBox<Bar>,
327/// }
328///
329/// struct Bar {
330/// x: u32,
331/// }
332///
333/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
334/// a <- new_mutex!(42),
335/// b: KBox::new(Bar {
336/// x: 64,
337/// }, GFP_KERNEL)?,
338/// }));
339/// pr_info!("a: {}", &*foo.a.lock());
340/// # Ok::<_, AllocError>(())
341/// ```
342///
343/// # Syntax
344///
345/// A normal `let` binding with optional type annotation. The expression is expected to implement
346/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
347/// `=` will propagate this error.
348#[macro_export]
349macro_rules! stack_try_pin_init {
350 (let $var:ident $(: $t:ty)? = $val:expr) => {
351 let val = $val;
352 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
353 let mut $var = $crate::init::__internal::StackInit::init($var, val);
354 };
355 (let $var:ident $(: $t:ty)? =? $val:expr) => {
356 let val = $val;
357 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
358 let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
359 };
360}
361
362/// Construct an in-place, pinned initializer for `struct`s.
363///
364/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
365/// [`try_pin_init!`].
366///
367/// The syntax is almost identical to that of a normal `struct` initializer:
368///
369/// ```rust
370/// # use kernel::{init, pin_init, macros::pin_data, init::*};
371/// # use core::pin::Pin;
372/// #[pin_data]
373/// struct Foo {
374/// a: usize,
375/// b: Bar,
376/// }
377///
378/// #[pin_data]
379/// struct Bar {
380/// x: u32,
381/// }
382///
383/// # fn demo() -> impl PinInit<Foo> {
384/// let a = 42;
385///
386/// let initializer = pin_init!(Foo {
387/// a,
388/// b: Bar {
389/// x: 64,
390/// },
391/// });
392/// # initializer }
393/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap();
394/// ```
395///
396/// Arbitrary Rust expressions can be used to set the value of a variable.
397///
398/// The fields are initialized in the order that they appear in the initializer. So it is possible
399/// to read already initialized fields using raw pointers.
400///
401/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
402/// initializer.
403///
404/// # Init-functions
405///
406/// When working with this API it is often desired to let others construct your types without
407/// giving access to all fields. This is where you would normally write a plain function `new`
408/// that would return a new instance of your type. With this API that is also possible.
409/// However, there are a few extra things to keep in mind.
410///
411/// To create an initializer function, simply declare it like this:
412///
413/// ```rust
414/// # use kernel::{init, pin_init, init::*};
415/// # use core::pin::Pin;
416/// # #[pin_data]
417/// # struct Foo {
418/// # a: usize,
419/// # b: Bar,
420/// # }
421/// # #[pin_data]
422/// # struct Bar {
423/// # x: u32,
424/// # }
425/// impl Foo {
426/// fn new() -> impl PinInit<Self> {
427/// pin_init!(Self {
428/// a: 42,
429/// b: Bar {
430/// x: 64,
431/// },
432/// })
433/// }
434/// }
435/// ```
436///
437/// Users of `Foo` can now create it like this:
438///
439/// ```rust
440/// # #![expect(clippy::disallowed_names)]
441/// # use kernel::{init, pin_init, macros::pin_data, init::*};
442/// # use core::pin::Pin;
443/// # #[pin_data]
444/// # struct Foo {
445/// # a: usize,
446/// # b: Bar,
447/// # }
448/// # #[pin_data]
449/// # struct Bar {
450/// # x: u32,
451/// # }
452/// # impl Foo {
453/// # fn new() -> impl PinInit<Self> {
454/// # pin_init!(Self {
455/// # a: 42,
456/// # b: Bar {
457/// # x: 64,
458/// # },
459/// # })
460/// # }
461/// # }
462/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL);
463/// ```
464///
465/// They can also easily embed it into their own `struct`s:
466///
467/// ```rust
468/// # use kernel::{init, pin_init, macros::pin_data, init::*};
469/// # use core::pin::Pin;
470/// # #[pin_data]
471/// # struct Foo {
472/// # a: usize,
473/// # b: Bar,
474/// # }
475/// # #[pin_data]
476/// # struct Bar {
477/// # x: u32,
478/// # }
479/// # impl Foo {
480/// # fn new() -> impl PinInit<Self> {
481/// # pin_init!(Self {
482/// # a: 42,
483/// # b: Bar {
484/// # x: 64,
485/// # },
486/// # })
487/// # }
488/// # }
489/// #[pin_data]
490/// struct FooContainer {
491/// #[pin]
492/// foo1: Foo,
493/// #[pin]
494/// foo2: Foo,
495/// other: u32,
496/// }
497///
498/// impl FooContainer {
499/// fn new(other: u32) -> impl PinInit<Self> {
500/// pin_init!(Self {
501/// foo1 <- Foo::new(),
502/// foo2 <- Foo::new(),
503/// other,
504/// })
505/// }
506/// }
507/// ```
508///
509/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
510/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
511/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
512///
513/// # Syntax
514///
515/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
516/// the following modifications is expected:
517/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
518/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
519/// pointer named `this` inside of the initializer.
520/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
521/// struct, this initializes every field with 0 and then runs all initializers specified in the
522/// body. This can only be done if [`Zeroable`] is implemented for the struct.
523///
524/// For instance:
525///
526/// ```rust
527/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
528/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
529/// #[pin_data]
530/// #[derive(Zeroable)]
531/// struct Buf {
532/// // `ptr` points into `buf`.
533/// ptr: *mut u8,
534/// buf: [u8; 64],
535/// #[pin]
536/// pin: PhantomPinned,
537/// }
538/// pin_init!(&this in Buf {
539/// buf: [0; 64],
540/// // SAFETY: TODO.
541/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
542/// pin: PhantomPinned,
543/// });
544/// pin_init!(Buf {
545/// buf: [1; 64],
546/// ..Zeroable::zeroed()
547/// });
548/// ```
549///
550/// [`try_pin_init!`]: kernel::try_pin_init
551/// [`NonNull<Self>`]: core::ptr::NonNull
552// For a detailed example of how this macro works, see the module documentation of the hidden
553// module `__internal` inside of `init/__internal.rs`.
554#[macro_export]
555macro_rules! pin_init {
556 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
557 $($fields:tt)*
558 }) => {
559 $crate::__init_internal!(
560 @this($($this)?),
561 @typ($t $(::<$($generics),*>)?),
562 @fields($($fields)*),
563 @error(::core::convert::Infallible),
564 @data(PinData, use_data),
565 @has_data(HasPinData, __pin_data),
566 @construct_closure(pin_init_from_closure),
567 @munch_fields($($fields)*),
568 )
569 };
570}
571
572/// Construct an in-place, fallible pinned initializer for `struct`s.
573///
574/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
575///
576/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
577/// initialization and return the error.
578///
579/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
580/// initialization fails, the memory can be safely deallocated without any further modifications.
581///
582/// This macro defaults the error to [`Error`].
583///
584/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
585/// after the `struct` initializer to specify the error type you want to use.
586///
587/// # Examples
588///
589/// ```rust
590/// use kernel::{init::{self, PinInit}, error::Error};
591/// #[pin_data]
592/// struct BigBuf {
593/// big: KBox<[u8; 1024 * 1024 * 1024]>,
594/// small: [u8; 1024 * 1024],
595/// ptr: *mut u8,
596/// }
597///
598/// impl BigBuf {
599/// fn new() -> impl PinInit<Self, Error> {
600/// try_pin_init!(Self {
601/// big: KBox::init(init::zeroed(), GFP_KERNEL)?,
602/// small: [0; 1024 * 1024],
603/// ptr: core::ptr::null_mut(),
604/// }? Error)
605/// }
606/// }
607/// ```
608// For a detailed example of how this macro works, see the module documentation of the hidden
609// module `__internal` inside of `init/__internal.rs`.
610#[macro_export]
611macro_rules! try_pin_init {
612 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
613 $($fields:tt)*
614 }) => {
615 $crate::__init_internal!(
616 @this($($this)?),
617 @typ($t $(::<$($generics),*>)? ),
618 @fields($($fields)*),
619 @error($crate::error::Error),
620 @data(PinData, use_data),
621 @has_data(HasPinData, __pin_data),
622 @construct_closure(pin_init_from_closure),
623 @munch_fields($($fields)*),
624 )
625 };
626 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
627 $($fields:tt)*
628 }? $err:ty) => {
629 $crate::__init_internal!(
630 @this($($this)?),
631 @typ($t $(::<$($generics),*>)? ),
632 @fields($($fields)*),
633 @error($err),
634 @data(PinData, use_data),
635 @has_data(HasPinData, __pin_data),
636 @construct_closure(pin_init_from_closure),
637 @munch_fields($($fields)*),
638 )
639 };
640}
641
642/// Construct an in-place initializer for `struct`s.
643///
644/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
645/// [`try_init!`].
646///
647/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
648/// - `unsafe` code must guarantee either full initialization or return an error and allow
649/// deallocation of the memory.
650/// - the fields are initialized in the order given in the initializer.
651/// - no references to fields are allowed to be created inside of the initializer.
652///
653/// This initializer is for initializing data in-place that might later be moved. If you want to
654/// pin-initialize, use [`pin_init!`].
655///
656/// [`try_init!`]: crate::try_init!
657// For a detailed example of how this macro works, see the module documentation of the hidden
658// module `__internal` inside of `init/__internal.rs`.
659#[macro_export]
660macro_rules! init {
661 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
662 $($fields:tt)*
663 }) => {
664 $crate::__init_internal!(
665 @this($($this)?),
666 @typ($t $(::<$($generics),*>)?),
667 @fields($($fields)*),
668 @error(::core::convert::Infallible),
669 @data(InitData, /*no use_data*/),
670 @has_data(HasInitData, __init_data),
671 @construct_closure(init_from_closure),
672 @munch_fields($($fields)*),
673 )
674 }
675}
676
677/// Construct an in-place fallible initializer for `struct`s.
678///
679/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
680/// [`init!`].
681///
682/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
683/// append `? $type` after the `struct` initializer.
684/// The safety caveats from [`try_pin_init!`] also apply:
685/// - `unsafe` code must guarantee either full initialization or return an error and allow
686/// deallocation of the memory.
687/// - the fields are initialized in the order given in the initializer.
688/// - no references to fields are allowed to be created inside of the initializer.
689///
690/// # Examples
691///
692/// ```rust
693/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error};
694/// struct BigBuf {
695/// big: KBox<[u8; 1024 * 1024 * 1024]>,
696/// small: [u8; 1024 * 1024],
697/// }
698///
699/// impl BigBuf {
700/// fn new() -> impl Init<Self, Error> {
701/// try_init!(Self {
702/// big: KBox::init(zeroed(), GFP_KERNEL)?,
703/// small: [0; 1024 * 1024],
704/// }? Error)
705/// }
706/// }
707/// ```
708// For a detailed example of how this macro works, see the module documentation of the hidden
709// module `__internal` inside of `init/__internal.rs`.
710#[macro_export]
711macro_rules! try_init {
712 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
713 $($fields:tt)*
714 }) => {
715 $crate::__init_internal!(
716 @this($($this)?),
717 @typ($t $(::<$($generics),*>)?),
718 @fields($($fields)*),
719 @error($crate::error::Error),
720 @data(InitData, /*no use_data*/),
721 @has_data(HasInitData, __init_data),
722 @construct_closure(init_from_closure),
723 @munch_fields($($fields)*),
724 )
725 };
726 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
727 $($fields:tt)*
728 }? $err:ty) => {
729 $crate::__init_internal!(
730 @this($($this)?),
731 @typ($t $(::<$($generics),*>)?),
732 @fields($($fields)*),
733 @error($err),
734 @data(InitData, /*no use_data*/),
735 @has_data(HasInitData, __init_data),
736 @construct_closure(init_from_closure),
737 @munch_fields($($fields)*),
738 )
739 };
740}
741
742/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
743/// structurally pinned.
744///
745/// # Example
746///
747/// This will succeed:
748/// ```
749/// use kernel::assert_pinned;
750/// #[pin_data]
751/// struct MyStruct {
752/// #[pin]
753/// some_field: u64,
754/// }
755///
756/// assert_pinned!(MyStruct, some_field, u64);
757/// ```
758///
759/// This will fail:
760// TODO: replace with `compile_fail` when supported.
761/// ```ignore
762/// use kernel::assert_pinned;
763/// #[pin_data]
764/// struct MyStruct {
765/// some_field: u64,
766/// }
767///
768/// assert_pinned!(MyStruct, some_field, u64);
769/// ```
770///
771/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
772/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
773/// only be used when the macro is invoked from a function body.
774/// ```
775/// use kernel::assert_pinned;
776/// #[pin_data]
777/// struct Foo<T> {
778/// #[pin]
779/// elem: T,
780/// }
781///
782/// impl<T> Foo<T> {
783/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
784/// assert_pinned!(Foo<T>, elem, T, inline);
785///
786/// // SAFETY: The field is structurally pinned.
787/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
788/// }
789/// }
790/// ```
791#[macro_export]
792macro_rules! assert_pinned {
793 ($ty:ty, $field:ident, $field_ty:ty, inline) => {
794 let _ = move |ptr: *mut $field_ty| {
795 // SAFETY: This code is unreachable.
796 let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
797 let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
798 // SAFETY: This code is unreachable.
799 unsafe { data.$field(ptr, init) }.ok();
800 };
801 };
802
803 ($ty:ty, $field:ident, $field_ty:ty) => {
804 const _: () = {
805 $crate::assert_pinned!($ty, $field, $field_ty, inline);
806 };
807 };
808}
809
810/// A pin-initializer for the type `T`.
811///
812/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
813/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
814/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
815///
816/// Also see the [module description](self).
817///
818/// # Safety
819///
820/// When implementing this trait you will need to take great care. Also there are probably very few
821/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
822///
823/// The [`PinInit::__pinned_init`] function:
824/// - returns `Ok(())` if it initialized every field of `slot`,
825/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
826/// - `slot` can be deallocated without UB occurring,
827/// - `slot` does not need to be dropped,
828/// - `slot` is not partially initialized.
829/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
830///
831/// [`Arc<T>`]: crate::sync::Arc
832/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
833#[must_use = "An initializer must be used in order to create its value."]
834pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
835 /// Initializes `slot`.
836 ///
837 /// # Safety
838 ///
839 /// - `slot` is a valid pointer to uninitialized memory.
840 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
841 /// deallocate.
842 /// - `slot` will not move until it is dropped, i.e. it will be pinned.
843 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
844
845 /// First initializes the value using `self` then calls the function `f` with the initialized
846 /// value.
847 ///
848 /// If `f` returns an error the value is dropped and the initializer will forward the error.
849 ///
850 /// # Examples
851 ///
852 /// ```rust
853 /// # #![expect(clippy::disallowed_names)]
854 /// use kernel::{types::Opaque, init::pin_init_from_closure};
855 /// #[repr(C)]
856 /// struct RawFoo([u8; 16]);
857 /// extern "C" {
858 /// fn init_foo(_: *mut RawFoo);
859 /// }
860 ///
861 /// #[pin_data]
862 /// struct Foo {
863 /// #[pin]
864 /// raw: Opaque<RawFoo>,
865 /// }
866 ///
867 /// impl Foo {
868 /// fn setup(self: Pin<&mut Self>) {
869 /// pr_info!("Setting up foo");
870 /// }
871 /// }
872 ///
873 /// let foo = pin_init!(Foo {
874 /// // SAFETY: TODO.
875 /// raw <- unsafe {
876 /// Opaque::ffi_init(|s| {
877 /// init_foo(s);
878 /// })
879 /// },
880 /// }).pin_chain(|foo| {
881 /// foo.setup();
882 /// Ok(())
883 /// });
884 /// ```
885 fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
886 where
887 F: FnOnce(Pin<&mut T>) -> Result<(), E>,
888 {
889 ChainPinInit(self, f, PhantomData)
890 }
891}
892
893/// An initializer returned by [`PinInit::pin_chain`].
894pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
895
896// SAFETY: The `__pinned_init` function is implemented such that it
897// - returns `Ok(())` on successful initialization,
898// - returns `Err(err)` on error and in this case `slot` will be dropped.
899// - considers `slot` pinned.
900unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
901where
902 I: PinInit<T, E>,
903 F: FnOnce(Pin<&mut T>) -> Result<(), E>,
904{
905 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
906 // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
907 unsafe { self.0.__pinned_init(slot)? };
908 // SAFETY: The above call initialized `slot` and we still have unique access.
909 let val = unsafe { &mut *slot };
910 // SAFETY: `slot` is considered pinned.
911 let val = unsafe { Pin::new_unchecked(val) };
912 // SAFETY: `slot` was initialized above.
913 (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
914 }
915}
916
917/// An initializer for `T`.
918///
919/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
920/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
921/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
922/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
923///
924/// Also see the [module description](self).
925///
926/// # Safety
927///
928/// When implementing this trait you will need to take great care. Also there are probably very few
929/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
930///
931/// The [`Init::__init`] function:
932/// - returns `Ok(())` if it initialized every field of `slot`,
933/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
934/// - `slot` can be deallocated without UB occurring,
935/// - `slot` does not need to be dropped,
936/// - `slot` is not partially initialized.
937/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
938///
939/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
940/// code as `__init`.
941///
942/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
943/// move the pointee after initialization.
944///
945/// [`Arc<T>`]: crate::sync::Arc
946#[must_use = "An initializer must be used in order to create its value."]
947pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
948 /// Initializes `slot`.
949 ///
950 /// # Safety
951 ///
952 /// - `slot` is a valid pointer to uninitialized memory.
953 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
954 /// deallocate.
955 unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
956
957 /// First initializes the value using `self` then calls the function `f` with the initialized
958 /// value.
959 ///
960 /// If `f` returns an error the value is dropped and the initializer will forward the error.
961 ///
962 /// # Examples
963 ///
964 /// ```rust
965 /// # #![expect(clippy::disallowed_names)]
966 /// use kernel::{types::Opaque, init::{self, init_from_closure}};
967 /// struct Foo {
968 /// buf: [u8; 1_000_000],
969 /// }
970 ///
971 /// impl Foo {
972 /// fn setup(&mut self) {
973 /// pr_info!("Setting up foo");
974 /// }
975 /// }
976 ///
977 /// let foo = init!(Foo {
978 /// buf <- init::zeroed()
979 /// }).chain(|foo| {
980 /// foo.setup();
981 /// Ok(())
982 /// });
983 /// ```
984 fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
985 where
986 F: FnOnce(&mut T) -> Result<(), E>,
987 {
988 ChainInit(self, f, PhantomData)
989 }
990}
991
992/// An initializer returned by [`Init::chain`].
993pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
994
995// SAFETY: The `__init` function is implemented such that it
996// - returns `Ok(())` on successful initialization,
997// - returns `Err(err)` on error and in this case `slot` will be dropped.
998unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
999where
1000 I: Init<T, E>,
1001 F: FnOnce(&mut T) -> Result<(), E>,
1002{
1003 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1004 // SAFETY: All requirements fulfilled since this function is `__init`.
1005 unsafe { self.0.__pinned_init(slot)? };
1006 // SAFETY: The above call initialized `slot` and we still have unique access.
1007 (self.1)(unsafe { &mut *slot }).inspect_err(|_|
1008 // SAFETY: `slot` was initialized above.
1009 unsafe { core::ptr::drop_in_place(slot) })
1010 }
1011}
1012
1013// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
1014unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
1015where
1016 I: Init<T, E>,
1017 F: FnOnce(&mut T) -> Result<(), E>,
1018{
1019 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1020 // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
1021 unsafe { self.__init(slot) }
1022 }
1023}
1024
1025/// Creates a new [`PinInit<T, E>`] from the given closure.
1026///
1027/// # Safety
1028///
1029/// The closure:
1030/// - returns `Ok(())` if it initialized every field of `slot`,
1031/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1032/// - `slot` can be deallocated without UB occurring,
1033/// - `slot` does not need to be dropped,
1034/// - `slot` is not partially initialized.
1035/// - may assume that the `slot` does not move if `T: !Unpin`,
1036/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1037#[inline]
1038pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1039 f: impl FnOnce(*mut T) -> Result<(), E>,
1040) -> impl PinInit<T, E> {
1041 __internal::InitClosure(f, PhantomData)
1042}
1043
1044/// Creates a new [`Init<T, E>`] from the given closure.
1045///
1046/// # Safety
1047///
1048/// The closure:
1049/// - returns `Ok(())` if it initialized every field of `slot`,
1050/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1051/// - `slot` can be deallocated without UB occurring,
1052/// - `slot` does not need to be dropped,
1053/// - `slot` is not partially initialized.
1054/// - the `slot` may move after initialization.
1055/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1056#[inline]
1057pub const unsafe fn init_from_closure<T: ?Sized, E>(
1058 f: impl FnOnce(*mut T) -> Result<(), E>,
1059) -> impl Init<T, E> {
1060 __internal::InitClosure(f, PhantomData)
1061}
1062
1063/// An initializer that leaves the memory uninitialized.
1064///
1065/// The initializer is a no-op. The `slot` memory is not changed.
1066#[inline]
1067pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1068 // SAFETY: The memory is allowed to be uninitialized.
1069 unsafe { init_from_closure(|_| Ok(())) }
1070}
1071
1072/// Initializes an array by initializing each element via the provided initializer.
1073///
1074/// # Examples
1075///
1076/// ```rust
1077/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn};
1078/// let array: KBox<[usize; 1_000]> =
1079/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap();
1080/// assert_eq!(array.len(), 1_000);
1081/// ```
1082pub fn init_array_from_fn<I, const N: usize, T, E>(
1083 mut make_init: impl FnMut(usize) -> I,
1084) -> impl Init<[T; N], E>
1085where
1086 I: Init<T, E>,
1087{
1088 let init = move |slot: *mut [T; N]| {
1089 let slot = slot.cast::<T>();
1090 // Counts the number of initialized elements and when dropped drops that many elements from
1091 // `slot`.
1092 let mut init_count = ScopeGuard::new_with_data(0, |i| {
1093 // We now free every element that has been initialized before.
1094 // SAFETY: The loop initialized exactly the values from 0..i and since we
1095 // return `Err` below, the caller will consider the memory at `slot` as
1096 // uninitialized.
1097 unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1098 });
1099 for i in 0..N {
1100 let init = make_init(i);
1101 // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1102 let ptr = unsafe { slot.add(i) };
1103 // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1104 // requirements.
1105 unsafe { init.__init(ptr) }?;
1106 *init_count += 1;
1107 }
1108 init_count.dismiss();
1109 Ok(())
1110 };
1111 // SAFETY: The initializer above initializes every element of the array. On failure it drops
1112 // any initialized elements and returns `Err`.
1113 unsafe { init_from_closure(init) }
1114}
1115
1116/// Initializes an array by initializing each element via the provided initializer.
1117///
1118/// # Examples
1119///
1120/// ```rust
1121/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
1122/// let array: Arc<[Mutex<usize>; 1_000]> =
1123/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap();
1124/// assert_eq!(array.len(), 1_000);
1125/// ```
1126pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
1127 mut make_init: impl FnMut(usize) -> I,
1128) -> impl PinInit<[T; N], E>
1129where
1130 I: PinInit<T, E>,
1131{
1132 let init = move |slot: *mut [T; N]| {
1133 let slot = slot.cast::<T>();
1134 // Counts the number of initialized elements and when dropped drops that many elements from
1135 // `slot`.
1136 let mut init_count = ScopeGuard::new_with_data(0, |i| {
1137 // We now free every element that has been initialized before.
1138 // SAFETY: The loop initialized exactly the values from 0..i and since we
1139 // return `Err` below, the caller will consider the memory at `slot` as
1140 // uninitialized.
1141 unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1142 });
1143 for i in 0..N {
1144 let init = make_init(i);
1145 // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1146 let ptr = unsafe { slot.add(i) };
1147 // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1148 // requirements.
1149 unsafe { init.__pinned_init(ptr) }?;
1150 *init_count += 1;
1151 }
1152 init_count.dismiss();
1153 Ok(())
1154 };
1155 // SAFETY: The initializer above initializes every element of the array. On failure it drops
1156 // any initialized elements and returns `Err`.
1157 unsafe { pin_init_from_closure(init) }
1158}
1159
1160// SAFETY: Every type can be initialized by-value.
1161unsafe impl<T, E> Init<T, E> for T {
1162 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1163 // SAFETY: TODO.
1164 unsafe { slot.write(self) };
1165 Ok(())
1166 }
1167}
1168
1169// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
1170unsafe impl<T, E> PinInit<T, E> for T {
1171 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1172 // SAFETY: TODO.
1173 unsafe { self.__init(slot) }
1174 }
1175}
1176
1177/// Smart pointer that can initialize memory in-place.
1178pub trait InPlaceInit<T>: Sized {
1179 /// Pinned version of `Self`.
1180 ///
1181 /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
1182 /// `Self`, otherwise just use `Pin<Self>`.
1183 type PinnedSelf;
1184
1185 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1186 /// type.
1187 ///
1188 /// If `T: !Unpin` it will not be able to move afterwards.
1189 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1190 where
1191 E: From<AllocError>;
1192
1193 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1194 /// type.
1195 ///
1196 /// If `T: !Unpin` it will not be able to move afterwards.
1197 fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
1198 where
1199 Error: From<E>,
1200 {
1201 // SAFETY: We delegate to `init` and only change the error type.
1202 let init = unsafe {
1203 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1204 };
1205 Self::try_pin_init(init, flags)
1206 }
1207
1208 /// Use the given initializer to in-place initialize a `T`.
1209 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1210 where
1211 E: From<AllocError>;
1212
1213 /// Use the given initializer to in-place initialize a `T`.
1214 fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
1215 where
1216 Error: From<E>,
1217 {
1218 // SAFETY: We delegate to `init` and only change the error type.
1219 let init = unsafe {
1220 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1221 };
1222 Self::try_init(init, flags)
1223 }
1224}
1225
1226impl<T> InPlaceInit<T> for Arc<T> {
1227 type PinnedSelf = Self;
1228
1229 #[inline]
1230 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1231 where
1232 E: From<AllocError>,
1233 {
1234 UniqueArc::try_pin_init(init, flags).map(|u| u.into())
1235 }
1236
1237 #[inline]
1238 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1239 where
1240 E: From<AllocError>,
1241 {
1242 UniqueArc::try_init(init, flags).map(|u| u.into())
1243 }
1244}
1245
1246impl<T> InPlaceInit<T> for UniqueArc<T> {
1247 type PinnedSelf = Pin<Self>;
1248
1249 #[inline]
1250 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1251 where
1252 E: From<AllocError>,
1253 {
1254 UniqueArc::new_uninit(flags)?.write_pin_init(init)
1255 }
1256
1257 #[inline]
1258 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1259 where
1260 E: From<AllocError>,
1261 {
1262 UniqueArc::new_uninit(flags)?.write_init(init)
1263 }
1264}
1265
1266/// Smart pointer containing uninitialized memory and that can write a value.
1267pub trait InPlaceWrite<T> {
1268 /// The type `Self` turns into when the contents are initialized.
1269 type Initialized;
1270
1271 /// Use the given initializer to write a value into `self`.
1272 ///
1273 /// Does not drop the current value and considers it as uninitialized memory.
1274 fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
1275
1276 /// Use the given pin-initializer to write a value into `self`.
1277 ///
1278 /// Does not drop the current value and considers it as uninitialized memory.
1279 fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
1280}
1281
1282impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
1283 type Initialized = UniqueArc<T>;
1284
1285 fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
1286 let slot = self.as_mut_ptr();
1287 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1288 // slot is valid.
1289 unsafe { init.__init(slot)? };
1290 // SAFETY: All fields have been initialized.
1291 Ok(unsafe { self.assume_init() })
1292 }
1293
1294 fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
1295 let slot = self.as_mut_ptr();
1296 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1297 // slot is valid and will not be moved, because we pin it later.
1298 unsafe { init.__pinned_init(slot)? };
1299 // SAFETY: All fields have been initialized.
1300 Ok(unsafe { self.assume_init() }.into())
1301 }
1302}
1303
1304/// Trait facilitating pinned destruction.
1305///
1306/// Use [`pinned_drop`] to implement this trait safely:
1307///
1308/// ```rust
1309/// # use kernel::sync::Mutex;
1310/// use kernel::macros::pinned_drop;
1311/// use core::pin::Pin;
1312/// #[pin_data(PinnedDrop)]
1313/// struct Foo {
1314/// #[pin]
1315/// mtx: Mutex<usize>,
1316/// }
1317///
1318/// #[pinned_drop]
1319/// impl PinnedDrop for Foo {
1320/// fn drop(self: Pin<&mut Self>) {
1321/// pr_info!("Foo is being dropped!");
1322/// }
1323/// }
1324/// ```
1325///
1326/// # Safety
1327///
1328/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1329///
1330/// [`pinned_drop`]: kernel::macros::pinned_drop
1331pub unsafe trait PinnedDrop: __internal::HasPinData {
1332 /// Executes the pinned destructor of this type.
1333 ///
1334 /// While this function is marked safe, it is actually unsafe to call it manually. For this
1335 /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1336 /// and thus prevents this function from being called where it should not.
1337 ///
1338 /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1339 /// automatically.
1340 fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1341}
1342
1343/// Marker trait for types that can be initialized by writing just zeroes.
1344///
1345/// # Safety
1346///
1347/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1348/// this is not UB:
1349///
1350/// ```rust,ignore
1351/// let val: Self = unsafe { core::mem::zeroed() };
1352/// ```
1353pub unsafe trait Zeroable {}
1354
1355/// Create a new zeroed T.
1356///
1357/// The returned initializer will write `0x00` to every byte of the given `slot`.
1358#[inline]
1359pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1360 // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1361 // and because we write all zeroes, the memory is initialized.
1362 unsafe {
1363 init_from_closure(|slot: *mut T| {
1364 slot.write_bytes(0, 1);
1365 Ok(())
1366 })
1367 }
1368}
1369
1370macro_rules! impl_zeroable {
1371 ($($({$($generics:tt)*})? $t:ty, )*) => {
1372 // SAFETY: Safety comments written in the macro invocation.
1373 $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1374 };
1375}
1376
1377impl_zeroable! {
1378 // SAFETY: All primitives that are allowed to be zero.
1379 bool,
1380 char,
1381 u8, u16, u32, u64, u128, usize,
1382 i8, i16, i32, i64, i128, isize,
1383 f32, f64,
1384
1385 // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
1386 // creating an instance of an uninhabited type is immediate undefined behavior. For more on
1387 // uninhabited/empty types, consult The Rustonomicon:
1388 // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
1389 // also has information on undefined behavior:
1390 // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
1391 //
1392 // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
1393 {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
1394
1395 // SAFETY: Type is allowed to take any value, including all zeros.
1396 {<T>} MaybeUninit<T>,
1397 // SAFETY: Type is allowed to take any value, including all zeros.
1398 {<T>} Opaque<T>,
1399
1400 // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
1401 {<T: ?Sized + Zeroable>} UnsafeCell<T>,
1402
1403 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1404 Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1405 Option<NonZeroU128>, Option<NonZeroUsize>,
1406 Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1407 Option<NonZeroI128>, Option<NonZeroIsize>,
1408
1409 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1410 //
1411 // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1412 {<T: ?Sized>} Option<NonNull<T>>,
1413 {<T: ?Sized>} Option<KBox<T>>,
1414
1415 // SAFETY: `null` pointer is valid.
1416 //
1417 // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1418 // null.
1419 //
1420 // When `Pointee` gets stabilized, we could use
1421 // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1422 {<T>} *mut T, {<T>} *const T,
1423
1424 // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1425 // zero.
1426 {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1427
1428 // SAFETY: `T` is `Zeroable`.
1429 {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1430}
1431
1432macro_rules! impl_tuple_zeroable {
1433 ($(,)?) => {};
1434 ($first:ident, $($t:ident),* $(,)?) => {
1435 // SAFETY: All elements are zeroable and padding can be zero.
1436 unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1437 impl_tuple_zeroable!($($t),* ,);
1438 }
1439}
1440
1441impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);