Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
5//! macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! # Macro expansion example
15//!
16//! This section is intended for readers trying to understand the macros in this module and the
17//! `pin_init!` macros from `init.rs`.
18//!
19//! We will look at the following example:
20//!
21//! ```rust,ignore
22//! # use kernel::init::*;
23//! # use core::pin::Pin;
24//! #[pin_data]
25//! #[repr(C)]
26//! struct Bar<T> {
27//! #[pin]
28//! t: T,
29//! pub x: usize,
30//! }
31//!
32//! impl<T> Bar<T> {
33//! fn new(t: T) -> impl PinInit<Self> {
34//! pin_init!(Self { t, x: 0 })
35//! }
36//! }
37//!
38//! #[pin_data(PinnedDrop)]
39//! struct Foo {
40//! a: usize,
41//! #[pin]
42//! b: Bar<u32>,
43//! }
44//!
45//! #[pinned_drop]
46//! impl PinnedDrop for Foo {
47//! fn drop(self: Pin<&mut Self>) {
48//! pr_info!("{self:p} is getting dropped.");
49//! }
50//! }
51//!
52//! let a = 42;
53//! let initializer = pin_init!(Foo {
54//! a,
55//! b <- Bar::new(36),
56//! });
57//! ```
58//!
59//! This example includes the most common and important features of the pin-init API.
60//!
61//! Below you can find individual section about the different macro invocations. Here are some
62//! general things we need to take into account when designing macros:
63//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
64//! this ensures that the correct item is used, since users could define their own `mod core {}`
65//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
66//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
67//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
68//! `unsafe` operations without an associated `unsafe` block.
69//!
70//! ## `#[pin_data]` on `Bar`
71//!
72//! This macro is used to specify which fields are structurally pinned and which fields are not. It
73//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
74//!
75//! Here is the definition of `Bar` from our example:
76//!
77//! ```rust,ignore
78//! # use kernel::init::*;
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//! #[pin]
83//! t: T,
84//! pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//! t: T,
95//! pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//! // since we need to implement access functions for each field and thus need to know its
102//! // type.
103//! struct __ThePinData<T> {
104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//! }
106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//! // `self` by value.
108//! impl<T> ::core::clone::Clone for __ThePinData<T> {
109//! fn clone(&self) -> Self {
110//! *self
111//! }
112//! }
113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//! // For every field of `Bar`, the pin-data struct will define a function with the same name
115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//! #[allow(dead_code)]
119//! impl<T> __ThePinData<T> {
120//! unsafe fn t<E>(
121//! self,
122//! slot: *mut T,
123//! // Since `t` is `#[pin]`, this is `PinInit`.
124//! init: impl ::kernel::init::PinInit<T, E>,
125//! ) -> ::core::result::Result<(), E> {
126//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
127//! }
128//! pub unsafe fn x<E>(
129//! self,
130//! slot: *mut usize,
131//! // Since `x` is not `#[pin]`, this is `Init`.
132//! init: impl ::kernel::init::Init<usize, E>,
133//! ) -> ::core::result::Result<(), E> {
134//! unsafe { ::kernel::init::Init::__init(init, slot) }
135//! }
136//! }
137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//! // that we constructed above.
139//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
140//! type PinData = __ThePinData<T>;
141//! unsafe fn __pin_data() -> Self::PinData {
142//! __ThePinData {
143//! __phantom: ::core::marker::PhantomData,
144//! }
145//! }
146//! }
147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
149//! // function without using `unsafe`.
150//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
151//! type Datee = Bar<T>;
152//! }
153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//! // for two reasons:
158//! // - `__phantom`: every generic must be used, since we cannot really know which generics
159//! // are used, we declare all and then use everything here once.
160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//! // over it. The lifetime is needed to work around the limitation that trait bounds must
162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//! // into accepting these bounds regardless.
165//! #[allow(dead_code)]
166//! struct __Unpin<'__pin, T> {
167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//! // Our only `#[pin]` field is `t`.
170//! t: T,
171//! }
172//! #[doc(hidden)]
173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//! where
175//! __Unpin<'__pin, T>: ::core::marker::Unpin,
176//! {}
177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//! // UB with only safe code, so we disallow this by giving a trait implementation error using
180//! // a direct impl and a blanket implementation.
181//! trait MustNotImplDrop {}
182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//! // (normally people want to know if a type has any kind of drop glue at all, here we want
184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//! #[expect(drop_bounds)]
186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//! impl<T> MustNotImplDrop for Bar<T> {}
188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//! #[expect(non_camel_case_types)]
192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//! impl<
194//! T: ::kernel::init::PinnedDrop,
195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//! fn new(t: T) -> impl PinInit<Self> {
210//! pin_init!(Self { t, x: 0 })
211//! }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//! fn new(t: T) -> impl PinInit<Self> {
220//! {
221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//! // return type and shadow it later when we insert the arbitrary user code. That way
223//! // there will be no possibility of returning without `unsafe`.
224//! struct __InitOk;
225//! // Get the data about fields from the supplied type.
226//! // - the function is unsafe, hence the unsafe block
227//! // - we `use` the `HasPinData` trait in the block, it is only available in that
228//! // scope.
229//! let data = unsafe {
230//! use ::kernel::init::__internal::HasPinData;
231//! Self::__pin_data()
232//! };
233//! // Ensure that `data` really is of type `PinData` and help with type inference:
234//! let init = ::kernel::init::__internal::PinData::make_closure::<
235//! _,
236//! __InitOk,
237//! ::core::convert::Infallible,
238//! >(data, move |slot| {
239//! {
240//! // Shadow the structure so it cannot be used to return early. If a user
241//! // tries to write `return Ok(__InitOk)`, then they get a type error,
242//! // since that will refer to this struct instead of the one defined
243//! // above.
244//! struct __InitOk;
245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//! {
247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//! }
249//! // Since initialization could fail later (not in this case, since the
250//! // error type is `Infallible`) we will need to drop this field if there
251//! // is an error later. This `DropGuard` will drop the field when it gets
252//! // dropped and has not yet been forgotten.
253//! let __t_guard = unsafe {
254//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//! };
256//! // Expansion of `x: 0,`:
257//! // Since this can be an arbitrary expression we cannot place it inside
258//! // of the `unsafe` block, so we bind it here.
259//! {
260//! let x = 0;
261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//! }
263//! // We again create a `DropGuard`.
264//! let __x_guard = unsafe {
265//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//! };
267//! // Since initialization has successfully completed, we can now forget
268//! // the guards. This is not `mem::forget`, since we only have
269//! // `&DropGuard`.
270//! ::core::mem::forget(__x_guard);
271//! ::core::mem::forget(__t_guard);
272//! // Here we use the type checker to ensure that every field has been
273//! // initialized exactly once, since this is `if false` it will never get
274//! // executed, but still type-checked.
275//! // Additionally we abuse `slot` to automatically infer the correct type
276//! // for the struct. This is also another check that every field is
277//! // accessible from this scope.
278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//! let _ = || {
280//! unsafe {
281//! ::core::ptr::write(
282//! slot,
283//! Self {
284//! // We only care about typecheck finding every field
285//! // here, the expression does not matter, just conjure
286//! // one using `panic!()`:
287//! t: ::core::panic!(),
288//! x: ::core::panic!(),
289//! },
290//! );
291//! };
292//! };
293//! }
294//! // We leave the scope above and gain access to the previously shadowed
295//! // `__InitOk` that we need to return.
296//! Ok(__InitOk)
297//! });
298//! // Change the return type from `__InitOk` to `()`.
299//! let init = move |
300//! slot,
301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//! init(slot).map(|__InitOk| ())
303//! };
304//! // Construct the initializer.
305//! let init = unsafe {
306//! ::kernel::init::pin_init_from_closure::<
307//! _,
308//! ::core::convert::Infallible,
309//! >(init)
310//! };
311//! init
312//! }
313//! }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//! a: usize,
326//! #[pin]
327//! b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//! a: usize,
336//! b: Bar<u32>,
337//! }
338//! const _: () = {
339//! struct __ThePinData {
340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//! }
342//! impl ::core::clone::Clone for __ThePinData {
343//! fn clone(&self) -> Self {
344//! *self
345//! }
346//! }
347//! impl ::core::marker::Copy for __ThePinData {}
348//! #[allow(dead_code)]
349//! impl __ThePinData {
350//! unsafe fn b<E>(
351//! self,
352//! slot: *mut Bar<u32>,
353//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
354//! ) -> ::core::result::Result<(), E> {
355//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
356//! }
357//! unsafe fn a<E>(
358//! self,
359//! slot: *mut usize,
360//! init: impl ::kernel::init::Init<usize, E>,
361//! ) -> ::core::result::Result<(), E> {
362//! unsafe { ::kernel::init::Init::__init(init, slot) }
363//! }
364//! }
365//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
366//! type PinData = __ThePinData;
367//! unsafe fn __pin_data() -> Self::PinData {
368//! __ThePinData {
369//! __phantom: ::core::marker::PhantomData,
370//! }
371//! }
372//! }
373//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
374//! type Datee = Foo;
375//! }
376//! #[allow(dead_code)]
377//! struct __Unpin<'__pin> {
378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//! b: Bar<u32>,
381//! }
382//! #[doc(hidden)]
383//! impl<'__pin> ::core::marker::Unpin for Foo
384//! where
385//! __Unpin<'__pin>: ::core::marker::Unpin,
386//! {}
387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//! impl ::core::ops::Drop for Foo {
391//! fn drop(&mut self) {
392//! // Since we are getting dropped, no one else has a reference to `self` and thus we
393//! // can assume that we never move.
394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//! // Create the unsafe token that proves that we are inside of a destructor, this
396//! // type is only allowed to be created in a destructor.
397//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
398//! ::kernel::init::PinnedDrop::drop(pinned, token);
399//! }
400//! }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//! fn drop(self: Pin<&mut Self>) {
415//! pr_info!("{self:p} is getting dropped.");
416//! }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::kernel::init::PinnedDrop for Foo {
425//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
426//! pr_info!("{self:p} is getting dropped.");
427//! }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//! a,
440//! b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//! struct __InitOk;
450//! let data = unsafe {
451//! use ::kernel::init::__internal::HasPinData;
452//! Foo::__pin_data()
453//! };
454//! let init = ::kernel::init::__internal::PinData::make_closure::<
455//! _,
456//! __InitOk,
457//! ::core::convert::Infallible,
458//! >(data, move |slot| {
459//! {
460//! struct __InitOk;
461//! {
462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//! }
464//! let __a_guard = unsafe {
465//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//! };
467//! let init = Bar::new(36);
468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//! let __b_guard = unsafe {
470//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//! };
472//! ::core::mem::forget(__b_guard);
473//! ::core::mem::forget(__a_guard);
474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//! let _ = || {
476//! unsafe {
477//! ::core::ptr::write(
478//! slot,
479//! Foo {
480//! a: ::core::panic!(),
481//! b: ::core::panic!(),
482//! },
483//! );
484//! };
485//! };
486//! }
487//! Ok(__InitOk)
488//! });
489//! let init = move |
490//! slot,
491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//! init(slot).map(|__InitOk| ())
493//! };
494//! let init = unsafe {
495//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//! };
497//! init
498//! };
499//! ```
500
501/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
502///
503/// See [`PinnedDrop`] for more information.
504#[doc(hidden)]
505#[macro_export]
506macro_rules! __pinned_drop {
507 (
508 @impl_sig($($impl_sig:tt)*),
509 @impl_body(
510 $(#[$($attr:tt)*])*
511 fn drop($($sig:tt)*) {
512 $($inner:tt)*
513 }
514 ),
515 ) => {
516 // SAFETY: TODO.
517 unsafe $($impl_sig)* {
518 // Inherit all attributes and the type/ident tokens for the signature.
519 $(#[$($attr)*])*
520 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
521 $($inner)*
522 }
523 }
524 }
525}
526
527/// This macro first parses the struct definition such that it separates pinned and not pinned
528/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
529#[doc(hidden)]
530#[macro_export]
531macro_rules! __pin_data {
532 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
533 (parse_input:
534 @args($($pinned_drop:ident)?),
535 @sig(
536 $(#[$($struct_attr:tt)*])*
537 $vis:vis struct $name:ident
538 $(where $($whr:tt)*)?
539 ),
540 @impl_generics($($impl_generics:tt)*),
541 @ty_generics($($ty_generics:tt)*),
542 @decl_generics($($decl_generics:tt)*),
543 @body({ $($fields:tt)* }),
544 ) => {
545 // We now use token munching to iterate through all of the fields. While doing this we
546 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
547 // wants these to be structurally pinned. The rest of the fields are the
548 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
549 // order to declare the struct.
550 //
551 // In this call we also put some explaining comments for the parameters.
552 $crate::__pin_data!(find_pinned_fields:
553 // Attributes on the struct itself, these will just be propagated to be put onto the
554 // struct definition.
555 @struct_attrs($(#[$($struct_attr)*])*),
556 // The visibility of the struct.
557 @vis($vis),
558 // The name of the struct.
559 @name($name),
560 // The 'impl generics', the generics that will need to be specified on the struct inside
561 // of an `impl<$ty_generics>` block.
562 @impl_generics($($impl_generics)*),
563 // The 'ty generics', the generics that will need to be specified on the impl blocks.
564 @ty_generics($($ty_generics)*),
565 // The 'decl generics', the generics that need to be specified on the struct
566 // definition.
567 @decl_generics($($decl_generics)*),
568 // The where clause of any impl block and the declaration.
569 @where($($($whr)*)?),
570 // The remaining fields tokens that need to be processed.
571 // We add a `,` at the end to ensure correct parsing.
572 @fields_munch($($fields)* ,),
573 // The pinned fields.
574 @pinned(),
575 // The not pinned fields.
576 @not_pinned(),
577 // All fields.
578 @fields(),
579 // The accumulator containing all attributes already parsed.
580 @accum(),
581 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
582 @is_pinned(),
583 // The proc-macro argument, this should be `PinnedDrop` or ``.
584 @pinned_drop($($pinned_drop)?),
585 );
586 };
587 (find_pinned_fields:
588 @struct_attrs($($struct_attrs:tt)*),
589 @vis($vis:vis),
590 @name($name:ident),
591 @impl_generics($($impl_generics:tt)*),
592 @ty_generics($($ty_generics:tt)*),
593 @decl_generics($($decl_generics:tt)*),
594 @where($($whr:tt)*),
595 // We found a PhantomPinned field, this should generally be pinned!
596 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
597 @pinned($($pinned:tt)*),
598 @not_pinned($($not_pinned:tt)*),
599 @fields($($fields:tt)*),
600 @accum($($accum:tt)*),
601 // This field is not pinned.
602 @is_pinned(),
603 @pinned_drop($($pinned_drop:ident)?),
604 ) => {
605 ::core::compile_error!(concat!(
606 "The field `",
607 stringify!($field),
608 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
609 ));
610 $crate::__pin_data!(find_pinned_fields:
611 @struct_attrs($($struct_attrs)*),
612 @vis($vis),
613 @name($name),
614 @impl_generics($($impl_generics)*),
615 @ty_generics($($ty_generics)*),
616 @decl_generics($($decl_generics)*),
617 @where($($whr)*),
618 @fields_munch($($rest)*),
619 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
620 @not_pinned($($not_pinned)*),
621 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
622 @accum(),
623 @is_pinned(),
624 @pinned_drop($($pinned_drop)?),
625 );
626 };
627 (find_pinned_fields:
628 @struct_attrs($($struct_attrs:tt)*),
629 @vis($vis:vis),
630 @name($name:ident),
631 @impl_generics($($impl_generics:tt)*),
632 @ty_generics($($ty_generics:tt)*),
633 @decl_generics($($decl_generics:tt)*),
634 @where($($whr:tt)*),
635 // We reached the field declaration.
636 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
637 @pinned($($pinned:tt)*),
638 @not_pinned($($not_pinned:tt)*),
639 @fields($($fields:tt)*),
640 @accum($($accum:tt)*),
641 // This field is pinned.
642 @is_pinned(yes),
643 @pinned_drop($($pinned_drop:ident)?),
644 ) => {
645 $crate::__pin_data!(find_pinned_fields:
646 @struct_attrs($($struct_attrs)*),
647 @vis($vis),
648 @name($name),
649 @impl_generics($($impl_generics)*),
650 @ty_generics($($ty_generics)*),
651 @decl_generics($($decl_generics)*),
652 @where($($whr)*),
653 @fields_munch($($rest)*),
654 @pinned($($pinned)* $($accum)* $field: $type,),
655 @not_pinned($($not_pinned)*),
656 @fields($($fields)* $($accum)* $field: $type,),
657 @accum(),
658 @is_pinned(),
659 @pinned_drop($($pinned_drop)?),
660 );
661 };
662 (find_pinned_fields:
663 @struct_attrs($($struct_attrs:tt)*),
664 @vis($vis:vis),
665 @name($name:ident),
666 @impl_generics($($impl_generics:tt)*),
667 @ty_generics($($ty_generics:tt)*),
668 @decl_generics($($decl_generics:tt)*),
669 @where($($whr:tt)*),
670 // We reached the field declaration.
671 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
672 @pinned($($pinned:tt)*),
673 @not_pinned($($not_pinned:tt)*),
674 @fields($($fields:tt)*),
675 @accum($($accum:tt)*),
676 // This field is not pinned.
677 @is_pinned(),
678 @pinned_drop($($pinned_drop:ident)?),
679 ) => {
680 $crate::__pin_data!(find_pinned_fields:
681 @struct_attrs($($struct_attrs)*),
682 @vis($vis),
683 @name($name),
684 @impl_generics($($impl_generics)*),
685 @ty_generics($($ty_generics)*),
686 @decl_generics($($decl_generics)*),
687 @where($($whr)*),
688 @fields_munch($($rest)*),
689 @pinned($($pinned)*),
690 @not_pinned($($not_pinned)* $($accum)* $field: $type,),
691 @fields($($fields)* $($accum)* $field: $type,),
692 @accum(),
693 @is_pinned(),
694 @pinned_drop($($pinned_drop)?),
695 );
696 };
697 (find_pinned_fields:
698 @struct_attrs($($struct_attrs:tt)*),
699 @vis($vis:vis),
700 @name($name:ident),
701 @impl_generics($($impl_generics:tt)*),
702 @ty_generics($($ty_generics:tt)*),
703 @decl_generics($($decl_generics:tt)*),
704 @where($($whr:tt)*),
705 // We found the `#[pin]` attr.
706 @fields_munch(#[pin] $($rest:tt)*),
707 @pinned($($pinned:tt)*),
708 @not_pinned($($not_pinned:tt)*),
709 @fields($($fields:tt)*),
710 @accum($($accum:tt)*),
711 @is_pinned($($is_pinned:ident)?),
712 @pinned_drop($($pinned_drop:ident)?),
713 ) => {
714 $crate::__pin_data!(find_pinned_fields:
715 @struct_attrs($($struct_attrs)*),
716 @vis($vis),
717 @name($name),
718 @impl_generics($($impl_generics)*),
719 @ty_generics($($ty_generics)*),
720 @decl_generics($($decl_generics)*),
721 @where($($whr)*),
722 @fields_munch($($rest)*),
723 // We do not include `#[pin]` in the list of attributes, since it is not actually an
724 // attribute that is defined somewhere.
725 @pinned($($pinned)*),
726 @not_pinned($($not_pinned)*),
727 @fields($($fields)*),
728 @accum($($accum)*),
729 // Set this to `yes`.
730 @is_pinned(yes),
731 @pinned_drop($($pinned_drop)?),
732 );
733 };
734 (find_pinned_fields:
735 @struct_attrs($($struct_attrs:tt)*),
736 @vis($vis:vis),
737 @name($name:ident),
738 @impl_generics($($impl_generics:tt)*),
739 @ty_generics($($ty_generics:tt)*),
740 @decl_generics($($decl_generics:tt)*),
741 @where($($whr:tt)*),
742 // We reached the field declaration with visibility, for simplicity we only munch the
743 // visibility and put it into `$accum`.
744 @fields_munch($fvis:vis $field:ident $($rest:tt)*),
745 @pinned($($pinned:tt)*),
746 @not_pinned($($not_pinned:tt)*),
747 @fields($($fields:tt)*),
748 @accum($($accum:tt)*),
749 @is_pinned($($is_pinned:ident)?),
750 @pinned_drop($($pinned_drop:ident)?),
751 ) => {
752 $crate::__pin_data!(find_pinned_fields:
753 @struct_attrs($($struct_attrs)*),
754 @vis($vis),
755 @name($name),
756 @impl_generics($($impl_generics)*),
757 @ty_generics($($ty_generics)*),
758 @decl_generics($($decl_generics)*),
759 @where($($whr)*),
760 @fields_munch($field $($rest)*),
761 @pinned($($pinned)*),
762 @not_pinned($($not_pinned)*),
763 @fields($($fields)*),
764 @accum($($accum)* $fvis),
765 @is_pinned($($is_pinned)?),
766 @pinned_drop($($pinned_drop)?),
767 );
768 };
769 (find_pinned_fields:
770 @struct_attrs($($struct_attrs:tt)*),
771 @vis($vis:vis),
772 @name($name:ident),
773 @impl_generics($($impl_generics:tt)*),
774 @ty_generics($($ty_generics:tt)*),
775 @decl_generics($($decl_generics:tt)*),
776 @where($($whr:tt)*),
777 // Some other attribute, just put it into `$accum`.
778 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
779 @pinned($($pinned:tt)*),
780 @not_pinned($($not_pinned:tt)*),
781 @fields($($fields:tt)*),
782 @accum($($accum:tt)*),
783 @is_pinned($($is_pinned:ident)?),
784 @pinned_drop($($pinned_drop:ident)?),
785 ) => {
786 $crate::__pin_data!(find_pinned_fields:
787 @struct_attrs($($struct_attrs)*),
788 @vis($vis),
789 @name($name),
790 @impl_generics($($impl_generics)*),
791 @ty_generics($($ty_generics)*),
792 @decl_generics($($decl_generics)*),
793 @where($($whr)*),
794 @fields_munch($($rest)*),
795 @pinned($($pinned)*),
796 @not_pinned($($not_pinned)*),
797 @fields($($fields)*),
798 @accum($($accum)* #[$($attr)*]),
799 @is_pinned($($is_pinned)?),
800 @pinned_drop($($pinned_drop)?),
801 );
802 };
803 (find_pinned_fields:
804 @struct_attrs($($struct_attrs:tt)*),
805 @vis($vis:vis),
806 @name($name:ident),
807 @impl_generics($($impl_generics:tt)*),
808 @ty_generics($($ty_generics:tt)*),
809 @decl_generics($($decl_generics:tt)*),
810 @where($($whr:tt)*),
811 // We reached the end of the fields, plus an optional additional comma, since we added one
812 // before and the user is also allowed to put a trailing comma.
813 @fields_munch($(,)?),
814 @pinned($($pinned:tt)*),
815 @not_pinned($($not_pinned:tt)*),
816 @fields($($fields:tt)*),
817 @accum(),
818 @is_pinned(),
819 @pinned_drop($($pinned_drop:ident)?),
820 ) => {
821 // Declare the struct with all fields in the correct order.
822 $($struct_attrs)*
823 $vis struct $name <$($decl_generics)*>
824 where $($whr)*
825 {
826 $($fields)*
827 }
828
829 // We put the rest into this const item, because it then will not be accessible to anything
830 // outside.
831 const _: () = {
832 // We declare this struct which will host all of the projection function for our type.
833 // it will be invariant over all generic parameters which are inherited from the
834 // struct.
835 $vis struct __ThePinData<$($impl_generics)*>
836 where $($whr)*
837 {
838 __phantom: ::core::marker::PhantomData<
839 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
840 >,
841 }
842
843 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
844 where $($whr)*
845 {
846 fn clone(&self) -> Self { *self }
847 }
848
849 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
850 where $($whr)*
851 {}
852
853 // Make all projection functions.
854 $crate::__pin_data!(make_pin_data:
855 @pin_data(__ThePinData),
856 @impl_generics($($impl_generics)*),
857 @ty_generics($($ty_generics)*),
858 @where($($whr)*),
859 @pinned($($pinned)*),
860 @not_pinned($($not_pinned)*),
861 );
862
863 // SAFETY: We have added the correct projection functions above to `__ThePinData` and
864 // we also use the least restrictive generics possible.
865 unsafe impl<$($impl_generics)*>
866 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
867 where $($whr)*
868 {
869 type PinData = __ThePinData<$($ty_generics)*>;
870
871 unsafe fn __pin_data() -> Self::PinData {
872 __ThePinData { __phantom: ::core::marker::PhantomData }
873 }
874 }
875
876 // SAFETY: TODO.
877 unsafe impl<$($impl_generics)*>
878 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
879 where $($whr)*
880 {
881 type Datee = $name<$($ty_generics)*>;
882 }
883
884 // This struct will be used for the unpin analysis. Since only structurally pinned
885 // fields are relevant whether the struct should implement `Unpin`.
886 #[allow(dead_code)]
887 struct __Unpin <'__pin, $($impl_generics)*>
888 where $($whr)*
889 {
890 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
891 __phantom: ::core::marker::PhantomData<
892 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
893 >,
894 // Only the pinned fields.
895 $($pinned)*
896 }
897
898 #[doc(hidden)]
899 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
900 where
901 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
902 $($whr)*
903 {}
904
905 // We need to disallow normal `Drop` implementation, the exact behavior depends on
906 // whether `PinnedDrop` was specified as the parameter.
907 $crate::__pin_data!(drop_prevention:
908 @name($name),
909 @impl_generics($($impl_generics)*),
910 @ty_generics($($ty_generics)*),
911 @where($($whr)*),
912 @pinned_drop($($pinned_drop)?),
913 );
914 };
915 };
916 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
917 (drop_prevention:
918 @name($name:ident),
919 @impl_generics($($impl_generics:tt)*),
920 @ty_generics($($ty_generics:tt)*),
921 @where($($whr:tt)*),
922 @pinned_drop(),
923 ) => {
924 // We prevent this by creating a trait that will be implemented for all types implementing
925 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
926 // if it also implements `Drop`
927 trait MustNotImplDrop {}
928 #[expect(drop_bounds)]
929 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
930 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
931 where $($whr)* {}
932 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
933 // They might implement `PinnedDrop` correctly for the struct, but forget to give
934 // `PinnedDrop` as the parameter to `#[pin_data]`.
935 #[expect(non_camel_case_types)]
936 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
937 impl<T: $crate::init::PinnedDrop>
938 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
939 impl<$($impl_generics)*>
940 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
941 where $($whr)* {}
942 };
943 // When `PinnedDrop` was specified we just implement `Drop` and delegate.
944 (drop_prevention:
945 @name($name:ident),
946 @impl_generics($($impl_generics:tt)*),
947 @ty_generics($($ty_generics:tt)*),
948 @where($($whr:tt)*),
949 @pinned_drop(PinnedDrop),
950 ) => {
951 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
952 where $($whr)*
953 {
954 fn drop(&mut self) {
955 // SAFETY: Since this is a destructor, `self` will not move after this function
956 // terminates, since it is inaccessible.
957 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
958 // SAFETY: Since this is a drop function, we can create this token to call the
959 // pinned destructor of this type.
960 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
961 $crate::init::PinnedDrop::drop(pinned, token);
962 }
963 }
964 };
965 // If some other parameter was specified, we emit a readable error.
966 (drop_prevention:
967 @name($name:ident),
968 @impl_generics($($impl_generics:tt)*),
969 @ty_generics($($ty_generics:tt)*),
970 @where($($whr:tt)*),
971 @pinned_drop($($rest:tt)*),
972 ) => {
973 compile_error!(
974 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
975 stringify!($($rest)*),
976 );
977 };
978 (make_pin_data:
979 @pin_data($pin_data:ident),
980 @impl_generics($($impl_generics:tt)*),
981 @ty_generics($($ty_generics:tt)*),
982 @where($($whr:tt)*),
983 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
984 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
985 ) => {
986 // For every field, we create a projection function according to its projection type. If a
987 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
988 // structurally pinned, then it can be initialized via `Init`.
989 //
990 // The functions are `unsafe` to prevent accidentally calling them.
991 #[allow(dead_code)]
992 #[expect(clippy::missing_safety_doc)]
993 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
994 where $($whr)*
995 {
996 $(
997 $(#[$($p_attr)*])*
998 $pvis unsafe fn $p_field<E>(
999 self,
1000 slot: *mut $p_type,
1001 init: impl $crate::init::PinInit<$p_type, E>,
1002 ) -> ::core::result::Result<(), E> {
1003 // SAFETY: TODO.
1004 unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
1005 }
1006 )*
1007 $(
1008 $(#[$($attr)*])*
1009 $fvis unsafe fn $field<E>(
1010 self,
1011 slot: *mut $type,
1012 init: impl $crate::init::Init<$type, E>,
1013 ) -> ::core::result::Result<(), E> {
1014 // SAFETY: TODO.
1015 unsafe { $crate::init::Init::__init(init, slot) }
1016 }
1017 )*
1018 }
1019 };
1020}
1021
1022/// The internal init macro. Do not call manually!
1023///
1024/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1025///
1026/// This macro has multiple internal call configurations, these are always the very first ident:
1027/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1028/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1029/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1030/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1031/// field has been initialized exactly once.
1032#[doc(hidden)]
1033#[macro_export]
1034macro_rules! __init_internal {
1035 (
1036 @this($($this:ident)?),
1037 @typ($t:path),
1038 @fields($($fields:tt)*),
1039 @error($err:ty),
1040 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1041 // case.
1042 @data($data:ident, $($use_data:ident)?),
1043 // `HasPinData` or `HasInitData`.
1044 @has_data($has_data:ident, $get_data:ident),
1045 // `pin_init_from_closure` or `init_from_closure`.
1046 @construct_closure($construct_closure:ident),
1047 @munch_fields(),
1048 ) => {
1049 $crate::__init_internal!(with_update_parsed:
1050 @this($($this)?),
1051 @typ($t),
1052 @fields($($fields)*),
1053 @error($err),
1054 @data($data, $($use_data)?),
1055 @has_data($has_data, $get_data),
1056 @construct_closure($construct_closure),
1057 @zeroed(), // Nothing means default behavior.
1058 )
1059 };
1060 (
1061 @this($($this:ident)?),
1062 @typ($t:path),
1063 @fields($($fields:tt)*),
1064 @error($err:ty),
1065 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1066 // case.
1067 @data($data:ident, $($use_data:ident)?),
1068 // `HasPinData` or `HasInitData`.
1069 @has_data($has_data:ident, $get_data:ident),
1070 // `pin_init_from_closure` or `init_from_closure`.
1071 @construct_closure($construct_closure:ident),
1072 @munch_fields(..Zeroable::zeroed()),
1073 ) => {
1074 $crate::__init_internal!(with_update_parsed:
1075 @this($($this)?),
1076 @typ($t),
1077 @fields($($fields)*),
1078 @error($err),
1079 @data($data, $($use_data)?),
1080 @has_data($has_data, $get_data),
1081 @construct_closure($construct_closure),
1082 @zeroed(()), // `()` means zero all fields not mentioned.
1083 )
1084 };
1085 (
1086 @this($($this:ident)?),
1087 @typ($t:path),
1088 @fields($($fields:tt)*),
1089 @error($err:ty),
1090 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1091 // case.
1092 @data($data:ident, $($use_data:ident)?),
1093 // `HasPinData` or `HasInitData`.
1094 @has_data($has_data:ident, $get_data:ident),
1095 // `pin_init_from_closure` or `init_from_closure`.
1096 @construct_closure($construct_closure:ident),
1097 @munch_fields($ignore:tt $($rest:tt)*),
1098 ) => {
1099 $crate::__init_internal!(
1100 @this($($this)?),
1101 @typ($t),
1102 @fields($($fields)*),
1103 @error($err),
1104 @data($data, $($use_data)?),
1105 @has_data($has_data, $get_data),
1106 @construct_closure($construct_closure),
1107 @munch_fields($($rest)*),
1108 )
1109 };
1110 (with_update_parsed:
1111 @this($($this:ident)?),
1112 @typ($t:path),
1113 @fields($($fields:tt)*),
1114 @error($err:ty),
1115 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1116 // case.
1117 @data($data:ident, $($use_data:ident)?),
1118 // `HasPinData` or `HasInitData`.
1119 @has_data($has_data:ident, $get_data:ident),
1120 // `pin_init_from_closure` or `init_from_closure`.
1121 @construct_closure($construct_closure:ident),
1122 @zeroed($($init_zeroed:expr)?),
1123 ) => {{
1124 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1125 // type and shadow it later when we insert the arbitrary user code. That way there will be
1126 // no possibility of returning without `unsafe`.
1127 struct __InitOk;
1128 // Get the data about fields from the supplied type.
1129 //
1130 // SAFETY: TODO.
1131 let data = unsafe {
1132 use $crate::init::__internal::$has_data;
1133 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1134 // information that is associated to already parsed fragments, so a path fragment
1135 // cannot be used in this position. Doing the retokenization results in valid rust
1136 // code.
1137 ::kernel::macros::paste!($t::$get_data())
1138 };
1139 // Ensure that `data` really is of type `$data` and help with type inference:
1140 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1141 data,
1142 move |slot| {
1143 {
1144 // Shadow the structure so it cannot be used to return early.
1145 struct __InitOk;
1146 // If `$init_zeroed` is present we should zero the slot now and not emit an
1147 // error when fields are missing (since they will be zeroed). We also have to
1148 // check that the type actually implements `Zeroable`.
1149 $({
1150 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1151 // Ensure that the struct is indeed `Zeroable`.
1152 assert_zeroable(slot);
1153 // SAFETY: The type implements `Zeroable` by the check above.
1154 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1155 $init_zeroed // This will be `()` if set.
1156 })?
1157 // Create the `this` so it can be referenced by the user inside of the
1158 // expressions creating the individual fields.
1159 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1160 // Initialize every field.
1161 $crate::__init_internal!(init_slot($($use_data)?):
1162 @data(data),
1163 @slot(slot),
1164 @guards(),
1165 @munch_fields($($fields)*,),
1166 );
1167 // We use unreachable code to ensure that all fields have been mentioned exactly
1168 // once, this struct initializer will still be type-checked and complain with a
1169 // very natural error message if a field is forgotten/mentioned more than once.
1170 #[allow(unreachable_code, clippy::diverging_sub_expression)]
1171 let _ = || {
1172 $crate::__init_internal!(make_initializer:
1173 @slot(slot),
1174 @type_name($t),
1175 @munch_fields($($fields)*,),
1176 @acc(),
1177 );
1178 };
1179 }
1180 Ok(__InitOk)
1181 }
1182 );
1183 let init = move |slot| -> ::core::result::Result<(), $err> {
1184 init(slot).map(|__InitOk| ())
1185 };
1186 // SAFETY: TODO.
1187 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1188 init
1189 }};
1190 (init_slot($($use_data:ident)?):
1191 @data($data:ident),
1192 @slot($slot:ident),
1193 @guards($($guards:ident,)*),
1194 @munch_fields($(..Zeroable::zeroed())? $(,)?),
1195 ) => {
1196 // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1197 // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1198 $(::core::mem::forget($guards);)*
1199 };
1200 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1201 @data($data:ident),
1202 @slot($slot:ident),
1203 @guards($($guards:ident,)*),
1204 // In-place initialization syntax.
1205 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1206 ) => {
1207 let init = $val;
1208 // Call the initializer.
1209 //
1210 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1211 // return when an error/panic occurs.
1212 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1213 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1214 // Create the drop guard:
1215 //
1216 // We rely on macro hygiene to make it impossible for users to access this local variable.
1217 // We use `paste!` to create new hygiene for `$field`.
1218 ::kernel::macros::paste! {
1219 // SAFETY: We forget the guard later when initialization has succeeded.
1220 let [< __ $field _guard >] = unsafe {
1221 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1222 };
1223
1224 $crate::__init_internal!(init_slot($use_data):
1225 @data($data),
1226 @slot($slot),
1227 @guards([< __ $field _guard >], $($guards,)*),
1228 @munch_fields($($rest)*),
1229 );
1230 }
1231 };
1232 (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1233 @data($data:ident),
1234 @slot($slot:ident),
1235 @guards($($guards:ident,)*),
1236 // In-place initialization syntax.
1237 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1238 ) => {
1239 let init = $val;
1240 // Call the initializer.
1241 //
1242 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1243 // return when an error/panic occurs.
1244 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1245 // Create the drop guard:
1246 //
1247 // We rely on macro hygiene to make it impossible for users to access this local variable.
1248 // We use `paste!` to create new hygiene for `$field`.
1249 ::kernel::macros::paste! {
1250 // SAFETY: We forget the guard later when initialization has succeeded.
1251 let [< __ $field _guard >] = unsafe {
1252 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1253 };
1254
1255 $crate::__init_internal!(init_slot():
1256 @data($data),
1257 @slot($slot),
1258 @guards([< __ $field _guard >], $($guards,)*),
1259 @munch_fields($($rest)*),
1260 );
1261 }
1262 };
1263 (init_slot($($use_data:ident)?):
1264 @data($data:ident),
1265 @slot($slot:ident),
1266 @guards($($guards:ident,)*),
1267 // Init by-value.
1268 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1269 ) => {
1270 {
1271 $(let $field = $val;)?
1272 // Initialize the field.
1273 //
1274 // SAFETY: The memory at `slot` is uninitialized.
1275 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1276 }
1277 // Create the drop guard:
1278 //
1279 // We rely on macro hygiene to make it impossible for users to access this local variable.
1280 // We use `paste!` to create new hygiene for `$field`.
1281 ::kernel::macros::paste! {
1282 // SAFETY: We forget the guard later when initialization has succeeded.
1283 let [< __ $field _guard >] = unsafe {
1284 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1285 };
1286
1287 $crate::__init_internal!(init_slot($($use_data)?):
1288 @data($data),
1289 @slot($slot),
1290 @guards([< __ $field _guard >], $($guards,)*),
1291 @munch_fields($($rest)*),
1292 );
1293 }
1294 };
1295 (make_initializer:
1296 @slot($slot:ident),
1297 @type_name($t:path),
1298 @munch_fields(..Zeroable::zeroed() $(,)?),
1299 @acc($($acc:tt)*),
1300 ) => {
1301 // Endpoint, nothing more to munch, create the initializer. Since the users specified
1302 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1303 // not been overwritten are thus zero and initialized. We still check that all fields are
1304 // actually accessible by using the struct update syntax ourselves.
1305 // We are inside of a closure that is never executed and thus we can abuse `slot` to
1306 // get the correct type inference here:
1307 #[allow(unused_assignments)]
1308 unsafe {
1309 let mut zeroed = ::core::mem::zeroed();
1310 // We have to use type inference here to make zeroed have the correct type. This does
1311 // not get executed, so it has no effect.
1312 ::core::ptr::write($slot, zeroed);
1313 zeroed = ::core::mem::zeroed();
1314 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1315 // information that is associated to already parsed fragments, so a path fragment
1316 // cannot be used in this position. Doing the retokenization results in valid rust
1317 // code.
1318 ::kernel::macros::paste!(
1319 ::core::ptr::write($slot, $t {
1320 $($acc)*
1321 ..zeroed
1322 });
1323 );
1324 }
1325 };
1326 (make_initializer:
1327 @slot($slot:ident),
1328 @type_name($t:path),
1329 @munch_fields($(,)?),
1330 @acc($($acc:tt)*),
1331 ) => {
1332 // Endpoint, nothing more to munch, create the initializer.
1333 // Since we are in the closure that is never called, this will never get executed.
1334 // We abuse `slot` to get the correct type inference here:
1335 //
1336 // SAFETY: TODO.
1337 unsafe {
1338 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1339 // information that is associated to already parsed fragments, so a path fragment
1340 // cannot be used in this position. Doing the retokenization results in valid rust
1341 // code.
1342 ::kernel::macros::paste!(
1343 ::core::ptr::write($slot, $t {
1344 $($acc)*
1345 });
1346 );
1347 }
1348 };
1349 (make_initializer:
1350 @slot($slot:ident),
1351 @type_name($t:path),
1352 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1353 @acc($($acc:tt)*),
1354 ) => {
1355 $crate::__init_internal!(make_initializer:
1356 @slot($slot),
1357 @type_name($t),
1358 @munch_fields($($rest)*),
1359 @acc($($acc)* $field: ::core::panic!(),),
1360 );
1361 };
1362 (make_initializer:
1363 @slot($slot:ident),
1364 @type_name($t:path),
1365 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1366 @acc($($acc:tt)*),
1367 ) => {
1368 $crate::__init_internal!(make_initializer:
1369 @slot($slot),
1370 @type_name($t),
1371 @munch_fields($($rest)*),
1372 @acc($($acc)* $field: ::core::panic!(),),
1373 );
1374 };
1375}
1376
1377#[doc(hidden)]
1378#[macro_export]
1379macro_rules! __derive_zeroable {
1380 (parse_input:
1381 @sig(
1382 $(#[$($struct_attr:tt)*])*
1383 $vis:vis struct $name:ident
1384 $(where $($whr:tt)*)?
1385 ),
1386 @impl_generics($($impl_generics:tt)*),
1387 @ty_generics($($ty_generics:tt)*),
1388 @body({
1389 $(
1390 $(#[$($field_attr:tt)*])*
1391 $field:ident : $field_ty:ty
1392 ),* $(,)?
1393 }),
1394 ) => {
1395 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1396 #[automatically_derived]
1397 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1398 where
1399 $($($whr)*)?
1400 {}
1401 const _: () = {
1402 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1403 fn ensure_zeroable<$($impl_generics)*>()
1404 where $($($whr)*)?
1405 {
1406 $(assert_zeroable::<$field_ty>();)*
1407 }
1408 };
1409 };
1410}