Loading...
Note: File does not exist in v6.2.
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
5//! macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! # Macro expansion example
15//!
16//! This section is intended for readers trying to understand the macros in this module and the
17//! `pin_init!` macros from `init.rs`.
18//!
19//! We will look at the following example:
20//!
21//! ```rust,ignore
22//! # use kernel::init::*;
23//! # use core::pin::Pin;
24//! #[pin_data]
25//! #[repr(C)]
26//! struct Bar<T> {
27//! #[pin]
28//! t: T,
29//! pub x: usize,
30//! }
31//!
32//! impl<T> Bar<T> {
33//! fn new(t: T) -> impl PinInit<Self> {
34//! pin_init!(Self { t, x: 0 })
35//! }
36//! }
37//!
38//! #[pin_data(PinnedDrop)]
39//! struct Foo {
40//! a: usize,
41//! #[pin]
42//! b: Bar<u32>,
43//! }
44//!
45//! #[pinned_drop]
46//! impl PinnedDrop for Foo {
47//! fn drop(self: Pin<&mut Self>) {
48//! pr_info!("{self:p} is getting dropped.");
49//! }
50//! }
51//!
52//! let a = 42;
53//! let initializer = pin_init!(Foo {
54//! a,
55//! b <- Bar::new(36),
56//! });
57//! ```
58//!
59//! This example includes the most common and important features of the pin-init API.
60//!
61//! Below you can find individual section about the different macro invocations. Here are some
62//! general things we need to take into account when designing macros:
63//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
64//! this ensures that the correct item is used, since users could define their own `mod core {}`
65//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
66//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
67//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
68//! `unsafe` operations without an associated `unsafe` block.
69//!
70//! ## `#[pin_data]` on `Bar`
71//!
72//! This macro is used to specify which fields are structurally pinned and which fields are not. It
73//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
74//!
75//! Here is the definition of `Bar` from our example:
76//!
77//! ```rust,ignore
78//! # use kernel::init::*;
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//! #[pin]
83//! t: T,
84//! pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//! t: T,
95//! pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//! // since we need to implement access functions for each field and thus need to know its
102//! // type.
103//! struct __ThePinData<T> {
104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//! }
106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//! // `self` by value.
108//! impl<T> ::core::clone::Clone for __ThePinData<T> {
109//! fn clone(&self) -> Self {
110//! *self
111//! }
112//! }
113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//! // For every field of `Bar`, the pin-data struct will define a function with the same name
115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//! #[allow(dead_code)]
119//! impl<T> __ThePinData<T> {
120//! unsafe fn t<E>(
121//! self,
122//! slot: *mut T,
123//! // Since `t` is `#[pin]`, this is `PinInit`.
124//! init: impl ::kernel::init::PinInit<T, E>,
125//! ) -> ::core::result::Result<(), E> {
126//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
127//! }
128//! pub unsafe fn x<E>(
129//! self,
130//! slot: *mut usize,
131//! // Since `x` is not `#[pin]`, this is `Init`.
132//! init: impl ::kernel::init::Init<usize, E>,
133//! ) -> ::core::result::Result<(), E> {
134//! unsafe { ::kernel::init::Init::__init(init, slot) }
135//! }
136//! }
137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//! // that we constructed above.
139//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
140//! type PinData = __ThePinData<T>;
141//! unsafe fn __pin_data() -> Self::PinData {
142//! __ThePinData {
143//! __phantom: ::core::marker::PhantomData,
144//! }
145//! }
146//! }
147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//! // struct. This is important to ensure that no user can implement a rouge `__pin_data`
149//! // function without using `unsafe`.
150//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
151//! type Datee = Bar<T>;
152//! }
153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//! // for two reasons:
158//! // - `__phantom`: every generic must be used, since we cannot really know which generics
159//! // are used, we declere all and then use everything here once.
160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//! // over it. The lifetime is needed to work around the limitation that trait bounds must
162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//! // into accepting these bounds regardless.
165//! #[allow(dead_code)]
166//! struct __Unpin<'__pin, T> {
167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//! // Our only `#[pin]` field is `t`.
170//! t: T,
171//! }
172//! #[doc(hidden)]
173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//! where
175//! __Unpin<'__pin, T>: ::core::marker::Unpin,
176//! {}
177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//! // UB with only safe code, so we disallow this by giving a trait implementation error using
180//! // a direct impl and a blanket implementation.
181//! trait MustNotImplDrop {}
182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//! // (normally people want to know if a type has any kind of drop glue at all, here we want
184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//! #[allow(drop_bounds)]
186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//! impl<T> MustNotImplDrop for Bar<T> {}
188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//! #[allow(non_camel_case_types)]
192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//! impl<
194//! T: ::kernel::init::PinnedDrop,
195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//! fn new(t: T) -> impl PinInit<Self> {
210//! pin_init!(Self { t, x: 0 })
211//! }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//! fn new(t: T) -> impl PinInit<Self> {
220//! {
221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//! // return type and shadow it later when we insert the arbitrary user code. That way
223//! // there will be no possibility of returning without `unsafe`.
224//! struct __InitOk;
225//! // Get the data about fields from the supplied type.
226//! // - the function is unsafe, hence the unsafe block
227//! // - we `use` the `HasPinData` trait in the block, it is only available in that
228//! // scope.
229//! let data = unsafe {
230//! use ::kernel::init::__internal::HasPinData;
231//! Self::__pin_data()
232//! };
233//! // Ensure that `data` really is of type `PinData` and help with type inference:
234//! let init = ::kernel::init::__internal::PinData::make_closure::<
235//! _,
236//! __InitOk,
237//! ::core::convert::Infallible,
238//! >(data, move |slot| {
239//! {
240//! // Shadow the structure so it cannot be used to return early. If a user
241//! // tries to write `return Ok(__InitOk)`, then they get a type error,
242//! // since that will refer to this struct instead of the one defined
243//! // above.
244//! struct __InitOk;
245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//! {
247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//! }
249//! // Since initialization could fail later (not in this case, since the
250//! // error type is `Infallible`) we will need to drop this field if there
251//! // is an error later. This `DropGuard` will drop the field when it gets
252//! // dropped and has not yet been forgotten.
253//! let t = unsafe {
254//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//! };
256//! // Expansion of `x: 0,`:
257//! // Since this can be an arbitrary expression we cannot place it inside
258//! // of the `unsafe` block, so we bind it here.
259//! {
260//! let x = 0;
261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//! }
263//! // We again create a `DropGuard`.
264//! let x = unsafe {
265//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//! };
267//! // Since initialization has successfully completed, we can now forget
268//! // the guards. This is not `mem::forget`, since we only have
269//! // `&DropGuard`.
270//! ::core::mem::forget(x);
271//! ::core::mem::forget(t);
272//! // Here we use the type checker to ensure that every field has been
273//! // initialized exactly once, since this is `if false` it will never get
274//! // executed, but still type-checked.
275//! // Additionally we abuse `slot` to automatically infer the correct type
276//! // for the struct. This is also another check that every field is
277//! // accessible from this scope.
278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//! let _ = || {
280//! unsafe {
281//! ::core::ptr::write(
282//! slot,
283//! Self {
284//! // We only care about typecheck finding every field
285//! // here, the expression does not matter, just conjure
286//! // one using `panic!()`:
287//! t: ::core::panic!(),
288//! x: ::core::panic!(),
289//! },
290//! );
291//! };
292//! };
293//! }
294//! // We leave the scope above and gain access to the previously shadowed
295//! // `__InitOk` that we need to return.
296//! Ok(__InitOk)
297//! });
298//! // Change the return type from `__InitOk` to `()`.
299//! let init = move |
300//! slot,
301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//! init(slot).map(|__InitOk| ())
303//! };
304//! // Construct the initializer.
305//! let init = unsafe {
306//! ::kernel::init::pin_init_from_closure::<
307//! _,
308//! ::core::convert::Infallible,
309//! >(init)
310//! };
311//! init
312//! }
313//! }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//! a: usize,
326//! #[pin]
327//! b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//! a: usize,
336//! b: Bar<u32>,
337//! }
338//! const _: () = {
339//! struct __ThePinData {
340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//! }
342//! impl ::core::clone::Clone for __ThePinData {
343//! fn clone(&self) -> Self {
344//! *self
345//! }
346//! }
347//! impl ::core::marker::Copy for __ThePinData {}
348//! #[allow(dead_code)]
349//! impl __ThePinData {
350//! unsafe fn b<E>(
351//! self,
352//! slot: *mut Bar<u32>,
353//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
354//! ) -> ::core::result::Result<(), E> {
355//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
356//! }
357//! unsafe fn a<E>(
358//! self,
359//! slot: *mut usize,
360//! init: impl ::kernel::init::Init<usize, E>,
361//! ) -> ::core::result::Result<(), E> {
362//! unsafe { ::kernel::init::Init::__init(init, slot) }
363//! }
364//! }
365//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
366//! type PinData = __ThePinData;
367//! unsafe fn __pin_data() -> Self::PinData {
368//! __ThePinData {
369//! __phantom: ::core::marker::PhantomData,
370//! }
371//! }
372//! }
373//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
374//! type Datee = Foo;
375//! }
376//! #[allow(dead_code)]
377//! struct __Unpin<'__pin> {
378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//! b: Bar<u32>,
381//! }
382//! #[doc(hidden)]
383//! impl<'__pin> ::core::marker::Unpin for Foo
384//! where
385//! __Unpin<'__pin>: ::core::marker::Unpin,
386//! {}
387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//! impl ::core::ops::Drop for Foo {
391//! fn drop(&mut self) {
392//! // Since we are getting dropped, no one else has a reference to `self` and thus we
393//! // can assume that we never move.
394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//! // Create the unsafe token that proves that we are inside of a destructor, this
396//! // type is only allowed to be created in a destructor.
397//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
398//! ::kernel::init::PinnedDrop::drop(pinned, token);
399//! }
400//! }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//! fn drop(self: Pin<&mut Self>) {
415//! pr_info!("{self:p} is getting dropped.");
416//! }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::kernel::init::PinnedDrop for Foo {
425//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
426//! pr_info!("{self:p} is getting dropped.");
427//! }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//! a,
440//! b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//! struct __InitOk;
450//! let data = unsafe {
451//! use ::kernel::init::__internal::HasPinData;
452//! Foo::__pin_data()
453//! };
454//! let init = ::kernel::init::__internal::PinData::make_closure::<
455//! _,
456//! __InitOk,
457//! ::core::convert::Infallible,
458//! >(data, move |slot| {
459//! {
460//! struct __InitOk;
461//! {
462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//! }
464//! let a = unsafe {
465//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//! };
467//! let init = Bar::new(36);
468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//! let b = unsafe {
470//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//! };
472//! ::core::mem::forget(b);
473//! ::core::mem::forget(a);
474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//! let _ = || {
476//! unsafe {
477//! ::core::ptr::write(
478//! slot,
479//! Foo {
480//! a: ::core::panic!(),
481//! b: ::core::panic!(),
482//! },
483//! );
484//! };
485//! };
486//! }
487//! Ok(__InitOk)
488//! });
489//! let init = move |
490//! slot,
491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//! init(slot).map(|__InitOk| ())
493//! };
494//! let init = unsafe {
495//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//! };
497//! init
498//! };
499//! ```
500
501/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
502///
503/// See [`PinnedDrop`] for more information.
504#[doc(hidden)]
505#[macro_export]
506macro_rules! __pinned_drop {
507 (
508 @impl_sig($($impl_sig:tt)*),
509 @impl_body(
510 $(#[$($attr:tt)*])*
511 fn drop($($sig:tt)*) {
512 $($inner:tt)*
513 }
514 ),
515 ) => {
516 unsafe $($impl_sig)* {
517 // Inherit all attributes and the type/ident tokens for the signature.
518 $(#[$($attr)*])*
519 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
520 $($inner)*
521 }
522 }
523 }
524}
525
526/// This macro first parses the struct definition such that it separates pinned and not pinned
527/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
528#[doc(hidden)]
529#[macro_export]
530macro_rules! __pin_data {
531 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
532 (parse_input:
533 @args($($pinned_drop:ident)?),
534 @sig(
535 $(#[$($struct_attr:tt)*])*
536 $vis:vis struct $name:ident
537 $(where $($whr:tt)*)?
538 ),
539 @impl_generics($($impl_generics:tt)*),
540 @ty_generics($($ty_generics:tt)*),
541 @body({ $($fields:tt)* }),
542 ) => {
543 // We now use token munching to iterate through all of the fields. While doing this we
544 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
545 // wants these to be structurally pinned. The rest of the fields are the
546 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
547 // order to declare the struct.
548 //
549 // In this call we also put some explaining comments for the parameters.
550 $crate::__pin_data!(find_pinned_fields:
551 // Attributes on the struct itself, these will just be propagated to be put onto the
552 // struct definition.
553 @struct_attrs($(#[$($struct_attr)*])*),
554 // The visibility of the struct.
555 @vis($vis),
556 // The name of the struct.
557 @name($name),
558 // The 'impl generics', the generics that will need to be specified on the struct inside
559 // of an `impl<$ty_generics>` block.
560 @impl_generics($($impl_generics)*),
561 // The 'ty generics', the generics that will need to be specified on the impl blocks.
562 @ty_generics($($ty_generics)*),
563 // The where clause of any impl block and the declaration.
564 @where($($($whr)*)?),
565 // The remaining fields tokens that need to be processed.
566 // We add a `,` at the end to ensure correct parsing.
567 @fields_munch($($fields)* ,),
568 // The pinned fields.
569 @pinned(),
570 // The not pinned fields.
571 @not_pinned(),
572 // All fields.
573 @fields(),
574 // The accumulator containing all attributes already parsed.
575 @accum(),
576 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
577 @is_pinned(),
578 // The proc-macro argument, this should be `PinnedDrop` or ``.
579 @pinned_drop($($pinned_drop)?),
580 );
581 };
582 (find_pinned_fields:
583 @struct_attrs($($struct_attrs:tt)*),
584 @vis($vis:vis),
585 @name($name:ident),
586 @impl_generics($($impl_generics:tt)*),
587 @ty_generics($($ty_generics:tt)*),
588 @where($($whr:tt)*),
589 // We found a PhantomPinned field, this should generally be pinned!
590 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
591 @pinned($($pinned:tt)*),
592 @not_pinned($($not_pinned:tt)*),
593 @fields($($fields:tt)*),
594 @accum($($accum:tt)*),
595 // This field is not pinned.
596 @is_pinned(),
597 @pinned_drop($($pinned_drop:ident)?),
598 ) => {
599 ::core::compile_error!(concat!(
600 "The field `",
601 stringify!($field),
602 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
603 ));
604 $crate::__pin_data!(find_pinned_fields:
605 @struct_attrs($($struct_attrs)*),
606 @vis($vis),
607 @name($name),
608 @impl_generics($($impl_generics)*),
609 @ty_generics($($ty_generics)*),
610 @where($($whr)*),
611 @fields_munch($($rest)*),
612 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
613 @not_pinned($($not_pinned)*),
614 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
615 @accum(),
616 @is_pinned(),
617 @pinned_drop($($pinned_drop)?),
618 );
619 };
620 (find_pinned_fields:
621 @struct_attrs($($struct_attrs:tt)*),
622 @vis($vis:vis),
623 @name($name:ident),
624 @impl_generics($($impl_generics:tt)*),
625 @ty_generics($($ty_generics:tt)*),
626 @where($($whr:tt)*),
627 // We reached the field declaration.
628 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
629 @pinned($($pinned:tt)*),
630 @not_pinned($($not_pinned:tt)*),
631 @fields($($fields:tt)*),
632 @accum($($accum:tt)*),
633 // This field is pinned.
634 @is_pinned(yes),
635 @pinned_drop($($pinned_drop:ident)?),
636 ) => {
637 $crate::__pin_data!(find_pinned_fields:
638 @struct_attrs($($struct_attrs)*),
639 @vis($vis),
640 @name($name),
641 @impl_generics($($impl_generics)*),
642 @ty_generics($($ty_generics)*),
643 @where($($whr)*),
644 @fields_munch($($rest)*),
645 @pinned($($pinned)* $($accum)* $field: $type,),
646 @not_pinned($($not_pinned)*),
647 @fields($($fields)* $($accum)* $field: $type,),
648 @accum(),
649 @is_pinned(),
650 @pinned_drop($($pinned_drop)?),
651 );
652 };
653 (find_pinned_fields:
654 @struct_attrs($($struct_attrs:tt)*),
655 @vis($vis:vis),
656 @name($name:ident),
657 @impl_generics($($impl_generics:tt)*),
658 @ty_generics($($ty_generics:tt)*),
659 @where($($whr:tt)*),
660 // We reached the field declaration.
661 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
662 @pinned($($pinned:tt)*),
663 @not_pinned($($not_pinned:tt)*),
664 @fields($($fields:tt)*),
665 @accum($($accum:tt)*),
666 // This field is not pinned.
667 @is_pinned(),
668 @pinned_drop($($pinned_drop:ident)?),
669 ) => {
670 $crate::__pin_data!(find_pinned_fields:
671 @struct_attrs($($struct_attrs)*),
672 @vis($vis),
673 @name($name),
674 @impl_generics($($impl_generics)*),
675 @ty_generics($($ty_generics)*),
676 @where($($whr)*),
677 @fields_munch($($rest)*),
678 @pinned($($pinned)*),
679 @not_pinned($($not_pinned)* $($accum)* $field: $type,),
680 @fields($($fields)* $($accum)* $field: $type,),
681 @accum(),
682 @is_pinned(),
683 @pinned_drop($($pinned_drop)?),
684 );
685 };
686 (find_pinned_fields:
687 @struct_attrs($($struct_attrs:tt)*),
688 @vis($vis:vis),
689 @name($name:ident),
690 @impl_generics($($impl_generics:tt)*),
691 @ty_generics($($ty_generics:tt)*),
692 @where($($whr:tt)*),
693 // We found the `#[pin]` attr.
694 @fields_munch(#[pin] $($rest:tt)*),
695 @pinned($($pinned:tt)*),
696 @not_pinned($($not_pinned:tt)*),
697 @fields($($fields:tt)*),
698 @accum($($accum:tt)*),
699 @is_pinned($($is_pinned:ident)?),
700 @pinned_drop($($pinned_drop:ident)?),
701 ) => {
702 $crate::__pin_data!(find_pinned_fields:
703 @struct_attrs($($struct_attrs)*),
704 @vis($vis),
705 @name($name),
706 @impl_generics($($impl_generics)*),
707 @ty_generics($($ty_generics)*),
708 @where($($whr)*),
709 @fields_munch($($rest)*),
710 // We do not include `#[pin]` in the list of attributes, since it is not actually an
711 // attribute that is defined somewhere.
712 @pinned($($pinned)*),
713 @not_pinned($($not_pinned)*),
714 @fields($($fields)*),
715 @accum($($accum)*),
716 // Set this to `yes`.
717 @is_pinned(yes),
718 @pinned_drop($($pinned_drop)?),
719 );
720 };
721 (find_pinned_fields:
722 @struct_attrs($($struct_attrs:tt)*),
723 @vis($vis:vis),
724 @name($name:ident),
725 @impl_generics($($impl_generics:tt)*),
726 @ty_generics($($ty_generics:tt)*),
727 @where($($whr:tt)*),
728 // We reached the field declaration with visibility, for simplicity we only munch the
729 // visibility and put it into `$accum`.
730 @fields_munch($fvis:vis $field:ident $($rest:tt)*),
731 @pinned($($pinned:tt)*),
732 @not_pinned($($not_pinned:tt)*),
733 @fields($($fields:tt)*),
734 @accum($($accum:tt)*),
735 @is_pinned($($is_pinned:ident)?),
736 @pinned_drop($($pinned_drop:ident)?),
737 ) => {
738 $crate::__pin_data!(find_pinned_fields:
739 @struct_attrs($($struct_attrs)*),
740 @vis($vis),
741 @name($name),
742 @impl_generics($($impl_generics)*),
743 @ty_generics($($ty_generics)*),
744 @where($($whr)*),
745 @fields_munch($field $($rest)*),
746 @pinned($($pinned)*),
747 @not_pinned($($not_pinned)*),
748 @fields($($fields)*),
749 @accum($($accum)* $fvis),
750 @is_pinned($($is_pinned)?),
751 @pinned_drop($($pinned_drop)?),
752 );
753 };
754 (find_pinned_fields:
755 @struct_attrs($($struct_attrs:tt)*),
756 @vis($vis:vis),
757 @name($name:ident),
758 @impl_generics($($impl_generics:tt)*),
759 @ty_generics($($ty_generics:tt)*),
760 @where($($whr:tt)*),
761 // Some other attribute, just put it into `$accum`.
762 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
763 @pinned($($pinned:tt)*),
764 @not_pinned($($not_pinned:tt)*),
765 @fields($($fields:tt)*),
766 @accum($($accum:tt)*),
767 @is_pinned($($is_pinned:ident)?),
768 @pinned_drop($($pinned_drop:ident)?),
769 ) => {
770 $crate::__pin_data!(find_pinned_fields:
771 @struct_attrs($($struct_attrs)*),
772 @vis($vis),
773 @name($name),
774 @impl_generics($($impl_generics)*),
775 @ty_generics($($ty_generics)*),
776 @where($($whr)*),
777 @fields_munch($($rest)*),
778 @pinned($($pinned)*),
779 @not_pinned($($not_pinned)*),
780 @fields($($fields)*),
781 @accum($($accum)* #[$($attr)*]),
782 @is_pinned($($is_pinned)?),
783 @pinned_drop($($pinned_drop)?),
784 );
785 };
786 (find_pinned_fields:
787 @struct_attrs($($struct_attrs:tt)*),
788 @vis($vis:vis),
789 @name($name:ident),
790 @impl_generics($($impl_generics:tt)*),
791 @ty_generics($($ty_generics:tt)*),
792 @where($($whr:tt)*),
793 // We reached the end of the fields, plus an optional additional comma, since we added one
794 // before and the user is also allowed to put a trailing comma.
795 @fields_munch($(,)?),
796 @pinned($($pinned:tt)*),
797 @not_pinned($($not_pinned:tt)*),
798 @fields($($fields:tt)*),
799 @accum(),
800 @is_pinned(),
801 @pinned_drop($($pinned_drop:ident)?),
802 ) => {
803 // Declare the struct with all fields in the correct order.
804 $($struct_attrs)*
805 $vis struct $name <$($impl_generics)*>
806 where $($whr)*
807 {
808 $($fields)*
809 }
810
811 // We put the rest into this const item, because it then will not be accessible to anything
812 // outside.
813 const _: () = {
814 // We declare this struct which will host all of the projection function for our type.
815 // it will be invariant over all generic parameters which are inherited from the
816 // struct.
817 $vis struct __ThePinData<$($impl_generics)*>
818 where $($whr)*
819 {
820 __phantom: ::core::marker::PhantomData<
821 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
822 >,
823 }
824
825 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
826 where $($whr)*
827 {
828 fn clone(&self) -> Self { *self }
829 }
830
831 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
832 where $($whr)*
833 {}
834
835 // Make all projection functions.
836 $crate::__pin_data!(make_pin_data:
837 @pin_data(__ThePinData),
838 @impl_generics($($impl_generics)*),
839 @ty_generics($($ty_generics)*),
840 @where($($whr)*),
841 @pinned($($pinned)*),
842 @not_pinned($($not_pinned)*),
843 );
844
845 // SAFETY: We have added the correct projection functions above to `__ThePinData` and
846 // we also use the least restrictive generics possible.
847 unsafe impl<$($impl_generics)*>
848 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
849 where $($whr)*
850 {
851 type PinData = __ThePinData<$($ty_generics)*>;
852
853 unsafe fn __pin_data() -> Self::PinData {
854 __ThePinData { __phantom: ::core::marker::PhantomData }
855 }
856 }
857
858 unsafe impl<$($impl_generics)*>
859 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
860 where $($whr)*
861 {
862 type Datee = $name<$($ty_generics)*>;
863 }
864
865 // This struct will be used for the unpin analysis. Since only structurally pinned
866 // fields are relevant whether the struct should implement `Unpin`.
867 #[allow(dead_code)]
868 struct __Unpin <'__pin, $($impl_generics)*>
869 where $($whr)*
870 {
871 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
872 __phantom: ::core::marker::PhantomData<
873 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
874 >,
875 // Only the pinned fields.
876 $($pinned)*
877 }
878
879 #[doc(hidden)]
880 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
881 where
882 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
883 $($whr)*
884 {}
885
886 // We need to disallow normal `Drop` implementation, the exact behavior depends on
887 // whether `PinnedDrop` was specified as the parameter.
888 $crate::__pin_data!(drop_prevention:
889 @name($name),
890 @impl_generics($($impl_generics)*),
891 @ty_generics($($ty_generics)*),
892 @where($($whr)*),
893 @pinned_drop($($pinned_drop)?),
894 );
895 };
896 };
897 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
898 (drop_prevention:
899 @name($name:ident),
900 @impl_generics($($impl_generics:tt)*),
901 @ty_generics($($ty_generics:tt)*),
902 @where($($whr:tt)*),
903 @pinned_drop(),
904 ) => {
905 // We prevent this by creating a trait that will be implemented for all types implementing
906 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
907 // if it also implements `Drop`
908 trait MustNotImplDrop {}
909 #[allow(drop_bounds)]
910 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
911 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
912 where $($whr)* {}
913 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
914 // They might implement `PinnedDrop` correctly for the struct, but forget to give
915 // `PinnedDrop` as the parameter to `#[pin_data]`.
916 #[allow(non_camel_case_types)]
917 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
918 impl<T: $crate::init::PinnedDrop>
919 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
920 impl<$($impl_generics)*>
921 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
922 where $($whr)* {}
923 };
924 // When `PinnedDrop` was specified we just implement `Drop` and delegate.
925 (drop_prevention:
926 @name($name:ident),
927 @impl_generics($($impl_generics:tt)*),
928 @ty_generics($($ty_generics:tt)*),
929 @where($($whr:tt)*),
930 @pinned_drop(PinnedDrop),
931 ) => {
932 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
933 where $($whr)*
934 {
935 fn drop(&mut self) {
936 // SAFETY: Since this is a destructor, `self` will not move after this function
937 // terminates, since it is inaccessible.
938 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
939 // SAFETY: Since this is a drop function, we can create this token to call the
940 // pinned destructor of this type.
941 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
942 $crate::init::PinnedDrop::drop(pinned, token);
943 }
944 }
945 };
946 // If some other parameter was specified, we emit a readable error.
947 (drop_prevention:
948 @name($name:ident),
949 @impl_generics($($impl_generics:tt)*),
950 @ty_generics($($ty_generics:tt)*),
951 @where($($whr:tt)*),
952 @pinned_drop($($rest:tt)*),
953 ) => {
954 compile_error!(
955 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
956 stringify!($($rest)*),
957 );
958 };
959 (make_pin_data:
960 @pin_data($pin_data:ident),
961 @impl_generics($($impl_generics:tt)*),
962 @ty_generics($($ty_generics:tt)*),
963 @where($($whr:tt)*),
964 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
965 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
966 ) => {
967 // For every field, we create a projection function according to its projection type. If a
968 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
969 // structurally pinned, then it can be initialized via `Init`.
970 //
971 // The functions are `unsafe` to prevent accidentally calling them.
972 #[allow(dead_code)]
973 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
974 where $($whr)*
975 {
976 $(
977 $(#[$($p_attr)*])*
978 $pvis unsafe fn $p_field<E>(
979 self,
980 slot: *mut $p_type,
981 init: impl $crate::init::PinInit<$p_type, E>,
982 ) -> ::core::result::Result<(), E> {
983 unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
984 }
985 )*
986 $(
987 $(#[$($attr)*])*
988 $fvis unsafe fn $field<E>(
989 self,
990 slot: *mut $type,
991 init: impl $crate::init::Init<$type, E>,
992 ) -> ::core::result::Result<(), E> {
993 unsafe { $crate::init::Init::__init(init, slot) }
994 }
995 )*
996 }
997 };
998}
999
1000/// The internal init macro. Do not call manually!
1001///
1002/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1003///
1004/// This macro has multiple internal call configurations, these are always the very first ident:
1005/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1006/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1007/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1008/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1009/// field has been initialized exactly once.
1010#[doc(hidden)]
1011#[macro_export]
1012macro_rules! __init_internal {
1013 (
1014 @this($($this:ident)?),
1015 @typ($t:path),
1016 @fields($($fields:tt)*),
1017 @error($err:ty),
1018 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1019 // case.
1020 @data($data:ident, $($use_data:ident)?),
1021 // `HasPinData` or `HasInitData`.
1022 @has_data($has_data:ident, $get_data:ident),
1023 // `pin_init_from_closure` or `init_from_closure`.
1024 @construct_closure($construct_closure:ident),
1025 @munch_fields(),
1026 ) => {
1027 $crate::__init_internal!(with_update_parsed:
1028 @this($($this)?),
1029 @typ($t),
1030 @fields($($fields)*),
1031 @error($err),
1032 @data($data, $($use_data)?),
1033 @has_data($has_data, $get_data),
1034 @construct_closure($construct_closure),
1035 @zeroed(), // Nothing means default behavior.
1036 )
1037 };
1038 (
1039 @this($($this:ident)?),
1040 @typ($t:path),
1041 @fields($($fields:tt)*),
1042 @error($err:ty),
1043 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1044 // case.
1045 @data($data:ident, $($use_data:ident)?),
1046 // `HasPinData` or `HasInitData`.
1047 @has_data($has_data:ident, $get_data:ident),
1048 // `pin_init_from_closure` or `init_from_closure`.
1049 @construct_closure($construct_closure:ident),
1050 @munch_fields(..Zeroable::zeroed()),
1051 ) => {
1052 $crate::__init_internal!(with_update_parsed:
1053 @this($($this)?),
1054 @typ($t),
1055 @fields($($fields)*),
1056 @error($err),
1057 @data($data, $($use_data)?),
1058 @has_data($has_data, $get_data),
1059 @construct_closure($construct_closure),
1060 @zeroed(()), // `()` means zero all fields not mentioned.
1061 )
1062 };
1063 (
1064 @this($($this:ident)?),
1065 @typ($t:path),
1066 @fields($($fields:tt)*),
1067 @error($err:ty),
1068 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1069 // case.
1070 @data($data:ident, $($use_data:ident)?),
1071 // `HasPinData` or `HasInitData`.
1072 @has_data($has_data:ident, $get_data:ident),
1073 // `pin_init_from_closure` or `init_from_closure`.
1074 @construct_closure($construct_closure:ident),
1075 @munch_fields($ignore:tt $($rest:tt)*),
1076 ) => {
1077 $crate::__init_internal!(
1078 @this($($this)?),
1079 @typ($t),
1080 @fields($($fields)*),
1081 @error($err),
1082 @data($data, $($use_data)?),
1083 @has_data($has_data, $get_data),
1084 @construct_closure($construct_closure),
1085 @munch_fields($($rest)*),
1086 )
1087 };
1088 (with_update_parsed:
1089 @this($($this:ident)?),
1090 @typ($t:path),
1091 @fields($($fields:tt)*),
1092 @error($err:ty),
1093 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1094 // case.
1095 @data($data:ident, $($use_data:ident)?),
1096 // `HasPinData` or `HasInitData`.
1097 @has_data($has_data:ident, $get_data:ident),
1098 // `pin_init_from_closure` or `init_from_closure`.
1099 @construct_closure($construct_closure:ident),
1100 @zeroed($($init_zeroed:expr)?),
1101 ) => {{
1102 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1103 // type and shadow it later when we insert the arbitrary user code. That way there will be
1104 // no possibility of returning without `unsafe`.
1105 struct __InitOk;
1106 // Get the data about fields from the supplied type.
1107 let data = unsafe {
1108 use $crate::init::__internal::$has_data;
1109 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1110 // information that is associated to already parsed fragments, so a path fragment
1111 // cannot be used in this position. Doing the retokenization results in valid rust
1112 // code.
1113 ::kernel::macros::paste!($t::$get_data())
1114 };
1115 // Ensure that `data` really is of type `$data` and help with type inference:
1116 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1117 data,
1118 move |slot| {
1119 {
1120 // Shadow the structure so it cannot be used to return early.
1121 struct __InitOk;
1122 // If `$init_zeroed` is present we should zero the slot now and not emit an
1123 // error when fields are missing (since they will be zeroed). We also have to
1124 // check that the type actually implements `Zeroable`.
1125 $({
1126 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1127 // Ensure that the struct is indeed `Zeroable`.
1128 assert_zeroable(slot);
1129 // SAFETY: The type implements `Zeroable` by the check above.
1130 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1131 $init_zeroed // This will be `()` if set.
1132 })?
1133 // Create the `this` so it can be referenced by the user inside of the
1134 // expressions creating the individual fields.
1135 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1136 // Initialize every field.
1137 $crate::__init_internal!(init_slot($($use_data)?):
1138 @data(data),
1139 @slot(slot),
1140 @guards(),
1141 @munch_fields($($fields)*,),
1142 );
1143 // We use unreachable code to ensure that all fields have been mentioned exactly
1144 // once, this struct initializer will still be type-checked and complain with a
1145 // very natural error message if a field is forgotten/mentioned more than once.
1146 #[allow(unreachable_code, clippy::diverging_sub_expression)]
1147 let _ = || {
1148 $crate::__init_internal!(make_initializer:
1149 @slot(slot),
1150 @type_name($t),
1151 @munch_fields($($fields)*,),
1152 @acc(),
1153 );
1154 };
1155 }
1156 Ok(__InitOk)
1157 }
1158 );
1159 let init = move |slot| -> ::core::result::Result<(), $err> {
1160 init(slot).map(|__InitOk| ())
1161 };
1162 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1163 init
1164 }};
1165 (init_slot($($use_data:ident)?):
1166 @data($data:ident),
1167 @slot($slot:ident),
1168 @guards($($guards:ident,)*),
1169 @munch_fields($(..Zeroable::zeroed())? $(,)?),
1170 ) => {
1171 // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1172 // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1173 $(::core::mem::forget($guards);)*
1174 };
1175 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1176 @data($data:ident),
1177 @slot($slot:ident),
1178 @guards($($guards:ident,)*),
1179 // In-place initialization syntax.
1180 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1181 ) => {
1182 let init = $val;
1183 // Call the initializer.
1184 //
1185 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1186 // return when an error/panic occurs.
1187 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1188 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1189 // Create the drop guard:
1190 //
1191 // We rely on macro hygiene to make it impossible for users to access this local variable.
1192 // We use `paste!` to create new hygiene for `$field`.
1193 ::kernel::macros::paste! {
1194 // SAFETY: We forget the guard later when initialization has succeeded.
1195 let [<$field>] = unsafe {
1196 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1197 };
1198
1199 $crate::__init_internal!(init_slot($use_data):
1200 @data($data),
1201 @slot($slot),
1202 @guards([<$field>], $($guards,)*),
1203 @munch_fields($($rest)*),
1204 );
1205 }
1206 };
1207 (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1208 @data($data:ident),
1209 @slot($slot:ident),
1210 @guards($($guards:ident,)*),
1211 // In-place initialization syntax.
1212 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1213 ) => {
1214 let init = $val;
1215 // Call the initializer.
1216 //
1217 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1218 // return when an error/panic occurs.
1219 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1220 // Create the drop guard:
1221 //
1222 // We rely on macro hygiene to make it impossible for users to access this local variable.
1223 // We use `paste!` to create new hygiene for `$field`.
1224 ::kernel::macros::paste! {
1225 // SAFETY: We forget the guard later when initialization has succeeded.
1226 let [<$field>] = unsafe {
1227 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1228 };
1229
1230 $crate::__init_internal!(init_slot():
1231 @data($data),
1232 @slot($slot),
1233 @guards([<$field>], $($guards,)*),
1234 @munch_fields($($rest)*),
1235 );
1236 }
1237 };
1238 (init_slot($($use_data:ident)?):
1239 @data($data:ident),
1240 @slot($slot:ident),
1241 @guards($($guards:ident,)*),
1242 // Init by-value.
1243 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1244 ) => {
1245 {
1246 $(let $field = $val;)?
1247 // Initialize the field.
1248 //
1249 // SAFETY: The memory at `slot` is uninitialized.
1250 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1251 }
1252 // Create the drop guard:
1253 //
1254 // We rely on macro hygiene to make it impossible for users to access this local variable.
1255 // We use `paste!` to create new hygiene for `$field`.
1256 ::kernel::macros::paste! {
1257 // SAFETY: We forget the guard later when initialization has succeeded.
1258 let [<$field>] = unsafe {
1259 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1260 };
1261
1262 $crate::__init_internal!(init_slot($($use_data)?):
1263 @data($data),
1264 @slot($slot),
1265 @guards([<$field>], $($guards,)*),
1266 @munch_fields($($rest)*),
1267 );
1268 }
1269 };
1270 (make_initializer:
1271 @slot($slot:ident),
1272 @type_name($t:path),
1273 @munch_fields(..Zeroable::zeroed() $(,)?),
1274 @acc($($acc:tt)*),
1275 ) => {
1276 // Endpoint, nothing more to munch, create the initializer. Since the users specified
1277 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1278 // not been overwritten are thus zero and initialized. We still check that all fields are
1279 // actually accessible by using the struct update syntax ourselves.
1280 // We are inside of a closure that is never executed and thus we can abuse `slot` to
1281 // get the correct type inference here:
1282 #[allow(unused_assignments)]
1283 unsafe {
1284 let mut zeroed = ::core::mem::zeroed();
1285 // We have to use type inference here to make zeroed have the correct type. This does
1286 // not get executed, so it has no effect.
1287 ::core::ptr::write($slot, zeroed);
1288 zeroed = ::core::mem::zeroed();
1289 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1290 // information that is associated to already parsed fragments, so a path fragment
1291 // cannot be used in this position. Doing the retokenization results in valid rust
1292 // code.
1293 ::kernel::macros::paste!(
1294 ::core::ptr::write($slot, $t {
1295 $($acc)*
1296 ..zeroed
1297 });
1298 );
1299 }
1300 };
1301 (make_initializer:
1302 @slot($slot:ident),
1303 @type_name($t:path),
1304 @munch_fields($(,)?),
1305 @acc($($acc:tt)*),
1306 ) => {
1307 // Endpoint, nothing more to munch, create the initializer.
1308 // Since we are in the closure that is never called, this will never get executed.
1309 // We abuse `slot` to get the correct type inference here:
1310 unsafe {
1311 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1312 // information that is associated to already parsed fragments, so a path fragment
1313 // cannot be used in this position. Doing the retokenization results in valid rust
1314 // code.
1315 ::kernel::macros::paste!(
1316 ::core::ptr::write($slot, $t {
1317 $($acc)*
1318 });
1319 );
1320 }
1321 };
1322 (make_initializer:
1323 @slot($slot:ident),
1324 @type_name($t:path),
1325 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1326 @acc($($acc:tt)*),
1327 ) => {
1328 $crate::__init_internal!(make_initializer:
1329 @slot($slot),
1330 @type_name($t),
1331 @munch_fields($($rest)*),
1332 @acc($($acc)* $field: ::core::panic!(),),
1333 );
1334 };
1335 (make_initializer:
1336 @slot($slot:ident),
1337 @type_name($t:path),
1338 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1339 @acc($($acc:tt)*),
1340 ) => {
1341 $crate::__init_internal!(make_initializer:
1342 @slot($slot),
1343 @type_name($t),
1344 @munch_fields($($rest)*),
1345 @acc($($acc)* $field: ::core::panic!(),),
1346 );
1347 };
1348}
1349
1350#[doc(hidden)]
1351#[macro_export]
1352macro_rules! __derive_zeroable {
1353 (parse_input:
1354 @sig(
1355 $(#[$($struct_attr:tt)*])*
1356 $vis:vis struct $name:ident
1357 $(where $($whr:tt)*)?
1358 ),
1359 @impl_generics($($impl_generics:tt)*),
1360 @ty_generics($($ty_generics:tt)*),
1361 @body({
1362 $(
1363 $(#[$($field_attr:tt)*])*
1364 $field:ident : $field_ty:ty
1365 ),* $(,)?
1366 }),
1367 ) => {
1368 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1369 #[automatically_derived]
1370 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1371 where
1372 $($($whr)*)?
1373 {}
1374 const _: () = {
1375 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1376 fn ensure_zeroable<$($impl_generics)*>()
1377 where $($($whr)*)?
1378 {
1379 $(assert_zeroable::<$field_ty>();)*
1380 }
1381 };
1382 };
1383}