1// SPDX-License-Identifier: GPL-2.0
  2
  3//! Implementation of the kernel's memory allocation infrastructure.
  4
  5#[cfg(not(any(test, testlib)))]
  6pub mod allocator;
  7pub mod kbox;
  8pub mod kvec;
  9pub mod layout;
 10
 11#[cfg(any(test, testlib))]
 12pub mod allocator_test;
 13
 14#[cfg(any(test, testlib))]
 15pub use self::allocator_test as allocator;
 16
 17pub use self::kbox::Box;
 18pub use self::kbox::KBox;
 19pub use self::kbox::KVBox;
 20pub use self::kbox::VBox;
 21
 22pub use self::kvec::IntoIter;
 23pub use self::kvec::KVVec;
 24pub use self::kvec::KVec;
 25pub use self::kvec::VVec;
 26pub use self::kvec::Vec;
 27
 28/// Indicates an allocation error.
 29#[derive(Copy, Clone, PartialEq, Eq, Debug)]
 30pub struct AllocError;
 31use core::{alloc::Layout, ptr::NonNull};
 32
 33/// Flags to be used when allocating memory.
 34///
 35/// They can be combined with the operators `|`, `&`, and `!`.
 36///
 37/// Values can be used from the [`flags`] module.
 38#[derive(Clone, Copy, PartialEq)]
 39pub struct Flags(u32);
 40
 41impl Flags {
 42    /// Get the raw representation of this flag.
 43    pub(crate) fn as_raw(self) -> u32 {
 44        self.0
 45    }
 46
 47    /// Check whether `flags` is contained in `self`.
 48    pub fn contains(self, flags: Flags) -> bool {
 49        (self & flags) == flags
 50    }
 51}
 52
 53impl core::ops::BitOr for Flags {
 54    type Output = Self;
 55    fn bitor(self, rhs: Self) -> Self::Output {
 56        Self(self.0 | rhs.0)
 57    }
 58}
 59
 60impl core::ops::BitAnd for Flags {
 61    type Output = Self;
 62    fn bitand(self, rhs: Self) -> Self::Output {
 63        Self(self.0 & rhs.0)
 64    }
 65}
 66
 67impl core::ops::Not for Flags {
 68    type Output = Self;
 69    fn not(self) -> Self::Output {
 70        Self(!self.0)
 71    }
 72}
 73
 74/// Allocation flags.
 75///
 76/// These are meant to be used in functions that can allocate memory.
 77pub mod flags {
 78    use super::Flags;
 79
 80    /// Zeroes out the allocated memory.
 81    ///
 82    /// This is normally or'd with other flags.
 83    pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO);
 84
 85    /// Allow the allocation to be in high memory.
 86    ///
 87    /// Allocations in high memory may not be mapped into the kernel's address space, so this can't
 88    /// be used with `kmalloc` and other similar methods.
 89    ///
 90    /// This is normally or'd with other flags.
 91    pub const __GFP_HIGHMEM: Flags = Flags(bindings::__GFP_HIGHMEM);
 92
 93    /// Users can not sleep and need the allocation to succeed.
 94    ///
 95    /// A lower watermark is applied to allow access to "atomic reserves". The current
 96    /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g.
 97    /// raw_spin_lock). The same applies to [`GFP_NOWAIT`].
 98    pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC);
 99
100    /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone
101    /// for direct access but can direct reclaim.
102    pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL);
103
104    /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg.
105    pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT);
106
107    /// For kernel allocations that should not stall for direct reclaim, start physical IO or
108    /// use any filesystem callback.  It is very likely to fail to allocate memory, even for very
109    /// small allocations.
110    pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT);
111
112    /// Suppresses allocation failure reports.
113    ///
114    /// This is normally or'd with other flags.
115    pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN);
116}
117
118/// The kernel's [`Allocator`] trait.
119///
120/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described
121/// via [`Layout`].
122///
123/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on
124/// an object instance.
125///
126/// In order to be able to support `#[derive(SmartPointer)]` later on, we need to avoid a design
127/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind
128/// of `self` parameter.
129///
130/// # Safety
131///
132/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed.
133///
134/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`]
135///   function of the same type.
136///
137/// - Implementers must ensure that all trait functions abide by the guarantees documented in the
138///   `# Guarantees` sections.
139pub unsafe trait Allocator {
140    /// Allocate memory based on `layout` and `flags`.
141    ///
142    /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout
143    /// constraints (i.e. minimum size and alignment as specified by `layout`).
144    ///
145    /// This function is equivalent to `realloc` when called with `None`.
146    ///
147    /// # Guarantees
148    ///
149    /// When the return value is `Ok(ptr)`, then `ptr` is
150    /// - valid for reads and writes for `layout.size()` bytes, until it is passed to
151    ///   [`Allocator::free`] or [`Allocator::realloc`],
152    /// - aligned to `layout.align()`,
153    ///
154    /// Additionally, `Flags` are honored as documented in
155    /// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>.
156    fn alloc(layout: Layout, flags: Flags) -> Result<NonNull<[u8]>, AllocError> {
157        // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a
158        // new memory allocation.
159        unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) }
160    }
161
162    /// Re-allocate an existing memory allocation to satisfy the requested `layout`.
163    ///
164    /// If the requested size is zero, `realloc` behaves equivalent to `free`.
165    ///
166    /// If the requested size is larger than the size of the existing allocation, a successful call
167    /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but
168    /// may also be larger.
169    ///
170    /// If the requested size is smaller than the size of the existing allocation, `realloc` may or
171    /// may not shrink the buffer; this is implementation specific to the allocator.
172    ///
173    /// On allocation failure, the existing buffer, if any, remains valid.
174    ///
175    /// The buffer is represented as `NonNull<[u8]>`.
176    ///
177    /// # Safety
178    ///
179    /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation
180    ///   created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a
181    ///   pointer returned by this [`Allocator`].
182    /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and
183    ///   `old_layout` is ignored.
184    /// - `old_layout` must match the `Layout` the allocation has been created with.
185    ///
186    /// # Guarantees
187    ///
188    /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then
189    /// it additionally guarantees that:
190    /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new
191    ///   and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] ==
192    ///   p[0..min(layout.size(), old_layout.size())]`.
193    /// - when the return value is `Err(AllocError)`, then `ptr` is still valid.
194    unsafe fn realloc(
195        ptr: Option<NonNull<u8>>,
196        layout: Layout,
197        old_layout: Layout,
198        flags: Flags,
199    ) -> Result<NonNull<[u8]>, AllocError>;
200
201    /// Free an existing memory allocation.
202    ///
203    /// # Safety
204    ///
205    /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`];
206    ///   if `old_layout` is zero-sized `p` does not need to be a pointer returned by this
207    ///   [`Allocator`].
208    /// - `layout` must match the `Layout` the allocation has been created with.
209    /// - The memory allocation at `ptr` must never again be read from or written to.
210    unsafe fn free(ptr: NonNull<u8>, layout: Layout) {
211        // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this
212        // allocator. We are passing a `Layout` with the smallest possible alignment, so it is
213        // smaller than or equal to the alignment previously used with this allocation.
214        let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) };
215    }
216}
217
218/// Returns a properly aligned dangling pointer from the given `layout`.
219pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull<u8> {
220    let ptr = layout.align() as *mut u8;
221
222    // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero.
223    unsafe { NonNull::new_unchecked(ptr) }
224}