1// SPDX-License-Identifier: GPL-2.0
  2
  3//! Allocator support.
  4//!
  5//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide"
  6//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the
  7//! typical application of the different kernel allocators.
  8//!
  9//! Reference: <https://docs.kernel.org/core-api/memory-allocation.html>
 10
 11use super::Flags;
 12use core::alloc::Layout;
 13use core::ptr;
 14use core::ptr::NonNull;
 15
 16use crate::alloc::{AllocError, Allocator};
 17use crate::bindings;
 18use crate::pr_warn;
 19
 20/// The contiguous kernel allocator.
 21///
 22/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also
 23/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific.
 24///
 25/// For more details see [self].
 26pub struct Kmalloc;
 27
 28/// The virtually contiguous kernel allocator.
 29///
 30/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel
 31/// virtual space. It is typically used for large allocations. The memory allocated with this
 32/// allocator is not physically contiguous.
 33///
 34/// For more details see [self].
 35pub struct Vmalloc;
 36
 37/// The kvmalloc kernel allocator.
 38///
 39/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon
 40/// failure. This allocator is typically used when the size for the requested allocation is not
 41/// known and may exceed the capabilities of `Kmalloc`.
 42///
 43/// For more details see [self].
 44pub struct KVmalloc;
 45
 46/// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment.
 47fn aligned_size(new_layout: Layout) -> usize {
 48    // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first.
 49    let layout = new_layout.pad_to_align();
 50
 51    // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()`
 52    // which together with the slab guarantees means the `krealloc` will return a properly aligned
 53    // object (see comments in `kmalloc()` for more information).
 54    layout.size()
 55}
 56
 57/// # Invariants
 58///
 59/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`.
 60struct ReallocFunc(
 61    unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void,
 62);
 63
 64impl ReallocFunc {
 65    // INVARIANT: `krealloc` satisfies the type invariants.
 66    const KREALLOC: Self = Self(bindings::krealloc);
 67
 68    // INVARIANT: `vrealloc` satisfies the type invariants.
 69    const VREALLOC: Self = Self(bindings::vrealloc);
 70
 71    // INVARIANT: `kvrealloc` satisfies the type invariants.
 72    const KVREALLOC: Self = Self(bindings::kvrealloc);
 73
 74    /// # Safety
 75    ///
 76    /// This method has the same safety requirements as [`Allocator::realloc`].
 77    ///
 78    /// # Guarantees
 79    ///
 80    /// This method has the same guarantees as `Allocator::realloc`. Additionally
 81    /// - it accepts any pointer to a valid memory allocation allocated by this function.
 82    /// - memory allocated by this function remains valid until it is passed to this function.
 83    unsafe fn call(
 84        &self,
 85        ptr: Option<NonNull<u8>>,
 86        layout: Layout,
 87        old_layout: Layout,
 88        flags: Flags,
 89    ) -> Result<NonNull<[u8]>, AllocError> {
 90        let size = aligned_size(layout);
 91        let ptr = match ptr {
 92            Some(ptr) => {
 93                if old_layout.size() == 0 {
 94                    ptr::null()
 95                } else {
 96                    ptr.as_ptr()
 97                }
 98            }
 99            None => ptr::null(),
100        };
101
102        // SAFETY:
103        // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that
104        //   `ptr` is NULL or valid.
105        // - `ptr` is either NULL or valid by the safety requirements of this function.
106        //
107        // GUARANTEE:
108        // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`.
109        // - Those functions provide the guarantees of this function.
110        let raw_ptr = unsafe {
111            // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed.
112            self.0(ptr.cast(), size, flags.0).cast()
113        };
114
115        let ptr = if size == 0 {
116            crate::alloc::dangling_from_layout(layout)
117        } else {
118            NonNull::new(raw_ptr).ok_or(AllocError)?
119        };
120
121        Ok(NonNull::slice_from_raw_parts(ptr, size))
122    }
123}
124
125// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
126// - memory remains valid until it is explicitly freed,
127// - passing a pointer to a valid memory allocation is OK,
128// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
129unsafe impl Allocator for Kmalloc {
130    #[inline]
131    unsafe fn realloc(
132        ptr: Option<NonNull<u8>>,
133        layout: Layout,
134        old_layout: Layout,
135        flags: Flags,
136    ) -> Result<NonNull<[u8]>, AllocError> {
137        // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`.
138        unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) }
139    }
140}
141
142// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
143// - memory remains valid until it is explicitly freed,
144// - passing a pointer to a valid memory allocation is OK,
145// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
146unsafe impl Allocator for Vmalloc {
147    #[inline]
148    unsafe fn realloc(
149        ptr: Option<NonNull<u8>>,
150        layout: Layout,
151        old_layout: Layout,
152        flags: Flags,
153    ) -> Result<NonNull<[u8]>, AllocError> {
154        // TODO: Support alignments larger than PAGE_SIZE.
155        if layout.align() > bindings::PAGE_SIZE {
156            pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n");
157            return Err(AllocError);
158        }
159
160        // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
161        // allocated with this `Allocator`.
162        unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) }
163    }
164}
165
166// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
167// - memory remains valid until it is explicitly freed,
168// - passing a pointer to a valid memory allocation is OK,
169// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
170unsafe impl Allocator for KVmalloc {
171    #[inline]
172    unsafe fn realloc(
173        ptr: Option<NonNull<u8>>,
174        layout: Layout,
175        old_layout: Layout,
176        flags: Flags,
177    ) -> Result<NonNull<[u8]>, AllocError> {
178        // TODO: Support alignments larger than PAGE_SIZE.
179        if layout.align() > bindings::PAGE_SIZE {
180            pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n");
181            return Err(AllocError);
182        }
183
184        // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
185        // allocated with this `Allocator`.
186        unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) }
187    }
188}