Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common EFI memory map functions.
4 */
5
6#define pr_fmt(fmt) "efi: " fmt
7
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/efi.h>
11#include <linux/io.h>
12#include <asm/early_ioremap.h>
13#include <linux/memblock.h>
14#include <linux/slab.h>
15
16static phys_addr_t __init __efi_memmap_alloc_early(unsigned long size)
17{
18 return memblock_phys_alloc(size, SMP_CACHE_BYTES);
19}
20
21static phys_addr_t __init __efi_memmap_alloc_late(unsigned long size)
22{
23 unsigned int order = get_order(size);
24 struct page *p = alloc_pages(GFP_KERNEL, order);
25
26 if (!p)
27 return 0;
28
29 return PFN_PHYS(page_to_pfn(p));
30}
31
32void __init __efi_memmap_free(u64 phys, unsigned long size, unsigned long flags)
33{
34 if (flags & EFI_MEMMAP_MEMBLOCK) {
35 if (slab_is_available())
36 memblock_free_late(phys, size);
37 else
38 memblock_free(phys, size);
39 } else if (flags & EFI_MEMMAP_SLAB) {
40 struct page *p = pfn_to_page(PHYS_PFN(phys));
41 unsigned int order = get_order(size);
42
43 free_pages((unsigned long) page_address(p), order);
44 }
45}
46
47static void __init efi_memmap_free(void)
48{
49 __efi_memmap_free(efi.memmap.phys_map,
50 efi.memmap.desc_size * efi.memmap.nr_map,
51 efi.memmap.flags);
52}
53
54/**
55 * efi_memmap_alloc - Allocate memory for the EFI memory map
56 * @num_entries: Number of entries in the allocated map.
57 * @data: efi memmap installation parameters
58 *
59 * Depending on whether mm_init() has already been invoked or not,
60 * either memblock or "normal" page allocation is used.
61 *
62 * Returns the physical address of the allocated memory map on
63 * success, zero on failure.
64 */
65int __init efi_memmap_alloc(unsigned int num_entries,
66 struct efi_memory_map_data *data)
67{
68 /* Expect allocation parameters are zero initialized */
69 WARN_ON(data->phys_map || data->size);
70
71 data->size = num_entries * efi.memmap.desc_size;
72 data->desc_version = efi.memmap.desc_version;
73 data->desc_size = efi.memmap.desc_size;
74 data->flags &= ~(EFI_MEMMAP_SLAB | EFI_MEMMAP_MEMBLOCK);
75 data->flags |= efi.memmap.flags & EFI_MEMMAP_LATE;
76
77 if (slab_is_available()) {
78 data->flags |= EFI_MEMMAP_SLAB;
79 data->phys_map = __efi_memmap_alloc_late(data->size);
80 } else {
81 data->flags |= EFI_MEMMAP_MEMBLOCK;
82 data->phys_map = __efi_memmap_alloc_early(data->size);
83 }
84
85 if (!data->phys_map)
86 return -ENOMEM;
87 return 0;
88}
89
90/**
91 * __efi_memmap_init - Common code for mapping the EFI memory map
92 * @data: EFI memory map data
93 *
94 * This function takes care of figuring out which function to use to
95 * map the EFI memory map in efi.memmap based on how far into the boot
96 * we are.
97 *
98 * During bootup EFI_MEMMAP_LATE in data->flags should be clear since we
99 * only have access to the early_memremap*() functions as the vmalloc
100 * space isn't setup. Once the kernel is fully booted we can fallback
101 * to the more robust memremap*() API.
102 *
103 * Returns zero on success, a negative error code on failure.
104 */
105static int __init __efi_memmap_init(struct efi_memory_map_data *data)
106{
107 struct efi_memory_map map;
108 phys_addr_t phys_map;
109
110 if (efi_enabled(EFI_PARAVIRT))
111 return 0;
112
113 phys_map = data->phys_map;
114
115 if (data->flags & EFI_MEMMAP_LATE)
116 map.map = memremap(phys_map, data->size, MEMREMAP_WB);
117 else
118 map.map = early_memremap(phys_map, data->size);
119
120 if (!map.map) {
121 pr_err("Could not map the memory map!\n");
122 return -ENOMEM;
123 }
124
125 /* NOP if data->flags & (EFI_MEMMAP_MEMBLOCK | EFI_MEMMAP_SLAB) == 0 */
126 efi_memmap_free();
127
128 map.phys_map = data->phys_map;
129 map.nr_map = data->size / data->desc_size;
130 map.map_end = map.map + data->size;
131
132 map.desc_version = data->desc_version;
133 map.desc_size = data->desc_size;
134 map.flags = data->flags;
135
136 set_bit(EFI_MEMMAP, &efi.flags);
137
138 efi.memmap = map;
139
140 return 0;
141}
142
143/**
144 * efi_memmap_init_early - Map the EFI memory map data structure
145 * @data: EFI memory map data
146 *
147 * Use early_memremap() to map the passed in EFI memory map and assign
148 * it to efi.memmap.
149 */
150int __init efi_memmap_init_early(struct efi_memory_map_data *data)
151{
152 /* Cannot go backwards */
153 WARN_ON(efi.memmap.flags & EFI_MEMMAP_LATE);
154
155 data->flags = 0;
156 return __efi_memmap_init(data);
157}
158
159void __init efi_memmap_unmap(void)
160{
161 if (!efi_enabled(EFI_MEMMAP))
162 return;
163
164 if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) {
165 unsigned long size;
166
167 size = efi.memmap.desc_size * efi.memmap.nr_map;
168 early_memunmap(efi.memmap.map, size);
169 } else {
170 memunmap(efi.memmap.map);
171 }
172
173 efi.memmap.map = NULL;
174 clear_bit(EFI_MEMMAP, &efi.flags);
175}
176
177/**
178 * efi_memmap_init_late - Map efi.memmap with memremap()
179 * @phys_addr: Physical address of the new EFI memory map
180 * @size: Size in bytes of the new EFI memory map
181 *
182 * Setup a mapping of the EFI memory map using ioremap_cache(). This
183 * function should only be called once the vmalloc space has been
184 * setup and is therefore not suitable for calling during early EFI
185 * initialise, e.g. in efi_init(). Additionally, it expects
186 * efi_memmap_init_early() to have already been called.
187 *
188 * The reason there are two EFI memmap initialisation
189 * (efi_memmap_init_early() and this late version) is because the
190 * early EFI memmap should be explicitly unmapped once EFI
191 * initialisation is complete as the fixmap space used to map the EFI
192 * memmap (via early_memremap()) is a scarce resource.
193 *
194 * This late mapping is intended to persist for the duration of
195 * runtime so that things like efi_mem_desc_lookup() and
196 * efi_mem_attributes() always work.
197 *
198 * Returns zero on success, a negative error code on failure.
199 */
200int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size)
201{
202 struct efi_memory_map_data data = {
203 .phys_map = addr,
204 .size = size,
205 .flags = EFI_MEMMAP_LATE,
206 };
207
208 /* Did we forget to unmap the early EFI memmap? */
209 WARN_ON(efi.memmap.map);
210
211 /* Were we already called? */
212 WARN_ON(efi.memmap.flags & EFI_MEMMAP_LATE);
213
214 /*
215 * It makes no sense to allow callers to register different
216 * values for the following fields. Copy them out of the
217 * existing early EFI memmap.
218 */
219 data.desc_version = efi.memmap.desc_version;
220 data.desc_size = efi.memmap.desc_size;
221
222 return __efi_memmap_init(&data);
223}
224
225/**
226 * efi_memmap_install - Install a new EFI memory map in efi.memmap
227 * @ctx: map allocation parameters (address, size, flags)
228 *
229 * Unlike efi_memmap_init_*(), this function does not allow the caller
230 * to switch from early to late mappings. It simply uses the existing
231 * mapping function and installs the new memmap.
232 *
233 * Returns zero on success, a negative error code on failure.
234 */
235int __init efi_memmap_install(struct efi_memory_map_data *data)
236{
237 efi_memmap_unmap();
238
239 return __efi_memmap_init(data);
240}
241
242/**
243 * efi_memmap_split_count - Count number of additional EFI memmap entries
244 * @md: EFI memory descriptor to split
245 * @range: Address range (start, end) to split around
246 *
247 * Returns the number of additional EFI memmap entries required to
248 * accomodate @range.
249 */
250int __init efi_memmap_split_count(efi_memory_desc_t *md, struct range *range)
251{
252 u64 m_start, m_end;
253 u64 start, end;
254 int count = 0;
255
256 start = md->phys_addr;
257 end = start + (md->num_pages << EFI_PAGE_SHIFT) - 1;
258
259 /* modifying range */
260 m_start = range->start;
261 m_end = range->end;
262
263 if (m_start <= start) {
264 /* split into 2 parts */
265 if (start < m_end && m_end < end)
266 count++;
267 }
268
269 if (start < m_start && m_start < end) {
270 /* split into 3 parts */
271 if (m_end < end)
272 count += 2;
273 /* split into 2 parts */
274 if (end <= m_end)
275 count++;
276 }
277
278 return count;
279}
280
281/**
282 * efi_memmap_insert - Insert a memory region in an EFI memmap
283 * @old_memmap: The existing EFI memory map structure
284 * @buf: Address of buffer to store new map
285 * @mem: Memory map entry to insert
286 *
287 * It is suggested that you call efi_memmap_split_count() first
288 * to see how large @buf needs to be.
289 */
290void __init efi_memmap_insert(struct efi_memory_map *old_memmap, void *buf,
291 struct efi_mem_range *mem)
292{
293 u64 m_start, m_end, m_attr;
294 efi_memory_desc_t *md;
295 u64 start, end;
296 void *old, *new;
297
298 /* modifying range */
299 m_start = mem->range.start;
300 m_end = mem->range.end;
301 m_attr = mem->attribute;
302
303 /*
304 * The EFI memory map deals with regions in EFI_PAGE_SIZE
305 * units. Ensure that the region described by 'mem' is aligned
306 * correctly.
307 */
308 if (!IS_ALIGNED(m_start, EFI_PAGE_SIZE) ||
309 !IS_ALIGNED(m_end + 1, EFI_PAGE_SIZE)) {
310 WARN_ON(1);
311 return;
312 }
313
314 for (old = old_memmap->map, new = buf;
315 old < old_memmap->map_end;
316 old += old_memmap->desc_size, new += old_memmap->desc_size) {
317
318 /* copy original EFI memory descriptor */
319 memcpy(new, old, old_memmap->desc_size);
320 md = new;
321 start = md->phys_addr;
322 end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1;
323
324 if (m_start <= start && end <= m_end)
325 md->attribute |= m_attr;
326
327 if (m_start <= start &&
328 (start < m_end && m_end < end)) {
329 /* first part */
330 md->attribute |= m_attr;
331 md->num_pages = (m_end - md->phys_addr + 1) >>
332 EFI_PAGE_SHIFT;
333 /* latter part */
334 new += old_memmap->desc_size;
335 memcpy(new, old, old_memmap->desc_size);
336 md = new;
337 md->phys_addr = m_end + 1;
338 md->num_pages = (end - md->phys_addr + 1) >>
339 EFI_PAGE_SHIFT;
340 }
341
342 if ((start < m_start && m_start < end) && m_end < end) {
343 /* first part */
344 md->num_pages = (m_start - md->phys_addr) >>
345 EFI_PAGE_SHIFT;
346 /* middle part */
347 new += old_memmap->desc_size;
348 memcpy(new, old, old_memmap->desc_size);
349 md = new;
350 md->attribute |= m_attr;
351 md->phys_addr = m_start;
352 md->num_pages = (m_end - m_start + 1) >>
353 EFI_PAGE_SHIFT;
354 /* last part */
355 new += old_memmap->desc_size;
356 memcpy(new, old, old_memmap->desc_size);
357 md = new;
358 md->phys_addr = m_end + 1;
359 md->num_pages = (end - m_end) >>
360 EFI_PAGE_SHIFT;
361 }
362
363 if ((start < m_start && m_start < end) &&
364 (end <= m_end)) {
365 /* first part */
366 md->num_pages = (m_start - md->phys_addr) >>
367 EFI_PAGE_SHIFT;
368 /* latter part */
369 new += old_memmap->desc_size;
370 memcpy(new, old, old_memmap->desc_size);
371 md = new;
372 md->phys_addr = m_start;
373 md->num_pages = (end - md->phys_addr + 1) >>
374 EFI_PAGE_SHIFT;
375 md->attribute |= m_attr;
376 }
377 }
378}
1/*
2 * Common EFI memory map functions.
3 */
4
5#define pr_fmt(fmt) "efi: " fmt
6
7#include <linux/init.h>
8#include <linux/kernel.h>
9#include <linux/efi.h>
10#include <linux/io.h>
11#include <asm/early_ioremap.h>
12#include <linux/memblock.h>
13#include <linux/slab.h>
14
15static phys_addr_t __init __efi_memmap_alloc_early(unsigned long size)
16{
17 return memblock_alloc(size, 0);
18}
19
20static phys_addr_t __init __efi_memmap_alloc_late(unsigned long size)
21{
22 unsigned int order = get_order(size);
23 struct page *p = alloc_pages(GFP_KERNEL, order);
24
25 if (!p)
26 return 0;
27
28 return PFN_PHYS(page_to_pfn(p));
29}
30
31/**
32 * efi_memmap_alloc - Allocate memory for the EFI memory map
33 * @num_entries: Number of entries in the allocated map.
34 *
35 * Depending on whether mm_init() has already been invoked or not,
36 * either memblock or "normal" page allocation is used.
37 *
38 * Returns the physical address of the allocated memory map on
39 * success, zero on failure.
40 */
41phys_addr_t __init efi_memmap_alloc(unsigned int num_entries)
42{
43 unsigned long size = num_entries * efi.memmap.desc_size;
44
45 if (slab_is_available())
46 return __efi_memmap_alloc_late(size);
47
48 return __efi_memmap_alloc_early(size);
49}
50
51/**
52 * __efi_memmap_init - Common code for mapping the EFI memory map
53 * @data: EFI memory map data
54 * @late: Use early or late mapping function?
55 *
56 * This function takes care of figuring out which function to use to
57 * map the EFI memory map in efi.memmap based on how far into the boot
58 * we are.
59 *
60 * During bootup @late should be %false since we only have access to
61 * the early_memremap*() functions as the vmalloc space isn't setup.
62 * Once the kernel is fully booted we can fallback to the more robust
63 * memremap*() API.
64 *
65 * Returns zero on success, a negative error code on failure.
66 */
67static int __init
68__efi_memmap_init(struct efi_memory_map_data *data, bool late)
69{
70 struct efi_memory_map map;
71 phys_addr_t phys_map;
72
73 if (efi_enabled(EFI_PARAVIRT))
74 return 0;
75
76 phys_map = data->phys_map;
77
78 if (late)
79 map.map = memremap(phys_map, data->size, MEMREMAP_WB);
80 else
81 map.map = early_memremap(phys_map, data->size);
82
83 if (!map.map) {
84 pr_err("Could not map the memory map!\n");
85 return -ENOMEM;
86 }
87
88 map.phys_map = data->phys_map;
89 map.nr_map = data->size / data->desc_size;
90 map.map_end = map.map + data->size;
91
92 map.desc_version = data->desc_version;
93 map.desc_size = data->desc_size;
94 map.late = late;
95
96 set_bit(EFI_MEMMAP, &efi.flags);
97
98 efi.memmap = map;
99
100 return 0;
101}
102
103/**
104 * efi_memmap_init_early - Map the EFI memory map data structure
105 * @data: EFI memory map data
106 *
107 * Use early_memremap() to map the passed in EFI memory map and assign
108 * it to efi.memmap.
109 */
110int __init efi_memmap_init_early(struct efi_memory_map_data *data)
111{
112 /* Cannot go backwards */
113 WARN_ON(efi.memmap.late);
114
115 return __efi_memmap_init(data, false);
116}
117
118void __init efi_memmap_unmap(void)
119{
120 if (!efi.memmap.late) {
121 unsigned long size;
122
123 size = efi.memmap.desc_size * efi.memmap.nr_map;
124 early_memunmap(efi.memmap.map, size);
125 } else {
126 memunmap(efi.memmap.map);
127 }
128
129 efi.memmap.map = NULL;
130 clear_bit(EFI_MEMMAP, &efi.flags);
131}
132
133/**
134 * efi_memmap_init_late - Map efi.memmap with memremap()
135 * @phys_addr: Physical address of the new EFI memory map
136 * @size: Size in bytes of the new EFI memory map
137 *
138 * Setup a mapping of the EFI memory map using ioremap_cache(). This
139 * function should only be called once the vmalloc space has been
140 * setup and is therefore not suitable for calling during early EFI
141 * initialise, e.g. in efi_init(). Additionally, it expects
142 * efi_memmap_init_early() to have already been called.
143 *
144 * The reason there are two EFI memmap initialisation
145 * (efi_memmap_init_early() and this late version) is because the
146 * early EFI memmap should be explicitly unmapped once EFI
147 * initialisation is complete as the fixmap space used to map the EFI
148 * memmap (via early_memremap()) is a scarce resource.
149 *
150 * This late mapping is intended to persist for the duration of
151 * runtime so that things like efi_mem_desc_lookup() and
152 * efi_mem_attributes() always work.
153 *
154 * Returns zero on success, a negative error code on failure.
155 */
156int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size)
157{
158 struct efi_memory_map_data data = {
159 .phys_map = addr,
160 .size = size,
161 };
162
163 /* Did we forget to unmap the early EFI memmap? */
164 WARN_ON(efi.memmap.map);
165
166 /* Were we already called? */
167 WARN_ON(efi.memmap.late);
168
169 /*
170 * It makes no sense to allow callers to register different
171 * values for the following fields. Copy them out of the
172 * existing early EFI memmap.
173 */
174 data.desc_version = efi.memmap.desc_version;
175 data.desc_size = efi.memmap.desc_size;
176
177 return __efi_memmap_init(&data, true);
178}
179
180/**
181 * efi_memmap_install - Install a new EFI memory map in efi.memmap
182 * @addr: Physical address of the memory map
183 * @nr_map: Number of entries in the memory map
184 *
185 * Unlike efi_memmap_init_*(), this function does not allow the caller
186 * to switch from early to late mappings. It simply uses the existing
187 * mapping function and installs the new memmap.
188 *
189 * Returns zero on success, a negative error code on failure.
190 */
191int __init efi_memmap_install(phys_addr_t addr, unsigned int nr_map)
192{
193 struct efi_memory_map_data data;
194
195 efi_memmap_unmap();
196
197 data.phys_map = addr;
198 data.size = efi.memmap.desc_size * nr_map;
199 data.desc_version = efi.memmap.desc_version;
200 data.desc_size = efi.memmap.desc_size;
201
202 return __efi_memmap_init(&data, efi.memmap.late);
203}
204
205/**
206 * efi_memmap_split_count - Count number of additional EFI memmap entries
207 * @md: EFI memory descriptor to split
208 * @range: Address range (start, end) to split around
209 *
210 * Returns the number of additional EFI memmap entries required to
211 * accomodate @range.
212 */
213int __init efi_memmap_split_count(efi_memory_desc_t *md, struct range *range)
214{
215 u64 m_start, m_end;
216 u64 start, end;
217 int count = 0;
218
219 start = md->phys_addr;
220 end = start + (md->num_pages << EFI_PAGE_SHIFT) - 1;
221
222 /* modifying range */
223 m_start = range->start;
224 m_end = range->end;
225
226 if (m_start <= start) {
227 /* split into 2 parts */
228 if (start < m_end && m_end < end)
229 count++;
230 }
231
232 if (start < m_start && m_start < end) {
233 /* split into 3 parts */
234 if (m_end < end)
235 count += 2;
236 /* split into 2 parts */
237 if (end <= m_end)
238 count++;
239 }
240
241 return count;
242}
243
244/**
245 * efi_memmap_insert - Insert a memory region in an EFI memmap
246 * @old_memmap: The existing EFI memory map structure
247 * @buf: Address of buffer to store new map
248 * @mem: Memory map entry to insert
249 *
250 * It is suggested that you call efi_memmap_split_count() first
251 * to see how large @buf needs to be.
252 */
253void __init efi_memmap_insert(struct efi_memory_map *old_memmap, void *buf,
254 struct efi_mem_range *mem)
255{
256 u64 m_start, m_end, m_attr;
257 efi_memory_desc_t *md;
258 u64 start, end;
259 void *old, *new;
260
261 /* modifying range */
262 m_start = mem->range.start;
263 m_end = mem->range.end;
264 m_attr = mem->attribute;
265
266 /*
267 * The EFI memory map deals with regions in EFI_PAGE_SIZE
268 * units. Ensure that the region described by 'mem' is aligned
269 * correctly.
270 */
271 if (!IS_ALIGNED(m_start, EFI_PAGE_SIZE) ||
272 !IS_ALIGNED(m_end + 1, EFI_PAGE_SIZE)) {
273 WARN_ON(1);
274 return;
275 }
276
277 for (old = old_memmap->map, new = buf;
278 old < old_memmap->map_end;
279 old += old_memmap->desc_size, new += old_memmap->desc_size) {
280
281 /* copy original EFI memory descriptor */
282 memcpy(new, old, old_memmap->desc_size);
283 md = new;
284 start = md->phys_addr;
285 end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1;
286
287 if (m_start <= start && end <= m_end)
288 md->attribute |= m_attr;
289
290 if (m_start <= start &&
291 (start < m_end && m_end < end)) {
292 /* first part */
293 md->attribute |= m_attr;
294 md->num_pages = (m_end - md->phys_addr + 1) >>
295 EFI_PAGE_SHIFT;
296 /* latter part */
297 new += old_memmap->desc_size;
298 memcpy(new, old, old_memmap->desc_size);
299 md = new;
300 md->phys_addr = m_end + 1;
301 md->num_pages = (end - md->phys_addr + 1) >>
302 EFI_PAGE_SHIFT;
303 }
304
305 if ((start < m_start && m_start < end) && m_end < end) {
306 /* first part */
307 md->num_pages = (m_start - md->phys_addr) >>
308 EFI_PAGE_SHIFT;
309 /* middle part */
310 new += old_memmap->desc_size;
311 memcpy(new, old, old_memmap->desc_size);
312 md = new;
313 md->attribute |= m_attr;
314 md->phys_addr = m_start;
315 md->num_pages = (m_end - m_start + 1) >>
316 EFI_PAGE_SHIFT;
317 /* last part */
318 new += old_memmap->desc_size;
319 memcpy(new, old, old_memmap->desc_size);
320 md = new;
321 md->phys_addr = m_end + 1;
322 md->num_pages = (end - m_end) >>
323 EFI_PAGE_SHIFT;
324 }
325
326 if ((start < m_start && m_start < end) &&
327 (end <= m_end)) {
328 /* first part */
329 md->num_pages = (m_start - md->phys_addr) >>
330 EFI_PAGE_SHIFT;
331 /* latter part */
332 new += old_memmap->desc_size;
333 memcpy(new, old, old_memmap->desc_size);
334 md = new;
335 md->phys_addr = m_start;
336 md->num_pages = (end - md->phys_addr + 1) >>
337 EFI_PAGE_SHIFT;
338 md->attribute |= m_attr;
339 }
340 }
341}