Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Bootmem core functions.
4 *
5 * Copyright (c) 2020, Bytedance.
6 *
7 * Author: Muchun Song <songmuchun@bytedance.com>
8 *
9 */
10#include <linux/mm.h>
11#include <linux/compiler.h>
12#include <linux/memblock.h>
13#include <linux/bootmem_info.h>
14#include <linux/memory_hotplug.h>
15
16void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)
17{
18 page->freelist = (void *)type;
19 SetPagePrivate(page);
20 set_page_private(page, info);
21 page_ref_inc(page);
22}
23
24void put_page_bootmem(struct page *page)
25{
26 unsigned long type;
27
28 type = (unsigned long) page->freelist;
29 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
30 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
31
32 if (page_ref_dec_return(page) == 1) {
33 page->freelist = NULL;
34 ClearPagePrivate(page);
35 set_page_private(page, 0);
36 INIT_LIST_HEAD(&page->lru);
37 free_reserved_page(page);
38 }
39}
40
41#ifndef CONFIG_SPARSEMEM_VMEMMAP
42static void register_page_bootmem_info_section(unsigned long start_pfn)
43{
44 unsigned long mapsize, section_nr, i;
45 struct mem_section *ms;
46 struct page *page, *memmap;
47 struct mem_section_usage *usage;
48
49 section_nr = pfn_to_section_nr(start_pfn);
50 ms = __nr_to_section(section_nr);
51
52 /* Get section's memmap address */
53 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
54
55 /*
56 * Get page for the memmap's phys address
57 * XXX: need more consideration for sparse_vmemmap...
58 */
59 page = virt_to_page(memmap);
60 mapsize = sizeof(struct page) * PAGES_PER_SECTION;
61 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
62
63 /* remember memmap's page */
64 for (i = 0; i < mapsize; i++, page++)
65 get_page_bootmem(section_nr, page, SECTION_INFO);
66
67 usage = ms->usage;
68 page = virt_to_page(usage);
69
70 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
71
72 for (i = 0; i < mapsize; i++, page++)
73 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
74
75}
76#else /* CONFIG_SPARSEMEM_VMEMMAP */
77static void register_page_bootmem_info_section(unsigned long start_pfn)
78{
79 unsigned long mapsize, section_nr, i;
80 struct mem_section *ms;
81 struct page *page, *memmap;
82 struct mem_section_usage *usage;
83
84 section_nr = pfn_to_section_nr(start_pfn);
85 ms = __nr_to_section(section_nr);
86
87 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
88
89 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
90
91 usage = ms->usage;
92 page = virt_to_page(usage);
93
94 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
95
96 for (i = 0; i < mapsize; i++, page++)
97 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
98}
99#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
100
101void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
102{
103 unsigned long i, pfn, end_pfn, nr_pages;
104 int node = pgdat->node_id;
105 struct page *page;
106
107 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
108 page = virt_to_page(pgdat);
109
110 for (i = 0; i < nr_pages; i++, page++)
111 get_page_bootmem(node, page, NODE_INFO);
112
113 pfn = pgdat->node_start_pfn;
114 end_pfn = pgdat_end_pfn(pgdat);
115
116 /* register section info */
117 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
118 /*
119 * Some platforms can assign the same pfn to multiple nodes - on
120 * node0 as well as nodeN. To avoid registering a pfn against
121 * multiple nodes we check that this pfn does not already
122 * reside in some other nodes.
123 */
124 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
125 register_page_bootmem_info_section(pfn);
126 }
127}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Bootmem core functions.
4 *
5 * Copyright (c) 2020, Bytedance.
6 *
7 * Author: Muchun Song <songmuchun@bytedance.com>
8 *
9 */
10#include <linux/mm.h>
11#include <linux/compiler.h>
12#include <linux/memblock.h>
13#include <linux/bootmem_info.h>
14#include <linux/memory_hotplug.h>
15#include <linux/kmemleak.h>
16
17void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)
18{
19 page->index = type;
20 SetPagePrivate(page);
21 set_page_private(page, info);
22 page_ref_inc(page);
23}
24
25void put_page_bootmem(struct page *page)
26{
27 unsigned long type = page->index;
28
29 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
30 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
31
32 if (page_ref_dec_return(page) == 1) {
33 page->index = 0;
34 ClearPagePrivate(page);
35 set_page_private(page, 0);
36 INIT_LIST_HEAD(&page->lru);
37 kmemleak_free_part(page_to_virt(page), PAGE_SIZE);
38 free_reserved_page(page);
39 }
40}
41
42#ifndef CONFIG_SPARSEMEM_VMEMMAP
43static void __init register_page_bootmem_info_section(unsigned long start_pfn)
44{
45 unsigned long mapsize, section_nr, i;
46 struct mem_section *ms;
47 struct page *page, *memmap;
48 struct mem_section_usage *usage;
49
50 section_nr = pfn_to_section_nr(start_pfn);
51 ms = __nr_to_section(section_nr);
52
53 /* Get section's memmap address */
54 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
55
56 /*
57 * Get page for the memmap's phys address
58 * XXX: need more consideration for sparse_vmemmap...
59 */
60 page = virt_to_page(memmap);
61 mapsize = sizeof(struct page) * PAGES_PER_SECTION;
62 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
63
64 /* remember memmap's page */
65 for (i = 0; i < mapsize; i++, page++)
66 get_page_bootmem(section_nr, page, SECTION_INFO);
67
68 usage = ms->usage;
69 page = virt_to_page(usage);
70
71 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
72
73 for (i = 0; i < mapsize; i++, page++)
74 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
75
76}
77#else /* CONFIG_SPARSEMEM_VMEMMAP */
78static void __init register_page_bootmem_info_section(unsigned long start_pfn)
79{
80 unsigned long mapsize, section_nr, i;
81 struct mem_section *ms;
82 struct page *page, *memmap;
83 struct mem_section_usage *usage;
84
85 section_nr = pfn_to_section_nr(start_pfn);
86 ms = __nr_to_section(section_nr);
87
88 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
89
90 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
91
92 usage = ms->usage;
93 page = virt_to_page(usage);
94
95 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
96
97 for (i = 0; i < mapsize; i++, page++)
98 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
99}
100#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
101
102void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
103{
104 unsigned long i, pfn, end_pfn, nr_pages;
105 int node = pgdat->node_id;
106 struct page *page;
107
108 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
109 page = virt_to_page(pgdat);
110
111 for (i = 0; i < nr_pages; i++, page++)
112 get_page_bootmem(node, page, NODE_INFO);
113
114 pfn = pgdat->node_start_pfn;
115 end_pfn = pgdat_end_pfn(pgdat);
116
117 /* register section info */
118 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
119 /*
120 * Some platforms can assign the same pfn to multiple nodes - on
121 * node0 as well as nodeN. To avoid registering a pfn against
122 * multiple nodes we check that this pfn does not already
123 * reside in some other nodes.
124 */
125 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
126 register_page_bootmem_info_section(pfn);
127 }
128}