Linux Audio

Check our new training course

Loading...
v6.13.7
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Helper routines for building identity mapping page tables. This is
  4 * included by both the compressed kernel and the regular kernel.
  5 */
  6
  7static void free_pte(struct x86_mapping_info *info, pmd_t *pmd)
  8{
  9	pte_t *pte = pte_offset_kernel(pmd, 0);
 10
 11	info->free_pgt_page(pte, info->context);
 12}
 13
 14static void free_pmd(struct x86_mapping_info *info, pud_t *pud)
 15{
 16	pmd_t *pmd = pmd_offset(pud, 0);
 17	int i;
 18
 19	for (i = 0; i < PTRS_PER_PMD; i++) {
 20		if (!pmd_present(pmd[i]))
 21			continue;
 22
 23		if (pmd_leaf(pmd[i]))
 24			continue;
 25
 26		free_pte(info, &pmd[i]);
 27	}
 28
 29	info->free_pgt_page(pmd, info->context);
 30}
 31
 32static void free_pud(struct x86_mapping_info *info, p4d_t *p4d)
 33{
 34	pud_t *pud = pud_offset(p4d, 0);
 35	int i;
 36
 37	for (i = 0; i < PTRS_PER_PUD; i++) {
 38		if (!pud_present(pud[i]))
 39			continue;
 40
 41		if (pud_leaf(pud[i]))
 42			continue;
 43
 44		free_pmd(info, &pud[i]);
 45	}
 46
 47	info->free_pgt_page(pud, info->context);
 48}
 49
 50static void free_p4d(struct x86_mapping_info *info, pgd_t *pgd)
 51{
 52	p4d_t *p4d = p4d_offset(pgd, 0);
 53	int i;
 54
 55	for (i = 0; i < PTRS_PER_P4D; i++) {
 56		if (!p4d_present(p4d[i]))
 57			continue;
 58
 59		free_pud(info, &p4d[i]);
 60	}
 61
 62	if (pgtable_l5_enabled())
 63		info->free_pgt_page(p4d, info->context);
 64}
 65
 66void kernel_ident_mapping_free(struct x86_mapping_info *info, pgd_t *pgd)
 67{
 68	int i;
 69
 70	for (i = 0; i < PTRS_PER_PGD; i++) {
 71		if (!pgd_present(pgd[i]))
 72			continue;
 73
 74		free_p4d(info, &pgd[i]);
 75	}
 76
 77	info->free_pgt_page(pgd, info->context);
 78}
 79
 80static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
 81			   unsigned long addr, unsigned long end)
 82{
 83	addr &= PMD_MASK;
 84	for (; addr < end; addr += PMD_SIZE) {
 85		pmd_t *pmd = pmd_page + pmd_index(addr);
 86
 87		if (pmd_present(*pmd))
 88			continue;
 89
 90		set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
 91	}
 92}
 93
 94static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
 95			  unsigned long addr, unsigned long end)
 96{
 97	unsigned long next;
 98
 99	for (; addr < end; addr = next) {
100		pud_t *pud = pud_page + pud_index(addr);
101		pmd_t *pmd;
102		bool use_gbpage;
103
104		next = (addr & PUD_MASK) + PUD_SIZE;
105		if (next > end)
106			next = end;
107
108		/* if this is already a gbpage, this portion is already mapped */
109		if (pud_leaf(*pud))
110			continue;
111
112		/* Is using a gbpage allowed? */
113		use_gbpage = info->direct_gbpages;
114
115		/* Don't use gbpage if it maps more than the requested region. */
116		/* at the begining: */
117		use_gbpage &= ((addr & ~PUD_MASK) == 0);
118		/* ... or at the end: */
119		use_gbpage &= ((next & ~PUD_MASK) == 0);
120
121		/* Never overwrite existing mappings */
122		use_gbpage &= !pud_present(*pud);
123
124		if (use_gbpage) {
125			pud_t pudval;
126
127			pudval = __pud((addr - info->offset) | info->page_flag);
128			set_pud(pud, pudval);
129			continue;
130		}
131
132		if (pud_present(*pud)) {
133			pmd = pmd_offset(pud, 0);
134			ident_pmd_init(info, pmd, addr, next);
135			continue;
136		}
137		pmd = (pmd_t *)info->alloc_pgt_page(info->context);
138		if (!pmd)
139			return -ENOMEM;
140		ident_pmd_init(info, pmd, addr, next);
141		set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
142	}
143
144	return 0;
145}
146
147static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
148			  unsigned long addr, unsigned long end)
149{
150	unsigned long next;
151	int result;
152
153	for (; addr < end; addr = next) {
154		p4d_t *p4d = p4d_page + p4d_index(addr);
155		pud_t *pud;
156
157		next = (addr & P4D_MASK) + P4D_SIZE;
158		if (next > end)
159			next = end;
160
161		if (p4d_present(*p4d)) {
162			pud = pud_offset(p4d, 0);
163			result = ident_pud_init(info, pud, addr, next);
164			if (result)
165				return result;
166
167			continue;
168		}
169		pud = (pud_t *)info->alloc_pgt_page(info->context);
170		if (!pud)
171			return -ENOMEM;
172
173		result = ident_pud_init(info, pud, addr, next);
174		if (result)
175			return result;
176
177		set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag | _PAGE_NOPTISHADOW));
178	}
179
180	return 0;
181}
182
183int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
184			      unsigned long pstart, unsigned long pend)
185{
186	unsigned long addr = pstart + info->offset;
187	unsigned long end = pend + info->offset;
188	unsigned long next;
189	int result;
190
191	/* Set the default pagetable flags if not supplied */
192	if (!info->kernpg_flag)
193		info->kernpg_flag = _KERNPG_TABLE;
194
195	/* Filter out unsupported __PAGE_KERNEL_* bits: */
196	info->kernpg_flag &= __default_kernel_pte_mask;
197
198	for (; addr < end; addr = next) {
199		pgd_t *pgd = pgd_page + pgd_index(addr);
200		p4d_t *p4d;
201
202		next = (addr & PGDIR_MASK) + PGDIR_SIZE;
203		if (next > end)
204			next = end;
205
206		if (pgd_present(*pgd)) {
207			p4d = p4d_offset(pgd, 0);
208			result = ident_p4d_init(info, p4d, addr, next);
209			if (result)
210				return result;
211			continue;
212		}
213
214		p4d = (p4d_t *)info->alloc_pgt_page(info->context);
215		if (!p4d)
216			return -ENOMEM;
217		result = ident_p4d_init(info, p4d, addr, next);
218		if (result)
219			return result;
220		if (pgtable_l5_enabled()) {
221			set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag | _PAGE_NOPTISHADOW));
222		} else {
223			/*
224			 * With p4d folded, pgd is equal to p4d.
225			 * The pgd entry has to point to the pud page table in this case.
226			 */
227			pud_t *pud = pud_offset(p4d, 0);
228			set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag | _PAGE_NOPTISHADOW));
229		}
230	}
231
232	return 0;
233}
v6.8
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Helper routines for building identity mapping page tables. This is
  4 * included by both the compressed kernel and the regular kernel.
  5 */
  6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  7static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
  8			   unsigned long addr, unsigned long end)
  9{
 10	addr &= PMD_MASK;
 11	for (; addr < end; addr += PMD_SIZE) {
 12		pmd_t *pmd = pmd_page + pmd_index(addr);
 13
 14		if (pmd_present(*pmd))
 15			continue;
 16
 17		set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
 18	}
 19}
 20
 21static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
 22			  unsigned long addr, unsigned long end)
 23{
 24	unsigned long next;
 25
 26	for (; addr < end; addr = next) {
 27		pud_t *pud = pud_page + pud_index(addr);
 28		pmd_t *pmd;
 29		bool use_gbpage;
 30
 31		next = (addr & PUD_MASK) + PUD_SIZE;
 32		if (next > end)
 33			next = end;
 34
 35		/* if this is already a gbpage, this portion is already mapped */
 36		if (pud_large(*pud))
 37			continue;
 38
 39		/* Is using a gbpage allowed? */
 40		use_gbpage = info->direct_gbpages;
 41
 42		/* Don't use gbpage if it maps more than the requested region. */
 43		/* at the begining: */
 44		use_gbpage &= ((addr & ~PUD_MASK) == 0);
 45		/* ... or at the end: */
 46		use_gbpage &= ((next & ~PUD_MASK) == 0);
 47
 48		/* Never overwrite existing mappings */
 49		use_gbpage &= !pud_present(*pud);
 50
 51		if (use_gbpage) {
 52			pud_t pudval;
 53
 54			pudval = __pud((addr - info->offset) | info->page_flag);
 55			set_pud(pud, pudval);
 56			continue;
 57		}
 58
 59		if (pud_present(*pud)) {
 60			pmd = pmd_offset(pud, 0);
 61			ident_pmd_init(info, pmd, addr, next);
 62			continue;
 63		}
 64		pmd = (pmd_t *)info->alloc_pgt_page(info->context);
 65		if (!pmd)
 66			return -ENOMEM;
 67		ident_pmd_init(info, pmd, addr, next);
 68		set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
 69	}
 70
 71	return 0;
 72}
 73
 74static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
 75			  unsigned long addr, unsigned long end)
 76{
 77	unsigned long next;
 78	int result;
 79
 80	for (; addr < end; addr = next) {
 81		p4d_t *p4d = p4d_page + p4d_index(addr);
 82		pud_t *pud;
 83
 84		next = (addr & P4D_MASK) + P4D_SIZE;
 85		if (next > end)
 86			next = end;
 87
 88		if (p4d_present(*p4d)) {
 89			pud = pud_offset(p4d, 0);
 90			result = ident_pud_init(info, pud, addr, next);
 91			if (result)
 92				return result;
 93
 94			continue;
 95		}
 96		pud = (pud_t *)info->alloc_pgt_page(info->context);
 97		if (!pud)
 98			return -ENOMEM;
 99
100		result = ident_pud_init(info, pud, addr, next);
101		if (result)
102			return result;
103
104		set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag));
105	}
106
107	return 0;
108}
109
110int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
111			      unsigned long pstart, unsigned long pend)
112{
113	unsigned long addr = pstart + info->offset;
114	unsigned long end = pend + info->offset;
115	unsigned long next;
116	int result;
117
118	/* Set the default pagetable flags if not supplied */
119	if (!info->kernpg_flag)
120		info->kernpg_flag = _KERNPG_TABLE;
121
122	/* Filter out unsupported __PAGE_KERNEL_* bits: */
123	info->kernpg_flag &= __default_kernel_pte_mask;
124
125	for (; addr < end; addr = next) {
126		pgd_t *pgd = pgd_page + pgd_index(addr);
127		p4d_t *p4d;
128
129		next = (addr & PGDIR_MASK) + PGDIR_SIZE;
130		if (next > end)
131			next = end;
132
133		if (pgd_present(*pgd)) {
134			p4d = p4d_offset(pgd, 0);
135			result = ident_p4d_init(info, p4d, addr, next);
136			if (result)
137				return result;
138			continue;
139		}
140
141		p4d = (p4d_t *)info->alloc_pgt_page(info->context);
142		if (!p4d)
143			return -ENOMEM;
144		result = ident_p4d_init(info, p4d, addr, next);
145		if (result)
146			return result;
147		if (pgtable_l5_enabled()) {
148			set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
149		} else {
150			/*
151			 * With p4d folded, pgd is equal to p4d.
152			 * The pgd entry has to point to the pud page table in this case.
153			 */
154			pud_t *pud = pud_offset(p4d, 0);
155			set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag));
156		}
157	}
158
159	return 0;
160}