1/*
  2 * Re-map IO memory to kernel address space so that we can access it.
  3 * This is needed for high PCI addresses that aren't mapped in the
  4 * 640k-1MB IO memory area on PC's
  5 *
  6 * (C) Copyright 1995 1996 Linus Torvalds
  7 */
  8#include <linux/vmalloc.h>
  9#include <linux/mm.h>
 10#include <linux/sched.h>
 11#include <linux/io.h>
 12#include <linux/export.h>
 13#include <asm/cacheflush.h>
 14#include <asm/pgtable.h>
 15
 16#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
 17static int __read_mostly ioremap_pud_capable;
 18static int __read_mostly ioremap_pmd_capable;
 19static int __read_mostly ioremap_huge_disabled;
 20
 21static int __init set_nohugeiomap(char *str)
 22{
 23	ioremap_huge_disabled = 1;
 24	return 0;
 25}
 26early_param("nohugeiomap", set_nohugeiomap);
 27
 28void __init ioremap_huge_init(void)
 29{
 30	if (!ioremap_huge_disabled) {
 31		if (arch_ioremap_pud_supported())
 32			ioremap_pud_capable = 1;
 33		if (arch_ioremap_pmd_supported())
 34			ioremap_pmd_capable = 1;
 35	}
 36}
 37
 38static inline int ioremap_pud_enabled(void)
 39{
 40	return ioremap_pud_capable;
 41}
 42
 43static inline int ioremap_pmd_enabled(void)
 44{
 45	return ioremap_pmd_capable;
 46}
 47
 48#else	/* !CONFIG_HAVE_ARCH_HUGE_VMAP */
 49static inline int ioremap_pud_enabled(void) { return 0; }
 50static inline int ioremap_pmd_enabled(void) { return 0; }
 51#endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */
 52
 53static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,
 54		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
 55{
 56	pte_t *pte;
 57	u64 pfn;
 58
 59	pfn = phys_addr >> PAGE_SHIFT;
 60	pte = pte_alloc_kernel(pmd, addr);
 61	if (!pte)
 62		return -ENOMEM;
 63	do {
 64		BUG_ON(!pte_none(*pte));
 65		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
 66		pfn++;
 67	} while (pte++, addr += PAGE_SIZE, addr != end);
 68	return 0;
 69}
 70
 71static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
 72		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
 73{
 74	pmd_t *pmd;
 75	unsigned long next;
 76
 77	phys_addr -= addr;
 78	pmd = pmd_alloc(&init_mm, pud, addr);
 79	if (!pmd)
 80		return -ENOMEM;
 81	do {
 82		next = pmd_addr_end(addr, end);
 83
 84		if (ioremap_pmd_enabled() &&
 85		    ((next - addr) == PMD_SIZE) &&
 86		    IS_ALIGNED(phys_addr + addr, PMD_SIZE)) {
 87			if (pmd_set_huge(pmd, phys_addr + addr, prot))
 88				continue;
 89		}
 90
 91		if (ioremap_pte_range(pmd, addr, next, phys_addr + addr, prot))
 92			return -ENOMEM;
 93	} while (pmd++, addr = next, addr != end);
 94	return 0;
 95}
 96
 97static inline int ioremap_pud_range(pgd_t *pgd, unsigned long addr,
 98		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
 99{
100	pud_t *pud;
101	unsigned long next;
102
103	phys_addr -= addr;
104	pud = pud_alloc(&init_mm, pgd, addr);
105	if (!pud)
106		return -ENOMEM;
107	do {
108		next = pud_addr_end(addr, end);
109
110		if (ioremap_pud_enabled() &&
111		    ((next - addr) == PUD_SIZE) &&
112		    IS_ALIGNED(phys_addr + addr, PUD_SIZE)) {
113			if (pud_set_huge(pud, phys_addr + addr, prot))
114				continue;
115		}
116
117		if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot))
118			return -ENOMEM;
119	} while (pud++, addr = next, addr != end);
120	return 0;
121}
122
123int ioremap_page_range(unsigned long addr,
124		       unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
125{
126	pgd_t *pgd;
127	unsigned long start;
128	unsigned long next;
129	int err;
130
131	BUG_ON(addr >= end);
132
133	start = addr;
134	phys_addr -= addr;
135	pgd = pgd_offset_k(addr);
136	do {
137		next = pgd_addr_end(addr, end);
138		err = ioremap_pud_range(pgd, addr, next, phys_addr+addr, prot);
139		if (err)
140			break;
141	} while (pgd++, addr = next, addr != end);
142
143	flush_cache_vmap(start, end);
144
145	return err;
146}

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Re-map IO memory to kernel address space so that we can access it.
  4 * This is needed for high PCI addresses that aren't mapped in the
  5 * 640k-1MB IO memory area on PC's
  6 *
  7 * (C) Copyright 1995 1996 Linus Torvalds
  8 */
  9#include <linux/vmalloc.h>
 10#include <linux/mm.h>
 11#include <linux/sched.h>
 12#include <linux/io.h>
 13#include <linux/export.h>
 14#include <asm/cacheflush.h>
 15#include <asm/pgtable.h>
 16
 17#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
 18static int __read_mostly ioremap_p4d_capable;
 19static int __read_mostly ioremap_pud_capable;
 20static int __read_mostly ioremap_pmd_capable;
 21static int __read_mostly ioremap_huge_disabled;
 22
 23static int __init set_nohugeiomap(char *str)
 24{
 25	ioremap_huge_disabled = 1;
 26	return 0;
 27}
 28early_param("nohugeiomap", set_nohugeiomap);
 29
 30void __init ioremap_huge_init(void)
 31{
 32	if (!ioremap_huge_disabled) {
 33		if (arch_ioremap_p4d_supported())
 34			ioremap_p4d_capable = 1;
 35		if (arch_ioremap_pud_supported())
 36			ioremap_pud_capable = 1;
 37		if (arch_ioremap_pmd_supported())
 38			ioremap_pmd_capable = 1;
 39	}
 40}
 41
 42static inline int ioremap_p4d_enabled(void)
 43{
 44	return ioremap_p4d_capable;
 45}
 46
 47static inline int ioremap_pud_enabled(void)
 48{
 49	return ioremap_pud_capable;
 50}
 51
 52static inline int ioremap_pmd_enabled(void)
 53{
 54	return ioremap_pmd_capable;
 55}
 56
 57#else	/* !CONFIG_HAVE_ARCH_HUGE_VMAP */
 58static inline int ioremap_p4d_enabled(void) { return 0; }
 59static inline int ioremap_pud_enabled(void) { return 0; }
 60static inline int ioremap_pmd_enabled(void) { return 0; }
 61#endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */
 62
 63static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,
 64		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
 65{
 66	pte_t *pte;
 67	u64 pfn;
 68
 69	pfn = phys_addr >> PAGE_SHIFT;
 70	pte = pte_alloc_kernel(pmd, addr);
 71	if (!pte)
 72		return -ENOMEM;
 73	do {
 74		BUG_ON(!pte_none(*pte));
 75		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
 76		pfn++;
 77	} while (pte++, addr += PAGE_SIZE, addr != end);
 78	return 0;
 79}
 80
 81static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr,
 82				unsigned long end, phys_addr_t phys_addr,
 83				pgprot_t prot)
 84{
 85	if (!ioremap_pmd_enabled())
 86		return 0;
 87
 88	if ((end - addr) != PMD_SIZE)
 89		return 0;
 90
 91	if (!IS_ALIGNED(addr, PMD_SIZE))
 92		return 0;
 93
 94	if (!IS_ALIGNED(phys_addr, PMD_SIZE))
 95		return 0;
 96
 97	if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))
 98		return 0;
 99
100	return pmd_set_huge(pmd, phys_addr, prot);
101}
102
103static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
104		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
105{
106	pmd_t *pmd;
107	unsigned long next;
108
109	pmd = pmd_alloc(&init_mm, pud, addr);
110	if (!pmd)
111		return -ENOMEM;
112	do {
113		next = pmd_addr_end(addr, end);
114
115		if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot))
116			continue;
117
118		if (ioremap_pte_range(pmd, addr, next, phys_addr, prot))
119			return -ENOMEM;
120	} while (pmd++, phys_addr += (next - addr), addr = next, addr != end);
121	return 0;
122}
123
124static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr,
125				unsigned long end, phys_addr_t phys_addr,
126				pgprot_t prot)
127{
128	if (!ioremap_pud_enabled())
129		return 0;
130
131	if ((end - addr) != PUD_SIZE)
132		return 0;
133
134	if (!IS_ALIGNED(addr, PUD_SIZE))
135		return 0;
136
137	if (!IS_ALIGNED(phys_addr, PUD_SIZE))
138		return 0;
139
140	if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))
141		return 0;
142
143	return pud_set_huge(pud, phys_addr, prot);
144}
145
146static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
147		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
148{
149	pud_t *pud;
150	unsigned long next;
151
152	pud = pud_alloc(&init_mm, p4d, addr);
153	if (!pud)
154		return -ENOMEM;
155	do {
156		next = pud_addr_end(addr, end);
157
158		if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot))
159			continue;
160
161		if (ioremap_pmd_range(pud, addr, next, phys_addr, prot))
162			return -ENOMEM;
163	} while (pud++, phys_addr += (next - addr), addr = next, addr != end);
164	return 0;
165}
166
167static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr,
168				unsigned long end, phys_addr_t phys_addr,
169				pgprot_t prot)
170{
171	if (!ioremap_p4d_enabled())
172		return 0;
173
174	if ((end - addr) != P4D_SIZE)
175		return 0;
176
177	if (!IS_ALIGNED(addr, P4D_SIZE))
178		return 0;
179
180	if (!IS_ALIGNED(phys_addr, P4D_SIZE))
181		return 0;
182
183	if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))
184		return 0;
185
186	return p4d_set_huge(p4d, phys_addr, prot);
187}
188
189static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
190		unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
191{
192	p4d_t *p4d;
193	unsigned long next;
194
195	p4d = p4d_alloc(&init_mm, pgd, addr);
196	if (!p4d)
197		return -ENOMEM;
198	do {
199		next = p4d_addr_end(addr, end);
200
201		if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot))
202			continue;
203
204		if (ioremap_pud_range(p4d, addr, next, phys_addr, prot))
205			return -ENOMEM;
206	} while (p4d++, phys_addr += (next - addr), addr = next, addr != end);
207	return 0;
208}
209
210int ioremap_page_range(unsigned long addr,
211		       unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
212{
213	pgd_t *pgd;
214	unsigned long start;
215	unsigned long next;
216	int err;
217
218	might_sleep();
219	BUG_ON(addr >= end);
220
221	start = addr;
222	pgd = pgd_offset_k(addr);
223	do {
224		next = pgd_addr_end(addr, end);
225		err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot);
226		if (err)
227			break;
228	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
229
230	flush_cache_vmap(start, end);
231
232	return err;
233}