1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/arch/arm/mm/mmu.c
   4 *
   5 *  Copyright (C) 1995-2005 Russell King
 
 
 
 
   6 */
   7#include <linux/module.h>
   8#include <linux/kernel.h>
   9#include <linux/errno.h>
  10#include <linux/init.h>
  11#include <linux/mman.h>
  12#include <linux/nodemask.h>
  13#include <linux/memblock.h>
  14#include <linux/fs.h>
  15#include <linux/vmalloc.h>
  16#include <linux/sizes.h>
  17
  18#include <asm/cp15.h>
  19#include <asm/cputype.h>
  20#include <asm/sections.h>
  21#include <asm/cachetype.h>
  22#include <asm/fixmap.h>
  23#include <asm/sections.h>
  24#include <asm/setup.h>
 
  25#include <asm/smp_plat.h>
  26#include <asm/tlb.h>
  27#include <asm/highmem.h>
  28#include <asm/system_info.h>
  29#include <asm/traps.h>
  30#include <asm/procinfo.h>
  31#include <asm/memory.h>
  32
  33#include <asm/mach/arch.h>
  34#include <asm/mach/map.h>
  35#include <asm/mach/pci.h>
  36#include <asm/fixmap.h>
  37
  38#include "fault.h"
  39#include "mm.h"
  40#include "tcm.h"
  41
  42/*
  43 * empty_zero_page is a special page that is used for
  44 * zero-initialized data and COW.
  45 */
  46struct page *empty_zero_page;
  47EXPORT_SYMBOL(empty_zero_page);
  48
  49/*
  50 * The pmd table for the upper-most set of pages.
  51 */
  52pmd_t *top_pmd;
  53
  54pmdval_t user_pmd_table = _PAGE_USER_TABLE;
  55
  56#define CPOLICY_UNCACHED	0
  57#define CPOLICY_BUFFERED	1
  58#define CPOLICY_WRITETHROUGH	2
  59#define CPOLICY_WRITEBACK	3
  60#define CPOLICY_WRITEALLOC	4
  61
  62static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
  63static unsigned int ecc_mask __initdata = 0;
  64pgprot_t pgprot_user;
  65pgprot_t pgprot_kernel;
  66pgprot_t pgprot_hyp_device;
  67pgprot_t pgprot_s2;
  68pgprot_t pgprot_s2_device;
  69
  70EXPORT_SYMBOL(pgprot_user);
  71EXPORT_SYMBOL(pgprot_kernel);
  72
  73struct cachepolicy {
  74	const char	policy[16];
  75	unsigned int	cr_mask;
  76	pmdval_t	pmd;
  77	pteval_t	pte;
  78	pteval_t	pte_s2;
  79};
  80
  81#ifdef CONFIG_ARM_LPAE
  82#define s2_policy(policy)	policy
  83#else
  84#define s2_policy(policy)	0
  85#endif
  86
  87unsigned long kimage_voffset __ro_after_init;
  88
  89static struct cachepolicy cache_policies[] __initdata = {
  90	{
  91		.policy		= "uncached",
  92		.cr_mask	= CR_W|CR_C,
  93		.pmd		= PMD_SECT_UNCACHED,
  94		.pte		= L_PTE_MT_UNCACHED,
  95		.pte_s2		= s2_policy(L_PTE_S2_MT_UNCACHED),
  96	}, {
  97		.policy		= "buffered",
  98		.cr_mask	= CR_C,
  99		.pmd		= PMD_SECT_BUFFERED,
 100		.pte		= L_PTE_MT_BUFFERABLE,
 101		.pte_s2		= s2_policy(L_PTE_S2_MT_UNCACHED),
 102	}, {
 103		.policy		= "writethrough",
 104		.cr_mask	= 0,
 105		.pmd		= PMD_SECT_WT,
 106		.pte		= L_PTE_MT_WRITETHROUGH,
 107		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITETHROUGH),
 108	}, {
 109		.policy		= "writeback",
 110		.cr_mask	= 0,
 111		.pmd		= PMD_SECT_WB,
 112		.pte		= L_PTE_MT_WRITEBACK,
 113		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITEBACK),
 114	}, {
 115		.policy		= "writealloc",
 116		.cr_mask	= 0,
 117		.pmd		= PMD_SECT_WBWA,
 118		.pte		= L_PTE_MT_WRITEALLOC,
 119		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITEBACK),
 120	}
 121};
 122
 123#ifdef CONFIG_CPU_CP15
 124static unsigned long initial_pmd_value __initdata = 0;
 125
 126/*
 127 * Initialise the cache_policy variable with the initial state specified
 128 * via the "pmd" value.  This is used to ensure that on ARMv6 and later,
 129 * the C code sets the page tables up with the same policy as the head
 130 * assembly code, which avoids an illegal state where the TLBs can get
 131 * confused.  See comments in early_cachepolicy() for more information.
 132 */
 133void __init init_default_cache_policy(unsigned long pmd)
 134{
 135	int i;
 136
 137	initial_pmd_value = pmd;
 138
 139	pmd &= PMD_SECT_CACHE_MASK;
 140
 141	for (i = 0; i < ARRAY_SIZE(cache_policies); i++)
 142		if (cache_policies[i].pmd == pmd) {
 143			cachepolicy = i;
 144			break;
 145		}
 146
 147	if (i == ARRAY_SIZE(cache_policies))
 148		pr_err("ERROR: could not find cache policy\n");
 149}
 150
 151/*
 152 * These are useful for identifying cache coherency problems by allowing
 153 * the cache or the cache and writebuffer to be turned off.  (Note: the
 154 * write buffer should not be on and the cache off).
 
 155 */
 156static int __init early_cachepolicy(char *p)
 157{
 158	int i, selected = -1;
 159
 160	for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
 161		int len = strlen(cache_policies[i].policy);
 162
 163		if (memcmp(p, cache_policies[i].policy, len) == 0) {
 164			selected = i;
 
 
 165			break;
 166		}
 167	}
 168
 169	if (selected == -1)
 170		pr_err("ERROR: unknown or unsupported cache policy\n");
 171
 172	/*
 173	 * This restriction is partly to do with the way we boot; it is
 174	 * unpredictable to have memory mapped using two different sets of
 175	 * memory attributes (shared, type, and cache attribs).  We can not
 176	 * change these attributes once the initial assembly has setup the
 177	 * page tables.
 178	 */
 179	if (cpu_architecture() >= CPU_ARCH_ARMv6 && selected != cachepolicy) {
 180		pr_warn("Only cachepolicy=%s supported on ARMv6 and later\n",
 181			cache_policies[cachepolicy].policy);
 182		return 0;
 183	}
 184
 185	if (selected != cachepolicy) {
 186		unsigned long cr = __clear_cr(cache_policies[selected].cr_mask);
 187		cachepolicy = selected;
 188		flush_cache_all();
 189		set_cr(cr);
 190	}
 
 
 191	return 0;
 192}
 193early_param("cachepolicy", early_cachepolicy);
 194
 195static int __init early_nocache(char *__unused)
 196{
 197	char *p = "buffered";
 198	pr_warn("nocache is deprecated; use cachepolicy=%s\n", p);
 199	early_cachepolicy(p);
 200	return 0;
 201}
 202early_param("nocache", early_nocache);
 203
 204static int __init early_nowrite(char *__unused)
 205{
 206	char *p = "uncached";
 207	pr_warn("nowb is deprecated; use cachepolicy=%s\n", p);
 208	early_cachepolicy(p);
 209	return 0;
 210}
 211early_param("nowb", early_nowrite);
 212
 213#ifndef CONFIG_ARM_LPAE
 214static int __init early_ecc(char *p)
 215{
 216	if (memcmp(p, "on", 2) == 0)
 217		ecc_mask = PMD_PROTECTION;
 218	else if (memcmp(p, "off", 3) == 0)
 219		ecc_mask = 0;
 220	return 0;
 221}
 222early_param("ecc", early_ecc);
 223#endif
 224
 225#else /* ifdef CONFIG_CPU_CP15 */
 226
 227static int __init early_cachepolicy(char *p)
 228{
 229	pr_warn("cachepolicy kernel parameter not supported without cp15\n");
 230}
 231early_param("cachepolicy", early_cachepolicy);
 232
 233static int __init noalign_setup(char *__unused)
 234{
 235	pr_warn("noalign kernel parameter not supported without cp15\n");
 
 
 
 236}
 237__setup("noalign", noalign_setup);
 238
 239#endif /* ifdef CONFIG_CPU_CP15 / else */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 240
 241#define PROT_PTE_DEVICE		L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
 242#define PROT_PTE_S2_DEVICE	PROT_PTE_DEVICE
 243#define PROT_SECT_DEVICE	PMD_TYPE_SECT|PMD_SECT_AP_WRITE
 244
 245static struct mem_type mem_types[] __ro_after_init = {
 246	[MT_DEVICE] = {		  /* Strongly ordered / ARMv6 shared device */
 247		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
 248				  L_PTE_SHARED,
 249		.prot_pte_s2	= s2_policy(PROT_PTE_S2_DEVICE) |
 250				  s2_policy(L_PTE_S2_MT_DEV_SHARED) |
 251				  L_PTE_SHARED,
 252		.prot_l1	= PMD_TYPE_TABLE,
 253		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_S,
 254		.domain		= DOMAIN_IO,
 255	},
 256	[MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
 257		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
 258		.prot_l1	= PMD_TYPE_TABLE,
 259		.prot_sect	= PROT_SECT_DEVICE,
 260		.domain		= DOMAIN_IO,
 261	},
 262	[MT_DEVICE_CACHED] = {	  /* ioremap_cached */
 263		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
 264		.prot_l1	= PMD_TYPE_TABLE,
 265		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_WB,
 266		.domain		= DOMAIN_IO,
 267	},
 268	[MT_DEVICE_WC] = {	/* ioremap_wc */
 269		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
 270		.prot_l1	= PMD_TYPE_TABLE,
 271		.prot_sect	= PROT_SECT_DEVICE,
 272		.domain		= DOMAIN_IO,
 273	},
 274	[MT_UNCACHED] = {
 275		.prot_pte	= PROT_PTE_DEVICE,
 276		.prot_l1	= PMD_TYPE_TABLE,
 277		.prot_sect	= PMD_TYPE_SECT | PMD_SECT_XN,
 278		.domain		= DOMAIN_IO,
 279	},
 280	[MT_CACHECLEAN] = {
 281		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
 282		.domain    = DOMAIN_KERNEL,
 283	},
 284#ifndef CONFIG_ARM_LPAE
 285	[MT_MINICLEAN] = {
 286		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
 287		.domain    = DOMAIN_KERNEL,
 288	},
 289#endif
 290	[MT_LOW_VECTORS] = {
 291		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 292				L_PTE_RDONLY,
 293		.prot_l1   = PMD_TYPE_TABLE,
 294		.domain    = DOMAIN_VECTORS,
 295	},
 296	[MT_HIGH_VECTORS] = {
 297		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 298				L_PTE_USER | L_PTE_RDONLY,
 299		.prot_l1   = PMD_TYPE_TABLE,
 300		.domain    = DOMAIN_VECTORS,
 301	},
 302	[MT_MEMORY_RWX] = {
 303		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
 304		.prot_l1   = PMD_TYPE_TABLE,
 305		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 306		.domain    = DOMAIN_KERNEL,
 307	},
 308	[MT_MEMORY_RW] = {
 309		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 310			     L_PTE_XN,
 311		.prot_l1   = PMD_TYPE_TABLE,
 312		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 313		.domain    = DOMAIN_KERNEL,
 314	},
 315	[MT_ROM] = {
 316		.prot_sect = PMD_TYPE_SECT,
 317		.domain    = DOMAIN_KERNEL,
 318	},
 319	[MT_MEMORY_RWX_NONCACHED] = {
 320		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 321				L_PTE_MT_BUFFERABLE,
 322		.prot_l1   = PMD_TYPE_TABLE,
 323		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 324		.domain    = DOMAIN_KERNEL,
 325	},
 326	[MT_MEMORY_RW_DTCM] = {
 327		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 328				L_PTE_XN,
 329		.prot_l1   = PMD_TYPE_TABLE,
 330		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
 331		.domain    = DOMAIN_KERNEL,
 332	},
 333	[MT_MEMORY_RWX_ITCM] = {
 334		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
 335		.prot_l1   = PMD_TYPE_TABLE,
 336		.domain    = DOMAIN_KERNEL,
 337	},
 338	[MT_MEMORY_RW_SO] = {
 339		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 340				L_PTE_MT_UNCACHED | L_PTE_XN,
 341		.prot_l1   = PMD_TYPE_TABLE,
 342		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
 343				PMD_SECT_UNCACHED | PMD_SECT_XN,
 344		.domain    = DOMAIN_KERNEL,
 345	},
 346	[MT_MEMORY_DMA_READY] = {
 347		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 348				L_PTE_XN,
 349		.prot_l1   = PMD_TYPE_TABLE,
 350		.domain    = DOMAIN_KERNEL,
 351	},
 352};
 353
 354const struct mem_type *get_mem_type(unsigned int type)
 355{
 356	return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
 357}
 358EXPORT_SYMBOL(get_mem_type);
 359
 360static pte_t *(*pte_offset_fixmap)(pmd_t *dir, unsigned long addr);
 361
 362static pte_t bm_pte[PTRS_PER_PTE + PTE_HWTABLE_PTRS]
 363	__aligned(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE) __initdata;
 364
 365static pte_t * __init pte_offset_early_fixmap(pmd_t *dir, unsigned long addr)
 366{
 367	return &bm_pte[pte_index(addr)];
 368}
 369
 370static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr)
 371{
 372	return pte_offset_kernel(dir, addr);
 373}
 374
 375static inline pmd_t * __init fixmap_pmd(unsigned long addr)
 376{
 377	pgd_t *pgd = pgd_offset_k(addr);
 378	pud_t *pud = pud_offset(pgd, addr);
 379	pmd_t *pmd = pmd_offset(pud, addr);
 380
 381	return pmd;
 382}
 383
 384void __init early_fixmap_init(void)
 385{
 386	pmd_t *pmd;
 387
 388	/*
 389	 * The early fixmap range spans multiple pmds, for which
 390	 * we are not prepared:
 391	 */
 392	BUILD_BUG_ON((__fix_to_virt(__end_of_early_ioremap_region) >> PMD_SHIFT)
 393		     != FIXADDR_TOP >> PMD_SHIFT);
 394
 395	pmd = fixmap_pmd(FIXADDR_TOP);
 396	pmd_populate_kernel(&init_mm, pmd, bm_pte);
 397
 398	pte_offset_fixmap = pte_offset_early_fixmap;
 399}
 400
 401/*
 402 * To avoid TLB flush broadcasts, this uses local_flush_tlb_kernel_range().
 403 * As a result, this can only be called with preemption disabled, as under
 404 * stop_machine().
 405 */
 406void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
 407{
 408	unsigned long vaddr = __fix_to_virt(idx);
 409	pte_t *pte = pte_offset_fixmap(pmd_off_k(vaddr), vaddr);
 410
 411	/* Make sure fixmap region does not exceed available allocation. */
 412	BUILD_BUG_ON(FIXADDR_START + (__end_of_fixed_addresses * PAGE_SIZE) >
 413		     FIXADDR_END);
 414	BUG_ON(idx >= __end_of_fixed_addresses);
 415
 416	/* we only support device mappings until pgprot_kernel has been set */
 417	if (WARN_ON(pgprot_val(prot) != pgprot_val(FIXMAP_PAGE_IO) &&
 418		    pgprot_val(pgprot_kernel) == 0))
 419		return;
 420
 421	if (pgprot_val(prot))
 422		set_pte_at(NULL, vaddr, pte,
 423			pfn_pte(phys >> PAGE_SHIFT, prot));
 424	else
 425		pte_clear(NULL, vaddr, pte);
 426	local_flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE);
 427}
 428
 429/*
 430 * Adjust the PMD section entries according to the CPU in use.
 431 */
 432static void __init build_mem_type_table(void)
 433{
 434	struct cachepolicy *cp;
 435	unsigned int cr = get_cr();
 436	pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
 437	pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
 438	int cpu_arch = cpu_architecture();
 439	int i;
 440
 441	if (cpu_arch < CPU_ARCH_ARMv6) {
 442#if defined(CONFIG_CPU_DCACHE_DISABLE)
 443		if (cachepolicy > CPOLICY_BUFFERED)
 444			cachepolicy = CPOLICY_BUFFERED;
 445#elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
 446		if (cachepolicy > CPOLICY_WRITETHROUGH)
 447			cachepolicy = CPOLICY_WRITETHROUGH;
 448#endif
 449	}
 450	if (cpu_arch < CPU_ARCH_ARMv5) {
 451		if (cachepolicy >= CPOLICY_WRITEALLOC)
 452			cachepolicy = CPOLICY_WRITEBACK;
 453		ecc_mask = 0;
 454	}
 455
 456	if (is_smp()) {
 457		if (cachepolicy != CPOLICY_WRITEALLOC) {
 458			pr_warn("Forcing write-allocate cache policy for SMP\n");
 459			cachepolicy = CPOLICY_WRITEALLOC;
 460		}
 461		if (!(initial_pmd_value & PMD_SECT_S)) {
 462			pr_warn("Forcing shared mappings for SMP\n");
 463			initial_pmd_value |= PMD_SECT_S;
 464		}
 465	}
 466
 467	/*
 468	 * Strip out features not present on earlier architectures.
 469	 * Pre-ARMv5 CPUs don't have TEX bits.  Pre-ARMv6 CPUs or those
 470	 * without extended page tables don't have the 'Shared' bit.
 471	 */
 472	if (cpu_arch < CPU_ARCH_ARMv5)
 473		for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 474			mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
 475	if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
 476		for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 477			mem_types[i].prot_sect &= ~PMD_SECT_S;
 478
 479	/*
 480	 * ARMv5 and lower, bit 4 must be set for page tables (was: cache
 481	 * "update-able on write" bit on ARM610).  However, Xscale and
 482	 * Xscale3 require this bit to be cleared.
 483	 */
 484	if (cpu_is_xscale_family()) {
 485		for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 486			mem_types[i].prot_sect &= ~PMD_BIT4;
 487			mem_types[i].prot_l1 &= ~PMD_BIT4;
 488		}
 489	} else if (cpu_arch < CPU_ARCH_ARMv6) {
 490		for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 491			if (mem_types[i].prot_l1)
 492				mem_types[i].prot_l1 |= PMD_BIT4;
 493			if (mem_types[i].prot_sect)
 494				mem_types[i].prot_sect |= PMD_BIT4;
 495		}
 496	}
 497
 498	/*
 499	 * Mark the device areas according to the CPU/architecture.
 500	 */
 501	if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
 502		if (!cpu_is_xsc3()) {
 503			/*
 504			 * Mark device regions on ARMv6+ as execute-never
 505			 * to prevent speculative instruction fetches.
 506			 */
 507			mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
 508			mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
 509			mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
 510			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
 511
 512			/* Also setup NX memory mapping */
 513			mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_XN;
 514		}
 515		if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
 516			/*
 517			 * For ARMv7 with TEX remapping,
 518			 * - shared device is SXCB=1100
 519			 * - nonshared device is SXCB=0100
 520			 * - write combine device mem is SXCB=0001
 521			 * (Uncached Normal memory)
 522			 */
 523			mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
 524			mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
 525			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
 526		} else if (cpu_is_xsc3()) {
 527			/*
 528			 * For Xscale3,
 529			 * - shared device is TEXCB=00101
 530			 * - nonshared device is TEXCB=01000
 531			 * - write combine device mem is TEXCB=00100
 532			 * (Inner/Outer Uncacheable in xsc3 parlance)
 533			 */
 534			mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
 535			mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
 536			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
 537		} else {
 538			/*
 539			 * For ARMv6 and ARMv7 without TEX remapping,
 540			 * - shared device is TEXCB=00001
 541			 * - nonshared device is TEXCB=01000
 542			 * - write combine device mem is TEXCB=00100
 543			 * (Uncached Normal in ARMv6 parlance).
 544			 */
 545			mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
 546			mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
 547			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
 548		}
 549	} else {
 550		/*
 551		 * On others, write combining is "Uncached/Buffered"
 552		 */
 553		mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
 554	}
 555
 556	/*
 557	 * Now deal with the memory-type mappings
 558	 */
 559	cp = &cache_policies[cachepolicy];
 560	vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
 561	s2_pgprot = cp->pte_s2;
 562	hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte;
 563	s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2;
 564
 565#ifndef CONFIG_ARM_LPAE
 566	/*
 567	 * We don't use domains on ARMv6 (since this causes problems with
 568	 * v6/v7 kernels), so we must use a separate memory type for user
 569	 * r/o, kernel r/w to map the vectors page.
 570	 */
 571	if (cpu_arch == CPU_ARCH_ARMv6)
 572		vecs_pgprot |= L_PTE_MT_VECTORS;
 573
 574	/*
 575	 * Check is it with support for the PXN bit
 576	 * in the Short-descriptor translation table format descriptors.
 577	 */
 578	if (cpu_arch == CPU_ARCH_ARMv7 &&
 579		(read_cpuid_ext(CPUID_EXT_MMFR0) & 0xF) >= 4) {
 580		user_pmd_table |= PMD_PXNTABLE;
 
 
 
 581	}
 582#endif
 583
 584	/*
 585	 * ARMv6 and above have extended page tables.
 586	 */
 587	if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
 588#ifndef CONFIG_ARM_LPAE
 589		/*
 590		 * Mark cache clean areas and XIP ROM read only
 591		 * from SVC mode and no access from userspace.
 592		 */
 593		mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 594		mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 595		mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 596#endif
 597
 598		/*
 599		 * If the initial page tables were created with the S bit
 600		 * set, then we need to do the same here for the same
 601		 * reasons given in early_cachepolicy().
 602		 */
 603		if (initial_pmd_value & PMD_SECT_S) {
 604			user_pgprot |= L_PTE_SHARED;
 605			kern_pgprot |= L_PTE_SHARED;
 606			vecs_pgprot |= L_PTE_SHARED;
 607			s2_pgprot |= L_PTE_SHARED;
 608			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
 609			mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
 610			mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
 611			mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
 612			mem_types[MT_MEMORY_RWX].prot_sect |= PMD_SECT_S;
 613			mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
 614			mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S;
 615			mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED;
 616			mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
 617			mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S;
 618			mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED;
 619		}
 620	}
 621
 622	/*
 623	 * Non-cacheable Normal - intended for memory areas that must
 624	 * not cause dirty cache line writebacks when used
 625	 */
 626	if (cpu_arch >= CPU_ARCH_ARMv6) {
 627		if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
 628			/* Non-cacheable Normal is XCB = 001 */
 629			mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
 630				PMD_SECT_BUFFERED;
 631		} else {
 632			/* For both ARMv6 and non-TEX-remapping ARMv7 */
 633			mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
 634				PMD_SECT_TEX(1);
 635		}
 636	} else {
 637		mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
 638	}
 639
 640#ifdef CONFIG_ARM_LPAE
 641	/*
 642	 * Do not generate access flag faults for the kernel mappings.
 643	 */
 644	for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 645		mem_types[i].prot_pte |= PTE_EXT_AF;
 646		if (mem_types[i].prot_sect)
 647			mem_types[i].prot_sect |= PMD_SECT_AF;
 648	}
 649	kern_pgprot |= PTE_EXT_AF;
 650	vecs_pgprot |= PTE_EXT_AF;
 651
 652	/*
 653	 * Set PXN for user mappings
 654	 */
 655	user_pgprot |= PTE_EXT_PXN;
 656#endif
 657
 658	for (i = 0; i < 16; i++) {
 659		pteval_t v = pgprot_val(protection_map[i]);
 660		protection_map[i] = __pgprot(v | user_pgprot);
 661	}
 662
 663	mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
 664	mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
 665
 666	pgprot_user   = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
 667	pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
 668				 L_PTE_DIRTY | kern_pgprot);
 669	pgprot_s2  = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
 670	pgprot_s2_device  = __pgprot(s2_device_pgprot);
 671	pgprot_hyp_device  = __pgprot(hyp_device_pgprot);
 672
 673	mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
 674	mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
 675	mem_types[MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;
 676	mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot;
 677	mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;
 678	mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot;
 679	mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
 680	mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask;
 681	mem_types[MT_ROM].prot_sect |= cp->pmd;
 682
 683	switch (cp->pmd) {
 684	case PMD_SECT_WT:
 685		mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
 686		break;
 687	case PMD_SECT_WB:
 688	case PMD_SECT_WBWA:
 689		mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
 690		break;
 691	}
 692	pr_info("Memory policy: %sData cache %s\n",
 693		ecc_mask ? "ECC enabled, " : "", cp->policy);
 694
 695	for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 696		struct mem_type *t = &mem_types[i];
 697		if (t->prot_l1)
 698			t->prot_l1 |= PMD_DOMAIN(t->domain);
 699		if (t->prot_sect)
 700			t->prot_sect |= PMD_DOMAIN(t->domain);
 701	}
 702}
 703
 704#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
 705pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 706			      unsigned long size, pgprot_t vma_prot)
 707{
 708	if (!pfn_valid(pfn))
 709		return pgprot_noncached(vma_prot);
 710	else if (file->f_flags & O_SYNC)
 711		return pgprot_writecombine(vma_prot);
 712	return vma_prot;
 713}
 714EXPORT_SYMBOL(phys_mem_access_prot);
 715#endif
 716
 717#define vectors_base()	(vectors_high() ? 0xffff0000 : 0)
 718
 719static void __init *early_alloc(unsigned long sz)
 720{
 721	void *ptr = memblock_alloc(sz, sz);
 722
 723	if (!ptr)
 724		panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
 725		      __func__, sz, sz);
 726
 727	return ptr;
 728}
 729
 730static void *__init late_alloc(unsigned long sz)
 731{
 732	void *ptr = (void *)__get_free_pages(GFP_PGTABLE_KERNEL, get_order(sz));
 733
 734	if (!ptr || !pgtable_pte_page_ctor(virt_to_page(ptr)))
 735		BUG();
 736	return ptr;
 737}
 738
 739static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr,
 740				unsigned long prot,
 741				void *(*alloc)(unsigned long sz))
 742{
 743	if (pmd_none(*pmd)) {
 744		pte_t *pte = alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
 745		__pmd_populate(pmd, __pa(pte), prot);
 746	}
 747	BUG_ON(pmd_bad(*pmd));
 748	return pte_offset_kernel(pmd, addr);
 749}
 750
 751static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,
 752				      unsigned long prot)
 753{
 754	return arm_pte_alloc(pmd, addr, prot, early_alloc);
 755}
 756
 757static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
 758				  unsigned long end, unsigned long pfn,
 759				  const struct mem_type *type,
 760				  void *(*alloc)(unsigned long sz),
 761				  bool ng)
 762{
 763	pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);
 764	do {
 765		set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),
 766			    ng ? PTE_EXT_NG : 0);
 767		pfn++;
 768	} while (pte++, addr += PAGE_SIZE, addr != end);
 769}
 770
 771static void __init __map_init_section(pmd_t *pmd, unsigned long addr,
 772			unsigned long end, phys_addr_t phys,
 773			const struct mem_type *type, bool ng)
 774{
 775	pmd_t *p = pmd;
 776
 777#ifndef CONFIG_ARM_LPAE
 778	/*
 779	 * In classic MMU format, puds and pmds are folded in to
 780	 * the pgds. pmd_offset gives the PGD entry. PGDs refer to a
 781	 * group of L1 entries making up one logical pointer to
 782	 * an L2 table (2MB), where as PMDs refer to the individual
 783	 * L1 entries (1MB). Hence increment to get the correct
 784	 * offset for odd 1MB sections.
 785	 * (See arch/arm/include/asm/pgtable-2level.h)
 786	 */
 787	if (addr & SECTION_SIZE)
 788		pmd++;
 789#endif
 790	do {
 791		*pmd = __pmd(phys | type->prot_sect | (ng ? PMD_SECT_nG : 0));
 792		phys += SECTION_SIZE;
 793	} while (pmd++, addr += SECTION_SIZE, addr != end);
 794
 795	flush_pmd_entry(p);
 796}
 797
 798static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
 799				      unsigned long end, phys_addr_t phys,
 800				      const struct mem_type *type,
 801				      void *(*alloc)(unsigned long sz), bool ng)
 802{
 803	pmd_t *pmd = pmd_offset(pud, addr);
 804	unsigned long next;
 805
 806	do {
 807		/*
 808		 * With LPAE, we must loop over to map
 809		 * all the pmds for the given range.
 810		 */
 811		next = pmd_addr_end(addr, end);
 812
 
 
 813		/*
 814		 * Try a section mapping - addr, next and phys must all be
 815		 * aligned to a section boundary.
 816		 */
 817		if (type->prot_sect &&
 818				((addr | next | phys) & ~SECTION_MASK) == 0) {
 819			__map_init_section(pmd, addr, next, phys, type, ng);
 820		} else {
 821			alloc_init_pte(pmd, addr, next,
 822				       __phys_to_pfn(phys), type, alloc, ng);
 823		}
 824
 825		phys += next - addr;
 826
 827	} while (pmd++, addr = next, addr != end);
 828}
 829
 830static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
 831				  unsigned long end, phys_addr_t phys,
 832				  const struct mem_type *type,
 833				  void *(*alloc)(unsigned long sz), bool ng)
 834{
 835	pud_t *pud = pud_offset(pgd, addr);
 836	unsigned long next;
 837
 838	do {
 839		next = pud_addr_end(addr, end);
 840		alloc_init_pmd(pud, addr, next, phys, type, alloc, ng);
 841		phys += next - addr;
 842	} while (pud++, addr = next, addr != end);
 843}
 844
 845#ifndef CONFIG_ARM_LPAE
 846static void __init create_36bit_mapping(struct mm_struct *mm,
 847					struct map_desc *md,
 848					const struct mem_type *type,
 849					bool ng)
 850{
 851	unsigned long addr, length, end;
 852	phys_addr_t phys;
 853	pgd_t *pgd;
 854
 855	addr = md->virtual;
 856	phys = __pfn_to_phys(md->pfn);
 857	length = PAGE_ALIGN(md->length);
 858
 859	if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
 860		pr_err("MM: CPU does not support supersection mapping for 0x%08llx at 0x%08lx\n",
 
 861		       (long long)__pfn_to_phys((u64)md->pfn), addr);
 862		return;
 863	}
 864
 865	/* N.B.	ARMv6 supersections are only defined to work with domain 0.
 866	 *	Since domain assignments can in fact be arbitrary, the
 867	 *	'domain == 0' check below is required to insure that ARMv6
 868	 *	supersections are only allocated for domain 0 regardless
 869	 *	of the actual domain assignments in use.
 870	 */
 871	if (type->domain) {
 872		pr_err("MM: invalid domain in supersection mapping for 0x%08llx at 0x%08lx\n",
 
 873		       (long long)__pfn_to_phys((u64)md->pfn), addr);
 874		return;
 875	}
 876
 877	if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
 878		pr_err("MM: cannot create mapping for 0x%08llx at 0x%08lx invalid alignment\n",
 
 879		       (long long)__pfn_to_phys((u64)md->pfn), addr);
 880		return;
 881	}
 882
 883	/*
 884	 * Shift bits [35:32] of address into bits [23:20] of PMD
 885	 * (See ARMv6 spec).
 886	 */
 887	phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
 888
 889	pgd = pgd_offset(mm, addr);
 890	end = addr + length;
 891	do {
 892		pud_t *pud = pud_offset(pgd, addr);
 893		pmd_t *pmd = pmd_offset(pud, addr);
 894		int i;
 895
 896		for (i = 0; i < 16; i++)
 897			*pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER |
 898				       (ng ? PMD_SECT_nG : 0));
 899
 900		addr += SUPERSECTION_SIZE;
 901		phys += SUPERSECTION_SIZE;
 902		pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
 903	} while (addr != end);
 904}
 905#endif	/* !CONFIG_ARM_LPAE */
 906
 907static void __init __create_mapping(struct mm_struct *mm, struct map_desc *md,
 908				    void *(*alloc)(unsigned long sz),
 909				    bool ng)
 
 
 
 
 
 910{
 911	unsigned long addr, length, end;
 912	phys_addr_t phys;
 913	const struct mem_type *type;
 914	pgd_t *pgd;
 915
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 916	type = &mem_types[md->type];
 917
 918#ifndef CONFIG_ARM_LPAE
 919	/*
 920	 * Catch 36-bit addresses
 921	 */
 922	if (md->pfn >= 0x100000) {
 923		create_36bit_mapping(mm, md, type, ng);
 924		return;
 925	}
 926#endif
 927
 928	addr = md->virtual & PAGE_MASK;
 929	phys = __pfn_to_phys(md->pfn);
 930	length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
 931
 932	if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
 933		pr_warn("BUG: map for 0x%08llx at 0x%08lx can not be mapped using pages, ignoring.\n",
 934			(long long)__pfn_to_phys(md->pfn), addr);
 
 935		return;
 936	}
 937
 938	pgd = pgd_offset(mm, addr);
 939	end = addr + length;
 940	do {
 941		unsigned long next = pgd_addr_end(addr, end);
 942
 943		alloc_init_pud(pgd, addr, next, phys, type, alloc, ng);
 944
 945		phys += next - addr;
 946		addr = next;
 947	} while (pgd++, addr != end);
 948}
 949
 950/*
 951 * Create the page directory entries and any necessary
 952 * page tables for the mapping specified by `md'.  We
 953 * are able to cope here with varying sizes and address
 954 * offsets, and we take full advantage of sections and
 955 * supersections.
 956 */
 957static void __init create_mapping(struct map_desc *md)
 958{
 959	if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
 960		pr_warn("BUG: not creating mapping for 0x%08llx at 0x%08lx in user region\n",
 961			(long long)__pfn_to_phys((u64)md->pfn), md->virtual);
 962		return;
 963	}
 964
 965	if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
 966	    md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START &&
 967	    (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
 968		pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n",
 969			(long long)__pfn_to_phys((u64)md->pfn), md->virtual);
 970	}
 971
 972	__create_mapping(&init_mm, md, early_alloc, false);
 973}
 974
 975void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md,
 976				bool ng)
 977{
 978#ifdef CONFIG_ARM_LPAE
 979	pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
 980	if (WARN_ON(!pud))
 981		return;
 982	pmd_alloc(mm, pud, 0);
 983#endif
 984	__create_mapping(mm, md, late_alloc, ng);
 985}
 986
 987/*
 988 * Create the architecture specific mappings
 989 */
 990void __init iotable_init(struct map_desc *io_desc, int nr)
 991{
 992	struct map_desc *md;
 993	struct vm_struct *vm;
 994	struct static_vm *svm;
 995
 996	if (!nr)
 997		return;
 998
 999	svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm));
1000	if (!svm)
1001		panic("%s: Failed to allocate %zu bytes align=0x%zx\n",
1002		      __func__, sizeof(*svm) * nr, __alignof__(*svm));
1003
1004	for (md = io_desc; nr; md++, nr--) {
1005		create_mapping(md);
1006
1007		vm = &svm->vm;
1008		vm->addr = (void *)(md->virtual & PAGE_MASK);
1009		vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
1010		vm->phys_addr = __pfn_to_phys(md->pfn);
1011		vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
1012		vm->flags |= VM_ARM_MTYPE(md->type);
1013		vm->caller = iotable_init;
1014		add_static_vm_early(svm++);
1015	}
1016}
1017
1018void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
1019				  void *caller)
1020{
1021	struct vm_struct *vm;
1022	struct static_vm *svm;
1023
1024	svm = memblock_alloc(sizeof(*svm), __alignof__(*svm));
1025	if (!svm)
1026		panic("%s: Failed to allocate %zu bytes align=0x%zx\n",
1027		      __func__, sizeof(*svm), __alignof__(*svm));
1028
1029	vm = &svm->vm;
1030	vm->addr = (void *)addr;
1031	vm->size = size;
1032	vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
1033	vm->caller = caller;
1034	add_static_vm_early(svm);
1035}
1036
1037#ifndef CONFIG_ARM_LPAE
1038
1039/*
1040 * The Linux PMD is made of two consecutive section entries covering 2MB
1041 * (see definition in include/asm/pgtable-2level.h).  However a call to
1042 * create_mapping() may optimize static mappings by using individual
1043 * 1MB section mappings.  This leaves the actual PMD potentially half
1044 * initialized if the top or bottom section entry isn't used, leaving it
1045 * open to problems if a subsequent ioremap() or vmalloc() tries to use
1046 * the virtual space left free by that unused section entry.
1047 *
1048 * Let's avoid the issue by inserting dummy vm entries covering the unused
1049 * PMD halves once the static mappings are in place.
1050 */
1051
1052static void __init pmd_empty_section_gap(unsigned long addr)
1053{
1054	vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
 
 
 
 
 
 
 
1055}
1056
1057static void __init fill_pmd_gaps(void)
1058{
1059	struct static_vm *svm;
1060	struct vm_struct *vm;
1061	unsigned long addr, next = 0;
1062	pmd_t *pmd;
1063
1064	list_for_each_entry(svm, &static_vmlist, list) {
1065		vm = &svm->vm;
 
 
1066		addr = (unsigned long)vm->addr;
1067		if (addr < next)
1068			continue;
1069
1070		/*
1071		 * Check if this vm starts on an odd section boundary.
1072		 * If so and the first section entry for this PMD is free
1073		 * then we block the corresponding virtual address.
1074		 */
1075		if ((addr & ~PMD_MASK) == SECTION_SIZE) {
1076			pmd = pmd_off_k(addr);
1077			if (pmd_none(*pmd))
1078				pmd_empty_section_gap(addr & PMD_MASK);
1079		}
1080
1081		/*
1082		 * Then check if this vm ends on an odd section boundary.
1083		 * If so and the second section entry for this PMD is empty
1084		 * then we block the corresponding virtual address.
1085		 */
1086		addr += vm->size;
1087		if ((addr & ~PMD_MASK) == SECTION_SIZE) {
1088			pmd = pmd_off_k(addr) + 1;
1089			if (pmd_none(*pmd))
1090				pmd_empty_section_gap(addr);
1091		}
1092
1093		/* no need to look at any vm entry until we hit the next PMD */
1094		next = (addr + PMD_SIZE - 1) & PMD_MASK;
1095	}
1096}
1097
1098#else
1099#define fill_pmd_gaps() do { } while (0)
1100#endif
1101
1102#if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
1103static void __init pci_reserve_io(void)
1104{
1105	struct static_vm *svm;
1106
1107	svm = find_static_vm_vaddr((void *)PCI_IO_VIRT_BASE);
1108	if (svm)
1109		return;
1110
1111	vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
1112}
1113#else
1114#define pci_reserve_io() do { } while (0)
1115#endif
1116
1117#ifdef CONFIG_DEBUG_LL
1118void __init debug_ll_io_init(void)
1119{
1120	struct map_desc map;
1121
1122	debug_ll_addr(&map.pfn, &map.virtual);
1123	if (!map.pfn || !map.virtual)
1124		return;
1125	map.pfn = __phys_to_pfn(map.pfn);
1126	map.virtual &= PAGE_MASK;
1127	map.length = PAGE_SIZE;
1128	map.type = MT_DEVICE;
1129	iotable_init(&map, 1);
1130}
1131#endif
1132
1133static void * __initdata vmalloc_min =
1134	(void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
1135
1136/*
1137 * vmalloc=size forces the vmalloc area to be exactly 'size'
1138 * bytes. This can be used to increase (or decrease) the vmalloc
1139 * area - the default is 240m.
1140 */
1141static int __init early_vmalloc(char *arg)
1142{
1143	unsigned long vmalloc_reserve = memparse(arg, NULL);
1144
1145	if (vmalloc_reserve < SZ_16M) {
1146		vmalloc_reserve = SZ_16M;
1147		pr_warn("vmalloc area too small, limiting to %luMB\n",
 
1148			vmalloc_reserve >> 20);
1149	}
1150
1151	if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
1152		vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
1153		pr_warn("vmalloc area is too big, limiting to %luMB\n",
 
1154			vmalloc_reserve >> 20);
1155	}
1156
1157	vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
1158	return 0;
1159}
1160early_param("vmalloc", early_vmalloc);
1161
1162phys_addr_t arm_lowmem_limit __initdata = 0;
1163
1164void __init adjust_lowmem_bounds(void)
1165{
1166	phys_addr_t memblock_limit = 0;
1167	u64 vmalloc_limit;
1168	struct memblock_region *reg;
1169	phys_addr_t lowmem_limit = 0;
1170
1171	/*
1172	 * Let's use our own (unoptimized) equivalent of __pa() that is
1173	 * not affected by wrap-arounds when sizeof(phys_addr_t) == 4.
1174	 * The result is used as the upper bound on physical memory address
1175	 * and may itself be outside the valid range for which phys_addr_t
1176	 * and therefore __pa() is defined.
1177	 */
1178	vmalloc_limit = (u64)(uintptr_t)vmalloc_min - PAGE_OFFSET + PHYS_OFFSET;
1179
1180	/*
1181	 * The first usable region must be PMD aligned. Mark its start
1182	 * as MEMBLOCK_NOMAP if it isn't
1183	 */
1184	for_each_memblock(memory, reg) {
1185		if (!memblock_is_nomap(reg)) {
1186			if (!IS_ALIGNED(reg->base, PMD_SIZE)) {
1187				phys_addr_t len;
1188
1189				len = round_up(reg->base, PMD_SIZE) - reg->base;
1190				memblock_mark_nomap(reg->base, len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1191			}
1192			break;
1193		}
1194	}
 
1195
1196	for_each_memblock(memory, reg) {
1197		phys_addr_t block_start = reg->base;
1198		phys_addr_t block_end = reg->base + reg->size;
 
 
 
 
 
 
 
1199
1200		if (memblock_is_nomap(reg))
 
 
 
 
 
 
 
 
 
1201			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1202
1203		if (reg->base < vmalloc_limit) {
1204			if (block_end > lowmem_limit)
1205				/*
1206				 * Compare as u64 to ensure vmalloc_limit does
1207				 * not get truncated. block_end should always
1208				 * fit in phys_addr_t so there should be no
1209				 * issue with assignment.
1210				 */
1211				lowmem_limit = min_t(u64,
1212							 vmalloc_limit,
1213							 block_end);
1214
 
1215			/*
1216			 * Find the first non-pmd-aligned page, and point
1217			 * memblock_limit at it. This relies on rounding the
1218			 * limit down to be pmd-aligned, which happens at the
1219			 * end of this function.
1220			 *
1221			 * With this algorithm, the start or end of almost any
1222			 * bank can be non-pmd-aligned. The only exception is
1223			 * that the start of the bank 0 must be section-
1224			 * aligned, since otherwise memory would need to be
1225			 * allocated when mapping the start of bank 0, which
1226			 * occurs before any free memory is mapped.
1227			 */
1228			if (!memblock_limit) {
1229				if (!IS_ALIGNED(block_start, PMD_SIZE))
1230					memblock_limit = block_start;
1231				else if (!IS_ALIGNED(block_end, PMD_SIZE))
1232					memblock_limit = lowmem_limit;
1233			}
1234
1235		}
1236	}
1237
1238	arm_lowmem_limit = lowmem_limit;
1239
1240	high_memory = __va(arm_lowmem_limit - 1) + 1;
1241
1242	if (!memblock_limit)
1243		memblock_limit = arm_lowmem_limit;
1244
1245	/*
1246	 * Round the memblock limit down to a pmd size.  This
1247	 * helps to ensure that we will allocate memory from the
1248	 * last full pmd, which should be mapped.
1249	 */
1250	memblock_limit = round_down(memblock_limit, PMD_SIZE);
1251
1252	if (!IS_ENABLED(CONFIG_HIGHMEM) || cache_is_vipt_aliasing()) {
1253		if (memblock_end_of_DRAM() > arm_lowmem_limit) {
1254			phys_addr_t end = memblock_end_of_DRAM();
1255
1256			pr_notice("Ignoring RAM at %pa-%pa\n",
1257				  &memblock_limit, &end);
1258			pr_notice("Consider using a HIGHMEM enabled kernel.\n");
1259
1260			memblock_remove(memblock_limit, end - memblock_limit);
1261		}
1262	}
1263
1264	memblock_set_current_limit(memblock_limit);
 
 
1265}
1266
1267static inline void prepare_page_table(void)
1268{
1269	unsigned long addr;
1270	phys_addr_t end;
1271
1272	/*
1273	 * Clear out all the mappings below the kernel image.
1274	 */
1275	for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
1276		pmd_clear(pmd_off_k(addr));
1277
1278#ifdef CONFIG_XIP_KERNEL
1279	/* The XIP kernel is mapped in the module area -- skip over it */
1280	addr = ((unsigned long)_exiprom + PMD_SIZE - 1) & PMD_MASK;
1281#endif
1282	for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
1283		pmd_clear(pmd_off_k(addr));
1284
1285	/*
1286	 * Find the end of the first block of lowmem.
1287	 */
1288	end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
1289	if (end >= arm_lowmem_limit)
1290		end = arm_lowmem_limit;
1291
1292	/*
1293	 * Clear out all the kernel space mappings, except for the first
1294	 * memory bank, up to the vmalloc region.
1295	 */
1296	for (addr = __phys_to_virt(end);
1297	     addr < VMALLOC_START; addr += PMD_SIZE)
1298		pmd_clear(pmd_off_k(addr));
1299}
1300
1301#ifdef CONFIG_ARM_LPAE
1302/* the first page is reserved for pgd */
1303#define SWAPPER_PG_DIR_SIZE	(PAGE_SIZE + \
1304				 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
1305#else
1306#define SWAPPER_PG_DIR_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))
1307#endif
1308
1309/*
1310 * Reserve the special regions of memory
1311 */
1312void __init arm_mm_memblock_reserve(void)
1313{
1314	/*
1315	 * Reserve the page tables.  These are already in use,
1316	 * and can only be in node 0.
1317	 */
1318	memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
1319
1320#ifdef CONFIG_SA1111
1321	/*
1322	 * Because of the SA1111 DMA bug, we want to preserve our
1323	 * precious DMA-able memory...
1324	 */
1325	memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
1326#endif
1327}
1328
1329/*
1330 * Set up the device mappings.  Since we clear out the page tables for all
1331 * mappings above VMALLOC_START, except early fixmap, we might remove debug
1332 * device mappings.  This means earlycon can be used to debug this function
1333 * Any other function or debugging method which may touch any device _will_
1334 * crash the kernel.
1335 */
1336static void __init devicemaps_init(const struct machine_desc *mdesc)
1337{
1338	struct map_desc map;
1339	unsigned long addr;
1340	void *vectors;
1341
1342	/*
1343	 * Allocate the vector page early.
1344	 */
1345	vectors = early_alloc(PAGE_SIZE * 2);
1346
1347	early_trap_init(vectors);
1348
1349	/*
1350	 * Clear page table except top pmd used by early fixmaps
1351	 */
1352	for (addr = VMALLOC_START; addr < (FIXADDR_TOP & PMD_MASK); addr += PMD_SIZE)
1353		pmd_clear(pmd_off_k(addr));
1354
1355	/*
1356	 * Map the kernel if it is XIP.
1357	 * It is always first in the modulearea.
1358	 */
1359#ifdef CONFIG_XIP_KERNEL
1360	map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
1361	map.virtual = MODULES_VADDR;
1362	map.length = ((unsigned long)_exiprom - map.virtual + ~SECTION_MASK) & SECTION_MASK;
1363	map.type = MT_ROM;
1364	create_mapping(&map);
1365#endif
1366
1367	/*
1368	 * Map the cache flushing regions.
1369	 */
1370#ifdef FLUSH_BASE
1371	map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
1372	map.virtual = FLUSH_BASE;
1373	map.length = SZ_1M;
1374	map.type = MT_CACHECLEAN;
1375	create_mapping(&map);
1376#endif
1377#ifdef FLUSH_BASE_MINICACHE
1378	map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
1379	map.virtual = FLUSH_BASE_MINICACHE;
1380	map.length = SZ_1M;
1381	map.type = MT_MINICLEAN;
1382	create_mapping(&map);
1383#endif
1384
1385	/*
1386	 * Create a mapping for the machine vectors at the high-vectors
1387	 * location (0xffff0000).  If we aren't using high-vectors, also
1388	 * create a mapping at the low-vectors virtual address.
1389	 */
1390	map.pfn = __phys_to_pfn(virt_to_phys(vectors));
1391	map.virtual = 0xffff0000;
1392	map.length = PAGE_SIZE;
1393#ifdef CONFIG_KUSER_HELPERS
1394	map.type = MT_HIGH_VECTORS;
1395#else
1396	map.type = MT_LOW_VECTORS;
1397#endif
1398	create_mapping(&map);
1399
1400	if (!vectors_high()) {
1401		map.virtual = 0;
1402		map.length = PAGE_SIZE * 2;
1403		map.type = MT_LOW_VECTORS;
1404		create_mapping(&map);
1405	}
1406
1407	/* Now create a kernel read-only mapping */
1408	map.pfn += 1;
1409	map.virtual = 0xffff0000 + PAGE_SIZE;
1410	map.length = PAGE_SIZE;
1411	map.type = MT_LOW_VECTORS;
1412	create_mapping(&map);
1413
1414	/*
1415	 * Ask the machine support to map in the statically mapped devices.
1416	 */
1417	if (mdesc->map_io)
1418		mdesc->map_io();
1419	else
1420		debug_ll_io_init();
1421	fill_pmd_gaps();
1422
1423	/* Reserve fixed i/o space in VMALLOC region */
1424	pci_reserve_io();
1425
1426	/*
1427	 * Finally flush the caches and tlb to ensure that we're in a
1428	 * consistent state wrt the writebuffer.  This also ensures that
1429	 * any write-allocated cache lines in the vector page are written
1430	 * back.  After this point, we can start to touch devices again.
1431	 */
1432	local_flush_tlb_all();
1433	flush_cache_all();
1434
1435	/* Enable asynchronous aborts */
1436	early_abt_enable();
1437}
1438
1439static void __init kmap_init(void)
1440{
1441#ifdef CONFIG_HIGHMEM
1442	pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
1443		PKMAP_BASE, _PAGE_KERNEL_TABLE);
1444#endif
1445
1446	early_pte_alloc(pmd_off_k(FIXADDR_START), FIXADDR_START,
1447			_PAGE_KERNEL_TABLE);
1448}
1449
1450static void __init map_lowmem(void)
1451{
1452	struct memblock_region *reg;
1453	phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE);
1454	phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
1455
1456	/* Map all the lowmem memory banks. */
1457	for_each_memblock(memory, reg) {
1458		phys_addr_t start = reg->base;
1459		phys_addr_t end = start + reg->size;
1460		struct map_desc map;
1461
1462		if (memblock_is_nomap(reg))
1463			continue;
1464
1465		if (end > arm_lowmem_limit)
1466			end = arm_lowmem_limit;
1467		if (start >= end)
1468			break;
1469
1470		if (end < kernel_x_start) {
1471			map.pfn = __phys_to_pfn(start);
1472			map.virtual = __phys_to_virt(start);
1473			map.length = end - start;
1474			map.type = MT_MEMORY_RWX;
1475
1476			create_mapping(&map);
1477		} else if (start >= kernel_x_end) {
1478			map.pfn = __phys_to_pfn(start);
1479			map.virtual = __phys_to_virt(start);
1480			map.length = end - start;
1481			map.type = MT_MEMORY_RW;
1482
1483			create_mapping(&map);
1484		} else {
1485			/* This better cover the entire kernel */
1486			if (start < kernel_x_start) {
1487				map.pfn = __phys_to_pfn(start);
1488				map.virtual = __phys_to_virt(start);
1489				map.length = kernel_x_start - start;
1490				map.type = MT_MEMORY_RW;
1491
1492				create_mapping(&map);
1493			}
1494
1495			map.pfn = __phys_to_pfn(kernel_x_start);
1496			map.virtual = __phys_to_virt(kernel_x_start);
1497			map.length = kernel_x_end - kernel_x_start;
1498			map.type = MT_MEMORY_RWX;
1499
1500			create_mapping(&map);
1501
1502			if (kernel_x_end < end) {
1503				map.pfn = __phys_to_pfn(kernel_x_end);
1504				map.virtual = __phys_to_virt(kernel_x_end);
1505				map.length = end - kernel_x_end;
1506				map.type = MT_MEMORY_RW;
1507
1508				create_mapping(&map);
1509			}
1510		}
1511	}
1512}
1513
1514#ifdef CONFIG_ARM_PV_FIXUP
1515extern unsigned long __atags_pointer;
1516typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata);
1517pgtables_remap lpae_pgtables_remap_asm;
1518
1519/*
1520 * early_paging_init() recreates boot time page table setup, allowing machines
1521 * to switch over to a high (>4G) address space on LPAE systems
1522 */
1523static void __init early_paging_init(const struct machine_desc *mdesc)
1524{
1525	pgtables_remap *lpae_pgtables_remap;
1526	unsigned long pa_pgd;
1527	unsigned int cr, ttbcr;
1528	long long offset;
1529	void *boot_data;
1530
1531	if (!mdesc->pv_fixup)
1532		return;
1533
1534	offset = mdesc->pv_fixup();
1535	if (offset == 0)
1536		return;
1537
1538	/*
1539	 * Get the address of the remap function in the 1:1 identity
1540	 * mapping setup by the early page table assembly code.  We
1541	 * must get this prior to the pv update.  The following barrier
1542	 * ensures that this is complete before we fixup any P:V offsets.
1543	 */
1544	lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm);
1545	pa_pgd = __pa(swapper_pg_dir);
1546	boot_data = __va(__atags_pointer);
1547	barrier();
1548
1549	pr_info("Switching physical address space to 0x%08llx\n",
1550		(u64)PHYS_OFFSET + offset);
1551
1552	/* Re-set the phys pfn offset, and the pv offset */
1553	__pv_offset += offset;
1554	__pv_phys_pfn_offset += PFN_DOWN(offset);
1555
1556	/* Run the patch stub to update the constants */
1557	fixup_pv_table(&__pv_table_begin,
1558		(&__pv_table_end - &__pv_table_begin) << 2);
1559
1560	/*
1561	 * We changing not only the virtual to physical mapping, but also
1562	 * the physical addresses used to access memory.  We need to flush
1563	 * all levels of cache in the system with caching disabled to
1564	 * ensure that all data is written back, and nothing is prefetched
1565	 * into the caches.  We also need to prevent the TLB walkers
1566	 * allocating into the caches too.  Note that this is ARMv7 LPAE
1567	 * specific.
1568	 */
1569	cr = get_cr();
1570	set_cr(cr & ~(CR_I | CR_C));
1571	asm("mrc p15, 0, %0, c2, c0, 2" : "=r" (ttbcr));
1572	asm volatile("mcr p15, 0, %0, c2, c0, 2"
1573		: : "r" (ttbcr & ~(3 << 8 | 3 << 10)));
1574	flush_cache_all();
1575
1576	/*
1577	 * Fixup the page tables - this must be in the idmap region as
1578	 * we need to disable the MMU to do this safely, and hence it
1579	 * needs to be assembly.  It's fairly simple, as we're using the
1580	 * temporary tables setup by the initial assembly code.
1581	 */
1582	lpae_pgtables_remap(offset, pa_pgd, boot_data);
1583
1584	/* Re-enable the caches and cacheable TLB walks */
1585	asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr));
1586	set_cr(cr);
1587}
1588
1589#else
1590
1591static void __init early_paging_init(const struct machine_desc *mdesc)
1592{
1593	long long offset;
1594
1595	if (!mdesc->pv_fixup)
1596		return;
1597
1598	offset = mdesc->pv_fixup();
1599	if (offset == 0)
1600		return;
1601
1602	pr_crit("Physical address space modification is only to support Keystone2.\n");
1603	pr_crit("Please enable ARM_LPAE and ARM_PATCH_PHYS_VIRT support to use this\n");
1604	pr_crit("feature. Your kernel may crash now, have a good day.\n");
1605	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1606}
1607
1608#endif
1609
1610static void __init early_fixmap_shutdown(void)
1611{
1612	int i;
1613	unsigned long va = fix_to_virt(__end_of_permanent_fixed_addresses - 1);
1614
1615	pte_offset_fixmap = pte_offset_late_fixmap;
1616	pmd_clear(fixmap_pmd(va));
1617	local_flush_tlb_kernel_page(va);
1618
1619	for (i = 0; i < __end_of_permanent_fixed_addresses; i++) {
1620		pte_t *pte;
1621		struct map_desc map;
1622
1623		map.virtual = fix_to_virt(i);
1624		pte = pte_offset_early_fixmap(pmd_off_k(map.virtual), map.virtual);
1625
1626		/* Only i/o device mappings are supported ATM */
1627		if (pte_none(*pte) ||
1628		    (pte_val(*pte) & L_PTE_MT_MASK) != L_PTE_MT_DEV_SHARED)
1629			continue;
1630
1631		map.pfn = pte_pfn(*pte);
1632		map.type = MT_DEVICE;
1633		map.length = PAGE_SIZE;
1634
1635		create_mapping(&map);
1636	}
1637}
1638
1639/*
1640 * paging_init() sets up the page tables, initialises the zone memory
1641 * maps, and sets up the zero page, bad page and bad page tables.
1642 */
1643void __init paging_init(const struct machine_desc *mdesc)
1644{
1645	void *zero_page;
1646
 
 
 
1647	prepare_page_table();
1648	map_lowmem();
1649	memblock_set_current_limit(arm_lowmem_limit);
1650	dma_contiguous_remap();
1651	early_fixmap_shutdown();
1652	devicemaps_init(mdesc);
1653	kmap_init();
1654	tcm_init();
1655
1656	top_pmd = pmd_off_k(0xffff0000);
1657
1658	/* allocate the zero page. */
1659	zero_page = early_alloc(PAGE_SIZE);
1660
1661	bootmem_init();
1662
1663	empty_zero_page = virt_to_page(zero_page);
1664	__flush_dcache_page(NULL, empty_zero_page);
1665
1666	/* Compute the virt/idmap offset, mostly for the sake of KVM */
1667	kimage_voffset = (unsigned long)&kimage_voffset - virt_to_idmap(&kimage_voffset);
1668}
1669
1670void __init early_mm_init(const struct machine_desc *mdesc)
1671{
1672	build_mem_type_table();
1673	early_paging_init(mdesc);
1674}