Loading...
1/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable-hwdef.h>
27#include <asm/pgtable.h>
28#include <asm/cpufeature.h>
29#include <asm/alternative.h>
30
31#include "proc-macros.S"
32
33#ifdef CONFIG_ARM64_64K_PAGES
34#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
35#elif defined(CONFIG_ARM64_16K_PAGES)
36#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
37#else /* CONFIG_ARM64_4K_PAGES */
38#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
39#endif
40
41#define TCR_SMP_FLAGS TCR_SHARED
42
43/* PTWs cacheable, inner/outer WBWA */
44#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
45
46#define MAIR(attr, mt) ((attr) << ((mt) * 8))
47
48/*
49 * cpu_do_idle()
50 *
51 * Idle the processor (wait for interrupt).
52 */
53ENTRY(cpu_do_idle)
54 dsb sy // WFI may enter a low-power mode
55 wfi
56 ret
57ENDPROC(cpu_do_idle)
58
59#ifdef CONFIG_CPU_PM
60/**
61 * cpu_do_suspend - save CPU registers context
62 *
63 * x0: virtual address of context pointer
64 */
65ENTRY(cpu_do_suspend)
66 mrs x2, tpidr_el0
67 mrs x3, tpidrro_el0
68 mrs x4, contextidr_el1
69 mrs x5, mair_el1
70 mrs x6, cpacr_el1
71 mrs x7, ttbr1_el1
72 mrs x8, tcr_el1
73 mrs x9, vbar_el1
74 mrs x10, mdscr_el1
75 mrs x11, oslsr_el1
76 mrs x12, sctlr_el1
77 stp x2, x3, [x0]
78 stp x4, x5, [x0, #16]
79 stp x6, x7, [x0, #32]
80 stp x8, x9, [x0, #48]
81 stp x10, x11, [x0, #64]
82 str x12, [x0, #80]
83 ret
84ENDPROC(cpu_do_suspend)
85
86/**
87 * cpu_do_resume - restore CPU register context
88 *
89 * x0: Physical address of context pointer
90 * x1: ttbr0_el1 to be restored
91 *
92 * Returns:
93 * sctlr_el1 value in x0
94 */
95ENTRY(cpu_do_resume)
96 /*
97 * Invalidate local tlb entries before turning on MMU
98 */
99 tlbi vmalle1
100 ldp x2, x3, [x0]
101 ldp x4, x5, [x0, #16]
102 ldp x6, x7, [x0, #32]
103 ldp x8, x9, [x0, #48]
104 ldp x10, x11, [x0, #64]
105 ldr x12, [x0, #80]
106 msr tpidr_el0, x2
107 msr tpidrro_el0, x3
108 msr contextidr_el1, x4
109 msr mair_el1, x5
110 msr cpacr_el1, x6
111 msr ttbr0_el1, x1
112 msr ttbr1_el1, x7
113 tcr_set_idmap_t0sz x8, x7
114 msr tcr_el1, x8
115 msr vbar_el1, x9
116 msr mdscr_el1, x10
117 /*
118 * Restore oslsr_el1 by writing oslar_el1
119 */
120 ubfx x11, x11, #1, #1
121 msr oslar_el1, x11
122 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
123 mov x0, x12
124 dsb nsh // Make sure local tlb invalidation completed
125 isb
126 ret
127ENDPROC(cpu_do_resume)
128#endif
129
130/*
131 * cpu_do_switch_mm(pgd_phys, tsk)
132 *
133 * Set the translation table base pointer to be pgd_phys.
134 *
135 * - pgd_phys - physical address of new TTB
136 */
137ENTRY(cpu_do_switch_mm)
138 mmid x1, x1 // get mm->context.id
139 bfi x0, x1, #48, #16 // set the ASID
140 msr ttbr0_el1, x0 // set TTBR0
141 isb
142alternative_if_not ARM64_WORKAROUND_CAVIUM_27456
143 ret
144 nop
145 nop
146 nop
147alternative_else
148 ic iallu
149 dsb nsh
150 isb
151 ret
152alternative_endif
153ENDPROC(cpu_do_switch_mm)
154
155 .pushsection ".idmap.text", "ax"
156/*
157 * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
158 *
159 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
160 * called by anything else. It can only be executed from a TTBR0 mapping.
161 */
162ENTRY(idmap_cpu_replace_ttbr1)
163 mrs x2, daif
164 msr daifset, #0xf
165
166 adrp x1, empty_zero_page
167 msr ttbr1_el1, x1
168 isb
169
170 tlbi vmalle1
171 dsb nsh
172 isb
173
174 msr ttbr1_el1, x0
175 isb
176
177 msr daif, x2
178
179 ret
180ENDPROC(idmap_cpu_replace_ttbr1)
181 .popsection
182
183/*
184 * __cpu_setup
185 *
186 * Initialise the processor for turning the MMU on. Return in x0 the
187 * value of the SCTLR_EL1 register.
188 */
189ENTRY(__cpu_setup)
190 tlbi vmalle1 // Invalidate local TLB
191 dsb nsh
192
193 mov x0, #3 << 20
194 msr cpacr_el1, x0 // Enable FP/ASIMD
195 mov x0, #1 << 12 // Reset mdscr_el1 and disable
196 msr mdscr_el1, x0 // access to the DCC from EL0
197 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
198 /*
199 * Memory region attributes for LPAE:
200 *
201 * n = AttrIndx[2:0]
202 * n MAIR
203 * DEVICE_nGnRnE 000 00000000
204 * DEVICE_nGnRE 001 00000100
205 * DEVICE_GRE 010 00001100
206 * NORMAL_NC 011 01000100
207 * NORMAL 100 11111111
208 * NORMAL_WT 101 10111011
209 */
210 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
211 MAIR(0x04, MT_DEVICE_nGnRE) | \
212 MAIR(0x0c, MT_DEVICE_GRE) | \
213 MAIR(0x44, MT_NORMAL_NC) | \
214 MAIR(0xff, MT_NORMAL) | \
215 MAIR(0xbb, MT_NORMAL_WT)
216 msr mair_el1, x5
217 /*
218 * Prepare SCTLR
219 */
220 adr x5, crval
221 ldp w5, w6, [x5]
222 mrs x0, sctlr_el1
223 bic x0, x0, x5 // clear bits
224 orr x0, x0, x6 // set bits
225 /*
226 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
227 * both user and kernel.
228 */
229 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
230 TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
231 tcr_set_idmap_t0sz x10, x9
232
233 /*
234 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
235 * TCR_EL1.
236 */
237 mrs x9, ID_AA64MMFR0_EL1
238 bfi x10, x9, #32, #3
239#ifdef CONFIG_ARM64_HW_AFDBM
240 /*
241 * Hardware update of the Access and Dirty bits.
242 */
243 mrs x9, ID_AA64MMFR1_EL1
244 and x9, x9, #0xf
245 cbz x9, 2f
246 cmp x9, #2
247 b.lt 1f
248 orr x10, x10, #TCR_HD // hardware Dirty flag update
2491: orr x10, x10, #TCR_HA // hardware Access flag update
2502:
251#endif /* CONFIG_ARM64_HW_AFDBM */
252 msr tcr_el1, x10
253 ret // return to head.S
254ENDPROC(__cpu_setup)
255
256 /*
257 * We set the desired value explicitly, including those of the
258 * reserved bits. The values of bits EE & E0E were set early in
259 * el2_setup, which are left untouched below.
260 *
261 * n n T
262 * U E WT T UD US IHBS
263 * CE0 XWHW CZ ME TEEA S
264 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
265 * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
266 * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
267 */
268 .type crval, #object
269crval:
270 .word 0xfcffffff // clear
271 .word 0x34d5d91d // set
1/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable.h>
27#include <asm/pgtable-hwdef.h>
28#include <asm/cpufeature.h>
29#include <asm/alternative.h>
30
31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
33#elif defined(CONFIG_ARM64_16K_PAGES)
34#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
35#else /* CONFIG_ARM64_4K_PAGES */
36#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
37#endif
38
39#ifdef CONFIG_RANDOMIZE_BASE
40#define TCR_KASLR_FLAGS TCR_NFD1
41#else
42#define TCR_KASLR_FLAGS 0
43#endif
44
45#define TCR_SMP_FLAGS TCR_SHARED
46
47/* PTWs cacheable, inner/outer WBWA */
48#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
49
50#define MAIR(attr, mt) ((attr) << ((mt) * 8))
51
52/*
53 * cpu_do_idle()
54 *
55 * Idle the processor (wait for interrupt).
56 */
57ENTRY(cpu_do_idle)
58 dsb sy // WFI may enter a low-power mode
59 wfi
60 ret
61ENDPROC(cpu_do_idle)
62
63#ifdef CONFIG_CPU_PM
64/**
65 * cpu_do_suspend - save CPU registers context
66 *
67 * x0: virtual address of context pointer
68 */
69ENTRY(cpu_do_suspend)
70 mrs x2, tpidr_el0
71 mrs x3, tpidrro_el0
72 mrs x4, contextidr_el1
73 mrs x5, cpacr_el1
74 mrs x6, tcr_el1
75 mrs x7, vbar_el1
76 mrs x8, mdscr_el1
77 mrs x9, oslsr_el1
78 mrs x10, sctlr_el1
79alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
80 mrs x11, tpidr_el1
81alternative_else
82 mrs x11, tpidr_el2
83alternative_endif
84 mrs x12, sp_el0
85 stp x2, x3, [x0]
86 stp x4, xzr, [x0, #16]
87 stp x5, x6, [x0, #32]
88 stp x7, x8, [x0, #48]
89 stp x9, x10, [x0, #64]
90 stp x11, x12, [x0, #80]
91 ret
92ENDPROC(cpu_do_suspend)
93
94/**
95 * cpu_do_resume - restore CPU register context
96 *
97 * x0: Address of context pointer
98 */
99 .pushsection ".idmap.text", "awx"
100ENTRY(cpu_do_resume)
101 ldp x2, x3, [x0]
102 ldp x4, x5, [x0, #16]
103 ldp x6, x8, [x0, #32]
104 ldp x9, x10, [x0, #48]
105 ldp x11, x12, [x0, #64]
106 ldp x13, x14, [x0, #80]
107 msr tpidr_el0, x2
108 msr tpidrro_el0, x3
109 msr contextidr_el1, x4
110 msr cpacr_el1, x6
111
112 /* Don't change t0sz here, mask those bits when restoring */
113 mrs x5, tcr_el1
114 bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
115
116 msr tcr_el1, x8
117 msr vbar_el1, x9
118
119 /*
120 * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking
121 * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug
122 * exception. Mask them until local_daif_restore() in cpu_suspend()
123 * resets them.
124 */
125 disable_daif
126 msr mdscr_el1, x10
127
128 msr sctlr_el1, x12
129alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
130 msr tpidr_el1, x13
131alternative_else
132 msr tpidr_el2, x13
133alternative_endif
134 msr sp_el0, x14
135 /*
136 * Restore oslsr_el1 by writing oslar_el1
137 */
138 ubfx x11, x11, #1, #1
139 msr oslar_el1, x11
140 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
141
142alternative_if ARM64_HAS_RAS_EXTN
143 msr_s SYS_DISR_EL1, xzr
144alternative_else_nop_endif
145
146 isb
147 ret
148ENDPROC(cpu_do_resume)
149 .popsection
150#endif
151
152/*
153 * cpu_do_switch_mm(pgd_phys, tsk)
154 *
155 * Set the translation table base pointer to be pgd_phys.
156 *
157 * - pgd_phys - physical address of new TTB
158 */
159ENTRY(cpu_do_switch_mm)
160 mrs x2, ttbr1_el1
161 mmid x1, x1 // get mm->context.id
162 phys_to_ttbr x3, x0
163#ifdef CONFIG_ARM64_SW_TTBR0_PAN
164 bfi x3, x1, #48, #16 // set the ASID field in TTBR0
165#endif
166 bfi x2, x1, #48, #16 // set the ASID
167 msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set)
168 isb
169 msr ttbr0_el1, x3 // now update TTBR0
170 isb
171 b post_ttbr_update_workaround // Back to C code...
172ENDPROC(cpu_do_switch_mm)
173
174 .pushsection ".idmap.text", "awx"
175
176.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
177 adrp \tmp1, empty_zero_page
178 phys_to_ttbr \tmp2, \tmp1
179 msr ttbr1_el1, \tmp2
180 isb
181 tlbi vmalle1
182 dsb nsh
183 isb
184.endm
185
186/*
187 * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
188 *
189 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
190 * called by anything else. It can only be executed from a TTBR0 mapping.
191 */
192ENTRY(idmap_cpu_replace_ttbr1)
193 save_and_disable_daif flags=x2
194
195 __idmap_cpu_set_reserved_ttbr1 x1, x3
196
197 phys_to_ttbr x3, x0
198 msr ttbr1_el1, x3
199 isb
200
201 restore_daif x2
202
203 ret
204ENDPROC(idmap_cpu_replace_ttbr1)
205 .popsection
206
207#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
208 .pushsection ".idmap.text", "awx"
209
210 .macro __idmap_kpti_get_pgtable_ent, type
211 dc cvac, cur_\()\type\()p // Ensure any existing dirty
212 dmb sy // lines are written back before
213 ldr \type, [cur_\()\type\()p] // loading the entry
214 tbz \type, #0, skip_\()\type // Skip invalid and
215 tbnz \type, #11, skip_\()\type // non-global entries
216 .endm
217
218 .macro __idmap_kpti_put_pgtable_ent_ng, type
219 orr \type, \type, #PTE_NG // Same bit for blocks and pages
220 str \type, [cur_\()\type\()p] // Update the entry and ensure it
221 dc civac, cur_\()\type\()p // is visible to all CPUs.
222 .endm
223
224/*
225 * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper)
226 *
227 * Called exactly once from stop_machine context by each CPU found during boot.
228 */
229__idmap_kpti_flag:
230 .long 1
231ENTRY(idmap_kpti_install_ng_mappings)
232 cpu .req w0
233 num_cpus .req w1
234 swapper_pa .req x2
235 swapper_ttb .req x3
236 flag_ptr .req x4
237 cur_pgdp .req x5
238 end_pgdp .req x6
239 pgd .req x7
240 cur_pudp .req x8
241 end_pudp .req x9
242 pud .req x10
243 cur_pmdp .req x11
244 end_pmdp .req x12
245 pmd .req x13
246 cur_ptep .req x14
247 end_ptep .req x15
248 pte .req x16
249
250 mrs swapper_ttb, ttbr1_el1
251 adr flag_ptr, __idmap_kpti_flag
252
253 cbnz cpu, __idmap_kpti_secondary
254
255 /* We're the boot CPU. Wait for the others to catch up */
256 sevl
2571: wfe
258 ldaxr w18, [flag_ptr]
259 eor w18, w18, num_cpus
260 cbnz w18, 1b
261
262 /* We need to walk swapper, so turn off the MMU. */
263 pre_disable_mmu_workaround
264 mrs x18, sctlr_el1
265 bic x18, x18, #SCTLR_ELx_M
266 msr sctlr_el1, x18
267 isb
268
269 /* Everybody is enjoying the idmap, so we can rewrite swapper. */
270 /* PGD */
271 mov cur_pgdp, swapper_pa
272 add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
273do_pgd: __idmap_kpti_get_pgtable_ent pgd
274 tbnz pgd, #1, walk_puds
275next_pgd:
276 __idmap_kpti_put_pgtable_ent_ng pgd
277skip_pgd:
278 add cur_pgdp, cur_pgdp, #8
279 cmp cur_pgdp, end_pgdp
280 b.ne do_pgd
281
282 /* Publish the updated tables and nuke all the TLBs */
283 dsb sy
284 tlbi vmalle1is
285 dsb ish
286 isb
287
288 /* We're done: fire up the MMU again */
289 mrs x18, sctlr_el1
290 orr x18, x18, #SCTLR_ELx_M
291 msr sctlr_el1, x18
292 isb
293
294 /* Set the flag to zero to indicate that we're all done */
295 str wzr, [flag_ptr]
296 ret
297
298 /* PUD */
299walk_puds:
300 .if CONFIG_PGTABLE_LEVELS > 3
301 pte_to_phys cur_pudp, pgd
302 add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
303do_pud: __idmap_kpti_get_pgtable_ent pud
304 tbnz pud, #1, walk_pmds
305next_pud:
306 __idmap_kpti_put_pgtable_ent_ng pud
307skip_pud:
308 add cur_pudp, cur_pudp, 8
309 cmp cur_pudp, end_pudp
310 b.ne do_pud
311 b next_pgd
312 .else /* CONFIG_PGTABLE_LEVELS <= 3 */
313 mov pud, pgd
314 b walk_pmds
315next_pud:
316 b next_pgd
317 .endif
318
319 /* PMD */
320walk_pmds:
321 .if CONFIG_PGTABLE_LEVELS > 2
322 pte_to_phys cur_pmdp, pud
323 add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
324do_pmd: __idmap_kpti_get_pgtable_ent pmd
325 tbnz pmd, #1, walk_ptes
326next_pmd:
327 __idmap_kpti_put_pgtable_ent_ng pmd
328skip_pmd:
329 add cur_pmdp, cur_pmdp, #8
330 cmp cur_pmdp, end_pmdp
331 b.ne do_pmd
332 b next_pud
333 .else /* CONFIG_PGTABLE_LEVELS <= 2 */
334 mov pmd, pud
335 b walk_ptes
336next_pmd:
337 b next_pud
338 .endif
339
340 /* PTE */
341walk_ptes:
342 pte_to_phys cur_ptep, pmd
343 add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
344do_pte: __idmap_kpti_get_pgtable_ent pte
345 __idmap_kpti_put_pgtable_ent_ng pte
346skip_pte:
347 add cur_ptep, cur_ptep, #8
348 cmp cur_ptep, end_ptep
349 b.ne do_pte
350 b next_pmd
351
352 /* Secondary CPUs end up here */
353__idmap_kpti_secondary:
354 /* Uninstall swapper before surgery begins */
355 __idmap_cpu_set_reserved_ttbr1 x18, x17
356
357 /* Increment the flag to let the boot CPU we're ready */
3581: ldxr w18, [flag_ptr]
359 add w18, w18, #1
360 stxr w17, w18, [flag_ptr]
361 cbnz w17, 1b
362
363 /* Wait for the boot CPU to finish messing around with swapper */
364 sevl
3651: wfe
366 ldxr w18, [flag_ptr]
367 cbnz w18, 1b
368
369 /* All done, act like nothing happened */
370 msr ttbr1_el1, swapper_ttb
371 isb
372 ret
373
374 .unreq cpu
375 .unreq num_cpus
376 .unreq swapper_pa
377 .unreq swapper_ttb
378 .unreq flag_ptr
379 .unreq cur_pgdp
380 .unreq end_pgdp
381 .unreq pgd
382 .unreq cur_pudp
383 .unreq end_pudp
384 .unreq pud
385 .unreq cur_pmdp
386 .unreq end_pmdp
387 .unreq pmd
388 .unreq cur_ptep
389 .unreq end_ptep
390 .unreq pte
391ENDPROC(idmap_kpti_install_ng_mappings)
392 .popsection
393#endif
394
395/*
396 * __cpu_setup
397 *
398 * Initialise the processor for turning the MMU on. Return in x0 the
399 * value of the SCTLR_EL1 register.
400 */
401 .pushsection ".idmap.text", "awx"
402ENTRY(__cpu_setup)
403 tlbi vmalle1 // Invalidate local TLB
404 dsb nsh
405
406 mov x0, #3 << 20
407 msr cpacr_el1, x0 // Enable FP/ASIMD
408 mov x0, #1 << 12 // Reset mdscr_el1 and disable
409 msr mdscr_el1, x0 // access to the DCC from EL0
410 isb // Unmask debug exceptions now,
411 enable_dbg // since this is per-cpu
412 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
413 /*
414 * Memory region attributes for LPAE:
415 *
416 * n = AttrIndx[2:0]
417 * n MAIR
418 * DEVICE_nGnRnE 000 00000000
419 * DEVICE_nGnRE 001 00000100
420 * DEVICE_GRE 010 00001100
421 * NORMAL_NC 011 01000100
422 * NORMAL 100 11111111
423 * NORMAL_WT 101 10111011
424 */
425 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
426 MAIR(0x04, MT_DEVICE_nGnRE) | \
427 MAIR(0x0c, MT_DEVICE_GRE) | \
428 MAIR(0x44, MT_NORMAL_NC) | \
429 MAIR(0xff, MT_NORMAL) | \
430 MAIR(0xbb, MT_NORMAL_WT)
431 msr mair_el1, x5
432 /*
433 * Prepare SCTLR
434 */
435 mov_q x0, SCTLR_EL1_SET
436 /*
437 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
438 * both user and kernel.
439 */
440 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
441 TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
442 TCR_TBI0 | TCR_A1
443 tcr_set_idmap_t0sz x10, x9
444
445 /*
446 * Set the IPS bits in TCR_EL1.
447 */
448 tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6
449#ifdef CONFIG_ARM64_HW_AFDBM
450 /*
451 * Enable hardware update of the Access Flags bit.
452 * Hardware dirty bit management is enabled later,
453 * via capabilities.
454 */
455 mrs x9, ID_AA64MMFR1_EL1
456 and x9, x9, #0xf
457 cbz x9, 1f
458 orr x10, x10, #TCR_HA // hardware Access flag update
4591:
460#endif /* CONFIG_ARM64_HW_AFDBM */
461 msr tcr_el1, x10
462 ret // return to head.S
463ENDPROC(__cpu_setup)