Loading...
1/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable-hwdef.h>
27#include <asm/pgtable.h>
28#include <asm/cpufeature.h>
29#include <asm/alternative.h>
30
31#include "proc-macros.S"
32
33#ifdef CONFIG_ARM64_64K_PAGES
34#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
35#elif defined(CONFIG_ARM64_16K_PAGES)
36#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
37#else /* CONFIG_ARM64_4K_PAGES */
38#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
39#endif
40
41#define TCR_SMP_FLAGS TCR_SHARED
42
43/* PTWs cacheable, inner/outer WBWA */
44#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
45
46#define MAIR(attr, mt) ((attr) << ((mt) * 8))
47
48/*
49 * cpu_do_idle()
50 *
51 * Idle the processor (wait for interrupt).
52 */
53ENTRY(cpu_do_idle)
54 dsb sy // WFI may enter a low-power mode
55 wfi
56 ret
57ENDPROC(cpu_do_idle)
58
59#ifdef CONFIG_CPU_PM
60/**
61 * cpu_do_suspend - save CPU registers context
62 *
63 * x0: virtual address of context pointer
64 */
65ENTRY(cpu_do_suspend)
66 mrs x2, tpidr_el0
67 mrs x3, tpidrro_el0
68 mrs x4, contextidr_el1
69 mrs x5, mair_el1
70 mrs x6, cpacr_el1
71 mrs x7, ttbr1_el1
72 mrs x8, tcr_el1
73 mrs x9, vbar_el1
74 mrs x10, mdscr_el1
75 mrs x11, oslsr_el1
76 mrs x12, sctlr_el1
77 stp x2, x3, [x0]
78 stp x4, x5, [x0, #16]
79 stp x6, x7, [x0, #32]
80 stp x8, x9, [x0, #48]
81 stp x10, x11, [x0, #64]
82 str x12, [x0, #80]
83 ret
84ENDPROC(cpu_do_suspend)
85
86/**
87 * cpu_do_resume - restore CPU register context
88 *
89 * x0: Physical address of context pointer
90 * x1: ttbr0_el1 to be restored
91 *
92 * Returns:
93 * sctlr_el1 value in x0
94 */
95ENTRY(cpu_do_resume)
96 /*
97 * Invalidate local tlb entries before turning on MMU
98 */
99 tlbi vmalle1
100 ldp x2, x3, [x0]
101 ldp x4, x5, [x0, #16]
102 ldp x6, x7, [x0, #32]
103 ldp x8, x9, [x0, #48]
104 ldp x10, x11, [x0, #64]
105 ldr x12, [x0, #80]
106 msr tpidr_el0, x2
107 msr tpidrro_el0, x3
108 msr contextidr_el1, x4
109 msr mair_el1, x5
110 msr cpacr_el1, x6
111 msr ttbr0_el1, x1
112 msr ttbr1_el1, x7
113 tcr_set_idmap_t0sz x8, x7
114 msr tcr_el1, x8
115 msr vbar_el1, x9
116 msr mdscr_el1, x10
117 /*
118 * Restore oslsr_el1 by writing oslar_el1
119 */
120 ubfx x11, x11, #1, #1
121 msr oslar_el1, x11
122 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
123 mov x0, x12
124 dsb nsh // Make sure local tlb invalidation completed
125 isb
126 ret
127ENDPROC(cpu_do_resume)
128#endif
129
130/*
131 * cpu_do_switch_mm(pgd_phys, tsk)
132 *
133 * Set the translation table base pointer to be pgd_phys.
134 *
135 * - pgd_phys - physical address of new TTB
136 */
137ENTRY(cpu_do_switch_mm)
138 mmid x1, x1 // get mm->context.id
139 bfi x0, x1, #48, #16 // set the ASID
140 msr ttbr0_el1, x0 // set TTBR0
141 isb
142alternative_if_not ARM64_WORKAROUND_CAVIUM_27456
143 ret
144 nop
145 nop
146 nop
147alternative_else
148 ic iallu
149 dsb nsh
150 isb
151 ret
152alternative_endif
153ENDPROC(cpu_do_switch_mm)
154
155 .pushsection ".idmap.text", "ax"
156/*
157 * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
158 *
159 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
160 * called by anything else. It can only be executed from a TTBR0 mapping.
161 */
162ENTRY(idmap_cpu_replace_ttbr1)
163 mrs x2, daif
164 msr daifset, #0xf
165
166 adrp x1, empty_zero_page
167 msr ttbr1_el1, x1
168 isb
169
170 tlbi vmalle1
171 dsb nsh
172 isb
173
174 msr ttbr1_el1, x0
175 isb
176
177 msr daif, x2
178
179 ret
180ENDPROC(idmap_cpu_replace_ttbr1)
181 .popsection
182
183/*
184 * __cpu_setup
185 *
186 * Initialise the processor for turning the MMU on. Return in x0 the
187 * value of the SCTLR_EL1 register.
188 */
189ENTRY(__cpu_setup)
190 tlbi vmalle1 // Invalidate local TLB
191 dsb nsh
192
193 mov x0, #3 << 20
194 msr cpacr_el1, x0 // Enable FP/ASIMD
195 mov x0, #1 << 12 // Reset mdscr_el1 and disable
196 msr mdscr_el1, x0 // access to the DCC from EL0
197 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
198 /*
199 * Memory region attributes for LPAE:
200 *
201 * n = AttrIndx[2:0]
202 * n MAIR
203 * DEVICE_nGnRnE 000 00000000
204 * DEVICE_nGnRE 001 00000100
205 * DEVICE_GRE 010 00001100
206 * NORMAL_NC 011 01000100
207 * NORMAL 100 11111111
208 * NORMAL_WT 101 10111011
209 */
210 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
211 MAIR(0x04, MT_DEVICE_nGnRE) | \
212 MAIR(0x0c, MT_DEVICE_GRE) | \
213 MAIR(0x44, MT_NORMAL_NC) | \
214 MAIR(0xff, MT_NORMAL) | \
215 MAIR(0xbb, MT_NORMAL_WT)
216 msr mair_el1, x5
217 /*
218 * Prepare SCTLR
219 */
220 adr x5, crval
221 ldp w5, w6, [x5]
222 mrs x0, sctlr_el1
223 bic x0, x0, x5 // clear bits
224 orr x0, x0, x6 // set bits
225 /*
226 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
227 * both user and kernel.
228 */
229 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
230 TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
231 tcr_set_idmap_t0sz x10, x9
232
233 /*
234 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
235 * TCR_EL1.
236 */
237 mrs x9, ID_AA64MMFR0_EL1
238 bfi x10, x9, #32, #3
239#ifdef CONFIG_ARM64_HW_AFDBM
240 /*
241 * Hardware update of the Access and Dirty bits.
242 */
243 mrs x9, ID_AA64MMFR1_EL1
244 and x9, x9, #0xf
245 cbz x9, 2f
246 cmp x9, #2
247 b.lt 1f
248 orr x10, x10, #TCR_HD // hardware Dirty flag update
2491: orr x10, x10, #TCR_HA // hardware Access flag update
2502:
251#endif /* CONFIG_ARM64_HW_AFDBM */
252 msr tcr_el1, x10
253 ret // return to head.S
254ENDPROC(__cpu_setup)
255
256 /*
257 * We set the desired value explicitly, including those of the
258 * reserved bits. The values of bits EE & E0E were set early in
259 * el2_setup, which are left untouched below.
260 *
261 * n n T
262 * U E WT T UD US IHBS
263 * CE0 XWHW CZ ME TEEA S
264 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
265 * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
266 * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
267 */
268 .type crval, #object
269crval:
270 .word 0xfcffffff // clear
271 .word 0x34d5d91d // set
1/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable-hwdef.h>
27#include <asm/pgtable.h>
28
29#include "proc-macros.S"
30
31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
33#else
34#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
35#endif
36
37#ifdef CONFIG_SMP
38#define TCR_SMP_FLAGS TCR_SHARED
39#else
40#define TCR_SMP_FLAGS 0
41#endif
42
43/* PTWs cacheable, inner/outer WBWA */
44#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
45
46#define MAIR(attr, mt) ((attr) << ((mt) * 8))
47
48/*
49 * cpu_cache_off()
50 *
51 * Turn the CPU D-cache off.
52 */
53ENTRY(cpu_cache_off)
54 mrs x0, sctlr_el1
55 bic x0, x0, #1 << 2 // clear SCTLR.C
56 msr sctlr_el1, x0
57 isb
58 ret
59ENDPROC(cpu_cache_off)
60
61/*
62 * cpu_reset(loc)
63 *
64 * Perform a soft reset of the system. Put the CPU into the same state
65 * as it would be if it had been reset, and branch to what would be the
66 * reset vector. It must be executed with the flat identity mapping.
67 *
68 * - loc - location to jump to for soft reset
69 */
70 .align 5
71ENTRY(cpu_reset)
72 mrs x1, sctlr_el1
73 bic x1, x1, #1
74 msr sctlr_el1, x1 // disable the MMU
75 isb
76 ret x0
77ENDPROC(cpu_reset)
78
79/*
80 * cpu_do_idle()
81 *
82 * Idle the processor (wait for interrupt).
83 */
84ENTRY(cpu_do_idle)
85 dsb sy // WFI may enter a low-power mode
86 wfi
87 ret
88ENDPROC(cpu_do_idle)
89
90#ifdef CONFIG_ARM64_CPU_SUSPEND
91/**
92 * cpu_do_suspend - save CPU registers context
93 *
94 * x0: virtual address of context pointer
95 */
96ENTRY(cpu_do_suspend)
97 mrs x2, tpidr_el0
98 mrs x3, tpidrro_el0
99 mrs x4, contextidr_el1
100 mrs x5, mair_el1
101 mrs x6, cpacr_el1
102 mrs x7, ttbr1_el1
103 mrs x8, tcr_el1
104 mrs x9, vbar_el1
105 mrs x10, mdscr_el1
106 mrs x11, oslsr_el1
107 mrs x12, sctlr_el1
108 stp x2, x3, [x0]
109 stp x4, x5, [x0, #16]
110 stp x6, x7, [x0, #32]
111 stp x8, x9, [x0, #48]
112 stp x10, x11, [x0, #64]
113 str x12, [x0, #80]
114 ret
115ENDPROC(cpu_do_suspend)
116
117/**
118 * cpu_do_resume - restore CPU register context
119 *
120 * x0: Physical address of context pointer
121 * x1: ttbr0_el1 to be restored
122 *
123 * Returns:
124 * sctlr_el1 value in x0
125 */
126ENTRY(cpu_do_resume)
127 /*
128 * Invalidate local tlb entries before turning on MMU
129 */
130 tlbi vmalle1
131 ldp x2, x3, [x0]
132 ldp x4, x5, [x0, #16]
133 ldp x6, x7, [x0, #32]
134 ldp x8, x9, [x0, #48]
135 ldp x10, x11, [x0, #64]
136 ldr x12, [x0, #80]
137 msr tpidr_el0, x2
138 msr tpidrro_el0, x3
139 msr contextidr_el1, x4
140 msr mair_el1, x5
141 msr cpacr_el1, x6
142 msr ttbr0_el1, x1
143 msr ttbr1_el1, x7
144 msr tcr_el1, x8
145 msr vbar_el1, x9
146 msr mdscr_el1, x10
147 /*
148 * Restore oslsr_el1 by writing oslar_el1
149 */
150 ubfx x11, x11, #1, #1
151 msr oslar_el1, x11
152 mov x0, x12
153 dsb nsh // Make sure local tlb invalidation completed
154 isb
155 ret
156ENDPROC(cpu_do_resume)
157#endif
158
159/*
160 * cpu_do_switch_mm(pgd_phys, tsk)
161 *
162 * Set the translation table base pointer to be pgd_phys.
163 *
164 * - pgd_phys - physical address of new TTB
165 */
166ENTRY(cpu_do_switch_mm)
167 mmid w1, x1 // get mm->context.id
168 bfi x0, x1, #48, #16 // set the ASID
169 msr ttbr0_el1, x0 // set TTBR0
170 isb
171 ret
172ENDPROC(cpu_do_switch_mm)
173
174 .section ".text.init", #alloc, #execinstr
175
176/*
177 * __cpu_setup
178 *
179 * Initialise the processor for turning the MMU on. Return in x0 the
180 * value of the SCTLR_EL1 register.
181 */
182ENTRY(__cpu_setup)
183 ic iallu // I+BTB cache invalidate
184 tlbi vmalle1is // invalidate I + D TLBs
185 dsb sy
186
187 mov x0, #3 << 20
188 msr cpacr_el1, x0 // Enable FP/ASIMD
189 msr mdscr_el1, xzr // Reset mdscr_el1
190 /*
191 * Memory region attributes for LPAE:
192 *
193 * n = AttrIndx[2:0]
194 * n MAIR
195 * DEVICE_nGnRnE 000 00000000
196 * DEVICE_nGnRE 001 00000100
197 * DEVICE_GRE 010 00001100
198 * NORMAL_NC 011 01000100
199 * NORMAL 100 11111111
200 */
201 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
202 MAIR(0x04, MT_DEVICE_nGnRE) | \
203 MAIR(0x0c, MT_DEVICE_GRE) | \
204 MAIR(0x44, MT_NORMAL_NC) | \
205 MAIR(0xff, MT_NORMAL)
206 msr mair_el1, x5
207 /*
208 * Prepare SCTLR
209 */
210 adr x5, crval
211 ldp w5, w6, [x5]
212 mrs x0, sctlr_el1
213 bic x0, x0, x5 // clear bits
214 orr x0, x0, x6 // set bits
215 /*
216 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
217 * both user and kernel.
218 */
219 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
220 TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
221 /*
222 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
223 * TCR_EL1.
224 */
225 mrs x9, ID_AA64MMFR0_EL1
226 bfi x10, x9, #32, #3
227 msr tcr_el1, x10
228 ret // return to head.S
229ENDPROC(__cpu_setup)
230
231 /*
232 * n n T
233 * U E WT T UD US IHBS
234 * CE0 XWHW CZ ME TEEA S
235 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
236 * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
237 * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
238 */
239 .type crval, #object
240crval:
241 .word 0x000802e2 // clear
242 .word 0x0405d11d // set