Loading...
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/arm-smccc.h>
3#include <linux/kernel.h>
4#include <linux/smp.h>
5
6#include <asm/cp15.h>
7#include <asm/cputype.h>
8#include <asm/proc-fns.h>
9#include <asm/spectre.h>
10#include <asm/system_misc.h>
11
12#ifdef CONFIG_ARM_PSCI
13static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
14{
15 struct arm_smccc_res res;
16
17 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
18 ARM_SMCCC_ARCH_WORKAROUND_1, &res);
19
20 switch ((int)res.a0) {
21 case SMCCC_RET_SUCCESS:
22 return SPECTRE_MITIGATED;
23
24 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
25 return SPECTRE_UNAFFECTED;
26
27 default:
28 return SPECTRE_VULNERABLE;
29 }
30}
31#else
32static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
33{
34 return SPECTRE_VULNERABLE;
35}
36#endif
37
38#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
39DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
40
41extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
42extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
43extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
44extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
45
46static void harden_branch_predictor_bpiall(void)
47{
48 write_sysreg(0, BPIALL);
49}
50
51static void harden_branch_predictor_iciallu(void)
52{
53 write_sysreg(0, ICIALLU);
54}
55
56static void __maybe_unused call_smc_arch_workaround_1(void)
57{
58 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
59}
60
61static void __maybe_unused call_hvc_arch_workaround_1(void)
62{
63 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
64}
65
66static unsigned int spectre_v2_install_workaround(unsigned int method)
67{
68 const char *spectre_v2_method = NULL;
69 int cpu = smp_processor_id();
70
71 if (per_cpu(harden_branch_predictor_fn, cpu))
72 return SPECTRE_MITIGATED;
73
74 switch (method) {
75 case SPECTRE_V2_METHOD_BPIALL:
76 per_cpu(harden_branch_predictor_fn, cpu) =
77 harden_branch_predictor_bpiall;
78 spectre_v2_method = "BPIALL";
79 break;
80
81 case SPECTRE_V2_METHOD_ICIALLU:
82 per_cpu(harden_branch_predictor_fn, cpu) =
83 harden_branch_predictor_iciallu;
84 spectre_v2_method = "ICIALLU";
85 break;
86
87 case SPECTRE_V2_METHOD_HVC:
88 per_cpu(harden_branch_predictor_fn, cpu) =
89 call_hvc_arch_workaround_1;
90 cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
91 spectre_v2_method = "hypervisor";
92 break;
93
94 case SPECTRE_V2_METHOD_SMC:
95 per_cpu(harden_branch_predictor_fn, cpu) =
96 call_smc_arch_workaround_1;
97 cpu_do_switch_mm = cpu_v7_smc_switch_mm;
98 spectre_v2_method = "firmware";
99 break;
100 }
101
102 if (spectre_v2_method)
103 pr_info("CPU%u: Spectre v2: using %s workaround\n",
104 smp_processor_id(), spectre_v2_method);
105
106 return SPECTRE_MITIGATED;
107}
108#else
109static unsigned int spectre_v2_install_workaround(unsigned int method)
110{
111 pr_info_once("Spectre V2: workarounds disabled by configuration\n");
112
113 return SPECTRE_VULNERABLE;
114}
115#endif
116
117static void cpu_v7_spectre_v2_init(void)
118{
119 unsigned int state, method = 0;
120
121 switch (read_cpuid_part()) {
122 case ARM_CPU_PART_CORTEX_A8:
123 case ARM_CPU_PART_CORTEX_A9:
124 case ARM_CPU_PART_CORTEX_A12:
125 case ARM_CPU_PART_CORTEX_A17:
126 case ARM_CPU_PART_CORTEX_A73:
127 case ARM_CPU_PART_CORTEX_A75:
128 state = SPECTRE_MITIGATED;
129 method = SPECTRE_V2_METHOD_BPIALL;
130 break;
131
132 case ARM_CPU_PART_CORTEX_A15:
133 case ARM_CPU_PART_BRAHMA_B15:
134 state = SPECTRE_MITIGATED;
135 method = SPECTRE_V2_METHOD_ICIALLU;
136 break;
137
138 case ARM_CPU_PART_BRAHMA_B53:
139 /* Requires no workaround */
140 state = SPECTRE_UNAFFECTED;
141 break;
142
143 default:
144 /* Other ARM CPUs require no workaround */
145 if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) {
146 state = SPECTRE_UNAFFECTED;
147 break;
148 }
149
150 fallthrough;
151
152 /* Cortex A57/A72 require firmware workaround */
153 case ARM_CPU_PART_CORTEX_A57:
154 case ARM_CPU_PART_CORTEX_A72:
155 state = spectre_v2_get_cpu_fw_mitigation_state();
156 if (state != SPECTRE_MITIGATED)
157 break;
158
159 switch (arm_smccc_1_1_get_conduit()) {
160 case SMCCC_CONDUIT_HVC:
161 method = SPECTRE_V2_METHOD_HVC;
162 break;
163
164 case SMCCC_CONDUIT_SMC:
165 method = SPECTRE_V2_METHOD_SMC;
166 break;
167
168 default:
169 state = SPECTRE_VULNERABLE;
170 break;
171 }
172 }
173
174 if (state == SPECTRE_MITIGATED)
175 state = spectre_v2_install_workaround(method);
176
177 spectre_v2_update_state(state, method);
178}
179
180#ifdef CONFIG_HARDEN_BRANCH_HISTORY
181static int spectre_bhb_method;
182
183static const char *spectre_bhb_method_name(int method)
184{
185 switch (method) {
186 case SPECTRE_V2_METHOD_LOOP8:
187 return "loop";
188
189 case SPECTRE_V2_METHOD_BPIALL:
190 return "BPIALL";
191
192 default:
193 return "unknown";
194 }
195}
196
197static int spectre_bhb_install_workaround(int method)
198{
199 if (spectre_bhb_method != method) {
200 if (spectre_bhb_method) {
201 pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n",
202 smp_processor_id());
203
204 return SPECTRE_VULNERABLE;
205 }
206
207 if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE)
208 return SPECTRE_VULNERABLE;
209
210 spectre_bhb_method = method;
211
212 pr_info("CPU%u: Spectre BHB: enabling %s workaround for all CPUs\n",
213 smp_processor_id(), spectre_bhb_method_name(method));
214 }
215
216 return SPECTRE_MITIGATED;
217}
218#else
219static int spectre_bhb_install_workaround(int method)
220{
221 return SPECTRE_VULNERABLE;
222}
223#endif
224
225static void cpu_v7_spectre_bhb_init(void)
226{
227 unsigned int state, method = 0;
228
229 switch (read_cpuid_part()) {
230 case ARM_CPU_PART_CORTEX_A15:
231 case ARM_CPU_PART_BRAHMA_B15:
232 case ARM_CPU_PART_CORTEX_A57:
233 case ARM_CPU_PART_CORTEX_A72:
234 state = SPECTRE_MITIGATED;
235 method = SPECTRE_V2_METHOD_LOOP8;
236 break;
237
238 case ARM_CPU_PART_CORTEX_A73:
239 case ARM_CPU_PART_CORTEX_A75:
240 state = SPECTRE_MITIGATED;
241 method = SPECTRE_V2_METHOD_BPIALL;
242 break;
243
244 default:
245 state = SPECTRE_UNAFFECTED;
246 break;
247 }
248
249 if (state == SPECTRE_MITIGATED)
250 state = spectre_bhb_install_workaround(method);
251
252 spectre_v2_update_state(state, method);
253}
254
255static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
256 u32 mask, const char *msg)
257{
258 u32 aux_cr;
259
260 asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr));
261
262 if ((aux_cr & mask) != mask) {
263 if (!*warned)
264 pr_err("CPU%u: %s", smp_processor_id(), msg);
265 *warned = true;
266 return false;
267 }
268 return true;
269}
270
271static DEFINE_PER_CPU(bool, spectre_warned);
272
273static bool check_spectre_auxcr(bool *warned, u32 bit)
274{
275 return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) &&
276 cpu_v7_check_auxcr_set(warned, bit,
277 "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n");
278}
279
280void cpu_v7_ca8_ibe(void)
281{
282 if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
283 cpu_v7_spectre_v2_init();
284}
285
286void cpu_v7_ca15_ibe(void)
287{
288 if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
289 cpu_v7_spectre_v2_init();
290 cpu_v7_spectre_bhb_init();
291}
292
293void cpu_v7_bugs_init(void)
294{
295 cpu_v7_spectre_v2_init();
296 cpu_v7_spectre_bhb_init();
297}
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/arm-smccc.h>
3#include <linux/kernel.h>
4#include <linux/smp.h>
5
6#include <asm/cp15.h>
7#include <asm/cputype.h>
8#include <asm/proc-fns.h>
9#include <asm/system_misc.h>
10
11#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
12DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
13
14extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
15extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
16extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
17extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
18
19static void harden_branch_predictor_bpiall(void)
20{
21 write_sysreg(0, BPIALL);
22}
23
24static void harden_branch_predictor_iciallu(void)
25{
26 write_sysreg(0, ICIALLU);
27}
28
29static void __maybe_unused call_smc_arch_workaround_1(void)
30{
31 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
32}
33
34static void __maybe_unused call_hvc_arch_workaround_1(void)
35{
36 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
37}
38
39static void cpu_v7_spectre_init(void)
40{
41 const char *spectre_v2_method = NULL;
42 int cpu = smp_processor_id();
43
44 if (per_cpu(harden_branch_predictor_fn, cpu))
45 return;
46
47 switch (read_cpuid_part()) {
48 case ARM_CPU_PART_CORTEX_A8:
49 case ARM_CPU_PART_CORTEX_A9:
50 case ARM_CPU_PART_CORTEX_A12:
51 case ARM_CPU_PART_CORTEX_A17:
52 case ARM_CPU_PART_CORTEX_A73:
53 case ARM_CPU_PART_CORTEX_A75:
54 per_cpu(harden_branch_predictor_fn, cpu) =
55 harden_branch_predictor_bpiall;
56 spectre_v2_method = "BPIALL";
57 break;
58
59 case ARM_CPU_PART_CORTEX_A15:
60 case ARM_CPU_PART_BRAHMA_B15:
61 per_cpu(harden_branch_predictor_fn, cpu) =
62 harden_branch_predictor_iciallu;
63 spectre_v2_method = "ICIALLU";
64 break;
65
66#ifdef CONFIG_ARM_PSCI
67 case ARM_CPU_PART_BRAHMA_B53:
68 /* Requires no workaround */
69 break;
70 default:
71 /* Other ARM CPUs require no workaround */
72 if (read_cpuid_implementor() == ARM_CPU_IMP_ARM)
73 break;
74 fallthrough;
75 /* Cortex A57/A72 require firmware workaround */
76 case ARM_CPU_PART_CORTEX_A57:
77 case ARM_CPU_PART_CORTEX_A72: {
78 struct arm_smccc_res res;
79
80 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
81 ARM_SMCCC_ARCH_WORKAROUND_1, &res);
82 if ((int)res.a0 != 0)
83 return;
84
85 switch (arm_smccc_1_1_get_conduit()) {
86 case SMCCC_CONDUIT_HVC:
87 per_cpu(harden_branch_predictor_fn, cpu) =
88 call_hvc_arch_workaround_1;
89 cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
90 spectre_v2_method = "hypervisor";
91 break;
92
93 case SMCCC_CONDUIT_SMC:
94 per_cpu(harden_branch_predictor_fn, cpu) =
95 call_smc_arch_workaround_1;
96 cpu_do_switch_mm = cpu_v7_smc_switch_mm;
97 spectre_v2_method = "firmware";
98 break;
99
100 default:
101 break;
102 }
103 }
104#endif
105 }
106
107 if (spectre_v2_method)
108 pr_info("CPU%u: Spectre v2: using %s workaround\n",
109 smp_processor_id(), spectre_v2_method);
110}
111#else
112static void cpu_v7_spectre_init(void)
113{
114}
115#endif
116
117static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
118 u32 mask, const char *msg)
119{
120 u32 aux_cr;
121
122 asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr));
123
124 if ((aux_cr & mask) != mask) {
125 if (!*warned)
126 pr_err("CPU%u: %s", smp_processor_id(), msg);
127 *warned = true;
128 return false;
129 }
130 return true;
131}
132
133static DEFINE_PER_CPU(bool, spectre_warned);
134
135static bool check_spectre_auxcr(bool *warned, u32 bit)
136{
137 return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) &&
138 cpu_v7_check_auxcr_set(warned, bit,
139 "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n");
140}
141
142void cpu_v7_ca8_ibe(void)
143{
144 if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
145 cpu_v7_spectre_init();
146}
147
148void cpu_v7_ca15_ibe(void)
149{
150 if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
151 cpu_v7_spectre_init();
152}
153
154void cpu_v7_bugs_init(void)
155{
156 cpu_v7_spectre_init();
157}