1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/arm-smccc.h>
  3#include <linux/kernel.h>
  4#include <linux/smp.h>
  5
  6#include <asm/cp15.h>
  7#include <asm/cputype.h>
  8#include <asm/proc-fns.h>
  9#include <asm/spectre.h>
 10#include <asm/system_misc.h>
 11
 12#ifdef CONFIG_ARM_PSCI
 13static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
 14{
 15	struct arm_smccc_res res;
 16
 17	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 18			     ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 19
 20	switch ((int)res.a0) {
 21	case SMCCC_RET_SUCCESS:
 22		return SPECTRE_MITIGATED;
 23
 24	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
 25		return SPECTRE_UNAFFECTED;
 26
 27	default:
 28		return SPECTRE_VULNERABLE;
 29	}
 30}
 31#else
 32static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
 33{
 34	return SPECTRE_VULNERABLE;
 35}
 36#endif
 37
 38#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 39DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
 40
 41extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
 42extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
 43extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
 44extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
 45
 46static void harden_branch_predictor_bpiall(void)
 47{
 48	write_sysreg(0, BPIALL);
 49}
 50
 51static void harden_branch_predictor_iciallu(void)
 52{
 53	write_sysreg(0, ICIALLU);
 54}
 55
 56static void __maybe_unused call_smc_arch_workaround_1(void)
 57{
 58	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 59}
 60
 61static void __maybe_unused call_hvc_arch_workaround_1(void)
 62{
 63	arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 64}
 65
 66static unsigned int spectre_v2_install_workaround(unsigned int method)
 67{
 68	const char *spectre_v2_method = NULL;
 69	int cpu = smp_processor_id();
 70
 71	if (per_cpu(harden_branch_predictor_fn, cpu))
 72		return SPECTRE_MITIGATED;
 73
 74	switch (method) {
 75	case SPECTRE_V2_METHOD_BPIALL:
 76		per_cpu(harden_branch_predictor_fn, cpu) =
 77			harden_branch_predictor_bpiall;
 78		spectre_v2_method = "BPIALL";
 79		break;
 80
 81	case SPECTRE_V2_METHOD_ICIALLU:
 82		per_cpu(harden_branch_predictor_fn, cpu) =
 83			harden_branch_predictor_iciallu;
 84		spectre_v2_method = "ICIALLU";
 85		break;
 86
 87	case SPECTRE_V2_METHOD_HVC:
 88		per_cpu(harden_branch_predictor_fn, cpu) =
 89			call_hvc_arch_workaround_1;
 90		cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
 91		spectre_v2_method = "hypervisor";
 92		break;
 93
 94	case SPECTRE_V2_METHOD_SMC:
 95		per_cpu(harden_branch_predictor_fn, cpu) =
 96			call_smc_arch_workaround_1;
 97		cpu_do_switch_mm = cpu_v7_smc_switch_mm;
 98		spectre_v2_method = "firmware";
 99		break;
100	}
101
102	if (spectre_v2_method)
103		pr_info("CPU%u: Spectre v2: using %s workaround\n",
104			smp_processor_id(), spectre_v2_method);
105
106	return SPECTRE_MITIGATED;
107}
108#else
109static unsigned int spectre_v2_install_workaround(unsigned int method)
110{
111	pr_info_once("Spectre V2: workarounds disabled by configuration\n");
112
113	return SPECTRE_VULNERABLE;
114}
115#endif
116
117static void cpu_v7_spectre_v2_init(void)
118{
119	unsigned int state, method = 0;
120
121	switch (read_cpuid_part()) {
122	case ARM_CPU_PART_CORTEX_A8:
123	case ARM_CPU_PART_CORTEX_A9:
124	case ARM_CPU_PART_CORTEX_A12:
125	case ARM_CPU_PART_CORTEX_A17:
126	case ARM_CPU_PART_CORTEX_A73:
127	case ARM_CPU_PART_CORTEX_A75:
128		state = SPECTRE_MITIGATED;
129		method = SPECTRE_V2_METHOD_BPIALL;
130		break;
131
132	case ARM_CPU_PART_CORTEX_A15:
133	case ARM_CPU_PART_BRAHMA_B15:
134		state = SPECTRE_MITIGATED;
135		method = SPECTRE_V2_METHOD_ICIALLU;
136		break;
137
138	case ARM_CPU_PART_BRAHMA_B53:
139		/* Requires no workaround */
140		state = SPECTRE_UNAFFECTED;
141		break;
142
143	default:
144		/* Other ARM CPUs require no workaround */
145		if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) {
146			state = SPECTRE_UNAFFECTED;
147			break;
148		}
149
150		fallthrough;
151
152	/* Cortex A57/A72 require firmware workaround */
153	case ARM_CPU_PART_CORTEX_A57:
154	case ARM_CPU_PART_CORTEX_A72:
155		state = spectre_v2_get_cpu_fw_mitigation_state();
156		if (state != SPECTRE_MITIGATED)
157			break;
158
159		switch (arm_smccc_1_1_get_conduit()) {
160		case SMCCC_CONDUIT_HVC:
161			method = SPECTRE_V2_METHOD_HVC;
162			break;
163
164		case SMCCC_CONDUIT_SMC:
165			method = SPECTRE_V2_METHOD_SMC;
166			break;
167
168		default:
169			state = SPECTRE_VULNERABLE;
170			break;
171		}
172	}
173
174	if (state == SPECTRE_MITIGATED)
175		state = spectre_v2_install_workaround(method);
176
177	spectre_v2_update_state(state, method);
178}
179
180#ifdef CONFIG_HARDEN_BRANCH_HISTORY
181static int spectre_bhb_method;
182
183static const char *spectre_bhb_method_name(int method)
184{
185	switch (method) {
186	case SPECTRE_V2_METHOD_LOOP8:
187		return "loop";
188
189	case SPECTRE_V2_METHOD_BPIALL:
190		return "BPIALL";
191
192	default:
193		return "unknown";
194	}
195}
196
197static int spectre_bhb_install_workaround(int method)
198{
199	if (spectre_bhb_method != method) {
200		if (spectre_bhb_method) {
201			pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n",
202			       smp_processor_id());
203
204			return SPECTRE_VULNERABLE;
205		}
206
207		if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE)
208			return SPECTRE_VULNERABLE;
209
210		spectre_bhb_method = method;
211
212		pr_info("CPU%u: Spectre BHB: enabling %s workaround for all CPUs\n",
213			smp_processor_id(), spectre_bhb_method_name(method));
214	}
215
216	return SPECTRE_MITIGATED;
217}
218#else
219static int spectre_bhb_install_workaround(int method)
220{
221	return SPECTRE_VULNERABLE;
222}
223#endif
224
225static void cpu_v7_spectre_bhb_init(void)
226{
227	unsigned int state, method = 0;
228
229	switch (read_cpuid_part()) {
230	case ARM_CPU_PART_CORTEX_A15:
231	case ARM_CPU_PART_BRAHMA_B15:
232	case ARM_CPU_PART_CORTEX_A57:
233	case ARM_CPU_PART_CORTEX_A72:
234		state = SPECTRE_MITIGATED;
235		method = SPECTRE_V2_METHOD_LOOP8;
236		break;
237
238	case ARM_CPU_PART_CORTEX_A73:
239	case ARM_CPU_PART_CORTEX_A75:
240		state = SPECTRE_MITIGATED;
241		method = SPECTRE_V2_METHOD_BPIALL;
242		break;
243
244	default:
245		state = SPECTRE_UNAFFECTED;
246		break;
247	}
248
249	if (state == SPECTRE_MITIGATED)
250		state = spectre_bhb_install_workaround(method);
251
252	spectre_v2_update_state(state, method);
253}
254
255static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
256						  u32 mask, const char *msg)
257{
258	u32 aux_cr;
259
260	asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr));
261
262	if ((aux_cr & mask) != mask) {
263		if (!*warned)
264			pr_err("CPU%u: %s", smp_processor_id(), msg);
265		*warned = true;
266		return false;
267	}
268	return true;
269}
270
271static DEFINE_PER_CPU(bool, spectre_warned);
272
273static bool check_spectre_auxcr(bool *warned, u32 bit)
274{
275	return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) &&
276		cpu_v7_check_auxcr_set(warned, bit,
277				       "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n");
278}
279
280void cpu_v7_ca8_ibe(void)
281{
282	if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
283		cpu_v7_spectre_v2_init();
284}
285
286void cpu_v7_ca15_ibe(void)
287{
288	if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
289		cpu_v7_spectre_v2_init();
290	cpu_v7_spectre_bhb_init();
291}
292
293void cpu_v7_bugs_init(void)
294{
295	cpu_v7_spectre_v2_init();
296	cpu_v7_spectre_bhb_init();
297}