Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5#include <linux/kernel.h>
6#include <linux/init.h>
7#include <linux/mm.h>
8#include <linux/module.h>
9#include <linux/perf_event.h>
10#include <linux/irq.h>
11#include <linux/stringify.h>
12
13#include <asm/processor.h>
14#include <asm/ptrace.h>
15#include <asm/csr.h>
16#include <asm/entry-common.h>
17#include <asm/hwprobe.h>
18#include <asm/cpufeature.h>
19#include <asm/vector.h>
20
21#define INSN_MATCH_LB 0x3
22#define INSN_MASK_LB 0x707f
23#define INSN_MATCH_LH 0x1003
24#define INSN_MASK_LH 0x707f
25#define INSN_MATCH_LW 0x2003
26#define INSN_MASK_LW 0x707f
27#define INSN_MATCH_LD 0x3003
28#define INSN_MASK_LD 0x707f
29#define INSN_MATCH_LBU 0x4003
30#define INSN_MASK_LBU 0x707f
31#define INSN_MATCH_LHU 0x5003
32#define INSN_MASK_LHU 0x707f
33#define INSN_MATCH_LWU 0x6003
34#define INSN_MASK_LWU 0x707f
35#define INSN_MATCH_SB 0x23
36#define INSN_MASK_SB 0x707f
37#define INSN_MATCH_SH 0x1023
38#define INSN_MASK_SH 0x707f
39#define INSN_MATCH_SW 0x2023
40#define INSN_MASK_SW 0x707f
41#define INSN_MATCH_SD 0x3023
42#define INSN_MASK_SD 0x707f
43
44#define INSN_MATCH_FLW 0x2007
45#define INSN_MASK_FLW 0x707f
46#define INSN_MATCH_FLD 0x3007
47#define INSN_MASK_FLD 0x707f
48#define INSN_MATCH_FLQ 0x4007
49#define INSN_MASK_FLQ 0x707f
50#define INSN_MATCH_FSW 0x2027
51#define INSN_MASK_FSW 0x707f
52#define INSN_MATCH_FSD 0x3027
53#define INSN_MASK_FSD 0x707f
54#define INSN_MATCH_FSQ 0x4027
55#define INSN_MASK_FSQ 0x707f
56
57#define INSN_MATCH_C_LD 0x6000
58#define INSN_MASK_C_LD 0xe003
59#define INSN_MATCH_C_SD 0xe000
60#define INSN_MASK_C_SD 0xe003
61#define INSN_MATCH_C_LW 0x4000
62#define INSN_MASK_C_LW 0xe003
63#define INSN_MATCH_C_SW 0xc000
64#define INSN_MASK_C_SW 0xe003
65#define INSN_MATCH_C_LDSP 0x6002
66#define INSN_MASK_C_LDSP 0xe003
67#define INSN_MATCH_C_SDSP 0xe002
68#define INSN_MASK_C_SDSP 0xe003
69#define INSN_MATCH_C_LWSP 0x4002
70#define INSN_MASK_C_LWSP 0xe003
71#define INSN_MATCH_C_SWSP 0xc002
72#define INSN_MASK_C_SWSP 0xe003
73
74#define INSN_MATCH_C_FLD 0x2000
75#define INSN_MASK_C_FLD 0xe003
76#define INSN_MATCH_C_FLW 0x6000
77#define INSN_MASK_C_FLW 0xe003
78#define INSN_MATCH_C_FSD 0xa000
79#define INSN_MASK_C_FSD 0xe003
80#define INSN_MATCH_C_FSW 0xe000
81#define INSN_MASK_C_FSW 0xe003
82#define INSN_MATCH_C_FLDSP 0x2002
83#define INSN_MASK_C_FLDSP 0xe003
84#define INSN_MATCH_C_FSDSP 0xa002
85#define INSN_MASK_C_FSDSP 0xe003
86#define INSN_MATCH_C_FLWSP 0x6002
87#define INSN_MASK_C_FLWSP 0xe003
88#define INSN_MATCH_C_FSWSP 0xe002
89#define INSN_MASK_C_FSWSP 0xe003
90
91#define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
92
93#if defined(CONFIG_64BIT)
94#define LOG_REGBYTES 3
95#define XLEN 64
96#else
97#define LOG_REGBYTES 2
98#define XLEN 32
99#endif
100#define REGBYTES (1 << LOG_REGBYTES)
101#define XLEN_MINUS_16 ((XLEN) - 16)
102
103#define SH_RD 7
104#define SH_RS1 15
105#define SH_RS2 20
106#define SH_RS2C 2
107
108#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
109#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
110 (RV_X(x, 10, 3) << 3) | \
111 (RV_X(x, 5, 1) << 6))
112#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
113 (RV_X(x, 5, 2) << 6))
114#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
115 (RV_X(x, 12, 1) << 5) | \
116 (RV_X(x, 2, 2) << 6))
117#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
118 (RV_X(x, 12, 1) << 5) | \
119 (RV_X(x, 2, 3) << 6))
120#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
121 (RV_X(x, 7, 2) << 6))
122#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
123 (RV_X(x, 7, 3) << 6))
124#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
125#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
126#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
127
128#define SHIFT_RIGHT(x, y) \
129 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
130
131#define REG_MASK \
132 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
133
134#define REG_OFFSET(insn, pos) \
135 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
136
137#define REG_PTR(insn, pos, regs) \
138 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
139
140#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
141#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
142#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
143#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
144#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
145#define GET_SP(regs) (*REG_PTR(2, 0, regs))
146#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
147#define IMM_I(insn) ((s32)(insn) >> 20)
148#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
149 (s32)(((insn) >> 7) & 0x1f))
150#define MASK_FUNCT3 0x7000
151
152#define GET_PRECISION(insn) (((insn) >> 25) & 3)
153#define GET_RM(insn) (((insn) >> 12) & 7)
154#define PRECISION_S 0
155#define PRECISION_D 1
156
157#ifdef CONFIG_FPU
158
159#define FP_GET_RD(insn) (insn >> 7 & 0x1F)
160
161extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
162
163static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
164 unsigned long val)
165{
166 unsigned long fp_reg = FP_GET_RD(insn);
167
168 put_f32_reg(fp_reg, val);
169 regs->status |= SR_FS_DIRTY;
170
171 return 0;
172}
173
174extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
175
176static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
177{
178 unsigned long fp_reg = FP_GET_RD(insn);
179 unsigned long value;
180
181#if __riscv_xlen == 32
182 value = (unsigned long) &val;
183#else
184 value = val;
185#endif
186 put_f64_reg(fp_reg, value);
187 regs->status |= SR_FS_DIRTY;
188
189 return 0;
190}
191
192#if __riscv_xlen == 32
193extern void get_f64_reg(unsigned long fp_reg, u64 *value);
194
195static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
196 struct pt_regs *regs)
197{
198 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
199 u64 val;
200
201 get_f64_reg(fp_reg, &val);
202 regs->status |= SR_FS_DIRTY;
203
204 return val;
205}
206#else
207
208extern unsigned long get_f64_reg(unsigned long fp_reg);
209
210static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
211 struct pt_regs *regs)
212{
213 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
214 unsigned long val;
215
216 val = get_f64_reg(fp_reg);
217 regs->status |= SR_FS_DIRTY;
218
219 return val;
220}
221
222#endif
223
224extern unsigned long get_f32_reg(unsigned long fp_reg);
225
226static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
227 struct pt_regs *regs)
228{
229 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
230 unsigned long val;
231
232 val = get_f32_reg(fp_reg);
233 regs->status |= SR_FS_DIRTY;
234
235 return val;
236}
237
238#else /* CONFIG_FPU */
239static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
240 unsigned long val) {}
241
242static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
243
244static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
245 struct pt_regs *regs)
246{
247 return 0;
248}
249
250static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
251 struct pt_regs *regs)
252{
253 return 0;
254}
255
256#endif
257
258#define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
259#define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
260#define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
261
262#define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
263#define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
264#define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
265
266#define __read_insn(regs, insn, insn_addr, type) \
267({ \
268 int __ret; \
269 \
270 if (user_mode(regs)) { \
271 __ret = __get_user(insn, (type __user *) insn_addr); \
272 } else { \
273 insn = *(type *)insn_addr; \
274 __ret = 0; \
275 } \
276 \
277 __ret; \
278})
279
280static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
281{
282 ulong insn = 0;
283
284 if (epc & 0x2) {
285 ulong tmp = 0;
286
287 if (__read_insn(regs, insn, epc, u16))
288 return -EFAULT;
289 /* __get_user() uses regular "lw" which sign extend the loaded
290 * value make sure to clear higher order bits in case we "or" it
291 * below with the upper 16 bits half.
292 */
293 insn &= GENMASK(15, 0);
294 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
295 *r_insn = insn;
296 return 0;
297 }
298 epc += sizeof(u16);
299 if (__read_insn(regs, tmp, epc, u16))
300 return -EFAULT;
301 *r_insn = (tmp << 16) | insn;
302
303 return 0;
304 } else {
305 if (__read_insn(regs, insn, epc, u32))
306 return -EFAULT;
307 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
308 *r_insn = insn;
309 return 0;
310 }
311 insn &= GENMASK(15, 0);
312 *r_insn = insn;
313
314 return 0;
315 }
316}
317
318union reg_data {
319 u8 data_bytes[8];
320 ulong data_ulong;
321 u64 data_u64;
322};
323
324/* sysctl hooks */
325int unaligned_enabled __read_mostly = 1; /* Enabled by default */
326
327#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
328static int handle_vector_misaligned_load(struct pt_regs *regs)
329{
330 unsigned long epc = regs->epc;
331 unsigned long insn;
332
333 if (get_insn(regs, epc, &insn))
334 return -1;
335
336 /* Only return 0 when in check_vector_unaligned_access_emulated */
337 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
338 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
339 regs->epc = epc + INSN_LEN(insn);
340 return 0;
341 }
342
343 /* If vector instruction we don't emulate it yet */
344 regs->epc = epc;
345 return -1;
346}
347#else
348static int handle_vector_misaligned_load(struct pt_regs *regs)
349{
350 return -1;
351}
352#endif
353
354static int handle_scalar_misaligned_load(struct pt_regs *regs)
355{
356 union reg_data val;
357 unsigned long epc = regs->epc;
358 unsigned long insn;
359 unsigned long addr = regs->badaddr;
360 int fp = 0, shift = 0, len = 0;
361
362 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
363
364#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
365 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
366#endif
367
368 if (!unaligned_enabled)
369 return -1;
370
371 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
372 return -1;
373
374 if (get_insn(regs, epc, &insn))
375 return -1;
376
377 regs->epc = 0;
378
379 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
380 len = 4;
381 shift = 8 * (sizeof(unsigned long) - len);
382#if defined(CONFIG_64BIT)
383 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
384 len = 8;
385 shift = 8 * (sizeof(unsigned long) - len);
386 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
387 len = 4;
388#endif
389 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
390 fp = 1;
391 len = 8;
392 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
393 fp = 1;
394 len = 4;
395 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
396 len = 2;
397 shift = 8 * (sizeof(unsigned long) - len);
398 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
399 len = 2;
400#if defined(CONFIG_64BIT)
401 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
402 len = 8;
403 shift = 8 * (sizeof(unsigned long) - len);
404 insn = RVC_RS2S(insn) << SH_RD;
405 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
406 ((insn >> SH_RD) & 0x1f)) {
407 len = 8;
408 shift = 8 * (sizeof(unsigned long) - len);
409#endif
410 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
411 len = 4;
412 shift = 8 * (sizeof(unsigned long) - len);
413 insn = RVC_RS2S(insn) << SH_RD;
414 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
415 ((insn >> SH_RD) & 0x1f)) {
416 len = 4;
417 shift = 8 * (sizeof(unsigned long) - len);
418 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
419 fp = 1;
420 len = 8;
421 insn = RVC_RS2S(insn) << SH_RD;
422 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
423 fp = 1;
424 len = 8;
425#if defined(CONFIG_32BIT)
426 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
427 fp = 1;
428 len = 4;
429 insn = RVC_RS2S(insn) << SH_RD;
430 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
431 fp = 1;
432 len = 4;
433#endif
434 } else {
435 regs->epc = epc;
436 return -1;
437 }
438
439 if (!IS_ENABLED(CONFIG_FPU) && fp)
440 return -EOPNOTSUPP;
441
442 val.data_u64 = 0;
443 if (user_mode(regs)) {
444 if (copy_from_user(&val, (u8 __user *)addr, len))
445 return -1;
446 } else {
447 memcpy(&val, (u8 *)addr, len);
448 }
449
450 if (!fp)
451 SET_RD(insn, regs, val.data_ulong << shift >> shift);
452 else if (len == 8)
453 set_f64_rd(insn, regs, val.data_u64);
454 else
455 set_f32_rd(insn, regs, val.data_ulong);
456
457 regs->epc = epc + INSN_LEN(insn);
458
459 return 0;
460}
461
462static int handle_scalar_misaligned_store(struct pt_regs *regs)
463{
464 union reg_data val;
465 unsigned long epc = regs->epc;
466 unsigned long insn;
467 unsigned long addr = regs->badaddr;
468 int len = 0, fp = 0;
469
470 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
471
472 if (!unaligned_enabled)
473 return -1;
474
475 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
476 return -1;
477
478 if (get_insn(regs, epc, &insn))
479 return -1;
480
481 regs->epc = 0;
482
483 val.data_ulong = GET_RS2(insn, regs);
484
485 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
486 len = 4;
487#if defined(CONFIG_64BIT)
488 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
489 len = 8;
490#endif
491 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
492 fp = 1;
493 len = 8;
494 val.data_u64 = GET_F64_RS2(insn, regs);
495 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
496 fp = 1;
497 len = 4;
498 val.data_ulong = GET_F32_RS2(insn, regs);
499 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
500 len = 2;
501#if defined(CONFIG_64BIT)
502 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
503 len = 8;
504 val.data_ulong = GET_RS2S(insn, regs);
505 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
506 len = 8;
507 val.data_ulong = GET_RS2C(insn, regs);
508#endif
509 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
510 len = 4;
511 val.data_ulong = GET_RS2S(insn, regs);
512 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
513 len = 4;
514 val.data_ulong = GET_RS2C(insn, regs);
515 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
516 fp = 1;
517 len = 8;
518 val.data_u64 = GET_F64_RS2S(insn, regs);
519 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
520 fp = 1;
521 len = 8;
522 val.data_u64 = GET_F64_RS2C(insn, regs);
523#if !defined(CONFIG_64BIT)
524 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
525 fp = 1;
526 len = 4;
527 val.data_ulong = GET_F32_RS2S(insn, regs);
528 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
529 fp = 1;
530 len = 4;
531 val.data_ulong = GET_F32_RS2C(insn, regs);
532#endif
533 } else {
534 regs->epc = epc;
535 return -1;
536 }
537
538 if (!IS_ENABLED(CONFIG_FPU) && fp)
539 return -EOPNOTSUPP;
540
541 if (user_mode(regs)) {
542 if (copy_to_user((u8 __user *)addr, &val, len))
543 return -1;
544 } else {
545 memcpy((u8 *)addr, &val, len);
546 }
547
548 regs->epc = epc + INSN_LEN(insn);
549
550 return 0;
551}
552
553int handle_misaligned_load(struct pt_regs *regs)
554{
555 unsigned long epc = regs->epc;
556 unsigned long insn;
557
558 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
559 if (get_insn(regs, epc, &insn))
560 return -1;
561
562 if (insn_is_vector(insn))
563 return handle_vector_misaligned_load(regs);
564 }
565
566 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
567 return handle_scalar_misaligned_load(regs);
568
569 return -1;
570}
571
572int handle_misaligned_store(struct pt_regs *regs)
573{
574 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
575 return handle_scalar_misaligned_store(regs);
576
577 return -1;
578}
579
580#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
581void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
582{
583 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
584 unsigned long tmp_var;
585
586 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
587
588 kernel_vector_begin();
589 /*
590 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
591 * the clobber list. This inline asm clobbers v0, but since we do not
592 * currently build the kernel with V enabled, the v0 clobber arg is not
593 * needed (as the compiler will not emit vector code itself). If the kernel
594 * is changed to build with V enabled, the clobber arg will need to be
595 * added here.
596 */
597 __asm__ __volatile__ (
598 ".balign 4\n\t"
599 ".option push\n\t"
600 ".option arch, +zve32x\n\t"
601 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
602 " vle16.v v0, (%[ptr])\n\t" // Load bytes
603 ".option pop\n\t"
604 : : [ptr] "r" ((u8 *)&tmp_var + 1));
605 kernel_vector_end();
606}
607
608bool check_vector_unaligned_access_emulated_all_cpus(void)
609{
610 int cpu;
611
612 if (!has_vector()) {
613 for_each_online_cpu(cpu)
614 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
615 return false;
616 }
617
618 schedule_on_each_cpu(check_vector_unaligned_access_emulated);
619
620 for_each_online_cpu(cpu)
621 if (per_cpu(vector_misaligned_access, cpu)
622 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
623 return false;
624
625 return true;
626}
627#else
628bool check_vector_unaligned_access_emulated_all_cpus(void)
629{
630 return false;
631}
632#endif
633
634#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
635
636static bool unaligned_ctl __read_mostly;
637
638void check_unaligned_access_emulated(struct work_struct *work __always_unused)
639{
640 int cpu = smp_processor_id();
641 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
642 unsigned long tmp_var, tmp_val;
643
644 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
645
646 __asm__ __volatile__ (
647 " "REG_L" %[tmp], 1(%[ptr])\n"
648 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
649
650 /*
651 * If unaligned_ctl is already set, this means that we detected that all
652 * CPUS uses emulated misaligned access at boot time. If that changed
653 * when hotplugging the new cpu, this is something we don't handle.
654 */
655 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
656 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
657 while (true)
658 cpu_relax();
659 }
660}
661
662bool check_unaligned_access_emulated_all_cpus(void)
663{
664 int cpu;
665
666 /*
667 * We can only support PR_UNALIGN controls if all CPUs have misaligned
668 * accesses emulated since tasks requesting such control can run on any
669 * CPU.
670 */
671 schedule_on_each_cpu(check_unaligned_access_emulated);
672
673 for_each_online_cpu(cpu)
674 if (per_cpu(misaligned_access_speed, cpu)
675 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
676 return false;
677
678 unaligned_ctl = true;
679 return true;
680}
681
682bool unaligned_ctl_available(void)
683{
684 return unaligned_ctl;
685}
686#else
687bool check_unaligned_access_emulated_all_cpus(void)
688{
689 return false;
690}
691#endif