Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5#include <linux/kernel.h>
6#include <linux/init.h>
7#include <linux/mm.h>
8#include <linux/module.h>
9#include <linux/irq.h>
10
11#include <asm/processor.h>
12#include <asm/ptrace.h>
13#include <asm/csr.h>
14
15#define INSN_MATCH_LB 0x3
16#define INSN_MASK_LB 0x707f
17#define INSN_MATCH_LH 0x1003
18#define INSN_MASK_LH 0x707f
19#define INSN_MATCH_LW 0x2003
20#define INSN_MASK_LW 0x707f
21#define INSN_MATCH_LD 0x3003
22#define INSN_MASK_LD 0x707f
23#define INSN_MATCH_LBU 0x4003
24#define INSN_MASK_LBU 0x707f
25#define INSN_MATCH_LHU 0x5003
26#define INSN_MASK_LHU 0x707f
27#define INSN_MATCH_LWU 0x6003
28#define INSN_MASK_LWU 0x707f
29#define INSN_MATCH_SB 0x23
30#define INSN_MASK_SB 0x707f
31#define INSN_MATCH_SH 0x1023
32#define INSN_MASK_SH 0x707f
33#define INSN_MATCH_SW 0x2023
34#define INSN_MASK_SW 0x707f
35#define INSN_MATCH_SD 0x3023
36#define INSN_MASK_SD 0x707f
37
38#define INSN_MATCH_FLW 0x2007
39#define INSN_MASK_FLW 0x707f
40#define INSN_MATCH_FLD 0x3007
41#define INSN_MASK_FLD 0x707f
42#define INSN_MATCH_FLQ 0x4007
43#define INSN_MASK_FLQ 0x707f
44#define INSN_MATCH_FSW 0x2027
45#define INSN_MASK_FSW 0x707f
46#define INSN_MATCH_FSD 0x3027
47#define INSN_MASK_FSD 0x707f
48#define INSN_MATCH_FSQ 0x4027
49#define INSN_MASK_FSQ 0x707f
50
51#define INSN_MATCH_C_LD 0x6000
52#define INSN_MASK_C_LD 0xe003
53#define INSN_MATCH_C_SD 0xe000
54#define INSN_MASK_C_SD 0xe003
55#define INSN_MATCH_C_LW 0x4000
56#define INSN_MASK_C_LW 0xe003
57#define INSN_MATCH_C_SW 0xc000
58#define INSN_MASK_C_SW 0xe003
59#define INSN_MATCH_C_LDSP 0x6002
60#define INSN_MASK_C_LDSP 0xe003
61#define INSN_MATCH_C_SDSP 0xe002
62#define INSN_MASK_C_SDSP 0xe003
63#define INSN_MATCH_C_LWSP 0x4002
64#define INSN_MASK_C_LWSP 0xe003
65#define INSN_MATCH_C_SWSP 0xc002
66#define INSN_MASK_C_SWSP 0xe003
67
68#define INSN_MATCH_C_FLD 0x2000
69#define INSN_MASK_C_FLD 0xe003
70#define INSN_MATCH_C_FLW 0x6000
71#define INSN_MASK_C_FLW 0xe003
72#define INSN_MATCH_C_FSD 0xa000
73#define INSN_MASK_C_FSD 0xe003
74#define INSN_MATCH_C_FSW 0xe000
75#define INSN_MASK_C_FSW 0xe003
76#define INSN_MATCH_C_FLDSP 0x2002
77#define INSN_MASK_C_FLDSP 0xe003
78#define INSN_MATCH_C_FSDSP 0xa002
79#define INSN_MASK_C_FSDSP 0xe003
80#define INSN_MATCH_C_FLWSP 0x6002
81#define INSN_MASK_C_FLWSP 0xe003
82#define INSN_MATCH_C_FSWSP 0xe002
83#define INSN_MASK_C_FSWSP 0xe003
84
85#define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
86
87#if defined(CONFIG_64BIT)
88#define LOG_REGBYTES 3
89#define XLEN 64
90#else
91#define LOG_REGBYTES 2
92#define XLEN 32
93#endif
94#define REGBYTES (1 << LOG_REGBYTES)
95#define XLEN_MINUS_16 ((XLEN) - 16)
96
97#define SH_RD 7
98#define SH_RS1 15
99#define SH_RS2 20
100#define SH_RS2C 2
101
102#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
103#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
104 (RV_X(x, 10, 3) << 3) | \
105 (RV_X(x, 5, 1) << 6))
106#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
107 (RV_X(x, 5, 2) << 6))
108#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
109 (RV_X(x, 12, 1) << 5) | \
110 (RV_X(x, 2, 2) << 6))
111#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
112 (RV_X(x, 12, 1) << 5) | \
113 (RV_X(x, 2, 3) << 6))
114#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
115 (RV_X(x, 7, 2) << 6))
116#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
117 (RV_X(x, 7, 3) << 6))
118#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
119#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
120#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
121
122#define SHIFT_RIGHT(x, y) \
123 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
124
125#define REG_MASK \
126 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
127
128#define REG_OFFSET(insn, pos) \
129 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
130
131#define REG_PTR(insn, pos, regs) \
132 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
133
134#define GET_RM(insn) (((insn) >> 12) & 7)
135
136#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
137#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
138#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
139#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
140#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
141#define GET_SP(regs) (*REG_PTR(2, 0, regs))
142#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
143#define IMM_I(insn) ((s32)(insn) >> 20)
144#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
145 (s32)(((insn) >> 7) & 0x1f))
146#define MASK_FUNCT3 0x7000
147
148#define GET_PRECISION(insn) (((insn) >> 25) & 3)
149#define GET_RM(insn) (((insn) >> 12) & 7)
150#define PRECISION_S 0
151#define PRECISION_D 1
152
153#define STR(x) XSTR(x)
154#define XSTR(x) #x
155
156#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \
157static inline type load_##type(const type *addr) \
158{ \
159 type val; \
160 asm (#insn " %0, %1" \
161 : "=&r" (val) : "m" (*addr)); \
162 return val; \
163}
164
165#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \
166static inline void store_##type(type *addr, type val) \
167{ \
168 asm volatile (#insn " %0, %1\n" \
169 : : "r" (val), "m" (*addr)); \
170}
171
172DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu)
173DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu)
174DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb)
175DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh)
176DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw)
177DECLARE_UNPRIVILEGED_STORE_FUNCTION(u8, sb)
178DECLARE_UNPRIVILEGED_STORE_FUNCTION(u16, sh)
179DECLARE_UNPRIVILEGED_STORE_FUNCTION(u32, sw)
180#if defined(CONFIG_64BIT)
181DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu)
182DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld)
183DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64, sd)
184DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld)
185#else
186DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw)
187DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw)
188
189static inline u64 load_u64(const u64 *addr)
190{
191 return load_u32((u32 *)addr)
192 + ((u64)load_u32((u32 *)addr + 1) << 32);
193}
194
195static inline void store_u64(u64 *addr, u64 val)
196{
197 store_u32((u32 *)addr, val);
198 store_u32((u32 *)addr + 1, val >> 32);
199}
200#endif
201
202static inline ulong get_insn(ulong mepc)
203{
204 register ulong __mepc asm ("a2") = mepc;
205 ulong val, rvc_mask = 3, tmp;
206
207 asm ("and %[tmp], %[addr], 2\n"
208 "bnez %[tmp], 1f\n"
209#if defined(CONFIG_64BIT)
210 STR(LWU) " %[insn], (%[addr])\n"
211#else
212 STR(LW) " %[insn], (%[addr])\n"
213#endif
214 "and %[tmp], %[insn], %[rvc_mask]\n"
215 "beq %[tmp], %[rvc_mask], 2f\n"
216 "sll %[insn], %[insn], %[xlen_minus_16]\n"
217 "srl %[insn], %[insn], %[xlen_minus_16]\n"
218 "j 2f\n"
219 "1:\n"
220 "lhu %[insn], (%[addr])\n"
221 "and %[tmp], %[insn], %[rvc_mask]\n"
222 "bne %[tmp], %[rvc_mask], 2f\n"
223 "lhu %[tmp], 2(%[addr])\n"
224 "sll %[tmp], %[tmp], 16\n"
225 "add %[insn], %[insn], %[tmp]\n"
226 "2:"
227 : [insn] "=&r" (val), [tmp] "=&r" (tmp)
228 : [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask),
229 [xlen_minus_16] "i" (XLEN_MINUS_16));
230
231 return val;
232}
233
234union reg_data {
235 u8 data_bytes[8];
236 ulong data_ulong;
237 u64 data_u64;
238};
239
240int handle_misaligned_load(struct pt_regs *regs)
241{
242 union reg_data val;
243 unsigned long epc = regs->epc;
244 unsigned long insn = get_insn(epc);
245 unsigned long addr = csr_read(mtval);
246 int i, fp = 0, shift = 0, len = 0;
247
248 regs->epc = 0;
249
250 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
251 len = 4;
252 shift = 8 * (sizeof(unsigned long) - len);
253#if defined(CONFIG_64BIT)
254 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
255 len = 8;
256 shift = 8 * (sizeof(unsigned long) - len);
257 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
258 len = 4;
259#endif
260 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
261 fp = 1;
262 len = 8;
263 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
264 fp = 1;
265 len = 4;
266 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
267 len = 2;
268 shift = 8 * (sizeof(unsigned long) - len);
269 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
270 len = 2;
271#if defined(CONFIG_64BIT)
272 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
273 len = 8;
274 shift = 8 * (sizeof(unsigned long) - len);
275 insn = RVC_RS2S(insn) << SH_RD;
276 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
277 ((insn >> SH_RD) & 0x1f)) {
278 len = 8;
279 shift = 8 * (sizeof(unsigned long) - len);
280#endif
281 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
282 len = 4;
283 shift = 8 * (sizeof(unsigned long) - len);
284 insn = RVC_RS2S(insn) << SH_RD;
285 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
286 ((insn >> SH_RD) & 0x1f)) {
287 len = 4;
288 shift = 8 * (sizeof(unsigned long) - len);
289 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
290 fp = 1;
291 len = 8;
292 insn = RVC_RS2S(insn) << SH_RD;
293 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
294 fp = 1;
295 len = 8;
296#if defined(CONFIG_32BIT)
297 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
298 fp = 1;
299 len = 4;
300 insn = RVC_RS2S(insn) << SH_RD;
301 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
302 fp = 1;
303 len = 4;
304#endif
305 } else {
306 regs->epc = epc;
307 return -1;
308 }
309
310 val.data_u64 = 0;
311 for (i = 0; i < len; i++)
312 val.data_bytes[i] = load_u8((void *)(addr + i));
313
314 if (fp)
315 return -1;
316 SET_RD(insn, regs, val.data_ulong << shift >> shift);
317
318 regs->epc = epc + INSN_LEN(insn);
319
320 return 0;
321}
322
323int handle_misaligned_store(struct pt_regs *regs)
324{
325 union reg_data val;
326 unsigned long epc = regs->epc;
327 unsigned long insn = get_insn(epc);
328 unsigned long addr = csr_read(mtval);
329 int i, len = 0;
330
331 regs->epc = 0;
332
333 val.data_ulong = GET_RS2(insn, regs);
334
335 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
336 len = 4;
337#if defined(CONFIG_64BIT)
338 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
339 len = 8;
340#endif
341 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
342 len = 2;
343#if defined(CONFIG_64BIT)
344 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
345 len = 8;
346 val.data_ulong = GET_RS2S(insn, regs);
347 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
348 ((insn >> SH_RD) & 0x1f)) {
349 len = 8;
350 val.data_ulong = GET_RS2C(insn, regs);
351#endif
352 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
353 len = 4;
354 val.data_ulong = GET_RS2S(insn, regs);
355 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
356 ((insn >> SH_RD) & 0x1f)) {
357 len = 4;
358 val.data_ulong = GET_RS2C(insn, regs);
359 } else {
360 regs->epc = epc;
361 return -1;
362 }
363
364 for (i = 0; i < len; i++)
365 store_u8((void *)(addr + i), val.data_bytes[i]);
366
367 regs->epc = epc + INSN_LEN(insn);
368
369 return 0;
370}