Loading...
1/*
2 * arch/xtensa/kernel/align.S
3 *
4 * Handle unalignment exceptions in kernel space.
5 *
6 * This file is subject to the terms and conditions of the GNU General
7 * Public License. See the file "COPYING" in the main directory of
8 * this archive for more details.
9 *
10 * Copyright (C) 2001 - 2005 Tensilica, Inc.
11 * Copyright (C) 2014 Cadence Design Systems Inc.
12 *
13 * Rewritten by Chris Zankel <chris@zankel.net>
14 *
15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
17 */
18
19#include <linux/linkage.h>
20#include <asm/current.h>
21#include <asm/asm-offsets.h>
22#include <asm/asmmacro.h>
23#include <asm/processor.h>
24
25#if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
26
27/* First-level exception handler for unaligned exceptions.
28 *
29 * Note: This handler works only for kernel exceptions. Unaligned user
30 * access should get a seg fault.
31 */
32
33/* Big and little endian 16-bit values are located in
34 * different halves of a register. HWORD_START helps to
35 * abstract the notion of extracting a 16-bit value from a
36 * register.
37 * We also have to define new shifting instructions because
38 * lsb and msb are on 'opposite' ends in a register for
39 * different endian machines.
40 *
41 * Assume a memory region in ascending address:
42 * 0 1 2 3|4 5 6 7
43 *
44 * When loading one word into a register, the content of that register is:
45 * LE 3 2 1 0, 7 6 5 4
46 * BE 0 1 2 3, 4 5 6 7
47 *
48 * Masking the bits of the higher/lower address means:
49 * LE X X 0 0, 0 0 X X
50 * BE 0 0 X X, X X 0 0
51 *
52 * Shifting to higher/lower addresses, means:
53 * LE shift left / shift right
54 * BE shift right / shift left
55 *
56 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
57 * LE mask 0 0 X X / shift left
58 * BE shift left / mask 0 0 X X
59 */
60
61#if XCHAL_HAVE_WINDOWED
62#define UNALIGNED_USER_EXCEPTION
63#endif
64
65#if XCHAL_HAVE_BE
66
67#define HWORD_START 16
68#define INSN_OP0 28
69#define INSN_T 24
70#define INSN_OP1 16
71
72.macro __ssa8r r; ssa8l \r; .endm
73.macro __sh r, s; srl \r, \s; .endm
74.macro __sl r, s; sll \r, \s; .endm
75.macro __exth r, s; extui \r, \s, 0, 16; .endm
76.macro __extl r, s; slli \r, \s, 16; .endm
77
78#else
79
80#define HWORD_START 0
81#define INSN_OP0 0
82#define INSN_T 4
83#define INSN_OP1 12
84
85.macro __ssa8r r; ssa8b \r; .endm
86.macro __sh r, s; sll \r, \s; .endm
87.macro __sl r, s; srl \r, \s; .endm
88.macro __exth r, s; slli \r, \s, 16; .endm
89.macro __extl r, s; extui \r, \s, 0, 16; .endm
90
91#endif
92
93/*
94 * xxxx xxxx = imm8 field
95 * yyyy = imm4 field
96 * ssss = s field
97 * tttt = t field
98 *
99 * 16 0
100 * -------------------
101 * L32I.N yyyy ssss tttt 1000
102 * S32I.N yyyy ssss tttt 1001
103 *
104 * 23 0
105 * -----------------------------
106 * res 0000 0010
107 * L16UI xxxx xxxx 0001 ssss tttt 0010
108 * L32I xxxx xxxx 0010 ssss tttt 0010
109 * XXX 0011 ssss tttt 0010
110 * XXX 0100 ssss tttt 0010
111 * S16I xxxx xxxx 0101 ssss tttt 0010
112 * S32I xxxx xxxx 0110 ssss tttt 0010
113 * XXX 0111 ssss tttt 0010
114 * XXX 1000 ssss tttt 0010
115 * L16SI xxxx xxxx 1001 ssss tttt 0010
116 * XXX 1010 0010
117 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported
118 * XXX 1100 0010
119 * XXX 1101 0010
120 * XXX 1110 0010
121 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported
122 * -----------------------------
123 * ^ ^ ^
124 * sub-opcode (NIBBLE_R) -+ | |
125 * t field (NIBBLE_T) -----------+ |
126 * major opcode (NIBBLE_OP0) --------------+
127 */
128
129#define OP0_L32I_N 0x8 /* load immediate narrow */
130#define OP0_S32I_N 0x9 /* store immediate narrow */
131#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */
132#define OP1_SI_BIT 2 /* OP1 bit number for stores */
133
134#define OP1_L32I 0x2
135#define OP1_L16UI 0x1
136#define OP1_L16SI 0x9
137#define OP1_L32AI 0xb
138
139#define OP1_S32I 0x6
140#define OP1_S16I 0x5
141#define OP1_S32RI 0xf
142
143/*
144 * Entry condition:
145 *
146 * a0: trashed, original value saved on stack (PT_AREG0)
147 * a1: a1
148 * a2: new stack pointer, original in DEPC
149 * a3: a3
150 * depc: a2, original value saved on stack (PT_DEPC)
151 * excsave_1: dispatch table
152 *
153 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
154 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
155 */
156
157 .literal_position
158ENTRY(fast_unaligned)
159
160 /* Note: We don't expect the address to be aligned on a word
161 * boundary. After all, the processor generated that exception
162 * and it would be a hardware fault.
163 */
164
165 /* Save some working register */
166
167 s32i a4, a2, PT_AREG4
168 s32i a5, a2, PT_AREG5
169 s32i a6, a2, PT_AREG6
170 s32i a7, a2, PT_AREG7
171 s32i a8, a2, PT_AREG8
172
173 rsr a0, depc
174 s32i a0, a2, PT_AREG2
175 s32i a3, a2, PT_AREG3
176
177 rsr a3, excsave1
178 movi a4, fast_unaligned_fixup
179 s32i a4, a3, EXC_TABLE_FIXUP
180
181 /* Keep value of SAR in a0 */
182
183 rsr a0, sar
184 rsr a8, excvaddr # load unaligned memory address
185
186 /* Now, identify one of the following load/store instructions.
187 *
188 * The only possible danger of a double exception on the
189 * following l32i instructions is kernel code in vmalloc
190 * memory. The processor was just executing at the EPC_1
191 * address, and indeed, already fetched the instruction. That
192 * guarantees a TLB mapping, which hasn't been replaced by
193 * this unaligned exception handler that uses only static TLB
194 * mappings. However, high-level interrupt handlers might
195 * modify TLB entries, so for the generic case, we register a
196 * TABLE_FIXUP handler here, too.
197 */
198
199 /* a3...a6 saved on stack, a2 = SP */
200
201 /* Extract the instruction that caused the unaligned access. */
202
203 rsr a7, epc1 # load exception address
204 movi a3, ~3
205 and a3, a3, a7 # mask lower bits
206
207 l32i a4, a3, 0 # load 2 words
208 l32i a5, a3, 4
209
210 __ssa8 a7
211 __src_b a4, a4, a5 # a4 has the instruction
212
213 /* Analyze the instruction (load or store?). */
214
215 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble
216
217#if XCHAL_HAVE_DENSITY
218 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump
219 addi a6, a5, -OP0_S32I_N
220 _beqz a6, .Lstore # S32I.N, do a store
221#endif
222 /* 'store indicator bit' not set, jump */
223 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload
224
225 /* Store: Jump to table entry to get the value in the source register.*/
226
227.Lstore:movi a5, .Lstore_table # table
228 extui a6, a4, INSN_T, 4 # get source register
229 addx8 a5, a6, a5
230 jx a5 # jump into table
231
232 /* Load: Load memory address. */
233
234.Lload: movi a3, ~3
235 and a3, a3, a8 # align memory address
236
237 __ssa8 a8
238#ifdef UNALIGNED_USER_EXCEPTION
239 addi a3, a3, 8
240 l32e a5, a3, -8
241 l32e a6, a3, -4
242#else
243 l32i a5, a3, 0
244 l32i a6, a3, 4
245#endif
246 __src_b a3, a5, a6 # a3 has the data word
247
248#if XCHAL_HAVE_DENSITY
249 addi a7, a7, 2 # increment PC (assume 16-bit insn)
250
251 extui a5, a4, INSN_OP0, 4
252 _beqi a5, OP0_L32I_N, 1f # l32i.n: jump
253
254 addi a7, a7, 1
255#else
256 addi a7, a7, 3
257#endif
258
259 extui a5, a4, INSN_OP1, 4
260 _beqi a5, OP1_L32I, 1f # l32i: jump
261
262 extui a3, a3, 0, 16 # extract lower 16 bits
263 _beqi a5, OP1_L16UI, 1f
264 addi a5, a5, -OP1_L16SI
265 _bnez a5, .Linvalid_instruction_load
266
267 /* sign extend value */
268
269 slli a3, a3, 16
270 srai a3, a3, 16
271
272 /* Set target register. */
273
2741:
275 extui a4, a4, INSN_T, 4 # extract target register
276 movi a5, .Lload_table
277 addx8 a4, a4, a5
278 jx a4 # jump to entry for target register
279
280 .align 8
281.Lload_table:
282 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8
283 mov a1, a3; _j .Lexit; .align 8 # fishy??
284 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8
285 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8
286 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8
287 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8
288 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8
289 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8
290 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8
291 mov a9, a3 ; _j .Lexit; .align 8
292 mov a10, a3 ; _j .Lexit; .align 8
293 mov a11, a3 ; _j .Lexit; .align 8
294 mov a12, a3 ; _j .Lexit; .align 8
295 mov a13, a3 ; _j .Lexit; .align 8
296 mov a14, a3 ; _j .Lexit; .align 8
297 mov a15, a3 ; _j .Lexit; .align 8
298
299.Lstore_table:
300 l32i a3, a2, PT_AREG0; _j 1f; .align 8
301 mov a3, a1; _j 1f; .align 8 # fishy??
302 l32i a3, a2, PT_AREG2; _j 1f; .align 8
303 l32i a3, a2, PT_AREG3; _j 1f; .align 8
304 l32i a3, a2, PT_AREG4; _j 1f; .align 8
305 l32i a3, a2, PT_AREG5; _j 1f; .align 8
306 l32i a3, a2, PT_AREG6; _j 1f; .align 8
307 l32i a3, a2, PT_AREG7; _j 1f; .align 8
308 l32i a3, a2, PT_AREG8; _j 1f; .align 8
309 mov a3, a9 ; _j 1f; .align 8
310 mov a3, a10 ; _j 1f; .align 8
311 mov a3, a11 ; _j 1f; .align 8
312 mov a3, a12 ; _j 1f; .align 8
313 mov a3, a13 ; _j 1f; .align 8
314 mov a3, a14 ; _j 1f; .align 8
315 mov a3, a15 ; _j 1f; .align 8
316
317 /* We cannot handle this exception. */
318
319 .extern _kernel_exception
320.Linvalid_instruction_load:
321.Linvalid_instruction_store:
322
323 movi a4, 0
324 rsr a3, excsave1
325 s32i a4, a3, EXC_TABLE_FIXUP
326
327 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */
328
329 l32i a8, a2, PT_AREG8
330 l32i a7, a2, PT_AREG7
331 l32i a6, a2, PT_AREG6
332 l32i a5, a2, PT_AREG5
333 l32i a4, a2, PT_AREG4
334 wsr a0, sar
335 mov a1, a2
336
337 rsr a0, ps
338 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode
339
340 movi a0, _kernel_exception
341 jx a0
342
3432: movi a0, _user_exception
344 jx a0
345
3461: # a7: instruction pointer, a4: instruction, a3: value
347
348 movi a6, 0 # mask: ffffffff:00000000
349
350#if XCHAL_HAVE_DENSITY
351 addi a7, a7, 2 # incr. PC,assume 16-bit instruction
352
353 extui a5, a4, INSN_OP0, 4 # extract OP0
354 addi a5, a5, -OP0_S32I_N
355 _beqz a5, 1f # s32i.n: jump
356
357 addi a7, a7, 1 # increment PC, 32-bit instruction
358#else
359 addi a7, a7, 3 # increment PC, 32-bit instruction
360#endif
361
362 extui a5, a4, INSN_OP1, 4 # extract OP1
363 _beqi a5, OP1_S32I, 1f # jump if 32 bit store
364 _bnei a5, OP1_S16I, .Linvalid_instruction_store
365
366 movi a5, -1
367 __extl a3, a3 # get 16-bit value
368 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000
369
370 /* Get memory address */
371
3721:
373 movi a4, ~3
374 and a4, a4, a8 # align memory address
375
376 /* Insert value into memory */
377
378 movi a5, -1 # mask: ffffffff:XXXX0000
379#ifdef UNALIGNED_USER_EXCEPTION
380 addi a4, a4, 8
381#endif
382
383 __ssa8r a8
384 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE)
385 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE)
386#ifdef UNALIGNED_USER_EXCEPTION
387 l32e a5, a4, -8
388#else
389 l32i a5, a4, 0 # load lower address word
390#endif
391 and a5, a5, a8 # mask
392 __sh a8, a3 # shift value
393 or a5, a5, a8 # or with original value
394#ifdef UNALIGNED_USER_EXCEPTION
395 s32e a5, a4, -8
396 l32e a8, a4, -4
397#else
398 s32i a5, a4, 0 # store
399 l32i a8, a4, 4 # same for upper address word
400#endif
401 __sl a5, a3
402 and a6, a8, a6
403 or a6, a6, a5
404#ifdef UNALIGNED_USER_EXCEPTION
405 s32e a6, a4, -4
406#else
407 s32i a6, a4, 4
408#endif
409
410.Lexit:
411#if XCHAL_HAVE_LOOPS
412 rsr a4, lend # check if we reached LEND
413 bne a7, a4, 1f
414 rsr a4, lcount # and LCOUNT != 0
415 beqz a4, 1f
416 addi a4, a4, -1 # decrement LCOUNT and set
417 rsr a7, lbeg # set PC to LBEGIN
418 wsr a4, lcount
419#endif
420
4211: wsr a7, epc1 # skip emulated instruction
422
423 /* Update icount if we're single-stepping in userspace. */
424 rsr a4, icountlevel
425 beqz a4, 1f
426 bgeui a4, LOCKLEVEL + 1, 1f
427 rsr a4, icount
428 addi a4, a4, 1
429 wsr a4, icount
4301:
431 movi a4, 0
432 rsr a3, excsave1
433 s32i a4, a3, EXC_TABLE_FIXUP
434
435 /* Restore working register */
436
437 l32i a8, a2, PT_AREG8
438 l32i a7, a2, PT_AREG7
439 l32i a6, a2, PT_AREG6
440 l32i a5, a2, PT_AREG5
441 l32i a4, a2, PT_AREG4
442 l32i a3, a2, PT_AREG3
443
444 /* restore SAR and return */
445
446 wsr a0, sar
447 l32i a0, a2, PT_AREG0
448 l32i a2, a2, PT_AREG2
449 rfe
450
451ENDPROC(fast_unaligned)
452
453ENTRY(fast_unaligned_fixup)
454
455 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
456 wsr a3, excsave1
457
458 l32i a8, a2, PT_AREG8
459 l32i a7, a2, PT_AREG7
460 l32i a6, a2, PT_AREG6
461 l32i a5, a2, PT_AREG5
462 l32i a4, a2, PT_AREG4
463 l32i a0, a2, PT_AREG2
464 xsr a0, depc # restore depc and a0
465 wsr a0, sar
466
467 rsr a0, exccause
468 s32i a0, a2, PT_DEPC # mark as a regular exception
469
470 rsr a0, ps
471 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode
472
473 rsr a0, exccause
474 addx4 a0, a0, a3 # find entry in table
475 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler
476 l32i a3, a2, PT_AREG3
477 jx a0
4781:
479 rsr a0, exccause
480 addx4 a0, a0, a3 # find entry in table
481 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
482 l32i a3, a2, PT_AREG3
483 jx a0
484
485ENDPROC(fast_unaligned_fixup)
486
487#endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */
1/*
2 * arch/xtensa/kernel/align.S
3 *
4 * Handle unalignment and load/store exceptions.
5 *
6 * This file is subject to the terms and conditions of the GNU General
7 * Public License. See the file "COPYING" in the main directory of
8 * this archive for more details.
9 *
10 * Copyright (C) 2001 - 2005 Tensilica, Inc.
11 * Copyright (C) 2014 Cadence Design Systems Inc.
12 *
13 * Rewritten by Chris Zankel <chris@zankel.net>
14 *
15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
17 */
18
19#include <linux/linkage.h>
20#include <asm/current.h>
21#include <asm/asm-offsets.h>
22#include <asm/asmmacro.h>
23#include <asm/processor.h>
24
25#if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
26#define LOAD_EXCEPTION_HANDLER
27#endif
28
29#if XCHAL_UNALIGNED_STORE_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
30#define STORE_EXCEPTION_HANDLER
31#endif
32
33#if defined LOAD_EXCEPTION_HANDLER || defined STORE_EXCEPTION_HANDLER
34#define ANY_EXCEPTION_HANDLER
35#endif
36
37#if XCHAL_HAVE_WINDOWED && defined CONFIG_MMU
38#define UNALIGNED_USER_EXCEPTION
39#endif
40
41/* Big and little endian 16-bit values are located in
42 * different halves of a register. HWORD_START helps to
43 * abstract the notion of extracting a 16-bit value from a
44 * register.
45 * We also have to define new shifting instructions because
46 * lsb and msb are on 'opposite' ends in a register for
47 * different endian machines.
48 *
49 * Assume a memory region in ascending address:
50 * 0 1 2 3|4 5 6 7
51 *
52 * When loading one word into a register, the content of that register is:
53 * LE 3 2 1 0, 7 6 5 4
54 * BE 0 1 2 3, 4 5 6 7
55 *
56 * Masking the bits of the higher/lower address means:
57 * LE X X 0 0, 0 0 X X
58 * BE 0 0 X X, X X 0 0
59 *
60 * Shifting to higher/lower addresses, means:
61 * LE shift left / shift right
62 * BE shift right / shift left
63 *
64 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
65 * LE mask 0 0 X X / shift left
66 * BE shift left / mask 0 0 X X
67 */
68
69#if XCHAL_HAVE_BE
70
71#define HWORD_START 16
72#define INSN_OP0 28
73#define INSN_T 24
74#define INSN_OP1 16
75
76.macro __ssa8r r; ssa8l \r; .endm
77.macro __sh r, s; srl \r, \s; .endm
78.macro __sl r, s; sll \r, \s; .endm
79.macro __exth r, s; extui \r, \s, 0, 16; .endm
80.macro __extl r, s; slli \r, \s, 16; .endm
81
82#else
83
84#define HWORD_START 0
85#define INSN_OP0 0
86#define INSN_T 4
87#define INSN_OP1 12
88
89.macro __ssa8r r; ssa8b \r; .endm
90.macro __sh r, s; sll \r, \s; .endm
91.macro __sl r, s; srl \r, \s; .endm
92.macro __exth r, s; slli \r, \s, 16; .endm
93.macro __extl r, s; extui \r, \s, 0, 16; .endm
94
95#endif
96
97/*
98 * xxxx xxxx = imm8 field
99 * yyyy = imm4 field
100 * ssss = s field
101 * tttt = t field
102 *
103 * 16 0
104 * -------------------
105 * L32I.N yyyy ssss tttt 1000
106 * S32I.N yyyy ssss tttt 1001
107 *
108 * 23 0
109 * -----------------------------
110 * L8UI xxxx xxxx 0000 ssss tttt 0010
111 * L16UI xxxx xxxx 0001 ssss tttt 0010
112 * L32I xxxx xxxx 0010 ssss tttt 0010
113 * XXX 0011 ssss tttt 0010
114 * XXX 0100 ssss tttt 0010
115 * S16I xxxx xxxx 0101 ssss tttt 0010
116 * S32I xxxx xxxx 0110 ssss tttt 0010
117 * XXX 0111 ssss tttt 0010
118 * XXX 1000 ssss tttt 0010
119 * L16SI xxxx xxxx 1001 ssss tttt 0010
120 * XXX 1010 0010
121 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported
122 * XXX 1100 0010
123 * XXX 1101 0010
124 * XXX 1110 0010
125 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported
126 * -----------------------------
127 * ^ ^ ^
128 * sub-opcode (NIBBLE_R) -+ | |
129 * t field (NIBBLE_T) -----------+ |
130 * major opcode (NIBBLE_OP0) --------------+
131 */
132
133#define OP0_L32I_N 0x8 /* load immediate narrow */
134#define OP0_S32I_N 0x9 /* store immediate narrow */
135#define OP0_LSAI 0x2 /* load/store */
136#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */
137#define OP1_SI_BIT 2 /* OP1 bit number for stores */
138
139#define OP1_L8UI 0x0
140#define OP1_L32I 0x2
141#define OP1_L16UI 0x1
142#define OP1_L16SI 0x9
143#define OP1_L32AI 0xb
144
145#define OP1_S32I 0x6
146#define OP1_S16I 0x5
147#define OP1_S32RI 0xf
148
149/*
150 * Entry condition:
151 *
152 * a0: trashed, original value saved on stack (PT_AREG0)
153 * a1: a1
154 * a2: new stack pointer, original in DEPC
155 * a3: a3
156 * depc: a2, original value saved on stack (PT_DEPC)
157 * excsave_1: dispatch table
158 *
159 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
160 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
161 */
162
163 .literal_position
164#ifdef CONFIG_XTENSA_LOAD_STORE
165ENTRY(fast_load_store)
166
167 call0 .Lsave_and_load_instruction
168
169 /* Analyze the instruction (load or store?). */
170
171 extui a0, a4, INSN_OP0, 4 # get insn.op0 nibble
172
173#if XCHAL_HAVE_DENSITY
174 _beqi a0, OP0_L32I_N, 1f # L32I.N, jump
175#endif
176 bnei a0, OP0_LSAI, .Linvalid_instruction
177 /* 'store indicator bit' set, jump */
178 bbsi.l a4, OP1_SI_BIT + INSN_OP1, .Linvalid_instruction
179
1801:
181 movi a3, ~3
182 and a3, a3, a8 # align memory address
183
184 __ssa8 a8
185
186#ifdef CONFIG_MMU
187 /* l32e can't be used here even when it's available. */
188 /* TODO access_ok(a3) could be used here */
189 j .Linvalid_instruction
190#endif
191 l32i a5, a3, 0
192 l32i a6, a3, 4
193 __src_b a3, a5, a6 # a3 has the data word
194
195#if XCHAL_HAVE_DENSITY
196 addi a7, a7, 2 # increment PC (assume 16-bit insn)
197 _beqi a0, OP0_L32I_N, .Lload_w# l32i.n: jump
198 addi a7, a7, 1
199#else
200 addi a7, a7, 3
201#endif
202
203 extui a5, a4, INSN_OP1, 4
204 _beqi a5, OP1_L32I, .Lload_w
205 bnei a5, OP1_L8UI, .Lload16
206 extui a3, a3, 0, 8
207 j .Lload_w
208
209ENDPROC(fast_load_store)
210#endif
211
212/*
213 * Entry condition:
214 *
215 * a0: trashed, original value saved on stack (PT_AREG0)
216 * a1: a1
217 * a2: new stack pointer, original in DEPC
218 * a3: a3
219 * depc: a2, original value saved on stack (PT_DEPC)
220 * excsave_1: dispatch table
221 *
222 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
223 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
224 */
225
226#ifdef ANY_EXCEPTION_HANDLER
227ENTRY(fast_unaligned)
228
229 call0 .Lsave_and_load_instruction
230
231 /* Analyze the instruction (load or store?). */
232
233 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble
234
235#if XCHAL_HAVE_DENSITY
236 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump
237 addi a6, a5, -OP0_S32I_N
238 _beqz a6, .Lstore # S32I.N, do a store
239#endif
240 /* 'store indicator bit' not set, jump */
241 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload
242
243#ifdef STORE_EXCEPTION_HANDLER
244
245 /* Store: Jump to table entry to get the value in the source register.*/
246
247.Lstore:movi a5, .Lstore_table # table
248 extui a6, a4, INSN_T, 4 # get source register
249 addx8 a5, a6, a5
250 jx a5 # jump into table
251#endif
252#ifdef LOAD_EXCEPTION_HANDLER
253
254 /* Load: Load memory address. */
255
256.Lload: movi a3, ~3
257 and a3, a3, a8 # align memory address
258
259 __ssa8 a8
260#ifdef UNALIGNED_USER_EXCEPTION
261 addi a3, a3, 8
262 l32e a5, a3, -8
263 l32e a6, a3, -4
264#else
265 l32i a5, a3, 0
266 l32i a6, a3, 4
267#endif
268 __src_b a3, a5, a6 # a3 has the data word
269
270#if XCHAL_HAVE_DENSITY
271 addi a7, a7, 2 # increment PC (assume 16-bit insn)
272
273 extui a5, a4, INSN_OP0, 4
274 _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump
275
276 addi a7, a7, 1
277#else
278 addi a7, a7, 3
279#endif
280
281 extui a5, a4, INSN_OP1, 4
282 _beqi a5, OP1_L32I, .Lload_w # l32i: jump
283#endif
284#ifdef LOAD_EXCEPTION_HANDLER
285.Lload16:
286 extui a3, a3, 0, 16 # extract lower 16 bits
287 _beqi a5, OP1_L16UI, .Lload_w
288 addi a5, a5, -OP1_L16SI
289 _bnez a5, .Linvalid_instruction
290
291 /* sign extend value */
292#if XCHAL_HAVE_SEXT
293 sext a3, a3, 15
294#else
295 slli a3, a3, 16
296 srai a3, a3, 16
297#endif
298
299 /* Set target register. */
300
301.Lload_w:
302 extui a4, a4, INSN_T, 4 # extract target register
303 movi a5, .Lload_table
304 addx8 a4, a4, a5
305 jx a4 # jump to entry for target register
306
307 .align 8
308.Lload_table:
309 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8
310 mov a1, a3; _j .Lexit; .align 8 # fishy??
311 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8
312 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8
313 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8
314 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8
315 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8
316 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8
317 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8
318 mov a9, a3 ; _j .Lexit; .align 8
319 mov a10, a3 ; _j .Lexit; .align 8
320 mov a11, a3 ; _j .Lexit; .align 8
321 mov a12, a3 ; _j .Lexit; .align 8
322 mov a13, a3 ; _j .Lexit; .align 8
323 mov a14, a3 ; _j .Lexit; .align 8
324 mov a15, a3 ; _j .Lexit; .align 8
325#endif
326#ifdef STORE_EXCEPTION_HANDLER
327.Lstore_table:
328 l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8
329 mov a3, a1; _j .Lstore_w; .align 8 # fishy??
330 l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8
331 l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8
332 l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8
333 l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8
334 l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8
335 l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8
336 l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8
337 mov a3, a9 ; _j .Lstore_w; .align 8
338 mov a3, a10 ; _j .Lstore_w; .align 8
339 mov a3, a11 ; _j .Lstore_w; .align 8
340 mov a3, a12 ; _j .Lstore_w; .align 8
341 mov a3, a13 ; _j .Lstore_w; .align 8
342 mov a3, a14 ; _j .Lstore_w; .align 8
343 mov a3, a15 ; _j .Lstore_w; .align 8
344#endif
345
346 /* We cannot handle this exception. */
347
348 .extern _kernel_exception
349.Linvalid_instruction:
350
351 movi a4, 0
352 rsr a3, excsave1
353 s32i a4, a3, EXC_TABLE_FIXUP
354
355 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */
356
357 l32i a0, a2, PT_SAR
358 l32i a8, a2, PT_AREG8
359 l32i a7, a2, PT_AREG7
360 l32i a6, a2, PT_AREG6
361 l32i a5, a2, PT_AREG5
362 l32i a4, a2, PT_AREG4
363 wsr a0, sar
364 mov a1, a2
365
366 rsr a0, ps
367 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode
368
369 movi a0, _kernel_exception
370 jx a0
371
3722: movi a0, _user_exception
373 jx a0
374
375#ifdef STORE_EXCEPTION_HANDLER
376
377 # a7: instruction pointer, a4: instruction, a3: value
378.Lstore_w:
379 movi a6, 0 # mask: ffffffff:00000000
380
381#if XCHAL_HAVE_DENSITY
382 addi a7, a7, 2 # incr. PC,assume 16-bit instruction
383
384 extui a5, a4, INSN_OP0, 4 # extract OP0
385 addi a5, a5, -OP0_S32I_N
386 _beqz a5, 1f # s32i.n: jump
387
388 addi a7, a7, 1 # increment PC, 32-bit instruction
389#else
390 addi a7, a7, 3 # increment PC, 32-bit instruction
391#endif
392
393 extui a5, a4, INSN_OP1, 4 # extract OP1
394 _beqi a5, OP1_S32I, 1f # jump if 32 bit store
395 _bnei a5, OP1_S16I, .Linvalid_instruction
396
397 movi a5, -1
398 __extl a3, a3 # get 16-bit value
399 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000
400
401 /* Get memory address */
402
4031:
404 movi a4, ~3
405 and a4, a4, a8 # align memory address
406
407 /* Insert value into memory */
408
409 movi a5, -1 # mask: ffffffff:XXXX0000
410#ifdef UNALIGNED_USER_EXCEPTION
411 addi a4, a4, 8
412#endif
413
414 __ssa8r a8
415 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE)
416 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE)
417#ifdef UNALIGNED_USER_EXCEPTION
418 l32e a5, a4, -8
419#else
420 l32i a5, a4, 0 # load lower address word
421#endif
422 and a5, a5, a8 # mask
423 __sh a8, a3 # shift value
424 or a5, a5, a8 # or with original value
425#ifdef UNALIGNED_USER_EXCEPTION
426 s32e a5, a4, -8
427 l32e a8, a4, -4
428#else
429 s32i a5, a4, 0 # store
430 l32i a8, a4, 4 # same for upper address word
431#endif
432 __sl a5, a3
433 and a6, a8, a6
434 or a6, a6, a5
435#ifdef UNALIGNED_USER_EXCEPTION
436 s32e a6, a4, -4
437#else
438 s32i a6, a4, 4
439#endif
440#endif
441
442.Lexit:
443#if XCHAL_HAVE_LOOPS
444 rsr a4, lend # check if we reached LEND
445 bne a7, a4, 1f
446 rsr a4, lcount # and LCOUNT != 0
447 beqz a4, 1f
448 addi a4, a4, -1 # decrement LCOUNT and set
449 rsr a7, lbeg # set PC to LBEGIN
450 wsr a4, lcount
451#endif
452
4531: wsr a7, epc1 # skip emulated instruction
454
455 /* Update icount if we're single-stepping in userspace. */
456 rsr a4, icountlevel
457 beqz a4, 1f
458 bgeui a4, LOCKLEVEL + 1, 1f
459 rsr a4, icount
460 addi a4, a4, 1
461 wsr a4, icount
4621:
463 movi a4, 0
464 rsr a3, excsave1
465 s32i a4, a3, EXC_TABLE_FIXUP
466
467 /* Restore working register */
468
469 l32i a0, a2, PT_SAR
470 l32i a8, a2, PT_AREG8
471 l32i a7, a2, PT_AREG7
472 l32i a6, a2, PT_AREG6
473 l32i a5, a2, PT_AREG5
474 l32i a4, a2, PT_AREG4
475 l32i a3, a2, PT_AREG3
476
477 /* restore SAR and return */
478
479 wsr a0, sar
480 l32i a0, a2, PT_AREG0
481 l32i a2, a2, PT_AREG2
482 rfe
483
484 .align 4
485.Lsave_and_load_instruction:
486
487 /* Save some working register */
488
489 s32i a3, a2, PT_AREG3
490 s32i a4, a2, PT_AREG4
491 s32i a5, a2, PT_AREG5
492 s32i a6, a2, PT_AREG6
493 s32i a7, a2, PT_AREG7
494 s32i a8, a2, PT_AREG8
495
496 rsr a4, depc
497 s32i a4, a2, PT_AREG2
498
499 rsr a5, sar
500 s32i a5, a2, PT_SAR
501
502 rsr a3, excsave1
503 movi a4, fast_unaligned_fixup
504 s32i a4, a3, EXC_TABLE_FIXUP
505
506 rsr a8, excvaddr # load unaligned memory address
507
508 /* Now, identify one of the following load/store instructions.
509 *
510 * The only possible danger of a double exception on the
511 * following l32i instructions is kernel code in vmalloc
512 * memory. The processor was just executing at the EPC_1
513 * address, and indeed, already fetched the instruction. That
514 * guarantees a TLB mapping, which hasn't been replaced by
515 * this unaligned exception handler that uses only static TLB
516 * mappings. However, high-level interrupt handlers might
517 * modify TLB entries, so for the generic case, we register a
518 * TABLE_FIXUP handler here, too.
519 */
520
521 /* a3...a6 saved on stack, a2 = SP */
522
523 /* Extract the instruction that caused the unaligned access. */
524
525 rsr a7, epc1 # load exception address
526 movi a3, ~3
527 and a3, a3, a7 # mask lower bits
528
529 l32i a4, a3, 0 # load 2 words
530 l32i a5, a3, 4
531
532 __ssa8 a7
533 __src_b a4, a4, a5 # a4 has the instruction
534
535 ret
536
537ENDPROC(fast_unaligned)
538
539ENTRY(fast_unaligned_fixup)
540
541 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
542 wsr a3, excsave1
543
544 l32i a8, a2, PT_AREG8
545 l32i a7, a2, PT_AREG7
546 l32i a6, a2, PT_AREG6
547 l32i a5, a2, PT_AREG5
548 l32i a4, a2, PT_SAR
549 l32i a0, a2, PT_AREG2
550 wsr a4, sar
551 wsr a0, depc # restore depc and a0
552 l32i a4, a2, PT_AREG4
553
554 rsr a0, exccause
555 s32i a0, a2, PT_DEPC # mark as a regular exception
556
557 rsr a0, ps
558 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode
559
560 rsr a0, exccause
561 addx4 a0, a0, a3 # find entry in table
562 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler
563 l32i a3, a2, PT_AREG3
564 jx a0
5651:
566 rsr a0, exccause
567 addx4 a0, a0, a3 # find entry in table
568 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
569 l32i a3, a2, PT_AREG3
570 jx a0
571
572ENDPROC(fast_unaligned_fixup)
573#endif