1// SPDX-License-Identifier: GPL-2.0-only
  2// Copyright (C) 2015-2019 ARM Limited.
  3// Original author: Dave Martin <Dave.Martin@arm.com>
  4//
  5// Simple Scalable Vector Extension context switch test
  6// Repeatedly writes unique test patterns into each SVE register
  7// and reads them back to verify integrity.
  8//
  9// for x in `seq 1 NR_CPUS`; do sve-test & pids=$pids\ $! ; done
 10// (leave it running for as long as you want...)
 11// kill $pids
 12
 13#include <asm/unistd.h>
 14#include "assembler.h"
 15#include "asm-offsets.h"
 16#include "sme-inst.h"
 17
 18#define NZR	32
 19#define NPR	16
 20#define MAXVL_B	(2048 / 8)
 21
 22.arch_extension sve
 23
 24.macro _sve_ldr_v zt, xn
 25	ldr	z\zt, [x\xn]
 26.endm
 27
 28.macro _sve_str_v zt, xn
 29	str	z\zt, [x\xn]
 30.endm
 31
 32.macro _sve_ldr_p pt, xn
 33	ldr	p\pt, [x\xn]
 34.endm
 35
 36.macro _sve_str_p pt, xn
 37	str	p\pt, [x\xn]
 38.endm
 39
 40// Generate accessor functions to read/write programmatically selected
 41// SVE registers.
 42// x0 is the register index to access
 43// x1 is the memory address to read from (getz,setp) or store to (setz,setp)
 44// All clobber x0-x2
 45define_accessor setz, NZR, _sve_ldr_v
 46define_accessor getz, NZR, _sve_str_v
 47define_accessor setp, NPR, _sve_ldr_p
 48define_accessor getp, NPR, _sve_str_p
 49
 50// Declare some storate space to shadow the SVE register contents:
 51.pushsection .text
 52.data
 53.align 4
 54zref:
 55	.space	MAXVL_B * NZR
 56pref:
 57	.space	MAXVL_B / 8 * NPR
 58ffrref:
 59	.space	MAXVL_B / 8
 60scratch:
 61	.space	MAXVL_B
 62.popsection
 63
 64// Generate a test pattern for storage in SVE registers
 65// x0: pid	(16 bits)
 66// x1: register number (6 bits)
 67// x2: generation (4 bits)
 68
 69// These values are used to constuct a 32-bit pattern that is repeated in the
 70// scratch buffer as many times as will fit:
 71// bits 31:28	generation number (increments once per test_loop)
 72// bits 27:22	32-bit lane index
 73// bits 21:16	register number
 74// bits 15: 0	pid
 75
 76function pattern
 77	orr	w1, w0, w1, lsl #16
 78	orr	w2, w1, w2, lsl #28
 79
 80	ldr	x0, =scratch
 81	mov	w1, #MAXVL_B / 4
 82
 830:	str	w2, [x0], #4
 84	add	w2, w2, #(1 << 22)
 85	subs	w1, w1, #1
 86	bne	0b
 87
 88	ret
 89endfunction
 90
 91// Get the address of shadow data for SVE Z-register Z<xn>
 92.macro _adrz xd, xn, nrtmp
 93	ldr	\xd, =zref
 94	rdvl	x\nrtmp, #1
 95	madd	\xd, x\nrtmp, \xn, \xd
 96.endm
 97
 98// Get the address of shadow data for SVE P-register P<xn - NZR>
 99.macro _adrp xd, xn, nrtmp
100	ldr	\xd, =pref
101	rdvl	x\nrtmp, #1
102	lsr	x\nrtmp, x\nrtmp, #3
103	sub	\xn, \xn, #NZR
104	madd	\xd, x\nrtmp, \xn, \xd
105.endm
106
107// Set up test pattern in a SVE Z-register
108// x0: pid
109// x1: register number
110// x2: generation
111function setup_zreg
112	mov	x4, x30
113
114	mov	x6, x1
115	bl	pattern
116	_adrz	x0, x6, 2
117	mov	x5, x0
118	ldr	x1, =scratch
119	bl	memcpy
120
121	mov	x0, x6
122	mov	x1, x5
123	bl	setz
124
125	ret	x4
126endfunction
127
128// Set up test pattern in a SVE P-register
129// x0: pid
130// x1: register number
131// x2: generation
132function setup_preg
133	mov	x4, x30
134
135	mov	x6, x1
136	bl	pattern
137	_adrp	x0, x6, 2
138	mov	x5, x0
139	ldr	x1, =scratch
140	bl	memcpy
141
142	mov	x0, x6
143	mov	x1, x5
144	bl	setp
145
146	ret	x4
147endfunction
148
149// Set up test pattern in the FFR
150// x0: pid
151// x2: generation
152//
153// We need to generate a canonical FFR value, which consists of a number of
154// low "1" bits, followed by a number of zeros. This gives us 17 unique values
155// per 16 bits of FFR, so we create a 4 bit signature out of the PID and
156// generation, and use that as the initial number of ones in the pattern.
157// We fill the upper lanes of FFR with zeros.
158// Beware: corrupts P0.
159function setup_ffr
160#ifndef SSVE
161	mov	x4, x30
162
163	and	w0, w0, #0x3
164	bfi	w0, w2, #2, #2
165	mov	w1, #1
166	lsl	w1, w1, w0
167	sub	w1, w1, #1
168
169	ldr	x0, =ffrref
170	strh	w1, [x0], 2
171	rdvl	x1, #1
172	lsr	x1, x1, #3
173	sub	x1, x1, #2
174	bl	memclr
175
176	mov	x0, #0
177	ldr	x1, =ffrref
178	bl	setp
179
180	wrffr	p0.b
181
182	ret	x4
183#else
184	ret
185#endif
186endfunction
187
188// Trivial memory compare: compare x2 bytes starting at address x0 with
189// bytes starting at address x1.
190// Returns only if all bytes match; otherwise, the program is aborted.
191// Clobbers x0-x5.
192function memcmp
193	cbz	x2, 2f
194
195	stp	x0, x1, [sp, #-0x20]!
196	str	x2, [sp, #0x10]
197
198	mov	x5, #0
1990:	ldrb	w3, [x0, x5]
200	ldrb	w4, [x1, x5]
201	add	x5, x5, #1
202	cmp	w3, w4
203	b.ne	1f
204	subs	x2, x2, #1
205	b.ne	0b
206
2071:	ldr	x2, [sp, #0x10]
208	ldp	x0, x1, [sp], #0x20
209	b.ne	barf
210
2112:	ret
212endfunction
213
214// Verify that a SVE Z-register matches its shadow in memory, else abort
215// x0: reg number
216// Clobbers x0-x7.
217function check_zreg
218	mov	x3, x30
219
220	_adrz	x5, x0, 6
221	mov	x4, x0
222	ldr	x7, =scratch
223
224	mov	x0, x7
225	mov	x1, x6
226	bl	memfill_ae
227
228	mov	x0, x4
229	mov	x1, x7
230	bl	getz
231
232	mov	x0, x5
233	mov	x1, x7
234	mov	x2, x6
235	mov	x30, x3
236	b	memcmp
237endfunction
238
239// Verify that a SVE P-register matches its shadow in memory, else abort
240// x0: reg number
241// Clobbers x0-x7.
242function check_preg
243	mov	x3, x30
244
245	_adrp	x5, x0, 6
246	mov	x4, x0
247	ldr	x7, =scratch
248
249	mov	x0, x7
250	mov	x1, x6
251	bl	memfill_ae
252
253	mov	x0, x4
254	mov	x1, x7
255	bl	getp
256
257	mov	x0, x5
258	mov	x1, x7
259	mov	x2, x6
260	mov	x30, x3
261	b	memcmp
262endfunction
263
264// Verify that the FFR matches its shadow in memory, else abort
265// Beware -- corrupts P0.
266// Clobbers x0-x5.
267function check_ffr
268#ifndef SSVE
269	mov	x3, x30
270
271	ldr	x4, =scratch
272	rdvl	x5, #1
273	lsr	x5, x5, #3
274
275	mov	x0, x4
276	mov	x1, x5
277	bl	memfill_ae
278
279	rdffr	p0.b
280	mov	x0, #0
281	mov	x1, x4
282	bl	getp
283
284	ldr	x0, =ffrref
285	mov	x1, x4
286	mov	x2, x5
287	mov	x30, x3
288	b	memcmp
289#else
290	ret
291#endif
292endfunction
293
294// Any SVE register modified here can cause corruption in the main
295// thread -- but *only* the registers modified here.
296function irritator_handler
297	// Increment the irritation signal count (x23):
298	ldr	x0, [x2, #ucontext_regs + 8 * 23]
299	add	x0, x0, #1
300	str	x0, [x2, #ucontext_regs + 8 * 23]
301
302	// Corrupt some random Z-regs
303	adr	x0, .text + (irritator_handler - .text) / 16 * 16
304	movi	v0.8b, #1
305	movi	v9.16b, #2
306	movi	v31.8b, #3
307#ifndef SSVE
308	// And P0
309	rdffr	p0.b
310	// And FFR
311	wrffr	p15.b
312#endif
313
314	ret
315endfunction
316
317function tickle_handler
318	// Increment the signal count (x23):
319	ldr	x0, [x2, #ucontext_regs + 8 * 23]
320	add	x0, x0, #1
321	str	x0, [x2, #ucontext_regs + 8 * 23]
322
323	ret
324endfunction
325
326function terminate_handler
327	mov	w21, w0
328	mov	x20, x2
329
330	puts	"Terminated by signal "
331	mov	w0, w21
332	bl	putdec
333	puts	", no error, iterations="
334	ldr	x0, [x20, #ucontext_regs + 8 * 22]
335	bl	putdec
336	puts	", signals="
337	ldr	x0, [x20, #ucontext_regs + 8 * 23]
338	bl	putdecn
339
340	mov	x0, #0
341	mov	x8, #__NR_exit
342	svc	#0
343endfunction
344
345// w0: signal number
346// x1: sa_action
347// w2: sa_flags
348// Clobbers x0-x6,x8
349function setsignal
350	str	x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]!
351
352	mov	w4, w0
353	mov	x5, x1
354	mov	w6, w2
355
356	add	x0, sp, #16
357	mov	x1, #sa_sz
358	bl	memclr
359
360	mov	w0, w4
361	add	x1, sp, #16
362	str	w6, [x1, #sa_flags]
363	str	x5, [x1, #sa_handler]
364	mov	x2, #0
365	mov	x3, #sa_mask_sz
366	mov	x8, #__NR_rt_sigaction
367	svc	#0
368
369	cbz	w0, 1f
370
371	puts	"sigaction failure\n"
372	b	.Labort
373
3741:	ldr	x30, [sp], #((sa_sz + 15) / 16 * 16 + 16)
375	ret
376endfunction
377
378// Main program entry point
379.globl _start
380function _start
381	mov	x23, #0		// Irritation signal count
382
383	mov	w0, #SIGINT
384	adr	x1, terminate_handler
385	mov	w2, #SA_SIGINFO
386	bl	setsignal
387
388	mov	w0, #SIGTERM
389	adr	x1, terminate_handler
390	mov	w2, #SA_SIGINFO
391	bl	setsignal
392
393	mov	w0, #SIGUSR1
394	adr	x1, irritator_handler
395	mov	w2, #SA_SIGINFO
396	orr	w2, w2, #SA_NODEFER
397	bl	setsignal
398
399	mov	w0, #SIGUSR2
400	adr	x1, tickle_handler
401	mov	w2, #SA_SIGINFO
402	orr	w2, w2, #SA_NODEFER
403	bl	setsignal
404
405#ifdef SSVE
406	puts	"Streaming mode "
407	smstart_sm
408#endif
409
410	// Sanity-check and report the vector length
411
412	rdvl	x19, #8
413	cmp	x19, #128
414	b.lo	1f
415	cmp	x19, #2048
416	b.hi	1f
417	tst	x19, #(8 - 1)
418	b.eq	2f
419
4201:	puts	"Bad vector length: "
421	mov	x0, x19
422	bl	putdecn
423	b	.Labort
424
4252:	puts	"Vector length:\t"
426	mov	x0, x19
427	bl	putdec
428	puts	" bits\n"
429
430	// Obtain our PID, to ensure test pattern uniqueness between processes
431
432	mov	x8, #__NR_getpid
433	svc	#0
434	mov	x20, x0
435
436	puts	"PID:\t"
437	mov	x0, x20
438	bl	putdecn
439
440#ifdef SSVE
441	smstart_sm		// syscalls will have exited streaming mode
442#endif
443
444	mov	x22, #0		// generation number, increments per iteration
445.Ltest_loop:
446	rdvl	x0, #8
447	cmp	x0, x19
448	b.ne	vl_barf
449
450	mov	x21, #0		// Set up Z-regs & shadow with test pattern
4510:	mov	x0, x20
452	mov	x1, x21
453	and	x2, x22, #0xf
454	bl	setup_zreg
455	add	x21, x21, #1
456	cmp	x21, #NZR
457	b.lo	0b
458
459	mov	x0, x20		// Set up FFR & shadow with test pattern
460	mov	x1, #NZR + NPR
461	and	x2, x22, #0xf
462	bl	setup_ffr
463
4640:	mov	x0, x20		// Set up P-regs & shadow with test pattern
465	mov	x1, x21
466	and	x2, x22, #0xf
467	bl	setup_preg
468	add	x21, x21, #1
469	cmp	x21, #NZR + NPR
470	b.lo	0b
471
472// Can't do this when SVE state is volatile across SVC:
473//	mov	x8, #__NR_sched_yield	// Encourage preemption
474//	svc	#0
475
476#ifdef SSVE
477	mrs	x0, S3_3_C4_C2_2	// SVCR should have ZA=0,SM=1
478	and	x1, x0, #3
479	cmp	x1, #1
480	b.ne	svcr_barf
481#endif
482
483	mov	x21, #0
4840:	mov	x0, x21
485	bl	check_zreg
486	add	x21, x21, #1
487	cmp	x21, #NZR
488	b.lo	0b
489
4900:	mov	x0, x21
491	bl	check_preg
492	add	x21, x21, #1
493	cmp	x21, #NZR + NPR
494	b.lo	0b
495
496	bl	check_ffr
497
498	add	x22, x22, #1
499	b	.Ltest_loop
500
501.Labort:
502	mov	x0, #0
503	mov	x1, #SIGABRT
504	mov	x8, #__NR_kill
505	svc	#0
506endfunction
507
508function barf
509// fpsimd.c acitivty log dump hack
510//	ldr	w0, =0xdeadc0de
511//	mov	w8, #__NR_exit
512//	svc	#0
513// end hack
514	mov	x10, x0	// expected data
515	mov	x11, x1	// actual data
516	mov	x12, x2	// data size
517
518#ifdef SSVE
519	mrs	x13, S3_3_C4_C2_2
520#endif
521
522	puts	"Mismatch: PID="
523	mov	x0, x20
524	bl	putdec
525	puts	", iteration="
526	mov	x0, x22
527	bl	putdec
528	puts	", reg="
529	mov	x0, x21
530	bl	putdecn
531	puts	"\tExpected ["
532	mov	x0, x10
533	mov	x1, x12
534	bl	dumphex
535	puts	"]\n\tGot      ["
536	mov	x0, x11
537	mov	x1, x12
538	bl	dumphex
539	puts	"]\n"
540
541#ifdef SSVE
542	puts	"\tSVCR: "
543	mov	x0, x13
544	bl	putdecn
545#endif
546
547	mov	x8, #__NR_getpid
548	svc	#0
549// fpsimd.c acitivty log dump hack
550//	ldr	w0, =0xdeadc0de
551//	mov	w8, #__NR_exit
552//	svc	#0
553// ^ end of hack
554	mov	x1, #SIGABRT
555	mov	x8, #__NR_kill
556	svc	#0
557//	mov	x8, #__NR_exit
558//	mov	x1, #1
559//	svc	#0
560endfunction
561
562function vl_barf
563	mov	x10, x0
564
565	puts	"Bad active VL: "
566	mov	x0, x10
567	bl	putdecn
568
569	mov	x8, #__NR_exit
570	mov	x1, #1
571	svc	#0
572endfunction
573
574function svcr_barf
575	mov	x10, x0
576
577	puts	"Bad SVCR: "
578	mov	x0, x10
579	bl	putdecn
580
581	mov	x8, #__NR_exit
582	mov	x1, #1
583	svc	#0
584endfunction