Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/processor.h>
3#include <asm/ppc_asm.h>
4#include <asm/reg.h>
5#include <asm/asm-offsets.h>
6#include <asm/cputable.h>
7#include <asm/thread_info.h>
8#include <asm/page.h>
9#include <asm/ptrace.h>
10#include <asm/export.h>
11#include <asm/asm-compat.h>
12
13/*
14 * Load state from memory into VMX registers including VSCR.
15 * Assumes the caller has enabled VMX in the MSR.
16 */
17_GLOBAL(load_vr_state)
18 li r4,VRSTATE_VSCR
19 lvx v0,r4,r3
20 mtvscr v0
21 REST_32VRS(0,r4,r3)
22 blr
23EXPORT_SYMBOL(load_vr_state)
24_ASM_NOKPROBE_SYMBOL(load_vr_state); /* used by restore_math */
25
26/*
27 * Store VMX state into memory, including VSCR.
28 * Assumes the caller has enabled VMX in the MSR.
29 */
30_GLOBAL(store_vr_state)
31 SAVE_32VRS(0, r4, r3)
32 mfvscr v0
33 li r4, VRSTATE_VSCR
34 stvx v0, r4, r3
35 blr
36EXPORT_SYMBOL(store_vr_state)
37
38/*
39 * Disable VMX for the task which had it previously,
40 * and save its vector registers in its thread_struct.
41 * Enables the VMX for use in the kernel on return.
42 * On SMP we know the VMX is free, since we give it up every
43 * switch (ie, no lazy save of the vector registers).
44 *
45 * Note that on 32-bit this can only use registers that will be
46 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
47 */
48_GLOBAL(load_up_altivec)
49 mfmsr r5 /* grab the current MSR */
50 oris r5,r5,MSR_VEC@h
51 MTMSRD(r5) /* enable use of AltiVec now */
52 isync
53
54 /*
55 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
56 * to optimise userspace context save/restore. Whenever we take an
57 * altivec unavailable exception we must set VRSAVE to something non
58 * zero. Set it to all 1s. See also the programming note in the ISA.
59 */
60 mfspr r4,SPRN_VRSAVE
61 cmpwi 0,r4,0
62 bne+ 1f
63 li r4,-1
64 mtspr SPRN_VRSAVE,r4
651:
66 /* enable use of VMX after return */
67#ifdef CONFIG_PPC32
68 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
69 oris r9,r9,MSR_VEC@h
70#ifdef CONFIG_VMAP_STACK
71 tovirt(r5, r5)
72#endif
73#else
74 ld r4,PACACURRENT(r13)
75 addi r5,r4,THREAD /* Get THREAD */
76 oris r12,r12,MSR_VEC@h
77 std r12,_MSR(r1)
78#endif
79 li r4,1
80 stb r4,THREAD_LOAD_VEC(r5)
81 addi r6,r5,THREAD_VRSTATE
82 li r4,1
83 li r10,VRSTATE_VSCR
84 stw r4,THREAD_USED_VR(r5)
85 lvx v0,r10,r6
86 mtvscr v0
87 REST_32VRS(0,r4,r6)
88 /* restore registers and return */
89 blr
90_ASM_NOKPROBE_SYMBOL(load_up_altivec)
91
92/*
93 * save_altivec(tsk)
94 * Save the vector registers to its thread_struct
95 */
96_GLOBAL(save_altivec)
97 addi r3,r3,THREAD /* want THREAD of task */
98 PPC_LL r7,THREAD_VRSAVEAREA(r3)
99 PPC_LL r5,PT_REGS(r3)
100 PPC_LCMPI 0,r7,0
101 bne 2f
102 addi r7,r3,THREAD_VRSTATE
1032: SAVE_32VRS(0,r4,r7)
104 mfvscr v0
105 li r4,VRSTATE_VSCR
106 stvx v0,r4,r7
107 blr
108
109#ifdef CONFIG_VSX
110
111#ifdef CONFIG_PPC32
112#error This asm code isn't ready for 32-bit kernels
113#endif
114
115/*
116 * load_up_vsx(unused, unused, tsk)
117 * Disable VSX for the task which had it previously,
118 * and save its vector registers in its thread_struct.
119 * Reuse the fp and vsx saves, but first check to see if they have
120 * been saved already.
121 */
122_GLOBAL(load_up_vsx)
123/* Load FP and VSX registers if they haven't been done yet */
124 andi. r5,r12,MSR_FP
125 beql+ load_up_fpu /* skip if already loaded */
126 andis. r5,r12,MSR_VEC@h
127 beql+ load_up_altivec /* skip if already loaded */
128
129 ld r4,PACACURRENT(r13)
130 addi r4,r4,THREAD /* Get THREAD */
131 li r6,1
132 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
133 /* enable use of VSX after return */
134 oris r12,r12,MSR_VSX@h
135 std r12,_MSR(r1)
136 b fast_interrupt_return
137
138#endif /* CONFIG_VSX */
139
140
141/*
142 * The routines below are in assembler so we can closely control the
143 * usage of floating-point registers. These routines must be called
144 * with preempt disabled.
145 */
146#ifdef CONFIG_PPC32
147 .data
148fpzero:
149 .long 0
150fpone:
151 .long 0x3f800000 /* 1.0 in single-precision FP */
152fphalf:
153 .long 0x3f000000 /* 0.5 in single-precision FP */
154
155#define LDCONST(fr, name) \
156 lis r11,name@ha; \
157 lfs fr,name@l(r11)
158#else
159
160 .section ".toc","aw"
161fpzero:
162 .tc FD_0_0[TC],0
163fpone:
164 .tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */
165fphalf:
166 .tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */
167
168#define LDCONST(fr, name) \
169 lfd fr,name@toc(r2)
170#endif
171
172 .text
173/*
174 * Internal routine to enable floating point and set FPSCR to 0.
175 * Don't call it from C; it doesn't use the normal calling convention.
176 */
177fpenable:
178#ifdef CONFIG_PPC32
179 stwu r1,-64(r1)
180#else
181 stdu r1,-64(r1)
182#endif
183 mfmsr r10
184 ori r11,r10,MSR_FP
185 mtmsr r11
186 isync
187 stfd fr0,24(r1)
188 stfd fr1,16(r1)
189 stfd fr31,8(r1)
190 LDCONST(fr1, fpzero)
191 mffs fr31
192 MTFSF_L(fr1)
193 blr
194
195fpdisable:
196 mtlr r12
197 MTFSF_L(fr31)
198 lfd fr31,8(r1)
199 lfd fr1,16(r1)
200 lfd fr0,24(r1)
201 mtmsr r10
202 isync
203 addi r1,r1,64
204 blr
205
206/*
207 * Vector add, floating point.
208 */
209_GLOBAL(vaddfp)
210 mflr r12
211 bl fpenable
212 li r0,4
213 mtctr r0
214 li r6,0
2151: lfsx fr0,r4,r6
216 lfsx fr1,r5,r6
217 fadds fr0,fr0,fr1
218 stfsx fr0,r3,r6
219 addi r6,r6,4
220 bdnz 1b
221 b fpdisable
222
223/*
224 * Vector subtract, floating point.
225 */
226_GLOBAL(vsubfp)
227 mflr r12
228 bl fpenable
229 li r0,4
230 mtctr r0
231 li r6,0
2321: lfsx fr0,r4,r6
233 lfsx fr1,r5,r6
234 fsubs fr0,fr0,fr1
235 stfsx fr0,r3,r6
236 addi r6,r6,4
237 bdnz 1b
238 b fpdisable
239
240/*
241 * Vector multiply and add, floating point.
242 */
243_GLOBAL(vmaddfp)
244 mflr r12
245 bl fpenable
246 stfd fr2,32(r1)
247 li r0,4
248 mtctr r0
249 li r7,0
2501: lfsx fr0,r4,r7
251 lfsx fr1,r5,r7
252 lfsx fr2,r6,r7
253 fmadds fr0,fr0,fr2,fr1
254 stfsx fr0,r3,r7
255 addi r7,r7,4
256 bdnz 1b
257 lfd fr2,32(r1)
258 b fpdisable
259
260/*
261 * Vector negative multiply and subtract, floating point.
262 */
263_GLOBAL(vnmsubfp)
264 mflr r12
265 bl fpenable
266 stfd fr2,32(r1)
267 li r0,4
268 mtctr r0
269 li r7,0
2701: lfsx fr0,r4,r7
271 lfsx fr1,r5,r7
272 lfsx fr2,r6,r7
273 fnmsubs fr0,fr0,fr2,fr1
274 stfsx fr0,r3,r7
275 addi r7,r7,4
276 bdnz 1b
277 lfd fr2,32(r1)
278 b fpdisable
279
280/*
281 * Vector reciprocal estimate. We just compute 1.0/x.
282 * r3 -> destination, r4 -> source.
283 */
284_GLOBAL(vrefp)
285 mflr r12
286 bl fpenable
287 li r0,4
288 LDCONST(fr1, fpone)
289 mtctr r0
290 li r6,0
2911: lfsx fr0,r4,r6
292 fdivs fr0,fr1,fr0
293 stfsx fr0,r3,r6
294 addi r6,r6,4
295 bdnz 1b
296 b fpdisable
297
298/*
299 * Vector reciprocal square-root estimate, floating point.
300 * We use the frsqrte instruction for the initial estimate followed
301 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
302 * r3 -> destination, r4 -> source.
303 */
304_GLOBAL(vrsqrtefp)
305 mflr r12
306 bl fpenable
307 stfd fr2,32(r1)
308 stfd fr3,40(r1)
309 stfd fr4,48(r1)
310 stfd fr5,56(r1)
311 li r0,4
312 LDCONST(fr4, fpone)
313 LDCONST(fr5, fphalf)
314 mtctr r0
315 li r6,0
3161: lfsx fr0,r4,r6
317 frsqrte fr1,fr0 /* r = frsqrte(s) */
318 fmuls fr3,fr1,fr0 /* r * s */
319 fmuls fr2,fr1,fr5 /* r * 0.5 */
320 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
321 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
322 fmuls fr3,fr1,fr0 /* r * s */
323 fmuls fr2,fr1,fr5 /* r * 0.5 */
324 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
325 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
326 stfsx fr1,r3,r6
327 addi r6,r6,4
328 bdnz 1b
329 lfd fr5,56(r1)
330 lfd fr4,48(r1)
331 lfd fr3,40(r1)
332 lfd fr2,32(r1)
333 b fpdisable
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/processor.h>
3#include <asm/ppc_asm.h>
4#include <asm/reg.h>
5#include <asm/asm-offsets.h>
6#include <asm/cputable.h>
7#include <asm/thread_info.h>
8#include <asm/page.h>
9#include <asm/ptrace.h>
10#include <asm/export.h>
11
12/*
13 * Load state from memory into VMX registers including VSCR.
14 * Assumes the caller has enabled VMX in the MSR.
15 */
16_GLOBAL(load_vr_state)
17 li r4,VRSTATE_VSCR
18 lvx v0,r4,r3
19 mtvscr v0
20 REST_32VRS(0,r4,r3)
21 blr
22EXPORT_SYMBOL(load_vr_state)
23
24/*
25 * Store VMX state into memory, including VSCR.
26 * Assumes the caller has enabled VMX in the MSR.
27 */
28_GLOBAL(store_vr_state)
29 SAVE_32VRS(0, r4, r3)
30 mfvscr v0
31 li r4, VRSTATE_VSCR
32 stvx v0, r4, r3
33 blr
34EXPORT_SYMBOL(store_vr_state)
35
36/*
37 * Disable VMX for the task which had it previously,
38 * and save its vector registers in its thread_struct.
39 * Enables the VMX for use in the kernel on return.
40 * On SMP we know the VMX is free, since we give it up every
41 * switch (ie, no lazy save of the vector registers).
42 *
43 * Note that on 32-bit this can only use registers that will be
44 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
45 */
46_GLOBAL(load_up_altivec)
47 mfmsr r5 /* grab the current MSR */
48 oris r5,r5,MSR_VEC@h
49 MTMSRD(r5) /* enable use of AltiVec now */
50 isync
51
52 /*
53 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
54 * to optimise userspace context save/restore. Whenever we take an
55 * altivec unavailable exception we must set VRSAVE to something non
56 * zero. Set it to all 1s. See also the programming note in the ISA.
57 */
58 mfspr r4,SPRN_VRSAVE
59 cmpwi 0,r4,0
60 bne+ 1f
61 li r4,-1
62 mtspr SPRN_VRSAVE,r4
631:
64 /* enable use of VMX after return */
65#ifdef CONFIG_PPC32
66 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
67 oris r9,r9,MSR_VEC@h
68#else
69 ld r4,PACACURRENT(r13)
70 addi r5,r4,THREAD /* Get THREAD */
71 oris r12,r12,MSR_VEC@h
72 std r12,_MSR(r1)
73#endif
74 /* Don't care if r4 overflows, this is desired behaviour */
75 lbz r4,THREAD_LOAD_VEC(r5)
76 addi r4,r4,1
77 stb r4,THREAD_LOAD_VEC(r5)
78 addi r6,r5,THREAD_VRSTATE
79 li r4,1
80 li r10,VRSTATE_VSCR
81 stw r4,THREAD_USED_VR(r5)
82 lvx v0,r10,r6
83 mtvscr v0
84 REST_32VRS(0,r4,r6)
85 /* restore registers and return */
86 blr
87
88/*
89 * save_altivec(tsk)
90 * Save the vector registers to its thread_struct
91 */
92_GLOBAL(save_altivec)
93 addi r3,r3,THREAD /* want THREAD of task */
94 PPC_LL r7,THREAD_VRSAVEAREA(r3)
95 PPC_LL r5,PT_REGS(r3)
96 PPC_LCMPI 0,r7,0
97 bne 2f
98 addi r7,r3,THREAD_VRSTATE
992: SAVE_32VRS(0,r4,r7)
100 mfvscr v0
101 li r4,VRSTATE_VSCR
102 stvx v0,r4,r7
103 blr
104
105#ifdef CONFIG_VSX
106
107#ifdef CONFIG_PPC32
108#error This asm code isn't ready for 32-bit kernels
109#endif
110
111/*
112 * load_up_vsx(unused, unused, tsk)
113 * Disable VSX for the task which had it previously,
114 * and save its vector registers in its thread_struct.
115 * Reuse the fp and vsx saves, but first check to see if they have
116 * been saved already.
117 */
118_GLOBAL(load_up_vsx)
119/* Load FP and VSX registers if they haven't been done yet */
120 andi. r5,r12,MSR_FP
121 beql+ load_up_fpu /* skip if already loaded */
122 andis. r5,r12,MSR_VEC@h
123 beql+ load_up_altivec /* skip if already loaded */
124
125 ld r4,PACACURRENT(r13)
126 addi r4,r4,THREAD /* Get THREAD */
127 li r6,1
128 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
129 /* enable use of VSX after return */
130 oris r12,r12,MSR_VSX@h
131 std r12,_MSR(r1)
132 b fast_exception_return
133
134#endif /* CONFIG_VSX */
135
136
137/*
138 * The routines below are in assembler so we can closely control the
139 * usage of floating-point registers. These routines must be called
140 * with preempt disabled.
141 */
142#ifdef CONFIG_PPC32
143 .data
144fpzero:
145 .long 0
146fpone:
147 .long 0x3f800000 /* 1.0 in single-precision FP */
148fphalf:
149 .long 0x3f000000 /* 0.5 in single-precision FP */
150
151#define LDCONST(fr, name) \
152 lis r11,name@ha; \
153 lfs fr,name@l(r11)
154#else
155
156 .section ".toc","aw"
157fpzero:
158 .tc FD_0_0[TC],0
159fpone:
160 .tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */
161fphalf:
162 .tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */
163
164#define LDCONST(fr, name) \
165 lfd fr,name@toc(r2)
166#endif
167
168 .text
169/*
170 * Internal routine to enable floating point and set FPSCR to 0.
171 * Don't call it from C; it doesn't use the normal calling convention.
172 */
173fpenable:
174#ifdef CONFIG_PPC32
175 stwu r1,-64(r1)
176#else
177 stdu r1,-64(r1)
178#endif
179 mfmsr r10
180 ori r11,r10,MSR_FP
181 mtmsr r11
182 isync
183 stfd fr0,24(r1)
184 stfd fr1,16(r1)
185 stfd fr31,8(r1)
186 LDCONST(fr1, fpzero)
187 mffs fr31
188 MTFSF_L(fr1)
189 blr
190
191fpdisable:
192 mtlr r12
193 MTFSF_L(fr31)
194 lfd fr31,8(r1)
195 lfd fr1,16(r1)
196 lfd fr0,24(r1)
197 mtmsr r10
198 isync
199 addi r1,r1,64
200 blr
201
202/*
203 * Vector add, floating point.
204 */
205_GLOBAL(vaddfp)
206 mflr r12
207 bl fpenable
208 li r0,4
209 mtctr r0
210 li r6,0
2111: lfsx fr0,r4,r6
212 lfsx fr1,r5,r6
213 fadds fr0,fr0,fr1
214 stfsx fr0,r3,r6
215 addi r6,r6,4
216 bdnz 1b
217 b fpdisable
218
219/*
220 * Vector subtract, floating point.
221 */
222_GLOBAL(vsubfp)
223 mflr r12
224 bl fpenable
225 li r0,4
226 mtctr r0
227 li r6,0
2281: lfsx fr0,r4,r6
229 lfsx fr1,r5,r6
230 fsubs fr0,fr0,fr1
231 stfsx fr0,r3,r6
232 addi r6,r6,4
233 bdnz 1b
234 b fpdisable
235
236/*
237 * Vector multiply and add, floating point.
238 */
239_GLOBAL(vmaddfp)
240 mflr r12
241 bl fpenable
242 stfd fr2,32(r1)
243 li r0,4
244 mtctr r0
245 li r7,0
2461: lfsx fr0,r4,r7
247 lfsx fr1,r5,r7
248 lfsx fr2,r6,r7
249 fmadds fr0,fr0,fr2,fr1
250 stfsx fr0,r3,r7
251 addi r7,r7,4
252 bdnz 1b
253 lfd fr2,32(r1)
254 b fpdisable
255
256/*
257 * Vector negative multiply and subtract, floating point.
258 */
259_GLOBAL(vnmsubfp)
260 mflr r12
261 bl fpenable
262 stfd fr2,32(r1)
263 li r0,4
264 mtctr r0
265 li r7,0
2661: lfsx fr0,r4,r7
267 lfsx fr1,r5,r7
268 lfsx fr2,r6,r7
269 fnmsubs fr0,fr0,fr2,fr1
270 stfsx fr0,r3,r7
271 addi r7,r7,4
272 bdnz 1b
273 lfd fr2,32(r1)
274 b fpdisable
275
276/*
277 * Vector reciprocal estimate. We just compute 1.0/x.
278 * r3 -> destination, r4 -> source.
279 */
280_GLOBAL(vrefp)
281 mflr r12
282 bl fpenable
283 li r0,4
284 LDCONST(fr1, fpone)
285 mtctr r0
286 li r6,0
2871: lfsx fr0,r4,r6
288 fdivs fr0,fr1,fr0
289 stfsx fr0,r3,r6
290 addi r6,r6,4
291 bdnz 1b
292 b fpdisable
293
294/*
295 * Vector reciprocal square-root estimate, floating point.
296 * We use the frsqrte instruction for the initial estimate followed
297 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
298 * r3 -> destination, r4 -> source.
299 */
300_GLOBAL(vrsqrtefp)
301 mflr r12
302 bl fpenable
303 stfd fr2,32(r1)
304 stfd fr3,40(r1)
305 stfd fr4,48(r1)
306 stfd fr5,56(r1)
307 li r0,4
308 LDCONST(fr4, fpone)
309 LDCONST(fr5, fphalf)
310 mtctr r0
311 li r6,0
3121: lfsx fr0,r4,r6
313 frsqrte fr1,fr0 /* r = frsqrte(s) */
314 fmuls fr3,fr1,fr0 /* r * s */
315 fmuls fr2,fr1,fr5 /* r * 0.5 */
316 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
317 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
318 fmuls fr3,fr1,fr0 /* r * s */
319 fmuls fr2,fr1,fr5 /* r * 0.5 */
320 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
321 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
322 stfsx fr1,r3,r6
323 addi r6,r6,4
324 bdnz 1b
325 lfd fr5,56(r1)
326 lfd fr4,48(r1)
327 lfd fr3,40(r1)
328 lfd fr2,32(r1)
329 b fpdisable