Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/linkage.h>
3#include <asm/processor.h>
4#include <asm/ppc_asm.h>
5#include <asm/reg.h>
6#include <asm/asm-offsets.h>
7#include <asm/cputable.h>
8#include <asm/thread_info.h>
9#include <asm/page.h>
10#include <asm/ptrace.h>
11#include <asm/export.h>
12#include <asm/asm-compat.h>
13
14/*
15 * Load state from memory into VMX registers including VSCR.
16 * Assumes the caller has enabled VMX in the MSR.
17 */
18_GLOBAL(load_vr_state)
19 li r4,VRSTATE_VSCR
20 lvx v0,r4,r3
21 mtvscr v0
22 REST_32VRS(0,r4,r3)
23 blr
24EXPORT_SYMBOL(load_vr_state)
25_ASM_NOKPROBE_SYMBOL(load_vr_state); /* used by restore_math */
26
27/*
28 * Store VMX state into memory, including VSCR.
29 * Assumes the caller has enabled VMX in the MSR.
30 */
31_GLOBAL(store_vr_state)
32 SAVE_32VRS(0, r4, r3)
33 mfvscr v0
34 li r4, VRSTATE_VSCR
35 stvx v0, r4, r3
36 blr
37EXPORT_SYMBOL(store_vr_state)
38
39/*
40 * Disable VMX for the task which had it previously,
41 * and save its vector registers in its thread_struct.
42 * Enables the VMX for use in the kernel on return.
43 * On SMP we know the VMX is free, since we give it up every
44 * switch (ie, no lazy save of the vector registers).
45 *
46 * Note that on 32-bit this can only use registers that will be
47 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
48 */
49_GLOBAL(load_up_altivec)
50 mfmsr r5 /* grab the current MSR */
51#ifdef CONFIG_PPC_BOOK3S_64
52 /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
53 ori r5,r5,MSR_RI
54#endif
55 oris r5,r5,MSR_VEC@h
56 MTMSRD(r5) /* enable use of AltiVec now */
57 isync
58
59 /*
60 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
61 * to optimise userspace context save/restore. Whenever we take an
62 * altivec unavailable exception we must set VRSAVE to something non
63 * zero. Set it to all 1s. See also the programming note in the ISA.
64 */
65 mfspr r4,SPRN_VRSAVE
66 cmpwi 0,r4,0
67 bne+ 1f
68 li r4,-1
69 mtspr SPRN_VRSAVE,r4
701:
71 /* enable use of VMX after return */
72#ifdef CONFIG_PPC32
73 addi r5,r2,THREAD
74 oris r9,r9,MSR_VEC@h
75#else
76 ld r4,PACACURRENT(r13)
77 addi r5,r4,THREAD /* Get THREAD */
78 oris r12,r12,MSR_VEC@h
79 std r12,_MSR(r1)
80#ifdef CONFIG_PPC_BOOK3S_64
81 li r4,0
82 stb r4,PACASRR_VALID(r13)
83#endif
84#endif
85 li r4,1
86 stb r4,THREAD_LOAD_VEC(r5)
87 addi r6,r5,THREAD_VRSTATE
88 li r10,VRSTATE_VSCR
89 stw r4,THREAD_USED_VR(r5)
90 lvx v0,r10,r6
91 mtvscr v0
92 REST_32VRS(0,r4,r6)
93 /* restore registers and return */
94 blr
95_ASM_NOKPROBE_SYMBOL(load_up_altivec)
96
97/*
98 * save_altivec(tsk)
99 * Save the vector registers to its thread_struct
100 */
101_GLOBAL(save_altivec)
102 addi r3,r3,THREAD /* want THREAD of task */
103 PPC_LL r7,THREAD_VRSAVEAREA(r3)
104 PPC_LL r5,PT_REGS(r3)
105 PPC_LCMPI 0,r7,0
106 bne 2f
107 addi r7,r3,THREAD_VRSTATE
1082: SAVE_32VRS(0,r4,r7)
109 mfvscr v0
110 li r4,VRSTATE_VSCR
111 stvx v0,r4,r7
112 blr
113
114#ifdef CONFIG_VSX
115
116#ifdef CONFIG_PPC32
117#error This asm code isn't ready for 32-bit kernels
118#endif
119
120/*
121 * load_up_vsx(unused, unused, tsk)
122 * Disable VSX for the task which had it previously,
123 * and save its vector registers in its thread_struct.
124 * Reuse the fp and vsx saves, but first check to see if they have
125 * been saved already.
126 */
127_GLOBAL(load_up_vsx)
128/* Load FP and VSX registers if they haven't been done yet */
129 andi. r5,r12,MSR_FP
130 beql+ load_up_fpu /* skip if already loaded */
131 andis. r5,r12,MSR_VEC@h
132 beql+ load_up_altivec /* skip if already loaded */
133
134#ifdef CONFIG_PPC_BOOK3S_64
135 /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
136 li r5,MSR_RI
137 mtmsrd r5,1
138#endif
139
140 ld r4,PACACURRENT(r13)
141 addi r4,r4,THREAD /* Get THREAD */
142 li r6,1
143 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
144 /* enable use of VSX after return */
145 oris r12,r12,MSR_VSX@h
146 std r12,_MSR(r1)
147 li r4,0
148 stb r4,PACASRR_VALID(r13)
149 b fast_interrupt_return_srr
150
151#endif /* CONFIG_VSX */
152
153
154/*
155 * The routines below are in assembler so we can closely control the
156 * usage of floating-point registers. These routines must be called
157 * with preempt disabled.
158 */
159 .data
160#ifdef CONFIG_PPC32
161fpzero:
162 .long 0
163fpone:
164 .long 0x3f800000 /* 1.0 in single-precision FP */
165fphalf:
166 .long 0x3f000000 /* 0.5 in single-precision FP */
167
168#define LDCONST(fr, name) \
169 lis r11,name@ha; \
170 lfs fr,name@l(r11)
171#else
172
173fpzero:
174 .quad 0
175fpone:
176 .quad 0x3ff0000000000000 /* 1.0 */
177fphalf:
178 .quad 0x3fe0000000000000 /* 0.5 */
179
180#define LDCONST(fr, name) \
181 addis r11,r2,name@toc@ha; \
182 lfd fr,name@toc@l(r11)
183#endif
184 .text
185/*
186 * Internal routine to enable floating point and set FPSCR to 0.
187 * Don't call it from C; it doesn't use the normal calling convention.
188 */
189SYM_FUNC_START_LOCAL(fpenable)
190#ifdef CONFIG_PPC32
191 stwu r1,-64(r1)
192#else
193 stdu r1,-64(r1)
194#endif
195 mfmsr r10
196 ori r11,r10,MSR_FP
197 mtmsr r11
198 isync
199 stfd fr0,24(r1)
200 stfd fr1,16(r1)
201 stfd fr31,8(r1)
202 LDCONST(fr1, fpzero)
203 mffs fr31
204 MTFSF_L(fr1)
205 blr
206SYM_FUNC_END(fpenable)
207
208fpdisable:
209 mtlr r12
210 MTFSF_L(fr31)
211 lfd fr31,8(r1)
212 lfd fr1,16(r1)
213 lfd fr0,24(r1)
214 mtmsr r10
215 isync
216 addi r1,r1,64
217 blr
218
219/*
220 * Vector add, floating point.
221 */
222_GLOBAL(vaddfp)
223 mflr r12
224 bl fpenable
225 li r0,4
226 mtctr r0
227 li r6,0
2281: lfsx fr0,r4,r6
229 lfsx fr1,r5,r6
230 fadds fr0,fr0,fr1
231 stfsx fr0,r3,r6
232 addi r6,r6,4
233 bdnz 1b
234 b fpdisable
235
236/*
237 * Vector subtract, floating point.
238 */
239_GLOBAL(vsubfp)
240 mflr r12
241 bl fpenable
242 li r0,4
243 mtctr r0
244 li r6,0
2451: lfsx fr0,r4,r6
246 lfsx fr1,r5,r6
247 fsubs fr0,fr0,fr1
248 stfsx fr0,r3,r6
249 addi r6,r6,4
250 bdnz 1b
251 b fpdisable
252
253/*
254 * Vector multiply and add, floating point.
255 */
256_GLOBAL(vmaddfp)
257 mflr r12
258 bl fpenable
259 stfd fr2,32(r1)
260 li r0,4
261 mtctr r0
262 li r7,0
2631: lfsx fr0,r4,r7
264 lfsx fr1,r5,r7
265 lfsx fr2,r6,r7
266 fmadds fr0,fr0,fr2,fr1
267 stfsx fr0,r3,r7
268 addi r7,r7,4
269 bdnz 1b
270 lfd fr2,32(r1)
271 b fpdisable
272
273/*
274 * Vector negative multiply and subtract, floating point.
275 */
276_GLOBAL(vnmsubfp)
277 mflr r12
278 bl fpenable
279 stfd fr2,32(r1)
280 li r0,4
281 mtctr r0
282 li r7,0
2831: lfsx fr0,r4,r7
284 lfsx fr1,r5,r7
285 lfsx fr2,r6,r7
286 fnmsubs fr0,fr0,fr2,fr1
287 stfsx fr0,r3,r7
288 addi r7,r7,4
289 bdnz 1b
290 lfd fr2,32(r1)
291 b fpdisable
292
293/*
294 * Vector reciprocal estimate. We just compute 1.0/x.
295 * r3 -> destination, r4 -> source.
296 */
297_GLOBAL(vrefp)
298 mflr r12
299 bl fpenable
300 li r0,4
301 LDCONST(fr1, fpone)
302 mtctr r0
303 li r6,0
3041: lfsx fr0,r4,r6
305 fdivs fr0,fr1,fr0
306 stfsx fr0,r3,r6
307 addi r6,r6,4
308 bdnz 1b
309 b fpdisable
310
311/*
312 * Vector reciprocal square-root estimate, floating point.
313 * We use the frsqrte instruction for the initial estimate followed
314 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
315 * r3 -> destination, r4 -> source.
316 */
317_GLOBAL(vrsqrtefp)
318 mflr r12
319 bl fpenable
320 stfd fr2,32(r1)
321 stfd fr3,40(r1)
322 stfd fr4,48(r1)
323 stfd fr5,56(r1)
324 li r0,4
325 LDCONST(fr4, fpone)
326 LDCONST(fr5, fphalf)
327 mtctr r0
328 li r6,0
3291: lfsx fr0,r4,r6
330 frsqrte fr1,fr0 /* r = frsqrte(s) */
331 fmuls fr3,fr1,fr0 /* r * s */
332 fmuls fr2,fr1,fr5 /* r * 0.5 */
333 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
334 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
335 fmuls fr3,fr1,fr0 /* r * s */
336 fmuls fr2,fr1,fr5 /* r * 0.5 */
337 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
338 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
339 stfsx fr1,r3,r6
340 addi r6,r6,4
341 bdnz 1b
342 lfd fr5,56(r1)
343 lfd fr4,48(r1)
344 lfd fr3,40(r1)
345 lfd fr2,32(r1)
346 b fpdisable
1#include <asm/processor.h>
2#include <asm/ppc_asm.h>
3#include <asm/reg.h>
4#include <asm/asm-offsets.h>
5#include <asm/cputable.h>
6#include <asm/thread_info.h>
7#include <asm/page.h>
8#include <asm/ptrace.h>
9
10#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
11/* void do_load_up_transact_altivec(struct thread_struct *thread)
12 *
13 * This is similar to load_up_altivec but for the transactional version of the
14 * vector regs. It doesn't mess with the task MSR or valid flags.
15 * Furthermore, VEC laziness is not supported with TM currently.
16 */
17_GLOBAL(do_load_up_transact_altivec)
18 mfmsr r6
19 oris r5,r6,MSR_VEC@h
20 MTMSRD(r5)
21 isync
22
23 li r4,1
24 stw r4,THREAD_USED_VR(r3)
25
26 li r10,THREAD_TRANSACT_VRSTATE+VRSTATE_VSCR
27 lvx vr0,r10,r3
28 mtvscr vr0
29 addi r10,r3,THREAD_TRANSACT_VRSTATE
30 REST_32VRS(0,r4,r10)
31
32 /* Disable VEC again. */
33 MTMSRD(r6)
34 isync
35
36 blr
37#endif
38
39/*
40 * Enable use of VMX/Altivec for the caller.
41 */
42_GLOBAL(vec_enable)
43 mfmsr r3
44 oris r3,r3,MSR_VEC@h
45 MTMSRD(r3)
46 isync
47 blr
48
49/*
50 * Load state from memory into VMX registers including VSCR.
51 * Assumes the caller has enabled VMX in the MSR.
52 */
53_GLOBAL(load_vr_state)
54 li r4,VRSTATE_VSCR
55 lvx vr0,r4,r3
56 mtvscr vr0
57 REST_32VRS(0,r4,r3)
58 blr
59
60/*
61 * Store VMX state into memory, including VSCR.
62 * Assumes the caller has enabled VMX in the MSR.
63 */
64_GLOBAL(store_vr_state)
65 SAVE_32VRS(0, r4, r3)
66 mfvscr vr0
67 li r4, VRSTATE_VSCR
68 stvx vr0, r4, r3
69 blr
70
71/*
72 * Disable VMX for the task which had it previously,
73 * and save its vector registers in its thread_struct.
74 * Enables the VMX for use in the kernel on return.
75 * On SMP we know the VMX is free, since we give it up every
76 * switch (ie, no lazy save of the vector registers).
77 *
78 * Note that on 32-bit this can only use registers that will be
79 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
80 */
81_GLOBAL(load_up_altivec)
82 mfmsr r5 /* grab the current MSR */
83 oris r5,r5,MSR_VEC@h
84 MTMSRD(r5) /* enable use of AltiVec now */
85 isync
86
87/*
88 * For SMP, we don't do lazy VMX switching because it just gets too
89 * horrendously complex, especially when a task switches from one CPU
90 * to another. Instead we call giveup_altvec in switch_to.
91 * VRSAVE isn't dealt with here, that is done in the normal context
92 * switch code. Note that we could rely on vrsave value to eventually
93 * avoid saving all of the VREGs here...
94 */
95#ifndef CONFIG_SMP
96 LOAD_REG_ADDRBASE(r3, last_task_used_altivec)
97 toreal(r3)
98 PPC_LL r4,ADDROFF(last_task_used_altivec)(r3)
99 PPC_LCMPI 0,r4,0
100 beq 1f
101
102 /* Save VMX state to last_task_used_altivec's THREAD struct */
103 toreal(r4)
104 addi r4,r4,THREAD
105 addi r6,r4,THREAD_VRSTATE
106 SAVE_32VRS(0,r5,r6)
107 mfvscr vr0
108 li r10,VRSTATE_VSCR
109 stvx vr0,r10,r6
110 /* Disable VMX for last_task_used_altivec */
111 PPC_LL r5,PT_REGS(r4)
112 toreal(r5)
113 PPC_LL r4,_MSR-STACK_FRAME_OVERHEAD(r5)
114 lis r10,MSR_VEC@h
115 andc r4,r4,r10
116 PPC_STL r4,_MSR-STACK_FRAME_OVERHEAD(r5)
1171:
118#endif /* CONFIG_SMP */
119
120 /* Hack: if we get an altivec unavailable trap with VRSAVE
121 * set to all zeros, we assume this is a broken application
122 * that fails to set it properly, and thus we switch it to
123 * all 1's
124 */
125 mfspr r4,SPRN_VRSAVE
126 cmpwi 0,r4,0
127 bne+ 1f
128 li r4,-1
129 mtspr SPRN_VRSAVE,r4
1301:
131 /* enable use of VMX after return */
132#ifdef CONFIG_PPC32
133 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
134 oris r9,r9,MSR_VEC@h
135#else
136 ld r4,PACACURRENT(r13)
137 addi r5,r4,THREAD /* Get THREAD */
138 oris r12,r12,MSR_VEC@h
139 std r12,_MSR(r1)
140#endif
141 addi r6,r5,THREAD_VRSTATE
142 li r4,1
143 li r10,VRSTATE_VSCR
144 stw r4,THREAD_USED_VR(r5)
145 lvx vr0,r10,r6
146 mtvscr vr0
147 REST_32VRS(0,r4,r6)
148#ifndef CONFIG_SMP
149 /* Update last_task_used_altivec to 'current' */
150 subi r4,r5,THREAD /* Back to 'current' */
151 fromreal(r4)
152 PPC_STL r4,ADDROFF(last_task_used_altivec)(r3)
153#endif /* CONFIG_SMP */
154 /* restore registers and return */
155 blr
156
157_GLOBAL(giveup_altivec_notask)
158 mfmsr r3
159 andis. r4,r3,MSR_VEC@h
160 bnelr /* Already enabled? */
161 oris r3,r3,MSR_VEC@h
162 SYNC
163 MTMSRD(r3) /* enable use of VMX now */
164 isync
165 blr
166
167/*
168 * giveup_altivec(tsk)
169 * Disable VMX for the task given as the argument,
170 * and save the vector registers in its thread_struct.
171 * Enables the VMX for use in the kernel on return.
172 */
173_GLOBAL(giveup_altivec)
174 mfmsr r5
175 oris r5,r5,MSR_VEC@h
176 SYNC
177 MTMSRD(r5) /* enable use of VMX now */
178 isync
179 PPC_LCMPI 0,r3,0
180 beqlr /* if no previous owner, done */
181 addi r3,r3,THREAD /* want THREAD of task */
182 PPC_LL r7,THREAD_VRSAVEAREA(r3)
183 PPC_LL r5,PT_REGS(r3)
184 PPC_LCMPI 0,r7,0
185 bne 2f
186 addi r7,r3,THREAD_VRSTATE
1872: PPC_LCMPI 0,r5,0
188 SAVE_32VRS(0,r4,r7)
189 mfvscr vr0
190 li r4,VRSTATE_VSCR
191 stvx vr0,r4,r7
192 beq 1f
193 PPC_LL r4,_MSR-STACK_FRAME_OVERHEAD(r5)
194#ifdef CONFIG_VSX
195BEGIN_FTR_SECTION
196 lis r3,(MSR_VEC|MSR_VSX)@h
197FTR_SECTION_ELSE
198 lis r3,MSR_VEC@h
199ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
200#else
201 lis r3,MSR_VEC@h
202#endif
203 andc r4,r4,r3 /* disable FP for previous task */
204 PPC_STL r4,_MSR-STACK_FRAME_OVERHEAD(r5)
2051:
206#ifndef CONFIG_SMP
207 li r5,0
208 LOAD_REG_ADDRBASE(r4,last_task_used_altivec)
209 PPC_STL r5,ADDROFF(last_task_used_altivec)(r4)
210#endif /* CONFIG_SMP */
211 blr
212
213#ifdef CONFIG_VSX
214
215#ifdef CONFIG_PPC32
216#error This asm code isn't ready for 32-bit kernels
217#endif
218
219/*
220 * load_up_vsx(unused, unused, tsk)
221 * Disable VSX for the task which had it previously,
222 * and save its vector registers in its thread_struct.
223 * Reuse the fp and vsx saves, but first check to see if they have
224 * been saved already.
225 */
226_GLOBAL(load_up_vsx)
227/* Load FP and VSX registers if they haven't been done yet */
228 andi. r5,r12,MSR_FP
229 beql+ load_up_fpu /* skip if already loaded */
230 andis. r5,r12,MSR_VEC@h
231 beql+ load_up_altivec /* skip if already loaded */
232
233#ifndef CONFIG_SMP
234 ld r3,last_task_used_vsx@got(r2)
235 ld r4,0(r3)
236 cmpdi 0,r4,0
237 beq 1f
238 /* Disable VSX for last_task_used_vsx */
239 addi r4,r4,THREAD
240 ld r5,PT_REGS(r4)
241 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5)
242 lis r6,MSR_VSX@h
243 andc r6,r4,r6
244 std r6,_MSR-STACK_FRAME_OVERHEAD(r5)
2451:
246#endif /* CONFIG_SMP */
247 ld r4,PACACURRENT(r13)
248 addi r4,r4,THREAD /* Get THREAD */
249 li r6,1
250 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
251 /* enable use of VSX after return */
252 oris r12,r12,MSR_VSX@h
253 std r12,_MSR(r1)
254#ifndef CONFIG_SMP
255 /* Update last_task_used_vsx to 'current' */
256 ld r4,PACACURRENT(r13)
257 std r4,0(r3)
258#endif /* CONFIG_SMP */
259 b fast_exception_return
260
261/*
262 * __giveup_vsx(tsk)
263 * Disable VSX for the task given as the argument.
264 * Does NOT save vsx registers.
265 * Enables the VSX for use in the kernel on return.
266 */
267_GLOBAL(__giveup_vsx)
268 mfmsr r5
269 oris r5,r5,MSR_VSX@h
270 mtmsrd r5 /* enable use of VSX now */
271 isync
272
273 cmpdi 0,r3,0
274 beqlr- /* if no previous owner, done */
275 addi r3,r3,THREAD /* want THREAD of task */
276 ld r5,PT_REGS(r3)
277 cmpdi 0,r5,0
278 beq 1f
279 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5)
280 lis r3,MSR_VSX@h
281 andc r4,r4,r3 /* disable VSX for previous task */
282 std r4,_MSR-STACK_FRAME_OVERHEAD(r5)
2831:
284#ifndef CONFIG_SMP
285 li r5,0
286 ld r4,last_task_used_vsx@got(r2)
287 std r5,0(r4)
288#endif /* CONFIG_SMP */
289 blr
290
291#endif /* CONFIG_VSX */
292
293
294/*
295 * The routines below are in assembler so we can closely control the
296 * usage of floating-point registers. These routines must be called
297 * with preempt disabled.
298 */
299#ifdef CONFIG_PPC32
300 .data
301fpzero:
302 .long 0
303fpone:
304 .long 0x3f800000 /* 1.0 in single-precision FP */
305fphalf:
306 .long 0x3f000000 /* 0.5 in single-precision FP */
307
308#define LDCONST(fr, name) \
309 lis r11,name@ha; \
310 lfs fr,name@l(r11)
311#else
312
313 .section ".toc","aw"
314fpzero:
315 .tc FD_0_0[TC],0
316fpone:
317 .tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */
318fphalf:
319 .tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */
320
321#define LDCONST(fr, name) \
322 lfd fr,name@toc(r2)
323#endif
324
325 .text
326/*
327 * Internal routine to enable floating point and set FPSCR to 0.
328 * Don't call it from C; it doesn't use the normal calling convention.
329 */
330fpenable:
331#ifdef CONFIG_PPC32
332 stwu r1,-64(r1)
333#else
334 stdu r1,-64(r1)
335#endif
336 mfmsr r10
337 ori r11,r10,MSR_FP
338 mtmsr r11
339 isync
340 stfd fr0,24(r1)
341 stfd fr1,16(r1)
342 stfd fr31,8(r1)
343 LDCONST(fr1, fpzero)
344 mffs fr31
345 MTFSF_L(fr1)
346 blr
347
348fpdisable:
349 mtlr r12
350 MTFSF_L(fr31)
351 lfd fr31,8(r1)
352 lfd fr1,16(r1)
353 lfd fr0,24(r1)
354 mtmsr r10
355 isync
356 addi r1,r1,64
357 blr
358
359/*
360 * Vector add, floating point.
361 */
362_GLOBAL(vaddfp)
363 mflr r12
364 bl fpenable
365 li r0,4
366 mtctr r0
367 li r6,0
3681: lfsx fr0,r4,r6
369 lfsx fr1,r5,r6
370 fadds fr0,fr0,fr1
371 stfsx fr0,r3,r6
372 addi r6,r6,4
373 bdnz 1b
374 b fpdisable
375
376/*
377 * Vector subtract, floating point.
378 */
379_GLOBAL(vsubfp)
380 mflr r12
381 bl fpenable
382 li r0,4
383 mtctr r0
384 li r6,0
3851: lfsx fr0,r4,r6
386 lfsx fr1,r5,r6
387 fsubs fr0,fr0,fr1
388 stfsx fr0,r3,r6
389 addi r6,r6,4
390 bdnz 1b
391 b fpdisable
392
393/*
394 * Vector multiply and add, floating point.
395 */
396_GLOBAL(vmaddfp)
397 mflr r12
398 bl fpenable
399 stfd fr2,32(r1)
400 li r0,4
401 mtctr r0
402 li r7,0
4031: lfsx fr0,r4,r7
404 lfsx fr1,r5,r7
405 lfsx fr2,r6,r7
406 fmadds fr0,fr0,fr2,fr1
407 stfsx fr0,r3,r7
408 addi r7,r7,4
409 bdnz 1b
410 lfd fr2,32(r1)
411 b fpdisable
412
413/*
414 * Vector negative multiply and subtract, floating point.
415 */
416_GLOBAL(vnmsubfp)
417 mflr r12
418 bl fpenable
419 stfd fr2,32(r1)
420 li r0,4
421 mtctr r0
422 li r7,0
4231: lfsx fr0,r4,r7
424 lfsx fr1,r5,r7
425 lfsx fr2,r6,r7
426 fnmsubs fr0,fr0,fr2,fr1
427 stfsx fr0,r3,r7
428 addi r7,r7,4
429 bdnz 1b
430 lfd fr2,32(r1)
431 b fpdisable
432
433/*
434 * Vector reciprocal estimate. We just compute 1.0/x.
435 * r3 -> destination, r4 -> source.
436 */
437_GLOBAL(vrefp)
438 mflr r12
439 bl fpenable
440 li r0,4
441 LDCONST(fr1, fpone)
442 mtctr r0
443 li r6,0
4441: lfsx fr0,r4,r6
445 fdivs fr0,fr1,fr0
446 stfsx fr0,r3,r6
447 addi r6,r6,4
448 bdnz 1b
449 b fpdisable
450
451/*
452 * Vector reciprocal square-root estimate, floating point.
453 * We use the frsqrte instruction for the initial estimate followed
454 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
455 * r3 -> destination, r4 -> source.
456 */
457_GLOBAL(vrsqrtefp)
458 mflr r12
459 bl fpenable
460 stfd fr2,32(r1)
461 stfd fr3,40(r1)
462 stfd fr4,48(r1)
463 stfd fr5,56(r1)
464 li r0,4
465 LDCONST(fr4, fpone)
466 LDCONST(fr5, fphalf)
467 mtctr r0
468 li r6,0
4691: lfsx fr0,r4,r6
470 frsqrte fr1,fr0 /* r = frsqrte(s) */
471 fmuls fr3,fr1,fr0 /* r * s */
472 fmuls fr2,fr1,fr5 /* r * 0.5 */
473 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
474 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
475 fmuls fr3,fr1,fr0 /* r * s */
476 fmuls fr2,fr1,fr5 /* r * 0.5 */
477 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
478 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
479 stfsx fr1,r3,r6
480 addi r6,r6,4
481 bdnz 1b
482 lfd fr5,56(r1)
483 lfd fr4,48(r1)
484 lfd fr3,40(r1)
485 lfd fr2,32(r1)
486 b fpdisable