1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * FPU signal frame handling routines.
  4 */
  5
  6#include <linux/compat.h>
  7#include <linux/cpu.h>
  8#include <linux/pagemap.h>
  9
 10#include <asm/fpu/internal.h>
 11#include <asm/fpu/signal.h>
 12#include <asm/fpu/regset.h>
 13#include <asm/fpu/xstate.h>
 14
 15#include <asm/sigframe.h>
 16#include <asm/trace/fpu.h>
 17
 18static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init;
 19static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init;
 20
 21/*
 22 * Check for the presence of extended state information in the
 23 * user fpstate pointer in the sigcontext.
 24 */
 25static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
 26					   struct _fpx_sw_bytes *fx_sw)
 27{
 28	int min_xstate_size = sizeof(struct fxregs_state) +
 29			      sizeof(struct xstate_header);
 30	void __user *fpstate = fxbuf;
 31	unsigned int magic2;
 32
 33	if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
 34		return -EFAULT;
 35
 36	/* Check for the first magic field and other error scenarios. */
 37	if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
 38	    fx_sw->xstate_size < min_xstate_size ||
 39	    fx_sw->xstate_size > fpu_user_xstate_size ||
 40	    fx_sw->xstate_size > fx_sw->extended_size)
 41		goto setfx;
 42
 43	/*
 44	 * Check for the presence of second magic word at the end of memory
 45	 * layout. This detects the case where the user just copied the legacy
 46	 * fpstate layout with out copying the extended state information
 47	 * in the memory layout.
 48	 */
 49	if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
 50		return -EFAULT;
 51
 52	if (likely(magic2 == FP_XSTATE_MAGIC2))
 53		return 0;
 54setfx:
 55	trace_x86_fpu_xstate_check_failed(&current->thread.fpu);
 56
 57	/* Set the parameters for fx only state */
 58	fx_sw->magic1 = 0;
 59	fx_sw->xstate_size = sizeof(struct fxregs_state);
 60	fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
 61	return 0;
 62}
 63
 64/*
 65 * Signal frame handlers.
 66 */
 67static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
 68{
 69	if (use_fxsr()) {
 70		struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;
 71		struct user_i387_ia32_struct env;
 72		struct _fpstate_32 __user *fp = buf;
 73
 74		fpregs_lock();
 75		if (!test_thread_flag(TIF_NEED_FPU_LOAD))
 76			fxsave(&tsk->thread.fpu.state.fxsave);
 77		fpregs_unlock();
 78
 79		convert_from_fxsr(&env, tsk);
 80
 81		if (__copy_to_user(buf, &env, sizeof(env)) ||
 82		    __put_user(xsave->i387.swd, &fp->status) ||
 83		    __put_user(X86_FXSR_MAGIC, &fp->magic))
 84			return -1;
 85	} else {
 86		struct fregs_state __user *fp = buf;
 87		u32 swd;
 88		if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
 89			return -1;
 90	}
 91
 92	return 0;
 93}
 94
 95static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
 96{
 97	struct xregs_state __user *x = buf;
 98	struct _fpx_sw_bytes *sw_bytes;
 99	u32 xfeatures;
100	int err;
101
102	/* Setup the bytes not touched by the [f]xsave and reserved for SW. */
103	sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
104	err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
105
106	if (!use_xsave())
107		return err;
108
109	err |= __put_user(FP_XSTATE_MAGIC2,
110			  (__u32 __user *)(buf + fpu_user_xstate_size));
111
112	/*
113	 * Read the xfeatures which we copied (directly from the cpu or
114	 * from the state in task struct) to the user buffers.
115	 */
116	err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
117
118	/*
119	 * For legacy compatible, we always set FP/SSE bits in the bit
120	 * vector while saving the state to the user context. This will
121	 * enable us capturing any changes(during sigreturn) to
122	 * the FP/SSE bits by the legacy applications which don't touch
123	 * xfeatures in the xsave header.
124	 *
125	 * xsave aware apps can change the xfeatures in the xsave
126	 * header as well as change any contents in the memory layout.
127	 * xrestore as part of sigreturn will capture all the changes.
128	 */
129	xfeatures |= XFEATURE_MASK_FPSSE;
130
131	err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
132
133	return err;
134}
135
136static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
137{
138	int err;
139
140	if (use_xsave())
141		err = xsave_to_user_sigframe(buf);
142	else if (use_fxsr())
143		err = fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
144	else
145		err = fnsave_to_user_sigframe((struct fregs_state __user *) buf);
146
147	if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size))
148		err = -EFAULT;
149	return err;
150}
151
152/*
153 * Save the fpu, extended register state to the user signal frame.
154 *
155 * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
156 *  state is copied.
157 *  'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
158 *
159 *	buf == buf_fx for 64-bit frames and 32-bit fsave frame.
160 *	buf != buf_fx for 32-bit frames with fxstate.
161 *
162 * Try to save it directly to the user frame with disabled page fault handler.
163 * If this fails then do the slow path where the FPU state is first saved to
164 * task's fpu->state and then copy it to the user frame pointed to by the
165 * aligned pointer 'buf_fx'.
166 *
167 * If this is a 32-bit frame with fxstate, put a fsave header before
168 * the aligned state at 'buf_fx'.
169 *
170 * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
171 * indicating the absence/presence of the extended state to the user.
172 */
173int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
174{
175	struct task_struct *tsk = current;
176	int ia32_fxstate = (buf != buf_fx);
177	int ret;
178
179	ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
180			 IS_ENABLED(CONFIG_IA32_EMULATION));
181
182	if (!static_cpu_has(X86_FEATURE_FPU)) {
183		struct user_i387_ia32_struct fp;
184		fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
185						.left = sizeof(fp)});
186		return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0;
187	}
188
189	if (!access_ok(buf, size))
190		return -EACCES;
191retry:
192	/*
193	 * Load the FPU registers if they are not valid for the current task.
194	 * With a valid FPU state we can attempt to save the state directly to
195	 * userland's stack frame which will likely succeed. If it does not,
196	 * resolve the fault in the user memory and try again.
197	 */
198	fpregs_lock();
199	if (test_thread_flag(TIF_NEED_FPU_LOAD))
200		fpregs_restore_userregs();
201
202	pagefault_disable();
203	ret = copy_fpregs_to_sigframe(buf_fx);
204	pagefault_enable();
205	fpregs_unlock();
206
207	if (ret) {
208		if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size))
209			goto retry;
210		return -EFAULT;
211	}
212
213	/* Save the fsave header for the 32-bit frames. */
214	if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
215		return -1;
216
217	if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
218		return -1;
219
220	return 0;
221}
222
223static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
224				      bool fx_only)
225{
226	if (use_xsave()) {
227		u64 init_bv = xfeatures_mask_uabi() & ~xrestore;
228		int ret;
229
230		if (likely(!fx_only))
231			ret = xrstor_from_user_sigframe(buf, xrestore);
232		else
233			ret = fxrstor_from_user_sigframe(buf);
234
235		if (!ret && unlikely(init_bv))
236			os_xrstor(&init_fpstate.xsave, init_bv);
237		return ret;
238	} else if (use_fxsr()) {
239		return fxrstor_from_user_sigframe(buf);
240	} else {
241		return frstor_from_user_sigframe(buf);
242	}
243}
244
245/*
246 * Attempt to restore the FPU registers directly from user memory.
247 * Pagefaults are handled and any errors returned are fatal.
248 */
249static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
250				    bool fx_only, unsigned int size)
251{
252	struct fpu *fpu = &current->thread.fpu;
253	int ret;
254
255retry:
256	fpregs_lock();
257	pagefault_disable();
258	ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
259	pagefault_enable();
260
261	if (unlikely(ret)) {
262		/*
263		 * The above did an FPU restore operation, restricted to
264		 * the user portion of the registers, and failed, but the
265		 * microcode might have modified the FPU registers
266		 * nevertheless.
267		 *
268		 * If the FPU registers do not belong to current, then
269		 * invalidate the FPU register state otherwise the task
270		 * might preempt current and return to user space with
271		 * corrupted FPU registers.
272		 */
273		if (test_thread_flag(TIF_NEED_FPU_LOAD))
274			__cpu_invalidate_fpregs_state();
275		fpregs_unlock();
276
277		/* Try to handle #PF, but anything else is fatal. */
278		if (ret != -EFAULT)
279			return -EINVAL;
280
281		ret = fault_in_pages_readable(buf, size);
282		if (!ret)
283			goto retry;
284		return ret;
285	}
286
287	/*
288	 * Restore supervisor states: previous context switch etc has done
289	 * XSAVES and saved the supervisor states in the kernel buffer from
290	 * which they can be restored now.
291	 *
292	 * It would be optimal to handle this with a single XRSTORS, but
293	 * this does not work because the rest of the FPU registers have
294	 * been restored from a user buffer directly.
295	 */
296	if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
297		os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
298
299	fpregs_mark_activate();
300	fpregs_unlock();
301	return 0;
302}
303
304static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
305			     bool ia32_fxstate)
306{
307	int state_size = fpu_kernel_xstate_size;
308	struct task_struct *tsk = current;
309	struct fpu *fpu = &tsk->thread.fpu;
310	struct user_i387_ia32_struct env;
311	u64 user_xfeatures = 0;
312	bool fx_only = false;
313	int ret;
314
315	if (use_xsave()) {
316		struct _fpx_sw_bytes fx_sw_user;
317
318		ret = check_xstate_in_sigframe(buf_fx, &fx_sw_user);
319		if (unlikely(ret))
320			return ret;
321
322		fx_only = !fx_sw_user.magic1;
323		state_size = fx_sw_user.xstate_size;
324		user_xfeatures = fx_sw_user.xfeatures;
325	} else {
326		user_xfeatures = XFEATURE_MASK_FPSSE;
327	}
328
329	if (likely(!ia32_fxstate)) {
330		/*
331		 * Attempt to restore the FPU registers directly from user
332		 * memory. For that to succeed, the user access cannot cause page
333		 * faults. If it does, fall back to the slow path below, going
334		 * through the kernel buffer with the enabled pagefault handler.
335		 */
336		return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
337						state_size);
338	}
339
340	/*
341	 * Copy the legacy state because the FP portion of the FX frame has
342	 * to be ignored for histerical raisins. The legacy state is folded
343	 * in once the larger state has been copied.
344	 */
345	ret = __copy_from_user(&env, buf, sizeof(env));
346	if (ret)
347		return ret;
348
349	/*
350	 * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
351	 * not modified on context switch and that the xstate is considered
352	 * to be loaded again on return to userland (overriding last_cpu avoids
353	 * the optimisation).
354	 */
355	fpregs_lock();
356	if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
357		/*
358		 * If supervisor states are available then save the
359		 * hardware state in current's fpstate so that the
360		 * supervisor state is preserved. Save the full state for
361		 * simplicity. There is no point in optimizing this by only
362		 * saving the supervisor states and then shuffle them to
363		 * the right place in memory. It's ia32 mode. Shrug.
364		 */
365		if (xfeatures_mask_supervisor())
366			os_xsave(&fpu->state.xsave);
367		set_thread_flag(TIF_NEED_FPU_LOAD);
368	}
369	__fpu_invalidate_fpregs_state(fpu);
370	__cpu_invalidate_fpregs_state();
371	fpregs_unlock();
372
373	if (use_xsave() && !fx_only) {
374		ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx);
375		if (ret)
376			return ret;
377	} else {
378		if (__copy_from_user(&fpu->state.fxsave, buf_fx,
379				     sizeof(fpu->state.fxsave)))
380			return -EFAULT;
381
382		if (IS_ENABLED(CONFIG_X86_64)) {
383			/* Reject invalid MXCSR values. */
384			if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask)
385				return -EINVAL;
386		} else {
387			/* Mask invalid bits out for historical reasons (broken hardware). */
388			fpu->state.fxsave.mxcsr &= mxcsr_feature_mask;
389		}
390
391		/* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
392		if (use_xsave())
393			fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
394	}
395
396	/* Fold the legacy FP storage */
397	convert_to_fxsr(&fpu->state.fxsave, &env);
398
399	fpregs_lock();
400	if (use_xsave()) {
401		/*
402		 * Remove all UABI feature bits not set in user_xfeatures
403		 * from the memory xstate header which makes the full
404		 * restore below bring them into init state. This works for
405		 * fx_only mode as well because that has only FP and SSE
406		 * set in user_xfeatures.
407		 *
408		 * Preserve supervisor states!
409		 */
410		u64 mask = user_xfeatures | xfeatures_mask_supervisor();
411
412		fpu->state.xsave.header.xfeatures &= mask;
413		ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
414	} else {
415		ret = fxrstor_safe(&fpu->state.fxsave);
416	}
417
418	if (likely(!ret))
419		fpregs_mark_activate();
420
421	fpregs_unlock();
422	return ret;
423}
424static inline int xstate_sigframe_size(void)
425{
426	return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
427			fpu_user_xstate_size;
428}
429
430/*
431 * Restore FPU state from a sigframe:
432 */
433int fpu__restore_sig(void __user *buf, int ia32_frame)
434{
435	unsigned int size = xstate_sigframe_size();
436	struct fpu *fpu = &current->thread.fpu;
437	void __user *buf_fx = buf;
438	bool ia32_fxstate = false;
439	int ret;
440
441	if (unlikely(!buf)) {
442		fpu__clear_user_states(fpu);
443		return 0;
444	}
445
446	ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
447		       IS_ENABLED(CONFIG_IA32_EMULATION));
448
449	/*
450	 * Only FXSR enabled systems need the FX state quirk.
451	 * FRSTOR does not need it and can use the fast path.
452	 */
453	if (ia32_frame && use_fxsr()) {
454		buf_fx = buf + sizeof(struct fregs_state);
455		size += sizeof(struct fregs_state);
456		ia32_fxstate = true;
457	}
458
459	if (!access_ok(buf, size)) {
460		ret = -EACCES;
461		goto out;
462	}
463
464	if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
465		ret = fpregs_soft_set(current, NULL, 0,
466				      sizeof(struct user_i387_ia32_struct),
467				      NULL, buf);
468	} else {
469		ret = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
470	}
471
472out:
473	if (unlikely(ret))
474		fpu__clear_user_states(fpu);
475	return ret;
476}
477
478unsigned long
479fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
480		     unsigned long *buf_fx, unsigned long *size)
481{
482	unsigned long frame_size = xstate_sigframe_size();
483
484	*buf_fx = sp = round_down(sp - frame_size, 64);
485	if (ia32_frame && use_fxsr()) {
486		frame_size += sizeof(struct fregs_state);
487		sp -= sizeof(struct fregs_state);
488	}
489
490	*size = frame_size;
491
492	return sp;
493}
494
495unsigned long fpu__get_fpstate_size(void)
496{
497	unsigned long ret = xstate_sigframe_size();
498
499	/*
500	 * This space is needed on (most) 32-bit kernels, or when a 32-bit
501	 * app is running on a 64-bit kernel. To keep things simple, just
502	 * assume the worst case and always include space for 'freg_state',
503	 * even for 64-bit apps on 64-bit kernels. This wastes a bit of
504	 * space, but keeps the code simple.
505	 */
506	if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
507	     IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
508		ret += sizeof(struct fregs_state);
509
510	return ret;
511}
512
513/*
514 * Prepare the SW reserved portion of the fxsave memory layout, indicating
515 * the presence of the extended state information in the memory layout
516 * pointed by the fpstate pointer in the sigcontext.
517 * This will be saved when ever the FP and extended state context is
518 * saved on the user stack during the signal handler delivery to the user.
519 */
520void fpu__init_prepare_fx_sw_frame(void)
521{
522	int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE;
523
524	fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
525	fx_sw_reserved.extended_size = size;
526	fx_sw_reserved.xfeatures = xfeatures_mask_uabi();
527	fx_sw_reserved.xstate_size = fpu_user_xstate_size;
528
529	if (IS_ENABLED(CONFIG_IA32_EMULATION) ||
530	    IS_ENABLED(CONFIG_X86_32)) {
531		int fsave_header_size = sizeof(struct fregs_state);
532
533		fx_sw_reserved_ia32 = fx_sw_reserved;
534		fx_sw_reserved_ia32.extended_size = size + fsave_header_size;
535	}
536}
537