Loading...
1/*
2 * xsave/xrstor support.
3 *
4 * Author: Suresh Siddha <suresh.b.siddha@intel.com>
5 */
6#include <linux/compat.h>
7#include <linux/cpu.h>
8#include <linux/mman.h>
9#include <linux/pkeys.h>
10
11#include <asm/fpu/api.h>
12#include <asm/fpu/internal.h>
13#include <asm/fpu/signal.h>
14#include <asm/fpu/regset.h>
15#include <asm/fpu/xstate.h>
16
17#include <asm/tlbflush.h>
18#include <asm/cpufeature.h>
19
20/*
21 * Although we spell it out in here, the Processor Trace
22 * xfeature is completely unused. We use other mechanisms
23 * to save/restore PT state in Linux.
24 */
25static const char *xfeature_names[] =
26{
27 "x87 floating point registers" ,
28 "SSE registers" ,
29 "AVX registers" ,
30 "MPX bounds registers" ,
31 "MPX CSR" ,
32 "AVX-512 opmask" ,
33 "AVX-512 Hi256" ,
34 "AVX-512 ZMM_Hi256" ,
35 "Processor Trace (unused)" ,
36 "Protection Keys User registers",
37 "unknown xstate feature" ,
38};
39
40static short xsave_cpuid_features[] __initdata = {
41 X86_FEATURE_FPU,
42 X86_FEATURE_XMM,
43 X86_FEATURE_AVX,
44 X86_FEATURE_MPX,
45 X86_FEATURE_MPX,
46 X86_FEATURE_AVX512F,
47 X86_FEATURE_AVX512F,
48 X86_FEATURE_AVX512F,
49 X86_FEATURE_INTEL_PT,
50 X86_FEATURE_PKU,
51};
52
53/*
54 * Mask of xstate features supported by the CPU and the kernel:
55 */
56u64 xfeatures_mask __read_mostly;
57
58static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
59static unsigned int xstate_sizes[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
60static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
61
62/*
63 * The XSAVE area of kernel can be in standard or compacted format;
64 * it is always in standard format for user mode. This is the user
65 * mode standard format size used for signal and ptrace frames.
66 */
67unsigned int fpu_user_xstate_size;
68
69/*
70 * Clear all of the X86_FEATURE_* bits that are unavailable
71 * when the CPU has no XSAVE support.
72 */
73void fpu__xstate_clear_all_cpu_caps(void)
74{
75 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
76}
77
78/*
79 * Return whether the system supports a given xfeature.
80 *
81 * Also return the name of the (most advanced) feature that the caller requested:
82 */
83int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
84{
85 u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;
86
87 if (unlikely(feature_name)) {
88 long xfeature_idx, max_idx;
89 u64 xfeatures_print;
90 /*
91 * So we use FLS here to be able to print the most advanced
92 * feature that was requested but is missing. So if a driver
93 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
94 * missing AVX feature - this is the most informative message
95 * to users:
96 */
97 if (xfeatures_missing)
98 xfeatures_print = xfeatures_missing;
99 else
100 xfeatures_print = xfeatures_needed;
101
102 xfeature_idx = fls64(xfeatures_print)-1;
103 max_idx = ARRAY_SIZE(xfeature_names)-1;
104 xfeature_idx = min(xfeature_idx, max_idx);
105
106 *feature_name = xfeature_names[xfeature_idx];
107 }
108
109 if (xfeatures_missing)
110 return 0;
111
112 return 1;
113}
114EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
115
116static int xfeature_is_supervisor(int xfeature_nr)
117{
118 /*
119 * We currently do not support supervisor states, but if
120 * we did, we could find out like this.
121 *
122 * SDM says: If state component 'i' is a user state component,
123 * ECX[0] return 0; if state component i is a supervisor
124 * state component, ECX[0] returns 1.
125 */
126 u32 eax, ebx, ecx, edx;
127
128 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
129 return !!(ecx & 1);
130}
131
132static int xfeature_is_user(int xfeature_nr)
133{
134 return !xfeature_is_supervisor(xfeature_nr);
135}
136
137/*
138 * When executing XSAVEOPT (or other optimized XSAVE instructions), if
139 * a processor implementation detects that an FPU state component is still
140 * (or is again) in its initialized state, it may clear the corresponding
141 * bit in the header.xfeatures field, and can skip the writeout of registers
142 * to the corresponding memory layout.
143 *
144 * This means that when the bit is zero, the state component might still contain
145 * some previous - non-initialized register state.
146 *
147 * Before writing xstate information to user-space we sanitize those components,
148 * to always ensure that the memory layout of a feature will be in the init state
149 * if the corresponding header bit is zero. This is to ensure that user-space doesn't
150 * see some stale state in the memory layout during signal handling, debugging etc.
151 */
152void fpstate_sanitize_xstate(struct fpu *fpu)
153{
154 struct fxregs_state *fx = &fpu->state.fxsave;
155 int feature_bit;
156 u64 xfeatures;
157
158 if (!use_xsaveopt())
159 return;
160
161 xfeatures = fpu->state.xsave.header.xfeatures;
162
163 /*
164 * None of the feature bits are in init state. So nothing else
165 * to do for us, as the memory layout is up to date.
166 */
167 if ((xfeatures & xfeatures_mask) == xfeatures_mask)
168 return;
169
170 /*
171 * FP is in init state
172 */
173 if (!(xfeatures & XFEATURE_MASK_FP)) {
174 fx->cwd = 0x37f;
175 fx->swd = 0;
176 fx->twd = 0;
177 fx->fop = 0;
178 fx->rip = 0;
179 fx->rdp = 0;
180 memset(&fx->st_space[0], 0, 128);
181 }
182
183 /*
184 * SSE is in init state
185 */
186 if (!(xfeatures & XFEATURE_MASK_SSE))
187 memset(&fx->xmm_space[0], 0, 256);
188
189 /*
190 * First two features are FPU and SSE, which above we handled
191 * in a special way already:
192 */
193 feature_bit = 0x2;
194 xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
195
196 /*
197 * Update all the remaining memory layouts according to their
198 * standard xstate layout, if their header bit is in the init
199 * state:
200 */
201 while (xfeatures) {
202 if (xfeatures & 0x1) {
203 int offset = xstate_comp_offsets[feature_bit];
204 int size = xstate_sizes[feature_bit];
205
206 memcpy((void *)fx + offset,
207 (void *)&init_fpstate.xsave + offset,
208 size);
209 }
210
211 xfeatures >>= 1;
212 feature_bit++;
213 }
214}
215
216/*
217 * Enable the extended processor state save/restore feature.
218 * Called once per CPU onlining.
219 */
220void fpu__init_cpu_xstate(void)
221{
222 if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask)
223 return;
224 /*
225 * Make it clear that XSAVES supervisor states are not yet
226 * implemented should anyone expect it to work by changing
227 * bits in XFEATURE_MASK_* macros and XCR0.
228 */
229 WARN_ONCE((xfeatures_mask & XFEATURE_MASK_SUPERVISOR),
230 "x86/fpu: XSAVES supervisor states are not yet implemented.\n");
231
232 xfeatures_mask &= ~XFEATURE_MASK_SUPERVISOR;
233
234 cr4_set_bits(X86_CR4_OSXSAVE);
235 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
236}
237
238/*
239 * Note that in the future we will likely need a pair of
240 * functions here: one for user xstates and the other for
241 * system xstates. For now, they are the same.
242 */
243static int xfeature_enabled(enum xfeature xfeature)
244{
245 return !!(xfeatures_mask & (1UL << xfeature));
246}
247
248/*
249 * Record the offsets and sizes of various xstates contained
250 * in the XSAVE state memory layout.
251 */
252static void __init setup_xstate_features(void)
253{
254 u32 eax, ebx, ecx, edx, i;
255 /* start at the beginnning of the "extended state" */
256 unsigned int last_good_offset = offsetof(struct xregs_state,
257 extended_state_area);
258 /*
259 * The FP xstates and SSE xstates are legacy states. They are always
260 * in the fixed offsets in the xsave area in either compacted form
261 * or standard form.
262 */
263 xstate_offsets[0] = 0;
264 xstate_sizes[0] = offsetof(struct fxregs_state, xmm_space);
265 xstate_offsets[1] = xstate_sizes[0];
266 xstate_sizes[1] = FIELD_SIZEOF(struct fxregs_state, xmm_space);
267
268 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
269 if (!xfeature_enabled(i))
270 continue;
271
272 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
273
274 /*
275 * If an xfeature is supervisor state, the offset
276 * in EBX is invalid. We leave it to -1.
277 */
278 if (xfeature_is_user(i))
279 xstate_offsets[i] = ebx;
280
281 xstate_sizes[i] = eax;
282 /*
283 * In our xstate size checks, we assume that the
284 * highest-numbered xstate feature has the
285 * highest offset in the buffer. Ensure it does.
286 */
287 WARN_ONCE(last_good_offset > xstate_offsets[i],
288 "x86/fpu: misordered xstate at %d\n", last_good_offset);
289 last_good_offset = xstate_offsets[i];
290 }
291}
292
293static void __init print_xstate_feature(u64 xstate_mask)
294{
295 const char *feature_name;
296
297 if (cpu_has_xfeatures(xstate_mask, &feature_name))
298 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
299}
300
301/*
302 * Print out all the supported xstate features:
303 */
304static void __init print_xstate_features(void)
305{
306 print_xstate_feature(XFEATURE_MASK_FP);
307 print_xstate_feature(XFEATURE_MASK_SSE);
308 print_xstate_feature(XFEATURE_MASK_YMM);
309 print_xstate_feature(XFEATURE_MASK_BNDREGS);
310 print_xstate_feature(XFEATURE_MASK_BNDCSR);
311 print_xstate_feature(XFEATURE_MASK_OPMASK);
312 print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
313 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
314 print_xstate_feature(XFEATURE_MASK_PKRU);
315}
316
317/*
318 * This check is important because it is easy to get XSTATE_*
319 * confused with XSTATE_BIT_*.
320 */
321#define CHECK_XFEATURE(nr) do { \
322 WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
323 WARN_ON(nr >= XFEATURE_MAX); \
324} while (0)
325
326/*
327 * We could cache this like xstate_size[], but we only use
328 * it here, so it would be a waste of space.
329 */
330static int xfeature_is_aligned(int xfeature_nr)
331{
332 u32 eax, ebx, ecx, edx;
333
334 CHECK_XFEATURE(xfeature_nr);
335 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
336 /*
337 * The value returned by ECX[1] indicates the alignment
338 * of state component 'i' when the compacted format
339 * of the extended region of an XSAVE area is used:
340 */
341 return !!(ecx & 2);
342}
343
344/*
345 * This function sets up offsets and sizes of all extended states in
346 * xsave area. This supports both standard format and compacted format
347 * of the xsave aread.
348 */
349static void __init setup_xstate_comp(void)
350{
351 unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
352 int i;
353
354 /*
355 * The FP xstates and SSE xstates are legacy states. They are always
356 * in the fixed offsets in the xsave area in either compacted form
357 * or standard form.
358 */
359 xstate_comp_offsets[0] = 0;
360 xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
361
362 if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
363 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
364 if (xfeature_enabled(i)) {
365 xstate_comp_offsets[i] = xstate_offsets[i];
366 xstate_comp_sizes[i] = xstate_sizes[i];
367 }
368 }
369 return;
370 }
371
372 xstate_comp_offsets[FIRST_EXTENDED_XFEATURE] =
373 FXSAVE_SIZE + XSAVE_HDR_SIZE;
374
375 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
376 if (xfeature_enabled(i))
377 xstate_comp_sizes[i] = xstate_sizes[i];
378 else
379 xstate_comp_sizes[i] = 0;
380
381 if (i > FIRST_EXTENDED_XFEATURE) {
382 xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
383 + xstate_comp_sizes[i-1];
384
385 if (xfeature_is_aligned(i))
386 xstate_comp_offsets[i] =
387 ALIGN(xstate_comp_offsets[i], 64);
388 }
389 }
390}
391
392/*
393 * Print out xstate component offsets and sizes
394 */
395static void __init print_xstate_offset_size(void)
396{
397 int i;
398
399 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
400 if (!xfeature_enabled(i))
401 continue;
402 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
403 i, xstate_comp_offsets[i], i, xstate_sizes[i]);
404 }
405}
406
407/*
408 * setup the xstate image representing the init state
409 */
410static void __init setup_init_fpu_buf(void)
411{
412 static int on_boot_cpu __initdata = 1;
413
414 WARN_ON_FPU(!on_boot_cpu);
415 on_boot_cpu = 0;
416
417 if (!boot_cpu_has(X86_FEATURE_XSAVE))
418 return;
419
420 setup_xstate_features();
421 print_xstate_features();
422
423 if (boot_cpu_has(X86_FEATURE_XSAVES))
424 init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
425
426 /*
427 * Init all the features state with header.xfeatures being 0x0
428 */
429 copy_kernel_to_xregs_booting(&init_fpstate.xsave);
430
431 /*
432 * Dump the init state again. This is to identify the init state
433 * of any feature which is not represented by all zero's.
434 */
435 copy_xregs_to_kernel_booting(&init_fpstate.xsave);
436}
437
438static int xfeature_uncompacted_offset(int xfeature_nr)
439{
440 u32 eax, ebx, ecx, edx;
441
442 /*
443 * Only XSAVES supports supervisor states and it uses compacted
444 * format. Checking a supervisor state's uncompacted offset is
445 * an error.
446 */
447 if (XFEATURE_MASK_SUPERVISOR & (1 << xfeature_nr)) {
448 WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
449 return -1;
450 }
451
452 CHECK_XFEATURE(xfeature_nr);
453 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
454 return ebx;
455}
456
457static int xfeature_size(int xfeature_nr)
458{
459 u32 eax, ebx, ecx, edx;
460
461 CHECK_XFEATURE(xfeature_nr);
462 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
463 return eax;
464}
465
466/*
467 * 'XSAVES' implies two different things:
468 * 1. saving of supervisor/system state
469 * 2. using the compacted format
470 *
471 * Use this function when dealing with the compacted format so
472 * that it is obvious which aspect of 'XSAVES' is being handled
473 * by the calling code.
474 */
475int using_compacted_format(void)
476{
477 return boot_cpu_has(X86_FEATURE_XSAVES);
478}
479
480/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
481int validate_xstate_header(const struct xstate_header *hdr)
482{
483 /* No unknown or supervisor features may be set */
484 if (hdr->xfeatures & (~xfeatures_mask | XFEATURE_MASK_SUPERVISOR))
485 return -EINVAL;
486
487 /* Userspace must use the uncompacted format */
488 if (hdr->xcomp_bv)
489 return -EINVAL;
490
491 /*
492 * If 'reserved' is shrunken to add a new field, make sure to validate
493 * that new field here!
494 */
495 BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
496
497 /* No reserved bits may be set */
498 if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
499 return -EINVAL;
500
501 return 0;
502}
503
504static void __xstate_dump_leaves(void)
505{
506 int i;
507 u32 eax, ebx, ecx, edx;
508 static int should_dump = 1;
509
510 if (!should_dump)
511 return;
512 should_dump = 0;
513 /*
514 * Dump out a few leaves past the ones that we support
515 * just in case there are some goodies up there
516 */
517 for (i = 0; i < XFEATURE_MAX + 10; i++) {
518 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
519 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
520 XSTATE_CPUID, i, eax, ebx, ecx, edx);
521 }
522}
523
524#define XSTATE_WARN_ON(x) do { \
525 if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \
526 __xstate_dump_leaves(); \
527 } \
528} while (0)
529
530#define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \
531 if ((nr == nr_macro) && \
532 WARN_ONCE(sz != sizeof(__struct), \
533 "%s: struct is %zu bytes, cpu state %d bytes\n", \
534 __stringify(nr_macro), sizeof(__struct), sz)) { \
535 __xstate_dump_leaves(); \
536 } \
537} while (0)
538
539/*
540 * We have a C struct for each 'xstate'. We need to ensure
541 * that our software representation matches what the CPU
542 * tells us about the state's size.
543 */
544static void check_xstate_against_struct(int nr)
545{
546 /*
547 * Ask the CPU for the size of the state.
548 */
549 int sz = xfeature_size(nr);
550 /*
551 * Match each CPU state with the corresponding software
552 * structure.
553 */
554 XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);
555 XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);
556 XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);
557 XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);
558 XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
559 XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
560 XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
561
562 /*
563 * Make *SURE* to add any feature numbers in below if
564 * there are "holes" in the xsave state component
565 * numbers.
566 */
567 if ((nr < XFEATURE_YMM) ||
568 (nr >= XFEATURE_MAX) ||
569 (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR)) {
570 WARN_ONCE(1, "no structure for xstate: %d\n", nr);
571 XSTATE_WARN_ON(1);
572 }
573}
574
575/*
576 * This essentially double-checks what the cpu told us about
577 * how large the XSAVE buffer needs to be. We are recalculating
578 * it to be safe.
579 */
580static void do_extra_xstate_size_checks(void)
581{
582 int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
583 int i;
584
585 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
586 if (!xfeature_enabled(i))
587 continue;
588
589 check_xstate_against_struct(i);
590 /*
591 * Supervisor state components can be managed only by
592 * XSAVES, which is compacted-format only.
593 */
594 if (!using_compacted_format())
595 XSTATE_WARN_ON(xfeature_is_supervisor(i));
596
597 /* Align from the end of the previous feature */
598 if (xfeature_is_aligned(i))
599 paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
600 /*
601 * The offset of a given state in the non-compacted
602 * format is given to us in a CPUID leaf. We check
603 * them for being ordered (increasing offsets) in
604 * setup_xstate_features().
605 */
606 if (!using_compacted_format())
607 paranoid_xstate_size = xfeature_uncompacted_offset(i);
608 /*
609 * The compacted-format offset always depends on where
610 * the previous state ended.
611 */
612 paranoid_xstate_size += xfeature_size(i);
613 }
614 XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
615}
616
617
618/*
619 * Get total size of enabled xstates in XCR0/xfeatures_mask.
620 *
621 * Note the SDM's wording here. "sub-function 0" only enumerates
622 * the size of the *user* states. If we use it to size a buffer
623 * that we use 'XSAVES' on, we could potentially overflow the
624 * buffer because 'XSAVES' saves system states too.
625 *
626 * Note that we do not currently set any bits on IA32_XSS so
627 * 'XCR0 | IA32_XSS == XCR0' for now.
628 */
629static unsigned int __init get_xsaves_size(void)
630{
631 unsigned int eax, ebx, ecx, edx;
632 /*
633 * - CPUID function 0DH, sub-function 1:
634 * EBX enumerates the size (in bytes) required by
635 * the XSAVES instruction for an XSAVE area
636 * containing all the state components
637 * corresponding to bits currently set in
638 * XCR0 | IA32_XSS.
639 */
640 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
641 return ebx;
642}
643
644static unsigned int __init get_xsave_size(void)
645{
646 unsigned int eax, ebx, ecx, edx;
647 /*
648 * - CPUID function 0DH, sub-function 0:
649 * EBX enumerates the size (in bytes) required by
650 * the XSAVE instruction for an XSAVE area
651 * containing all the *user* state components
652 * corresponding to bits currently set in XCR0.
653 */
654 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
655 return ebx;
656}
657
658/*
659 * Will the runtime-enumerated 'xstate_size' fit in the init
660 * task's statically-allocated buffer?
661 */
662static bool is_supported_xstate_size(unsigned int test_xstate_size)
663{
664 if (test_xstate_size <= sizeof(union fpregs_state))
665 return true;
666
667 pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
668 sizeof(union fpregs_state), test_xstate_size);
669 return false;
670}
671
672static int init_xstate_size(void)
673{
674 /* Recompute the context size for enabled features: */
675 unsigned int possible_xstate_size;
676 unsigned int xsave_size;
677
678 xsave_size = get_xsave_size();
679
680 if (boot_cpu_has(X86_FEATURE_XSAVES))
681 possible_xstate_size = get_xsaves_size();
682 else
683 possible_xstate_size = xsave_size;
684
685 /* Ensure we have the space to store all enabled: */
686 if (!is_supported_xstate_size(possible_xstate_size))
687 return -EINVAL;
688
689 /*
690 * The size is OK, we are definitely going to use xsave,
691 * make it known to the world that we need more space.
692 */
693 fpu_kernel_xstate_size = possible_xstate_size;
694 do_extra_xstate_size_checks();
695
696 /*
697 * User space is always in standard format.
698 */
699 fpu_user_xstate_size = xsave_size;
700 return 0;
701}
702
703/*
704 * We enabled the XSAVE hardware, but something went wrong and
705 * we can not use it. Disable it.
706 */
707static void fpu__init_disable_system_xstate(void)
708{
709 xfeatures_mask = 0;
710 cr4_clear_bits(X86_CR4_OSXSAVE);
711 fpu__xstate_clear_all_cpu_caps();
712}
713
714/*
715 * Enable and initialize the xsave feature.
716 * Called once per system bootup.
717 */
718void __init fpu__init_system_xstate(void)
719{
720 unsigned int eax, ebx, ecx, edx;
721 static int on_boot_cpu __initdata = 1;
722 int err;
723 int i;
724
725 WARN_ON_FPU(!on_boot_cpu);
726 on_boot_cpu = 0;
727
728 if (!boot_cpu_has(X86_FEATURE_FPU)) {
729 pr_info("x86/fpu: No FPU detected\n");
730 return;
731 }
732
733 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
734 pr_info("x86/fpu: x87 FPU will use %s\n",
735 boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
736 return;
737 }
738
739 if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
740 WARN_ON_FPU(1);
741 return;
742 }
743
744 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
745 xfeatures_mask = eax + ((u64)edx << 32);
746
747 if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
748 /*
749 * This indicates that something really unexpected happened
750 * with the enumeration. Disable XSAVE and try to continue
751 * booting without it. This is too early to BUG().
752 */
753 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
754 goto out_disable;
755 }
756
757 /*
758 * Clear XSAVE features that are disabled in the normal CPUID.
759 */
760 for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
761 if (!boot_cpu_has(xsave_cpuid_features[i]))
762 xfeatures_mask &= ~BIT(i);
763 }
764
765 xfeatures_mask &= fpu__get_supported_xfeatures_mask();
766
767 /* Enable xstate instructions to be able to continue with initialization: */
768 fpu__init_cpu_xstate();
769 err = init_xstate_size();
770 if (err)
771 goto out_disable;
772
773 /*
774 * Update info used for ptrace frames; use standard-format size and no
775 * supervisor xstates:
776 */
777 update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR);
778
779 fpu__init_prepare_fx_sw_frame();
780 setup_init_fpu_buf();
781 setup_xstate_comp();
782 print_xstate_offset_size();
783
784 pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
785 xfeatures_mask,
786 fpu_kernel_xstate_size,
787 boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
788 return;
789
790out_disable:
791 /* something went wrong, try to boot without any XSAVE support */
792 fpu__init_disable_system_xstate();
793}
794
795/*
796 * Restore minimal FPU state after suspend:
797 */
798void fpu__resume_cpu(void)
799{
800 /*
801 * Restore XCR0 on xsave capable CPUs:
802 */
803 if (boot_cpu_has(X86_FEATURE_XSAVE))
804 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
805}
806
807/*
808 * Given an xstate feature mask, calculate where in the xsave
809 * buffer the state is. Callers should ensure that the buffer
810 * is valid.
811 *
812 * Note: does not work for compacted buffers.
813 */
814void *__raw_xsave_addr(struct xregs_state *xsave, int xstate_feature_mask)
815{
816 int feature_nr = fls64(xstate_feature_mask) - 1;
817
818 if (!xfeature_enabled(feature_nr)) {
819 WARN_ON_FPU(1);
820 return NULL;
821 }
822
823 return (void *)xsave + xstate_comp_offsets[feature_nr];
824}
825/*
826 * Given the xsave area and a state inside, this function returns the
827 * address of the state.
828 *
829 * This is the API that is called to get xstate address in either
830 * standard format or compacted format of xsave area.
831 *
832 * Note that if there is no data for the field in the xsave buffer
833 * this will return NULL.
834 *
835 * Inputs:
836 * xstate: the thread's storage area for all FPU data
837 * xstate_feature: state which is defined in xsave.h (e.g.
838 * XFEATURE_MASK_FP, XFEATURE_MASK_SSE, etc...)
839 * Output:
840 * address of the state in the xsave area, or NULL if the
841 * field is not present in the xsave buffer.
842 */
843void *get_xsave_addr(struct xregs_state *xsave, int xstate_feature)
844{
845 /*
846 * Do we even *have* xsave state?
847 */
848 if (!boot_cpu_has(X86_FEATURE_XSAVE))
849 return NULL;
850
851 /*
852 * We should not ever be requesting features that we
853 * have not enabled. Remember that pcntxt_mask is
854 * what we write to the XCR0 register.
855 */
856 WARN_ONCE(!(xfeatures_mask & xstate_feature),
857 "get of unsupported state");
858 /*
859 * This assumes the last 'xsave*' instruction to
860 * have requested that 'xstate_feature' be saved.
861 * If it did not, we might be seeing and old value
862 * of the field in the buffer.
863 *
864 * This can happen because the last 'xsave' did not
865 * request that this feature be saved (unlikely)
866 * or because the "init optimization" caused it
867 * to not be saved.
868 */
869 if (!(xsave->header.xfeatures & xstate_feature))
870 return NULL;
871
872 return __raw_xsave_addr(xsave, xstate_feature);
873}
874EXPORT_SYMBOL_GPL(get_xsave_addr);
875
876/*
877 * This wraps up the common operations that need to occur when retrieving
878 * data from xsave state. It first ensures that the current task was
879 * using the FPU and retrieves the data in to a buffer. It then calculates
880 * the offset of the requested field in the buffer.
881 *
882 * This function is safe to call whether the FPU is in use or not.
883 *
884 * Note that this only works on the current task.
885 *
886 * Inputs:
887 * @xsave_state: state which is defined in xsave.h (e.g. XFEATURE_MASK_FP,
888 * XFEATURE_MASK_SSE, etc...)
889 * Output:
890 * address of the state in the xsave area or NULL if the state
891 * is not present or is in its 'init state'.
892 */
893const void *get_xsave_field_ptr(int xsave_state)
894{
895 struct fpu *fpu = ¤t->thread.fpu;
896
897 if (!fpu->initialized)
898 return NULL;
899 /*
900 * fpu__save() takes the CPU's xstate registers
901 * and saves them off to the 'fpu memory buffer.
902 */
903 fpu__save(fpu);
904
905 return get_xsave_addr(&fpu->state.xsave, xsave_state);
906}
907
908#ifdef CONFIG_ARCH_HAS_PKEYS
909
910#define NR_VALID_PKRU_BITS (CONFIG_NR_PROTECTION_KEYS * 2)
911#define PKRU_VALID_MASK (NR_VALID_PKRU_BITS - 1)
912/*
913 * This will go out and modify PKRU register to set the access
914 * rights for @pkey to @init_val.
915 */
916int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
917 unsigned long init_val)
918{
919 u32 old_pkru;
920 int pkey_shift = (pkey * PKRU_BITS_PER_PKEY);
921 u32 new_pkru_bits = 0;
922
923 /*
924 * This check implies XSAVE support. OSPKE only gets
925 * set if we enable XSAVE and we enable PKU in XCR0.
926 */
927 if (!boot_cpu_has(X86_FEATURE_OSPKE))
928 return -EINVAL;
929
930 /* Set the bits we need in PKRU: */
931 if (init_val & PKEY_DISABLE_ACCESS)
932 new_pkru_bits |= PKRU_AD_BIT;
933 if (init_val & PKEY_DISABLE_WRITE)
934 new_pkru_bits |= PKRU_WD_BIT;
935
936 /* Shift the bits in to the correct place in PKRU for pkey: */
937 new_pkru_bits <<= pkey_shift;
938
939 /* Get old PKRU and mask off any old bits in place: */
940 old_pkru = read_pkru();
941 old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
942
943 /* Write old part along with new part: */
944 write_pkru(old_pkru | new_pkru_bits);
945
946 return 0;
947}
948#endif /* ! CONFIG_ARCH_HAS_PKEYS */
949
950/*
951 * Weird legacy quirk: SSE and YMM states store information in the
952 * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP
953 * area is marked as unused in the xfeatures header, we need to copy
954 * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use.
955 */
956static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
957{
958 if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
959 return false;
960
961 if (xfeatures & XFEATURE_MASK_FP)
962 return false;
963
964 return true;
965}
966
967/*
968 * This is similar to user_regset_copyout(), but will not add offset to
969 * the source data pointer or increment pos, count, kbuf, and ubuf.
970 */
971static inline void
972__copy_xstate_to_kernel(void *kbuf, const void *data,
973 unsigned int offset, unsigned int size, unsigned int size_total)
974{
975 if (offset < size_total) {
976 unsigned int copy = min(size, size_total - offset);
977
978 memcpy(kbuf + offset, data, copy);
979 }
980}
981
982/*
983 * Convert from kernel XSAVES compacted format to standard format and copy
984 * to a kernel-space ptrace buffer.
985 *
986 * It supports partial copy but pos always starts from zero. This is called
987 * from xstateregs_get() and there we check the CPU has XSAVES.
988 */
989int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
990{
991 unsigned int offset, size;
992 struct xstate_header header;
993 int i;
994
995 /*
996 * Currently copy_regset_to_user() starts from pos 0:
997 */
998 if (unlikely(offset_start != 0))
999 return -EFAULT;
1000
1001 /*
1002 * The destination is a ptrace buffer; we put in only user xstates:
1003 */
1004 memset(&header, 0, sizeof(header));
1005 header.xfeatures = xsave->header.xfeatures;
1006 header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
1007
1008 /*
1009 * Copy xregs_state->header:
1010 */
1011 offset = offsetof(struct xregs_state, header);
1012 size = sizeof(header);
1013
1014 __copy_xstate_to_kernel(kbuf, &header, offset, size, size_total);
1015
1016 for (i = 0; i < XFEATURE_MAX; i++) {
1017 /*
1018 * Copy only in-use xstates:
1019 */
1020 if ((header.xfeatures >> i) & 1) {
1021 void *src = __raw_xsave_addr(xsave, 1 << i);
1022
1023 offset = xstate_offsets[i];
1024 size = xstate_sizes[i];
1025
1026 /* The next component has to fit fully into the output buffer: */
1027 if (offset + size > size_total)
1028 break;
1029
1030 __copy_xstate_to_kernel(kbuf, src, offset, size, size_total);
1031 }
1032
1033 }
1034
1035 if (xfeatures_mxcsr_quirk(header.xfeatures)) {
1036 offset = offsetof(struct fxregs_state, mxcsr);
1037 size = MXCSR_AND_FLAGS_SIZE;
1038 __copy_xstate_to_kernel(kbuf, &xsave->i387.mxcsr, offset, size, size_total);
1039 }
1040
1041 /*
1042 * Fill xsave->i387.sw_reserved value for ptrace frame:
1043 */
1044 offset = offsetof(struct fxregs_state, sw_reserved);
1045 size = sizeof(xstate_fx_sw_bytes);
1046
1047 __copy_xstate_to_kernel(kbuf, xstate_fx_sw_bytes, offset, size, size_total);
1048
1049 return 0;
1050}
1051
1052static inline int
1053__copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total)
1054{
1055 if (!size)
1056 return 0;
1057
1058 if (offset < size_total) {
1059 unsigned int copy = min(size, size_total - offset);
1060
1061 if (__copy_to_user(ubuf + offset, data, copy))
1062 return -EFAULT;
1063 }
1064 return 0;
1065}
1066
1067/*
1068 * Convert from kernel XSAVES compacted format to standard format and copy
1069 * to a user-space buffer. It supports partial copy but pos always starts from
1070 * zero. This is called from xstateregs_get() and there we check the CPU
1071 * has XSAVES.
1072 */
1073int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1074{
1075 unsigned int offset, size;
1076 int ret, i;
1077 struct xstate_header header;
1078
1079 /*
1080 * Currently copy_regset_to_user() starts from pos 0:
1081 */
1082 if (unlikely(offset_start != 0))
1083 return -EFAULT;
1084
1085 /*
1086 * The destination is a ptrace buffer; we put in only user xstates:
1087 */
1088 memset(&header, 0, sizeof(header));
1089 header.xfeatures = xsave->header.xfeatures;
1090 header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
1091
1092 /*
1093 * Copy xregs_state->header:
1094 */
1095 offset = offsetof(struct xregs_state, header);
1096 size = sizeof(header);
1097
1098 ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total);
1099 if (ret)
1100 return ret;
1101
1102 for (i = 0; i < XFEATURE_MAX; i++) {
1103 /*
1104 * Copy only in-use xstates:
1105 */
1106 if ((header.xfeatures >> i) & 1) {
1107 void *src = __raw_xsave_addr(xsave, 1 << i);
1108
1109 offset = xstate_offsets[i];
1110 size = xstate_sizes[i];
1111
1112 /* The next component has to fit fully into the output buffer: */
1113 if (offset + size > size_total)
1114 break;
1115
1116 ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total);
1117 if (ret)
1118 return ret;
1119 }
1120
1121 }
1122
1123 if (xfeatures_mxcsr_quirk(header.xfeatures)) {
1124 offset = offsetof(struct fxregs_state, mxcsr);
1125 size = MXCSR_AND_FLAGS_SIZE;
1126 __copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total);
1127 }
1128
1129 /*
1130 * Fill xsave->i387.sw_reserved value for ptrace frame:
1131 */
1132 offset = offsetof(struct fxregs_state, sw_reserved);
1133 size = sizeof(xstate_fx_sw_bytes);
1134
1135 ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total);
1136 if (ret)
1137 return ret;
1138
1139 return 0;
1140}
1141
1142/*
1143 * Convert from a ptrace standard-format kernel buffer to kernel XSAVES format
1144 * and copy to the target thread. This is called from xstateregs_set().
1145 */
1146int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1147{
1148 unsigned int offset, size;
1149 int i;
1150 struct xstate_header hdr;
1151
1152 offset = offsetof(struct xregs_state, header);
1153 size = sizeof(hdr);
1154
1155 memcpy(&hdr, kbuf + offset, size);
1156
1157 if (validate_xstate_header(&hdr))
1158 return -EINVAL;
1159
1160 for (i = 0; i < XFEATURE_MAX; i++) {
1161 u64 mask = ((u64)1 << i);
1162
1163 if (hdr.xfeatures & mask) {
1164 void *dst = __raw_xsave_addr(xsave, 1 << i);
1165
1166 offset = xstate_offsets[i];
1167 size = xstate_sizes[i];
1168
1169 memcpy(dst, kbuf + offset, size);
1170 }
1171 }
1172
1173 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1174 offset = offsetof(struct fxregs_state, mxcsr);
1175 size = MXCSR_AND_FLAGS_SIZE;
1176 memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
1177 }
1178
1179 /*
1180 * The state that came in from userspace was user-state only.
1181 * Mask all the user states out of 'xfeatures':
1182 */
1183 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1184
1185 /*
1186 * Add back in the features that came in from userspace:
1187 */
1188 xsave->header.xfeatures |= hdr.xfeatures;
1189
1190 return 0;
1191}
1192
1193/*
1194 * Convert from a ptrace or sigreturn standard-format user-space buffer to
1195 * kernel XSAVES format and copy to the target thread. This is called from
1196 * xstateregs_set(), as well as potentially from the sigreturn() and
1197 * rt_sigreturn() system calls.
1198 */
1199int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1200{
1201 unsigned int offset, size;
1202 int i;
1203 struct xstate_header hdr;
1204
1205 offset = offsetof(struct xregs_state, header);
1206 size = sizeof(hdr);
1207
1208 if (__copy_from_user(&hdr, ubuf + offset, size))
1209 return -EFAULT;
1210
1211 if (validate_xstate_header(&hdr))
1212 return -EINVAL;
1213
1214 for (i = 0; i < XFEATURE_MAX; i++) {
1215 u64 mask = ((u64)1 << i);
1216
1217 if (hdr.xfeatures & mask) {
1218 void *dst = __raw_xsave_addr(xsave, 1 << i);
1219
1220 offset = xstate_offsets[i];
1221 size = xstate_sizes[i];
1222
1223 if (__copy_from_user(dst, ubuf + offset, size))
1224 return -EFAULT;
1225 }
1226 }
1227
1228 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1229 offset = offsetof(struct fxregs_state, mxcsr);
1230 size = MXCSR_AND_FLAGS_SIZE;
1231 if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
1232 return -EFAULT;
1233 }
1234
1235 /*
1236 * The state that came in from userspace was user-state only.
1237 * Mask all the user states out of 'xfeatures':
1238 */
1239 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1240
1241 /*
1242 * Add back in the features that came in from userspace:
1243 */
1244 xsave->header.xfeatures |= hdr.xfeatures;
1245
1246 return 0;
1247}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * xsave/xrstor support.
4 *
5 * Author: Suresh Siddha <suresh.b.siddha@intel.com>
6 */
7#include <linux/bitops.h>
8#include <linux/compat.h>
9#include <linux/cpu.h>
10#include <linux/mman.h>
11#include <linux/nospec.h>
12#include <linux/pkeys.h>
13#include <linux/seq_file.h>
14#include <linux/proc_fs.h>
15#include <linux/vmalloc.h>
16
17#include <asm/fpu/api.h>
18#include <asm/fpu/regset.h>
19#include <asm/fpu/signal.h>
20#include <asm/fpu/xcr.h>
21
22#include <asm/tlbflush.h>
23#include <asm/prctl.h>
24#include <asm/elf.h>
25
26#include "context.h"
27#include "internal.h"
28#include "legacy.h"
29#include "xstate.h"
30
31#define for_each_extended_xfeature(bit, mask) \
32 (bit) = FIRST_EXTENDED_XFEATURE; \
33 for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask))
34
35/*
36 * Although we spell it out in here, the Processor Trace
37 * xfeature is completely unused. We use other mechanisms
38 * to save/restore PT state in Linux.
39 */
40static const char *xfeature_names[] =
41{
42 "x87 floating point registers",
43 "SSE registers",
44 "AVX registers",
45 "MPX bounds registers",
46 "MPX CSR",
47 "AVX-512 opmask",
48 "AVX-512 Hi256",
49 "AVX-512 ZMM_Hi256",
50 "Processor Trace (unused)",
51 "Protection Keys User registers",
52 "PASID state",
53 "Control-flow User registers",
54 "Control-flow Kernel registers (unused)",
55 "unknown xstate feature",
56 "unknown xstate feature",
57 "unknown xstate feature",
58 "unknown xstate feature",
59 "AMX Tile config",
60 "AMX Tile data",
61 "unknown xstate feature",
62};
63
64static unsigned short xsave_cpuid_features[] __initdata = {
65 [XFEATURE_FP] = X86_FEATURE_FPU,
66 [XFEATURE_SSE] = X86_FEATURE_XMM,
67 [XFEATURE_YMM] = X86_FEATURE_AVX,
68 [XFEATURE_BNDREGS] = X86_FEATURE_MPX,
69 [XFEATURE_BNDCSR] = X86_FEATURE_MPX,
70 [XFEATURE_OPMASK] = X86_FEATURE_AVX512F,
71 [XFEATURE_ZMM_Hi256] = X86_FEATURE_AVX512F,
72 [XFEATURE_Hi16_ZMM] = X86_FEATURE_AVX512F,
73 [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT,
74 [XFEATURE_PKRU] = X86_FEATURE_OSPKE,
75 [XFEATURE_PASID] = X86_FEATURE_ENQCMD,
76 [XFEATURE_CET_USER] = X86_FEATURE_SHSTK,
77 [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE,
78 [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE,
79};
80
81static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =
82 { [ 0 ... XFEATURE_MAX - 1] = -1};
83static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =
84 { [ 0 ... XFEATURE_MAX - 1] = -1};
85static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init;
86
87#define XSTATE_FLAG_SUPERVISOR BIT(0)
88#define XSTATE_FLAG_ALIGNED64 BIT(1)
89
90/*
91 * Return whether the system supports a given xfeature.
92 *
93 * Also return the name of the (most advanced) feature that the caller requested:
94 */
95int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
96{
97 u64 xfeatures_missing = xfeatures_needed & ~fpu_kernel_cfg.max_features;
98
99 if (unlikely(feature_name)) {
100 long xfeature_idx, max_idx;
101 u64 xfeatures_print;
102 /*
103 * So we use FLS here to be able to print the most advanced
104 * feature that was requested but is missing. So if a driver
105 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
106 * missing AVX feature - this is the most informative message
107 * to users:
108 */
109 if (xfeatures_missing)
110 xfeatures_print = xfeatures_missing;
111 else
112 xfeatures_print = xfeatures_needed;
113
114 xfeature_idx = fls64(xfeatures_print)-1;
115 max_idx = ARRAY_SIZE(xfeature_names)-1;
116 xfeature_idx = min(xfeature_idx, max_idx);
117
118 *feature_name = xfeature_names[xfeature_idx];
119 }
120
121 if (xfeatures_missing)
122 return 0;
123
124 return 1;
125}
126EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
127
128static bool xfeature_is_aligned64(int xfeature_nr)
129{
130 return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64;
131}
132
133static bool xfeature_is_supervisor(int xfeature_nr)
134{
135 return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR;
136}
137
138static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature)
139{
140 unsigned int offs, i;
141
142 /*
143 * Non-compacted format and legacy features use the cached fixed
144 * offsets.
145 */
146 if (!cpu_feature_enabled(X86_FEATURE_XCOMPACTED) ||
147 xfeature <= XFEATURE_SSE)
148 return xstate_offsets[xfeature];
149
150 /*
151 * Compacted format offsets depend on the actual content of the
152 * compacted xsave area which is determined by the xcomp_bv header
153 * field.
154 */
155 offs = FXSAVE_SIZE + XSAVE_HDR_SIZE;
156 for_each_extended_xfeature(i, xcomp_bv) {
157 if (xfeature_is_aligned64(i))
158 offs = ALIGN(offs, 64);
159 if (i == xfeature)
160 break;
161 offs += xstate_sizes[i];
162 }
163 return offs;
164}
165
166/*
167 * Enable the extended processor state save/restore feature.
168 * Called once per CPU onlining.
169 */
170void fpu__init_cpu_xstate(void)
171{
172 if (!boot_cpu_has(X86_FEATURE_XSAVE) || !fpu_kernel_cfg.max_features)
173 return;
174
175 cr4_set_bits(X86_CR4_OSXSAVE);
176
177 /*
178 * Must happen after CR4 setup and before xsetbv() to allow KVM
179 * lazy passthrough. Write independent of the dynamic state static
180 * key as that does not work on the boot CPU. This also ensures
181 * that any stale state is wiped out from XFD.
182 */
183 if (cpu_feature_enabled(X86_FEATURE_XFD))
184 wrmsrl(MSR_IA32_XFD, init_fpstate.xfd);
185
186 /*
187 * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
188 * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user
189 * states can be set here.
190 */
191 xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features);
192
193 /*
194 * MSR_IA32_XSS sets supervisor states managed by XSAVES.
195 */
196 if (boot_cpu_has(X86_FEATURE_XSAVES)) {
197 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
198 xfeatures_mask_independent());
199 }
200}
201
202static bool xfeature_enabled(enum xfeature xfeature)
203{
204 return fpu_kernel_cfg.max_features & BIT_ULL(xfeature);
205}
206
207/*
208 * Record the offsets and sizes of various xstates contained
209 * in the XSAVE state memory layout.
210 */
211static void __init setup_xstate_cache(void)
212{
213 u32 eax, ebx, ecx, edx, i;
214 /* start at the beginning of the "extended state" */
215 unsigned int last_good_offset = offsetof(struct xregs_state,
216 extended_state_area);
217 /*
218 * The FP xstates and SSE xstates are legacy states. They are always
219 * in the fixed offsets in the xsave area in either compacted form
220 * or standard form.
221 */
222 xstate_offsets[XFEATURE_FP] = 0;
223 xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state,
224 xmm_space);
225
226 xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP];
227 xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state,
228 xmm_space);
229
230 for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
231 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
232
233 xstate_sizes[i] = eax;
234 xstate_flags[i] = ecx;
235
236 /*
237 * If an xfeature is supervisor state, the offset in EBX is
238 * invalid, leave it to -1.
239 */
240 if (xfeature_is_supervisor(i))
241 continue;
242
243 xstate_offsets[i] = ebx;
244
245 /*
246 * In our xstate size checks, we assume that the highest-numbered
247 * xstate feature has the highest offset in the buffer. Ensure
248 * it does.
249 */
250 WARN_ONCE(last_good_offset > xstate_offsets[i],
251 "x86/fpu: misordered xstate at %d\n", last_good_offset);
252
253 last_good_offset = xstate_offsets[i];
254 }
255}
256
257static void __init print_xstate_feature(u64 xstate_mask)
258{
259 const char *feature_name;
260
261 if (cpu_has_xfeatures(xstate_mask, &feature_name))
262 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
263}
264
265/*
266 * Print out all the supported xstate features:
267 */
268static void __init print_xstate_features(void)
269{
270 print_xstate_feature(XFEATURE_MASK_FP);
271 print_xstate_feature(XFEATURE_MASK_SSE);
272 print_xstate_feature(XFEATURE_MASK_YMM);
273 print_xstate_feature(XFEATURE_MASK_BNDREGS);
274 print_xstate_feature(XFEATURE_MASK_BNDCSR);
275 print_xstate_feature(XFEATURE_MASK_OPMASK);
276 print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
277 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
278 print_xstate_feature(XFEATURE_MASK_PKRU);
279 print_xstate_feature(XFEATURE_MASK_PASID);
280 print_xstate_feature(XFEATURE_MASK_CET_USER);
281 print_xstate_feature(XFEATURE_MASK_XTILE_CFG);
282 print_xstate_feature(XFEATURE_MASK_XTILE_DATA);
283}
284
285/*
286 * This check is important because it is easy to get XSTATE_*
287 * confused with XSTATE_BIT_*.
288 */
289#define CHECK_XFEATURE(nr) do { \
290 WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
291 WARN_ON(nr >= XFEATURE_MAX); \
292} while (0)
293
294/*
295 * Print out xstate component offsets and sizes
296 */
297static void __init print_xstate_offset_size(void)
298{
299 int i;
300
301 for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
302 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
303 i, xfeature_get_offset(fpu_kernel_cfg.max_features, i),
304 i, xstate_sizes[i]);
305 }
306}
307
308/*
309 * This function is called only during boot time when x86 caps are not set
310 * up and alternative can not be used yet.
311 */
312static __init void os_xrstor_booting(struct xregs_state *xstate)
313{
314 u64 mask = fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSTATE;
315 u32 lmask = mask;
316 u32 hmask = mask >> 32;
317 int err;
318
319 if (cpu_feature_enabled(X86_FEATURE_XSAVES))
320 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
321 else
322 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
323
324 /*
325 * We should never fault when copying from a kernel buffer, and the FPU
326 * state we set at boot time should be valid.
327 */
328 WARN_ON_FPU(err);
329}
330
331/*
332 * All supported features have either init state all zeros or are
333 * handled in setup_init_fpu() individually. This is an explicit
334 * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
335 * newly added supported features at build time and make people
336 * actually look at the init state for the new feature.
337 */
338#define XFEATURES_INIT_FPSTATE_HANDLED \
339 (XFEATURE_MASK_FP | \
340 XFEATURE_MASK_SSE | \
341 XFEATURE_MASK_YMM | \
342 XFEATURE_MASK_OPMASK | \
343 XFEATURE_MASK_ZMM_Hi256 | \
344 XFEATURE_MASK_Hi16_ZMM | \
345 XFEATURE_MASK_PKRU | \
346 XFEATURE_MASK_BNDREGS | \
347 XFEATURE_MASK_BNDCSR | \
348 XFEATURE_MASK_PASID | \
349 XFEATURE_MASK_CET_USER | \
350 XFEATURE_MASK_XTILE)
351
352/*
353 * setup the xstate image representing the init state
354 */
355static void __init setup_init_fpu_buf(void)
356{
357 BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
358 XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
359 XFEATURES_INIT_FPSTATE_HANDLED);
360
361 if (!boot_cpu_has(X86_FEATURE_XSAVE))
362 return;
363
364 print_xstate_features();
365
366 xstate_init_xcomp_bv(&init_fpstate.regs.xsave, init_fpstate.xfeatures);
367
368 /*
369 * Init all the features state with header.xfeatures being 0x0
370 */
371 os_xrstor_booting(&init_fpstate.regs.xsave);
372
373 /*
374 * All components are now in init state. Read the state back so
375 * that init_fpstate contains all non-zero init state. This only
376 * works with XSAVE, but not with XSAVEOPT and XSAVEC/S because
377 * those use the init optimization which skips writing data for
378 * components in init state.
379 *
380 * XSAVE could be used, but that would require to reshuffle the
381 * data when XSAVEC/S is available because XSAVEC/S uses xstate
382 * compaction. But doing so is a pointless exercise because most
383 * components have an all zeros init state except for the legacy
384 * ones (FP and SSE). Those can be saved with FXSAVE into the
385 * legacy area. Adding new features requires to ensure that init
386 * state is all zeroes or if not to add the necessary handling
387 * here.
388 */
389 fxsave(&init_fpstate.regs.fxsave);
390}
391
392int xfeature_size(int xfeature_nr)
393{
394 u32 eax, ebx, ecx, edx;
395
396 CHECK_XFEATURE(xfeature_nr);
397 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
398 return eax;
399}
400
401/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
402static int validate_user_xstate_header(const struct xstate_header *hdr,
403 struct fpstate *fpstate)
404{
405 /* No unknown or supervisor features may be set */
406 if (hdr->xfeatures & ~fpstate->user_xfeatures)
407 return -EINVAL;
408
409 /* Userspace must use the uncompacted format */
410 if (hdr->xcomp_bv)
411 return -EINVAL;
412
413 /*
414 * If 'reserved' is shrunken to add a new field, make sure to validate
415 * that new field here!
416 */
417 BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
418
419 /* No reserved bits may be set */
420 if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
421 return -EINVAL;
422
423 return 0;
424}
425
426static void __init __xstate_dump_leaves(void)
427{
428 int i;
429 u32 eax, ebx, ecx, edx;
430 static int should_dump = 1;
431
432 if (!should_dump)
433 return;
434 should_dump = 0;
435 /*
436 * Dump out a few leaves past the ones that we support
437 * just in case there are some goodies up there
438 */
439 for (i = 0; i < XFEATURE_MAX + 10; i++) {
440 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
441 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
442 XSTATE_CPUID, i, eax, ebx, ecx, edx);
443 }
444}
445
446#define XSTATE_WARN_ON(x, fmt, ...) do { \
447 if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) { \
448 __xstate_dump_leaves(); \
449 } \
450} while (0)
451
452#define XCHECK_SZ(sz, nr, __struct) ({ \
453 if (WARN_ONCE(sz != sizeof(__struct), \
454 "[%s]: struct is %zu bytes, cpu state %d bytes\n", \
455 xfeature_names[nr], sizeof(__struct), sz)) { \
456 __xstate_dump_leaves(); \
457 } \
458 true; \
459})
460
461
462/**
463 * check_xtile_data_against_struct - Check tile data state size.
464 *
465 * Calculate the state size by multiplying the single tile size which is
466 * recorded in a C struct, and the number of tiles that the CPU informs.
467 * Compare the provided size with the calculation.
468 *
469 * @size: The tile data state size
470 *
471 * Returns: 0 on success, -EINVAL on mismatch.
472 */
473static int __init check_xtile_data_against_struct(int size)
474{
475 u32 max_palid, palid, state_size;
476 u32 eax, ebx, ecx, edx;
477 u16 max_tile;
478
479 /*
480 * Check the maximum palette id:
481 * eax: the highest numbered palette subleaf.
482 */
483 cpuid_count(TILE_CPUID, 0, &max_palid, &ebx, &ecx, &edx);
484
485 /*
486 * Cross-check each tile size and find the maximum number of
487 * supported tiles.
488 */
489 for (palid = 1, max_tile = 0; palid <= max_palid; palid++) {
490 u16 tile_size, max;
491
492 /*
493 * Check the tile size info:
494 * eax[31:16]: bytes per title
495 * ebx[31:16]: the max names (or max number of tiles)
496 */
497 cpuid_count(TILE_CPUID, palid, &eax, &ebx, &edx, &edx);
498 tile_size = eax >> 16;
499 max = ebx >> 16;
500
501 if (tile_size != sizeof(struct xtile_data)) {
502 pr_err("%s: struct is %zu bytes, cpu xtile %d bytes\n",
503 __stringify(XFEATURE_XTILE_DATA),
504 sizeof(struct xtile_data), tile_size);
505 __xstate_dump_leaves();
506 return -EINVAL;
507 }
508
509 if (max > max_tile)
510 max_tile = max;
511 }
512
513 state_size = sizeof(struct xtile_data) * max_tile;
514 if (size != state_size) {
515 pr_err("%s: calculated size is %u bytes, cpu state %d bytes\n",
516 __stringify(XFEATURE_XTILE_DATA), state_size, size);
517 __xstate_dump_leaves();
518 return -EINVAL;
519 }
520 return 0;
521}
522
523/*
524 * We have a C struct for each 'xstate'. We need to ensure
525 * that our software representation matches what the CPU
526 * tells us about the state's size.
527 */
528static bool __init check_xstate_against_struct(int nr)
529{
530 /*
531 * Ask the CPU for the size of the state.
532 */
533 int sz = xfeature_size(nr);
534
535 /*
536 * Match each CPU state with the corresponding software
537 * structure.
538 */
539 switch (nr) {
540 case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct);
541 case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state);
542 case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state);
543 case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state);
544 case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state);
545 case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state);
546 case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state);
547 case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state);
548 case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg);
549 case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state);
550 case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true;
551 default:
552 XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
553 return false;
554 }
555
556 return true;
557}
558
559static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted)
560{
561 unsigned int topmost = fls64(xfeatures) - 1;
562 unsigned int offset = xstate_offsets[topmost];
563
564 if (topmost <= XFEATURE_SSE)
565 return sizeof(struct xregs_state);
566
567 if (compacted)
568 offset = xfeature_get_offset(xfeatures, topmost);
569 return offset + xstate_sizes[topmost];
570}
571
572/*
573 * This essentially double-checks what the cpu told us about
574 * how large the XSAVE buffer needs to be. We are recalculating
575 * it to be safe.
576 *
577 * Independent XSAVE features allocate their own buffers and are not
578 * covered by these checks. Only the size of the buffer for task->fpu
579 * is checked here.
580 */
581static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)
582{
583 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
584 bool xsaves = cpu_feature_enabled(X86_FEATURE_XSAVES);
585 unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
586 int i;
587
588 for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
589 if (!check_xstate_against_struct(i))
590 return false;
591 /*
592 * Supervisor state components can be managed only by
593 * XSAVES.
594 */
595 if (!xsaves && xfeature_is_supervisor(i)) {
596 XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i);
597 return false;
598 }
599 }
600 size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted);
601 XSTATE_WARN_ON(size != kernel_size,
602 "size %u != kernel_size %u\n", size, kernel_size);
603 return size == kernel_size;
604}
605
606/*
607 * Get total size of enabled xstates in XCR0 | IA32_XSS.
608 *
609 * Note the SDM's wording here. "sub-function 0" only enumerates
610 * the size of the *user* states. If we use it to size a buffer
611 * that we use 'XSAVES' on, we could potentially overflow the
612 * buffer because 'XSAVES' saves system states too.
613 *
614 * This also takes compaction into account. So this works for
615 * XSAVEC as well.
616 */
617static unsigned int __init get_compacted_size(void)
618{
619 unsigned int eax, ebx, ecx, edx;
620 /*
621 * - CPUID function 0DH, sub-function 1:
622 * EBX enumerates the size (in bytes) required by
623 * the XSAVES instruction for an XSAVE area
624 * containing all the state components
625 * corresponding to bits currently set in
626 * XCR0 | IA32_XSS.
627 *
628 * When XSAVES is not available but XSAVEC is (virt), then there
629 * are no supervisor states, but XSAVEC still uses compacted
630 * format.
631 */
632 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
633 return ebx;
634}
635
636/*
637 * Get the total size of the enabled xstates without the independent supervisor
638 * features.
639 */
640static unsigned int __init get_xsave_compacted_size(void)
641{
642 u64 mask = xfeatures_mask_independent();
643 unsigned int size;
644
645 if (!mask)
646 return get_compacted_size();
647
648 /* Disable independent features. */
649 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
650
651 /*
652 * Ask the hardware what size is required of the buffer.
653 * This is the size required for the task->fpu buffer.
654 */
655 size = get_compacted_size();
656
657 /* Re-enable independent features so XSAVES will work on them again. */
658 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
659
660 return size;
661}
662
663static unsigned int __init get_xsave_size_user(void)
664{
665 unsigned int eax, ebx, ecx, edx;
666 /*
667 * - CPUID function 0DH, sub-function 0:
668 * EBX enumerates the size (in bytes) required by
669 * the XSAVE instruction for an XSAVE area
670 * containing all the *user* state components
671 * corresponding to bits currently set in XCR0.
672 */
673 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
674 return ebx;
675}
676
677static int __init init_xstate_size(void)
678{
679 /* Recompute the context size for enabled features: */
680 unsigned int user_size, kernel_size, kernel_default_size;
681 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
682
683 /* Uncompacted user space size */
684 user_size = get_xsave_size_user();
685
686 /*
687 * XSAVES kernel size includes supervisor states and uses compacted
688 * format. XSAVEC uses compacted format, but does not save
689 * supervisor states.
690 *
691 * XSAVE[OPT] do not support supervisor states so kernel and user
692 * size is identical.
693 */
694 if (compacted)
695 kernel_size = get_xsave_compacted_size();
696 else
697 kernel_size = user_size;
698
699 kernel_default_size =
700 xstate_calculate_size(fpu_kernel_cfg.default_features, compacted);
701
702 if (!paranoid_xstate_size_valid(kernel_size))
703 return -EINVAL;
704
705 fpu_kernel_cfg.max_size = kernel_size;
706 fpu_user_cfg.max_size = user_size;
707
708 fpu_kernel_cfg.default_size = kernel_default_size;
709 fpu_user_cfg.default_size =
710 xstate_calculate_size(fpu_user_cfg.default_features, false);
711
712 return 0;
713}
714
715/*
716 * We enabled the XSAVE hardware, but something went wrong and
717 * we can not use it. Disable it.
718 */
719static void __init fpu__init_disable_system_xstate(unsigned int legacy_size)
720{
721 fpu_kernel_cfg.max_features = 0;
722 cr4_clear_bits(X86_CR4_OSXSAVE);
723 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
724
725 /* Restore the legacy size.*/
726 fpu_kernel_cfg.max_size = legacy_size;
727 fpu_kernel_cfg.default_size = legacy_size;
728 fpu_user_cfg.max_size = legacy_size;
729 fpu_user_cfg.default_size = legacy_size;
730
731 /*
732 * Prevent enabling the static branch which enables writes to the
733 * XFD MSR.
734 */
735 init_fpstate.xfd = 0;
736
737 fpstate_reset(¤t->thread.fpu);
738}
739
740/*
741 * Enable and initialize the xsave feature.
742 * Called once per system bootup.
743 */
744void __init fpu__init_system_xstate(unsigned int legacy_size)
745{
746 unsigned int eax, ebx, ecx, edx;
747 u64 xfeatures;
748 int err;
749 int i;
750
751 if (!boot_cpu_has(X86_FEATURE_FPU)) {
752 pr_info("x86/fpu: No FPU detected\n");
753 return;
754 }
755
756 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
757 pr_info("x86/fpu: x87 FPU will use %s\n",
758 boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
759 return;
760 }
761
762 if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
763 WARN_ON_FPU(1);
764 return;
765 }
766
767 /*
768 * Find user xstates supported by the processor.
769 */
770 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
771 fpu_kernel_cfg.max_features = eax + ((u64)edx << 32);
772
773 /*
774 * Find supervisor xstates supported by the processor.
775 */
776 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
777 fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32);
778
779 if ((fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
780 /*
781 * This indicates that something really unexpected happened
782 * with the enumeration. Disable XSAVE and try to continue
783 * booting without it. This is too early to BUG().
784 */
785 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
786 fpu_kernel_cfg.max_features);
787 goto out_disable;
788 }
789
790 /*
791 * Clear XSAVE features that are disabled in the normal CPUID.
792 */
793 for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
794 unsigned short cid = xsave_cpuid_features[i];
795
796 /* Careful: X86_FEATURE_FPU is 0! */
797 if ((i != XFEATURE_FP && !cid) || !boot_cpu_has(cid))
798 fpu_kernel_cfg.max_features &= ~BIT_ULL(i);
799 }
800
801 if (!cpu_feature_enabled(X86_FEATURE_XFD))
802 fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC;
803
804 if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
805 fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED;
806 else
807 fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED |
808 XFEATURE_MASK_SUPERVISOR_SUPPORTED;
809
810 fpu_user_cfg.max_features = fpu_kernel_cfg.max_features;
811 fpu_user_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED;
812
813 /* Clean out dynamic features from default */
814 fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features;
815 fpu_kernel_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC;
816
817 fpu_user_cfg.default_features = fpu_user_cfg.max_features;
818 fpu_user_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC;
819
820 /* Store it for paranoia check at the end */
821 xfeatures = fpu_kernel_cfg.max_features;
822
823 /*
824 * Initialize the default XFD state in initfp_state and enable the
825 * dynamic sizing mechanism if dynamic states are available. The
826 * static key cannot be enabled here because this runs before
827 * jump_label_init(). This is delayed to an initcall.
828 */
829 init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC;
830
831 /* Set up compaction feature bit */
832 if (cpu_feature_enabled(X86_FEATURE_XSAVEC) ||
833 cpu_feature_enabled(X86_FEATURE_XSAVES))
834 setup_force_cpu_cap(X86_FEATURE_XCOMPACTED);
835
836 /* Enable xstate instructions to be able to continue with initialization: */
837 fpu__init_cpu_xstate();
838
839 /* Cache size, offset and flags for initialization */
840 setup_xstate_cache();
841
842 err = init_xstate_size();
843 if (err)
844 goto out_disable;
845
846 /* Reset the state for the current task */
847 fpstate_reset(¤t->thread.fpu);
848
849 /*
850 * Update info used for ptrace frames; use standard-format size and no
851 * supervisor xstates:
852 */
853 update_regset_xstate_info(fpu_user_cfg.max_size,
854 fpu_user_cfg.max_features);
855
856 /*
857 * init_fpstate excludes dynamic states as they are large but init
858 * state is zero.
859 */
860 init_fpstate.size = fpu_kernel_cfg.default_size;
861 init_fpstate.xfeatures = fpu_kernel_cfg.default_features;
862
863 if (init_fpstate.size > sizeof(init_fpstate.regs)) {
864 pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d), disabling XSAVE\n",
865 sizeof(init_fpstate.regs), init_fpstate.size);
866 goto out_disable;
867 }
868
869 setup_init_fpu_buf();
870
871 /*
872 * Paranoia check whether something in the setup modified the
873 * xfeatures mask.
874 */
875 if (xfeatures != fpu_kernel_cfg.max_features) {
876 pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n",
877 xfeatures, fpu_kernel_cfg.max_features);
878 goto out_disable;
879 }
880
881 /*
882 * CPU capabilities initialization runs before FPU init. So
883 * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely
884 * functional, set the feature bit so depending code works.
885 */
886 setup_force_cpu_cap(X86_FEATURE_OSXSAVE);
887
888 print_xstate_offset_size();
889 pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
890 fpu_kernel_cfg.max_features,
891 fpu_kernel_cfg.max_size,
892 boot_cpu_has(X86_FEATURE_XCOMPACTED) ? "compacted" : "standard");
893 return;
894
895out_disable:
896 /* something went wrong, try to boot without any XSAVE support */
897 fpu__init_disable_system_xstate(legacy_size);
898}
899
900/*
901 * Restore minimal FPU state after suspend:
902 */
903void fpu__resume_cpu(void)
904{
905 /*
906 * Restore XCR0 on xsave capable CPUs:
907 */
908 if (cpu_feature_enabled(X86_FEATURE_XSAVE))
909 xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features);
910
911 /*
912 * Restore IA32_XSS. The same CPUID bit enumerates support
913 * of XSAVES and MSR_IA32_XSS.
914 */
915 if (cpu_feature_enabled(X86_FEATURE_XSAVES)) {
916 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
917 xfeatures_mask_independent());
918 }
919
920 if (fpu_state_size_dynamic())
921 wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd);
922}
923
924/*
925 * Given an xstate feature nr, calculate where in the xsave
926 * buffer the state is. Callers should ensure that the buffer
927 * is valid.
928 */
929static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
930{
931 u64 xcomp_bv = xsave->header.xcomp_bv;
932
933 if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))
934 return NULL;
935
936 if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) {
937 if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr))))
938 return NULL;
939 }
940
941 return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr);
942}
943
944/*
945 * Given the xsave area and a state inside, this function returns the
946 * address of the state.
947 *
948 * This is the API that is called to get xstate address in either
949 * standard format or compacted format of xsave area.
950 *
951 * Note that if there is no data for the field in the xsave buffer
952 * this will return NULL.
953 *
954 * Inputs:
955 * xstate: the thread's storage area for all FPU data
956 * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
957 * XFEATURE_SSE, etc...)
958 * Output:
959 * address of the state in the xsave area, or NULL if the
960 * field is not present in the xsave buffer.
961 */
962void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
963{
964 /*
965 * Do we even *have* xsave state?
966 */
967 if (!boot_cpu_has(X86_FEATURE_XSAVE))
968 return NULL;
969
970 /*
971 * We should not ever be requesting features that we
972 * have not enabled.
973 */
974 if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))
975 return NULL;
976
977 /*
978 * This assumes the last 'xsave*' instruction to
979 * have requested that 'xfeature_nr' be saved.
980 * If it did not, we might be seeing and old value
981 * of the field in the buffer.
982 *
983 * This can happen because the last 'xsave' did not
984 * request that this feature be saved (unlikely)
985 * or because the "init optimization" caused it
986 * to not be saved.
987 */
988 if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
989 return NULL;
990
991 return __raw_xsave_addr(xsave, xfeature_nr);
992}
993
994#ifdef CONFIG_ARCH_HAS_PKEYS
995
996/*
997 * This will go out and modify PKRU register to set the access
998 * rights for @pkey to @init_val.
999 */
1000int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
1001 unsigned long init_val)
1002{
1003 u32 old_pkru, new_pkru_bits = 0;
1004 int pkey_shift;
1005
1006 /*
1007 * This check implies XSAVE support. OSPKE only gets
1008 * set if we enable XSAVE and we enable PKU in XCR0.
1009 */
1010 if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
1011 return -EINVAL;
1012
1013 /*
1014 * This code should only be called with valid 'pkey'
1015 * values originating from in-kernel users. Complain
1016 * if a bad value is observed.
1017 */
1018 if (WARN_ON_ONCE(pkey >= arch_max_pkey()))
1019 return -EINVAL;
1020
1021 /* Set the bits we need in PKRU: */
1022 if (init_val & PKEY_DISABLE_ACCESS)
1023 new_pkru_bits |= PKRU_AD_BIT;
1024 if (init_val & PKEY_DISABLE_WRITE)
1025 new_pkru_bits |= PKRU_WD_BIT;
1026
1027 /* Shift the bits in to the correct place in PKRU for pkey: */
1028 pkey_shift = pkey * PKRU_BITS_PER_PKEY;
1029 new_pkru_bits <<= pkey_shift;
1030
1031 /* Get old PKRU and mask off any old bits in place: */
1032 old_pkru = read_pkru();
1033 old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
1034
1035 /* Write old part along with new part: */
1036 write_pkru(old_pkru | new_pkru_bits);
1037
1038 return 0;
1039}
1040#endif /* ! CONFIG_ARCH_HAS_PKEYS */
1041
1042static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
1043 void *init_xstate, unsigned int size)
1044{
1045 membuf_write(to, from_xstate ? xstate : init_xstate, size);
1046}
1047
1048/**
1049 * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
1050 * @to: membuf descriptor
1051 * @fpstate: The fpstate buffer from which to copy
1052 * @xfeatures: The mask of xfeatures to save (XSAVE mode only)
1053 * @pkru_val: The PKRU value to store in the PKRU component
1054 * @copy_mode: The requested copy mode
1055 *
1056 * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
1057 * format, i.e. from the kernel internal hardware dependent storage format
1058 * to the requested @mode. UABI XSTATE is always uncompacted!
1059 *
1060 * It supports partial copy but @to.pos always starts from zero.
1061 */
1062void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate,
1063 u64 xfeatures, u32 pkru_val,
1064 enum xstate_copy_mode copy_mode)
1065{
1066 const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
1067 struct xregs_state *xinit = &init_fpstate.regs.xsave;
1068 struct xregs_state *xsave = &fpstate->regs.xsave;
1069 struct xstate_header header;
1070 unsigned int zerofrom;
1071 u64 mask;
1072 int i;
1073
1074 memset(&header, 0, sizeof(header));
1075 header.xfeatures = xsave->header.xfeatures;
1076
1077 /* Mask out the feature bits depending on copy mode */
1078 switch (copy_mode) {
1079 case XSTATE_COPY_FP:
1080 header.xfeatures &= XFEATURE_MASK_FP;
1081 break;
1082
1083 case XSTATE_COPY_FX:
1084 header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE;
1085 break;
1086
1087 case XSTATE_COPY_XSAVE:
1088 header.xfeatures &= fpstate->user_xfeatures & xfeatures;
1089 break;
1090 }
1091
1092 /* Copy FP state up to MXCSR */
1093 copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
1094 &xinit->i387, off_mxcsr);
1095
1096 /* Copy MXCSR when SSE or YMM are set in the feature mask */
1097 copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
1098 &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
1099 MXCSR_AND_FLAGS_SIZE);
1100
1101 /* Copy the remaining FP state */
1102 copy_feature(header.xfeatures & XFEATURE_MASK_FP,
1103 &to, &xsave->i387.st_space, &xinit->i387.st_space,
1104 sizeof(xsave->i387.st_space));
1105
1106 /* Copy the SSE state - shared with YMM, but independently managed */
1107 copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
1108 &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
1109 sizeof(xsave->i387.xmm_space));
1110
1111 if (copy_mode != XSTATE_COPY_XSAVE)
1112 goto out;
1113
1114 /* Zero the padding area */
1115 membuf_zero(&to, sizeof(xsave->i387.padding));
1116
1117 /* Copy xsave->i387.sw_reserved */
1118 membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
1119
1120 /* Copy the user space relevant state of @xsave->header */
1121 membuf_write(&to, &header, sizeof(header));
1122
1123 zerofrom = offsetof(struct xregs_state, extended_state_area);
1124
1125 /*
1126 * This 'mask' indicates which states to copy from fpstate.
1127 * Those extended states that are not present in fpstate are
1128 * either disabled or initialized:
1129 *
1130 * In non-compacted format, disabled features still occupy
1131 * state space but there is no state to copy from in the
1132 * compacted init_fpstate. The gap tracking will zero these
1133 * states.
1134 *
1135 * The extended features have an all zeroes init state. Thus,
1136 * remove them from 'mask' to zero those features in the user
1137 * buffer instead of retrieving them from init_fpstate.
1138 */
1139 mask = header.xfeatures;
1140
1141 for_each_extended_xfeature(i, mask) {
1142 /*
1143 * If there was a feature or alignment gap, zero the space
1144 * in the destination buffer.
1145 */
1146 if (zerofrom < xstate_offsets[i])
1147 membuf_zero(&to, xstate_offsets[i] - zerofrom);
1148
1149 if (i == XFEATURE_PKRU) {
1150 struct pkru_state pkru = {0};
1151 /*
1152 * PKRU is not necessarily up to date in the
1153 * XSAVE buffer. Use the provided value.
1154 */
1155 pkru.pkru = pkru_val;
1156 membuf_write(&to, &pkru, sizeof(pkru));
1157 } else {
1158 membuf_write(&to,
1159 __raw_xsave_addr(xsave, i),
1160 xstate_sizes[i]);
1161 }
1162 /*
1163 * Keep track of the last copied state in the non-compacted
1164 * target buffer for gap zeroing.
1165 */
1166 zerofrom = xstate_offsets[i] + xstate_sizes[i];
1167 }
1168
1169out:
1170 if (to.left)
1171 membuf_zero(&to, to.left);
1172}
1173
1174/**
1175 * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
1176 * @to: membuf descriptor
1177 * @tsk: The task from which to copy the saved xstate
1178 * @copy_mode: The requested copy mode
1179 *
1180 * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
1181 * format, i.e. from the kernel internal hardware dependent storage format
1182 * to the requested @mode. UABI XSTATE is always uncompacted!
1183 *
1184 * It supports partial copy but @to.pos always starts from zero.
1185 */
1186void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,
1187 enum xstate_copy_mode copy_mode)
1188{
1189 __copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate,
1190 tsk->thread.fpu.fpstate->user_xfeatures,
1191 tsk->thread.pkru, copy_mode);
1192}
1193
1194static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size,
1195 const void *kbuf, const void __user *ubuf)
1196{
1197 if (kbuf) {
1198 memcpy(dst, kbuf + offset, size);
1199 } else {
1200 if (copy_from_user(dst, ubuf + offset, size))
1201 return -EFAULT;
1202 }
1203 return 0;
1204}
1205
1206
1207/**
1208 * copy_uabi_to_xstate - Copy a UABI format buffer to the kernel xstate
1209 * @fpstate: The fpstate buffer to copy to
1210 * @kbuf: The UABI format buffer, if it comes from the kernel
1211 * @ubuf: The UABI format buffer, if it comes from userspace
1212 * @pkru: The location to write the PKRU value to
1213 *
1214 * Converts from the UABI format into the kernel internal hardware
1215 * dependent format.
1216 *
1217 * This function ultimately has three different callers with distinct PKRU
1218 * behavior.
1219 * 1. When called from sigreturn the PKRU register will be restored from
1220 * @fpstate via an XRSTOR. Correctly copying the UABI format buffer to
1221 * @fpstate is sufficient to cover this case, but the caller will also
1222 * pass a pointer to the thread_struct's pkru field in @pkru and updating
1223 * it is harmless.
1224 * 2. When called from ptrace the PKRU register will be restored from the
1225 * thread_struct's pkru field. A pointer to that is passed in @pkru.
1226 * The kernel will restore it manually, so the XRSTOR behavior that resets
1227 * the PKRU register to the hardware init value (0) if the corresponding
1228 * xfeatures bit is not set is emulated here.
1229 * 3. When called from KVM the PKRU register will be restored from the vcpu's
1230 * pkru field. A pointer to that is passed in @pkru. KVM hasn't used
1231 * XRSTOR and hasn't had the PKRU resetting behavior described above. To
1232 * preserve that KVM behavior, it passes NULL for @pkru if the xfeatures
1233 * bit is not set.
1234 */
1235static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf,
1236 const void __user *ubuf, u32 *pkru)
1237{
1238 struct xregs_state *xsave = &fpstate->regs.xsave;
1239 unsigned int offset, size;
1240 struct xstate_header hdr;
1241 u64 mask;
1242 int i;
1243
1244 offset = offsetof(struct xregs_state, header);
1245 if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf))
1246 return -EFAULT;
1247
1248 if (validate_user_xstate_header(&hdr, fpstate))
1249 return -EINVAL;
1250
1251 /* Validate MXCSR when any of the related features is in use */
1252 mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM;
1253 if (hdr.xfeatures & mask) {
1254 u32 mxcsr[2];
1255
1256 offset = offsetof(struct fxregs_state, mxcsr);
1257 if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf))
1258 return -EFAULT;
1259
1260 /* Reserved bits in MXCSR must be zero. */
1261 if (mxcsr[0] & ~mxcsr_feature_mask)
1262 return -EINVAL;
1263
1264 /* SSE and YMM require MXCSR even when FP is not in use. */
1265 if (!(hdr.xfeatures & XFEATURE_MASK_FP)) {
1266 xsave->i387.mxcsr = mxcsr[0];
1267 xsave->i387.mxcsr_mask = mxcsr[1];
1268 }
1269 }
1270
1271 for (i = 0; i < XFEATURE_MAX; i++) {
1272 mask = BIT_ULL(i);
1273
1274 if (hdr.xfeatures & mask) {
1275 void *dst = __raw_xsave_addr(xsave, i);
1276
1277 offset = xstate_offsets[i];
1278 size = xstate_sizes[i];
1279
1280 if (copy_from_buffer(dst, offset, size, kbuf, ubuf))
1281 return -EFAULT;
1282 }
1283 }
1284
1285 if (hdr.xfeatures & XFEATURE_MASK_PKRU) {
1286 struct pkru_state *xpkru;
1287
1288 xpkru = __raw_xsave_addr(xsave, XFEATURE_PKRU);
1289 *pkru = xpkru->pkru;
1290 } else {
1291 /*
1292 * KVM may pass NULL here to indicate that it does not need
1293 * PKRU updated.
1294 */
1295 if (pkru)
1296 *pkru = 0;
1297 }
1298
1299 /*
1300 * The state that came in from userspace was user-state only.
1301 * Mask all the user states out of 'xfeatures':
1302 */
1303 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1304
1305 /*
1306 * Add back in the features that came in from userspace:
1307 */
1308 xsave->header.xfeatures |= hdr.xfeatures;
1309
1310 return 0;
1311}
1312
1313/*
1314 * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S]
1315 * format and copy to the target thread. Used by ptrace and KVM.
1316 */
1317int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru)
1318{
1319 return copy_uabi_to_xstate(fpstate, kbuf, NULL, pkru);
1320}
1321
1322/*
1323 * Convert from a sigreturn standard-format user-space buffer to kernel
1324 * XSAVE[S] format and copy to the target thread. This is called from the
1325 * sigreturn() and rt_sigreturn() system calls.
1326 */
1327int copy_sigframe_from_user_to_xstate(struct task_struct *tsk,
1328 const void __user *ubuf)
1329{
1330 return copy_uabi_to_xstate(tsk->thread.fpu.fpstate, NULL, ubuf, &tsk->thread.pkru);
1331}
1332
1333static bool validate_independent_components(u64 mask)
1334{
1335 u64 xchk;
1336
1337 if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES)))
1338 return false;
1339
1340 xchk = ~xfeatures_mask_independent();
1341
1342 if (WARN_ON_ONCE(!mask || mask & xchk))
1343 return false;
1344
1345 return true;
1346}
1347
1348/**
1349 * xsaves - Save selected components to a kernel xstate buffer
1350 * @xstate: Pointer to the buffer
1351 * @mask: Feature mask to select the components to save
1352 *
1353 * The @xstate buffer must be 64 byte aligned and correctly initialized as
1354 * XSAVES does not write the full xstate header. Before first use the
1355 * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer
1356 * can #GP.
1357 *
1358 * The feature mask must be a subset of the independent features.
1359 */
1360void xsaves(struct xregs_state *xstate, u64 mask)
1361{
1362 int err;
1363
1364 if (!validate_independent_components(mask))
1365 return;
1366
1367 XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err);
1368 WARN_ON_ONCE(err);
1369}
1370
1371/**
1372 * xrstors - Restore selected components from a kernel xstate buffer
1373 * @xstate: Pointer to the buffer
1374 * @mask: Feature mask to select the components to restore
1375 *
1376 * The @xstate buffer must be 64 byte aligned and correctly initialized
1377 * otherwise XRSTORS from that buffer can #GP.
1378 *
1379 * Proper usage is to restore the state which was saved with
1380 * xsaves() into @xstate.
1381 *
1382 * The feature mask must be a subset of the independent features.
1383 */
1384void xrstors(struct xregs_state *xstate, u64 mask)
1385{
1386 int err;
1387
1388 if (!validate_independent_components(mask))
1389 return;
1390
1391 XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err);
1392 WARN_ON_ONCE(err);
1393}
1394
1395#if IS_ENABLED(CONFIG_KVM)
1396void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature)
1397{
1398 void *addr = get_xsave_addr(&fps->regs.xsave, xfeature);
1399
1400 if (addr)
1401 memset(addr, 0, xstate_sizes[xfeature]);
1402}
1403EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component);
1404#endif
1405
1406#ifdef CONFIG_X86_64
1407
1408#ifdef CONFIG_X86_DEBUG_FPU
1409/*
1410 * Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask
1411 * can safely operate on the @fpstate buffer.
1412 */
1413static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor)
1414{
1415 u64 xfd = __this_cpu_read(xfd_state);
1416
1417 if (fpstate->xfd == xfd)
1418 return true;
1419
1420 /*
1421 * The XFD MSR does not match fpstate->xfd. That's invalid when
1422 * the passed in fpstate is current's fpstate.
1423 */
1424 if (fpstate->xfd == current->thread.fpu.fpstate->xfd)
1425 return false;
1426
1427 /*
1428 * XRSTOR(S) from init_fpstate are always correct as it will just
1429 * bring all components into init state and not read from the
1430 * buffer. XSAVE(S) raises #PF after init.
1431 */
1432 if (fpstate == &init_fpstate)
1433 return rstor;
1434
1435 /*
1436 * XSAVE(S): clone(), fpu_swap_kvm_fpu()
1437 * XRSTORS(S): fpu_swap_kvm_fpu()
1438 */
1439
1440 /*
1441 * No XSAVE/XRSTOR instructions (except XSAVE itself) touch
1442 * the buffer area for XFD-disabled state components.
1443 */
1444 mask &= ~xfd;
1445
1446 /*
1447 * Remove features which are valid in fpstate. They
1448 * have space allocated in fpstate.
1449 */
1450 mask &= ~fpstate->xfeatures;
1451
1452 /*
1453 * Any remaining state components in 'mask' might be written
1454 * by XSAVE/XRSTOR. Fail validation it found.
1455 */
1456 return !mask;
1457}
1458
1459void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor)
1460{
1461 WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor));
1462}
1463#endif /* CONFIG_X86_DEBUG_FPU */
1464
1465static int __init xfd_update_static_branch(void)
1466{
1467 /*
1468 * If init_fpstate.xfd has bits set then dynamic features are
1469 * available and the dynamic sizing must be enabled.
1470 */
1471 if (init_fpstate.xfd)
1472 static_branch_enable(&__fpu_state_size_dynamic);
1473 return 0;
1474}
1475arch_initcall(xfd_update_static_branch)
1476
1477void fpstate_free(struct fpu *fpu)
1478{
1479 if (fpu->fpstate && fpu->fpstate != &fpu->__fpstate)
1480 vfree(fpu->fpstate);
1481}
1482
1483/**
1484 * fpstate_realloc - Reallocate struct fpstate for the requested new features
1485 *
1486 * @xfeatures: A bitmap of xstate features which extend the enabled features
1487 * of that task
1488 * @ksize: The required size for the kernel buffer
1489 * @usize: The required size for user space buffers
1490 * @guest_fpu: Pointer to a guest FPU container. NULL for host allocations
1491 *
1492 * Note vs. vmalloc(): If the task with a vzalloc()-allocated buffer
1493 * terminates quickly, vfree()-induced IPIs may be a concern, but tasks
1494 * with large states are likely to live longer.
1495 *
1496 * Returns: 0 on success, -ENOMEM on allocation error.
1497 */
1498static int fpstate_realloc(u64 xfeatures, unsigned int ksize,
1499 unsigned int usize, struct fpu_guest *guest_fpu)
1500{
1501 struct fpu *fpu = ¤t->thread.fpu;
1502 struct fpstate *curfps, *newfps = NULL;
1503 unsigned int fpsize;
1504 bool in_use;
1505
1506 fpsize = ksize + ALIGN(offsetof(struct fpstate, regs), 64);
1507
1508 newfps = vzalloc(fpsize);
1509 if (!newfps)
1510 return -ENOMEM;
1511 newfps->size = ksize;
1512 newfps->user_size = usize;
1513 newfps->is_valloc = true;
1514
1515 /*
1516 * When a guest FPU is supplied, use @guest_fpu->fpstate
1517 * as reference independent whether it is in use or not.
1518 */
1519 curfps = guest_fpu ? guest_fpu->fpstate : fpu->fpstate;
1520
1521 /* Determine whether @curfps is the active fpstate */
1522 in_use = fpu->fpstate == curfps;
1523
1524 if (guest_fpu) {
1525 newfps->is_guest = true;
1526 newfps->is_confidential = curfps->is_confidential;
1527 newfps->in_use = curfps->in_use;
1528 guest_fpu->xfeatures |= xfeatures;
1529 guest_fpu->uabi_size = usize;
1530 }
1531
1532 fpregs_lock();
1533 /*
1534 * If @curfps is in use, ensure that the current state is in the
1535 * registers before swapping fpstate as that might invalidate it
1536 * due to layout changes.
1537 */
1538 if (in_use && test_thread_flag(TIF_NEED_FPU_LOAD))
1539 fpregs_restore_userregs();
1540
1541 newfps->xfeatures = curfps->xfeatures | xfeatures;
1542 newfps->user_xfeatures = curfps->user_xfeatures | xfeatures;
1543 newfps->xfd = curfps->xfd & ~xfeatures;
1544
1545 /* Do the final updates within the locked region */
1546 xstate_init_xcomp_bv(&newfps->regs.xsave, newfps->xfeatures);
1547
1548 if (guest_fpu) {
1549 guest_fpu->fpstate = newfps;
1550 /* If curfps is active, update the FPU fpstate pointer */
1551 if (in_use)
1552 fpu->fpstate = newfps;
1553 } else {
1554 fpu->fpstate = newfps;
1555 }
1556
1557 if (in_use)
1558 xfd_update_state(fpu->fpstate);
1559 fpregs_unlock();
1560
1561 /* Only free valloc'ed state */
1562 if (curfps && curfps->is_valloc)
1563 vfree(curfps);
1564
1565 return 0;
1566}
1567
1568static int validate_sigaltstack(unsigned int usize)
1569{
1570 struct task_struct *thread, *leader = current->group_leader;
1571 unsigned long framesize = get_sigframe_size();
1572
1573 lockdep_assert_held(¤t->sighand->siglock);
1574
1575 /* get_sigframe_size() is based on fpu_user_cfg.max_size */
1576 framesize -= fpu_user_cfg.max_size;
1577 framesize += usize;
1578 for_each_thread(leader, thread) {
1579 if (thread->sas_ss_size && thread->sas_ss_size < framesize)
1580 return -ENOSPC;
1581 }
1582 return 0;
1583}
1584
1585static int __xstate_request_perm(u64 permitted, u64 requested, bool guest)
1586{
1587 /*
1588 * This deliberately does not exclude !XSAVES as we still might
1589 * decide to optionally context switch XCR0 or talk the silicon
1590 * vendors into extending XFD for the pre AMX states, especially
1591 * AVX512.
1592 */
1593 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
1594 struct fpu *fpu = ¤t->group_leader->thread.fpu;
1595 struct fpu_state_perm *perm;
1596 unsigned int ksize, usize;
1597 u64 mask;
1598 int ret = 0;
1599
1600 /* Check whether fully enabled */
1601 if ((permitted & requested) == requested)
1602 return 0;
1603
1604 /* Calculate the resulting kernel state size */
1605 mask = permitted | requested;
1606 /* Take supervisor states into account on the host */
1607 if (!guest)
1608 mask |= xfeatures_mask_supervisor();
1609 ksize = xstate_calculate_size(mask, compacted);
1610
1611 /* Calculate the resulting user state size */
1612 mask &= XFEATURE_MASK_USER_SUPPORTED;
1613 usize = xstate_calculate_size(mask, false);
1614
1615 if (!guest) {
1616 ret = validate_sigaltstack(usize);
1617 if (ret)
1618 return ret;
1619 }
1620
1621 perm = guest ? &fpu->guest_perm : &fpu->perm;
1622 /* Pairs with the READ_ONCE() in xstate_get_group_perm() */
1623 WRITE_ONCE(perm->__state_perm, mask);
1624 /* Protected by sighand lock */
1625 perm->__state_size = ksize;
1626 perm->__user_state_size = usize;
1627 return ret;
1628}
1629
1630/*
1631 * Permissions array to map facilities with more than one component
1632 */
1633static const u64 xstate_prctl_req[XFEATURE_MAX] = {
1634 [XFEATURE_XTILE_DATA] = XFEATURE_MASK_XTILE_DATA,
1635};
1636
1637static int xstate_request_perm(unsigned long idx, bool guest)
1638{
1639 u64 permitted, requested;
1640 int ret;
1641
1642 if (idx >= XFEATURE_MAX)
1643 return -EINVAL;
1644
1645 /*
1646 * Look up the facility mask which can require more than
1647 * one xstate component.
1648 */
1649 idx = array_index_nospec(idx, ARRAY_SIZE(xstate_prctl_req));
1650 requested = xstate_prctl_req[idx];
1651 if (!requested)
1652 return -EOPNOTSUPP;
1653
1654 if ((fpu_user_cfg.max_features & requested) != requested)
1655 return -EOPNOTSUPP;
1656
1657 /* Lockless quick check */
1658 permitted = xstate_get_group_perm(guest);
1659 if ((permitted & requested) == requested)
1660 return 0;
1661
1662 /* Protect against concurrent modifications */
1663 spin_lock_irq(¤t->sighand->siglock);
1664 permitted = xstate_get_group_perm(guest);
1665
1666 /* First vCPU allocation locks the permissions. */
1667 if (guest && (permitted & FPU_GUEST_PERM_LOCKED))
1668 ret = -EBUSY;
1669 else
1670 ret = __xstate_request_perm(permitted, requested, guest);
1671 spin_unlock_irq(¤t->sighand->siglock);
1672 return ret;
1673}
1674
1675int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu)
1676{
1677 u64 xfd_event = xfd_err & XFEATURE_MASK_USER_DYNAMIC;
1678 struct fpu_state_perm *perm;
1679 unsigned int ksize, usize;
1680 struct fpu *fpu;
1681
1682 if (!xfd_event) {
1683 if (!guest_fpu)
1684 pr_err_once("XFD: Invalid xfd error: %016llx\n", xfd_err);
1685 return 0;
1686 }
1687
1688 /* Protect against concurrent modifications */
1689 spin_lock_irq(¤t->sighand->siglock);
1690
1691 /* If not permitted let it die */
1692 if ((xstate_get_group_perm(!!guest_fpu) & xfd_event) != xfd_event) {
1693 spin_unlock_irq(¤t->sighand->siglock);
1694 return -EPERM;
1695 }
1696
1697 fpu = ¤t->group_leader->thread.fpu;
1698 perm = guest_fpu ? &fpu->guest_perm : &fpu->perm;
1699 ksize = perm->__state_size;
1700 usize = perm->__user_state_size;
1701
1702 /*
1703 * The feature is permitted. State size is sufficient. Dropping
1704 * the lock is safe here even if more features are added from
1705 * another task, the retrieved buffer sizes are valid for the
1706 * currently requested feature(s).
1707 */
1708 spin_unlock_irq(¤t->sighand->siglock);
1709
1710 /*
1711 * Try to allocate a new fpstate. If that fails there is no way
1712 * out.
1713 */
1714 if (fpstate_realloc(xfd_event, ksize, usize, guest_fpu))
1715 return -EFAULT;
1716 return 0;
1717}
1718
1719int xfd_enable_feature(u64 xfd_err)
1720{
1721 return __xfd_enable_feature(xfd_err, NULL);
1722}
1723
1724#else /* CONFIG_X86_64 */
1725static inline int xstate_request_perm(unsigned long idx, bool guest)
1726{
1727 return -EPERM;
1728}
1729#endif /* !CONFIG_X86_64 */
1730
1731u64 xstate_get_guest_group_perm(void)
1732{
1733 return xstate_get_group_perm(true);
1734}
1735EXPORT_SYMBOL_GPL(xstate_get_guest_group_perm);
1736
1737/**
1738 * fpu_xstate_prctl - xstate permission operations
1739 * @option: A subfunction of arch_prctl()
1740 * @arg2: option argument
1741 * Return: 0 if successful; otherwise, an error code
1742 *
1743 * Option arguments:
1744 *
1745 * ARCH_GET_XCOMP_SUPP: Pointer to user space u64 to store the info
1746 * ARCH_GET_XCOMP_PERM: Pointer to user space u64 to store the info
1747 * ARCH_REQ_XCOMP_PERM: Facility number requested
1748 *
1749 * For facilities which require more than one XSTATE component, the request
1750 * must be the highest state component number related to that facility,
1751 * e.g. for AMX which requires XFEATURE_XTILE_CFG(17) and
1752 * XFEATURE_XTILE_DATA(18) this would be XFEATURE_XTILE_DATA(18).
1753 */
1754long fpu_xstate_prctl(int option, unsigned long arg2)
1755{
1756 u64 __user *uptr = (u64 __user *)arg2;
1757 u64 permitted, supported;
1758 unsigned long idx = arg2;
1759 bool guest = false;
1760
1761 switch (option) {
1762 case ARCH_GET_XCOMP_SUPP:
1763 supported = fpu_user_cfg.max_features | fpu_user_cfg.legacy_features;
1764 return put_user(supported, uptr);
1765
1766 case ARCH_GET_XCOMP_PERM:
1767 /*
1768 * Lockless snapshot as it can also change right after the
1769 * dropping the lock.
1770 */
1771 permitted = xstate_get_host_group_perm();
1772 permitted &= XFEATURE_MASK_USER_SUPPORTED;
1773 return put_user(permitted, uptr);
1774
1775 case ARCH_GET_XCOMP_GUEST_PERM:
1776 permitted = xstate_get_guest_group_perm();
1777 permitted &= XFEATURE_MASK_USER_SUPPORTED;
1778 return put_user(permitted, uptr);
1779
1780 case ARCH_REQ_XCOMP_GUEST_PERM:
1781 guest = true;
1782 fallthrough;
1783
1784 case ARCH_REQ_XCOMP_PERM:
1785 if (!IS_ENABLED(CONFIG_X86_64))
1786 return -EOPNOTSUPP;
1787
1788 return xstate_request_perm(idx, guest);
1789
1790 default:
1791 return -EINVAL;
1792 }
1793}
1794
1795#ifdef CONFIG_PROC_PID_ARCH_STATUS
1796/*
1797 * Report the amount of time elapsed in millisecond since last AVX512
1798 * use in the task.
1799 */
1800static void avx512_status(struct seq_file *m, struct task_struct *task)
1801{
1802 unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1803 long delta;
1804
1805 if (!timestamp) {
1806 /*
1807 * Report -1 if no AVX512 usage
1808 */
1809 delta = -1;
1810 } else {
1811 delta = (long)(jiffies - timestamp);
1812 /*
1813 * Cap to LONG_MAX if time difference > LONG_MAX
1814 */
1815 if (delta < 0)
1816 delta = LONG_MAX;
1817 delta = jiffies_to_msecs(delta);
1818 }
1819
1820 seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1821 seq_putc(m, '\n');
1822}
1823
1824/*
1825 * Report architecture specific information
1826 */
1827int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1828 struct pid *pid, struct task_struct *task)
1829{
1830 /*
1831 * Report AVX512 state if the processor and build option supported.
1832 */
1833 if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1834 avx512_status(m, task);
1835
1836 return 0;
1837}
1838#endif /* CONFIG_PROC_PID_ARCH_STATUS */