1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _ASM_X86_PROCESSOR_H
  3#define _ASM_X86_PROCESSOR_H
  4
  5#include <asm/processor-flags.h>
  6
  7/* Forward declaration, a strange C thing */
  8struct task_struct;
  9struct mm_struct;
 
 10struct vm86;
 11
 12#include <asm/math_emu.h>
 13#include <asm/segment.h>
 14#include <asm/types.h>
 15#include <uapi/asm/sigcontext.h>
 16#include <asm/current.h>
 17#include <asm/cpufeatures.h>
 
 18#include <asm/page.h>
 19#include <asm/pgtable_types.h>
 20#include <asm/percpu.h>
 21#include <asm/msr.h>
 22#include <asm/desc_defs.h>
 23#include <asm/nops.h>
 24#include <asm/special_insns.h>
 25#include <asm/fpu/types.h>
 26#include <asm/unwind_hints.h>
 
 
 27
 28#include <linux/personality.h>
 29#include <linux/cache.h>
 30#include <linux/threads.h>
 31#include <linux/math64.h>
 32#include <linux/err.h>
 33#include <linux/irqflags.h>
 34#include <linux/mem_encrypt.h>
 35
 36/*
 37 * We handle most unaligned accesses in hardware.  On the other hand
 38 * unaligned DMA can be quite expensive on some Nehalem processors.
 39 *
 40 * Based on this we disable the IP header alignment in network drivers.
 41 */
 42#define NET_IP_ALIGN	0
 43
 44#define HBP_NUM 4
 45/*
 46 * Default implementation of macro that returns current
 47 * instruction pointer ("program counter").
 48 */
 49static inline void *current_text_addr(void)
 50{
 51	void *pc;
 52
 53	asm volatile("mov $1f, %0; 1:":"=r" (pc));
 54
 55	return pc;
 56}
 57
 58/*
 59 * These alignment constraints are for performance in the vSMP case,
 60 * but in the task_struct case we must also meet hardware imposed
 61 * alignment requirements of the FPU state:
 62 */
 63#ifdef CONFIG_X86_VSMP
 64# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
 65# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
 66#else
 67# define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
 68# define ARCH_MIN_MMSTRUCT_ALIGN	0
 69#endif
 70
 71enum tlb_infos {
 72	ENTRIES,
 73	NR_INFO
 74};
 75
 76extern u16 __read_mostly tlb_lli_4k[NR_INFO];
 77extern u16 __read_mostly tlb_lli_2m[NR_INFO];
 78extern u16 __read_mostly tlb_lli_4m[NR_INFO];
 79extern u16 __read_mostly tlb_lld_4k[NR_INFO];
 80extern u16 __read_mostly tlb_lld_2m[NR_INFO];
 81extern u16 __read_mostly tlb_lld_4m[NR_INFO];
 82extern u16 __read_mostly tlb_lld_1g[NR_INFO];
 83
 84/*
 85 *  CPU type and hardware bug flags. Kept separately for each CPU.
 86 *  Members of this structure are referenced in head_32.S, so think twice
 87 *  before touching them. [mj]
 88 */
 89
 90struct cpuinfo_x86 {
 91	__u8			x86;		/* CPU family */
 92	__u8			x86_vendor;	/* CPU vendor */
 93	__u8			x86_model;
 94	__u8			x86_stepping;
 95#ifdef CONFIG_X86_64
 96	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
 97	int			x86_tlbsize;
 98#endif
 
 
 
 99	__u8			x86_virt_bits;
100	__u8			x86_phys_bits;
101	/* CPUID returned core id bits: */
102	__u8			x86_coreid_bits;
103	__u8			cu_id;
104	/* Max extended CPUID function supported: */
105	__u32			extended_cpuid_level;
106	/* Maximum supported CPUID level, -1=no CPUID: */
107	int			cpuid_level;
108	__u32			x86_capability[NCAPINTS + NBUGINTS];
 
 
 
 
 
 
 
 
109	char			x86_vendor_id[16];
110	char			x86_model_id[64];
111	/* in KB - valid for CPUS which support this call: */
112	unsigned int		x86_cache_size;
113	int			x86_cache_alignment;	/* In bytes */
114	/* Cache QoS architectural values: */
115	int			x86_cache_max_rmid;	/* max index */
116	int			x86_cache_occ_scale;	/* scale to bytes */
 
117	int			x86_power;
118	unsigned long		loops_per_jiffy;
 
 
119	/* cpuid returned max cores value: */
120	u16			 x86_max_cores;
121	u16			apicid;
122	u16			initial_apicid;
123	u16			x86_clflush_size;
124	/* number of cores as seen by the OS: */
125	u16			booted_cores;
126	/* Physical processor id: */
127	u16			phys_proc_id;
128	/* Logical processor id: */
129	u16			logical_proc_id;
130	/* Core id: */
131	u16			cpu_core_id;
 
 
132	/* Index into per_cpu list: */
133	u16			cpu_index;
 
 
134	u32			microcode;
 
 
135	unsigned		initialized : 1;
136} __randomize_layout;
137
138struct cpuid_regs {
139	u32 eax, ebx, ecx, edx;
140};
141
142enum cpuid_regs_idx {
143	CPUID_EAX = 0,
144	CPUID_EBX,
145	CPUID_ECX,
146	CPUID_EDX,
147};
148
149#define X86_VENDOR_INTEL	0
150#define X86_VENDOR_CYRIX	1
151#define X86_VENDOR_AMD		2
152#define X86_VENDOR_UMC		3
153#define X86_VENDOR_CENTAUR	5
154#define X86_VENDOR_TRANSMETA	7
155#define X86_VENDOR_NSC		8
156#define X86_VENDOR_NUM		9
 
 
 
157
158#define X86_VENDOR_UNKNOWN	0xff
159
160/*
161 * capabilities of CPUs
162 */
163extern struct cpuinfo_x86	boot_cpu_data;
164extern struct cpuinfo_x86	new_cpu_data;
165
166extern struct x86_hw_tss	doublefault_tss;
167extern __u32			cpu_caps_cleared[NCAPINTS + NBUGINTS];
168extern __u32			cpu_caps_set[NCAPINTS + NBUGINTS];
169
170#ifdef CONFIG_SMP
171DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
172#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
173#else
174#define cpu_info		boot_cpu_data
175#define cpu_data(cpu)		boot_cpu_data
176#endif
177
178extern const struct seq_operations cpuinfo_op;
179
180#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)
181
182extern void cpu_detect(struct cpuinfo_x86 *c);
183
 
 
 
 
 
184extern void early_cpu_init(void);
185extern void identify_boot_cpu(void);
186extern void identify_secondary_cpu(struct cpuinfo_x86 *);
187extern void print_cpu_info(struct cpuinfo_x86 *);
188void print_cpu_msr(struct cpuinfo_x86 *);
189extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
190extern u32 get_scattered_cpuid_leaf(unsigned int level,
191				    unsigned int sub_leaf,
192				    enum cpuid_regs_idx reg);
193extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
194extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
195
196extern void detect_extended_topology(struct cpuinfo_x86 *c);
197extern void detect_ht(struct cpuinfo_x86 *c);
198
199#ifdef CONFIG_X86_32
200extern int have_cpuid_p(void);
201#else
202static inline int have_cpuid_p(void)
203{
204	return 1;
205}
206#endif
207static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
208				unsigned int *ecx, unsigned int *edx)
209{
210	/* ecx is often an input as well as an output. */
211	asm volatile("cpuid"
212	    : "=a" (*eax),
213	      "=b" (*ebx),
214	      "=c" (*ecx),
215	      "=d" (*edx)
216	    : "0" (*eax), "2" (*ecx)
217	    : "memory");
218}
219
220#define native_cpuid_reg(reg)					\
221static inline unsigned int native_cpuid_##reg(unsigned int op)	\
222{								\
223	unsigned int eax = op, ebx, ecx = 0, edx;		\
224								\
225	native_cpuid(&eax, &ebx, &ecx, &edx);			\
226								\
227	return reg;						\
228}
229
230/*
231 * Native CPUID functions returning a single datum.
232 */
233native_cpuid_reg(eax)
234native_cpuid_reg(ebx)
235native_cpuid_reg(ecx)
236native_cpuid_reg(edx)
237
238/*
239 * Friendlier CR3 helpers.
240 */
241static inline unsigned long read_cr3_pa(void)
242{
243	return __read_cr3() & CR3_ADDR_MASK;
244}
245
246static inline unsigned long native_read_cr3_pa(void)
247{
248	return __native_read_cr3() & CR3_ADDR_MASK;
249}
250
251static inline void load_cr3(pgd_t *pgdir)
252{
253	write_cr3(__sme_pa(pgdir));
254}
255
256/*
257 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
258 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
259 * unrelated to the task-switch mechanism:
260 */
261#ifdef CONFIG_X86_32
262/* This is the TSS defined by the hardware. */
263struct x86_hw_tss {
264	unsigned short		back_link, __blh;
265	unsigned long		sp0;
266	unsigned short		ss0, __ss0h;
267	unsigned long		sp1;
268
269	/*
270	 * We don't use ring 1, so ss1 is a convenient scratch space in
271	 * the same cacheline as sp0.  We use ss1 to cache the value in
272	 * MSR_IA32_SYSENTER_CS.  When we context switch
273	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
274	 * written matches ss1, and, if it's not, then we wrmsr the new
275	 * value and update ss1.
276	 *
277	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
278	 * that we set it to zero in vm86 tasks to avoid corrupting the
279	 * stack if we were to go through the sysenter path from vm86
280	 * mode.
281	 */
282	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */
283
284	unsigned short		__ss1h;
285	unsigned long		sp2;
286	unsigned short		ss2, __ss2h;
287	unsigned long		__cr3;
288	unsigned long		ip;
289	unsigned long		flags;
290	unsigned long		ax;
291	unsigned long		cx;
292	unsigned long		dx;
293	unsigned long		bx;
294	unsigned long		sp;
295	unsigned long		bp;
296	unsigned long		si;
297	unsigned long		di;
298	unsigned short		es, __esh;
299	unsigned short		cs, __csh;
300	unsigned short		ss, __ssh;
301	unsigned short		ds, __dsh;
302	unsigned short		fs, __fsh;
303	unsigned short		gs, __gsh;
304	unsigned short		ldt, __ldth;
305	unsigned short		trace;
306	unsigned short		io_bitmap_base;
307
308} __attribute__((packed));
309#else
310struct x86_hw_tss {
311	u32			reserved1;
312	u64			sp0;
 
313
314	/*
315	 * We store cpu_current_top_of_stack in sp1 so it's always accessible.
316	 * Linux does not use ring 1, so sp1 is not otherwise needed.
 
317	 */
318	u64			sp1;
319
320	u64			sp2;
 
321	u64			reserved2;
322	u64			ist[7];
323	u32			reserved3;
324	u32			reserved4;
325	u16			reserved5;
326	u16			io_bitmap_base;
327
328} __attribute__((packed));
329#endif
330
331/*
332 * IO-bitmap sizes:
333 */
334#define IO_BITMAP_BITS			65536
335#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
336#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
337#define IO_BITMAP_OFFSET		(offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
338#define INVALID_IO_BITMAP_OFFSET	0x8000
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
339
340struct entry_stack {
341	unsigned long		words[64];
342};
343
344struct entry_stack_page {
345	struct entry_stack stack;
346} __aligned(PAGE_SIZE);
347
348struct tss_struct {
 
 
 
 
 
 
349	/*
350	 * The fixed hardware portion.  This must not cross a page boundary
351	 * at risk of violating the SDM's advice and potentially triggering
352	 * errata.
 
 
353	 */
354	struct x86_hw_tss	x86_tss;
355
356	/*
357	 * The extra 1 is there because the CPU will access an
358	 * additional byte beyond the end of the IO permission
359	 * bitmap. The extra byte must be all 1 bits, and must
360	 * be within the limit.
361	 */
362	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];
363} __aligned(PAGE_SIZE);
364
365DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
 
 
 
 
 
366
367/*
368 * sizeof(unsigned long) coming from an extra "long" at the end
369 * of the iobitmap.
370 *
371 * -1? seg base+limit should be pointing to the address of the
372 * last valid byte
373 */
374#define __KERNEL_TSS_LIMIT	\
375	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)
376
377#ifdef CONFIG_X86_32
378DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
379#else
380/* The RO copy can't be accessed with this_cpu_xyz(), so use the RW copy. */
381#define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1
382#endif
383
384/*
385 * Save the original ist values for checking stack pointers during debugging
386 */
387struct orig_ist {
388	unsigned long		ist[7];
389};
390
391#ifdef CONFIG_X86_64
392DECLARE_PER_CPU(struct orig_ist, orig_ist);
 
 
393
394union irq_stack_union {
395	char irq_stack[IRQ_STACK_SIZE];
396	/*
397	 * GCC hardcodes the stack canary as %gs:40.  Since the
398	 * irq_stack is the object at %gs:0, we reserve the bottom
399	 * 48 bytes of the irq stack for the canary.
 
 
 
400	 */
401	struct {
402		char gs_base[40];
403		unsigned long stack_canary;
404	};
405};
406
407DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
408DECLARE_INIT_PER_CPU(irq_stack_union);
409
410static inline unsigned long cpu_kernelmode_gs_base(int cpu)
411{
412	return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu);
413}
414
415DECLARE_PER_CPU(char *, irq_stack_ptr);
416DECLARE_PER_CPU(unsigned int, irq_count);
417extern asmlinkage void ignore_sysret(void);
418
419#if IS_ENABLED(CONFIG_KVM)
420/* Save actual FS/GS selectors and bases to current->thread */
421void save_fsgs_for_kvm(void);
422#endif
423#else	/* X86_64 */
424#ifdef CONFIG_CC_STACKPROTECTOR
425/*
426 * Make sure stack canary segment base is cached-aligned:
427 *   "For Intel Atom processors, avoid non zero segment base address
428 *    that is not aligned to cache line boundary at all cost."
429 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
430 */
431struct stack_canary {
432	char __pad[20];		/* canary at %gs:20 */
433	unsigned long canary;
434};
435DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
436#endif
437/*
438 * per-CPU IRQ handling stacks
439 */
440struct irq_stack {
441	u32                     stack[THREAD_SIZE/sizeof(u32)];
442} __aligned(THREAD_SIZE);
443
444DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
445DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
446#endif	/* X86_64 */
447
448extern unsigned int fpu_kernel_xstate_size;
449extern unsigned int fpu_user_xstate_size;
450
451struct perf_event;
452
453typedef struct {
454	unsigned long		seg;
455} mm_segment_t;
456
457struct thread_struct {
458	/* Cached TLS descriptors: */
459	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
460#ifdef CONFIG_X86_32
461	unsigned long		sp0;
462#endif
463	unsigned long		sp;
464#ifdef CONFIG_X86_32
465	unsigned long		sysenter_cs;
466#else
467	unsigned short		es;
468	unsigned short		ds;
469	unsigned short		fsindex;
470	unsigned short		gsindex;
471#endif
472
473#ifdef CONFIG_X86_64
474	unsigned long		fsbase;
475	unsigned long		gsbase;
476#else
477	/*
478	 * XXX: this could presumably be unsigned short.  Alternatively,
479	 * 32-bit kernels could be taught to use fsindex instead.
480	 */
481	unsigned long fs;
482	unsigned long gs;
483#endif
484
485	/* Save middle states of ptrace breakpoints */
486	struct perf_event	*ptrace_bps[HBP_NUM];
487	/* Debug status used for traps, single steps, etc... */
488	unsigned long           debugreg6;
489	/* Keep track of the exact dr7 value set by the user */
490	unsigned long           ptrace_dr7;
491	/* Fault info: */
492	unsigned long		cr2;
493	unsigned long		trap_nr;
494	unsigned long		error_code;
495#ifdef CONFIG_VM86
496	/* Virtual 86 mode info */
497	struct vm86		*vm86;
498#endif
499	/* IO permissions: */
500	unsigned long		*io_bitmap_ptr;
501	unsigned long		iopl;
502	/* Max allowed port in the bitmap, in bytes: */
503	unsigned		io_bitmap_max;
504
505	mm_segment_t		addr_limit;
 
 
 
 
 
506
 
507	unsigned int		sig_on_uaccess_err:1;
508	unsigned int		uaccess_err:1;	/* uaccess failed */
 
 
 
 
 
 
 
 
509
510	/* Floating point and extended processor state */
511	struct fpu		fpu;
512	/*
513	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
514	 * the end.
515	 */
516};
517
518/* Whitelist the FPU state from the task_struct for hardened usercopy. */
 
519static inline void arch_thread_struct_whitelist(unsigned long *offset,
520						unsigned long *size)
521{
522	*offset = offsetof(struct thread_struct, fpu.state);
523	*size = fpu_kernel_xstate_size;
524}
525
526/*
527 * Thread-synchronous status.
528 *
529 * This is different from the flags in that nobody else
530 * ever touches our thread-synchronous status, so we don't
531 * have to worry about atomic accesses.
532 */
533#define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/
534
535/*
536 * Set IOPL bits in EFLAGS from given mask
537 */
538static inline void native_set_iopl_mask(unsigned mask)
539{
540#ifdef CONFIG_X86_32
541	unsigned int reg;
542
543	asm volatile ("pushfl;"
544		      "popl %0;"
545		      "andl %1, %0;"
546		      "orl %2, %0;"
547		      "pushl %0;"
548		      "popfl"
549		      : "=&r" (reg)
550		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
551#endif
552}
553
554static inline void
555native_load_sp0(unsigned long sp0)
556{
557	this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
558}
559
560static inline void native_swapgs(void)
561{
562#ifdef CONFIG_X86_64
563	asm volatile("swapgs" ::: "memory");
564#endif
565}
566
567static inline unsigned long current_top_of_stack(void)
568{
569	/*
570	 *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
571	 *  and around vm86 mode and sp0 on x86_64 is special because of the
572	 *  entry trampoline.
573	 */
574	return this_cpu_read_stable(cpu_current_top_of_stack);
575}
576
577static inline bool on_thread_stack(void)
578{
579	return (unsigned long)(current_top_of_stack() -
580			       current_stack_pointer) < THREAD_SIZE;
581}
582
583#ifdef CONFIG_PARAVIRT
584#include <asm/paravirt.h>
585#else
586#define __cpuid			native_cpuid
587
588static inline void load_sp0(unsigned long sp0)
589{
590	native_load_sp0(sp0);
591}
592
593#define set_iopl_mask native_set_iopl_mask
594#endif /* CONFIG_PARAVIRT */
595
596/* Free all resources held by a thread. */
597extern void release_thread(struct task_struct *);
598
599unsigned long get_wchan(struct task_struct *p);
600
601/*
602 * Generic CPUID function
603 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
604 * resulting in stale register contents being returned.
605 */
606static inline void cpuid(unsigned int op,
607			 unsigned int *eax, unsigned int *ebx,
608			 unsigned int *ecx, unsigned int *edx)
609{
610	*eax = op;
611	*ecx = 0;
612	__cpuid(eax, ebx, ecx, edx);
613}
614
615/* Some CPUID calls want 'count' to be placed in ecx */
616static inline void cpuid_count(unsigned int op, int count,
617			       unsigned int *eax, unsigned int *ebx,
618			       unsigned int *ecx, unsigned int *edx)
619{
620	*eax = op;
621	*ecx = count;
622	__cpuid(eax, ebx, ecx, edx);
623}
624
625/*
626 * CPUID functions returning a single datum
627 */
628static inline unsigned int cpuid_eax(unsigned int op)
629{
630	unsigned int eax, ebx, ecx, edx;
631
632	cpuid(op, &eax, &ebx, &ecx, &edx);
633
634	return eax;
635}
636
637static inline unsigned int cpuid_ebx(unsigned int op)
638{
639	unsigned int eax, ebx, ecx, edx;
640
641	cpuid(op, &eax, &ebx, &ecx, &edx);
642
643	return ebx;
644}
645
646static inline unsigned int cpuid_ecx(unsigned int op)
647{
648	unsigned int eax, ebx, ecx, edx;
649
650	cpuid(op, &eax, &ebx, &ecx, &edx);
651
652	return ecx;
653}
654
655static inline unsigned int cpuid_edx(unsigned int op)
656{
657	unsigned int eax, ebx, ecx, edx;
658
659	cpuid(op, &eax, &ebx, &ecx, &edx);
660
661	return edx;
662}
663
664/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
665static __always_inline void rep_nop(void)
666{
667	asm volatile("rep; nop" ::: "memory");
668}
669
670static __always_inline void cpu_relax(void)
671{
672	rep_nop();
673}
674
675/*
676 * This function forces the icache and prefetched instruction stream to
677 * catch up with reality in two very specific cases:
678 *
679 *  a) Text was modified using one virtual address and is about to be executed
680 *     from the same physical page at a different virtual address.
681 *
682 *  b) Text was modified on a different CPU, may subsequently be
683 *     executed on this CPU, and you want to make sure the new version
684 *     gets executed.  This generally means you're calling this in a IPI.
685 *
686 * If you're calling this for a different reason, you're probably doing
687 * it wrong.
688 */
689static inline void sync_core(void)
690{
691	/*
692	 * There are quite a few ways to do this.  IRET-to-self is nice
693	 * because it works on every CPU, at any CPL (so it's compatible
694	 * with paravirtualization), and it never exits to a hypervisor.
695	 * The only down sides are that it's a bit slow (it seems to be
696	 * a bit more than 2x slower than the fastest options) and that
697	 * it unmasks NMIs.  The "push %cs" is needed because, in
698	 * paravirtual environments, __KERNEL_CS may not be a valid CS
699	 * value when we do IRET directly.
700	 *
701	 * In case NMI unmasking or performance ever becomes a problem,
702	 * the next best option appears to be MOV-to-CR2 and an
703	 * unconditional jump.  That sequence also works on all CPUs,
704	 * but it will fault at CPL3 (i.e. Xen PV).
705	 *
706	 * CPUID is the conventional way, but it's nasty: it doesn't
707	 * exist on some 486-like CPUs, and it usually exits to a
708	 * hypervisor.
709	 *
710	 * Like all of Linux's memory ordering operations, this is a
711	 * compiler barrier as well.
712	 */
713#ifdef CONFIG_X86_32
714	asm volatile (
715		"pushfl\n\t"
716		"pushl %%cs\n\t"
717		"pushl $1f\n\t"
718		"iret\n\t"
719		"1:"
720		: ASM_CALL_CONSTRAINT : : "memory");
721#else
722	unsigned int tmp;
723
724	asm volatile (
725		UNWIND_HINT_SAVE
726		"mov %%ss, %0\n\t"
727		"pushq %q0\n\t"
728		"pushq %%rsp\n\t"
729		"addq $8, (%%rsp)\n\t"
730		"pushfq\n\t"
731		"mov %%cs, %0\n\t"
732		"pushq %q0\n\t"
733		"pushq $1f\n\t"
734		"iretq\n\t"
735		UNWIND_HINT_RESTORE
736		"1:"
737		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
738#endif
739}
740
741extern void select_idle_routine(const struct cpuinfo_x86 *c);
742extern void amd_e400_c1e_apic_setup(void);
743
744extern unsigned long		boot_option_idle_override;
745
746enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
747			 IDLE_POLL};
748
749extern void enable_sep_cpu(void);
750extern int sysenter_setup(void);
751
752void early_trap_pf_init(void);
753
754/* Defined in head.S */
755extern struct desc_ptr		early_gdt_descr;
756
757extern void switch_to_new_gdt(int);
758extern void load_direct_gdt(int);
759extern void load_fixmap_gdt(int);
760extern void load_percpu_segment(int);
761extern void cpu_init(void);
 
 
 
762
763static inline unsigned long get_debugctlmsr(void)
764{
765	unsigned long debugctlmsr = 0;
766
767#ifndef CONFIG_X86_DEBUGCTLMSR
768	if (boot_cpu_data.x86 < 6)
769		return 0;
770#endif
771	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
772
773	return debugctlmsr;
774}
775
776static inline void update_debugctlmsr(unsigned long debugctlmsr)
777{
778#ifndef CONFIG_X86_DEBUGCTLMSR
779	if (boot_cpu_data.x86 < 6)
780		return;
781#endif
782	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
783}
784
785extern void set_task_blockstep(struct task_struct *task, bool on);
786
787/* Boot loader type from the setup header: */
788extern int			bootloader_type;
789extern int			bootloader_version;
790
791extern char			ignore_fpu_irq;
792
793#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
794#define ARCH_HAS_PREFETCHW
795#define ARCH_HAS_SPINLOCK_PREFETCH
796
797#ifdef CONFIG_X86_32
798# define BASE_PREFETCH		""
799# define ARCH_HAS_PREFETCH
800#else
801# define BASE_PREFETCH		"prefetcht0 %P1"
802#endif
803
804/*
805 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
806 *
807 * It's not worth to care about 3dnow prefetches for the K6
808 * because they are microcoded there and very slow.
809 */
810static inline void prefetch(const void *x)
811{
812	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
813			  X86_FEATURE_XMM,
814			  "m" (*(const char *)x));
815}
816
817/*
818 * 3dnow prefetch to get an exclusive cache line.
819 * Useful for spinlocks to avoid one state transition in the
820 * cache coherency protocol:
821 */
822static inline void prefetchw(const void *x)
823{
824	alternative_input(BASE_PREFETCH, "prefetchw %P1",
825			  X86_FEATURE_3DNOWPREFETCH,
826			  "m" (*(const char *)x));
827}
828
829static inline void spin_lock_prefetch(const void *x)
830{
831	prefetchw(x);
832}
833
834#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
835			   TOP_OF_KERNEL_STACK_PADDING)
836
837#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))
838
839#define task_pt_regs(task) \
840({									\
841	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
842	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
843	((struct pt_regs *)__ptr) - 1;					\
844})
845
846#ifdef CONFIG_X86_32
847/*
848 * User space process size: 3GB (default).
849 */
850#define IA32_PAGE_OFFSET	PAGE_OFFSET
851#define TASK_SIZE		PAGE_OFFSET
852#define TASK_SIZE_LOW		TASK_SIZE
853#define TASK_SIZE_MAX		TASK_SIZE
854#define DEFAULT_MAP_WINDOW	TASK_SIZE
855#define STACK_TOP		TASK_SIZE
856#define STACK_TOP_MAX		STACK_TOP
857
858#define INIT_THREAD  {							  \
859	.sp0			= TOP_OF_INIT_STACK,			  \
860	.sysenter_cs		= __KERNEL_CS,				  \
861	.io_bitmap_ptr		= NULL,					  \
862	.addr_limit		= KERNEL_DS,				  \
863}
864
865#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
866
867#else
868/*
869 * User space process size.  This is the first address outside the user range.
870 * There are a few constraints that determine this:
871 *
872 * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
873 * address, then that syscall will enter the kernel with a
874 * non-canonical return address, and SYSRET will explode dangerously.
875 * We avoid this particular problem by preventing anything executable
876 * from being mapped at the maximum canonical address.
877 *
878 * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
879 * CPUs malfunction if they execute code from the highest canonical page.
880 * They'll speculate right off the end of the canonical space, and
881 * bad things happen.  This is worked around in the same way as the
882 * Intel problem.
883 *
884 * With page table isolation enabled, we map the LDT in ... [stay tuned]
885 */
886#define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
887
888#define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
889
890/* This decides where the kernel will search for a free chunk of vm
891 * space during mmap's.
892 */
893#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
894					0xc0000000 : 0xFFFFe000)
895
896#define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
897					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
898#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
899					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
900#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
901					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
902
903#define STACK_TOP		TASK_SIZE_LOW
904#define STACK_TOP_MAX		TASK_SIZE_MAX
905
906#define INIT_THREAD  {						\
907	.addr_limit		= KERNEL_DS,			\
908}
909
910extern unsigned long KSTK_ESP(struct task_struct *task);
911
912#endif /* CONFIG_X86_64 */
913
914extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
915					       unsigned long new_sp);
916
917/*
918 * This decides where the kernel will search for a free chunk of vm
919 * space during mmap's.
920 */
921#define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
922#define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
923
924#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
925
926/* Get/set a process' ability to use the timestamp counter instruction */
927#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
928#define SET_TSC_CTL(val)	set_tsc_mode((val))
929
930extern int get_tsc_mode(unsigned long adr);
931extern int set_tsc_mode(unsigned int val);
932
933DECLARE_PER_CPU(u64, msr_misc_features_shadow);
934
935/* Register/unregister a process' MPX related resource */
936#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
937#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
938
939#ifdef CONFIG_X86_INTEL_MPX
940extern int mpx_enable_management(void);
941extern int mpx_disable_management(void);
942#else
943static inline int mpx_enable_management(void)
944{
945	return -EINVAL;
946}
947static inline int mpx_disable_management(void)
948{
949	return -EINVAL;
950}
951#endif /* CONFIG_X86_INTEL_MPX */
952
953#ifdef CONFIG_CPU_SUP_AMD
954extern u16 amd_get_nb_id(int cpu);
955extern u32 amd_get_nodes_per_socket(void);
 
956#else
957static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
958static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
 
959#endif
960
961static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
962{
963	uint32_t base, eax, signature[3];
964
965	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
966		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
967
968		if (!memcmp(sig, signature, 12) &&
969		    (leaves == 0 || ((eax - base) >= leaves)))
970			return base;
971	}
972
973	return 0;
974}
975
976extern unsigned long arch_align_stack(unsigned long sp);
977extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
 
978
979void default_idle(void);
980#ifdef	CONFIG_XEN
981bool xen_set_default_idle(void);
982#else
983#define xen_set_default_idle 0
984#endif
985
986void stop_this_cpu(void *dummy);
987void df_debug(struct pt_regs *regs, long error_code);
988void microcode_check(void);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
989#endif /* _ASM_X86_PROCESSOR_H */