Linux Audio

Check our new training course

Yocto / OpenEmbedded training

Feb 10-13, 2025
Register
Loading...
v3.1
  1/*
  2 *  linux/boot/head.S
  3 *
  4 *  Copyright (C) 1991, 1992, 1993  Linus Torvalds
  5 */
  6
  7/*
  8 *  head.S contains the 32-bit startup code.
  9 *
 10 * NOTE!!! Startup happens at absolute address 0x00001000, which is also where
 11 * the page directory will exist. The startup code will be overwritten by
 12 * the page directory. [According to comments etc elsewhere on a compressed
 13 * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
 14 *
 15 * Page 0 is deliberately kept safe, since System Management Mode code in 
 16 * laptops may need to access the BIOS data stored there.  This is also
 17 * useful for future device drivers that either access the BIOS via VM86 
 18 * mode.
 19 */
 20
 21/*
 22 * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
 23 */
 24	.code32
 25	.text
 26
 27#include <linux/init.h>
 28#include <linux/linkage.h>
 29#include <asm/segment.h>
 30#include <asm/pgtable_types.h>
 31#include <asm/page_types.h>
 32#include <asm/boot.h>
 33#include <asm/msr.h>
 34#include <asm/processor-flags.h>
 35#include <asm/asm-offsets.h>
 
 
 
 
 
 
 
 
 
 36
 37	__HEAD
 38	.code32
 39ENTRY(startup_32)
 
 
 
 
 
 
 40	cld
 41	/*
 42	 * Test KEEP_SEGMENTS flag to see if the bootloader is asking
 43	 * us to not reload segments
 44	 */
 45	testb $(1<<6), BP_loadflags(%esi)
 46	jnz 1f
 47
 48	cli
 49	movl	$(__KERNEL_DS), %eax
 50	movl	%eax, %ds
 51	movl	%eax, %es
 52	movl	%eax, %ss
 531:
 54
 55/*
 56 * Calculate the delta between where we were compiled to run
 57 * at and where we were actually loaded at.  This can only be done
 58 * with a short local call on x86.  Nothing  else will tell us what
 59 * address we are running at.  The reserved chunk of the real-mode
 60 * data at 0x1e4 (defined as a scratch field) are used as the stack
 61 * for this calculation. Only 4 bytes are needed.
 62 */
 63	leal	(BP_scratch+4)(%esi), %esp
 64	call	1f
 651:	popl	%ebp
 66	subl	$1b, %ebp
 67
 68/* setup a stack and make sure cpu supports long mode. */
 69	movl	$boot_stack_end, %eax
 70	addl	%ebp, %eax
 71	movl	%eax, %esp
 72
 73	call	verify_cpu
 74	testl	%eax, %eax
 75	jnz	no_longmode
 76
 77/*
 78 * Compute the delta between where we were compiled to run at
 79 * and where the code will actually run at.
 80 *
 81 * %ebp contains the address we are loaded at by the boot loader and %ebx
 82 * contains the address where we should move the kernel image temporarily
 83 * for safe in-place decompression.
 84 */
 85
 86#ifdef CONFIG_RELOCATABLE
 87	movl	%ebp, %ebx
 88	movl	BP_kernel_alignment(%esi), %eax
 89	decl	%eax
 90	addl	%eax, %ebx
 91	notl	%eax
 92	andl	%eax, %ebx
 93#else
 94	movl	$LOAD_PHYSICAL_ADDR, %ebx
 95#endif
 
 
 96
 97	/* Target address to relocate to for decompression */
 98	addl	$z_extract_offset, %ebx
 99
100/*
101 * Prepare for entering 64 bit mode
102 */
103
104	/* Load new GDT with the 64bit segments using 32bit descriptor */
105	leal	gdt(%ebp), %eax
106	movl	%eax, gdt+2(%ebp)
107	lgdt	gdt(%ebp)
108
109	/* Enable PAE mode */
110	movl	$(X86_CR4_PAE), %eax
 
111	movl	%eax, %cr4
112
113 /*
114  * Build early 4G boot pagetable
115  */
116	/* Initialize Page tables to 0 */
117	leal	pgtable(%ebx), %edi
118	xorl	%eax, %eax
119	movl	$((4096*6)/4), %ecx
120	rep	stosl
121
122	/* Build Level 4 */
123	leal	pgtable + 0(%ebx), %edi
124	leal	0x1007 (%edi), %eax
125	movl	%eax, 0(%edi)
126
127	/* Build Level 3 */
128	leal	pgtable + 0x1000(%ebx), %edi
129	leal	0x1007(%edi), %eax
130	movl	$4, %ecx
1311:	movl	%eax, 0x00(%edi)
132	addl	$0x00001000, %eax
133	addl	$8, %edi
134	decl	%ecx
135	jnz	1b
136
137	/* Build Level 2 */
138	leal	pgtable + 0x2000(%ebx), %edi
139	movl	$0x00000183, %eax
140	movl	$2048, %ecx
1411:	movl	%eax, 0(%edi)
142	addl	$0x00200000, %eax
143	addl	$8, %edi
144	decl	%ecx
145	jnz	1b
146
147	/* Enable the boot page tables */
148	leal	pgtable(%ebx), %eax
149	movl	%eax, %cr3
150
151	/* Enable Long mode in EFER (Extended Feature Enable Register) */
152	movl	$MSR_EFER, %ecx
153	rdmsr
154	btsl	$_EFER_LME, %eax
155	wrmsr
156
 
 
 
 
 
 
157	/*
158	 * Setup for the jump to 64bit mode
159	 *
160	 * When the jump is performend we will be in long mode but
161	 * in 32bit compatibility mode with EFER.LME = 1, CS.L = 0, CS.D = 1
162	 * (and in turn EFER.LMA = 1).	To jump into 64bit mode we use
163	 * the new gdt/idt that has __KERNEL_CS with CS.L = 1.
164	 * We place all of the values on our mini stack so lret can
165	 * used to perform that far jump.
166	 */
167	pushl	$__KERNEL_CS
168	leal	startup_64(%ebp), %eax
 
 
 
 
 
 
 
169	pushl	%eax
170
171	/* Enter paged protected Mode, activating Long Mode */
172	movl	$(X86_CR0_PG | X86_CR0_PE), %eax /* Enable Paging and Protected mode */
173	movl	%eax, %cr0
174
175	/* Jump from 32bit compatibility mode into 64bit mode. */
176	lret
177ENDPROC(startup_32)
178
179no_longmode:
180	/* This isn't an x86-64 CPU so hang */
1811:
182	hlt
183	jmp     1b
 
 
184
185#include "../../kernel/verify_cpu.S"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
187	/*
188	 * Be careful here startup_64 needs to be at a predictable
189	 * address so I can export it in an ELF header.  Bootloaders
190	 * should look at the ELF header to find this address, as
191	 * it may change in the future.
192	 */
193	.code64
194	.org 0x200
195ENTRY(startup_64)
196	/*
 
197	 * We come here either from startup_32 or directly from a
198	 * 64bit bootloader.  If we come here from a bootloader we depend on
199	 * an identity mapped page table being provied that maps our
200	 * entire text+data+bss and hopefully all of memory.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201	 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
203	/* Setup data segments. */
204	xorl	%eax, %eax
205	movl	%eax, %ds
206	movl	%eax, %es
207	movl	%eax, %ss
208	movl	%eax, %fs
209	movl	%eax, %gs
210	lldt	%ax
211	movl    $0x20, %eax
212	ltr	%ax
213
214	/*
215	 * Compute the decompressed kernel start address.  It is where
216	 * we were loaded at aligned to a 2M boundary. %rbp contains the
217	 * decompressed kernel start address.
218	 *
219	 * If it is a relocatable kernel then decompress and run the kernel
220	 * from load address aligned to 2MB addr, otherwise decompress and
221	 * run the kernel from LOAD_PHYSICAL_ADDR
222	 *
223	 * We cannot rely on the calculation done in 32-bit mode, since we
224	 * may have been invoked via the 64-bit entry point.
225	 */
226
227	/* Start with the delta to where the kernel will run at. */
228#ifdef CONFIG_RELOCATABLE
229	leaq	startup_32(%rip) /* - $startup_32 */, %rbp
230	movl	BP_kernel_alignment(%rsi), %eax
231	decl	%eax
232	addq	%rax, %rbp
233	notq	%rax
234	andq	%rax, %rbp
235#else
236	movq	$LOAD_PHYSICAL_ADDR, %rbp
237#endif
 
 
238
239	/* Target address to relocate to for decompression */
240	leaq	z_extract_offset(%rbp), %rbx
241
242	/* Set up the stack */
243	leaq	boot_stack_end(%rbx), %rsp
244
245	/* Zero EFLAGS */
246	pushq	$0
247	popfq
248
249/*
250 * Copy the compressed kernel to the end of our buffer
251 * where decompression in place becomes safe.
252 */
253	pushq	%rsi
254	leaq	(_bss-8)(%rip), %rsi
255	leaq	(_bss-8)(%rbx), %rdi
256	movq	$_bss /* - $startup_32 */, %rcx
257	shrq	$3, %rcx
258	std
259	rep	movsq
260	cld
261	popq	%rsi
262
263/*
264 * Jump to the relocated address.
265 */
266	leaq	relocated(%rbx), %rax
267	jmp	*%rax
268
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269	.text
270relocated:
271
272/*
273 * Clear BSS (stack is currently empty)
274 */
275	xorl	%eax, %eax
276	leaq    _bss(%rip), %rdi
277	leaq    _ebss(%rip), %rcx
278	subq	%rdi, %rcx
279	shrq	$3, %rcx
280	rep	stosq
281
282/*
283 * Adjust our own GOT
284 */
285	leaq	_got(%rip), %rdx
286	leaq	_egot(%rip), %rcx
2871:
288	cmpq	%rcx, %rdx
289	jae	2f
290	addq	%rbx, (%rdx)
291	addq	$8, %rdx
292	jmp	1b
2932:
294	
295/*
296 * Do the decompression, and jump to the new kernel..
297 */
298	pushq	%rsi			/* Save the real mode argument */
 
 
299	movq	%rsi, %rdi		/* real mode address */
300	leaq	boot_heap(%rip), %rsi	/* malloc area for uncompression */
301	leaq	input_data(%rip), %rdx  /* input_data */
302	movl	$z_input_len, %ecx	/* input_len */
303	movq	%rbp, %r8		/* output target address */
304	call	decompress_kernel
 
 
305	popq	%rsi
306
307/*
308 * Jump to the decompressed kernel.
309 */
310	jmp	*%rbp
 
 
 
 
 
 
 
 
 
311
312	.data
313gdt:
314	.word	gdt_end - gdt
315	.long	gdt
316	.word	0
317	.quad	0x0000000000000000	/* NULL descriptor */
318	.quad	0x00af9a000000ffff	/* __KERNEL_CS */
319	.quad	0x00cf92000000ffff	/* __KERNEL_DS */
320	.quad	0x0080890000000000	/* TS descriptor */
321	.quad   0x0000000000000000	/* TS continued */
322gdt_end:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
324/*
325 * Stack and heap for uncompression
326 */
327	.bss
328	.balign 4
329boot_heap:
330	.fill BOOT_HEAP_SIZE, 1, 0
331boot_stack:
332	.fill BOOT_STACK_SIZE, 1, 0
333boot_stack_end:
334
335/*
336 * Space for page tables (not in .bss so not zeroed)
337 */
338	.section ".pgtable","a",@nobits
339	.balign 4096
340pgtable:
341	.fill 6*4096, 1, 0
v4.6
  1/*
  2 *  linux/boot/head.S
  3 *
  4 *  Copyright (C) 1991, 1992, 1993  Linus Torvalds
  5 */
  6
  7/*
  8 *  head.S contains the 32-bit startup code.
  9 *
 10 * NOTE!!! Startup happens at absolute address 0x00001000, which is also where
 11 * the page directory will exist. The startup code will be overwritten by
 12 * the page directory. [According to comments etc elsewhere on a compressed
 13 * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
 14 *
 15 * Page 0 is deliberately kept safe, since System Management Mode code in 
 16 * laptops may need to access the BIOS data stored there.  This is also
 17 * useful for future device drivers that either access the BIOS via VM86 
 18 * mode.
 19 */
 20
 21/*
 22 * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
 23 */
 24	.code32
 25	.text
 26
 27#include <linux/init.h>
 28#include <linux/linkage.h>
 29#include <asm/segment.h>
 
 
 30#include <asm/boot.h>
 31#include <asm/msr.h>
 32#include <asm/processor-flags.h>
 33#include <asm/asm-offsets.h>
 34#include <asm/bootparam.h>
 35
 36/*
 37 * Locally defined symbols should be marked hidden:
 38 */
 39	.hidden _bss
 40	.hidden _ebss
 41	.hidden _got
 42	.hidden _egot
 43
 44	__HEAD
 45	.code32
 46ENTRY(startup_32)
 47	/*
 48	 * 32bit entry is 0 and it is ABI so immutable!
 49	 * If we come here directly from a bootloader,
 50	 * kernel(text+data+bss+brk) ramdisk, zero_page, command line
 51	 * all need to be under the 4G limit.
 52	 */
 53	cld
 54	/*
 55	 * Test KEEP_SEGMENTS flag to see if the bootloader is asking
 56	 * us to not reload segments
 57	 */
 58	testb $KEEP_SEGMENTS, BP_loadflags(%esi)
 59	jnz 1f
 60
 61	cli
 62	movl	$(__BOOT_DS), %eax
 63	movl	%eax, %ds
 64	movl	%eax, %es
 65	movl	%eax, %ss
 661:
 67
 68/*
 69 * Calculate the delta between where we were compiled to run
 70 * at and where we were actually loaded at.  This can only be done
 71 * with a short local call on x86.  Nothing  else will tell us what
 72 * address we are running at.  The reserved chunk of the real-mode
 73 * data at 0x1e4 (defined as a scratch field) are used as the stack
 74 * for this calculation. Only 4 bytes are needed.
 75 */
 76	leal	(BP_scratch+4)(%esi), %esp
 77	call	1f
 781:	popl	%ebp
 79	subl	$1b, %ebp
 80
 81/* setup a stack and make sure cpu supports long mode. */
 82	movl	$boot_stack_end, %eax
 83	addl	%ebp, %eax
 84	movl	%eax, %esp
 85
 86	call	verify_cpu
 87	testl	%eax, %eax
 88	jnz	no_longmode
 89
 90/*
 91 * Compute the delta between where we were compiled to run at
 92 * and where the code will actually run at.
 93 *
 94 * %ebp contains the address we are loaded at by the boot loader and %ebx
 95 * contains the address where we should move the kernel image temporarily
 96 * for safe in-place decompression.
 97 */
 98
 99#ifdef CONFIG_RELOCATABLE
100	movl	%ebp, %ebx
101	movl	BP_kernel_alignment(%esi), %eax
102	decl	%eax
103	addl	%eax, %ebx
104	notl	%eax
105	andl	%eax, %ebx
106	cmpl	$LOAD_PHYSICAL_ADDR, %ebx
107	jge	1f
108#endif
109	movl	$LOAD_PHYSICAL_ADDR, %ebx
1101:
111
112	/* Target address to relocate to for decompression */
113	addl	$z_extract_offset, %ebx
114
115/*
116 * Prepare for entering 64 bit mode
117 */
118
119	/* Load new GDT with the 64bit segments using 32bit descriptor */
120	leal	gdt(%ebp), %eax
121	movl	%eax, gdt+2(%ebp)
122	lgdt	gdt(%ebp)
123
124	/* Enable PAE mode */
125	movl	%cr4, %eax
126	orl	$X86_CR4_PAE, %eax
127	movl	%eax, %cr4
128
129 /*
130  * Build early 4G boot pagetable
131  */
132	/* Initialize Page tables to 0 */
133	leal	pgtable(%ebx), %edi
134	xorl	%eax, %eax
135	movl	$((4096*6)/4), %ecx
136	rep	stosl
137
138	/* Build Level 4 */
139	leal	pgtable + 0(%ebx), %edi
140	leal	0x1007 (%edi), %eax
141	movl	%eax, 0(%edi)
142
143	/* Build Level 3 */
144	leal	pgtable + 0x1000(%ebx), %edi
145	leal	0x1007(%edi), %eax
146	movl	$4, %ecx
1471:	movl	%eax, 0x00(%edi)
148	addl	$0x00001000, %eax
149	addl	$8, %edi
150	decl	%ecx
151	jnz	1b
152
153	/* Build Level 2 */
154	leal	pgtable + 0x2000(%ebx), %edi
155	movl	$0x00000183, %eax
156	movl	$2048, %ecx
1571:	movl	%eax, 0(%edi)
158	addl	$0x00200000, %eax
159	addl	$8, %edi
160	decl	%ecx
161	jnz	1b
162
163	/* Enable the boot page tables */
164	leal	pgtable(%ebx), %eax
165	movl	%eax, %cr3
166
167	/* Enable Long mode in EFER (Extended Feature Enable Register) */
168	movl	$MSR_EFER, %ecx
169	rdmsr
170	btsl	$_EFER_LME, %eax
171	wrmsr
172
173	/* After gdt is loaded */
174	xorl	%eax, %eax
175	lldt	%ax
176	movl    $__BOOT_TSS, %eax
177	ltr	%ax
178
179	/*
180	 * Setup for the jump to 64bit mode
181	 *
182	 * When the jump is performend we will be in long mode but
183	 * in 32bit compatibility mode with EFER.LME = 1, CS.L = 0, CS.D = 1
184	 * (and in turn EFER.LMA = 1).	To jump into 64bit mode we use
185	 * the new gdt/idt that has __KERNEL_CS with CS.L = 1.
186	 * We place all of the values on our mini stack so lret can
187	 * used to perform that far jump.
188	 */
189	pushl	$__KERNEL_CS
190	leal	startup_64(%ebp), %eax
191#ifdef CONFIG_EFI_MIXED
192	movl	efi32_config(%ebp), %ebx
193	cmp	$0, %ebx
194	jz	1f
195	leal	handover_entry(%ebp), %eax
1961:
197#endif
198	pushl	%eax
199
200	/* Enter paged protected Mode, activating Long Mode */
201	movl	$(X86_CR0_PG | X86_CR0_PE), %eax /* Enable Paging and Protected mode */
202	movl	%eax, %cr0
203
204	/* Jump from 32bit compatibility mode into 64bit mode. */
205	lret
206ENDPROC(startup_32)
207
208#ifdef CONFIG_EFI_MIXED
209	.org 0x190
210ENTRY(efi32_stub_entry)
211	add	$0x4, %esp		/* Discard return address */
212	popl	%ecx
213	popl	%edx
214	popl	%esi
215
216	leal	(BP_scratch+4)(%esi), %esp
217	call	1f
2181:	pop	%ebp
219	subl	$1b, %ebp
220
221	movl	%ecx, efi32_config(%ebp)
222	movl	%edx, efi32_config+8(%ebp)
223	sgdtl	efi32_boot_gdt(%ebp)
224
225	leal	efi32_config(%ebp), %eax
226	movl	%eax, efi_config(%ebp)
227
228	jmp	startup_32
229ENDPROC(efi32_stub_entry)
230#endif
231
 
 
 
 
 
 
232	.code64
233	.org 0x200
234ENTRY(startup_64)
235	/*
236	 * 64bit entry is 0x200 and it is ABI so immutable!
237	 * We come here either from startup_32 or directly from a
238	 * 64bit bootloader.
239	 * If we come here from a bootloader, kernel(text+data+bss+brk),
240	 * ramdisk, zero_page, command line could be above 4G.
241	 * We depend on an identity mapped page table being provided
242	 * that maps our entire kernel(text+data+bss+brk), zero page
243	 * and command line.
244	 */
245#ifdef CONFIG_EFI_STUB
246	/*
247	 * The entry point for the PE/COFF executable is efi_pe_entry, so
248	 * only legacy boot loaders will execute this jmp.
249	 */
250	jmp	preferred_addr
251
252ENTRY(efi_pe_entry)
253	movq	%rcx, efi64_config(%rip)	/* Handle */
254	movq	%rdx, efi64_config+8(%rip) /* EFI System table pointer */
255
256	leaq	efi64_config(%rip), %rax
257	movq	%rax, efi_config(%rip)
258
259	call	1f
2601:	popq	%rbp
261	subq	$1b, %rbp
262
263	/*
264	 * Relocate efi_config->call().
265	 */
266	addq	%rbp, efi64_config+88(%rip)
267
268	movq	%rax, %rdi
269	call	make_boot_params
270	cmpq	$0,%rax
271	je	fail
272	mov	%rax, %rsi
273	leaq	startup_32(%rip), %rax
274	movl	%eax, BP_code32_start(%rsi)
275	jmp	2f		/* Skip the relocation */
276
277handover_entry:
278	call	1f
2791:	popq	%rbp
280	subq	$1b, %rbp
281
282	/*
283	 * Relocate efi_config->call().
284	 */
285	movq	efi_config(%rip), %rax
286	addq	%rbp, 88(%rax)
2872:
288	movq	efi_config(%rip), %rdi
289	call	efi_main
290	movq	%rax,%rsi
291	cmpq	$0,%rax
292	jne	2f
293fail:
294	/* EFI init failed, so hang. */
295	hlt
296	jmp	fail
2972:
298	movl	BP_code32_start(%esi), %eax
299	leaq	preferred_addr(%rax), %rax
300	jmp	*%rax
301
302preferred_addr:
303#endif
304
305	/* Setup data segments. */
306	xorl	%eax, %eax
307	movl	%eax, %ds
308	movl	%eax, %es
309	movl	%eax, %ss
310	movl	%eax, %fs
311	movl	%eax, %gs
 
 
 
312
313	/*
314	 * Compute the decompressed kernel start address.  It is where
315	 * we were loaded at aligned to a 2M boundary. %rbp contains the
316	 * decompressed kernel start address.
317	 *
318	 * If it is a relocatable kernel then decompress and run the kernel
319	 * from load address aligned to 2MB addr, otherwise decompress and
320	 * run the kernel from LOAD_PHYSICAL_ADDR
321	 *
322	 * We cannot rely on the calculation done in 32-bit mode, since we
323	 * may have been invoked via the 64-bit entry point.
324	 */
325
326	/* Start with the delta to where the kernel will run at. */
327#ifdef CONFIG_RELOCATABLE
328	leaq	startup_32(%rip) /* - $startup_32 */, %rbp
329	movl	BP_kernel_alignment(%rsi), %eax
330	decl	%eax
331	addq	%rax, %rbp
332	notq	%rax
333	andq	%rax, %rbp
334	cmpq	$LOAD_PHYSICAL_ADDR, %rbp
335	jge	1f
336#endif
337	movq	$LOAD_PHYSICAL_ADDR, %rbp
3381:
339
340	/* Target address to relocate to for decompression */
341	leaq	z_extract_offset(%rbp), %rbx
342
343	/* Set up the stack */
344	leaq	boot_stack_end(%rbx), %rsp
345
346	/* Zero EFLAGS */
347	pushq	$0
348	popfq
349
350/*
351 * Copy the compressed kernel to the end of our buffer
352 * where decompression in place becomes safe.
353 */
354	pushq	%rsi
355	leaq	(_bss-8)(%rip), %rsi
356	leaq	(_bss-8)(%rbx), %rdi
357	movq	$_bss /* - $startup_32 */, %rcx
358	shrq	$3, %rcx
359	std
360	rep	movsq
361	cld
362	popq	%rsi
363
364/*
365 * Jump to the relocated address.
366 */
367	leaq	relocated(%rbx), %rax
368	jmp	*%rax
369
370#ifdef CONFIG_EFI_STUB
371	.org 0x390
372ENTRY(efi64_stub_entry)
373	movq	%rdi, efi64_config(%rip)	/* Handle */
374	movq	%rsi, efi64_config+8(%rip) /* EFI System table pointer */
375
376	leaq	efi64_config(%rip), %rax
377	movq	%rax, efi_config(%rip)
378
379	movq	%rdx, %rsi
380	jmp	handover_entry
381ENDPROC(efi64_stub_entry)
382#endif
383
384	.text
385relocated:
386
387/*
388 * Clear BSS (stack is currently empty)
389 */
390	xorl	%eax, %eax
391	leaq    _bss(%rip), %rdi
392	leaq    _ebss(%rip), %rcx
393	subq	%rdi, %rcx
394	shrq	$3, %rcx
395	rep	stosq
396
397/*
398 * Adjust our own GOT
399 */
400	leaq	_got(%rip), %rdx
401	leaq	_egot(%rip), %rcx
4021:
403	cmpq	%rcx, %rdx
404	jae	2f
405	addq	%rbx, (%rdx)
406	addq	$8, %rdx
407	jmp	1b
4082:
409	
410/*
411 * Do the decompression, and jump to the new kernel..
412 */
413	pushq	%rsi			/* Save the real mode argument */
414	movq	$z_run_size, %r9	/* size of kernel with .bss and .brk */
415	pushq	%r9
416	movq	%rsi, %rdi		/* real mode address */
417	leaq	boot_heap(%rip), %rsi	/* malloc area for uncompression */
418	leaq	input_data(%rip), %rdx  /* input_data */
419	movl	$z_input_len, %ecx	/* input_len */
420	movq	%rbp, %r8		/* output target address */
421	movq	$z_output_len, %r9	/* decompressed length, end of relocs */
422	call	decompress_kernel	/* returns kernel location in %rax */
423	popq	%r9
424	popq	%rsi
425
426/*
427 * Jump to the decompressed kernel.
428 */
429	jmp	*%rax
430
431	.code32
432no_longmode:
433	/* This isn't an x86-64 CPU so hang */
4341:
435	hlt
436	jmp     1b
437
438#include "../../kernel/verify_cpu.S"
439
440	.data
441gdt:
442	.word	gdt_end - gdt
443	.long	gdt
444	.word	0
445	.quad	0x0000000000000000	/* NULL descriptor */
446	.quad	0x00af9a000000ffff	/* __KERNEL_CS */
447	.quad	0x00cf92000000ffff	/* __KERNEL_DS */
448	.quad	0x0080890000000000	/* TS descriptor */
449	.quad   0x0000000000000000	/* TS continued */
450gdt_end:
451
452#ifdef CONFIG_EFI_STUB
453efi_config:
454	.quad	0
455
456#ifdef CONFIG_EFI_MIXED
457	.global efi32_config
458efi32_config:
459	.fill	11,8,0
460	.quad	efi64_thunk
461	.byte	0
462#endif
463
464	.global efi64_config
465efi64_config:
466	.fill	11,8,0
467	.quad	efi_call
468	.byte	1
469#endif /* CONFIG_EFI_STUB */
470
471/*
472 * Stack and heap for uncompression
473 */
474	.bss
475	.balign 4
476boot_heap:
477	.fill BOOT_HEAP_SIZE, 1, 0
478boot_stack:
479	.fill BOOT_STACK_SIZE, 1, 0
480boot_stack_end:
481
482/*
483 * Space for page tables (not in .bss so not zeroed)
484 */
485	.section ".pgtable","a",@nobits
486	.balign 4096
487pgtable:
488	.fill 6*4096, 1, 0