1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012 Regents of the University of California
  4 */
  5
  6#include <asm/thread_info.h>
  7#include <asm/asm-offsets.h>
  8#include <asm/asm.h>
  9#include <linux/init.h>
 10#include <linux/linkage.h>
 11#include <asm/thread_info.h>
 12#include <asm/page.h>
 
 13#include <asm/csr.h>
 
 14#include <asm/image.h>
 
 
 
 
 
 
 
 
 
 
 
 
 15
 16__INIT
 17ENTRY(_start)
 18	/*
 19	 * Image header expected by Linux boot-loaders. The image header data
 20	 * structure is described in asm/image.h.
 21	 * Do not modify it without modifying the structure and all bootloaders
 22	 * that expects this header format!!
 23	 */
 
 
 
 
 
 
 
 24	/* jump to start kernel */
 25	j _start_kernel
 26	/* reserved */
 27	.word 0
 
 28	.balign 8
 
 
 
 
 29#if __riscv_xlen == 64
 30	/* Image load offset(2MB) from start of RAM */
 31	.dword 0x200000
 32#else
 33	/* Image load offset(4MB) from start of RAM */
 34	.dword 0x400000
 35#endif
 
 36	/* Effective size of kernel image */
 37	.dword _end - _start
 38	.dword __HEAD_FLAGS
 39	.word RISCV_HEADER_VERSION
 40	.word 0
 41	.dword 0
 42	.ascii RISCV_IMAGE_MAGIC
 43	.balign 4
 44	.ascii RISCV_IMAGE_MAGIC2
 
 
 
 
 
 
 45	.word 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 46
 47.global _start_kernel
 48_start_kernel:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 49	/* Mask all interrupts */
 50	csrw CSR_SIE, zero
 51	csrw CSR_SIP, zero
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 52
 53	/* Load the global pointer */
 54.option push
 55.option norelax
 56	la gp, __global_pointer$
 57.option pop
 58
 59	/*
 60	 * Disable FPU to detect illegal usage of
 61	 * floating point in kernel space
 62	 */
 63	li t0, SR_FS
 64	csrc CSR_SSTATUS, t0
 65
 66#ifdef CONFIG_SMP
 67	li t0, CONFIG_NR_CPUS
 68	bgeu a0, t0, .Lsecondary_park
 
 
 69#endif
 70
 
 71	/* Pick one hart to run the main boot sequence */
 72	la a3, hart_lottery
 73	li a2, 1
 74	amoadd.w a3, a2, (a3)
 75	bnez a3, .Lsecondary_start
 76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 77	/* Clear BSS for flat non-ELF images */
 78	la a3, __bss_start
 79	la a4, __bss_stop
 80	ble a4, a3, clear_bss_done
 81clear_bss:
 82	REG_S zero, (a3)
 83	add a3, a3, RISCV_SZPTR
 84	blt a3, a4, clear_bss
 85clear_bss_done:
 86
 87	/* Save hart ID and DTB physical address */
 88	mv s0, a0
 89	mv s1, a1
 
 90	la a2, boot_cpu_hartid
 
 91	REG_S a0, (a2)
 92
 93	/* Initialize page tables and relocate to virtual addresses */
 94	la sp, init_thread_union + THREAD_SIZE
 
 
 
 
 95	mv a0, s1
 
 96	call setup_vm
 
 97	la a0, early_pg_dir
 
 98	call relocate
 
 99
 
100	/* Restore C environment */
101	la tp, init_task
102	sw zero, TASK_TI_CPU(tp)
103	la sp, init_thread_union + THREAD_SIZE
104
 
 
 
105	/* Start the kernel */
106	call parse_dtb
107	tail start_kernel
108
109relocate:
110	/* Relocate return address */
111	li a1, PAGE_OFFSET
112	la a2, _start
113	sub a1, a1, a2
114	add ra, ra, a1
115
116	/* Point stvec to virtual address of intruction after satp write */
117	la a2, 1f
118	add a2, a2, a1
119	csrw CSR_STVEC, a2
120
121	/* Compute satp for kernel page tables, but don't load it yet */
122	srl a2, a0, PAGE_SHIFT
123	li a1, SATP_MODE
124	or a2, a2, a1
125
126	/*
127	 * Load trampoline page directory, which will cause us to trap to
128	 * stvec if VA != PA, or simply fall through if VA == PA.  We need a
129	 * full fence here because setup_vm() just wrote these PTEs and we need
130	 * to ensure the new translations are in use.
131	 */
132	la a0, trampoline_pg_dir
133	srl a0, a0, PAGE_SHIFT
134	or a0, a0, a1
135	sfence.vma
136	csrw CSR_SATP, a0
137.align 2
1381:
139	/* Set trap vector to spin forever to help debug */
140	la a0, .Lsecondary_park
141	csrw CSR_STVEC, a0
142
143	/* Reload the global pointer */
144.option push
145.option norelax
146	la gp, __global_pointer$
147.option pop
148
149	/*
150	 * Switch to kernel page tables.  A full fence is necessary in order to
151	 * avoid using the trampoline translations, which are only correct for
152	 * the first superpage.  Fetching the fence is guarnteed to work
153	 * because that first superpage is translated the same way.
154	 */
155	csrw CSR_SATP, a2
156	sfence.vma
157
158	ret
159
160.Lsecondary_start:
161#ifdef CONFIG_SMP
162	/* Set trap vector to spin forever to help debug */
163	la a3, .Lsecondary_park
164	csrw CSR_STVEC, a3
165
166	slli a3, a0, LGREG
167	la a1, __cpu_up_stack_pointer
 
168	la a2, __cpu_up_task_pointer
 
169	add a1, a3, a1
170	add a2, a3, a2
171
172	/*
173	 * This hart didn't win the lottery, so we wait for the winning hart to
174	 * get far enough along the boot process that it should continue.
175	 */
176.Lwait_for_cpu_up:
177	/* FIXME: We should WFI to save some energy here. */
178	REG_L sp, (a1)
179	REG_L tp, (a2)
180	beqz sp, .Lwait_for_cpu_up
181	beqz tp, .Lwait_for_cpu_up
182	fence
183
184	/* Enable virtual memory and relocate to virtual address */
185	la a0, swapper_pg_dir
186	call relocate
187
188	tail smp_callin
189#endif
190
191.align 2
192.Lsecondary_park:
193	/* We lack SMP support or have too many harts, so park this hart */
194	wfi
195	j .Lsecondary_park
196END(_start)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
197
198__PAGE_ALIGNED_BSS
199	/* Empty zero page */
200	.balign PAGE_SIZE

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012 Regents of the University of California
  4 */
  5
 
  6#include <asm/asm-offsets.h>
  7#include <asm/asm.h>
  8#include <linux/init.h>
  9#include <linux/linkage.h>
 10#include <asm/thread_info.h>
 11#include <asm/page.h>
 12#include <asm/pgtable.h>
 13#include <asm/csr.h>
 14#include <asm/hwcap.h>
 15#include <asm/image.h>
 16#include "efi-header.S"
 17
 18#ifdef CONFIG_XIP_KERNEL
 19.macro XIP_FIXUP_OFFSET reg
 20	REG_L t0, _xip_fixup
 21	add \reg, \reg, t0
 22.endm
 23_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET
 24#else
 25.macro XIP_FIXUP_OFFSET reg
 26.endm
 27#endif /* CONFIG_XIP_KERNEL */
 28
 29__HEAD
 30ENTRY(_start)
 31	/*
 32	 * Image header expected by Linux boot-loaders. The image header data
 33	 * structure is described in asm/image.h.
 34	 * Do not modify it without modifying the structure and all bootloaders
 35	 * that expects this header format!!
 36	 */
 37#ifdef CONFIG_EFI
 38	/*
 39	 * This instruction decodes to "MZ" ASCII required by UEFI.
 40	 */
 41	c.li s4,-13
 42	j _start_kernel
 43#else
 44	/* jump to start kernel */
 45	j _start_kernel
 46	/* reserved */
 47	.word 0
 48#endif
 49	.balign 8
 50#ifdef CONFIG_RISCV_M_MODE
 51	/* Image load offset (0MB) from start of RAM for M-mode */
 52	.dword 0
 53#else
 54#if __riscv_xlen == 64
 55	/* Image load offset(2MB) from start of RAM */
 56	.dword 0x200000
 57#else
 58	/* Image load offset(4MB) from start of RAM */
 59	.dword 0x400000
 60#endif
 61#endif
 62	/* Effective size of kernel image */
 63	.dword _end - _start
 64	.dword __HEAD_FLAGS
 65	.word RISCV_HEADER_VERSION
 66	.word 0
 67	.dword 0
 68	.ascii RISCV_IMAGE_MAGIC
 69	.balign 4
 70	.ascii RISCV_IMAGE_MAGIC2
 71#ifdef CONFIG_EFI
 72	.word pe_head_start - _start
 73pe_head_start:
 74
 75	__EFI_PE_HEADER
 76#else
 77	.word 0
 78#endif
 79
 80.align 2
 81#ifdef CONFIG_MMU
 82relocate:
 83	/* Relocate return address */
 84	la a1, kernel_map
 85	XIP_FIXUP_OFFSET a1
 86	REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
 87	la a2, _start
 88	sub a1, a1, a2
 89	add ra, ra, a1
 90
 91	/* Point stvec to virtual address of intruction after satp write */
 92	la a2, 1f
 93	add a2, a2, a1
 94	csrw CSR_TVEC, a2
 95
 96	/* Compute satp for kernel page tables, but don't load it yet */
 97	srl a2, a0, PAGE_SHIFT
 98	li a1, SATP_MODE
 99	or a2, a2, a1
100
101	/*
102	 * Load trampoline page directory, which will cause us to trap to
103	 * stvec if VA != PA, or simply fall through if VA == PA.  We need a
104	 * full fence here because setup_vm() just wrote these PTEs and we need
105	 * to ensure the new translations are in use.
106	 */
107	la a0, trampoline_pg_dir
108	XIP_FIXUP_OFFSET a0
109	srl a0, a0, PAGE_SHIFT
110	or a0, a0, a1
111	sfence.vma
112	csrw CSR_SATP, a0
113.align 2
1141:
115	/* Set trap vector to spin forever to help debug */
116	la a0, .Lsecondary_park
117	csrw CSR_TVEC, a0
118
119	/* Reload the global pointer */
120.option push
121.option norelax
122	la gp, __global_pointer$
123.option pop
124
125	/*
126	 * Switch to kernel page tables.  A full fence is necessary in order to
127	 * avoid using the trampoline translations, which are only correct for
128	 * the first superpage.  Fetching the fence is guarnteed to work
129	 * because that first superpage is translated the same way.
130	 */
131	csrw CSR_SATP, a2
132	sfence.vma
133
134	ret
135#endif /* CONFIG_MMU */
136#ifdef CONFIG_SMP
137	.global secondary_start_sbi
138secondary_start_sbi:
139	/* Mask all interrupts */
140	csrw CSR_IE, zero
141	csrw CSR_IP, zero
142
143	/* Load the global pointer */
144	.option push
145	.option norelax
146		la gp, __global_pointer$
147	.option pop
148
149	/*
150	 * Disable FPU to detect illegal usage of
151	 * floating point in kernel space
152	 */
153	li t0, SR_FS
154	csrc CSR_STATUS, t0
155
156	/* Set trap vector to spin forever to help debug */
157	la a3, .Lsecondary_park
158	csrw CSR_TVEC, a3
159
160	slli a3, a0, LGREG
161	la a4, __cpu_up_stack_pointer
162	XIP_FIXUP_OFFSET a4
163	la a5, __cpu_up_task_pointer
164	XIP_FIXUP_OFFSET a5
165	add a4, a3, a4
166	add a5, a3, a5
167	REG_L sp, (a4)
168	REG_L tp, (a5)
169
170	.global secondary_start_common
171secondary_start_common:
172
173#ifdef CONFIG_MMU
174	/* Enable virtual memory and relocate to virtual address */
175	la a0, swapper_pg_dir
176	XIP_FIXUP_OFFSET a0
177	call relocate
178#endif
179	call setup_trap_vector
180	tail smp_callin
181#endif /* CONFIG_SMP */
182
183.align 2
184setup_trap_vector:
185	/* Set trap vector to exception handler */
186	la a0, handle_exception
187	csrw CSR_TVEC, a0
188
189	/*
190	 * Set sup0 scratch register to 0, indicating to exception vector that
191	 * we are presently executing in kernel.
192	 */
193	csrw CSR_SCRATCH, zero
194	ret
195
196.Lsecondary_park:
197	/* We lack SMP support or have too many harts, so park this hart */
198	wfi
199	j .Lsecondary_park
200
201END(_start)
202
203ENTRY(_start_kernel)
204	/* Mask all interrupts */
205	csrw CSR_IE, zero
206	csrw CSR_IP, zero
207
208#ifdef CONFIG_RISCV_M_MODE
209	/* flush the instruction cache */
210	fence.i
211
212	/* Reset all registers except ra, a0, a1 */
213	call reset_regs
214
215	/*
216	 * Setup a PMP to permit access to all of memory.  Some machines may
217	 * not implement PMPs, so we set up a quick trap handler to just skip
218	 * touching the PMPs on any trap.
219	 */
220	la a0, pmp_done
221	csrw CSR_TVEC, a0
222
223	li a0, -1
224	csrw CSR_PMPADDR0, a0
225	li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X)
226	csrw CSR_PMPCFG0, a0
227.align 2
228pmp_done:
229
230	/*
231	 * The hartid in a0 is expected later on, and we have no firmware
232	 * to hand it to us.
233	 */
234	csrr a0, CSR_MHARTID
235#endif /* CONFIG_RISCV_M_MODE */
236
237	/* Load the global pointer */
238.option push
239.option norelax
240	la gp, __global_pointer$
241.option pop
242
243	/*
244	 * Disable FPU to detect illegal usage of
245	 * floating point in kernel space
246	 */
247	li t0, SR_FS
248	csrc CSR_STATUS, t0
249
250#ifdef CONFIG_SMP
251	li t0, CONFIG_NR_CPUS
252	blt a0, t0, .Lgood_cores
253	tail .Lsecondary_park
254.Lgood_cores:
255#endif
256
257#ifndef CONFIG_XIP_KERNEL
258	/* Pick one hart to run the main boot sequence */
259	la a3, hart_lottery
260	li a2, 1
261	amoadd.w a3, a2, (a3)
262	bnez a3, .Lsecondary_start
263
264#else
265	/* hart_lottery in flash contains a magic number */
266	la a3, hart_lottery
267	mv a2, a3
268	XIP_FIXUP_OFFSET a2
269	lw t1, (a3)
270	amoswap.w t0, t1, (a2)
271	/* first time here if hart_lottery in RAM is not set */
272	beq t0, t1, .Lsecondary_start
273
274	la sp, _end + THREAD_SIZE
275	XIP_FIXUP_OFFSET sp
276	mv s0, a0
277	call __copy_data
278
279	/* Restore a0 copy */
280	mv a0, s0
281#endif
282
283#ifndef CONFIG_XIP_KERNEL
284	/* Clear BSS for flat non-ELF images */
285	la a3, __bss_start
286	la a4, __bss_stop
287	ble a4, a3, clear_bss_done
288clear_bss:
289	REG_S zero, (a3)
290	add a3, a3, RISCV_SZPTR
291	blt a3, a4, clear_bss
292clear_bss_done:
293#endif
294	/* Save hart ID and DTB physical address */
295	mv s0, a0
296	mv s1, a1
297
298	la a2, boot_cpu_hartid
299	XIP_FIXUP_OFFSET a2
300	REG_S a0, (a2)
301
302	/* Initialize page tables and relocate to virtual addresses */
303	la sp, init_thread_union + THREAD_SIZE
304	XIP_FIXUP_OFFSET sp
305#ifdef CONFIG_BUILTIN_DTB
306	la a0, __dtb_start
307#else
308	mv a0, s1
309#endif /* CONFIG_BUILTIN_DTB */
310	call setup_vm
311#ifdef CONFIG_MMU
312	la a0, early_pg_dir
313	XIP_FIXUP_OFFSET a0
314	call relocate
315#endif /* CONFIG_MMU */
316
317	call setup_trap_vector
318	/* Restore C environment */
319	la tp, init_task
320	sw zero, TASK_TI_CPU(tp)
321	la sp, init_thread_union + THREAD_SIZE
322
323#ifdef CONFIG_KASAN
324	call kasan_early_init
325#endif
326	/* Start the kernel */
327	call soc_early_init
328	tail start_kernel
329
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330.Lsecondary_start:
331#ifdef CONFIG_SMP
332	/* Set trap vector to spin forever to help debug */
333	la a3, .Lsecondary_park
334	csrw CSR_TVEC, a3
335
336	slli a3, a0, LGREG
337	la a1, __cpu_up_stack_pointer
338	XIP_FIXUP_OFFSET a1
339	la a2, __cpu_up_task_pointer
340	XIP_FIXUP_OFFSET a2
341	add a1, a3, a1
342	add a2, a3, a2
343
344	/*
345	 * This hart didn't win the lottery, so we wait for the winning hart to
346	 * get far enough along the boot process that it should continue.
347	 */
348.Lwait_for_cpu_up:
349	/* FIXME: We should WFI to save some energy here. */
350	REG_L sp, (a1)
351	REG_L tp, (a2)
352	beqz sp, .Lwait_for_cpu_up
353	beqz tp, .Lwait_for_cpu_up
354	fence
355
356	tail secondary_start_common
 
 
 
 
357#endif
358
359END(_start_kernel)
360
361#ifdef CONFIG_RISCV_M_MODE
362ENTRY(reset_regs)
363	li	sp, 0
364	li	gp, 0
365	li	tp, 0
366	li	t0, 0
367	li	t1, 0
368	li	t2, 0
369	li	s0, 0
370	li	s1, 0
371	li	a2, 0
372	li	a3, 0
373	li	a4, 0
374	li	a5, 0
375	li	a6, 0
376	li	a7, 0
377	li	s2, 0
378	li	s3, 0
379	li	s4, 0
380	li	s5, 0
381	li	s6, 0
382	li	s7, 0
383	li	s8, 0
384	li	s9, 0
385	li	s10, 0
386	li	s11, 0
387	li	t3, 0
388	li	t4, 0
389	li	t5, 0
390	li	t6, 0
391	csrw	CSR_SCRATCH, 0
392
393#ifdef CONFIG_FPU
394	csrr	t0, CSR_MISA
395	andi	t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
396	beqz	t0, .Lreset_regs_done
397
398	li	t1, SR_FS
399	csrs	CSR_STATUS, t1
400	fmv.s.x	f0, zero
401	fmv.s.x	f1, zero
402	fmv.s.x	f2, zero
403	fmv.s.x	f3, zero
404	fmv.s.x	f4, zero
405	fmv.s.x	f5, zero
406	fmv.s.x	f6, zero
407	fmv.s.x	f7, zero
408	fmv.s.x	f8, zero
409	fmv.s.x	f9, zero
410	fmv.s.x	f10, zero
411	fmv.s.x	f11, zero
412	fmv.s.x	f12, zero
413	fmv.s.x	f13, zero
414	fmv.s.x	f14, zero
415	fmv.s.x	f15, zero
416	fmv.s.x	f16, zero
417	fmv.s.x	f17, zero
418	fmv.s.x	f18, zero
419	fmv.s.x	f19, zero
420	fmv.s.x	f20, zero
421	fmv.s.x	f21, zero
422	fmv.s.x	f22, zero
423	fmv.s.x	f23, zero
424	fmv.s.x	f24, zero
425	fmv.s.x	f25, zero
426	fmv.s.x	f26, zero
427	fmv.s.x	f27, zero
428	fmv.s.x	f28, zero
429	fmv.s.x	f29, zero
430	fmv.s.x	f30, zero
431	fmv.s.x	f31, zero
432	csrw	fcsr, 0
433	/* note that the caller must clear SR_FS */
434#endif /* CONFIG_FPU */
435.Lreset_regs_done:
436	ret
437END(reset_regs)
438#endif /* CONFIG_RISCV_M_MODE */
439
440__PAGE_ALIGNED_BSS
441	/* Empty zero page */
442	.balign PAGE_SIZE