1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * KVM nVHE hypervisor stack tracing support.
  4 *
  5 * The unwinder implementation depends on the nVHE mode:
  6 *
  7 *   1) Non-protected nVHE mode - the host can directly access the
  8 *      HYP stack pages and unwind the HYP stack in EL1. This saves having
  9 *      to allocate shared buffers for the host to read the unwinded
 10 *      stacktrace.
 11 *
 12 *   2) pKVM (protected nVHE) mode - the host cannot directly access
 13 *      the HYP memory. The stack is unwinded in EL2 and dumped to a shared
 14 *      buffer where the host can read and print the stacktrace.
 15 *
 16 * Copyright (C) 2022 Google LLC
 17 */
 18
 19#include <linux/kvm.h>
 20#include <linux/kvm_host.h>
 21
 22#include <asm/stacktrace/nvhe.h>
 23
 24static struct stack_info stackinfo_get_overflow(void)
 25{
 26	struct kvm_nvhe_stacktrace_info *stacktrace_info
 27				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
 28	unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
 29	unsigned long high = low + OVERFLOW_STACK_SIZE;
 30
 31	return (struct stack_info) {
 32		.low = low,
 33		.high = high,
 34	};
 35}
 36
 37static struct stack_info stackinfo_get_overflow_kern_va(void)
 38{
 39	unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
 40	unsigned long high = low + OVERFLOW_STACK_SIZE;
 41
 42	return (struct stack_info) {
 43		.low = low,
 44		.high = high,
 45	};
 46}
 47
 48static struct stack_info stackinfo_get_hyp(void)
 49{
 50	struct kvm_nvhe_stacktrace_info *stacktrace_info
 51				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
 52	unsigned long low = (unsigned long)stacktrace_info->stack_base;
 53	unsigned long high = low + PAGE_SIZE;
 54
 55	return (struct stack_info) {
 56		.low = low,
 57		.high = high,
 58	};
 59}
 60
 61static struct stack_info stackinfo_get_hyp_kern_va(void)
 62{
 63	unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
 64	unsigned long high = low + PAGE_SIZE;
 65
 66	return (struct stack_info) {
 67		.low = low,
 68		.high = high,
 69	};
 70}
 71
 72/*
 73 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
 74 *
 75 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
 76 * allow for guard pages below the stack. Consequently, the fixed offset address
 77 * translation macros won't work here.
 78 *
 79 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
 80 * stack base.
 81 *
 82 * Returns true on success and updates @addr to its corresponding kernel VA;
 83 * otherwise returns false.
 84 */
 85static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
 86{
 87	struct stack_info stack_hyp, stack_kern;
 88
 89	stack_hyp = stackinfo_get_hyp();
 90	stack_kern = stackinfo_get_hyp_kern_va();
 91	if (stackinfo_on_stack(&stack_hyp, *addr, size))
 92		goto found;
 93
 94	stack_hyp = stackinfo_get_overflow();
 95	stack_kern = stackinfo_get_overflow_kern_va();
 96	if (stackinfo_on_stack(&stack_hyp, *addr, size))
 97		goto found;
 98
 99	return false;
100
101found:
102	*addr = *addr - stack_hyp.low + stack_kern.low;
103	return true;
104}
105
106/*
107 * Convert a KVN nVHE HYP frame record address to a kernel VA
108 */
109static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
110{
111	return kvm_nvhe_stack_kern_va(addr, 16);
112}
113
114static int unwind_next(struct unwind_state *state)
115{
116	/*
117	 * The FP is in the hypervisor VA space. Convert it to the kernel VA
118	 * space so it can be unwound by the regular unwind functions.
119	 */
120	if (!kvm_nvhe_stack_kern_record_va(&state->fp))
121		return -EINVAL;
122
123	return unwind_next_frame_record(state);
124}
125
126static void unwind(struct unwind_state *state,
127		   stack_trace_consume_fn consume_entry, void *cookie)
128{
129	while (1) {
130		int ret;
131
132		if (!consume_entry(cookie, state->pc))
133			break;
134		ret = unwind_next(state);
135		if (ret < 0)
136			break;
137	}
138}
139
140/*
141 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
142 *
143 * @arg    : the hypervisor offset, used for address translation
144 * @where  : the program counter corresponding to the stack frame
145 */
146static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
147{
148	unsigned long va_mask = GENMASK_ULL(vabits_actual - 1, 0);
149	unsigned long hyp_offset = (unsigned long)arg;
150
151	/* Mask tags and convert to kern addr */
152	where = (where & va_mask) + hyp_offset;
153	kvm_err(" [<%016lx>] %pB\n", where, (void *)(where + kaslr_offset()));
154
155	return true;
156}
157
158static void kvm_nvhe_dump_backtrace_start(void)
159{
160	kvm_err("nVHE call trace:\n");
161}
162
163static void kvm_nvhe_dump_backtrace_end(void)
164{
165	kvm_err("---[ end nVHE call trace ]---\n");
166}
167
168/*
169 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
170 *
171 * @hyp_offset: hypervisor offset, used for address translation.
172 *
173 * The host can directly access HYP stack pages in non-protected
174 * mode, so the unwinding is done directly from EL1. This removes
175 * the need for shared buffers between host and hypervisor for
176 * the stacktrace.
177 */
178static void hyp_dump_backtrace(unsigned long hyp_offset)
179{
180	struct kvm_nvhe_stacktrace_info *stacktrace_info;
181	struct stack_info stacks[] = {
182		stackinfo_get_overflow_kern_va(),
183		stackinfo_get_hyp_kern_va(),
184	};
185	struct unwind_state state = {
186		.stacks = stacks,
187		.nr_stacks = ARRAY_SIZE(stacks),
188	};
189
190	stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
191
192	kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
193
194	kvm_nvhe_dump_backtrace_start();
195	unwind(&state, kvm_nvhe_dump_backtrace_entry, (void *)hyp_offset);
196	kvm_nvhe_dump_backtrace_end();
197}
198
199#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
200DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
201			 pkvm_stacktrace);
202
203/*
204 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
205 *
206 * @hyp_offset: hypervisor offset, used for address translation.
207 *
208 * Dumping of the pKVM HYP backtrace is done by reading the
209 * stack addresses from the shared stacktrace buffer, since the
210 * host cannot directly access hypervisor memory in protected
211 * mode.
212 */
213static void pkvm_dump_backtrace(unsigned long hyp_offset)
214{
215	unsigned long *stacktrace
216		= (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
217	int i;
218
219	kvm_nvhe_dump_backtrace_start();
220	/* The saved stacktrace is terminated by a null entry */
221	for (i = 0;
222	     i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
223	     i++)
224		kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
225	kvm_nvhe_dump_backtrace_end();
226}
227#else	/* !CONFIG_PROTECTED_NVHE_STACKTRACE */
228static void pkvm_dump_backtrace(unsigned long hyp_offset)
229{
230	kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
231}
232#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
233
234/*
235 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
236 *
237 * @hyp_offset: hypervisor offset, used for address translation.
238 */
239void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
240{
241	if (is_protected_kvm_enabled())
242		pkvm_dump_backtrace(hyp_offset);
243	else
244		hyp_dump_backtrace(hyp_offset);
245}