1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Hyper-V Isolation VM interface with paravisor and hypervisor
  4 *
  5 * Author:
  6 *  Tianyu Lan <Tianyu.Lan@microsoft.com>
  7 */
  8
  9#include <linux/bitfield.h>
 10#include <linux/hyperv.h>
 11#include <linux/types.h>
 12#include <linux/slab.h>
 13#include <asm/svm.h>
 14#include <asm/sev.h>
 15#include <asm/io.h>
 16#include <asm/mshyperv.h>
 17#include <asm/hypervisor.h>
 18
 19#ifdef CONFIG_AMD_MEM_ENCRYPT
 20
 21#define GHCB_USAGE_HYPERV_CALL	1
 22
 23union hv_ghcb {
 24	struct ghcb ghcb;
 25	struct {
 26		u64 hypercalldata[509];
 27		u64 outputgpa;
 28		union {
 29			union {
 30				struct {
 31					u32 callcode        : 16;
 32					u32 isfast          : 1;
 33					u32 reserved1       : 14;
 34					u32 isnested        : 1;
 35					u32 countofelements : 12;
 36					u32 reserved2       : 4;
 37					u32 repstartindex   : 12;
 38					u32 reserved3       : 4;
 39				};
 40				u64 asuint64;
 41			} hypercallinput;
 42			union {
 43				struct {
 44					u16 callstatus;
 45					u16 reserved1;
 46					u32 elementsprocessed : 12;
 47					u32 reserved2         : 20;
 48				};
 49				u64 asunit64;
 50			} hypercalloutput;
 51		};
 52		u64 reserved2;
 53	} hypercall;
 54} __packed __aligned(HV_HYP_PAGE_SIZE);
 55
 56static u16 hv_ghcb_version __ro_after_init;
 57
 58u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size)
 59{
 60	union hv_ghcb *hv_ghcb;
 61	void **ghcb_base;
 62	unsigned long flags;
 63	u64 status;
 64
 65	if (!hv_ghcb_pg)
 66		return -EFAULT;
 67
 68	WARN_ON(in_nmi());
 69
 70	local_irq_save(flags);
 71	ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
 72	hv_ghcb = (union hv_ghcb *)*ghcb_base;
 73	if (!hv_ghcb) {
 74		local_irq_restore(flags);
 75		return -EFAULT;
 76	}
 77
 78	hv_ghcb->ghcb.protocol_version = GHCB_PROTOCOL_MAX;
 79	hv_ghcb->ghcb.ghcb_usage = GHCB_USAGE_HYPERV_CALL;
 80
 81	hv_ghcb->hypercall.outputgpa = (u64)output;
 82	hv_ghcb->hypercall.hypercallinput.asuint64 = 0;
 83	hv_ghcb->hypercall.hypercallinput.callcode = control;
 84
 85	if (input_size)
 86		memcpy(hv_ghcb->hypercall.hypercalldata, input, input_size);
 87
 88	VMGEXIT();
 89
 90	hv_ghcb->ghcb.ghcb_usage = 0xffffffff;
 91	memset(hv_ghcb->ghcb.save.valid_bitmap, 0,
 92	       sizeof(hv_ghcb->ghcb.save.valid_bitmap));
 93
 94	status = hv_ghcb->hypercall.hypercalloutput.callstatus;
 95
 96	local_irq_restore(flags);
 97
 98	return status;
 99}
100
101static inline u64 rd_ghcb_msr(void)
102{
103	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
104}
105
106static inline void wr_ghcb_msr(u64 val)
107{
108	native_wrmsrl(MSR_AMD64_SEV_ES_GHCB, val);
109}
110
111static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code,
112				   u64 exit_info_1, u64 exit_info_2)
113{
114	/* Fill in protocol and format specifiers */
115	ghcb->protocol_version = hv_ghcb_version;
116	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
117
118	ghcb_set_sw_exit_code(ghcb, exit_code);
119	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
120	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
121
122	VMGEXIT();
123
124	if (ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0))
125		return ES_VMM_ERROR;
126	else
127		return ES_OK;
128}
129
130void hv_ghcb_terminate(unsigned int set, unsigned int reason)
131{
132	u64 val = GHCB_MSR_TERM_REQ;
133
134	/* Tell the hypervisor what went wrong. */
135	val |= GHCB_SEV_TERM_REASON(set, reason);
136
137	/* Request Guest Termination from Hypvervisor */
138	wr_ghcb_msr(val);
139	VMGEXIT();
140
141	while (true)
142		asm volatile("hlt\n" : : : "memory");
143}
144
145bool hv_ghcb_negotiate_protocol(void)
146{
147	u64 ghcb_gpa;
148	u64 val;
149
150	/* Save ghcb page gpa. */
151	ghcb_gpa = rd_ghcb_msr();
152
153	/* Do the GHCB protocol version negotiation */
154	wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
155	VMGEXIT();
156	val = rd_ghcb_msr();
157
158	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
159		return false;
160
161	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
162	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
163		return false;
164
165	hv_ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val),
166			     GHCB_PROTOCOL_MAX);
167
168	/* Write ghcb page back after negotiating protocol. */
169	wr_ghcb_msr(ghcb_gpa);
170	VMGEXIT();
171
172	return true;
173}
174
175void hv_ghcb_msr_write(u64 msr, u64 value)
176{
177	union hv_ghcb *hv_ghcb;
178	void **ghcb_base;
179	unsigned long flags;
180
181	if (!hv_ghcb_pg)
182		return;
183
184	WARN_ON(in_nmi());
185
186	local_irq_save(flags);
187	ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
188	hv_ghcb = (union hv_ghcb *)*ghcb_base;
189	if (!hv_ghcb) {
190		local_irq_restore(flags);
191		return;
192	}
193
194	ghcb_set_rcx(&hv_ghcb->ghcb, msr);
195	ghcb_set_rax(&hv_ghcb->ghcb, lower_32_bits(value));
196	ghcb_set_rdx(&hv_ghcb->ghcb, upper_32_bits(value));
197
198	if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 1, 0))
199		pr_warn("Fail to write msr via ghcb %llx.\n", msr);
200
201	local_irq_restore(flags);
202}
203EXPORT_SYMBOL_GPL(hv_ghcb_msr_write);
204
205void hv_ghcb_msr_read(u64 msr, u64 *value)
206{
207	union hv_ghcb *hv_ghcb;
208	void **ghcb_base;
209	unsigned long flags;
210
211	/* Check size of union hv_ghcb here. */
212	BUILD_BUG_ON(sizeof(union hv_ghcb) != HV_HYP_PAGE_SIZE);
213
214	if (!hv_ghcb_pg)
215		return;
216
217	WARN_ON(in_nmi());
218
219	local_irq_save(flags);
220	ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
221	hv_ghcb = (union hv_ghcb *)*ghcb_base;
222	if (!hv_ghcb) {
223		local_irq_restore(flags);
224		return;
225	}
226
227	ghcb_set_rcx(&hv_ghcb->ghcb, msr);
228	if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 0, 0))
229		pr_warn("Fail to read msr via ghcb %llx.\n", msr);
230	else
231		*value = (u64)lower_32_bits(hv_ghcb->ghcb.save.rax)
232			| ((u64)lower_32_bits(hv_ghcb->ghcb.save.rdx) << 32);
233	local_irq_restore(flags);
234}
235EXPORT_SYMBOL_GPL(hv_ghcb_msr_read);
236#endif
237
238enum hv_isolation_type hv_get_isolation_type(void)
239{
240	if (!(ms_hyperv.priv_high & HV_ISOLATION))
241		return HV_ISOLATION_TYPE_NONE;
242	return FIELD_GET(HV_ISOLATION_TYPE, ms_hyperv.isolation_config_b);
243}
244EXPORT_SYMBOL_GPL(hv_get_isolation_type);
245
246/*
247 * hv_is_isolation_supported - Check system runs in the Hyper-V
248 * isolation VM.
249 */
250bool hv_is_isolation_supported(void)
251{
252	if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
253		return false;
254
255	if (!hypervisor_is_type(X86_HYPER_MS_HYPERV))
256		return false;
257
258	return hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE;
259}
260
261DEFINE_STATIC_KEY_FALSE(isolation_type_snp);
262
263/*
264 * hv_isolation_type_snp - Check system runs in the AMD SEV-SNP based
265 * isolation VM.
266 */
267bool hv_isolation_type_snp(void)
268{
269	return static_branch_unlikely(&isolation_type_snp);
270}
271
272/*
273 * hv_mark_gpa_visibility - Set pages visible to host via hvcall.
274 *
275 * In Isolation VM, all guest memory is encrypted from host and guest
276 * needs to set memory visible to host via hvcall before sharing memory
277 * with host.
278 */
279static int hv_mark_gpa_visibility(u16 count, const u64 pfn[],
280			   enum hv_mem_host_visibility visibility)
281{
282	struct hv_gpa_range_for_visibility **input_pcpu, *input;
283	u16 pages_processed;
284	u64 hv_status;
285	unsigned long flags;
286
287	/* no-op if partition isolation is not enabled */
288	if (!hv_is_isolation_supported())
289		return 0;
290
291	if (count > HV_MAX_MODIFY_GPA_REP_COUNT) {
292		pr_err("Hyper-V: GPA count:%d exceeds supported:%lu\n", count,
293			HV_MAX_MODIFY_GPA_REP_COUNT);
294		return -EINVAL;
295	}
296
297	local_irq_save(flags);
298	input_pcpu = (struct hv_gpa_range_for_visibility **)
299			this_cpu_ptr(hyperv_pcpu_input_arg);
300	input = *input_pcpu;
301	if (unlikely(!input)) {
302		local_irq_restore(flags);
303		return -EINVAL;
304	}
305
306	input->partition_id = HV_PARTITION_ID_SELF;
307	input->host_visibility = visibility;
308	input->reserved0 = 0;
309	input->reserved1 = 0;
310	memcpy((void *)input->gpa_page_list, pfn, count * sizeof(*pfn));
311	hv_status = hv_do_rep_hypercall(
312			HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY, count,
313			0, input, &pages_processed);
314	local_irq_restore(flags);
315
316	if (hv_result_success(hv_status))
317		return 0;
318	else
319		return -EFAULT;
320}
321
322/*
323 * hv_set_mem_host_visibility - Set specified memory visible to host.
324 *
325 * In Isolation VM, all guest memory is encrypted from host and guest
326 * needs to set memory visible to host via hvcall before sharing memory
327 * with host. This function works as wrap of hv_mark_gpa_visibility()
328 * with memory base and size.
329 */
330int hv_set_mem_host_visibility(unsigned long kbuffer, int pagecount, bool visible)
331{
332	enum hv_mem_host_visibility visibility = visible ?
333			VMBUS_PAGE_VISIBLE_READ_WRITE : VMBUS_PAGE_NOT_VISIBLE;
334	u64 *pfn_array;
335	int ret = 0;
336	int i, pfn;
337
338	if (!hv_is_isolation_supported() || !hv_hypercall_pg)
339		return 0;
340
341	pfn_array = kmalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL);
342	if (!pfn_array)
343		return -ENOMEM;
344
345	for (i = 0, pfn = 0; i < pagecount; i++) {
346		pfn_array[pfn] = virt_to_hvpfn((void *)kbuffer + i * HV_HYP_PAGE_SIZE);
347		pfn++;
348
349		if (pfn == HV_MAX_MODIFY_GPA_REP_COUNT || i == pagecount - 1) {
350			ret = hv_mark_gpa_visibility(pfn, pfn_array,
351						     visibility);
352			if (ret)
353				goto err_free_pfn_array;
354			pfn = 0;
355		}
356	}
357
358 err_free_pfn_array:
359	kfree(pfn_array);
360	return ret;
361}
362
363/*
364 * hv_map_memory - map memory to extra space in the AMD SEV-SNP Isolation VM.
365 */
366void *hv_map_memory(void *addr, unsigned long size)
367{
368	unsigned long *pfns = kcalloc(size / PAGE_SIZE,
369				      sizeof(unsigned long), GFP_KERNEL);
370	void *vaddr;
371	int i;
372
373	if (!pfns)
374		return NULL;
375
376	for (i = 0; i < size / PAGE_SIZE; i++)
377		pfns[i] = vmalloc_to_pfn(addr + i * PAGE_SIZE) +
378			(ms_hyperv.shared_gpa_boundary >> PAGE_SHIFT);
379
380	vaddr = vmap_pfn(pfns, size / PAGE_SIZE, PAGE_KERNEL_IO);
381	kfree(pfns);
382
383	return vaddr;
384}
385
386void hv_unmap_memory(void *addr)
387{
388	vunmap(addr);
389}