1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2020 - Google LLC
  4 * Author: Quentin Perret <qperret@google.com>
  5 */
  6
  7#include <linux/kvm_host.h>
  8#include <linux/memblock.h>
  9#include <linux/mutex.h>
 10#include <linux/sort.h>
 11
 12#include <asm/kvm_pkvm.h>
 13
 14#include "hyp_constants.h"
 15
 16static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
 17static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
 18
 19phys_addr_t hyp_mem_base;
 20phys_addr_t hyp_mem_size;
 21
 22static int cmp_hyp_memblock(const void *p1, const void *p2)
 23{
 24	const struct memblock_region *r1 = p1;
 25	const struct memblock_region *r2 = p2;
 26
 27	return r1->base < r2->base ? -1 : (r1->base > r2->base);
 28}
 29
 30static void __init sort_memblock_regions(void)
 31{
 32	sort(hyp_memory,
 33	     *hyp_memblock_nr_ptr,
 34	     sizeof(struct memblock_region),
 35	     cmp_hyp_memblock,
 36	     NULL);
 37}
 38
 39static int __init register_memblock_regions(void)
 40{
 41	struct memblock_region *reg;
 42
 43	for_each_mem_region(reg) {
 44		if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
 45			return -ENOMEM;
 46
 47		hyp_memory[*hyp_memblock_nr_ptr] = *reg;
 48		(*hyp_memblock_nr_ptr)++;
 49	}
 50	sort_memblock_regions();
 51
 52	return 0;
 53}
 54
 55void __init kvm_hyp_reserve(void)
 56{
 57	u64 hyp_mem_pages = 0;
 58	int ret;
 59
 60	if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
 61		return;
 62
 63	if (kvm_get_mode() != KVM_MODE_PROTECTED)
 64		return;
 65
 66	ret = register_memblock_regions();
 67	if (ret) {
 68		*hyp_memblock_nr_ptr = 0;
 69		kvm_err("Failed to register hyp memblocks: %d\n", ret);
 70		return;
 71	}
 72
 73	hyp_mem_pages += hyp_s1_pgtable_pages();
 74	hyp_mem_pages += host_s2_pgtable_pages();
 75	hyp_mem_pages += hyp_vm_table_pages();
 76	hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
 77
 78	/*
 79	 * Try to allocate a PMD-aligned region to reduce TLB pressure once
 80	 * this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
 81	 */
 82	hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
 83	hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE),
 84					   PMD_SIZE);
 85	if (!hyp_mem_base)
 86		hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE);
 87	else
 88		hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
 89
 90	if (!hyp_mem_base) {
 91		kvm_err("Failed to reserve hyp memory\n");
 92		return;
 93	}
 94
 95	kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
 96		 hyp_mem_base);
 97}
 98
 99/*
100 * Allocates and donates memory for hypervisor VM structs at EL2.
101 *
102 * Allocates space for the VM state, which includes the hyp vm as well as
103 * the hyp vcpus.
104 *
105 * Stores an opaque handler in the kvm struct for future reference.
106 *
107 * Return 0 on success, negative error code on failure.
108 */
109static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
110{
111	size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz;
112	struct kvm_vcpu *host_vcpu;
113	pkvm_handle_t handle;
114	void *pgd, *hyp_vm;
115	unsigned long idx;
116	int ret;
117
118	if (host_kvm->created_vcpus < 1)
119		return -EINVAL;
120
121	pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.vtcr);
122
123	/*
124	 * The PGD pages will be reclaimed using a hyp_memcache which implies
125	 * page granularity. So, use alloc_pages_exact() to get individual
126	 * refcounts.
127	 */
128	pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
129	if (!pgd)
130		return -ENOMEM;
131
132	/* Allocate memory to donate to hyp for vm and vcpu pointers. */
133	hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
134					size_mul(sizeof(void *),
135						 host_kvm->created_vcpus)));
136	hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
137	if (!hyp_vm) {
138		ret = -ENOMEM;
139		goto free_pgd;
140	}
141
142	/* Donate the VM memory to hyp and let hyp initialize it. */
143	ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd);
144	if (ret < 0)
145		goto free_vm;
146
147	handle = ret;
148
149	host_kvm->arch.pkvm.handle = handle;
150
151	/* Donate memory for the vcpus at hyp and initialize it. */
152	hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
153	kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
154		void *hyp_vcpu;
155
156		/* Indexing of the vcpus to be sequential starting at 0. */
157		if (WARN_ON(host_vcpu->vcpu_idx != idx)) {
158			ret = -EINVAL;
159			goto destroy_vm;
160		}
161
162		hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
163		if (!hyp_vcpu) {
164			ret = -ENOMEM;
165			goto destroy_vm;
166		}
167
168		ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu,
169					hyp_vcpu);
170		if (ret) {
171			free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
172			goto destroy_vm;
173		}
174	}
175
176	return 0;
177
178destroy_vm:
179	pkvm_destroy_hyp_vm(host_kvm);
180	return ret;
181free_vm:
182	free_pages_exact(hyp_vm, hyp_vm_sz);
183free_pgd:
184	free_pages_exact(pgd, pgd_sz);
185	return ret;
186}
187
188int pkvm_create_hyp_vm(struct kvm *host_kvm)
189{
190	int ret = 0;
191
192	mutex_lock(&host_kvm->lock);
193	if (!host_kvm->arch.pkvm.handle)
194		ret = __pkvm_create_hyp_vm(host_kvm);
195	mutex_unlock(&host_kvm->lock);
196
197	return ret;
198}
199
200void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
201{
202	if (host_kvm->arch.pkvm.handle) {
203		WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm,
204					  host_kvm->arch.pkvm.handle));
205	}
206
207	host_kvm->arch.pkvm.handle = 0;
208	free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
209}
210
211int pkvm_init_host_vm(struct kvm *host_kvm)
212{
213	mutex_init(&host_kvm->lock);
214	return 0;
215}