1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
  4 *
  5 * Authors:
  6 *     Anup Patel <anup.patel@wdc.com>
  7 */
  8
  9#include <linux/bitops.h>
 10#include <linux/cpumask.h>
 11#include <linux/errno.h>
 12#include <linux/err.h>
 13#include <linux/module.h>
 14#include <linux/smp.h>
 15#include <linux/kvm_host.h>
 16#include <asm/csr.h>
 17
 18static unsigned long vmid_version = 1;
 19static unsigned long vmid_next;
 20static unsigned long vmid_bits __ro_after_init;
 21static DEFINE_SPINLOCK(vmid_lock);
 22
 23void __init kvm_riscv_gstage_vmid_detect(void)
 24{
 25	unsigned long old;
 26
 27	/* Figure-out number of VMID bits in HW */
 28	old = csr_read(CSR_HGATP);
 29	csr_write(CSR_HGATP, old | HGATP_VMID);
 30	vmid_bits = csr_read(CSR_HGATP);
 31	vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT;
 32	vmid_bits = fls_long(vmid_bits);
 33	csr_write(CSR_HGATP, old);
 34
 35	/* We polluted local TLB so flush all guest TLB */
 36	kvm_riscv_local_hfence_gvma_all();
 37
 38	/* We don't use VMID bits if they are not sufficient */
 39	if ((1UL << vmid_bits) < num_possible_cpus())
 40		vmid_bits = 0;
 41}
 42
 43unsigned long kvm_riscv_gstage_vmid_bits(void)
 44{
 45	return vmid_bits;
 46}
 47
 48int kvm_riscv_gstage_vmid_init(struct kvm *kvm)
 49{
 50	/* Mark the initial VMID and VMID version invalid */
 51	kvm->arch.vmid.vmid_version = 0;
 52	kvm->arch.vmid.vmid = 0;
 53
 54	return 0;
 55}
 56
 57bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid)
 58{
 59	if (!vmid_bits)
 60		return false;
 61
 62	return unlikely(READ_ONCE(vmid->vmid_version) !=
 63			READ_ONCE(vmid_version));
 64}
 65
 66static void __local_hfence_gvma_all(void *info)
 67{
 68	kvm_riscv_local_hfence_gvma_all();
 69}
 70
 71void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
 72{
 73	unsigned long i;
 74	struct kvm_vcpu *v;
 75	struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;
 76
 77	if (!kvm_riscv_gstage_vmid_ver_changed(vmid))
 78		return;
 79
 80	spin_lock(&vmid_lock);
 81
 82	/*
 83	 * We need to re-check the vmid_version here to ensure that if
 84	 * another vcpu already allocated a valid vmid for this vm.
 85	 */
 86	if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) {
 87		spin_unlock(&vmid_lock);
 88		return;
 89	}
 90
 91	/* First user of a new VMID version? */
 92	if (unlikely(vmid_next == 0)) {
 93		WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1);
 94		vmid_next = 1;
 95
 96		/*
 97		 * We ran out of VMIDs so we increment vmid_version and
 98		 * start assigning VMIDs from 1.
 99		 *
100		 * This also means existing VMIDs assignment to all Guest
101		 * instances is invalid and we have force VMID re-assignement
102		 * for all Guest instances. The Guest instances that were not
103		 * running will automatically pick-up new VMIDs because will
104		 * call kvm_riscv_gstage_vmid_update() whenever they enter
105		 * in-kernel run loop. For Guest instances that are already
106		 * running, we force VM exits on all host CPUs using IPI and
107		 * flush all Guest TLBs.
108		 */
109		on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all,
110				 NULL, 1);
111	}
112
113	vmid->vmid = vmid_next;
114	vmid_next++;
115	vmid_next &= (1 << vmid_bits) - 1;
116
117	WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version));
118
119	spin_unlock(&vmid_lock);
120
121	/* Request G-stage page table update for all VCPUs */
122	kvm_for_each_vcpu(i, v, vcpu->kvm)
123		kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
124}