1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * A stand-alone ticket spinlock implementation for use by the non-VHE
  4 * KVM hypervisor code running at EL2.
  5 *
  6 * Copyright (C) 2020 Google LLC
  7 * Author: Will Deacon <will@kernel.org>
  8 *
  9 * Heavily based on the implementation removed by c11090474d70 which was:
 10 * Copyright (C) 2012 ARM Ltd.
 11 */
 12
 13#ifndef __ARM64_KVM_NVHE_SPINLOCK_H__
 14#define __ARM64_KVM_NVHE_SPINLOCK_H__
 15
 16#include <asm/alternative.h>
 17#include <asm/lse.h>
 18#include <asm/rwonce.h>
 19
 20typedef union hyp_spinlock {
 21	u32	__val;
 22	struct {
 23#ifdef __AARCH64EB__
 24		u16 next, owner;
 25#else
 26		u16 owner, next;
 27#endif
 28	};
 29} hyp_spinlock_t;
 30
 31#define __HYP_SPIN_LOCK_INITIALIZER \
 32	{ .__val = 0 }
 33
 34#define __HYP_SPIN_LOCK_UNLOCKED \
 35	((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER)
 36
 37#define DEFINE_HYP_SPINLOCK(x)	hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED
 38
 39#define hyp_spin_lock_init(l)						\
 40do {									\
 41	*(l) = __HYP_SPIN_LOCK_UNLOCKED;				\
 42} while (0)
 43
 44static inline void hyp_spin_lock(hyp_spinlock_t *lock)
 45{
 46	u32 tmp;
 47	hyp_spinlock_t lockval, newval;
 48
 49	asm volatile(
 50	/* Atomically increment the next ticket. */
 51	ARM64_LSE_ATOMIC_INSN(
 52	/* LL/SC */
 53"	prfm	pstl1strm, %3\n"
 54"1:	ldaxr	%w0, %3\n"
 55"	add	%w1, %w0, #(1 << 16)\n"
 56"	stxr	%w2, %w1, %3\n"
 57"	cbnz	%w2, 1b\n",
 58	/* LSE atomics */
 59"	mov	%w2, #(1 << 16)\n"
 60"	ldadda	%w2, %w0, %3\n"
 61	__nops(3))
 62
 63	/* Did we get the lock? */
 64"	eor	%w1, %w0, %w0, ror #16\n"
 65"	cbz	%w1, 3f\n"
 66	/*
 67	 * No: spin on the owner. Send a local event to avoid missing an
 68	 * unlock before the exclusive load.
 69	 */
 70"	sevl\n"
 71"2:	wfe\n"
 72"	ldaxrh	%w2, %4\n"
 73"	eor	%w1, %w2, %w0, lsr #16\n"
 74"	cbnz	%w1, 2b\n"
 75	/* We got the lock. Critical section starts here. */
 76"3:"
 77	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
 78	: "Q" (lock->owner)
 79	: "memory");
 80}
 81
 82static inline void hyp_spin_unlock(hyp_spinlock_t *lock)
 83{
 84	u64 tmp;
 85
 86	asm volatile(
 87	ARM64_LSE_ATOMIC_INSN(
 88	/* LL/SC */
 89	"	ldrh	%w1, %0\n"
 90	"	add	%w1, %w1, #1\n"
 91	"	stlrh	%w1, %0",
 92	/* LSE atomics */
 93	"	mov	%w1, #1\n"
 94	"	staddlh	%w1, %0\n"
 95	__nops(1))
 96	: "=Q" (lock->owner), "=&r" (tmp)
 97	:
 98	: "memory");
 99}
100
101static inline bool hyp_spin_is_locked(hyp_spinlock_t *lock)
102{
103	hyp_spinlock_t lockval = READ_ONCE(*lock);
104
105	return lockval.owner != lockval.next;
106}
107
108#ifdef CONFIG_NVHE_EL2_DEBUG
109static inline void hyp_assert_lock_held(hyp_spinlock_t *lock)
110{
111	/*
112	 * The __pkvm_init() path accesses protected data-structures without
113	 * holding locks as the other CPUs are guaranteed to not enter EL2
114	 * concurrently at this point in time. The point by which EL2 is
115	 * initialized on all CPUs is reflected in the pkvm static key, so
116	 * wait until it is set before checking the lock state.
117	 */
118	if (static_branch_likely(&kvm_protected_mode_initialized))
119		BUG_ON(!hyp_spin_is_locked(lock));
120}
121#else
122static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { }
123#endif
124
125#endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */