1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2023 Ventana Micro Systems Inc.
  4 */
  5
  6#define pr_fmt(fmt) "riscv-pv: " fmt
  7
  8#include <linux/cpuhotplug.h>
  9#include <linux/compiler.h>
 10#include <linux/errno.h>
 11#include <linux/init.h>
 12#include <linux/jump_label.h>
 13#include <linux/kconfig.h>
 14#include <linux/kernel.h>
 15#include <linux/percpu-defs.h>
 16#include <linux/printk.h>
 17#include <linux/static_call.h>
 18#include <linux/types.h>
 19
 20#include <asm/barrier.h>
 21#include <asm/page.h>
 22#include <asm/paravirt.h>
 23#include <asm/sbi.h>
 24
 25struct static_key paravirt_steal_enabled;
 26struct static_key paravirt_steal_rq_enabled;
 27
 28static u64 native_steal_clock(int cpu)
 29{
 30	return 0;
 31}
 32
 33DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
 34
 35static bool steal_acc = true;
 36static int __init parse_no_stealacc(char *arg)
 37{
 38	steal_acc = false;
 39	return 0;
 40}
 41
 42early_param("no-steal-acc", parse_no_stealacc);
 43
 44static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64);
 45
 46static bool __init has_pv_steal_clock(void)
 47{
 48	if (sbi_spec_version >= sbi_mk_version(2, 0) &&
 49	    sbi_probe_extension(SBI_EXT_STA) > 0) {
 50		pr_info("SBI STA extension detected\n");
 51		return true;
 52	}
 53
 54	return false;
 55}
 56
 57static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi,
 58					unsigned long flags)
 59{
 60	struct sbiret ret;
 61
 62	ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM,
 63			lo, hi, flags, 0, 0, 0);
 64	if (ret.error) {
 65		if (lo == SBI_STA_SHMEM_DISABLE && hi == SBI_STA_SHMEM_DISABLE)
 66			pr_warn("Failed to disable steal-time shmem");
 67		else
 68			pr_warn("Failed to set steal-time shmem");
 69		return sbi_err_map_linux_errno(ret.error);
 70	}
 71
 72	return 0;
 73}
 74
 75static int pv_time_cpu_online(unsigned int cpu)
 76{
 77	struct sbi_sta_struct *st = this_cpu_ptr(&steal_time);
 78	phys_addr_t pa = __pa(st);
 79	unsigned long lo = (unsigned long)pa;
 80	unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0;
 81
 82	return sbi_sta_steal_time_set_shmem(lo, hi, 0);
 83}
 84
 85static int pv_time_cpu_down_prepare(unsigned int cpu)
 86{
 87	return sbi_sta_steal_time_set_shmem(SBI_STA_SHMEM_DISABLE,
 88					    SBI_STA_SHMEM_DISABLE, 0);
 89}
 90
 91static u64 pv_time_steal_clock(int cpu)
 92{
 93	struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu);
 94	__le32 sequence;
 95	__le64 steal;
 96
 97	/*
 98	 * Check the sequence field before and after reading the steal
 99	 * field. Repeat the read if it is different or odd.
100	 */
101	do {
102		sequence = READ_ONCE(st->sequence);
103		virt_rmb();
104		steal = READ_ONCE(st->steal);
105		virt_rmb();
106	} while ((le32_to_cpu(sequence) & 1) ||
107		 sequence != READ_ONCE(st->sequence));
108
109	return le64_to_cpu(steal);
110}
111
112int __init pv_time_init(void)
113{
114	int ret;
115
116	if (!has_pv_steal_clock())
117		return 0;
118
119	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
120				"riscv/pv_time:online",
121				pv_time_cpu_online,
122				pv_time_cpu_down_prepare);
123	if (ret < 0)
124		return ret;
125
126	static_call_update(pv_steal_clock, pv_time_steal_clock);
127
128	static_key_slow_inc(&paravirt_steal_enabled);
129	if (steal_acc)
130		static_key_slow_inc(&paravirt_steal_rq_enabled);
131
132	pr_info("Computing paravirt steal-time\n");
133
134	return 0;
135}