1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Copyright (C) 2023 SiFive
  4 * Author: Andy Chiu <andy.chiu@sifive.com>
  5 */
  6#include <linux/export.h>
  7#include <linux/sched/signal.h>
  8#include <linux/types.h>
  9#include <linux/slab.h>
 10#include <linux/sched.h>
 11#include <linux/uaccess.h>
 12#include <linux/prctl.h>
 13
 14#include <asm/thread_info.h>
 15#include <asm/processor.h>
 16#include <asm/insn.h>
 17#include <asm/vector.h>
 18#include <asm/csr.h>
 19#include <asm/elf.h>
 20#include <asm/ptrace.h>
 21#include <asm/bug.h>
 22
 23static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
 24static struct kmem_cache *riscv_v_user_cachep;
 25#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 26static struct kmem_cache *riscv_v_kernel_cachep;
 27#endif
 28
 29unsigned long riscv_v_vsize __read_mostly;
 30EXPORT_SYMBOL_GPL(riscv_v_vsize);
 31
 32int riscv_v_setup_vsize(void)
 33{
 34	unsigned long this_vsize;
 35
 36	/* There are 32 vector registers with vlenb length. */
 37	riscv_v_enable();
 38	this_vsize = csr_read(CSR_VLENB) * 32;
 39	riscv_v_disable();
 40
 41	if (!riscv_v_vsize) {
 42		riscv_v_vsize = this_vsize;
 43		return 0;
 44	}
 45
 46	if (riscv_v_vsize != this_vsize) {
 47		WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
 48		return -EOPNOTSUPP;
 49	}
 50
 51	return 0;
 52}
 53
 54void __init riscv_v_setup_ctx_cache(void)
 55{
 56	if (!has_vector())
 57		return;
 58
 59	riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx",
 60							 riscv_v_vsize, 16, SLAB_PANIC,
 61							 0, riscv_v_vsize, NULL);
 62#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 63	riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
 64						  riscv_v_vsize, 16,
 65						  SLAB_PANIC, NULL);
 66#endif
 67}
 68
 69bool insn_is_vector(u32 insn_buf)
 70{
 71	u32 opcode = insn_buf & __INSN_OPCODE_MASK;
 72	u32 width, csr;
 73
 74	/*
 75	 * All V-related instructions, including CSR operations are 4-Byte. So,
 76	 * do not handle if the instruction length is not 4-Byte.
 77	 */
 78	if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
 79		return false;
 80
 81	switch (opcode) {
 82	case RVV_OPCODE_VECTOR:
 83		return true;
 84	case RVV_OPCODE_VL:
 85	case RVV_OPCODE_VS:
 86		width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
 87		if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
 88		    width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
 89			return true;
 90
 91		break;
 92	case RVG_OPCODE_SYSTEM:
 93		csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
 94		if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
 95		    (csr >= CSR_VL && csr <= CSR_VLENB))
 96			return true;
 97	}
 98
 99	return false;
100}
101
102static int riscv_v_thread_zalloc(struct kmem_cache *cache,
103				 struct __riscv_v_ext_state *ctx)
104{
105	void *datap;
106
107	datap = kmem_cache_zalloc(cache, GFP_KERNEL);
108	if (!datap)
109		return -ENOMEM;
110
111	ctx->datap = datap;
112	memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
113	return 0;
114}
115
116void riscv_v_thread_alloc(struct task_struct *tsk)
117{
118#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
119	riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
120#endif
121}
122
123void riscv_v_thread_free(struct task_struct *tsk)
124{
125	if (tsk->thread.vstate.datap)
126		kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);
127#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
128	if (tsk->thread.kernel_vstate.datap)
129		kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
130#endif
131}
132
133#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
134#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
135#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
136#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
137static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
138{
139	return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
140}
141
142static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
143{
144	return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
145}
146
147static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
148{
149	return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
150}
151
152static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
153				    bool inherit)
154{
155	unsigned long ctrl;
156
157	ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
158	ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
159	if (inherit)
160		ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
161	tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
162	tsk->thread.vstate_ctrl |= ctrl;
163}
164
165bool riscv_v_vstate_ctrl_user_allowed(void)
166{
167	return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
168}
169EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
170
171bool riscv_v_first_use_handler(struct pt_regs *regs)
172{
173	u32 __user *epc = (u32 __user *)regs->epc;
174	u32 insn = (u32)regs->badaddr;
175
176	if (!has_vector())
177		return false;
178
179	/* Do not handle if V is not supported, or disabled */
180	if (!riscv_v_vstate_ctrl_user_allowed())
181		return false;
182
183	/* If V has been enabled then it is not the first-use trap */
184	if (riscv_v_vstate_query(regs))
185		return false;
186
187	/* Get the instruction */
188	if (!insn) {
189		if (__get_user(insn, epc))
190			return false;
191	}
192
193	/* Filter out non-V instructions */
194	if (!insn_is_vector(insn))
195		return false;
196
197	/* Sanity check. datap should be null by the time of the first-use trap */
198	WARN_ON(current->thread.vstate.datap);
199
200	/*
201	 * Now we sure that this is a V instruction. And it executes in the
202	 * context where VS has been off. So, try to allocate the user's V
203	 * context and resume execution.
204	 */
205	if (riscv_v_thread_zalloc(riscv_v_user_cachep, &current->thread.vstate)) {
206		force_sig(SIGBUS);
207		return true;
208	}
209	riscv_v_vstate_on(regs);
210	riscv_v_vstate_set_restore(current, regs);
211	return true;
212}
213
214void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
215{
216	bool inherit;
217	int cur, next;
218
219	if (!has_vector())
220		return;
221
222	next = riscv_v_ctrl_get_next(tsk);
223	if (!next) {
224		if (READ_ONCE(riscv_v_implicit_uacc))
225			cur = PR_RISCV_V_VSTATE_CTRL_ON;
226		else
227			cur = PR_RISCV_V_VSTATE_CTRL_OFF;
228	} else {
229		cur = next;
230	}
231	/* Clear next mask if inherit-bit is not set */
232	inherit = riscv_v_ctrl_test_inherit(tsk);
233	if (!inherit)
234		next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
235
236	riscv_v_ctrl_set(tsk, cur, next, inherit);
237}
238
239long riscv_v_vstate_ctrl_get_current(void)
240{
241	if (!has_vector())
242		return -EINVAL;
243
244	return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
245}
246
247long riscv_v_vstate_ctrl_set_current(unsigned long arg)
248{
249	bool inherit;
250	int cur, next;
251
252	if (!has_vector())
253		return -EINVAL;
254
255	if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
256		return -EINVAL;
257
258	cur = VSTATE_CTRL_GET_CUR(arg);
259	switch (cur) {
260	case PR_RISCV_V_VSTATE_CTRL_OFF:
261		/* Do not allow user to turn off V if current is not off */
262		if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
263			return -EPERM;
264
265		break;
266	case PR_RISCV_V_VSTATE_CTRL_ON:
267		break;
268	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
269		cur = riscv_v_ctrl_get_cur(current);
270		break;
271	default:
272		return -EINVAL;
273	}
274
275	next = VSTATE_CTRL_GET_NEXT(arg);
276	inherit = VSTATE_CTRL_GET_INHERIT(arg);
277	switch (next) {
278	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
279	case PR_RISCV_V_VSTATE_CTRL_OFF:
280	case PR_RISCV_V_VSTATE_CTRL_ON:
281		riscv_v_ctrl_set(current, cur, next, inherit);
282		return 0;
283	}
284
285	return -EINVAL;
286}
287
288#ifdef CONFIG_SYSCTL
289
290static struct ctl_table riscv_v_default_vstate_table[] = {
291	{
292		.procname	= "riscv_v_default_allow",
293		.data		= &riscv_v_implicit_uacc,
294		.maxlen		= sizeof(riscv_v_implicit_uacc),
295		.mode		= 0644,
296		.proc_handler	= proc_dobool,
297	},
298};
299
300static int __init riscv_v_sysctl_init(void)
301{
302	if (has_vector())
303		if (!register_sysctl("abi", riscv_v_default_vstate_table))
304			return -EINVAL;
305	return 0;
306}
307
308#else /* ! CONFIG_SYSCTL */
309static int __init riscv_v_sysctl_init(void) { return 0; }
310#endif /* ! CONFIG_SYSCTL */
311
312static int __init riscv_v_init(void)
313{
314	return riscv_v_sysctl_init();
315}
316core_initcall(riscv_v_init);