Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * The hwprobe interface, for allowing userspace to probe to see which features
4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for
5 * more details.
6 */
7#include <linux/syscalls.h>
8#include <asm/cacheflush.h>
9#include <asm/cpufeature.h>
10#include <asm/hwprobe.h>
11#include <asm/processor.h>
12#include <asm/delay.h>
13#include <asm/sbi.h>
14#include <asm/switch_to.h>
15#include <asm/uaccess.h>
16#include <asm/unistd.h>
17#include <asm/vector.h>
18#include <vdso/vsyscall.h>
19
20
21static void hwprobe_arch_id(struct riscv_hwprobe *pair,
22 const struct cpumask *cpus)
23{
24 u64 id = -1ULL;
25 bool first = true;
26 int cpu;
27
28 for_each_cpu(cpu, cpus) {
29 u64 cpu_id;
30
31 switch (pair->key) {
32 case RISCV_HWPROBE_KEY_MVENDORID:
33 cpu_id = riscv_cached_mvendorid(cpu);
34 break;
35 case RISCV_HWPROBE_KEY_MIMPID:
36 cpu_id = riscv_cached_mimpid(cpu);
37 break;
38 case RISCV_HWPROBE_KEY_MARCHID:
39 cpu_id = riscv_cached_marchid(cpu);
40 break;
41 }
42
43 if (first) {
44 id = cpu_id;
45 first = false;
46 }
47
48 /*
49 * If there's a mismatch for the given set, return -1 in the
50 * value.
51 */
52 if (id != cpu_id) {
53 id = -1ULL;
54 break;
55 }
56 }
57
58 pair->value = id;
59}
60
61static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
62 const struct cpumask *cpus)
63{
64 int cpu;
65 u64 missing = 0;
66
67 pair->value = 0;
68 if (has_fpu())
69 pair->value |= RISCV_HWPROBE_IMA_FD;
70
71 if (riscv_isa_extension_available(NULL, c))
72 pair->value |= RISCV_HWPROBE_IMA_C;
73
74 if (has_vector() && riscv_isa_extension_available(NULL, v))
75 pair->value |= RISCV_HWPROBE_IMA_V;
76
77 /*
78 * Loop through and record extensions that 1) anyone has, and 2) anyone
79 * doesn't have.
80 */
81 for_each_cpu(cpu, cpus) {
82 struct riscv_isainfo *isainfo = &hart_isa[cpu];
83
84#define EXT_KEY(ext) \
85 do { \
86 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \
87 pair->value |= RISCV_HWPROBE_EXT_##ext; \
88 else \
89 missing |= RISCV_HWPROBE_EXT_##ext; \
90 } while (false)
91
92 /*
93 * Only use EXT_KEY() for extensions which can be exposed to userspace,
94 * regardless of the kernel's configuration, as no other checks, besides
95 * presence in the hart_isa bitmap, are made.
96 */
97 EXT_KEY(ZACAS);
98 EXT_KEY(ZAWRS);
99 EXT_KEY(ZBA);
100 EXT_KEY(ZBB);
101 EXT_KEY(ZBC);
102 EXT_KEY(ZBKB);
103 EXT_KEY(ZBKC);
104 EXT_KEY(ZBKX);
105 EXT_KEY(ZBS);
106 EXT_KEY(ZCA);
107 EXT_KEY(ZCB);
108 EXT_KEY(ZCMOP);
109 EXT_KEY(ZICBOZ);
110 EXT_KEY(ZICOND);
111 EXT_KEY(ZIHINTNTL);
112 EXT_KEY(ZIHINTPAUSE);
113 EXT_KEY(ZIMOP);
114 EXT_KEY(ZKND);
115 EXT_KEY(ZKNE);
116 EXT_KEY(ZKNH);
117 EXT_KEY(ZKSED);
118 EXT_KEY(ZKSH);
119 EXT_KEY(ZKT);
120 EXT_KEY(ZTSO);
121
122 /*
123 * All the following extensions must depend on the kernel
124 * support of V.
125 */
126 if (has_vector()) {
127 EXT_KEY(ZVBB);
128 EXT_KEY(ZVBC);
129 EXT_KEY(ZVE32F);
130 EXT_KEY(ZVE32X);
131 EXT_KEY(ZVE64D);
132 EXT_KEY(ZVE64F);
133 EXT_KEY(ZVE64X);
134 EXT_KEY(ZVFH);
135 EXT_KEY(ZVFHMIN);
136 EXT_KEY(ZVKB);
137 EXT_KEY(ZVKG);
138 EXT_KEY(ZVKNED);
139 EXT_KEY(ZVKNHA);
140 EXT_KEY(ZVKNHB);
141 EXT_KEY(ZVKSED);
142 EXT_KEY(ZVKSH);
143 EXT_KEY(ZVKT);
144 }
145
146 if (has_fpu()) {
147 EXT_KEY(ZCD);
148 EXT_KEY(ZCF);
149 EXT_KEY(ZFA);
150 EXT_KEY(ZFH);
151 EXT_KEY(ZFHMIN);
152 }
153
154 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
155 EXT_KEY(SUPM);
156#undef EXT_KEY
157 }
158
159 /* Now turn off reporting features if any CPU is missing it. */
160 pair->value &= ~missing;
161}
162
163static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
164{
165 struct riscv_hwprobe pair;
166
167 hwprobe_isa_ext0(&pair, cpus);
168 return (pair.value & ext);
169}
170
171#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
172static u64 hwprobe_misaligned(const struct cpumask *cpus)
173{
174 int cpu;
175 u64 perf = -1ULL;
176
177 for_each_cpu(cpu, cpus) {
178 int this_perf = per_cpu(misaligned_access_speed, cpu);
179
180 if (perf == -1ULL)
181 perf = this_perf;
182
183 if (perf != this_perf) {
184 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
185 break;
186 }
187 }
188
189 if (perf == -1ULL)
190 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
191
192 return perf;
193}
194#else
195static u64 hwprobe_misaligned(const struct cpumask *cpus)
196{
197 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
198 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
199
200 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
201 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
202
203 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
204}
205#endif
206
207#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
208static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
209{
210 int cpu;
211 u64 perf = -1ULL;
212
213 /* Return if supported or not even if speed wasn't probed */
214 for_each_cpu(cpu, cpus) {
215 int this_perf = per_cpu(vector_misaligned_access, cpu);
216
217 if (perf == -1ULL)
218 perf = this_perf;
219
220 if (perf != this_perf) {
221 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
222 break;
223 }
224 }
225
226 if (perf == -1ULL)
227 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
228
229 return perf;
230}
231#else
232static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
233{
234 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
235 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
236
237 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
238 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
239
240 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
241}
242#endif
243
244static void hwprobe_one_pair(struct riscv_hwprobe *pair,
245 const struct cpumask *cpus)
246{
247 switch (pair->key) {
248 case RISCV_HWPROBE_KEY_MVENDORID:
249 case RISCV_HWPROBE_KEY_MARCHID:
250 case RISCV_HWPROBE_KEY_MIMPID:
251 hwprobe_arch_id(pair, cpus);
252 break;
253 /*
254 * The kernel already assumes that the base single-letter ISA
255 * extensions are supported on all harts, and only supports the
256 * IMA base, so just cheat a bit here and tell that to
257 * userspace.
258 */
259 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
260 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
261 break;
262
263 case RISCV_HWPROBE_KEY_IMA_EXT_0:
264 hwprobe_isa_ext0(pair, cpus);
265 break;
266
267 case RISCV_HWPROBE_KEY_CPUPERF_0:
268 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
269 pair->value = hwprobe_misaligned(cpus);
270 break;
271
272 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
273 pair->value = hwprobe_vec_misaligned(cpus);
274 break;
275
276 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
277 pair->value = 0;
278 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
279 pair->value = riscv_cboz_block_size;
280 break;
281 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
282 pair->value = user_max_virt_addr();
283 break;
284
285 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
286 pair->value = riscv_timebase;
287 break;
288
289 /*
290 * For forward compatibility, unknown keys don't fail the whole
291 * call, but get their element key set to -1 and value set to 0
292 * indicating they're unrecognized.
293 */
294 default:
295 pair->key = -1;
296 pair->value = 0;
297 break;
298 }
299}
300
301static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
302 size_t pair_count, size_t cpusetsize,
303 unsigned long __user *cpus_user,
304 unsigned int flags)
305{
306 size_t out;
307 int ret;
308 cpumask_t cpus;
309
310 /* Check the reserved flags. */
311 if (flags != 0)
312 return -EINVAL;
313
314 /*
315 * The interface supports taking in a CPU mask, and returns values that
316 * are consistent across that mask. Allow userspace to specify NULL and
317 * 0 as a shortcut to all online CPUs.
318 */
319 cpumask_clear(&cpus);
320 if (!cpusetsize && !cpus_user) {
321 cpumask_copy(&cpus, cpu_online_mask);
322 } else {
323 if (cpusetsize > cpumask_size())
324 cpusetsize = cpumask_size();
325
326 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
327 if (ret)
328 return -EFAULT;
329
330 /*
331 * Userspace must provide at least one online CPU, without that
332 * there's no way to define what is supported.
333 */
334 cpumask_and(&cpus, &cpus, cpu_online_mask);
335 if (cpumask_empty(&cpus))
336 return -EINVAL;
337 }
338
339 for (out = 0; out < pair_count; out++, pairs++) {
340 struct riscv_hwprobe pair;
341
342 if (get_user(pair.key, &pairs->key))
343 return -EFAULT;
344
345 pair.value = 0;
346 hwprobe_one_pair(&pair, &cpus);
347 ret = put_user(pair.key, &pairs->key);
348 if (ret == 0)
349 ret = put_user(pair.value, &pairs->value);
350
351 if (ret)
352 return -EFAULT;
353 }
354
355 return 0;
356}
357
358static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
359 size_t pair_count, size_t cpusetsize,
360 unsigned long __user *cpus_user,
361 unsigned int flags)
362{
363 cpumask_t cpus, one_cpu;
364 bool clear_all = false;
365 size_t i;
366 int ret;
367
368 if (flags != RISCV_HWPROBE_WHICH_CPUS)
369 return -EINVAL;
370
371 if (!cpusetsize || !cpus_user)
372 return -EINVAL;
373
374 if (cpusetsize > cpumask_size())
375 cpusetsize = cpumask_size();
376
377 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
378 if (ret)
379 return -EFAULT;
380
381 if (cpumask_empty(&cpus))
382 cpumask_copy(&cpus, cpu_online_mask);
383
384 cpumask_and(&cpus, &cpus, cpu_online_mask);
385
386 cpumask_clear(&one_cpu);
387
388 for (i = 0; i < pair_count; i++) {
389 struct riscv_hwprobe pair, tmp;
390 int cpu;
391
392 ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
393 if (ret)
394 return -EFAULT;
395
396 if (!riscv_hwprobe_key_is_valid(pair.key)) {
397 clear_all = true;
398 pair = (struct riscv_hwprobe){ .key = -1, };
399 ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
400 if (ret)
401 return -EFAULT;
402 }
403
404 if (clear_all)
405 continue;
406
407 tmp = (struct riscv_hwprobe){ .key = pair.key, };
408
409 for_each_cpu(cpu, &cpus) {
410 cpumask_set_cpu(cpu, &one_cpu);
411
412 hwprobe_one_pair(&tmp, &one_cpu);
413
414 if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
415 cpumask_clear_cpu(cpu, &cpus);
416
417 cpumask_clear_cpu(cpu, &one_cpu);
418 }
419 }
420
421 if (clear_all)
422 cpumask_clear(&cpus);
423
424 ret = copy_to_user(cpus_user, &cpus, cpusetsize);
425 if (ret)
426 return -EFAULT;
427
428 return 0;
429}
430
431static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
432 size_t pair_count, size_t cpusetsize,
433 unsigned long __user *cpus_user,
434 unsigned int flags)
435{
436 if (flags & RISCV_HWPROBE_WHICH_CPUS)
437 return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
438 cpus_user, flags);
439
440 return hwprobe_get_values(pairs, pair_count, cpusetsize,
441 cpus_user, flags);
442}
443
444#ifdef CONFIG_MMU
445
446static int __init init_hwprobe_vdso_data(void)
447{
448 struct vdso_data *vd = __arch_get_k_vdso_data();
449 struct arch_vdso_time_data *avd = &vd->arch_data;
450 u64 id_bitsmash = 0;
451 struct riscv_hwprobe pair;
452 int key;
453
454 /*
455 * Initialize vDSO data with the answers for the "all CPUs" case, to
456 * save a syscall in the common case.
457 */
458 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
459 pair.key = key;
460 hwprobe_one_pair(&pair, cpu_online_mask);
461
462 WARN_ON_ONCE(pair.key < 0);
463
464 avd->all_cpu_hwprobe_values[key] = pair.value;
465 /*
466 * Smash together the vendor, arch, and impl IDs to see if
467 * they're all 0 or any negative.
468 */
469 if (key <= RISCV_HWPROBE_KEY_MIMPID)
470 id_bitsmash |= pair.value;
471 }
472
473 /*
474 * If the arch, vendor, and implementation ID are all the same across
475 * all harts, then assume all CPUs are the same, and allow the vDSO to
476 * answer queries for arbitrary masks. However if all values are 0 (not
477 * populated) or any value returns -1 (varies across CPUs), then the
478 * vDSO should defer to the kernel for exotic cpu masks.
479 */
480 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
481 return 0;
482}
483
484arch_initcall_sync(init_hwprobe_vdso_data);
485
486#endif /* CONFIG_MMU */
487
488SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
489 size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
490 cpus, unsigned int, flags)
491{
492 return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
493 cpus, flags);
494}