1// SPDX-License-Identifier: GPL-2.0-only
 2// Copyright 2024 Google LLC
 3// Author: Ard Biesheuvel <ardb@google.com>
 4
 5#include <linux/types.h>
 6#include <asm/tlbflush.h>
 7
 8#ifdef CONFIG_CPU_TLB_V4WT
 9void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
10void v4_flush_kern_tlb_range(unsigned long, unsigned long);
11
12struct cpu_tlb_fns v4_tlb_fns __initconst = {
13	.flush_user_range	= v4_flush_user_tlb_range,
14	.flush_kern_range	= v4_flush_kern_tlb_range,
15	.tlb_flags		= v4_tlb_flags,
16};
17#endif
18
19#ifdef CONFIG_CPU_TLB_V4WB
20void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
21void v4wb_flush_kern_tlb_range(unsigned long, unsigned long);
22
23struct cpu_tlb_fns v4wb_tlb_fns __initconst = {
24	.flush_user_range	= v4wb_flush_user_tlb_range,
25	.flush_kern_range	= v4wb_flush_kern_tlb_range,
26	.tlb_flags		= v4wb_tlb_flags,
27};
28#endif
29
30#if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON)
31void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
32void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long);
33
34struct cpu_tlb_fns v4wbi_tlb_fns __initconst = {
35	.flush_user_range	= v4wbi_flush_user_tlb_range,
36	.flush_kern_range	= v4wbi_flush_kern_tlb_range,
37	.tlb_flags		= v4wbi_tlb_flags,
38};
39#endif
40
41#ifdef CONFIG_CPU_TLB_V6
42void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
43void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long);
44
45struct cpu_tlb_fns v6wbi_tlb_fns __initconst = {
46	.flush_user_range	= v6wbi_flush_user_tlb_range,
47	.flush_kern_range	= v6wbi_flush_kern_tlb_range,
48	.tlb_flags		= v6wbi_tlb_flags,
49};
50#endif
51
52#ifdef CONFIG_CPU_TLB_V7
53void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
54void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long);
55
56struct cpu_tlb_fns v7wbi_tlb_fns __initconst = {
57	.flush_user_range	= v7wbi_flush_user_tlb_range,
58	.flush_kern_range	= v7wbi_flush_kern_tlb_range,
59	.tlb_flags		= IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp
60							 : v7wbi_tlb_flags_up,
61};
62
63#ifdef CONFIG_SMP_ON_UP
64/* This will be run-time patched so the offset better be right */
65static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8);
66
67asm("	.pushsection	\".alt.smp.init\", \"a\"		\n" \
68    "	.align		2					\n" \
69    "	.long		v7wbi_tlb_fns + 8 - .			\n" \
70    "	.long "  	__stringify(v7wbi_tlb_flags_up) "	\n" \
71    "	.popsection						\n");
72#endif
73#endif
74
75#ifdef CONFIG_CPU_TLB_FA
76void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
77void fa_flush_kern_tlb_range(unsigned long, unsigned long);
78
79struct cpu_tlb_fns fa_tlb_fns __initconst = {
80	.flush_user_range	= fa_flush_user_tlb_range,
81	.flush_kern_range	= fa_flush_kern_tlb_range,
82	.tlb_flags		= fa_tlb_flags,
83};
84#endif