Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2017 SiFive
4 */
5
6#include <asm/cacheflush.h>
7
8#ifdef CONFIG_SMP
9
10#include <asm/sbi.h>
11
12static void ipi_remote_fence_i(void *info)
13{
14 return local_flush_icache_all();
15}
16
17void flush_icache_all(void)
18{
19 if (IS_ENABLED(CONFIG_RISCV_SBI))
20 sbi_remote_fence_i(NULL);
21 else
22 on_each_cpu(ipi_remote_fence_i, NULL, 1);
23}
24EXPORT_SYMBOL(flush_icache_all);
25
26/*
27 * Performs an icache flush for the given MM context. RISC-V has no direct
28 * mechanism for instruction cache shoot downs, so instead we send an IPI that
29 * informs the remote harts they need to flush their local instruction caches.
30 * To avoid pathologically slow behavior in a common case (a bunch of
31 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
32 * IPIs for harts that are not currently executing a MM context and instead
33 * schedule a deferred local instruction cache flush to be performed before
34 * execution resumes on each hart.
35 */
36void flush_icache_mm(struct mm_struct *mm, bool local)
37{
38 unsigned int cpu;
39 cpumask_t others, *mask;
40
41 preempt_disable();
42
43 /* Mark every hart's icache as needing a flush for this MM. */
44 mask = &mm->context.icache_stale_mask;
45 cpumask_setall(mask);
46 /* Flush this hart's I$ now, and mark it as flushed. */
47 cpu = smp_processor_id();
48 cpumask_clear_cpu(cpu, mask);
49 local_flush_icache_all();
50
51 /*
52 * Flush the I$ of other harts concurrently executing, and mark them as
53 * flushed.
54 */
55 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
56 local |= cpumask_empty(&others);
57 if (mm == current->active_mm && local) {
58 /*
59 * It's assumed that at least one strongly ordered operation is
60 * performed on this hart between setting a hart's cpumask bit
61 * and scheduling this MM context on that hart. Sending an SBI
62 * remote message will do this, but in the case where no
63 * messages are sent we still need to order this hart's writes
64 * with flush_icache_deferred().
65 */
66 smp_mb();
67 } else if (IS_ENABLED(CONFIG_RISCV_SBI)) {
68 cpumask_t hartid_mask;
69
70 riscv_cpuid_to_hartid_mask(&others, &hartid_mask);
71 sbi_remote_fence_i(cpumask_bits(&hartid_mask));
72 } else {
73 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
74 }
75
76 preempt_enable();
77}
78
79#endif /* CONFIG_SMP */
80
81#ifdef CONFIG_MMU
82void flush_icache_pte(pte_t pte)
83{
84 struct page *page = pte_page(pte);
85
86 if (!test_and_set_bit(PG_dcache_clean, &page->flags))
87 flush_icache_all();
88}
89#endif /* CONFIG_MMU */
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2017 SiFive
4 */
5
6#include <linux/acpi.h>
7#include <linux/of.h>
8#include <linux/prctl.h>
9#include <asm/acpi.h>
10#include <asm/cacheflush.h>
11
12#ifdef CONFIG_SMP
13
14#include <asm/sbi.h>
15
16static void ipi_remote_fence_i(void *info)
17{
18 return local_flush_icache_all();
19}
20
21void flush_icache_all(void)
22{
23 local_flush_icache_all();
24
25 if (num_online_cpus() < 2)
26 return;
27 else if (riscv_use_sbi_for_rfence())
28 sbi_remote_fence_i(NULL);
29 else
30 on_each_cpu(ipi_remote_fence_i, NULL, 1);
31}
32EXPORT_SYMBOL(flush_icache_all);
33
34/*
35 * Performs an icache flush for the given MM context. RISC-V has no direct
36 * mechanism for instruction cache shoot downs, so instead we send an IPI that
37 * informs the remote harts they need to flush their local instruction caches.
38 * To avoid pathologically slow behavior in a common case (a bunch of
39 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
40 * IPIs for harts that are not currently executing a MM context and instead
41 * schedule a deferred local instruction cache flush to be performed before
42 * execution resumes on each hart.
43 */
44void flush_icache_mm(struct mm_struct *mm, bool local)
45{
46 unsigned int cpu;
47 cpumask_t others, *mask;
48
49 preempt_disable();
50
51 /* Mark every hart's icache as needing a flush for this MM. */
52 mask = &mm->context.icache_stale_mask;
53 cpumask_setall(mask);
54 /* Flush this hart's I$ now, and mark it as flushed. */
55 cpu = smp_processor_id();
56 cpumask_clear_cpu(cpu, mask);
57 local_flush_icache_all();
58
59 /*
60 * Flush the I$ of other harts concurrently executing, and mark them as
61 * flushed.
62 */
63 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
64 local |= cpumask_empty(&others);
65 if (mm == current->active_mm && local) {
66 /*
67 * It's assumed that at least one strongly ordered operation is
68 * performed on this hart between setting a hart's cpumask bit
69 * and scheduling this MM context on that hart. Sending an SBI
70 * remote message will do this, but in the case where no
71 * messages are sent we still need to order this hart's writes
72 * with flush_icache_deferred().
73 */
74 smp_mb();
75 } else if (riscv_use_sbi_for_rfence()) {
76 sbi_remote_fence_i(&others);
77 } else {
78 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
79 }
80
81 preempt_enable();
82}
83
84#endif /* CONFIG_SMP */
85
86#ifdef CONFIG_MMU
87void flush_icache_pte(struct mm_struct *mm, pte_t pte)
88{
89 struct folio *folio = page_folio(pte_page(pte));
90
91 if (!test_bit(PG_dcache_clean, &folio->flags)) {
92 flush_icache_mm(mm, false);
93 set_bit(PG_dcache_clean, &folio->flags);
94 }
95}
96#endif /* CONFIG_MMU */
97
98unsigned int riscv_cbom_block_size;
99EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
100
101unsigned int riscv_cboz_block_size;
102EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
103
104static void __init cbo_get_block_size(struct device_node *node,
105 const char *name, u32 *block_size,
106 unsigned long *first_hartid)
107{
108 unsigned long hartid;
109 u32 val;
110
111 if (riscv_of_processor_hartid(node, &hartid))
112 return;
113
114 if (of_property_read_u32(node, name, &val))
115 return;
116
117 if (!*block_size) {
118 *block_size = val;
119 *first_hartid = hartid;
120 } else if (*block_size != val) {
121 pr_warn("%s mismatched between harts %lu and %lu\n",
122 name, *first_hartid, hartid);
123 }
124}
125
126void __init riscv_init_cbo_blocksizes(void)
127{
128 unsigned long cbom_hartid, cboz_hartid;
129 u32 cbom_block_size = 0, cboz_block_size = 0;
130 struct device_node *node;
131 struct acpi_table_header *rhct;
132 acpi_status status;
133
134 if (acpi_disabled) {
135 for_each_of_cpu_node(node) {
136 /* set block-size for cbom and/or cboz extension if available */
137 cbo_get_block_size(node, "riscv,cbom-block-size",
138 &cbom_block_size, &cbom_hartid);
139 cbo_get_block_size(node, "riscv,cboz-block-size",
140 &cboz_block_size, &cboz_hartid);
141 }
142 } else {
143 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
144 if (ACPI_FAILURE(status))
145 return;
146
147 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL);
148 acpi_put_table((struct acpi_table_header *)rhct);
149 }
150
151 if (cbom_block_size)
152 riscv_cbom_block_size = cbom_block_size;
153
154 if (cboz_block_size)
155 riscv_cboz_block_size = cboz_block_size;
156}
157
158#ifdef CONFIG_SMP
159static void set_icache_stale_mask(void)
160{
161 int cpu = get_cpu();
162 cpumask_t *mask;
163 bool stale_cpu;
164
165 /*
166 * Mark every other hart's icache as needing a flush for
167 * this MM. Maintain the previous value of the current
168 * cpu to handle the case when this function is called
169 * concurrently on different harts.
170 */
171 mask = ¤t->mm->context.icache_stale_mask;
172 stale_cpu = cpumask_test_cpu(cpu, mask);
173
174 cpumask_setall(mask);
175 cpumask_assign_cpu(cpu, mask, stale_cpu);
176 put_cpu();
177}
178#endif
179
180/**
181 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in
182 * userspace.
183 * @ctx: Set the type of icache flushing instructions permitted/prohibited in
184 * userspace. Supported values described below.
185 *
186 * Supported values for ctx:
187 *
188 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space.
189 *
190 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in
191 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``.
192 * Therefore, caution must be taken; use this flag only when you can guarantee
193 * that no thread in the process will emit fence.i from this point onward.
194 *
195 * @scope: Set scope of where icache flushing instructions are allowed to be
196 * emitted. Supported values described below.
197 *
198 * Supported values for scope:
199 *
200 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process
201 * is coherent with instruction storage upon
202 * migration.
203 *
204 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is
205 * coherent with instruction storage upon
206 * migration.
207 *
208 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are
209 * permitted to emit icache flushing instructions. Whenever any thread in the
210 * process is migrated, the corresponding hart's icache will be guaranteed to be
211 * consistent with instruction storage. This does not enforce any guarantees
212 * outside of migration. If a thread modifies an instruction that another thread
213 * may attempt to execute, the other thread must still emit an icache flushing
214 * instruction before attempting to execute the potentially modified
215 * instruction. This must be performed by the user-space program.
216 *
217 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the
218 * thread calling this function is permitted to emit icache flushing
219 * instructions. When the thread is migrated, the corresponding hart's icache
220 * will be guaranteed to be consistent with instruction storage.
221 *
222 * On kernels configured without SMP, this function is a nop as migrations
223 * across harts will not occur.
224 */
225int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope)
226{
227#ifdef CONFIG_SMP
228 switch (ctx) {
229 case PR_RISCV_CTX_SW_FENCEI_ON:
230 switch (scope) {
231 case PR_RISCV_SCOPE_PER_PROCESS:
232 current->mm->context.force_icache_flush = true;
233 break;
234 case PR_RISCV_SCOPE_PER_THREAD:
235 current->thread.force_icache_flush = true;
236 break;
237 default:
238 return -EINVAL;
239 }
240 break;
241 case PR_RISCV_CTX_SW_FENCEI_OFF:
242 switch (scope) {
243 case PR_RISCV_SCOPE_PER_PROCESS:
244 set_icache_stale_mask();
245 current->mm->context.force_icache_flush = false;
246 break;
247 case PR_RISCV_SCOPE_PER_THREAD:
248 set_icache_stale_mask();
249 current->thread.force_icache_flush = false;
250 break;
251 default:
252 return -EINVAL;
253 }
254 break;
255 default:
256 return -EINVAL;
257 }
258 return 0;
259#else
260 switch (ctx) {
261 case PR_RISCV_CTX_SW_FENCEI_ON:
262 case PR_RISCV_CTX_SW_FENCEI_OFF:
263 return 0;
264 default:
265 return -EINVAL;
266 }
267#endif
268}