1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/atomic.h>
  3#include <linux/mmu_context.h>
  4#include <linux/percpu.h>
  5#include <linux/spinlock.h>
  6
  7static DEFINE_RAW_SPINLOCK(cpu_mmid_lock);
  8
  9static atomic64_t mmid_version;
 10static unsigned int num_mmids;
 11static unsigned long *mmid_map;
 12
 13static DEFINE_PER_CPU(u64, reserved_mmids);
 14static cpumask_t tlb_flush_pending;
 15
 16static bool asid_versions_eq(int cpu, u64 a, u64 b)
 17{
 18	return ((a ^ b) & asid_version_mask(cpu)) == 0;
 19}
 20
 21void get_new_mmu_context(struct mm_struct *mm)
 22{
 23	unsigned int cpu;
 24	u64 asid;
 25
 26	/*
 27	 * This function is specific to ASIDs, and should not be called when
 28	 * MMIDs are in use.
 29	 */
 30	if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
 31		return;
 32
 33	cpu = smp_processor_id();
 34	asid = asid_cache(cpu);
 35
 36	if (!((asid += cpu_asid_inc()) & cpu_asid_mask(&cpu_data[cpu]))) {
 37		if (cpu_has_vtag_icache)
 38			flush_icache_all();
 39		local_flush_tlb_all();	/* start new asid cycle */
 40	}
 41
 42	set_cpu_context(cpu, mm, asid);
 43	asid_cache(cpu) = asid;
 44}
 45EXPORT_SYMBOL_GPL(get_new_mmu_context);
 46
 47void check_mmu_context(struct mm_struct *mm)
 48{
 49	unsigned int cpu = smp_processor_id();
 50
 51	/*
 52	 * This function is specific to ASIDs, and should not be called when
 53	 * MMIDs are in use.
 54	 */
 55	if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
 56		return;
 57
 58	/* Check if our ASID is of an older version and thus invalid */
 59	if (!asid_versions_eq(cpu, cpu_context(cpu, mm), asid_cache(cpu)))
 60		get_new_mmu_context(mm);
 61}
 62EXPORT_SYMBOL_GPL(check_mmu_context);
 63
 64static void flush_context(void)
 65{
 66	u64 mmid;
 67	int cpu;
 68
 69	/* Update the list of reserved MMIDs and the MMID bitmap */
 70	bitmap_zero(mmid_map, num_mmids);
 71
 72	/* Reserve an MMID for kmap/wired entries */
 73	__set_bit(MMID_KERNEL_WIRED, mmid_map);
 74
 75	for_each_possible_cpu(cpu) {
 76		mmid = xchg_relaxed(&cpu_data[cpu].asid_cache, 0);
 77
 78		/*
 79		 * If this CPU has already been through a
 80		 * rollover, but hasn't run another task in
 81		 * the meantime, we must preserve its reserved
 82		 * MMID, as this is the only trace we have of
 83		 * the process it is still running.
 84		 */
 85		if (mmid == 0)
 86			mmid = per_cpu(reserved_mmids, cpu);
 87
 88		__set_bit(mmid & cpu_asid_mask(&cpu_data[cpu]), mmid_map);
 89		per_cpu(reserved_mmids, cpu) = mmid;
 90	}
 91
 92	/*
 93	 * Queue a TLB invalidation for each CPU to perform on next
 94	 * context-switch
 95	 */
 96	cpumask_setall(&tlb_flush_pending);
 97}
 98
 99static bool check_update_reserved_mmid(u64 mmid, u64 newmmid)
100{
101	bool hit;
102	int cpu;
103
104	/*
105	 * Iterate over the set of reserved MMIDs looking for a match.
106	 * If we find one, then we can update our mm to use newmmid
107	 * (i.e. the same MMID in the current generation) but we can't
108	 * exit the loop early, since we need to ensure that all copies
109	 * of the old MMID are updated to reflect the mm. Failure to do
110	 * so could result in us missing the reserved MMID in a future
111	 * generation.
112	 */
113	hit = false;
114	for_each_possible_cpu(cpu) {
115		if (per_cpu(reserved_mmids, cpu) == mmid) {
116			hit = true;
117			per_cpu(reserved_mmids, cpu) = newmmid;
118		}
119	}
120
121	return hit;
122}
123
124static u64 get_new_mmid(struct mm_struct *mm)
125{
126	static u32 cur_idx = MMID_KERNEL_WIRED + 1;
127	u64 mmid, version, mmid_mask;
128
129	mmid = cpu_context(0, mm);
130	version = atomic64_read(&mmid_version);
131	mmid_mask = cpu_asid_mask(&boot_cpu_data);
132
133	if (!asid_versions_eq(0, mmid, 0)) {
134		u64 newmmid = version | (mmid & mmid_mask);
135
136		/*
137		 * If our current MMID was active during a rollover, we
138		 * can continue to use it and this was just a false alarm.
139		 */
140		if (check_update_reserved_mmid(mmid, newmmid)) {
141			mmid = newmmid;
142			goto set_context;
143		}
144
145		/*
146		 * We had a valid MMID in a previous life, so try to re-use
147		 * it if possible.
148		 */
149		if (!__test_and_set_bit(mmid & mmid_mask, mmid_map)) {
150			mmid = newmmid;
151			goto set_context;
152		}
153	}
154
155	/* Allocate a free MMID */
156	mmid = find_next_zero_bit(mmid_map, num_mmids, cur_idx);
157	if (mmid != num_mmids)
158		goto reserve_mmid;
159
160	/* We're out of MMIDs, so increment the global version */
161	version = atomic64_add_return_relaxed(asid_first_version(0),
162					      &mmid_version);
163
164	/* Note currently active MMIDs & mark TLBs as requiring flushes */
165	flush_context();
166
167	/* We have more MMIDs than CPUs, so this will always succeed */
168	mmid = find_first_zero_bit(mmid_map, num_mmids);
169
170reserve_mmid:
171	__set_bit(mmid, mmid_map);
172	cur_idx = mmid;
173	mmid |= version;
174set_context:
175	set_cpu_context(0, mm, mmid);
176	return mmid;
177}
178
179void check_switch_mmu_context(struct mm_struct *mm)
180{
181	unsigned int cpu = smp_processor_id();
182	u64 ctx, old_active_mmid;
183	unsigned long flags;
184
185	if (!cpu_has_mmid) {
186		check_mmu_context(mm);
187		write_c0_entryhi(cpu_asid(cpu, mm));
188		goto setup_pgd;
189	}
190
191	/*
192	 * MMID switch fast-path, to avoid acquiring cpu_mmid_lock when it's
193	 * unnecessary.
194	 *
195	 * The memory ordering here is subtle. If our active_mmids is non-zero
196	 * and the MMID matches the current version, then we update the CPU's
197	 * asid_cache with a relaxed cmpxchg. Racing with a concurrent rollover
198	 * means that either:
199	 *
200	 * - We get a zero back from the cmpxchg and end up waiting on
201	 *   cpu_mmid_lock in check_mmu_context(). Taking the lock synchronises
202	 *   with the rollover and so we are forced to see the updated
203	 *   generation.
204	 *
205	 * - We get a valid MMID back from the cmpxchg, which means the
206	 *   relaxed xchg in flush_context will treat us as reserved
207	 *   because atomic RmWs are totally ordered for a given location.
208	 */
209	ctx = cpu_context(cpu, mm);
210	old_active_mmid = READ_ONCE(cpu_data[cpu].asid_cache);
211	if (!old_active_mmid ||
212	    !asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)) ||
213	    !cmpxchg_relaxed(&cpu_data[cpu].asid_cache, old_active_mmid, ctx)) {
214		raw_spin_lock_irqsave(&cpu_mmid_lock, flags);
215
216		ctx = cpu_context(cpu, mm);
217		if (!asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)))
218			ctx = get_new_mmid(mm);
219
220		WRITE_ONCE(cpu_data[cpu].asid_cache, ctx);
221		raw_spin_unlock_irqrestore(&cpu_mmid_lock, flags);
222	}
223
224	/*
225	 * Invalidate the local TLB if needed. Note that we must only clear our
226	 * bit in tlb_flush_pending after this is complete, so that the
227	 * cpu_has_shared_ftlb_entries case below isn't misled.
228	 */
229	if (cpumask_test_cpu(cpu, &tlb_flush_pending)) {
230		if (cpu_has_vtag_icache)
231			flush_icache_all();
232		local_flush_tlb_all();
233		cpumask_clear_cpu(cpu, &tlb_flush_pending);
234	}
235
236	write_c0_memorymapid(ctx & cpu_asid_mask(&boot_cpu_data));
237
238	/*
239	 * If this CPU shares FTLB entries with its siblings and one or more of
240	 * those siblings hasn't yet invalidated its TLB following a version
241	 * increase then we need to invalidate any TLB entries for our MMID
242	 * that we might otherwise pick up from a sibling.
243	 *
244	 * We ifdef on CONFIG_SMP because cpu_sibling_map isn't defined in
245	 * CONFIG_SMP=n kernels.
246	 */
247#ifdef CONFIG_SMP
248	if (cpu_has_shared_ftlb_entries &&
249	    cpumask_intersects(&tlb_flush_pending, &cpu_sibling_map[cpu])) {
250		/* Ensure we operate on the new MMID */
251		mtc0_tlbw_hazard();
252
253		/*
254		 * Invalidate all TLB entries associated with the new
255		 * MMID, and wait for the invalidation to complete.
256		 */
257		ginvt_mmid();
258		sync_ginv();
259	}
260#endif
261
262setup_pgd:
263	TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
264}
265EXPORT_SYMBOL_GPL(check_switch_mmu_context);
266
267static int mmid_init(void)
268{
269	if (!cpu_has_mmid)
270		return 0;
271
272	/*
273	 * Expect allocation after rollover to fail if we don't have at least
274	 * one more MMID than CPUs.
275	 */
276	num_mmids = asid_first_version(0);
277	WARN_ON(num_mmids <= num_possible_cpus());
278
279	atomic64_set(&mmid_version, asid_first_version(0));
280	mmid_map = bitmap_zalloc(num_mmids, GFP_KERNEL);
281	if (!mmid_map)
282		panic("Failed to allocate bitmap for %u MMIDs\n", num_mmids);
283
284	/* Reserve an MMID for kmap/wired entries */
285	__set_bit(MMID_KERNEL_WIRED, mmid_map);
286
287	pr_info("MMID allocator initialised with %u entries\n", num_mmids);
288	return 0;
289}
290early_initcall(mmid_init);