Linux Audio

Check our new training course

Loading...
Note: File does not exist in v4.10.11.
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Generic ASID allocator.
  4 *
  5 * Based on arch/arm/mm/context.c
  6 *
  7 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
  8 * Copyright (C) 2012 ARM Ltd.
  9 */
 10
 11#include <linux/slab.h>
 12#include <linux/mm_types.h>
 13
 14#include <asm/asid.h>
 15
 16#define reserved_asid(info, cpu) *per_cpu_ptr((info)->reserved, cpu)
 17
 18#define ASID_MASK(info)			(~GENMASK((info)->bits - 1, 0))
 19#define ASID_FIRST_VERSION(info)	(1UL << ((info)->bits))
 20
 21#define asid2idx(info, asid)		(((asid) & ~ASID_MASK(info)) >> (info)->ctxt_shift)
 22#define idx2asid(info, idx)		(((idx) << (info)->ctxt_shift) & ~ASID_MASK(info))
 23
 24static void flush_context(struct asid_info *info)
 25{
 26	int i;
 27	u64 asid;
 28
 29	/* Update the list of reserved ASIDs and the ASID bitmap. */
 30	bitmap_zero(info->map, NUM_CTXT_ASIDS(info));
 31
 32	for_each_possible_cpu(i) {
 33		asid = atomic64_xchg_relaxed(&active_asid(info, i), 0);
 34		/*
 35		 * If this CPU has already been through a
 36		 * rollover, but hasn't run another task in
 37		 * the meantime, we must preserve its reserved
 38		 * ASID, as this is the only trace we have of
 39		 * the process it is still running.
 40		 */
 41		if (asid == 0)
 42			asid = reserved_asid(info, i);
 43		__set_bit(asid2idx(info, asid), info->map);
 44		reserved_asid(info, i) = asid;
 45	}
 46
 47	/*
 48	 * Queue a TLB invalidation for each CPU to perform on next
 49	 * context-switch
 50	 */
 51	cpumask_setall(&info->flush_pending);
 52}
 53
 54static bool check_update_reserved_asid(struct asid_info *info, u64 asid,
 55				       u64 newasid)
 56{
 57	int cpu;
 58	bool hit = false;
 59
 60	/*
 61	 * Iterate over the set of reserved ASIDs looking for a match.
 62	 * If we find one, then we can update our mm to use newasid
 63	 * (i.e. the same ASID in the current generation) but we can't
 64	 * exit the loop early, since we need to ensure that all copies
 65	 * of the old ASID are updated to reflect the mm. Failure to do
 66	 * so could result in us missing the reserved ASID in a future
 67	 * generation.
 68	 */
 69	for_each_possible_cpu(cpu) {
 70		if (reserved_asid(info, cpu) == asid) {
 71			hit = true;
 72			reserved_asid(info, cpu) = newasid;
 73		}
 74	}
 75
 76	return hit;
 77}
 78
 79static u64 new_context(struct asid_info *info, atomic64_t *pasid,
 80		       struct mm_struct *mm)
 81{
 82	static u32 cur_idx = 1;
 83	u64 asid = atomic64_read(pasid);
 84	u64 generation = atomic64_read(&info->generation);
 85
 86	if (asid != 0) {
 87		u64 newasid = generation | (asid & ~ASID_MASK(info));
 88
 89		/*
 90		 * If our current ASID was active during a rollover, we
 91		 * can continue to use it and this was just a false alarm.
 92		 */
 93		if (check_update_reserved_asid(info, asid, newasid))
 94			return newasid;
 95
 96		/*
 97		 * We had a valid ASID in a previous life, so try to re-use
 98		 * it if possible.
 99		 */
100		if (!__test_and_set_bit(asid2idx(info, asid), info->map))
101			return newasid;
102	}
103
104	/*
105	 * Allocate a free ASID. If we can't find one, take a note of the
106	 * currently active ASIDs and mark the TLBs as requiring flushes.  We
107	 * always count from ASID #2 (index 1), as we use ASID #0 when setting
108	 * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
109	 * pairs.
110	 */
111	asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), cur_idx);
112	if (asid != NUM_CTXT_ASIDS(info))
113		goto set_asid;
114
115	/* We're out of ASIDs, so increment the global generation count */
116	generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION(info),
117						 &info->generation);
118	flush_context(info);
119
120	/* We have more ASIDs than CPUs, so this will always succeed */
121	asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), 1);
122
123set_asid:
124	__set_bit(asid, info->map);
125	cur_idx = asid;
126	cpumask_clear(mm_cpumask(mm));
127	return idx2asid(info, asid) | generation;
128}
129
130/*
131 * Generate a new ASID for the context.
132 *
133 * @pasid: Pointer to the current ASID batch allocated. It will be updated
134 * with the new ASID batch.
135 * @cpu: current CPU ID. Must have been acquired through get_cpu()
136 */
137void asid_new_context(struct asid_info *info, atomic64_t *pasid,
138		      unsigned int cpu, struct mm_struct *mm)
139{
140	unsigned long flags;
141	u64 asid;
142
143	raw_spin_lock_irqsave(&info->lock, flags);
144	/* Check that our ASID belongs to the current generation. */
145	asid = atomic64_read(pasid);
146	if ((asid ^ atomic64_read(&info->generation)) >> info->bits) {
147		asid = new_context(info, pasid, mm);
148		atomic64_set(pasid, asid);
149	}
150
151	if (cpumask_test_and_clear_cpu(cpu, &info->flush_pending))
152		info->flush_cpu_ctxt_cb();
153
154	atomic64_set(&active_asid(info, cpu), asid);
155	cpumask_set_cpu(cpu, mm_cpumask(mm));
156	raw_spin_unlock_irqrestore(&info->lock, flags);
157}
158
159/*
160 * Initialize the ASID allocator
161 *
162 * @info: Pointer to the asid allocator structure
163 * @bits: Number of ASIDs available
164 * @asid_per_ctxt: Number of ASIDs to allocate per-context. ASIDs are
165 * allocated contiguously for a given context. This value should be a power of
166 * 2.
167 */
168int asid_allocator_init(struct asid_info *info,
169			u32 bits, unsigned int asid_per_ctxt,
170			void (*flush_cpu_ctxt_cb)(void))
171{
172	info->bits = bits;
173	info->ctxt_shift = ilog2(asid_per_ctxt);
174	info->flush_cpu_ctxt_cb = flush_cpu_ctxt_cb;
175	/*
176	 * Expect allocation after rollover to fail if we don't have at least
177	 * one more ASID than CPUs. ASID #0 is always reserved.
178	 */
179	WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus());
180	atomic64_set(&info->generation, ASID_FIRST_VERSION(info));
181	info->map = bitmap_zalloc(NUM_CTXT_ASIDS(info), GFP_KERNEL);
182	if (!info->map)
183		return -ENOMEM;
184
185	raw_spin_lock_init(&info->lock);
186
187	return 0;
188}