Loading...
1/*
2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * vineetg: May 2011
9 * -Refactored get_new_mmu_context( ) to only handle live-mm.
10 * retiring-mm handled in other hooks
11 *
12 * Vineetg: March 25th, 2008: Bug #92690
13 * -Major rewrite of Core ASID allocation routine get_new_mmu_context
14 *
15 * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
16 */
17
18#ifndef _ASM_ARC_MMU_CONTEXT_H
19#define _ASM_ARC_MMU_CONTEXT_H
20
21#include <asm/arcregs.h>
22#include <asm/tlb.h>
23
24#include <asm-generic/mm_hooks.h>
25
26/* ARC700 ASID Management
27 *
28 * ARC MMU provides 8-bit ASID (0..255) to TAG TLB entries, allowing entries
29 * with same vaddr (different tasks) to co-exit. This provides for
30 * "Fast Context Switch" i.e. no TLB flush on ctxt-switch
31 *
32 * Linux assigns each task a unique ASID. A simple round-robin allocation
33 * of H/w ASID is done using software tracker @asid_cpu.
34 * When it reaches max 255, the allocation cycle starts afresh by flushing
35 * the entire TLB and wrapping ASID back to zero.
36 *
37 * A new allocation cycle, post rollover, could potentially reassign an ASID
38 * to a different task. Thus the rule is to refresh the ASID in a new cycle.
39 * The 32 bit @asid_cpu (and mm->asid) have 8 bits MMU PID and rest 24 bits
40 * serve as cycle/generation indicator and natural 32 bit unsigned math
41 * automagically increments the generation when lower 8 bits rollover.
42 */
43
44#define MM_CTXT_ASID_MASK 0x000000ff /* MMU PID reg :8 bit PID */
45#define MM_CTXT_CYCLE_MASK (~MM_CTXT_ASID_MASK)
46
47#define MM_CTXT_FIRST_CYCLE (MM_CTXT_ASID_MASK + 1)
48#define MM_CTXT_NO_ASID 0UL
49
50#define asid_mm(mm, cpu) mm->context.asid[cpu]
51#define hw_pid(mm, cpu) (asid_mm(mm, cpu) & MM_CTXT_ASID_MASK)
52
53DECLARE_PER_CPU(unsigned int, asid_cache);
54#define asid_cpu(cpu) per_cpu(asid_cache, cpu)
55
56/*
57 * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
58 * Also set the MMU PID register to existing/updated ASID
59 */
60static inline void get_new_mmu_context(struct mm_struct *mm)
61{
62 const unsigned int cpu = smp_processor_id();
63 unsigned long flags;
64
65 local_irq_save(flags);
66
67 /*
68 * Move to new ASID if it was not from current alloc-cycle/generation.
69 * This is done by ensuring that the generation bits in both mm->ASID
70 * and cpu's ASID counter are exactly same.
71 *
72 * Note: Callers needing new ASID unconditionally, independent of
73 * generation, e.g. local_flush_tlb_mm() for forking parent,
74 * first need to destroy the context, setting it to invalid
75 * value.
76 */
77 if (!((asid_mm(mm, cpu) ^ asid_cpu(cpu)) & MM_CTXT_CYCLE_MASK))
78 goto set_hw;
79
80 /* move to new ASID and handle rollover */
81 if (unlikely(!(++asid_cpu(cpu) & MM_CTXT_ASID_MASK))) {
82
83 local_flush_tlb_all();
84
85 /*
86 * Above check for rollover of 8 bit ASID in 32 bit container.
87 * If the container itself wrapped around, set it to a non zero
88 * "generation" to distinguish from no context
89 */
90 if (!asid_cpu(cpu))
91 asid_cpu(cpu) = MM_CTXT_FIRST_CYCLE;
92 }
93
94 /* Assign new ASID to tsk */
95 asid_mm(mm, cpu) = asid_cpu(cpu);
96
97set_hw:
98 write_aux_reg(ARC_REG_PID, hw_pid(mm, cpu) | MMU_ENABLE);
99
100 local_irq_restore(flags);
101}
102
103/*
104 * Initialize the context related info for a new mm_struct
105 * instance.
106 */
107static inline int
108init_new_context(struct task_struct *tsk, struct mm_struct *mm)
109{
110 int i;
111
112 for_each_possible_cpu(i)
113 asid_mm(mm, i) = MM_CTXT_NO_ASID;
114
115 return 0;
116}
117
118static inline void destroy_context(struct mm_struct *mm)
119{
120 unsigned long flags;
121
122 /* Needed to elide CONFIG_DEBUG_PREEMPT warning */
123 local_irq_save(flags);
124 asid_mm(mm, smp_processor_id()) = MM_CTXT_NO_ASID;
125 local_irq_restore(flags);
126}
127
128/* Prepare the MMU for task: setup PID reg with allocated ASID
129 If task doesn't have an ASID (never alloc or stolen, get a new ASID)
130*/
131static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
132 struct task_struct *tsk)
133{
134 const int cpu = smp_processor_id();
135
136 /*
137 * Note that the mm_cpumask is "aggregating" only, we don't clear it
138 * for the switched-out task, unlike some other arches.
139 * It is used to enlist cpus for sending TLB flush IPIs and not sending
140 * it to CPUs where a task once ran-on, could cause stale TLB entry
141 * re-use, specially for a multi-threaded task.
142 * e.g. T1 runs on C1, migrates to C3. T2 running on C2 munmaps.
143 * For a non-aggregating mm_cpumask, IPI not sent C1, and if T1
144 * were to re-migrate to C1, it could access the unmapped region
145 * via any existing stale TLB entries.
146 */
147 cpumask_set_cpu(cpu, mm_cpumask(next));
148
149#ifndef CONFIG_SMP
150 /* PGD cached in MMU reg to avoid 3 mem lookups: task->mm->pgd */
151 write_aux_reg(ARC_REG_SCRATCH_DATA0, next->pgd);
152#endif
153
154 get_new_mmu_context(next);
155}
156
157/*
158 * Called at the time of execve() to get a new ASID
159 * Note the subtlety here: get_new_mmu_context() behaves differently here
160 * vs. in switch_mm(). Here it always returns a new ASID, because mm has
161 * an unallocated "initial" value, while in latter, it moves to a new ASID,
162 * only if it was unallocated
163 */
164#define activate_mm(prev, next) switch_mm(prev, next, NULL)
165
166/* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
167 * for retiring-mm. However destroy_context( ) still needs to do that because
168 * between mm_release( ) = >deactive_mm( ) and
169 * mmput => .. => __mmdrop( ) => destroy_context( )
170 * there is a good chance that task gets sched-out/in, making it's ASID valid
171 * again (this teased me for a whole day).
172 */
173#define deactivate_mm(tsk, mm) do { } while (0)
174
175#define enter_lazy_tlb(mm, tsk)
176
177#endif /* __ASM_ARC_MMU_CONTEXT_H */
1/*
2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * vineetg: May 2011
9 * -Refactored get_new_mmu_context( ) to only handle live-mm.
10 * retiring-mm handled in other hooks
11 *
12 * Vineetg: March 25th, 2008: Bug #92690
13 * -Major rewrite of Core ASID allocation routine get_new_mmu_context
14 *
15 * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
16 */
17
18#ifndef _ASM_ARC_MMU_CONTEXT_H
19#define _ASM_ARC_MMU_CONTEXT_H
20
21#include <asm/arcregs.h>
22#include <asm/tlb.h>
23#include <linux/sched/mm.h>
24
25#include <asm-generic/mm_hooks.h>
26
27/* ARC700 ASID Management
28 *
29 * ARC MMU provides 8-bit ASID (0..255) to TAG TLB entries, allowing entries
30 * with same vaddr (different tasks) to co-exit. This provides for
31 * "Fast Context Switch" i.e. no TLB flush on ctxt-switch
32 *
33 * Linux assigns each task a unique ASID. A simple round-robin allocation
34 * of H/w ASID is done using software tracker @asid_cpu.
35 * When it reaches max 255, the allocation cycle starts afresh by flushing
36 * the entire TLB and wrapping ASID back to zero.
37 *
38 * A new allocation cycle, post rollover, could potentially reassign an ASID
39 * to a different task. Thus the rule is to refresh the ASID in a new cycle.
40 * The 32 bit @asid_cpu (and mm->asid) have 8 bits MMU PID and rest 24 bits
41 * serve as cycle/generation indicator and natural 32 bit unsigned math
42 * automagically increments the generation when lower 8 bits rollover.
43 */
44
45#define MM_CTXT_ASID_MASK 0x000000ff /* MMU PID reg :8 bit PID */
46#define MM_CTXT_CYCLE_MASK (~MM_CTXT_ASID_MASK)
47
48#define MM_CTXT_FIRST_CYCLE (MM_CTXT_ASID_MASK + 1)
49#define MM_CTXT_NO_ASID 0UL
50
51#define asid_mm(mm, cpu) mm->context.asid[cpu]
52#define hw_pid(mm, cpu) (asid_mm(mm, cpu) & MM_CTXT_ASID_MASK)
53
54DECLARE_PER_CPU(unsigned int, asid_cache);
55#define asid_cpu(cpu) per_cpu(asid_cache, cpu)
56
57/*
58 * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
59 * Also set the MMU PID register to existing/updated ASID
60 */
61static inline void get_new_mmu_context(struct mm_struct *mm)
62{
63 const unsigned int cpu = smp_processor_id();
64 unsigned long flags;
65
66 local_irq_save(flags);
67
68 /*
69 * Move to new ASID if it was not from current alloc-cycle/generation.
70 * This is done by ensuring that the generation bits in both mm->ASID
71 * and cpu's ASID counter are exactly same.
72 *
73 * Note: Callers needing new ASID unconditionally, independent of
74 * generation, e.g. local_flush_tlb_mm() for forking parent,
75 * first need to destroy the context, setting it to invalid
76 * value.
77 */
78 if (!((asid_mm(mm, cpu) ^ asid_cpu(cpu)) & MM_CTXT_CYCLE_MASK))
79 goto set_hw;
80
81 /* move to new ASID and handle rollover */
82 if (unlikely(!(++asid_cpu(cpu) & MM_CTXT_ASID_MASK))) {
83
84 local_flush_tlb_all();
85
86 /*
87 * Above check for rollover of 8 bit ASID in 32 bit container.
88 * If the container itself wrapped around, set it to a non zero
89 * "generation" to distinguish from no context
90 */
91 if (!asid_cpu(cpu))
92 asid_cpu(cpu) = MM_CTXT_FIRST_CYCLE;
93 }
94
95 /* Assign new ASID to tsk */
96 asid_mm(mm, cpu) = asid_cpu(cpu);
97
98set_hw:
99 write_aux_reg(ARC_REG_PID, hw_pid(mm, cpu) | MMU_ENABLE);
100
101 local_irq_restore(flags);
102}
103
104/*
105 * Initialize the context related info for a new mm_struct
106 * instance.
107 */
108static inline int
109init_new_context(struct task_struct *tsk, struct mm_struct *mm)
110{
111 int i;
112
113 for_each_possible_cpu(i)
114 asid_mm(mm, i) = MM_CTXT_NO_ASID;
115
116 return 0;
117}
118
119static inline void destroy_context(struct mm_struct *mm)
120{
121 unsigned long flags;
122
123 /* Needed to elide CONFIG_DEBUG_PREEMPT warning */
124 local_irq_save(flags);
125 asid_mm(mm, smp_processor_id()) = MM_CTXT_NO_ASID;
126 local_irq_restore(flags);
127}
128
129/* Prepare the MMU for task: setup PID reg with allocated ASID
130 If task doesn't have an ASID (never alloc or stolen, get a new ASID)
131*/
132static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
133 struct task_struct *tsk)
134{
135 const int cpu = smp_processor_id();
136
137 /*
138 * Note that the mm_cpumask is "aggregating" only, we don't clear it
139 * for the switched-out task, unlike some other arches.
140 * It is used to enlist cpus for sending TLB flush IPIs and not sending
141 * it to CPUs where a task once ran-on, could cause stale TLB entry
142 * re-use, specially for a multi-threaded task.
143 * e.g. T1 runs on C1, migrates to C3. T2 running on C2 munmaps.
144 * For a non-aggregating mm_cpumask, IPI not sent C1, and if T1
145 * were to re-migrate to C1, it could access the unmapped region
146 * via any existing stale TLB entries.
147 */
148 cpumask_set_cpu(cpu, mm_cpumask(next));
149
150#ifndef CONFIG_SMP
151 /* PGD cached in MMU reg to avoid 3 mem lookups: task->mm->pgd */
152 write_aux_reg(ARC_REG_SCRATCH_DATA0, next->pgd);
153#endif
154
155 get_new_mmu_context(next);
156}
157
158/*
159 * Called at the time of execve() to get a new ASID
160 * Note the subtlety here: get_new_mmu_context() behaves differently here
161 * vs. in switch_mm(). Here it always returns a new ASID, because mm has
162 * an unallocated "initial" value, while in latter, it moves to a new ASID,
163 * only if it was unallocated
164 */
165#define activate_mm(prev, next) switch_mm(prev, next, NULL)
166
167/* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
168 * for retiring-mm. However destroy_context( ) still needs to do that because
169 * between mm_release( ) = >deactive_mm( ) and
170 * mmput => .. => __mmdrop( ) => destroy_context( )
171 * there is a good chance that task gets sched-out/in, making it's ASID valid
172 * again (this teased me for a whole day).
173 */
174#define deactivate_mm(tsk, mm) do { } while (0)
175
176#define enter_lazy_tlb(mm, tsk)
177
178#endif /* __ASM_ARC_MMU_CONTEXT_H */