Loading...
1/*
2 * Copyright (C) 2014 Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
3 * Copyright (C) 2017 Stafford Horne <shorne@gmail.com>
4 *
5 * Based on arm64 and arc implementations
6 * Copyright (C) 2013 ARM Ltd.
7 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
8 *
9 * This file is licensed under the terms of the GNU General Public License
10 * version 2. This program is licensed "as is" without any warranty of any
11 * kind, whether express or implied.
12 */
13
14#include <linux/smp.h>
15#include <linux/cpu.h>
16#include <linux/sched.h>
17#include <linux/sched/mm.h>
18#include <linux/irq.h>
19#include <linux/of.h>
20#include <asm/cpuinfo.h>
21#include <asm/mmu_context.h>
22#include <asm/tlbflush.h>
23#include <asm/cacheflush.h>
24#include <asm/time.h>
25
26asmlinkage __init void secondary_start_kernel(void);
27
28static void (*smp_cross_call)(const struct cpumask *, unsigned int);
29
30unsigned long secondary_release = -1;
31struct thread_info *secondary_thread_info;
32
33enum ipi_msg_type {
34 IPI_WAKEUP,
35 IPI_RESCHEDULE,
36 IPI_CALL_FUNC,
37 IPI_CALL_FUNC_SINGLE,
38};
39
40static DEFINE_SPINLOCK(boot_lock);
41
42static void boot_secondary(unsigned int cpu, struct task_struct *idle)
43{
44 /*
45 * set synchronisation state between this boot processor
46 * and the secondary one
47 */
48 spin_lock(&boot_lock);
49
50 secondary_release = cpu;
51 smp_cross_call(cpumask_of(cpu), IPI_WAKEUP);
52
53 /*
54 * now the secondary core is starting up let it run its
55 * calibrations, then wait for it to finish
56 */
57 spin_unlock(&boot_lock);
58}
59
60void __init smp_init_cpus(void)
61{
62 struct device_node *cpu;
63 u32 cpu_id;
64
65 for_each_of_cpu_node(cpu) {
66 cpu_id = of_get_cpu_hwid(cpu, 0);
67 if (cpu_id < NR_CPUS)
68 set_cpu_possible(cpu_id, true);
69 }
70}
71
72void __init smp_prepare_cpus(unsigned int max_cpus)
73{
74 unsigned int cpu;
75
76 /*
77 * Initialise the present map, which describes the set of CPUs
78 * actually populated at the present time.
79 */
80 for_each_possible_cpu(cpu) {
81 if (cpu < max_cpus)
82 set_cpu_present(cpu, true);
83 }
84}
85
86void __init smp_cpus_done(unsigned int max_cpus)
87{
88}
89
90static DECLARE_COMPLETION(cpu_running);
91
92int __cpu_up(unsigned int cpu, struct task_struct *idle)
93{
94 if (smp_cross_call == NULL) {
95 pr_warn("CPU%u: failed to start, IPI controller missing",
96 cpu);
97 return -EIO;
98 }
99
100 secondary_thread_info = task_thread_info(idle);
101 current_pgd[cpu] = init_mm.pgd;
102
103 boot_secondary(cpu, idle);
104 if (!wait_for_completion_timeout(&cpu_running,
105 msecs_to_jiffies(1000))) {
106 pr_crit("CPU%u: failed to start\n", cpu);
107 return -EIO;
108 }
109 synchronise_count_master(cpu);
110
111 return 0;
112}
113
114asmlinkage __init void secondary_start_kernel(void)
115{
116 struct mm_struct *mm = &init_mm;
117 unsigned int cpu = smp_processor_id();
118 /*
119 * All kernel threads share the same mm context; grab a
120 * reference and switch to it.
121 */
122 mmgrab(mm);
123 current->active_mm = mm;
124 cpumask_set_cpu(cpu, mm_cpumask(mm));
125
126 pr_info("CPU%u: Booted secondary processor\n", cpu);
127
128 setup_cpuinfo();
129 openrisc_clockevent_init();
130
131 notify_cpu_starting(cpu);
132
133 /*
134 * OK, now it's safe to let the boot CPU continue
135 */
136 complete(&cpu_running);
137
138 synchronise_count_slave(cpu);
139 set_cpu_online(cpu, true);
140
141 local_irq_enable();
142 /*
143 * OK, it's off to the idle thread for us
144 */
145 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
146}
147
148void handle_IPI(unsigned int ipi_msg)
149{
150 unsigned int cpu = smp_processor_id();
151
152 switch (ipi_msg) {
153 case IPI_WAKEUP:
154 break;
155
156 case IPI_RESCHEDULE:
157 scheduler_ipi();
158 break;
159
160 case IPI_CALL_FUNC:
161 generic_smp_call_function_interrupt();
162 break;
163
164 case IPI_CALL_FUNC_SINGLE:
165 generic_smp_call_function_single_interrupt();
166 break;
167
168 default:
169 WARN(1, "CPU%u: Unknown IPI message 0x%x\n", cpu, ipi_msg);
170 break;
171 }
172}
173
174void arch_smp_send_reschedule(int cpu)
175{
176 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
177}
178
179static void stop_this_cpu(void *dummy)
180{
181 /* Remove this CPU */
182 set_cpu_online(smp_processor_id(), false);
183
184 local_irq_disable();
185 /* CPU Doze */
186 if (mfspr(SPR_UPR) & SPR_UPR_PMP)
187 mtspr(SPR_PMR, mfspr(SPR_PMR) | SPR_PMR_DME);
188 /* If that didn't work, infinite loop */
189 while (1)
190 ;
191}
192
193void smp_send_stop(void)
194{
195 smp_call_function(stop_this_cpu, NULL, 0);
196}
197
198void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
199{
200 smp_cross_call = fn;
201}
202
203void arch_send_call_function_single_ipi(int cpu)
204{
205 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
206}
207
208void arch_send_call_function_ipi_mask(const struct cpumask *mask)
209{
210 smp_cross_call(mask, IPI_CALL_FUNC);
211}
212
213/* TLB flush operations - Performed on each CPU*/
214static inline void ipi_flush_tlb_all(void *ignored)
215{
216 local_flush_tlb_all();
217}
218
219static inline void ipi_flush_tlb_mm(void *info)
220{
221 struct mm_struct *mm = (struct mm_struct *)info;
222
223 local_flush_tlb_mm(mm);
224}
225
226static void smp_flush_tlb_mm(struct cpumask *cmask, struct mm_struct *mm)
227{
228 unsigned int cpuid;
229
230 if (cpumask_empty(cmask))
231 return;
232
233 cpuid = get_cpu();
234
235 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
236 /* local cpu is the only cpu present in cpumask */
237 local_flush_tlb_mm(mm);
238 } else {
239 on_each_cpu_mask(cmask, ipi_flush_tlb_mm, mm, 1);
240 }
241 put_cpu();
242}
243
244struct flush_tlb_data {
245 unsigned long addr1;
246 unsigned long addr2;
247};
248
249static inline void ipi_flush_tlb_page(void *info)
250{
251 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
252
253 local_flush_tlb_page(NULL, fd->addr1);
254}
255
256static inline void ipi_flush_tlb_range(void *info)
257{
258 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
259
260 local_flush_tlb_range(NULL, fd->addr1, fd->addr2);
261}
262
263static void smp_flush_tlb_range(const struct cpumask *cmask, unsigned long start,
264 unsigned long end)
265{
266 unsigned int cpuid;
267
268 if (cpumask_empty(cmask))
269 return;
270
271 cpuid = get_cpu();
272
273 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
274 /* local cpu is the only cpu present in cpumask */
275 if ((end - start) <= PAGE_SIZE)
276 local_flush_tlb_page(NULL, start);
277 else
278 local_flush_tlb_range(NULL, start, end);
279 } else {
280 struct flush_tlb_data fd;
281
282 fd.addr1 = start;
283 fd.addr2 = end;
284
285 if ((end - start) <= PAGE_SIZE)
286 on_each_cpu_mask(cmask, ipi_flush_tlb_page, &fd, 1);
287 else
288 on_each_cpu_mask(cmask, ipi_flush_tlb_range, &fd, 1);
289 }
290 put_cpu();
291}
292
293void flush_tlb_all(void)
294{
295 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
296}
297
298void flush_tlb_mm(struct mm_struct *mm)
299{
300 smp_flush_tlb_mm(mm_cpumask(mm), mm);
301}
302
303void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
304{
305 smp_flush_tlb_range(mm_cpumask(vma->vm_mm), uaddr, uaddr + PAGE_SIZE);
306}
307
308void flush_tlb_range(struct vm_area_struct *vma,
309 unsigned long start, unsigned long end)
310{
311 const struct cpumask *cmask = vma ? mm_cpumask(vma->vm_mm)
312 : cpu_online_mask;
313 smp_flush_tlb_range(cmask, start, end);
314}
315
316/* Instruction cache invalidate - performed on each cpu */
317static void ipi_icache_page_inv(void *arg)
318{
319 struct page *page = arg;
320
321 local_icache_page_inv(page);
322}
323
324void smp_icache_page_inv(struct page *page)
325{
326 on_each_cpu(ipi_icache_page_inv, page, 1);
327}
328EXPORT_SYMBOL(smp_icache_page_inv);
1/*
2 * Copyright (C) 2014 Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
3 * Copyright (C) 2017 Stafford Horne <shorne@gmail.com>
4 *
5 * Based on arm64 and arc implementations
6 * Copyright (C) 2013 ARM Ltd.
7 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
8 *
9 * This file is licensed under the terms of the GNU General Public License
10 * version 2. This program is licensed "as is" without any warranty of any
11 * kind, whether express or implied.
12 */
13
14#include <linux/smp.h>
15#include <linux/cpu.h>
16#include <linux/sched.h>
17#include <linux/sched/mm.h>
18#include <linux/irq.h>
19#include <linux/of.h>
20#include <asm/cpuinfo.h>
21#include <asm/mmu_context.h>
22#include <asm/tlbflush.h>
23#include <asm/cacheflush.h>
24#include <asm/time.h>
25
26static void (*smp_cross_call)(const struct cpumask *, unsigned int);
27
28unsigned long secondary_release = -1;
29struct thread_info *secondary_thread_info;
30
31enum ipi_msg_type {
32 IPI_WAKEUP,
33 IPI_RESCHEDULE,
34 IPI_CALL_FUNC,
35 IPI_CALL_FUNC_SINGLE,
36};
37
38static DEFINE_SPINLOCK(boot_lock);
39
40static void boot_secondary(unsigned int cpu, struct task_struct *idle)
41{
42 /*
43 * set synchronisation state between this boot processor
44 * and the secondary one
45 */
46 spin_lock(&boot_lock);
47
48 secondary_release = cpu;
49 smp_cross_call(cpumask_of(cpu), IPI_WAKEUP);
50
51 /*
52 * now the secondary core is starting up let it run its
53 * calibrations, then wait for it to finish
54 */
55 spin_unlock(&boot_lock);
56}
57
58void __init smp_prepare_boot_cpu(void)
59{
60}
61
62void __init smp_init_cpus(void)
63{
64 struct device_node *cpu;
65 u32 cpu_id;
66
67 for_each_of_cpu_node(cpu) {
68 if (of_property_read_u32(cpu, "reg", &cpu_id)) {
69 pr_warn("%s missing reg property", cpu->full_name);
70 continue;
71 }
72
73 if (cpu_id < NR_CPUS)
74 set_cpu_possible(cpu_id, true);
75 }
76}
77
78void __init smp_prepare_cpus(unsigned int max_cpus)
79{
80 unsigned int cpu;
81
82 /*
83 * Initialise the present map, which describes the set of CPUs
84 * actually populated at the present time.
85 */
86 for_each_possible_cpu(cpu) {
87 if (cpu < max_cpus)
88 set_cpu_present(cpu, true);
89 }
90}
91
92void __init smp_cpus_done(unsigned int max_cpus)
93{
94}
95
96static DECLARE_COMPLETION(cpu_running);
97
98int __cpu_up(unsigned int cpu, struct task_struct *idle)
99{
100 if (smp_cross_call == NULL) {
101 pr_warn("CPU%u: failed to start, IPI controller missing",
102 cpu);
103 return -EIO;
104 }
105
106 secondary_thread_info = task_thread_info(idle);
107 current_pgd[cpu] = init_mm.pgd;
108
109 boot_secondary(cpu, idle);
110 if (!wait_for_completion_timeout(&cpu_running,
111 msecs_to_jiffies(1000))) {
112 pr_crit("CPU%u: failed to start\n", cpu);
113 return -EIO;
114 }
115 synchronise_count_master(cpu);
116
117 return 0;
118}
119
120asmlinkage __init void secondary_start_kernel(void)
121{
122 struct mm_struct *mm = &init_mm;
123 unsigned int cpu = smp_processor_id();
124 /*
125 * All kernel threads share the same mm context; grab a
126 * reference and switch to it.
127 */
128 mmgrab(mm);
129 current->active_mm = mm;
130 cpumask_set_cpu(cpu, mm_cpumask(mm));
131
132 pr_info("CPU%u: Booted secondary processor\n", cpu);
133
134 setup_cpuinfo();
135 openrisc_clockevent_init();
136
137 notify_cpu_starting(cpu);
138
139 /*
140 * OK, now it's safe to let the boot CPU continue
141 */
142 complete(&cpu_running);
143
144 synchronise_count_slave(cpu);
145 set_cpu_online(cpu, true);
146
147 local_irq_enable();
148 /*
149 * OK, it's off to the idle thread for us
150 */
151 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
152}
153
154void handle_IPI(unsigned int ipi_msg)
155{
156 unsigned int cpu = smp_processor_id();
157
158 switch (ipi_msg) {
159 case IPI_WAKEUP:
160 break;
161
162 case IPI_RESCHEDULE:
163 scheduler_ipi();
164 break;
165
166 case IPI_CALL_FUNC:
167 generic_smp_call_function_interrupt();
168 break;
169
170 case IPI_CALL_FUNC_SINGLE:
171 generic_smp_call_function_single_interrupt();
172 break;
173
174 default:
175 WARN(1, "CPU%u: Unknown IPI message 0x%x\n", cpu, ipi_msg);
176 break;
177 }
178}
179
180void smp_send_reschedule(int cpu)
181{
182 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
183}
184
185static void stop_this_cpu(void *dummy)
186{
187 /* Remove this CPU */
188 set_cpu_online(smp_processor_id(), false);
189
190 local_irq_disable();
191 /* CPU Doze */
192 if (mfspr(SPR_UPR) & SPR_UPR_PMP)
193 mtspr(SPR_PMR, mfspr(SPR_PMR) | SPR_PMR_DME);
194 /* If that didn't work, infinite loop */
195 while (1)
196 ;
197}
198
199void smp_send_stop(void)
200{
201 smp_call_function(stop_this_cpu, NULL, 0);
202}
203
204/* not supported, yet */
205int setup_profiling_timer(unsigned int multiplier)
206{
207 return -EINVAL;
208}
209
210void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
211{
212 smp_cross_call = fn;
213}
214
215void arch_send_call_function_single_ipi(int cpu)
216{
217 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
218}
219
220void arch_send_call_function_ipi_mask(const struct cpumask *mask)
221{
222 smp_cross_call(mask, IPI_CALL_FUNC);
223}
224
225/* TLB flush operations - Performed on each CPU*/
226static inline void ipi_flush_tlb_all(void *ignored)
227{
228 local_flush_tlb_all();
229}
230
231static inline void ipi_flush_tlb_mm(void *info)
232{
233 struct mm_struct *mm = (struct mm_struct *)info;
234
235 local_flush_tlb_mm(mm);
236}
237
238static void smp_flush_tlb_mm(struct cpumask *cmask, struct mm_struct *mm)
239{
240 unsigned int cpuid;
241
242 if (cpumask_empty(cmask))
243 return;
244
245 cpuid = get_cpu();
246
247 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
248 /* local cpu is the only cpu present in cpumask */
249 local_flush_tlb_mm(mm);
250 } else {
251 on_each_cpu_mask(cmask, ipi_flush_tlb_mm, mm, 1);
252 }
253 put_cpu();
254}
255
256struct flush_tlb_data {
257 unsigned long addr1;
258 unsigned long addr2;
259};
260
261static inline void ipi_flush_tlb_page(void *info)
262{
263 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
264
265 local_flush_tlb_page(NULL, fd->addr1);
266}
267
268static inline void ipi_flush_tlb_range(void *info)
269{
270 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
271
272 local_flush_tlb_range(NULL, fd->addr1, fd->addr2);
273}
274
275static void smp_flush_tlb_range(struct cpumask *cmask, unsigned long start,
276 unsigned long end)
277{
278 unsigned int cpuid;
279
280 if (cpumask_empty(cmask))
281 return;
282
283 cpuid = get_cpu();
284
285 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
286 /* local cpu is the only cpu present in cpumask */
287 if ((end - start) <= PAGE_SIZE)
288 local_flush_tlb_page(NULL, start);
289 else
290 local_flush_tlb_range(NULL, start, end);
291 } else {
292 struct flush_tlb_data fd;
293
294 fd.addr1 = start;
295 fd.addr2 = end;
296
297 if ((end - start) <= PAGE_SIZE)
298 on_each_cpu_mask(cmask, ipi_flush_tlb_page, &fd, 1);
299 else
300 on_each_cpu_mask(cmask, ipi_flush_tlb_range, &fd, 1);
301 }
302 put_cpu();
303}
304
305void flush_tlb_all(void)
306{
307 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
308}
309
310void flush_tlb_mm(struct mm_struct *mm)
311{
312 smp_flush_tlb_mm(mm_cpumask(mm), mm);
313}
314
315void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
316{
317 smp_flush_tlb_range(mm_cpumask(vma->vm_mm), uaddr, uaddr + PAGE_SIZE);
318}
319
320void flush_tlb_range(struct vm_area_struct *vma,
321 unsigned long start, unsigned long end)
322{
323 smp_flush_tlb_range(mm_cpumask(vma->vm_mm), start, end);
324}
325
326/* Instruction cache invalidate - performed on each cpu */
327static void ipi_icache_page_inv(void *arg)
328{
329 struct page *page = arg;
330
331 local_icache_page_inv(page);
332}
333
334void smp_icache_page_inv(struct page *page)
335{
336 on_each_cpu(ipi_icache_page_inv, page, 1);
337}
338EXPORT_SYMBOL(smp_icache_page_inv);