Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
4 *
5 * Created by: Nicolas Pitre, March 2012
6 * Copyright: (C) 2012-2013 Linaro Limited
7 */
8
9#include <linux/export.h>
10#include <linux/kernel.h>
11#include <linux/init.h>
12#include <linux/irqflags.h>
13#include <linux/cpu_pm.h>
14
15#include <asm/mcpm.h>
16#include <asm/cacheflush.h>
17#include <asm/idmap.h>
18#include <asm/cputype.h>
19#include <asm/suspend.h>
20
21/*
22 * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
23 * For a comprehensive description of the main algorithm used here, please
24 * see Documentation/arch/arm/cluster-pm-race-avoidance.rst.
25 */
26
27struct sync_struct mcpm_sync;
28
29/*
30 * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
31 * This must be called at the point of committing to teardown of a CPU.
32 * The CPU cache (SCTRL.C bit) is expected to still be active.
33 */
34static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
35{
36 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
37 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
38}
39
40/*
41 * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
42 * cluster can be torn down without disrupting this CPU.
43 * To avoid deadlocks, this must be called before a CPU is powered down.
44 * The CPU cache (SCTRL.C bit) is expected to be off.
45 * However L2 cache might or might not be active.
46 */
47static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
48{
49 dmb();
50 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
51 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
52 sev();
53}
54
55/*
56 * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
57 * @state: the final state of the cluster:
58 * CLUSTER_UP: no destructive teardown was done and the cluster has been
59 * restored to the previous state (CPU cache still active); or
60 * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
61 * (CPU cache disabled, L2 cache either enabled or disabled).
62 */
63static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
64{
65 dmb();
66 mcpm_sync.clusters[cluster].cluster = state;
67 sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
68 sev();
69}
70
71/*
72 * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
73 * This function should be called by the last man, after local CPU teardown
74 * is complete. CPU cache expected to be active.
75 *
76 * Returns:
77 * false: the critical section was not entered because an inbound CPU was
78 * observed, or the cluster is already being set up;
79 * true: the critical section was entered: it is now safe to tear down the
80 * cluster.
81 */
82static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
83{
84 unsigned int i;
85 struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
86
87 /* Warn inbound CPUs that the cluster is being torn down: */
88 c->cluster = CLUSTER_GOING_DOWN;
89 sync_cache_w(&c->cluster);
90
91 /* Back out if the inbound cluster is already in the critical region: */
92 sync_cache_r(&c->inbound);
93 if (c->inbound == INBOUND_COMING_UP)
94 goto abort;
95
96 /*
97 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
98 * teardown is complete on each CPU before tearing down the cluster.
99 *
100 * If any CPU has been woken up again from the DOWN state, then we
101 * shouldn't be taking the cluster down at all: abort in that case.
102 */
103 sync_cache_r(&c->cpus);
104 for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
105 int cpustate;
106
107 if (i == cpu)
108 continue;
109
110 while (1) {
111 cpustate = c->cpus[i].cpu;
112 if (cpustate != CPU_GOING_DOWN)
113 break;
114
115 wfe();
116 sync_cache_r(&c->cpus[i].cpu);
117 }
118
119 switch (cpustate) {
120 case CPU_DOWN:
121 continue;
122
123 default:
124 goto abort;
125 }
126 }
127
128 return true;
129
130abort:
131 __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
132 return false;
133}
134
135static int __mcpm_cluster_state(unsigned int cluster)
136{
137 sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
138 return mcpm_sync.clusters[cluster].cluster;
139}
140
141extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
142
143void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
144{
145 unsigned long val = ptr ? __pa_symbol(ptr) : 0;
146 mcpm_entry_vectors[cluster][cpu] = val;
147 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
148}
149
150extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
151
152void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
153 unsigned long poke_phys_addr, unsigned long poke_val)
154{
155 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
156 poke[0] = poke_phys_addr;
157 poke[1] = poke_val;
158 __sync_cache_range_w(poke, 2 * sizeof(*poke));
159}
160
161static const struct mcpm_platform_ops *platform_ops;
162
163int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
164{
165 if (platform_ops)
166 return -EBUSY;
167 platform_ops = ops;
168 return 0;
169}
170
171bool mcpm_is_available(void)
172{
173 return (platform_ops) ? true : false;
174}
175EXPORT_SYMBOL_GPL(mcpm_is_available);
176
177/*
178 * We can't use regular spinlocks. In the switcher case, it is possible
179 * for an outbound CPU to call power_down() after its inbound counterpart
180 * is already live using the same logical CPU number which trips lockdep
181 * debugging.
182 */
183static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
184
185static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
186
187static inline bool mcpm_cluster_unused(unsigned int cluster)
188{
189 int i, cnt;
190 for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
191 cnt |= mcpm_cpu_use_count[cluster][i];
192 return !cnt;
193}
194
195int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
196{
197 bool cpu_is_down, cluster_is_down;
198 int ret = 0;
199
200 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
201 if (!platform_ops)
202 return -EUNATCH; /* try not to shadow power_up errors */
203 might_sleep();
204
205 /*
206 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
207 * variant exists, we need to disable IRQs manually here.
208 */
209 local_irq_disable();
210 arch_spin_lock(&mcpm_lock);
211
212 cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
213 cluster_is_down = mcpm_cluster_unused(cluster);
214
215 mcpm_cpu_use_count[cluster][cpu]++;
216 /*
217 * The only possible values are:
218 * 0 = CPU down
219 * 1 = CPU (still) up
220 * 2 = CPU requested to be up before it had a chance
221 * to actually make itself down.
222 * Any other value is a bug.
223 */
224 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
225 mcpm_cpu_use_count[cluster][cpu] != 2);
226
227 if (cluster_is_down)
228 ret = platform_ops->cluster_powerup(cluster);
229 if (cpu_is_down && !ret)
230 ret = platform_ops->cpu_powerup(cpu, cluster);
231
232 arch_spin_unlock(&mcpm_lock);
233 local_irq_enable();
234 return ret;
235}
236
237typedef typeof(cpu_reset) phys_reset_t;
238
239void mcpm_cpu_power_down(void)
240{
241 unsigned int mpidr, cpu, cluster;
242 bool cpu_going_down, last_man;
243 phys_reset_t phys_reset;
244
245 mpidr = read_cpuid_mpidr();
246 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
247 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
248 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
249 if (WARN_ON_ONCE(!platform_ops))
250 return;
251 BUG_ON(!irqs_disabled());
252
253 setup_mm_for_reboot();
254
255 __mcpm_cpu_going_down(cpu, cluster);
256 arch_spin_lock(&mcpm_lock);
257 BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
258
259 mcpm_cpu_use_count[cluster][cpu]--;
260 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
261 mcpm_cpu_use_count[cluster][cpu] != 1);
262 cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
263 last_man = mcpm_cluster_unused(cluster);
264
265 if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
266 platform_ops->cpu_powerdown_prepare(cpu, cluster);
267 platform_ops->cluster_powerdown_prepare(cluster);
268 arch_spin_unlock(&mcpm_lock);
269 platform_ops->cluster_cache_disable();
270 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
271 } else {
272 if (cpu_going_down)
273 platform_ops->cpu_powerdown_prepare(cpu, cluster);
274 arch_spin_unlock(&mcpm_lock);
275 /*
276 * If cpu_going_down is false here, that means a power_up
277 * request raced ahead of us. Even if we do not want to
278 * shut this CPU down, the caller still expects execution
279 * to return through the system resume entry path, like
280 * when the WFI is aborted due to a new IRQ or the like..
281 * So let's continue with cache cleaning in all cases.
282 */
283 platform_ops->cpu_cache_disable();
284 }
285
286 __mcpm_cpu_down(cpu, cluster);
287
288 /* Now we are prepared for power-down, do it: */
289 if (cpu_going_down)
290 wfi();
291
292 /*
293 * It is possible for a power_up request to happen concurrently
294 * with a power_down request for the same CPU. In this case the
295 * CPU might not be able to actually enter a powered down state
296 * with the WFI instruction if the power_up request has removed
297 * the required reset condition. We must perform a re-entry in
298 * the kernel as if the power_up method just had deasserted reset
299 * on the CPU.
300 */
301 phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
302 phys_reset(__pa_symbol(mcpm_entry_point), false);
303
304 /* should never get here */
305 BUG();
306}
307
308int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
309{
310 int ret;
311
312 if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
313 return -EUNATCH;
314
315 ret = platform_ops->wait_for_powerdown(cpu, cluster);
316 if (ret)
317 pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
318 __func__, cpu, cluster, ret);
319
320 return ret;
321}
322
323void mcpm_cpu_suspend(void)
324{
325 if (WARN_ON_ONCE(!platform_ops))
326 return;
327
328 /* Some platforms might have to enable special resume modes, etc. */
329 if (platform_ops->cpu_suspend_prepare) {
330 unsigned int mpidr = read_cpuid_mpidr();
331 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
332 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
333 arch_spin_lock(&mcpm_lock);
334 platform_ops->cpu_suspend_prepare(cpu, cluster);
335 arch_spin_unlock(&mcpm_lock);
336 }
337 mcpm_cpu_power_down();
338}
339
340int mcpm_cpu_powered_up(void)
341{
342 unsigned int mpidr, cpu, cluster;
343 bool cpu_was_down, first_man;
344 unsigned long flags;
345
346 if (!platform_ops)
347 return -EUNATCH;
348
349 mpidr = read_cpuid_mpidr();
350 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
351 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
352 local_irq_save(flags);
353 arch_spin_lock(&mcpm_lock);
354
355 cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
356 first_man = mcpm_cluster_unused(cluster);
357
358 if (first_man && platform_ops->cluster_is_up)
359 platform_ops->cluster_is_up(cluster);
360 if (cpu_was_down)
361 mcpm_cpu_use_count[cluster][cpu] = 1;
362 if (platform_ops->cpu_is_up)
363 platform_ops->cpu_is_up(cpu, cluster);
364
365 arch_spin_unlock(&mcpm_lock);
366 local_irq_restore(flags);
367
368 return 0;
369}
370
371#ifdef CONFIG_ARM_CPU_SUSPEND
372
373static int __init nocache_trampoline(unsigned long _arg)
374{
375 void (*cache_disable)(void) = (void *)_arg;
376 unsigned int mpidr = read_cpuid_mpidr();
377 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
378 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
379 phys_reset_t phys_reset;
380
381 mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp);
382 setup_mm_for_reboot();
383
384 __mcpm_cpu_going_down(cpu, cluster);
385 BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
386 cache_disable();
387 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
388 __mcpm_cpu_down(cpu, cluster);
389
390 phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
391 phys_reset(__pa_symbol(mcpm_entry_point), false);
392 BUG();
393}
394
395int __init mcpm_loopback(void (*cache_disable)(void))
396{
397 int ret;
398
399 /*
400 * We're going to soft-restart the current CPU through the
401 * low-level MCPM code by leveraging the suspend/resume
402 * infrastructure. Let's play it safe by using cpu_pm_enter()
403 * in case the CPU init code path resets the VFP or similar.
404 */
405 local_irq_disable();
406 local_fiq_disable();
407 ret = cpu_pm_enter();
408 if (!ret) {
409 ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
410 cpu_pm_exit();
411 }
412 local_fiq_enable();
413 local_irq_enable();
414 if (ret)
415 pr_err("%s returned %d\n", __func__, ret);
416 return ret;
417}
418
419#endif
420
421extern unsigned long mcpm_power_up_setup_phys;
422
423int __init mcpm_sync_init(
424 void (*power_up_setup)(unsigned int affinity_level))
425{
426 unsigned int i, j, mpidr, this_cluster;
427
428 BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
429 BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
430
431 /*
432 * Set initial CPU and cluster states.
433 * Only one cluster is assumed to be active at this point.
434 */
435 for (i = 0; i < MAX_NR_CLUSTERS; i++) {
436 mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
437 mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
438 for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
439 mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
440 }
441 mpidr = read_cpuid_mpidr();
442 this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
443 for_each_online_cpu(i) {
444 mcpm_cpu_use_count[this_cluster][i] = 1;
445 mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
446 }
447 mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
448 sync_cache_w(&mcpm_sync);
449
450 if (power_up_setup) {
451 mcpm_power_up_setup_phys = __pa_symbol(power_up_setup);
452 sync_cache_w(&mcpm_power_up_setup_phys);
453 }
454
455 return 0;
456}
1/*
2 * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
3 *
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/kernel.h>
13#include <linux/init.h>
14#include <linux/irqflags.h>
15#include <linux/cpu_pm.h>
16
17#include <asm/mcpm.h>
18#include <asm/cacheflush.h>
19#include <asm/idmap.h>
20#include <asm/cputype.h>
21#include <asm/suspend.h>
22
23/*
24 * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
25 * For a comprehensive description of the main algorithm used here, please
26 * see Documentation/arm/cluster-pm-race-avoidance.txt.
27 */
28
29struct sync_struct mcpm_sync;
30
31/*
32 * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
33 * This must be called at the point of committing to teardown of a CPU.
34 * The CPU cache (SCTRL.C bit) is expected to still be active.
35 */
36static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
37{
38 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
39 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
40}
41
42/*
43 * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
44 * cluster can be torn down without disrupting this CPU.
45 * To avoid deadlocks, this must be called before a CPU is powered down.
46 * The CPU cache (SCTRL.C bit) is expected to be off.
47 * However L2 cache might or might not be active.
48 */
49static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
50{
51 dmb();
52 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
53 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
54 sev();
55}
56
57/*
58 * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
59 * @state: the final state of the cluster:
60 * CLUSTER_UP: no destructive teardown was done and the cluster has been
61 * restored to the previous state (CPU cache still active); or
62 * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
63 * (CPU cache disabled, L2 cache either enabled or disabled).
64 */
65static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
66{
67 dmb();
68 mcpm_sync.clusters[cluster].cluster = state;
69 sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
70 sev();
71}
72
73/*
74 * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
75 * This function should be called by the last man, after local CPU teardown
76 * is complete. CPU cache expected to be active.
77 *
78 * Returns:
79 * false: the critical section was not entered because an inbound CPU was
80 * observed, or the cluster is already being set up;
81 * true: the critical section was entered: it is now safe to tear down the
82 * cluster.
83 */
84static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
85{
86 unsigned int i;
87 struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
88
89 /* Warn inbound CPUs that the cluster is being torn down: */
90 c->cluster = CLUSTER_GOING_DOWN;
91 sync_cache_w(&c->cluster);
92
93 /* Back out if the inbound cluster is already in the critical region: */
94 sync_cache_r(&c->inbound);
95 if (c->inbound == INBOUND_COMING_UP)
96 goto abort;
97
98 /*
99 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
100 * teardown is complete on each CPU before tearing down the cluster.
101 *
102 * If any CPU has been woken up again from the DOWN state, then we
103 * shouldn't be taking the cluster down at all: abort in that case.
104 */
105 sync_cache_r(&c->cpus);
106 for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
107 int cpustate;
108
109 if (i == cpu)
110 continue;
111
112 while (1) {
113 cpustate = c->cpus[i].cpu;
114 if (cpustate != CPU_GOING_DOWN)
115 break;
116
117 wfe();
118 sync_cache_r(&c->cpus[i].cpu);
119 }
120
121 switch (cpustate) {
122 case CPU_DOWN:
123 continue;
124
125 default:
126 goto abort;
127 }
128 }
129
130 return true;
131
132abort:
133 __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
134 return false;
135}
136
137static int __mcpm_cluster_state(unsigned int cluster)
138{
139 sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
140 return mcpm_sync.clusters[cluster].cluster;
141}
142
143extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
144
145void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
146{
147 unsigned long val = ptr ? virt_to_phys(ptr) : 0;
148 mcpm_entry_vectors[cluster][cpu] = val;
149 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
150}
151
152extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
153
154void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
155 unsigned long poke_phys_addr, unsigned long poke_val)
156{
157 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
158 poke[0] = poke_phys_addr;
159 poke[1] = poke_val;
160 __sync_cache_range_w(poke, 2 * sizeof(*poke));
161}
162
163static const struct mcpm_platform_ops *platform_ops;
164
165int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
166{
167 if (platform_ops)
168 return -EBUSY;
169 platform_ops = ops;
170 return 0;
171}
172
173bool mcpm_is_available(void)
174{
175 return (platform_ops) ? true : false;
176}
177
178/*
179 * We can't use regular spinlocks. In the switcher case, it is possible
180 * for an outbound CPU to call power_down() after its inbound counterpart
181 * is already live using the same logical CPU number which trips lockdep
182 * debugging.
183 */
184static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
185
186static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
187
188static inline bool mcpm_cluster_unused(unsigned int cluster)
189{
190 int i, cnt;
191 for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
192 cnt |= mcpm_cpu_use_count[cluster][i];
193 return !cnt;
194}
195
196int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
197{
198 bool cpu_is_down, cluster_is_down;
199 int ret = 0;
200
201 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
202 if (!platform_ops)
203 return -EUNATCH; /* try not to shadow power_up errors */
204 might_sleep();
205
206 /*
207 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
208 * variant exists, we need to disable IRQs manually here.
209 */
210 local_irq_disable();
211 arch_spin_lock(&mcpm_lock);
212
213 cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
214 cluster_is_down = mcpm_cluster_unused(cluster);
215
216 mcpm_cpu_use_count[cluster][cpu]++;
217 /*
218 * The only possible values are:
219 * 0 = CPU down
220 * 1 = CPU (still) up
221 * 2 = CPU requested to be up before it had a chance
222 * to actually make itself down.
223 * Any other value is a bug.
224 */
225 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
226 mcpm_cpu_use_count[cluster][cpu] != 2);
227
228 if (cluster_is_down)
229 ret = platform_ops->cluster_powerup(cluster);
230 if (cpu_is_down && !ret)
231 ret = platform_ops->cpu_powerup(cpu, cluster);
232
233 arch_spin_unlock(&mcpm_lock);
234 local_irq_enable();
235 return ret;
236}
237
238typedef void (*phys_reset_t)(unsigned long);
239
240void mcpm_cpu_power_down(void)
241{
242 unsigned int mpidr, cpu, cluster;
243 bool cpu_going_down, last_man;
244 phys_reset_t phys_reset;
245
246 mpidr = read_cpuid_mpidr();
247 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
248 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
249 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
250 if (WARN_ON_ONCE(!platform_ops))
251 return;
252 BUG_ON(!irqs_disabled());
253
254 setup_mm_for_reboot();
255
256 __mcpm_cpu_going_down(cpu, cluster);
257 arch_spin_lock(&mcpm_lock);
258 BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
259
260 mcpm_cpu_use_count[cluster][cpu]--;
261 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
262 mcpm_cpu_use_count[cluster][cpu] != 1);
263 cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
264 last_man = mcpm_cluster_unused(cluster);
265
266 if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
267 platform_ops->cpu_powerdown_prepare(cpu, cluster);
268 platform_ops->cluster_powerdown_prepare(cluster);
269 arch_spin_unlock(&mcpm_lock);
270 platform_ops->cluster_cache_disable();
271 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
272 } else {
273 if (cpu_going_down)
274 platform_ops->cpu_powerdown_prepare(cpu, cluster);
275 arch_spin_unlock(&mcpm_lock);
276 /*
277 * If cpu_going_down is false here, that means a power_up
278 * request raced ahead of us. Even if we do not want to
279 * shut this CPU down, the caller still expects execution
280 * to return through the system resume entry path, like
281 * when the WFI is aborted due to a new IRQ or the like..
282 * So let's continue with cache cleaning in all cases.
283 */
284 platform_ops->cpu_cache_disable();
285 }
286
287 __mcpm_cpu_down(cpu, cluster);
288
289 /* Now we are prepared for power-down, do it: */
290 if (cpu_going_down)
291 wfi();
292
293 /*
294 * It is possible for a power_up request to happen concurrently
295 * with a power_down request for the same CPU. In this case the
296 * CPU might not be able to actually enter a powered down state
297 * with the WFI instruction if the power_up request has removed
298 * the required reset condition. We must perform a re-entry in
299 * the kernel as if the power_up method just had deasserted reset
300 * on the CPU.
301 */
302 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
303 phys_reset(virt_to_phys(mcpm_entry_point));
304
305 /* should never get here */
306 BUG();
307}
308
309int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
310{
311 int ret;
312
313 if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
314 return -EUNATCH;
315
316 ret = platform_ops->wait_for_powerdown(cpu, cluster);
317 if (ret)
318 pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
319 __func__, cpu, cluster, ret);
320
321 return ret;
322}
323
324void mcpm_cpu_suspend(void)
325{
326 if (WARN_ON_ONCE(!platform_ops))
327 return;
328
329 /* Some platforms might have to enable special resume modes, etc. */
330 if (platform_ops->cpu_suspend_prepare) {
331 unsigned int mpidr = read_cpuid_mpidr();
332 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
333 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
334 arch_spin_lock(&mcpm_lock);
335 platform_ops->cpu_suspend_prepare(cpu, cluster);
336 arch_spin_unlock(&mcpm_lock);
337 }
338 mcpm_cpu_power_down();
339}
340
341int mcpm_cpu_powered_up(void)
342{
343 unsigned int mpidr, cpu, cluster;
344 bool cpu_was_down, first_man;
345 unsigned long flags;
346
347 if (!platform_ops)
348 return -EUNATCH;
349
350 mpidr = read_cpuid_mpidr();
351 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
352 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
353 local_irq_save(flags);
354 arch_spin_lock(&mcpm_lock);
355
356 cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
357 first_man = mcpm_cluster_unused(cluster);
358
359 if (first_man && platform_ops->cluster_is_up)
360 platform_ops->cluster_is_up(cluster);
361 if (cpu_was_down)
362 mcpm_cpu_use_count[cluster][cpu] = 1;
363 if (platform_ops->cpu_is_up)
364 platform_ops->cpu_is_up(cpu, cluster);
365
366 arch_spin_unlock(&mcpm_lock);
367 local_irq_restore(flags);
368
369 return 0;
370}
371
372#ifdef CONFIG_ARM_CPU_SUSPEND
373
374static int __init nocache_trampoline(unsigned long _arg)
375{
376 void (*cache_disable)(void) = (void *)_arg;
377 unsigned int mpidr = read_cpuid_mpidr();
378 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
379 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
380 phys_reset_t phys_reset;
381
382 mcpm_set_entry_vector(cpu, cluster, cpu_resume);
383 setup_mm_for_reboot();
384
385 __mcpm_cpu_going_down(cpu, cluster);
386 BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
387 cache_disable();
388 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
389 __mcpm_cpu_down(cpu, cluster);
390
391 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
392 phys_reset(virt_to_phys(mcpm_entry_point));
393 BUG();
394}
395
396int __init mcpm_loopback(void (*cache_disable)(void))
397{
398 int ret;
399
400 /*
401 * We're going to soft-restart the current CPU through the
402 * low-level MCPM code by leveraging the suspend/resume
403 * infrastructure. Let's play it safe by using cpu_pm_enter()
404 * in case the CPU init code path resets the VFP or similar.
405 */
406 local_irq_disable();
407 local_fiq_disable();
408 ret = cpu_pm_enter();
409 if (!ret) {
410 ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
411 cpu_pm_exit();
412 }
413 local_fiq_enable();
414 local_irq_enable();
415 if (ret)
416 pr_err("%s returned %d\n", __func__, ret);
417 return ret;
418}
419
420#endif
421
422extern unsigned long mcpm_power_up_setup_phys;
423
424int __init mcpm_sync_init(
425 void (*power_up_setup)(unsigned int affinity_level))
426{
427 unsigned int i, j, mpidr, this_cluster;
428
429 BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
430 BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
431
432 /*
433 * Set initial CPU and cluster states.
434 * Only one cluster is assumed to be active at this point.
435 */
436 for (i = 0; i < MAX_NR_CLUSTERS; i++) {
437 mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
438 mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
439 for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
440 mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
441 }
442 mpidr = read_cpuid_mpidr();
443 this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
444 for_each_online_cpu(i) {
445 mcpm_cpu_use_count[this_cluster][i] = 1;
446 mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
447 }
448 mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
449 sync_cache_w(&mcpm_sync);
450
451 if (power_up_setup) {
452 mcpm_power_up_setup_phys = virt_to_phys(power_up_setup);
453 sync_cache_w(&mcpm_power_up_setup_phys);
454 }
455
456 return 0;
457}