Loading...
1/*
2 * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
3 *
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/kernel.h>
13#include <linux/init.h>
14#include <linux/irqflags.h>
15
16#include <asm/mcpm.h>
17#include <asm/cacheflush.h>
18#include <asm/idmap.h>
19#include <asm/cputype.h>
20
21extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
22
23void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
24{
25 unsigned long val = ptr ? virt_to_phys(ptr) : 0;
26 mcpm_entry_vectors[cluster][cpu] = val;
27 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
28}
29
30extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
31
32void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
33 unsigned long poke_phys_addr, unsigned long poke_val)
34{
35 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
36 poke[0] = poke_phys_addr;
37 poke[1] = poke_val;
38 __sync_cache_range_w(poke, 2 * sizeof(*poke));
39}
40
41static const struct mcpm_platform_ops *platform_ops;
42
43int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
44{
45 if (platform_ops)
46 return -EBUSY;
47 platform_ops = ops;
48 return 0;
49}
50
51bool mcpm_is_available(void)
52{
53 return (platform_ops) ? true : false;
54}
55
56int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
57{
58 if (!platform_ops)
59 return -EUNATCH; /* try not to shadow power_up errors */
60 might_sleep();
61 return platform_ops->power_up(cpu, cluster);
62}
63
64typedef void (*phys_reset_t)(unsigned long);
65
66void mcpm_cpu_power_down(void)
67{
68 phys_reset_t phys_reset;
69
70 if (WARN_ON_ONCE(!platform_ops || !platform_ops->power_down))
71 return;
72 BUG_ON(!irqs_disabled());
73
74 /*
75 * Do this before calling into the power_down method,
76 * as it might not always be safe to do afterwards.
77 */
78 setup_mm_for_reboot();
79
80 platform_ops->power_down();
81
82 /*
83 * It is possible for a power_up request to happen concurrently
84 * with a power_down request for the same CPU. In this case the
85 * power_down method might not be able to actually enter a
86 * powered down state with the WFI instruction if the power_up
87 * method has removed the required reset condition. The
88 * power_down method is then allowed to return. We must perform
89 * a re-entry in the kernel as if the power_up method just had
90 * deasserted reset on the CPU.
91 *
92 * To simplify race issues, the platform specific implementation
93 * must accommodate for the possibility of unordered calls to
94 * power_down and power_up with a usage count. Therefore, if a
95 * call to power_up is issued for a CPU that is not down, then
96 * the next call to power_down must not attempt a full shutdown
97 * but only do the minimum (normally disabling L1 cache and CPU
98 * coherency) and return just as if a concurrent power_up request
99 * had happened as described above.
100 */
101
102 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
103 phys_reset(virt_to_phys(mcpm_entry_point));
104
105 /* should never get here */
106 BUG();
107}
108
109int mcpm_cpu_power_down_finish(unsigned int cpu, unsigned int cluster)
110{
111 int ret;
112
113 if (WARN_ON_ONCE(!platform_ops || !platform_ops->power_down_finish))
114 return -EUNATCH;
115
116 ret = platform_ops->power_down_finish(cpu, cluster);
117 if (ret)
118 pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
119 __func__, cpu, cluster, ret);
120
121 return ret;
122}
123
124void mcpm_cpu_suspend(u64 expected_residency)
125{
126 phys_reset_t phys_reset;
127
128 if (WARN_ON_ONCE(!platform_ops || !platform_ops->suspend))
129 return;
130 BUG_ON(!irqs_disabled());
131
132 /* Very similar to mcpm_cpu_power_down() */
133 setup_mm_for_reboot();
134 platform_ops->suspend(expected_residency);
135 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
136 phys_reset(virt_to_phys(mcpm_entry_point));
137 BUG();
138}
139
140int mcpm_cpu_powered_up(void)
141{
142 if (!platform_ops)
143 return -EUNATCH;
144 if (platform_ops->powered_up)
145 platform_ops->powered_up();
146 return 0;
147}
148
149struct sync_struct mcpm_sync;
150
151/*
152 * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
153 * This must be called at the point of committing to teardown of a CPU.
154 * The CPU cache (SCTRL.C bit) is expected to still be active.
155 */
156void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
157{
158 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
159 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
160}
161
162/*
163 * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
164 * cluster can be torn down without disrupting this CPU.
165 * To avoid deadlocks, this must be called before a CPU is powered down.
166 * The CPU cache (SCTRL.C bit) is expected to be off.
167 * However L2 cache might or might not be active.
168 */
169void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
170{
171 dmb();
172 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
173 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
174 sev();
175}
176
177/*
178 * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
179 * @state: the final state of the cluster:
180 * CLUSTER_UP: no destructive teardown was done and the cluster has been
181 * restored to the previous state (CPU cache still active); or
182 * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
183 * (CPU cache disabled, L2 cache either enabled or disabled).
184 */
185void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
186{
187 dmb();
188 mcpm_sync.clusters[cluster].cluster = state;
189 sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
190 sev();
191}
192
193/*
194 * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
195 * This function should be called by the last man, after local CPU teardown
196 * is complete. CPU cache expected to be active.
197 *
198 * Returns:
199 * false: the critical section was not entered because an inbound CPU was
200 * observed, or the cluster is already being set up;
201 * true: the critical section was entered: it is now safe to tear down the
202 * cluster.
203 */
204bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
205{
206 unsigned int i;
207 struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
208
209 /* Warn inbound CPUs that the cluster is being torn down: */
210 c->cluster = CLUSTER_GOING_DOWN;
211 sync_cache_w(&c->cluster);
212
213 /* Back out if the inbound cluster is already in the critical region: */
214 sync_cache_r(&c->inbound);
215 if (c->inbound == INBOUND_COMING_UP)
216 goto abort;
217
218 /*
219 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
220 * teardown is complete on each CPU before tearing down the cluster.
221 *
222 * If any CPU has been woken up again from the DOWN state, then we
223 * shouldn't be taking the cluster down at all: abort in that case.
224 */
225 sync_cache_r(&c->cpus);
226 for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
227 int cpustate;
228
229 if (i == cpu)
230 continue;
231
232 while (1) {
233 cpustate = c->cpus[i].cpu;
234 if (cpustate != CPU_GOING_DOWN)
235 break;
236
237 wfe();
238 sync_cache_r(&c->cpus[i].cpu);
239 }
240
241 switch (cpustate) {
242 case CPU_DOWN:
243 continue;
244
245 default:
246 goto abort;
247 }
248 }
249
250 return true;
251
252abort:
253 __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
254 return false;
255}
256
257int __mcpm_cluster_state(unsigned int cluster)
258{
259 sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
260 return mcpm_sync.clusters[cluster].cluster;
261}
262
263extern unsigned long mcpm_power_up_setup_phys;
264
265int __init mcpm_sync_init(
266 void (*power_up_setup)(unsigned int affinity_level))
267{
268 unsigned int i, j, mpidr, this_cluster;
269
270 BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
271 BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
272
273 /*
274 * Set initial CPU and cluster states.
275 * Only one cluster is assumed to be active at this point.
276 */
277 for (i = 0; i < MAX_NR_CLUSTERS; i++) {
278 mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
279 mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
280 for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
281 mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
282 }
283 mpidr = read_cpuid_mpidr();
284 this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
285 for_each_online_cpu(i)
286 mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
287 mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
288 sync_cache_w(&mcpm_sync);
289
290 if (power_up_setup) {
291 mcpm_power_up_setup_phys = virt_to_phys(power_up_setup);
292 sync_cache_w(&mcpm_power_up_setup_phys);
293 }
294
295 return 0;
296}
1/*
2 * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
3 *
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/kernel.h>
13#include <linux/init.h>
14#include <linux/irqflags.h>
15#include <linux/cpu_pm.h>
16
17#include <asm/mcpm.h>
18#include <asm/cacheflush.h>
19#include <asm/idmap.h>
20#include <asm/cputype.h>
21#include <asm/suspend.h>
22
23/*
24 * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
25 * For a comprehensive description of the main algorithm used here, please
26 * see Documentation/arm/cluster-pm-race-avoidance.txt.
27 */
28
29struct sync_struct mcpm_sync;
30
31/*
32 * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
33 * This must be called at the point of committing to teardown of a CPU.
34 * The CPU cache (SCTRL.C bit) is expected to still be active.
35 */
36static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
37{
38 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
39 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
40}
41
42/*
43 * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
44 * cluster can be torn down without disrupting this CPU.
45 * To avoid deadlocks, this must be called before a CPU is powered down.
46 * The CPU cache (SCTRL.C bit) is expected to be off.
47 * However L2 cache might or might not be active.
48 */
49static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
50{
51 dmb();
52 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
53 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
54 sev();
55}
56
57/*
58 * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
59 * @state: the final state of the cluster:
60 * CLUSTER_UP: no destructive teardown was done and the cluster has been
61 * restored to the previous state (CPU cache still active); or
62 * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
63 * (CPU cache disabled, L2 cache either enabled or disabled).
64 */
65static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
66{
67 dmb();
68 mcpm_sync.clusters[cluster].cluster = state;
69 sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
70 sev();
71}
72
73/*
74 * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
75 * This function should be called by the last man, after local CPU teardown
76 * is complete. CPU cache expected to be active.
77 *
78 * Returns:
79 * false: the critical section was not entered because an inbound CPU was
80 * observed, or the cluster is already being set up;
81 * true: the critical section was entered: it is now safe to tear down the
82 * cluster.
83 */
84static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
85{
86 unsigned int i;
87 struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
88
89 /* Warn inbound CPUs that the cluster is being torn down: */
90 c->cluster = CLUSTER_GOING_DOWN;
91 sync_cache_w(&c->cluster);
92
93 /* Back out if the inbound cluster is already in the critical region: */
94 sync_cache_r(&c->inbound);
95 if (c->inbound == INBOUND_COMING_UP)
96 goto abort;
97
98 /*
99 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
100 * teardown is complete on each CPU before tearing down the cluster.
101 *
102 * If any CPU has been woken up again from the DOWN state, then we
103 * shouldn't be taking the cluster down at all: abort in that case.
104 */
105 sync_cache_r(&c->cpus);
106 for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
107 int cpustate;
108
109 if (i == cpu)
110 continue;
111
112 while (1) {
113 cpustate = c->cpus[i].cpu;
114 if (cpustate != CPU_GOING_DOWN)
115 break;
116
117 wfe();
118 sync_cache_r(&c->cpus[i].cpu);
119 }
120
121 switch (cpustate) {
122 case CPU_DOWN:
123 continue;
124
125 default:
126 goto abort;
127 }
128 }
129
130 return true;
131
132abort:
133 __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
134 return false;
135}
136
137static int __mcpm_cluster_state(unsigned int cluster)
138{
139 sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
140 return mcpm_sync.clusters[cluster].cluster;
141}
142
143extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
144
145void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
146{
147 unsigned long val = ptr ? virt_to_phys(ptr) : 0;
148 mcpm_entry_vectors[cluster][cpu] = val;
149 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
150}
151
152extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
153
154void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
155 unsigned long poke_phys_addr, unsigned long poke_val)
156{
157 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
158 poke[0] = poke_phys_addr;
159 poke[1] = poke_val;
160 __sync_cache_range_w(poke, 2 * sizeof(*poke));
161}
162
163static const struct mcpm_platform_ops *platform_ops;
164
165int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
166{
167 if (platform_ops)
168 return -EBUSY;
169 platform_ops = ops;
170 return 0;
171}
172
173bool mcpm_is_available(void)
174{
175 return (platform_ops) ? true : false;
176}
177
178/*
179 * We can't use regular spinlocks. In the switcher case, it is possible
180 * for an outbound CPU to call power_down() after its inbound counterpart
181 * is already live using the same logical CPU number which trips lockdep
182 * debugging.
183 */
184static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
185
186static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
187
188static inline bool mcpm_cluster_unused(unsigned int cluster)
189{
190 int i, cnt;
191 for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
192 cnt |= mcpm_cpu_use_count[cluster][i];
193 return !cnt;
194}
195
196int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
197{
198 bool cpu_is_down, cluster_is_down;
199 int ret = 0;
200
201 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
202 if (!platform_ops)
203 return -EUNATCH; /* try not to shadow power_up errors */
204 might_sleep();
205
206 /*
207 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
208 * variant exists, we need to disable IRQs manually here.
209 */
210 local_irq_disable();
211 arch_spin_lock(&mcpm_lock);
212
213 cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
214 cluster_is_down = mcpm_cluster_unused(cluster);
215
216 mcpm_cpu_use_count[cluster][cpu]++;
217 /*
218 * The only possible values are:
219 * 0 = CPU down
220 * 1 = CPU (still) up
221 * 2 = CPU requested to be up before it had a chance
222 * to actually make itself down.
223 * Any other value is a bug.
224 */
225 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
226 mcpm_cpu_use_count[cluster][cpu] != 2);
227
228 if (cluster_is_down)
229 ret = platform_ops->cluster_powerup(cluster);
230 if (cpu_is_down && !ret)
231 ret = platform_ops->cpu_powerup(cpu, cluster);
232
233 arch_spin_unlock(&mcpm_lock);
234 local_irq_enable();
235 return ret;
236}
237
238typedef void (*phys_reset_t)(unsigned long);
239
240void mcpm_cpu_power_down(void)
241{
242 unsigned int mpidr, cpu, cluster;
243 bool cpu_going_down, last_man;
244 phys_reset_t phys_reset;
245
246 mpidr = read_cpuid_mpidr();
247 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
248 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
249 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
250 if (WARN_ON_ONCE(!platform_ops))
251 return;
252 BUG_ON(!irqs_disabled());
253
254 setup_mm_for_reboot();
255
256 __mcpm_cpu_going_down(cpu, cluster);
257 arch_spin_lock(&mcpm_lock);
258 BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
259
260 mcpm_cpu_use_count[cluster][cpu]--;
261 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
262 mcpm_cpu_use_count[cluster][cpu] != 1);
263 cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
264 last_man = mcpm_cluster_unused(cluster);
265
266 if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
267 platform_ops->cpu_powerdown_prepare(cpu, cluster);
268 platform_ops->cluster_powerdown_prepare(cluster);
269 arch_spin_unlock(&mcpm_lock);
270 platform_ops->cluster_cache_disable();
271 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
272 } else {
273 if (cpu_going_down)
274 platform_ops->cpu_powerdown_prepare(cpu, cluster);
275 arch_spin_unlock(&mcpm_lock);
276 /*
277 * If cpu_going_down is false here, that means a power_up
278 * request raced ahead of us. Even if we do not want to
279 * shut this CPU down, the caller still expects execution
280 * to return through the system resume entry path, like
281 * when the WFI is aborted due to a new IRQ or the like..
282 * So let's continue with cache cleaning in all cases.
283 */
284 platform_ops->cpu_cache_disable();
285 }
286
287 __mcpm_cpu_down(cpu, cluster);
288
289 /* Now we are prepared for power-down, do it: */
290 if (cpu_going_down)
291 wfi();
292
293 /*
294 * It is possible for a power_up request to happen concurrently
295 * with a power_down request for the same CPU. In this case the
296 * CPU might not be able to actually enter a powered down state
297 * with the WFI instruction if the power_up request has removed
298 * the required reset condition. We must perform a re-entry in
299 * the kernel as if the power_up method just had deasserted reset
300 * on the CPU.
301 */
302 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
303 phys_reset(virt_to_phys(mcpm_entry_point));
304
305 /* should never get here */
306 BUG();
307}
308
309int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
310{
311 int ret;
312
313 if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
314 return -EUNATCH;
315
316 ret = platform_ops->wait_for_powerdown(cpu, cluster);
317 if (ret)
318 pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
319 __func__, cpu, cluster, ret);
320
321 return ret;
322}
323
324void mcpm_cpu_suspend(void)
325{
326 if (WARN_ON_ONCE(!platform_ops))
327 return;
328
329 /* Some platforms might have to enable special resume modes, etc. */
330 if (platform_ops->cpu_suspend_prepare) {
331 unsigned int mpidr = read_cpuid_mpidr();
332 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
333 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
334 arch_spin_lock(&mcpm_lock);
335 platform_ops->cpu_suspend_prepare(cpu, cluster);
336 arch_spin_unlock(&mcpm_lock);
337 }
338 mcpm_cpu_power_down();
339}
340
341int mcpm_cpu_powered_up(void)
342{
343 unsigned int mpidr, cpu, cluster;
344 bool cpu_was_down, first_man;
345 unsigned long flags;
346
347 if (!platform_ops)
348 return -EUNATCH;
349
350 mpidr = read_cpuid_mpidr();
351 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
352 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
353 local_irq_save(flags);
354 arch_spin_lock(&mcpm_lock);
355
356 cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
357 first_man = mcpm_cluster_unused(cluster);
358
359 if (first_man && platform_ops->cluster_is_up)
360 platform_ops->cluster_is_up(cluster);
361 if (cpu_was_down)
362 mcpm_cpu_use_count[cluster][cpu] = 1;
363 if (platform_ops->cpu_is_up)
364 platform_ops->cpu_is_up(cpu, cluster);
365
366 arch_spin_unlock(&mcpm_lock);
367 local_irq_restore(flags);
368
369 return 0;
370}
371
372#ifdef CONFIG_ARM_CPU_SUSPEND
373
374static int __init nocache_trampoline(unsigned long _arg)
375{
376 void (*cache_disable)(void) = (void *)_arg;
377 unsigned int mpidr = read_cpuid_mpidr();
378 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
379 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
380 phys_reset_t phys_reset;
381
382 mcpm_set_entry_vector(cpu, cluster, cpu_resume);
383 setup_mm_for_reboot();
384
385 __mcpm_cpu_going_down(cpu, cluster);
386 BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
387 cache_disable();
388 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
389 __mcpm_cpu_down(cpu, cluster);
390
391 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
392 phys_reset(virt_to_phys(mcpm_entry_point));
393 BUG();
394}
395
396int __init mcpm_loopback(void (*cache_disable)(void))
397{
398 int ret;
399
400 /*
401 * We're going to soft-restart the current CPU through the
402 * low-level MCPM code by leveraging the suspend/resume
403 * infrastructure. Let's play it safe by using cpu_pm_enter()
404 * in case the CPU init code path resets the VFP or similar.
405 */
406 local_irq_disable();
407 local_fiq_disable();
408 ret = cpu_pm_enter();
409 if (!ret) {
410 ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
411 cpu_pm_exit();
412 }
413 local_fiq_enable();
414 local_irq_enable();
415 if (ret)
416 pr_err("%s returned %d\n", __func__, ret);
417 return ret;
418}
419
420#endif
421
422extern unsigned long mcpm_power_up_setup_phys;
423
424int __init mcpm_sync_init(
425 void (*power_up_setup)(unsigned int affinity_level))
426{
427 unsigned int i, j, mpidr, this_cluster;
428
429 BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
430 BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
431
432 /*
433 * Set initial CPU and cluster states.
434 * Only one cluster is assumed to be active at this point.
435 */
436 for (i = 0; i < MAX_NR_CLUSTERS; i++) {
437 mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
438 mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
439 for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
440 mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
441 }
442 mpidr = read_cpuid_mpidr();
443 this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
444 for_each_online_cpu(i) {
445 mcpm_cpu_use_count[this_cluster][i] = 1;
446 mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
447 }
448 mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
449 sync_cache_w(&mcpm_sync);
450
451 if (power_up_setup) {
452 mcpm_power_up_setup_phys = virt_to_phys(power_up_setup);
453 sync_cache_w(&mcpm_power_up_setup_phys);
454 }
455
456 return 0;
457}