Loading...
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/cpumask.h>
4#include <linux/delay.h>
5#include <linux/smp.h>
6
7#include <asm/io_apic.h>
8
9#include "local.h"
10
11DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
12
13#ifdef CONFIG_SMP
14static int apic_ipi_shorthand_off __ro_after_init;
15
16static __init int apic_ipi_shorthand(char *str)
17{
18 get_option(&str, &apic_ipi_shorthand_off);
19 return 1;
20}
21__setup("no_ipi_broadcast=", apic_ipi_shorthand);
22
23static int __init print_ipi_mode(void)
24{
25 pr_info("IPI shorthand broadcast: %s\n",
26 apic_ipi_shorthand_off ? "disabled" : "enabled");
27 return 0;
28}
29late_initcall(print_ipi_mode);
30
31void apic_smt_update(void)
32{
33 /*
34 * Do not switch to broadcast mode if:
35 * - Disabled on the command line
36 * - Only a single CPU is online
37 * - Not all present CPUs have been at least booted once
38 *
39 * The latter is important as the local APIC might be in some
40 * random state and a broadcast might cause havoc. That's
41 * especially true for NMI broadcasting.
42 */
43 if (apic_ipi_shorthand_off || num_online_cpus() == 1 ||
44 !cpumask_equal(cpu_present_mask, &cpus_booted_once_mask)) {
45 static_branch_disable(&apic_use_ipi_shorthand);
46 } else {
47 static_branch_enable(&apic_use_ipi_shorthand);
48 }
49}
50
51void apic_send_IPI_allbutself(unsigned int vector)
52{
53 if (num_online_cpus() < 2)
54 return;
55
56 if (static_branch_likely(&apic_use_ipi_shorthand))
57 __apic_send_IPI_allbutself(vector);
58 else
59 __apic_send_IPI_mask_allbutself(cpu_online_mask, vector);
60}
61
62/*
63 * Send a 'reschedule' IPI to another CPU. It goes straight through and
64 * wastes no time serializing anything. Worst case is that we lose a
65 * reschedule ...
66 */
67void native_smp_send_reschedule(int cpu)
68{
69 if (unlikely(cpu_is_offline(cpu))) {
70 WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
71 return;
72 }
73 __apic_send_IPI(cpu, RESCHEDULE_VECTOR);
74}
75
76void native_send_call_func_single_ipi(int cpu)
77{
78 __apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
79}
80
81void native_send_call_func_ipi(const struct cpumask *mask)
82{
83 if (static_branch_likely(&apic_use_ipi_shorthand)) {
84 unsigned int cpu = smp_processor_id();
85
86 if (!cpumask_or_equal(mask, cpumask_of(cpu), cpu_online_mask))
87 goto sendmask;
88
89 if (cpumask_test_cpu(cpu, mask))
90 __apic_send_IPI_all(CALL_FUNCTION_VECTOR);
91 else if (num_online_cpus() > 1)
92 __apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR);
93 return;
94 }
95
96sendmask:
97 __apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
98}
99
100void apic_send_nmi_to_offline_cpu(unsigned int cpu)
101{
102 if (WARN_ON_ONCE(!apic->nmi_to_offline_cpu))
103 return;
104 if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, &cpus_booted_once_mask)))
105 return;
106 apic->send_IPI(cpu, NMI_VECTOR);
107}
108#endif /* CONFIG_SMP */
109
110static inline int __prepare_ICR2(unsigned int mask)
111{
112 return SET_XAPIC_DEST_FIELD(mask);
113}
114
115u32 apic_mem_wait_icr_idle_timeout(void)
116{
117 int cnt;
118
119 for (cnt = 0; cnt < 1000; cnt++) {
120 if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
121 return 0;
122 inc_irq_stat(icr_read_retry_count);
123 udelay(100);
124 }
125 return APIC_ICR_BUSY;
126}
127
128void apic_mem_wait_icr_idle(void)
129{
130 while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
131 cpu_relax();
132}
133
134/*
135 * This is safe against interruption because it only writes the lower 32
136 * bits of the APIC_ICR register. The destination field is ignored for
137 * short hand IPIs.
138 *
139 * wait_icr_idle()
140 * write(ICR2, dest)
141 * NMI
142 * wait_icr_idle()
143 * write(ICR)
144 * wait_icr_idle()
145 * write(ICR)
146 *
147 * This function does not need to disable interrupts as there is no ICR2
148 * interaction. The memory write is direct except when the machine is
149 * affected by the 11AP Pentium erratum, which turns the plain write into
150 * an XCHG operation.
151 */
152static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
153{
154 /*
155 * Wait for the previous ICR command to complete. Use
156 * safe_apic_wait_icr_idle() for the NMI vector as there have been
157 * issues where otherwise the system hangs when the panic CPU tries
158 * to stop the others before launching the kdump kernel.
159 */
160 if (unlikely(vector == NMI_VECTOR))
161 apic_mem_wait_icr_idle_timeout();
162 else
163 apic_mem_wait_icr_idle();
164
165 /* Destination field (ICR2) and the destination mode are ignored */
166 native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0));
167}
168
169/*
170 * This is used to send an IPI with no shorthand notation (the destination is
171 * specified in bits 56 to 63 of the ICR).
172 */
173void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
174 unsigned int dest_mode)
175{
176 /* See comment in __default_send_IPI_shortcut() */
177 if (unlikely(vector == NMI_VECTOR))
178 apic_mem_wait_icr_idle_timeout();
179 else
180 apic_mem_wait_icr_idle();
181
182 /* Set the IPI destination field in the ICR */
183 native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask));
184 /* Send it with the proper destination mode */
185 native_apic_mem_write(APIC_ICR, __prepare_ICR(0, vector, dest_mode));
186}
187
188void default_send_IPI_single_phys(int cpu, int vector)
189{
190 unsigned long flags;
191
192 local_irq_save(flags);
193 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
194 vector, APIC_DEST_PHYSICAL);
195 local_irq_restore(flags);
196}
197
198void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
199{
200 unsigned long flags;
201 unsigned long cpu;
202
203 local_irq_save(flags);
204 for_each_cpu(cpu, mask) {
205 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
206 cpu), vector, APIC_DEST_PHYSICAL);
207 }
208 local_irq_restore(flags);
209}
210
211void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
212 int vector)
213{
214 unsigned int cpu, this_cpu = smp_processor_id();
215 unsigned long flags;
216
217 local_irq_save(flags);
218 for_each_cpu(cpu, mask) {
219 if (cpu == this_cpu)
220 continue;
221 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
222 cpu), vector, APIC_DEST_PHYSICAL);
223 }
224 local_irq_restore(flags);
225}
226
227/*
228 * Helper function for APICs which insist on cpumasks
229 */
230void default_send_IPI_single(int cpu, int vector)
231{
232 __apic_send_IPI_mask(cpumask_of(cpu), vector);
233}
234
235void default_send_IPI_allbutself(int vector)
236{
237 __default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
238}
239
240void default_send_IPI_all(int vector)
241{
242 __default_send_IPI_shortcut(APIC_DEST_ALLINC, vector);
243}
244
245void default_send_IPI_self(int vector)
246{
247 __default_send_IPI_shortcut(APIC_DEST_SELF, vector);
248}
249
250#ifdef CONFIG_X86_32
251void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
252{
253 unsigned long flags;
254 unsigned int cpu;
255
256 local_irq_save(flags);
257 for_each_cpu(cpu, mask)
258 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
259 local_irq_restore(flags);
260}
261
262void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
263 int vector)
264{
265 unsigned int cpu, this_cpu = smp_processor_id();
266 unsigned long flags;
267
268 local_irq_save(flags);
269 for_each_cpu(cpu, mask) {
270 if (cpu == this_cpu)
271 continue;
272 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
273 }
274 local_irq_restore(flags);
275}
276
277void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
278{
279 unsigned long mask = cpumask_bits(cpumask)[0];
280 unsigned long flags;
281
282 if (!mask)
283 return;
284
285 local_irq_save(flags);
286 WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
287 __default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
288 local_irq_restore(flags);
289}
290
291#ifdef CONFIG_SMP
292static int convert_apicid_to_cpu(u32 apic_id)
293{
294 int i;
295
296 for_each_possible_cpu(i) {
297 if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
298 return i;
299 }
300 return -1;
301}
302
303int safe_smp_processor_id(void)
304{
305 u32 apicid;
306 int cpuid;
307
308 if (!boot_cpu_has(X86_FEATURE_APIC))
309 return 0;
310
311 apicid = read_apic_id();
312 if (apicid == BAD_APICID)
313 return 0;
314
315 cpuid = convert_apicid_to_cpu(apicid);
316
317 return cpuid >= 0 ? cpuid : 0;
318}
319#endif
320#endif
1#include <linux/cpumask.h>
2#include <linux/interrupt.h>
3#include <linux/init.h>
4
5#include <linux/mm.h>
6#include <linux/delay.h>
7#include <linux/spinlock.h>
8#include <linux/kernel_stat.h>
9#include <linux/mc146818rtc.h>
10#include <linux/cache.h>
11#include <linux/cpu.h>
12#include <linux/module.h>
13
14#include <asm/smp.h>
15#include <asm/mtrr.h>
16#include <asm/tlbflush.h>
17#include <asm/mmu_context.h>
18#include <asm/apic.h>
19#include <asm/proto.h>
20#include <asm/ipi.h>
21
22void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
23{
24 unsigned long query_cpu;
25 unsigned long flags;
26
27 /*
28 * Hack. The clustered APIC addressing mode doesn't allow us to send
29 * to an arbitrary mask, so I do a unicast to each CPU instead.
30 * - mbligh
31 */
32 local_irq_save(flags);
33 for_each_cpu(query_cpu, mask) {
34 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
35 query_cpu), vector, APIC_DEST_PHYSICAL);
36 }
37 local_irq_restore(flags);
38}
39
40void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
41 int vector)
42{
43 unsigned int this_cpu = smp_processor_id();
44 unsigned int query_cpu;
45 unsigned long flags;
46
47 /* See Hack comment above */
48
49 local_irq_save(flags);
50 for_each_cpu(query_cpu, mask) {
51 if (query_cpu == this_cpu)
52 continue;
53 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
54 query_cpu), vector, APIC_DEST_PHYSICAL);
55 }
56 local_irq_restore(flags);
57}
58
59#ifdef CONFIG_X86_32
60
61void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
62 int vector)
63{
64 unsigned long flags;
65 unsigned int query_cpu;
66
67 /*
68 * Hack. The clustered APIC addressing mode doesn't allow us to send
69 * to an arbitrary mask, so I do a unicasts to each CPU instead. This
70 * should be modified to do 1 message per cluster ID - mbligh
71 */
72
73 local_irq_save(flags);
74 for_each_cpu(query_cpu, mask)
75 __default_send_IPI_dest_field(
76 early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
77 vector, apic->dest_logical);
78 local_irq_restore(flags);
79}
80
81void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
82 int vector)
83{
84 unsigned long flags;
85 unsigned int query_cpu;
86 unsigned int this_cpu = smp_processor_id();
87
88 /* See Hack comment above */
89
90 local_irq_save(flags);
91 for_each_cpu(query_cpu, mask) {
92 if (query_cpu == this_cpu)
93 continue;
94 __default_send_IPI_dest_field(
95 early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
96 vector, apic->dest_logical);
97 }
98 local_irq_restore(flags);
99}
100
101/*
102 * This is only used on smaller machines.
103 */
104void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
105{
106 unsigned long mask = cpumask_bits(cpumask)[0];
107 unsigned long flags;
108
109 if (WARN_ONCE(!mask, "empty IPI mask"))
110 return;
111
112 local_irq_save(flags);
113 WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
114 __default_send_IPI_dest_field(mask, vector, apic->dest_logical);
115 local_irq_restore(flags);
116}
117
118void default_send_IPI_allbutself(int vector)
119{
120 /*
121 * if there are no other CPUs in the system then we get an APIC send
122 * error if we try to broadcast, thus avoid sending IPIs in this case.
123 */
124 if (!(num_online_cpus() > 1))
125 return;
126
127 __default_local_send_IPI_allbutself(vector);
128}
129
130void default_send_IPI_all(int vector)
131{
132 __default_local_send_IPI_all(vector);
133}
134
135void default_send_IPI_self(int vector)
136{
137 __default_send_IPI_shortcut(APIC_DEST_SELF, vector, apic->dest_logical);
138}
139
140/* must come after the send_IPI functions above for inlining */
141static int convert_apicid_to_cpu(int apic_id)
142{
143 int i;
144
145 for_each_possible_cpu(i) {
146 if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
147 return i;
148 }
149 return -1;
150}
151
152int safe_smp_processor_id(void)
153{
154 int apicid, cpuid;
155
156 if (!cpu_has_apic)
157 return 0;
158
159 apicid = hard_smp_processor_id();
160 if (apicid == BAD_APICID)
161 return 0;
162
163 cpuid = convert_apicid_to_cpu(apicid);
164
165 return cpuid >= 0 ? cpuid : 0;
166}
167#endif