Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * acpi_numa.c - ACPI NUMA support
4 *
5 * Copyright (C) 2002 Takayoshi Kochi <t-kochi@bq.jp.nec.com>
6 */
7
8#define pr_fmt(fmt) "ACPI: " fmt
9
10#include <linux/module.h>
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/types.h>
14#include <linux/errno.h>
15#include <linux/acpi.h>
16#include <linux/memblock.h>
17#include <linux/numa.h>
18#include <linux/nodemask.h>
19#include <linux/topology.h>
20#include <linux/numa_memblks.h>
21
22static nodemask_t nodes_found_map = NODE_MASK_NONE;
23
24/* maps to convert between proximity domain and logical node ID */
25static int pxm_to_node_map[MAX_PXM_DOMAINS]
26 = { [0 ... MAX_PXM_DOMAINS - 1] = NUMA_NO_NODE };
27static int node_to_pxm_map[MAX_NUMNODES]
28 = { [0 ... MAX_NUMNODES - 1] = PXM_INVAL };
29
30unsigned char acpi_srat_revision __initdata;
31static int acpi_numa __initdata;
32
33static int last_real_pxm;
34
35void __init disable_srat(void)
36{
37 acpi_numa = -1;
38}
39
40int pxm_to_node(int pxm)
41{
42 if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
43 return NUMA_NO_NODE;
44 return pxm_to_node_map[pxm];
45}
46EXPORT_SYMBOL(pxm_to_node);
47
48int node_to_pxm(int node)
49{
50 if (node < 0)
51 return PXM_INVAL;
52 return node_to_pxm_map[node];
53}
54
55static void __acpi_map_pxm_to_node(int pxm, int node)
56{
57 if (pxm_to_node_map[pxm] == NUMA_NO_NODE || node < pxm_to_node_map[pxm])
58 pxm_to_node_map[pxm] = node;
59 if (node_to_pxm_map[node] == PXM_INVAL || pxm < node_to_pxm_map[node])
60 node_to_pxm_map[node] = pxm;
61}
62
63int acpi_map_pxm_to_node(int pxm)
64{
65 int node;
66
67 if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
68 return NUMA_NO_NODE;
69
70 node = pxm_to_node_map[pxm];
71
72 if (node == NUMA_NO_NODE) {
73 node = first_unset_node(nodes_found_map);
74 if (node >= MAX_NUMNODES)
75 return NUMA_NO_NODE;
76 __acpi_map_pxm_to_node(pxm, node);
77 node_set(node, nodes_found_map);
78 }
79
80 return node;
81}
82EXPORT_SYMBOL(acpi_map_pxm_to_node);
83
84static void __init
85acpi_table_print_srat_entry(struct acpi_subtable_header *header)
86{
87 switch (header->type) {
88 case ACPI_SRAT_TYPE_CPU_AFFINITY:
89 {
90 struct acpi_srat_cpu_affinity *p =
91 (struct acpi_srat_cpu_affinity *)header;
92 pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n",
93 p->apic_id, p->local_sapic_eid,
94 p->proximity_domain_lo,
95 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
96 "enabled" : "disabled");
97 }
98 break;
99
100 case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
101 {
102 struct acpi_srat_mem_affinity *p =
103 (struct acpi_srat_mem_affinity *)header;
104 pr_debug("SRAT Memory (0x%llx length 0x%llx) in proximity domain %d %s%s%s\n",
105 (unsigned long long)p->base_address,
106 (unsigned long long)p->length,
107 p->proximity_domain,
108 (p->flags & ACPI_SRAT_MEM_ENABLED) ?
109 "enabled" : "disabled",
110 (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ?
111 " hot-pluggable" : "",
112 (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ?
113 " non-volatile" : "");
114 }
115 break;
116
117 case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY:
118 {
119 struct acpi_srat_x2apic_cpu_affinity *p =
120 (struct acpi_srat_x2apic_cpu_affinity *)header;
121 pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n",
122 p->apic_id,
123 p->proximity_domain,
124 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
125 "enabled" : "disabled");
126 }
127 break;
128
129 case ACPI_SRAT_TYPE_GICC_AFFINITY:
130 {
131 struct acpi_srat_gicc_affinity *p =
132 (struct acpi_srat_gicc_affinity *)header;
133 pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
134 p->acpi_processor_uid,
135 p->proximity_domain,
136 (p->flags & ACPI_SRAT_GICC_ENABLED) ?
137 "enabled" : "disabled");
138 }
139 break;
140
141 case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
142 {
143 struct acpi_srat_generic_affinity *p =
144 (struct acpi_srat_generic_affinity *)header;
145
146 if (p->device_handle_type == 0) {
147 /*
148 * For pci devices this may be the only place they
149 * are assigned a proximity domain
150 */
151 pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n",
152 *(u16 *)(&p->device_handle[0]),
153 *(u16 *)(&p->device_handle[2]),
154 p->proximity_domain,
155 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
156 "enabled" : "disabled");
157 } else {
158 /*
159 * In this case we can rely on the device having a
160 * proximity domain reference
161 */
162 pr_debug("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n",
163 (char *)(&p->device_handle[0]),
164 (char *)(&p->device_handle[8]),
165 p->proximity_domain,
166 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
167 "enabled" : "disabled");
168 }
169 }
170 break;
171
172 case ACPI_SRAT_TYPE_RINTC_AFFINITY:
173 {
174 struct acpi_srat_rintc_affinity *p =
175 (struct acpi_srat_rintc_affinity *)header;
176 pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
177 p->acpi_processor_uid,
178 p->proximity_domain,
179 (p->flags & ACPI_SRAT_RINTC_ENABLED) ?
180 "enabled" : "disabled");
181 }
182 break;
183
184 default:
185 pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
186 header->type);
187 break;
188 }
189}
190
191/*
192 * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
193 * up the NUMA heuristics which wants the local node to have a smaller
194 * distance than the others.
195 * Do some quick checks here and only use the SLIT if it passes.
196 */
197static int __init slit_valid(struct acpi_table_slit *slit)
198{
199 int i, j;
200 int d = slit->locality_count;
201 for (i = 0; i < d; i++) {
202 for (j = 0; j < d; j++) {
203 u8 val = slit->entry[d*i + j];
204 if (i == j) {
205 if (val != LOCAL_DISTANCE)
206 return 0;
207 } else if (val <= LOCAL_DISTANCE)
208 return 0;
209 }
210 }
211 return 1;
212}
213
214void __init bad_srat(void)
215{
216 pr_err("SRAT: SRAT not used.\n");
217 disable_srat();
218}
219
220int __init srat_disabled(void)
221{
222 return acpi_numa < 0;
223}
224
225__weak int __init numa_fill_memblks(u64 start, u64 end)
226{
227 return NUMA_NO_MEMBLK;
228}
229
230/*
231 * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for
232 * I/O localities since SRAT does not list them. I/O localities are
233 * not supported at this point.
234 */
235static int __init acpi_parse_slit(struct acpi_table_header *table)
236{
237 struct acpi_table_slit *slit = (struct acpi_table_slit *)table;
238 int i, j;
239
240 if (!slit_valid(slit)) {
241 pr_info("SLIT table looks invalid. Not used.\n");
242 return -EINVAL;
243 }
244
245 for (i = 0; i < slit->locality_count; i++) {
246 const int from_node = pxm_to_node(i);
247
248 if (from_node == NUMA_NO_NODE)
249 continue;
250
251 for (j = 0; j < slit->locality_count; j++) {
252 const int to_node = pxm_to_node(j);
253
254 if (to_node == NUMA_NO_NODE)
255 continue;
256
257 numa_set_distance(from_node, to_node,
258 slit->entry[slit->locality_count * i + j]);
259 }
260 }
261
262 return 0;
263}
264
265static int parsed_numa_memblks __initdata;
266
267static int __init
268acpi_parse_memory_affinity(union acpi_subtable_headers *header,
269 const unsigned long table_end)
270{
271 struct acpi_srat_mem_affinity *ma;
272 u64 start, end;
273 u32 hotpluggable;
274 int node, pxm;
275
276 ma = (struct acpi_srat_mem_affinity *)header;
277
278 acpi_table_print_srat_entry(&header->common);
279
280 if (srat_disabled())
281 return 0;
282 if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) {
283 pr_err("SRAT: Unexpected header length: %d\n",
284 ma->header.length);
285 goto out_err_bad_srat;
286 }
287 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
288 return 0;
289 hotpluggable = IS_ENABLED(CONFIG_MEMORY_HOTPLUG) &&
290 (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE);
291
292 start = ma->base_address;
293 end = start + ma->length;
294 pxm = ma->proximity_domain;
295 if (acpi_srat_revision <= 1)
296 pxm &= 0xff;
297
298 node = acpi_map_pxm_to_node(pxm);
299 if (node == NUMA_NO_NODE) {
300 pr_err("SRAT: Too many proximity domains.\n");
301 goto out_err_bad_srat;
302 }
303
304 if (numa_add_memblk(node, start, end) < 0) {
305 pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n",
306 node, (unsigned long long) start,
307 (unsigned long long) end - 1);
308 goto out_err_bad_srat;
309 }
310
311 node_set(node, numa_nodes_parsed);
312
313 pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
314 node, pxm,
315 (unsigned long long) start, (unsigned long long) end - 1,
316 hotpluggable ? " hotplug" : "",
317 ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : "");
318
319 /* Mark hotplug range in memblock. */
320 if (hotpluggable && memblock_mark_hotplug(start, ma->length))
321 pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n",
322 (unsigned long long)start, (unsigned long long)end - 1);
323
324 max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1));
325
326 parsed_numa_memblks++;
327
328 return 0;
329
330out_err_bad_srat:
331 /* Just disable SRAT, but do not fail and ignore errors. */
332 bad_srat();
333
334 return 0;
335}
336
337static int __init acpi_parse_cfmws(union acpi_subtable_headers *header,
338 void *arg, const unsigned long table_end)
339{
340 struct acpi_cedt_cfmws *cfmws;
341 int *fake_pxm = arg;
342 u64 start, end;
343 int node;
344
345 cfmws = (struct acpi_cedt_cfmws *)header;
346 start = cfmws->base_hpa;
347 end = cfmws->base_hpa + cfmws->window_size;
348
349 /*
350 * The SRAT may have already described NUMA details for all,
351 * or a portion of, this CFMWS HPA range. Extend the memblks
352 * found for any portion of the window to cover the entire
353 * window.
354 */
355 if (!numa_fill_memblks(start, end))
356 return 0;
357
358 /* No SRAT description. Create a new node. */
359 node = acpi_map_pxm_to_node(*fake_pxm);
360
361 if (node == NUMA_NO_NODE) {
362 pr_err("ACPI NUMA: Too many proximity domains while processing CFMWS.\n");
363 return -EINVAL;
364 }
365
366 if (numa_add_memblk(node, start, end) < 0) {
367 /* CXL driver must handle the NUMA_NO_NODE case */
368 pr_warn("ACPI NUMA: Failed to add memblk for CFMWS node %d [mem %#llx-%#llx]\n",
369 node, start, end);
370 }
371 node_set(node, numa_nodes_parsed);
372
373 /* Set the next available fake_pxm value */
374 (*fake_pxm)++;
375 return 0;
376}
377
378void __init __weak
379acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
380{
381 pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id);
382}
383
384static int __init
385acpi_parse_x2apic_affinity(union acpi_subtable_headers *header,
386 const unsigned long end)
387{
388 struct acpi_srat_x2apic_cpu_affinity *processor_affinity;
389
390 processor_affinity = (struct acpi_srat_x2apic_cpu_affinity *)header;
391
392 acpi_table_print_srat_entry(&header->common);
393
394 /* let architecture-dependent part to do it */
395 acpi_numa_x2apic_affinity_init(processor_affinity);
396
397 return 0;
398}
399
400static int __init
401acpi_parse_processor_affinity(union acpi_subtable_headers *header,
402 const unsigned long end)
403{
404 struct acpi_srat_cpu_affinity *processor_affinity;
405
406 processor_affinity = (struct acpi_srat_cpu_affinity *)header;
407
408 acpi_table_print_srat_entry(&header->common);
409
410 /* let architecture-dependent part to do it */
411 acpi_numa_processor_affinity_init(processor_affinity);
412
413 return 0;
414}
415
416static int __init
417acpi_parse_gicc_affinity(union acpi_subtable_headers *header,
418 const unsigned long end)
419{
420 struct acpi_srat_gicc_affinity *processor_affinity;
421
422 processor_affinity = (struct acpi_srat_gicc_affinity *)header;
423
424 acpi_table_print_srat_entry(&header->common);
425
426 /* let architecture-dependent part to do it */
427 acpi_numa_gicc_affinity_init(processor_affinity);
428
429 return 0;
430}
431
432#if defined(CONFIG_X86) || defined(CONFIG_ARM64)
433static int __init
434acpi_parse_gi_affinity(union acpi_subtable_headers *header,
435 const unsigned long end)
436{
437 struct acpi_srat_generic_affinity *gi_affinity;
438 int node;
439
440 gi_affinity = (struct acpi_srat_generic_affinity *)header;
441 if (!gi_affinity)
442 return -EINVAL;
443 acpi_table_print_srat_entry(&header->common);
444
445 if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
446 return -EINVAL;
447
448 node = acpi_map_pxm_to_node(gi_affinity->proximity_domain);
449 if (node == NUMA_NO_NODE) {
450 pr_err("SRAT: Too many proximity domains.\n");
451 return -EINVAL;
452 }
453 node_set(node, numa_nodes_parsed);
454 node_set_state(node, N_GENERIC_INITIATOR);
455
456 return 0;
457}
458#else
459static int __init
460acpi_parse_gi_affinity(union acpi_subtable_headers *header,
461 const unsigned long end)
462{
463 return 0;
464}
465#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */
466
467static int __init
468acpi_parse_rintc_affinity(union acpi_subtable_headers *header,
469 const unsigned long end)
470{
471 struct acpi_srat_rintc_affinity *rintc_affinity;
472
473 rintc_affinity = (struct acpi_srat_rintc_affinity *)header;
474 acpi_table_print_srat_entry(&header->common);
475
476 /* let architecture-dependent part to do it */
477 acpi_numa_rintc_affinity_init(rintc_affinity);
478
479 return 0;
480}
481
482static int __init acpi_parse_srat(struct acpi_table_header *table)
483{
484 struct acpi_table_srat *srat = (struct acpi_table_srat *)table;
485
486 acpi_srat_revision = srat->header.revision;
487
488 /* Real work done in acpi_table_parse_srat below. */
489
490 return 0;
491}
492
493static int __init
494acpi_table_parse_srat(enum acpi_srat_type id,
495 acpi_tbl_entry_handler handler, unsigned int max_entries)
496{
497 return acpi_table_parse_entries(ACPI_SIG_SRAT,
498 sizeof(struct acpi_table_srat), id,
499 handler, max_entries);
500}
501
502int __init acpi_numa_init(void)
503{
504 int i, fake_pxm, cnt = 0;
505
506 if (acpi_disabled)
507 return -EINVAL;
508
509 /*
510 * Should not limit number with cpu num that is from NR_CPUS or nr_cpus=
511 * SRAT cpu entries could have different order with that in MADT.
512 * So go over all cpu entries in SRAT to get apicid to node mapping.
513 */
514
515 /* SRAT: System Resource Affinity Table */
516 if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
517 struct acpi_subtable_proc srat_proc[5];
518
519 memset(srat_proc, 0, sizeof(srat_proc));
520 srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;
521 srat_proc[0].handler = acpi_parse_processor_affinity;
522 srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY;
523 srat_proc[1].handler = acpi_parse_x2apic_affinity;
524 srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
525 srat_proc[2].handler = acpi_parse_gicc_affinity;
526 srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY;
527 srat_proc[3].handler = acpi_parse_gi_affinity;
528 srat_proc[4].id = ACPI_SRAT_TYPE_RINTC_AFFINITY;
529 srat_proc[4].handler = acpi_parse_rintc_affinity;
530
531 acpi_table_parse_entries_array(ACPI_SIG_SRAT,
532 sizeof(struct acpi_table_srat),
533 srat_proc, ARRAY_SIZE(srat_proc), 0);
534
535 cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
536 acpi_parse_memory_affinity, 0);
537 }
538
539 /* SLIT: System Locality Information Table */
540 acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit);
541
542 /*
543 * CXL Fixed Memory Window Structures (CFMWS) must be parsed
544 * after the SRAT. Create NUMA Nodes for CXL memory ranges that
545 * are defined in the CFMWS and not already defined in the SRAT.
546 * Initialize a fake_pxm as the first available PXM to emulate.
547 */
548
549 /* fake_pxm is the next unused PXM value after SRAT parsing */
550 for (i = 0, fake_pxm = -1; i < MAX_NUMNODES; i++) {
551 if (node_to_pxm_map[i] > fake_pxm)
552 fake_pxm = node_to_pxm_map[i];
553 }
554 last_real_pxm = fake_pxm;
555 fake_pxm++;
556 acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws,
557 &fake_pxm);
558
559 if (cnt < 0)
560 return cnt;
561 else if (!parsed_numa_memblks)
562 return -ENOENT;
563 return 0;
564}
565
566bool acpi_node_backed_by_real_pxm(int nid)
567{
568 int pxm = node_to_pxm(nid);
569
570 return pxm <= last_real_pxm;
571}
572EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm);
573
574static int acpi_get_pxm(acpi_handle h)
575{
576 unsigned long long pxm;
577 acpi_status status;
578 acpi_handle handle;
579 acpi_handle phandle = h;
580
581 do {
582 handle = phandle;
583 status = acpi_evaluate_integer(handle, "_PXM", NULL, &pxm);
584 if (ACPI_SUCCESS(status))
585 return pxm;
586 status = acpi_get_parent(handle, &phandle);
587 } while (ACPI_SUCCESS(status));
588 return -1;
589}
590
591int acpi_get_node(acpi_handle handle)
592{
593 int pxm;
594
595 pxm = acpi_get_pxm(handle);
596
597 return pxm_to_node(pxm);
598}
599EXPORT_SYMBOL(acpi_get_node);