1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 *  acpi_numa.c - ACPI NUMA support
  4 *
  5 *  Copyright (C) 2002 Takayoshi Kochi <t-kochi@bq.jp.nec.com>
  6 */
  7
  8#define pr_fmt(fmt) "ACPI: " fmt
  9
 10#include <linux/module.h>
 11#include <linux/init.h>
 12#include <linux/kernel.h>
 13#include <linux/types.h>
 14#include <linux/errno.h>
 15#include <linux/acpi.h>
 16#include <linux/memblock.h>
 17#include <linux/numa.h>
 18#include <linux/nodemask.h>
 19#include <linux/topology.h>
 20
 21static nodemask_t nodes_found_map = NODE_MASK_NONE;
 22
 23/* maps to convert between proximity domain and logical node ID */
 24static int pxm_to_node_map[MAX_PXM_DOMAINS]
 25			= { [0 ... MAX_PXM_DOMAINS - 1] = NUMA_NO_NODE };
 26static int node_to_pxm_map[MAX_NUMNODES]
 27			= { [0 ... MAX_NUMNODES - 1] = PXM_INVAL };
 28
 29unsigned char acpi_srat_revision __initdata;
 30static int acpi_numa __initdata;
 31
 32static int last_real_pxm;
 33
 34void __init disable_srat(void)
 35{
 36	acpi_numa = -1;
 37}
 38
 39int pxm_to_node(int pxm)
 40{
 41	if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
 42		return NUMA_NO_NODE;
 43	return pxm_to_node_map[pxm];
 44}
 45EXPORT_SYMBOL(pxm_to_node);
 46
 47int node_to_pxm(int node)
 48{
 49	if (node < 0)
 50		return PXM_INVAL;
 51	return node_to_pxm_map[node];
 52}
 53
 54static void __acpi_map_pxm_to_node(int pxm, int node)
 55{
 56	if (pxm_to_node_map[pxm] == NUMA_NO_NODE || node < pxm_to_node_map[pxm])
 57		pxm_to_node_map[pxm] = node;
 58	if (node_to_pxm_map[node] == PXM_INVAL || pxm < node_to_pxm_map[node])
 59		node_to_pxm_map[node] = pxm;
 60}
 61
 62int acpi_map_pxm_to_node(int pxm)
 63{
 64	int node;
 65
 66	if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
 67		return NUMA_NO_NODE;
 68
 69	node = pxm_to_node_map[pxm];
 70
 71	if (node == NUMA_NO_NODE) {
 72		node = first_unset_node(nodes_found_map);
 73		if (node >= MAX_NUMNODES)
 74			return NUMA_NO_NODE;
 75		__acpi_map_pxm_to_node(pxm, node);
 76		node_set(node, nodes_found_map);
 77	}
 78
 79	return node;
 80}
 81EXPORT_SYMBOL(acpi_map_pxm_to_node);
 82
 83static void __init
 84acpi_table_print_srat_entry(struct acpi_subtable_header *header)
 85{
 86	switch (header->type) {
 87	case ACPI_SRAT_TYPE_CPU_AFFINITY:
 88		{
 89			struct acpi_srat_cpu_affinity *p =
 90			    (struct acpi_srat_cpu_affinity *)header;
 91			pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n",
 92				 p->apic_id, p->local_sapic_eid,
 93				 p->proximity_domain_lo,
 94				 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
 95				 "enabled" : "disabled");
 96		}
 97		break;
 98
 99	case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
100		{
101			struct acpi_srat_mem_affinity *p =
102			    (struct acpi_srat_mem_affinity *)header;
103			pr_debug("SRAT Memory (0x%llx length 0x%llx) in proximity domain %d %s%s%s\n",
104				 (unsigned long long)p->base_address,
105				 (unsigned long long)p->length,
106				 p->proximity_domain,
107				 (p->flags & ACPI_SRAT_MEM_ENABLED) ?
108				 "enabled" : "disabled",
109				 (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ?
110				 " hot-pluggable" : "",
111				 (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ?
112				 " non-volatile" : "");
113		}
114		break;
115
116	case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY:
117		{
118			struct acpi_srat_x2apic_cpu_affinity *p =
119			    (struct acpi_srat_x2apic_cpu_affinity *)header;
120			pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n",
121				 p->apic_id,
122				 p->proximity_domain,
123				 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
124				 "enabled" : "disabled");
125		}
126		break;
127
128	case ACPI_SRAT_TYPE_GICC_AFFINITY:
129		{
130			struct acpi_srat_gicc_affinity *p =
131			    (struct acpi_srat_gicc_affinity *)header;
132			pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
133				 p->acpi_processor_uid,
134				 p->proximity_domain,
135				 (p->flags & ACPI_SRAT_GICC_ENABLED) ?
136				 "enabled" : "disabled");
137		}
138		break;
139
140	case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
141	{
142		struct acpi_srat_generic_affinity *p =
143			(struct acpi_srat_generic_affinity *)header;
144
145		if (p->device_handle_type == 0) {
146			/*
147			 * For pci devices this may be the only place they
148			 * are assigned a proximity domain
149			 */
150			pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n",
151				 *(u16 *)(&p->device_handle[0]),
152				 *(u16 *)(&p->device_handle[2]),
153				 p->proximity_domain,
154				 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
155				"enabled" : "disabled");
156		} else {
157			/*
158			 * In this case we can rely on the device having a
159			 * proximity domain reference
160			 */
161			pr_debug("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n",
162				(char *)(&p->device_handle[0]),
163				(char *)(&p->device_handle[8]),
164				p->proximity_domain,
165				(p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
166				"enabled" : "disabled");
167		}
168	}
169	break;
170	default:
171		pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
172			header->type);
173		break;
174	}
175}
176
177/*
178 * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
179 * up the NUMA heuristics which wants the local node to have a smaller
180 * distance than the others.
181 * Do some quick checks here and only use the SLIT if it passes.
182 */
183static int __init slit_valid(struct acpi_table_slit *slit)
184{
185	int i, j;
186	int d = slit->locality_count;
187	for (i = 0; i < d; i++) {
188		for (j = 0; j < d; j++) {
189			u8 val = slit->entry[d*i + j];
190			if (i == j) {
191				if (val != LOCAL_DISTANCE)
192					return 0;
193			} else if (val <= LOCAL_DISTANCE)
194				return 0;
195		}
196	}
197	return 1;
198}
199
200void __init bad_srat(void)
201{
202	pr_err("SRAT: SRAT not used.\n");
203	disable_srat();
204}
205
206int __init srat_disabled(void)
207{
208	return acpi_numa < 0;
209}
210
211__weak int __init numa_fill_memblks(u64 start, u64 end)
212{
213	return NUMA_NO_MEMBLK;
214}
215
216#if defined(CONFIG_X86) || defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH)
217/*
218 * Callback for SLIT parsing.  pxm_to_node() returns NUMA_NO_NODE for
219 * I/O localities since SRAT does not list them.  I/O localities are
220 * not supported at this point.
221 */
222void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
223{
224	int i, j;
225
226	for (i = 0; i < slit->locality_count; i++) {
227		const int from_node = pxm_to_node(i);
228
229		if (from_node == NUMA_NO_NODE)
230			continue;
231
232		for (j = 0; j < slit->locality_count; j++) {
233			const int to_node = pxm_to_node(j);
234
235			if (to_node == NUMA_NO_NODE)
236				continue;
237
238			numa_set_distance(from_node, to_node,
239				slit->entry[slit->locality_count * i + j]);
240		}
241	}
242}
243
244/*
245 * Default callback for parsing of the Proximity Domain <-> Memory
246 * Area mappings
247 */
248int __init
249acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
250{
251	u64 start, end;
252	u32 hotpluggable;
253	int node, pxm;
254
255	if (srat_disabled())
256		goto out_err;
257	if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) {
258		pr_err("SRAT: Unexpected header length: %d\n",
259		       ma->header.length);
260		goto out_err_bad_srat;
261	}
262	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
263		goto out_err;
264	hotpluggable = IS_ENABLED(CONFIG_MEMORY_HOTPLUG) &&
265		(ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE);
266
267	start = ma->base_address;
268	end = start + ma->length;
269	pxm = ma->proximity_domain;
270	if (acpi_srat_revision <= 1)
271		pxm &= 0xff;
272
273	node = acpi_map_pxm_to_node(pxm);
274	if (node == NUMA_NO_NODE) {
275		pr_err("SRAT: Too many proximity domains.\n");
276		goto out_err_bad_srat;
277	}
278
279	if (numa_add_memblk(node, start, end) < 0) {
280		pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n",
281		       node, (unsigned long long) start,
282		       (unsigned long long) end - 1);
283		goto out_err_bad_srat;
284	}
285
286	node_set(node, numa_nodes_parsed);
287
288	pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
289		node, pxm,
290		(unsigned long long) start, (unsigned long long) end - 1,
291		hotpluggable ? " hotplug" : "",
292		ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : "");
293
294	/* Mark hotplug range in memblock. */
295	if (hotpluggable && memblock_mark_hotplug(start, ma->length))
296		pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n",
297			(unsigned long long)start, (unsigned long long)end - 1);
298
299	max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1));
300
301	return 0;
302out_err_bad_srat:
303	bad_srat();
304out_err:
305	return -EINVAL;
306}
307
308static int __init acpi_parse_cfmws(union acpi_subtable_headers *header,
309				   void *arg, const unsigned long table_end)
310{
311	struct acpi_cedt_cfmws *cfmws;
312	int *fake_pxm = arg;
313	u64 start, end;
314	int node;
315
316	cfmws = (struct acpi_cedt_cfmws *)header;
317	start = cfmws->base_hpa;
318	end = cfmws->base_hpa + cfmws->window_size;
319
320	/*
321	 * The SRAT may have already described NUMA details for all,
322	 * or a portion of, this CFMWS HPA range. Extend the memblks
323	 * found for any portion of the window to cover the entire
324	 * window.
325	 */
326	if (!numa_fill_memblks(start, end))
327		return 0;
328
329	/* No SRAT description. Create a new node. */
330	node = acpi_map_pxm_to_node(*fake_pxm);
331
332	if (node == NUMA_NO_NODE) {
333		pr_err("ACPI NUMA: Too many proximity domains while processing CFMWS.\n");
334		return -EINVAL;
335	}
336
337	if (numa_add_memblk(node, start, end) < 0) {
338		/* CXL driver must handle the NUMA_NO_NODE case */
339		pr_warn("ACPI NUMA: Failed to add memblk for CFMWS node %d [mem %#llx-%#llx]\n",
340			node, start, end);
341	}
342	node_set(node, numa_nodes_parsed);
343
344	/* Set the next available fake_pxm value */
345	(*fake_pxm)++;
346	return 0;
347}
348#else
349static int __init acpi_parse_cfmws(union acpi_subtable_headers *header,
350				   void *arg, const unsigned long table_end)
351{
352	return 0;
353}
354#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */
355
356static int __init acpi_parse_slit(struct acpi_table_header *table)
357{
358	struct acpi_table_slit *slit = (struct acpi_table_slit *)table;
359
360	if (!slit_valid(slit)) {
361		pr_info("SLIT table looks invalid. Not used.\n");
362		return -EINVAL;
363	}
364	acpi_numa_slit_init(slit);
365
366	return 0;
367}
368
369void __init __weak
370acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
371{
372	pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id);
373}
374
375static int __init
376acpi_parse_x2apic_affinity(union acpi_subtable_headers *header,
377			   const unsigned long end)
378{
379	struct acpi_srat_x2apic_cpu_affinity *processor_affinity;
380
381	processor_affinity = (struct acpi_srat_x2apic_cpu_affinity *)header;
382
383	acpi_table_print_srat_entry(&header->common);
384
385	/* let architecture-dependent part to do it */
386	acpi_numa_x2apic_affinity_init(processor_affinity);
387
388	return 0;
389}
390
391static int __init
392acpi_parse_processor_affinity(union acpi_subtable_headers *header,
393			      const unsigned long end)
394{
395	struct acpi_srat_cpu_affinity *processor_affinity;
396
397	processor_affinity = (struct acpi_srat_cpu_affinity *)header;
398
399	acpi_table_print_srat_entry(&header->common);
400
401	/* let architecture-dependent part to do it */
402	acpi_numa_processor_affinity_init(processor_affinity);
403
404	return 0;
405}
406
407static int __init
408acpi_parse_gicc_affinity(union acpi_subtable_headers *header,
409			 const unsigned long end)
410{
411	struct acpi_srat_gicc_affinity *processor_affinity;
412
413	processor_affinity = (struct acpi_srat_gicc_affinity *)header;
414
415	acpi_table_print_srat_entry(&header->common);
416
417	/* let architecture-dependent part to do it */
418	acpi_numa_gicc_affinity_init(processor_affinity);
419
420	return 0;
421}
422
423#if defined(CONFIG_X86) || defined(CONFIG_ARM64)
424static int __init
425acpi_parse_gi_affinity(union acpi_subtable_headers *header,
426		       const unsigned long end)
427{
428	struct acpi_srat_generic_affinity *gi_affinity;
429	int node;
430
431	gi_affinity = (struct acpi_srat_generic_affinity *)header;
432	if (!gi_affinity)
433		return -EINVAL;
434	acpi_table_print_srat_entry(&header->common);
435
436	if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
437		return -EINVAL;
438
439	node = acpi_map_pxm_to_node(gi_affinity->proximity_domain);
440	if (node == NUMA_NO_NODE) {
441		pr_err("SRAT: Too many proximity domains.\n");
442		return -EINVAL;
443	}
444	node_set(node, numa_nodes_parsed);
445	node_set_state(node, N_GENERIC_INITIATOR);
446
447	return 0;
448}
449#else
450static int __init
451acpi_parse_gi_affinity(union acpi_subtable_headers *header,
452		       const unsigned long end)
453{
454	return 0;
455}
456#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */
457
458static int __initdata parsed_numa_memblks;
459
460static int __init
461acpi_parse_memory_affinity(union acpi_subtable_headers * header,
462			   const unsigned long end)
463{
464	struct acpi_srat_mem_affinity *memory_affinity;
465
466	memory_affinity = (struct acpi_srat_mem_affinity *)header;
467
468	acpi_table_print_srat_entry(&header->common);
469
470	/* let architecture-dependent part to do it */
471	if (!acpi_numa_memory_affinity_init(memory_affinity))
472		parsed_numa_memblks++;
473	return 0;
474}
475
476static int __init acpi_parse_srat(struct acpi_table_header *table)
477{
478	struct acpi_table_srat *srat = (struct acpi_table_srat *)table;
479
480	acpi_srat_revision = srat->header.revision;
481
482	/* Real work done in acpi_table_parse_srat below. */
483
484	return 0;
485}
486
487static int __init
488acpi_table_parse_srat(enum acpi_srat_type id,
489		      acpi_tbl_entry_handler handler, unsigned int max_entries)
490{
491	return acpi_table_parse_entries(ACPI_SIG_SRAT,
492					    sizeof(struct acpi_table_srat), id,
493					    handler, max_entries);
494}
495
496int __init acpi_numa_init(void)
497{
498	int i, fake_pxm, cnt = 0;
499
500	if (acpi_disabled)
501		return -EINVAL;
502
503	/*
504	 * Should not limit number with cpu num that is from NR_CPUS or nr_cpus=
505	 * SRAT cpu entries could have different order with that in MADT.
506	 * So go over all cpu entries in SRAT to get apicid to node mapping.
507	 */
508
509	/* SRAT: System Resource Affinity Table */
510	if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
511		struct acpi_subtable_proc srat_proc[4];
512
513		memset(srat_proc, 0, sizeof(srat_proc));
514		srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;
515		srat_proc[0].handler = acpi_parse_processor_affinity;
516		srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY;
517		srat_proc[1].handler = acpi_parse_x2apic_affinity;
518		srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
519		srat_proc[2].handler = acpi_parse_gicc_affinity;
520		srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY;
521		srat_proc[3].handler = acpi_parse_gi_affinity;
522
523		acpi_table_parse_entries_array(ACPI_SIG_SRAT,
524					sizeof(struct acpi_table_srat),
525					srat_proc, ARRAY_SIZE(srat_proc), 0);
526
527		cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
528					    acpi_parse_memory_affinity, 0);
529	}
530
531	/* SLIT: System Locality Information Table */
532	acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit);
533
534	/*
535	 * CXL Fixed Memory Window Structures (CFMWS) must be parsed
536	 * after the SRAT. Create NUMA Nodes for CXL memory ranges that
537	 * are defined in the CFMWS and not already defined in the SRAT.
538	 * Initialize a fake_pxm as the first available PXM to emulate.
539	 */
540
541	/* fake_pxm is the next unused PXM value after SRAT parsing */
542	for (i = 0, fake_pxm = -1; i < MAX_NUMNODES; i++) {
543		if (node_to_pxm_map[i] > fake_pxm)
544			fake_pxm = node_to_pxm_map[i];
545	}
546	last_real_pxm = fake_pxm;
547	fake_pxm++;
548	acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws,
549			      &fake_pxm);
550
551	if (cnt < 0)
552		return cnt;
553	else if (!parsed_numa_memblks)
554		return -ENOENT;
555	return 0;
556}
557
558bool acpi_node_backed_by_real_pxm(int nid)
559{
560	int pxm = node_to_pxm(nid);
561
562	return pxm <= last_real_pxm;
563}
564EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm);
565
566static int acpi_get_pxm(acpi_handle h)
567{
568	unsigned long long pxm;
569	acpi_status status;
570	acpi_handle handle;
571	acpi_handle phandle = h;
572
573	do {
574		handle = phandle;
575		status = acpi_evaluate_integer(handle, "_PXM", NULL, &pxm);
576		if (ACPI_SUCCESS(status))
577			return pxm;
578		status = acpi_get_parent(handle, &phandle);
579	} while (ACPI_SUCCESS(status));
580	return -1;
581}
582
583int acpi_get_node(acpi_handle handle)
584{
585	int pxm;
586
587	pxm = acpi_get_pxm(handle);
588
589	return pxm_to_node(pxm);
590}
591EXPORT_SYMBOL(acpi_get_node);