1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Author:  Xiang Gao <gaoxiang@loongson.cn>
  4 *          Huacai Chen <chenhuacai@loongson.cn>
  5 *
  6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  7 */
  8#include <linux/init.h>
  9#include <linux/kernel.h>
 10#include <linux/mm.h>
 11#include <linux/mmzone.h>
 12#include <linux/export.h>
 13#include <linux/nodemask.h>
 14#include <linux/swap.h>
 15#include <linux/memblock.h>
 16#include <linux/pfn.h>
 17#include <linux/acpi.h>
 18#include <linux/efi.h>
 19#include <linux/irq.h>
 20#include <linux/pci.h>
 21#include <asm/bootinfo.h>
 22#include <asm/loongson.h>
 23#include <asm/numa.h>
 24#include <asm/page.h>
 25#include <asm/pgalloc.h>
 26#include <asm/sections.h>
 27#include <asm/time.h>
 28
 29int numa_off;
 30unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES];
 31EXPORT_SYMBOL(node_distances);
 32
 33static struct numa_meminfo numa_meminfo;
 34cpumask_t cpus_on_node[MAX_NUMNODES];
 35cpumask_t phys_cpus_on_node[MAX_NUMNODES];
 36EXPORT_SYMBOL(cpus_on_node);
 37
 38/*
 39 * apicid, cpu, node mappings
 40 */
 41s16 __cpuid_to_node[CONFIG_NR_CPUS] = {
 42	[0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE
 43};
 44EXPORT_SYMBOL(__cpuid_to_node);
 45
 46nodemask_t numa_nodes_parsed __initdata;
 47
 48#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 49unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 50EXPORT_SYMBOL(__per_cpu_offset);
 51
 52static int __init pcpu_cpu_to_node(int cpu)
 53{
 54	return early_cpu_to_node(cpu);
 55}
 56
 57static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
 58{
 59	if (early_cpu_to_node(from) == early_cpu_to_node(to))
 60		return LOCAL_DISTANCE;
 61	else
 62		return REMOTE_DISTANCE;
 63}
 64
 65void __init pcpu_populate_pte(unsigned long addr)
 66{
 67	populate_kernel_pte(addr);
 68}
 69
 70void __init setup_per_cpu_areas(void)
 71{
 72	unsigned long delta;
 73	unsigned int cpu;
 74	int rc = -EINVAL;
 75
 76	if (pcpu_chosen_fc == PCPU_FC_AUTO) {
 77		if (nr_node_ids >= 8)
 78			pcpu_chosen_fc = PCPU_FC_PAGE;
 79		else
 80			pcpu_chosen_fc = PCPU_FC_EMBED;
 81	}
 82
 83	/*
 84	 * Always reserve area for module percpu variables.  That's
 85	 * what the legacy allocator did.
 86	 */
 87	if (pcpu_chosen_fc != PCPU_FC_PAGE) {
 88		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
 89					    PERCPU_DYNAMIC_RESERVE, PMD_SIZE,
 90					    pcpu_cpu_distance, pcpu_cpu_to_node);
 91		if (rc < 0)
 92			pr_warn("%s allocator failed (%d), falling back to page size\n",
 93				pcpu_fc_names[pcpu_chosen_fc], rc);
 94	}
 95	if (rc < 0)
 96		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node);
 97	if (rc < 0)
 98		panic("cannot initialize percpu area (err=%d)", rc);
 99
100	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
101	for_each_possible_cpu(cpu)
102		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
103}
104#endif
105
106/*
107 * Get nodeid by logical cpu number.
108 * __cpuid_to_node maps phyical cpu id to node, so we
109 * should use cpu_logical_map(cpu) to index it.
110 *
111 * This routine is only used in early phase during
112 * booting, after setup_per_cpu_areas calling and numa_node
113 * initialization, cpu_to_node will be used instead.
114 */
115int early_cpu_to_node(int cpu)
116{
117	int physid = cpu_logical_map(cpu);
118
119	if (physid < 0)
120		return NUMA_NO_NODE;
121
122	return __cpuid_to_node[physid];
123}
124
125void __init early_numa_add_cpu(int cpuid, s16 node)
126{
127	int cpu = __cpu_number_map[cpuid];
128
129	if (cpu < 0)
130		return;
131
132	cpumask_set_cpu(cpu, &cpus_on_node[node]);
133	cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]);
134}
135
136void numa_add_cpu(unsigned int cpu)
137{
138	int nid = cpu_to_node(cpu);
139	cpumask_set_cpu(cpu, &cpus_on_node[nid]);
140}
141
142void numa_remove_cpu(unsigned int cpu)
143{
144	int nid = cpu_to_node(cpu);
145	cpumask_clear_cpu(cpu, &cpus_on_node[nid]);
146}
147
148static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
149				     struct numa_meminfo *mi)
150{
151	/* ignore zero length blks */
152	if (start == end)
153		return 0;
154
155	/* whine about and ignore invalid blks */
156	if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
157		pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
158			   nid, start, end - 1);
159		return 0;
160	}
161
162	if (mi->nr_blks >= NR_NODE_MEMBLKS) {
163		pr_err("NUMA: too many memblk ranges\n");
164		return -EINVAL;
165	}
166
167	mi->blk[mi->nr_blks].start = PFN_ALIGN(start);
168	mi->blk[mi->nr_blks].end = PFN_ALIGN(end - PAGE_SIZE + 1);
169	mi->blk[mi->nr_blks].nid = nid;
170	mi->nr_blks++;
171	return 0;
172}
173
174/**
175 * numa_add_memblk - Add one numa_memblk to numa_meminfo
176 * @nid: NUMA node ID of the new memblk
177 * @start: Start address of the new memblk
178 * @end: End address of the new memblk
179 *
180 * Add a new memblk to the default numa_meminfo.
181 *
182 * RETURNS:
183 * 0 on success, -errno on failure.
184 */
185int __init numa_add_memblk(int nid, u64 start, u64 end)
186{
187	return numa_add_memblk_to(nid, start, end, &numa_meminfo);
188}
189
190static void __init node_mem_init(unsigned int node)
191{
192	unsigned long start_pfn, end_pfn;
193	unsigned long node_addrspace_offset;
194
195	node_addrspace_offset = nid_to_addrbase(node);
196	pr_info("Node%d's addrspace_offset is 0x%lx\n",
197			node, node_addrspace_offset);
198
199	get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
200	pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n",
201		node, start_pfn, end_pfn);
202
203	alloc_node_data(node);
204}
205
206#ifdef CONFIG_ACPI_NUMA
207
208static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type)
209{
210	static unsigned long num_physpages;
211
212	num_physpages += (size >> PAGE_SHIFT);
213	pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
214		node, type, start, size);
215	pr_info("       start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
216		start >> PAGE_SHIFT, (start + size) >> PAGE_SHIFT, num_physpages);
217	memblock_set_node(start, size, &memblock.memory, node);
218}
219
220/*
221 * add_numamem_region
222 *
223 * Add a uasable memory region described by BIOS. The
224 * routine gets each intersection between BIOS's region
225 * and node's region, and adds them into node's memblock
226 * pool.
227 *
228 */
229static void __init add_numamem_region(u64 start, u64 end, u32 type)
230{
231	u32 i;
232	u64 ofs = start;
233
234	if (start >= end) {
235		pr_debug("Invalid region: %016llx-%016llx\n", start, end);
236		return;
237	}
238
239	for (i = 0; i < numa_meminfo.nr_blks; i++) {
240		struct numa_memblk *mb = &numa_meminfo.blk[i];
241
242		if (ofs > mb->end)
243			continue;
244
245		if (end > mb->end) {
246			add_node_intersection(mb->nid, ofs, mb->end - ofs, type);
247			ofs = mb->end;
248		} else {
249			add_node_intersection(mb->nid, ofs, end - ofs, type);
250			break;
251		}
252	}
253}
254
255static void __init init_node_memblock(void)
256{
257	u32 mem_type;
258	u64 mem_end, mem_start, mem_size;
259	efi_memory_desc_t *md;
260
261	/* Parse memory information and activate */
262	for_each_efi_memory_desc(md) {
263		mem_type = md->type;
264		mem_start = md->phys_addr;
265		mem_size = md->num_pages << EFI_PAGE_SHIFT;
266		mem_end = mem_start + mem_size;
267
268		switch (mem_type) {
269		case EFI_LOADER_CODE:
270		case EFI_LOADER_DATA:
271		case EFI_BOOT_SERVICES_CODE:
272		case EFI_BOOT_SERVICES_DATA:
273		case EFI_PERSISTENT_MEMORY:
274		case EFI_CONVENTIONAL_MEMORY:
275			add_numamem_region(mem_start, mem_end, mem_type);
276			break;
277		case EFI_PAL_CODE:
278		case EFI_UNUSABLE_MEMORY:
279		case EFI_ACPI_RECLAIM_MEMORY:
280			add_numamem_region(mem_start, mem_end, mem_type);
281			fallthrough;
282		case EFI_RESERVED_TYPE:
283		case EFI_RUNTIME_SERVICES_CODE:
284		case EFI_RUNTIME_SERVICES_DATA:
285		case EFI_MEMORY_MAPPED_IO:
286		case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
287			pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
288					mem_type, mem_start, mem_size);
289			break;
290		}
291	}
292}
293
294static void __init numa_default_distance(void)
295{
296	int row, col;
297
298	for (row = 0; row < MAX_NUMNODES; row++)
299		for (col = 0; col < MAX_NUMNODES; col++) {
300			if (col == row)
301				node_distances[row][col] = LOCAL_DISTANCE;
302			else
303				/* We assume that one node per package here!
304				 *
305				 * A SLIT should be used for multiple nodes
306				 * per package to override default setting.
307				 */
308				node_distances[row][col] = REMOTE_DISTANCE;
309	}
310}
311
312/*
313 * fake_numa_init() - For Non-ACPI systems
314 * Return: 0 on success, -errno on failure.
315 */
316static int __init fake_numa_init(void)
317{
318	phys_addr_t start = memblock_start_of_DRAM();
319	phys_addr_t end = memblock_end_of_DRAM() - 1;
320
321	node_set(0, numa_nodes_parsed);
322	pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
323
324	return numa_add_memblk(0, start, end + 1);
325}
326
327int __init init_numa_memory(void)
328{
329	int i;
330	int ret;
331	int node;
332
333	for (i = 0; i < NR_CPUS; i++)
334		set_cpuid_to_node(i, NUMA_NO_NODE);
335
336	numa_default_distance();
337	nodes_clear(numa_nodes_parsed);
338	nodes_clear(node_possible_map);
339	nodes_clear(node_online_map);
340	memset(&numa_meminfo, 0, sizeof(numa_meminfo));
341
342	/* Parse SRAT and SLIT if provided by firmware. */
343	ret = acpi_disabled ? fake_numa_init() : acpi_numa_init();
344	if (ret < 0)
345		return ret;
346
347	node_possible_map = numa_nodes_parsed;
348	if (WARN_ON(nodes_empty(node_possible_map)))
349		return -EINVAL;
350
351	init_node_memblock();
352	if (!memblock_validate_numa_coverage(SZ_1M))
353		return -EINVAL;
354
355	for_each_node_mask(node, node_possible_map) {
356		node_mem_init(node);
357		node_set_online(node);
358	}
359	max_low_pfn = PHYS_PFN(memblock_end_of_DRAM());
360
361	setup_nr_node_ids();
362	loongson_sysconf.nr_nodes = nr_node_ids;
363	loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]);
364
365	return 0;
366}
367
368#endif
369
370void __init paging_init(void)
371{
372	unsigned int node;
373	unsigned long zones_size[MAX_NR_ZONES] = {0, };
374
375	for_each_online_node(node) {
376		unsigned long start_pfn, end_pfn;
377
378		get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
379
380		if (end_pfn > max_low_pfn)
381			max_low_pfn = end_pfn;
382	}
383#ifdef CONFIG_ZONE_DMA32
384	zones_size[ZONE_DMA32] = MAX_DMA32_PFN;
385#endif
386	zones_size[ZONE_NORMAL] = max_low_pfn;
387	free_area_init(zones_size);
388}
389
390void __init mem_init(void)
391{
392	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
393	memblock_free_all();
394}
395
396int pcibus_to_node(struct pci_bus *bus)
397{
398	return dev_to_node(&bus->dev);
399}
400EXPORT_SYMBOL(pcibus_to_node);