1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * IA-64 Huge TLB Page Support for Kernel.
  4 *
  5 * Copyright (C) 2002-2004 Rohit Seth <rohit.seth@intel.com>
  6 * Copyright (C) 2003-2004 Ken Chen <kenneth.w.chen@intel.com>
  7 *
  8 * Sep, 2003: add numa support
  9 * Feb, 2004: dynamic hugetlb page size via boot parameter
 10 */
 11
 12#include <linux/init.h>
 13#include <linux/fs.h>
 14#include <linux/mm.h>
 15#include <linux/hugetlb.h>
 16#include <linux/pagemap.h>
 17#include <linux/module.h>
 18#include <linux/sysctl.h>
 19#include <linux/log2.h>
 20#include <asm/mman.h>
 21#include <asm/pgalloc.h>
 22#include <asm/tlb.h>
 23#include <asm/tlbflush.h>
 24
 25unsigned int hpage_shift = HPAGE_SHIFT_DEFAULT;
 26EXPORT_SYMBOL(hpage_shift);
 27
 28pte_t *
 29huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
 30{
 31	unsigned long taddr = htlbpage_to_page(addr);
 32	pgd_t *pgd;
 33	pud_t *pud;
 34	pmd_t *pmd;
 35	pte_t *pte = NULL;
 36
 37	pgd = pgd_offset(mm, taddr);
 38	pud = pud_alloc(mm, pgd, taddr);
 39	if (pud) {
 40		pmd = pmd_alloc(mm, pud, taddr);
 41		if (pmd)
 42			pte = pte_alloc_map(mm, pmd, taddr);
 43	}
 44	return pte;
 45}
 46
 47pte_t *
 48huge_pte_offset (struct mm_struct *mm, unsigned long addr, unsigned long sz)
 49{
 50	unsigned long taddr = htlbpage_to_page(addr);
 51	pgd_t *pgd;
 52	pud_t *pud;
 53	pmd_t *pmd;
 54	pte_t *pte = NULL;
 55
 56	pgd = pgd_offset(mm, taddr);
 57	if (pgd_present(*pgd)) {
 58		pud = pud_offset(pgd, taddr);
 59		if (pud_present(*pud)) {
 60			pmd = pmd_offset(pud, taddr);
 61			if (pmd_present(*pmd))
 62				pte = pte_offset_map(pmd, taddr);
 63		}
 64	}
 65
 66	return pte;
 67}
 68
 69#define mk_pte_huge(entry) { pte_val(entry) |= _PAGE_P; }
 70
 71/*
 72 * Don't actually need to do any preparation, but need to make sure
 73 * the address is in the right region.
 74 */
 75int prepare_hugepage_range(struct file *file,
 76			unsigned long addr, unsigned long len)
 77{
 78	if (len & ~HPAGE_MASK)
 79		return -EINVAL;
 80	if (addr & ~HPAGE_MASK)
 81		return -EINVAL;
 82	if (REGION_NUMBER(addr) != RGN_HPAGE)
 83		return -EINVAL;
 84
 85	return 0;
 86}
 87
 88struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int write)
 89{
 90	struct page *page;
 91	pte_t *ptep;
 92
 93	if (REGION_NUMBER(addr) != RGN_HPAGE)
 94		return ERR_PTR(-EINVAL);
 95
 96	ptep = huge_pte_offset(mm, addr, HPAGE_SIZE);
 97	if (!ptep || pte_none(*ptep))
 98		return NULL;
 99	page = pte_page(*ptep);
100	page += ((addr & ~HPAGE_MASK) >> PAGE_SHIFT);
101	return page;
102}
103int pmd_huge(pmd_t pmd)
104{
105	return 0;
106}
107
108int pud_huge(pud_t pud)
109{
110	return 0;
111}
112
113void hugetlb_free_pgd_range(struct mmu_gather *tlb,
114			unsigned long addr, unsigned long end,
115			unsigned long floor, unsigned long ceiling)
116{
117	/*
118	 * This is called to free hugetlb page tables.
119	 *
120	 * The offset of these addresses from the base of the hugetlb
121	 * region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that
122	 * the standard free_pgd_range will free the right page tables.
123	 *
124	 * If floor and ceiling are also in the hugetlb region, they
125	 * must likewise be scaled down; but if outside, left unchanged.
126	 */
127
128	addr = htlbpage_to_page(addr);
129	end  = htlbpage_to_page(end);
130	if (REGION_NUMBER(floor) == RGN_HPAGE)
131		floor = htlbpage_to_page(floor);
132	if (REGION_NUMBER(ceiling) == RGN_HPAGE)
133		ceiling = htlbpage_to_page(ceiling);
134
135	free_pgd_range(tlb, addr, end, floor, ceiling);
136}
137
138unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
139		unsigned long pgoff, unsigned long flags)
140{
141	struct vm_unmapped_area_info info;
142
143	if (len > RGN_MAP_LIMIT)
144		return -ENOMEM;
145	if (len & ~HPAGE_MASK)
146		return -EINVAL;
147
148	/* Handle MAP_FIXED */
149	if (flags & MAP_FIXED) {
150		if (prepare_hugepage_range(file, addr, len))
151			return -EINVAL;
152		return addr;
153	}
154
155	/* This code assumes that RGN_HPAGE != 0. */
156	if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
157		addr = HPAGE_REGION_BASE;
158
159	info.flags = 0;
160	info.length = len;
161	info.low_limit = addr;
162	info.high_limit = HPAGE_REGION_BASE + RGN_MAP_LIMIT;
163	info.align_mask = PAGE_MASK & (HPAGE_SIZE - 1);
164	info.align_offset = 0;
165	return vm_unmapped_area(&info);
166}
167
168static int __init hugetlb_setup_sz(char *str)
169{
170	u64 tr_pages;
171	unsigned long long size;
172
173	if (ia64_pal_vm_page_size(&tr_pages, NULL) != 0)
174		/*
175		 * shouldn't happen, but just in case.
176		 */
177		tr_pages = 0x15557000UL;
178
179	size = memparse(str, &str);
180	if (*str || !is_power_of_2(size) || !(tr_pages & size) ||
181		size <= PAGE_SIZE ||
182		size >= (1UL << PAGE_SHIFT << MAX_ORDER)) {
183		printk(KERN_WARNING "Invalid huge page size specified\n");
184		return 1;
185	}
186
187	hpage_shift = __ffs(size);
188	/*
189	 * boot cpu already executed ia64_mmu_init, and has HPAGE_SHIFT_DEFAULT
190	 * override here with new page shift.
191	 */
192	ia64_set_rr(HPAGE_REGION_BASE, hpage_shift << 2);
193	return 0;
194}
195early_param("hugepagesz", hugetlb_setup_sz);