Loading...
1/*
2 * PPC Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
11#include <linux/mm.h>
12#include <linux/io.h>
13#include <linux/slab.h>
14#include <linux/hugetlb.h>
15#include <linux/export.h>
16#include <linux/of_fdt.h>
17#include <linux/memblock.h>
18#include <linux/moduleparam.h>
19#include <linux/swap.h>
20#include <linux/swapops.h>
21#include <linux/kmemleak.h>
22#include <asm/pgalloc.h>
23#include <asm/tlb.h>
24#include <asm/setup.h>
25#include <asm/hugetlb.h>
26#include <asm/pte-walk.h>
27#include <asm/firmware.h>
28
29bool hugetlb_disabled = false;
30
31#define hugepd_none(hpd) (hpd_val(hpd) == 0)
32
33#define PTE_T_ORDER (__builtin_ffs(sizeof(pte_basic_t)) - \
34 __builtin_ffs(sizeof(void *)))
35
36pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz)
37{
38 /*
39 * Only called for hugetlbfs pages, hence can ignore THP and the
40 * irq disabled walk.
41 */
42 return __find_linux_pte(mm->pgd, addr, NULL, NULL);
43}
44
45static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
46 unsigned long address, unsigned int pdshift,
47 unsigned int pshift, spinlock_t *ptl)
48{
49 struct kmem_cache *cachep;
50 pte_t *new;
51 int i;
52 int num_hugepd;
53
54 if (pshift >= pdshift) {
55 cachep = PGT_CACHE(PTE_T_ORDER);
56 num_hugepd = 1 << (pshift - pdshift);
57 } else {
58 cachep = PGT_CACHE(pdshift - pshift);
59 num_hugepd = 1;
60 }
61
62 if (!cachep) {
63 WARN_ONCE(1, "No page table cache created for hugetlb tables");
64 return -ENOMEM;
65 }
66
67 new = kmem_cache_alloc(cachep, pgtable_gfp_flags(mm, GFP_KERNEL));
68
69 BUG_ON(pshift > HUGEPD_SHIFT_MASK);
70 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
71
72 if (!new)
73 return -ENOMEM;
74
75 /*
76 * Make sure other cpus find the hugepd set only after a
77 * properly initialized page table is visible to them.
78 * For more details look for comment in __pte_alloc().
79 */
80 smp_wmb();
81
82 spin_lock(ptl);
83 /*
84 * We have multiple higher-level entries that point to the same
85 * actual pte location. Fill in each as we go and backtrack on error.
86 * We need all of these so the DTLB pgtable walk code can find the
87 * right higher-level entry without knowing if it's a hugepage or not.
88 */
89 for (i = 0; i < num_hugepd; i++, hpdp++) {
90 if (unlikely(!hugepd_none(*hpdp)))
91 break;
92 hugepd_populate(hpdp, new, pshift);
93 }
94 /* If we bailed from the for loop early, an error occurred, clean up */
95 if (i < num_hugepd) {
96 for (i = i - 1 ; i >= 0; i--, hpdp--)
97 *hpdp = __hugepd(0);
98 kmem_cache_free(cachep, new);
99 } else {
100 kmemleak_ignore(new);
101 }
102 spin_unlock(ptl);
103 return 0;
104}
105
106/*
107 * At this point we do the placement change only for BOOK3S 64. This would
108 * possibly work on other subarchs.
109 */
110pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
111 unsigned long addr, unsigned long sz)
112{
113 pgd_t *pg;
114 p4d_t *p4;
115 pud_t *pu;
116 pmd_t *pm;
117 hugepd_t *hpdp = NULL;
118 unsigned pshift = __ffs(sz);
119 unsigned pdshift = PGDIR_SHIFT;
120 spinlock_t *ptl;
121
122 addr &= ~(sz-1);
123 pg = pgd_offset(mm, addr);
124 p4 = p4d_offset(pg, addr);
125
126#ifdef CONFIG_PPC_BOOK3S_64
127 if (pshift == PGDIR_SHIFT)
128 /* 16GB huge page */
129 return (pte_t *) p4;
130 else if (pshift > PUD_SHIFT) {
131 /*
132 * We need to use hugepd table
133 */
134 ptl = &mm->page_table_lock;
135 hpdp = (hugepd_t *)p4;
136 } else {
137 pdshift = PUD_SHIFT;
138 pu = pud_alloc(mm, p4, addr);
139 if (!pu)
140 return NULL;
141 if (pshift == PUD_SHIFT)
142 return (pte_t *)pu;
143 else if (pshift > PMD_SHIFT) {
144 ptl = pud_lockptr(mm, pu);
145 hpdp = (hugepd_t *)pu;
146 } else {
147 pdshift = PMD_SHIFT;
148 pm = pmd_alloc(mm, pu, addr);
149 if (!pm)
150 return NULL;
151 if (pshift == PMD_SHIFT)
152 /* 16MB hugepage */
153 return (pte_t *)pm;
154 else {
155 ptl = pmd_lockptr(mm, pm);
156 hpdp = (hugepd_t *)pm;
157 }
158 }
159 }
160#else
161 if (pshift >= PGDIR_SHIFT) {
162 ptl = &mm->page_table_lock;
163 hpdp = (hugepd_t *)p4;
164 } else {
165 pdshift = PUD_SHIFT;
166 pu = pud_alloc(mm, p4, addr);
167 if (!pu)
168 return NULL;
169 if (pshift >= PUD_SHIFT) {
170 ptl = pud_lockptr(mm, pu);
171 hpdp = (hugepd_t *)pu;
172 } else {
173 pdshift = PMD_SHIFT;
174 pm = pmd_alloc(mm, pu, addr);
175 if (!pm)
176 return NULL;
177 ptl = pmd_lockptr(mm, pm);
178 hpdp = (hugepd_t *)pm;
179 }
180 }
181#endif
182 if (!hpdp)
183 return NULL;
184
185 if (IS_ENABLED(CONFIG_PPC_8xx) && pshift < PMD_SHIFT)
186 return pte_alloc_huge(mm, (pmd_t *)hpdp, addr);
187
188 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
189
190 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr,
191 pdshift, pshift, ptl))
192 return NULL;
193
194 return hugepte_offset(*hpdp, addr, pdshift);
195}
196
197#ifdef CONFIG_PPC_BOOK3S_64
198/*
199 * Tracks gpages after the device tree is scanned and before the
200 * huge_boot_pages list is ready on pseries.
201 */
202#define MAX_NUMBER_GPAGES 1024
203__initdata static u64 gpage_freearray[MAX_NUMBER_GPAGES];
204__initdata static unsigned nr_gpages;
205
206/*
207 * Build list of addresses of gigantic pages. This function is used in early
208 * boot before the buddy allocator is setup.
209 */
210void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
211{
212 if (!addr)
213 return;
214 while (number_of_pages > 0) {
215 gpage_freearray[nr_gpages] = addr;
216 nr_gpages++;
217 number_of_pages--;
218 addr += page_size;
219 }
220}
221
222static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate)
223{
224 struct huge_bootmem_page *m;
225 if (nr_gpages == 0)
226 return 0;
227 m = phys_to_virt(gpage_freearray[--nr_gpages]);
228 gpage_freearray[nr_gpages] = 0;
229 list_add(&m->list, &huge_boot_pages);
230 m->hstate = hstate;
231 return 1;
232}
233
234bool __init hugetlb_node_alloc_supported(void)
235{
236 return false;
237}
238#endif
239
240
241int __init alloc_bootmem_huge_page(struct hstate *h, int nid)
242{
243
244#ifdef CONFIG_PPC_BOOK3S_64
245 if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled())
246 return pseries_alloc_bootmem_huge_page(h);
247#endif
248 return __alloc_bootmem_huge_page(h, nid);
249}
250
251#ifndef CONFIG_PPC_BOOK3S_64
252#define HUGEPD_FREELIST_SIZE \
253 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
254
255struct hugepd_freelist {
256 struct rcu_head rcu;
257 unsigned int index;
258 void *ptes[];
259};
260
261static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
262
263static void hugepd_free_rcu_callback(struct rcu_head *head)
264{
265 struct hugepd_freelist *batch =
266 container_of(head, struct hugepd_freelist, rcu);
267 unsigned int i;
268
269 for (i = 0; i < batch->index; i++)
270 kmem_cache_free(PGT_CACHE(PTE_T_ORDER), batch->ptes[i]);
271
272 free_page((unsigned long)batch);
273}
274
275static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
276{
277 struct hugepd_freelist **batchp;
278
279 batchp = &get_cpu_var(hugepd_freelist_cur);
280
281 if (atomic_read(&tlb->mm->mm_users) < 2 ||
282 mm_is_thread_local(tlb->mm)) {
283 kmem_cache_free(PGT_CACHE(PTE_T_ORDER), hugepte);
284 put_cpu_var(hugepd_freelist_cur);
285 return;
286 }
287
288 if (*batchp == NULL) {
289 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
290 (*batchp)->index = 0;
291 }
292
293 (*batchp)->ptes[(*batchp)->index++] = hugepte;
294 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
295 call_rcu(&(*batchp)->rcu, hugepd_free_rcu_callback);
296 *batchp = NULL;
297 }
298 put_cpu_var(hugepd_freelist_cur);
299}
300#else
301static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {}
302#endif
303
304/* Return true when the entry to be freed maps more than the area being freed */
305static bool range_is_outside_limits(unsigned long start, unsigned long end,
306 unsigned long floor, unsigned long ceiling,
307 unsigned long mask)
308{
309 if ((start & mask) < floor)
310 return true;
311 if (ceiling) {
312 ceiling &= mask;
313 if (!ceiling)
314 return true;
315 }
316 return end - 1 > ceiling - 1;
317}
318
319static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
320 unsigned long start, unsigned long end,
321 unsigned long floor, unsigned long ceiling)
322{
323 pte_t *hugepte = hugepd_page(*hpdp);
324 int i;
325
326 unsigned long pdmask = ~((1UL << pdshift) - 1);
327 unsigned int num_hugepd = 1;
328 unsigned int shift = hugepd_shift(*hpdp);
329
330 /* Note: On fsl the hpdp may be the first of several */
331 if (shift > pdshift)
332 num_hugepd = 1 << (shift - pdshift);
333
334 if (range_is_outside_limits(start, end, floor, ceiling, pdmask))
335 return;
336
337 for (i = 0; i < num_hugepd; i++, hpdp++)
338 *hpdp = __hugepd(0);
339
340 if (shift >= pdshift)
341 hugepd_free(tlb, hugepte);
342 else
343 pgtable_free_tlb(tlb, hugepte,
344 get_hugepd_cache_index(pdshift - shift));
345}
346
347static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
348 unsigned long addr, unsigned long end,
349 unsigned long floor, unsigned long ceiling)
350{
351 pgtable_t token = pmd_pgtable(*pmd);
352
353 if (range_is_outside_limits(addr, end, floor, ceiling, PMD_MASK))
354 return;
355
356 pmd_clear(pmd);
357 pte_free_tlb(tlb, token, addr);
358 mm_dec_nr_ptes(tlb->mm);
359}
360
361static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
362 unsigned long addr, unsigned long end,
363 unsigned long floor, unsigned long ceiling)
364{
365 pmd_t *pmd;
366 unsigned long next;
367 unsigned long start;
368
369 start = addr;
370 do {
371 unsigned long more;
372
373 pmd = pmd_offset(pud, addr);
374 next = pmd_addr_end(addr, end);
375 if (!is_hugepd(__hugepd(pmd_val(*pmd)))) {
376 if (pmd_none_or_clear_bad(pmd))
377 continue;
378
379 /*
380 * if it is not hugepd pointer, we should already find
381 * it cleared.
382 */
383 WARN_ON(!IS_ENABLED(CONFIG_PPC_8xx));
384
385 hugetlb_free_pte_range(tlb, pmd, addr, end, floor, ceiling);
386
387 continue;
388 }
389 /*
390 * Increment next by the size of the huge mapping since
391 * there may be more than one entry at this level for a
392 * single hugepage, but all of them point to
393 * the same kmem cache that holds the hugepte.
394 */
395 more = addr + (1UL << hugepd_shift(*(hugepd_t *)pmd));
396 if (more > next)
397 next = more;
398
399 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
400 addr, next, floor, ceiling);
401 } while (addr = next, addr != end);
402
403 if (range_is_outside_limits(start, end, floor, ceiling, PUD_MASK))
404 return;
405
406 pmd = pmd_offset(pud, start & PUD_MASK);
407 pud_clear(pud);
408 pmd_free_tlb(tlb, pmd, start & PUD_MASK);
409 mm_dec_nr_pmds(tlb->mm);
410}
411
412static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
413 unsigned long addr, unsigned long end,
414 unsigned long floor, unsigned long ceiling)
415{
416 pud_t *pud;
417 unsigned long next;
418 unsigned long start;
419
420 start = addr;
421 do {
422 pud = pud_offset(p4d, addr);
423 next = pud_addr_end(addr, end);
424 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
425 if (pud_none_or_clear_bad(pud))
426 continue;
427 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
428 ceiling);
429 } else {
430 unsigned long more;
431 /*
432 * Increment next by the size of the huge mapping since
433 * there may be more than one entry at this level for a
434 * single hugepage, but all of them point to
435 * the same kmem cache that holds the hugepte.
436 */
437 more = addr + (1UL << hugepd_shift(*(hugepd_t *)pud));
438 if (more > next)
439 next = more;
440
441 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
442 addr, next, floor, ceiling);
443 }
444 } while (addr = next, addr != end);
445
446 if (range_is_outside_limits(start, end, floor, ceiling, PGDIR_MASK))
447 return;
448
449 pud = pud_offset(p4d, start & PGDIR_MASK);
450 p4d_clear(p4d);
451 pud_free_tlb(tlb, pud, start & PGDIR_MASK);
452 mm_dec_nr_puds(tlb->mm);
453}
454
455/*
456 * This function frees user-level page tables of a process.
457 */
458void hugetlb_free_pgd_range(struct mmu_gather *tlb,
459 unsigned long addr, unsigned long end,
460 unsigned long floor, unsigned long ceiling)
461{
462 pgd_t *pgd;
463 p4d_t *p4d;
464 unsigned long next;
465
466 /*
467 * Because there are a number of different possible pagetable
468 * layouts for hugepage ranges, we limit knowledge of how
469 * things should be laid out to the allocation path
470 * (huge_pte_alloc(), above). Everything else works out the
471 * structure as it goes from information in the hugepd
472 * pointers. That means that we can't here use the
473 * optimization used in the normal page free_pgd_range(), of
474 * checking whether we're actually covering a large enough
475 * range to have to do anything at the top level of the walk
476 * instead of at the bottom.
477 *
478 * To make sense of this, you should probably go read the big
479 * block comment at the top of the normal free_pgd_range(),
480 * too.
481 */
482
483 do {
484 next = pgd_addr_end(addr, end);
485 pgd = pgd_offset(tlb->mm, addr);
486 p4d = p4d_offset(pgd, addr);
487 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
488 if (p4d_none_or_clear_bad(p4d))
489 continue;
490 hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
491 } else {
492 unsigned long more;
493 /*
494 * Increment next by the size of the huge mapping since
495 * there may be more than one entry at the pgd level
496 * for a single hugepage, but all of them point to the
497 * same kmem cache that holds the hugepte.
498 */
499 more = addr + (1UL << hugepd_shift(*(hugepd_t *)pgd));
500 if (more > next)
501 next = more;
502
503 free_hugepd_range(tlb, (hugepd_t *)p4d, PGDIR_SHIFT,
504 addr, next, floor, ceiling);
505 }
506 } while (addr = next, addr != end);
507}
508
509bool __init arch_hugetlb_valid_size(unsigned long size)
510{
511 int shift = __ffs(size);
512 int mmu_psize;
513
514 /* Check that it is a page size supported by the hardware and
515 * that it fits within pagetable and slice limits. */
516 if (size <= PAGE_SIZE || !is_power_of_2(size))
517 return false;
518
519 mmu_psize = check_and_get_huge_psize(shift);
520 if (mmu_psize < 0)
521 return false;
522
523 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
524
525 return true;
526}
527
528static int __init add_huge_page_size(unsigned long long size)
529{
530 int shift = __ffs(size);
531
532 if (!arch_hugetlb_valid_size((unsigned long)size))
533 return -EINVAL;
534
535 hugetlb_add_hstate(shift - PAGE_SHIFT);
536 return 0;
537}
538
539static int __init hugetlbpage_init(void)
540{
541 bool configured = false;
542 int psize;
543
544 if (hugetlb_disabled) {
545 pr_info("HugeTLB support is disabled!\n");
546 return 0;
547 }
548
549 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !radix_enabled() &&
550 !mmu_has_feature(MMU_FTR_16M_PAGE))
551 return -ENODEV;
552
553 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
554 unsigned shift;
555 unsigned pdshift;
556
557 if (!mmu_psize_defs[psize].shift)
558 continue;
559
560 shift = mmu_psize_to_shift(psize);
561
562#ifdef CONFIG_PPC_BOOK3S_64
563 if (shift > PGDIR_SHIFT)
564 continue;
565 else if (shift > PUD_SHIFT)
566 pdshift = PGDIR_SHIFT;
567 else if (shift > PMD_SHIFT)
568 pdshift = PUD_SHIFT;
569 else
570 pdshift = PMD_SHIFT;
571#else
572 if (shift < PUD_SHIFT)
573 pdshift = PMD_SHIFT;
574 else if (shift < PGDIR_SHIFT)
575 pdshift = PUD_SHIFT;
576 else
577 pdshift = PGDIR_SHIFT;
578#endif
579
580 if (add_huge_page_size(1ULL << shift) < 0)
581 continue;
582 /*
583 * if we have pdshift and shift value same, we don't
584 * use pgt cache for hugepd.
585 */
586 if (pdshift > shift) {
587 if (!IS_ENABLED(CONFIG_PPC_8xx))
588 pgtable_cache_add(pdshift - shift);
589 } else if (IS_ENABLED(CONFIG_PPC_E500) ||
590 IS_ENABLED(CONFIG_PPC_8xx)) {
591 pgtable_cache_add(PTE_T_ORDER);
592 }
593
594 configured = true;
595 }
596
597 if (!configured)
598 pr_info("Failed to initialize. Disabling HugeTLB");
599
600 return 0;
601}
602
603arch_initcall(hugetlbpage_init);
604
605void __init gigantic_hugetlb_cma_reserve(void)
606{
607 unsigned long order = 0;
608
609 if (radix_enabled())
610 order = PUD_SHIFT - PAGE_SHIFT;
611 else if (!firmware_has_feature(FW_FEATURE_LPAR) && mmu_psize_defs[MMU_PAGE_16G].shift)
612 /*
613 * For pseries we do use ibm,expected#pages for reserving 16G pages.
614 */
615 order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT;
616
617 if (order) {
618 VM_WARN_ON(order <= MAX_PAGE_ORDER);
619 hugetlb_cma_reserve(order);
620 }
621}
1/*
2 * PPC Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
11#include <linux/mm.h>
12#include <linux/io.h>
13#include <linux/slab.h>
14#include <linux/hugetlb.h>
15#include <linux/export.h>
16#include <linux/of_fdt.h>
17#include <linux/memblock.h>
18#include <linux/bootmem.h>
19#include <linux/moduleparam.h>
20#include <asm/pgtable.h>
21#include <asm/pgalloc.h>
22#include <asm/tlb.h>
23#include <asm/setup.h>
24#include <asm/hugetlb.h>
25
26#ifdef CONFIG_HUGETLB_PAGE
27
28#define PAGE_SHIFT_64K 16
29#define PAGE_SHIFT_512K 19
30#define PAGE_SHIFT_8M 23
31#define PAGE_SHIFT_16M 24
32#define PAGE_SHIFT_16G 34
33
34unsigned int HPAGE_SHIFT;
35
36/*
37 * Tracks gpages after the device tree is scanned and before the
38 * huge_boot_pages list is ready. On non-Freescale implementations, this is
39 * just used to track 16G pages and so is a single array. FSL-based
40 * implementations may have more than one gpage size, so we need multiple
41 * arrays
42 */
43#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
44#define MAX_NUMBER_GPAGES 128
45struct psize_gpages {
46 u64 gpage_list[MAX_NUMBER_GPAGES];
47 unsigned int nr_gpages;
48};
49static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
50#else
51#define MAX_NUMBER_GPAGES 1024
52static u64 gpage_freearray[MAX_NUMBER_GPAGES];
53static unsigned nr_gpages;
54#endif
55
56#define hugepd_none(hpd) (hpd_val(hpd) == 0)
57
58pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
59{
60 /* Only called for hugetlbfs pages, hence can ignore THP */
61 return __find_linux_pte_or_hugepte(mm->pgd, addr, NULL, NULL);
62}
63
64static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
65 unsigned long address, unsigned pdshift, unsigned pshift)
66{
67 struct kmem_cache *cachep;
68 pte_t *new;
69 int i;
70 int num_hugepd;
71
72 if (pshift >= pdshift) {
73 cachep = hugepte_cache;
74 num_hugepd = 1 << (pshift - pdshift);
75 } else {
76 cachep = PGT_CACHE(pdshift - pshift);
77 num_hugepd = 1;
78 }
79
80 new = kmem_cache_zalloc(cachep, GFP_KERNEL);
81
82 BUG_ON(pshift > HUGEPD_SHIFT_MASK);
83 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
84
85 if (! new)
86 return -ENOMEM;
87
88 /*
89 * Make sure other cpus find the hugepd set only after a
90 * properly initialized page table is visible to them.
91 * For more details look for comment in __pte_alloc().
92 */
93 smp_wmb();
94
95 spin_lock(&mm->page_table_lock);
96
97 /*
98 * We have multiple higher-level entries that point to the same
99 * actual pte location. Fill in each as we go and backtrack on error.
100 * We need all of these so the DTLB pgtable walk code can find the
101 * right higher-level entry without knowing if it's a hugepage or not.
102 */
103 for (i = 0; i < num_hugepd; i++, hpdp++) {
104 if (unlikely(!hugepd_none(*hpdp)))
105 break;
106 else {
107#ifdef CONFIG_PPC_BOOK3S_64
108 *hpdp = __hugepd(__pa(new) |
109 (shift_to_mmu_psize(pshift) << 2));
110#elif defined(CONFIG_PPC_8xx)
111 *hpdp = __hugepd(__pa(new) |
112 (pshift == PAGE_SHIFT_8M ? _PMD_PAGE_8M :
113 _PMD_PAGE_512K) | _PMD_PRESENT);
114#else
115 /* We use the old format for PPC_FSL_BOOK3E */
116 *hpdp = __hugepd(((unsigned long)new & ~PD_HUGE) | pshift);
117#endif
118 }
119 }
120 /* If we bailed from the for loop early, an error occurred, clean up */
121 if (i < num_hugepd) {
122 for (i = i - 1 ; i >= 0; i--, hpdp--)
123 *hpdp = __hugepd(0);
124 kmem_cache_free(cachep, new);
125 }
126 spin_unlock(&mm->page_table_lock);
127 return 0;
128}
129
130/*
131 * These macros define how to determine which level of the page table holds
132 * the hpdp.
133 */
134#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
135#define HUGEPD_PGD_SHIFT PGDIR_SHIFT
136#define HUGEPD_PUD_SHIFT PUD_SHIFT
137#else
138#define HUGEPD_PGD_SHIFT PUD_SHIFT
139#define HUGEPD_PUD_SHIFT PMD_SHIFT
140#endif
141
142/*
143 * At this point we do the placement change only for BOOK3S 64. This would
144 * possibly work on other subarchs.
145 */
146pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
147{
148 pgd_t *pg;
149 pud_t *pu;
150 pmd_t *pm;
151 hugepd_t *hpdp = NULL;
152 unsigned pshift = __ffs(sz);
153 unsigned pdshift = PGDIR_SHIFT;
154
155 addr &= ~(sz-1);
156 pg = pgd_offset(mm, addr);
157
158#ifdef CONFIG_PPC_BOOK3S_64
159 if (pshift == PGDIR_SHIFT)
160 /* 16GB huge page */
161 return (pte_t *) pg;
162 else if (pshift > PUD_SHIFT)
163 /*
164 * We need to use hugepd table
165 */
166 hpdp = (hugepd_t *)pg;
167 else {
168 pdshift = PUD_SHIFT;
169 pu = pud_alloc(mm, pg, addr);
170 if (pshift == PUD_SHIFT)
171 return (pte_t *)pu;
172 else if (pshift > PMD_SHIFT)
173 hpdp = (hugepd_t *)pu;
174 else {
175 pdshift = PMD_SHIFT;
176 pm = pmd_alloc(mm, pu, addr);
177 if (pshift == PMD_SHIFT)
178 /* 16MB hugepage */
179 return (pte_t *)pm;
180 else
181 hpdp = (hugepd_t *)pm;
182 }
183 }
184#else
185 if (pshift >= HUGEPD_PGD_SHIFT) {
186 hpdp = (hugepd_t *)pg;
187 } else {
188 pdshift = PUD_SHIFT;
189 pu = pud_alloc(mm, pg, addr);
190 if (pshift >= HUGEPD_PUD_SHIFT) {
191 hpdp = (hugepd_t *)pu;
192 } else {
193 pdshift = PMD_SHIFT;
194 pm = pmd_alloc(mm, pu, addr);
195 hpdp = (hugepd_t *)pm;
196 }
197 }
198#endif
199 if (!hpdp)
200 return NULL;
201
202 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
203
204 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
205 return NULL;
206
207 return hugepte_offset(*hpdp, addr, pdshift);
208}
209
210#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
211/* Build list of addresses of gigantic pages. This function is used in early
212 * boot before the buddy allocator is setup.
213 */
214void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
215{
216 unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
217 int i;
218
219 if (addr == 0)
220 return;
221
222 gpage_freearray[idx].nr_gpages = number_of_pages;
223
224 for (i = 0; i < number_of_pages; i++) {
225 gpage_freearray[idx].gpage_list[i] = addr;
226 addr += page_size;
227 }
228}
229
230/*
231 * Moves the gigantic page addresses from the temporary list to the
232 * huge_boot_pages list.
233 */
234int alloc_bootmem_huge_page(struct hstate *hstate)
235{
236 struct huge_bootmem_page *m;
237 int idx = shift_to_mmu_psize(huge_page_shift(hstate));
238 int nr_gpages = gpage_freearray[idx].nr_gpages;
239
240 if (nr_gpages == 0)
241 return 0;
242
243#ifdef CONFIG_HIGHMEM
244 /*
245 * If gpages can be in highmem we can't use the trick of storing the
246 * data structure in the page; allocate space for this
247 */
248 m = memblock_virt_alloc(sizeof(struct huge_bootmem_page), 0);
249 m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
250#else
251 m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
252#endif
253
254 list_add(&m->list, &huge_boot_pages);
255 gpage_freearray[idx].nr_gpages = nr_gpages;
256 gpage_freearray[idx].gpage_list[nr_gpages] = 0;
257 m->hstate = hstate;
258
259 return 1;
260}
261/*
262 * Scan the command line hugepagesz= options for gigantic pages; store those in
263 * a list that we use to allocate the memory once all options are parsed.
264 */
265
266unsigned long gpage_npages[MMU_PAGE_COUNT];
267
268static int __init do_gpage_early_setup(char *param, char *val,
269 const char *unused, void *arg)
270{
271 static phys_addr_t size;
272 unsigned long npages;
273
274 /*
275 * The hugepagesz and hugepages cmdline options are interleaved. We
276 * use the size variable to keep track of whether or not this was done
277 * properly and skip over instances where it is incorrect. Other
278 * command-line parsing code will issue warnings, so we don't need to.
279 *
280 */
281 if ((strcmp(param, "default_hugepagesz") == 0) ||
282 (strcmp(param, "hugepagesz") == 0)) {
283 size = memparse(val, NULL);
284 } else if (strcmp(param, "hugepages") == 0) {
285 if (size != 0) {
286 if (sscanf(val, "%lu", &npages) <= 0)
287 npages = 0;
288 if (npages > MAX_NUMBER_GPAGES) {
289 pr_warn("MMU: %lu pages requested for page "
290#ifdef CONFIG_PHYS_ADDR_T_64BIT
291 "size %llu KB, limiting to "
292#else
293 "size %u KB, limiting to "
294#endif
295 __stringify(MAX_NUMBER_GPAGES) "\n",
296 npages, size / 1024);
297 npages = MAX_NUMBER_GPAGES;
298 }
299 gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
300 size = 0;
301 }
302 }
303 return 0;
304}
305
306
307/*
308 * This function allocates physical space for pages that are larger than the
309 * buddy allocator can handle. We want to allocate these in highmem because
310 * the amount of lowmem is limited. This means that this function MUST be
311 * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
312 * allocate to grab highmem.
313 */
314void __init reserve_hugetlb_gpages(void)
315{
316 static __initdata char cmdline[COMMAND_LINE_SIZE];
317 phys_addr_t size, base;
318 int i;
319
320 strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
321 parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
322 NULL, &do_gpage_early_setup);
323
324 /*
325 * Walk gpage list in reverse, allocating larger page sizes first.
326 * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
327 * When we reach the point in the list where pages are no longer
328 * considered gpages, we're done.
329 */
330 for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
331 if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
332 continue;
333 else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
334 break;
335
336 size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
337 base = memblock_alloc_base(size * gpage_npages[i], size,
338 MEMBLOCK_ALLOC_ANYWHERE);
339 add_gpage(base, size, gpage_npages[i]);
340 }
341}
342
343#else /* !PPC_FSL_BOOK3E */
344
345/* Build list of addresses of gigantic pages. This function is used in early
346 * boot before the buddy allocator is setup.
347 */
348void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
349{
350 if (!addr)
351 return;
352 while (number_of_pages > 0) {
353 gpage_freearray[nr_gpages] = addr;
354 nr_gpages++;
355 number_of_pages--;
356 addr += page_size;
357 }
358}
359
360/* Moves the gigantic page addresses from the temporary list to the
361 * huge_boot_pages list.
362 */
363int alloc_bootmem_huge_page(struct hstate *hstate)
364{
365 struct huge_bootmem_page *m;
366 if (nr_gpages == 0)
367 return 0;
368 m = phys_to_virt(gpage_freearray[--nr_gpages]);
369 gpage_freearray[nr_gpages] = 0;
370 list_add(&m->list, &huge_boot_pages);
371 m->hstate = hstate;
372 return 1;
373}
374#endif
375
376#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
377#define HUGEPD_FREELIST_SIZE \
378 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
379
380struct hugepd_freelist {
381 struct rcu_head rcu;
382 unsigned int index;
383 void *ptes[0];
384};
385
386static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
387
388static void hugepd_free_rcu_callback(struct rcu_head *head)
389{
390 struct hugepd_freelist *batch =
391 container_of(head, struct hugepd_freelist, rcu);
392 unsigned int i;
393
394 for (i = 0; i < batch->index; i++)
395 kmem_cache_free(hugepte_cache, batch->ptes[i]);
396
397 free_page((unsigned long)batch);
398}
399
400static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
401{
402 struct hugepd_freelist **batchp;
403
404 batchp = &get_cpu_var(hugepd_freelist_cur);
405
406 if (atomic_read(&tlb->mm->mm_users) < 2 ||
407 cpumask_equal(mm_cpumask(tlb->mm),
408 cpumask_of(smp_processor_id()))) {
409 kmem_cache_free(hugepte_cache, hugepte);
410 put_cpu_var(hugepd_freelist_cur);
411 return;
412 }
413
414 if (*batchp == NULL) {
415 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
416 (*batchp)->index = 0;
417 }
418
419 (*batchp)->ptes[(*batchp)->index++] = hugepte;
420 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
421 call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
422 *batchp = NULL;
423 }
424 put_cpu_var(hugepd_freelist_cur);
425}
426#else
427static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {}
428#endif
429
430static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
431 unsigned long start, unsigned long end,
432 unsigned long floor, unsigned long ceiling)
433{
434 pte_t *hugepte = hugepd_page(*hpdp);
435 int i;
436
437 unsigned long pdmask = ~((1UL << pdshift) - 1);
438 unsigned int num_hugepd = 1;
439 unsigned int shift = hugepd_shift(*hpdp);
440
441 /* Note: On fsl the hpdp may be the first of several */
442 if (shift > pdshift)
443 num_hugepd = 1 << (shift - pdshift);
444
445 start &= pdmask;
446 if (start < floor)
447 return;
448 if (ceiling) {
449 ceiling &= pdmask;
450 if (! ceiling)
451 return;
452 }
453 if (end - 1 > ceiling - 1)
454 return;
455
456 for (i = 0; i < num_hugepd; i++, hpdp++)
457 *hpdp = __hugepd(0);
458
459 if (shift >= pdshift)
460 hugepd_free(tlb, hugepte);
461 else
462 pgtable_free_tlb(tlb, hugepte, pdshift - shift);
463}
464
465static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
466 unsigned long addr, unsigned long end,
467 unsigned long floor, unsigned long ceiling)
468{
469 pmd_t *pmd;
470 unsigned long next;
471 unsigned long start;
472
473 start = addr;
474 do {
475 unsigned long more;
476
477 pmd = pmd_offset(pud, addr);
478 next = pmd_addr_end(addr, end);
479 if (!is_hugepd(__hugepd(pmd_val(*pmd)))) {
480 /*
481 * if it is not hugepd pointer, we should already find
482 * it cleared.
483 */
484 WARN_ON(!pmd_none_or_clear_bad(pmd));
485 continue;
486 }
487 /*
488 * Increment next by the size of the huge mapping since
489 * there may be more than one entry at this level for a
490 * single hugepage, but all of them point to
491 * the same kmem cache that holds the hugepte.
492 */
493 more = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
494 if (more > next)
495 next = more;
496
497 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
498 addr, next, floor, ceiling);
499 } while (addr = next, addr != end);
500
501 start &= PUD_MASK;
502 if (start < floor)
503 return;
504 if (ceiling) {
505 ceiling &= PUD_MASK;
506 if (!ceiling)
507 return;
508 }
509 if (end - 1 > ceiling - 1)
510 return;
511
512 pmd = pmd_offset(pud, start);
513 pud_clear(pud);
514 pmd_free_tlb(tlb, pmd, start);
515 mm_dec_nr_pmds(tlb->mm);
516}
517
518static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
519 unsigned long addr, unsigned long end,
520 unsigned long floor, unsigned long ceiling)
521{
522 pud_t *pud;
523 unsigned long next;
524 unsigned long start;
525
526 start = addr;
527 do {
528 pud = pud_offset(pgd, addr);
529 next = pud_addr_end(addr, end);
530 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
531 if (pud_none_or_clear_bad(pud))
532 continue;
533 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
534 ceiling);
535 } else {
536 unsigned long more;
537 /*
538 * Increment next by the size of the huge mapping since
539 * there may be more than one entry at this level for a
540 * single hugepage, but all of them point to
541 * the same kmem cache that holds the hugepte.
542 */
543 more = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
544 if (more > next)
545 next = more;
546
547 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
548 addr, next, floor, ceiling);
549 }
550 } while (addr = next, addr != end);
551
552 start &= PGDIR_MASK;
553 if (start < floor)
554 return;
555 if (ceiling) {
556 ceiling &= PGDIR_MASK;
557 if (!ceiling)
558 return;
559 }
560 if (end - 1 > ceiling - 1)
561 return;
562
563 pud = pud_offset(pgd, start);
564 pgd_clear(pgd);
565 pud_free_tlb(tlb, pud, start);
566}
567
568/*
569 * This function frees user-level page tables of a process.
570 */
571void hugetlb_free_pgd_range(struct mmu_gather *tlb,
572 unsigned long addr, unsigned long end,
573 unsigned long floor, unsigned long ceiling)
574{
575 pgd_t *pgd;
576 unsigned long next;
577
578 /*
579 * Because there are a number of different possible pagetable
580 * layouts for hugepage ranges, we limit knowledge of how
581 * things should be laid out to the allocation path
582 * (huge_pte_alloc(), above). Everything else works out the
583 * structure as it goes from information in the hugepd
584 * pointers. That means that we can't here use the
585 * optimization used in the normal page free_pgd_range(), of
586 * checking whether we're actually covering a large enough
587 * range to have to do anything at the top level of the walk
588 * instead of at the bottom.
589 *
590 * To make sense of this, you should probably go read the big
591 * block comment at the top of the normal free_pgd_range(),
592 * too.
593 */
594
595 do {
596 next = pgd_addr_end(addr, end);
597 pgd = pgd_offset(tlb->mm, addr);
598 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
599 if (pgd_none_or_clear_bad(pgd))
600 continue;
601 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
602 } else {
603 unsigned long more;
604 /*
605 * Increment next by the size of the huge mapping since
606 * there may be more than one entry at the pgd level
607 * for a single hugepage, but all of them point to the
608 * same kmem cache that holds the hugepte.
609 */
610 more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
611 if (more > next)
612 next = more;
613
614 free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
615 addr, next, floor, ceiling);
616 }
617 } while (addr = next, addr != end);
618}
619
620/*
621 * We are holding mmap_sem, so a parallel huge page collapse cannot run.
622 * To prevent hugepage split, disable irq.
623 */
624struct page *
625follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
626{
627 bool is_thp;
628 pte_t *ptep, pte;
629 unsigned shift;
630 unsigned long mask, flags;
631 struct page *page = ERR_PTR(-EINVAL);
632
633 local_irq_save(flags);
634 ptep = find_linux_pte_or_hugepte(mm->pgd, address, &is_thp, &shift);
635 if (!ptep)
636 goto no_page;
637 pte = READ_ONCE(*ptep);
638 /*
639 * Verify it is a huge page else bail.
640 * Transparent hugepages are handled by generic code. We can skip them
641 * here.
642 */
643 if (!shift || is_thp)
644 goto no_page;
645
646 if (!pte_present(pte)) {
647 page = NULL;
648 goto no_page;
649 }
650 mask = (1UL << shift) - 1;
651 page = pte_page(pte);
652 if (page)
653 page += (address & mask) / PAGE_SIZE;
654
655no_page:
656 local_irq_restore(flags);
657 return page;
658}
659
660struct page *
661follow_huge_pmd(struct mm_struct *mm, unsigned long address,
662 pmd_t *pmd, int write)
663{
664 BUG();
665 return NULL;
666}
667
668struct page *
669follow_huge_pud(struct mm_struct *mm, unsigned long address,
670 pud_t *pud, int write)
671{
672 BUG();
673 return NULL;
674}
675
676static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
677 unsigned long sz)
678{
679 unsigned long __boundary = (addr + sz) & ~(sz-1);
680 return (__boundary - 1 < end - 1) ? __boundary : end;
681}
682
683int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned pdshift,
684 unsigned long end, int write, struct page **pages, int *nr)
685{
686 pte_t *ptep;
687 unsigned long sz = 1UL << hugepd_shift(hugepd);
688 unsigned long next;
689
690 ptep = hugepte_offset(hugepd, addr, pdshift);
691 do {
692 next = hugepte_addr_end(addr, end, sz);
693 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
694 return 0;
695 } while (ptep++, addr = next, addr != end);
696
697 return 1;
698}
699
700#ifdef CONFIG_PPC_MM_SLICES
701unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
702 unsigned long len, unsigned long pgoff,
703 unsigned long flags)
704{
705 struct hstate *hstate = hstate_file(file);
706 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
707
708 if (radix_enabled())
709 return radix__hugetlb_get_unmapped_area(file, addr, len,
710 pgoff, flags);
711 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
712}
713#endif
714
715unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
716{
717#ifdef CONFIG_PPC_MM_SLICES
718 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
719 /* With radix we don't use slice, so derive it from vma*/
720 if (!radix_enabled())
721 return 1UL << mmu_psize_to_shift(psize);
722#endif
723 if (!is_vm_hugetlb_page(vma))
724 return PAGE_SIZE;
725
726 return huge_page_size(hstate_vma(vma));
727}
728
729static inline bool is_power_of_4(unsigned long x)
730{
731 if (is_power_of_2(x))
732 return (__ilog2(x) % 2) ? false : true;
733 return false;
734}
735
736static int __init add_huge_page_size(unsigned long long size)
737{
738 int shift = __ffs(size);
739 int mmu_psize;
740
741 /* Check that it is a page size supported by the hardware and
742 * that it fits within pagetable and slice limits. */
743 if (size <= PAGE_SIZE)
744 return -EINVAL;
745#if defined(CONFIG_PPC_FSL_BOOK3E)
746 if (!is_power_of_4(size))
747 return -EINVAL;
748#elif !defined(CONFIG_PPC_8xx)
749 if (!is_power_of_2(size) || (shift > SLICE_HIGH_SHIFT))
750 return -EINVAL;
751#endif
752
753 if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
754 return -EINVAL;
755
756 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
757
758 /* Return if huge page size has already been setup */
759 if (size_to_hstate(size))
760 return 0;
761
762 hugetlb_add_hstate(shift - PAGE_SHIFT);
763
764 return 0;
765}
766
767static int __init hugepage_setup_sz(char *str)
768{
769 unsigned long long size;
770
771 size = memparse(str, &str);
772
773 if (add_huge_page_size(size) != 0) {
774 hugetlb_bad_size();
775 pr_err("Invalid huge page size specified(%llu)\n", size);
776 }
777
778 return 1;
779}
780__setup("hugepagesz=", hugepage_setup_sz);
781
782struct kmem_cache *hugepte_cache;
783static int __init hugetlbpage_init(void)
784{
785 int psize;
786
787#if !defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_PPC_8xx)
788 if (!radix_enabled() && !mmu_has_feature(MMU_FTR_16M_PAGE))
789 return -ENODEV;
790#endif
791 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
792 unsigned shift;
793 unsigned pdshift;
794
795 if (!mmu_psize_defs[psize].shift)
796 continue;
797
798 shift = mmu_psize_to_shift(psize);
799
800 if (add_huge_page_size(1ULL << shift) < 0)
801 continue;
802
803 if (shift < HUGEPD_PUD_SHIFT)
804 pdshift = PMD_SHIFT;
805 else if (shift < HUGEPD_PGD_SHIFT)
806 pdshift = PUD_SHIFT;
807 else
808 pdshift = PGDIR_SHIFT;
809 /*
810 * if we have pdshift and shift value same, we don't
811 * use pgt cache for hugepd.
812 */
813 if (pdshift > shift)
814 pgtable_cache_add(pdshift - shift, NULL);
815#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
816 else if (!hugepte_cache) {
817 /*
818 * Create a kmem cache for hugeptes. The bottom bits in
819 * the pte have size information encoded in them, so
820 * align them to allow this
821 */
822 hugepte_cache = kmem_cache_create("hugepte-cache",
823 sizeof(pte_t),
824 HUGEPD_SHIFT_MASK + 1,
825 0, NULL);
826 if (hugepte_cache == NULL)
827 panic("%s: Unable to create kmem cache "
828 "for hugeptes\n", __func__);
829
830 }
831#endif
832 }
833
834#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
835 /* Default hpage size = 4M on FSL_BOOK3E and 512k on 8xx */
836 if (mmu_psize_defs[MMU_PAGE_4M].shift)
837 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
838 else if (mmu_psize_defs[MMU_PAGE_512K].shift)
839 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_512K].shift;
840#else
841 /* Set default large page size. Currently, we pick 16M or 1M
842 * depending on what is available
843 */
844 if (mmu_psize_defs[MMU_PAGE_16M].shift)
845 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
846 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
847 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
848 else if (mmu_psize_defs[MMU_PAGE_2M].shift)
849 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_2M].shift;
850#endif
851 return 0;
852}
853
854arch_initcall(hugetlbpage_init);
855
856void flush_dcache_icache_hugepage(struct page *page)
857{
858 int i;
859 void *start;
860
861 BUG_ON(!PageCompound(page));
862
863 for (i = 0; i < (1UL << compound_order(page)); i++) {
864 if (!PageHighMem(page)) {
865 __flush_dcache_icache(page_address(page+i));
866 } else {
867 start = kmap_atomic(page+i);
868 __flush_dcache_icache(start);
869 kunmap_atomic(start);
870 }
871 }
872}
873
874#endif /* CONFIG_HUGETLB_PAGE */
875
876/*
877 * We have 4 cases for pgds and pmds:
878 * (1) invalid (all zeroes)
879 * (2) pointer to next table, as normal; bottom 6 bits == 0
880 * (3) leaf pte for huge page _PAGE_PTE set
881 * (4) hugepd pointer, _PAGE_PTE = 0 and bits [2..6] indicate size of table
882 *
883 * So long as we atomically load page table pointers we are safe against teardown,
884 * we can follow the address down to the the page and take a ref on it.
885 * This function need to be called with interrupts disabled. We use this variant
886 * when we have MSR[EE] = 0 but the paca->soft_enabled = 1
887 */
888
889pte_t *__find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea,
890 bool *is_thp, unsigned *shift)
891{
892 pgd_t pgd, *pgdp;
893 pud_t pud, *pudp;
894 pmd_t pmd, *pmdp;
895 pte_t *ret_pte;
896 hugepd_t *hpdp = NULL;
897 unsigned pdshift = PGDIR_SHIFT;
898
899 if (shift)
900 *shift = 0;
901
902 if (is_thp)
903 *is_thp = false;
904
905 pgdp = pgdir + pgd_index(ea);
906 pgd = READ_ONCE(*pgdp);
907 /*
908 * Always operate on the local stack value. This make sure the
909 * value don't get updated by a parallel THP split/collapse,
910 * page fault or a page unmap. The return pte_t * is still not
911 * stable. So should be checked there for above conditions.
912 */
913 if (pgd_none(pgd))
914 return NULL;
915 else if (pgd_huge(pgd)) {
916 ret_pte = (pte_t *) pgdp;
917 goto out;
918 } else if (is_hugepd(__hugepd(pgd_val(pgd))))
919 hpdp = (hugepd_t *)&pgd;
920 else {
921 /*
922 * Even if we end up with an unmap, the pgtable will not
923 * be freed, because we do an rcu free and here we are
924 * irq disabled
925 */
926 pdshift = PUD_SHIFT;
927 pudp = pud_offset(&pgd, ea);
928 pud = READ_ONCE(*pudp);
929
930 if (pud_none(pud))
931 return NULL;
932 else if (pud_huge(pud)) {
933 ret_pte = (pte_t *) pudp;
934 goto out;
935 } else if (is_hugepd(__hugepd(pud_val(pud))))
936 hpdp = (hugepd_t *)&pud;
937 else {
938 pdshift = PMD_SHIFT;
939 pmdp = pmd_offset(&pud, ea);
940 pmd = READ_ONCE(*pmdp);
941 /*
942 * A hugepage collapse is captured by pmd_none, because
943 * it mark the pmd none and do a hpte invalidate.
944 */
945 if (pmd_none(pmd))
946 return NULL;
947
948 if (pmd_trans_huge(pmd)) {
949 if (is_thp)
950 *is_thp = true;
951 ret_pte = (pte_t *) pmdp;
952 goto out;
953 }
954
955 if (pmd_huge(pmd)) {
956 ret_pte = (pte_t *) pmdp;
957 goto out;
958 } else if (is_hugepd(__hugepd(pmd_val(pmd))))
959 hpdp = (hugepd_t *)&pmd;
960 else
961 return pte_offset_kernel(&pmd, ea);
962 }
963 }
964 if (!hpdp)
965 return NULL;
966
967 ret_pte = hugepte_offset(*hpdp, ea, pdshift);
968 pdshift = hugepd_shift(*hpdp);
969out:
970 if (shift)
971 *shift = pdshift;
972 return ret_pte;
973}
974EXPORT_SYMBOL_GPL(__find_linux_pte_or_hugepte);
975
976int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
977 unsigned long end, int write, struct page **pages, int *nr)
978{
979 unsigned long mask;
980 unsigned long pte_end;
981 struct page *head, *page;
982 pte_t pte;
983 int refs;
984
985 pte_end = (addr + sz) & ~(sz-1);
986 if (pte_end < end)
987 end = pte_end;
988
989 pte = READ_ONCE(*ptep);
990 mask = _PAGE_PRESENT | _PAGE_READ;
991
992 /*
993 * On some CPUs like the 8xx, _PAGE_RW hence _PAGE_WRITE is defined
994 * as 0 and _PAGE_RO has to be set when a page is not writable
995 */
996 if (write)
997 mask |= _PAGE_WRITE;
998 else
999 mask |= _PAGE_RO;
1000
1001 if ((pte_val(pte) & mask) != mask)
1002 return 0;
1003
1004 /* hugepages are never "special" */
1005 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1006
1007 refs = 0;
1008 head = pte_page(pte);
1009
1010 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
1011 do {
1012 VM_BUG_ON(compound_head(page) != head);
1013 pages[*nr] = page;
1014 (*nr)++;
1015 page++;
1016 refs++;
1017 } while (addr += PAGE_SIZE, addr != end);
1018
1019 if (!page_cache_add_speculative(head, refs)) {
1020 *nr -= refs;
1021 return 0;
1022 }
1023
1024 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1025 /* Could be optimized better */
1026 *nr -= refs;
1027 while (refs--)
1028 put_page(head);
1029 return 0;
1030 }
1031
1032 return 1;
1033}