Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2015 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#include <linux/memblock.h>
7#include <linux/export.h>
8#include <linux/highmem.h>
9#include <linux/pgtable.h>
10#include <asm/processor.h>
11#include <asm/pgalloc.h>
12#include <asm/tlbflush.h>
13
14/*
15 * HIGHMEM API:
16 *
17 * kmap() API provides sleep semantics hence referred to as "permanent maps"
18 * It allows mapping LAST_PKMAP pages, using @last_pkmap_nr as the cursor
19 * for book-keeping
20 *
21 * kmap_atomic() can't sleep (calls pagefault_disable()), thus it provides
22 * shortlived ala "temporary mappings" which historically were implemented as
23 * fixmaps (compile time addr etc). Their book-keeping is done per cpu.
24 *
25 * Both these facts combined (preemption disabled and per-cpu allocation)
26 * means the total number of concurrent fixmaps will be limited to max
27 * such allocations in a single control path. Thus KM_TYPE_NR (another
28 * historic relic) is a small'ish number which caps max percpu fixmaps
29 *
30 * ARC HIGHMEM Details
31 *
32 * - the kernel vaddr space from 0x7z to 0x8z (currently used by vmalloc/module)
33 * is now shared between vmalloc and kmap (non overlapping though)
34 *
35 * - Both fixmap/pkmap use a dedicated page table each, hooked up to swapper PGD
36 * This means each only has 1 PGDIR_SIZE worth of kvaddr mappings, which means
37 * 2M of kvaddr space for typical config (8K page and 11:8:13 traversal split)
38 *
39 * - fixmap anyhow needs a limited number of mappings. So 2M kvaddr == 256 PTE
40 * slots across NR_CPUS would be more than sufficient (generic code defines
41 * KM_TYPE_NR as 20).
42 *
43 * - pkmap being preemptible, in theory could do with more than 256 concurrent
44 * mappings. However, generic pkmap code: map_new_virtual(), doesn't traverse
45 * the PGD and only works with a single page table @pkmap_page_table, hence
46 * sets the limit
47 */
48
49extern pte_t * pkmap_page_table;
50static pte_t * fixmap_page_table;
51
52void *kmap_atomic_high_prot(struct page *page, pgprot_t prot)
53{
54 int idx, cpu_idx;
55 unsigned long vaddr;
56
57 cpu_idx = kmap_atomic_idx_push();
58 idx = cpu_idx + KM_TYPE_NR * smp_processor_id();
59 vaddr = FIXMAP_ADDR(idx);
60
61 set_pte_at(&init_mm, vaddr, fixmap_page_table + idx,
62 mk_pte(page, prot));
63
64 return (void *)vaddr;
65}
66EXPORT_SYMBOL(kmap_atomic_high_prot);
67
68void kunmap_atomic_high(void *kv)
69{
70 unsigned long kvaddr = (unsigned long)kv;
71
72 if (kvaddr >= FIXMAP_BASE && kvaddr < (FIXMAP_BASE + FIXMAP_SIZE)) {
73
74 /*
75 * Because preemption is disabled, this vaddr can be associated
76 * with the current allocated index.
77 * But in case of multiple live kmap_atomic(), it still relies on
78 * callers to unmap in right order.
79 */
80 int cpu_idx = kmap_atomic_idx();
81 int idx = cpu_idx + KM_TYPE_NR * smp_processor_id();
82
83 WARN_ON(kvaddr != FIXMAP_ADDR(idx));
84
85 pte_clear(&init_mm, kvaddr, fixmap_page_table + idx);
86 local_flush_tlb_kernel_range(kvaddr, kvaddr + PAGE_SIZE);
87
88 kmap_atomic_idx_pop();
89 }
90}
91EXPORT_SYMBOL(kunmap_atomic_high);
92
93static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr)
94{
95 pmd_t *pmd_k = pmd_off_k(kvaddr);
96 pte_t *pte_k;
97
98 pte_k = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
99 if (!pte_k)
100 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
101 __func__, PAGE_SIZE, PAGE_SIZE);
102
103 pmd_populate_kernel(&init_mm, pmd_k, pte_k);
104 return pte_k;
105}
106
107void __init kmap_init(void)
108{
109 /* Due to recursive include hell, we can't do this in processor.h */
110 BUILD_BUG_ON(PAGE_OFFSET < (VMALLOC_END + FIXMAP_SIZE + PKMAP_SIZE));
111
112 BUILD_BUG_ON(KM_TYPE_NR > PTRS_PER_PTE);
113 pkmap_page_table = alloc_kmap_pgtable(PKMAP_BASE);
114
115 BUILD_BUG_ON(LAST_PKMAP > PTRS_PER_PTE);
116 fixmap_page_table = alloc_kmap_pgtable(FIXMAP_BASE);
117}
1/*
2 * Copyright (C) 2015 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#include <linux/bootmem.h>
11#include <linux/export.h>
12#include <linux/highmem.h>
13#include <asm/processor.h>
14#include <asm/pgtable.h>
15#include <asm/pgalloc.h>
16#include <asm/tlbflush.h>
17
18/*
19 * HIGHMEM API:
20 *
21 * kmap() API provides sleep semantics hence referred to as "permanent maps"
22 * It allows mapping LAST_PKMAP pages, using @last_pkmap_nr as the cursor
23 * for book-keeping
24 *
25 * kmap_atomic() can't sleep (calls pagefault_disable()), thus it provides
26 * shortlived ala "temporary mappings" which historically were implemented as
27 * fixmaps (compile time addr etc). Their book-keeping is done per cpu.
28 *
29 * Both these facts combined (preemption disabled and per-cpu allocation)
30 * means the total number of concurrent fixmaps will be limited to max
31 * such allocations in a single control path. Thus KM_TYPE_NR (another
32 * historic relic) is a small'ish number which caps max percpu fixmaps
33 *
34 * ARC HIGHMEM Details
35 *
36 * - the kernel vaddr space from 0x7z to 0x8z (currently used by vmalloc/module)
37 * is now shared between vmalloc and kmap (non overlapping though)
38 *
39 * - Both fixmap/pkmap use a dedicated page table each, hooked up to swapper PGD
40 * This means each only has 1 PGDIR_SIZE worth of kvaddr mappings, which means
41 * 2M of kvaddr space for typical config (8K page and 11:8:13 traversal split)
42 *
43 * - fixmap anyhow needs a limited number of mappings. So 2M kvaddr == 256 PTE
44 * slots across NR_CPUS would be more than sufficient (generic code defines
45 * KM_TYPE_NR as 20).
46 *
47 * - pkmap being preemptible, in theory could do with more than 256 concurrent
48 * mappings. However, generic pkmap code: map_new_virtual(), doesn't traverse
49 * the PGD and only works with a single page table @pkmap_page_table, hence
50 * sets the limit
51 */
52
53extern pte_t * pkmap_page_table;
54static pte_t * fixmap_page_table;
55
56void *kmap(struct page *page)
57{
58 BUG_ON(in_interrupt());
59 if (!PageHighMem(page))
60 return page_address(page);
61
62 return kmap_high(page);
63}
64EXPORT_SYMBOL(kmap);
65
66void *kmap_atomic(struct page *page)
67{
68 int idx, cpu_idx;
69 unsigned long vaddr;
70
71 preempt_disable();
72 pagefault_disable();
73 if (!PageHighMem(page))
74 return page_address(page);
75
76 cpu_idx = kmap_atomic_idx_push();
77 idx = cpu_idx + KM_TYPE_NR * smp_processor_id();
78 vaddr = FIXMAP_ADDR(idx);
79
80 set_pte_at(&init_mm, vaddr, fixmap_page_table + idx,
81 mk_pte(page, kmap_prot));
82
83 return (void *)vaddr;
84}
85EXPORT_SYMBOL(kmap_atomic);
86
87void __kunmap_atomic(void *kv)
88{
89 unsigned long kvaddr = (unsigned long)kv;
90
91 if (kvaddr >= FIXMAP_BASE && kvaddr < (FIXMAP_BASE + FIXMAP_SIZE)) {
92
93 /*
94 * Because preemption is disabled, this vaddr can be associated
95 * with the current allocated index.
96 * But in case of multiple live kmap_atomic(), it still relies on
97 * callers to unmap in right order.
98 */
99 int cpu_idx = kmap_atomic_idx();
100 int idx = cpu_idx + KM_TYPE_NR * smp_processor_id();
101
102 WARN_ON(kvaddr != FIXMAP_ADDR(idx));
103
104 pte_clear(&init_mm, kvaddr, fixmap_page_table + idx);
105 local_flush_tlb_kernel_range(kvaddr, kvaddr + PAGE_SIZE);
106
107 kmap_atomic_idx_pop();
108 }
109
110 pagefault_enable();
111 preempt_enable();
112}
113EXPORT_SYMBOL(__kunmap_atomic);
114
115static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr)
116{
117 pgd_t *pgd_k;
118 pud_t *pud_k;
119 pmd_t *pmd_k;
120 pte_t *pte_k;
121
122 pgd_k = pgd_offset_k(kvaddr);
123 pud_k = pud_offset(pgd_k, kvaddr);
124 pmd_k = pmd_offset(pud_k, kvaddr);
125
126 pte_k = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
127 pmd_populate_kernel(&init_mm, pmd_k, pte_k);
128 return pte_k;
129}
130
131void __init kmap_init(void)
132{
133 /* Due to recursive include hell, we can't do this in processor.h */
134 BUILD_BUG_ON(PAGE_OFFSET < (VMALLOC_END + FIXMAP_SIZE + PKMAP_SIZE));
135
136 BUILD_BUG_ON(KM_TYPE_NR > PTRS_PER_PTE);
137 pkmap_page_table = alloc_kmap_pgtable(PKMAP_BASE);
138
139 BUILD_BUG_ON(LAST_PKMAP > PTRS_PER_PTE);
140 fixmap_page_table = alloc_kmap_pgtable(FIXMAP_BASE);
141}