Loading...
1/*
2 * Cache maintenance
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/errno.h>
21#include <linux/linkage.h>
22#include <linux/init.h>
23#include <asm/assembler.h>
24#include <asm/cpufeature.h>
25#include <asm/alternative.h>
26
27#include "proc-macros.S"
28
29/*
30 * flush_icache_range(start,end)
31 *
32 * Ensure that the I and D caches are coherent within specified region.
33 * This is typically used when code has been written to a memory region,
34 * and will be executed.
35 *
36 * - start - virtual start address of region
37 * - end - virtual end address of region
38 */
39ENTRY(flush_icache_range)
40 /* FALLTHROUGH */
41
42/*
43 * __flush_cache_user_range(start,end)
44 *
45 * Ensure that the I and D caches are coherent within specified region.
46 * This is typically used when code has been written to a memory region,
47 * and will be executed.
48 *
49 * - start - virtual start address of region
50 * - end - virtual end address of region
51 */
52ENTRY(__flush_cache_user_range)
53 dcache_line_size x2, x3
54 sub x3, x2, #1
55 bic x4, x0, x3
561:
57USER(9f, dc cvau, x4 ) // clean D line to PoU
58 add x4, x4, x2
59 cmp x4, x1
60 b.lo 1b
61 dsb ish
62
63 icache_line_size x2, x3
64 sub x3, x2, #1
65 bic x4, x0, x3
661:
67USER(9f, ic ivau, x4 ) // invalidate I line PoU
68 add x4, x4, x2
69 cmp x4, x1
70 b.lo 1b
71 dsb ish
72 isb
73 mov x0, #0
74 ret
759:
76 mov x0, #-EFAULT
77 ret
78ENDPROC(flush_icache_range)
79ENDPROC(__flush_cache_user_range)
80
81/*
82 * __flush_dcache_area(kaddr, size)
83 *
84 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
85 * are cleaned and invalidated to the PoC.
86 *
87 * - kaddr - kernel address
88 * - size - size in question
89 */
90ENTRY(__flush_dcache_area)
91 dcache_by_line_op civac, sy, x0, x1, x2, x3
92 ret
93ENDPIPROC(__flush_dcache_area)
94
95/*
96 * __clean_dcache_area_pou(kaddr, size)
97 *
98 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
99 * are cleaned to the PoU.
100 *
101 * - kaddr - kernel address
102 * - size - size in question
103 */
104ENTRY(__clean_dcache_area_pou)
105 dcache_by_line_op cvau, ish, x0, x1, x2, x3
106 ret
107ENDPROC(__clean_dcache_area_pou)
108
109/*
110 * __inval_cache_range(start, end)
111 * - start - start address of region
112 * - end - end address of region
113 */
114ENTRY(__inval_cache_range)
115 /* FALLTHROUGH */
116
117/*
118 * __dma_inv_range(start, end)
119 * - start - virtual start address of region
120 * - end - virtual end address of region
121 */
122__dma_inv_range:
123 dcache_line_size x2, x3
124 sub x3, x2, #1
125 tst x1, x3 // end cache line aligned?
126 bic x1, x1, x3
127 b.eq 1f
128 dc civac, x1 // clean & invalidate D / U line
1291: tst x0, x3 // start cache line aligned?
130 bic x0, x0, x3
131 b.eq 2f
132 dc civac, x0 // clean & invalidate D / U line
133 b 3f
1342: dc ivac, x0 // invalidate D / U line
1353: add x0, x0, x2
136 cmp x0, x1
137 b.lo 2b
138 dsb sy
139 ret
140ENDPIPROC(__inval_cache_range)
141ENDPROC(__dma_inv_range)
142
143/*
144 * __dma_clean_range(start, end)
145 * - start - virtual start address of region
146 * - end - virtual end address of region
147 */
148__dma_clean_range:
149 dcache_line_size x2, x3
150 sub x3, x2, #1
151 bic x0, x0, x3
1521:
153alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
154 dc cvac, x0
155alternative_else
156 dc civac, x0
157alternative_endif
158 add x0, x0, x2
159 cmp x0, x1
160 b.lo 1b
161 dsb sy
162 ret
163ENDPROC(__dma_clean_range)
164
165/*
166 * __dma_flush_range(start, end)
167 * - start - virtual start address of region
168 * - end - virtual end address of region
169 */
170ENTRY(__dma_flush_range)
171 dcache_line_size x2, x3
172 sub x3, x2, #1
173 bic x0, x0, x3
1741: dc civac, x0 // clean & invalidate D / U line
175 add x0, x0, x2
176 cmp x0, x1
177 b.lo 1b
178 dsb sy
179 ret
180ENDPIPROC(__dma_flush_range)
181
182/*
183 * __dma_map_area(start, size, dir)
184 * - start - kernel virtual start address
185 * - size - size of region
186 * - dir - DMA direction
187 */
188ENTRY(__dma_map_area)
189 add x1, x1, x0
190 cmp w2, #DMA_FROM_DEVICE
191 b.eq __dma_inv_range
192 b __dma_clean_range
193ENDPIPROC(__dma_map_area)
194
195/*
196 * __dma_unmap_area(start, size, dir)
197 * - start - kernel virtual start address
198 * - size - size of region
199 * - dir - DMA direction
200 */
201ENTRY(__dma_unmap_area)
202 add x1, x1, x0
203 cmp w2, #DMA_TO_DEVICE
204 b.ne __dma_inv_range
205 ret
206ENDPIPROC(__dma_unmap_area)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Cache maintenance
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9#include <linux/errno.h>
10#include <linux/linkage.h>
11#include <linux/init.h>
12#include <asm/assembler.h>
13#include <asm/cpufeature.h>
14#include <asm/alternative.h>
15#include <asm/asm-uaccess.h>
16
17/*
18 * caches_clean_inval_pou_macro(start,end) [fixup]
19 *
20 * Ensure that the I and D caches are coherent within specified region.
21 * This is typically used when code has been written to a memory region,
22 * and will be executed.
23 *
24 * - start - virtual start address of region
25 * - end - virtual end address of region
26 * - fixup - optional label to branch to on user fault
27 */
28.macro caches_clean_inval_pou_macro, fixup
29alternative_if ARM64_HAS_CACHE_IDC
30 dsb ishst
31 b .Ldc_skip_\@
32alternative_else_nop_endif
33 mov x2, x0
34 mov x3, x1
35 dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup
36.Ldc_skip_\@:
37alternative_if ARM64_HAS_CACHE_DIC
38 isb
39 b .Lic_skip_\@
40alternative_else_nop_endif
41 invalidate_icache_by_line x0, x1, x2, x3, \fixup
42.Lic_skip_\@:
43.endm
44
45/*
46 * caches_clean_inval_pou(start,end)
47 *
48 * Ensure that the I and D caches are coherent within specified region.
49 * This is typically used when code has been written to a memory region,
50 * and will be executed.
51 *
52 * - start - virtual start address of region
53 * - end - virtual end address of region
54 */
55SYM_FUNC_START(caches_clean_inval_pou)
56 caches_clean_inval_pou_macro
57 ret
58SYM_FUNC_END(caches_clean_inval_pou)
59
60/*
61 * caches_clean_inval_user_pou(start,end)
62 *
63 * Ensure that the I and D caches are coherent within specified region.
64 * This is typically used when code has been written to a memory region,
65 * and will be executed.
66 *
67 * - start - virtual start address of region
68 * - end - virtual end address of region
69 */
70SYM_FUNC_START(caches_clean_inval_user_pou)
71 uaccess_ttbr0_enable x2, x3, x4
72
73 caches_clean_inval_pou_macro 2f
74 mov x0, xzr
751:
76 uaccess_ttbr0_disable x1, x2
77 ret
782:
79 mov x0, #-EFAULT
80 b 1b
81SYM_FUNC_END(caches_clean_inval_user_pou)
82
83/*
84 * icache_inval_pou(start,end)
85 *
86 * Ensure that the I cache is invalid within specified region.
87 *
88 * - start - virtual start address of region
89 * - end - virtual end address of region
90 */
91SYM_FUNC_START(icache_inval_pou)
92alternative_if ARM64_HAS_CACHE_DIC
93 isb
94 ret
95alternative_else_nop_endif
96
97 invalidate_icache_by_line x0, x1, x2, x3
98 ret
99SYM_FUNC_END(icache_inval_pou)
100
101/*
102 * dcache_clean_inval_poc(start, end)
103 *
104 * Ensure that any D-cache lines for the interval [start, end)
105 * are cleaned and invalidated to the PoC.
106 *
107 * - start - virtual start address of region
108 * - end - virtual end address of region
109 */
110SYM_FUNC_START_PI(dcache_clean_inval_poc)
111 dcache_by_line_op civac, sy, x0, x1, x2, x3
112 ret
113SYM_FUNC_END_PI(dcache_clean_inval_poc)
114
115/*
116 * dcache_clean_pou(start, end)
117 *
118 * Ensure that any D-cache lines for the interval [start, end)
119 * are cleaned to the PoU.
120 *
121 * - start - virtual start address of region
122 * - end - virtual end address of region
123 */
124SYM_FUNC_START(dcache_clean_pou)
125alternative_if ARM64_HAS_CACHE_IDC
126 dsb ishst
127 ret
128alternative_else_nop_endif
129 dcache_by_line_op cvau, ish, x0, x1, x2, x3
130 ret
131SYM_FUNC_END(dcache_clean_pou)
132
133/*
134 * dcache_inval_poc(start, end)
135 *
136 * Ensure that any D-cache lines for the interval [start, end)
137 * are invalidated. Any partial lines at the ends of the interval are
138 * also cleaned to PoC to prevent data loss.
139 *
140 * - start - kernel start address of region
141 * - end - kernel end address of region
142 */
143SYM_FUNC_START_LOCAL(__dma_inv_area)
144SYM_FUNC_START_PI(dcache_inval_poc)
145 /* FALLTHROUGH */
146
147/*
148 * __dma_inv_area(start, end)
149 * - start - virtual start address of region
150 * - end - virtual end address of region
151 */
152 dcache_line_size x2, x3
153 sub x3, x2, #1
154 tst x1, x3 // end cache line aligned?
155 bic x1, x1, x3
156 b.eq 1f
157 dc civac, x1 // clean & invalidate D / U line
1581: tst x0, x3 // start cache line aligned?
159 bic x0, x0, x3
160 b.eq 2f
161 dc civac, x0 // clean & invalidate D / U line
162 b 3f
1632: dc ivac, x0 // invalidate D / U line
1643: add x0, x0, x2
165 cmp x0, x1
166 b.lo 2b
167 dsb sy
168 ret
169SYM_FUNC_END_PI(dcache_inval_poc)
170SYM_FUNC_END(__dma_inv_area)
171
172/*
173 * dcache_clean_poc(start, end)
174 *
175 * Ensure that any D-cache lines for the interval [start, end)
176 * are cleaned to the PoC.
177 *
178 * - start - virtual start address of region
179 * - end - virtual end address of region
180 */
181SYM_FUNC_START_LOCAL(__dma_clean_area)
182SYM_FUNC_START_PI(dcache_clean_poc)
183 /* FALLTHROUGH */
184
185/*
186 * __dma_clean_area(start, end)
187 * - start - virtual start address of region
188 * - end - virtual end address of region
189 */
190 dcache_by_line_op cvac, sy, x0, x1, x2, x3
191 ret
192SYM_FUNC_END_PI(dcache_clean_poc)
193SYM_FUNC_END(__dma_clean_area)
194
195/*
196 * dcache_clean_pop(start, end)
197 *
198 * Ensure that any D-cache lines for the interval [start, end)
199 * are cleaned to the PoP.
200 *
201 * - start - virtual start address of region
202 * - end - virtual end address of region
203 */
204SYM_FUNC_START_PI(dcache_clean_pop)
205 alternative_if_not ARM64_HAS_DCPOP
206 b dcache_clean_poc
207 alternative_else_nop_endif
208 dcache_by_line_op cvap, sy, x0, x1, x2, x3
209 ret
210SYM_FUNC_END_PI(dcache_clean_pop)
211
212/*
213 * __dma_flush_area(start, size)
214 *
215 * clean & invalidate D / U line
216 *
217 * - start - virtual start address of region
218 * - size - size in question
219 */
220SYM_FUNC_START_PI(__dma_flush_area)
221 add x1, x0, x1
222 dcache_by_line_op civac, sy, x0, x1, x2, x3
223 ret
224SYM_FUNC_END_PI(__dma_flush_area)
225
226/*
227 * __dma_map_area(start, size, dir)
228 * - start - kernel virtual start address
229 * - size - size of region
230 * - dir - DMA direction
231 */
232SYM_FUNC_START_PI(__dma_map_area)
233 add x1, x0, x1
234 cmp w2, #DMA_FROM_DEVICE
235 b.eq __dma_inv_area
236 b __dma_clean_area
237SYM_FUNC_END_PI(__dma_map_area)
238
239/*
240 * __dma_unmap_area(start, size, dir)
241 * - start - kernel virtual start address
242 * - size - size of region
243 * - dir - DMA direction
244 */
245SYM_FUNC_START_PI(__dma_unmap_area)
246 add x1, x0, x1
247 cmp w2, #DMA_TO_DEVICE
248 b.ne __dma_inv_area
249 ret
250SYM_FUNC_END_PI(__dma_unmap_area)