Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/mm/cache-v6.S
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 *
7 * This is the "shell" of the ARMv6 processor support.
8 */
9#include <linux/linkage.h>
10#include <linux/init.h>
11#include <asm/assembler.h>
12#include <asm/errno.h>
13#include <asm/unwind.h>
14
15#include "proc-macros.S"
16
17#define HARVARD_CACHE
18#define CACHE_LINE_SIZE 32
19#define D_CACHE_LINE_SIZE 32
20#define BTB_FLUSH_SIZE 8
21
22/*
23 * v6_flush_icache_all()
24 *
25 * Flush the whole I-cache.
26 *
27 * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail.
28 * This erratum is present in 1136, 1156 and 1176. It does not affect the
29 * MPCore.
30 *
31 * Registers:
32 * r0 - set to 0
33 * r1 - corrupted
34 */
35ENTRY(v6_flush_icache_all)
36 mov r0, #0
37#ifdef CONFIG_ARM_ERRATA_411920
38 mrs r1, cpsr
39 cpsid ifa @ disable interrupts
40 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
41 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
42 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
43 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
44 msr cpsr_cx, r1 @ restore interrupts
45 .rept 11 @ ARM Ltd recommends at least
46 nop @ 11 NOPs
47 .endr
48#else
49 mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache
50#endif
51 ret lr
52ENDPROC(v6_flush_icache_all)
53
54/*
55 * v6_flush_cache_all()
56 *
57 * Flush the entire cache.
58 *
59 * It is assumed that:
60 */
61ENTRY(v6_flush_kern_cache_all)
62 mov r0, #0
63#ifdef HARVARD_CACHE
64 mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate
65#ifndef CONFIG_ARM_ERRATA_411920
66 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
67#else
68 b v6_flush_icache_all
69#endif
70#else
71 mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate
72#endif
73 ret lr
74
75/*
76 * v6_flush_cache_all()
77 *
78 * Flush all TLB entries in a particular address space
79 *
80 * - mm - mm_struct describing address space
81 */
82ENTRY(v6_flush_user_cache_all)
83 /*FALLTHROUGH*/
84
85/*
86 * v6_flush_cache_range(start, end, flags)
87 *
88 * Flush a range of TLB entries in the specified address space.
89 *
90 * - start - start address (may not be aligned)
91 * - end - end address (exclusive, may not be aligned)
92 * - flags - vm_area_struct flags describing address space
93 *
94 * It is assumed that:
95 * - we have a VIPT cache.
96 */
97ENTRY(v6_flush_user_cache_range)
98 ret lr
99
100/*
101 * v6_coherent_kern_range(start,end)
102 *
103 * Ensure that the I and D caches are coherent within specified
104 * region. This is typically used when code has been written to
105 * a memory region, and will be executed.
106 *
107 * - start - virtual start address of region
108 * - end - virtual end address of region
109 *
110 * It is assumed that:
111 * - the Icache does not read data from the write buffer
112 */
113ENTRY(v6_coherent_kern_range)
114 /* FALLTHROUGH */
115
116/*
117 * v6_coherent_user_range(start,end)
118 *
119 * Ensure that the I and D caches are coherent within specified
120 * region. This is typically used when code has been written to
121 * a memory region, and will be executed.
122 *
123 * - start - virtual start address of region
124 * - end - virtual end address of region
125 *
126 * It is assumed that:
127 * - the Icache does not read data from the write buffer
128 */
129ENTRY(v6_coherent_user_range)
130 UNWIND(.fnstart )
131#ifdef HARVARD_CACHE
132 bic r0, r0, #CACHE_LINE_SIZE - 1
1331:
134 USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line
135 add r0, r0, #CACHE_LINE_SIZE
136 cmp r0, r1
137 blo 1b
138#endif
139 mov r0, #0
140#ifdef HARVARD_CACHE
141 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
142#ifndef CONFIG_ARM_ERRATA_411920
143 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
144#else
145 b v6_flush_icache_all
146#endif
147#else
148 mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB
149#endif
150 ret lr
151
152/*
153 * Fault handling for the cache operation above. If the virtual address in r0
154 * isn't mapped, fail with -EFAULT.
155 */
1569001:
157 mov r0, #-EFAULT
158 ret lr
159 UNWIND(.fnend )
160ENDPROC(v6_coherent_user_range)
161ENDPROC(v6_coherent_kern_range)
162
163/*
164 * v6_flush_kern_dcache_area(void *addr, size_t size)
165 *
166 * Ensure that the data held in the page kaddr is written back
167 * to the page in question.
168 *
169 * - addr - kernel address
170 * - size - region size
171 */
172ENTRY(v6_flush_kern_dcache_area)
173 add r1, r0, r1
174 bic r0, r0, #D_CACHE_LINE_SIZE - 1
1751:
176#ifdef HARVARD_CACHE
177 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
178#else
179 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line
180#endif
181 add r0, r0, #D_CACHE_LINE_SIZE
182 cmp r0, r1
183 blo 1b
184#ifdef HARVARD_CACHE
185 mov r0, #0
186 mcr p15, 0, r0, c7, c10, 4
187#endif
188 ret lr
189
190
191/*
192 * v6_dma_inv_range(start,end)
193 *
194 * Invalidate the data cache within the specified region; we will
195 * be performing a DMA operation in this region and we want to
196 * purge old data in the cache.
197 *
198 * - start - virtual start address of region
199 * - end - virtual end address of region
200 */
201v6_dma_inv_range:
202#ifdef CONFIG_DMA_CACHE_RWFO
203 ldrb r2, [r0] @ read for ownership
204 strb r2, [r0] @ write for ownership
205#endif
206 tst r0, #D_CACHE_LINE_SIZE - 1
207 bic r0, r0, #D_CACHE_LINE_SIZE - 1
208#ifdef HARVARD_CACHE
209 mcrne p15, 0, r0, c7, c10, 1 @ clean D line
210#else
211 mcrne p15, 0, r0, c7, c11, 1 @ clean unified line
212#endif
213 tst r1, #D_CACHE_LINE_SIZE - 1
214#ifdef CONFIG_DMA_CACHE_RWFO
215 ldrbne r2, [r1, #-1] @ read for ownership
216 strbne r2, [r1, #-1] @ write for ownership
217#endif
218 bic r1, r1, #D_CACHE_LINE_SIZE - 1
219#ifdef HARVARD_CACHE
220 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line
221#else
222 mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line
223#endif
2241:
225#ifdef HARVARD_CACHE
226 mcr p15, 0, r0, c7, c6, 1 @ invalidate D line
227#else
228 mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line
229#endif
230 add r0, r0, #D_CACHE_LINE_SIZE
231 cmp r0, r1
232#ifdef CONFIG_DMA_CACHE_RWFO
233 ldrlo r2, [r0] @ read for ownership
234 strlo r2, [r0] @ write for ownership
235#endif
236 blo 1b
237 mov r0, #0
238 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
239 ret lr
240
241/*
242 * v6_dma_clean_range(start,end)
243 * - start - virtual start address of region
244 * - end - virtual end address of region
245 */
246v6_dma_clean_range:
247 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2481:
249#ifdef CONFIG_DMA_CACHE_RWFO
250 ldr r2, [r0] @ read for ownership
251#endif
252#ifdef HARVARD_CACHE
253 mcr p15, 0, r0, c7, c10, 1 @ clean D line
254#else
255 mcr p15, 0, r0, c7, c11, 1 @ clean unified line
256#endif
257 add r0, r0, #D_CACHE_LINE_SIZE
258 cmp r0, r1
259 blo 1b
260 mov r0, #0
261 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
262 ret lr
263
264/*
265 * v6_dma_flush_range(start,end)
266 * - start - virtual start address of region
267 * - end - virtual end address of region
268 */
269ENTRY(v6_dma_flush_range)
270#ifdef CONFIG_DMA_CACHE_RWFO
271 ldrb r2, [r0] @ read for ownership
272 strb r2, [r0] @ write for ownership
273#endif
274 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2751:
276#ifdef HARVARD_CACHE
277 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
278#else
279 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line
280#endif
281 add r0, r0, #D_CACHE_LINE_SIZE
282 cmp r0, r1
283#ifdef CONFIG_DMA_CACHE_RWFO
284 ldrblo r2, [r0] @ read for ownership
285 strblo r2, [r0] @ write for ownership
286#endif
287 blo 1b
288 mov r0, #0
289 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
290 ret lr
291
292/*
293 * dma_map_area(start, size, dir)
294 * - start - kernel virtual start address
295 * - size - size of region
296 * - dir - DMA direction
297 */
298ENTRY(v6_dma_map_area)
299 add r1, r1, r0
300 teq r2, #DMA_FROM_DEVICE
301 beq v6_dma_inv_range
302#ifndef CONFIG_DMA_CACHE_RWFO
303 b v6_dma_clean_range
304#else
305 teq r2, #DMA_TO_DEVICE
306 beq v6_dma_clean_range
307 b v6_dma_flush_range
308#endif
309ENDPROC(v6_dma_map_area)
310
311/*
312 * dma_unmap_area(start, size, dir)
313 * - start - kernel virtual start address
314 * - size - size of region
315 * - dir - DMA direction
316 */
317ENTRY(v6_dma_unmap_area)
318#ifndef CONFIG_DMA_CACHE_RWFO
319 add r1, r1, r0
320 teq r2, #DMA_TO_DEVICE
321 bne v6_dma_inv_range
322#endif
323 ret lr
324ENDPROC(v6_dma_unmap_area)
325
326 .globl v6_flush_kern_cache_louis
327 .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all
328
329 __INITDATA
330
331 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
332 define_cache_functions v6
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/mm/cache-v6.S
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 *
7 * This is the "shell" of the ARMv6 processor support.
8 */
9#include <linux/linkage.h>
10#include <linux/init.h>
11#include <asm/assembler.h>
12#include <asm/errno.h>
13#include <asm/unwind.h>
14
15#include "proc-macros.S"
16
17#define HARVARD_CACHE
18#define CACHE_LINE_SIZE 32
19#define D_CACHE_LINE_SIZE 32
20#define BTB_FLUSH_SIZE 8
21
22.arch armv6
23
24/*
25 * v6_flush_icache_all()
26 *
27 * Flush the whole I-cache.
28 *
29 * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail.
30 * This erratum is present in 1136, 1156 and 1176. It does not affect the
31 * MPCore.
32 *
33 * Registers:
34 * r0 - set to 0
35 * r1 - corrupted
36 */
37ENTRY(v6_flush_icache_all)
38 mov r0, #0
39#ifdef CONFIG_ARM_ERRATA_411920
40 mrs r1, cpsr
41 cpsid ifa @ disable interrupts
42 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
43 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
44 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
45 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
46 msr cpsr_cx, r1 @ restore interrupts
47 .rept 11 @ ARM Ltd recommends at least
48 nop @ 11 NOPs
49 .endr
50#else
51 mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache
52#endif
53 ret lr
54ENDPROC(v6_flush_icache_all)
55
56/*
57 * v6_flush_cache_all()
58 *
59 * Flush the entire cache.
60 *
61 * It is assumed that:
62 */
63ENTRY(v6_flush_kern_cache_all)
64 mov r0, #0
65#ifdef HARVARD_CACHE
66 mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate
67#ifndef CONFIG_ARM_ERRATA_411920
68 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
69#else
70 b v6_flush_icache_all
71#endif
72#else
73 mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate
74#endif
75 ret lr
76
77/*
78 * v6_flush_cache_all()
79 *
80 * Flush all TLB entries in a particular address space
81 *
82 * - mm - mm_struct describing address space
83 */
84ENTRY(v6_flush_user_cache_all)
85 /*FALLTHROUGH*/
86
87/*
88 * v6_flush_cache_range(start, end, flags)
89 *
90 * Flush a range of TLB entries in the specified address space.
91 *
92 * - start - start address (may not be aligned)
93 * - end - end address (exclusive, may not be aligned)
94 * - flags - vm_area_struct flags describing address space
95 *
96 * It is assumed that:
97 * - we have a VIPT cache.
98 */
99ENTRY(v6_flush_user_cache_range)
100 ret lr
101
102/*
103 * v6_coherent_kern_range(start,end)
104 *
105 * Ensure that the I and D caches are coherent within specified
106 * region. This is typically used when code has been written to
107 * a memory region, and will be executed.
108 *
109 * - start - virtual start address of region
110 * - end - virtual end address of region
111 *
112 * It is assumed that:
113 * - the Icache does not read data from the write buffer
114 */
115ENTRY(v6_coherent_kern_range)
116 /* FALLTHROUGH */
117
118/*
119 * v6_coherent_user_range(start,end)
120 *
121 * Ensure that the I and D caches are coherent within specified
122 * region. This is typically used when code has been written to
123 * a memory region, and will be executed.
124 *
125 * - start - virtual start address of region
126 * - end - virtual end address of region
127 *
128 * It is assumed that:
129 * - the Icache does not read data from the write buffer
130 */
131ENTRY(v6_coherent_user_range)
132 UNWIND(.fnstart )
133#ifdef HARVARD_CACHE
134 bic r0, r0, #CACHE_LINE_SIZE - 1
1351:
136 USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line
137 add r0, r0, #CACHE_LINE_SIZE
138 cmp r0, r1
139 blo 1b
140#endif
141 mov r0, #0
142#ifdef HARVARD_CACHE
143 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
144#ifndef CONFIG_ARM_ERRATA_411920
145 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
146#else
147 b v6_flush_icache_all
148#endif
149#else
150 mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB
151#endif
152 ret lr
153
154/*
155 * Fault handling for the cache operation above. If the virtual address in r0
156 * isn't mapped, fail with -EFAULT.
157 */
1589001:
159 mov r0, #-EFAULT
160 ret lr
161 UNWIND(.fnend )
162ENDPROC(v6_coherent_user_range)
163ENDPROC(v6_coherent_kern_range)
164
165/*
166 * v6_flush_kern_dcache_area(void *addr, size_t size)
167 *
168 * Ensure that the data held in the page kaddr is written back
169 * to the page in question.
170 *
171 * - addr - kernel address
172 * - size - region size
173 */
174ENTRY(v6_flush_kern_dcache_area)
175 add r1, r0, r1
176 bic r0, r0, #D_CACHE_LINE_SIZE - 1
1771:
178#ifdef HARVARD_CACHE
179 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
180#else
181 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line
182#endif
183 add r0, r0, #D_CACHE_LINE_SIZE
184 cmp r0, r1
185 blo 1b
186#ifdef HARVARD_CACHE
187 mov r0, #0
188 mcr p15, 0, r0, c7, c10, 4
189#endif
190 ret lr
191
192
193/*
194 * v6_dma_inv_range(start,end)
195 *
196 * Invalidate the data cache within the specified region; we will
197 * be performing a DMA operation in this region and we want to
198 * purge old data in the cache.
199 *
200 * - start - virtual start address of region
201 * - end - virtual end address of region
202 */
203v6_dma_inv_range:
204#ifdef CONFIG_DMA_CACHE_RWFO
205 ldrb r2, [r0] @ read for ownership
206 strb r2, [r0] @ write for ownership
207#endif
208 tst r0, #D_CACHE_LINE_SIZE - 1
209 bic r0, r0, #D_CACHE_LINE_SIZE - 1
210#ifdef HARVARD_CACHE
211 mcrne p15, 0, r0, c7, c10, 1 @ clean D line
212#else
213 mcrne p15, 0, r0, c7, c11, 1 @ clean unified line
214#endif
215 tst r1, #D_CACHE_LINE_SIZE - 1
216#ifdef CONFIG_DMA_CACHE_RWFO
217 ldrbne r2, [r1, #-1] @ read for ownership
218 strbne r2, [r1, #-1] @ write for ownership
219#endif
220 bic r1, r1, #D_CACHE_LINE_SIZE - 1
221#ifdef HARVARD_CACHE
222 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line
223#else
224 mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line
225#endif
2261:
227#ifdef HARVARD_CACHE
228 mcr p15, 0, r0, c7, c6, 1 @ invalidate D line
229#else
230 mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line
231#endif
232 add r0, r0, #D_CACHE_LINE_SIZE
233 cmp r0, r1
234#ifdef CONFIG_DMA_CACHE_RWFO
235 ldrlo r2, [r0] @ read for ownership
236 strlo r2, [r0] @ write for ownership
237#endif
238 blo 1b
239 mov r0, #0
240 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
241 ret lr
242
243/*
244 * v6_dma_clean_range(start,end)
245 * - start - virtual start address of region
246 * - end - virtual end address of region
247 */
248v6_dma_clean_range:
249 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2501:
251#ifdef CONFIG_DMA_CACHE_RWFO
252 ldr r2, [r0] @ read for ownership
253#endif
254#ifdef HARVARD_CACHE
255 mcr p15, 0, r0, c7, c10, 1 @ clean D line
256#else
257 mcr p15, 0, r0, c7, c11, 1 @ clean unified line
258#endif
259 add r0, r0, #D_CACHE_LINE_SIZE
260 cmp r0, r1
261 blo 1b
262 mov r0, #0
263 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
264 ret lr
265
266/*
267 * v6_dma_flush_range(start,end)
268 * - start - virtual start address of region
269 * - end - virtual end address of region
270 */
271ENTRY(v6_dma_flush_range)
272#ifdef CONFIG_DMA_CACHE_RWFO
273 ldrb r2, [r0] @ read for ownership
274 strb r2, [r0] @ write for ownership
275#endif
276 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2771:
278#ifdef HARVARD_CACHE
279 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
280#else
281 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line
282#endif
283 add r0, r0, #D_CACHE_LINE_SIZE
284 cmp r0, r1
285#ifdef CONFIG_DMA_CACHE_RWFO
286 ldrblo r2, [r0] @ read for ownership
287 strblo r2, [r0] @ write for ownership
288#endif
289 blo 1b
290 mov r0, #0
291 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
292 ret lr
293
294/*
295 * dma_map_area(start, size, dir)
296 * - start - kernel virtual start address
297 * - size - size of region
298 * - dir - DMA direction
299 */
300ENTRY(v6_dma_map_area)
301 add r1, r1, r0
302 teq r2, #DMA_FROM_DEVICE
303 beq v6_dma_inv_range
304#ifndef CONFIG_DMA_CACHE_RWFO
305 b v6_dma_clean_range
306#else
307 teq r2, #DMA_TO_DEVICE
308 beq v6_dma_clean_range
309 b v6_dma_flush_range
310#endif
311ENDPROC(v6_dma_map_area)
312
313/*
314 * dma_unmap_area(start, size, dir)
315 * - start - kernel virtual start address
316 * - size - size of region
317 * - dir - DMA direction
318 */
319ENTRY(v6_dma_unmap_area)
320#ifndef CONFIG_DMA_CACHE_RWFO
321 add r1, r1, r0
322 teq r2, #DMA_TO_DEVICE
323 bne v6_dma_inv_range
324#endif
325 ret lr
326ENDPROC(v6_dma_unmap_area)
327
328 .globl v6_flush_kern_cache_louis
329 .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all
330
331 __INITDATA
332
333 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
334 define_cache_functions v6