Loading...
1/*
2 * PARISC TLB and cache flushing support
3 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2, or (at your option)
10 * any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 * should only use index and base registers that are not shadowed,
25 * so that the fast path emulation in the non access miss handler
26 * can be used.
27 */
28
29#ifdef CONFIG_64BIT
30 .level 2.0w
31#else
32 .level 2.0
33#endif
34
35#include <asm/psw.h>
36#include <asm/assembly.h>
37#include <asm/pgtable.h>
38#include <asm/cache.h>
39#include <linux/linkage.h>
40
41 .text
42 .align 128
43
44ENTRY(flush_tlb_all_local)
45 .proc
46 .callinfo NO_CALLS
47 .entry
48
49 /*
50 * The pitlbe and pdtlbe instructions should only be used to
51 * flush the entire tlb. Also, there needs to be no intervening
52 * tlb operations, e.g. tlb misses, so the operation needs
53 * to happen in real mode with all interruptions disabled.
54 */
55
56 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
57 rsm PSW_SM_I, %r19 /* save I-bit state */
58 load32 PA(1f), %r1
59 nop
60 nop
61 nop
62 nop
63 nop
64
65 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
66 mtctl %r0, %cr17 /* Clear IIASQ tail */
67 mtctl %r0, %cr17 /* Clear IIASQ head */
68 mtctl %r1, %cr18 /* IIAOQ head */
69 ldo 4(%r1), %r1
70 mtctl %r1, %cr18 /* IIAOQ tail */
71 load32 REAL_MODE_PSW, %r1
72 mtctl %r1, %ipsw
73 rfi
74 nop
75
761: load32 PA(cache_info), %r1
77
78 /* Flush Instruction Tlb */
79
80 LDREG ITLB_SID_BASE(%r1), %r20
81 LDREG ITLB_SID_STRIDE(%r1), %r21
82 LDREG ITLB_SID_COUNT(%r1), %r22
83 LDREG ITLB_OFF_BASE(%r1), %arg0
84 LDREG ITLB_OFF_STRIDE(%r1), %arg1
85 LDREG ITLB_OFF_COUNT(%r1), %arg2
86 LDREG ITLB_LOOP(%r1), %arg3
87
88 addib,COND(=) -1, %arg3, fitoneloop /* Preadjust and test */
89 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
90 copy %arg0, %r28 /* Init base addr */
91
92fitmanyloop: /* Loop if LOOP >= 2 */
93 mtsp %r20, %sr1
94 add %r21, %r20, %r20 /* increment space */
95 copy %arg2, %r29 /* Init middle loop count */
96
97fitmanymiddle: /* Loop if LOOP >= 2 */
98 addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
99 pitlbe 0(%sr1, %r28)
100 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
101 addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */
102 copy %arg3, %r31 /* Re-init inner loop count */
103
104 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
105 addib,COND(<=),n -1, %r22, fitdone /* Outer loop count decr */
106
107fitoneloop: /* Loop if LOOP = 1 */
108 mtsp %r20, %sr1
109 copy %arg0, %r28 /* init base addr */
110 copy %arg2, %r29 /* init middle loop count */
111
112fitonemiddle: /* Loop if LOOP = 1 */
113 addib,COND(>) -1, %r29, fitonemiddle /* Middle loop count decr */
114 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
115
116 addib,COND(>) -1, %r22, fitoneloop /* Outer loop count decr */
117 add %r21, %r20, %r20 /* increment space */
118
119fitdone:
120
121 /* Flush Data Tlb */
122
123 LDREG DTLB_SID_BASE(%r1), %r20
124 LDREG DTLB_SID_STRIDE(%r1), %r21
125 LDREG DTLB_SID_COUNT(%r1), %r22
126 LDREG DTLB_OFF_BASE(%r1), %arg0
127 LDREG DTLB_OFF_STRIDE(%r1), %arg1
128 LDREG DTLB_OFF_COUNT(%r1), %arg2
129 LDREG DTLB_LOOP(%r1), %arg3
130
131 addib,COND(=) -1, %arg3, fdtoneloop /* Preadjust and test */
132 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
133 copy %arg0, %r28 /* Init base addr */
134
135fdtmanyloop: /* Loop if LOOP >= 2 */
136 mtsp %r20, %sr1
137 add %r21, %r20, %r20 /* increment space */
138 copy %arg2, %r29 /* Init middle loop count */
139
140fdtmanymiddle: /* Loop if LOOP >= 2 */
141 addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
142 pdtlbe 0(%sr1, %r28)
143 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
144 addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */
145 copy %arg3, %r31 /* Re-init inner loop count */
146
147 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
148 addib,COND(<=),n -1, %r22,fdtdone /* Outer loop count decr */
149
150fdtoneloop: /* Loop if LOOP = 1 */
151 mtsp %r20, %sr1
152 copy %arg0, %r28 /* init base addr */
153 copy %arg2, %r29 /* init middle loop count */
154
155fdtonemiddle: /* Loop if LOOP = 1 */
156 addib,COND(>) -1, %r29, fdtonemiddle /* Middle loop count decr */
157 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
158
159 addib,COND(>) -1, %r22, fdtoneloop /* Outer loop count decr */
160 add %r21, %r20, %r20 /* increment space */
161
162
163fdtdone:
164 /*
165 * Switch back to virtual mode
166 */
167 /* pcxt_ssm_bug */
168 rsm PSW_SM_I, %r0
169 load32 2f, %r1
170 nop
171 nop
172 nop
173 nop
174 nop
175
176 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
177 mtctl %r0, %cr17 /* Clear IIASQ tail */
178 mtctl %r0, %cr17 /* Clear IIASQ head */
179 mtctl %r1, %cr18 /* IIAOQ head */
180 ldo 4(%r1), %r1
181 mtctl %r1, %cr18 /* IIAOQ tail */
182 load32 KERNEL_PSW, %r1
183 or %r1, %r19, %r1 /* I-bit to state on entry */
184 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
185 rfi
186 nop
187
1882: bv %r0(%r2)
189 nop
190
191 .exit
192 .procend
193ENDPROC(flush_tlb_all_local)
194
195 .import cache_info,data
196
197ENTRY(flush_instruction_cache_local)
198 .proc
199 .callinfo NO_CALLS
200 .entry
201
202 load32 cache_info, %r1
203
204 /* Flush Instruction Cache */
205
206 LDREG ICACHE_BASE(%r1), %arg0
207 LDREG ICACHE_STRIDE(%r1), %arg1
208 LDREG ICACHE_COUNT(%r1), %arg2
209 LDREG ICACHE_LOOP(%r1), %arg3
210 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
211 mtsp %r0, %sr1
212 addib,COND(=) -1, %arg3, fioneloop /* Preadjust and test */
213 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
214
215fimanyloop: /* Loop if LOOP >= 2 */
216 addib,COND(>) -1, %r31, fimanyloop /* Adjusted inner loop decr */
217 fice %r0(%sr1, %arg0)
218 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
219 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
220 addib,COND(<=),n -1, %arg2, fisync /* Outer loop decr */
221
222fioneloop: /* Loop if LOOP = 1 */
223 /* Some implementations may flush with a single fice instruction */
224 cmpib,COND(>>=),n 15, %arg2, fioneloop2
225
226fioneloop1:
227 fice,m %arg1(%sr1, %arg0)
228 fice,m %arg1(%sr1, %arg0)
229 fice,m %arg1(%sr1, %arg0)
230 fice,m %arg1(%sr1, %arg0)
231 fice,m %arg1(%sr1, %arg0)
232 fice,m %arg1(%sr1, %arg0)
233 fice,m %arg1(%sr1, %arg0)
234 fice,m %arg1(%sr1, %arg0)
235 fice,m %arg1(%sr1, %arg0)
236 fice,m %arg1(%sr1, %arg0)
237 fice,m %arg1(%sr1, %arg0)
238 fice,m %arg1(%sr1, %arg0)
239 fice,m %arg1(%sr1, %arg0)
240 fice,m %arg1(%sr1, %arg0)
241 fice,m %arg1(%sr1, %arg0)
242 addib,COND(>) -16, %arg2, fioneloop1
243 fice,m %arg1(%sr1, %arg0)
244
245 /* Check if done */
246 cmpb,COND(=),n %arg2, %r0, fisync /* Predict branch taken */
247
248fioneloop2:
249 addib,COND(>) -1, %arg2, fioneloop2 /* Outer loop count decr */
250 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
251
252fisync:
253 sync
254 mtsm %r22 /* restore I-bit */
255 bv %r0(%r2)
256 nop
257 .exit
258
259 .procend
260ENDPROC(flush_instruction_cache_local)
261
262
263 .import cache_info, data
264ENTRY(flush_data_cache_local)
265 .proc
266 .callinfo NO_CALLS
267 .entry
268
269 load32 cache_info, %r1
270
271 /* Flush Data Cache */
272
273 LDREG DCACHE_BASE(%r1), %arg0
274 LDREG DCACHE_STRIDE(%r1), %arg1
275 LDREG DCACHE_COUNT(%r1), %arg2
276 LDREG DCACHE_LOOP(%r1), %arg3
277 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
278 mtsp %r0, %sr1
279 addib,COND(=) -1, %arg3, fdoneloop /* Preadjust and test */
280 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
281
282fdmanyloop: /* Loop if LOOP >= 2 */
283 addib,COND(>) -1, %r31, fdmanyloop /* Adjusted inner loop decr */
284 fdce %r0(%sr1, %arg0)
285 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
286 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
287 addib,COND(<=),n -1, %arg2, fdsync /* Outer loop decr */
288
289fdoneloop: /* Loop if LOOP = 1 */
290 /* Some implementations may flush with a single fdce instruction */
291 cmpib,COND(>>=),n 15, %arg2, fdoneloop2
292
293fdoneloop1:
294 fdce,m %arg1(%sr1, %arg0)
295 fdce,m %arg1(%sr1, %arg0)
296 fdce,m %arg1(%sr1, %arg0)
297 fdce,m %arg1(%sr1, %arg0)
298 fdce,m %arg1(%sr1, %arg0)
299 fdce,m %arg1(%sr1, %arg0)
300 fdce,m %arg1(%sr1, %arg0)
301 fdce,m %arg1(%sr1, %arg0)
302 fdce,m %arg1(%sr1, %arg0)
303 fdce,m %arg1(%sr1, %arg0)
304 fdce,m %arg1(%sr1, %arg0)
305 fdce,m %arg1(%sr1, %arg0)
306 fdce,m %arg1(%sr1, %arg0)
307 fdce,m %arg1(%sr1, %arg0)
308 fdce,m %arg1(%sr1, %arg0)
309 addib,COND(>) -16, %arg2, fdoneloop1
310 fdce,m %arg1(%sr1, %arg0)
311
312 /* Check if done */
313 cmpb,COND(=),n %arg2, %r0, fdsync /* Predict branch taken */
314
315fdoneloop2:
316 addib,COND(>) -1, %arg2, fdoneloop2 /* Outer loop count decr */
317 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
318
319fdsync:
320 syncdma
321 sync
322 mtsm %r22 /* restore I-bit */
323 bv %r0(%r2)
324 nop
325 .exit
326
327 .procend
328ENDPROC(flush_data_cache_local)
329
330 .align 16
331
332/* Macros to serialize TLB purge operations on SMP. */
333
334 .macro tlb_lock la,flags,tmp
335#ifdef CONFIG_SMP
336 ldil L%pa_tlb_lock,%r1
337 ldo R%pa_tlb_lock(%r1),\la
338 rsm PSW_SM_I,\flags
3391: LDCW 0(\la),\tmp
340 cmpib,<>,n 0,\tmp,3f
3412: ldw 0(\la),\tmp
342 cmpb,<> %r0,\tmp,1b
343 nop
344 b,n 2b
3453:
346#endif
347 .endm
348
349 .macro tlb_unlock la,flags,tmp
350#ifdef CONFIG_SMP
351 ldi 1,\tmp
352 stw \tmp,0(\la)
353 mtsm \flags
354#endif
355 .endm
356
357/* Clear page using kernel mapping. */
358
359ENTRY(clear_page_asm)
360 .proc
361 .callinfo NO_CALLS
362 .entry
363
364#ifdef CONFIG_64BIT
365
366 /* Unroll the loop. */
367 ldi (PAGE_SIZE / 128), %r1
368
3691:
370 std %r0, 0(%r26)
371 std %r0, 8(%r26)
372 std %r0, 16(%r26)
373 std %r0, 24(%r26)
374 std %r0, 32(%r26)
375 std %r0, 40(%r26)
376 std %r0, 48(%r26)
377 std %r0, 56(%r26)
378 std %r0, 64(%r26)
379 std %r0, 72(%r26)
380 std %r0, 80(%r26)
381 std %r0, 88(%r26)
382 std %r0, 96(%r26)
383 std %r0, 104(%r26)
384 std %r0, 112(%r26)
385 std %r0, 120(%r26)
386
387 /* Note reverse branch hint for addib is taken. */
388 addib,COND(>),n -1, %r1, 1b
389 ldo 128(%r26), %r26
390
391#else
392
393 /*
394 * Note that until (if) we start saving the full 64-bit register
395 * values on interrupt, we can't use std on a 32 bit kernel.
396 */
397 ldi (PAGE_SIZE / 64), %r1
398
3991:
400 stw %r0, 0(%r26)
401 stw %r0, 4(%r26)
402 stw %r0, 8(%r26)
403 stw %r0, 12(%r26)
404 stw %r0, 16(%r26)
405 stw %r0, 20(%r26)
406 stw %r0, 24(%r26)
407 stw %r0, 28(%r26)
408 stw %r0, 32(%r26)
409 stw %r0, 36(%r26)
410 stw %r0, 40(%r26)
411 stw %r0, 44(%r26)
412 stw %r0, 48(%r26)
413 stw %r0, 52(%r26)
414 stw %r0, 56(%r26)
415 stw %r0, 60(%r26)
416
417 addib,COND(>),n -1, %r1, 1b
418 ldo 64(%r26), %r26
419#endif
420 bv %r0(%r2)
421 nop
422 .exit
423
424 .procend
425ENDPROC(clear_page_asm)
426
427/* Copy page using kernel mapping. */
428
429ENTRY(copy_page_asm)
430 .proc
431 .callinfo NO_CALLS
432 .entry
433
434#ifdef CONFIG_64BIT
435 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
436 * Unroll the loop by hand and arrange insn appropriately.
437 * Prefetch doesn't improve performance on rp3440.
438 * GCC probably can do this just as well...
439 */
440
441 ldi (PAGE_SIZE / 128), %r1
442
4431: ldd 0(%r25), %r19
444 ldd 8(%r25), %r20
445
446 ldd 16(%r25), %r21
447 ldd 24(%r25), %r22
448 std %r19, 0(%r26)
449 std %r20, 8(%r26)
450
451 ldd 32(%r25), %r19
452 ldd 40(%r25), %r20
453 std %r21, 16(%r26)
454 std %r22, 24(%r26)
455
456 ldd 48(%r25), %r21
457 ldd 56(%r25), %r22
458 std %r19, 32(%r26)
459 std %r20, 40(%r26)
460
461 ldd 64(%r25), %r19
462 ldd 72(%r25), %r20
463 std %r21, 48(%r26)
464 std %r22, 56(%r26)
465
466 ldd 80(%r25), %r21
467 ldd 88(%r25), %r22
468 std %r19, 64(%r26)
469 std %r20, 72(%r26)
470
471 ldd 96(%r25), %r19
472 ldd 104(%r25), %r20
473 std %r21, 80(%r26)
474 std %r22, 88(%r26)
475
476 ldd 112(%r25), %r21
477 ldd 120(%r25), %r22
478 ldo 128(%r25), %r25
479 std %r19, 96(%r26)
480 std %r20, 104(%r26)
481
482 std %r21, 112(%r26)
483 std %r22, 120(%r26)
484
485 /* Note reverse branch hint for addib is taken. */
486 addib,COND(>),n -1, %r1, 1b
487 ldo 128(%r26), %r26
488
489#else
490
491 /*
492 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
493 * bundles (very restricted rules for bundling).
494 * Note that until (if) we start saving
495 * the full 64 bit register values on interrupt, we can't
496 * use ldd/std on a 32 bit kernel.
497 */
498 ldw 0(%r25), %r19
499 ldi (PAGE_SIZE / 64), %r1
500
5011:
502 ldw 4(%r25), %r20
503 ldw 8(%r25), %r21
504 ldw 12(%r25), %r22
505 stw %r19, 0(%r26)
506 stw %r20, 4(%r26)
507 stw %r21, 8(%r26)
508 stw %r22, 12(%r26)
509 ldw 16(%r25), %r19
510 ldw 20(%r25), %r20
511 ldw 24(%r25), %r21
512 ldw 28(%r25), %r22
513 stw %r19, 16(%r26)
514 stw %r20, 20(%r26)
515 stw %r21, 24(%r26)
516 stw %r22, 28(%r26)
517 ldw 32(%r25), %r19
518 ldw 36(%r25), %r20
519 ldw 40(%r25), %r21
520 ldw 44(%r25), %r22
521 stw %r19, 32(%r26)
522 stw %r20, 36(%r26)
523 stw %r21, 40(%r26)
524 stw %r22, 44(%r26)
525 ldw 48(%r25), %r19
526 ldw 52(%r25), %r20
527 ldw 56(%r25), %r21
528 ldw 60(%r25), %r22
529 stw %r19, 48(%r26)
530 stw %r20, 52(%r26)
531 ldo 64(%r25), %r25
532 stw %r21, 56(%r26)
533 stw %r22, 60(%r26)
534 ldo 64(%r26), %r26
535 addib,COND(>),n -1, %r1, 1b
536 ldw 0(%r25), %r19
537#endif
538 bv %r0(%r2)
539 nop
540 .exit
541
542 .procend
543ENDPROC(copy_page_asm)
544
545/*
546 * NOTE: Code in clear_user_page has a hard coded dependency on the
547 * maximum alias boundary being 4 Mb. We've been assured by the
548 * parisc chip designers that there will not ever be a parisc
549 * chip with a larger alias boundary (Never say never :-) ).
550 *
551 * Subtle: the dtlb miss handlers support the temp alias region by
552 * "knowing" that if a dtlb miss happens within the temp alias
553 * region it must have occurred while in clear_user_page. Since
554 * this routine makes use of processor local translations, we
555 * don't want to insert them into the kernel page table. Instead,
556 * we load up some general registers (they need to be registers
557 * which aren't shadowed) with the physical page numbers (preshifted
558 * for tlb insertion) needed to insert the translations. When we
559 * miss on the translation, the dtlb miss handler inserts the
560 * translation into the tlb using these values:
561 *
562 * %r26 physical page (shifted for tlb insert) of "to" translation
563 * %r23 physical page (shifted for tlb insert) of "from" translation
564 */
565
566 /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
567 #define PAGE_ADD_SHIFT (PAGE_SHIFT-12)
568 .macro convert_phys_for_tlb_insert20 phys
569 extrd,u \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
570#if _PAGE_SIZE_ENCODING_DEFAULT
571 depdi _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
572#endif
573 .endm
574
575 /*
576 * We can't do this since copy_user_page is used to bring in
577 * file data that might have instructions. Since the data would
578 * then need to be flushed out so the i-fetch can see it, it
579 * makes more sense to just copy through the kernel translation
580 * and flush it.
581 *
582 * I'm still keeping this around because it may be possible to
583 * use it if more information is passed into copy_user_page().
584 * Have to do some measurements to see if it is worthwhile to
585 * lobby for such a change.
586 *
587 */
588
589ENTRY(copy_user_page_asm)
590 .proc
591 .callinfo NO_CALLS
592 .entry
593
594 /* Convert virtual `to' and `from' addresses to physical addresses.
595 Move `from' physical address to non shadowed register. */
596 ldil L%(__PAGE_OFFSET), %r1
597 sub %r26, %r1, %r26
598 sub %r25, %r1, %r23
599
600 ldil L%(TMPALIAS_MAP_START), %r28
601#ifdef CONFIG_64BIT
602#if (TMPALIAS_MAP_START >= 0x80000000)
603 depdi 0, 31,32, %r28 /* clear any sign extension */
604#endif
605 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
606 convert_phys_for_tlb_insert20 %r23 /* convert phys addr to tlb insert format */
607 depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
608 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
609 copy %r28, %r29
610 depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
611#else
612 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
613 extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
614 depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
615 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
616 copy %r28, %r29
617 depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
618#endif
619
620 /* Purge any old translations */
621
622#ifdef CONFIG_PA20
623 pdtlb,l 0(%r28)
624 pdtlb,l 0(%r29)
625#else
626 tlb_lock %r20,%r21,%r22
627 pdtlb 0(%r28)
628 pdtlb 0(%r29)
629 tlb_unlock %r20,%r21,%r22
630#endif
631
632#ifdef CONFIG_64BIT
633 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
634 * Unroll the loop by hand and arrange insn appropriately.
635 * GCC probably can do this just as well.
636 */
637
638 ldd 0(%r29), %r19
639 ldi (PAGE_SIZE / 128), %r1
640
6411: ldd 8(%r29), %r20
642
643 ldd 16(%r29), %r21
644 ldd 24(%r29), %r22
645 std %r19, 0(%r28)
646 std %r20, 8(%r28)
647
648 ldd 32(%r29), %r19
649 ldd 40(%r29), %r20
650 std %r21, 16(%r28)
651 std %r22, 24(%r28)
652
653 ldd 48(%r29), %r21
654 ldd 56(%r29), %r22
655 std %r19, 32(%r28)
656 std %r20, 40(%r28)
657
658 ldd 64(%r29), %r19
659 ldd 72(%r29), %r20
660 std %r21, 48(%r28)
661 std %r22, 56(%r28)
662
663 ldd 80(%r29), %r21
664 ldd 88(%r29), %r22
665 std %r19, 64(%r28)
666 std %r20, 72(%r28)
667
668 ldd 96(%r29), %r19
669 ldd 104(%r29), %r20
670 std %r21, 80(%r28)
671 std %r22, 88(%r28)
672
673 ldd 112(%r29), %r21
674 ldd 120(%r29), %r22
675 std %r19, 96(%r28)
676 std %r20, 104(%r28)
677
678 ldo 128(%r29), %r29
679 std %r21, 112(%r28)
680 std %r22, 120(%r28)
681 ldo 128(%r28), %r28
682
683 /* conditional branches nullify on forward taken branch, and on
684 * non-taken backward branch. Note that .+4 is a backwards branch.
685 * The ldd should only get executed if the branch is taken.
686 */
687 addib,COND(>),n -1, %r1, 1b /* bundle 10 */
688 ldd 0(%r29), %r19 /* start next loads */
689
690#else
691 ldi (PAGE_SIZE / 64), %r1
692
693 /*
694 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
695 * bundles (very restricted rules for bundling). It probably
696 * does OK on PCXU and better, but we could do better with
697 * ldd/std instructions. Note that until (if) we start saving
698 * the full 64 bit register values on interrupt, we can't
699 * use ldd/std on a 32 bit kernel.
700 */
701
7021: ldw 0(%r29), %r19
703 ldw 4(%r29), %r20
704 ldw 8(%r29), %r21
705 ldw 12(%r29), %r22
706 stw %r19, 0(%r28)
707 stw %r20, 4(%r28)
708 stw %r21, 8(%r28)
709 stw %r22, 12(%r28)
710 ldw 16(%r29), %r19
711 ldw 20(%r29), %r20
712 ldw 24(%r29), %r21
713 ldw 28(%r29), %r22
714 stw %r19, 16(%r28)
715 stw %r20, 20(%r28)
716 stw %r21, 24(%r28)
717 stw %r22, 28(%r28)
718 ldw 32(%r29), %r19
719 ldw 36(%r29), %r20
720 ldw 40(%r29), %r21
721 ldw 44(%r29), %r22
722 stw %r19, 32(%r28)
723 stw %r20, 36(%r28)
724 stw %r21, 40(%r28)
725 stw %r22, 44(%r28)
726 ldw 48(%r29), %r19
727 ldw 52(%r29), %r20
728 ldw 56(%r29), %r21
729 ldw 60(%r29), %r22
730 stw %r19, 48(%r28)
731 stw %r20, 52(%r28)
732 stw %r21, 56(%r28)
733 stw %r22, 60(%r28)
734 ldo 64(%r28), %r28
735
736 addib,COND(>) -1, %r1,1b
737 ldo 64(%r29), %r29
738#endif
739
740 bv %r0(%r2)
741 nop
742 .exit
743
744 .procend
745ENDPROC(copy_user_page_asm)
746
747ENTRY(clear_user_page_asm)
748 .proc
749 .callinfo NO_CALLS
750 .entry
751
752 tophys_r1 %r26
753
754 ldil L%(TMPALIAS_MAP_START), %r28
755#ifdef CONFIG_64BIT
756#if (TMPALIAS_MAP_START >= 0x80000000)
757 depdi 0, 31,32, %r28 /* clear any sign extension */
758#endif
759 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
760 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
761 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
762#else
763 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
764 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
765 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
766#endif
767
768 /* Purge any old translation */
769
770#ifdef CONFIG_PA20
771 pdtlb,l 0(%r28)
772#else
773 tlb_lock %r20,%r21,%r22
774 pdtlb 0(%r28)
775 tlb_unlock %r20,%r21,%r22
776#endif
777
778#ifdef CONFIG_64BIT
779 ldi (PAGE_SIZE / 128), %r1
780
781 /* PREFETCH (Write) has not (yet) been proven to help here */
782 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
783
7841: std %r0, 0(%r28)
785 std %r0, 8(%r28)
786 std %r0, 16(%r28)
787 std %r0, 24(%r28)
788 std %r0, 32(%r28)
789 std %r0, 40(%r28)
790 std %r0, 48(%r28)
791 std %r0, 56(%r28)
792 std %r0, 64(%r28)
793 std %r0, 72(%r28)
794 std %r0, 80(%r28)
795 std %r0, 88(%r28)
796 std %r0, 96(%r28)
797 std %r0, 104(%r28)
798 std %r0, 112(%r28)
799 std %r0, 120(%r28)
800 addib,COND(>) -1, %r1, 1b
801 ldo 128(%r28), %r28
802
803#else /* ! CONFIG_64BIT */
804 ldi (PAGE_SIZE / 64), %r1
805
8061: stw %r0, 0(%r28)
807 stw %r0, 4(%r28)
808 stw %r0, 8(%r28)
809 stw %r0, 12(%r28)
810 stw %r0, 16(%r28)
811 stw %r0, 20(%r28)
812 stw %r0, 24(%r28)
813 stw %r0, 28(%r28)
814 stw %r0, 32(%r28)
815 stw %r0, 36(%r28)
816 stw %r0, 40(%r28)
817 stw %r0, 44(%r28)
818 stw %r0, 48(%r28)
819 stw %r0, 52(%r28)
820 stw %r0, 56(%r28)
821 stw %r0, 60(%r28)
822 addib,COND(>) -1, %r1, 1b
823 ldo 64(%r28), %r28
824#endif /* CONFIG_64BIT */
825
826 bv %r0(%r2)
827 nop
828 .exit
829
830 .procend
831ENDPROC(clear_user_page_asm)
832
833ENTRY(flush_dcache_page_asm)
834 .proc
835 .callinfo NO_CALLS
836 .entry
837
838 ldil L%(TMPALIAS_MAP_START), %r28
839#ifdef CONFIG_64BIT
840#if (TMPALIAS_MAP_START >= 0x80000000)
841 depdi 0, 31,32, %r28 /* clear any sign extension */
842#endif
843 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
844 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
845 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
846#else
847 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
848 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
849 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
850#endif
851
852 /* Purge any old translation */
853
854#ifdef CONFIG_PA20
855 pdtlb,l 0(%r28)
856#else
857 tlb_lock %r20,%r21,%r22
858 pdtlb 0(%r28)
859 tlb_unlock %r20,%r21,%r22
860#endif
861
862 ldil L%dcache_stride, %r1
863 ldw R%dcache_stride(%r1), r31
864
865#ifdef CONFIG_64BIT
866 depdi,z 1, 63-PAGE_SHIFT,1, %r25
867#else
868 depwi,z 1, 31-PAGE_SHIFT,1, %r25
869#endif
870 add %r28, %r25, %r25
871 sub %r25, r31, %r25
872
873
8741: fdc,m r31(%r28)
875 fdc,m r31(%r28)
876 fdc,m r31(%r28)
877 fdc,m r31(%r28)
878 fdc,m r31(%r28)
879 fdc,m r31(%r28)
880 fdc,m r31(%r28)
881 fdc,m r31(%r28)
882 fdc,m r31(%r28)
883 fdc,m r31(%r28)
884 fdc,m r31(%r28)
885 fdc,m r31(%r28)
886 fdc,m r31(%r28)
887 fdc,m r31(%r28)
888 fdc,m r31(%r28)
889 cmpb,COND(<<) %r28, %r25,1b
890 fdc,m r31(%r28)
891
892 sync
893
894#ifdef CONFIG_PA20
895 pdtlb,l 0(%r25)
896#else
897 tlb_lock %r20,%r21,%r22
898 pdtlb 0(%r25)
899 tlb_unlock %r20,%r21,%r22
900#endif
901
902 bv %r0(%r2)
903 nop
904 .exit
905
906 .procend
907ENDPROC(flush_dcache_page_asm)
908
909ENTRY(flush_icache_page_asm)
910 .proc
911 .callinfo NO_CALLS
912 .entry
913
914 ldil L%(TMPALIAS_MAP_START), %r28
915#ifdef CONFIG_64BIT
916#if (TMPALIAS_MAP_START >= 0x80000000)
917 depdi 0, 31,32, %r28 /* clear any sign extension */
918#endif
919 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
920 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
921 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
922#else
923 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
924 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
925 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
926#endif
927
928 /* Purge any old translation */
929
930#ifdef CONFIG_PA20
931 pitlb,l %r0(%sr4,%r28)
932#else
933 tlb_lock %r20,%r21,%r22
934 pitlb (%sr4,%r28)
935 tlb_unlock %r20,%r21,%r22
936#endif
937
938 ldil L%icache_stride, %r1
939 ldw R%icache_stride(%r1), %r31
940
941#ifdef CONFIG_64BIT
942 depdi,z 1, 63-PAGE_SHIFT,1, %r25
943#else
944 depwi,z 1, 31-PAGE_SHIFT,1, %r25
945#endif
946 add %r28, %r25, %r25
947 sub %r25, %r31, %r25
948
949
950 /* fic only has the type 26 form on PA1.1, requiring an
951 * explicit space specification, so use %sr4 */
9521: fic,m %r31(%sr4,%r28)
953 fic,m %r31(%sr4,%r28)
954 fic,m %r31(%sr4,%r28)
955 fic,m %r31(%sr4,%r28)
956 fic,m %r31(%sr4,%r28)
957 fic,m %r31(%sr4,%r28)
958 fic,m %r31(%sr4,%r28)
959 fic,m %r31(%sr4,%r28)
960 fic,m %r31(%sr4,%r28)
961 fic,m %r31(%sr4,%r28)
962 fic,m %r31(%sr4,%r28)
963 fic,m %r31(%sr4,%r28)
964 fic,m %r31(%sr4,%r28)
965 fic,m %r31(%sr4,%r28)
966 fic,m %r31(%sr4,%r28)
967 cmpb,COND(<<) %r28, %r25,1b
968 fic,m %r31(%sr4,%r28)
969
970 sync
971
972#ifdef CONFIG_PA20
973 pitlb,l %r0(%sr4,%r25)
974#else
975 tlb_lock %r20,%r21,%r22
976 pitlb (%sr4,%r25)
977 tlb_unlock %r20,%r21,%r22
978#endif
979
980 bv %r0(%r2)
981 nop
982 .exit
983
984 .procend
985ENDPROC(flush_icache_page_asm)
986
987ENTRY(flush_kernel_dcache_page_asm)
988 .proc
989 .callinfo NO_CALLS
990 .entry
991
992 ldil L%dcache_stride, %r1
993 ldw R%dcache_stride(%r1), %r23
994
995#ifdef CONFIG_64BIT
996 depdi,z 1, 63-PAGE_SHIFT,1, %r25
997#else
998 depwi,z 1, 31-PAGE_SHIFT,1, %r25
999#endif
1000 add %r26, %r25, %r25
1001 sub %r25, %r23, %r25
1002
1003
10041: fdc,m %r23(%r26)
1005 fdc,m %r23(%r26)
1006 fdc,m %r23(%r26)
1007 fdc,m %r23(%r26)
1008 fdc,m %r23(%r26)
1009 fdc,m %r23(%r26)
1010 fdc,m %r23(%r26)
1011 fdc,m %r23(%r26)
1012 fdc,m %r23(%r26)
1013 fdc,m %r23(%r26)
1014 fdc,m %r23(%r26)
1015 fdc,m %r23(%r26)
1016 fdc,m %r23(%r26)
1017 fdc,m %r23(%r26)
1018 fdc,m %r23(%r26)
1019 cmpb,COND(<<) %r26, %r25,1b
1020 fdc,m %r23(%r26)
1021
1022 sync
1023 bv %r0(%r2)
1024 nop
1025 .exit
1026
1027 .procend
1028ENDPROC(flush_kernel_dcache_page_asm)
1029
1030ENTRY(purge_kernel_dcache_page_asm)
1031 .proc
1032 .callinfo NO_CALLS
1033 .entry
1034
1035 ldil L%dcache_stride, %r1
1036 ldw R%dcache_stride(%r1), %r23
1037
1038#ifdef CONFIG_64BIT
1039 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1040#else
1041 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1042#endif
1043 add %r26, %r25, %r25
1044 sub %r25, %r23, %r25
1045
10461: pdc,m %r23(%r26)
1047 pdc,m %r23(%r26)
1048 pdc,m %r23(%r26)
1049 pdc,m %r23(%r26)
1050 pdc,m %r23(%r26)
1051 pdc,m %r23(%r26)
1052 pdc,m %r23(%r26)
1053 pdc,m %r23(%r26)
1054 pdc,m %r23(%r26)
1055 pdc,m %r23(%r26)
1056 pdc,m %r23(%r26)
1057 pdc,m %r23(%r26)
1058 pdc,m %r23(%r26)
1059 pdc,m %r23(%r26)
1060 pdc,m %r23(%r26)
1061 cmpb,COND(<<) %r26, %r25, 1b
1062 pdc,m %r23(%r26)
1063
1064 sync
1065 bv %r0(%r2)
1066 nop
1067 .exit
1068
1069 .procend
1070ENDPROC(purge_kernel_dcache_page_asm)
1071
1072ENTRY(flush_user_dcache_range_asm)
1073 .proc
1074 .callinfo NO_CALLS
1075 .entry
1076
1077 ldil L%dcache_stride, %r1
1078 ldw R%dcache_stride(%r1), %r23
1079 ldo -1(%r23), %r21
1080 ANDCM %r26, %r21, %r26
1081
10821: cmpb,COND(<<),n %r26, %r25, 1b
1083 fdc,m %r23(%sr3, %r26)
1084
1085 sync
1086 bv %r0(%r2)
1087 nop
1088 .exit
1089
1090 .procend
1091ENDPROC(flush_user_dcache_range_asm)
1092
1093ENTRY(flush_kernel_dcache_range_asm)
1094 .proc
1095 .callinfo NO_CALLS
1096 .entry
1097
1098 ldil L%dcache_stride, %r1
1099 ldw R%dcache_stride(%r1), %r23
1100 ldo -1(%r23), %r21
1101 ANDCM %r26, %r21, %r26
1102
11031: cmpb,COND(<<),n %r26, %r25,1b
1104 fdc,m %r23(%r26)
1105
1106 sync
1107 syncdma
1108 bv %r0(%r2)
1109 nop
1110 .exit
1111
1112 .procend
1113ENDPROC(flush_kernel_dcache_range_asm)
1114
1115ENTRY(flush_user_icache_range_asm)
1116 .proc
1117 .callinfo NO_CALLS
1118 .entry
1119
1120 ldil L%icache_stride, %r1
1121 ldw R%icache_stride(%r1), %r23
1122 ldo -1(%r23), %r21
1123 ANDCM %r26, %r21, %r26
1124
11251: cmpb,COND(<<),n %r26, %r25,1b
1126 fic,m %r23(%sr3, %r26)
1127
1128 sync
1129 bv %r0(%r2)
1130 nop
1131 .exit
1132
1133 .procend
1134ENDPROC(flush_user_icache_range_asm)
1135
1136ENTRY(flush_kernel_icache_page)
1137 .proc
1138 .callinfo NO_CALLS
1139 .entry
1140
1141 ldil L%icache_stride, %r1
1142 ldw R%icache_stride(%r1), %r23
1143
1144#ifdef CONFIG_64BIT
1145 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1146#else
1147 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1148#endif
1149 add %r26, %r25, %r25
1150 sub %r25, %r23, %r25
1151
1152
11531: fic,m %r23(%sr4, %r26)
1154 fic,m %r23(%sr4, %r26)
1155 fic,m %r23(%sr4, %r26)
1156 fic,m %r23(%sr4, %r26)
1157 fic,m %r23(%sr4, %r26)
1158 fic,m %r23(%sr4, %r26)
1159 fic,m %r23(%sr4, %r26)
1160 fic,m %r23(%sr4, %r26)
1161 fic,m %r23(%sr4, %r26)
1162 fic,m %r23(%sr4, %r26)
1163 fic,m %r23(%sr4, %r26)
1164 fic,m %r23(%sr4, %r26)
1165 fic,m %r23(%sr4, %r26)
1166 fic,m %r23(%sr4, %r26)
1167 fic,m %r23(%sr4, %r26)
1168 cmpb,COND(<<) %r26, %r25, 1b
1169 fic,m %r23(%sr4, %r26)
1170
1171 sync
1172 bv %r0(%r2)
1173 nop
1174 .exit
1175
1176 .procend
1177ENDPROC(flush_kernel_icache_page)
1178
1179ENTRY(flush_kernel_icache_range_asm)
1180 .proc
1181 .callinfo NO_CALLS
1182 .entry
1183
1184 ldil L%icache_stride, %r1
1185 ldw R%icache_stride(%r1), %r23
1186 ldo -1(%r23), %r21
1187 ANDCM %r26, %r21, %r26
1188
11891: cmpb,COND(<<),n %r26, %r25, 1b
1190 fic,m %r23(%sr4, %r26)
1191
1192 sync
1193 bv %r0(%r2)
1194 nop
1195 .exit
1196 .procend
1197ENDPROC(flush_kernel_icache_range_asm)
1198
1199 /* align should cover use of rfi in disable_sr_hashing_asm and
1200 * srdis_done.
1201 */
1202 .align 256
1203ENTRY(disable_sr_hashing_asm)
1204 .proc
1205 .callinfo NO_CALLS
1206 .entry
1207
1208 /*
1209 * Switch to real mode
1210 */
1211 /* pcxt_ssm_bug */
1212 rsm PSW_SM_I, %r0
1213 load32 PA(1f), %r1
1214 nop
1215 nop
1216 nop
1217 nop
1218 nop
1219
1220 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1221 mtctl %r0, %cr17 /* Clear IIASQ tail */
1222 mtctl %r0, %cr17 /* Clear IIASQ head */
1223 mtctl %r1, %cr18 /* IIAOQ head */
1224 ldo 4(%r1), %r1
1225 mtctl %r1, %cr18 /* IIAOQ tail */
1226 load32 REAL_MODE_PSW, %r1
1227 mtctl %r1, %ipsw
1228 rfi
1229 nop
1230
12311: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1232 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1233 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1234 b,n srdis_done
1235
1236srdis_pcxs:
1237
1238 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1239
1240 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1241 .word 0x141c1a00 /* must issue twice */
1242 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1243 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1244 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1245 .word 0x141c1600 /* must issue twice */
1246 b,n srdis_done
1247
1248srdis_pcxl:
1249
1250 /* Disable Space Register Hashing for PCXL */
1251
1252 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1253 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1254 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1255 b,n srdis_done
1256
1257srdis_pa20:
1258
1259 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1260
1261 .word 0x144008bc /* mfdiag %dr2, %r28 */
1262 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1263 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1264
1265
1266srdis_done:
1267 /* Switch back to virtual mode */
1268 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1269 load32 2f, %r1
1270 nop
1271 nop
1272 nop
1273 nop
1274 nop
1275
1276 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1277 mtctl %r0, %cr17 /* Clear IIASQ tail */
1278 mtctl %r0, %cr17 /* Clear IIASQ head */
1279 mtctl %r1, %cr18 /* IIAOQ head */
1280 ldo 4(%r1), %r1
1281 mtctl %r1, %cr18 /* IIAOQ tail */
1282 load32 KERNEL_PSW, %r1
1283 mtctl %r1, %ipsw
1284 rfi
1285 nop
1286
12872: bv %r0(%r2)
1288 nop
1289 .exit
1290
1291 .procend
1292ENDPROC(disable_sr_hashing_asm)
1293
1294 .end
1/*
2 * PARISC TLB and cache flushing support
3 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2, or (at your option)
10 * any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 * should only use index and base registers that are not shadowed,
25 * so that the fast path emulation in the non access miss handler
26 * can be used.
27 */
28
29#ifdef CONFIG_64BIT
30 .level 2.0w
31#else
32 .level 2.0
33#endif
34
35#include <asm/psw.h>
36#include <asm/assembly.h>
37#include <asm/pgtable.h>
38#include <asm/cache.h>
39#include <asm/ldcw.h>
40#include <linux/linkage.h>
41#include <linux/init.h>
42
43 .section .text.hot
44 .align 16
45
46ENTRY_CFI(flush_tlb_all_local)
47 .proc
48 .callinfo NO_CALLS
49 .entry
50
51 /*
52 * The pitlbe and pdtlbe instructions should only be used to
53 * flush the entire tlb. Also, there needs to be no intervening
54 * tlb operations, e.g. tlb misses, so the operation needs
55 * to happen in real mode with all interruptions disabled.
56 */
57
58 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
59 rsm PSW_SM_I, %r19 /* save I-bit state */
60 load32 PA(1f), %r1
61 nop
62 nop
63 nop
64 nop
65 nop
66
67 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
68 mtctl %r0, %cr17 /* Clear IIASQ tail */
69 mtctl %r0, %cr17 /* Clear IIASQ head */
70 mtctl %r1, %cr18 /* IIAOQ head */
71 ldo 4(%r1), %r1
72 mtctl %r1, %cr18 /* IIAOQ tail */
73 load32 REAL_MODE_PSW, %r1
74 mtctl %r1, %ipsw
75 rfi
76 nop
77
781: load32 PA(cache_info), %r1
79
80 /* Flush Instruction Tlb */
81
82 LDREG ITLB_SID_BASE(%r1), %r20
83 LDREG ITLB_SID_STRIDE(%r1), %r21
84 LDREG ITLB_SID_COUNT(%r1), %r22
85 LDREG ITLB_OFF_BASE(%r1), %arg0
86 LDREG ITLB_OFF_STRIDE(%r1), %arg1
87 LDREG ITLB_OFF_COUNT(%r1), %arg2
88 LDREG ITLB_LOOP(%r1), %arg3
89
90 addib,COND(=) -1, %arg3, fitoneloop /* Preadjust and test */
91 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
92 copy %arg0, %r28 /* Init base addr */
93
94fitmanyloop: /* Loop if LOOP >= 2 */
95 mtsp %r20, %sr1
96 add %r21, %r20, %r20 /* increment space */
97 copy %arg2, %r29 /* Init middle loop count */
98
99fitmanymiddle: /* Loop if LOOP >= 2 */
100 addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
101 pitlbe %r0(%sr1, %r28)
102 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
103 addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */
104 copy %arg3, %r31 /* Re-init inner loop count */
105
106 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
107 addib,COND(<=),n -1, %r22, fitdone /* Outer loop count decr */
108
109fitoneloop: /* Loop if LOOP = 1 */
110 mtsp %r20, %sr1
111 copy %arg0, %r28 /* init base addr */
112 copy %arg2, %r29 /* init middle loop count */
113
114fitonemiddle: /* Loop if LOOP = 1 */
115 addib,COND(>) -1, %r29, fitonemiddle /* Middle loop count decr */
116 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
117
118 addib,COND(>) -1, %r22, fitoneloop /* Outer loop count decr */
119 add %r21, %r20, %r20 /* increment space */
120
121fitdone:
122
123 /* Flush Data Tlb */
124
125 LDREG DTLB_SID_BASE(%r1), %r20
126 LDREG DTLB_SID_STRIDE(%r1), %r21
127 LDREG DTLB_SID_COUNT(%r1), %r22
128 LDREG DTLB_OFF_BASE(%r1), %arg0
129 LDREG DTLB_OFF_STRIDE(%r1), %arg1
130 LDREG DTLB_OFF_COUNT(%r1), %arg2
131 LDREG DTLB_LOOP(%r1), %arg3
132
133 addib,COND(=) -1, %arg3, fdtoneloop /* Preadjust and test */
134 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
135 copy %arg0, %r28 /* Init base addr */
136
137fdtmanyloop: /* Loop if LOOP >= 2 */
138 mtsp %r20, %sr1
139 add %r21, %r20, %r20 /* increment space */
140 copy %arg2, %r29 /* Init middle loop count */
141
142fdtmanymiddle: /* Loop if LOOP >= 2 */
143 addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
144 pdtlbe %r0(%sr1, %r28)
145 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
146 addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */
147 copy %arg3, %r31 /* Re-init inner loop count */
148
149 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
150 addib,COND(<=),n -1, %r22,fdtdone /* Outer loop count decr */
151
152fdtoneloop: /* Loop if LOOP = 1 */
153 mtsp %r20, %sr1
154 copy %arg0, %r28 /* init base addr */
155 copy %arg2, %r29 /* init middle loop count */
156
157fdtonemiddle: /* Loop if LOOP = 1 */
158 addib,COND(>) -1, %r29, fdtonemiddle /* Middle loop count decr */
159 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
160
161 addib,COND(>) -1, %r22, fdtoneloop /* Outer loop count decr */
162 add %r21, %r20, %r20 /* increment space */
163
164
165fdtdone:
166 /*
167 * Switch back to virtual mode
168 */
169 /* pcxt_ssm_bug */
170 rsm PSW_SM_I, %r0
171 load32 2f, %r1
172 nop
173 nop
174 nop
175 nop
176 nop
177
178 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
179 mtctl %r0, %cr17 /* Clear IIASQ tail */
180 mtctl %r0, %cr17 /* Clear IIASQ head */
181 mtctl %r1, %cr18 /* IIAOQ head */
182 ldo 4(%r1), %r1
183 mtctl %r1, %cr18 /* IIAOQ tail */
184 load32 KERNEL_PSW, %r1
185 or %r1, %r19, %r1 /* I-bit to state on entry */
186 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
187 rfi
188 nop
189
1902: bv %r0(%r2)
191 nop
192
193 .exit
194 .procend
195ENDPROC_CFI(flush_tlb_all_local)
196
197 .import cache_info,data
198
199ENTRY_CFI(flush_instruction_cache_local)
200 .proc
201 .callinfo NO_CALLS
202 .entry
203
204 load32 cache_info, %r1
205
206 /* Flush Instruction Cache */
207
208 LDREG ICACHE_BASE(%r1), %arg0
209 LDREG ICACHE_STRIDE(%r1), %arg1
210 LDREG ICACHE_COUNT(%r1), %arg2
211 LDREG ICACHE_LOOP(%r1), %arg3
212 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
213 mtsp %r0, %sr1
214 addib,COND(=) -1, %arg3, fioneloop /* Preadjust and test */
215 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
216
217fimanyloop: /* Loop if LOOP >= 2 */
218 addib,COND(>) -1, %r31, fimanyloop /* Adjusted inner loop decr */
219 fice %r0(%sr1, %arg0)
220 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
221 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
222 addib,COND(<=),n -1, %arg2, fisync /* Outer loop decr */
223
224fioneloop: /* Loop if LOOP = 1 */
225 /* Some implementations may flush with a single fice instruction */
226 cmpib,COND(>>=),n 15, %arg2, fioneloop2
227
228fioneloop1:
229 fice,m %arg1(%sr1, %arg0)
230 fice,m %arg1(%sr1, %arg0)
231 fice,m %arg1(%sr1, %arg0)
232 fice,m %arg1(%sr1, %arg0)
233 fice,m %arg1(%sr1, %arg0)
234 fice,m %arg1(%sr1, %arg0)
235 fice,m %arg1(%sr1, %arg0)
236 fice,m %arg1(%sr1, %arg0)
237 fice,m %arg1(%sr1, %arg0)
238 fice,m %arg1(%sr1, %arg0)
239 fice,m %arg1(%sr1, %arg0)
240 fice,m %arg1(%sr1, %arg0)
241 fice,m %arg1(%sr1, %arg0)
242 fice,m %arg1(%sr1, %arg0)
243 fice,m %arg1(%sr1, %arg0)
244 addib,COND(>) -16, %arg2, fioneloop1
245 fice,m %arg1(%sr1, %arg0)
246
247 /* Check if done */
248 cmpb,COND(=),n %arg2, %r0, fisync /* Predict branch taken */
249
250fioneloop2:
251 addib,COND(>) -1, %arg2, fioneloop2 /* Outer loop count decr */
252 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
253
254fisync:
255 sync
256 mtsm %r22 /* restore I-bit */
257 bv %r0(%r2)
258 nop
259 .exit
260
261 .procend
262ENDPROC_CFI(flush_instruction_cache_local)
263
264
265 .import cache_info, data
266ENTRY_CFI(flush_data_cache_local)
267 .proc
268 .callinfo NO_CALLS
269 .entry
270
271 load32 cache_info, %r1
272
273 /* Flush Data Cache */
274
275 LDREG DCACHE_BASE(%r1), %arg0
276 LDREG DCACHE_STRIDE(%r1), %arg1
277 LDREG DCACHE_COUNT(%r1), %arg2
278 LDREG DCACHE_LOOP(%r1), %arg3
279 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
280 mtsp %r0, %sr1
281 addib,COND(=) -1, %arg3, fdoneloop /* Preadjust and test */
282 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
283
284fdmanyloop: /* Loop if LOOP >= 2 */
285 addib,COND(>) -1, %r31, fdmanyloop /* Adjusted inner loop decr */
286 fdce %r0(%sr1, %arg0)
287 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
288 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
289 addib,COND(<=),n -1, %arg2, fdsync /* Outer loop decr */
290
291fdoneloop: /* Loop if LOOP = 1 */
292 /* Some implementations may flush with a single fdce instruction */
293 cmpib,COND(>>=),n 15, %arg2, fdoneloop2
294
295fdoneloop1:
296 fdce,m %arg1(%sr1, %arg0)
297 fdce,m %arg1(%sr1, %arg0)
298 fdce,m %arg1(%sr1, %arg0)
299 fdce,m %arg1(%sr1, %arg0)
300 fdce,m %arg1(%sr1, %arg0)
301 fdce,m %arg1(%sr1, %arg0)
302 fdce,m %arg1(%sr1, %arg0)
303 fdce,m %arg1(%sr1, %arg0)
304 fdce,m %arg1(%sr1, %arg0)
305 fdce,m %arg1(%sr1, %arg0)
306 fdce,m %arg1(%sr1, %arg0)
307 fdce,m %arg1(%sr1, %arg0)
308 fdce,m %arg1(%sr1, %arg0)
309 fdce,m %arg1(%sr1, %arg0)
310 fdce,m %arg1(%sr1, %arg0)
311 addib,COND(>) -16, %arg2, fdoneloop1
312 fdce,m %arg1(%sr1, %arg0)
313
314 /* Check if done */
315 cmpb,COND(=),n %arg2, %r0, fdsync /* Predict branch taken */
316
317fdoneloop2:
318 addib,COND(>) -1, %arg2, fdoneloop2 /* Outer loop count decr */
319 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
320
321fdsync:
322 syncdma
323 sync
324 mtsm %r22 /* restore I-bit */
325 bv %r0(%r2)
326 nop
327 .exit
328
329 .procend
330ENDPROC_CFI(flush_data_cache_local)
331
332/* Macros to serialize TLB purge operations on SMP. */
333
334 .macro tlb_lock la,flags,tmp
335#ifdef CONFIG_SMP
336#if __PA_LDCW_ALIGNMENT > 4
337 load32 pa_tlb_lock + __PA_LDCW_ALIGNMENT-1, \la
338 depi 0,31,__PA_LDCW_ALIGN_ORDER, \la
339#else
340 load32 pa_tlb_lock, \la
341#endif
342 rsm PSW_SM_I,\flags
3431: LDCW 0(\la),\tmp
344 cmpib,<>,n 0,\tmp,3f
3452: ldw 0(\la),\tmp
346 cmpb,<> %r0,\tmp,1b
347 nop
348 b,n 2b
3493:
350#endif
351 .endm
352
353 .macro tlb_unlock la,flags,tmp
354#ifdef CONFIG_SMP
355 ldi 1,\tmp
356 stw \tmp,0(\la)
357 mtsm \flags
358#endif
359 .endm
360
361/* Clear page using kernel mapping. */
362
363ENTRY_CFI(clear_page_asm)
364 .proc
365 .callinfo NO_CALLS
366 .entry
367
368#ifdef CONFIG_64BIT
369
370 /* Unroll the loop. */
371 ldi (PAGE_SIZE / 128), %r1
372
3731:
374 std %r0, 0(%r26)
375 std %r0, 8(%r26)
376 std %r0, 16(%r26)
377 std %r0, 24(%r26)
378 std %r0, 32(%r26)
379 std %r0, 40(%r26)
380 std %r0, 48(%r26)
381 std %r0, 56(%r26)
382 std %r0, 64(%r26)
383 std %r0, 72(%r26)
384 std %r0, 80(%r26)
385 std %r0, 88(%r26)
386 std %r0, 96(%r26)
387 std %r0, 104(%r26)
388 std %r0, 112(%r26)
389 std %r0, 120(%r26)
390
391 /* Note reverse branch hint for addib is taken. */
392 addib,COND(>),n -1, %r1, 1b
393 ldo 128(%r26), %r26
394
395#else
396
397 /*
398 * Note that until (if) we start saving the full 64-bit register
399 * values on interrupt, we can't use std on a 32 bit kernel.
400 */
401 ldi (PAGE_SIZE / 64), %r1
402
4031:
404 stw %r0, 0(%r26)
405 stw %r0, 4(%r26)
406 stw %r0, 8(%r26)
407 stw %r0, 12(%r26)
408 stw %r0, 16(%r26)
409 stw %r0, 20(%r26)
410 stw %r0, 24(%r26)
411 stw %r0, 28(%r26)
412 stw %r0, 32(%r26)
413 stw %r0, 36(%r26)
414 stw %r0, 40(%r26)
415 stw %r0, 44(%r26)
416 stw %r0, 48(%r26)
417 stw %r0, 52(%r26)
418 stw %r0, 56(%r26)
419 stw %r0, 60(%r26)
420
421 addib,COND(>),n -1, %r1, 1b
422 ldo 64(%r26), %r26
423#endif
424 bv %r0(%r2)
425 nop
426 .exit
427
428 .procend
429ENDPROC_CFI(clear_page_asm)
430
431/* Copy page using kernel mapping. */
432
433ENTRY_CFI(copy_page_asm)
434 .proc
435 .callinfo NO_CALLS
436 .entry
437
438#ifdef CONFIG_64BIT
439 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
440 * Unroll the loop by hand and arrange insn appropriately.
441 * Prefetch doesn't improve performance on rp3440.
442 * GCC probably can do this just as well...
443 */
444
445 ldi (PAGE_SIZE / 128), %r1
446
4471: ldd 0(%r25), %r19
448 ldd 8(%r25), %r20
449
450 ldd 16(%r25), %r21
451 ldd 24(%r25), %r22
452 std %r19, 0(%r26)
453 std %r20, 8(%r26)
454
455 ldd 32(%r25), %r19
456 ldd 40(%r25), %r20
457 std %r21, 16(%r26)
458 std %r22, 24(%r26)
459
460 ldd 48(%r25), %r21
461 ldd 56(%r25), %r22
462 std %r19, 32(%r26)
463 std %r20, 40(%r26)
464
465 ldd 64(%r25), %r19
466 ldd 72(%r25), %r20
467 std %r21, 48(%r26)
468 std %r22, 56(%r26)
469
470 ldd 80(%r25), %r21
471 ldd 88(%r25), %r22
472 std %r19, 64(%r26)
473 std %r20, 72(%r26)
474
475 ldd 96(%r25), %r19
476 ldd 104(%r25), %r20
477 std %r21, 80(%r26)
478 std %r22, 88(%r26)
479
480 ldd 112(%r25), %r21
481 ldd 120(%r25), %r22
482 ldo 128(%r25), %r25
483 std %r19, 96(%r26)
484 std %r20, 104(%r26)
485
486 std %r21, 112(%r26)
487 std %r22, 120(%r26)
488
489 /* Note reverse branch hint for addib is taken. */
490 addib,COND(>),n -1, %r1, 1b
491 ldo 128(%r26), %r26
492
493#else
494
495 /*
496 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
497 * bundles (very restricted rules for bundling).
498 * Note that until (if) we start saving
499 * the full 64 bit register values on interrupt, we can't
500 * use ldd/std on a 32 bit kernel.
501 */
502 ldw 0(%r25), %r19
503 ldi (PAGE_SIZE / 64), %r1
504
5051:
506 ldw 4(%r25), %r20
507 ldw 8(%r25), %r21
508 ldw 12(%r25), %r22
509 stw %r19, 0(%r26)
510 stw %r20, 4(%r26)
511 stw %r21, 8(%r26)
512 stw %r22, 12(%r26)
513 ldw 16(%r25), %r19
514 ldw 20(%r25), %r20
515 ldw 24(%r25), %r21
516 ldw 28(%r25), %r22
517 stw %r19, 16(%r26)
518 stw %r20, 20(%r26)
519 stw %r21, 24(%r26)
520 stw %r22, 28(%r26)
521 ldw 32(%r25), %r19
522 ldw 36(%r25), %r20
523 ldw 40(%r25), %r21
524 ldw 44(%r25), %r22
525 stw %r19, 32(%r26)
526 stw %r20, 36(%r26)
527 stw %r21, 40(%r26)
528 stw %r22, 44(%r26)
529 ldw 48(%r25), %r19
530 ldw 52(%r25), %r20
531 ldw 56(%r25), %r21
532 ldw 60(%r25), %r22
533 stw %r19, 48(%r26)
534 stw %r20, 52(%r26)
535 ldo 64(%r25), %r25
536 stw %r21, 56(%r26)
537 stw %r22, 60(%r26)
538 ldo 64(%r26), %r26
539 addib,COND(>),n -1, %r1, 1b
540 ldw 0(%r25), %r19
541#endif
542 bv %r0(%r2)
543 nop
544 .exit
545
546 .procend
547ENDPROC_CFI(copy_page_asm)
548
549/*
550 * NOTE: Code in clear_user_page has a hard coded dependency on the
551 * maximum alias boundary being 4 Mb. We've been assured by the
552 * parisc chip designers that there will not ever be a parisc
553 * chip with a larger alias boundary (Never say never :-) ).
554 *
555 * Subtle: the dtlb miss handlers support the temp alias region by
556 * "knowing" that if a dtlb miss happens within the temp alias
557 * region it must have occurred while in clear_user_page. Since
558 * this routine makes use of processor local translations, we
559 * don't want to insert them into the kernel page table. Instead,
560 * we load up some general registers (they need to be registers
561 * which aren't shadowed) with the physical page numbers (preshifted
562 * for tlb insertion) needed to insert the translations. When we
563 * miss on the translation, the dtlb miss handler inserts the
564 * translation into the tlb using these values:
565 *
566 * %r26 physical page (shifted for tlb insert) of "to" translation
567 * %r23 physical page (shifted for tlb insert) of "from" translation
568 */
569
570 /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
571 #define PAGE_ADD_SHIFT (PAGE_SHIFT-12)
572 .macro convert_phys_for_tlb_insert20 phys
573 extrd,u \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
574#if _PAGE_SIZE_ENCODING_DEFAULT
575 depdi _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
576#endif
577 .endm
578
579 /*
580 * copy_user_page_asm() performs a page copy using mappings
581 * equivalent to the user page mappings. It can be used to
582 * implement copy_user_page() but unfortunately both the `from'
583 * and `to' pages need to be flushed through mappings equivalent
584 * to the user mappings after the copy because the kernel accesses
585 * the `from' page through the kmap kernel mapping and the `to'
586 * page needs to be flushed since code can be copied. As a
587 * result, this implementation is less efficient than the simpler
588 * copy using the kernel mapping. It only needs the `from' page
589 * to flushed via the user mapping. The kunmap routines handle
590 * the flushes needed for the kernel mapping.
591 *
592 * I'm still keeping this around because it may be possible to
593 * use it if more information is passed into copy_user_page().
594 * Have to do some measurements to see if it is worthwhile to
595 * lobby for such a change.
596 *
597 */
598
599ENTRY_CFI(copy_user_page_asm)
600 .proc
601 .callinfo NO_CALLS
602 .entry
603
604 /* Convert virtual `to' and `from' addresses to physical addresses.
605 Move `from' physical address to non shadowed register. */
606 ldil L%(__PAGE_OFFSET), %r1
607 sub %r26, %r1, %r26
608 sub %r25, %r1, %r23
609
610 ldil L%(TMPALIAS_MAP_START), %r28
611#ifdef CONFIG_64BIT
612#if (TMPALIAS_MAP_START >= 0x80000000)
613 depdi 0, 31,32, %r28 /* clear any sign extension */
614#endif
615 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
616 convert_phys_for_tlb_insert20 %r23 /* convert phys addr to tlb insert format */
617 depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
618 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
619 copy %r28, %r29
620 depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
621#else
622 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
623 extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
624 depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
625 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
626 copy %r28, %r29
627 depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
628#endif
629
630 /* Purge any old translations */
631
632#ifdef CONFIG_PA20
633 pdtlb,l %r0(%r28)
634 pdtlb,l %r0(%r29)
635#else
636 tlb_lock %r20,%r21,%r22
637 pdtlb %r0(%r28)
638 pdtlb %r0(%r29)
639 tlb_unlock %r20,%r21,%r22
640#endif
641
642#ifdef CONFIG_64BIT
643 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
644 * Unroll the loop by hand and arrange insn appropriately.
645 * GCC probably can do this just as well.
646 */
647
648 ldd 0(%r29), %r19
649 ldi (PAGE_SIZE / 128), %r1
650
6511: ldd 8(%r29), %r20
652
653 ldd 16(%r29), %r21
654 ldd 24(%r29), %r22
655 std %r19, 0(%r28)
656 std %r20, 8(%r28)
657
658 ldd 32(%r29), %r19
659 ldd 40(%r29), %r20
660 std %r21, 16(%r28)
661 std %r22, 24(%r28)
662
663 ldd 48(%r29), %r21
664 ldd 56(%r29), %r22
665 std %r19, 32(%r28)
666 std %r20, 40(%r28)
667
668 ldd 64(%r29), %r19
669 ldd 72(%r29), %r20
670 std %r21, 48(%r28)
671 std %r22, 56(%r28)
672
673 ldd 80(%r29), %r21
674 ldd 88(%r29), %r22
675 std %r19, 64(%r28)
676 std %r20, 72(%r28)
677
678 ldd 96(%r29), %r19
679 ldd 104(%r29), %r20
680 std %r21, 80(%r28)
681 std %r22, 88(%r28)
682
683 ldd 112(%r29), %r21
684 ldd 120(%r29), %r22
685 std %r19, 96(%r28)
686 std %r20, 104(%r28)
687
688 ldo 128(%r29), %r29
689 std %r21, 112(%r28)
690 std %r22, 120(%r28)
691 ldo 128(%r28), %r28
692
693 /* conditional branches nullify on forward taken branch, and on
694 * non-taken backward branch. Note that .+4 is a backwards branch.
695 * The ldd should only get executed if the branch is taken.
696 */
697 addib,COND(>),n -1, %r1, 1b /* bundle 10 */
698 ldd 0(%r29), %r19 /* start next loads */
699
700#else
701 ldi (PAGE_SIZE / 64), %r1
702
703 /*
704 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
705 * bundles (very restricted rules for bundling). It probably
706 * does OK on PCXU and better, but we could do better with
707 * ldd/std instructions. Note that until (if) we start saving
708 * the full 64 bit register values on interrupt, we can't
709 * use ldd/std on a 32 bit kernel.
710 */
711
7121: ldw 0(%r29), %r19
713 ldw 4(%r29), %r20
714 ldw 8(%r29), %r21
715 ldw 12(%r29), %r22
716 stw %r19, 0(%r28)
717 stw %r20, 4(%r28)
718 stw %r21, 8(%r28)
719 stw %r22, 12(%r28)
720 ldw 16(%r29), %r19
721 ldw 20(%r29), %r20
722 ldw 24(%r29), %r21
723 ldw 28(%r29), %r22
724 stw %r19, 16(%r28)
725 stw %r20, 20(%r28)
726 stw %r21, 24(%r28)
727 stw %r22, 28(%r28)
728 ldw 32(%r29), %r19
729 ldw 36(%r29), %r20
730 ldw 40(%r29), %r21
731 ldw 44(%r29), %r22
732 stw %r19, 32(%r28)
733 stw %r20, 36(%r28)
734 stw %r21, 40(%r28)
735 stw %r22, 44(%r28)
736 ldw 48(%r29), %r19
737 ldw 52(%r29), %r20
738 ldw 56(%r29), %r21
739 ldw 60(%r29), %r22
740 stw %r19, 48(%r28)
741 stw %r20, 52(%r28)
742 stw %r21, 56(%r28)
743 stw %r22, 60(%r28)
744 ldo 64(%r28), %r28
745
746 addib,COND(>) -1, %r1,1b
747 ldo 64(%r29), %r29
748#endif
749
750 bv %r0(%r2)
751 nop
752 .exit
753
754 .procend
755ENDPROC_CFI(copy_user_page_asm)
756
757ENTRY_CFI(clear_user_page_asm)
758 .proc
759 .callinfo NO_CALLS
760 .entry
761
762 tophys_r1 %r26
763
764 ldil L%(TMPALIAS_MAP_START), %r28
765#ifdef CONFIG_64BIT
766#if (TMPALIAS_MAP_START >= 0x80000000)
767 depdi 0, 31,32, %r28 /* clear any sign extension */
768#endif
769 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
770 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
771 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
772#else
773 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
774 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
775 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
776#endif
777
778 /* Purge any old translation */
779
780#ifdef CONFIG_PA20
781 pdtlb,l %r0(%r28)
782#else
783 tlb_lock %r20,%r21,%r22
784 pdtlb %r0(%r28)
785 tlb_unlock %r20,%r21,%r22
786#endif
787
788#ifdef CONFIG_64BIT
789 ldi (PAGE_SIZE / 128), %r1
790
791 /* PREFETCH (Write) has not (yet) been proven to help here */
792 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
793
7941: std %r0, 0(%r28)
795 std %r0, 8(%r28)
796 std %r0, 16(%r28)
797 std %r0, 24(%r28)
798 std %r0, 32(%r28)
799 std %r0, 40(%r28)
800 std %r0, 48(%r28)
801 std %r0, 56(%r28)
802 std %r0, 64(%r28)
803 std %r0, 72(%r28)
804 std %r0, 80(%r28)
805 std %r0, 88(%r28)
806 std %r0, 96(%r28)
807 std %r0, 104(%r28)
808 std %r0, 112(%r28)
809 std %r0, 120(%r28)
810 addib,COND(>) -1, %r1, 1b
811 ldo 128(%r28), %r28
812
813#else /* ! CONFIG_64BIT */
814 ldi (PAGE_SIZE / 64), %r1
815
8161: stw %r0, 0(%r28)
817 stw %r0, 4(%r28)
818 stw %r0, 8(%r28)
819 stw %r0, 12(%r28)
820 stw %r0, 16(%r28)
821 stw %r0, 20(%r28)
822 stw %r0, 24(%r28)
823 stw %r0, 28(%r28)
824 stw %r0, 32(%r28)
825 stw %r0, 36(%r28)
826 stw %r0, 40(%r28)
827 stw %r0, 44(%r28)
828 stw %r0, 48(%r28)
829 stw %r0, 52(%r28)
830 stw %r0, 56(%r28)
831 stw %r0, 60(%r28)
832 addib,COND(>) -1, %r1, 1b
833 ldo 64(%r28), %r28
834#endif /* CONFIG_64BIT */
835
836 bv %r0(%r2)
837 nop
838 .exit
839
840 .procend
841ENDPROC_CFI(clear_user_page_asm)
842
843ENTRY_CFI(flush_dcache_page_asm)
844 .proc
845 .callinfo NO_CALLS
846 .entry
847
848 ldil L%(TMPALIAS_MAP_START), %r28
849#ifdef CONFIG_64BIT
850#if (TMPALIAS_MAP_START >= 0x80000000)
851 depdi 0, 31,32, %r28 /* clear any sign extension */
852#endif
853 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
854 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
855 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
856#else
857 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
858 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
859 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
860#endif
861
862 /* Purge any old translation */
863
864#ifdef CONFIG_PA20
865 pdtlb,l %r0(%r28)
866#else
867 tlb_lock %r20,%r21,%r22
868 pdtlb %r0(%r28)
869 tlb_unlock %r20,%r21,%r22
870#endif
871
872 ldil L%dcache_stride, %r1
873 ldw R%dcache_stride(%r1), r31
874
875#ifdef CONFIG_64BIT
876 depdi,z 1, 63-PAGE_SHIFT,1, %r25
877#else
878 depwi,z 1, 31-PAGE_SHIFT,1, %r25
879#endif
880 add %r28, %r25, %r25
881 sub %r25, r31, %r25
882
883
8841: fdc,m r31(%r28)
885 fdc,m r31(%r28)
886 fdc,m r31(%r28)
887 fdc,m r31(%r28)
888 fdc,m r31(%r28)
889 fdc,m r31(%r28)
890 fdc,m r31(%r28)
891 fdc,m r31(%r28)
892 fdc,m r31(%r28)
893 fdc,m r31(%r28)
894 fdc,m r31(%r28)
895 fdc,m r31(%r28)
896 fdc,m r31(%r28)
897 fdc,m r31(%r28)
898 fdc,m r31(%r28)
899 cmpb,COND(<<) %r28, %r25,1b
900 fdc,m r31(%r28)
901
902 sync
903 bv %r0(%r2)
904 nop
905 .exit
906
907 .procend
908ENDPROC_CFI(flush_dcache_page_asm)
909
910ENTRY_CFI(flush_icache_page_asm)
911 .proc
912 .callinfo NO_CALLS
913 .entry
914
915 ldil L%(TMPALIAS_MAP_START), %r28
916#ifdef CONFIG_64BIT
917#if (TMPALIAS_MAP_START >= 0x80000000)
918 depdi 0, 31,32, %r28 /* clear any sign extension */
919#endif
920 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
921 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
922 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
923#else
924 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
925 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
926 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
927#endif
928
929 /* Purge any old translation. Note that the FIC instruction
930 * may use either the instruction or data TLB. Given that we
931 * have a flat address space, it's not clear which TLB will be
932 * used. So, we purge both entries. */
933
934#ifdef CONFIG_PA20
935 pdtlb,l %r0(%r28)
936 pitlb,l %r0(%sr4,%r28)
937#else
938 tlb_lock %r20,%r21,%r22
939 pdtlb %r0(%r28)
940 pitlb %r0(%sr4,%r28)
941 tlb_unlock %r20,%r21,%r22
942#endif
943
944 ldil L%icache_stride, %r1
945 ldw R%icache_stride(%r1), %r31
946
947#ifdef CONFIG_64BIT
948 depdi,z 1, 63-PAGE_SHIFT,1, %r25
949#else
950 depwi,z 1, 31-PAGE_SHIFT,1, %r25
951#endif
952 add %r28, %r25, %r25
953 sub %r25, %r31, %r25
954
955
956 /* fic only has the type 26 form on PA1.1, requiring an
957 * explicit space specification, so use %sr4 */
9581: fic,m %r31(%sr4,%r28)
959 fic,m %r31(%sr4,%r28)
960 fic,m %r31(%sr4,%r28)
961 fic,m %r31(%sr4,%r28)
962 fic,m %r31(%sr4,%r28)
963 fic,m %r31(%sr4,%r28)
964 fic,m %r31(%sr4,%r28)
965 fic,m %r31(%sr4,%r28)
966 fic,m %r31(%sr4,%r28)
967 fic,m %r31(%sr4,%r28)
968 fic,m %r31(%sr4,%r28)
969 fic,m %r31(%sr4,%r28)
970 fic,m %r31(%sr4,%r28)
971 fic,m %r31(%sr4,%r28)
972 fic,m %r31(%sr4,%r28)
973 cmpb,COND(<<) %r28, %r25,1b
974 fic,m %r31(%sr4,%r28)
975
976 sync
977 bv %r0(%r2)
978 nop
979 .exit
980
981 .procend
982ENDPROC_CFI(flush_icache_page_asm)
983
984ENTRY_CFI(flush_kernel_dcache_page_asm)
985 .proc
986 .callinfo NO_CALLS
987 .entry
988
989 ldil L%dcache_stride, %r1
990 ldw R%dcache_stride(%r1), %r23
991
992#ifdef CONFIG_64BIT
993 depdi,z 1, 63-PAGE_SHIFT,1, %r25
994#else
995 depwi,z 1, 31-PAGE_SHIFT,1, %r25
996#endif
997 add %r26, %r25, %r25
998 sub %r25, %r23, %r25
999
1000
10011: fdc,m %r23(%r26)
1002 fdc,m %r23(%r26)
1003 fdc,m %r23(%r26)
1004 fdc,m %r23(%r26)
1005 fdc,m %r23(%r26)
1006 fdc,m %r23(%r26)
1007 fdc,m %r23(%r26)
1008 fdc,m %r23(%r26)
1009 fdc,m %r23(%r26)
1010 fdc,m %r23(%r26)
1011 fdc,m %r23(%r26)
1012 fdc,m %r23(%r26)
1013 fdc,m %r23(%r26)
1014 fdc,m %r23(%r26)
1015 fdc,m %r23(%r26)
1016 cmpb,COND(<<) %r26, %r25,1b
1017 fdc,m %r23(%r26)
1018
1019 sync
1020 bv %r0(%r2)
1021 nop
1022 .exit
1023
1024 .procend
1025ENDPROC_CFI(flush_kernel_dcache_page_asm)
1026
1027ENTRY_CFI(purge_kernel_dcache_page_asm)
1028 .proc
1029 .callinfo NO_CALLS
1030 .entry
1031
1032 ldil L%dcache_stride, %r1
1033 ldw R%dcache_stride(%r1), %r23
1034
1035#ifdef CONFIG_64BIT
1036 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1037#else
1038 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1039#endif
1040 add %r26, %r25, %r25
1041 sub %r25, %r23, %r25
1042
10431: pdc,m %r23(%r26)
1044 pdc,m %r23(%r26)
1045 pdc,m %r23(%r26)
1046 pdc,m %r23(%r26)
1047 pdc,m %r23(%r26)
1048 pdc,m %r23(%r26)
1049 pdc,m %r23(%r26)
1050 pdc,m %r23(%r26)
1051 pdc,m %r23(%r26)
1052 pdc,m %r23(%r26)
1053 pdc,m %r23(%r26)
1054 pdc,m %r23(%r26)
1055 pdc,m %r23(%r26)
1056 pdc,m %r23(%r26)
1057 pdc,m %r23(%r26)
1058 cmpb,COND(<<) %r26, %r25, 1b
1059 pdc,m %r23(%r26)
1060
1061 sync
1062 bv %r0(%r2)
1063 nop
1064 .exit
1065
1066 .procend
1067ENDPROC_CFI(purge_kernel_dcache_page_asm)
1068
1069ENTRY_CFI(flush_user_dcache_range_asm)
1070 .proc
1071 .callinfo NO_CALLS
1072 .entry
1073
1074 ldil L%dcache_stride, %r1
1075 ldw R%dcache_stride(%r1), %r23
1076 ldo -1(%r23), %r21
1077 ANDCM %r26, %r21, %r26
1078
10791: cmpb,COND(<<),n %r26, %r25, 1b
1080 fdc,m %r23(%sr3, %r26)
1081
1082 sync
1083 bv %r0(%r2)
1084 nop
1085 .exit
1086
1087 .procend
1088ENDPROC_CFI(flush_user_dcache_range_asm)
1089
1090ENTRY_CFI(flush_kernel_dcache_range_asm)
1091 .proc
1092 .callinfo NO_CALLS
1093 .entry
1094
1095 ldil L%dcache_stride, %r1
1096 ldw R%dcache_stride(%r1), %r23
1097 ldo -1(%r23), %r21
1098 ANDCM %r26, %r21, %r26
1099
11001: cmpb,COND(<<),n %r26, %r25,1b
1101 fdc,m %r23(%r26)
1102
1103 sync
1104 syncdma
1105 bv %r0(%r2)
1106 nop
1107 .exit
1108
1109 .procend
1110ENDPROC_CFI(flush_kernel_dcache_range_asm)
1111
1112ENTRY_CFI(purge_kernel_dcache_range_asm)
1113 .proc
1114 .callinfo NO_CALLS
1115 .entry
1116
1117 ldil L%dcache_stride, %r1
1118 ldw R%dcache_stride(%r1), %r23
1119 ldo -1(%r23), %r21
1120 ANDCM %r26, %r21, %r26
1121
11221: cmpb,COND(<<),n %r26, %r25,1b
1123 pdc,m %r23(%r26)
1124
1125 sync
1126 syncdma
1127 bv %r0(%r2)
1128 nop
1129 .exit
1130
1131 .procend
1132ENDPROC_CFI(purge_kernel_dcache_range_asm)
1133
1134ENTRY_CFI(flush_user_icache_range_asm)
1135 .proc
1136 .callinfo NO_CALLS
1137 .entry
1138
1139 ldil L%icache_stride, %r1
1140 ldw R%icache_stride(%r1), %r23
1141 ldo -1(%r23), %r21
1142 ANDCM %r26, %r21, %r26
1143
11441: cmpb,COND(<<),n %r26, %r25,1b
1145 fic,m %r23(%sr3, %r26)
1146
1147 sync
1148 bv %r0(%r2)
1149 nop
1150 .exit
1151
1152 .procend
1153ENDPROC_CFI(flush_user_icache_range_asm)
1154
1155ENTRY_CFI(flush_kernel_icache_page)
1156 .proc
1157 .callinfo NO_CALLS
1158 .entry
1159
1160 ldil L%icache_stride, %r1
1161 ldw R%icache_stride(%r1), %r23
1162
1163#ifdef CONFIG_64BIT
1164 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1165#else
1166 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1167#endif
1168 add %r26, %r25, %r25
1169 sub %r25, %r23, %r25
1170
1171
11721: fic,m %r23(%sr4, %r26)
1173 fic,m %r23(%sr4, %r26)
1174 fic,m %r23(%sr4, %r26)
1175 fic,m %r23(%sr4, %r26)
1176 fic,m %r23(%sr4, %r26)
1177 fic,m %r23(%sr4, %r26)
1178 fic,m %r23(%sr4, %r26)
1179 fic,m %r23(%sr4, %r26)
1180 fic,m %r23(%sr4, %r26)
1181 fic,m %r23(%sr4, %r26)
1182 fic,m %r23(%sr4, %r26)
1183 fic,m %r23(%sr4, %r26)
1184 fic,m %r23(%sr4, %r26)
1185 fic,m %r23(%sr4, %r26)
1186 fic,m %r23(%sr4, %r26)
1187 cmpb,COND(<<) %r26, %r25, 1b
1188 fic,m %r23(%sr4, %r26)
1189
1190 sync
1191 bv %r0(%r2)
1192 nop
1193 .exit
1194
1195 .procend
1196ENDPROC_CFI(flush_kernel_icache_page)
1197
1198ENTRY_CFI(flush_kernel_icache_range_asm)
1199 .proc
1200 .callinfo NO_CALLS
1201 .entry
1202
1203 ldil L%icache_stride, %r1
1204 ldw R%icache_stride(%r1), %r23
1205 ldo -1(%r23), %r21
1206 ANDCM %r26, %r21, %r26
1207
12081: cmpb,COND(<<),n %r26, %r25, 1b
1209 fic,m %r23(%sr4, %r26)
1210
1211 sync
1212 bv %r0(%r2)
1213 nop
1214 .exit
1215 .procend
1216ENDPROC_CFI(flush_kernel_icache_range_asm)
1217
1218 __INIT
1219
1220 /* align should cover use of rfi in disable_sr_hashing_asm and
1221 * srdis_done.
1222 */
1223 .align 256
1224ENTRY_CFI(disable_sr_hashing_asm)
1225 .proc
1226 .callinfo NO_CALLS
1227 .entry
1228
1229 /*
1230 * Switch to real mode
1231 */
1232 /* pcxt_ssm_bug */
1233 rsm PSW_SM_I, %r0
1234 load32 PA(1f), %r1
1235 nop
1236 nop
1237 nop
1238 nop
1239 nop
1240
1241 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1242 mtctl %r0, %cr17 /* Clear IIASQ tail */
1243 mtctl %r0, %cr17 /* Clear IIASQ head */
1244 mtctl %r1, %cr18 /* IIAOQ head */
1245 ldo 4(%r1), %r1
1246 mtctl %r1, %cr18 /* IIAOQ tail */
1247 load32 REAL_MODE_PSW, %r1
1248 mtctl %r1, %ipsw
1249 rfi
1250 nop
1251
12521: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1253 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1254 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1255 b,n srdis_done
1256
1257srdis_pcxs:
1258
1259 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1260
1261 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1262 .word 0x141c1a00 /* must issue twice */
1263 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1264 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1265 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1266 .word 0x141c1600 /* must issue twice */
1267 b,n srdis_done
1268
1269srdis_pcxl:
1270
1271 /* Disable Space Register Hashing for PCXL */
1272
1273 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1274 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1275 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1276 b,n srdis_done
1277
1278srdis_pa20:
1279
1280 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1281
1282 .word 0x144008bc /* mfdiag %dr2, %r28 */
1283 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1284 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1285
1286
1287srdis_done:
1288 /* Switch back to virtual mode */
1289 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1290 load32 2f, %r1
1291 nop
1292 nop
1293 nop
1294 nop
1295 nop
1296
1297 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1298 mtctl %r0, %cr17 /* Clear IIASQ tail */
1299 mtctl %r0, %cr17 /* Clear IIASQ head */
1300 mtctl %r1, %cr18 /* IIAOQ head */
1301 ldo 4(%r1), %r1
1302 mtctl %r1, %cr18 /* IIAOQ tail */
1303 load32 KERNEL_PSW, %r1
1304 mtctl %r1, %ipsw
1305 rfi
1306 nop
1307
13082: bv %r0(%r2)
1309 nop
1310 .exit
1311
1312 .procend
1313ENDPROC_CFI(disable_sr_hashing_asm)
1314
1315 .end