Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * This file contains miscellaneous low-level functions.
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 *
6 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
7 * and Paul Mackerras.
8 *
9 */
10
11#include <linux/sys.h>
12#include <asm/unistd.h>
13#include <asm/errno.h>
14#include <asm/reg.h>
15#include <asm/page.h>
16#include <asm/cache.h>
17#include <asm/cputable.h>
18#include <asm/mmu.h>
19#include <asm/ppc_asm.h>
20#include <asm/thread_info.h>
21#include <asm/asm-offsets.h>
22#include <asm/processor.h>
23#include <asm/bug.h>
24#include <asm/ptrace.h>
25#include <asm/export.h>
26#include <asm/feature-fixups.h>
27
28 .text
29
30/*
31 * We store the saved ksp_limit in the unused part
32 * of the STACK_FRAME_OVERHEAD
33 */
34_GLOBAL(call_do_softirq)
35 mflr r0
36 stw r0,4(r1)
37 lwz r10,THREAD+KSP_LIMIT(r2)
38 stw r3, THREAD+KSP_LIMIT(r2)
39 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
40 mr r1,r3
41 stw r10,8(r1)
42 bl __do_softirq
43 lwz r10,8(r1)
44 lwz r1,0(r1)
45 lwz r0,4(r1)
46 stw r10,THREAD+KSP_LIMIT(r2)
47 mtlr r0
48 blr
49
50/*
51 * void call_do_irq(struct pt_regs *regs, void *sp);
52 */
53_GLOBAL(call_do_irq)
54 mflr r0
55 stw r0,4(r1)
56 lwz r10,THREAD+KSP_LIMIT(r2)
57 stw r4, THREAD+KSP_LIMIT(r2)
58 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
59 mr r1,r4
60 stw r10,8(r1)
61 bl __do_irq
62 lwz r10,8(r1)
63 lwz r1,0(r1)
64 lwz r0,4(r1)
65 stw r10,THREAD+KSP_LIMIT(r2)
66 mtlr r0
67 blr
68
69/*
70 * This returns the high 64 bits of the product of two 64-bit numbers.
71 */
72_GLOBAL(mulhdu)
73 cmpwi r6,0
74 cmpwi cr1,r3,0
75 mr r10,r4
76 mulhwu r4,r4,r5
77 beq 1f
78 mulhwu r0,r10,r6
79 mullw r7,r10,r5
80 addc r7,r0,r7
81 addze r4,r4
821: beqlr cr1 /* all done if high part of A is 0 */
83 mullw r9,r3,r5
84 mulhwu r10,r3,r5
85 beq 2f
86 mullw r0,r3,r6
87 mulhwu r8,r3,r6
88 addc r7,r0,r7
89 adde r4,r4,r8
90 addze r10,r10
912: addc r4,r4,r9
92 addze r3,r10
93 blr
94
95/*
96 * reloc_got2 runs through the .got2 section adding an offset
97 * to each entry.
98 */
99_GLOBAL(reloc_got2)
100 mflr r11
101 lis r7,__got2_start@ha
102 addi r7,r7,__got2_start@l
103 lis r8,__got2_end@ha
104 addi r8,r8,__got2_end@l
105 subf r8,r7,r8
106 srwi. r8,r8,2
107 beqlr
108 mtctr r8
109 bl 1f
1101: mflr r0
111 lis r4,1b@ha
112 addi r4,r4,1b@l
113 subf r0,r4,r0
114 add r7,r0,r7
1152: lwz r0,0(r7)
116 add r0,r0,r3
117 stw r0,0(r7)
118 addi r7,r7,4
119 bdnz 2b
120 mtlr r11
121 blr
122
123/*
124 * call_setup_cpu - call the setup_cpu function for this cpu
125 * r3 = data offset, r24 = cpu number
126 *
127 * Setup function is called with:
128 * r3 = data offset
129 * r4 = ptr to CPU spec (relocated)
130 */
131_GLOBAL(call_setup_cpu)
132 addis r4,r3,cur_cpu_spec@ha
133 addi r4,r4,cur_cpu_spec@l
134 lwz r4,0(r4)
135 add r4,r4,r3
136 lwz r5,CPU_SPEC_SETUP(r4)
137 cmpwi 0,r5,0
138 add r5,r5,r3
139 beqlr
140 mtctr r5
141 bctr
142
143#if defined(CONFIG_CPU_FREQ_PMAC) && defined(CONFIG_PPC_BOOK3S_32)
144
145/* This gets called by via-pmu.c to switch the PLL selection
146 * on 750fx CPU. This function should really be moved to some
147 * other place (as most of the cpufreq code in via-pmu
148 */
149_GLOBAL(low_choose_750fx_pll)
150 /* Clear MSR:EE */
151 mfmsr r7
152 rlwinm r0,r7,0,17,15
153 mtmsr r0
154
155 /* If switching to PLL1, disable HID0:BTIC */
156 cmplwi cr0,r3,0
157 beq 1f
158 mfspr r5,SPRN_HID0
159 rlwinm r5,r5,0,27,25
160 sync
161 mtspr SPRN_HID0,r5
162 isync
163 sync
164
1651:
166 /* Calc new HID1 value */
167 mfspr r4,SPRN_HID1 /* Build a HID1:PS bit from parameter */
168 rlwinm r5,r3,16,15,15 /* Clear out HID1:PS from value read */
169 rlwinm r4,r4,0,16,14 /* Could have I used rlwimi here ? */
170 or r4,r4,r5
171 mtspr SPRN_HID1,r4
172
173#ifdef CONFIG_SMP
174 /* Store new HID1 image */
175 lwz r6,TASK_CPU(r2)
176 slwi r6,r6,2
177#else
178 li r6, 0
179#endif
180 addis r6,r6,nap_save_hid1@ha
181 stw r4,nap_save_hid1@l(r6)
182
183 /* If switching to PLL0, enable HID0:BTIC */
184 cmplwi cr0,r3,0
185 bne 1f
186 mfspr r5,SPRN_HID0
187 ori r5,r5,HID0_BTIC
188 sync
189 mtspr SPRN_HID0,r5
190 isync
191 sync
192
1931:
194 /* Return */
195 mtmsr r7
196 blr
197
198_GLOBAL(low_choose_7447a_dfs)
199 /* Clear MSR:EE */
200 mfmsr r7
201 rlwinm r0,r7,0,17,15
202 mtmsr r0
203
204 /* Calc new HID1 value */
205 mfspr r4,SPRN_HID1
206 insrwi r4,r3,1,9 /* insert parameter into bit 9 */
207 sync
208 mtspr SPRN_HID1,r4
209 sync
210 isync
211
212 /* Return */
213 mtmsr r7
214 blr
215
216#endif /* CONFIG_CPU_FREQ_PMAC && CONFIG_PPC_BOOK3S_32 */
217
218/*
219 * complement mask on the msr then "or" some values on.
220 * _nmask_and_or_msr(nmask, value_to_or)
221 */
222_GLOBAL(_nmask_and_or_msr)
223 mfmsr r0 /* Get current msr */
224 andc r0,r0,r3 /* And off the bits set in r3 (first parm) */
225 or r0,r0,r4 /* Or on the bits in r4 (second parm) */
226 SYNC /* Some chip revs have problems here... */
227 mtmsr r0 /* Update machine state */
228 isync
229 blr /* Done */
230
231#ifdef CONFIG_40x
232
233/*
234 * Do an IO access in real mode
235 */
236_GLOBAL(real_readb)
237 mfmsr r7
238 rlwinm r0,r7,0,~MSR_DR
239 sync
240 mtmsr r0
241 sync
242 isync
243 lbz r3,0(r3)
244 sync
245 mtmsr r7
246 sync
247 isync
248 blr
249_ASM_NOKPROBE_SYMBOL(real_readb)
250
251 /*
252 * Do an IO access in real mode
253 */
254_GLOBAL(real_writeb)
255 mfmsr r7
256 rlwinm r0,r7,0,~MSR_DR
257 sync
258 mtmsr r0
259 sync
260 isync
261 stb r3,0(r4)
262 sync
263 mtmsr r7
264 sync
265 isync
266 blr
267_ASM_NOKPROBE_SYMBOL(real_writeb)
268
269#endif /* CONFIG_40x */
270
271
272/*
273 * Flush instruction cache.
274 * This is a no-op on the 601.
275 */
276#ifndef CONFIG_PPC_8xx
277_GLOBAL(flush_instruction_cache)
278#if defined(CONFIG_4xx)
279 lis r3, KERNELBASE@h
280 iccci 0,r3
281#elif defined(CONFIG_FSL_BOOKE)
282#ifdef CONFIG_E200
283 mfspr r3,SPRN_L1CSR0
284 ori r3,r3,L1CSR0_CFI|L1CSR0_CLFC
285 /* msync; isync recommended here */
286 mtspr SPRN_L1CSR0,r3
287 isync
288 blr
289#endif
290 mfspr r3,SPRN_L1CSR1
291 ori r3,r3,L1CSR1_ICFI|L1CSR1_ICLFR
292 mtspr SPRN_L1CSR1,r3
293#elif defined(CONFIG_PPC_BOOK3S_601)
294 blr /* for 601, do nothing */
295#else
296 /* 603/604 processor - use invalidate-all bit in HID0 */
297 mfspr r3,SPRN_HID0
298 ori r3,r3,HID0_ICFI
299 mtspr SPRN_HID0,r3
300#endif /* CONFIG_4xx */
301 isync
302 blr
303EXPORT_SYMBOL(flush_instruction_cache)
304#endif /* CONFIG_PPC_8xx */
305
306/*
307 * Copy a whole page. We use the dcbz instruction on the destination
308 * to reduce memory traffic (it eliminates the unnecessary reads of
309 * the destination into cache). This requires that the destination
310 * is cacheable.
311 */
312#define COPY_16_BYTES \
313 lwz r6,4(r4); \
314 lwz r7,8(r4); \
315 lwz r8,12(r4); \
316 lwzu r9,16(r4); \
317 stw r6,4(r3); \
318 stw r7,8(r3); \
319 stw r8,12(r3); \
320 stwu r9,16(r3)
321
322_GLOBAL(copy_page)
323 rlwinm r5, r3, 0, L1_CACHE_BYTES - 1
324 addi r3,r3,-4
325
3260: twnei r5, 0 /* WARN if r3 is not cache aligned */
327 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
328
329 addi r4,r4,-4
330
331 li r5,4
332
333#if MAX_COPY_PREFETCH > 1
334 li r0,MAX_COPY_PREFETCH
335 li r11,4
336 mtctr r0
33711: dcbt r11,r4
338 addi r11,r11,L1_CACHE_BYTES
339 bdnz 11b
340#else /* MAX_COPY_PREFETCH == 1 */
341 dcbt r5,r4
342 li r11,L1_CACHE_BYTES+4
343#endif /* MAX_COPY_PREFETCH */
344 li r0,PAGE_SIZE/L1_CACHE_BYTES - MAX_COPY_PREFETCH
345 crclr 4*cr0+eq
3462:
347 mtctr r0
3481:
349 dcbt r11,r4
350 dcbz r5,r3
351 COPY_16_BYTES
352#if L1_CACHE_BYTES >= 32
353 COPY_16_BYTES
354#if L1_CACHE_BYTES >= 64
355 COPY_16_BYTES
356 COPY_16_BYTES
357#if L1_CACHE_BYTES >= 128
358 COPY_16_BYTES
359 COPY_16_BYTES
360 COPY_16_BYTES
361 COPY_16_BYTES
362#endif
363#endif
364#endif
365 bdnz 1b
366 beqlr
367 crnot 4*cr0+eq,4*cr0+eq
368 li r0,MAX_COPY_PREFETCH
369 li r11,4
370 b 2b
371EXPORT_SYMBOL(copy_page)
372
373/*
374 * Extended precision shifts.
375 *
376 * Updated to be valid for shift counts from 0 to 63 inclusive.
377 * -- Gabriel
378 *
379 * R3/R4 has 64 bit value
380 * R5 has shift count
381 * result in R3/R4
382 *
383 * ashrdi3: arithmetic right shift (sign propagation)
384 * lshrdi3: logical right shift
385 * ashldi3: left shift
386 */
387_GLOBAL(__ashrdi3)
388 subfic r6,r5,32
389 srw r4,r4,r5 # LSW = count > 31 ? 0 : LSW >> count
390 addi r7,r5,32 # could be xori, or addi with -32
391 slw r6,r3,r6 # t1 = count > 31 ? 0 : MSW << (32-count)
392 rlwinm r8,r7,0,32 # t3 = (count < 32) ? 32 : 0
393 sraw r7,r3,r7 # t2 = MSW >> (count-32)
394 or r4,r4,r6 # LSW |= t1
395 slw r7,r7,r8 # t2 = (count < 32) ? 0 : t2
396 sraw r3,r3,r5 # MSW = MSW >> count
397 or r4,r4,r7 # LSW |= t2
398 blr
399EXPORT_SYMBOL(__ashrdi3)
400
401_GLOBAL(__ashldi3)
402 subfic r6,r5,32
403 slw r3,r3,r5 # MSW = count > 31 ? 0 : MSW << count
404 addi r7,r5,32 # could be xori, or addi with -32
405 srw r6,r4,r6 # t1 = count > 31 ? 0 : LSW >> (32-count)
406 slw r7,r4,r7 # t2 = count < 32 ? 0 : LSW << (count-32)
407 or r3,r3,r6 # MSW |= t1
408 slw r4,r4,r5 # LSW = LSW << count
409 or r3,r3,r7 # MSW |= t2
410 blr
411EXPORT_SYMBOL(__ashldi3)
412
413_GLOBAL(__lshrdi3)
414 subfic r6,r5,32
415 srw r4,r4,r5 # LSW = count > 31 ? 0 : LSW >> count
416 addi r7,r5,32 # could be xori, or addi with -32
417 slw r6,r3,r6 # t1 = count > 31 ? 0 : MSW << (32-count)
418 srw r7,r3,r7 # t2 = count < 32 ? 0 : MSW >> (count-32)
419 or r4,r4,r6 # LSW |= t1
420 srw r3,r3,r5 # MSW = MSW >> count
421 or r4,r4,r7 # LSW |= t2
422 blr
423EXPORT_SYMBOL(__lshrdi3)
424
425/*
426 * 64-bit comparison: __cmpdi2(s64 a, s64 b)
427 * Returns 0 if a < b, 1 if a == b, 2 if a > b.
428 */
429_GLOBAL(__cmpdi2)
430 cmpw r3,r5
431 li r3,1
432 bne 1f
433 cmplw r4,r6
434 beqlr
4351: li r3,0
436 bltlr
437 li r3,2
438 blr
439EXPORT_SYMBOL(__cmpdi2)
440/*
441 * 64-bit comparison: __ucmpdi2(u64 a, u64 b)
442 * Returns 0 if a < b, 1 if a == b, 2 if a > b.
443 */
444_GLOBAL(__ucmpdi2)
445 cmplw r3,r5
446 li r3,1
447 bne 1f
448 cmplw r4,r6
449 beqlr
4501: li r3,0
451 bltlr
452 li r3,2
453 blr
454EXPORT_SYMBOL(__ucmpdi2)
455
456_GLOBAL(__bswapdi2)
457 rotlwi r9,r4,8
458 rotlwi r10,r3,8
459 rlwimi r9,r4,24,0,7
460 rlwimi r10,r3,24,0,7
461 rlwimi r9,r4,24,16,23
462 rlwimi r10,r3,24,16,23
463 mr r3,r9
464 mr r4,r10
465 blr
466EXPORT_SYMBOL(__bswapdi2)
467
468#ifdef CONFIG_SMP
469_GLOBAL(start_secondary_resume)
470 /* Reset stack */
471 rlwinm r1, r1, 0, 0, 31 - THREAD_SHIFT
472 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
473 li r3,0
474 stw r3,0(r1) /* Zero the stack frame pointer */
475 bl start_secondary
476 b .
477#endif /* CONFIG_SMP */
478
479/*
480 * This routine is just here to keep GCC happy - sigh...
481 */
482_GLOBAL(__main)
483 blr
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * This file contains miscellaneous low-level functions.
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 *
6 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
7 * and Paul Mackerras.
8 *
9 * kexec bits:
10 * Copyright (C) 2002-2003 Eric Biederman <ebiederm@xmission.com>
11 * GameCube/ppc32 port Copyright (C) 2004 Albert Herranz
12 * PPC44x port. Copyright (C) 2011, IBM Corporation
13 * Author: Suzuki Poulose <suzuki@in.ibm.com>
14 */
15
16#include <linux/sys.h>
17#include <asm/unistd.h>
18#include <asm/errno.h>
19#include <asm/reg.h>
20#include <asm/page.h>
21#include <asm/cache.h>
22#include <asm/cputable.h>
23#include <asm/mmu.h>
24#include <asm/ppc_asm.h>
25#include <asm/thread_info.h>
26#include <asm/asm-offsets.h>
27#include <asm/processor.h>
28#include <asm/kexec.h>
29#include <asm/bug.h>
30#include <asm/ptrace.h>
31#include <asm/export.h>
32#include <asm/feature-fixups.h>
33
34 .text
35
36/*
37 * We store the saved ksp_limit in the unused part
38 * of the STACK_FRAME_OVERHEAD
39 */
40_GLOBAL(call_do_softirq)
41 mflr r0
42 stw r0,4(r1)
43 lwz r10,THREAD+KSP_LIMIT(r2)
44 stw r3, THREAD+KSP_LIMIT(r2)
45 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
46 mr r1,r3
47 stw r10,8(r1)
48 bl __do_softirq
49 lwz r10,8(r1)
50 lwz r1,0(r1)
51 lwz r0,4(r1)
52 stw r10,THREAD+KSP_LIMIT(r2)
53 mtlr r0
54 blr
55
56/*
57 * void call_do_irq(struct pt_regs *regs, void *sp);
58 */
59_GLOBAL(call_do_irq)
60 mflr r0
61 stw r0,4(r1)
62 lwz r10,THREAD+KSP_LIMIT(r2)
63 stw r4, THREAD+KSP_LIMIT(r2)
64 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
65 mr r1,r4
66 stw r10,8(r1)
67 bl __do_irq
68 lwz r10,8(r1)
69 lwz r1,0(r1)
70 lwz r0,4(r1)
71 stw r10,THREAD+KSP_LIMIT(r2)
72 mtlr r0
73 blr
74
75/*
76 * This returns the high 64 bits of the product of two 64-bit numbers.
77 */
78_GLOBAL(mulhdu)
79 cmpwi r6,0
80 cmpwi cr1,r3,0
81 mr r10,r4
82 mulhwu r4,r4,r5
83 beq 1f
84 mulhwu r0,r10,r6
85 mullw r7,r10,r5
86 addc r7,r0,r7
87 addze r4,r4
881: beqlr cr1 /* all done if high part of A is 0 */
89 mullw r9,r3,r5
90 mulhwu r10,r3,r5
91 beq 2f
92 mullw r0,r3,r6
93 mulhwu r8,r3,r6
94 addc r7,r0,r7
95 adde r4,r4,r8
96 addze r10,r10
972: addc r4,r4,r9
98 addze r3,r10
99 blr
100
101/*
102 * reloc_got2 runs through the .got2 section adding an offset
103 * to each entry.
104 */
105_GLOBAL(reloc_got2)
106 mflr r11
107 lis r7,__got2_start@ha
108 addi r7,r7,__got2_start@l
109 lis r8,__got2_end@ha
110 addi r8,r8,__got2_end@l
111 subf r8,r7,r8
112 srwi. r8,r8,2
113 beqlr
114 mtctr r8
115 bl 1f
1161: mflr r0
117 lis r4,1b@ha
118 addi r4,r4,1b@l
119 subf r0,r4,r0
120 add r7,r0,r7
1212: lwz r0,0(r7)
122 add r0,r0,r3
123 stw r0,0(r7)
124 addi r7,r7,4
125 bdnz 2b
126 mtlr r11
127 blr
128
129/*
130 * call_setup_cpu - call the setup_cpu function for this cpu
131 * r3 = data offset, r24 = cpu number
132 *
133 * Setup function is called with:
134 * r3 = data offset
135 * r4 = ptr to CPU spec (relocated)
136 */
137_GLOBAL(call_setup_cpu)
138 addis r4,r3,cur_cpu_spec@ha
139 addi r4,r4,cur_cpu_spec@l
140 lwz r4,0(r4)
141 add r4,r4,r3
142 lwz r5,CPU_SPEC_SETUP(r4)
143 cmpwi 0,r5,0
144 add r5,r5,r3
145 beqlr
146 mtctr r5
147 bctr
148
149#if defined(CONFIG_CPU_FREQ_PMAC) && defined(CONFIG_PPC_BOOK3S_32)
150
151/* This gets called by via-pmu.c to switch the PLL selection
152 * on 750fx CPU. This function should really be moved to some
153 * other place (as most of the cpufreq code in via-pmu
154 */
155_GLOBAL(low_choose_750fx_pll)
156 /* Clear MSR:EE */
157 mfmsr r7
158 rlwinm r0,r7,0,17,15
159 mtmsr r0
160
161 /* If switching to PLL1, disable HID0:BTIC */
162 cmplwi cr0,r3,0
163 beq 1f
164 mfspr r5,SPRN_HID0
165 rlwinm r5,r5,0,27,25
166 sync
167 mtspr SPRN_HID0,r5
168 isync
169 sync
170
1711:
172 /* Calc new HID1 value */
173 mfspr r4,SPRN_HID1 /* Build a HID1:PS bit from parameter */
174 rlwinm r5,r3,16,15,15 /* Clear out HID1:PS from value read */
175 rlwinm r4,r4,0,16,14 /* Could have I used rlwimi here ? */
176 or r4,r4,r5
177 mtspr SPRN_HID1,r4
178
179#ifdef CONFIG_SMP
180 /* Store new HID1 image */
181 lwz r6,TASK_CPU(r2)
182 slwi r6,r6,2
183#else
184 li r6, 0
185#endif
186 addis r6,r6,nap_save_hid1@ha
187 stw r4,nap_save_hid1@l(r6)
188
189 /* If switching to PLL0, enable HID0:BTIC */
190 cmplwi cr0,r3,0
191 bne 1f
192 mfspr r5,SPRN_HID0
193 ori r5,r5,HID0_BTIC
194 sync
195 mtspr SPRN_HID0,r5
196 isync
197 sync
198
1991:
200 /* Return */
201 mtmsr r7
202 blr
203
204_GLOBAL(low_choose_7447a_dfs)
205 /* Clear MSR:EE */
206 mfmsr r7
207 rlwinm r0,r7,0,17,15
208 mtmsr r0
209
210 /* Calc new HID1 value */
211 mfspr r4,SPRN_HID1
212 insrwi r4,r3,1,9 /* insert parameter into bit 9 */
213 sync
214 mtspr SPRN_HID1,r4
215 sync
216 isync
217
218 /* Return */
219 mtmsr r7
220 blr
221
222#endif /* CONFIG_CPU_FREQ_PMAC && CONFIG_PPC_BOOK3S_32 */
223
224/*
225 * complement mask on the msr then "or" some values on.
226 * _nmask_and_or_msr(nmask, value_to_or)
227 */
228_GLOBAL(_nmask_and_or_msr)
229 mfmsr r0 /* Get current msr */
230 andc r0,r0,r3 /* And off the bits set in r3 (first parm) */
231 or r0,r0,r4 /* Or on the bits in r4 (second parm) */
232 SYNC /* Some chip revs have problems here... */
233 mtmsr r0 /* Update machine state */
234 isync
235 blr /* Done */
236
237#ifdef CONFIG_40x
238
239/*
240 * Do an IO access in real mode
241 */
242_GLOBAL(real_readb)
243 mfmsr r7
244 rlwinm r0,r7,0,~MSR_DR
245 sync
246 mtmsr r0
247 sync
248 isync
249 lbz r3,0(r3)
250 sync
251 mtmsr r7
252 sync
253 isync
254 blr
255
256 /*
257 * Do an IO access in real mode
258 */
259_GLOBAL(real_writeb)
260 mfmsr r7
261 rlwinm r0,r7,0,~MSR_DR
262 sync
263 mtmsr r0
264 sync
265 isync
266 stb r3,0(r4)
267 sync
268 mtmsr r7
269 sync
270 isync
271 blr
272
273#endif /* CONFIG_40x */
274
275
276/*
277 * Flush instruction cache.
278 * This is a no-op on the 601.
279 */
280#ifndef CONFIG_PPC_8xx
281_GLOBAL(flush_instruction_cache)
282#if defined(CONFIG_4xx)
283#ifdef CONFIG_403GCX
284 li r3, 512
285 mtctr r3
286 lis r4, KERNELBASE@h
2871: iccci 0, r4
288 addi r4, r4, 16
289 bdnz 1b
290#else
291 lis r3, KERNELBASE@h
292 iccci 0,r3
293#endif
294#elif defined(CONFIG_FSL_BOOKE)
295#ifdef CONFIG_E200
296 mfspr r3,SPRN_L1CSR0
297 ori r3,r3,L1CSR0_CFI|L1CSR0_CLFC
298 /* msync; isync recommended here */
299 mtspr SPRN_L1CSR0,r3
300 isync
301 blr
302#endif
303 mfspr r3,SPRN_L1CSR1
304 ori r3,r3,L1CSR1_ICFI|L1CSR1_ICLFR
305 mtspr SPRN_L1CSR1,r3
306#elif defined(CONFIG_PPC_BOOK3S_601)
307 blr /* for 601, do nothing */
308#else
309 /* 603/604 processor - use invalidate-all bit in HID0 */
310 mfspr r3,SPRN_HID0
311 ori r3,r3,HID0_ICFI
312 mtspr SPRN_HID0,r3
313#endif /* CONFIG_4xx */
314 isync
315 blr
316EXPORT_SYMBOL(flush_instruction_cache)
317#endif /* CONFIG_PPC_8xx */
318
319/*
320 * Write any modified data cache blocks out to memory
321 * and invalidate the corresponding instruction cache blocks.
322 * This is a no-op on the 601.
323 *
324 * flush_icache_range(unsigned long start, unsigned long stop)
325 */
326_GLOBAL(flush_icache_range)
327#if defined(CONFIG_PPC_BOOK3S_601) || defined(CONFIG_E200)
328 PURGE_PREFETCHED_INS
329 blr /* for 601 and e200, do nothing */
330#else
331 rlwinm r3,r3,0,0,31 - L1_CACHE_SHIFT
332 subf r4,r3,r4
333 addi r4,r4,L1_CACHE_BYTES - 1
334 srwi. r4,r4,L1_CACHE_SHIFT
335 beqlr
336 mtctr r4
337 mr r6,r3
3381: dcbst 0,r3
339 addi r3,r3,L1_CACHE_BYTES
340 bdnz 1b
341 sync /* wait for dcbst's to get to ram */
342#ifndef CONFIG_44x
343 mtctr r4
3442: icbi 0,r6
345 addi r6,r6,L1_CACHE_BYTES
346 bdnz 2b
347#else
348 /* Flash invalidate on 44x because we are passed kmapped addresses and
349 this doesn't work for userspace pages due to the virtually tagged
350 icache. Sigh. */
351 iccci 0, r0
352#endif
353 sync /* additional sync needed on g4 */
354 isync
355 blr
356#endif
357_ASM_NOKPROBE_SYMBOL(flush_icache_range)
358EXPORT_SYMBOL(flush_icache_range)
359
360/*
361 * Flush a particular page from the data cache to RAM.
362 * Note: this is necessary because the instruction cache does *not*
363 * snoop from the data cache.
364 * This is a no-op on the 601 and e200 which have a unified cache.
365 *
366 * void __flush_dcache_icache(void *page)
367 */
368_GLOBAL(__flush_dcache_icache)
369#if defined(CONFIG_PPC_BOOK3S_601) || defined(CONFIG_E200)
370 PURGE_PREFETCHED_INS
371 blr
372#else
373 rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */
374 li r4,PAGE_SIZE/L1_CACHE_BYTES /* Number of lines in a page */
375 mtctr r4
376 mr r6,r3
3770: dcbst 0,r3 /* Write line to ram */
378 addi r3,r3,L1_CACHE_BYTES
379 bdnz 0b
380 sync
381#ifdef CONFIG_44x
382 /* We don't flush the icache on 44x. Those have a virtual icache
383 * and we don't have access to the virtual address here (it's
384 * not the page vaddr but where it's mapped in user space). The
385 * flushing of the icache on these is handled elsewhere, when
386 * a change in the address space occurs, before returning to
387 * user space
388 */
389BEGIN_MMU_FTR_SECTION
390 blr
391END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_44x)
392#endif /* CONFIG_44x */
393 mtctr r4
3941: icbi 0,r6
395 addi r6,r6,L1_CACHE_BYTES
396 bdnz 1b
397 sync
398 isync
399 blr
400#endif
401
402#ifndef CONFIG_BOOKE
403/*
404 * Flush a particular page from the data cache to RAM, identified
405 * by its physical address. We turn off the MMU so we can just use
406 * the physical address (this may be a highmem page without a kernel
407 * mapping).
408 *
409 * void __flush_dcache_icache_phys(unsigned long physaddr)
410 */
411_GLOBAL(__flush_dcache_icache_phys)
412#if defined(CONFIG_PPC_BOOK3S_601) || defined(CONFIG_E200)
413 PURGE_PREFETCHED_INS
414 blr /* for 601 and e200, do nothing */
415#else
416 mfmsr r10
417 rlwinm r0,r10,0,28,26 /* clear DR */
418 mtmsr r0
419 isync
420 rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */
421 li r4,PAGE_SIZE/L1_CACHE_BYTES /* Number of lines in a page */
422 mtctr r4
423 mr r6,r3
4240: dcbst 0,r3 /* Write line to ram */
425 addi r3,r3,L1_CACHE_BYTES
426 bdnz 0b
427 sync
428 mtctr r4
4291: icbi 0,r6
430 addi r6,r6,L1_CACHE_BYTES
431 bdnz 1b
432 sync
433 mtmsr r10 /* restore DR */
434 isync
435 blr
436#endif
437#endif /* CONFIG_BOOKE */
438
439/*
440 * Copy a whole page. We use the dcbz instruction on the destination
441 * to reduce memory traffic (it eliminates the unnecessary reads of
442 * the destination into cache). This requires that the destination
443 * is cacheable.
444 */
445#define COPY_16_BYTES \
446 lwz r6,4(r4); \
447 lwz r7,8(r4); \
448 lwz r8,12(r4); \
449 lwzu r9,16(r4); \
450 stw r6,4(r3); \
451 stw r7,8(r3); \
452 stw r8,12(r3); \
453 stwu r9,16(r3)
454
455_GLOBAL(copy_page)
456 rlwinm r5, r3, 0, L1_CACHE_BYTES - 1
457 addi r3,r3,-4
458
4590: twnei r5, 0 /* WARN if r3 is not cache aligned */
460 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
461
462 addi r4,r4,-4
463
464 li r5,4
465
466#if MAX_COPY_PREFETCH > 1
467 li r0,MAX_COPY_PREFETCH
468 li r11,4
469 mtctr r0
47011: dcbt r11,r4
471 addi r11,r11,L1_CACHE_BYTES
472 bdnz 11b
473#else /* MAX_COPY_PREFETCH == 1 */
474 dcbt r5,r4
475 li r11,L1_CACHE_BYTES+4
476#endif /* MAX_COPY_PREFETCH */
477 li r0,PAGE_SIZE/L1_CACHE_BYTES - MAX_COPY_PREFETCH
478 crclr 4*cr0+eq
4792:
480 mtctr r0
4811:
482 dcbt r11,r4
483 dcbz r5,r3
484 COPY_16_BYTES
485#if L1_CACHE_BYTES >= 32
486 COPY_16_BYTES
487#if L1_CACHE_BYTES >= 64
488 COPY_16_BYTES
489 COPY_16_BYTES
490#if L1_CACHE_BYTES >= 128
491 COPY_16_BYTES
492 COPY_16_BYTES
493 COPY_16_BYTES
494 COPY_16_BYTES
495#endif
496#endif
497#endif
498 bdnz 1b
499 beqlr
500 crnot 4*cr0+eq,4*cr0+eq
501 li r0,MAX_COPY_PREFETCH
502 li r11,4
503 b 2b
504EXPORT_SYMBOL(copy_page)
505
506/*
507 * Extended precision shifts.
508 *
509 * Updated to be valid for shift counts from 0 to 63 inclusive.
510 * -- Gabriel
511 *
512 * R3/R4 has 64 bit value
513 * R5 has shift count
514 * result in R3/R4
515 *
516 * ashrdi3: arithmetic right shift (sign propagation)
517 * lshrdi3: logical right shift
518 * ashldi3: left shift
519 */
520_GLOBAL(__ashrdi3)
521 subfic r6,r5,32
522 srw r4,r4,r5 # LSW = count > 31 ? 0 : LSW >> count
523 addi r7,r5,32 # could be xori, or addi with -32
524 slw r6,r3,r6 # t1 = count > 31 ? 0 : MSW << (32-count)
525 rlwinm r8,r7,0,32 # t3 = (count < 32) ? 32 : 0
526 sraw r7,r3,r7 # t2 = MSW >> (count-32)
527 or r4,r4,r6 # LSW |= t1
528 slw r7,r7,r8 # t2 = (count < 32) ? 0 : t2
529 sraw r3,r3,r5 # MSW = MSW >> count
530 or r4,r4,r7 # LSW |= t2
531 blr
532EXPORT_SYMBOL(__ashrdi3)
533
534_GLOBAL(__ashldi3)
535 subfic r6,r5,32
536 slw r3,r3,r5 # MSW = count > 31 ? 0 : MSW << count
537 addi r7,r5,32 # could be xori, or addi with -32
538 srw r6,r4,r6 # t1 = count > 31 ? 0 : LSW >> (32-count)
539 slw r7,r4,r7 # t2 = count < 32 ? 0 : LSW << (count-32)
540 or r3,r3,r6 # MSW |= t1
541 slw r4,r4,r5 # LSW = LSW << count
542 or r3,r3,r7 # MSW |= t2
543 blr
544EXPORT_SYMBOL(__ashldi3)
545
546_GLOBAL(__lshrdi3)
547 subfic r6,r5,32
548 srw r4,r4,r5 # LSW = count > 31 ? 0 : LSW >> count
549 addi r7,r5,32 # could be xori, or addi with -32
550 slw r6,r3,r6 # t1 = count > 31 ? 0 : MSW << (32-count)
551 srw r7,r3,r7 # t2 = count < 32 ? 0 : MSW >> (count-32)
552 or r4,r4,r6 # LSW |= t1
553 srw r3,r3,r5 # MSW = MSW >> count
554 or r4,r4,r7 # LSW |= t2
555 blr
556EXPORT_SYMBOL(__lshrdi3)
557
558/*
559 * 64-bit comparison: __cmpdi2(s64 a, s64 b)
560 * Returns 0 if a < b, 1 if a == b, 2 if a > b.
561 */
562_GLOBAL(__cmpdi2)
563 cmpw r3,r5
564 li r3,1
565 bne 1f
566 cmplw r4,r6
567 beqlr
5681: li r3,0
569 bltlr
570 li r3,2
571 blr
572EXPORT_SYMBOL(__cmpdi2)
573/*
574 * 64-bit comparison: __ucmpdi2(u64 a, u64 b)
575 * Returns 0 if a < b, 1 if a == b, 2 if a > b.
576 */
577_GLOBAL(__ucmpdi2)
578 cmplw r3,r5
579 li r3,1
580 bne 1f
581 cmplw r4,r6
582 beqlr
5831: li r3,0
584 bltlr
585 li r3,2
586 blr
587EXPORT_SYMBOL(__ucmpdi2)
588
589_GLOBAL(__bswapdi2)
590 rotlwi r9,r4,8
591 rotlwi r10,r3,8
592 rlwimi r9,r4,24,0,7
593 rlwimi r10,r3,24,0,7
594 rlwimi r9,r4,24,16,23
595 rlwimi r10,r3,24,16,23
596 mr r3,r9
597 mr r4,r10
598 blr
599EXPORT_SYMBOL(__bswapdi2)
600
601#ifdef CONFIG_SMP
602_GLOBAL(start_secondary_resume)
603 /* Reset stack */
604 rlwinm r1, r1, 0, 0, 31 - THREAD_SHIFT
605 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
606 li r3,0
607 stw r3,0(r1) /* Zero the stack frame pointer */
608 bl start_secondary
609 b .
610#endif /* CONFIG_SMP */
611
612/*
613 * This routine is just here to keep GCC happy - sigh...
614 */
615_GLOBAL(__main)
616 blr
617
618#ifdef CONFIG_KEXEC_CORE
619 /*
620 * Must be relocatable PIC code callable as a C function.
621 */
622 .globl relocate_new_kernel
623relocate_new_kernel:
624 /* r3 = page_list */
625 /* r4 = reboot_code_buffer */
626 /* r5 = start_address */
627
628#ifdef CONFIG_FSL_BOOKE
629
630 mr r29, r3
631 mr r30, r4
632 mr r31, r5
633
634#define ENTRY_MAPPING_KEXEC_SETUP
635#include "fsl_booke_entry_mapping.S"
636#undef ENTRY_MAPPING_KEXEC_SETUP
637
638 mr r3, r29
639 mr r4, r30
640 mr r5, r31
641
642 li r0, 0
643#elif defined(CONFIG_44x)
644
645 /* Save our parameters */
646 mr r29, r3
647 mr r30, r4
648 mr r31, r5
649
650#ifdef CONFIG_PPC_47x
651 /* Check for 47x cores */
652 mfspr r3,SPRN_PVR
653 srwi r3,r3,16
654 cmplwi cr0,r3,PVR_476FPE@h
655 beq setup_map_47x
656 cmplwi cr0,r3,PVR_476@h
657 beq setup_map_47x
658 cmplwi cr0,r3,PVR_476_ISS@h
659 beq setup_map_47x
660#endif /* CONFIG_PPC_47x */
661
662/*
663 * Code for setting up 1:1 mapping for PPC440x for KEXEC
664 *
665 * We cannot switch off the MMU on PPC44x.
666 * So we:
667 * 1) Invalidate all the mappings except the one we are running from.
668 * 2) Create a tmp mapping for our code in the other address space(TS) and
669 * jump to it. Invalidate the entry we started in.
670 * 3) Create a 1:1 mapping for 0-2GiB in chunks of 256M in original TS.
671 * 4) Jump to the 1:1 mapping in original TS.
672 * 5) Invalidate the tmp mapping.
673 *
674 * - Based on the kexec support code for FSL BookE
675 *
676 */
677
678 /*
679 * Load the PID with kernel PID (0).
680 * Also load our MSR_IS and TID to MMUCR for TLB search.
681 */
682 li r3, 0
683 mtspr SPRN_PID, r3
684 mfmsr r4
685 andi. r4,r4,MSR_IS@l
686 beq wmmucr
687 oris r3,r3,PPC44x_MMUCR_STS@h
688wmmucr:
689 mtspr SPRN_MMUCR,r3
690 sync
691
692 /*
693 * Invalidate all the TLB entries except the current entry
694 * where we are running from
695 */
696 bl 0f /* Find our address */
6970: mflr r5 /* Make it accessible */
698 tlbsx r23,0,r5 /* Find entry we are in */
699 li r4,0 /* Start at TLB entry 0 */
700 li r3,0 /* Set PAGEID inval value */
7011: cmpw r23,r4 /* Is this our entry? */
702 beq skip /* If so, skip the inval */
703 tlbwe r3,r4,PPC44x_TLB_PAGEID /* If not, inval the entry */
704skip:
705 addi r4,r4,1 /* Increment */
706 cmpwi r4,64 /* Are we done? */
707 bne 1b /* If not, repeat */
708 isync
709
710 /* Create a temp mapping and jump to it */
711 andi. r6, r23, 1 /* Find the index to use */
712 addi r24, r6, 1 /* r24 will contain 1 or 2 */
713
714 mfmsr r9 /* get the MSR */
715 rlwinm r5, r9, 27, 31, 31 /* Extract the MSR[IS] */
716 xori r7, r5, 1 /* Use the other address space */
717
718 /* Read the current mapping entries */
719 tlbre r3, r23, PPC44x_TLB_PAGEID
720 tlbre r4, r23, PPC44x_TLB_XLAT
721 tlbre r5, r23, PPC44x_TLB_ATTRIB
722
723 /* Save our current XLAT entry */
724 mr r25, r4
725
726 /* Extract the TLB PageSize */
727 li r10, 1 /* r10 will hold PageSize */
728 rlwinm r11, r3, 0, 24, 27 /* bits 24-27 */
729
730 /* XXX: As of now we use 256M, 4K pages */
731 cmpwi r11, PPC44x_TLB_256M
732 bne tlb_4k
733 rotlwi r10, r10, 28 /* r10 = 256M */
734 b write_out
735tlb_4k:
736 cmpwi r11, PPC44x_TLB_4K
737 bne default
738 rotlwi r10, r10, 12 /* r10 = 4K */
739 b write_out
740default:
741 rotlwi r10, r10, 10 /* r10 = 1K */
742
743write_out:
744 /*
745 * Write out the tmp 1:1 mapping for this code in other address space
746 * Fixup EPN = RPN , TS=other address space
747 */
748 insrwi r3, r7, 1, 23 /* Bit 23 is TS for PAGEID field */
749
750 /* Write out the tmp mapping entries */
751 tlbwe r3, r24, PPC44x_TLB_PAGEID
752 tlbwe r4, r24, PPC44x_TLB_XLAT
753 tlbwe r5, r24, PPC44x_TLB_ATTRIB
754
755 subi r11, r10, 1 /* PageOffset Mask = PageSize - 1 */
756 not r10, r11 /* Mask for PageNum */
757
758 /* Switch to other address space in MSR */
759 insrwi r9, r7, 1, 26 /* Set MSR[IS] = r7 */
760
761 bl 1f
7621: mflr r8
763 addi r8, r8, (2f-1b) /* Find the target offset */
764
765 /* Jump to the tmp mapping */
766 mtspr SPRN_SRR0, r8
767 mtspr SPRN_SRR1, r9
768 rfi
769
7702:
771 /* Invalidate the entry we were executing from */
772 li r3, 0
773 tlbwe r3, r23, PPC44x_TLB_PAGEID
774
775 /* attribute fields. rwx for SUPERVISOR mode */
776 li r5, 0
777 ori r5, r5, (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G)
778
779 /* Create 1:1 mapping in 256M pages */
780 xori r7, r7, 1 /* Revert back to Original TS */
781
782 li r8, 0 /* PageNumber */
783 li r6, 3 /* TLB Index, start at 3 */
784
785next_tlb:
786 rotlwi r3, r8, 28 /* Create EPN (bits 0-3) */
787 mr r4, r3 /* RPN = EPN */
788 ori r3, r3, (PPC44x_TLB_VALID | PPC44x_TLB_256M) /* SIZE = 256M, Valid */
789 insrwi r3, r7, 1, 23 /* Set TS from r7 */
790
791 tlbwe r3, r6, PPC44x_TLB_PAGEID /* PageID field : EPN, V, SIZE */
792 tlbwe r4, r6, PPC44x_TLB_XLAT /* Address translation : RPN */
793 tlbwe r5, r6, PPC44x_TLB_ATTRIB /* Attributes */
794
795 addi r8, r8, 1 /* Increment PN */
796 addi r6, r6, 1 /* Increment TLB Index */
797 cmpwi r8, 8 /* Are we done ? */
798 bne next_tlb
799 isync
800
801 /* Jump to the new mapping 1:1 */
802 li r9,0
803 insrwi r9, r7, 1, 26 /* Set MSR[IS] = r7 */
804
805 bl 1f
8061: mflr r8
807 and r8, r8, r11 /* Get our offset within page */
808 addi r8, r8, (2f-1b)
809
810 and r5, r25, r10 /* Get our target PageNum */
811 or r8, r8, r5 /* Target jump address */
812
813 mtspr SPRN_SRR0, r8
814 mtspr SPRN_SRR1, r9
815 rfi
8162:
817 /* Invalidate the tmp entry we used */
818 li r3, 0
819 tlbwe r3, r24, PPC44x_TLB_PAGEID
820 sync
821 b ppc44x_map_done
822
823#ifdef CONFIG_PPC_47x
824
825 /* 1:1 mapping for 47x */
826
827setup_map_47x:
828
829 /*
830 * Load the kernel pid (0) to PID and also to MMUCR[TID].
831 * Also set the MSR IS->MMUCR STS
832 */
833 li r3, 0
834 mtspr SPRN_PID, r3 /* Set PID */
835 mfmsr r4 /* Get MSR */
836 andi. r4, r4, MSR_IS@l /* TS=1? */
837 beq 1f /* If not, leave STS=0 */
838 oris r3, r3, PPC47x_MMUCR_STS@h /* Set STS=1 */
8391: mtspr SPRN_MMUCR, r3 /* Put MMUCR */
840 sync
841
842 /* Find the entry we are running from */
843 bl 2f
8442: mflr r23
845 tlbsx r23, 0, r23
846 tlbre r24, r23, 0 /* TLB Word 0 */
847 tlbre r25, r23, 1 /* TLB Word 1 */
848 tlbre r26, r23, 2 /* TLB Word 2 */
849
850
851 /*
852 * Invalidates all the tlb entries by writing to 256 RPNs(r4)
853 * of 4k page size in all 4 ways (0-3 in r3).
854 * This would invalidate the entire UTLB including the one we are
855 * running from. However the shadow TLB entries would help us
856 * to continue the execution, until we flush them (rfi/isync).
857 */
858 addis r3, 0, 0x8000 /* specify the way */
859 addi r4, 0, 0 /* TLB Word0 = (EPN=0, VALID = 0) */
860 addi r5, 0, 0
861 b clear_utlb_entry
862
863 /* Align the loop to speed things up. from head_44x.S */
864 .align 6
865
866clear_utlb_entry:
867
868 tlbwe r4, r3, 0
869 tlbwe r5, r3, 1
870 tlbwe r5, r3, 2
871 addis r3, r3, 0x2000 /* Increment the way */
872 cmpwi r3, 0
873 bne clear_utlb_entry
874 addis r3, 0, 0x8000
875 addis r4, r4, 0x100 /* Increment the EPN */
876 cmpwi r4, 0
877 bne clear_utlb_entry
878
879 /* Create the entries in the other address space */
880 mfmsr r5
881 rlwinm r7, r5, 27, 31, 31 /* Get the TS (Bit 26) from MSR */
882 xori r7, r7, 1 /* r7 = !TS */
883
884 insrwi r24, r7, 1, 21 /* Change the TS in the saved TLB word 0 */
885
886 /*
887 * write out the TLB entries for the tmp mapping
888 * Use way '0' so that we could easily invalidate it later.
889 */
890 lis r3, 0x8000 /* Way '0' */
891
892 tlbwe r24, r3, 0
893 tlbwe r25, r3, 1
894 tlbwe r26, r3, 2
895
896 /* Update the msr to the new TS */
897 insrwi r5, r7, 1, 26
898
899 bl 1f
9001: mflr r6
901 addi r6, r6, (2f-1b)
902
903 mtspr SPRN_SRR0, r6
904 mtspr SPRN_SRR1, r5
905 rfi
906
907 /*
908 * Now we are in the tmp address space.
909 * Create a 1:1 mapping for 0-2GiB in the original TS.
910 */
9112:
912 li r3, 0
913 li r4, 0 /* TLB Word 0 */
914 li r5, 0 /* TLB Word 1 */
915 li r6, 0
916 ori r6, r6, PPC47x_TLB2_S_RWX /* TLB word 2 */
917
918 li r8, 0 /* PageIndex */
919
920 xori r7, r7, 1 /* revert back to original TS */
921
922write_utlb:
923 rotlwi r5, r8, 28 /* RPN = PageIndex * 256M */
924 /* ERPN = 0 as we don't use memory above 2G */
925
926 mr r4, r5 /* EPN = RPN */
927 ori r4, r4, (PPC47x_TLB0_VALID | PPC47x_TLB0_256M)
928 insrwi r4, r7, 1, 21 /* Insert the TS to Word 0 */
929
930 tlbwe r4, r3, 0 /* Write out the entries */
931 tlbwe r5, r3, 1
932 tlbwe r6, r3, 2
933 addi r8, r8, 1
934 cmpwi r8, 8 /* Have we completed ? */
935 bne write_utlb
936
937 /* make sure we complete the TLB write up */
938 isync
939
940 /*
941 * Prepare to jump to the 1:1 mapping.
942 * 1) Extract page size of the tmp mapping
943 * DSIZ = TLB_Word0[22:27]
944 * 2) Calculate the physical address of the address
945 * to jump to.
946 */
947 rlwinm r10, r24, 0, 22, 27
948
949 cmpwi r10, PPC47x_TLB0_4K
950 bne 0f
951 li r10, 0x1000 /* r10 = 4k */
952 bl 1f
953
9540:
955 /* Defaults to 256M */
956 lis r10, 0x1000
957
958 bl 1f
9591: mflr r4
960 addi r4, r4, (2f-1b) /* virtual address of 2f */
961
962 subi r11, r10, 1 /* offsetmask = Pagesize - 1 */
963 not r10, r11 /* Pagemask = ~(offsetmask) */
964
965 and r5, r25, r10 /* Physical page */
966 and r6, r4, r11 /* offset within the current page */
967
968 or r5, r5, r6 /* Physical address for 2f */
969
970 /* Switch the TS in MSR to the original one */
971 mfmsr r8
972 insrwi r8, r7, 1, 26
973
974 mtspr SPRN_SRR1, r8
975 mtspr SPRN_SRR0, r5
976 rfi
977
9782:
979 /* Invalidate the tmp mapping */
980 lis r3, 0x8000 /* Way '0' */
981
982 clrrwi r24, r24, 12 /* Clear the valid bit */
983 tlbwe r24, r3, 0
984 tlbwe r25, r3, 1
985 tlbwe r26, r3, 2
986
987 /* Make sure we complete the TLB write and flush the shadow TLB */
988 isync
989
990#endif
991
992ppc44x_map_done:
993
994
995 /* Restore the parameters */
996 mr r3, r29
997 mr r4, r30
998 mr r5, r31
999
1000 li r0, 0
1001#else
1002 li r0, 0
1003
1004 /*
1005 * Set Machine Status Register to a known status,
1006 * switch the MMU off and jump to 1: in a single step.
1007 */
1008
1009 mr r8, r0
1010 ori r8, r8, MSR_RI|MSR_ME
1011 mtspr SPRN_SRR1, r8
1012 addi r8, r4, 1f - relocate_new_kernel
1013 mtspr SPRN_SRR0, r8
1014 sync
1015 rfi
1016
10171:
1018#endif
1019 /* from this point address translation is turned off */
1020 /* and interrupts are disabled */
1021
1022 /* set a new stack at the bottom of our page... */
1023 /* (not really needed now) */
1024 addi r1, r4, KEXEC_CONTROL_PAGE_SIZE - 8 /* for LR Save+Back Chain */
1025 stw r0, 0(r1)
1026
1027 /* Do the copies */
1028 li r6, 0 /* checksum */
1029 mr r0, r3
1030 b 1f
1031
10320: /* top, read another word for the indirection page */
1033 lwzu r0, 4(r3)
1034
10351:
1036 /* is it a destination page? (r8) */
1037 rlwinm. r7, r0, 0, 31, 31 /* IND_DESTINATION (1<<0) */
1038 beq 2f
1039
1040 rlwinm r8, r0, 0, 0, 19 /* clear kexec flags, page align */
1041 b 0b
1042
10432: /* is it an indirection page? (r3) */
1044 rlwinm. r7, r0, 0, 30, 30 /* IND_INDIRECTION (1<<1) */
1045 beq 2f
1046
1047 rlwinm r3, r0, 0, 0, 19 /* clear kexec flags, page align */
1048 subi r3, r3, 4
1049 b 0b
1050
10512: /* are we done? */
1052 rlwinm. r7, r0, 0, 29, 29 /* IND_DONE (1<<2) */
1053 beq 2f
1054 b 3f
1055
10562: /* is it a source page? (r9) */
1057 rlwinm. r7, r0, 0, 28, 28 /* IND_SOURCE (1<<3) */
1058 beq 0b
1059
1060 rlwinm r9, r0, 0, 0, 19 /* clear kexec flags, page align */
1061
1062 li r7, PAGE_SIZE / 4
1063 mtctr r7
1064 subi r9, r9, 4
1065 subi r8, r8, 4
10669:
1067 lwzu r0, 4(r9) /* do the copy */
1068 xor r6, r6, r0
1069 stwu r0, 4(r8)
1070 dcbst 0, r8
1071 sync
1072 icbi 0, r8
1073 bdnz 9b
1074
1075 addi r9, r9, 4
1076 addi r8, r8, 4
1077 b 0b
1078
10793:
1080
1081 /* To be certain of avoiding problems with self-modifying code
1082 * execute a serializing instruction here.
1083 */
1084 isync
1085 sync
1086
1087 mfspr r3, SPRN_PIR /* current core we are running on */
1088 mr r4, r5 /* load physical address of chunk called */
1089
1090 /* jump to the entry point, usually the setup routine */
1091 mtlr r5
1092 blrl
1093
10941: b 1b
1095
1096relocate_new_kernel_end:
1097
1098 .globl relocate_new_kernel_size
1099relocate_new_kernel_size:
1100 .long relocate_new_kernel_end - relocate_new_kernel
1101#endif