xref: /openbmc/linux/arch/powerpc/kernel/misc_64.S (revision 6391503b) 
1/*
2 * This file contains miscellaneous low-level functions.
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 */
16
17#include <linux/sys.h>
18#include <asm/unistd.h>
19#include <asm/errno.h>
20#include <asm/processor.h>
21#include <asm/page.h>
22#include <asm/cache.h>
23#include <asm/ppc_asm.h>
24#include <asm/asm-offsets.h>
25#include <asm/cputable.h>
26#include <asm/thread_info.h>
27#include <asm/kexec.h>
28#include <asm/ptrace.h>
29#include <asm/mmu.h>
30
31	.text
32
33_GLOBAL(call_do_softirq)
34	mflr	r0
35	std	r0,16(r1)
36	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
37	mr	r1,r3
38	bl	__do_softirq
39	ld	r1,0(r1)
40	ld	r0,16(r1)
41	mtlr	r0
42	blr
43
44_GLOBAL(call_do_irq)
45	mflr	r0
46	std	r0,16(r1)
47	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
48	mr	r1,r4
49	bl	__do_irq
50	ld	r1,0(r1)
51	ld	r0,16(r1)
52	mtlr	r0
53	blr
54
55	.section	".toc","aw"
56PPC64_CACHES:
57	.tc		ppc64_caches[TC],ppc64_caches
58	.section	".text"
59
60/*
61 * Write any modified data cache blocks out to memory
62 * and invalidate the corresponding instruction cache blocks.
63 *
64 * flush_icache_range(unsigned long start, unsigned long stop)
65 *
66 *   flush all bytes from start through stop-1 inclusive
67 */
68
69_KPROBE(flush_icache_range)
70BEGIN_FTR_SECTION
71	PURGE_PREFETCHED_INS
72	blr
73END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
74/*
75 * Flush the data cache to memory
76 *
77 * Different systems have different cache line sizes
78 * and in some cases i-cache and d-cache line sizes differ from
79 * each other.
80 */
81 	ld	r10,PPC64_CACHES@toc(r2)
82	lwz	r7,DCACHEL1LINESIZE(r10)/* Get cache line size */
83	addi	r5,r7,-1
84	andc	r6,r3,r5		/* round low to line bdy */
85	subf	r8,r6,r4		/* compute length */
86	add	r8,r8,r5		/* ensure we get enough */
87	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of cache line size */
88	srw.	r8,r8,r9		/* compute line count */
89	beqlr				/* nothing to do? */
90	mtctr	r8
911:	dcbst	0,r6
92	add	r6,r6,r7
93	bdnz	1b
94	sync
95
96/* Now invalidate the instruction cache */
97
98	lwz	r7,ICACHEL1LINESIZE(r10)	/* Get Icache line size */
99	addi	r5,r7,-1
100	andc	r6,r3,r5		/* round low to line bdy */
101	subf	r8,r6,r4		/* compute length */
102	add	r8,r8,r5
103	lwz	r9,ICACHEL1LOGLINESIZE(r10)	/* Get log-2 of Icache line size */
104	srw.	r8,r8,r9		/* compute line count */
105	beqlr				/* nothing to do? */
106	mtctr	r8
1072:	icbi	0,r6
108	add	r6,r6,r7
109	bdnz	2b
110	isync
111	blr
112	.previous .text
113/*
114 * Like above, but only do the D-cache.
115 *
116 * flush_dcache_range(unsigned long start, unsigned long stop)
117 *
118 *    flush all bytes from start to stop-1 inclusive
119 */
120_GLOBAL(flush_dcache_range)
121
122/*
123 * Flush the data cache to memory
124 *
125 * Different systems have different cache line sizes
126 */
127 	ld	r10,PPC64_CACHES@toc(r2)
128	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
129	addi	r5,r7,-1
130	andc	r6,r3,r5		/* round low to line bdy */
131	subf	r8,r6,r4		/* compute length */
132	add	r8,r8,r5		/* ensure we get enough */
133	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
134	srw.	r8,r8,r9		/* compute line count */
135	beqlr				/* nothing to do? */
136	mtctr	r8
1370:	dcbst	0,r6
138	add	r6,r6,r7
139	bdnz	0b
140	sync
141	blr
142
143/*
144 * Like above, but works on non-mapped physical addresses.
145 * Use only for non-LPAR setups ! It also assumes real mode
146 * is cacheable. Used for flushing out the DART before using
147 * it as uncacheable memory
148 *
149 * flush_dcache_phys_range(unsigned long start, unsigned long stop)
150 *
151 *    flush all bytes from start to stop-1 inclusive
152 */
153_GLOBAL(flush_dcache_phys_range)
154 	ld	r10,PPC64_CACHES@toc(r2)
155	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
156	addi	r5,r7,-1
157	andc	r6,r3,r5		/* round low to line bdy */
158	subf	r8,r6,r4		/* compute length */
159	add	r8,r8,r5		/* ensure we get enough */
160	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
161	srw.	r8,r8,r9		/* compute line count */
162	beqlr				/* nothing to do? */
163	mfmsr	r5			/* Disable MMU Data Relocation */
164	ori	r0,r5,MSR_DR
165	xori	r0,r0,MSR_DR
166	sync
167	mtmsr	r0
168	sync
169	isync
170	mtctr	r8
1710:	dcbst	0,r6
172	add	r6,r6,r7
173	bdnz	0b
174	sync
175	isync
176	mtmsr	r5			/* Re-enable MMU Data Relocation */
177	sync
178	isync
179	blr
180
181_GLOBAL(flush_inval_dcache_range)
182 	ld	r10,PPC64_CACHES@toc(r2)
183	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
184	addi	r5,r7,-1
185	andc	r6,r3,r5		/* round low to line bdy */
186	subf	r8,r6,r4		/* compute length */
187	add	r8,r8,r5		/* ensure we get enough */
188	lwz	r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */
189	srw.	r8,r8,r9		/* compute line count */
190	beqlr				/* nothing to do? */
191	sync
192	isync
193	mtctr	r8
1940:	dcbf	0,r6
195	add	r6,r6,r7
196	bdnz	0b
197	sync
198	isync
199	blr
200
201
202/*
203 * Flush a particular page from the data cache to RAM.
204 * Note: this is necessary because the instruction cache does *not*
205 * snoop from the data cache.
206 *
207 *	void __flush_dcache_icache(void *page)
208 */
209_GLOBAL(__flush_dcache_icache)
210/*
211 * Flush the data cache to memory
212 *
213 * Different systems have different cache line sizes
214 */
215
216BEGIN_FTR_SECTION
217	PURGE_PREFETCHED_INS
218	blr
219END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
220
221/* Flush the dcache */
222 	ld	r7,PPC64_CACHES@toc(r2)
223	clrrdi	r3,r3,PAGE_SHIFT           	    /* Page align */
224	lwz	r4,DCACHEL1LINESPERPAGE(r7)	/* Get # dcache lines per page */
225	lwz	r5,DCACHEL1LINESIZE(r7)		/* Get dcache line size */
226	mr	r6,r3
227	mtctr	r4
2280:	dcbst	0,r6
229	add	r6,r6,r5
230	bdnz	0b
231	sync
232
233/* Now invalidate the icache */
234
235	lwz	r4,ICACHEL1LINESPERPAGE(r7)	/* Get # icache lines per page */
236	lwz	r5,ICACHEL1LINESIZE(r7)		/* Get icache line size */
237	mtctr	r4
2381:	icbi	0,r3
239	add	r3,r3,r5
240	bdnz	1b
241	isync
242	blr
243
244_GLOBAL(__bswapdi2)
245	srdi	r8,r3,32
246	rlwinm	r7,r3,8,0xffffffff
247	rlwimi	r7,r3,24,0,7
248	rlwinm	r9,r8,8,0xffffffff
249	rlwimi	r7,r3,24,16,23
250	rlwimi	r9,r8,24,0,7
251	rlwimi	r9,r8,24,16,23
252	sldi	r7,r7,32
253	or	r3,r7,r9
254	blr
255
256
257#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
258_GLOBAL(rmci_on)
259	sync
260	isync
261	li	r3,0x100
262	rldicl	r3,r3,32,0
263	mfspr	r5,SPRN_HID4
264	or	r5,r5,r3
265	sync
266	mtspr	SPRN_HID4,r5
267	isync
268	slbia
269	isync
270	sync
271	blr
272
273_GLOBAL(rmci_off)
274	sync
275	isync
276	li	r3,0x100
277	rldicl	r3,r3,32,0
278	mfspr	r5,SPRN_HID4
279	andc	r5,r5,r3
280	sync
281	mtspr	SPRN_HID4,r5
282	isync
283	slbia
284	isync
285	sync
286	blr
287#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
288
289#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
290
291/*
292 * Do an IO access in real mode
293 */
294_GLOBAL(real_readb)
295	mfmsr	r7
296	ori	r0,r7,MSR_DR
297	xori	r0,r0,MSR_DR
298	sync
299	mtmsrd	r0
300	sync
301	isync
302	mfspr	r6,SPRN_HID4
303	rldicl	r5,r6,32,0
304	ori	r5,r5,0x100
305	rldicl	r5,r5,32,0
306	sync
307	mtspr	SPRN_HID4,r5
308	isync
309	slbia
310	isync
311	lbz	r3,0(r3)
312	sync
313	mtspr	SPRN_HID4,r6
314	isync
315	slbia
316	isync
317	mtmsrd	r7
318	sync
319	isync
320	blr
321
322	/*
323 * Do an IO access in real mode
324 */
325_GLOBAL(real_writeb)
326	mfmsr	r7
327	ori	r0,r7,MSR_DR
328	xori	r0,r0,MSR_DR
329	sync
330	mtmsrd	r0
331	sync
332	isync
333	mfspr	r6,SPRN_HID4
334	rldicl	r5,r6,32,0
335	ori	r5,r5,0x100
336	rldicl	r5,r5,32,0
337	sync
338	mtspr	SPRN_HID4,r5
339	isync
340	slbia
341	isync
342	stb	r3,0(r4)
343	sync
344	mtspr	SPRN_HID4,r6
345	isync
346	slbia
347	isync
348	mtmsrd	r7
349	sync
350	isync
351	blr
352#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
353
354#ifdef CONFIG_PPC_PASEMI
355
356_GLOBAL(real_205_readb)
357	mfmsr	r7
358	ori	r0,r7,MSR_DR
359	xori	r0,r0,MSR_DR
360	sync
361	mtmsrd	r0
362	sync
363	isync
364	LBZCIX(R3,R0,R3)
365	isync
366	mtmsrd	r7
367	sync
368	isync
369	blr
370
371_GLOBAL(real_205_writeb)
372	mfmsr	r7
373	ori	r0,r7,MSR_DR
374	xori	r0,r0,MSR_DR
375	sync
376	mtmsrd	r0
377	sync
378	isync
379	STBCIX(R3,R0,R4)
380	isync
381	mtmsrd	r7
382	sync
383	isync
384	blr
385
386#endif /* CONFIG_PPC_PASEMI */
387
388
389#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
390/*
391 * SCOM access functions for 970 (FX only for now)
392 *
393 * unsigned long scom970_read(unsigned int address);
394 * void scom970_write(unsigned int address, unsigned long value);
395 *
396 * The address passed in is the 24 bits register address. This code
397 * is 970 specific and will not check the status bits, so you should
398 * know what you are doing.
399 */
400_GLOBAL(scom970_read)
401	/* interrupts off */
402	mfmsr	r4
403	ori	r0,r4,MSR_EE
404	xori	r0,r0,MSR_EE
405	mtmsrd	r0,1
406
407	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
408	 * (including parity). On current CPUs they must be 0'd,
409	 * and finally or in RW bit
410	 */
411	rlwinm	r3,r3,8,0,15
412	ori	r3,r3,0x8000
413
414	/* do the actual scom read */
415	sync
416	mtspr	SPRN_SCOMC,r3
417	isync
418	mfspr	r3,SPRN_SCOMD
419	isync
420	mfspr	r0,SPRN_SCOMC
421	isync
422
423	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
424	 * that's the best we can do). Not implemented yet as we don't use
425	 * the scom on any of the bogus CPUs yet, but may have to be done
426	 * ultimately
427	 */
428
429	/* restore interrupts */
430	mtmsrd	r4,1
431	blr
432
433
434_GLOBAL(scom970_write)
435	/* interrupts off */
436	mfmsr	r5
437	ori	r0,r5,MSR_EE
438	xori	r0,r0,MSR_EE
439	mtmsrd	r0,1
440
441	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
442	 * (including parity). On current CPUs they must be 0'd.
443	 */
444
445	rlwinm	r3,r3,8,0,15
446
447	sync
448	mtspr	SPRN_SCOMD,r4      /* write data */
449	isync
450	mtspr	SPRN_SCOMC,r3      /* write command */
451	isync
452	mfspr	3,SPRN_SCOMC
453	isync
454
455	/* restore interrupts */
456	mtmsrd	r5,1
457	blr
458#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
459
460/* kexec_wait(phys_cpu)
461 *
462 * wait for the flag to change, indicating this kernel is going away but
463 * the slave code for the next one is at addresses 0 to 100.
464 *
465 * This is used by all slaves, even those that did not find a matching
466 * paca in the secondary startup code.
467 *
468 * Physical (hardware) cpu id should be in r3.
469 */
470_GLOBAL(kexec_wait)
471	bl	1f
4721:	mflr	r5
473	addi	r5,r5,kexec_flag-1b
474
47599:	HMT_LOW
476#ifdef CONFIG_KEXEC		/* use no memory without kexec */
477	lwz	r4,0(r5)
478	cmpwi	0,r4,0
479	beq	99b
480#ifdef CONFIG_PPC_BOOK3S_64
481	li	r10,0x60
482	mfmsr	r11
483	clrrdi	r11,r11,1	/* Clear MSR_LE */
484	mtsrr0	r10
485	mtsrr1	r11
486	rfid
487#else
488	/* Create TLB entry in book3e_secondary_core_init */
489	li	r4,0
490	ba	0x60
491#endif
492#endif
493
494/* this can be in text because we won't change it until we are
495 * running in real anyways
496 */
497kexec_flag:
498	.long	0
499
500
501#ifdef CONFIG_KEXEC
502#ifdef CONFIG_PPC_BOOK3E
503/*
504 * BOOK3E has no real MMU mode, so we have to setup the initial TLB
505 * for a core to identity map v:0 to p:0.  This current implementation
506 * assumes that 1G is enough for kexec.
507 */
508kexec_create_tlb:
509	/*
510	 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
511	 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
512	 */
513	PPC_TLBILX_ALL(0,R0)
514	sync
515	isync
516
517	mfspr	r10,SPRN_TLB1CFG
518	andi.	r10,r10,TLBnCFG_N_ENTRY	/* Extract # entries */
519	subi	r10,r10,1	/* Last entry: no conflict with kernel text */
520	lis	r9,MAS0_TLBSEL(1)@h
521	rlwimi	r9,r10,16,4,15		/* Setup MAS0 = TLBSEL | ESEL(r9) */
522
523/* Set up a temp identity mapping v:0 to p:0 and return to it. */
524#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
525#define M_IF_NEEDED	MAS2_M
526#else
527#define M_IF_NEEDED	0
528#endif
529	mtspr	SPRN_MAS0,r9
530
531	lis	r9,(MAS1_VALID|MAS1_IPROT)@h
532	ori	r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
533	mtspr	SPRN_MAS1,r9
534
535	LOAD_REG_IMMEDIATE(r9, 0x0 | M_IF_NEEDED)
536	mtspr	SPRN_MAS2,r9
537
538	LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
539	mtspr	SPRN_MAS3,r9
540	li	r9,0
541	mtspr	SPRN_MAS7,r9
542
543	tlbwe
544	isync
545	blr
546#endif
547
548/* kexec_smp_wait(void)
549 *
550 * call with interrupts off
551 * note: this is a terminal routine, it does not save lr
552 *
553 * get phys id from paca
554 * switch to real mode
555 * mark the paca as no longer used
556 * join other cpus in kexec_wait(phys_id)
557 */
558_GLOBAL(kexec_smp_wait)
559	lhz	r3,PACAHWCPUID(r13)
560	bl	real_mode
561
562	li	r4,KEXEC_STATE_REAL_MODE
563	stb	r4,PACAKEXECSTATE(r13)
564	SYNC
565
566	b	kexec_wait
567
568/*
569 * switch to real mode (turn mmu off)
570 * we use the early kernel trick that the hardware ignores bits
571 * 0 and 1 (big endian) of the effective address in real mode
572 *
573 * don't overwrite r3 here, it is live for kexec_wait above.
574 */
575real_mode:	/* assume normal blr return */
576#ifdef CONFIG_PPC_BOOK3E
577	/* Create an identity mapping. */
578	b	kexec_create_tlb
579#else
5801:	li	r9,MSR_RI
581	li	r10,MSR_DR|MSR_IR
582	mflr	r11		/* return address to SRR0 */
583	mfmsr	r12
584	andc	r9,r12,r9
585	andc	r10,r12,r10
586
587	mtmsrd	r9,1
588	mtspr	SPRN_SRR1,r10
589	mtspr	SPRN_SRR0,r11
590	rfid
591#endif
592
593/*
594 * kexec_sequence(newstack, start, image, control, clear_all())
595 *
596 * does the grungy work with stack switching and real mode switches
597 * also does simple calls to other code
598 */
599
600_GLOBAL(kexec_sequence)
601	mflr	r0
602	std	r0,16(r1)
603
604	/* switch stacks to newstack -- &kexec_stack.stack */
605	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
606	mr	r1,r3
607
608	li	r0,0
609	std	r0,16(r1)
610
611	/* save regs for local vars on new stack.
612	 * yes, we won't go back, but ...
613	 */
614	std	r31,-8(r1)
615	std	r30,-16(r1)
616	std	r29,-24(r1)
617	std	r28,-32(r1)
618	std	r27,-40(r1)
619	std	r26,-48(r1)
620	std	r25,-56(r1)
621
622	stdu	r1,-STACK_FRAME_OVERHEAD-64(r1)
623
624	/* save args into preserved regs */
625	mr	r31,r3			/* newstack (both) */
626	mr	r30,r4			/* start (real) */
627	mr	r29,r5			/* image (virt) */
628	mr	r28,r6			/* control, unused */
629	mr	r27,r7			/* clear_all() fn desc */
630	mr	r26,r8			/* spare */
631	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */
632
633	/* disable interrupts, we are overwriting kernel data next */
634#ifdef CONFIG_PPC_BOOK3E
635	wrteei	0
636#else
637	mfmsr	r3
638	rlwinm	r3,r3,0,17,15
639	mtmsrd	r3,1
640#endif
641
642	/* copy dest pages, flush whole dest image */
643	mr	r3,r29
644	bl	kexec_copy_flush	/* (image) */
645
646	/* turn off mmu */
647	bl	real_mode
648
649	/* copy  0x100 bytes starting at start to 0 */
650	li	r3,0
651	mr	r4,r30		/* start, aka phys mem offset */
652	li	r5,0x100
653	li	r6,0
654	bl	copy_and_flush	/* (dest, src, copy limit, start offset) */
6551:	/* assume normal blr return */
656
657	/* release other cpus to the new kernel secondary start at 0x60 */
658	mflr	r5
659	li	r6,1
660	stw	r6,kexec_flag-1b(5)
661
662#ifndef CONFIG_PPC_BOOK3E
663	/* clear out hardware hash page table and tlb */
664#if !defined(_CALL_ELF) || _CALL_ELF != 2
665	ld	r12,0(r27)		/* deref function descriptor */
666#else
667	mr	r12,r27
668#endif
669	mtctr	r12
670	bctrl				/* ppc_md.hpte_clear_all(void); */
671#endif /* !CONFIG_PPC_BOOK3E */
672
673/*
674 *   kexec image calling is:
675 *      the first 0x100 bytes of the entry point are copied to 0
676 *
677 *      all slaves branch to slave = 0x60 (absolute)
678 *              slave(phys_cpu_id);
679 *
680 *      master goes to start = entry point
681 *              start(phys_cpu_id, start, 0);
682 *
683 *
684 *   a wrapper is needed to call existing kernels, here is an approximate
685 *   description of one method:
686 *
687 * v2: (2.6.10)
688 *   start will be near the boot_block (maybe 0x100 bytes before it?)
689 *   it will have a 0x60, which will b to boot_block, where it will wait
690 *   and 0 will store phys into struct boot-block and load r3 from there,
691 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
692 *
693 * v1: (2.6.9)
694 *    boot block will have all cpus scanning device tree to see if they
695 *    are the boot cpu ?????
696 *    other device tree differences (prop sizes, va vs pa, etc)...
697 */
698	mr	r3,r25	# my phys cpu
699	mr	r4,r30	# start, aka phys mem offset
700	mtlr	4
701	li	r5,0
702	blr	/* image->start(physid, image->start, 0); */
703#endif /* CONFIG_KEXEC */
704
705#ifdef CONFIG_MODULES
706#if defined(_CALL_ELF) && _CALL_ELF == 2
707
708#ifdef CONFIG_MODVERSIONS
709.weak __crc_TOC.
710.section "___kcrctab+TOC.","a"
711.globl __kcrctab_TOC.
712__kcrctab_TOC.:
713	.llong	__crc_TOC.
714#endif
715
716/*
717 * Export a fake .TOC. since both modpost and depmod will complain otherwise.
718 * Both modpost and depmod strip the leading . so we do the same here.
719 */
720.section "__ksymtab_strings","a"
721__kstrtab_TOC.:
722	.asciz "TOC."
723
724.section "___ksymtab+TOC.","a"
725/* This symbol name is important: it's used by modpost to find exported syms */
726.globl __ksymtab_TOC.
727__ksymtab_TOC.:
728	.llong 0 /* .value */
729	.llong __kstrtab_TOC.
730#endif /* ELFv2 */
731#endif /* MODULES */
732