xref: /openbmc/linux/arch/powerpc/kernel/misc_64.S (revision f42b3800) 
1/*
2 * This file contains miscellaneous low-level functions.
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 */
16
17#include <linux/sys.h>
18#include <asm/unistd.h>
19#include <asm/errno.h>
20#include <asm/processor.h>
21#include <asm/page.h>
22#include <asm/cache.h>
23#include <asm/ppc_asm.h>
24#include <asm/asm-offsets.h>
25#include <asm/cputable.h>
26#include <asm/thread_info.h>
27
28	.text
29
30_GLOBAL(get_msr)
31	mfmsr	r3
32	blr
33
34_GLOBAL(get_srr0)
35	mfsrr0  r3
36	blr
37
38_GLOBAL(get_srr1)
39	mfsrr1  r3
40	blr
41
42#ifdef CONFIG_IRQSTACKS
43_GLOBAL(call_do_softirq)
44	mflr	r0
45	std	r0,16(r1)
46	stdu	r1,THREAD_SIZE-112(r3)
47	mr	r1,r3
48	bl	.__do_softirq
49	ld	r1,0(r1)
50	ld	r0,16(r1)
51	mtlr	r0
52	blr
53
54_GLOBAL(call_handle_irq)
55	ld	r8,0(r6)
56	mflr	r0
57	std	r0,16(r1)
58	mtctr	r8
59	stdu	r1,THREAD_SIZE-112(r5)
60	mr	r1,r5
61	bctrl
62	ld	r1,0(r1)
63	ld	r0,16(r1)
64	mtlr	r0
65	blr
66#endif /* CONFIG_IRQSTACKS */
67
68	.section	".toc","aw"
69PPC64_CACHES:
70	.tc		ppc64_caches[TC],ppc64_caches
71	.section	".text"
72
73/*
74 * Write any modified data cache blocks out to memory
75 * and invalidate the corresponding instruction cache blocks.
76 *
77 * flush_icache_range(unsigned long start, unsigned long stop)
78 *
79 *   flush all bytes from start through stop-1 inclusive
80 */
81
82_KPROBE(__flush_icache_range)
83
84/*
85 * Flush the data cache to memory
86 *
87 * Different systems have different cache line sizes
88 * and in some cases i-cache and d-cache line sizes differ from
89 * each other.
90 */
91 	ld	r10,PPC64_CACHES@toc(r2)
92	lwz	r7,DCACHEL1LINESIZE(r10)/* Get cache line size */
93	addi	r5,r7,-1
94	andc	r6,r3,r5		/* round low to line bdy */
95	subf	r8,r6,r4		/* compute length */
96	add	r8,r8,r5		/* ensure we get enough */
97	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of cache line size */
98	srw.	r8,r8,r9		/* compute line count */
99	beqlr				/* nothing to do? */
100	mtctr	r8
1011:	dcbst	0,r6
102	add	r6,r6,r7
103	bdnz	1b
104	sync
105
106/* Now invalidate the instruction cache */
107
108	lwz	r7,ICACHEL1LINESIZE(r10)	/* Get Icache line size */
109	addi	r5,r7,-1
110	andc	r6,r3,r5		/* round low to line bdy */
111	subf	r8,r6,r4		/* compute length */
112	add	r8,r8,r5
113	lwz	r9,ICACHEL1LOGLINESIZE(r10)	/* Get log-2 of Icache line size */
114	srw.	r8,r8,r9		/* compute line count */
115	beqlr				/* nothing to do? */
116	mtctr	r8
1172:	icbi	0,r6
118	add	r6,r6,r7
119	bdnz	2b
120	isync
121	blr
122	.previous .text
123/*
124 * Like above, but only do the D-cache.
125 *
126 * flush_dcache_range(unsigned long start, unsigned long stop)
127 *
128 *    flush all bytes from start to stop-1 inclusive
129 */
130_GLOBAL(flush_dcache_range)
131
132/*
133 * Flush the data cache to memory
134 *
135 * Different systems have different cache line sizes
136 */
137 	ld	r10,PPC64_CACHES@toc(r2)
138	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
139	addi	r5,r7,-1
140	andc	r6,r3,r5		/* round low to line bdy */
141	subf	r8,r6,r4		/* compute length */
142	add	r8,r8,r5		/* ensure we get enough */
143	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
144	srw.	r8,r8,r9		/* compute line count */
145	beqlr				/* nothing to do? */
146	mtctr	r8
1470:	dcbst	0,r6
148	add	r6,r6,r7
149	bdnz	0b
150	sync
151	blr
152
153/*
154 * Like above, but works on non-mapped physical addresses.
155 * Use only for non-LPAR setups ! It also assumes real mode
156 * is cacheable. Used for flushing out the DART before using
157 * it as uncacheable memory
158 *
159 * flush_dcache_phys_range(unsigned long start, unsigned long stop)
160 *
161 *    flush all bytes from start to stop-1 inclusive
162 */
163_GLOBAL(flush_dcache_phys_range)
164 	ld	r10,PPC64_CACHES@toc(r2)
165	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
166	addi	r5,r7,-1
167	andc	r6,r3,r5		/* round low to line bdy */
168	subf	r8,r6,r4		/* compute length */
169	add	r8,r8,r5		/* ensure we get enough */
170	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
171	srw.	r8,r8,r9		/* compute line count */
172	beqlr				/* nothing to do? */
173	mfmsr	r5			/* Disable MMU Data Relocation */
174	ori	r0,r5,MSR_DR
175	xori	r0,r0,MSR_DR
176	sync
177	mtmsr	r0
178	sync
179	isync
180	mtctr	r8
1810:	dcbst	0,r6
182	add	r6,r6,r7
183	bdnz	0b
184	sync
185	isync
186	mtmsr	r5			/* Re-enable MMU Data Relocation */
187	sync
188	isync
189	blr
190
191_GLOBAL(flush_inval_dcache_range)
192 	ld	r10,PPC64_CACHES@toc(r2)
193	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
194	addi	r5,r7,-1
195	andc	r6,r3,r5		/* round low to line bdy */
196	subf	r8,r6,r4		/* compute length */
197	add	r8,r8,r5		/* ensure we get enough */
198	lwz	r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */
199	srw.	r8,r8,r9		/* compute line count */
200	beqlr				/* nothing to do? */
201	sync
202	isync
203	mtctr	r8
2040:	dcbf	0,r6
205	add	r6,r6,r7
206	bdnz	0b
207	sync
208	isync
209	blr
210
211
212/*
213 * Flush a particular page from the data cache to RAM.
214 * Note: this is necessary because the instruction cache does *not*
215 * snoop from the data cache.
216 *
217 *	void __flush_dcache_icache(void *page)
218 */
219_GLOBAL(__flush_dcache_icache)
220/*
221 * Flush the data cache to memory
222 *
223 * Different systems have different cache line sizes
224 */
225
226/* Flush the dcache */
227 	ld	r7,PPC64_CACHES@toc(r2)
228	clrrdi	r3,r3,PAGE_SHIFT           	    /* Page align */
229	lwz	r4,DCACHEL1LINESPERPAGE(r7)	/* Get # dcache lines per page */
230	lwz	r5,DCACHEL1LINESIZE(r7)		/* Get dcache line size */
231	mr	r6,r3
232	mtctr	r4
2330:	dcbst	0,r6
234	add	r6,r6,r5
235	bdnz	0b
236	sync
237
238/* Now invalidate the icache */
239
240	lwz	r4,ICACHEL1LINESPERPAGE(r7)	/* Get # icache lines per page */
241	lwz	r5,ICACHEL1LINESIZE(r7)		/* Get icache line size */
242	mtctr	r4
2431:	icbi	0,r3
244	add	r3,r3,r5
245	bdnz	1b
246	isync
247	blr
248
249
250#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
251/*
252 * Do an IO access in real mode
253 */
254_GLOBAL(real_readb)
255	mfmsr	r7
256	ori	r0,r7,MSR_DR
257	xori	r0,r0,MSR_DR
258	sync
259	mtmsrd	r0
260	sync
261	isync
262	mfspr	r6,SPRN_HID4
263	rldicl	r5,r6,32,0
264	ori	r5,r5,0x100
265	rldicl	r5,r5,32,0
266	sync
267	mtspr	SPRN_HID4,r5
268	isync
269	slbia
270	isync
271	lbz	r3,0(r3)
272	sync
273	mtspr	SPRN_HID4,r6
274	isync
275	slbia
276	isync
277	mtmsrd	r7
278	sync
279	isync
280	blr
281
282	/*
283 * Do an IO access in real mode
284 */
285_GLOBAL(real_writeb)
286	mfmsr	r7
287	ori	r0,r7,MSR_DR
288	xori	r0,r0,MSR_DR
289	sync
290	mtmsrd	r0
291	sync
292	isync
293	mfspr	r6,SPRN_HID4
294	rldicl	r5,r6,32,0
295	ori	r5,r5,0x100
296	rldicl	r5,r5,32,0
297	sync
298	mtspr	SPRN_HID4,r5
299	isync
300	slbia
301	isync
302	stb	r3,0(r4)
303	sync
304	mtspr	SPRN_HID4,r6
305	isync
306	slbia
307	isync
308	mtmsrd	r7
309	sync
310	isync
311	blr
312#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
313
314#ifdef CONFIG_PPC_PASEMI
315
316/* No support in all binutils for these yet, so use defines */
317#define LBZCIX(RT,RA,RB)  .long (0x7c0006aa|(RT<<21)|(RA<<16)|(RB << 11))
318#define STBCIX(RS,RA,RB)  .long (0x7c0007aa|(RS<<21)|(RA<<16)|(RB << 11))
319
320
321_GLOBAL(real_205_readb)
322	mfmsr	r7
323	ori	r0,r7,MSR_DR
324	xori	r0,r0,MSR_DR
325	sync
326	mtmsrd	r0
327	sync
328	isync
329	LBZCIX(r3,0,r3)
330	isync
331	mtmsrd	r7
332	sync
333	isync
334	blr
335
336_GLOBAL(real_205_writeb)
337	mfmsr	r7
338	ori	r0,r7,MSR_DR
339	xori	r0,r0,MSR_DR
340	sync
341	mtmsrd	r0
342	sync
343	isync
344	STBCIX(r3,0,r4)
345	isync
346	mtmsrd	r7
347	sync
348	isync
349	blr
350
351#endif /* CONFIG_PPC_PASEMI */
352
353
354#ifdef CONFIG_CPU_FREQ_PMAC64
355/*
356 * SCOM access functions for 970 (FX only for now)
357 *
358 * unsigned long scom970_read(unsigned int address);
359 * void scom970_write(unsigned int address, unsigned long value);
360 *
361 * The address passed in is the 24 bits register address. This code
362 * is 970 specific and will not check the status bits, so you should
363 * know what you are doing.
364 */
365_GLOBAL(scom970_read)
366	/* interrupts off */
367	mfmsr	r4
368	ori	r0,r4,MSR_EE
369	xori	r0,r0,MSR_EE
370	mtmsrd	r0,1
371
372	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
373	 * (including parity). On current CPUs they must be 0'd,
374	 * and finally or in RW bit
375	 */
376	rlwinm	r3,r3,8,0,15
377	ori	r3,r3,0x8000
378
379	/* do the actual scom read */
380	sync
381	mtspr	SPRN_SCOMC,r3
382	isync
383	mfspr	r3,SPRN_SCOMD
384	isync
385	mfspr	r0,SPRN_SCOMC
386	isync
387
388	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
389	 * that's the best we can do). Not implemented yet as we don't use
390	 * the scom on any of the bogus CPUs yet, but may have to be done
391	 * ultimately
392	 */
393
394	/* restore interrupts */
395	mtmsrd	r4,1
396	blr
397
398
399_GLOBAL(scom970_write)
400	/* interrupts off */
401	mfmsr	r5
402	ori	r0,r5,MSR_EE
403	xori	r0,r0,MSR_EE
404	mtmsrd	r0,1
405
406	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
407	 * (including parity). On current CPUs they must be 0'd.
408	 */
409
410	rlwinm	r3,r3,8,0,15
411
412	sync
413	mtspr	SPRN_SCOMD,r4      /* write data */
414	isync
415	mtspr	SPRN_SCOMC,r3      /* write command */
416	isync
417	mfspr	3,SPRN_SCOMC
418	isync
419
420	/* restore interrupts */
421	mtmsrd	r5,1
422	blr
423#endif /* CONFIG_CPU_FREQ_PMAC64 */
424
425
426/*
427 * Create a kernel thread
428 *   kernel_thread(fn, arg, flags)
429 */
430_GLOBAL(kernel_thread)
431	std	r29,-24(r1)
432	std	r30,-16(r1)
433	stdu	r1,-STACK_FRAME_OVERHEAD(r1)
434	mr	r29,r3
435	mr	r30,r4
436	ori	r3,r5,CLONE_VM	/* flags */
437	oris	r3,r3,(CLONE_UNTRACED>>16)
438	li	r4,0		/* new sp (unused) */
439	li	r0,__NR_clone
440	sc
441	cmpdi	0,r3,0		/* parent or child? */
442	bne	1f		/* return if parent */
443	li	r0,0
444	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
445	ld	r2,8(r29)
446	ld	r29,0(r29)
447	mtlr	r29              /* fn addr in lr */
448	mr	r3,r30	        /* load arg and call fn */
449	blrl
450	li	r0,__NR_exit	/* exit after child exits */
451        li	r3,0
452	sc
4531:	addi	r1,r1,STACK_FRAME_OVERHEAD
454	ld	r29,-24(r1)
455	ld	r30,-16(r1)
456	blr
457
458/*
459 * disable_kernel_fp()
460 * Disable the FPU.
461 */
462_GLOBAL(disable_kernel_fp)
463	mfmsr	r3
464	rldicl	r0,r3,(63-MSR_FP_LG),1
465	rldicl	r3,r0,(MSR_FP_LG+1),0
466	mtmsrd	r3			/* disable use of fpu now */
467	isync
468	blr
469
470#ifdef CONFIG_ALTIVEC
471
472#if 0 /* this has no callers for now */
473/*
474 * disable_kernel_altivec()
475 * Disable the VMX.
476 */
477_GLOBAL(disable_kernel_altivec)
478	mfmsr	r3
479	rldicl	r0,r3,(63-MSR_VEC_LG),1
480	rldicl	r3,r0,(MSR_VEC_LG+1),0
481	mtmsrd	r3			/* disable use of VMX now */
482	isync
483	blr
484#endif /* 0 */
485
486/*
487 * giveup_altivec(tsk)
488 * Disable VMX for the task given as the argument,
489 * and save the vector registers in its thread_struct.
490 * Enables the VMX for use in the kernel on return.
491 */
492_GLOBAL(giveup_altivec)
493	mfmsr	r5
494	oris	r5,r5,MSR_VEC@h
495	mtmsrd	r5			/* enable use of VMX now */
496	isync
497	cmpdi	0,r3,0
498	beqlr-				/* if no previous owner, done */
499	addi	r3,r3,THREAD		/* want THREAD of task */
500	ld	r5,PT_REGS(r3)
501	cmpdi	0,r5,0
502	SAVE_32VRS(0,r4,r3)
503	mfvscr	vr0
504	li	r4,THREAD_VSCR
505	stvx	vr0,r4,r3
506	beq	1f
507	ld	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
508	lis	r3,MSR_VEC@h
509	andc	r4,r4,r3		/* disable FP for previous task */
510	std	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
5111:
512#ifndef CONFIG_SMP
513	li	r5,0
514	ld	r4,last_task_used_altivec@got(r2)
515	std	r5,0(r4)
516#endif /* CONFIG_SMP */
517	blr
518
519#endif /* CONFIG_ALTIVEC */
520
521/* kexec_wait(phys_cpu)
522 *
523 * wait for the flag to change, indicating this kernel is going away but
524 * the slave code for the next one is at addresses 0 to 100.
525 *
526 * This is used by all slaves.
527 *
528 * Physical (hardware) cpu id should be in r3.
529 */
530_GLOBAL(kexec_wait)
531	bl	1f
5321:	mflr	r5
533	addi	r5,r5,kexec_flag-1b
534
53599:	HMT_LOW
536#ifdef CONFIG_KEXEC		/* use no memory without kexec */
537	lwz	r4,0(r5)
538	cmpwi	0,r4,0
539	bnea	0x60
540#endif
541	b	99b
542
543/* this can be in text because we won't change it until we are
544 * running in real anyways
545 */
546kexec_flag:
547	.long	0
548
549
550#ifdef CONFIG_KEXEC
551
552/* kexec_smp_wait(void)
553 *
554 * call with interrupts off
555 * note: this is a terminal routine, it does not save lr
556 *
557 * get phys id from paca
558 * set paca id to -1 to say we got here
559 * switch to real mode
560 * join other cpus in kexec_wait(phys_id)
561 */
562_GLOBAL(kexec_smp_wait)
563	lhz	r3,PACAHWCPUID(r13)
564	li	r4,-1
565	sth	r4,PACAHWCPUID(r13)	/* let others know we left */
566	bl	real_mode
567	b	.kexec_wait
568
569/*
570 * switch to real mode (turn mmu off)
571 * we use the early kernel trick that the hardware ignores bits
572 * 0 and 1 (big endian) of the effective address in real mode
573 *
574 * don't overwrite r3 here, it is live for kexec_wait above.
575 */
576real_mode:	/* assume normal blr return */
5771:	li	r9,MSR_RI
578	li	r10,MSR_DR|MSR_IR
579	mflr	r11		/* return address to SRR0 */
580	mfmsr	r12
581	andc	r9,r12,r9
582	andc	r10,r12,r10
583
584	mtmsrd	r9,1
585	mtspr	SPRN_SRR1,r10
586	mtspr	SPRN_SRR0,r11
587	rfid
588
589
590/*
591 * kexec_sequence(newstack, start, image, control, clear_all())
592 *
593 * does the grungy work with stack switching and real mode switches
594 * also does simple calls to other code
595 */
596
597_GLOBAL(kexec_sequence)
598	mflr	r0
599	std	r0,16(r1)
600
601	/* switch stacks to newstack -- &kexec_stack.stack */
602	stdu	r1,THREAD_SIZE-112(r3)
603	mr	r1,r3
604
605	li	r0,0
606	std	r0,16(r1)
607
608	/* save regs for local vars on new stack.
609	 * yes, we won't go back, but ...
610	 */
611	std	r31,-8(r1)
612	std	r30,-16(r1)
613	std	r29,-24(r1)
614	std	r28,-32(r1)
615	std	r27,-40(r1)
616	std	r26,-48(r1)
617	std	r25,-56(r1)
618
619	stdu	r1,-112-64(r1)
620
621	/* save args into preserved regs */
622	mr	r31,r3			/* newstack (both) */
623	mr	r30,r4			/* start (real) */
624	mr	r29,r5			/* image (virt) */
625	mr	r28,r6			/* control, unused */
626	mr	r27,r7			/* clear_all() fn desc */
627	mr	r26,r8			/* spare */
628	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */
629
630	/* disable interrupts, we are overwriting kernel data next */
631	mfmsr	r3
632	rlwinm	r3,r3,0,17,15
633	mtmsrd	r3,1
634
635	/* copy dest pages, flush whole dest image */
636	mr	r3,r29
637	bl	.kexec_copy_flush	/* (image) */
638
639	/* turn off mmu */
640	bl	real_mode
641
642	/* copy  0x100 bytes starting at start to 0 */
643	li	r3,0
644	mr	r4,r30		/* start, aka phys mem offset */
645	li	r5,0x100
646	li	r6,0
647	bl	.copy_and_flush	/* (dest, src, copy limit, start offset) */
6481:	/* assume normal blr return */
649
650	/* release other cpus to the new kernel secondary start at 0x60 */
651	mflr	r5
652	li	r6,1
653	stw	r6,kexec_flag-1b(5)
654
655	/* clear out hardware hash page table and tlb */
656	ld	r5,0(r27)		/* deref function descriptor */
657	mtctr	r5
658	bctrl				/* ppc_md.hpte_clear_all(void); */
659
660/*
661 *   kexec image calling is:
662 *      the first 0x100 bytes of the entry point are copied to 0
663 *
664 *      all slaves branch to slave = 0x60 (absolute)
665 *              slave(phys_cpu_id);
666 *
667 *      master goes to start = entry point
668 *              start(phys_cpu_id, start, 0);
669 *
670 *
671 *   a wrapper is needed to call existing kernels, here is an approximate
672 *   description of one method:
673 *
674 * v2: (2.6.10)
675 *   start will be near the boot_block (maybe 0x100 bytes before it?)
676 *   it will have a 0x60, which will b to boot_block, where it will wait
677 *   and 0 will store phys into struct boot-block and load r3 from there,
678 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
679 *
680 * v1: (2.6.9)
681 *    boot block will have all cpus scanning device tree to see if they
682 *    are the boot cpu ?????
683 *    other device tree differences (prop sizes, va vs pa, etc)...
684 */
685	mr	r3,r25	# my phys cpu
686	mr	r4,r30	# start, aka phys mem offset
687	mtlr	4
688	li	r5,0
689	blr	/* image->start(physid, image->start, 0); */
690#endif /* CONFIG_KEXEC */
691