xref: /openbmc/linux/arch/powerpc/kernel/head_64.S (revision c6622f63)
114cf11afSPaul Mackerras/*
214cf11afSPaul Mackerras *  PowerPC version
314cf11afSPaul Mackerras *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
414cf11afSPaul Mackerras *
514cf11afSPaul Mackerras *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
614cf11afSPaul Mackerras *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
714cf11afSPaul Mackerras *  Adapted for Power Macintosh by Paul Mackerras.
814cf11afSPaul Mackerras *  Low-level exception handlers and MMU support
914cf11afSPaul Mackerras *  rewritten by Paul Mackerras.
1014cf11afSPaul Mackerras *    Copyright (C) 1996 Paul Mackerras.
1114cf11afSPaul Mackerras *
1214cf11afSPaul Mackerras *  Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
1314cf11afSPaul Mackerras *    Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
1414cf11afSPaul Mackerras *
1514cf11afSPaul Mackerras *  This file contains the low-level support and setup for the
1614cf11afSPaul Mackerras *  PowerPC-64 platform, including trap and interrupt dispatch.
1714cf11afSPaul Mackerras *
1814cf11afSPaul Mackerras *  This program is free software; you can redistribute it and/or
1914cf11afSPaul Mackerras *  modify it under the terms of the GNU General Public License
2014cf11afSPaul Mackerras *  as published by the Free Software Foundation; either version
2114cf11afSPaul Mackerras *  2 of the License, or (at your option) any later version.
2214cf11afSPaul Mackerras */
2314cf11afSPaul Mackerras
2414cf11afSPaul Mackerras#include <linux/config.h>
2514cf11afSPaul Mackerras#include <linux/threads.h>
26b5bbeb23SPaul Mackerras#include <asm/reg.h>
2714cf11afSPaul Mackerras#include <asm/page.h>
2814cf11afSPaul Mackerras#include <asm/mmu.h>
2914cf11afSPaul Mackerras#include <asm/ppc_asm.h>
3014cf11afSPaul Mackerras#include <asm/asm-offsets.h>
3114cf11afSPaul Mackerras#include <asm/bug.h>
3214cf11afSPaul Mackerras#include <asm/cputable.h>
3314cf11afSPaul Mackerras#include <asm/setup.h>
3414cf11afSPaul Mackerras#include <asm/hvcall.h>
35c43a55ffSKelly Daly#include <asm/iseries/lpar_map.h>
366cb7bfebSDavid Gibson#include <asm/thread_info.h>
3714cf11afSPaul Mackerras
3814cf11afSPaul Mackerras#ifdef CONFIG_PPC_ISERIES
3914cf11afSPaul Mackerras#define DO_SOFT_DISABLE
4014cf11afSPaul Mackerras#endif
4114cf11afSPaul Mackerras
4214cf11afSPaul Mackerras/*
4314cf11afSPaul Mackerras * We layout physical memory as follows:
4414cf11afSPaul Mackerras * 0x0000 - 0x00ff : Secondary processor spin code
4514cf11afSPaul Mackerras * 0x0100 - 0x2fff : pSeries Interrupt prologs
4614cf11afSPaul Mackerras * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
4714cf11afSPaul Mackerras * 0x6000 - 0x6fff : Initial (CPU0) segment table
4814cf11afSPaul Mackerras * 0x7000 - 0x7fff : FWNMI data area
4914cf11afSPaul Mackerras * 0x8000 -        : Early init and support code
5014cf11afSPaul Mackerras */
5114cf11afSPaul Mackerras
5214cf11afSPaul Mackerras/*
5314cf11afSPaul Mackerras *   SPRG Usage
5414cf11afSPaul Mackerras *
5514cf11afSPaul Mackerras *   Register	Definition
5614cf11afSPaul Mackerras *
5714cf11afSPaul Mackerras *   SPRG0	reserved for hypervisor
5814cf11afSPaul Mackerras *   SPRG1	temp - used to save gpr
5914cf11afSPaul Mackerras *   SPRG2	temp - used to save gpr
6014cf11afSPaul Mackerras *   SPRG3	virt addr of paca
6114cf11afSPaul Mackerras */
6214cf11afSPaul Mackerras
6314cf11afSPaul Mackerras/*
6414cf11afSPaul Mackerras * Entering into this code we make the following assumptions:
6514cf11afSPaul Mackerras *  For pSeries:
6614cf11afSPaul Mackerras *   1. The MMU is off & open firmware is running in real mode.
6714cf11afSPaul Mackerras *   2. The kernel is entered at __start
6814cf11afSPaul Mackerras *
6914cf11afSPaul Mackerras *  For iSeries:
7014cf11afSPaul Mackerras *   1. The MMU is on (as it always is for iSeries)
7114cf11afSPaul Mackerras *   2. The kernel is entered at system_reset_iSeries
7214cf11afSPaul Mackerras */
7314cf11afSPaul Mackerras
7414cf11afSPaul Mackerras	.text
7514cf11afSPaul Mackerras	.globl  _stext
7614cf11afSPaul Mackerras_stext:
7714cf11afSPaul Mackerras#ifdef CONFIG_PPC_MULTIPLATFORM
7814cf11afSPaul Mackerras_GLOBAL(__start)
7914cf11afSPaul Mackerras	/* NOP this out unconditionally */
8014cf11afSPaul MackerrasBEGIN_FTR_SECTION
8114cf11afSPaul Mackerras	b	.__start_initialization_multiplatform
8214cf11afSPaul MackerrasEND_FTR_SECTION(0, 1)
8314cf11afSPaul Mackerras#endif /* CONFIG_PPC_MULTIPLATFORM */
8414cf11afSPaul Mackerras
8514cf11afSPaul Mackerras	/* Catch branch to 0 in real mode */
8614cf11afSPaul Mackerras	trap
8714cf11afSPaul Mackerras
8814cf11afSPaul Mackerras#ifdef CONFIG_PPC_ISERIES
8914cf11afSPaul Mackerras	/*
9014cf11afSPaul Mackerras	 * At offset 0x20, there is a pointer to iSeries LPAR data.
9114cf11afSPaul Mackerras	 * This is required by the hypervisor
9214cf11afSPaul Mackerras	 */
9314cf11afSPaul Mackerras	. = 0x20
9414cf11afSPaul Mackerras	.llong hvReleaseData-KERNELBASE
9514cf11afSPaul Mackerras
9614cf11afSPaul Mackerras	/*
9714cf11afSPaul Mackerras	 * At offset 0x28 and 0x30 are offsets to the mschunks_map
9814cf11afSPaul Mackerras	 * array (used by the iSeries LPAR debugger to do translation
9914cf11afSPaul Mackerras	 * between physical addresses and absolute addresses) and
10014cf11afSPaul Mackerras	 * to the pidhash table (also used by the debugger)
10114cf11afSPaul Mackerras	 */
10214cf11afSPaul Mackerras	.llong mschunks_map-KERNELBASE
10314cf11afSPaul Mackerras	.llong 0	/* pidhash-KERNELBASE SFRXXX */
10414cf11afSPaul Mackerras
10514cf11afSPaul Mackerras	/* Offset 0x38 - Pointer to start of embedded System.map */
10614cf11afSPaul Mackerras	.globl	embedded_sysmap_start
10714cf11afSPaul Mackerrasembedded_sysmap_start:
10814cf11afSPaul Mackerras	.llong	0
10914cf11afSPaul Mackerras	/* Offset 0x40 - Pointer to end of embedded System.map */
11014cf11afSPaul Mackerras	.globl	embedded_sysmap_end
11114cf11afSPaul Mackerrasembedded_sysmap_end:
11214cf11afSPaul Mackerras	.llong	0
11314cf11afSPaul Mackerras
11414cf11afSPaul Mackerras#endif /* CONFIG_PPC_ISERIES */
11514cf11afSPaul Mackerras
11614cf11afSPaul Mackerras	/* Secondary processors spin on this value until it goes to 1. */
11714cf11afSPaul Mackerras	.globl  __secondary_hold_spinloop
11814cf11afSPaul Mackerras__secondary_hold_spinloop:
11914cf11afSPaul Mackerras	.llong	0x0
12014cf11afSPaul Mackerras
12114cf11afSPaul Mackerras	/* Secondary processors write this value with their cpu # */
12214cf11afSPaul Mackerras	/* after they enter the spin loop immediately below.	  */
12314cf11afSPaul Mackerras	.globl	__secondary_hold_acknowledge
12414cf11afSPaul Mackerras__secondary_hold_acknowledge:
12514cf11afSPaul Mackerras	.llong	0x0
12614cf11afSPaul Mackerras
12714cf11afSPaul Mackerras	. = 0x60
12814cf11afSPaul Mackerras/*
12914cf11afSPaul Mackerras * The following code is used on pSeries to hold secondary processors
13014cf11afSPaul Mackerras * in a spin loop after they have been freed from OpenFirmware, but
13114cf11afSPaul Mackerras * before the bulk of the kernel has been relocated.  This code
13214cf11afSPaul Mackerras * is relocated to physical address 0x60 before prom_init is run.
13314cf11afSPaul Mackerras * All of it must fit below the first exception vector at 0x100.
13414cf11afSPaul Mackerras */
13514cf11afSPaul Mackerras_GLOBAL(__secondary_hold)
13614cf11afSPaul Mackerras	mfmsr	r24
13714cf11afSPaul Mackerras	ori	r24,r24,MSR_RI
13814cf11afSPaul Mackerras	mtmsrd	r24			/* RI on */
13914cf11afSPaul Mackerras
140f1870f77SAnton Blanchard	/* Grab our physical cpu number */
14114cf11afSPaul Mackerras	mr	r24,r3
14214cf11afSPaul Mackerras
14314cf11afSPaul Mackerras	/* Tell the master cpu we're here */
14414cf11afSPaul Mackerras	/* Relocation is off & we are located at an address less */
14514cf11afSPaul Mackerras	/* than 0x100, so only need to grab low order offset.    */
14614cf11afSPaul Mackerras	std	r24,__secondary_hold_acknowledge@l(0)
14714cf11afSPaul Mackerras	sync
14814cf11afSPaul Mackerras
14914cf11afSPaul Mackerras	/* All secondary cpus wait here until told to start. */
15014cf11afSPaul Mackerras100:	ld	r4,__secondary_hold_spinloop@l(0)
15114cf11afSPaul Mackerras	cmpdi	0,r4,1
15214cf11afSPaul Mackerras	bne	100b
15314cf11afSPaul Mackerras
154f1870f77SAnton Blanchard#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
155e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, .pSeries_secondary_smp_init)
156758438a7SMichael Ellerman	mtctr	r4
15714cf11afSPaul Mackerras	mr	r3,r24
158758438a7SMichael Ellerman	bctr
15914cf11afSPaul Mackerras#else
16014cf11afSPaul Mackerras	BUG_OPCODE
16114cf11afSPaul Mackerras#endif
16214cf11afSPaul Mackerras
16314cf11afSPaul Mackerras/* This value is used to mark exception frames on the stack. */
16414cf11afSPaul Mackerras	.section ".toc","aw"
16514cf11afSPaul Mackerrasexception_marker:
16614cf11afSPaul Mackerras	.tc	ID_72656773_68657265[TC],0x7265677368657265
16714cf11afSPaul Mackerras	.text
16814cf11afSPaul Mackerras
16914cf11afSPaul Mackerras/*
17014cf11afSPaul Mackerras * The following macros define the code that appears as
17114cf11afSPaul Mackerras * the prologue to each of the exception handlers.  They
17214cf11afSPaul Mackerras * are split into two parts to allow a single kernel binary
17314cf11afSPaul Mackerras * to be used for pSeries and iSeries.
17414cf11afSPaul Mackerras * LOL.  One day... - paulus
17514cf11afSPaul Mackerras */
17614cf11afSPaul Mackerras
17714cf11afSPaul Mackerras/*
17814cf11afSPaul Mackerras * We make as much of the exception code common between native
17914cf11afSPaul Mackerras * exception handlers (including pSeries LPAR) and iSeries LPAR
18014cf11afSPaul Mackerras * implementations as possible.
18114cf11afSPaul Mackerras */
18214cf11afSPaul Mackerras
18314cf11afSPaul Mackerras/*
18414cf11afSPaul Mackerras * This is the start of the interrupt handlers for pSeries
18514cf11afSPaul Mackerras * This code runs with relocation off.
18614cf11afSPaul Mackerras */
18714cf11afSPaul Mackerras#define EX_R9		0
18814cf11afSPaul Mackerras#define EX_R10		8
18914cf11afSPaul Mackerras#define EX_R11		16
19014cf11afSPaul Mackerras#define EX_R12		24
19114cf11afSPaul Mackerras#define EX_R13		32
19214cf11afSPaul Mackerras#define EX_SRR0		40
19314cf11afSPaul Mackerras#define EX_DAR		48
19414cf11afSPaul Mackerras#define EX_DSISR	56
19514cf11afSPaul Mackerras#define EX_CCR		60
1963c726f8dSBenjamin Herrenschmidt#define EX_R3		64
1973c726f8dSBenjamin Herrenschmidt#define EX_LR		72
19814cf11afSPaul Mackerras
199758438a7SMichael Ellerman/*
200e58c3495SDavid Gibson * We're short on space and time in the exception prolog, so we can't
201e58c3495SDavid Gibson * use the normal SET_REG_IMMEDIATE macro. Normally we just need the
202e58c3495SDavid Gibson * low halfword of the address, but for Kdump we need the whole low
203e58c3495SDavid Gibson * word.
204758438a7SMichael Ellerman */
205758438a7SMichael Ellerman#ifdef CONFIG_CRASH_DUMP
206758438a7SMichael Ellerman#define LOAD_HANDLER(reg, label)					\
207758438a7SMichael Ellerman	oris	reg,reg,(label)@h;	/* virt addr of handler ... */	\
208758438a7SMichael Ellerman	ori	reg,reg,(label)@l;	/* .. and the rest */
209758438a7SMichael Ellerman#else
210758438a7SMichael Ellerman#define LOAD_HANDLER(reg, label)					\
211758438a7SMichael Ellerman	ori	reg,reg,(label)@l;	/* virt addr of handler ... */
212758438a7SMichael Ellerman#endif
213758438a7SMichael Ellerman
21414cf11afSPaul Mackerras#define EXCEPTION_PROLOG_PSERIES(area, label)				\
215b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
21614cf11afSPaul Mackerras	std	r9,area+EX_R9(r13);	/* save r9 - r12 */		\
21714cf11afSPaul Mackerras	std	r10,area+EX_R10(r13);					\
21814cf11afSPaul Mackerras	std	r11,area+EX_R11(r13);					\
21914cf11afSPaul Mackerras	std	r12,area+EX_R12(r13);					\
220b5bbeb23SPaul Mackerras	mfspr	r9,SPRN_SPRG1;						\
22114cf11afSPaul Mackerras	std	r9,area+EX_R13(r13);					\
22214cf11afSPaul Mackerras	mfcr	r9;							\
22314cf11afSPaul Mackerras	clrrdi	r12,r13,32;		/* get high part of &label */	\
22414cf11afSPaul Mackerras	mfmsr	r10;							\
225b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_SRR0;		/* save SRR0 */			\
226758438a7SMichael Ellerman	LOAD_HANDLER(r12,label)						\
22714cf11afSPaul Mackerras	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI;				\
228b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r12;						\
229b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SRR1;		/* and SRR1 */			\
230b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r10;						\
23114cf11afSPaul Mackerras	rfid;								\
23214cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
23314cf11afSPaul Mackerras
23414cf11afSPaul Mackerras/*
23514cf11afSPaul Mackerras * This is the start of the interrupt handlers for iSeries
23614cf11afSPaul Mackerras * This code runs with relocation on.
23714cf11afSPaul Mackerras */
23814cf11afSPaul Mackerras#define EXCEPTION_PROLOG_ISERIES_1(area)				\
239b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
24014cf11afSPaul Mackerras	std	r9,area+EX_R9(r13);	/* save r9 - r12 */		\
24114cf11afSPaul Mackerras	std	r10,area+EX_R10(r13);					\
24214cf11afSPaul Mackerras	std	r11,area+EX_R11(r13);					\
24314cf11afSPaul Mackerras	std	r12,area+EX_R12(r13);					\
244b5bbeb23SPaul Mackerras	mfspr	r9,SPRN_SPRG1;						\
24514cf11afSPaul Mackerras	std	r9,area+EX_R13(r13);					\
24614cf11afSPaul Mackerras	mfcr	r9
24714cf11afSPaul Mackerras
24814cf11afSPaul Mackerras#define EXCEPTION_PROLOG_ISERIES_2					\
24914cf11afSPaul Mackerras	mfmsr	r10;							\
2503356bb9fSDavid Gibson	ld	r12,PACALPPACAPTR(r13);					\
2513356bb9fSDavid Gibson	ld	r11,LPPACASRR0(r12);					\
2523356bb9fSDavid Gibson	ld	r12,LPPACASRR1(r12);					\
25314cf11afSPaul Mackerras	ori	r10,r10,MSR_RI;						\
25414cf11afSPaul Mackerras	mtmsrd	r10,1
25514cf11afSPaul Mackerras
25614cf11afSPaul Mackerras/*
25714cf11afSPaul Mackerras * The common exception prolog is used for all except a few exceptions
25814cf11afSPaul Mackerras * such as a segment miss on a kernel address.  We have to be prepared
25914cf11afSPaul Mackerras * to take another exception from the point where we first touch the
26014cf11afSPaul Mackerras * kernel stack onwards.
26114cf11afSPaul Mackerras *
26214cf11afSPaul Mackerras * On entry r13 points to the paca, r9-r13 are saved in the paca,
26314cf11afSPaul Mackerras * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
26414cf11afSPaul Mackerras * SRR1, and relocation is on.
26514cf11afSPaul Mackerras */
26614cf11afSPaul Mackerras#define EXCEPTION_PROLOG_COMMON(n, area)				   \
26714cf11afSPaul Mackerras	andi.	r10,r12,MSR_PR;		/* See if coming from user	*/ \
26814cf11afSPaul Mackerras	mr	r10,r1;			/* Save r1			*/ \
26914cf11afSPaul Mackerras	subi	r1,r1,INT_FRAME_SIZE;	/* alloc frame on kernel stack	*/ \
27014cf11afSPaul Mackerras	beq-	1f;							   \
27114cf11afSPaul Mackerras	ld	r1,PACAKSAVE(r13);	/* kernel stack to use		*/ \
27214cf11afSPaul Mackerras1:	cmpdi	cr1,r1,0;		/* check if r1 is in userspace	*/ \
27314cf11afSPaul Mackerras	bge-	cr1,bad_stack;		/* abort if it is		*/ \
27414cf11afSPaul Mackerras	std	r9,_CCR(r1);		/* save CR in stackframe	*/ \
27514cf11afSPaul Mackerras	std	r11,_NIP(r1);		/* save SRR0 in stackframe	*/ \
27614cf11afSPaul Mackerras	std	r12,_MSR(r1);		/* save SRR1 in stackframe	*/ \
27714cf11afSPaul Mackerras	std	r10,0(r1);		/* make stack chain pointer	*/ \
27814cf11afSPaul Mackerras	std	r0,GPR0(r1);		/* save r0 in stackframe	*/ \
27914cf11afSPaul Mackerras	std	r10,GPR1(r1);		/* save r1 in stackframe	*/ \
280c6622f63SPaul Mackerras	ACCOUNT_CPU_USER_ENTRY(r9, r10);				   \
28114cf11afSPaul Mackerras	std	r2,GPR2(r1);		/* save r2 in stackframe	*/ \
28214cf11afSPaul Mackerras	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe	*/ \
28314cf11afSPaul Mackerras	SAVE_2GPRS(7, r1);		/* save r7, r8 in stackframe	*/ \
28414cf11afSPaul Mackerras	ld	r9,area+EX_R9(r13);	/* move r9, r10 to stackframe	*/ \
28514cf11afSPaul Mackerras	ld	r10,area+EX_R10(r13);					   \
28614cf11afSPaul Mackerras	std	r9,GPR9(r1);						   \
28714cf11afSPaul Mackerras	std	r10,GPR10(r1);						   \
28814cf11afSPaul Mackerras	ld	r9,area+EX_R11(r13);	/* move r11 - r13 to stackframe	*/ \
28914cf11afSPaul Mackerras	ld	r10,area+EX_R12(r13);					   \
29014cf11afSPaul Mackerras	ld	r11,area+EX_R13(r13);					   \
29114cf11afSPaul Mackerras	std	r9,GPR11(r1);						   \
29214cf11afSPaul Mackerras	std	r10,GPR12(r1);						   \
29314cf11afSPaul Mackerras	std	r11,GPR13(r1);						   \
29414cf11afSPaul Mackerras	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
29514cf11afSPaul Mackerras	mflr	r9;			/* save LR in stackframe	*/ \
29614cf11afSPaul Mackerras	std	r9,_LINK(r1);						   \
29714cf11afSPaul Mackerras	mfctr	r10;			/* save CTR in stackframe	*/ \
29814cf11afSPaul Mackerras	std	r10,_CTR(r1);						   \
299b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
30014cf11afSPaul Mackerras	std	r11,_XER(r1);						   \
30114cf11afSPaul Mackerras	li	r9,(n)+1;						   \
30214cf11afSPaul Mackerras	std	r9,_TRAP(r1);		/* set trap number		*/ \
30314cf11afSPaul Mackerras	li	r10,0;							   \
30414cf11afSPaul Mackerras	ld	r11,exception_marker@toc(r2);				   \
30514cf11afSPaul Mackerras	std	r10,RESULT(r1);		/* clear regs->result		*/ \
30614cf11afSPaul Mackerras	std	r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame	*/
30714cf11afSPaul Mackerras
30814cf11afSPaul Mackerras/*
30914cf11afSPaul Mackerras * Exception vectors.
31014cf11afSPaul Mackerras */
31114cf11afSPaul Mackerras#define STD_EXCEPTION_PSERIES(n, label)			\
31214cf11afSPaul Mackerras	. = n;						\
31314cf11afSPaul Mackerras	.globl label##_pSeries;				\
31414cf11afSPaul Mackerraslabel##_pSeries:					\
31514cf11afSPaul Mackerras	HMT_MEDIUM;					\
316b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13;		/* save r13 */	\
31714cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
31814cf11afSPaul Mackerras
31914cf11afSPaul Mackerras#define STD_EXCEPTION_ISERIES(n, label, area)		\
32014cf11afSPaul Mackerras	.globl label##_iSeries;				\
32114cf11afSPaul Mackerraslabel##_iSeries:					\
32214cf11afSPaul Mackerras	HMT_MEDIUM;					\
323b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13;		/* save r13 */	\
32414cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_1(area);		\
32514cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_2;			\
32614cf11afSPaul Mackerras	b	label##_common
32714cf11afSPaul Mackerras
32814cf11afSPaul Mackerras#define MASKABLE_EXCEPTION_ISERIES(n, label)				\
32914cf11afSPaul Mackerras	.globl label##_iSeries;						\
33014cf11afSPaul Mackerraslabel##_iSeries:							\
33114cf11afSPaul Mackerras	HMT_MEDIUM;							\
332b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13;		/* save r13 */			\
33314cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);				\
33414cf11afSPaul Mackerras	lbz	r10,PACAPROCENABLED(r13);				\
33514cf11afSPaul Mackerras	cmpwi	0,r10,0;						\
33614cf11afSPaul Mackerras	beq-	label##_iSeries_masked;					\
33714cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_2;					\
33814cf11afSPaul Mackerras	b	label##_common;						\
33914cf11afSPaul Mackerras
34014cf11afSPaul Mackerras#ifdef DO_SOFT_DISABLE
34114cf11afSPaul Mackerras#define DISABLE_INTS				\
34214cf11afSPaul Mackerras	lbz	r10,PACAPROCENABLED(r13);	\
34314cf11afSPaul Mackerras	li	r11,0;				\
34414cf11afSPaul Mackerras	std	r10,SOFTE(r1);			\
34514cf11afSPaul Mackerras	mfmsr	r10;				\
34614cf11afSPaul Mackerras	stb	r11,PACAPROCENABLED(r13);	\
34714cf11afSPaul Mackerras	ori	r10,r10,MSR_EE;			\
34814cf11afSPaul Mackerras	mtmsrd	r10,1
34914cf11afSPaul Mackerras
35014cf11afSPaul Mackerras#define ENABLE_INTS				\
35114cf11afSPaul Mackerras	lbz	r10,PACAPROCENABLED(r13);	\
35214cf11afSPaul Mackerras	mfmsr	r11;				\
35314cf11afSPaul Mackerras	std	r10,SOFTE(r1);			\
35414cf11afSPaul Mackerras	ori	r11,r11,MSR_EE;			\
35514cf11afSPaul Mackerras	mtmsrd	r11,1
35614cf11afSPaul Mackerras
35714cf11afSPaul Mackerras#else	/* hard enable/disable interrupts */
35814cf11afSPaul Mackerras#define DISABLE_INTS
35914cf11afSPaul Mackerras
36014cf11afSPaul Mackerras#define ENABLE_INTS				\
36114cf11afSPaul Mackerras	ld	r12,_MSR(r1);			\
36214cf11afSPaul Mackerras	mfmsr	r11;				\
36314cf11afSPaul Mackerras	rlwimi	r11,r12,0,MSR_EE;		\
36414cf11afSPaul Mackerras	mtmsrd	r11,1
36514cf11afSPaul Mackerras
36614cf11afSPaul Mackerras#endif
36714cf11afSPaul Mackerras
36814cf11afSPaul Mackerras#define STD_EXCEPTION_COMMON(trap, label, hdlr)		\
36914cf11afSPaul Mackerras	.align	7;					\
37014cf11afSPaul Mackerras	.globl label##_common;				\
37114cf11afSPaul Mackerraslabel##_common:						\
37214cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);	\
37314cf11afSPaul Mackerras	DISABLE_INTS;					\
37414cf11afSPaul Mackerras	bl	.save_nvgprs;				\
37514cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD;		\
37614cf11afSPaul Mackerras	bl	hdlr;					\
37714cf11afSPaul Mackerras	b	.ret_from_except
37814cf11afSPaul Mackerras
37914cf11afSPaul Mackerras#define STD_EXCEPTION_COMMON_LITE(trap, label, hdlr)	\
38014cf11afSPaul Mackerras	.align	7;					\
38114cf11afSPaul Mackerras	.globl label##_common;				\
38214cf11afSPaul Mackerraslabel##_common:						\
38314cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);	\
38414cf11afSPaul Mackerras	DISABLE_INTS;					\
385cb2c9b27SAnton Blanchard	bl	.ppc64_runlatch_on;			\
38614cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD;		\
38714cf11afSPaul Mackerras	bl	hdlr;					\
38814cf11afSPaul Mackerras	b	.ret_from_except_lite
38914cf11afSPaul Mackerras
39014cf11afSPaul Mackerras/*
39114cf11afSPaul Mackerras * Start of pSeries system interrupt routines
39214cf11afSPaul Mackerras */
39314cf11afSPaul Mackerras	. = 0x100
39414cf11afSPaul Mackerras	.globl __start_interrupts
39514cf11afSPaul Mackerras__start_interrupts:
39614cf11afSPaul Mackerras
39714cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x100, system_reset)
39814cf11afSPaul Mackerras
39914cf11afSPaul Mackerras	. = 0x200
40014cf11afSPaul Mackerras_machine_check_pSeries:
40114cf11afSPaul Mackerras	HMT_MEDIUM
402b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13		/* save r13 */
40314cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
40414cf11afSPaul Mackerras
40514cf11afSPaul Mackerras	. = 0x300
40614cf11afSPaul Mackerras	.globl data_access_pSeries
40714cf11afSPaul Mackerrasdata_access_pSeries:
40814cf11afSPaul Mackerras	HMT_MEDIUM
409b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13
41014cf11afSPaul MackerrasBEGIN_FTR_SECTION
411b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG2,r12
412b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_DAR
413b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_DSISR
41414cf11afSPaul Mackerras	srdi	r13,r13,60
41514cf11afSPaul Mackerras	rlwimi	r13,r12,16,0x20
41614cf11afSPaul Mackerras	mfcr	r12
41714cf11afSPaul Mackerras	cmpwi	r13,0x2c
41814cf11afSPaul Mackerras	beq	.do_stab_bolted_pSeries
41914cf11afSPaul Mackerras	mtcrf	0x80,r12
420b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SPRG2
42114cf11afSPaul MackerrasEND_FTR_SECTION_IFCLR(CPU_FTR_SLB)
42214cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
42314cf11afSPaul Mackerras
42414cf11afSPaul Mackerras	. = 0x380
42514cf11afSPaul Mackerras	.globl data_access_slb_pSeries
42614cf11afSPaul Mackerrasdata_access_slb_pSeries:
42714cf11afSPaul Mackerras	HMT_MEDIUM
428b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13
429b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
4303c726f8dSBenjamin Herrenschmidt	std	r3,PACA_EXSLB+EX_R3(r13)
4313c726f8dSBenjamin Herrenschmidt	mfspr	r3,SPRN_DAR
43214cf11afSPaul Mackerras	std	r9,PACA_EXSLB+EX_R9(r13)	/* save r9 - r12 */
4333c726f8dSBenjamin Herrenschmidt	mfcr	r9
4343c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
4353c726f8dSBenjamin Herrenschmidt	/* Keep that around for when we re-implement dynamic VSIDs */
4363c726f8dSBenjamin Herrenschmidt	cmpdi	r3,0
4373c726f8dSBenjamin Herrenschmidt	bge	slb_miss_user_pseries
4383c726f8dSBenjamin Herrenschmidt#endif /* __DISABLED__ */
43914cf11afSPaul Mackerras	std	r10,PACA_EXSLB+EX_R10(r13)
44014cf11afSPaul Mackerras	std	r11,PACA_EXSLB+EX_R11(r13)
44114cf11afSPaul Mackerras	std	r12,PACA_EXSLB+EX_R12(r13)
4423c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRN_SPRG1
4433c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R13(r13)
444b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SRR1		/* and SRR1 */
4453c726f8dSBenjamin Herrenschmidt	b	.slb_miss_realmode	/* Rel. branch works in real mode */
44614cf11afSPaul Mackerras
44714cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x400, instruction_access)
44814cf11afSPaul Mackerras
44914cf11afSPaul Mackerras	. = 0x480
45014cf11afSPaul Mackerras	.globl instruction_access_slb_pSeries
45114cf11afSPaul Mackerrasinstruction_access_slb_pSeries:
45214cf11afSPaul Mackerras	HMT_MEDIUM
453b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13
454b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
4553c726f8dSBenjamin Herrenschmidt	std	r3,PACA_EXSLB+EX_R3(r13)
4563c726f8dSBenjamin Herrenschmidt	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
45714cf11afSPaul Mackerras	std	r9,PACA_EXSLB+EX_R9(r13)	/* save r9 - r12 */
4583c726f8dSBenjamin Herrenschmidt	mfcr	r9
4593c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
4603c726f8dSBenjamin Herrenschmidt	/* Keep that around for when we re-implement dynamic VSIDs */
4613c726f8dSBenjamin Herrenschmidt	cmpdi	r3,0
4623c726f8dSBenjamin Herrenschmidt	bge	slb_miss_user_pseries
4633c726f8dSBenjamin Herrenschmidt#endif /* __DISABLED__ */
46414cf11afSPaul Mackerras	std	r10,PACA_EXSLB+EX_R10(r13)
46514cf11afSPaul Mackerras	std	r11,PACA_EXSLB+EX_R11(r13)
46614cf11afSPaul Mackerras	std	r12,PACA_EXSLB+EX_R12(r13)
4673c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRN_SPRG1
4683c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R13(r13)
469b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SRR1		/* and SRR1 */
4703c726f8dSBenjamin Herrenschmidt	b	.slb_miss_realmode	/* Rel. branch works in real mode */
47114cf11afSPaul Mackerras
47214cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
47314cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x600, alignment)
47414cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x700, program_check)
47514cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
47614cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x900, decrementer)
47714cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0xa00, trap_0a)
47814cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0xb00, trap_0b)
47914cf11afSPaul Mackerras
48014cf11afSPaul Mackerras	. = 0xc00
48114cf11afSPaul Mackerras	.globl	system_call_pSeries
48214cf11afSPaul Mackerrassystem_call_pSeries:
48314cf11afSPaul Mackerras	HMT_MEDIUM
48414cf11afSPaul Mackerras	mr	r9,r13
48514cf11afSPaul Mackerras	mfmsr	r10
486b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3
487b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_SRR0
48814cf11afSPaul Mackerras	clrrdi	r12,r13,32
48914cf11afSPaul Mackerras	oris	r12,r12,system_call_common@h
49014cf11afSPaul Mackerras	ori	r12,r12,system_call_common@l
491b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r12
49214cf11afSPaul Mackerras	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI
493b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SRR1
494b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r10
49514cf11afSPaul Mackerras	rfid
49614cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
49714cf11afSPaul Mackerras
49814cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0xd00, single_step)
49914cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0xe00, trap_0e)
50014cf11afSPaul Mackerras
50114cf11afSPaul Mackerras	/* We need to deal with the Altivec unavailable exception
50214cf11afSPaul Mackerras	 * here which is at 0xf20, thus in the middle of the
50314cf11afSPaul Mackerras	 * prolog code of the PerformanceMonitor one. A little
50414cf11afSPaul Mackerras	 * trickery is thus necessary
50514cf11afSPaul Mackerras	 */
50614cf11afSPaul Mackerras	. = 0xf00
50714cf11afSPaul Mackerras	b	performance_monitor_pSeries
50814cf11afSPaul Mackerras
50914cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0xf20, altivec_unavailable)
51014cf11afSPaul Mackerras
51114cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
51214cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
51314cf11afSPaul Mackerras
51414cf11afSPaul Mackerras	. = 0x3000
51514cf11afSPaul Mackerras
51614cf11afSPaul Mackerras/*** pSeries interrupt support ***/
51714cf11afSPaul Mackerras
51814cf11afSPaul Mackerras	/* moved from 0xf00 */
51914cf11afSPaul Mackerras	STD_EXCEPTION_PSERIES(., performance_monitor)
52014cf11afSPaul Mackerras
52114cf11afSPaul Mackerras	.align	7
52214cf11afSPaul Mackerras_GLOBAL(do_stab_bolted_pSeries)
52314cf11afSPaul Mackerras	mtcrf	0x80,r12
524b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SPRG2
52514cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
52614cf11afSPaul Mackerras
52714cf11afSPaul Mackerras/*
5283c726f8dSBenjamin Herrenschmidt * We have some room here  we use that to put
5293c726f8dSBenjamin Herrenschmidt * the peries slb miss user trampoline code so it's reasonably
5303c726f8dSBenjamin Herrenschmidt * away from slb_miss_user_common to avoid problems with rfid
5313c726f8dSBenjamin Herrenschmidt *
5323c726f8dSBenjamin Herrenschmidt * This is used for when the SLB miss handler has to go virtual,
5333c726f8dSBenjamin Herrenschmidt * which doesn't happen for now anymore but will once we re-implement
5343c726f8dSBenjamin Herrenschmidt * dynamic VSIDs for shared page tables
5353c726f8dSBenjamin Herrenschmidt */
5363c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
5373c726f8dSBenjamin Herrenschmidtslb_miss_user_pseries:
5383c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXGEN+EX_R10(r13)
5393c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXGEN+EX_R11(r13)
5403c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXGEN+EX_R12(r13)
5413c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRG1
5423c726f8dSBenjamin Herrenschmidt	ld	r11,PACA_EXSLB+EX_R9(r13)
5433c726f8dSBenjamin Herrenschmidt	ld	r12,PACA_EXSLB+EX_R3(r13)
5443c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXGEN+EX_R13(r13)
5453c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXGEN+EX_R9(r13)
5463c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXGEN+EX_R3(r13)
5473c726f8dSBenjamin Herrenschmidt	clrrdi	r12,r13,32
5483c726f8dSBenjamin Herrenschmidt	mfmsr	r10
5493c726f8dSBenjamin Herrenschmidt	mfspr	r11,SRR0			/* save SRR0 */
5503c726f8dSBenjamin Herrenschmidt	ori	r12,r12,slb_miss_user_common@l	/* virt addr of handler */
5513c726f8dSBenjamin Herrenschmidt	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI
5523c726f8dSBenjamin Herrenschmidt	mtspr	SRR0,r12
5533c726f8dSBenjamin Herrenschmidt	mfspr	r12,SRR1			/* and SRR1 */
5543c726f8dSBenjamin Herrenschmidt	mtspr	SRR1,r10
5553c726f8dSBenjamin Herrenschmidt	rfid
5563c726f8dSBenjamin Herrenschmidt	b	.				/* prevent spec. execution */
5573c726f8dSBenjamin Herrenschmidt#endif /* __DISABLED__ */
5583c726f8dSBenjamin Herrenschmidt
5593c726f8dSBenjamin Herrenschmidt/*
56014cf11afSPaul Mackerras * Vectors for the FWNMI option.  Share common code.
56114cf11afSPaul Mackerras */
56214cf11afSPaul Mackerras	.globl system_reset_fwnmi
5638c4f1f29SMichael Ellerman      .align 7
56414cf11afSPaul Mackerrassystem_reset_fwnmi:
56514cf11afSPaul Mackerras	HMT_MEDIUM
566b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13		/* save r13 */
56714cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
56814cf11afSPaul Mackerras
56914cf11afSPaul Mackerras	.globl machine_check_fwnmi
5708c4f1f29SMichael Ellerman      .align 7
57114cf11afSPaul Mackerrasmachine_check_fwnmi:
57214cf11afSPaul Mackerras	HMT_MEDIUM
573b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13		/* save r13 */
57414cf11afSPaul Mackerras	EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
57514cf11afSPaul Mackerras
57614cf11afSPaul Mackerras#ifdef CONFIG_PPC_ISERIES
57714cf11afSPaul Mackerras/***  ISeries-LPAR interrupt handlers ***/
57814cf11afSPaul Mackerras
57914cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0x200, machine_check, PACA_EXMC)
58014cf11afSPaul Mackerras
58114cf11afSPaul Mackerras	.globl data_access_iSeries
58214cf11afSPaul Mackerrasdata_access_iSeries:
583b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13
58414cf11afSPaul MackerrasBEGIN_FTR_SECTION
585b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG2,r12
586b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_DAR
587b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_DSISR
58814cf11afSPaul Mackerras	srdi	r13,r13,60
58914cf11afSPaul Mackerras	rlwimi	r13,r12,16,0x20
59014cf11afSPaul Mackerras	mfcr	r12
59114cf11afSPaul Mackerras	cmpwi	r13,0x2c
59214cf11afSPaul Mackerras	beq	.do_stab_bolted_iSeries
59314cf11afSPaul Mackerras	mtcrf	0x80,r12
594b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SPRG2
59514cf11afSPaul MackerrasEND_FTR_SECTION_IFCLR(CPU_FTR_SLB)
59614cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN)
59714cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_2
59814cf11afSPaul Mackerras	b	data_access_common
59914cf11afSPaul Mackerras
60014cf11afSPaul Mackerras.do_stab_bolted_iSeries:
60114cf11afSPaul Mackerras	mtcrf	0x80,r12
602b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_SPRG2
60314cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_1(PACA_EXSLB)
60414cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_2
60514cf11afSPaul Mackerras	b	.do_stab_bolted
60614cf11afSPaul Mackerras
60714cf11afSPaul Mackerras	.globl	data_access_slb_iSeries
60814cf11afSPaul Mackerrasdata_access_slb_iSeries:
609b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13		/* save r13 */
6103c726f8dSBenjamin Herrenschmidt	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
61114cf11afSPaul Mackerras	std	r3,PACA_EXSLB+EX_R3(r13)
612b5bbeb23SPaul Mackerras	mfspr	r3,SPRN_DAR
6133c726f8dSBenjamin Herrenschmidt	std	r9,PACA_EXSLB+EX_R9(r13)
6143c726f8dSBenjamin Herrenschmidt	mfcr	r9
6153c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
6163c726f8dSBenjamin Herrenschmidt	cmpdi	r3,0
6173c726f8dSBenjamin Herrenschmidt	bge	slb_miss_user_iseries
6183c726f8dSBenjamin Herrenschmidt#endif
6193c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R10(r13)
6203c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXSLB+EX_R11(r13)
6213c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXSLB+EX_R12(r13)
6223c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRN_SPRG1
6233c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R13(r13)
6243356bb9fSDavid Gibson	ld	r12,PACALPPACAPTR(r13)
6253356bb9fSDavid Gibson	ld	r12,LPPACASRR1(r12)
6263c726f8dSBenjamin Herrenschmidt	b	.slb_miss_realmode
62714cf11afSPaul Mackerras
62814cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0x400, instruction_access, PACA_EXGEN)
62914cf11afSPaul Mackerras
63014cf11afSPaul Mackerras	.globl	instruction_access_slb_iSeries
63114cf11afSPaul Mackerrasinstruction_access_slb_iSeries:
632b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG1,r13		/* save r13 */
6333c726f8dSBenjamin Herrenschmidt	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
63414cf11afSPaul Mackerras	std	r3,PACA_EXSLB+EX_R3(r13)
6353356bb9fSDavid Gibson	ld	r3,PACALPPACAPTR(r13)
6363356bb9fSDavid Gibson	ld	r3,LPPACASRR0(r3)	/* get SRR0 value */
6373c726f8dSBenjamin Herrenschmidt	std	r9,PACA_EXSLB+EX_R9(r13)
6383c726f8dSBenjamin Herrenschmidt	mfcr	r9
6393c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
6403c726f8dSBenjamin Herrenschmidt	cmpdi	r3,0
6413c726f8dSBenjamin Herrenschmidt	bge	.slb_miss_user_iseries
6423c726f8dSBenjamin Herrenschmidt#endif
6433c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R10(r13)
6443c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXSLB+EX_R11(r13)
6453c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXSLB+EX_R12(r13)
6463c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRN_SPRG1
6473c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_R13(r13)
6483356bb9fSDavid Gibson	ld	r12,PACALPPACAPTR(r13)
6493356bb9fSDavid Gibson	ld	r12,LPPACASRR1(r12)
6503c726f8dSBenjamin Herrenschmidt	b	.slb_miss_realmode
6513c726f8dSBenjamin Herrenschmidt
6523c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
6533c726f8dSBenjamin Herrenschmidtslb_miss_user_iseries:
6543c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXGEN+EX_R10(r13)
6553c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXGEN+EX_R11(r13)
6563c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXGEN+EX_R12(r13)
6573c726f8dSBenjamin Herrenschmidt	mfspr	r10,SPRG1
6583c726f8dSBenjamin Herrenschmidt	ld	r11,PACA_EXSLB+EX_R9(r13)
6593c726f8dSBenjamin Herrenschmidt	ld	r12,PACA_EXSLB+EX_R3(r13)
6603c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXGEN+EX_R13(r13)
6613c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXGEN+EX_R9(r13)
6623c726f8dSBenjamin Herrenschmidt	std	r12,PACA_EXGEN+EX_R3(r13)
6633c726f8dSBenjamin Herrenschmidt	EXCEPTION_PROLOG_ISERIES_2
6643c726f8dSBenjamin Herrenschmidt	b	slb_miss_user_common
6653c726f8dSBenjamin Herrenschmidt#endif
66614cf11afSPaul Mackerras
66714cf11afSPaul Mackerras	MASKABLE_EXCEPTION_ISERIES(0x500, hardware_interrupt)
66814cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0x600, alignment, PACA_EXGEN)
66914cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0x700, program_check, PACA_EXGEN)
67014cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0x800, fp_unavailable, PACA_EXGEN)
67114cf11afSPaul Mackerras	MASKABLE_EXCEPTION_ISERIES(0x900, decrementer)
67214cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0xa00, trap_0a, PACA_EXGEN)
67314cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES(0xb00, trap_0b, PACA_EXGEN)
67414cf11afSPaul Mackerras
67514cf11afSPaul Mackerras	.globl	system_call_iSeries
67614cf11afSPaul Mackerrassystem_call_iSeries:
67714cf11afSPaul Mackerras	mr	r9,r13
678b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3
67914cf11afSPaul Mackerras	EXCEPTION_PROLOG_ISERIES_2
68014cf11afSPaul Mackerras	b	system_call_common
68114cf11afSPaul Mackerras
68214cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES( 0xd00, single_step, PACA_EXGEN)
68314cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES( 0xe00, trap_0e, PACA_EXGEN)
68414cf11afSPaul Mackerras	STD_EXCEPTION_ISERIES( 0xf00, performance_monitor, PACA_EXGEN)
68514cf11afSPaul Mackerras
68614cf11afSPaul Mackerras	.globl system_reset_iSeries
68714cf11afSPaul Mackerrassystem_reset_iSeries:
688b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3		/* Get paca address */
68914cf11afSPaul Mackerras	mfmsr	r24
69014cf11afSPaul Mackerras	ori	r24,r24,MSR_RI
69114cf11afSPaul Mackerras	mtmsrd	r24			/* RI on */
69214cf11afSPaul Mackerras	lhz	r24,PACAPACAINDEX(r13)	/* Get processor # */
69314cf11afSPaul Mackerras	cmpwi	0,r24,0			/* Are we processor 0? */
69414cf11afSPaul Mackerras	beq	.__start_initialization_iSeries	/* Start up the first processor */
69514cf11afSPaul Mackerras	mfspr	r4,SPRN_CTRLF
69614cf11afSPaul Mackerras	li	r5,CTRL_RUNLATCH	/* Turn off the run light */
69714cf11afSPaul Mackerras	andc	r4,r4,r5
69814cf11afSPaul Mackerras	mtspr	SPRN_CTRLT,r4
69914cf11afSPaul Mackerras
70014cf11afSPaul Mackerras1:
70114cf11afSPaul Mackerras	HMT_LOW
70214cf11afSPaul Mackerras#ifdef CONFIG_SMP
70314cf11afSPaul Mackerras	lbz	r23,PACAPROCSTART(r13)	/* Test if this processor
70414cf11afSPaul Mackerras					 * should start */
70514cf11afSPaul Mackerras	sync
706e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3,current_set)
70714cf11afSPaul Mackerras	sldi	r28,r24,3		/* get current_set[cpu#] */
70814cf11afSPaul Mackerras	ldx	r3,r3,r28
70914cf11afSPaul Mackerras	addi	r1,r3,THREAD_SIZE
71014cf11afSPaul Mackerras	subi	r1,r1,STACK_FRAME_OVERHEAD
71114cf11afSPaul Mackerras
71214cf11afSPaul Mackerras	cmpwi	0,r23,0
71314cf11afSPaul Mackerras	beq	iSeries_secondary_smp_loop	/* Loop until told to go */
71414cf11afSPaul Mackerras	bne	.__secondary_start		/* Loop until told to go */
71514cf11afSPaul MackerrasiSeries_secondary_smp_loop:
71614cf11afSPaul Mackerras	/* Let the Hypervisor know we are alive */
71714cf11afSPaul Mackerras	/* 8002 is a call to HvCallCfg::getLps, a harmless Hypervisor function */
71814cf11afSPaul Mackerras	lis	r3,0x8002
71914cf11afSPaul Mackerras	rldicr	r3,r3,32,15		/* r0 = (r3 << 32) & 0xffff000000000000 */
72014cf11afSPaul Mackerras#else /* CONFIG_SMP */
72114cf11afSPaul Mackerras	/* Yield the processor.  This is required for non-SMP kernels
72214cf11afSPaul Mackerras		which are running on multi-threaded machines. */
72314cf11afSPaul Mackerras	lis	r3,0x8000
72414cf11afSPaul Mackerras	rldicr	r3,r3,32,15		/* r3 = (r3 << 32) & 0xffff000000000000 */
72514cf11afSPaul Mackerras	addi	r3,r3,18		/* r3 = 0x8000000000000012 which is "yield" */
72614cf11afSPaul Mackerras	li	r4,0			/* "yield timed" */
72714cf11afSPaul Mackerras	li	r5,-1			/* "yield forever" */
72814cf11afSPaul Mackerras#endif /* CONFIG_SMP */
72914cf11afSPaul Mackerras	li	r0,-1			/* r0=-1 indicates a Hypervisor call */
73014cf11afSPaul Mackerras	sc				/* Invoke the hypervisor via a system call */
731b5bbeb23SPaul Mackerras	mfspr	r13,SPRN_SPRG3		/* Put r13 back ???? */
73214cf11afSPaul Mackerras	b	1b			/* If SMP not configured, secondaries
73314cf11afSPaul Mackerras					 * loop forever */
73414cf11afSPaul Mackerras
73514cf11afSPaul Mackerras	.globl decrementer_iSeries_masked
73614cf11afSPaul Mackerrasdecrementer_iSeries_masked:
737f9b4045dSMichael Ellerman	/* We may not have a valid TOC pointer in here. */
73814cf11afSPaul Mackerras	li	r11,1
7393356bb9fSDavid Gibson	ld	r12,PACALPPACAPTR(r13)
7403356bb9fSDavid Gibson	stb	r11,LPPACADECRINT(r12)
741f9b4045dSMichael Ellerman	LOAD_REG_IMMEDIATE(r12, tb_ticks_per_jiffy)
742f9b4045dSMichael Ellerman	lwz	r12,0(r12)
74314cf11afSPaul Mackerras	mtspr	SPRN_DEC,r12
74414cf11afSPaul Mackerras	/* fall through */
74514cf11afSPaul Mackerras
74614cf11afSPaul Mackerras	.globl hardware_interrupt_iSeries_masked
74714cf11afSPaul Mackerrashardware_interrupt_iSeries_masked:
74814cf11afSPaul Mackerras	mtcrf	0x80,r9		/* Restore regs */
7493356bb9fSDavid Gibson	ld	r12,PACALPPACAPTR(r13)
7503356bb9fSDavid Gibson	ld	r11,LPPACASRR0(r12)
7513356bb9fSDavid Gibson	ld	r12,LPPACASRR1(r12)
752b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r11
753b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r12
75414cf11afSPaul Mackerras	ld	r9,PACA_EXGEN+EX_R9(r13)
75514cf11afSPaul Mackerras	ld	r10,PACA_EXGEN+EX_R10(r13)
75614cf11afSPaul Mackerras	ld	r11,PACA_EXGEN+EX_R11(r13)
75714cf11afSPaul Mackerras	ld	r12,PACA_EXGEN+EX_R12(r13)
75814cf11afSPaul Mackerras	ld	r13,PACA_EXGEN+EX_R13(r13)
75914cf11afSPaul Mackerras	rfid
76014cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
76114cf11afSPaul Mackerras#endif /* CONFIG_PPC_ISERIES */
76214cf11afSPaul Mackerras
76314cf11afSPaul Mackerras/*** Common interrupt handlers ***/
76414cf11afSPaul Mackerras
76514cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
76614cf11afSPaul Mackerras
76714cf11afSPaul Mackerras	/*
76814cf11afSPaul Mackerras	 * Machine check is different because we use a different
76914cf11afSPaul Mackerras	 * save area: PACA_EXMC instead of PACA_EXGEN.
77014cf11afSPaul Mackerras	 */
77114cf11afSPaul Mackerras	.align	7
77214cf11afSPaul Mackerras	.globl machine_check_common
77314cf11afSPaul Mackerrasmachine_check_common:
77414cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
77514cf11afSPaul Mackerras	DISABLE_INTS
77614cf11afSPaul Mackerras	bl	.save_nvgprs
77714cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
77814cf11afSPaul Mackerras	bl	.machine_check_exception
77914cf11afSPaul Mackerras	b	.ret_from_except
78014cf11afSPaul Mackerras
78114cf11afSPaul Mackerras	STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
78214cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
78314cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
78414cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
78514cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
78614cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0xf00, performance_monitor, .performance_monitor_exception)
78714cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
78814cf11afSPaul Mackerras#ifdef CONFIG_ALTIVEC
78914cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
79014cf11afSPaul Mackerras#else
79114cf11afSPaul Mackerras	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
79214cf11afSPaul Mackerras#endif
79314cf11afSPaul Mackerras
79414cf11afSPaul Mackerras/*
79514cf11afSPaul Mackerras * Here we have detected that the kernel stack pointer is bad.
79614cf11afSPaul Mackerras * R9 contains the saved CR, r13 points to the paca,
79714cf11afSPaul Mackerras * r10 contains the (bad) kernel stack pointer,
79814cf11afSPaul Mackerras * r11 and r12 contain the saved SRR0 and SRR1.
79914cf11afSPaul Mackerras * We switch to using an emergency stack, save the registers there,
80014cf11afSPaul Mackerras * and call kernel_bad_stack(), which panics.
80114cf11afSPaul Mackerras */
80214cf11afSPaul Mackerrasbad_stack:
80314cf11afSPaul Mackerras	ld	r1,PACAEMERGSP(r13)
80414cf11afSPaul Mackerras	subi	r1,r1,64+INT_FRAME_SIZE
80514cf11afSPaul Mackerras	std	r9,_CCR(r1)
80614cf11afSPaul Mackerras	std	r10,GPR1(r1)
80714cf11afSPaul Mackerras	std	r11,_NIP(r1)
80814cf11afSPaul Mackerras	std	r12,_MSR(r1)
809b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_DAR
810b5bbeb23SPaul Mackerras	mfspr	r12,SPRN_DSISR
81114cf11afSPaul Mackerras	std	r11,_DAR(r1)
81214cf11afSPaul Mackerras	std	r12,_DSISR(r1)
81314cf11afSPaul Mackerras	mflr	r10
81414cf11afSPaul Mackerras	mfctr	r11
81514cf11afSPaul Mackerras	mfxer	r12
81614cf11afSPaul Mackerras	std	r10,_LINK(r1)
81714cf11afSPaul Mackerras	std	r11,_CTR(r1)
81814cf11afSPaul Mackerras	std	r12,_XER(r1)
81914cf11afSPaul Mackerras	SAVE_GPR(0,r1)
82014cf11afSPaul Mackerras	SAVE_GPR(2,r1)
82114cf11afSPaul Mackerras	SAVE_4GPRS(3,r1)
82214cf11afSPaul Mackerras	SAVE_2GPRS(7,r1)
82314cf11afSPaul Mackerras	SAVE_10GPRS(12,r1)
82414cf11afSPaul Mackerras	SAVE_10GPRS(22,r1)
82514cf11afSPaul Mackerras	addi	r11,r1,INT_FRAME_SIZE
82614cf11afSPaul Mackerras	std	r11,0(r1)
82714cf11afSPaul Mackerras	li	r12,0
82814cf11afSPaul Mackerras	std	r12,0(r11)
82914cf11afSPaul Mackerras	ld	r2,PACATOC(r13)
83014cf11afSPaul Mackerras1:	addi	r3,r1,STACK_FRAME_OVERHEAD
83114cf11afSPaul Mackerras	bl	.kernel_bad_stack
83214cf11afSPaul Mackerras	b	1b
83314cf11afSPaul Mackerras
83414cf11afSPaul Mackerras/*
83514cf11afSPaul Mackerras * Return from an exception with minimal checks.
83614cf11afSPaul Mackerras * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
83714cf11afSPaul Mackerras * If interrupts have been enabled, or anything has been
83814cf11afSPaul Mackerras * done that might have changed the scheduling status of
83914cf11afSPaul Mackerras * any task or sent any task a signal, you should use
84014cf11afSPaul Mackerras * ret_from_except or ret_from_except_lite instead of this.
84114cf11afSPaul Mackerras */
84240ef8cbcSPaul Mackerras	.globl	fast_exception_return
84314cf11afSPaul Mackerrasfast_exception_return:
84414cf11afSPaul Mackerras	ld	r12,_MSR(r1)
84514cf11afSPaul Mackerras	ld	r11,_NIP(r1)
84614cf11afSPaul Mackerras	andi.	r3,r12,MSR_RI		/* check if RI is set */
84714cf11afSPaul Mackerras	beq-	unrecov_fer
848c6622f63SPaul Mackerras
849c6622f63SPaul Mackerras#ifdef CONFIG_VIRT_CPU_ACCOUNTING
850c6622f63SPaul Mackerras	andi.	r3,r12,MSR_PR
851c6622f63SPaul Mackerras	beq	2f
852c6622f63SPaul Mackerras	ACCOUNT_CPU_USER_EXIT(r3, r4)
853c6622f63SPaul Mackerras2:
854c6622f63SPaul Mackerras#endif
855c6622f63SPaul Mackerras
85614cf11afSPaul Mackerras	ld	r3,_CCR(r1)
85714cf11afSPaul Mackerras	ld	r4,_LINK(r1)
85814cf11afSPaul Mackerras	ld	r5,_CTR(r1)
85914cf11afSPaul Mackerras	ld	r6,_XER(r1)
86014cf11afSPaul Mackerras	mtcr	r3
86114cf11afSPaul Mackerras	mtlr	r4
86214cf11afSPaul Mackerras	mtctr	r5
86314cf11afSPaul Mackerras	mtxer	r6
86414cf11afSPaul Mackerras	REST_GPR(0, r1)
86514cf11afSPaul Mackerras	REST_8GPRS(2, r1)
86614cf11afSPaul Mackerras
86714cf11afSPaul Mackerras	mfmsr	r10
86814cf11afSPaul Mackerras	clrrdi	r10,r10,2		/* clear RI (LE is 0 already) */
86914cf11afSPaul Mackerras	mtmsrd	r10,1
87014cf11afSPaul Mackerras
871b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r12
872b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r11
87314cf11afSPaul Mackerras	REST_4GPRS(10, r1)
87414cf11afSPaul Mackerras	ld	r1,GPR1(r1)
87514cf11afSPaul Mackerras	rfid
87614cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
87714cf11afSPaul Mackerras
87814cf11afSPaul Mackerrasunrecov_fer:
87914cf11afSPaul Mackerras	bl	.save_nvgprs
88014cf11afSPaul Mackerras1:	addi	r3,r1,STACK_FRAME_OVERHEAD
88114cf11afSPaul Mackerras	bl	.unrecoverable_exception
88214cf11afSPaul Mackerras	b	1b
88314cf11afSPaul Mackerras
88414cf11afSPaul Mackerras/*
88514cf11afSPaul Mackerras * Here r13 points to the paca, r9 contains the saved CR,
88614cf11afSPaul Mackerras * SRR0 and SRR1 are saved in r11 and r12,
88714cf11afSPaul Mackerras * r9 - r13 are saved in paca->exgen.
88814cf11afSPaul Mackerras */
88914cf11afSPaul Mackerras	.align	7
89014cf11afSPaul Mackerras	.globl data_access_common
89114cf11afSPaul Mackerrasdata_access_common:
892b5bbeb23SPaul Mackerras	mfspr	r10,SPRN_DAR
89314cf11afSPaul Mackerras	std	r10,PACA_EXGEN+EX_DAR(r13)
894b5bbeb23SPaul Mackerras	mfspr	r10,SPRN_DSISR
89514cf11afSPaul Mackerras	stw	r10,PACA_EXGEN+EX_DSISR(r13)
89614cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
89714cf11afSPaul Mackerras	ld	r3,PACA_EXGEN+EX_DAR(r13)
89814cf11afSPaul Mackerras	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
89914cf11afSPaul Mackerras	li	r5,0x300
90014cf11afSPaul Mackerras	b	.do_hash_page	 	/* Try to handle as hpte fault */
90114cf11afSPaul Mackerras
90214cf11afSPaul Mackerras	.align	7
90314cf11afSPaul Mackerras	.globl instruction_access_common
90414cf11afSPaul Mackerrasinstruction_access_common:
90514cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
90614cf11afSPaul Mackerras	ld	r3,_NIP(r1)
90714cf11afSPaul Mackerras	andis.	r4,r12,0x5820
90814cf11afSPaul Mackerras	li	r5,0x400
90914cf11afSPaul Mackerras	b	.do_hash_page		/* Try to handle as hpte fault */
91014cf11afSPaul Mackerras
9113c726f8dSBenjamin Herrenschmidt/*
9123c726f8dSBenjamin Herrenschmidt * Here is the common SLB miss user that is used when going to virtual
9133c726f8dSBenjamin Herrenschmidt * mode for SLB misses, that is currently not used
9143c726f8dSBenjamin Herrenschmidt */
9153c726f8dSBenjamin Herrenschmidt#ifdef __DISABLED__
9163c726f8dSBenjamin Herrenschmidt	.align	7
9173c726f8dSBenjamin Herrenschmidt	.globl	slb_miss_user_common
9183c726f8dSBenjamin Herrenschmidtslb_miss_user_common:
9193c726f8dSBenjamin Herrenschmidt	mflr	r10
9203c726f8dSBenjamin Herrenschmidt	std	r3,PACA_EXGEN+EX_DAR(r13)
9213c726f8dSBenjamin Herrenschmidt	stw	r9,PACA_EXGEN+EX_CCR(r13)
9223c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXGEN+EX_LR(r13)
9233c726f8dSBenjamin Herrenschmidt	std	r11,PACA_EXGEN+EX_SRR0(r13)
9243c726f8dSBenjamin Herrenschmidt	bl	.slb_allocate_user
9253c726f8dSBenjamin Herrenschmidt
9263c726f8dSBenjamin Herrenschmidt	ld	r10,PACA_EXGEN+EX_LR(r13)
9273c726f8dSBenjamin Herrenschmidt	ld	r3,PACA_EXGEN+EX_R3(r13)
9283c726f8dSBenjamin Herrenschmidt	lwz	r9,PACA_EXGEN+EX_CCR(r13)
9293c726f8dSBenjamin Herrenschmidt	ld	r11,PACA_EXGEN+EX_SRR0(r13)
9303c726f8dSBenjamin Herrenschmidt	mtlr	r10
9313c726f8dSBenjamin Herrenschmidt	beq-	slb_miss_fault
9323c726f8dSBenjamin Herrenschmidt
9333c726f8dSBenjamin Herrenschmidt	andi.	r10,r12,MSR_RI		/* check for unrecoverable exception */
9343c726f8dSBenjamin Herrenschmidt	beq-	unrecov_user_slb
9353c726f8dSBenjamin Herrenschmidt	mfmsr	r10
9363c726f8dSBenjamin Herrenschmidt
9373c726f8dSBenjamin Herrenschmidt.machine push
9383c726f8dSBenjamin Herrenschmidt.machine "power4"
9393c726f8dSBenjamin Herrenschmidt	mtcrf	0x80,r9
9403c726f8dSBenjamin Herrenschmidt.machine pop
9413c726f8dSBenjamin Herrenschmidt
9423c726f8dSBenjamin Herrenschmidt	clrrdi	r10,r10,2		/* clear RI before setting SRR0/1 */
9433c726f8dSBenjamin Herrenschmidt	mtmsrd	r10,1
9443c726f8dSBenjamin Herrenschmidt
9453c726f8dSBenjamin Herrenschmidt	mtspr	SRR0,r11
9463c726f8dSBenjamin Herrenschmidt	mtspr	SRR1,r12
9473c726f8dSBenjamin Herrenschmidt
9483c726f8dSBenjamin Herrenschmidt	ld	r9,PACA_EXGEN+EX_R9(r13)
9493c726f8dSBenjamin Herrenschmidt	ld	r10,PACA_EXGEN+EX_R10(r13)
9503c726f8dSBenjamin Herrenschmidt	ld	r11,PACA_EXGEN+EX_R11(r13)
9513c726f8dSBenjamin Herrenschmidt	ld	r12,PACA_EXGEN+EX_R12(r13)
9523c726f8dSBenjamin Herrenschmidt	ld	r13,PACA_EXGEN+EX_R13(r13)
9533c726f8dSBenjamin Herrenschmidt	rfid
9543c726f8dSBenjamin Herrenschmidt	b	.
9553c726f8dSBenjamin Herrenschmidt
9563c726f8dSBenjamin Herrenschmidtslb_miss_fault:
9573c726f8dSBenjamin Herrenschmidt	EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
9583c726f8dSBenjamin Herrenschmidt	ld	r4,PACA_EXGEN+EX_DAR(r13)
9593c726f8dSBenjamin Herrenschmidt	li	r5,0
9603c726f8dSBenjamin Herrenschmidt	std	r4,_DAR(r1)
9613c726f8dSBenjamin Herrenschmidt	std	r5,_DSISR(r1)
9623c726f8dSBenjamin Herrenschmidt	b	.handle_page_fault
9633c726f8dSBenjamin Herrenschmidt
9643c726f8dSBenjamin Herrenschmidtunrecov_user_slb:
9653c726f8dSBenjamin Herrenschmidt	EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
9663c726f8dSBenjamin Herrenschmidt	DISABLE_INTS
9673c726f8dSBenjamin Herrenschmidt	bl	.save_nvgprs
9683c726f8dSBenjamin Herrenschmidt1:	addi	r3,r1,STACK_FRAME_OVERHEAD
9693c726f8dSBenjamin Herrenschmidt	bl	.unrecoverable_exception
9703c726f8dSBenjamin Herrenschmidt	b	1b
9713c726f8dSBenjamin Herrenschmidt
9723c726f8dSBenjamin Herrenschmidt#endif /* __DISABLED__ */
9733c726f8dSBenjamin Herrenschmidt
9743c726f8dSBenjamin Herrenschmidt
9753c726f8dSBenjamin Herrenschmidt/*
9763c726f8dSBenjamin Herrenschmidt * r13 points to the PACA, r9 contains the saved CR,
9773c726f8dSBenjamin Herrenschmidt * r12 contain the saved SRR1, SRR0 is still ready for return
9783c726f8dSBenjamin Herrenschmidt * r3 has the faulting address
9793c726f8dSBenjamin Herrenschmidt * r9 - r13 are saved in paca->exslb.
9803c726f8dSBenjamin Herrenschmidt * r3 is saved in paca->slb_r3
9813c726f8dSBenjamin Herrenschmidt * We assume we aren't going to take any exceptions during this procedure.
9823c726f8dSBenjamin Herrenschmidt */
9833c726f8dSBenjamin Herrenschmidt_GLOBAL(slb_miss_realmode)
9843c726f8dSBenjamin Herrenschmidt	mflr	r10
9853c726f8dSBenjamin Herrenschmidt
9863c726f8dSBenjamin Herrenschmidt	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
9873c726f8dSBenjamin Herrenschmidt	std	r10,PACA_EXSLB+EX_LR(r13)	/* save LR */
9883c726f8dSBenjamin Herrenschmidt
9893c726f8dSBenjamin Herrenschmidt	bl	.slb_allocate_realmode
9903c726f8dSBenjamin Herrenschmidt
9913c726f8dSBenjamin Herrenschmidt	/* All done -- return from exception. */
9923c726f8dSBenjamin Herrenschmidt
9933c726f8dSBenjamin Herrenschmidt	ld	r10,PACA_EXSLB+EX_LR(r13)
9943c726f8dSBenjamin Herrenschmidt	ld	r3,PACA_EXSLB+EX_R3(r13)
9953c726f8dSBenjamin Herrenschmidt	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
9963c726f8dSBenjamin Herrenschmidt#ifdef CONFIG_PPC_ISERIES
9973356bb9fSDavid Gibson	ld	r11,PACALPPACAPTR(r13)
9983356bb9fSDavid Gibson	ld	r11,LPPACASRR0(r11)		/* get SRR0 value */
9993c726f8dSBenjamin Herrenschmidt#endif /* CONFIG_PPC_ISERIES */
10003c726f8dSBenjamin Herrenschmidt
10013c726f8dSBenjamin Herrenschmidt	mtlr	r10
10023c726f8dSBenjamin Herrenschmidt
10033c726f8dSBenjamin Herrenschmidt	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
10043c726f8dSBenjamin Herrenschmidt	beq-	unrecov_slb
10053c726f8dSBenjamin Herrenschmidt
10063c726f8dSBenjamin Herrenschmidt.machine	push
10073c726f8dSBenjamin Herrenschmidt.machine	"power4"
10083c726f8dSBenjamin Herrenschmidt	mtcrf	0x80,r9
10093c726f8dSBenjamin Herrenschmidt	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
10103c726f8dSBenjamin Herrenschmidt.machine	pop
10113c726f8dSBenjamin Herrenschmidt
10123c726f8dSBenjamin Herrenschmidt#ifdef CONFIG_PPC_ISERIES
10133c726f8dSBenjamin Herrenschmidt	mtspr	SPRN_SRR0,r11
10143c726f8dSBenjamin Herrenschmidt	mtspr	SPRN_SRR1,r12
10153c726f8dSBenjamin Herrenschmidt#endif /* CONFIG_PPC_ISERIES */
10163c726f8dSBenjamin Herrenschmidt	ld	r9,PACA_EXSLB+EX_R9(r13)
10173c726f8dSBenjamin Herrenschmidt	ld	r10,PACA_EXSLB+EX_R10(r13)
10183c726f8dSBenjamin Herrenschmidt	ld	r11,PACA_EXSLB+EX_R11(r13)
10193c726f8dSBenjamin Herrenschmidt	ld	r12,PACA_EXSLB+EX_R12(r13)
10203c726f8dSBenjamin Herrenschmidt	ld	r13,PACA_EXSLB+EX_R13(r13)
10213c726f8dSBenjamin Herrenschmidt	rfid
10223c726f8dSBenjamin Herrenschmidt	b	.	/* prevent speculative execution */
10233c726f8dSBenjamin Herrenschmidt
10243c726f8dSBenjamin Herrenschmidtunrecov_slb:
10253c726f8dSBenjamin Herrenschmidt	EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
10263c726f8dSBenjamin Herrenschmidt	DISABLE_INTS
10273c726f8dSBenjamin Herrenschmidt	bl	.save_nvgprs
10283c726f8dSBenjamin Herrenschmidt1:	addi	r3,r1,STACK_FRAME_OVERHEAD
10293c726f8dSBenjamin Herrenschmidt	bl	.unrecoverable_exception
10303c726f8dSBenjamin Herrenschmidt	b	1b
10313c726f8dSBenjamin Herrenschmidt
103214cf11afSPaul Mackerras	.align	7
103314cf11afSPaul Mackerras	.globl hardware_interrupt_common
103414cf11afSPaul Mackerras	.globl hardware_interrupt_entry
103514cf11afSPaul Mackerrashardware_interrupt_common:
103614cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
103714cf11afSPaul Mackerrashardware_interrupt_entry:
103814cf11afSPaul Mackerras	DISABLE_INTS
1039cb2c9b27SAnton Blanchard	bl	.ppc64_runlatch_on
104014cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
104114cf11afSPaul Mackerras	bl	.do_IRQ
104214cf11afSPaul Mackerras	b	.ret_from_except_lite
104314cf11afSPaul Mackerras
104414cf11afSPaul Mackerras	.align	7
104514cf11afSPaul Mackerras	.globl alignment_common
104614cf11afSPaul Mackerrasalignment_common:
1047b5bbeb23SPaul Mackerras	mfspr	r10,SPRN_DAR
104814cf11afSPaul Mackerras	std	r10,PACA_EXGEN+EX_DAR(r13)
1049b5bbeb23SPaul Mackerras	mfspr	r10,SPRN_DSISR
105014cf11afSPaul Mackerras	stw	r10,PACA_EXGEN+EX_DSISR(r13)
105114cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
105214cf11afSPaul Mackerras	ld	r3,PACA_EXGEN+EX_DAR(r13)
105314cf11afSPaul Mackerras	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
105414cf11afSPaul Mackerras	std	r3,_DAR(r1)
105514cf11afSPaul Mackerras	std	r4,_DSISR(r1)
105614cf11afSPaul Mackerras	bl	.save_nvgprs
105714cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
105814cf11afSPaul Mackerras	ENABLE_INTS
105914cf11afSPaul Mackerras	bl	.alignment_exception
106014cf11afSPaul Mackerras	b	.ret_from_except
106114cf11afSPaul Mackerras
106214cf11afSPaul Mackerras	.align	7
106314cf11afSPaul Mackerras	.globl program_check_common
106414cf11afSPaul Mackerrasprogram_check_common:
106514cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
106614cf11afSPaul Mackerras	bl	.save_nvgprs
106714cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
106814cf11afSPaul Mackerras	ENABLE_INTS
106914cf11afSPaul Mackerras	bl	.program_check_exception
107014cf11afSPaul Mackerras	b	.ret_from_except
107114cf11afSPaul Mackerras
107214cf11afSPaul Mackerras	.align	7
107314cf11afSPaul Mackerras	.globl fp_unavailable_common
107414cf11afSPaul Mackerrasfp_unavailable_common:
107514cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
107614cf11afSPaul Mackerras	bne	.load_up_fpu		/* if from user, just load it up */
107714cf11afSPaul Mackerras	bl	.save_nvgprs
107814cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
107914cf11afSPaul Mackerras	ENABLE_INTS
108014cf11afSPaul Mackerras	bl	.kernel_fp_unavailable_exception
108114cf11afSPaul Mackerras	BUG_OPCODE
108214cf11afSPaul Mackerras
108314cf11afSPaul Mackerras	.align	7
108414cf11afSPaul Mackerras	.globl altivec_unavailable_common
108514cf11afSPaul Mackerrasaltivec_unavailable_common:
108614cf11afSPaul Mackerras	EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
108714cf11afSPaul Mackerras#ifdef CONFIG_ALTIVEC
108814cf11afSPaul MackerrasBEGIN_FTR_SECTION
108914cf11afSPaul Mackerras	bne	.load_up_altivec	/* if from user, just load it up */
109014cf11afSPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
109114cf11afSPaul Mackerras#endif
109214cf11afSPaul Mackerras	bl	.save_nvgprs
109314cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
109414cf11afSPaul Mackerras	ENABLE_INTS
109514cf11afSPaul Mackerras	bl	.altivec_unavailable_exception
109614cf11afSPaul Mackerras	b	.ret_from_except
109714cf11afSPaul Mackerras
109814cf11afSPaul Mackerras#ifdef CONFIG_ALTIVEC
109914cf11afSPaul Mackerras/*
110014cf11afSPaul Mackerras * load_up_altivec(unused, unused, tsk)
110114cf11afSPaul Mackerras * Disable VMX for the task which had it previously,
110214cf11afSPaul Mackerras * and save its vector registers in its thread_struct.
110314cf11afSPaul Mackerras * Enables the VMX for use in the kernel on return.
110414cf11afSPaul Mackerras * On SMP we know the VMX is free, since we give it up every
110514cf11afSPaul Mackerras * switch (ie, no lazy save of the vector registers).
110614cf11afSPaul Mackerras * On entry: r13 == 'current' && last_task_used_altivec != 'current'
110714cf11afSPaul Mackerras */
110814cf11afSPaul Mackerras_STATIC(load_up_altivec)
110914cf11afSPaul Mackerras	mfmsr	r5			/* grab the current MSR */
111014cf11afSPaul Mackerras	oris	r5,r5,MSR_VEC@h
111114cf11afSPaul Mackerras	mtmsrd	r5			/* enable use of VMX now */
111214cf11afSPaul Mackerras	isync
111314cf11afSPaul Mackerras
111414cf11afSPaul Mackerras/*
111514cf11afSPaul Mackerras * For SMP, we don't do lazy VMX switching because it just gets too
111614cf11afSPaul Mackerras * horrendously complex, especially when a task switches from one CPU
111714cf11afSPaul Mackerras * to another.  Instead we call giveup_altvec in switch_to.
111814cf11afSPaul Mackerras * VRSAVE isn't dealt with here, that is done in the normal context
111914cf11afSPaul Mackerras * switch code. Note that we could rely on vrsave value to eventually
112014cf11afSPaul Mackerras * avoid saving all of the VREGs here...
112114cf11afSPaul Mackerras */
112214cf11afSPaul Mackerras#ifndef CONFIG_SMP
112314cf11afSPaul Mackerras	ld	r3,last_task_used_altivec@got(r2)
112414cf11afSPaul Mackerras	ld	r4,0(r3)
112514cf11afSPaul Mackerras	cmpdi	0,r4,0
112614cf11afSPaul Mackerras	beq	1f
112714cf11afSPaul Mackerras	/* Save VMX state to last_task_used_altivec's THREAD struct */
112814cf11afSPaul Mackerras	addi	r4,r4,THREAD
112914cf11afSPaul Mackerras	SAVE_32VRS(0,r5,r4)
113014cf11afSPaul Mackerras	mfvscr	vr0
113114cf11afSPaul Mackerras	li	r10,THREAD_VSCR
113214cf11afSPaul Mackerras	stvx	vr0,r10,r4
113314cf11afSPaul Mackerras	/* Disable VMX for last_task_used_altivec */
113414cf11afSPaul Mackerras	ld	r5,PT_REGS(r4)
113514cf11afSPaul Mackerras	ld	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
113614cf11afSPaul Mackerras	lis	r6,MSR_VEC@h
113714cf11afSPaul Mackerras	andc	r4,r4,r6
113814cf11afSPaul Mackerras	std	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
113914cf11afSPaul Mackerras1:
114014cf11afSPaul Mackerras#endif /* CONFIG_SMP */
114114cf11afSPaul Mackerras	/* Hack: if we get an altivec unavailable trap with VRSAVE
114214cf11afSPaul Mackerras	 * set to all zeros, we assume this is a broken application
114314cf11afSPaul Mackerras	 * that fails to set it properly, and thus we switch it to
114414cf11afSPaul Mackerras	 * all 1's
114514cf11afSPaul Mackerras	 */
114614cf11afSPaul Mackerras	mfspr	r4,SPRN_VRSAVE
114714cf11afSPaul Mackerras	cmpdi	0,r4,0
114814cf11afSPaul Mackerras	bne+	1f
114914cf11afSPaul Mackerras	li	r4,-1
115014cf11afSPaul Mackerras	mtspr	SPRN_VRSAVE,r4
115114cf11afSPaul Mackerras1:
115214cf11afSPaul Mackerras	/* enable use of VMX after return */
115314cf11afSPaul Mackerras	ld	r4,PACACURRENT(r13)
115414cf11afSPaul Mackerras	addi	r5,r4,THREAD		/* Get THREAD */
115514cf11afSPaul Mackerras	oris	r12,r12,MSR_VEC@h
115614cf11afSPaul Mackerras	std	r12,_MSR(r1)
115714cf11afSPaul Mackerras	li	r4,1
115814cf11afSPaul Mackerras	li	r10,THREAD_VSCR
115914cf11afSPaul Mackerras	stw	r4,THREAD_USED_VR(r5)
116014cf11afSPaul Mackerras	lvx	vr0,r10,r5
116114cf11afSPaul Mackerras	mtvscr	vr0
116214cf11afSPaul Mackerras	REST_32VRS(0,r4,r5)
116314cf11afSPaul Mackerras#ifndef CONFIG_SMP
116414cf11afSPaul Mackerras	/* Update last_task_used_math to 'current' */
116514cf11afSPaul Mackerras	subi	r4,r5,THREAD		/* Back to 'current' */
116614cf11afSPaul Mackerras	std	r4,0(r3)
116714cf11afSPaul Mackerras#endif /* CONFIG_SMP */
116814cf11afSPaul Mackerras	/* restore registers and return */
116914cf11afSPaul Mackerras	b	fast_exception_return
117014cf11afSPaul Mackerras#endif /* CONFIG_ALTIVEC */
117114cf11afSPaul Mackerras
117214cf11afSPaul Mackerras/*
117314cf11afSPaul Mackerras * Hash table stuff
117414cf11afSPaul Mackerras */
117514cf11afSPaul Mackerras	.align	7
117614cf11afSPaul Mackerras_GLOBAL(do_hash_page)
117714cf11afSPaul Mackerras	std	r3,_DAR(r1)
117814cf11afSPaul Mackerras	std	r4,_DSISR(r1)
117914cf11afSPaul Mackerras
118014cf11afSPaul Mackerras	andis.	r0,r4,0xa450		/* weird error? */
118114cf11afSPaul Mackerras	bne-	.handle_page_fault	/* if not, try to insert a HPTE */
118214cf11afSPaul MackerrasBEGIN_FTR_SECTION
118314cf11afSPaul Mackerras	andis.	r0,r4,0x0020		/* Is it a segment table fault? */
118414cf11afSPaul Mackerras	bne-	.do_ste_alloc		/* If so handle it */
118514cf11afSPaul MackerrasEND_FTR_SECTION_IFCLR(CPU_FTR_SLB)
118614cf11afSPaul Mackerras
118714cf11afSPaul Mackerras	/*
118814cf11afSPaul Mackerras	 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
118914cf11afSPaul Mackerras	 * accessing a userspace segment (even from the kernel). We assume
119014cf11afSPaul Mackerras	 * kernel addresses always have the high bit set.
119114cf11afSPaul Mackerras	 */
119214cf11afSPaul Mackerras	rlwinm	r4,r4,32-25+9,31-9,31-9	/* DSISR_STORE -> _PAGE_RW */
119314cf11afSPaul Mackerras	rotldi	r0,r3,15		/* Move high bit into MSR_PR posn */
119414cf11afSPaul Mackerras	orc	r0,r12,r0		/* MSR_PR | ~high_bit */
119514cf11afSPaul Mackerras	rlwimi	r4,r0,32-13,30,30	/* becomes _PAGE_USER access bit */
119614cf11afSPaul Mackerras	ori	r4,r4,1			/* add _PAGE_PRESENT */
119714cf11afSPaul Mackerras	rlwimi	r4,r5,22+2,31-2,31-2	/* Set _PAGE_EXEC if trap is 0x400 */
119814cf11afSPaul Mackerras
119914cf11afSPaul Mackerras	/*
120014cf11afSPaul Mackerras	 * On iSeries, we soft-disable interrupts here, then
120114cf11afSPaul Mackerras	 * hard-enable interrupts so that the hash_page code can spin on
120214cf11afSPaul Mackerras	 * the hash_table_lock without problems on a shared processor.
120314cf11afSPaul Mackerras	 */
120414cf11afSPaul Mackerras	DISABLE_INTS
120514cf11afSPaul Mackerras
120614cf11afSPaul Mackerras	/*
120714cf11afSPaul Mackerras	 * r3 contains the faulting address
120814cf11afSPaul Mackerras	 * r4 contains the required access permissions
120914cf11afSPaul Mackerras	 * r5 contains the trap number
121014cf11afSPaul Mackerras	 *
121114cf11afSPaul Mackerras	 * at return r3 = 0 for success
121214cf11afSPaul Mackerras	 */
121314cf11afSPaul Mackerras	bl	.hash_page		/* build HPTE if possible */
121414cf11afSPaul Mackerras	cmpdi	r3,0			/* see if hash_page succeeded */
121514cf11afSPaul Mackerras
121614cf11afSPaul Mackerras#ifdef DO_SOFT_DISABLE
121714cf11afSPaul Mackerras	/*
121814cf11afSPaul Mackerras	 * If we had interrupts soft-enabled at the point where the
121914cf11afSPaul Mackerras	 * DSI/ISI occurred, and an interrupt came in during hash_page,
122014cf11afSPaul Mackerras	 * handle it now.
122114cf11afSPaul Mackerras	 * We jump to ret_from_except_lite rather than fast_exception_return
122214cf11afSPaul Mackerras	 * because ret_from_except_lite will check for and handle pending
122314cf11afSPaul Mackerras	 * interrupts if necessary.
122414cf11afSPaul Mackerras	 */
122514cf11afSPaul Mackerras	beq	.ret_from_except_lite
122614cf11afSPaul Mackerras	/* For a hash failure, we don't bother re-enabling interrupts */
122714cf11afSPaul Mackerras	ble-	12f
122814cf11afSPaul Mackerras
122914cf11afSPaul Mackerras	/*
123014cf11afSPaul Mackerras	 * hash_page couldn't handle it, set soft interrupt enable back
123114cf11afSPaul Mackerras	 * to what it was before the trap.  Note that .local_irq_restore
123214cf11afSPaul Mackerras	 * handles any interrupts pending at this point.
123314cf11afSPaul Mackerras	 */
123414cf11afSPaul Mackerras	ld	r3,SOFTE(r1)
123514cf11afSPaul Mackerras	bl	.local_irq_restore
123614cf11afSPaul Mackerras	b	11f
123714cf11afSPaul Mackerras#else
123814cf11afSPaul Mackerras	beq	fast_exception_return   /* Return from exception on success */
123914cf11afSPaul Mackerras	ble-	12f			/* Failure return from hash_page */
124014cf11afSPaul Mackerras
124114cf11afSPaul Mackerras	/* fall through */
124214cf11afSPaul Mackerras#endif
124314cf11afSPaul Mackerras
124414cf11afSPaul Mackerras/* Here we have a page fault that hash_page can't handle. */
124514cf11afSPaul Mackerras_GLOBAL(handle_page_fault)
124614cf11afSPaul Mackerras	ENABLE_INTS
124714cf11afSPaul Mackerras11:	ld	r4,_DAR(r1)
124814cf11afSPaul Mackerras	ld	r5,_DSISR(r1)
124914cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
125014cf11afSPaul Mackerras	bl	.do_page_fault
125114cf11afSPaul Mackerras	cmpdi	r3,0
125214cf11afSPaul Mackerras	beq+	.ret_from_except_lite
125314cf11afSPaul Mackerras	bl	.save_nvgprs
125414cf11afSPaul Mackerras	mr	r5,r3
125514cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
125614cf11afSPaul Mackerras	lwz	r4,_DAR(r1)
125714cf11afSPaul Mackerras	bl	.bad_page_fault
125814cf11afSPaul Mackerras	b	.ret_from_except
125914cf11afSPaul Mackerras
126014cf11afSPaul Mackerras/* We have a page fault that hash_page could handle but HV refused
126114cf11afSPaul Mackerras * the PTE insertion
126214cf11afSPaul Mackerras */
126314cf11afSPaul Mackerras12:	bl	.save_nvgprs
126414cf11afSPaul Mackerras	addi	r3,r1,STACK_FRAME_OVERHEAD
126514cf11afSPaul Mackerras	lwz	r4,_DAR(r1)
126614cf11afSPaul Mackerras	bl	.low_hash_fault
126714cf11afSPaul Mackerras	b	.ret_from_except
126814cf11afSPaul Mackerras
126914cf11afSPaul Mackerras	/* here we have a segment miss */
127014cf11afSPaul Mackerras_GLOBAL(do_ste_alloc)
127114cf11afSPaul Mackerras	bl	.ste_allocate		/* try to insert stab entry */
127214cf11afSPaul Mackerras	cmpdi	r3,0
127314cf11afSPaul Mackerras	beq+	fast_exception_return
127414cf11afSPaul Mackerras	b	.handle_page_fault
127514cf11afSPaul Mackerras
127614cf11afSPaul Mackerras/*
127714cf11afSPaul Mackerras * r13 points to the PACA, r9 contains the saved CR,
127814cf11afSPaul Mackerras * r11 and r12 contain the saved SRR0 and SRR1.
127914cf11afSPaul Mackerras * r9 - r13 are saved in paca->exslb.
128014cf11afSPaul Mackerras * We assume we aren't going to take any exceptions during this procedure.
128114cf11afSPaul Mackerras * We assume (DAR >> 60) == 0xc.
128214cf11afSPaul Mackerras */
128314cf11afSPaul Mackerras	.align	7
128414cf11afSPaul Mackerras_GLOBAL(do_stab_bolted)
128514cf11afSPaul Mackerras	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
128614cf11afSPaul Mackerras	std	r11,PACA_EXSLB+EX_SRR0(r13)	/* save SRR0 in exc. frame */
128714cf11afSPaul Mackerras
128814cf11afSPaul Mackerras	/* Hash to the primary group */
128914cf11afSPaul Mackerras	ld	r10,PACASTABVIRT(r13)
1290b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_DAR
129114cf11afSPaul Mackerras	srdi	r11,r11,28
129214cf11afSPaul Mackerras	rldimi	r10,r11,7,52	/* r10 = first ste of the group */
129314cf11afSPaul Mackerras
129414cf11afSPaul Mackerras	/* Calculate VSID */
129514cf11afSPaul Mackerras	/* This is a kernel address, so protovsid = ESID */
129614cf11afSPaul Mackerras	ASM_VSID_SCRAMBLE(r11, r9)
129714cf11afSPaul Mackerras	rldic	r9,r11,12,16	/* r9 = vsid << 12 */
129814cf11afSPaul Mackerras
129914cf11afSPaul Mackerras	/* Search the primary group for a free entry */
130014cf11afSPaul Mackerras1:	ld	r11,0(r10)	/* Test valid bit of the current ste	*/
130114cf11afSPaul Mackerras	andi.	r11,r11,0x80
130214cf11afSPaul Mackerras	beq	2f
130314cf11afSPaul Mackerras	addi	r10,r10,16
130414cf11afSPaul Mackerras	andi.	r11,r10,0x70
130514cf11afSPaul Mackerras	bne	1b
130614cf11afSPaul Mackerras
130714cf11afSPaul Mackerras	/* Stick for only searching the primary group for now.		*/
130814cf11afSPaul Mackerras	/* At least for now, we use a very simple random castout scheme */
130914cf11afSPaul Mackerras	/* Use the TB as a random number ;  OR in 1 to avoid entry 0	*/
131014cf11afSPaul Mackerras	mftb	r11
131114cf11afSPaul Mackerras	rldic	r11,r11,4,57	/* r11 = (r11 << 4) & 0x70 */
131214cf11afSPaul Mackerras	ori	r11,r11,0x10
131314cf11afSPaul Mackerras
131414cf11afSPaul Mackerras	/* r10 currently points to an ste one past the group of interest */
131514cf11afSPaul Mackerras	/* make it point to the randomly selected entry			*/
131614cf11afSPaul Mackerras	subi	r10,r10,128
131714cf11afSPaul Mackerras	or 	r10,r10,r11	/* r10 is the entry to invalidate	*/
131814cf11afSPaul Mackerras
131914cf11afSPaul Mackerras	isync			/* mark the entry invalid		*/
132014cf11afSPaul Mackerras	ld	r11,0(r10)
132114cf11afSPaul Mackerras	rldicl	r11,r11,56,1	/* clear the valid bit */
132214cf11afSPaul Mackerras	rotldi	r11,r11,8
132314cf11afSPaul Mackerras	std	r11,0(r10)
132414cf11afSPaul Mackerras	sync
132514cf11afSPaul Mackerras
132614cf11afSPaul Mackerras	clrrdi	r11,r11,28	/* Get the esid part of the ste		*/
132714cf11afSPaul Mackerras	slbie	r11
132814cf11afSPaul Mackerras
132914cf11afSPaul Mackerras2:	std	r9,8(r10)	/* Store the vsid part of the ste	*/
133014cf11afSPaul Mackerras	eieio
133114cf11afSPaul Mackerras
1332b5bbeb23SPaul Mackerras	mfspr	r11,SPRN_DAR		/* Get the new esid			*/
133314cf11afSPaul Mackerras	clrrdi	r11,r11,28	/* Permits a full 32b of ESID		*/
133414cf11afSPaul Mackerras	ori	r11,r11,0x90	/* Turn on valid and kp			*/
133514cf11afSPaul Mackerras	std	r11,0(r10)	/* Put new entry back into the stab	*/
133614cf11afSPaul Mackerras
133714cf11afSPaul Mackerras	sync
133814cf11afSPaul Mackerras
133914cf11afSPaul Mackerras	/* All done -- return from exception. */
134014cf11afSPaul Mackerras	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
134114cf11afSPaul Mackerras	ld	r11,PACA_EXSLB+EX_SRR0(r13)	/* get saved SRR0 */
134214cf11afSPaul Mackerras
134314cf11afSPaul Mackerras	andi.	r10,r12,MSR_RI
134414cf11afSPaul Mackerras	beq-	unrecov_slb
134514cf11afSPaul Mackerras
134614cf11afSPaul Mackerras	mtcrf	0x80,r9			/* restore CR */
134714cf11afSPaul Mackerras
134814cf11afSPaul Mackerras	mfmsr	r10
134914cf11afSPaul Mackerras	clrrdi	r10,r10,2
135014cf11afSPaul Mackerras	mtmsrd	r10,1
135114cf11afSPaul Mackerras
1352b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r11
1353b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r12
135414cf11afSPaul Mackerras	ld	r9,PACA_EXSLB+EX_R9(r13)
135514cf11afSPaul Mackerras	ld	r10,PACA_EXSLB+EX_R10(r13)
135614cf11afSPaul Mackerras	ld	r11,PACA_EXSLB+EX_R11(r13)
135714cf11afSPaul Mackerras	ld	r12,PACA_EXSLB+EX_R12(r13)
135814cf11afSPaul Mackerras	ld	r13,PACA_EXSLB+EX_R13(r13)
135914cf11afSPaul Mackerras	rfid
136014cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
136114cf11afSPaul Mackerras
136214cf11afSPaul Mackerras/*
136314cf11afSPaul Mackerras * Space for CPU0's segment table.
136414cf11afSPaul Mackerras *
136514cf11afSPaul Mackerras * On iSeries, the hypervisor must fill in at least one entry before
136614cf11afSPaul Mackerras * we get control (with relocate on).  The address is give to the hv
1367ee400b63SStephen Rothwell * as a page number (see xLparMap in lpardata.c), so this must be at a
136814cf11afSPaul Mackerras * fixed address (the linker can't compute (u64)&initial_stab >>
136914cf11afSPaul Mackerras * PAGE_SHIFT).
137014cf11afSPaul Mackerras */
1371758438a7SMichael Ellerman	. = STAB0_OFFSET	/* 0x6000 */
137214cf11afSPaul Mackerras	.globl initial_stab
137314cf11afSPaul Mackerrasinitial_stab:
137414cf11afSPaul Mackerras	.space	4096
137514cf11afSPaul Mackerras
137614cf11afSPaul Mackerras/*
137714cf11afSPaul Mackerras * Data area reserved for FWNMI option.
137814cf11afSPaul Mackerras * This address (0x7000) is fixed by the RPA.
137914cf11afSPaul Mackerras */
138014cf11afSPaul Mackerras	.= 0x7000
138114cf11afSPaul Mackerras	.globl fwnmi_data_area
138214cf11afSPaul Mackerrasfwnmi_data_area:
138314cf11afSPaul Mackerras
138414cf11afSPaul Mackerras	/* iSeries does not use the FWNMI stuff, so it is safe to put
138514cf11afSPaul Mackerras	 * this here, even if we later allow kernels that will boot on
138614cf11afSPaul Mackerras	 * both pSeries and iSeries */
138714cf11afSPaul Mackerras#ifdef CONFIG_PPC_ISERIES
138814cf11afSPaul Mackerras        . = LPARMAP_PHYS
138914cf11afSPaul Mackerras#include "lparmap.s"
139014cf11afSPaul Mackerras/*
139114cf11afSPaul Mackerras * This ".text" is here for old compilers that generate a trailing
139214cf11afSPaul Mackerras * .note section when compiling .c files to .s
139314cf11afSPaul Mackerras */
139414cf11afSPaul Mackerras	.text
139514cf11afSPaul Mackerras#endif /* CONFIG_PPC_ISERIES */
139614cf11afSPaul Mackerras
139714cf11afSPaul Mackerras        . = 0x8000
139814cf11afSPaul Mackerras
139914cf11afSPaul Mackerras/*
140014cf11afSPaul Mackerras * On pSeries, secondary processors spin in the following code.
140114cf11afSPaul Mackerras * At entry, r3 = this processor's number (physical cpu id)
140214cf11afSPaul Mackerras */
140314cf11afSPaul Mackerras_GLOBAL(pSeries_secondary_smp_init)
140414cf11afSPaul Mackerras	mr	r24,r3
140514cf11afSPaul Mackerras
140614cf11afSPaul Mackerras	/* turn on 64-bit mode */
140714cf11afSPaul Mackerras	bl	.enable_64b_mode
140814cf11afSPaul Mackerras	isync
140914cf11afSPaul Mackerras
141014cf11afSPaul Mackerras	/* Copy some CPU settings from CPU 0 */
141114cf11afSPaul Mackerras	bl	.__restore_cpu_setup
141214cf11afSPaul Mackerras
141314cf11afSPaul Mackerras	/* Set up a paca value for this processor. Since we have the
141414cf11afSPaul Mackerras	 * physical cpu id in r24, we need to search the pacas to find
141514cf11afSPaul Mackerras	 * which logical id maps to our physical one.
141614cf11afSPaul Mackerras	 */
1417e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r13, paca)	/* Get base vaddr of paca array	 */
141814cf11afSPaul Mackerras	li	r5,0			/* logical cpu id                */
141914cf11afSPaul Mackerras1:	lhz	r6,PACAHWCPUID(r13)	/* Load HW procid from paca      */
142014cf11afSPaul Mackerras	cmpw	r6,r24			/* Compare to our id             */
142114cf11afSPaul Mackerras	beq	2f
142214cf11afSPaul Mackerras	addi	r13,r13,PACA_SIZE	/* Loop to next PACA on miss     */
142314cf11afSPaul Mackerras	addi	r5,r5,1
142414cf11afSPaul Mackerras	cmpwi	r5,NR_CPUS
142514cf11afSPaul Mackerras	blt	1b
142614cf11afSPaul Mackerras
142714cf11afSPaul Mackerras	mr	r3,r24			/* not found, copy phys to r3	 */
142814cf11afSPaul Mackerras	b	.kexec_wait		/* next kernel might do better	 */
142914cf11afSPaul Mackerras
1430b5bbeb23SPaul Mackerras2:	mtspr	SPRN_SPRG3,r13		/* Save vaddr of paca in SPRG3	 */
143114cf11afSPaul Mackerras	/* From now on, r24 is expected to be logical cpuid */
143214cf11afSPaul Mackerras	mr	r24,r5
143314cf11afSPaul Mackerras3:	HMT_LOW
143414cf11afSPaul Mackerras	lbz	r23,PACAPROCSTART(r13)	/* Test if this processor should */
143514cf11afSPaul Mackerras					/* start.			 */
143614cf11afSPaul Mackerras	sync
143714cf11afSPaul Mackerras
143814cf11afSPaul Mackerras	/* Create a temp kernel stack for use before relocation is on.	*/
143914cf11afSPaul Mackerras	ld	r1,PACAEMERGSP(r13)
144014cf11afSPaul Mackerras	subi	r1,r1,STACK_FRAME_OVERHEAD
144114cf11afSPaul Mackerras
144214cf11afSPaul Mackerras	cmpwi	0,r23,0
144314cf11afSPaul Mackerras#ifdef CONFIG_SMP
144414cf11afSPaul Mackerras	bne	.__secondary_start
144514cf11afSPaul Mackerras#endif
144614cf11afSPaul Mackerras	b 	3b			/* Loop until told to go	 */
144714cf11afSPaul Mackerras
144814cf11afSPaul Mackerras#ifdef CONFIG_PPC_ISERIES
144914cf11afSPaul Mackerras_STATIC(__start_initialization_iSeries)
145014cf11afSPaul Mackerras	/* Clear out the BSS */
1451e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r11,__bss_stop)
1452e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r8,__bss_start)
145314cf11afSPaul Mackerras	sub	r11,r11,r8		/* bss size			*/
145414cf11afSPaul Mackerras	addi	r11,r11,7		/* round up to an even double word */
145514cf11afSPaul Mackerras	rldicl. r11,r11,61,3		/* shift right by 3		*/
145614cf11afSPaul Mackerras	beq	4f
145714cf11afSPaul Mackerras	addi	r8,r8,-8
145814cf11afSPaul Mackerras	li	r0,0
145914cf11afSPaul Mackerras	mtctr	r11			/* zero this many doublewords	*/
146014cf11afSPaul Mackerras3:	stdu	r0,8(r8)
146114cf11afSPaul Mackerras	bdnz	3b
146214cf11afSPaul Mackerras4:
1463e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r1,init_thread_union)
146414cf11afSPaul Mackerras	addi	r1,r1,THREAD_SIZE
146514cf11afSPaul Mackerras	li	r0,0
146614cf11afSPaul Mackerras	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
146714cf11afSPaul Mackerras
1468e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3,cpu_specs)
1469e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
147014cf11afSPaul Mackerras	li	r5,0
147114cf11afSPaul Mackerras	bl	.identify_cpu
147214cf11afSPaul Mackerras
1473e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r2,__toc_start)
147414cf11afSPaul Mackerras	addi	r2,r2,0x4000
147514cf11afSPaul Mackerras	addi	r2,r2,0x4000
147614cf11afSPaul Mackerras
147714cf11afSPaul Mackerras	bl	.iSeries_early_setup
1478ee400b63SStephen Rothwell	bl	.early_setup
147914cf11afSPaul Mackerras
148014cf11afSPaul Mackerras	/* relocation is on at this point */
148114cf11afSPaul Mackerras
148214cf11afSPaul Mackerras	b	.start_here_common
148314cf11afSPaul Mackerras#endif /* CONFIG_PPC_ISERIES */
148414cf11afSPaul Mackerras
148514cf11afSPaul Mackerras#ifdef CONFIG_PPC_MULTIPLATFORM
148614cf11afSPaul Mackerras
148714cf11afSPaul Mackerras_STATIC(__mmu_off)
148814cf11afSPaul Mackerras	mfmsr	r3
148914cf11afSPaul Mackerras	andi.	r0,r3,MSR_IR|MSR_DR
149014cf11afSPaul Mackerras	beqlr
149114cf11afSPaul Mackerras	andc	r3,r3,r0
149214cf11afSPaul Mackerras	mtspr	SPRN_SRR0,r4
149314cf11afSPaul Mackerras	mtspr	SPRN_SRR1,r3
149414cf11afSPaul Mackerras	sync
149514cf11afSPaul Mackerras	rfid
149614cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
149714cf11afSPaul Mackerras
149814cf11afSPaul Mackerras
149914cf11afSPaul Mackerras/*
150014cf11afSPaul Mackerras * Here is our main kernel entry point. We support currently 2 kind of entries
150114cf11afSPaul Mackerras * depending on the value of r5.
150214cf11afSPaul Mackerras *
150314cf11afSPaul Mackerras *   r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
150414cf11afSPaul Mackerras *                 in r3...r7
150514cf11afSPaul Mackerras *
150614cf11afSPaul Mackerras *   r5 == NULL -> kexec style entry. r3 is a physical pointer to the
150714cf11afSPaul Mackerras *                 DT block, r4 is a physical pointer to the kernel itself
150814cf11afSPaul Mackerras *
150914cf11afSPaul Mackerras */
151014cf11afSPaul Mackerras_GLOBAL(__start_initialization_multiplatform)
1511be42d5faSPaul Mackerras#ifdef CONFIG_PPC_MULTIPLATFORM
151214cf11afSPaul Mackerras	/*
151314cf11afSPaul Mackerras	 * Are we booted from a PROM Of-type client-interface ?
151414cf11afSPaul Mackerras	 */
151514cf11afSPaul Mackerras	cmpldi	cr0,r5,0
151614cf11afSPaul Mackerras	bne	.__boot_from_prom		/* yes -> prom */
1517be42d5faSPaul Mackerras#endif
151814cf11afSPaul Mackerras
151914cf11afSPaul Mackerras	/* Save parameters */
152014cf11afSPaul Mackerras	mr	r31,r3
152114cf11afSPaul Mackerras	mr	r30,r4
152214cf11afSPaul Mackerras
152314cf11afSPaul Mackerras	/* Make sure we are running in 64 bits mode */
152414cf11afSPaul Mackerras	bl	.enable_64b_mode
152514cf11afSPaul Mackerras
152614cf11afSPaul Mackerras	/* Setup some critical 970 SPRs before switching MMU off */
152714cf11afSPaul Mackerras	bl	.__970_cpu_preinit
152814cf11afSPaul Mackerras
152914cf11afSPaul Mackerras	/* cpu # */
153014cf11afSPaul Mackerras	li	r24,0
153114cf11afSPaul Mackerras
153214cf11afSPaul Mackerras	/* Switch off MMU if not already */
1533e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, .__after_prom_start - KERNELBASE)
153414cf11afSPaul Mackerras	add	r4,r4,r30
153514cf11afSPaul Mackerras	bl	.__mmu_off
153614cf11afSPaul Mackerras	b	.__after_prom_start
153714cf11afSPaul Mackerras
1538be42d5faSPaul Mackerras#ifdef CONFIG_PPC_MULTIPLATFORM
153914cf11afSPaul Mackerras_STATIC(__boot_from_prom)
154014cf11afSPaul Mackerras	/* Save parameters */
154114cf11afSPaul Mackerras	mr	r31,r3
154214cf11afSPaul Mackerras	mr	r30,r4
154314cf11afSPaul Mackerras	mr	r29,r5
154414cf11afSPaul Mackerras	mr	r28,r6
154514cf11afSPaul Mackerras	mr	r27,r7
154614cf11afSPaul Mackerras
154714cf11afSPaul Mackerras	/* Make sure we are running in 64 bits mode */
154814cf11afSPaul Mackerras	bl	.enable_64b_mode
154914cf11afSPaul Mackerras
155014cf11afSPaul Mackerras	/* put a relocation offset into r3 */
155114cf11afSPaul Mackerras	bl	.reloc_offset
155214cf11afSPaul Mackerras
1553e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r2,__toc_start)
155414cf11afSPaul Mackerras	addi	r2,r2,0x4000
155514cf11afSPaul Mackerras	addi	r2,r2,0x4000
155614cf11afSPaul Mackerras
155714cf11afSPaul Mackerras	/* Relocate the TOC from a virt addr to a real addr */
15585a408329SPaul Mackerras	add	r2,r2,r3
155914cf11afSPaul Mackerras
156014cf11afSPaul Mackerras	/* Restore parameters */
156114cf11afSPaul Mackerras	mr	r3,r31
156214cf11afSPaul Mackerras	mr	r4,r30
156314cf11afSPaul Mackerras	mr	r5,r29
156414cf11afSPaul Mackerras	mr	r6,r28
156514cf11afSPaul Mackerras	mr	r7,r27
156614cf11afSPaul Mackerras
156714cf11afSPaul Mackerras	/* Do all of the interaction with OF client interface */
156814cf11afSPaul Mackerras	bl	.prom_init
156914cf11afSPaul Mackerras	/* We never return */
157014cf11afSPaul Mackerras	trap
1571be42d5faSPaul Mackerras#endif
157214cf11afSPaul Mackerras
157314cf11afSPaul Mackerras/*
157414cf11afSPaul Mackerras * At this point, r3 contains the physical address we are running at,
157514cf11afSPaul Mackerras * returned by prom_init()
157614cf11afSPaul Mackerras */
157714cf11afSPaul Mackerras_STATIC(__after_prom_start)
157814cf11afSPaul Mackerras
157914cf11afSPaul Mackerras/*
1580758438a7SMichael Ellerman * We need to run with __start at physical address PHYSICAL_START.
158114cf11afSPaul Mackerras * This will leave some code in the first 256B of
158214cf11afSPaul Mackerras * real memory, which are reserved for software use.
158314cf11afSPaul Mackerras * The remainder of the first page is loaded with the fixed
158414cf11afSPaul Mackerras * interrupt vectors.  The next two pages are filled with
158514cf11afSPaul Mackerras * unknown exception placeholders.
158614cf11afSPaul Mackerras *
158714cf11afSPaul Mackerras * Note: This process overwrites the OF exception vectors.
158814cf11afSPaul Mackerras *	r26 == relocation offset
158914cf11afSPaul Mackerras *	r27 == KERNELBASE
159014cf11afSPaul Mackerras */
159114cf11afSPaul Mackerras	bl	.reloc_offset
159214cf11afSPaul Mackerras	mr	r26,r3
1593e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r27, KERNELBASE)
159414cf11afSPaul Mackerras
1595e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3, PHYSICAL_START)	/* target addr */
159614cf11afSPaul Mackerras
159714cf11afSPaul Mackerras	// XXX FIXME: Use phys returned by OF (r30)
15985a408329SPaul Mackerras	add	r4,r27,r26 		/* source addr			 */
159914cf11afSPaul Mackerras					/* current address of _start	 */
160014cf11afSPaul Mackerras					/*   i.e. where we are running	 */
160114cf11afSPaul Mackerras					/*	the source addr		 */
160214cf11afSPaul Mackerras
1603e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r5,copy_to_here) /* # bytes of memory to copy */
160414cf11afSPaul Mackerras	sub	r5,r5,r27
160514cf11afSPaul Mackerras
160614cf11afSPaul Mackerras	li	r6,0x100		/* Start offset, the first 0x100 */
160714cf11afSPaul Mackerras					/* bytes were copied earlier.	 */
160814cf11afSPaul Mackerras
160914cf11afSPaul Mackerras	bl	.copy_and_flush		/* copy the first n bytes	 */
161014cf11afSPaul Mackerras					/* this includes the code being	 */
161114cf11afSPaul Mackerras					/* executed here.		 */
161214cf11afSPaul Mackerras
1613e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r0, 4f)	/* Jump to the copy of this code */
161414cf11afSPaul Mackerras	mtctr	r0			/* that we just made/relocated	 */
161514cf11afSPaul Mackerras	bctr
161614cf11afSPaul Mackerras
1617e58c3495SDavid Gibson4:	LOAD_REG_IMMEDIATE(r5,klimit)
16185a408329SPaul Mackerras	add	r5,r5,r26
161914cf11afSPaul Mackerras	ld	r5,0(r5)		/* get the value of klimit */
162014cf11afSPaul Mackerras	sub	r5,r5,r27
162114cf11afSPaul Mackerras	bl	.copy_and_flush		/* copy the rest */
162214cf11afSPaul Mackerras	b	.start_here_multiplatform
162314cf11afSPaul Mackerras
162414cf11afSPaul Mackerras#endif /* CONFIG_PPC_MULTIPLATFORM */
162514cf11afSPaul Mackerras
162614cf11afSPaul Mackerras/*
162714cf11afSPaul Mackerras * Copy routine used to copy the kernel to start at physical address 0
162814cf11afSPaul Mackerras * and flush and invalidate the caches as needed.
162914cf11afSPaul Mackerras * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
163014cf11afSPaul Mackerras * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
163114cf11afSPaul Mackerras *
163214cf11afSPaul Mackerras * Note: this routine *only* clobbers r0, r6 and lr
163314cf11afSPaul Mackerras */
163414cf11afSPaul Mackerras_GLOBAL(copy_and_flush)
163514cf11afSPaul Mackerras	addi	r5,r5,-8
163614cf11afSPaul Mackerras	addi	r6,r6,-8
163714cf11afSPaul Mackerras4:	li	r0,16			/* Use the least common		*/
163814cf11afSPaul Mackerras					/* denominator cache line	*/
163914cf11afSPaul Mackerras					/* size.  This results in	*/
164014cf11afSPaul Mackerras					/* extra cache line flushes	*/
164114cf11afSPaul Mackerras					/* but operation is correct.	*/
164214cf11afSPaul Mackerras					/* Can't get cache line size	*/
164314cf11afSPaul Mackerras					/* from NACA as it is being	*/
164414cf11afSPaul Mackerras					/* moved too.			*/
164514cf11afSPaul Mackerras
164614cf11afSPaul Mackerras	mtctr	r0			/* put # words/line in ctr	*/
164714cf11afSPaul Mackerras3:	addi	r6,r6,8			/* copy a cache line		*/
164814cf11afSPaul Mackerras	ldx	r0,r6,r4
164914cf11afSPaul Mackerras	stdx	r0,r6,r3
165014cf11afSPaul Mackerras	bdnz	3b
165114cf11afSPaul Mackerras	dcbst	r6,r3			/* write it to memory		*/
165214cf11afSPaul Mackerras	sync
165314cf11afSPaul Mackerras	icbi	r6,r3			/* flush the icache line	*/
165414cf11afSPaul Mackerras	cmpld	0,r6,r5
165514cf11afSPaul Mackerras	blt	4b
165614cf11afSPaul Mackerras	sync
165714cf11afSPaul Mackerras	addi	r5,r5,8
165814cf11afSPaul Mackerras	addi	r6,r6,8
165914cf11afSPaul Mackerras	blr
166014cf11afSPaul Mackerras
166114cf11afSPaul Mackerras.align 8
166214cf11afSPaul Mackerrascopy_to_here:
166314cf11afSPaul Mackerras
166414cf11afSPaul Mackerras#ifdef CONFIG_SMP
166514cf11afSPaul Mackerras#ifdef CONFIG_PPC_PMAC
166614cf11afSPaul Mackerras/*
166714cf11afSPaul Mackerras * On PowerMac, secondary processors starts from the reset vector, which
166814cf11afSPaul Mackerras * is temporarily turned into a call to one of the functions below.
166914cf11afSPaul Mackerras */
167014cf11afSPaul Mackerras	.section ".text";
167114cf11afSPaul Mackerras	.align 2 ;
167214cf11afSPaul Mackerras
167335499c01SPaul Mackerras	.globl	__secondary_start_pmac_0
167435499c01SPaul Mackerras__secondary_start_pmac_0:
167535499c01SPaul Mackerras	/* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
167635499c01SPaul Mackerras	li	r24,0
167735499c01SPaul Mackerras	b	1f
167814cf11afSPaul Mackerras	li	r24,1
167935499c01SPaul Mackerras	b	1f
168014cf11afSPaul Mackerras	li	r24,2
168135499c01SPaul Mackerras	b	1f
168214cf11afSPaul Mackerras	li	r24,3
168335499c01SPaul Mackerras1:
168414cf11afSPaul Mackerras
168514cf11afSPaul Mackerras_GLOBAL(pmac_secondary_start)
168614cf11afSPaul Mackerras	/* turn on 64-bit mode */
168714cf11afSPaul Mackerras	bl	.enable_64b_mode
168814cf11afSPaul Mackerras	isync
168914cf11afSPaul Mackerras
169014cf11afSPaul Mackerras	/* Copy some CPU settings from CPU 0 */
169114cf11afSPaul Mackerras	bl	.__restore_cpu_setup
169214cf11afSPaul Mackerras
169314cf11afSPaul Mackerras	/* pSeries do that early though I don't think we really need it */
169414cf11afSPaul Mackerras	mfmsr	r3
169514cf11afSPaul Mackerras	ori	r3,r3,MSR_RI
169614cf11afSPaul Mackerras	mtmsrd	r3			/* RI on */
169714cf11afSPaul Mackerras
169814cf11afSPaul Mackerras	/* Set up a paca value for this processor. */
1699e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, paca)	/* Get base vaddr of paca array	*/
170014cf11afSPaul Mackerras	mulli	r13,r24,PACA_SIZE	 /* Calculate vaddr of right paca */
170114cf11afSPaul Mackerras	add	r13,r13,r4		/* for this processor.		*/
1702b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG3,r13		 /* Save vaddr of paca in SPRG3	*/
170314cf11afSPaul Mackerras
170414cf11afSPaul Mackerras	/* Create a temp kernel stack for use before relocation is on.	*/
170514cf11afSPaul Mackerras	ld	r1,PACAEMERGSP(r13)
170614cf11afSPaul Mackerras	subi	r1,r1,STACK_FRAME_OVERHEAD
170714cf11afSPaul Mackerras
170814cf11afSPaul Mackerras	b	.__secondary_start
170914cf11afSPaul Mackerras
171014cf11afSPaul Mackerras#endif /* CONFIG_PPC_PMAC */
171114cf11afSPaul Mackerras
171214cf11afSPaul Mackerras/*
171314cf11afSPaul Mackerras * This function is called after the master CPU has released the
171414cf11afSPaul Mackerras * secondary processors.  The execution environment is relocation off.
171514cf11afSPaul Mackerras * The paca for this processor has the following fields initialized at
171614cf11afSPaul Mackerras * this point:
171714cf11afSPaul Mackerras *   1. Processor number
171814cf11afSPaul Mackerras *   2. Segment table pointer (virtual address)
171914cf11afSPaul Mackerras * On entry the following are set:
172014cf11afSPaul Mackerras *   r1	= stack pointer.  vaddr for iSeries, raddr (temp stack) for pSeries
172114cf11afSPaul Mackerras *   r24   = cpu# (in Linux terms)
172214cf11afSPaul Mackerras *   r13   = paca virtual address
172314cf11afSPaul Mackerras *   SPRG3 = paca virtual address
172414cf11afSPaul Mackerras */
172514cf11afSPaul Mackerras_GLOBAL(__secondary_start)
1726799d6046SPaul Mackerras	/* Set thread priority to MEDIUM */
1727799d6046SPaul Mackerras	HMT_MEDIUM
172814cf11afSPaul Mackerras
1729799d6046SPaul Mackerras	/* Load TOC */
173014cf11afSPaul Mackerras	ld	r2,PACATOC(r13)
173114cf11afSPaul Mackerras
1732799d6046SPaul Mackerras	/* Do early setup for that CPU (stab, slb, hash table pointer) */
1733799d6046SPaul Mackerras	bl	.early_setup_secondary
173414cf11afSPaul Mackerras
173514cf11afSPaul Mackerras	/* Initialize the kernel stack.  Just a repeat for iSeries.	 */
1736e58c3495SDavid Gibson	LOAD_REG_ADDR(r3, current_set)
173714cf11afSPaul Mackerras	sldi	r28,r24,3		/* get current_set[cpu#]	 */
173814cf11afSPaul Mackerras	ldx	r1,r3,r28
173914cf11afSPaul Mackerras	addi	r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
174014cf11afSPaul Mackerras	std	r1,PACAKSAVE(r13)
174114cf11afSPaul Mackerras
1742799d6046SPaul Mackerras	/* Clear backchain so we get nice backtraces */
174314cf11afSPaul Mackerras	li	r7,0
174414cf11afSPaul Mackerras	mtlr	r7
174514cf11afSPaul Mackerras
174614cf11afSPaul Mackerras	/* enable MMU and jump to start_secondary */
1747e58c3495SDavid Gibson	LOAD_REG_ADDR(r3, .start_secondary_prolog)
1748e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
174914cf11afSPaul Mackerras#ifdef DO_SOFT_DISABLE
175014cf11afSPaul Mackerras	ori	r4,r4,MSR_EE
175114cf11afSPaul Mackerras#endif
1752b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r3
1753b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r4
175414cf11afSPaul Mackerras	rfid
175514cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
175614cf11afSPaul Mackerras
175714cf11afSPaul Mackerras/*
175814cf11afSPaul Mackerras * Running with relocation on at this point.  All we want to do is
175914cf11afSPaul Mackerras * zero the stack back-chain pointer before going into C code.
176014cf11afSPaul Mackerras */
176114cf11afSPaul Mackerras_GLOBAL(start_secondary_prolog)
176214cf11afSPaul Mackerras	li	r3,0
176314cf11afSPaul Mackerras	std	r3,0(r1)		/* Zero the stack frame pointer	*/
176414cf11afSPaul Mackerras	bl	.start_secondary
1765799d6046SPaul Mackerras	b	.
176614cf11afSPaul Mackerras#endif
176714cf11afSPaul Mackerras
176814cf11afSPaul Mackerras/*
176914cf11afSPaul Mackerras * This subroutine clobbers r11 and r12
177014cf11afSPaul Mackerras */
177114cf11afSPaul Mackerras_GLOBAL(enable_64b_mode)
177214cf11afSPaul Mackerras	mfmsr	r11			/* grab the current MSR */
177314cf11afSPaul Mackerras	li	r12,1
177414cf11afSPaul Mackerras	rldicr	r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
177514cf11afSPaul Mackerras	or	r11,r11,r12
177614cf11afSPaul Mackerras	li	r12,1
177714cf11afSPaul Mackerras	rldicr	r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
177814cf11afSPaul Mackerras	or	r11,r11,r12
177914cf11afSPaul Mackerras	mtmsrd	r11
178014cf11afSPaul Mackerras	isync
178114cf11afSPaul Mackerras	blr
178214cf11afSPaul Mackerras
178314cf11afSPaul Mackerras#ifdef CONFIG_PPC_MULTIPLATFORM
178414cf11afSPaul Mackerras/*
178514cf11afSPaul Mackerras * This is where the main kernel code starts.
178614cf11afSPaul Mackerras */
178714cf11afSPaul Mackerras_STATIC(start_here_multiplatform)
178814cf11afSPaul Mackerras	/* get a new offset, now that the kernel has moved. */
178914cf11afSPaul Mackerras	bl	.reloc_offset
179014cf11afSPaul Mackerras	mr	r26,r3
179114cf11afSPaul Mackerras
179214cf11afSPaul Mackerras	/* Clear out the BSS. It may have been done in prom_init,
179314cf11afSPaul Mackerras	 * already but that's irrelevant since prom_init will soon
179414cf11afSPaul Mackerras	 * be detached from the kernel completely. Besides, we need
179514cf11afSPaul Mackerras	 * to clear it now for kexec-style entry.
179614cf11afSPaul Mackerras	 */
1797e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r11,__bss_stop)
1798e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r8,__bss_start)
179914cf11afSPaul Mackerras	sub	r11,r11,r8		/* bss size			*/
180014cf11afSPaul Mackerras	addi	r11,r11,7		/* round up to an even double word */
180114cf11afSPaul Mackerras	rldicl. r11,r11,61,3		/* shift right by 3		*/
180214cf11afSPaul Mackerras	beq	4f
180314cf11afSPaul Mackerras	addi	r8,r8,-8
180414cf11afSPaul Mackerras	li	r0,0
180514cf11afSPaul Mackerras	mtctr	r11			/* zero this many doublewords	*/
180614cf11afSPaul Mackerras3:	stdu	r0,8(r8)
180714cf11afSPaul Mackerras	bdnz	3b
180814cf11afSPaul Mackerras4:
180914cf11afSPaul Mackerras
181014cf11afSPaul Mackerras	mfmsr	r6
181114cf11afSPaul Mackerras	ori	r6,r6,MSR_RI
181214cf11afSPaul Mackerras	mtmsrd	r6			/* RI on */
181314cf11afSPaul Mackerras
181414cf11afSPaul Mackerras	/* The following gets the stack and TOC set up with the regs */
181514cf11afSPaul Mackerras	/* pointing to the real addr of the kernel stack.  This is   */
181614cf11afSPaul Mackerras	/* all done to support the C function call below which sets  */
181714cf11afSPaul Mackerras	/* up the htab.  This is done because we have relocated the  */
181814cf11afSPaul Mackerras	/* kernel but are still running in real mode. */
181914cf11afSPaul Mackerras
1820e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3,init_thread_union)
18215a408329SPaul Mackerras	add	r3,r3,r26
182214cf11afSPaul Mackerras
182314cf11afSPaul Mackerras	/* set up a stack pointer (physical address) */
182414cf11afSPaul Mackerras	addi	r1,r3,THREAD_SIZE
182514cf11afSPaul Mackerras	li	r0,0
182614cf11afSPaul Mackerras	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
182714cf11afSPaul Mackerras
182814cf11afSPaul Mackerras	/* set up the TOC (physical address) */
1829e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r2,__toc_start)
183014cf11afSPaul Mackerras	addi	r2,r2,0x4000
183114cf11afSPaul Mackerras	addi	r2,r2,0x4000
18325a408329SPaul Mackerras	add	r2,r2,r26
183314cf11afSPaul Mackerras
1834e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3, cpu_specs)
18355a408329SPaul Mackerras	add	r3,r3,r26
1836e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
18375a408329SPaul Mackerras	add	r4,r4,r26
183814cf11afSPaul Mackerras	mr	r5,r26
183914cf11afSPaul Mackerras	bl	.identify_cpu
184014cf11afSPaul Mackerras
184114cf11afSPaul Mackerras	/* Save some low level config HIDs of CPU0 to be copied to
184214cf11afSPaul Mackerras	 * other CPUs later on, or used for suspend/resume
184314cf11afSPaul Mackerras	 */
184414cf11afSPaul Mackerras	bl	.__save_cpu_setup
184514cf11afSPaul Mackerras	sync
184614cf11afSPaul Mackerras
184714cf11afSPaul Mackerras	/* Setup a valid physical PACA pointer in SPRG3 for early_setup
184814cf11afSPaul Mackerras	 * note that boot_cpuid can always be 0 nowadays since there is
184914cf11afSPaul Mackerras	 * nowhere it can be initialized differently before we reach this
185014cf11afSPaul Mackerras	 * code
185114cf11afSPaul Mackerras	 */
1852e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r27, boot_cpuid)
18535a408329SPaul Mackerras	add	r27,r27,r26
185414cf11afSPaul Mackerras	lwz	r27,0(r27)
185514cf11afSPaul Mackerras
1856e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r24, paca) 	/* Get base vaddr of paca array	 */
185714cf11afSPaul Mackerras	mulli	r13,r27,PACA_SIZE	/* Calculate vaddr of right paca */
185814cf11afSPaul Mackerras	add	r13,r13,r24		/* for this processor.		 */
18595a408329SPaul Mackerras	add	r13,r13,r26		/* convert to physical addr	 */
1860448b2719SAnton Blanchard	mtspr	SPRN_SPRG3,r13
186114cf11afSPaul Mackerras
186214cf11afSPaul Mackerras	/* Do very early kernel initializations, including initial hash table,
186314cf11afSPaul Mackerras	 * stab and slb setup before we turn on relocation.	*/
186414cf11afSPaul Mackerras
186514cf11afSPaul Mackerras	/* Restore parameters passed from prom_init/kexec */
186614cf11afSPaul Mackerras	mr	r3,r31
186714cf11afSPaul Mackerras 	bl	.early_setup
186814cf11afSPaul Mackerras
1869e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3, .start_here_common)
1870e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1871b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR0,r3
1872b5bbeb23SPaul Mackerras	mtspr	SPRN_SRR1,r4
187314cf11afSPaul Mackerras	rfid
187414cf11afSPaul Mackerras	b	.	/* prevent speculative execution */
187514cf11afSPaul Mackerras#endif /* CONFIG_PPC_MULTIPLATFORM */
187614cf11afSPaul Mackerras
187714cf11afSPaul Mackerras	/* This is where all platforms converge execution */
187814cf11afSPaul Mackerras_STATIC(start_here_common)
187914cf11afSPaul Mackerras	/* relocation is on at this point */
188014cf11afSPaul Mackerras
188114cf11afSPaul Mackerras	/* The following code sets up the SP and TOC now that we are */
188214cf11afSPaul Mackerras	/* running with translation enabled. */
188314cf11afSPaul Mackerras
1884e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r3,init_thread_union)
188514cf11afSPaul Mackerras
188614cf11afSPaul Mackerras	/* set up the stack */
188714cf11afSPaul Mackerras	addi	r1,r3,THREAD_SIZE
188814cf11afSPaul Mackerras	li	r0,0
188914cf11afSPaul Mackerras	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
189014cf11afSPaul Mackerras
189114cf11afSPaul Mackerras	/* Apply the CPUs-specific fixups (nop out sections not relevant
189214cf11afSPaul Mackerras	 * to this CPU
189314cf11afSPaul Mackerras	 */
189414cf11afSPaul Mackerras	li	r3,0
189514cf11afSPaul Mackerras	bl	.do_cpu_ftr_fixups
189614cf11afSPaul Mackerras
1897e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r26, boot_cpuid)
189814cf11afSPaul Mackerras	lwz	r26,0(r26)
189914cf11afSPaul Mackerras
1900e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r24, paca)	/* Get base vaddr of paca array  */
190114cf11afSPaul Mackerras	mulli	r13,r26,PACA_SIZE	/* Calculate vaddr of right paca */
190214cf11afSPaul Mackerras	add	r13,r13,r24		/* for this processor.		 */
1903b5bbeb23SPaul Mackerras	mtspr	SPRN_SPRG3,r13
190414cf11afSPaul Mackerras
190514cf11afSPaul Mackerras	/* ptr to current */
1906e58c3495SDavid Gibson	LOAD_REG_IMMEDIATE(r4, init_task)
190714cf11afSPaul Mackerras	std	r4,PACACURRENT(r13)
190814cf11afSPaul Mackerras
190914cf11afSPaul Mackerras	/* Load the TOC */
191014cf11afSPaul Mackerras	ld	r2,PACATOC(r13)
191114cf11afSPaul Mackerras	std	r1,PACAKSAVE(r13)
191214cf11afSPaul Mackerras
191314cf11afSPaul Mackerras	bl	.setup_system
191414cf11afSPaul Mackerras
191514cf11afSPaul Mackerras	/* Load up the kernel context */
191614cf11afSPaul Mackerras5:
191714cf11afSPaul Mackerras#ifdef DO_SOFT_DISABLE
191814cf11afSPaul Mackerras	li	r5,0
191914cf11afSPaul Mackerras	stb	r5,PACAPROCENABLED(r13)	/* Soft Disabled */
192014cf11afSPaul Mackerras	mfmsr	r5
192114cf11afSPaul Mackerras	ori	r5,r5,MSR_EE		/* Hard Enabled */
192214cf11afSPaul Mackerras	mtmsrd	r5
192314cf11afSPaul Mackerras#endif
192414cf11afSPaul Mackerras
192514cf11afSPaul Mackerras	bl .start_kernel
192614cf11afSPaul Mackerras
1927f1870f77SAnton Blanchard	/* Not reached */
1928f1870f77SAnton Blanchard	BUG_OPCODE
192914cf11afSPaul Mackerras
193014cf11afSPaul Mackerras/*
193114cf11afSPaul Mackerras * We put a few things here that have to be page-aligned.
193214cf11afSPaul Mackerras * This stuff goes at the beginning of the bss, which is page-aligned.
193314cf11afSPaul Mackerras */
193414cf11afSPaul Mackerras	.section ".bss"
193514cf11afSPaul Mackerras
193614cf11afSPaul Mackerras	.align	PAGE_SHIFT
193714cf11afSPaul Mackerras
193814cf11afSPaul Mackerras	.globl	empty_zero_page
193914cf11afSPaul Mackerrasempty_zero_page:
194014cf11afSPaul Mackerras	.space	PAGE_SIZE
194114cf11afSPaul Mackerras
194214cf11afSPaul Mackerras	.globl	swapper_pg_dir
194314cf11afSPaul Mackerrasswapper_pg_dir:
194414cf11afSPaul Mackerras	.space	PAGE_SIZE
194514cf11afSPaul Mackerras
194614cf11afSPaul Mackerras/*
194714cf11afSPaul Mackerras * This space gets a copy of optional info passed to us by the bootstrap
194814cf11afSPaul Mackerras * Used to pass parameters into the kernel like root=/dev/sda1, etc.
194914cf11afSPaul Mackerras */
195014cf11afSPaul Mackerras	.globl	cmd_line
195114cf11afSPaul Mackerrascmd_line:
195214cf11afSPaul Mackerras	.space	COMMAND_LINE_SIZE
1953