xref: /openbmc/linux/arch/powerpc/kernel/head_8xx.S (revision e7253313) 
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *  PowerPC version
4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 *  Low-level exception handlers and MMU support
8 *  rewritten by Paul Mackerras.
9 *    Copyright (C) 1996 Paul Mackerras.
10 *  MPC8xx modifications by Dan Malek
11 *    Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 *  This file contains low-level support and setup for PowerPC 8xx
14 *  embedded processors, including trap and interrupt dispatch.
15 */
16
17#include <linux/init.h>
18#include <linux/magic.h>
19#include <asm/processor.h>
20#include <asm/page.h>
21#include <asm/mmu.h>
22#include <asm/cache.h>
23#include <asm/pgtable.h>
24#include <asm/cputable.h>
25#include <asm/thread_info.h>
26#include <asm/ppc_asm.h>
27#include <asm/asm-offsets.h>
28#include <asm/ptrace.h>
29#include <asm/export.h>
30#include <asm/code-patching-asm.h>
31
32#include "head_32.h"
33
34#if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
35/* By simply checking Address >= 0x80000000, we know if its a kernel address */
36#define SIMPLE_KERNEL_ADDRESS		1
37#endif
38
39/*
40 * We need an ITLB miss handler for kernel addresses if:
41 * - Either we have modules
42 * - Or we have not pinned the first 8M
43 */
44#if defined(CONFIG_MODULES) || !defined(CONFIG_PIN_TLB_TEXT) || \
45    defined(CONFIG_DEBUG_PAGEALLOC)
46#define ITLB_MISS_KERNEL	1
47#endif
48
49/*
50 * Value for the bits that have fixed value in RPN entries.
51 * Also used for tagging DAR for DTLBerror.
52 */
53#define RPN_PATTERN	0x00f0
54
55#define PAGE_SHIFT_512K		19
56#define PAGE_SHIFT_8M		23
57
58	__HEAD
59_ENTRY(_stext);
60_ENTRY(_start);
61
62/* MPC8xx
63 * This port was done on an MBX board with an 860.  Right now I only
64 * support an ELF compressed (zImage) boot from EPPC-Bug because the
65 * code there loads up some registers before calling us:
66 *   r3: ptr to board info data
67 *   r4: initrd_start or if no initrd then 0
68 *   r5: initrd_end - unused if r4 is 0
69 *   r6: Start of command line string
70 *   r7: End of command line string
71 *
72 * I decided to use conditional compilation instead of checking PVR and
73 * adding more processor specific branches around code I don't need.
74 * Since this is an embedded processor, I also appreciate any memory
75 * savings I can get.
76 *
77 * The MPC8xx does not have any BATs, but it supports large page sizes.
78 * We first initialize the MMU to support 8M byte pages, then load one
79 * entry into each of the instruction and data TLBs to map the first
80 * 8M 1:1.  I also mapped an additional I/O space 1:1 so we can get to
81 * the "internal" processor registers before MMU_init is called.
82 *
83 *	-- Dan
84 */
85	.globl	__start
86__start:
87	mr	r31,r3			/* save device tree ptr */
88
89	/* We have to turn on the MMU right away so we get cache modes
90	 * set correctly.
91	 */
92	bl	initial_mmu
93
94/* We now have the lower 8 Meg mapped into TLB entries, and the caches
95 * ready to work.
96 */
97
98turn_on_mmu:
99	mfmsr	r0
100	ori	r0,r0,MSR_DR|MSR_IR
101	mtspr	SPRN_SRR1,r0
102	lis	r0,start_here@h
103	ori	r0,r0,start_here@l
104	mtspr	SPRN_SRR0,r0
105	rfi				/* enables MMU */
106
107
108#ifdef CONFIG_PERF_EVENTS
109	.align	4
110
111	.globl	itlb_miss_counter
112itlb_miss_counter:
113	.space	4
114
115	.globl	dtlb_miss_counter
116dtlb_miss_counter:
117	.space	4
118
119	.globl	instruction_counter
120instruction_counter:
121	.space	4
122#endif
123
124/* System reset */
125	EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
126
127/* Machine check */
128	. = 0x200
129MachineCheck:
130	EXCEPTION_PROLOG
131	mfspr r4,SPRN_DAR
132	stw r4,_DAR(r11)
133	li r5,RPN_PATTERN
134	mtspr SPRN_DAR,r5	/* Tag DAR, to be used in DTLB Error */
135	mfspr r5,SPRN_DSISR
136	stw r5,_DSISR(r11)
137	addi r3,r1,STACK_FRAME_OVERHEAD
138	EXC_XFER_STD(0x200, machine_check_exception)
139
140/* Data access exception.
141 * This is "never generated" by the MPC8xx.
142 */
143	. = 0x300
144DataAccess:
145
146/* Instruction access exception.
147 * This is "never generated" by the MPC8xx.
148 */
149	. = 0x400
150InstructionAccess:
151
152/* External interrupt */
153	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
154
155/* Alignment exception */
156	. = 0x600
157Alignment:
158	EXCEPTION_PROLOG
159	mfspr	r4,SPRN_DAR
160	stw	r4,_DAR(r11)
161	li	r5,RPN_PATTERN
162	mtspr	SPRN_DAR,r5	/* Tag DAR, to be used in DTLB Error */
163	mfspr	r5,SPRN_DSISR
164	stw	r5,_DSISR(r11)
165	addi	r3,r1,STACK_FRAME_OVERHEAD
166	EXC_XFER_STD(0x600, alignment_exception)
167
168/* Program check exception */
169	EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
170
171/* No FPU on MPC8xx.  This exception is not supposed to happen.
172*/
173	EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD)
174
175/* Decrementer */
176	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
177
178	EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)
179	EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)
180
181/* System call */
182	. = 0xc00
183SystemCall:
184	SYSCALL_ENTRY	0xc00
185
186/* Single step - not used on 601 */
187	EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
188	EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)
189	EXCEPTION(0xf00, Trap_0f, unknown_exception, EXC_XFER_STD)
190
191/* On the MPC8xx, this is a software emulation interrupt.  It occurs
192 * for all unimplemented and illegal instructions.
193 */
194	EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD)
195
196/* Called from DataStoreTLBMiss when perf TLB misses events are activated */
197#ifdef CONFIG_PERF_EVENTS
198	patch_site	0f, patch__dtlbmiss_perf
1990:	lwz	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
200	addi	r10, r10, 1
201	stw	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
202	mfspr	r10, SPRN_SPRG_SCRATCH0
203	mfspr	r11, SPRN_SPRG_SCRATCH1
204	rfi
205#endif
206
207	. = 0x1100
208/*
209 * For the MPC8xx, this is a software tablewalk to load the instruction
210 * TLB.  The task switch loads the M_TWB register with the pointer to the first
211 * level table.
212 * If we discover there is no second level table (value is zero) or if there
213 * is an invalid pte, we load that into the TLB, which causes another fault
214 * into the TLB Error interrupt where we can handle such problems.
215 * We have to use the MD_xxx registers for the tablewalk because the
216 * equivalent MI_xxx registers only perform the attribute functions.
217 */
218
219#ifdef CONFIG_8xx_CPU15
220#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)	\
221	addi	addr, addr, PAGE_SIZE;	\
222	tlbie	addr;			\
223	addi	addr, addr, -(PAGE_SIZE << 1);	\
224	tlbie	addr;			\
225	addi	addr, addr, PAGE_SIZE
226#else
227#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)
228#endif
229
230InstructionTLBMiss:
231	mtspr	SPRN_SPRG_SCRATCH0, r10
232#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
233	mtspr	SPRN_SPRG_SCRATCH1, r11
234#endif
235
236	/* If we are faulting a kernel address, we have to use the
237	 * kernel page tables.
238	 */
239	mfspr	r10, SPRN_SRR0	/* Get effective address of fault */
240	INVALIDATE_ADJACENT_PAGES_CPU15(r10)
241	mtspr	SPRN_MD_EPN, r10
242	/* Only modules will cause ITLB Misses as we always
243	 * pin the first 8MB of kernel memory */
244#ifdef ITLB_MISS_KERNEL
245	mfcr	r11
246#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
247	cmpi	cr0, r10, 0	/* Address >= 0x80000000 */
248#else
249	rlwinm	r10, r10, 16, 0xfff8
250	cmpli	cr0, r10, PAGE_OFFSET@h
251#ifndef CONFIG_PIN_TLB_TEXT
252	/* It is assumed that kernel code fits into the first 32M */
2530:	cmpli	cr7, r10, (PAGE_OFFSET + 0x2000000)@h
254	patch_site	0b, patch__itlbmiss_linmem_top
255#endif
256#endif
257#endif
258	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
259#ifdef ITLB_MISS_KERNEL
260#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
261	bge+	3f
262#else
263	blt+	3f
264#endif
265#ifndef CONFIG_PIN_TLB_TEXT
266	blt	cr7, ITLBMissLinear
267#endif
268	rlwinm	r10, r10, 0, 20, 31
269	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
2703:
271#endif
272	lwz	r10, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
273	mtspr	SPRN_MI_TWC, r10	/* Set segment attributes */
274
275	mtspr	SPRN_MD_TWC, r10
276	mfspr	r10, SPRN_MD_TWC
277	lwz	r10, 0(r10)	/* Get the pte */
278#ifdef ITLB_MISS_KERNEL
279	mtcr	r11
280#endif
281#ifdef CONFIG_SWAP
282	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
283	and	r11, r11, r10
284	rlwimi	r10, r11, 0, _PAGE_PRESENT
285#endif
286	/* The Linux PTE won't go exactly into the MMU TLB.
287	 * Software indicator bits 20 and 23 must be clear.
288	 * Software indicator bits 22, 24, 25, 26, and 27 must be
289	 * set.  All other Linux PTE bits control the behavior
290	 * of the MMU.
291	 */
292	rlwimi	r10, r10, 0, 0x0f00	/* Clear bits 20-23 */
293	rlwimi	r10, r10, 4, 0x0400	/* Copy _PAGE_EXEC into bit 21 */
294	ori	r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
295	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
296
297	/* Restore registers */
2980:	mfspr	r10, SPRN_SPRG_SCRATCH0
299#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
300	mfspr	r11, SPRN_SPRG_SCRATCH1
301#endif
302	rfi
303	patch_site	0b, patch__itlbmiss_exit_1
304
305#ifdef CONFIG_PERF_EVENTS
306	patch_site	0f, patch__itlbmiss_perf
3070:	lwz	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
308	addi	r10, r10, 1
309	stw	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
310	mfspr	r10, SPRN_SPRG_SCRATCH0
311#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
312	mfspr	r11, SPRN_SPRG_SCRATCH1
313#endif
314	rfi
315#endif
316
317#ifndef CONFIG_PIN_TLB_TEXT
318ITLBMissLinear:
319	mtcr	r11
320#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_ETEXT_SHIFT < 23
321	patch_site	0f, patch__itlbmiss_linmem_top8
322
323	mfspr	r10, SPRN_SRR0
3240:	subis	r11, r10, (PAGE_OFFSET - 0x80000000)@ha
325	rlwinm	r11, r11, 4, MI_PS8MEG ^ MI_PS512K
326	ori	r11, r11, MI_PS512K | MI_SVALID
327	rlwinm	r10, r10, 0, 0x0ff80000	/* 8xx supports max 256Mb RAM */
328#else
329	/* Set 8M byte page and mark it valid */
330	li	r11, MI_PS8MEG | MI_SVALID
331	rlwinm	r10, r10, 20, 0x0f800000	/* 8xx supports max 256Mb RAM */
332#endif
333	mtspr	SPRN_MI_TWC, r11
334	ori	r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
335			  _PAGE_PRESENT
336	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
337
3380:	mfspr	r10, SPRN_SPRG_SCRATCH0
339	mfspr	r11, SPRN_SPRG_SCRATCH1
340	rfi
341	patch_site	0b, patch__itlbmiss_exit_2
342#endif
343
344	. = 0x1200
345DataStoreTLBMiss:
346	mtspr	SPRN_SPRG_SCRATCH0, r10
347	mtspr	SPRN_SPRG_SCRATCH1, r11
348	mfcr	r11
349
350	/* If we are faulting a kernel address, we have to use the
351	 * kernel page tables.
352	 */
353	mfspr	r10, SPRN_MD_EPN
354	rlwinm	r10, r10, 16, 0xfff8
355	cmpli	cr0, r10, PAGE_OFFSET@h
356#ifndef CONFIG_PIN_TLB_IMMR
357	cmpli	cr6, r10, VIRT_IMMR_BASE@h
358#endif
3590:	cmpli	cr7, r10, (PAGE_OFFSET + 0x2000000)@h
360	patch_site	0b, patch__dtlbmiss_linmem_top
361
362	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
363	blt+	3f
364#ifndef CONFIG_PIN_TLB_IMMR
3650:	beq-	cr6, DTLBMissIMMR
366	patch_site	0b, patch__dtlbmiss_immr_jmp
367#endif
368	blt	cr7, DTLBMissLinear
369	rlwinm	r10, r10, 0, 20, 31
370	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
3713:
372	mtcr	r11
373	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
374
375	mtspr	SPRN_MD_TWC, r11
376	mfspr	r10, SPRN_MD_TWC
377	lwz	r10, 0(r10)	/* Get the pte */
378
379	/* Insert the Guarded flag into the TWC from the Linux PTE.
380	 * It is bit 27 of both the Linux PTE and the TWC (at least
381	 * I got that right :-).  It will be better when we can put
382	 * this into the Linux pgd/pmd and load it in the operation
383	 * above.
384	 */
385	rlwimi	r11, r10, 0, _PAGE_GUARDED
386	mtspr	SPRN_MD_TWC, r11
387
388	/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
389	 * We also need to know if the insn is a load/store, so:
390	 * Clear _PAGE_PRESENT and load that which will
391	 * trap into DTLB Error with store bit set accordinly.
392	 */
393	/* PRESENT=0x1, ACCESSED=0x20
394	 * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5));
395	 * r10 = (r10 & ~PRESENT) | r11;
396	 */
397#ifdef CONFIG_SWAP
398	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
399	and	r11, r11, r10
400	rlwimi	r10, r11, 0, _PAGE_PRESENT
401#endif
402	/* The Linux PTE won't go exactly into the MMU TLB.
403	 * Software indicator bits 24, 25, 26, and 27 must be
404	 * set.  All other Linux PTE bits control the behavior
405	 * of the MMU.
406	 */
407	li	r11, RPN_PATTERN
408	rlwimi	r10, r11, 0, 24, 27	/* Set 24-27 */
409	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
410
411	/* Restore registers */
412	mtspr	SPRN_DAR, r11	/* Tag DAR */
413
4140:	mfspr	r10, SPRN_SPRG_SCRATCH0
415	mfspr	r11, SPRN_SPRG_SCRATCH1
416	rfi
417	patch_site	0b, patch__dtlbmiss_exit_1
418
419DTLBMissIMMR:
420	mtcr	r11
421	/* Set 512k byte guarded page and mark it valid */
422	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
423	mtspr	SPRN_MD_TWC, r10
424	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
425	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
426	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
427			  _PAGE_PRESENT | _PAGE_NO_CACHE
428	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
429
430	li	r11, RPN_PATTERN
431	mtspr	SPRN_DAR, r11	/* Tag DAR */
432
4330:	mfspr	r10, SPRN_SPRG_SCRATCH0
434	mfspr	r11, SPRN_SPRG_SCRATCH1
435	rfi
436	patch_site	0b, patch__dtlbmiss_exit_2
437
438DTLBMissLinear:
439	mtcr	r11
440	rlwinm	r10, r10, 20, 0x0f800000	/* 8xx supports max 256Mb RAM */
441#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_DATA_SHIFT < 23
442	patch_site	0f, patch__dtlbmiss_romem_top8
443
4440:	subis	r11, r10, (PAGE_OFFSET - 0x80000000)@ha
445	rlwinm	r11, r11, 0, 0xff800000
446	neg	r10, r11
447	or	r11, r11, r10
448	rlwinm	r11, r11, 4, MI_PS8MEG ^ MI_PS512K
449	ori	r11, r11, MI_PS512K | MI_SVALID
450	mfspr	r10, SPRN_MD_EPN
451	rlwinm	r10, r10, 0, 0x0ff80000	/* 8xx supports max 256Mb RAM */
452#else
453	/* Set 8M byte page and mark it valid */
454	li	r11, MD_PS8MEG | MD_SVALID
455#endif
456	mtspr	SPRN_MD_TWC, r11
457#ifdef CONFIG_STRICT_KERNEL_RWX
458	patch_site	0f, patch__dtlbmiss_romem_top
459
4600:	subis	r11, r10, 0
461	rlwimi	r10, r11, 11, _PAGE_RO
462#endif
463	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
464			  _PAGE_PRESENT
465	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
466
467	li	r11, RPN_PATTERN
468	mtspr	SPRN_DAR, r11	/* Tag DAR */
469
4700:	mfspr	r10, SPRN_SPRG_SCRATCH0
471	mfspr	r11, SPRN_SPRG_SCRATCH1
472	rfi
473	patch_site	0b, patch__dtlbmiss_exit_3
474
475/* This is an instruction TLB error on the MPC8xx.  This could be due
476 * to many reasons, such as executing guarded memory or illegal instruction
477 * addresses.  There is nothing to do but handle a big time error fault.
478 */
479	. = 0x1300
480InstructionTLBError:
481	EXCEPTION_PROLOG
482	mr	r4,r12
483	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
484	andis.	r10,r9,SRR1_ISI_NOPT@h
485	beq+	.Litlbie
486	tlbie	r4
487	/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
488.Litlbie:
489	EXC_XFER_LITE(0x400, handle_page_fault)
490
491/* This is the data TLB error on the MPC8xx.  This could be due to
492 * many reasons, including a dirty update to a pte.  We bail out to
493 * a higher level function that can handle it.
494 */
495	. = 0x1400
496DataTLBError:
497	mtspr	SPRN_SPRG_SCRATCH0, r10
498	mtspr	SPRN_SPRG_SCRATCH1, r11
499	mfcr	r10
500
501	mfspr	r11, SPRN_DAR
502	cmpwi	cr0, r11, RPN_PATTERN
503	beq-	FixupDAR	/* must be a buggy dcbX, icbi insn. */
504DARFixed:/* Return from dcbx instruction bug workaround */
505	EXCEPTION_PROLOG_1
506	EXCEPTION_PROLOG_2
507	mfspr	r5,SPRN_DSISR
508	stw	r5,_DSISR(r11)
509	mfspr	r4,SPRN_DAR
510	andis.	r10,r5,DSISR_NOHPTE@h
511	beq+	.Ldtlbie
512	tlbie	r4
513.Ldtlbie:
514	li	r10,RPN_PATTERN
515	mtspr	SPRN_DAR,r10	/* Tag DAR, to be used in DTLB Error */
516	/* 0x300 is DataAccess exception, needed by bad_page_fault() */
517	EXC_XFER_LITE(0x300, handle_page_fault)
518
519	EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
520	EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD)
521	EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD)
522	EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
523	EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
524	EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)
525	EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)
526
527/* On the MPC8xx, these next four traps are used for development
528 * support of breakpoints and such.  Someday I will get around to
529 * using them.
530 */
531	. = 0x1c00
532DataBreakpoint:
533	mtspr	SPRN_SPRG_SCRATCH0, r10
534	mtspr	SPRN_SPRG_SCRATCH1, r11
535	mfcr	r10
536	mfspr	r11, SPRN_SRR0
537	cmplwi	cr0, r11, (.Ldtlbie - PAGE_OFFSET)@l
538	cmplwi	cr7, r11, (.Litlbie - PAGE_OFFSET)@l
539	beq-	cr0, 11f
540	beq-	cr7, 11f
541	EXCEPTION_PROLOG_1
542	EXCEPTION_PROLOG_2
543	addi	r3,r1,STACK_FRAME_OVERHEAD
544	mfspr	r4,SPRN_BAR
545	stw	r4,_DAR(r11)
546	mfspr	r5,SPRN_DSISR
547	EXC_XFER_STD(0x1c00, do_break)
54811:
549	mtcr	r10
550	mfspr	r10, SPRN_SPRG_SCRATCH0
551	mfspr	r11, SPRN_SPRG_SCRATCH1
552	rfi
553
554#ifdef CONFIG_PERF_EVENTS
555	. = 0x1d00
556InstructionBreakpoint:
557	mtspr	SPRN_SPRG_SCRATCH0, r10
558	lwz	r10, (instruction_counter - PAGE_OFFSET)@l(0)
559	addi	r10, r10, -1
560	stw	r10, (instruction_counter - PAGE_OFFSET)@l(0)
561	lis	r10, 0xffff
562	ori	r10, r10, 0x01
563	mtspr	SPRN_COUNTA, r10
564	mfspr	r10, SPRN_SPRG_SCRATCH0
565	rfi
566#else
567	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
568#endif
569	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
570	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
571
572	. = 0x2000
573
574/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
575 * by decoding the registers used by the dcbx instruction and adding them.
576 * DAR is set to the calculated address.
577 */
578FixupDAR:/* Entry point for dcbx workaround. */
579	mtspr	SPRN_M_TW, r10
580	/* fetch instruction from memory. */
581	mfspr	r10, SPRN_SRR0
582	mtspr	SPRN_MD_EPN, r10
583	rlwinm	r11, r10, 16, 0xfff8
584	cmpli	cr0, r11, PAGE_OFFSET@h
585	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
586	blt+	3f
587	rlwinm	r11, r10, 16, 0xfff8
588
5890:	cmpli	cr7, r11, (PAGE_OFFSET + 0x1800000)@h
590	patch_site	0b, patch__fixupdar_linmem_top
591
592	/* create physical page address from effective address */
593	tophys(r11, r10)
594	blt-	cr7, 201f
595	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
596	rlwinm	r11, r11, 0, 20, 31
597	oris	r11, r11, (swapper_pg_dir - PAGE_OFFSET)@ha
5983:
599	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
600	mtspr	SPRN_MD_TWC, r11
601	mtcr	r11
602	mfspr	r11, SPRN_MD_TWC
603	lwz	r11, 0(r11)	/* Get the pte */
604	bt	28,200f		/* bit 28 = Large page (8M) */
605	bt	29,202f		/* bit 29 = Large page (8M or 512K) */
606	/* concat physical page address(r11) and page offset(r10) */
607	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT, 31
608201:	lwz	r11,0(r11)
609/* Check if it really is a dcbx instruction. */
610/* dcbt and dcbtst does not generate DTLB Misses/Errors,
611 * no need to include them here */
612	xoris	r10, r11, 0x7c00	/* check if major OP code is 31 */
613	rlwinm	r10, r10, 0, 21, 5
614	cmpwi	cr0, r10, 2028	/* Is dcbz? */
615	beq+	142f
616	cmpwi	cr0, r10, 940	/* Is dcbi? */
617	beq+	142f
618	cmpwi	cr0, r10, 108	/* Is dcbst? */
619	beq+	144f		/* Fix up store bit! */
620	cmpwi	cr0, r10, 172	/* Is dcbf? */
621	beq+	142f
622	cmpwi	cr0, r10, 1964	/* Is icbi? */
623	beq+	142f
624141:	mfspr	r10,SPRN_M_TW
625	b	DARFixed	/* Nope, go back to normal TLB processing */
626
627200:
628	/* concat physical page address(r11) and page offset(r10) */
629	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
630	b	201b
631
632202:
633	/* concat physical page address(r11) and page offset(r10) */
634	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
635	b	201b
636
637144:	mfspr	r10, SPRN_DSISR
638	rlwinm	r10, r10,0,7,5	/* Clear store bit for buggy dcbst insn */
639	mtspr	SPRN_DSISR, r10
640142:	/* continue, it was a dcbx, dcbi instruction. */
641	mfctr	r10
642	mtdar	r10			/* save ctr reg in DAR */
643	rlwinm	r10, r11, 24, 24, 28	/* offset into jump table for reg RB */
644	addi	r10, r10, 150f@l	/* add start of table */
645	mtctr	r10			/* load ctr with jump address */
646	xor	r10, r10, r10		/* sum starts at zero */
647	bctr				/* jump into table */
648150:
649	add	r10, r10, r0	;b	151f
650	add	r10, r10, r1	;b	151f
651	add	r10, r10, r2	;b	151f
652	add	r10, r10, r3	;b	151f
653	add	r10, r10, r4	;b	151f
654	add	r10, r10, r5	;b	151f
655	add	r10, r10, r6	;b	151f
656	add	r10, r10, r7	;b	151f
657	add	r10, r10, r8	;b	151f
658	add	r10, r10, r9	;b	151f
659	mtctr	r11	;b	154f	/* r10 needs special handling */
660	mtctr	r11	;b	153f	/* r11 needs special handling */
661	add	r10, r10, r12	;b	151f
662	add	r10, r10, r13	;b	151f
663	add	r10, r10, r14	;b	151f
664	add	r10, r10, r15	;b	151f
665	add	r10, r10, r16	;b	151f
666	add	r10, r10, r17	;b	151f
667	add	r10, r10, r18	;b	151f
668	add	r10, r10, r19	;b	151f
669	add	r10, r10, r20	;b	151f
670	add	r10, r10, r21	;b	151f
671	add	r10, r10, r22	;b	151f
672	add	r10, r10, r23	;b	151f
673	add	r10, r10, r24	;b	151f
674	add	r10, r10, r25	;b	151f
675	add	r10, r10, r26	;b	151f
676	add	r10, r10, r27	;b	151f
677	add	r10, r10, r28	;b	151f
678	add	r10, r10, r29	;b	151f
679	add	r10, r10, r30	;b	151f
680	add	r10, r10, r31
681151:
682	rlwinm. r11,r11,19,24,28	/* offset into jump table for reg RA */
683	beq	152f			/* if reg RA is zero, don't add it */
684	addi	r11, r11, 150b@l	/* add start of table */
685	mtctr	r11			/* load ctr with jump address */
686	rlwinm	r11,r11,0,16,10		/* make sure we don't execute this more than once */
687	bctr				/* jump into table */
688152:
689	mfdar	r11
690	mtctr	r11			/* restore ctr reg from DAR */
691	mtdar	r10			/* save fault EA to DAR */
692	mfspr	r10,SPRN_M_TW
693	b	DARFixed		/* Go back to normal TLB handling */
694
695	/* special handling for r10,r11 since these are modified already */
696153:	mfspr	r11, SPRN_SPRG_SCRATCH1	/* load r11 from SPRN_SPRG_SCRATCH1 */
697	add	r10, r10, r11	/* add it */
698	mfctr	r11		/* restore r11 */
699	b	151b
700154:	mfspr	r11, SPRN_SPRG_SCRATCH0	/* load r10 from SPRN_SPRG_SCRATCH0 */
701	add	r10, r10, r11	/* add it */
702	mfctr	r11		/* restore r11 */
703	b	151b
704
705/*
706 * This is where the main kernel code starts.
707 */
708start_here:
709	/* ptr to current */
710	lis	r2,init_task@h
711	ori	r2,r2,init_task@l
712
713	/* ptr to phys current thread */
714	tophys(r4,r2)
715	addi	r4,r4,THREAD	/* init task's THREAD */
716	mtspr	SPRN_SPRG_THREAD,r4
717
718	/* stack */
719	lis	r1,init_thread_union@ha
720	addi	r1,r1,init_thread_union@l
721	lis	r0, STACK_END_MAGIC@h
722	ori	r0, r0, STACK_END_MAGIC@l
723	stw	r0, 0(r1)
724	li	r0,0
725	stwu	r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
726
727	lis	r6, swapper_pg_dir@ha
728	tophys(r6,r6)
729	mtspr	SPRN_M_TWB, r6
730
731	bl	early_init	/* We have to do this with MMU on */
732
733/*
734 * Decide what sort of machine this is and initialize the MMU.
735 */
736#ifdef CONFIG_KASAN
737	bl	kasan_early_init
738#endif
739	li	r3,0
740	mr	r4,r31
741	bl	machine_init
742	bl	MMU_init
743
744/*
745 * Go back to running unmapped so we can load up new values
746 * and change to using our exception vectors.
747 * On the 8xx, all we have to do is invalidate the TLB to clear
748 * the old 8M byte TLB mappings and load the page table base register.
749 */
750	/* The right way to do this would be to track it down through
751	 * init's THREAD like the context switch code does, but this is
752	 * easier......until someone changes init's static structures.
753	 */
754	lis	r4,2f@h
755	ori	r4,r4,2f@l
756	tophys(r4,r4)
757	li	r3,MSR_KERNEL & ~(MSR_IR|MSR_DR)
758	mtspr	SPRN_SRR0,r4
759	mtspr	SPRN_SRR1,r3
760	rfi
761/* Load up the kernel context */
7622:
763	tlbia			/* Clear all TLB entries */
764	sync			/* wait for tlbia/tlbie to finish */
765
766	/* set up the PTE pointers for the Abatron bdiGDB.
767	*/
768	lis	r5, abatron_pteptrs@h
769	ori	r5, r5, abatron_pteptrs@l
770	stw	r5, 0xf0(0)	/* Must match your Abatron config file */
771	tophys(r5,r5)
772	lis	r6, swapper_pg_dir@h
773	ori	r6, r6, swapper_pg_dir@l
774	stw	r6, 0(r5)
775
776/* Now turn on the MMU for real! */
777	li	r4,MSR_KERNEL
778	lis	r3,start_kernel@h
779	ori	r3,r3,start_kernel@l
780	mtspr	SPRN_SRR0,r3
781	mtspr	SPRN_SRR1,r4
782	rfi			/* enable MMU and jump to start_kernel */
783
784/* Set up the initial MMU state so we can do the first level of
785 * kernel initialization.  This maps the first 8 MBytes of memory 1:1
786 * virtual to physical.  Also, set the cache mode since that is defined
787 * by TLB entries and perform any additional mapping (like of the IMMR).
788 * If configured to pin some TLBs, we pin the first 8 Mbytes of kernel,
789 * 24 Mbytes of data, and the 512k IMMR space.  Anything not covered by
790 * these mappings is mapped by page tables.
791 */
792initial_mmu:
793	li	r8, 0
794	mtspr	SPRN_MI_CTR, r8		/* remove PINNED ITLB entries */
795	lis	r10, MD_RESETVAL@h
796#ifndef CONFIG_8xx_COPYBACK
797	oris	r10, r10, MD_WTDEF@h
798#endif
799	mtspr	SPRN_MD_CTR, r10	/* remove PINNED DTLB entries */
800
801	tlbia			/* Invalidate all TLB entries */
802#ifdef CONFIG_PIN_TLB_DATA
803	oris	r10, r10, MD_RSV4I@h
804	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
805#endif
806
807	lis	r8, MI_APG_INIT@h	/* Set protection modes */
808	ori	r8, r8, MI_APG_INIT@l
809	mtspr	SPRN_MI_AP, r8
810	lis	r8, MD_APG_INIT@h
811	ori	r8, r8, MD_APG_INIT@l
812	mtspr	SPRN_MD_AP, r8
813
814	/* Map a 512k page for the IMMR to get the processor
815	 * internal registers (among other things).
816	 */
817#ifdef CONFIG_PIN_TLB_IMMR
818	oris	r10, r10, MD_RSV4I@h
819	ori	r10, r10, 0x1c00
820	mtspr	SPRN_MD_CTR, r10
821
822	mfspr	r9, 638			/* Get current IMMR */
823	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
824
825	lis	r8, VIRT_IMMR_BASE@h	/* Create vaddr for TLB */
826	ori	r8, r8, MD_EVALID	/* Mark it valid */
827	mtspr	SPRN_MD_EPN, r8
828	li	r8, MD_PS512K | MD_GUARDED	/* Set 512k byte page */
829	ori	r8, r8, MD_SVALID	/* Make it valid */
830	mtspr	SPRN_MD_TWC, r8
831	mr	r8, r9			/* Create paddr for TLB */
832	ori	r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
833	mtspr	SPRN_MD_RPN, r8
834#endif
835
836	/* Now map the lower RAM (up to 32 Mbytes) into the ITLB. */
837#ifdef CONFIG_PIN_TLB_TEXT
838	lis	r8, MI_RSV4I@h
839	ori	r8, r8, 0x1c00
840#endif
841	li	r9, 4				/* up to 4 pages of 8M */
842	mtctr	r9
843	lis	r9, KERNELBASE@h		/* Create vaddr for TLB */
844	li	r10, MI_PS8MEG | MI_SVALID	/* Set 8M byte page */
845	li	r11, MI_BOOTINIT		/* Create RPN for address 0 */
846	lis	r12, _einittext@h
847	ori	r12, r12, _einittext@l
8481:
849#ifdef CONFIG_PIN_TLB_TEXT
850	mtspr	SPRN_MI_CTR, r8	/* Set instruction MMU control */
851	addi	r8, r8, 0x100
852#endif
853
854	ori	r0, r9, MI_EVALID		/* Mark it valid */
855	mtspr	SPRN_MI_EPN, r0
856	mtspr	SPRN_MI_TWC, r10
857	mtspr	SPRN_MI_RPN, r11		/* Store TLB entry */
858	addis	r9, r9, 0x80
859	addis	r11, r11, 0x80
860
861	cmpl	cr0, r9, r12
862	bdnzf	gt, 1b
863
864	/* Since the cache is enabled according to the information we
865	 * just loaded into the TLB, invalidate and enable the caches here.
866	 * We should probably check/set other modes....later.
867	 */
868	lis	r8, IDC_INVALL@h
869	mtspr	SPRN_IC_CST, r8
870	mtspr	SPRN_DC_CST, r8
871	lis	r8, IDC_ENABLE@h
872	mtspr	SPRN_IC_CST, r8
873#ifdef CONFIG_8xx_COPYBACK
874	mtspr	SPRN_DC_CST, r8
875#else
876	/* For a debug option, I left this here to easily enable
877	 * the write through cache mode
878	 */
879	lis	r8, DC_SFWT@h
880	mtspr	SPRN_DC_CST, r8
881	lis	r8, IDC_ENABLE@h
882	mtspr	SPRN_DC_CST, r8
883#endif
884	/* Disable debug mode entry on breakpoints */
885	mfspr	r8, SPRN_DER
886#ifdef CONFIG_PERF_EVENTS
887	rlwinm	r8, r8, 0, ~0xc
888#else
889	rlwinm	r8, r8, 0, ~0x8
890#endif
891	mtspr	SPRN_DER, r8
892	blr
893
894
895/*
896 * We put a few things here that have to be page-aligned.
897 * This stuff goes at the beginning of the data segment,
898 * which is page-aligned.
899 */
900	.data
901	.globl	sdata
902sdata:
903	.globl	empty_zero_page
904	.align	PAGE_SHIFT
905empty_zero_page:
906	.space	PAGE_SIZE
907EXPORT_SYMBOL(empty_zero_page)
908
909	.globl	swapper_pg_dir
910swapper_pg_dir:
911	.space	PGD_TABLE_SIZE
912
913/* Room for two PTE table poiners, usually the kernel and current user
914 * pointer to their respective root page table (pgdir).
915 */
916	.globl	abatron_pteptrs
917abatron_pteptrs:
918	.space	8
919