xref: /openbmc/linux/arch/powerpc/kernel/head_8xx.S (revision 15e3ae36) 
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *  PowerPC version
4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 *  Low-level exception handlers and MMU support
8 *  rewritten by Paul Mackerras.
9 *    Copyright (C) 1996 Paul Mackerras.
10 *  MPC8xx modifications by Dan Malek
11 *    Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 *  This file contains low-level support and setup for PowerPC 8xx
14 *  embedded processors, including trap and interrupt dispatch.
15 */
16
17#include <linux/init.h>
18#include <linux/magic.h>
19#include <asm/processor.h>
20#include <asm/page.h>
21#include <asm/mmu.h>
22#include <asm/cache.h>
23#include <asm/pgtable.h>
24#include <asm/cputable.h>
25#include <asm/thread_info.h>
26#include <asm/ppc_asm.h>
27#include <asm/asm-offsets.h>
28#include <asm/ptrace.h>
29#include <asm/export.h>
30#include <asm/code-patching-asm.h>
31
32#include "head_32.h"
33
34#if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
35/* By simply checking Address >= 0x80000000, we know if its a kernel address */
36#define SIMPLE_KERNEL_ADDRESS		1
37#endif
38
39/*
40 * We need an ITLB miss handler for kernel addresses if:
41 * - Either we have modules
42 * - Or we have not pinned the first 8M
43 */
44#if defined(CONFIG_MODULES) || !defined(CONFIG_PIN_TLB_TEXT) || \
45    defined(CONFIG_DEBUG_PAGEALLOC)
46#define ITLB_MISS_KERNEL	1
47#endif
48
49/*
50 * Value for the bits that have fixed value in RPN entries.
51 * Also used for tagging DAR for DTLBerror.
52 */
53#define RPN_PATTERN	0x00f0
54
55#define PAGE_SHIFT_512K		19
56#define PAGE_SHIFT_8M		23
57
58	__HEAD
59_ENTRY(_stext);
60_ENTRY(_start);
61
62/* MPC8xx
63 * This port was done on an MBX board with an 860.  Right now I only
64 * support an ELF compressed (zImage) boot from EPPC-Bug because the
65 * code there loads up some registers before calling us:
66 *   r3: ptr to board info data
67 *   r4: initrd_start or if no initrd then 0
68 *   r5: initrd_end - unused if r4 is 0
69 *   r6: Start of command line string
70 *   r7: End of command line string
71 *
72 * I decided to use conditional compilation instead of checking PVR and
73 * adding more processor specific branches around code I don't need.
74 * Since this is an embedded processor, I also appreciate any memory
75 * savings I can get.
76 *
77 * The MPC8xx does not have any BATs, but it supports large page sizes.
78 * We first initialize the MMU to support 8M byte pages, then load one
79 * entry into each of the instruction and data TLBs to map the first
80 * 8M 1:1.  I also mapped an additional I/O space 1:1 so we can get to
81 * the "internal" processor registers before MMU_init is called.
82 *
83 *	-- Dan
84 */
85	.globl	__start
86__start:
87	mr	r31,r3			/* save device tree ptr */
88
89	/* We have to turn on the MMU right away so we get cache modes
90	 * set correctly.
91	 */
92	bl	initial_mmu
93
94/* We now have the lower 8 Meg mapped into TLB entries, and the caches
95 * ready to work.
96 */
97
98turn_on_mmu:
99	mfmsr	r0
100	ori	r0,r0,MSR_DR|MSR_IR
101	mtspr	SPRN_SRR1,r0
102	lis	r0,start_here@h
103	ori	r0,r0,start_here@l
104	mtspr	SPRN_SRR0,r0
105	rfi				/* enables MMU */
106
107
108#ifdef CONFIG_PERF_EVENTS
109	.align	4
110
111	.globl	itlb_miss_counter
112itlb_miss_counter:
113	.space	4
114
115	.globl	dtlb_miss_counter
116dtlb_miss_counter:
117	.space	4
118
119	.globl	instruction_counter
120instruction_counter:
121	.space	4
122#endif
123
124/* System reset */
125	EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
126
127/* Machine check */
128	. = 0x200
129MachineCheck:
130	EXCEPTION_PROLOG handle_dar_dsisr=1
131	save_dar_dsisr_on_stack r4, r5, r11
132	li	r6, RPN_PATTERN
133	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
134	addi r3,r1,STACK_FRAME_OVERHEAD
135	EXC_XFER_STD(0x200, machine_check_exception)
136
137/* External interrupt */
138	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
139
140/* Alignment exception */
141	. = 0x600
142Alignment:
143	EXCEPTION_PROLOG handle_dar_dsisr=1
144	save_dar_dsisr_on_stack r4, r5, r11
145	li	r6, RPN_PATTERN
146	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
147	addi	r3,r1,STACK_FRAME_OVERHEAD
148	b	.Lalignment_exception_ool
149
150/* Program check exception */
151	EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
152
153/* Decrementer */
154	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
155
156	/* With VMAP_STACK there's not enough room for this at 0x600 */
157	. = 0xa00
158.Lalignment_exception_ool:
159	EXC_XFER_STD(0x600, alignment_exception)
160
161/* System call */
162	. = 0xc00
163SystemCall:
164	SYSCALL_ENTRY	0xc00
165
166/* Single step - not used on 601 */
167	EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
168
169/* On the MPC8xx, this is a software emulation interrupt.  It occurs
170 * for all unimplemented and illegal instructions.
171 */
172	EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD)
173
174	. = 0x1100
175/*
176 * For the MPC8xx, this is a software tablewalk to load the instruction
177 * TLB.  The task switch loads the M_TWB register with the pointer to the first
178 * level table.
179 * If we discover there is no second level table (value is zero) or if there
180 * is an invalid pte, we load that into the TLB, which causes another fault
181 * into the TLB Error interrupt where we can handle such problems.
182 * We have to use the MD_xxx registers for the tablewalk because the
183 * equivalent MI_xxx registers only perform the attribute functions.
184 */
185
186#ifdef CONFIG_8xx_CPU15
187#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)	\
188	addi	addr, addr, PAGE_SIZE;	\
189	tlbie	addr;			\
190	addi	addr, addr, -(PAGE_SIZE << 1);	\
191	tlbie	addr;			\
192	addi	addr, addr, PAGE_SIZE
193#else
194#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)
195#endif
196
197InstructionTLBMiss:
198	mtspr	SPRN_SPRG_SCRATCH0, r10
199#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
200	mtspr	SPRN_SPRG_SCRATCH1, r11
201#endif
202
203	/* If we are faulting a kernel address, we have to use the
204	 * kernel page tables.
205	 */
206	mfspr	r10, SPRN_SRR0	/* Get effective address of fault */
207	INVALIDATE_ADJACENT_PAGES_CPU15(r10)
208	mtspr	SPRN_MD_EPN, r10
209	/* Only modules will cause ITLB Misses as we always
210	 * pin the first 8MB of kernel memory */
211#ifdef ITLB_MISS_KERNEL
212	mfcr	r11
213#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
214	cmpi	cr0, r10, 0	/* Address >= 0x80000000 */
215#else
216	rlwinm	r10, r10, 16, 0xfff8
217	cmpli	cr0, r10, PAGE_OFFSET@h
218#ifndef CONFIG_PIN_TLB_TEXT
219	/* It is assumed that kernel code fits into the first 32M */
2200:	cmpli	cr7, r10, (PAGE_OFFSET + 0x2000000)@h
221	patch_site	0b, patch__itlbmiss_linmem_top
222#endif
223#endif
224#endif
225	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
226#ifdef ITLB_MISS_KERNEL
227#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
228	bge+	3f
229#else
230	blt+	3f
231#endif
232#ifndef CONFIG_PIN_TLB_TEXT
233	blt	cr7, ITLBMissLinear
234#endif
235	rlwinm	r10, r10, 0, 20, 31
236	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
2373:
238#endif
239	lwz	r10, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
240	mtspr	SPRN_MI_TWC, r10	/* Set segment attributes */
241
242	mtspr	SPRN_MD_TWC, r10
243	mfspr	r10, SPRN_MD_TWC
244	lwz	r10, 0(r10)	/* Get the pte */
245#ifdef ITLB_MISS_KERNEL
246	mtcr	r11
247#endif
248#ifdef CONFIG_SWAP
249	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
250	and	r11, r11, r10
251	rlwimi	r10, r11, 0, _PAGE_PRESENT
252#endif
253	/* The Linux PTE won't go exactly into the MMU TLB.
254	 * Software indicator bits 20 and 23 must be clear.
255	 * Software indicator bits 22, 24, 25, 26, and 27 must be
256	 * set.  All other Linux PTE bits control the behavior
257	 * of the MMU.
258	 */
259	rlwinm	r10, r10, 0, ~0x0f00	/* Clear bits 20-23 */
260	rlwimi	r10, r10, 4, 0x0400	/* Copy _PAGE_EXEC into bit 21 */
261	ori	r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
262	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
263
264	/* Restore registers */
2650:	mfspr	r10, SPRN_SPRG_SCRATCH0
266#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
267	mfspr	r11, SPRN_SPRG_SCRATCH1
268#endif
269	rfi
270	patch_site	0b, patch__itlbmiss_exit_1
271
272#ifdef CONFIG_PERF_EVENTS
273	patch_site	0f, patch__itlbmiss_perf
2740:	lwz	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
275	addi	r10, r10, 1
276	stw	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
277	mfspr	r10, SPRN_SPRG_SCRATCH0
278#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
279	mfspr	r11, SPRN_SPRG_SCRATCH1
280#endif
281	rfi
282#endif
283
284#ifndef CONFIG_PIN_TLB_TEXT
285ITLBMissLinear:
286	mtcr	r11
287#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_ETEXT_SHIFT < 23
288	patch_site	0f, patch__itlbmiss_linmem_top8
289
290	mfspr	r10, SPRN_SRR0
2910:	subis	r11, r10, (PAGE_OFFSET - 0x80000000)@ha
292	rlwinm	r11, r11, 4, MI_PS8MEG ^ MI_PS512K
293	ori	r11, r11, MI_PS512K | MI_SVALID
294	rlwinm	r10, r10, 0, 0x0ff80000	/* 8xx supports max 256Mb RAM */
295#else
296	/* Set 8M byte page and mark it valid */
297	li	r11, MI_PS8MEG | MI_SVALID
298	rlwinm	r10, r10, 20, 0x0f800000	/* 8xx supports max 256Mb RAM */
299#endif
300	mtspr	SPRN_MI_TWC, r11
301	ori	r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
302			  _PAGE_PRESENT
303	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
304
3050:	mfspr	r10, SPRN_SPRG_SCRATCH0
306	mfspr	r11, SPRN_SPRG_SCRATCH1
307	rfi
308	patch_site	0b, patch__itlbmiss_exit_2
309#endif
310
311	. = 0x1200
312DataStoreTLBMiss:
313	mtspr	SPRN_DAR, r10
314	mtspr	SPRN_M_TW, r11
315	mfcr	r11
316
317	/* If we are faulting a kernel address, we have to use the
318	 * kernel page tables.
319	 */
320	mfspr	r10, SPRN_MD_EPN
321	rlwinm	r10, r10, 16, 0xfff8
322	cmpli	cr0, r10, PAGE_OFFSET@h
323#ifndef CONFIG_PIN_TLB_IMMR
324	cmpli	cr6, r10, VIRT_IMMR_BASE@h
325#endif
3260:	cmpli	cr7, r10, (PAGE_OFFSET + 0x2000000)@h
327	patch_site	0b, patch__dtlbmiss_linmem_top
328
329	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
330	blt+	3f
331#ifndef CONFIG_PIN_TLB_IMMR
3320:	beq-	cr6, DTLBMissIMMR
333	patch_site	0b, patch__dtlbmiss_immr_jmp
334#endif
335	blt	cr7, DTLBMissLinear
336	rlwinm	r10, r10, 0, 20, 31
337	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
3383:
339	mtcr	r11
340	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
341
342	mtspr	SPRN_MD_TWC, r11
343	mfspr	r10, SPRN_MD_TWC
344	lwz	r10, 0(r10)	/* Get the pte */
345
346	/* Insert the Guarded flag into the TWC from the Linux PTE.
347	 * It is bit 27 of both the Linux PTE and the TWC (at least
348	 * I got that right :-).  It will be better when we can put
349	 * this into the Linux pgd/pmd and load it in the operation
350	 * above.
351	 */
352	rlwimi	r11, r10, 0, _PAGE_GUARDED
353	mtspr	SPRN_MD_TWC, r11
354
355	/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
356	 * We also need to know if the insn is a load/store, so:
357	 * Clear _PAGE_PRESENT and load that which will
358	 * trap into DTLB Error with store bit set accordinly.
359	 */
360	/* PRESENT=0x1, ACCESSED=0x20
361	 * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5));
362	 * r10 = (r10 & ~PRESENT) | r11;
363	 */
364#ifdef CONFIG_SWAP
365	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
366	and	r11, r11, r10
367	rlwimi	r10, r11, 0, _PAGE_PRESENT
368#endif
369	/* The Linux PTE won't go exactly into the MMU TLB.
370	 * Software indicator bits 24, 25, 26, and 27 must be
371	 * set.  All other Linux PTE bits control the behavior
372	 * of the MMU.
373	 */
374	li	r11, RPN_PATTERN
375	rlwimi	r10, r11, 0, 24, 27	/* Set 24-27 */
376	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
377
378	/* Restore registers */
379
3800:	mfspr	r10, SPRN_DAR
381	mtspr	SPRN_DAR, r11	/* Tag DAR */
382	mfspr	r11, SPRN_M_TW
383	rfi
384	patch_site	0b, patch__dtlbmiss_exit_1
385
386DTLBMissIMMR:
387	mtcr	r11
388	/* Set 512k byte guarded page and mark it valid */
389	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
390	mtspr	SPRN_MD_TWC, r10
391	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
392	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
393	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
394			  _PAGE_PRESENT | _PAGE_NO_CACHE
395	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
396
397	li	r11, RPN_PATTERN
398
3990:	mfspr	r10, SPRN_DAR
400	mtspr	SPRN_DAR, r11	/* Tag DAR */
401	mfspr	r11, SPRN_M_TW
402	rfi
403	patch_site	0b, patch__dtlbmiss_exit_2
404
405DTLBMissLinear:
406	mtcr	r11
407	rlwinm	r10, r10, 20, 0x0f800000	/* 8xx supports max 256Mb RAM */
408#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_DATA_SHIFT < 23
409	patch_site	0f, patch__dtlbmiss_romem_top8
410
4110:	subis	r11, r10, (PAGE_OFFSET - 0x80000000)@ha
412	rlwinm	r11, r11, 0, 0xff800000
413	neg	r10, r11
414	or	r11, r11, r10
415	rlwinm	r11, r11, 4, MI_PS8MEG ^ MI_PS512K
416	ori	r11, r11, MI_PS512K | MI_SVALID
417	mfspr	r10, SPRN_MD_EPN
418	rlwinm	r10, r10, 0, 0x0ff80000	/* 8xx supports max 256Mb RAM */
419#else
420	/* Set 8M byte page and mark it valid */
421	li	r11, MD_PS8MEG | MD_SVALID
422#endif
423	mtspr	SPRN_MD_TWC, r11
424#ifdef CONFIG_STRICT_KERNEL_RWX
425	patch_site	0f, patch__dtlbmiss_romem_top
426
4270:	subis	r11, r10, 0
428	rlwimi	r10, r11, 11, _PAGE_RO
429#endif
430	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
431			  _PAGE_PRESENT
432	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
433
434	li	r11, RPN_PATTERN
435
4360:	mfspr	r10, SPRN_DAR
437	mtspr	SPRN_DAR, r11	/* Tag DAR */
438	mfspr	r11, SPRN_M_TW
439	rfi
440	patch_site	0b, patch__dtlbmiss_exit_3
441
442/* This is an instruction TLB error on the MPC8xx.  This could be due
443 * to many reasons, such as executing guarded memory or illegal instruction
444 * addresses.  There is nothing to do but handle a big time error fault.
445 */
446	. = 0x1300
447InstructionTLBError:
448	EXCEPTION_PROLOG
449	mr	r4,r12
450	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
451	andis.	r10,r9,SRR1_ISI_NOPT@h
452	beq+	.Litlbie
453	tlbie	r4
454	/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
455.Litlbie:
456	stw	r4, _DAR(r11)
457	EXC_XFER_LITE(0x400, handle_page_fault)
458
459/* This is the data TLB error on the MPC8xx.  This could be due to
460 * many reasons, including a dirty update to a pte.  We bail out to
461 * a higher level function that can handle it.
462 */
463	. = 0x1400
464DataTLBError:
465	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
466	mfspr	r11, SPRN_DAR
467	cmpwi	cr1, r11, RPN_PATTERN
468	beq-	cr1, FixupDAR	/* must be a buggy dcbX, icbi insn. */
469DARFixed:/* Return from dcbx instruction bug workaround */
470#ifdef CONFIG_VMAP_STACK
471	li	r11, RPN_PATTERN
472	mtspr	SPRN_DAR, r11	/* Tag DAR, to be used in DTLB Error */
473#endif
474	EXCEPTION_PROLOG_1
475	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
476	get_and_save_dar_dsisr_on_stack r4, r5, r11
477	andis.	r10,r5,DSISR_NOHPTE@h
478	beq+	.Ldtlbie
479	tlbie	r4
480.Ldtlbie:
481#ifndef CONFIG_VMAP_STACK
482	li	r10,RPN_PATTERN
483	mtspr	SPRN_DAR,r10	/* Tag DAR, to be used in DTLB Error */
484#endif
485	/* 0x300 is DataAccess exception, needed by bad_page_fault() */
486	EXC_XFER_LITE(0x300, handle_page_fault)
487
488/* Called from DataStoreTLBMiss when perf TLB misses events are activated */
489#ifdef CONFIG_PERF_EVENTS
490	patch_site	0f, patch__dtlbmiss_perf
4910:	lwz	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
492	addi	r10, r10, 1
493	stw	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
494	mfspr	r10, SPRN_DAR
495	mtspr	SPRN_DAR, r11	/* Tag DAR */
496	mfspr	r11, SPRN_M_TW
497	rfi
498#endif
499
500stack_overflow:
501	vmap_stack_overflow_exception
502
503/* On the MPC8xx, these next four traps are used for development
504 * support of breakpoints and such.  Someday I will get around to
505 * using them.
506 */
507do_databreakpoint:
508	EXCEPTION_PROLOG_1
509	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
510	addi	r3,r1,STACK_FRAME_OVERHEAD
511	mfspr	r4,SPRN_BAR
512	stw	r4,_DAR(r11)
513#ifdef CONFIG_VMAP_STACK
514	lwz	r5,_DSISR(r11)
515#else
516	mfspr	r5,SPRN_DSISR
517#endif
518	EXC_XFER_STD(0x1c00, do_break)
519
520	. = 0x1c00
521DataBreakpoint:
522	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
523	mfspr	r11, SPRN_SRR0
524	cmplwi	cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l
525	cmplwi	cr7, r11, (.Litlbie - PAGE_OFFSET)@l
526	cror	4*cr1+eq, 4*cr1+eq, 4*cr7+eq
527	bne	cr1, do_databreakpoint
528	mtcr	r10
529	mfspr	r10, SPRN_SPRG_SCRATCH0
530	mfspr	r11, SPRN_SPRG_SCRATCH1
531	rfi
532
533#ifdef CONFIG_PERF_EVENTS
534	. = 0x1d00
535InstructionBreakpoint:
536	mtspr	SPRN_SPRG_SCRATCH0, r10
537	lwz	r10, (instruction_counter - PAGE_OFFSET)@l(0)
538	addi	r10, r10, -1
539	stw	r10, (instruction_counter - PAGE_OFFSET)@l(0)
540	lis	r10, 0xffff
541	ori	r10, r10, 0x01
542	mtspr	SPRN_COUNTA, r10
543	mfspr	r10, SPRN_SPRG_SCRATCH0
544	rfi
545#else
546	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
547#endif
548	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
549	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
550
551	. = 0x2000
552
553/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
554 * by decoding the registers used by the dcbx instruction and adding them.
555 * DAR is set to the calculated address.
556 */
557FixupDAR:/* Entry point for dcbx workaround. */
558	mtspr	SPRN_M_TW, r10
559	/* fetch instruction from memory. */
560	mfspr	r10, SPRN_SRR0
561	mtspr	SPRN_MD_EPN, r10
562	rlwinm	r11, r10, 16, 0xfff8
563	cmpli	cr1, r11, PAGE_OFFSET@h
564	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
565	blt+	cr1, 3f
566	rlwinm	r11, r10, 16, 0xfff8
567
5680:	cmpli	cr7, r11, (PAGE_OFFSET + 0x1800000)@h
569	patch_site	0b, patch__fixupdar_linmem_top
570
571	/* create physical page address from effective address */
572	tophys(r11, r10)
573	blt-	cr7, 201f
574	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
575	rlwinm	r11, r11, 0, 20, 31
576	oris	r11, r11, (swapper_pg_dir - PAGE_OFFSET)@ha
5773:
578	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
579	mtspr	SPRN_MD_TWC, r11
580	mtcrf	0x01, r11
581	mfspr	r11, SPRN_MD_TWC
582	lwz	r11, 0(r11)	/* Get the pte */
583	bt	28,200f		/* bit 28 = Large page (8M) */
584	bt	29,202f		/* bit 29 = Large page (8M or 512K) */
585	/* concat physical page address(r11) and page offset(r10) */
586	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT, 31
587201:	lwz	r11,0(r11)
588/* Check if it really is a dcbx instruction. */
589/* dcbt and dcbtst does not generate DTLB Misses/Errors,
590 * no need to include them here */
591	xoris	r10, r11, 0x7c00	/* check if major OP code is 31 */
592	rlwinm	r10, r10, 0, 21, 5
593	cmpwi	cr1, r10, 2028	/* Is dcbz? */
594	beq+	cr1, 142f
595	cmpwi	cr1, r10, 940	/* Is dcbi? */
596	beq+	cr1, 142f
597	cmpwi	cr1, r10, 108	/* Is dcbst? */
598	beq+	cr1, 144f		/* Fix up store bit! */
599	cmpwi	cr1, r10, 172	/* Is dcbf? */
600	beq+	cr1, 142f
601	cmpwi	cr1, r10, 1964	/* Is icbi? */
602	beq+	cr1, 142f
603141:	mfspr	r10,SPRN_M_TW
604	b	DARFixed	/* Nope, go back to normal TLB processing */
605
606200:
607	/* concat physical page address(r11) and page offset(r10) */
608	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
609	b	201b
610
611202:
612	/* concat physical page address(r11) and page offset(r10) */
613	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
614	b	201b
615
616144:	mfspr	r10, SPRN_DSISR
617	rlwinm	r10, r10,0,7,5	/* Clear store bit for buggy dcbst insn */
618	mtspr	SPRN_DSISR, r10
619142:	/* continue, it was a dcbx, dcbi instruction. */
620	mfctr	r10
621	mtdar	r10			/* save ctr reg in DAR */
622	rlwinm	r10, r11, 24, 24, 28	/* offset into jump table for reg RB */
623	addi	r10, r10, 150f@l	/* add start of table */
624	mtctr	r10			/* load ctr with jump address */
625	xor	r10, r10, r10		/* sum starts at zero */
626	bctr				/* jump into table */
627150:
628	add	r10, r10, r0	;b	151f
629	add	r10, r10, r1	;b	151f
630	add	r10, r10, r2	;b	151f
631	add	r10, r10, r3	;b	151f
632	add	r10, r10, r4	;b	151f
633	add	r10, r10, r5	;b	151f
634	add	r10, r10, r6	;b	151f
635	add	r10, r10, r7	;b	151f
636	add	r10, r10, r8	;b	151f
637	add	r10, r10, r9	;b	151f
638	mtctr	r11	;b	154f	/* r10 needs special handling */
639	mtctr	r11	;b	153f	/* r11 needs special handling */
640	add	r10, r10, r12	;b	151f
641	add	r10, r10, r13	;b	151f
642	add	r10, r10, r14	;b	151f
643	add	r10, r10, r15	;b	151f
644	add	r10, r10, r16	;b	151f
645	add	r10, r10, r17	;b	151f
646	add	r10, r10, r18	;b	151f
647	add	r10, r10, r19	;b	151f
648	add	r10, r10, r20	;b	151f
649	add	r10, r10, r21	;b	151f
650	add	r10, r10, r22	;b	151f
651	add	r10, r10, r23	;b	151f
652	add	r10, r10, r24	;b	151f
653	add	r10, r10, r25	;b	151f
654	add	r10, r10, r26	;b	151f
655	add	r10, r10, r27	;b	151f
656	add	r10, r10, r28	;b	151f
657	add	r10, r10, r29	;b	151f
658	add	r10, r10, r30	;b	151f
659	add	r10, r10, r31
660151:
661	rlwinm	r11,r11,19,24,28	/* offset into jump table for reg RA */
662	cmpwi	cr1, r11, 0
663	beq	cr1, 152f		/* if reg RA is zero, don't add it */
664	addi	r11, r11, 150b@l	/* add start of table */
665	mtctr	r11			/* load ctr with jump address */
666	rlwinm	r11,r11,0,16,10		/* make sure we don't execute this more than once */
667	bctr				/* jump into table */
668152:
669	mfdar	r11
670	mtctr	r11			/* restore ctr reg from DAR */
671#ifdef CONFIG_VMAP_STACK
672	mfspr	r11, SPRN_SPRG_THREAD
673	stw	r10, DAR(r11)
674	mfspr	r10, SPRN_DSISR
675	stw	r10, DSISR(r11)
676#else
677	mtdar	r10			/* save fault EA to DAR */
678#endif
679	mfspr	r10,SPRN_M_TW
680	b	DARFixed		/* Go back to normal TLB handling */
681
682	/* special handling for r10,r11 since these are modified already */
683153:	mfspr	r11, SPRN_SPRG_SCRATCH1	/* load r11 from SPRN_SPRG_SCRATCH1 */
684	add	r10, r10, r11	/* add it */
685	mfctr	r11		/* restore r11 */
686	b	151b
687154:	mfspr	r11, SPRN_SPRG_SCRATCH0	/* load r10 from SPRN_SPRG_SCRATCH0 */
688	add	r10, r10, r11	/* add it */
689	mfctr	r11		/* restore r11 */
690	b	151b
691
692/*
693 * This is where the main kernel code starts.
694 */
695start_here:
696	/* ptr to current */
697	lis	r2,init_task@h
698	ori	r2,r2,init_task@l
699
700	/* ptr to phys current thread */
701	tophys(r4,r2)
702	addi	r4,r4,THREAD	/* init task's THREAD */
703	mtspr	SPRN_SPRG_THREAD,r4
704
705	/* stack */
706	lis	r1,init_thread_union@ha
707	addi	r1,r1,init_thread_union@l
708	lis	r0, STACK_END_MAGIC@h
709	ori	r0, r0, STACK_END_MAGIC@l
710	stw	r0, 0(r1)
711	li	r0,0
712	stwu	r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
713
714	lis	r6, swapper_pg_dir@ha
715	tophys(r6,r6)
716	mtspr	SPRN_M_TWB, r6
717
718	bl	early_init	/* We have to do this with MMU on */
719
720/*
721 * Decide what sort of machine this is and initialize the MMU.
722 */
723#ifdef CONFIG_KASAN
724	bl	kasan_early_init
725#endif
726	li	r3,0
727	mr	r4,r31
728	bl	machine_init
729	bl	MMU_init
730
731/*
732 * Go back to running unmapped so we can load up new values
733 * and change to using our exception vectors.
734 * On the 8xx, all we have to do is invalidate the TLB to clear
735 * the old 8M byte TLB mappings and load the page table base register.
736 */
737	/* The right way to do this would be to track it down through
738	 * init's THREAD like the context switch code does, but this is
739	 * easier......until someone changes init's static structures.
740	 */
741	lis	r4,2f@h
742	ori	r4,r4,2f@l
743	tophys(r4,r4)
744	li	r3,MSR_KERNEL & ~(MSR_IR|MSR_DR)
745	mtspr	SPRN_SRR0,r4
746	mtspr	SPRN_SRR1,r3
747	rfi
748/* Load up the kernel context */
7492:
750	tlbia			/* Clear all TLB entries */
751	sync			/* wait for tlbia/tlbie to finish */
752
753	/* set up the PTE pointers for the Abatron bdiGDB.
754	*/
755	lis	r5, abatron_pteptrs@h
756	ori	r5, r5, abatron_pteptrs@l
757	stw	r5, 0xf0(0)	/* Must match your Abatron config file */
758	tophys(r5,r5)
759	lis	r6, swapper_pg_dir@h
760	ori	r6, r6, swapper_pg_dir@l
761	stw	r6, 0(r5)
762
763/* Now turn on the MMU for real! */
764	li	r4,MSR_KERNEL
765	lis	r3,start_kernel@h
766	ori	r3,r3,start_kernel@l
767	mtspr	SPRN_SRR0,r3
768	mtspr	SPRN_SRR1,r4
769	rfi			/* enable MMU and jump to start_kernel */
770
771/* Set up the initial MMU state so we can do the first level of
772 * kernel initialization.  This maps the first 8 MBytes of memory 1:1
773 * virtual to physical.  Also, set the cache mode since that is defined
774 * by TLB entries and perform any additional mapping (like of the IMMR).
775 * If configured to pin some TLBs, we pin the first 8 Mbytes of kernel,
776 * 24 Mbytes of data, and the 512k IMMR space.  Anything not covered by
777 * these mappings is mapped by page tables.
778 */
779initial_mmu:
780	li	r8, 0
781	mtspr	SPRN_MI_CTR, r8		/* remove PINNED ITLB entries */
782	lis	r10, MD_RESETVAL@h
783#ifndef CONFIG_8xx_COPYBACK
784	oris	r10, r10, MD_WTDEF@h
785#endif
786	mtspr	SPRN_MD_CTR, r10	/* remove PINNED DTLB entries */
787
788	tlbia			/* Invalidate all TLB entries */
789#ifdef CONFIG_PIN_TLB_DATA
790	oris	r10, r10, MD_RSV4I@h
791	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
792#endif
793
794	lis	r8, MI_APG_INIT@h	/* Set protection modes */
795	ori	r8, r8, MI_APG_INIT@l
796	mtspr	SPRN_MI_AP, r8
797	lis	r8, MD_APG_INIT@h
798	ori	r8, r8, MD_APG_INIT@l
799	mtspr	SPRN_MD_AP, r8
800
801	/* Map a 512k page for the IMMR to get the processor
802	 * internal registers (among other things).
803	 */
804#ifdef CONFIG_PIN_TLB_IMMR
805	oris	r10, r10, MD_RSV4I@h
806	ori	r10, r10, 0x1c00
807	mtspr	SPRN_MD_CTR, r10
808
809	mfspr	r9, 638			/* Get current IMMR */
810	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
811
812	lis	r8, VIRT_IMMR_BASE@h	/* Create vaddr for TLB */
813	ori	r8, r8, MD_EVALID	/* Mark it valid */
814	mtspr	SPRN_MD_EPN, r8
815	li	r8, MD_PS512K | MD_GUARDED	/* Set 512k byte page */
816	ori	r8, r8, MD_SVALID	/* Make it valid */
817	mtspr	SPRN_MD_TWC, r8
818	mr	r8, r9			/* Create paddr for TLB */
819	ori	r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
820	mtspr	SPRN_MD_RPN, r8
821#endif
822
823	/* Now map the lower RAM (up to 32 Mbytes) into the ITLB. */
824#ifdef CONFIG_PIN_TLB_TEXT
825	lis	r8, MI_RSV4I@h
826	ori	r8, r8, 0x1c00
827#endif
828	li	r9, 4				/* up to 4 pages of 8M */
829	mtctr	r9
830	lis	r9, KERNELBASE@h		/* Create vaddr for TLB */
831	li	r10, MI_PS8MEG | MI_SVALID	/* Set 8M byte page */
832	li	r11, MI_BOOTINIT		/* Create RPN for address 0 */
833	lis	r12, _einittext@h
834	ori	r12, r12, _einittext@l
8351:
836#ifdef CONFIG_PIN_TLB_TEXT
837	mtspr	SPRN_MI_CTR, r8	/* Set instruction MMU control */
838	addi	r8, r8, 0x100
839#endif
840
841	ori	r0, r9, MI_EVALID		/* Mark it valid */
842	mtspr	SPRN_MI_EPN, r0
843	mtspr	SPRN_MI_TWC, r10
844	mtspr	SPRN_MI_RPN, r11		/* Store TLB entry */
845	addis	r9, r9, 0x80
846	addis	r11, r11, 0x80
847
848	cmpl	cr0, r9, r12
849	bdnzf	gt, 1b
850
851	/* Since the cache is enabled according to the information we
852	 * just loaded into the TLB, invalidate and enable the caches here.
853	 * We should probably check/set other modes....later.
854	 */
855	lis	r8, IDC_INVALL@h
856	mtspr	SPRN_IC_CST, r8
857	mtspr	SPRN_DC_CST, r8
858	lis	r8, IDC_ENABLE@h
859	mtspr	SPRN_IC_CST, r8
860#ifdef CONFIG_8xx_COPYBACK
861	mtspr	SPRN_DC_CST, r8
862#else
863	/* For a debug option, I left this here to easily enable
864	 * the write through cache mode
865	 */
866	lis	r8, DC_SFWT@h
867	mtspr	SPRN_DC_CST, r8
868	lis	r8, IDC_ENABLE@h
869	mtspr	SPRN_DC_CST, r8
870#endif
871	/* Disable debug mode entry on breakpoints */
872	mfspr	r8, SPRN_DER
873#ifdef CONFIG_PERF_EVENTS
874	rlwinm	r8, r8, 0, ~0xc
875#else
876	rlwinm	r8, r8, 0, ~0x8
877#endif
878	mtspr	SPRN_DER, r8
879	blr
880
881
882/*
883 * We put a few things here that have to be page-aligned.
884 * This stuff goes at the beginning of the data segment,
885 * which is page-aligned.
886 */
887	.data
888	.globl	sdata
889sdata:
890	.globl	empty_zero_page
891	.align	PAGE_SHIFT
892empty_zero_page:
893	.space	PAGE_SIZE
894EXPORT_SYMBOL(empty_zero_page)
895
896	.globl	swapper_pg_dir
897swapper_pg_dir:
898	.space	PGD_TABLE_SIZE
899
900/* Room for two PTE table poiners, usually the kernel and current user
901 * pointer to their respective root page table (pgdir).
902 */
903	.globl	abatron_pteptrs
904abatron_pteptrs:
905	.space	8
906