xref: /openbmc/linux/arch/arm/boot/compressed/head.S (revision 78560d41) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *  linux/arch/arm/boot/compressed/head.S
4 *
5 *  Copyright (C) 1996-2002 Russell King
6 *  Copyright (C) 2004 Hyok S. Choi (MPU support)
7 */
8#include <linux/linkage.h>
9#include <asm/assembler.h>
10#include <asm/v7m.h>
11
12#include "efi-header.S"
13
14 AR_CLASS(	.arch	armv7-a	)
15 M_CLASS(	.arch	armv7-m	)
16
17/*
18 * Debugging stuff
19 *
20 * Note that these macros must not contain any code which is not
21 * 100% relocatable.  Any attempt to do so will result in a crash.
22 * Please select one of the following when turning on debugging.
23 */
24#ifdef DEBUG
25
26#if defined(CONFIG_DEBUG_ICEDCC)
27
28#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7)
29		.macro	loadsp, rb, tmp1, tmp2
30		.endm
31		.macro	writeb, ch, rb
32		mcr	p14, 0, \ch, c0, c5, 0
33		.endm
34#elif defined(CONFIG_CPU_XSCALE)
35		.macro	loadsp, rb, tmp1, tmp2
36		.endm
37		.macro	writeb, ch, rb
38		mcr	p14, 0, \ch, c8, c0, 0
39		.endm
40#else
41		.macro	loadsp, rb, tmp1, tmp2
42		.endm
43		.macro	writeb, ch, rb
44		mcr	p14, 0, \ch, c1, c0, 0
45		.endm
46#endif
47
48#else
49
50#include CONFIG_DEBUG_LL_INCLUDE
51
52		.macro	writeb,	ch, rb
53		senduart \ch, \rb
54		.endm
55
56#if defined(CONFIG_ARCH_SA1100)
57		.macro	loadsp, rb, tmp1, tmp2
58		mov	\rb, #0x80000000	@ physical base address
59#ifdef CONFIG_DEBUG_LL_SER3
60		add	\rb, \rb, #0x00050000	@ Ser3
61#else
62		add	\rb, \rb, #0x00010000	@ Ser1
63#endif
64		.endm
65#else
66		.macro	loadsp,	rb, tmp1, tmp2
67		addruart \rb, \tmp1, \tmp2
68		.endm
69#endif
70#endif
71#endif
72
73		.macro	kputc,val
74		mov	r0, \val
75		bl	putc
76		.endm
77
78		.macro	kphex,val,len
79		mov	r0, \val
80		mov	r1, #\len
81		bl	phex
82		.endm
83
84		.macro	debug_reloc_start
85#ifdef DEBUG
86		kputc	#'\n'
87		kphex	r6, 8		/* processor id */
88		kputc	#':'
89		kphex	r7, 8		/* architecture id */
90#ifdef CONFIG_CPU_CP15
91		kputc	#':'
92		mrc	p15, 0, r0, c1, c0
93		kphex	r0, 8		/* control reg */
94#endif
95		kputc	#'\n'
96		kphex	r5, 8		/* decompressed kernel start */
97		kputc	#'-'
98		kphex	r9, 8		/* decompressed kernel end  */
99		kputc	#'>'
100		kphex	r4, 8		/* kernel execution address */
101		kputc	#'\n'
102#endif
103		.endm
104
105		.macro	debug_reloc_end
106#ifdef DEBUG
107		kphex	r5, 8		/* end of kernel */
108		kputc	#'\n'
109		mov	r0, r4
110		bl	memdump		/* dump 256 bytes at start of kernel */
111#endif
112		.endm
113
114		/*
115		 * Debug kernel copy by printing the memory addresses involved
116		 */
117		.macro dbgkc, begin, end, cbegin, cend
118#ifdef DEBUG
119		kputc   #'\n'
120		kputc   #'C'
121		kputc   #':'
122		kputc   #'0'
123		kputc   #'x'
124		kphex   \begin, 8	/* Start of compressed kernel */
125		kputc	#'-'
126		kputc	#'0'
127		kputc	#'x'
128		kphex	\end, 8		/* End of compressed kernel */
129		kputc	#'-'
130		kputc	#'>'
131		kputc   #'0'
132		kputc   #'x'
133		kphex   \cbegin, 8	/* Start of kernel copy */
134		kputc	#'-'
135		kputc	#'0'
136		kputc	#'x'
137		kphex	\cend, 8	/* End of kernel copy */
138		kputc	#'\n'
139		kputc	#'\r'
140#endif
141		.endm
142
143		.macro	enable_cp15_barriers, reg
144		mrc	p15, 0, \reg, c1, c0, 0	@ read SCTLR
145		tst	\reg, #(1 << 5)		@ CP15BEN bit set?
146		bne	.L_\@
147		orr	\reg, \reg, #(1 << 5)	@ CP15 barrier instructions
148		mcr	p15, 0, \reg, c1, c0, 0	@ write SCTLR
149 ARM(		.inst   0xf57ff06f		@ v7+ isb	)
150 THUMB(		isb						)
151.L_\@:
152		.endm
153
154		.section ".start", "ax"
155/*
156 * sort out different calling conventions
157 */
158		.align
159		/*
160		 * Always enter in ARM state for CPUs that support the ARM ISA.
161		 * As of today (2014) that's exactly the members of the A and R
162		 * classes.
163		 */
164 AR_CLASS(	.arm	)
165start:
166		.type	start,#function
167		/*
168		 * These 7 nops along with the 1 nop immediately below for
169		 * !THUMB2 form 8 nops that make the compressed kernel bootable
170		 * on legacy ARM systems that were assuming the kernel in a.out
171		 * binary format. The boot loaders on these systems would
172		 * jump 32 bytes into the image to skip the a.out header.
173		 * with these 8 nops filling exactly 32 bytes, things still
174		 * work as expected on these legacy systems. Thumb2 mode keeps
175		 * 7 of the nops as it turns out that some boot loaders
176		 * were patching the initial instructions of the kernel, i.e
177		 * had started to exploit this "patch area".
178		 */
179		.rept	7
180		__nop
181		.endr
182#ifndef CONFIG_THUMB2_KERNEL
183		__nop
184#else
185 AR_CLASS(	sub	pc, pc, #3	)	@ A/R: switch to Thumb2 mode
186  M_CLASS(	nop.w			)	@ M: already in Thumb2 mode
187		.thumb
188#endif
189		W(b)	1f
190
191		.word	_magic_sig	@ Magic numbers to help the loader
192		.word	_magic_start	@ absolute load/run zImage address
193		.word	_magic_end	@ zImage end address
194		.word	0x04030201	@ endianness flag
195		.word	0x45454545	@ another magic number to indicate
196		.word	_magic_table	@ additional data table
197
198		__EFI_HEADER
1991:
200 ARM_BE8(	setend	be		)	@ go BE8 if compiled for BE8
201 AR_CLASS(	mrs	r9, cpsr	)
202#ifdef CONFIG_ARM_VIRT_EXT
203		bl	__hyp_stub_install	@ get into SVC mode, reversibly
204#endif
205		mov	r7, r1			@ save architecture ID
206		mov	r8, r2			@ save atags pointer
207
208#ifndef CONFIG_CPU_V7M
209		/*
210		 * Booting from Angel - need to enter SVC mode and disable
211		 * FIQs/IRQs (numeric definitions from angel arm.h source).
212		 * We only do this if we were in user mode on entry.
213		 */
214		mrs	r2, cpsr		@ get current mode
215		tst	r2, #3			@ not user?
216		bne	not_angel
217		mov	r0, #0x17		@ angel_SWIreason_EnterSVC
218 ARM(		swi	0x123456	)	@ angel_SWI_ARM
219 THUMB(		svc	0xab		)	@ angel_SWI_THUMB
220not_angel:
221		safe_svcmode_maskall r0
222		msr	spsr_cxsf, r9		@ Save the CPU boot mode in
223						@ SPSR
224#endif
225		/*
226		 * Note that some cache flushing and other stuff may
227		 * be needed here - is there an Angel SWI call for this?
228		 */
229
230		/*
231		 * some architecture specific code can be inserted
232		 * by the linker here, but it should preserve r7, r8, and r9.
233		 */
234
235		.text
236
237#ifdef CONFIG_AUTO_ZRELADDR
238		/*
239		 * Find the start of physical memory.  As we are executing
240		 * without the MMU on, we are in the physical address space.
241		 * We just need to get rid of any offset by aligning the
242		 * address.
243		 *
244		 * This alignment is a balance between the requirements of
245		 * different platforms - we have chosen 128MB to allow
246		 * platforms which align the start of their physical memory
247		 * to 128MB to use this feature, while allowing the zImage
248		 * to be placed within the first 128MB of memory on other
249		 * platforms.  Increasing the alignment means we place
250		 * stricter alignment requirements on the start of physical
251		 * memory, but relaxing it means that we break people who
252		 * are already placing their zImage in (eg) the top 64MB
253		 * of this range.
254		 */
255		mov	r4, pc
256		and	r4, r4, #0xf8000000
257		/* Determine final kernel image address. */
258		add	r4, r4, #TEXT_OFFSET
259#else
260		ldr	r4, =zreladdr
261#endif
262
263		/*
264		 * Set up a page table only if it won't overwrite ourself.
265		 * That means r4 < pc || r4 - 16k page directory > &_end.
266		 * Given that r4 > &_end is most unfrequent, we add a rough
267		 * additional 1MB of room for a possible appended DTB.
268		 */
269		mov	r0, pc
270		cmp	r0, r4
271		ldrcc	r0, LC0+32
272		addcc	r0, r0, pc
273		cmpcc	r4, r0
274		orrcc	r4, r4, #1		@ remember we skipped cache_on
275		blcs	cache_on
276
277restart:	adr	r0, LC0
278		ldmia	r0, {r1, r2, r3, r6, r10, r11, r12}
279		ldr	sp, [r0, #28]
280
281		/*
282		 * We might be running at a different address.  We need
283		 * to fix up various pointers.
284		 */
285		sub	r0, r0, r1		@ calculate the delta offset
286		add	r6, r6, r0		@ _edata
287		add	r10, r10, r0		@ inflated kernel size location
288
289		/*
290		 * The kernel build system appends the size of the
291		 * decompressed kernel at the end of the compressed data
292		 * in little-endian form.
293		 */
294		ldrb	r9, [r10, #0]
295		ldrb	lr, [r10, #1]
296		orr	r9, r9, lr, lsl #8
297		ldrb	lr, [r10, #2]
298		ldrb	r10, [r10, #3]
299		orr	r9, r9, lr, lsl #16
300		orr	r9, r9, r10, lsl #24
301
302#ifndef CONFIG_ZBOOT_ROM
303		/* malloc space is above the relocated stack (64k max) */
304		add	sp, sp, r0
305		add	r10, sp, #0x10000
306#else
307		/*
308		 * With ZBOOT_ROM the bss/stack is non relocatable,
309		 * but someone could still run this code from RAM,
310		 * in which case our reference is _edata.
311		 */
312		mov	r10, r6
313#endif
314
315		mov	r5, #0			@ init dtb size to 0
316#ifdef CONFIG_ARM_APPENDED_DTB
317/*
318 *   r0  = delta
319 *   r2  = BSS start
320 *   r3  = BSS end
321 *   r4  = final kernel address (possibly with LSB set)
322 *   r5  = appended dtb size (still unknown)
323 *   r6  = _edata
324 *   r7  = architecture ID
325 *   r8  = atags/device tree pointer
326 *   r9  = size of decompressed image
327 *   r10 = end of this image, including  bss/stack/malloc space if non XIP
328 *   r11 = GOT start
329 *   r12 = GOT end
330 *   sp  = stack pointer
331 *
332 * if there are device trees (dtb) appended to zImage, advance r10 so that the
333 * dtb data will get relocated along with the kernel if necessary.
334 */
335
336		ldr	lr, [r6, #0]
337#ifndef __ARMEB__
338		ldr	r1, =0xedfe0dd0		@ sig is 0xd00dfeed big endian
339#else
340		ldr	r1, =0xd00dfeed
341#endif
342		cmp	lr, r1
343		bne	dtb_check_done		@ not found
344
345#ifdef CONFIG_ARM_ATAG_DTB_COMPAT
346		/*
347		 * OK... Let's do some funky business here.
348		 * If we do have a DTB appended to zImage, and we do have
349		 * an ATAG list around, we want the later to be translated
350		 * and folded into the former here. No GOT fixup has occurred
351		 * yet, but none of the code we're about to call uses any
352		 * global variable.
353		*/
354
355		/* Get the initial DTB size */
356		ldr	r5, [r6, #4]
357#ifndef __ARMEB__
358		/* convert to little endian */
359		eor	r1, r5, r5, ror #16
360		bic	r1, r1, #0x00ff0000
361		mov	r5, r5, ror #8
362		eor	r5, r5, r1, lsr #8
363#endif
364		/* 50% DTB growth should be good enough */
365		add	r5, r5, r5, lsr #1
366		/* preserve 64-bit alignment */
367		add	r5, r5, #7
368		bic	r5, r5, #7
369		/* clamp to 32KB min and 1MB max */
370		cmp	r5, #(1 << 15)
371		movlo	r5, #(1 << 15)
372		cmp	r5, #(1 << 20)
373		movhi	r5, #(1 << 20)
374		/* temporarily relocate the stack past the DTB work space */
375		add	sp, sp, r5
376
377		stmfd	sp!, {r0-r3, ip, lr}
378		mov	r0, r8
379		mov	r1, r6
380		mov	r2, r5
381		bl	atags_to_fdt
382
383		/*
384		 * If returned value is 1, there is no ATAG at the location
385		 * pointed by r8.  Try the typical 0x100 offset from start
386		 * of RAM and hope for the best.
387		 */
388		cmp	r0, #1
389		sub	r0, r4, #TEXT_OFFSET
390		bic	r0, r0, #1
391		add	r0, r0, #0x100
392		mov	r1, r6
393		mov	r2, r5
394		bleq	atags_to_fdt
395
396		ldmfd	sp!, {r0-r3, ip, lr}
397		sub	sp, sp, r5
398#endif
399
400		mov	r8, r6			@ use the appended device tree
401
402		/*
403		 * Make sure that the DTB doesn't end up in the final
404		 * kernel's .bss area. To do so, we adjust the decompressed
405		 * kernel size to compensate if that .bss size is larger
406		 * than the relocated code.
407		 */
408		ldr	r5, =_kernel_bss_size
409		adr	r1, wont_overwrite
410		sub	r1, r6, r1
411		subs	r1, r5, r1
412		addhi	r9, r9, r1
413
414		/* Get the current DTB size */
415		ldr	r5, [r6, #4]
416#ifndef __ARMEB__
417		/* convert r5 (dtb size) to little endian */
418		eor	r1, r5, r5, ror #16
419		bic	r1, r1, #0x00ff0000
420		mov	r5, r5, ror #8
421		eor	r5, r5, r1, lsr #8
422#endif
423
424		/* preserve 64-bit alignment */
425		add	r5, r5, #7
426		bic	r5, r5, #7
427
428		/* relocate some pointers past the appended dtb */
429		add	r6, r6, r5
430		add	r10, r10, r5
431		add	sp, sp, r5
432dtb_check_done:
433#endif
434
435/*
436 * Check to see if we will overwrite ourselves.
437 *   r4  = final kernel address (possibly with LSB set)
438 *   r9  = size of decompressed image
439 *   r10 = end of this image, including  bss/stack/malloc space if non XIP
440 * We basically want:
441 *   r4 - 16k page directory >= r10 -> OK
442 *   r4 + image length <= address of wont_overwrite -> OK
443 * Note: the possible LSB in r4 is harmless here.
444 */
445		add	r10, r10, #16384
446		cmp	r4, r10
447		bhs	wont_overwrite
448		add	r10, r4, r9
449		adr	r9, wont_overwrite
450		cmp	r10, r9
451		bls	wont_overwrite
452
453/*
454 * Relocate ourselves past the end of the decompressed kernel.
455 *   r6  = _edata
456 *   r10 = end of the decompressed kernel
457 * Because we always copy ahead, we need to do it from the end and go
458 * backward in case the source and destination overlap.
459 */
460		/*
461		 * Bump to the next 256-byte boundary with the size of
462		 * the relocation code added. This avoids overwriting
463		 * ourself when the offset is small.
464		 */
465		add	r10, r10, #((reloc_code_end - restart + 256) & ~255)
466		bic	r10, r10, #255
467
468		/* Get start of code we want to copy and align it down. */
469		adr	r5, restart
470		bic	r5, r5, #31
471
472/* Relocate the hyp vector base if necessary */
473#ifdef CONFIG_ARM_VIRT_EXT
474		mrs	r0, spsr
475		and	r0, r0, #MODE_MASK
476		cmp	r0, #HYP_MODE
477		bne	1f
478
479		/*
480		 * Compute the address of the hyp vectors after relocation.
481		 * This requires some arithmetic since we cannot directly
482		 * reference __hyp_stub_vectors in a PC-relative way.
483		 * Call __hyp_set_vectors with the new address so that we
484		 * can HVC again after the copy.
485		 */
4860:		adr	r0, 0b
487		movw	r1, #:lower16:__hyp_stub_vectors - 0b
488		movt	r1, #:upper16:__hyp_stub_vectors - 0b
489		add	r0, r0, r1
490		sub	r0, r0, r5
491		add	r0, r0, r10
492		bl	__hyp_set_vectors
4931:
494#endif
495
496		sub	r9, r6, r5		@ size to copy
497		add	r9, r9, #31		@ rounded up to a multiple
498		bic	r9, r9, #31		@ ... of 32 bytes
499		add	r6, r9, r5
500		add	r9, r9, r10
501
502#ifdef DEBUG
503		sub     r10, r6, r5
504		sub     r10, r9, r10
505		/*
506		 * We are about to copy the kernel to a new memory area.
507		 * The boundaries of the new memory area can be found in
508		 * r10 and r9, whilst r5 and r6 contain the boundaries
509		 * of the memory we are going to copy.
510		 * Calling dbgkc will help with the printing of this
511		 * information.
512		 */
513		dbgkc	r5, r6, r10, r9
514#endif
515
5161:		ldmdb	r6!, {r0 - r3, r10 - r12, lr}
517		cmp	r6, r5
518		stmdb	r9!, {r0 - r3, r10 - r12, lr}
519		bhi	1b
520
521		/* Preserve offset to relocated code. */
522		sub	r6, r9, r6
523
524#ifndef CONFIG_ZBOOT_ROM
525		/* cache_clean_flush may use the stack, so relocate it */
526		add	sp, sp, r6
527#endif
528
529		bl	cache_clean_flush
530
531		badr	r0, restart
532		add	r0, r0, r6
533		mov	pc, r0
534
535wont_overwrite:
536/*
537 * If delta is zero, we are running at the address we were linked at.
538 *   r0  = delta
539 *   r2  = BSS start
540 *   r3  = BSS end
541 *   r4  = kernel execution address (possibly with LSB set)
542 *   r5  = appended dtb size (0 if not present)
543 *   r7  = architecture ID
544 *   r8  = atags pointer
545 *   r11 = GOT start
546 *   r12 = GOT end
547 *   sp  = stack pointer
548 */
549		orrs	r1, r0, r5
550		beq	not_relocated
551
552		add	r11, r11, r0
553		add	r12, r12, r0
554
555#ifndef CONFIG_ZBOOT_ROM
556		/*
557		 * If we're running fully PIC === CONFIG_ZBOOT_ROM = n,
558		 * we need to fix up pointers into the BSS region.
559		 * Note that the stack pointer has already been fixed up.
560		 */
561		add	r2, r2, r0
562		add	r3, r3, r0
563
564		/*
565		 * Relocate all entries in the GOT table.
566		 * Bump bss entries to _edata + dtb size
567		 */
5681:		ldr	r1, [r11, #0]		@ relocate entries in the GOT
569		add	r1, r1, r0		@ This fixes up C references
570		cmp	r1, r2			@ if entry >= bss_start &&
571		cmphs	r3, r1			@       bss_end > entry
572		addhi	r1, r1, r5		@    entry += dtb size
573		str	r1, [r11], #4		@ next entry
574		cmp	r11, r12
575		blo	1b
576
577		/* bump our bss pointers too */
578		add	r2, r2, r5
579		add	r3, r3, r5
580
581#else
582
583		/*
584		 * Relocate entries in the GOT table.  We only relocate
585		 * the entries that are outside the (relocated) BSS region.
586		 */
5871:		ldr	r1, [r11, #0]		@ relocate entries in the GOT
588		cmp	r1, r2			@ entry < bss_start ||
589		cmphs	r3, r1			@ _end < entry
590		addlo	r1, r1, r0		@ table.  This fixes up the
591		str	r1, [r11], #4		@ C references.
592		cmp	r11, r12
593		blo	1b
594#endif
595
596not_relocated:	mov	r0, #0
5971:		str	r0, [r2], #4		@ clear bss
598		str	r0, [r2], #4
599		str	r0, [r2], #4
600		str	r0, [r2], #4
601		cmp	r2, r3
602		blo	1b
603
604		/*
605		 * Did we skip the cache setup earlier?
606		 * That is indicated by the LSB in r4.
607		 * Do it now if so.
608		 */
609		tst	r4, #1
610		bic	r4, r4, #1
611		blne	cache_on
612
613/*
614 * The C runtime environment should now be setup sufficiently.
615 * Set up some pointers, and start decompressing.
616 *   r4  = kernel execution address
617 *   r7  = architecture ID
618 *   r8  = atags pointer
619 */
620		mov	r0, r4
621		mov	r1, sp			@ malloc space above stack
622		add	r2, sp, #0x10000	@ 64k max
623		mov	r3, r7
624		bl	decompress_kernel
625		bl	cache_clean_flush
626		bl	cache_off
627
628#ifdef CONFIG_ARM_VIRT_EXT
629		mrs	r0, spsr		@ Get saved CPU boot mode
630		and	r0, r0, #MODE_MASK
631		cmp	r0, #HYP_MODE		@ if not booted in HYP mode...
632		bne	__enter_kernel		@ boot kernel directly
633
634		adr	r12, .L__hyp_reentry_vectors_offset
635		ldr	r0, [r12]
636		add	r0, r0, r12
637
638		bl	__hyp_set_vectors
639		__HVC(0)			@ otherwise bounce to hyp mode
640
641		b	.			@ should never be reached
642
643		.align	2
644.L__hyp_reentry_vectors_offset:	.long	__hyp_reentry_vectors - .
645#else
646		b	__enter_kernel
647#endif
648
649		.align	2
650		.type	LC0, #object
651LC0:		.word	LC0			@ r1
652		.word	__bss_start		@ r2
653		.word	_end			@ r3
654		.word	_edata			@ r6
655		.word	input_data_end - 4	@ r10 (inflated size location)
656		.word	_got_start		@ r11
657		.word	_got_end		@ ip
658		.word	.L_user_stack_end	@ sp
659		.word	_end - restart + 16384 + 1024*1024
660		.size	LC0, . - LC0
661
662#ifdef CONFIG_ARCH_RPC
663		.globl	params
664params:		ldr	r0, =0x10000100		@ params_phys for RPC
665		mov	pc, lr
666		.ltorg
667		.align
668#endif
669
670/*
671 * Turn on the cache.  We need to setup some page tables so that we
672 * can have both the I and D caches on.
673 *
674 * We place the page tables 16k down from the kernel execution address,
675 * and we hope that nothing else is using it.  If we're using it, we
676 * will go pop!
677 *
678 * On entry,
679 *  r4 = kernel execution address
680 *  r7 = architecture number
681 *  r8 = atags pointer
682 * On exit,
683 *  r0, r1, r2, r3, r9, r10, r12 corrupted
684 * This routine must preserve:
685 *  r4, r7, r8
686 */
687		.align	5
688cache_on:	mov	r3, #8			@ cache_on function
689		b	call_cache_fn
690
691/*
692 * Initialize the highest priority protection region, PR7
693 * to cover all 32bit address and cacheable and bufferable.
694 */
695__armv4_mpu_cache_on:
696		mov	r0, #0x3f		@ 4G, the whole
697		mcr	p15, 0, r0, c6, c7, 0	@ PR7 Area Setting
698		mcr 	p15, 0, r0, c6, c7, 1
699
700		mov	r0, #0x80		@ PR7
701		mcr	p15, 0, r0, c2, c0, 0	@ D-cache on
702		mcr	p15, 0, r0, c2, c0, 1	@ I-cache on
703		mcr	p15, 0, r0, c3, c0, 0	@ write-buffer on
704
705		mov	r0, #0xc000
706		mcr	p15, 0, r0, c5, c0, 1	@ I-access permission
707		mcr	p15, 0, r0, c5, c0, 0	@ D-access permission
708
709		mov	r0, #0
710		mcr	p15, 0, r0, c7, c10, 4	@ drain write buffer
711		mcr	p15, 0, r0, c7, c5, 0	@ flush(inval) I-Cache
712		mcr	p15, 0, r0, c7, c6, 0	@ flush(inval) D-Cache
713		mrc	p15, 0, r0, c1, c0, 0	@ read control reg
714						@ ...I .... ..D. WC.M
715		orr	r0, r0, #0x002d		@ .... .... ..1. 11.1
716		orr	r0, r0, #0x1000		@ ...1 .... .... ....
717
718		mcr	p15, 0, r0, c1, c0, 0	@ write control reg
719
720		mov	r0, #0
721		mcr	p15, 0, r0, c7, c5, 0	@ flush(inval) I-Cache
722		mcr	p15, 0, r0, c7, c6, 0	@ flush(inval) D-Cache
723		mov	pc, lr
724
725__armv3_mpu_cache_on:
726		mov	r0, #0x3f		@ 4G, the whole
727		mcr	p15, 0, r0, c6, c7, 0	@ PR7 Area Setting
728
729		mov	r0, #0x80		@ PR7
730		mcr	p15, 0, r0, c2, c0, 0	@ cache on
731		mcr	p15, 0, r0, c3, c0, 0	@ write-buffer on
732
733		mov	r0, #0xc000
734		mcr	p15, 0, r0, c5, c0, 0	@ access permission
735
736		mov	r0, #0
737		mcr	p15, 0, r0, c7, c0, 0	@ invalidate whole cache v3
738		/*
739		 * ?? ARMv3 MMU does not allow reading the control register,
740		 * does this really work on ARMv3 MPU?
741		 */
742		mrc	p15, 0, r0, c1, c0, 0	@ read control reg
743						@ .... .... .... WC.M
744		orr	r0, r0, #0x000d		@ .... .... .... 11.1
745		/* ?? this overwrites the value constructed above? */
746		mov	r0, #0
747		mcr	p15, 0, r0, c1, c0, 0	@ write control reg
748
749		/* ?? invalidate for the second time? */
750		mcr	p15, 0, r0, c7, c0, 0	@ invalidate whole cache v3
751		mov	pc, lr
752
753#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
754#define CB_BITS 0x08
755#else
756#define CB_BITS 0x0c
757#endif
758
759__setup_mmu:	sub	r3, r4, #16384		@ Page directory size
760		bic	r3, r3, #0xff		@ Align the pointer
761		bic	r3, r3, #0x3f00
762/*
763 * Initialise the page tables, turning on the cacheable and bufferable
764 * bits for the RAM area only.
765 */
766		mov	r0, r3
767		mov	r9, r0, lsr #18
768		mov	r9, r9, lsl #18		@ start of RAM
769		add	r10, r9, #0x10000000	@ a reasonable RAM size
770		mov	r1, #0x12		@ XN|U + section mapping
771		orr	r1, r1, #3 << 10	@ AP=11
772		add	r2, r3, #16384
7731:		cmp	r1, r9			@ if virt > start of RAM
774		cmphs	r10, r1			@   && end of RAM > virt
775		bic	r1, r1, #0x1c		@ clear XN|U + C + B
776		orrlo	r1, r1, #0x10		@ Set XN|U for non-RAM
777		orrhs	r1, r1, r6		@ set RAM section settings
778		str	r1, [r0], #4		@ 1:1 mapping
779		add	r1, r1, #1048576
780		teq	r0, r2
781		bne	1b
782/*
783 * If ever we are running from Flash, then we surely want the cache
784 * to be enabled also for our execution instance...  We map 2MB of it
785 * so there is no map overlap problem for up to 1 MB compressed kernel.
786 * If the execution is in RAM then we would only be duplicating the above.
787 */
788		orr	r1, r6, #0x04		@ ensure B is set for this
789		orr	r1, r1, #3 << 10
790		mov	r2, pc
791		mov	r2, r2, lsr #20
792		orr	r1, r1, r2, lsl #20
793		add	r0, r3, r2, lsl #2
794		str	r1, [r0], #4
795		add	r1, r1, #1048576
796		str	r1, [r0]
797		mov	pc, lr
798ENDPROC(__setup_mmu)
799
800@ Enable unaligned access on v6, to allow better code generation
801@ for the decompressor C code:
802__armv6_mmu_cache_on:
803		mrc	p15, 0, r0, c1, c0, 0	@ read SCTLR
804		bic	r0, r0, #2		@ A (no unaligned access fault)
805		orr	r0, r0, #1 << 22	@ U (v6 unaligned access model)
806		mcr	p15, 0, r0, c1, c0, 0	@ write SCTLR
807		b	__armv4_mmu_cache_on
808
809__arm926ejs_mmu_cache_on:
810#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
811		mov	r0, #4			@ put dcache in WT mode
812		mcr	p15, 7, r0, c15, c0, 0
813#endif
814
815__armv4_mmu_cache_on:
816		mov	r12, lr
817#ifdef CONFIG_MMU
818		mov	r6, #CB_BITS | 0x12	@ U
819		bl	__setup_mmu
820		mov	r0, #0
821		mcr	p15, 0, r0, c7, c10, 4	@ drain write buffer
822		mcr	p15, 0, r0, c8, c7, 0	@ flush I,D TLBs
823		mrc	p15, 0, r0, c1, c0, 0	@ read control reg
824		orr	r0, r0, #0x5000		@ I-cache enable, RR cache replacement
825		orr	r0, r0, #0x0030
826 ARM_BE8(	orr	r0, r0, #1 << 25 )	@ big-endian page tables
827		bl	__common_mmu_cache_on
828		mov	r0, #0
829		mcr	p15, 0, r0, c8, c7, 0	@ flush I,D TLBs
830#endif
831		mov	pc, r12
832
833__armv7_mmu_cache_on:
834		enable_cp15_barriers	r11
835		mov	r12, lr
836#ifdef CONFIG_MMU
837		mrc	p15, 0, r11, c0, c1, 4	@ read ID_MMFR0
838		tst	r11, #0xf		@ VMSA
839		movne	r6, #CB_BITS | 0x02	@ !XN
840		blne	__setup_mmu
841		mov	r0, #0
842		mcr	p15, 0, r0, c7, c10, 4	@ drain write buffer
843		tst	r11, #0xf		@ VMSA
844		mcrne	p15, 0, r0, c8, c7, 0	@ flush I,D TLBs
845#endif
846		mrc	p15, 0, r0, c1, c0, 0	@ read control reg
847		bic	r0, r0, #1 << 28	@ clear SCTLR.TRE
848		orr	r0, r0, #0x5000		@ I-cache enable, RR cache replacement
849		orr	r0, r0, #0x003c		@ write buffer
850		bic	r0, r0, #2		@ A (no unaligned access fault)
851		orr	r0, r0, #1 << 22	@ U (v6 unaligned access model)
852						@ (needed for ARM1176)
853#ifdef CONFIG_MMU
854 ARM_BE8(	orr	r0, r0, #1 << 25 )	@ big-endian page tables
855		mrcne   p15, 0, r6, c2, c0, 2   @ read ttb control reg
856		orrne	r0, r0, #1		@ MMU enabled
857		movne	r1, #0xfffffffd		@ domain 0 = client
858		bic     r6, r6, #1 << 31        @ 32-bit translation system
859		bic     r6, r6, #(7 << 0) | (1 << 4)	@ use only ttbr0
860		mcrne	p15, 0, r3, c2, c0, 0	@ load page table pointer
861		mcrne	p15, 0, r1, c3, c0, 0	@ load domain access control
862		mcrne   p15, 0, r6, c2, c0, 2   @ load ttb control
863#endif
864		mcr	p15, 0, r0, c7, c5, 4	@ ISB
865		mcr	p15, 0, r0, c1, c0, 0	@ load control register
866		mrc	p15, 0, r0, c1, c0, 0	@ and read it back
867		mov	r0, #0
868		mcr	p15, 0, r0, c7, c5, 4	@ ISB
869		mov	pc, r12
870
871__fa526_cache_on:
872		mov	r12, lr
873		mov	r6, #CB_BITS | 0x12	@ U
874		bl	__setup_mmu
875		mov	r0, #0
876		mcr	p15, 0, r0, c7, c7, 0	@ Invalidate whole cache
877		mcr	p15, 0, r0, c7, c10, 4	@ drain write buffer
878		mcr	p15, 0, r0, c8, c7, 0	@ flush UTLB
879		mrc	p15, 0, r0, c1, c0, 0	@ read control reg
880		orr	r0, r0, #0x1000		@ I-cache enable
881		bl	__common_mmu_cache_on
882		mov	r0, #0
883		mcr	p15, 0, r0, c8, c7, 0	@ flush UTLB
884		mov	pc, r12
885
886__common_mmu_cache_on:
887#ifndef CONFIG_THUMB2_KERNEL
888#ifndef DEBUG
889		orr	r0, r0, #0x000d		@ Write buffer, mmu
890#endif
891		mov	r1, #-1
892		mcr	p15, 0, r3, c2, c0, 0	@ load page table pointer
893		mcr	p15, 0, r1, c3, c0, 0	@ load domain access control
894		b	1f
895		.align	5			@ cache line aligned
8961:		mcr	p15, 0, r0, c1, c0, 0	@ load control register
897		mrc	p15, 0, r0, c1, c0, 0	@ and read it back to
898		sub	pc, lr, r0, lsr #32	@ properly flush pipeline
899#endif
900
901#define PROC_ENTRY_SIZE (4*5)
902
903/*
904 * Here follow the relocatable cache support functions for the
905 * various processors.  This is a generic hook for locating an
906 * entry and jumping to an instruction at the specified offset
907 * from the start of the block.  Please note this is all position
908 * independent code.
909 *
910 *  r1  = corrupted
911 *  r2  = corrupted
912 *  r3  = block offset
913 *  r9  = corrupted
914 *  r12 = corrupted
915 */
916
917call_cache_fn:	adr	r12, proc_types
918#ifdef CONFIG_CPU_CP15
919		mrc	p15, 0, r9, c0, c0	@ get processor ID
920#elif defined(CONFIG_CPU_V7M)
921		/*
922		 * On v7-M the processor id is located in the V7M_SCB_CPUID
923		 * register, but as cache handling is IMPLEMENTATION DEFINED on
924		 * v7-M (if existant at all) we just return early here.
925		 * If V7M_SCB_CPUID were used the cpu ID functions (i.e.
926		 * __armv7_mmu_cache_{on,off,flush}) would be selected which
927		 * use cp15 registers that are not implemented on v7-M.
928		 */
929		bx	lr
930#else
931		ldr	r9, =CONFIG_PROCESSOR_ID
932#endif
9331:		ldr	r1, [r12, #0]		@ get value
934		ldr	r2, [r12, #4]		@ get mask
935		eor	r1, r1, r9		@ (real ^ match)
936		tst	r1, r2			@       & mask
937 ARM(		addeq	pc, r12, r3		) @ call cache function
938 THUMB(		addeq	r12, r3			)
939 THUMB(		moveq	pc, r12			) @ call cache function
940		add	r12, r12, #PROC_ENTRY_SIZE
941		b	1b
942
943/*
944 * Table for cache operations.  This is basically:
945 *   - CPU ID match
946 *   - CPU ID mask
947 *   - 'cache on' method instruction
948 *   - 'cache off' method instruction
949 *   - 'cache flush' method instruction
950 *
951 * We match an entry using: ((real_id ^ match) & mask) == 0
952 *
953 * Writethrough caches generally only need 'on' and 'off'
954 * methods.  Writeback caches _must_ have the flush method
955 * defined.
956 */
957		.align	2
958		.type	proc_types,#object
959proc_types:
960		.word	0x41000000		@ old ARM ID
961		.word	0xff00f000
962		mov	pc, lr
963 THUMB(		nop				)
964		mov	pc, lr
965 THUMB(		nop				)
966		mov	pc, lr
967 THUMB(		nop				)
968
969		.word	0x41007000		@ ARM7/710
970		.word	0xfff8fe00
971		mov	pc, lr
972 THUMB(		nop				)
973		mov	pc, lr
974 THUMB(		nop				)
975		mov	pc, lr
976 THUMB(		nop				)
977
978		.word	0x41807200		@ ARM720T (writethrough)
979		.word	0xffffff00
980		W(b)	__armv4_mmu_cache_on
981		W(b)	__armv4_mmu_cache_off
982		mov	pc, lr
983 THUMB(		nop				)
984
985		.word	0x41007400		@ ARM74x
986		.word	0xff00ff00
987		W(b)	__armv3_mpu_cache_on
988		W(b)	__armv3_mpu_cache_off
989		W(b)	__armv3_mpu_cache_flush
990
991		.word	0x41009400		@ ARM94x
992		.word	0xff00ff00
993		W(b)	__armv4_mpu_cache_on
994		W(b)	__armv4_mpu_cache_off
995		W(b)	__armv4_mpu_cache_flush
996
997		.word	0x41069260		@ ARM926EJ-S (v5TEJ)
998		.word	0xff0ffff0
999		W(b)	__arm926ejs_mmu_cache_on
1000		W(b)	__armv4_mmu_cache_off
1001		W(b)	__armv5tej_mmu_cache_flush
1002
1003		.word	0x00007000		@ ARM7 IDs
1004		.word	0x0000f000
1005		mov	pc, lr
1006 THUMB(		nop				)
1007		mov	pc, lr
1008 THUMB(		nop				)
1009		mov	pc, lr
1010 THUMB(		nop				)
1011
1012		@ Everything from here on will be the new ID system.
1013
1014		.word	0x4401a100		@ sa110 / sa1100
1015		.word	0xffffffe0
1016		W(b)	__armv4_mmu_cache_on
1017		W(b)	__armv4_mmu_cache_off
1018		W(b)	__armv4_mmu_cache_flush
1019
1020		.word	0x6901b110		@ sa1110
1021		.word	0xfffffff0
1022		W(b)	__armv4_mmu_cache_on
1023		W(b)	__armv4_mmu_cache_off
1024		W(b)	__armv4_mmu_cache_flush
1025
1026		.word	0x56056900
1027		.word	0xffffff00		@ PXA9xx
1028		W(b)	__armv4_mmu_cache_on
1029		W(b)	__armv4_mmu_cache_off
1030		W(b)	__armv4_mmu_cache_flush
1031
1032		.word	0x56158000		@ PXA168
1033		.word	0xfffff000
1034		W(b)	__armv4_mmu_cache_on
1035		W(b)	__armv4_mmu_cache_off
1036		W(b)	__armv5tej_mmu_cache_flush
1037
1038		.word	0x56050000		@ Feroceon
1039		.word	0xff0f0000
1040		W(b)	__armv4_mmu_cache_on
1041		W(b)	__armv4_mmu_cache_off
1042		W(b)	__armv5tej_mmu_cache_flush
1043
1044#ifdef CONFIG_CPU_FEROCEON_OLD_ID
1045		/* this conflicts with the standard ARMv5TE entry */
1046		.long	0x41009260		@ Old Feroceon
1047		.long	0xff00fff0
1048		b	__armv4_mmu_cache_on
1049		b	__armv4_mmu_cache_off
1050		b	__armv5tej_mmu_cache_flush
1051#endif
1052
1053		.word	0x66015261		@ FA526
1054		.word	0xff01fff1
1055		W(b)	__fa526_cache_on
1056		W(b)	__armv4_mmu_cache_off
1057		W(b)	__fa526_cache_flush
1058
1059		@ These match on the architecture ID
1060
1061		.word	0x00020000		@ ARMv4T
1062		.word	0x000f0000
1063		W(b)	__armv4_mmu_cache_on
1064		W(b)	__armv4_mmu_cache_off
1065		W(b)	__armv4_mmu_cache_flush
1066
1067		.word	0x00050000		@ ARMv5TE
1068		.word	0x000f0000
1069		W(b)	__armv4_mmu_cache_on
1070		W(b)	__armv4_mmu_cache_off
1071		W(b)	__armv4_mmu_cache_flush
1072
1073		.word	0x00060000		@ ARMv5TEJ
1074		.word	0x000f0000
1075		W(b)	__armv4_mmu_cache_on
1076		W(b)	__armv4_mmu_cache_off
1077		W(b)	__armv5tej_mmu_cache_flush
1078
1079		.word	0x0007b000		@ ARMv6
1080		.word	0x000ff000
1081		W(b)	__armv6_mmu_cache_on
1082		W(b)	__armv4_mmu_cache_off
1083		W(b)	__armv6_mmu_cache_flush
1084
1085		.word	0x000f0000		@ new CPU Id
1086		.word	0x000f0000
1087		W(b)	__armv7_mmu_cache_on
1088		W(b)	__armv7_mmu_cache_off
1089		W(b)	__armv7_mmu_cache_flush
1090
1091		.word	0			@ unrecognised type
1092		.word	0
1093		mov	pc, lr
1094 THUMB(		nop				)
1095		mov	pc, lr
1096 THUMB(		nop				)
1097		mov	pc, lr
1098 THUMB(		nop				)
1099
1100		.size	proc_types, . - proc_types
1101
1102		/*
1103		 * If you get a "non-constant expression in ".if" statement"
1104		 * error from the assembler on this line, check that you have
1105		 * not accidentally written a "b" instruction where you should
1106		 * have written W(b).
1107		 */
1108		.if (. - proc_types) % PROC_ENTRY_SIZE != 0
1109		.error "The size of one or more proc_types entries is wrong."
1110		.endif
1111
1112/*
1113 * Turn off the Cache and MMU.  ARMv3 does not support
1114 * reading the control register, but ARMv4 does.
1115 *
1116 * On exit,
1117 *  r0, r1, r2, r3, r9, r12 corrupted
1118 * This routine must preserve:
1119 *  r4, r7, r8
1120 */
1121		.align	5
1122cache_off:	mov	r3, #12			@ cache_off function
1123		b	call_cache_fn
1124
1125__armv4_mpu_cache_off:
1126		mrc	p15, 0, r0, c1, c0
1127		bic	r0, r0, #0x000d
1128		mcr	p15, 0, r0, c1, c0	@ turn MPU and cache off
1129		mov	r0, #0
1130		mcr	p15, 0, r0, c7, c10, 4	@ drain write buffer
1131		mcr	p15, 0, r0, c7, c6, 0	@ flush D-Cache
1132		mcr	p15, 0, r0, c7, c5, 0	@ flush I-Cache
1133		mov	pc, lr
1134
1135__armv3_mpu_cache_off:
1136		mrc	p15, 0, r0, c1, c0
1137		bic	r0, r0, #0x000d
1138		mcr	p15, 0, r0, c1, c0, 0	@ turn MPU and cache off
1139		mov	r0, #0
1140		mcr	p15, 0, r0, c7, c0, 0	@ invalidate whole cache v3
1141		mov	pc, lr
1142
1143__armv4_mmu_cache_off:
1144#ifdef CONFIG_MMU
1145		mrc	p15, 0, r0, c1, c0
1146		bic	r0, r0, #0x000d
1147		mcr	p15, 0, r0, c1, c0	@ turn MMU and cache off
1148		mov	r0, #0
1149		mcr	p15, 0, r0, c7, c7	@ invalidate whole cache v4
1150		mcr	p15, 0, r0, c8, c7	@ invalidate whole TLB v4
1151#endif
1152		mov	pc, lr
1153
1154__armv7_mmu_cache_off:
1155		mrc	p15, 0, r0, c1, c0
1156#ifdef CONFIG_MMU
1157		bic	r0, r0, #0x000d
1158#else
1159		bic	r0, r0, #0x000c
1160#endif
1161		mcr	p15, 0, r0, c1, c0	@ turn MMU and cache off
1162		mov	r12, lr
1163		bl	__armv7_mmu_cache_flush
1164		mov	r0, #0
1165#ifdef CONFIG_MMU
1166		mcr	p15, 0, r0, c8, c7, 0	@ invalidate whole TLB
1167#endif
1168		mcr	p15, 0, r0, c7, c5, 6	@ invalidate BTC
1169		mcr	p15, 0, r0, c7, c10, 4	@ DSB
1170		mcr	p15, 0, r0, c7, c5, 4	@ ISB
1171		mov	pc, r12
1172
1173/*
1174 * Clean and flush the cache to maintain consistency.
1175 *
1176 * On exit,
1177 *  r1, r2, r3, r9, r10, r11, r12 corrupted
1178 * This routine must preserve:
1179 *  r4, r6, r7, r8
1180 */
1181		.align	5
1182cache_clean_flush:
1183		mov	r3, #16
1184		b	call_cache_fn
1185
1186__armv4_mpu_cache_flush:
1187		tst	r4, #1
1188		movne	pc, lr
1189		mov	r2, #1
1190		mov	r3, #0
1191		mcr	p15, 0, ip, c7, c6, 0	@ invalidate D cache
1192		mov	r1, #7 << 5		@ 8 segments
11931:		orr	r3, r1, #63 << 26	@ 64 entries
11942:		mcr	p15, 0, r3, c7, c14, 2	@ clean & invalidate D index
1195		subs	r3, r3, #1 << 26
1196		bcs	2b			@ entries 63 to 0
1197		subs 	r1, r1, #1 << 5
1198		bcs	1b			@ segments 7 to 0
1199
1200		teq	r2, #0
1201		mcrne	p15, 0, ip, c7, c5, 0	@ invalidate I cache
1202		mcr	p15, 0, ip, c7, c10, 4	@ drain WB
1203		mov	pc, lr
1204
1205__fa526_cache_flush:
1206		tst	r4, #1
1207		movne	pc, lr
1208		mov	r1, #0
1209		mcr	p15, 0, r1, c7, c14, 0	@ clean and invalidate D cache
1210		mcr	p15, 0, r1, c7, c5, 0	@ flush I cache
1211		mcr	p15, 0, r1, c7, c10, 4	@ drain WB
1212		mov	pc, lr
1213
1214__armv6_mmu_cache_flush:
1215		mov	r1, #0
1216		tst	r4, #1
1217		mcreq	p15, 0, r1, c7, c14, 0	@ clean+invalidate D
1218		mcr	p15, 0, r1, c7, c5, 0	@ invalidate I+BTB
1219		mcreq	p15, 0, r1, c7, c15, 0	@ clean+invalidate unified
1220		mcr	p15, 0, r1, c7, c10, 4	@ drain WB
1221		mov	pc, lr
1222
1223__armv7_mmu_cache_flush:
1224		enable_cp15_barriers	r10
1225		tst	r4, #1
1226		bne	iflush
1227		mrc	p15, 0, r10, c0, c1, 5	@ read ID_MMFR1
1228		tst	r10, #0xf << 16		@ hierarchical cache (ARMv7)
1229		mov	r10, #0
1230		beq	hierarchical
1231		mcr	p15, 0, r10, c7, c14, 0	@ clean+invalidate D
1232		b	iflush
1233hierarchical:
1234		mcr	p15, 0, r10, c7, c10, 5	@ DMB
1235		stmfd	sp!, {r0-r7, r9-r11}
1236		mrc	p15, 1, r0, c0, c0, 1	@ read clidr
1237		ands	r3, r0, #0x7000000	@ extract loc from clidr
1238		mov	r3, r3, lsr #23		@ left align loc bit field
1239		beq	finished		@ if loc is 0, then no need to clean
1240		mov	r10, #0			@ start clean at cache level 0
1241loop1:
1242		add	r2, r10, r10, lsr #1	@ work out 3x current cache level
1243		mov	r1, r0, lsr r2		@ extract cache type bits from clidr
1244		and	r1, r1, #7		@ mask of the bits for current cache only
1245		cmp	r1, #2			@ see what cache we have at this level
1246		blt	skip			@ skip if no cache, or just i-cache
1247		mcr	p15, 2, r10, c0, c0, 0	@ select current cache level in cssr
1248		mcr	p15, 0, r10, c7, c5, 4	@ isb to sych the new cssr&csidr
1249		mrc	p15, 1, r1, c0, c0, 0	@ read the new csidr
1250		and	r2, r1, #7		@ extract the length of the cache lines
1251		add	r2, r2, #4		@ add 4 (line length offset)
1252		ldr	r4, =0x3ff
1253		ands	r4, r4, r1, lsr #3	@ find maximum number on the way size
1254		clz	r5, r4			@ find bit position of way size increment
1255		ldr	r7, =0x7fff
1256		ands	r7, r7, r1, lsr #13	@ extract max number of the index size
1257loop2:
1258		mov	r9, r4			@ create working copy of max way size
1259loop3:
1260 ARM(		orr	r11, r10, r9, lsl r5	) @ factor way and cache number into r11
1261 ARM(		orr	r11, r11, r7, lsl r2	) @ factor index number into r11
1262 THUMB(		lsl	r6, r9, r5		)
1263 THUMB(		orr	r11, r10, r6		) @ factor way and cache number into r11
1264 THUMB(		lsl	r6, r7, r2		)
1265 THUMB(		orr	r11, r11, r6		) @ factor index number into r11
1266		mcr	p15, 0, r11, c7, c14, 2	@ clean & invalidate by set/way
1267		subs	r9, r9, #1		@ decrement the way
1268		bge	loop3
1269		subs	r7, r7, #1		@ decrement the index
1270		bge	loop2
1271skip:
1272		add	r10, r10, #2		@ increment cache number
1273		cmp	r3, r10
1274		bgt	loop1
1275finished:
1276		ldmfd	sp!, {r0-r7, r9-r11}
1277		mov	r10, #0			@ switch back to cache level 0
1278		mcr	p15, 2, r10, c0, c0, 0	@ select current cache level in cssr
1279iflush:
1280		mcr	p15, 0, r10, c7, c10, 4	@ DSB
1281		mcr	p15, 0, r10, c7, c5, 0	@ invalidate I+BTB
1282		mcr	p15, 0, r10, c7, c10, 4	@ DSB
1283		mcr	p15, 0, r10, c7, c5, 4	@ ISB
1284		mov	pc, lr
1285
1286__armv5tej_mmu_cache_flush:
1287		tst	r4, #1
1288		movne	pc, lr
12891:		mrc	p15, 0, APSR_nzcv, c7, c14, 3	@ test,clean,invalidate D cache
1290		bne	1b
1291		mcr	p15, 0, r0, c7, c5, 0	@ flush I cache
1292		mcr	p15, 0, r0, c7, c10, 4	@ drain WB
1293		mov	pc, lr
1294
1295__armv4_mmu_cache_flush:
1296		tst	r4, #1
1297		movne	pc, lr
1298		mov	r2, #64*1024		@ default: 32K dcache size (*2)
1299		mov	r11, #32		@ default: 32 byte line size
1300		mrc	p15, 0, r3, c0, c0, 1	@ read cache type
1301		teq	r3, r9			@ cache ID register present?
1302		beq	no_cache_id
1303		mov	r1, r3, lsr #18
1304		and	r1, r1, #7
1305		mov	r2, #1024
1306		mov	r2, r2, lsl r1		@ base dcache size *2
1307		tst	r3, #1 << 14		@ test M bit
1308		addne	r2, r2, r2, lsr #1	@ +1/2 size if M == 1
1309		mov	r3, r3, lsr #12
1310		and	r3, r3, #3
1311		mov	r11, #8
1312		mov	r11, r11, lsl r3	@ cache line size in bytes
1313no_cache_id:
1314		mov	r1, pc
1315		bic	r1, r1, #63		@ align to longest cache line
1316		add	r2, r1, r2
13171:
1318 ARM(		ldr	r3, [r1], r11		) @ s/w flush D cache
1319 THUMB(		ldr     r3, [r1]		) @ s/w flush D cache
1320 THUMB(		add     r1, r1, r11		)
1321		teq	r1, r2
1322		bne	1b
1323
1324		mcr	p15, 0, r1, c7, c5, 0	@ flush I cache
1325		mcr	p15, 0, r1, c7, c6, 0	@ flush D cache
1326		mcr	p15, 0, r1, c7, c10, 4	@ drain WB
1327		mov	pc, lr
1328
1329__armv3_mmu_cache_flush:
1330__armv3_mpu_cache_flush:
1331		tst	r4, #1
1332		movne	pc, lr
1333		mov	r1, #0
1334		mcr	p15, 0, r1, c7, c0, 0	@ invalidate whole cache v3
1335		mov	pc, lr
1336
1337/*
1338 * Various debugging routines for printing hex characters and
1339 * memory, which again must be relocatable.
1340 */
1341#ifdef DEBUG
1342		.align	2
1343		.type	phexbuf,#object
1344phexbuf:	.space	12
1345		.size	phexbuf, . - phexbuf
1346
1347@ phex corrupts {r0, r1, r2, r3}
1348phex:		adr	r3, phexbuf
1349		mov	r2, #0
1350		strb	r2, [r3, r1]
13511:		subs	r1, r1, #1
1352		movmi	r0, r3
1353		bmi	puts
1354		and	r2, r0, #15
1355		mov	r0, r0, lsr #4
1356		cmp	r2, #10
1357		addge	r2, r2, #7
1358		add	r2, r2, #'0'
1359		strb	r2, [r3, r1]
1360		b	1b
1361
1362@ puts corrupts {r0, r1, r2, r3}
1363puts:		loadsp	r3, r2, r1
13641:		ldrb	r2, [r0], #1
1365		teq	r2, #0
1366		moveq	pc, lr
13672:		writeb	r2, r3
1368		mov	r1, #0x00020000
13693:		subs	r1, r1, #1
1370		bne	3b
1371		teq	r2, #'\n'
1372		moveq	r2, #'\r'
1373		beq	2b
1374		teq	r0, #0
1375		bne	1b
1376		mov	pc, lr
1377@ putc corrupts {r0, r1, r2, r3}
1378putc:
1379		mov	r2, r0
1380		loadsp	r3, r1, r0
1381		mov	r0, #0
1382		b	2b
1383
1384@ memdump corrupts {r0, r1, r2, r3, r10, r11, r12, lr}
1385memdump:	mov	r12, r0
1386		mov	r10, lr
1387		mov	r11, #0
13882:		mov	r0, r11, lsl #2
1389		add	r0, r0, r12
1390		mov	r1, #8
1391		bl	phex
1392		mov	r0, #':'
1393		bl	putc
13941:		mov	r0, #' '
1395		bl	putc
1396		ldr	r0, [r12, r11, lsl #2]
1397		mov	r1, #8
1398		bl	phex
1399		and	r0, r11, #7
1400		teq	r0, #3
1401		moveq	r0, #' '
1402		bleq	putc
1403		and	r0, r11, #7
1404		add	r11, r11, #1
1405		teq	r0, #7
1406		bne	1b
1407		mov	r0, #'\n'
1408		bl	putc
1409		cmp	r11, #64
1410		blt	2b
1411		mov	pc, r10
1412#endif
1413
1414		.ltorg
1415
1416#ifdef CONFIG_ARM_VIRT_EXT
1417.align 5
1418__hyp_reentry_vectors:
1419		W(b)	.			@ reset
1420		W(b)	.			@ undef
1421		W(b)	.			@ svc
1422		W(b)	.			@ pabort
1423		W(b)	.			@ dabort
1424		W(b)	__enter_kernel		@ hyp
1425		W(b)	.			@ irq
1426		W(b)	.			@ fiq
1427#endif /* CONFIG_ARM_VIRT_EXT */
1428
1429__enter_kernel:
1430		mov	r0, #0			@ must be 0
1431		mov	r1, r7			@ restore architecture number
1432		mov	r2, r8			@ restore atags pointer
1433 ARM(		mov	pc, r4		)	@ call kernel
1434 M_CLASS(	add	r4, r4, #1	)	@ enter in Thumb mode for M class
1435 THUMB(		bx	r4		)	@ entry point is always ARM for A/R classes
1436
1437reloc_code_end:
1438
1439#ifdef CONFIG_EFI_STUB
1440		.align	2
1441_start:		.long	start - .
1442
1443ENTRY(efi_stub_entry)
1444		@ allocate space on stack for passing current zImage address
1445		@ and for the EFI stub to return of new entry point of
1446		@ zImage, as EFI stub may copy the kernel. Pointer address
1447		@ is passed in r2. r0 and r1 are passed through from the
1448		@ EFI firmware to efi_entry
1449		adr	ip, _start
1450		ldr	r3, [ip]
1451		add	r3, r3, ip
1452		stmfd	sp!, {r3, lr}
1453		mov	r2, sp			@ pass zImage address in r2
1454		bl	efi_entry
1455
1456		@ Check for error return from EFI stub. r0 has FDT address
1457		@ or error code.
1458		cmn	r0, #1
1459		beq	efi_load_fail
1460
1461		@ Preserve return value of efi_entry() in r4
1462		mov	r4, r0
1463		bl	cache_clean_flush
1464		bl	cache_off
1465
1466		@ Set parameters for booting zImage according to boot protocol
1467		@ put FDT address in r2, it was returned by efi_entry()
1468		@ r1 is the machine type, and r0 needs to be 0
1469		mov	r0, #0
1470		mov	r1, #0xFFFFFFFF
1471		mov	r2, r4
1472
1473		@ Branch to (possibly) relocated zImage that is in [sp]
1474		ldr	lr, [sp]
1475		ldr	ip, =start_offset
1476		add	lr, lr, ip
1477		mov	pc, lr				@ no mode switch
1478
1479efi_load_fail:
1480		@ Return EFI_LOAD_ERROR to EFI firmware on error.
1481		ldr	r0, =0x80000001
1482		ldmfd	sp!, {ip, pc}
1483ENDPROC(efi_stub_entry)
1484#endif
1485
1486		.align
1487		.section ".stack", "aw", %nobits
1488.L_user_stack:	.space	4096
1489.L_user_stack_end:
1490