xref: /openbmc/linux/arch/parisc/kernel/pacache.S (revision c4c3c32d) 
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *  PARISC TLB and cache flushing support
4 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
5 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
6 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
7 */
8
9/*
10 * NOTE: fdc,fic, and pdc instructions that use base register modification
11 *       should only use index and base registers that are not shadowed,
12 *       so that the fast path emulation in the non access miss handler
13 *       can be used.
14 */
15
16#ifdef CONFIG_64BIT
17	.level	2.0w
18#else
19	.level	2.0
20#endif
21
22#include <asm/psw.h>
23#include <asm/assembly.h>
24#include <asm/cache.h>
25#include <asm/ldcw.h>
26#include <asm/alternative.h>
27#include <linux/linkage.h>
28#include <linux/init.h>
29#include <linux/pgtable.h>
30
31	.section .text.hot
32	.align	16
33
34ENTRY_CFI(flush_tlb_all_local)
35	/*
36	 * The pitlbe and pdtlbe instructions should only be used to
37	 * flush the entire tlb. Also, there needs to be no intervening
38	 * tlb operations, e.g. tlb misses, so the operation needs
39	 * to happen in real mode with all interruptions disabled.
40	 */
41
42	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
43	rsm		PSW_SM_I, %r19		/* save I-bit state */
44	load32		PA(1f), %r1
45	nop
46	nop
47	nop
48	nop
49	nop
50
51	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
52	mtctl		%r0, %cr17		/* Clear IIASQ tail */
53	mtctl		%r0, %cr17		/* Clear IIASQ head */
54	mtctl		%r1, %cr18		/* IIAOQ head */
55	ldo		4(%r1), %r1
56	mtctl		%r1, %cr18		/* IIAOQ tail */
57	load32		REAL_MODE_PSW, %r1
58	mtctl           %r1, %ipsw
59	rfi
60	nop
61
621:      load32		PA(cache_info), %r1
63
64	/* Flush Instruction Tlb */
65
6688:	LDREG		ITLB_SID_BASE(%r1), %r20
67	LDREG		ITLB_SID_STRIDE(%r1), %r21
68	LDREG		ITLB_SID_COUNT(%r1), %r22
69	LDREG		ITLB_OFF_BASE(%r1), %arg0
70	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
71	LDREG		ITLB_OFF_COUNT(%r1), %arg2
72	LDREG		ITLB_LOOP(%r1), %arg3
73
74	addib,COND(=)		-1, %arg3, fitoneloop	/* Preadjust and test */
75	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
76	copy		%arg0, %r28		/* Init base addr */
77
78fitmanyloop:					/* Loop if LOOP >= 2 */
79	mtsp		%r20, %sr1
80	add		%r21, %r20, %r20	/* increment space */
81	copy		%arg2, %r29		/* Init middle loop count */
82
83fitmanymiddle:					/* Loop if LOOP >= 2 */
84	addib,COND(>)		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
85	pitlbe		%r0(%sr1, %r28)
86	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
87	addib,COND(>)		-1, %r29, fitmanymiddle	/* Middle loop decr */
88	copy		%arg3, %r31		/* Re-init inner loop count */
89
90	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
91	addib,COND(<=),n	-1, %r22, fitdone	/* Outer loop count decr */
92
93fitoneloop:					/* Loop if LOOP = 1 */
94	mtsp		%r20, %sr1
95	copy		%arg0, %r28		/* init base addr */
96	copy		%arg2, %r29		/* init middle loop count */
97
98fitonemiddle:					/* Loop if LOOP = 1 */
99	addib,COND(>)		-1, %r29, fitonemiddle	/* Middle loop count decr */
100	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
101
102	addib,COND(>)		-1, %r22, fitoneloop	/* Outer loop count decr */
103	add		%r21, %r20, %r20		/* increment space */
104
105fitdone:
106	ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
107
108	/* Flush Data Tlb */
109
110	LDREG		DTLB_SID_BASE(%r1), %r20
111	LDREG		DTLB_SID_STRIDE(%r1), %r21
112	LDREG		DTLB_SID_COUNT(%r1), %r22
113	LDREG		DTLB_OFF_BASE(%r1), %arg0
114	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
115	LDREG		DTLB_OFF_COUNT(%r1), %arg2
116	LDREG		DTLB_LOOP(%r1), %arg3
117
118	addib,COND(=)		-1, %arg3, fdtoneloop	/* Preadjust and test */
119	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
120	copy		%arg0, %r28		/* Init base addr */
121
122fdtmanyloop:					/* Loop if LOOP >= 2 */
123	mtsp		%r20, %sr1
124	add		%r21, %r20, %r20	/* increment space */
125	copy		%arg2, %r29		/* Init middle loop count */
126
127fdtmanymiddle:					/* Loop if LOOP >= 2 */
128	addib,COND(>)		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
129	pdtlbe		%r0(%sr1, %r28)
130	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
131	addib,COND(>)		-1, %r29, fdtmanymiddle	/* Middle loop decr */
132	copy		%arg3, %r31		/* Re-init inner loop count */
133
134	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
135	addib,COND(<=),n	-1, %r22,fdtdone	/* Outer loop count decr */
136
137fdtoneloop:					/* Loop if LOOP = 1 */
138	mtsp		%r20, %sr1
139	copy		%arg0, %r28		/* init base addr */
140	copy		%arg2, %r29		/* init middle loop count */
141
142fdtonemiddle:					/* Loop if LOOP = 1 */
143	addib,COND(>)		-1, %r29, fdtonemiddle	/* Middle loop count decr */
144	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
145
146	addib,COND(>)		-1, %r22, fdtoneloop	/* Outer loop count decr */
147	add		%r21, %r20, %r20	/* increment space */
148
149
150fdtdone:
151	/*
152	 * Switch back to virtual mode
153	 */
154	/* pcxt_ssm_bug */
155	rsm		PSW_SM_I, %r0
156	load32		2f, %r1
157	nop
158	nop
159	nop
160	nop
161	nop
162
163	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
164	mtctl		%r0, %cr17		/* Clear IIASQ tail */
165	mtctl		%r0, %cr17		/* Clear IIASQ head */
166	mtctl		%r1, %cr18		/* IIAOQ head */
167	ldo		4(%r1), %r1
168	mtctl		%r1, %cr18		/* IIAOQ tail */
169	load32		KERNEL_PSW, %r1
170	or		%r1, %r19, %r1	/* I-bit to state on entry */
171	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
172	rfi
173	nop
174
1752:      bv		%r0(%r2)
176	nop
177
178	/*
179	 * When running in qemu, drop whole flush_tlb_all_local function and
180	 * replace by one pdtlbe instruction, for which QEMU will drop all
181	 * local TLB entries.
182	 */
1833:	pdtlbe		%r0(%sr1,%r0)
184	bv,n		%r0(%r2)
185	ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
186ENDPROC_CFI(flush_tlb_all_local)
187
188	.import cache_info,data
189
190ENTRY_CFI(flush_instruction_cache_local)
19188:	load32		cache_info, %r1
192
193	/* Flush Instruction Cache */
194
195	LDREG		ICACHE_BASE(%r1), %arg0
196	LDREG		ICACHE_STRIDE(%r1), %arg1
197	LDREG		ICACHE_COUNT(%r1), %arg2
198	LDREG		ICACHE_LOOP(%r1), %arg3
199	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
200	mtsp		%r0, %sr1
201	addib,COND(=)		-1, %arg3, fioneloop	/* Preadjust and test */
202	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
203
204fimanyloop:					/* Loop if LOOP >= 2 */
205	addib,COND(>)		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
206	fice            %r0(%sr1, %arg0)
207	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
208	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
209	addib,COND(<=),n	-1, %arg2, fisync	/* Outer loop decr */
210
211fioneloop:					/* Loop if LOOP = 1 */
212	/* Some implementations may flush with a single fice instruction */
213	cmpib,COND(>>=),n	15, %arg2, fioneloop2
214
215fioneloop1:
216	fice,m		%arg1(%sr1, %arg0)
217	fice,m		%arg1(%sr1, %arg0)
218	fice,m		%arg1(%sr1, %arg0)
219	fice,m		%arg1(%sr1, %arg0)
220	fice,m		%arg1(%sr1, %arg0)
221	fice,m		%arg1(%sr1, %arg0)
222	fice,m		%arg1(%sr1, %arg0)
223	fice,m		%arg1(%sr1, %arg0)
224	fice,m		%arg1(%sr1, %arg0)
225	fice,m		%arg1(%sr1, %arg0)
226	fice,m		%arg1(%sr1, %arg0)
227	fice,m		%arg1(%sr1, %arg0)
228	fice,m		%arg1(%sr1, %arg0)
229	fice,m		%arg1(%sr1, %arg0)
230	fice,m		%arg1(%sr1, %arg0)
231	addib,COND(>)	-16, %arg2, fioneloop1
232	fice,m		%arg1(%sr1, %arg0)
233
234	/* Check if done */
235	cmpb,COND(=),n	%arg2, %r0, fisync	/* Predict branch taken */
236
237fioneloop2:
238	addib,COND(>)	-1, %arg2, fioneloop2	/* Outer loop count decr */
239	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
240
241fisync:
242	sync
243	mtsm		%r22			/* restore I-bit */
24489:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
245	bv		%r0(%r2)
246	nop
247ENDPROC_CFI(flush_instruction_cache_local)
248
249
250	.import cache_info, data
251ENTRY_CFI(flush_data_cache_local)
25288:	load32		cache_info, %r1
253
254	/* Flush Data Cache */
255
256	LDREG		DCACHE_BASE(%r1), %arg0
257	LDREG		DCACHE_STRIDE(%r1), %arg1
258	LDREG		DCACHE_COUNT(%r1), %arg2
259	LDREG		DCACHE_LOOP(%r1), %arg3
260	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
261	mtsp		%r0, %sr1
262	addib,COND(=)		-1, %arg3, fdoneloop	/* Preadjust and test */
263	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
264
265fdmanyloop:					/* Loop if LOOP >= 2 */
266	addib,COND(>)		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
267	fdce		%r0(%sr1, %arg0)
268	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
269	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
270	addib,COND(<=),n	-1, %arg2, fdsync	/* Outer loop decr */
271
272fdoneloop:					/* Loop if LOOP = 1 */
273	/* Some implementations may flush with a single fdce instruction */
274	cmpib,COND(>>=),n	15, %arg2, fdoneloop2
275
276fdoneloop1:
277	fdce,m		%arg1(%sr1, %arg0)
278	fdce,m		%arg1(%sr1, %arg0)
279	fdce,m		%arg1(%sr1, %arg0)
280	fdce,m		%arg1(%sr1, %arg0)
281	fdce,m		%arg1(%sr1, %arg0)
282	fdce,m		%arg1(%sr1, %arg0)
283	fdce,m		%arg1(%sr1, %arg0)
284	fdce,m		%arg1(%sr1, %arg0)
285	fdce,m		%arg1(%sr1, %arg0)
286	fdce,m		%arg1(%sr1, %arg0)
287	fdce,m		%arg1(%sr1, %arg0)
288	fdce,m		%arg1(%sr1, %arg0)
289	fdce,m		%arg1(%sr1, %arg0)
290	fdce,m		%arg1(%sr1, %arg0)
291	fdce,m		%arg1(%sr1, %arg0)
292	addib,COND(>)	-16, %arg2, fdoneloop1
293	fdce,m		%arg1(%sr1, %arg0)
294
295	/* Check if done */
296	cmpb,COND(=),n	%arg2, %r0, fdsync	/* Predict branch taken */
297
298fdoneloop2:
299	addib,COND(>)	-1, %arg2, fdoneloop2	/* Outer loop count decr */
300	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
301
302fdsync:
303	sync
304	mtsm		%r22			/* restore I-bit */
30589:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
306	bv		%r0(%r2)
307	nop
308ENDPROC_CFI(flush_data_cache_local)
309
310/* Clear page using kernel mapping.  */
311
312ENTRY_CFI(clear_page_asm)
313#ifdef CONFIG_64BIT
314
315	/* Unroll the loop.  */
316	ldi		(PAGE_SIZE / 128), %r1
317
3181:
319	std		%r0, 0(%r26)
320	std		%r0, 8(%r26)
321	std		%r0, 16(%r26)
322	std		%r0, 24(%r26)
323	std		%r0, 32(%r26)
324	std		%r0, 40(%r26)
325	std		%r0, 48(%r26)
326	std		%r0, 56(%r26)
327	std		%r0, 64(%r26)
328	std		%r0, 72(%r26)
329	std		%r0, 80(%r26)
330	std		%r0, 88(%r26)
331	std		%r0, 96(%r26)
332	std		%r0, 104(%r26)
333	std		%r0, 112(%r26)
334	std		%r0, 120(%r26)
335
336	/* Note reverse branch hint for addib is taken.  */
337	addib,COND(>),n	-1, %r1, 1b
338	ldo		128(%r26), %r26
339
340#else
341
342	/*
343	 * Note that until (if) we start saving the full 64-bit register
344	 * values on interrupt, we can't use std on a 32 bit kernel.
345	 */
346	ldi		(PAGE_SIZE / 64), %r1
347
3481:
349	stw		%r0, 0(%r26)
350	stw		%r0, 4(%r26)
351	stw		%r0, 8(%r26)
352	stw		%r0, 12(%r26)
353	stw		%r0, 16(%r26)
354	stw		%r0, 20(%r26)
355	stw		%r0, 24(%r26)
356	stw		%r0, 28(%r26)
357	stw		%r0, 32(%r26)
358	stw		%r0, 36(%r26)
359	stw		%r0, 40(%r26)
360	stw		%r0, 44(%r26)
361	stw		%r0, 48(%r26)
362	stw		%r0, 52(%r26)
363	stw		%r0, 56(%r26)
364	stw		%r0, 60(%r26)
365
366	addib,COND(>),n	-1, %r1, 1b
367	ldo		64(%r26), %r26
368#endif
369	bv		%r0(%r2)
370	nop
371ENDPROC_CFI(clear_page_asm)
372
373/* Copy page using kernel mapping.  */
374
375ENTRY_CFI(copy_page_asm)
376#ifdef CONFIG_64BIT
377	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
378	 * Unroll the loop by hand and arrange insn appropriately.
379	 * Prefetch doesn't improve performance on rp3440.
380	 * GCC probably can do this just as well...
381	 */
382
383	ldi		(PAGE_SIZE / 128), %r1
384
3851:	ldd		0(%r25), %r19
386	ldd		8(%r25), %r20
387
388	ldd		16(%r25), %r21
389	ldd		24(%r25), %r22
390	std		%r19, 0(%r26)
391	std		%r20, 8(%r26)
392
393	ldd		32(%r25), %r19
394	ldd		40(%r25), %r20
395	std		%r21, 16(%r26)
396	std		%r22, 24(%r26)
397
398	ldd		48(%r25), %r21
399	ldd		56(%r25), %r22
400	std		%r19, 32(%r26)
401	std		%r20, 40(%r26)
402
403	ldd		64(%r25), %r19
404	ldd		72(%r25), %r20
405	std		%r21, 48(%r26)
406	std		%r22, 56(%r26)
407
408	ldd		80(%r25), %r21
409	ldd		88(%r25), %r22
410	std		%r19, 64(%r26)
411	std		%r20, 72(%r26)
412
413	ldd		 96(%r25), %r19
414	ldd		104(%r25), %r20
415	std		%r21, 80(%r26)
416	std		%r22, 88(%r26)
417
418	ldd		112(%r25), %r21
419	ldd		120(%r25), %r22
420	ldo		128(%r25), %r25
421	std		%r19, 96(%r26)
422	std		%r20, 104(%r26)
423
424	std		%r21, 112(%r26)
425	std		%r22, 120(%r26)
426
427	/* Note reverse branch hint for addib is taken.  */
428	addib,COND(>),n	-1, %r1, 1b
429	ldo		128(%r26), %r26
430
431#else
432
433	/*
434	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
435	 * bundles (very restricted rules for bundling).
436	 * Note that until (if) we start saving
437	 * the full 64 bit register values on interrupt, we can't
438	 * use ldd/std on a 32 bit kernel.
439	 */
440	ldw		0(%r25), %r19
441	ldi		(PAGE_SIZE / 64), %r1
442
4431:
444	ldw		4(%r25), %r20
445	ldw		8(%r25), %r21
446	ldw		12(%r25), %r22
447	stw		%r19, 0(%r26)
448	stw		%r20, 4(%r26)
449	stw		%r21, 8(%r26)
450	stw		%r22, 12(%r26)
451	ldw		16(%r25), %r19
452	ldw		20(%r25), %r20
453	ldw		24(%r25), %r21
454	ldw		28(%r25), %r22
455	stw		%r19, 16(%r26)
456	stw		%r20, 20(%r26)
457	stw		%r21, 24(%r26)
458	stw		%r22, 28(%r26)
459	ldw		32(%r25), %r19
460	ldw		36(%r25), %r20
461	ldw		40(%r25), %r21
462	ldw		44(%r25), %r22
463	stw		%r19, 32(%r26)
464	stw		%r20, 36(%r26)
465	stw		%r21, 40(%r26)
466	stw		%r22, 44(%r26)
467	ldw		48(%r25), %r19
468	ldw		52(%r25), %r20
469	ldw		56(%r25), %r21
470	ldw		60(%r25), %r22
471	stw		%r19, 48(%r26)
472	stw		%r20, 52(%r26)
473	ldo		64(%r25), %r25
474	stw		%r21, 56(%r26)
475	stw		%r22, 60(%r26)
476	ldo		64(%r26), %r26
477	addib,COND(>),n	-1, %r1, 1b
478	ldw		0(%r25), %r19
479#endif
480	bv		%r0(%r2)
481	nop
482ENDPROC_CFI(copy_page_asm)
483
484/*
485 * NOTE: Code in clear_user_page has a hard coded dependency on the
486 *       maximum alias boundary being 4 Mb. We've been assured by the
487 *       parisc chip designers that there will not ever be a parisc
488 *       chip with a larger alias boundary (Never say never :-) ).
489 *
490 *       Yah, what about the PA8800 and PA8900 processors?
491 *
492 *       Subtle: the dtlb miss handlers support the temp alias region by
493 *       "knowing" that if a dtlb miss happens within the temp alias
494 *       region it must have occurred while in clear_user_page. Since
495 *       this routine makes use of processor local translations, we
496 *       don't want to insert them into the kernel page table. Instead,
497 *       we load up some general registers (they need to be registers
498 *       which aren't shadowed) with the physical page numbers (preshifted
499 *       for tlb insertion) needed to insert the translations. When we
500 *       miss on the translation, the dtlb miss handler inserts the
501 *       translation into the tlb using these values:
502 *
503 *          %r26 physical address of "to" translation
504 *          %r23 physical address of "from" translation
505 */
506
507	/*
508	 * copy_user_page_asm() performs a page copy using mappings
509	 * equivalent to the user page mappings.  It can be used to
510	 * implement copy_user_page() but unfortunately both the `from'
511	 * and `to' pages need to be flushed through mappings equivalent
512	 * to the user mappings after the copy because the kernel accesses
513	 * the `from' page through the kmap kernel mapping and the `to'
514	 * page needs to be flushed since code can be copied.  As a
515	 * result, this implementation is less efficient than the simpler
516	 * copy using the kernel mapping.  It only needs the `from' page
517	 * to flushed via the user mapping.  The kunmap routines handle
518	 * the flushes needed for the kernel mapping.
519	 *
520	 * I'm still keeping this around because it may be possible to
521	 * use it if more information is passed into copy_user_page().
522	 * Have to do some measurements to see if it is worthwhile to
523	 * lobby for such a change.
524	 *
525	 */
526
527ENTRY_CFI(copy_user_page_asm)
528	/* Convert virtual `to' and `from' addresses to physical addresses.
529	   Move `from' physical address to non shadowed register.  */
530	ldil		L%(__PAGE_OFFSET), %r1
531	sub		%r26, %r1, %r26
532	sub		%r25, %r1, %r23
533
534	ldil		L%(TMPALIAS_MAP_START), %r28
535	dep_safe	%r24, 31,TMPALIAS_SIZE_BITS, %r28	/* Form aliased virtual address 'to' */
536	depi_safe	0, 31,PAGE_SHIFT, %r28			/* Clear any offset bits */
537	copy		%r28, %r29
538	depi_safe	1, 31-TMPALIAS_SIZE_BITS,1, %r29	/* Form aliased virtual address 'from' */
539
540	/* Purge any old translations */
541
542#ifdef CONFIG_PA20
543	pdtlb,l		%r0(%r28)
544	pdtlb,l		%r0(%r29)
545#else
5460:	pdtlb		%r0(%r28)
5471:	pdtlb		%r0(%r29)
548	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
549	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
550#endif
551
552#ifdef CONFIG_64BIT
553	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
554	 * Unroll the loop by hand and arrange insn appropriately.
555	 * GCC probably can do this just as well.
556	 */
557
558	ldd		0(%r29), %r19
559	ldi		(PAGE_SIZE / 128), %r1
560
5611:	ldd		8(%r29), %r20
562
563	ldd		16(%r29), %r21
564	ldd		24(%r29), %r22
565	std		%r19, 0(%r28)
566	std		%r20, 8(%r28)
567
568	ldd		32(%r29), %r19
569	ldd		40(%r29), %r20
570	std		%r21, 16(%r28)
571	std		%r22, 24(%r28)
572
573	ldd		48(%r29), %r21
574	ldd		56(%r29), %r22
575	std		%r19, 32(%r28)
576	std		%r20, 40(%r28)
577
578	ldd		64(%r29), %r19
579	ldd		72(%r29), %r20
580	std		%r21, 48(%r28)
581	std		%r22, 56(%r28)
582
583	ldd		80(%r29), %r21
584	ldd		88(%r29), %r22
585	std		%r19, 64(%r28)
586	std		%r20, 72(%r28)
587
588	ldd		 96(%r29), %r19
589	ldd		104(%r29), %r20
590	std		%r21, 80(%r28)
591	std		%r22, 88(%r28)
592
593	ldd		112(%r29), %r21
594	ldd		120(%r29), %r22
595	std		%r19, 96(%r28)
596	std		%r20, 104(%r28)
597
598	ldo		128(%r29), %r29
599	std		%r21, 112(%r28)
600	std		%r22, 120(%r28)
601	ldo		128(%r28), %r28
602
603	/* conditional branches nullify on forward taken branch, and on
604	 * non-taken backward branch. Note that .+4 is a backwards branch.
605	 * The ldd should only get executed if the branch is taken.
606	 */
607	addib,COND(>),n	-1, %r1, 1b		/* bundle 10 */
608	ldd		0(%r29), %r19		/* start next loads */
609
610#else
611	ldi		(PAGE_SIZE / 64), %r1
612
613	/*
614	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
615	 * bundles (very restricted rules for bundling). It probably
616	 * does OK on PCXU and better, but we could do better with
617	 * ldd/std instructions. Note that until (if) we start saving
618	 * the full 64 bit register values on interrupt, we can't
619	 * use ldd/std on a 32 bit kernel.
620	 */
621
6221:	ldw		0(%r29), %r19
623	ldw		4(%r29), %r20
624	ldw		8(%r29), %r21
625	ldw		12(%r29), %r22
626	stw		%r19, 0(%r28)
627	stw		%r20, 4(%r28)
628	stw		%r21, 8(%r28)
629	stw		%r22, 12(%r28)
630	ldw		16(%r29), %r19
631	ldw		20(%r29), %r20
632	ldw		24(%r29), %r21
633	ldw		28(%r29), %r22
634	stw		%r19, 16(%r28)
635	stw		%r20, 20(%r28)
636	stw		%r21, 24(%r28)
637	stw		%r22, 28(%r28)
638	ldw		32(%r29), %r19
639	ldw		36(%r29), %r20
640	ldw		40(%r29), %r21
641	ldw		44(%r29), %r22
642	stw		%r19, 32(%r28)
643	stw		%r20, 36(%r28)
644	stw		%r21, 40(%r28)
645	stw		%r22, 44(%r28)
646	ldw		48(%r29), %r19
647	ldw		52(%r29), %r20
648	ldw		56(%r29), %r21
649	ldw		60(%r29), %r22
650	stw		%r19, 48(%r28)
651	stw		%r20, 52(%r28)
652	stw		%r21, 56(%r28)
653	stw		%r22, 60(%r28)
654	ldo		64(%r28), %r28
655
656	addib,COND(>)		-1, %r1,1b
657	ldo		64(%r29), %r29
658#endif
659
660	bv		%r0(%r2)
661	nop
662ENDPROC_CFI(copy_user_page_asm)
663
664ENTRY_CFI(clear_user_page_asm)
665	tophys_r1	%r26
666
667	ldil		L%(TMPALIAS_MAP_START), %r28
668	dep_safe	%r25, 31,TMPALIAS_SIZE_BITS, %r28	/* Form aliased virtual address 'to' */
669	depi_safe	0, 31,PAGE_SHIFT, %r28			/* Clear any offset bits */
670
671	/* Purge any old translation */
672
673#ifdef CONFIG_PA20
674	pdtlb,l		%r0(%r28)
675#else
6760:	pdtlb		%r0(%r28)
677	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
678#endif
679
680#ifdef CONFIG_64BIT
681	ldi		(PAGE_SIZE / 128), %r1
682
683	/* PREFETCH (Write) has not (yet) been proven to help here */
684	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
685
6861:	std		%r0, 0(%r28)
687	std		%r0, 8(%r28)
688	std		%r0, 16(%r28)
689	std		%r0, 24(%r28)
690	std		%r0, 32(%r28)
691	std		%r0, 40(%r28)
692	std		%r0, 48(%r28)
693	std		%r0, 56(%r28)
694	std		%r0, 64(%r28)
695	std		%r0, 72(%r28)
696	std		%r0, 80(%r28)
697	std		%r0, 88(%r28)
698	std		%r0, 96(%r28)
699	std		%r0, 104(%r28)
700	std		%r0, 112(%r28)
701	std		%r0, 120(%r28)
702	addib,COND(>)		-1, %r1, 1b
703	ldo		128(%r28), %r28
704
705#else	/* ! CONFIG_64BIT */
706	ldi		(PAGE_SIZE / 64), %r1
707
7081:	stw		%r0, 0(%r28)
709	stw		%r0, 4(%r28)
710	stw		%r0, 8(%r28)
711	stw		%r0, 12(%r28)
712	stw		%r0, 16(%r28)
713	stw		%r0, 20(%r28)
714	stw		%r0, 24(%r28)
715	stw		%r0, 28(%r28)
716	stw		%r0, 32(%r28)
717	stw		%r0, 36(%r28)
718	stw		%r0, 40(%r28)
719	stw		%r0, 44(%r28)
720	stw		%r0, 48(%r28)
721	stw		%r0, 52(%r28)
722	stw		%r0, 56(%r28)
723	stw		%r0, 60(%r28)
724	addib,COND(>)		-1, %r1, 1b
725	ldo		64(%r28), %r28
726#endif	/* CONFIG_64BIT */
727
728	bv		%r0(%r2)
729	nop
730ENDPROC_CFI(clear_user_page_asm)
731
732ENTRY_CFI(flush_dcache_page_asm)
733	ldil		L%(TMPALIAS_MAP_START), %r28
734	dep_safe	%r25, 31,TMPALIAS_SIZE_BITS, %r28	/* Form aliased virtual address 'to' */
735	depi_safe	0, 31,PAGE_SHIFT, %r28			/* Clear any offset bits */
736
737	/* Purge any old translation */
738
739#ifdef CONFIG_PA20
740	pdtlb,l		%r0(%r28)
741#else
7420:	pdtlb		%r0(%r28)
743	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
744#endif
745
74688:	ldil		L%dcache_stride, %r1
747	ldw		R%dcache_stride(%r1), r31
748
749#ifdef CONFIG_64BIT
750	depdi,z		1, 63-PAGE_SHIFT,1, %r25
751#else
752	depwi,z		1, 31-PAGE_SHIFT,1, %r25
753#endif
754	add		%r28, %r25, %r25
755	sub		%r25, r31, %r25
756
7571:	fdc,m		r31(%r28)
758	fdc,m		r31(%r28)
759	fdc,m		r31(%r28)
760	fdc,m		r31(%r28)
761	fdc,m		r31(%r28)
762	fdc,m		r31(%r28)
763	fdc,m		r31(%r28)
764	fdc,m		r31(%r28)
765	fdc,m		r31(%r28)
766	fdc,m		r31(%r28)
767	fdc,m		r31(%r28)
768	fdc,m		r31(%r28)
769	fdc,m		r31(%r28)
770	fdc,m		r31(%r28)
771	fdc,m		r31(%r28)
772	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
773	fdc,m		r31(%r28)
774
77589:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
776	sync
777	bv		%r0(%r2)
778	nop
779ENDPROC_CFI(flush_dcache_page_asm)
780
781ENTRY_CFI(purge_dcache_page_asm)
782	ldil		L%(TMPALIAS_MAP_START), %r28
783	dep_safe	%r25, 31,TMPALIAS_SIZE_BITS, %r28	/* Form aliased virtual address 'to' */
784	depi_safe	0, 31,PAGE_SHIFT, %r28			/* Clear any offset bits */
785
786	/* Purge any old translation */
787
788#ifdef CONFIG_PA20
789	pdtlb,l		%r0(%r28)
790#else
7910:	pdtlb		%r0(%r28)
792	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
793#endif
794
79588:	ldil		L%dcache_stride, %r1
796	ldw		R%dcache_stride(%r1), r31
797
798#ifdef CONFIG_64BIT
799	depdi,z		1, 63-PAGE_SHIFT,1, %r25
800#else
801	depwi,z		1, 31-PAGE_SHIFT,1, %r25
802#endif
803	add		%r28, %r25, %r25
804	sub		%r25, r31, %r25
805
8061:      pdc,m		r31(%r28)
807	pdc,m		r31(%r28)
808	pdc,m		r31(%r28)
809	pdc,m		r31(%r28)
810	pdc,m		r31(%r28)
811	pdc,m		r31(%r28)
812	pdc,m		r31(%r28)
813	pdc,m		r31(%r28)
814	pdc,m		r31(%r28)
815	pdc,m		r31(%r28)
816	pdc,m		r31(%r28)
817	pdc,m		r31(%r28)
818	pdc,m		r31(%r28)
819	pdc,m		r31(%r28)
820	pdc,m		r31(%r28)
821	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
822	pdc,m		r31(%r28)
823
82489:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
825	sync
826	bv		%r0(%r2)
827	nop
828ENDPROC_CFI(purge_dcache_page_asm)
829
830ENTRY_CFI(flush_icache_page_asm)
831	ldil		L%(TMPALIAS_MAP_START), %r28
832	dep_safe	%r25, 31,TMPALIAS_SIZE_BITS, %r28	/* Form aliased virtual address 'to' */
833	depi_safe	0, 31,PAGE_SHIFT, %r28			/* Clear any offset bits */
834
835	/* Purge any old translation.  Note that the FIC instruction
836	 * may use either the instruction or data TLB.  Given that we
837	 * have a flat address space, it's not clear which TLB will be
838	 * used.  So, we purge both entries.  */
839
840#ifdef CONFIG_PA20
841	pdtlb,l		%r0(%r28)
8421:	pitlb,l         %r0(%sr4,%r28)
843	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
844#else
8450:	pdtlb		%r0(%r28)
8461:	pitlb           %r0(%sr4,%r28)
847	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
848	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
849	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
850#endif
851
85288:	ldil		L%icache_stride, %r1
853	ldw		R%icache_stride(%r1), %r31
854
855#ifdef CONFIG_64BIT
856	depdi,z		1, 63-PAGE_SHIFT,1, %r25
857#else
858	depwi,z		1, 31-PAGE_SHIFT,1, %r25
859#endif
860	add		%r28, %r25, %r25
861	sub		%r25, %r31, %r25
862
863	/* fic only has the type 26 form on PA1.1, requiring an
864	 * explicit space specification, so use %sr4 */
8651:      fic,m		%r31(%sr4,%r28)
866	fic,m		%r31(%sr4,%r28)
867	fic,m		%r31(%sr4,%r28)
868	fic,m		%r31(%sr4,%r28)
869	fic,m		%r31(%sr4,%r28)
870	fic,m		%r31(%sr4,%r28)
871	fic,m		%r31(%sr4,%r28)
872	fic,m		%r31(%sr4,%r28)
873	fic,m		%r31(%sr4,%r28)
874	fic,m		%r31(%sr4,%r28)
875	fic,m		%r31(%sr4,%r28)
876	fic,m		%r31(%sr4,%r28)
877	fic,m		%r31(%sr4,%r28)
878	fic,m		%r31(%sr4,%r28)
879	fic,m		%r31(%sr4,%r28)
880	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
881	fic,m		%r31(%sr4,%r28)
882
88389:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
884	sync
885	bv		%r0(%r2)
886	nop
887ENDPROC_CFI(flush_icache_page_asm)
888
889ENTRY_CFI(flush_kernel_dcache_page_asm)
89088:	ldil		L%dcache_stride, %r1
891	ldw		R%dcache_stride(%r1), %r23
892	depi_safe	0, 31,PAGE_SHIFT, %r26	/* Clear any offset bits */
893
894#ifdef CONFIG_64BIT
895	depdi,z		1, 63-PAGE_SHIFT,1, %r25
896#else
897	depwi,z		1, 31-PAGE_SHIFT,1, %r25
898#endif
899	add		%r26, %r25, %r25
900	sub		%r25, %r23, %r25
901
9021:      fdc,m		%r23(%r26)
903	fdc,m		%r23(%r26)
904	fdc,m		%r23(%r26)
905	fdc,m		%r23(%r26)
906	fdc,m		%r23(%r26)
907	fdc,m		%r23(%r26)
908	fdc,m		%r23(%r26)
909	fdc,m		%r23(%r26)
910	fdc,m		%r23(%r26)
911	fdc,m		%r23(%r26)
912	fdc,m		%r23(%r26)
913	fdc,m		%r23(%r26)
914	fdc,m		%r23(%r26)
915	fdc,m		%r23(%r26)
916	fdc,m		%r23(%r26)
917	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
918	fdc,m		%r23(%r26)
919
92089:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
921	sync
922	bv		%r0(%r2)
923	nop
924ENDPROC_CFI(flush_kernel_dcache_page_asm)
925
926ENTRY_CFI(purge_kernel_dcache_page_asm)
92788:	ldil		L%dcache_stride, %r1
928	ldw		R%dcache_stride(%r1), %r23
929	depi_safe	0, 31,PAGE_SHIFT, %r26	/* Clear any offset bits */
930
931#ifdef CONFIG_64BIT
932	depdi,z		1, 63-PAGE_SHIFT,1, %r25
933#else
934	depwi,z		1, 31-PAGE_SHIFT,1, %r25
935#endif
936	add		%r26, %r25, %r25
937	sub		%r25, %r23, %r25
938
9391:      pdc,m		%r23(%r26)
940	pdc,m		%r23(%r26)
941	pdc,m		%r23(%r26)
942	pdc,m		%r23(%r26)
943	pdc,m		%r23(%r26)
944	pdc,m		%r23(%r26)
945	pdc,m		%r23(%r26)
946	pdc,m		%r23(%r26)
947	pdc,m		%r23(%r26)
948	pdc,m		%r23(%r26)
949	pdc,m		%r23(%r26)
950	pdc,m		%r23(%r26)
951	pdc,m		%r23(%r26)
952	pdc,m		%r23(%r26)
953	pdc,m		%r23(%r26)
954	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
955	pdc,m		%r23(%r26)
956
95789:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
958	sync
959	bv		%r0(%r2)
960	nop
961ENDPROC_CFI(purge_kernel_dcache_page_asm)
962
963ENTRY_CFI(flush_user_dcache_range_asm)
96488:	ldil		L%dcache_stride, %r1
965	ldw		R%dcache_stride(%r1), %r23
966	ldo		-1(%r23), %r21
967	ANDCM		%r26, %r21, %r26
968
969#ifdef CONFIG_64BIT
970	depd,z		%r23, 59, 60, %r21
971#else
972	depw,z		%r23, 27, 28, %r21
973#endif
974	add		%r26, %r21, %r22
975	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
9761:	add		%r22, %r21, %r22
977	fdc,m		%r23(%sr3, %r26)
978	fdc,m		%r23(%sr3, %r26)
979	fdc,m		%r23(%sr3, %r26)
980	fdc,m		%r23(%sr3, %r26)
981	fdc,m		%r23(%sr3, %r26)
982	fdc,m		%r23(%sr3, %r26)
983	fdc,m		%r23(%sr3, %r26)
984	fdc,m		%r23(%sr3, %r26)
985	fdc,m		%r23(%sr3, %r26)
986	fdc,m		%r23(%sr3, %r26)
987	fdc,m		%r23(%sr3, %r26)
988	fdc,m		%r23(%sr3, %r26)
989	fdc,m		%r23(%sr3, %r26)
990	fdc,m		%r23(%sr3, %r26)
991	fdc,m		%r23(%sr3, %r26)
992	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
993	fdc,m		%r23(%sr3, %r26)
994
9952:	cmpb,COND(>>),n	%r25, %r26, 2b
996	fdc,m		%r23(%sr3, %r26)
997
99889:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
999	sync
1000	bv		%r0(%r2)
1001	nop
1002ENDPROC_CFI(flush_user_dcache_range_asm)
1003
1004ENTRY_CFI(flush_kernel_dcache_range_asm)
100588:	ldil		L%dcache_stride, %r1
1006	ldw		R%dcache_stride(%r1), %r23
1007	ldo		-1(%r23), %r21
1008	ANDCM		%r26, %r21, %r26
1009
1010#ifdef CONFIG_64BIT
1011	depd,z		%r23, 59, 60, %r21
1012#else
1013	depw,z		%r23, 27, 28, %r21
1014#endif
1015	add		%r26, %r21, %r22
1016	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
10171:	add		%r22, %r21, %r22
1018	fdc,m		%r23(%r26)
1019	fdc,m		%r23(%r26)
1020	fdc,m		%r23(%r26)
1021	fdc,m		%r23(%r26)
1022	fdc,m		%r23(%r26)
1023	fdc,m		%r23(%r26)
1024	fdc,m		%r23(%r26)
1025	fdc,m		%r23(%r26)
1026	fdc,m		%r23(%r26)
1027	fdc,m		%r23(%r26)
1028	fdc,m		%r23(%r26)
1029	fdc,m		%r23(%r26)
1030	fdc,m		%r23(%r26)
1031	fdc,m		%r23(%r26)
1032	fdc,m		%r23(%r26)
1033	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1034	fdc,m		%r23(%r26)
1035
10362:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1037	fdc,m		%r23(%r26)
1038
1039	sync
104089:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1041	bv		%r0(%r2)
1042	nop
1043ENDPROC_CFI(flush_kernel_dcache_range_asm)
1044
1045ENTRY_CFI(purge_kernel_dcache_range_asm)
104688:	ldil		L%dcache_stride, %r1
1047	ldw		R%dcache_stride(%r1), %r23
1048	ldo		-1(%r23), %r21
1049	ANDCM		%r26, %r21, %r26
1050
1051#ifdef CONFIG_64BIT
1052	depd,z		%r23, 59, 60, %r21
1053#else
1054	depw,z		%r23, 27, 28, %r21
1055#endif
1056	add		%r26, %r21, %r22
1057	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
10581:	add		%r22, %r21, %r22
1059	pdc,m		%r23(%r26)
1060	pdc,m		%r23(%r26)
1061	pdc,m		%r23(%r26)
1062	pdc,m		%r23(%r26)
1063	pdc,m		%r23(%r26)
1064	pdc,m		%r23(%r26)
1065	pdc,m		%r23(%r26)
1066	pdc,m		%r23(%r26)
1067	pdc,m		%r23(%r26)
1068	pdc,m		%r23(%r26)
1069	pdc,m		%r23(%r26)
1070	pdc,m		%r23(%r26)
1071	pdc,m		%r23(%r26)
1072	pdc,m		%r23(%r26)
1073	pdc,m		%r23(%r26)
1074	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1075	pdc,m		%r23(%r26)
1076
10772:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1078	pdc,m		%r23(%r26)
1079
1080	sync
108189:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1082	bv		%r0(%r2)
1083	nop
1084ENDPROC_CFI(purge_kernel_dcache_range_asm)
1085
1086ENTRY_CFI(flush_user_icache_range_asm)
108788:	ldil		L%icache_stride, %r1
1088	ldw		R%icache_stride(%r1), %r23
1089	ldo		-1(%r23), %r21
1090	ANDCM		%r26, %r21, %r26
1091
1092#ifdef CONFIG_64BIT
1093	depd,z		%r23, 59, 60, %r21
1094#else
1095	depw,z		%r23, 27, 28, %r21
1096#endif
1097	add		%r26, %r21, %r22
1098	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
10991:	add		%r22, %r21, %r22
1100	fic,m		%r23(%sr3, %r26)
1101	fic,m		%r23(%sr3, %r26)
1102	fic,m		%r23(%sr3, %r26)
1103	fic,m		%r23(%sr3, %r26)
1104	fic,m		%r23(%sr3, %r26)
1105	fic,m		%r23(%sr3, %r26)
1106	fic,m		%r23(%sr3, %r26)
1107	fic,m		%r23(%sr3, %r26)
1108	fic,m		%r23(%sr3, %r26)
1109	fic,m		%r23(%sr3, %r26)
1110	fic,m		%r23(%sr3, %r26)
1111	fic,m		%r23(%sr3, %r26)
1112	fic,m		%r23(%sr3, %r26)
1113	fic,m		%r23(%sr3, %r26)
1114	fic,m		%r23(%sr3, %r26)
1115	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1116	fic,m		%r23(%sr3, %r26)
1117
11182:	cmpb,COND(>>),n	%r25, %r26, 2b
1119	fic,m		%r23(%sr3, %r26)
1120
112189:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1122	sync
1123	bv		%r0(%r2)
1124	nop
1125ENDPROC_CFI(flush_user_icache_range_asm)
1126
1127ENTRY_CFI(flush_kernel_icache_page)
112888:	ldil		L%icache_stride, %r1
1129	ldw		R%icache_stride(%r1), %r23
1130
1131#ifdef CONFIG_64BIT
1132	depdi,z		1, 63-PAGE_SHIFT,1, %r25
1133#else
1134	depwi,z		1, 31-PAGE_SHIFT,1, %r25
1135#endif
1136	add		%r26, %r25, %r25
1137	sub		%r25, %r23, %r25
1138
1139
11401:      fic,m		%r23(%sr4, %r26)
1141	fic,m		%r23(%sr4, %r26)
1142	fic,m		%r23(%sr4, %r26)
1143	fic,m		%r23(%sr4, %r26)
1144	fic,m		%r23(%sr4, %r26)
1145	fic,m		%r23(%sr4, %r26)
1146	fic,m		%r23(%sr4, %r26)
1147	fic,m		%r23(%sr4, %r26)
1148	fic,m		%r23(%sr4, %r26)
1149	fic,m		%r23(%sr4, %r26)
1150	fic,m		%r23(%sr4, %r26)
1151	fic,m		%r23(%sr4, %r26)
1152	fic,m		%r23(%sr4, %r26)
1153	fic,m		%r23(%sr4, %r26)
1154	fic,m		%r23(%sr4, %r26)
1155	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
1156	fic,m		%r23(%sr4, %r26)
1157
115889:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1159	sync
1160	bv		%r0(%r2)
1161	nop
1162ENDPROC_CFI(flush_kernel_icache_page)
1163
1164ENTRY_CFI(flush_kernel_icache_range_asm)
116588:	ldil		L%icache_stride, %r1
1166	ldw		R%icache_stride(%r1), %r23
1167	ldo		-1(%r23), %r21
1168	ANDCM		%r26, %r21, %r26
1169
1170#ifdef CONFIG_64BIT
1171	depd,z		%r23, 59, 60, %r21
1172#else
1173	depw,z		%r23, 27, 28, %r21
1174#endif
1175	add		%r26, %r21, %r22
1176	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
11771:	add		%r22, %r21, %r22
1178	fic,m		%r23(%sr4, %r26)
1179	fic,m		%r23(%sr4, %r26)
1180	fic,m		%r23(%sr4, %r26)
1181	fic,m		%r23(%sr4, %r26)
1182	fic,m		%r23(%sr4, %r26)
1183	fic,m		%r23(%sr4, %r26)
1184	fic,m		%r23(%sr4, %r26)
1185	fic,m		%r23(%sr4, %r26)
1186	fic,m		%r23(%sr4, %r26)
1187	fic,m		%r23(%sr4, %r26)
1188	fic,m		%r23(%sr4, %r26)
1189	fic,m		%r23(%sr4, %r26)
1190	fic,m		%r23(%sr4, %r26)
1191	fic,m		%r23(%sr4, %r26)
1192	fic,m		%r23(%sr4, %r26)
1193	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1194	fic,m		%r23(%sr4, %r26)
1195
11962:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1197	fic,m		%r23(%sr4, %r26)
1198
119989:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1200	sync
1201	bv		%r0(%r2)
1202	nop
1203ENDPROC_CFI(flush_kernel_icache_range_asm)
1204
1205	.text
1206
1207	/* align should cover use of rfi in disable_sr_hashing_asm and
1208	 * srdis_done.
1209	 */
1210	.align	256
1211ENTRY_CFI(disable_sr_hashing_asm)
1212	/*
1213	 * Switch to real mode
1214	 */
1215	/* pcxt_ssm_bug */
1216	rsm		PSW_SM_I, %r0
1217	load32		PA(1f), %r1
1218	nop
1219	nop
1220	nop
1221	nop
1222	nop
1223
1224	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1225	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1226	mtctl		%r0, %cr17		/* Clear IIASQ head */
1227	mtctl		%r1, %cr18		/* IIAOQ head */
1228	ldo		4(%r1), %r1
1229	mtctl		%r1, %cr18		/* IIAOQ tail */
1230	load32		REAL_MODE_PSW, %r1
1231	mtctl		%r1, %ipsw
1232	rfi
1233	nop
1234
12351:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1236	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1237	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1238	b,n		srdis_done
1239
1240srdis_pcxs:
1241
1242	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1243
1244	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1245	.word		0x141c1a00		/* must issue twice */
1246	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1247	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1248	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1249	.word		0x141c1600		/* must issue twice */
1250	b,n		srdis_done
1251
1252srdis_pcxl:
1253
1254	/* Disable Space Register Hashing for PCXL */
1255
1256	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1257	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1258	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1259	b,n		srdis_done
1260
1261srdis_pa20:
1262
1263	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1264
1265	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1266	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1267	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1268
1269
1270srdis_done:
1271	/* Switch back to virtual mode */
1272	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1273	load32 	   	2f, %r1
1274	nop
1275	nop
1276	nop
1277	nop
1278	nop
1279
1280	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1281	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1282	mtctl		%r0, %cr17		/* Clear IIASQ head */
1283	mtctl		%r1, %cr18		/* IIAOQ head */
1284	ldo		4(%r1), %r1
1285	mtctl		%r1, %cr18		/* IIAOQ tail */
1286	load32		KERNEL_PSW, %r1
1287	mtctl		%r1, %ipsw
1288	rfi
1289	nop
1290
12912:      bv		%r0(%r2)
1292	nop
1293ENDPROC_CFI(disable_sr_hashing_asm)
1294
1295	.end
1296