xref: /openbmc/linux/arch/parisc/kernel/pacache.S (revision 87c2ce3b) 
1/*
2 *  PARISC TLB and cache flushing support
3 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 *    This program is free software; you can redistribute it and/or modify
8 *    it under the terms of the GNU General Public License as published by
9 *    the Free Software Foundation; either version 2, or (at your option)
10 *    any later version.
11 *
12 *    This program is distributed in the hope that it will be useful,
13 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 *    GNU General Public License for more details.
16 *
17 *    You should have received a copy of the GNU General Public License
18 *    along with this program; if not, write to the Free Software
19 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 *       should only use index and base registers that are not shadowed,
25 *       so that the fast path emulation in the non access miss handler
26 *       can be used.
27 */
28
29#ifdef CONFIG_64BIT
30#define ADDIB	addib,*
31#define CMPB	cmpb,*
32#define ANDCM	andcm,*
33
34	.level	2.0w
35#else
36#define ADDIB	addib,
37#define CMPB	cmpb,
38#define ANDCM	andcm
39
40	.level	2.0
41#endif
42
43#include <linux/config.h>
44
45#include <asm/psw.h>
46#include <asm/assembly.h>
47#include <asm/pgtable.h>
48#include <asm/cache.h>
49
50	.text
51	.align	128
52
53	.export flush_tlb_all_local,code
54
55flush_tlb_all_local:
56	.proc
57	.callinfo NO_CALLS
58	.entry
59
60	/*
61	 * The pitlbe and pdtlbe instructions should only be used to
62	 * flush the entire tlb. Also, there needs to be no intervening
63	 * tlb operations, e.g. tlb misses, so the operation needs
64	 * to happen in real mode with all interruptions disabled.
65	 */
66
67	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
68	rsm	PSW_SM_I, %r19		/* save I-bit state */
69	load32		PA(1f), %r1
70	nop
71	nop
72	nop
73	nop
74	nop
75
76	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
77	mtctl		%r0, %cr17		/* Clear IIASQ tail */
78	mtctl		%r0, %cr17		/* Clear IIASQ head */
79	mtctl		%r1, %cr18		/* IIAOQ head */
80	ldo		4(%r1), %r1
81	mtctl		%r1, %cr18		/* IIAOQ tail */
82	load32		REAL_MODE_PSW, %r1
83	mtctl           %r1, %ipsw
84	rfi
85	nop
86
871:      ldil		L%PA(cache_info), %r1
88	ldo		R%PA(cache_info)(%r1), %r1
89
90	/* Flush Instruction Tlb */
91
92	LDREG		ITLB_SID_BASE(%r1), %r20
93	LDREG		ITLB_SID_STRIDE(%r1), %r21
94	LDREG		ITLB_SID_COUNT(%r1), %r22
95	LDREG		ITLB_OFF_BASE(%r1), %arg0
96	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
97	LDREG		ITLB_OFF_COUNT(%r1), %arg2
98	LDREG		ITLB_LOOP(%r1), %arg3
99
100	ADDIB=		-1, %arg3, fitoneloop	/* Preadjust and test */
101	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
102	copy		%arg0, %r28		/* Init base addr */
103
104fitmanyloop:					/* Loop if LOOP >= 2 */
105	mtsp		%r20, %sr1
106	add		%r21, %r20, %r20	/* increment space */
107	copy		%arg2, %r29		/* Init middle loop count */
108
109fitmanymiddle:					/* Loop if LOOP >= 2 */
110	ADDIB>		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
111	pitlbe		0(%sr1, %r28)
112	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
113	ADDIB>		-1, %r29, fitmanymiddle	/* Middle loop decr */
114	copy		%arg3, %r31		/* Re-init inner loop count */
115
116	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
117	ADDIB<=,n	-1, %r22, fitdone	/* Outer loop count decr */
118
119fitoneloop:					/* Loop if LOOP = 1 */
120	mtsp		%r20, %sr1
121	copy		%arg0, %r28		/* init base addr */
122	copy		%arg2, %r29		/* init middle loop count */
123
124fitonemiddle:					/* Loop if LOOP = 1 */
125	ADDIB>		-1, %r29, fitonemiddle	/* Middle loop count decr */
126	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
127
128	ADDIB>		-1, %r22, fitoneloop	/* Outer loop count decr */
129	add		%r21, %r20, %r20		/* increment space */
130
131fitdone:
132
133	/* Flush Data Tlb */
134
135	LDREG		DTLB_SID_BASE(%r1), %r20
136	LDREG		DTLB_SID_STRIDE(%r1), %r21
137	LDREG		DTLB_SID_COUNT(%r1), %r22
138	LDREG		DTLB_OFF_BASE(%r1), %arg0
139	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
140	LDREG		DTLB_OFF_COUNT(%r1), %arg2
141	LDREG		DTLB_LOOP(%r1), %arg3
142
143	ADDIB=		-1, %arg3, fdtoneloop	/* Preadjust and test */
144	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
145	copy		%arg0, %r28		/* Init base addr */
146
147fdtmanyloop:					/* Loop if LOOP >= 2 */
148	mtsp		%r20, %sr1
149	add		%r21, %r20, %r20	/* increment space */
150	copy		%arg2, %r29		/* Init middle loop count */
151
152fdtmanymiddle:					/* Loop if LOOP >= 2 */
153	ADDIB>		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
154	pdtlbe		0(%sr1, %r28)
155	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
156	ADDIB>		-1, %r29, fdtmanymiddle	/* Middle loop decr */
157	copy		%arg3, %r31		/* Re-init inner loop count */
158
159	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
160	ADDIB<=,n	-1, %r22,fdtdone	/* Outer loop count decr */
161
162fdtoneloop:					/* Loop if LOOP = 1 */
163	mtsp		%r20, %sr1
164	copy		%arg0, %r28		/* init base addr */
165	copy		%arg2, %r29		/* init middle loop count */
166
167fdtonemiddle:					/* Loop if LOOP = 1 */
168	ADDIB>		-1, %r29, fdtonemiddle	/* Middle loop count decr */
169	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
170
171	ADDIB>		-1, %r22, fdtoneloop	/* Outer loop count decr */
172	add		%r21, %r20, %r20	/* increment space */
173
174
175fdtdone:
176	/*
177	 * Switch back to virtual mode
178	 */
179	/* pcxt_ssm_bug */
180	rsm		PSW_SM_I, %r0
181	load32		2f, %r1
182	nop
183	nop
184	nop
185	nop
186	nop
187
188	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
189	mtctl		%r0, %cr17		/* Clear IIASQ tail */
190	mtctl		%r0, %cr17		/* Clear IIASQ head */
191	mtctl		%r1, %cr18		/* IIAOQ head */
192	ldo		4(%r1), %r1
193	mtctl		%r1, %cr18		/* IIAOQ tail */
194	load32		KERNEL_PSW, %r1
195	or		%r1, %r19, %r1	/* I-bit to state on entry */
196	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
197	rfi
198	nop
199
2002:      bv		%r0(%r2)
201	nop
202
203	.exit
204	.procend
205
206	.export flush_instruction_cache_local,code
207	.import cache_info,data
208
209flush_instruction_cache_local:
210	.proc
211	.callinfo NO_CALLS
212	.entry
213
214	mtsp		%r0, %sr1
215	ldil		L%cache_info, %r1
216	ldo		R%cache_info(%r1), %r1
217
218	/* Flush Instruction Cache */
219
220	LDREG		ICACHE_BASE(%r1), %arg0
221	LDREG		ICACHE_STRIDE(%r1), %arg1
222	LDREG		ICACHE_COUNT(%r1), %arg2
223	LDREG		ICACHE_LOOP(%r1), %arg3
224	rsm             PSW_SM_I, %r22		/* No mmgt ops during loop*/
225	ADDIB=		-1, %arg3, fioneloop	/* Preadjust and test */
226	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
227
228fimanyloop:					/* Loop if LOOP >= 2 */
229	ADDIB>		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
230	fice            %r0(%sr1, %arg0)
231	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
232	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
233	ADDIB<=,n	-1, %arg2, fisync	/* Outer loop decr */
234
235fioneloop:					/* Loop if LOOP = 1 */
236	ADDIB>		-1, %arg2, fioneloop	/* Outer loop count decr */
237	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
238
239fisync:
240	sync
241	mtsm		%r22			/* restore I-bit */
242	bv		%r0(%r2)
243	nop
244	.exit
245
246	.procend
247
248	.export flush_data_cache_local, code
249	.import cache_info, data
250
251flush_data_cache_local:
252	.proc
253	.callinfo NO_CALLS
254	.entry
255
256	mtsp		%r0, %sr1
257	ldil		L%cache_info, %r1
258	ldo		R%cache_info(%r1), %r1
259
260	/* Flush Data Cache */
261
262	LDREG		DCACHE_BASE(%r1), %arg0
263	LDREG		DCACHE_STRIDE(%r1), %arg1
264	LDREG		DCACHE_COUNT(%r1), %arg2
265	LDREG		DCACHE_LOOP(%r1), %arg3
266	rsm		PSW_SM_I, %r22
267	ADDIB=		-1, %arg3, fdoneloop	/* Preadjust and test */
268	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
269
270fdmanyloop:					/* Loop if LOOP >= 2 */
271	ADDIB>		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
272	fdce		%r0(%sr1, %arg0)
273	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
274	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
275	ADDIB<=,n	-1, %arg2, fdsync	/* Outer loop decr */
276
277fdoneloop:					/* Loop if LOOP = 1 */
278	ADDIB>		-1, %arg2, fdoneloop	/* Outer loop count decr */
279	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
280
281fdsync:
282	syncdma
283	sync
284	mtsm		%r22			/* restore I-bit */
285	bv		%r0(%r2)
286	nop
287	.exit
288
289	.procend
290
291	.export copy_user_page_asm,code
292	.align	16
293
294copy_user_page_asm:
295	.proc
296	.callinfo NO_CALLS
297	.entry
298
299#ifdef CONFIG_64BIT
300	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
301	 * Unroll the loop by hand and arrange insn appropriately.
302	 * GCC probably can do this just as well.
303	 */
304
305	ldd		0(%r25), %r19
306	ldi		32, %r1                 /* PAGE_SIZE/128 == 32 */
307	ldw		64(%r25), %r0		/* prefetch 1 cacheline ahead */
308	ldw		128(%r25), %r0		/* prefetch 2 */
309
3101:	ldd		8(%r25), %r20
311	ldw		192(%r25), %r0		/* prefetch 3 */
312	ldw		256(%r25), %r0		/* prefetch 4 */
313
314	ldd		16(%r25), %r21
315	ldd		24(%r25), %r22
316	std		%r19, 0(%r26)
317	std		%r20, 8(%r26)
318
319	ldd		32(%r25), %r19
320	ldd		40(%r25), %r20
321	std		%r21, 16(%r26)
322	std		%r22, 24(%r26)
323
324	ldd		48(%r25), %r21
325	ldd		56(%r25), %r22
326	std		%r19, 32(%r26)
327	std		%r20, 40(%r26)
328
329	ldd		64(%r25), %r19
330	ldd		72(%r25), %r20
331	std		%r21, 48(%r26)
332	std		%r22, 56(%r26)
333
334	ldd		80(%r25), %r21
335	ldd		88(%r25), %r22
336	std		%r19, 64(%r26)
337	std		%r20, 72(%r26)
338
339	ldd		 96(%r25), %r19
340	ldd		104(%r25), %r20
341	std		%r21, 80(%r26)
342	std		%r22, 88(%r26)
343
344	ldd		112(%r25), %r21
345	ldd		120(%r25), %r22
346	std		%r19, 96(%r26)
347	std		%r20, 104(%r26)
348
349	ldo		128(%r25), %r25
350	std		%r21, 112(%r26)
351	std		%r22, 120(%r26)
352	ldo		128(%r26), %r26
353
354	/* conditional branches nullify on forward taken branch, and on
355	 * non-taken backward branch. Note that .+4 is a backwards branch.
356	 * The ldd should only get executed if the branch is taken.
357	 */
358	ADDIB>,n	-1, %r1, 1b		/* bundle 10 */
359	ldd		0(%r25), %r19		/* start next loads */
360
361#else
362
363	/*
364	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
365	 * bundles (very restricted rules for bundling).
366	 * Note that until (if) we start saving
367	 * the full 64 bit register values on interrupt, we can't
368	 * use ldd/std on a 32 bit kernel.
369	 */
370	ldw		0(%r25), %r19
371	ldi		64, %r1		/* PAGE_SIZE/64 == 64 */
372
3731:
374	ldw		4(%r25), %r20
375	ldw		8(%r25), %r21
376	ldw		12(%r25), %r22
377	stw		%r19, 0(%r26)
378	stw		%r20, 4(%r26)
379	stw		%r21, 8(%r26)
380	stw		%r22, 12(%r26)
381	ldw		16(%r25), %r19
382	ldw		20(%r25), %r20
383	ldw		24(%r25), %r21
384	ldw		28(%r25), %r22
385	stw		%r19, 16(%r26)
386	stw		%r20, 20(%r26)
387	stw		%r21, 24(%r26)
388	stw		%r22, 28(%r26)
389	ldw		32(%r25), %r19
390	ldw		36(%r25), %r20
391	ldw		40(%r25), %r21
392	ldw		44(%r25), %r22
393	stw		%r19, 32(%r26)
394	stw		%r20, 36(%r26)
395	stw		%r21, 40(%r26)
396	stw		%r22, 44(%r26)
397	ldw		48(%r25), %r19
398	ldw		52(%r25), %r20
399	ldw		56(%r25), %r21
400	ldw		60(%r25), %r22
401	stw		%r19, 48(%r26)
402	stw		%r20, 52(%r26)
403	ldo		64(%r25), %r25
404	stw		%r21, 56(%r26)
405	stw		%r22, 60(%r26)
406	ldo		64(%r26), %r26
407	ADDIB>,n	-1, %r1, 1b
408	ldw		0(%r25), %r19
409#endif
410	bv		%r0(%r2)
411	nop
412	.exit
413
414	.procend
415
416/*
417 * NOTE: Code in clear_user_page has a hard coded dependency on the
418 *       maximum alias boundary being 4 Mb. We've been assured by the
419 *       parisc chip designers that there will not ever be a parisc
420 *       chip with a larger alias boundary (Never say never :-) ).
421 *
422 *       Subtle: the dtlb miss handlers support the temp alias region by
423 *       "knowing" that if a dtlb miss happens within the temp alias
424 *       region it must have occurred while in clear_user_page. Since
425 *       this routine makes use of processor local translations, we
426 *       don't want to insert them into the kernel page table. Instead,
427 *       we load up some general registers (they need to be registers
428 *       which aren't shadowed) with the physical page numbers (preshifted
429 *       for tlb insertion) needed to insert the translations. When we
430 *       miss on the translation, the dtlb miss handler inserts the
431 *       translation into the tlb using these values:
432 *
433 *          %r26 physical page (shifted for tlb insert) of "to" translation
434 *          %r23 physical page (shifted for tlb insert) of "from" translation
435 */
436
437#if 0
438
439	/*
440	 * We can't do this since copy_user_page is used to bring in
441	 * file data that might have instructions. Since the data would
442	 * then need to be flushed out so the i-fetch can see it, it
443	 * makes more sense to just copy through the kernel translation
444	 * and flush it.
445	 *
446	 * I'm still keeping this around because it may be possible to
447	 * use it if more information is passed into copy_user_page().
448	 * Have to do some measurements to see if it is worthwhile to
449	 * lobby for such a change.
450	 */
451
452	.export copy_user_page_asm,code
453
454copy_user_page_asm:
455	.proc
456	.callinfo NO_CALLS
457	.entry
458
459	ldil		L%(__PAGE_OFFSET), %r1
460	sub		%r26, %r1, %r26
461	sub		%r25, %r1, %r23		/* move physical addr into non shadowed reg */
462
463	ldil		L%(TMPALIAS_MAP_START), %r28
464#ifdef CONFIG_64BIT
465	extrd,u		%r26,56,32, %r26		/* convert phys addr to tlb insert format */
466	extrd,u		%r23,56,32, %r23		/* convert phys addr to tlb insert format */
467	depd		%r24,63,22, %r28		/* Form aliased virtual address 'to' */
468	depdi		0, 63,12, %r28		/* Clear any offset bits */
469	copy		%r28, %r29
470	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
471#else
472	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
473	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
474	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
475	depwi		0, 31,12, %r28		/* Clear any offset bits */
476	copy		%r28, %r29
477	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
478#endif
479
480	/* Purge any old translations */
481
482	pdtlb		0(%r28)
483	pdtlb		0(%r29)
484
485	ldi		64, %r1
486
487	/*
488	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
489	 * bundles (very restricted rules for bundling). It probably
490	 * does OK on PCXU and better, but we could do better with
491	 * ldd/std instructions. Note that until (if) we start saving
492	 * the full 64 bit register values on interrupt, we can't
493	 * use ldd/std on a 32 bit kernel.
494	 */
495
496
4971:
498	ldw		0(%r29), %r19
499	ldw		4(%r29), %r20
500	ldw		8(%r29), %r21
501	ldw		12(%r29), %r22
502	stw		%r19, 0(%r28)
503	stw		%r20, 4(%r28)
504	stw		%r21, 8(%r28)
505	stw		%r22, 12(%r28)
506	ldw		16(%r29), %r19
507	ldw		20(%r29), %r20
508	ldw		24(%r29), %r21
509	ldw		28(%r29), %r22
510	stw		%r19, 16(%r28)
511	stw		%r20, 20(%r28)
512	stw		%r21, 24(%r28)
513	stw		%r22, 28(%r28)
514	ldw		32(%r29), %r19
515	ldw		36(%r29), %r20
516	ldw		40(%r29), %r21
517	ldw		44(%r29), %r22
518	stw		%r19, 32(%r28)
519	stw		%r20, 36(%r28)
520	stw		%r21, 40(%r28)
521	stw		%r22, 44(%r28)
522	ldw		48(%r29), %r19
523	ldw		52(%r29), %r20
524	ldw		56(%r29), %r21
525	ldw		60(%r29), %r22
526	stw		%r19, 48(%r28)
527	stw		%r20, 52(%r28)
528	stw		%r21, 56(%r28)
529	stw		%r22, 60(%r28)
530	ldo		64(%r28), %r28
531	ADDIB>		-1, %r1,1b
532	ldo		64(%r29), %r29
533
534	bv		%r0(%r2)
535	nop
536	.exit
537
538	.procend
539#endif
540
541	.export __clear_user_page_asm,code
542
543__clear_user_page_asm:
544	.proc
545	.callinfo NO_CALLS
546	.entry
547
548	tophys_r1	%r26
549
550	ldil		L%(TMPALIAS_MAP_START), %r28
551#ifdef CONFIG_64BIT
552#if (TMPALIAS_MAP_START >= 0x80000000)
553	depdi		0, 31,32, %r28		/* clear any sign extension */
554#endif
555	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
556	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
557	depdi		0, 63,12, %r28		/* Clear any offset bits */
558#else
559	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
560	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
561	depwi		0, 31,12, %r28		/* Clear any offset bits */
562#endif
563
564	/* Purge any old translation */
565
566	pdtlb		0(%r28)
567
568#ifdef CONFIG_64BIT
569	ldi		32, %r1			/* PAGE_SIZE/128 == 32 */
570
571	/* PREFETCH (Write) has not (yet) been proven to help here */
572/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
573
5741:	std		%r0, 0(%r28)
575	std		%r0, 8(%r28)
576	std		%r0, 16(%r28)
577	std		%r0, 24(%r28)
578	std		%r0, 32(%r28)
579	std		%r0, 40(%r28)
580	std		%r0, 48(%r28)
581	std		%r0, 56(%r28)
582	std		%r0, 64(%r28)
583	std		%r0, 72(%r28)
584	std		%r0, 80(%r28)
585	std		%r0, 88(%r28)
586	std		%r0, 96(%r28)
587	std		%r0, 104(%r28)
588	std		%r0, 112(%r28)
589	std		%r0, 120(%r28)
590	ADDIB>		-1, %r1, 1b
591	ldo		128(%r28), %r28
592
593#else	/* ! CONFIG_64BIT */
594
595	ldi		64, %r1			/* PAGE_SIZE/64 == 64 */
596
5971:
598	stw		%r0, 0(%r28)
599	stw		%r0, 4(%r28)
600	stw		%r0, 8(%r28)
601	stw		%r0, 12(%r28)
602	stw		%r0, 16(%r28)
603	stw		%r0, 20(%r28)
604	stw		%r0, 24(%r28)
605	stw		%r0, 28(%r28)
606	stw		%r0, 32(%r28)
607	stw		%r0, 36(%r28)
608	stw		%r0, 40(%r28)
609	stw		%r0, 44(%r28)
610	stw		%r0, 48(%r28)
611	stw		%r0, 52(%r28)
612	stw		%r0, 56(%r28)
613	stw		%r0, 60(%r28)
614	ADDIB>		-1, %r1, 1b
615	ldo		64(%r28), %r28
616#endif	/* CONFIG_64BIT */
617
618	bv		%r0(%r2)
619	nop
620	.exit
621
622	.procend
623
624	.export flush_kernel_dcache_page
625
626flush_kernel_dcache_page:
627	.proc
628	.callinfo NO_CALLS
629	.entry
630
631	ldil		L%dcache_stride, %r1
632	ldw		R%dcache_stride(%r1), %r23
633
634#ifdef CONFIG_64BIT
635	depdi,z		1, 63-PAGE_SHIFT,1, %r25
636#else
637	depwi,z		1, 31-PAGE_SHIFT,1, %r25
638#endif
639	add		%r26, %r25, %r25
640	sub		%r25, %r23, %r25
641
642
6431:      fdc,m		%r23(%r26)
644	fdc,m		%r23(%r26)
645	fdc,m		%r23(%r26)
646	fdc,m		%r23(%r26)
647	fdc,m		%r23(%r26)
648	fdc,m		%r23(%r26)
649	fdc,m		%r23(%r26)
650	fdc,m		%r23(%r26)
651	fdc,m		%r23(%r26)
652	fdc,m		%r23(%r26)
653	fdc,m		%r23(%r26)
654	fdc,m		%r23(%r26)
655	fdc,m		%r23(%r26)
656	fdc,m		%r23(%r26)
657	fdc,m		%r23(%r26)
658	CMPB<<		%r26, %r25,1b
659	fdc,m		%r23(%r26)
660
661	sync
662	bv		%r0(%r2)
663	nop
664	.exit
665
666	.procend
667
668	.export flush_user_dcache_page
669
670flush_user_dcache_page:
671	.proc
672	.callinfo NO_CALLS
673	.entry
674
675	ldil		L%dcache_stride, %r1
676	ldw		R%dcache_stride(%r1), %r23
677
678#ifdef CONFIG_64BIT
679	depdi,z		1,63-PAGE_SHIFT,1, %r25
680#else
681	depwi,z		1,31-PAGE_SHIFT,1, %r25
682#endif
683	add		%r26, %r25, %r25
684	sub		%r25, %r23, %r25
685
686
6871:      fdc,m		%r23(%sr3, %r26)
688	fdc,m		%r23(%sr3, %r26)
689	fdc,m		%r23(%sr3, %r26)
690	fdc,m		%r23(%sr3, %r26)
691	fdc,m		%r23(%sr3, %r26)
692	fdc,m		%r23(%sr3, %r26)
693	fdc,m		%r23(%sr3, %r26)
694	fdc,m		%r23(%sr3, %r26)
695	fdc,m		%r23(%sr3, %r26)
696	fdc,m		%r23(%sr3, %r26)
697	fdc,m		%r23(%sr3, %r26)
698	fdc,m		%r23(%sr3, %r26)
699	fdc,m		%r23(%sr3, %r26)
700	fdc,m		%r23(%sr3, %r26)
701	fdc,m		%r23(%sr3, %r26)
702	CMPB<<		%r26, %r25,1b
703	fdc,m		%r23(%sr3, %r26)
704
705	sync
706	bv		%r0(%r2)
707	nop
708	.exit
709
710	.procend
711
712	.export flush_user_icache_page
713
714flush_user_icache_page:
715	.proc
716	.callinfo NO_CALLS
717	.entry
718
719	ldil		L%dcache_stride, %r1
720	ldw		R%dcache_stride(%r1), %r23
721
722#ifdef CONFIG_64BIT
723	depdi,z		1, 63-PAGE_SHIFT,1, %r25
724#else
725	depwi,z		1, 31-PAGE_SHIFT,1, %r25
726#endif
727	add		%r26, %r25, %r25
728	sub		%r25, %r23, %r25
729
730
7311:      fic,m		%r23(%sr3, %r26)
732	fic,m		%r23(%sr3, %r26)
733	fic,m		%r23(%sr3, %r26)
734	fic,m		%r23(%sr3, %r26)
735	fic,m		%r23(%sr3, %r26)
736	fic,m		%r23(%sr3, %r26)
737	fic,m		%r23(%sr3, %r26)
738	fic,m		%r23(%sr3, %r26)
739	fic,m		%r23(%sr3, %r26)
740	fic,m		%r23(%sr3, %r26)
741	fic,m		%r23(%sr3, %r26)
742	fic,m		%r23(%sr3, %r26)
743	fic,m		%r23(%sr3, %r26)
744	fic,m		%r23(%sr3, %r26)
745	fic,m		%r23(%sr3, %r26)
746	CMPB<<		%r26, %r25,1b
747	fic,m		%r23(%sr3, %r26)
748
749	sync
750	bv		%r0(%r2)
751	nop
752	.exit
753
754	.procend
755
756
757	.export purge_kernel_dcache_page
758
759purge_kernel_dcache_page:
760	.proc
761	.callinfo NO_CALLS
762	.entry
763
764	ldil		L%dcache_stride, %r1
765	ldw		R%dcache_stride(%r1), %r23
766
767#ifdef CONFIG_64BIT
768	depdi,z		1, 63-PAGE_SHIFT,1, %r25
769#else
770	depwi,z		1, 31-PAGE_SHIFT,1, %r25
771#endif
772	add		%r26, %r25, %r25
773	sub		%r25, %r23, %r25
774
7751:      pdc,m		%r23(%r26)
776	pdc,m		%r23(%r26)
777	pdc,m		%r23(%r26)
778	pdc,m		%r23(%r26)
779	pdc,m		%r23(%r26)
780	pdc,m		%r23(%r26)
781	pdc,m		%r23(%r26)
782	pdc,m		%r23(%r26)
783	pdc,m		%r23(%r26)
784	pdc,m		%r23(%r26)
785	pdc,m		%r23(%r26)
786	pdc,m		%r23(%r26)
787	pdc,m		%r23(%r26)
788	pdc,m		%r23(%r26)
789	pdc,m		%r23(%r26)
790	CMPB<<		%r26, %r25, 1b
791	pdc,m		%r23(%r26)
792
793	sync
794	bv		%r0(%r2)
795	nop
796	.exit
797
798	.procend
799
800#if 0
801	/* Currently not used, but it still is a possible alternate
802	 * solution.
803	 */
804
805	.export flush_alias_page
806
807flush_alias_page:
808	.proc
809	.callinfo NO_CALLS
810	.entry
811
812	tophys_r1		%r26
813
814	ldil		L%(TMPALIAS_MAP_START), %r28
815#ifdef CONFIG_64BIT
816	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
817	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
818	depdi		0, 63,12, %r28		/* Clear any offset bits */
819#else
820	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
821	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
822	depwi		0, 31,12, %r28		/* Clear any offset bits */
823#endif
824
825	/* Purge any old translation */
826
827	pdtlb		0(%r28)
828
829	ldil		L%dcache_stride, %r1
830	ldw		R%dcache_stride(%r1), %r23
831
832#ifdef CONFIG_64BIT
833	depdi,z		1, 63-PAGE_SHIFT,1, %r29
834#else
835	depwi,z		1, 31-PAGE_SHIFT,1, %r29
836#endif
837	add		%r28, %r29, %r29
838	sub		%r29, %r23, %r29
839
8401:      fdc,m		%r23(%r28)
841	fdc,m		%r23(%r28)
842	fdc,m		%r23(%r28)
843	fdc,m		%r23(%r28)
844	fdc,m		%r23(%r28)
845	fdc,m		%r23(%r28)
846	fdc,m		%r23(%r28)
847	fdc,m		%r23(%r28)
848	fdc,m		%r23(%r28)
849	fdc,m		%r23(%r28)
850	fdc,m		%r23(%r28)
851	fdc,m		%r23(%r28)
852	fdc,m		%r23(%r28)
853	fdc,m		%r23(%r28)
854	fdc,m		%r23(%r28)
855	CMPB<<		%r28, %r29, 1b
856	fdc,m		%r23(%r28)
857
858	sync
859	bv		%r0(%r2)
860	nop
861	.exit
862
863	.procend
864#endif
865
866	.export flush_user_dcache_range_asm
867
868flush_user_dcache_range_asm:
869	.proc
870	.callinfo NO_CALLS
871	.entry
872
873	ldil		L%dcache_stride, %r1
874	ldw		R%dcache_stride(%r1), %r23
875	ldo		-1(%r23), %r21
876	ANDCM		%r26, %r21, %r26
877
8781:      CMPB<<,n	%r26, %r25, 1b
879	fdc,m		%r23(%sr3, %r26)
880
881	sync
882	bv		%r0(%r2)
883	nop
884	.exit
885
886	.procend
887
888	.export flush_kernel_dcache_range_asm
889
890flush_kernel_dcache_range_asm:
891	.proc
892	.callinfo NO_CALLS
893	.entry
894
895	ldil		L%dcache_stride, %r1
896	ldw		R%dcache_stride(%r1), %r23
897	ldo		-1(%r23), %r21
898	ANDCM		%r26, %r21, %r26
899
9001:      CMPB<<,n	%r26, %r25,1b
901	fdc,m		%r23(%r26)
902
903	sync
904	syncdma
905	bv		%r0(%r2)
906	nop
907	.exit
908
909	.procend
910
911	.export flush_user_icache_range_asm
912
913flush_user_icache_range_asm:
914	.proc
915	.callinfo NO_CALLS
916	.entry
917
918	ldil		L%icache_stride, %r1
919	ldw		R%icache_stride(%r1), %r23
920	ldo		-1(%r23), %r21
921	ANDCM		%r26, %r21, %r26
922
9231:      CMPB<<,n	%r26, %r25,1b
924	fic,m		%r23(%sr3, %r26)
925
926	sync
927	bv		%r0(%r2)
928	nop
929	.exit
930
931	.procend
932
933	.export flush_kernel_icache_page
934
935flush_kernel_icache_page:
936	.proc
937	.callinfo NO_CALLS
938	.entry
939
940	ldil		L%icache_stride, %r1
941	ldw		R%icache_stride(%r1), %r23
942
943#ifdef CONFIG_64BIT
944	depdi,z		1, 63-PAGE_SHIFT,1, %r25
945#else
946	depwi,z		1, 31-PAGE_SHIFT,1, %r25
947#endif
948	add		%r26, %r25, %r25
949	sub		%r25, %r23, %r25
950
951
9521:      fic,m		%r23(%sr4, %r26)
953	fic,m		%r23(%sr4, %r26)
954	fic,m		%r23(%sr4, %r26)
955	fic,m		%r23(%sr4, %r26)
956	fic,m		%r23(%sr4, %r26)
957	fic,m		%r23(%sr4, %r26)
958	fic,m		%r23(%sr4, %r26)
959	fic,m		%r23(%sr4, %r26)
960	fic,m		%r23(%sr4, %r26)
961	fic,m		%r23(%sr4, %r26)
962	fic,m		%r23(%sr4, %r26)
963	fic,m		%r23(%sr4, %r26)
964	fic,m		%r23(%sr4, %r26)
965	fic,m		%r23(%sr4, %r26)
966	fic,m		%r23(%sr4, %r26)
967	CMPB<<		%r26, %r25, 1b
968	fic,m		%r23(%sr4, %r26)
969
970	sync
971	bv		%r0(%r2)
972	nop
973	.exit
974
975	.procend
976
977	.export flush_kernel_icache_range_asm
978
979flush_kernel_icache_range_asm:
980	.proc
981	.callinfo NO_CALLS
982	.entry
983
984	ldil		L%icache_stride, %r1
985	ldw		R%icache_stride(%r1), %r23
986	ldo		-1(%r23), %r21
987	ANDCM		%r26, %r21, %r26
988
9891:      CMPB<<,n	%r26, %r25, 1b
990	fic,m		%r23(%sr4, %r26)
991
992	sync
993	bv		%r0(%r2)
994	nop
995	.exit
996	.procend
997
998	/* align should cover use of rfi in disable_sr_hashing_asm and
999	 * srdis_done.
1000	 */
1001	.align	256
1002	.export disable_sr_hashing_asm,code
1003
1004disable_sr_hashing_asm:
1005	.proc
1006	.callinfo NO_CALLS
1007	.entry
1008
1009	/*
1010	 * Switch to real mode
1011	 */
1012	/* pcxt_ssm_bug */
1013	rsm		PSW_SM_I, %r0
1014	load32		PA(1f), %r1
1015	nop
1016	nop
1017	nop
1018	nop
1019	nop
1020
1021	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1022	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1023	mtctl		%r0, %cr17		/* Clear IIASQ head */
1024	mtctl		%r1, %cr18		/* IIAOQ head */
1025	ldo		4(%r1), %r1
1026	mtctl		%r1, %cr18		/* IIAOQ tail */
1027	load32		REAL_MODE_PSW, %r1
1028	mtctl		%r1, %ipsw
1029	rfi
1030	nop
1031
10321:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1033	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1034	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1035	b,n		srdis_done
1036
1037srdis_pcxs:
1038
1039	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1040
1041	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1042	.word		0x141c1a00		/* must issue twice */
1043	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1044	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1045	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1046	.word		0x141c1600		/* must issue twice */
1047	b,n		srdis_done
1048
1049srdis_pcxl:
1050
1051	/* Disable Space Register Hashing for PCXL */
1052
1053	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1054	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1055	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1056	b,n		srdis_done
1057
1058srdis_pa20:
1059
1060	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1061
1062	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1063	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1064	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1065
1066
1067srdis_done:
1068	/* Switch back to virtual mode */
1069	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1070	load32 	   	2f, %r1
1071	nop
1072	nop
1073	nop
1074	nop
1075	nop
1076
1077	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1078	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1079	mtctl		%r0, %cr17		/* Clear IIASQ head */
1080	mtctl		%r1, %cr18		/* IIAOQ head */
1081	ldo		4(%r1), %r1
1082	mtctl		%r1, %cr18		/* IIAOQ tail */
1083	load32		KERNEL_PSW, %r1
1084	mtctl		%r1, %ipsw
1085	rfi
1086	nop
1087
10882:      bv		%r0(%r2)
1089	nop
1090	.exit
1091
1092	.procend
1093
1094	.end
1095