xref: /openbmc/linux/arch/parisc/kernel/pacache.S (revision 1da177e4c3f41524e886b7f1b8a0c1fc7321cac2) 
1/*
2 *  PARISC TLB and cache flushing support
3 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 *    This program is free software; you can redistribute it and/or modify
8 *    it under the terms of the GNU General Public License as published by
9 *    the Free Software Foundation; either version 2, or (at your option)
10 *    any later version.
11 *
12 *    This program is distributed in the hope that it will be useful,
13 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 *    GNU General Public License for more details.
16 *
17 *    You should have received a copy of the GNU General Public License
18 *    along with this program; if not, write to the Free Software
19 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 *       should only use index and base registers that are not shadowed,
25 *       so that the fast path emulation in the non access miss handler
26 *       can be used.
27 */
28
29#ifdef __LP64__
30#define ADDIB	addib,*
31#define CMPB	cmpb,*
32#define ANDCM	andcm,*
33
34	.level	2.0w
35#else
36#define ADDIB	addib,
37#define CMPB	cmpb,
38#define ANDCM	andcm
39
40	.level	2.0
41#endif
42
43#include <asm/assembly.h>
44#include <asm/psw.h>
45#include <asm/pgtable.h>
46#include <asm/cache.h>
47
48	.text
49	.align	128
50
51	.export flush_tlb_all_local,code
52
53flush_tlb_all_local:
54	.proc
55	.callinfo NO_CALLS
56	.entry
57
58	/*
59	 * The pitlbe and pdtlbe instructions should only be used to
60	 * flush the entire tlb. Also, there needs to be no intervening
61	 * tlb operations, e.g. tlb misses, so the operation needs
62	 * to happen in real mode with all interruptions disabled.
63	 */
64
65	/*
66	 * Once again, we do the rfi dance ... some day we need examine
67	 * all of our uses of this type of code and see what can be
68	 * consolidated.
69	 */
70
71	rsm		PSW_SM_I, %r19		/* relied upon translation! PA 2.0 Arch. F-5 */
72	nop
73	nop
74	nop
75	nop
76	nop
77	nop
78	nop
79
80	rsm		PSW_SM_Q, %r0		/* Turn off Q bit to load iia queue */
81	ldil		L%REAL_MODE_PSW, %r1
82	ldo		R%REAL_MODE_PSW(%r1), %r1
83	mtctl		%r1, %cr22
84	mtctl		%r0, %cr17		/* Clear IIASQ tail */
85	mtctl		%r0, %cr17		/* Clear IIASQ head */
86	ldil		L%PA(1f), %r1
87	ldo		R%PA(1f)(%r1), %r1
88	mtctl		%r1, %cr18		/* IIAOQ head */
89	ldo		4(%r1), %r1
90	mtctl		%r1, %cr18		/* IIAOQ tail */
91	rfi
92	nop
93
941:      ldil		L%PA(cache_info), %r1
95	ldo		R%PA(cache_info)(%r1), %r1
96
97	/* Flush Instruction Tlb */
98
99	LDREG		ITLB_SID_BASE(%r1), %r20
100	LDREG		ITLB_SID_STRIDE(%r1), %r21
101	LDREG		ITLB_SID_COUNT(%r1), %r22
102	LDREG		ITLB_OFF_BASE(%r1), %arg0
103	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
104	LDREG		ITLB_OFF_COUNT(%r1), %arg2
105	LDREG		ITLB_LOOP(%r1), %arg3
106
107	ADDIB=		-1, %arg3, fitoneloop	/* Preadjust and test */
108	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
109	copy		%arg0, %r28		/* Init base addr */
110
111fitmanyloop:					/* Loop if LOOP >= 2 */
112	mtsp		%r20, %sr1
113	add		%r21, %r20, %r20	/* increment space */
114	copy		%arg2, %r29		/* Init middle loop count */
115
116fitmanymiddle:					/* Loop if LOOP >= 2 */
117	ADDIB>		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
118	pitlbe		0(%sr1, %r28)
119	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
120	ADDIB>		-1, %r29, fitmanymiddle	/* Middle loop decr */
121	copy		%arg3, %r31		/* Re-init inner loop count */
122
123	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
124	ADDIB<=,n	-1, %r22, fitdone	/* Outer loop count decr */
125
126fitoneloop:					/* Loop if LOOP = 1 */
127	mtsp		%r20, %sr1
128	copy		%arg0, %r28		/* init base addr */
129	copy		%arg2, %r29		/* init middle loop count */
130
131fitonemiddle:					/* Loop if LOOP = 1 */
132	ADDIB>		-1, %r29, fitonemiddle	/* Middle loop count decr */
133	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
134
135	ADDIB>		-1, %r22, fitoneloop	/* Outer loop count decr */
136	add		%r21, %r20, %r20		/* increment space */
137
138fitdone:
139
140	/* Flush Data Tlb */
141
142	LDREG		DTLB_SID_BASE(%r1), %r20
143	LDREG		DTLB_SID_STRIDE(%r1), %r21
144	LDREG		DTLB_SID_COUNT(%r1), %r22
145	LDREG		DTLB_OFF_BASE(%r1), %arg0
146	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
147	LDREG		DTLB_OFF_COUNT(%r1), %arg2
148	LDREG		DTLB_LOOP(%r1), %arg3
149
150	ADDIB=		-1, %arg3, fdtoneloop	/* Preadjust and test */
151	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
152	copy		%arg0, %r28		/* Init base addr */
153
154fdtmanyloop:					/* Loop if LOOP >= 2 */
155	mtsp		%r20, %sr1
156	add		%r21, %r20, %r20	/* increment space */
157	copy		%arg2, %r29		/* Init middle loop count */
158
159fdtmanymiddle:					/* Loop if LOOP >= 2 */
160	ADDIB>		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
161	pdtlbe		0(%sr1, %r28)
162	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
163	ADDIB>		-1, %r29, fdtmanymiddle	/* Middle loop decr */
164	copy		%arg3, %r31		/* Re-init inner loop count */
165
166	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
167	ADDIB<=,n	-1, %r22,fdtdone	/* Outer loop count decr */
168
169fdtoneloop:					/* Loop if LOOP = 1 */
170	mtsp		%r20, %sr1
171	copy		%arg0, %r28		/* init base addr */
172	copy		%arg2, %r29		/* init middle loop count */
173
174fdtonemiddle:					/* Loop if LOOP = 1 */
175	ADDIB>		-1, %r29, fdtonemiddle	/* Middle loop count decr */
176	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
177
178	ADDIB>		-1, %r22, fdtoneloop	/* Outer loop count decr */
179	add		%r21, %r20, %r20	/* increment space */
180
181fdtdone:
182
183	/* Switch back to virtual mode */
184
185	rsm		PSW_SM_Q, %r0		/* clear Q bit to load iia queue */
186	ldil		L%KERNEL_PSW, %r1
187	ldo		R%KERNEL_PSW(%r1), %r1
188	or		%r1, %r19, %r1		/* Set I bit if set on entry */
189	mtctl		%r1, %cr22
190	mtctl		%r0, %cr17		/* Clear IIASQ tail */
191	mtctl		%r0, %cr17		/* Clear IIASQ head */
192	ldil		L%(2f), %r1
193	ldo		R%(2f)(%r1), %r1
194	mtctl		%r1, %cr18		/* IIAOQ head */
195	ldo		4(%r1), %r1
196	mtctl		%r1, %cr18		/* IIAOQ tail */
197	rfi
198	nop
199
2002:      bv		%r0(%r2)
201	nop
202	.exit
203
204	.procend
205
206	.export flush_instruction_cache_local,code
207	.import cache_info,data
208
209flush_instruction_cache_local:
210	.proc
211	.callinfo NO_CALLS
212	.entry
213
214	mtsp		%r0, %sr1
215	ldil		L%cache_info, %r1
216	ldo		R%cache_info(%r1), %r1
217
218	/* Flush Instruction Cache */
219
220	LDREG		ICACHE_BASE(%r1), %arg0
221	LDREG		ICACHE_STRIDE(%r1), %arg1
222	LDREG		ICACHE_COUNT(%r1), %arg2
223	LDREG		ICACHE_LOOP(%r1), %arg3
224	rsm             PSW_SM_I, %r22		/* No mmgt ops during loop*/
225	ADDIB=		-1, %arg3, fioneloop	/* Preadjust and test */
226	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
227
228fimanyloop:					/* Loop if LOOP >= 2 */
229	ADDIB>		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
230	fice            0(%sr1, %arg0)
231	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
232	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
233	ADDIB<=,n	-1, %arg2, fisync	/* Outer loop decr */
234
235fioneloop:					/* Loop if LOOP = 1 */
236	ADDIB>		-1, %arg2, fioneloop	/* Outer loop count decr */
237	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
238
239fisync:
240	sync
241	mtsm		%r22
242	bv		%r0(%r2)
243	nop
244	.exit
245
246	.procend
247
248	.export flush_data_cache_local, code
249	.import cache_info, data
250
251flush_data_cache_local:
252	.proc
253	.callinfo NO_CALLS
254	.entry
255
256	mtsp		%r0, %sr1
257	ldil		L%cache_info, %r1
258	ldo		R%cache_info(%r1), %r1
259
260	/* Flush Data Cache */
261
262	LDREG		DCACHE_BASE(%r1), %arg0
263	LDREG		DCACHE_STRIDE(%r1), %arg1
264	LDREG		DCACHE_COUNT(%r1), %arg2
265	LDREG		DCACHE_LOOP(%r1), %arg3
266	rsm		PSW_SM_I, %r22
267	ADDIB=		-1, %arg3, fdoneloop	/* Preadjust and test */
268	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
269
270fdmanyloop:					/* Loop if LOOP >= 2 */
271	ADDIB>		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
272	fdce		0(%sr1, %arg0)
273	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
274	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
275	ADDIB<=,n	-1, %arg2, fdsync	/* Outer loop decr */
276
277fdoneloop:					/* Loop if LOOP = 1 */
278	ADDIB>		-1, %arg2, fdoneloop	/* Outer loop count decr */
279	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
280
281fdsync:
282	syncdma
283	sync
284	mtsm		%r22
285	bv		%r0(%r2)
286	nop
287	.exit
288
289	.procend
290
291	.export copy_user_page_asm,code
292	.align	16
293
294copy_user_page_asm:
295	.proc
296	.callinfo NO_CALLS
297	.entry
298
299#ifdef __LP64__
300	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
301	 * Unroll the loop by hand and arrange insn appropriately.
302	 * GCC probably can do this just as well.
303	 */
304
305	ldd		0(%r25), %r19
306	ldi		32, %r1                 /* PAGE_SIZE/128 == 32 */
307	ldw		64(%r25), %r0		/* prefetch 1 cacheline ahead */
308	ldw		128(%r25), %r0		/* prefetch 2 */
309
3101:	ldd		8(%r25), %r20
311	ldw		192(%r25), %r0		/* prefetch 3 */
312	ldw		256(%r25), %r0		/* prefetch 4 */
313
314	ldd		16(%r25), %r21
315	ldd		24(%r25), %r22
316	std		%r19, 0(%r26)
317	std		%r20, 8(%r26)
318
319	ldd		32(%r25), %r19
320	ldd		40(%r25), %r20
321	std		%r21, 16(%r26)
322	std		%r22, 24(%r26)
323
324	ldd		48(%r25), %r21
325	ldd		56(%r25), %r22
326	std		%r19, 32(%r26)
327	std		%r20, 40(%r26)
328
329	ldd		64(%r25), %r19
330	ldd		72(%r25), %r20
331	std		%r21, 48(%r26)
332	std		%r22, 56(%r26)
333
334	ldd		80(%r25), %r21
335	ldd		88(%r25), %r22
336	std		%r19, 64(%r26)
337	std		%r20, 72(%r26)
338
339	ldd		 96(%r25), %r19
340	ldd		104(%r25), %r20
341	std		%r21, 80(%r26)
342	std		%r22, 88(%r26)
343
344	ldd		112(%r25), %r21
345	ldd		120(%r25), %r22
346	std		%r19, 96(%r26)
347	std		%r20, 104(%r26)
348
349	ldo		128(%r25), %r25
350	std		%r21, 112(%r26)
351	std		%r22, 120(%r26)
352	ldo		128(%r26), %r26
353
354	ADDIB>		-1, %r1, 1b		/* bundle 10 */
355	ldd		0(%r25), %r19		/* start next loads */
356
357#else
358
359	/*
360	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
361	 * bundles (very restricted rules for bundling).
362	 * Note that until (if) we start saving
363	 * the full 64 bit register values on interrupt, we can't
364	 * use ldd/std on a 32 bit kernel.
365	 */
366	ldi		64, %r1		/* PAGE_SIZE/64 == 64 */
367
3681:
369	ldw		0(%r25), %r19
370	ldw		4(%r25), %r20
371	ldw		8(%r25), %r21
372	ldw		12(%r25), %r22
373	stw		%r19, 0(%r26)
374	stw		%r20, 4(%r26)
375	stw		%r21, 8(%r26)
376	stw		%r22, 12(%r26)
377	ldw		16(%r25), %r19
378	ldw		20(%r25), %r20
379	ldw		24(%r25), %r21
380	ldw		28(%r25), %r22
381	stw		%r19, 16(%r26)
382	stw		%r20, 20(%r26)
383	stw		%r21, 24(%r26)
384	stw		%r22, 28(%r26)
385	ldw		32(%r25), %r19
386	ldw		36(%r25), %r20
387	ldw		40(%r25), %r21
388	ldw		44(%r25), %r22
389	stw		%r19, 32(%r26)
390	stw		%r20, 36(%r26)
391	stw		%r21, 40(%r26)
392	stw		%r22, 44(%r26)
393	ldw		48(%r25), %r19
394	ldw		52(%r25), %r20
395	ldw		56(%r25), %r21
396	ldw		60(%r25), %r22
397	stw		%r19, 48(%r26)
398	stw		%r20, 52(%r26)
399	stw		%r21, 56(%r26)
400	stw		%r22, 60(%r26)
401	ldo		64(%r26), %r26
402	ADDIB>		-1, %r1, 1b
403	ldo		64(%r25), %r25
404#endif
405	bv		%r0(%r2)
406	nop
407	.exit
408
409	.procend
410
411/*
412 * NOTE: Code in clear_user_page has a hard coded dependency on the
413 *       maximum alias boundary being 4 Mb. We've been assured by the
414 *       parisc chip designers that there will not ever be a parisc
415 *       chip with a larger alias boundary (Never say never :-) ).
416 *
417 *       Subtle: the dtlb miss handlers support the temp alias region by
418 *       "knowing" that if a dtlb miss happens within the temp alias
419 *       region it must have occurred while in clear_user_page. Since
420 *       this routine makes use of processor local translations, we
421 *       don't want to insert them into the kernel page table. Instead,
422 *       we load up some general registers (they need to be registers
423 *       which aren't shadowed) with the physical page numbers (preshifted
424 *       for tlb insertion) needed to insert the translations. When we
425 *       miss on the translation, the dtlb miss handler inserts the
426 *       translation into the tlb using these values:
427 *
428 *          %r26 physical page (shifted for tlb insert) of "to" translation
429 *          %r23 physical page (shifted for tlb insert) of "from" translation
430 */
431
432#if 0
433
434	/*
435	 * We can't do this since copy_user_page is used to bring in
436	 * file data that might have instructions. Since the data would
437	 * then need to be flushed out so the i-fetch can see it, it
438	 * makes more sense to just copy through the kernel translation
439	 * and flush it.
440	 *
441	 * I'm still keeping this around because it may be possible to
442	 * use it if more information is passed into copy_user_page().
443	 * Have to do some measurements to see if it is worthwhile to
444	 * lobby for such a change.
445	 */
446
447	.export copy_user_page_asm,code
448
449copy_user_page_asm:
450	.proc
451	.callinfo NO_CALLS
452	.entry
453
454	ldil		L%(__PAGE_OFFSET), %r1
455	sub		%r26, %r1, %r26
456	sub		%r25, %r1, %r23		/* move physical addr into non shadowed reg */
457
458	ldil		L%(TMPALIAS_MAP_START), %r28
459#ifdef __LP64__
460	extrd,u		%r26,56,32, %r26		/* convert phys addr to tlb insert format */
461	extrd,u		%r23,56,32, %r23		/* convert phys addr to tlb insert format */
462	depd		%r24,63,22, %r28		/* Form aliased virtual address 'to' */
463	depdi		0, 63,12, %r28		/* Clear any offset bits */
464	copy		%r28, %r29
465	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
466#else
467	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
468	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
469	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
470	depwi		0, 31,12, %r28		/* Clear any offset bits */
471	copy		%r28, %r29
472	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
473#endif
474
475	/* Purge any old translations */
476
477	pdtlb		0(%r28)
478	pdtlb		0(%r29)
479
480	ldi		64, %r1
481
482	/*
483	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
484	 * bundles (very restricted rules for bundling). It probably
485	 * does OK on PCXU and better, but we could do better with
486	 * ldd/std instructions. Note that until (if) we start saving
487	 * the full 64 bit register values on interrupt, we can't
488	 * use ldd/std on a 32 bit kernel.
489	 */
490
491
4921:
493	ldw		0(%r29), %r19
494	ldw		4(%r29), %r20
495	ldw		8(%r29), %r21
496	ldw		12(%r29), %r22
497	stw		%r19, 0(%r28)
498	stw		%r20, 4(%r28)
499	stw		%r21, 8(%r28)
500	stw		%r22, 12(%r28)
501	ldw		16(%r29), %r19
502	ldw		20(%r29), %r20
503	ldw		24(%r29), %r21
504	ldw		28(%r29), %r22
505	stw		%r19, 16(%r28)
506	stw		%r20, 20(%r28)
507	stw		%r21, 24(%r28)
508	stw		%r22, 28(%r28)
509	ldw		32(%r29), %r19
510	ldw		36(%r29), %r20
511	ldw		40(%r29), %r21
512	ldw		44(%r29), %r22
513	stw		%r19, 32(%r28)
514	stw		%r20, 36(%r28)
515	stw		%r21, 40(%r28)
516	stw		%r22, 44(%r28)
517	ldw		48(%r29), %r19
518	ldw		52(%r29), %r20
519	ldw		56(%r29), %r21
520	ldw		60(%r29), %r22
521	stw		%r19, 48(%r28)
522	stw		%r20, 52(%r28)
523	stw		%r21, 56(%r28)
524	stw		%r22, 60(%r28)
525	ldo		64(%r28), %r28
526	ADDIB>		-1, %r1,1b
527	ldo		64(%r29), %r29
528
529	bv		%r0(%r2)
530	nop
531	.exit
532
533	.procend
534#endif
535
536	.export __clear_user_page_asm,code
537
538__clear_user_page_asm:
539	.proc
540	.callinfo NO_CALLS
541	.entry
542
543	tophys_r1	%r26
544
545	ldil		L%(TMPALIAS_MAP_START), %r28
546#ifdef __LP64__
547#if (TMPALIAS_MAP_START >= 0x80000000)
548	depdi		0, 31,32, %r28		/* clear any sign extension */
549#endif
550	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
551	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
552	depdi		0, 63,12, %r28		/* Clear any offset bits */
553#else
554	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
555	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
556	depwi		0, 31,12, %r28		/* Clear any offset bits */
557#endif
558
559	/* Purge any old translation */
560
561	pdtlb		0(%r28)
562
563#ifdef __LP64__
564	ldi		32, %r1			/* PAGE_SIZE/128 == 32 */
565
566	/* PREFETCH (Write) has not (yet) been proven to help here */
567/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
568
5691:	std		%r0, 0(%r28)
570	std		%r0, 8(%r28)
571	std		%r0, 16(%r28)
572	std		%r0, 24(%r28)
573	std		%r0, 32(%r28)
574	std		%r0, 40(%r28)
575	std		%r0, 48(%r28)
576	std		%r0, 56(%r28)
577	std		%r0, 64(%r28)
578	std		%r0, 72(%r28)
579	std		%r0, 80(%r28)
580	std		%r0, 88(%r28)
581	std		%r0, 96(%r28)
582	std		%r0, 104(%r28)
583	std		%r0, 112(%r28)
584	std		%r0, 120(%r28)
585	ADDIB>		-1, %r1, 1b
586	ldo		128(%r28), %r28
587
588#else	/* ! __LP64 */
589
590	ldi		64, %r1			/* PAGE_SIZE/64 == 64 */
591
5921:
593	stw		%r0, 0(%r28)
594	stw		%r0, 4(%r28)
595	stw		%r0, 8(%r28)
596	stw		%r0, 12(%r28)
597	stw		%r0, 16(%r28)
598	stw		%r0, 20(%r28)
599	stw		%r0, 24(%r28)
600	stw		%r0, 28(%r28)
601	stw		%r0, 32(%r28)
602	stw		%r0, 36(%r28)
603	stw		%r0, 40(%r28)
604	stw		%r0, 44(%r28)
605	stw		%r0, 48(%r28)
606	stw		%r0, 52(%r28)
607	stw		%r0, 56(%r28)
608	stw		%r0, 60(%r28)
609	ADDIB>		-1, %r1, 1b
610	ldo		64(%r28), %r28
611#endif	/* __LP64 */
612
613	bv		%r0(%r2)
614	nop
615	.exit
616
617	.procend
618
619	.export flush_kernel_dcache_page
620
621flush_kernel_dcache_page:
622	.proc
623	.callinfo NO_CALLS
624	.entry
625
626	ldil		L%dcache_stride, %r1
627	ldw		R%dcache_stride(%r1), %r23
628
629#ifdef __LP64__
630	depdi,z		1, 63-PAGE_SHIFT,1, %r25
631#else
632	depwi,z		1, 31-PAGE_SHIFT,1, %r25
633#endif
634	add		%r26, %r25, %r25
635	sub		%r25, %r23, %r25
636
637
6381:      fdc,m		%r23(%r26)
639	fdc,m		%r23(%r26)
640	fdc,m		%r23(%r26)
641	fdc,m		%r23(%r26)
642	fdc,m		%r23(%r26)
643	fdc,m		%r23(%r26)
644	fdc,m		%r23(%r26)
645	fdc,m		%r23(%r26)
646	fdc,m		%r23(%r26)
647	fdc,m		%r23(%r26)
648	fdc,m		%r23(%r26)
649	fdc,m		%r23(%r26)
650	fdc,m		%r23(%r26)
651	fdc,m		%r23(%r26)
652	fdc,m		%r23(%r26)
653	CMPB<<		%r26, %r25,1b
654	fdc,m		%r23(%r26)
655
656	sync
657	bv		%r0(%r2)
658	nop
659	.exit
660
661	.procend
662
663	.export flush_user_dcache_page
664
665flush_user_dcache_page:
666	.proc
667	.callinfo NO_CALLS
668	.entry
669
670	ldil		L%dcache_stride, %r1
671	ldw		R%dcache_stride(%r1), %r23
672
673#ifdef __LP64__
674	depdi,z		1,63-PAGE_SHIFT,1, %r25
675#else
676	depwi,z		1,31-PAGE_SHIFT,1, %r25
677#endif
678	add		%r26, %r25, %r25
679	sub		%r25, %r23, %r25
680
681
6821:      fdc,m		%r23(%sr3, %r26)
683	fdc,m		%r23(%sr3, %r26)
684	fdc,m		%r23(%sr3, %r26)
685	fdc,m		%r23(%sr3, %r26)
686	fdc,m		%r23(%sr3, %r26)
687	fdc,m		%r23(%sr3, %r26)
688	fdc,m		%r23(%sr3, %r26)
689	fdc,m		%r23(%sr3, %r26)
690	fdc,m		%r23(%sr3, %r26)
691	fdc,m		%r23(%sr3, %r26)
692	fdc,m		%r23(%sr3, %r26)
693	fdc,m		%r23(%sr3, %r26)
694	fdc,m		%r23(%sr3, %r26)
695	fdc,m		%r23(%sr3, %r26)
696	fdc,m		%r23(%sr3, %r26)
697	CMPB<<		%r26, %r25,1b
698	fdc,m		%r23(%sr3, %r26)
699
700	sync
701	bv		%r0(%r2)
702	nop
703	.exit
704
705	.procend
706
707	.export flush_user_icache_page
708
709flush_user_icache_page:
710	.proc
711	.callinfo NO_CALLS
712	.entry
713
714	ldil		L%dcache_stride, %r1
715	ldw		R%dcache_stride(%r1), %r23
716
717#ifdef __LP64__
718	depdi,z		1, 63-PAGE_SHIFT,1, %r25
719#else
720	depwi,z		1, 31-PAGE_SHIFT,1, %r25
721#endif
722	add		%r26, %r25, %r25
723	sub		%r25, %r23, %r25
724
725
7261:      fic,m		%r23(%sr3, %r26)
727	fic,m		%r23(%sr3, %r26)
728	fic,m		%r23(%sr3, %r26)
729	fic,m		%r23(%sr3, %r26)
730	fic,m		%r23(%sr3, %r26)
731	fic,m		%r23(%sr3, %r26)
732	fic,m		%r23(%sr3, %r26)
733	fic,m		%r23(%sr3, %r26)
734	fic,m		%r23(%sr3, %r26)
735	fic,m		%r23(%sr3, %r26)
736	fic,m		%r23(%sr3, %r26)
737	fic,m		%r23(%sr3, %r26)
738	fic,m		%r23(%sr3, %r26)
739	fic,m		%r23(%sr3, %r26)
740	fic,m		%r23(%sr3, %r26)
741	CMPB<<		%r26, %r25,1b
742	fic,m		%r23(%sr3, %r26)
743
744	sync
745	bv		%r0(%r2)
746	nop
747	.exit
748
749	.procend
750
751
752	.export purge_kernel_dcache_page
753
754purge_kernel_dcache_page:
755	.proc
756	.callinfo NO_CALLS
757	.entry
758
759	ldil		L%dcache_stride, %r1
760	ldw		R%dcache_stride(%r1), %r23
761
762#ifdef __LP64__
763	depdi,z		1, 63-PAGE_SHIFT,1, %r25
764#else
765	depwi,z		1, 31-PAGE_SHIFT,1, %r25
766#endif
767	add		%r26, %r25, %r25
768	sub		%r25, %r23, %r25
769
7701:      pdc,m		%r23(%r26)
771	pdc,m		%r23(%r26)
772	pdc,m		%r23(%r26)
773	pdc,m		%r23(%r26)
774	pdc,m		%r23(%r26)
775	pdc,m		%r23(%r26)
776	pdc,m		%r23(%r26)
777	pdc,m		%r23(%r26)
778	pdc,m		%r23(%r26)
779	pdc,m		%r23(%r26)
780	pdc,m		%r23(%r26)
781	pdc,m		%r23(%r26)
782	pdc,m		%r23(%r26)
783	pdc,m		%r23(%r26)
784	pdc,m		%r23(%r26)
785	CMPB<<		%r26, %r25, 1b
786	pdc,m		%r23(%r26)
787
788	sync
789	bv		%r0(%r2)
790	nop
791	.exit
792
793	.procend
794
795#if 0
796	/* Currently not used, but it still is a possible alternate
797	 * solution.
798	 */
799
800	.export flush_alias_page
801
802flush_alias_page:
803	.proc
804	.callinfo NO_CALLS
805	.entry
806
807	tophys_r1		%r26
808
809	ldil		L%(TMPALIAS_MAP_START), %r28
810#ifdef __LP64__
811	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
812	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
813	depdi		0, 63,12, %r28		/* Clear any offset bits */
814#else
815	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
816	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
817	depwi		0, 31,12, %r28		/* Clear any offset bits */
818#endif
819
820	/* Purge any old translation */
821
822	pdtlb		0(%r28)
823
824	ldil		L%dcache_stride, %r1
825	ldw		R%dcache_stride(%r1), %r23
826
827#ifdef __LP64__
828	depdi,z		1, 63-PAGE_SHIFT,1, %r29
829#else
830	depwi,z		1, 31-PAGE_SHIFT,1, %r29
831#endif
832	add		%r28, %r29, %r29
833	sub		%r29, %r23, %r29
834
8351:      fdc,m		%r23(%r28)
836	fdc,m		%r23(%r28)
837	fdc,m		%r23(%r28)
838	fdc,m		%r23(%r28)
839	fdc,m		%r23(%r28)
840	fdc,m		%r23(%r28)
841	fdc,m		%r23(%r28)
842	fdc,m		%r23(%r28)
843	fdc,m		%r23(%r28)
844	fdc,m		%r23(%r28)
845	fdc,m		%r23(%r28)
846	fdc,m		%r23(%r28)
847	fdc,m		%r23(%r28)
848	fdc,m		%r23(%r28)
849	fdc,m		%r23(%r28)
850	CMPB<<		%r28, %r29, 1b
851	fdc,m		%r23(%r28)
852
853	sync
854	bv		%r0(%r2)
855	nop
856	.exit
857
858	.procend
859#endif
860
861	.export flush_user_dcache_range_asm
862
863flush_user_dcache_range_asm:
864	.proc
865	.callinfo NO_CALLS
866	.entry
867
868	ldil		L%dcache_stride, %r1
869	ldw		R%dcache_stride(%r1), %r23
870	ldo		-1(%r23), %r21
871	ANDCM		%r26, %r21, %r26
872
8731:      CMPB<<,n	%r26, %r25, 1b
874	fdc,m		%r23(%sr3, %r26)
875
876	sync
877	bv		%r0(%r2)
878	nop
879	.exit
880
881	.procend
882
883	.export flush_kernel_dcache_range_asm
884
885flush_kernel_dcache_range_asm:
886	.proc
887	.callinfo NO_CALLS
888	.entry
889
890	ldil		L%dcache_stride, %r1
891	ldw		R%dcache_stride(%r1), %r23
892	ldo		-1(%r23), %r21
893	ANDCM		%r26, %r21, %r26
894
8951:      CMPB<<,n	%r26, %r25,1b
896	fdc,m		%r23(%r26)
897
898	sync
899	syncdma
900	bv		%r0(%r2)
901	nop
902	.exit
903
904	.procend
905
906	.export flush_user_icache_range_asm
907
908flush_user_icache_range_asm:
909	.proc
910	.callinfo NO_CALLS
911	.entry
912
913	ldil		L%icache_stride, %r1
914	ldw		R%icache_stride(%r1), %r23
915	ldo		-1(%r23), %r21
916	ANDCM		%r26, %r21, %r26
917
9181:      CMPB<<,n	%r26, %r25,1b
919	fic,m		%r23(%sr3, %r26)
920
921	sync
922	bv		%r0(%r2)
923	nop
924	.exit
925
926	.procend
927
928	.export flush_kernel_icache_page
929
930flush_kernel_icache_page:
931	.proc
932	.callinfo NO_CALLS
933	.entry
934
935	ldil		L%icache_stride, %r1
936	ldw		R%icache_stride(%r1), %r23
937
938#ifdef __LP64__
939	depdi,z		1, 63-PAGE_SHIFT,1, %r25
940#else
941	depwi,z		1, 31-PAGE_SHIFT,1, %r25
942#endif
943	add		%r26, %r25, %r25
944	sub		%r25, %r23, %r25
945
946
9471:      fic,m		%r23(%r26)
948	fic,m		%r23(%r26)
949	fic,m		%r23(%r26)
950	fic,m		%r23(%r26)
951	fic,m		%r23(%r26)
952	fic,m		%r23(%r26)
953	fic,m		%r23(%r26)
954	fic,m		%r23(%r26)
955	fic,m		%r23(%r26)
956	fic,m		%r23(%r26)
957	fic,m		%r23(%r26)
958	fic,m		%r23(%r26)
959	fic,m		%r23(%r26)
960	fic,m		%r23(%r26)
961	fic,m		%r23(%r26)
962	CMPB<<		%r26, %r25, 1b
963	fic,m		%r23(%r26)
964
965	sync
966	bv		%r0(%r2)
967	nop
968	.exit
969
970	.procend
971
972	.export flush_kernel_icache_range_asm
973
974flush_kernel_icache_range_asm:
975	.proc
976	.callinfo NO_CALLS
977	.entry
978
979	ldil		L%icache_stride, %r1
980	ldw		R%icache_stride(%r1), %r23
981	ldo		-1(%r23), %r21
982	ANDCM		%r26, %r21, %r26
983
9841:      CMPB<<,n	%r26, %r25, 1b
985	fic,m		%r23(%r26)
986
987	sync
988	bv		%r0(%r2)
989	nop
990	.exit
991
992	.procend
993
994	.align	128
995
996	.export disable_sr_hashing_asm,code
997
998disable_sr_hashing_asm:
999	.proc
1000	.callinfo NO_CALLS
1001	.entry
1002
1003	/* Switch to real mode */
1004
1005	ssm		0, %r0			/* relied upon translation! */
1006	nop
1007	nop
1008	nop
1009	nop
1010	nop
1011	nop
1012	nop
1013
1014	rsm		(PSW_SM_Q|PSW_SM_I), %r0 /* disable Q&I to load the iia queue */
1015	ldil		L%REAL_MODE_PSW, %r1
1016	ldo		R%REAL_MODE_PSW(%r1), %r1
1017	mtctl		%r1, %cr22
1018	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1019	mtctl		%r0, %cr17		/* Clear IIASQ head */
1020	ldil		L%PA(1f), %r1
1021	ldo		R%PA(1f)(%r1), %r1
1022	mtctl		%r1, %cr18		/* IIAOQ head */
1023	ldo		4(%r1), %r1
1024	mtctl		%r1, %cr18		/* IIAOQ tail */
1025	rfi
1026	nop
1027
10281:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1029	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1030	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1031	b,n		srdis_done
1032
1033srdis_pcxs:
1034
1035	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1036
1037	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1038	.word		0x141c1a00		/* must issue twice */
1039	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1040	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1041	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1042	.word		0x141c1600		/* must issue twice */
1043	b,n		srdis_done
1044
1045srdis_pcxl:
1046
1047	/* Disable Space Register Hashing for PCXL */
1048
1049	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1050	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1051	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1052	b,n		srdis_done
1053
1054srdis_pa20:
1055
1056	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+ */
1057
1058	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1059	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1060	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1061
1062srdis_done:
1063
1064	/* Switch back to virtual mode */
1065
1066	rsm		PSW_SM_Q, %r0		/* clear Q bit to load iia queue */
1067	ldil		L%KERNEL_PSW, %r1
1068	ldo		R%KERNEL_PSW(%r1), %r1
1069	mtctl		%r1, %cr22
1070	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1071	mtctl		%r0, %cr17		/* Clear IIASQ head */
1072	ldil 	   	L%(2f), %r1
1073	ldo     	R%(2f)(%r1), %r1
1074	mtctl		%r1, %cr18		/* IIAOQ head */
1075	ldo		4(%r1), %r1
1076	mtctl		%r1, %cr18		/* IIAOQ tail */
1077	rfi
1078	nop
1079
10802:      bv		%r0(%r2)
1081	nop
1082	.exit
1083
1084	.procend
1085
1086	.end
1087