xref: /openbmc/linux/arch/parisc/kernel/pacache.S (revision 22246614) 
1/*
2 *  PARISC TLB and cache flushing support
3 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 *    This program is free software; you can redistribute it and/or modify
8 *    it under the terms of the GNU General Public License as published by
9 *    the Free Software Foundation; either version 2, or (at your option)
10 *    any later version.
11 *
12 *    This program is distributed in the hope that it will be useful,
13 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 *    GNU General Public License for more details.
16 *
17 *    You should have received a copy of the GNU General Public License
18 *    along with this program; if not, write to the Free Software
19 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 *       should only use index and base registers that are not shadowed,
25 *       so that the fast path emulation in the non access miss handler
26 *       can be used.
27 */
28
29#ifdef CONFIG_64BIT
30	.level	2.0w
31#else
32	.level	2.0
33#endif
34
35#include <asm/psw.h>
36#include <asm/assembly.h>
37#include <asm/pgtable.h>
38#include <asm/cache.h>
39#include <linux/linkage.h>
40
41	.text
42	.align	128
43
44ENTRY(flush_tlb_all_local)
45	.proc
46	.callinfo NO_CALLS
47	.entry
48
49	/*
50	 * The pitlbe and pdtlbe instructions should only be used to
51	 * flush the entire tlb. Also, there needs to be no intervening
52	 * tlb operations, e.g. tlb misses, so the operation needs
53	 * to happen in real mode with all interruptions disabled.
54	 */
55
56	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
57	rsm		PSW_SM_I, %r19		/* save I-bit state */
58	load32		PA(1f), %r1
59	nop
60	nop
61	nop
62	nop
63	nop
64
65	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
66	mtctl		%r0, %cr17		/* Clear IIASQ tail */
67	mtctl		%r0, %cr17		/* Clear IIASQ head */
68	mtctl		%r1, %cr18		/* IIAOQ head */
69	ldo		4(%r1), %r1
70	mtctl		%r1, %cr18		/* IIAOQ tail */
71	load32		REAL_MODE_PSW, %r1
72	mtctl           %r1, %ipsw
73	rfi
74	nop
75
761:      load32		PA(cache_info), %r1
77
78	/* Flush Instruction Tlb */
79
80	LDREG		ITLB_SID_BASE(%r1), %r20
81	LDREG		ITLB_SID_STRIDE(%r1), %r21
82	LDREG		ITLB_SID_COUNT(%r1), %r22
83	LDREG		ITLB_OFF_BASE(%r1), %arg0
84	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
85	LDREG		ITLB_OFF_COUNT(%r1), %arg2
86	LDREG		ITLB_LOOP(%r1), %arg3
87
88	ADDIB=		-1, %arg3, fitoneloop	/* Preadjust and test */
89	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
90	copy		%arg0, %r28		/* Init base addr */
91
92fitmanyloop:					/* Loop if LOOP >= 2 */
93	mtsp		%r20, %sr1
94	add		%r21, %r20, %r20	/* increment space */
95	copy		%arg2, %r29		/* Init middle loop count */
96
97fitmanymiddle:					/* Loop if LOOP >= 2 */
98	ADDIB>		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
99	pitlbe		0(%sr1, %r28)
100	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
101	ADDIB>		-1, %r29, fitmanymiddle	/* Middle loop decr */
102	copy		%arg3, %r31		/* Re-init inner loop count */
103
104	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
105	ADDIB<=,n	-1, %r22, fitdone	/* Outer loop count decr */
106
107fitoneloop:					/* Loop if LOOP = 1 */
108	mtsp		%r20, %sr1
109	copy		%arg0, %r28		/* init base addr */
110	copy		%arg2, %r29		/* init middle loop count */
111
112fitonemiddle:					/* Loop if LOOP = 1 */
113	ADDIB>		-1, %r29, fitonemiddle	/* Middle loop count decr */
114	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
115
116	ADDIB>		-1, %r22, fitoneloop	/* Outer loop count decr */
117	add		%r21, %r20, %r20		/* increment space */
118
119fitdone:
120
121	/* Flush Data Tlb */
122
123	LDREG		DTLB_SID_BASE(%r1), %r20
124	LDREG		DTLB_SID_STRIDE(%r1), %r21
125	LDREG		DTLB_SID_COUNT(%r1), %r22
126	LDREG		DTLB_OFF_BASE(%r1), %arg0
127	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
128	LDREG		DTLB_OFF_COUNT(%r1), %arg2
129	LDREG		DTLB_LOOP(%r1), %arg3
130
131	ADDIB=		-1, %arg3, fdtoneloop	/* Preadjust and test */
132	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
133	copy		%arg0, %r28		/* Init base addr */
134
135fdtmanyloop:					/* Loop if LOOP >= 2 */
136	mtsp		%r20, %sr1
137	add		%r21, %r20, %r20	/* increment space */
138	copy		%arg2, %r29		/* Init middle loop count */
139
140fdtmanymiddle:					/* Loop if LOOP >= 2 */
141	ADDIB>		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
142	pdtlbe		0(%sr1, %r28)
143	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
144	ADDIB>		-1, %r29, fdtmanymiddle	/* Middle loop decr */
145	copy		%arg3, %r31		/* Re-init inner loop count */
146
147	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
148	ADDIB<=,n	-1, %r22,fdtdone	/* Outer loop count decr */
149
150fdtoneloop:					/* Loop if LOOP = 1 */
151	mtsp		%r20, %sr1
152	copy		%arg0, %r28		/* init base addr */
153	copy		%arg2, %r29		/* init middle loop count */
154
155fdtonemiddle:					/* Loop if LOOP = 1 */
156	ADDIB>		-1, %r29, fdtonemiddle	/* Middle loop count decr */
157	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
158
159	ADDIB>		-1, %r22, fdtoneloop	/* Outer loop count decr */
160	add		%r21, %r20, %r20	/* increment space */
161
162
163fdtdone:
164	/*
165	 * Switch back to virtual mode
166	 */
167	/* pcxt_ssm_bug */
168	rsm		PSW_SM_I, %r0
169	load32		2f, %r1
170	nop
171	nop
172	nop
173	nop
174	nop
175
176	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
177	mtctl		%r0, %cr17		/* Clear IIASQ tail */
178	mtctl		%r0, %cr17		/* Clear IIASQ head */
179	mtctl		%r1, %cr18		/* IIAOQ head */
180	ldo		4(%r1), %r1
181	mtctl		%r1, %cr18		/* IIAOQ tail */
182	load32		KERNEL_PSW, %r1
183	or		%r1, %r19, %r1	/* I-bit to state on entry */
184	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
185	rfi
186	nop
187
1882:      bv		%r0(%r2)
189	nop
190
191	.exit
192	.procend
193ENDPROC(flush_tlb_all_local)
194
195	.import cache_info,data
196
197ENTRY(flush_instruction_cache_local)
198	.proc
199	.callinfo NO_CALLS
200	.entry
201
202	mtsp		%r0, %sr1
203	load32		cache_info, %r1
204
205	/* Flush Instruction Cache */
206
207	LDREG		ICACHE_BASE(%r1), %arg0
208	LDREG		ICACHE_STRIDE(%r1), %arg1
209	LDREG		ICACHE_COUNT(%r1), %arg2
210	LDREG		ICACHE_LOOP(%r1), %arg3
211	rsm             PSW_SM_I, %r22		/* No mmgt ops during loop*/
212	ADDIB=		-1, %arg3, fioneloop	/* Preadjust and test */
213	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
214
215fimanyloop:					/* Loop if LOOP >= 2 */
216	ADDIB>		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
217	fice            %r0(%sr1, %arg0)
218	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
219	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
220	ADDIB<=,n	-1, %arg2, fisync	/* Outer loop decr */
221
222fioneloop:					/* Loop if LOOP = 1 */
223	ADDIB>		-1, %arg2, fioneloop	/* Outer loop count decr */
224	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
225
226fisync:
227	sync
228	mtsm		%r22			/* restore I-bit */
229	bv		%r0(%r2)
230	nop
231	.exit
232
233	.procend
234ENDPROC(flush_instruction_cache_local)
235
236
237	.import cache_info, data
238ENTRY(flush_data_cache_local)
239	.proc
240	.callinfo NO_CALLS
241	.entry
242
243	mtsp		%r0, %sr1
244	load32 		cache_info, %r1
245
246	/* Flush Data Cache */
247
248	LDREG		DCACHE_BASE(%r1), %arg0
249	LDREG		DCACHE_STRIDE(%r1), %arg1
250	LDREG		DCACHE_COUNT(%r1), %arg2
251	LDREG		DCACHE_LOOP(%r1), %arg3
252	rsm		PSW_SM_I, %r22
253	ADDIB=		-1, %arg3, fdoneloop	/* Preadjust and test */
254	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
255
256fdmanyloop:					/* Loop if LOOP >= 2 */
257	ADDIB>		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
258	fdce		%r0(%sr1, %arg0)
259	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
260	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
261	ADDIB<=,n	-1, %arg2, fdsync	/* Outer loop decr */
262
263fdoneloop:					/* Loop if LOOP = 1 */
264	ADDIB>		-1, %arg2, fdoneloop	/* Outer loop count decr */
265	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
266
267fdsync:
268	syncdma
269	sync
270	mtsm		%r22			/* restore I-bit */
271	bv		%r0(%r2)
272	nop
273	.exit
274
275	.procend
276ENDPROC(flush_data_cache_local)
277
278	.align	16
279
280ENTRY(copy_user_page_asm)
281	.proc
282	.callinfo NO_CALLS
283	.entry
284
285#ifdef CONFIG_64BIT
286	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
287	 * Unroll the loop by hand and arrange insn appropriately.
288	 * GCC probably can do this just as well.
289	 */
290
291	ldd		0(%r25), %r19
292	ldi		(PAGE_SIZE / 128), %r1
293
294	ldw		64(%r25), %r0		/* prefetch 1 cacheline ahead */
295	ldw		128(%r25), %r0		/* prefetch 2 */
296
2971:	ldd		8(%r25), %r20
298	ldw		192(%r25), %r0		/* prefetch 3 */
299	ldw		256(%r25), %r0		/* prefetch 4 */
300
301	ldd		16(%r25), %r21
302	ldd		24(%r25), %r22
303	std		%r19, 0(%r26)
304	std		%r20, 8(%r26)
305
306	ldd		32(%r25), %r19
307	ldd		40(%r25), %r20
308	std		%r21, 16(%r26)
309	std		%r22, 24(%r26)
310
311	ldd		48(%r25), %r21
312	ldd		56(%r25), %r22
313	std		%r19, 32(%r26)
314	std		%r20, 40(%r26)
315
316	ldd		64(%r25), %r19
317	ldd		72(%r25), %r20
318	std		%r21, 48(%r26)
319	std		%r22, 56(%r26)
320
321	ldd		80(%r25), %r21
322	ldd		88(%r25), %r22
323	std		%r19, 64(%r26)
324	std		%r20, 72(%r26)
325
326	ldd		 96(%r25), %r19
327	ldd		104(%r25), %r20
328	std		%r21, 80(%r26)
329	std		%r22, 88(%r26)
330
331	ldd		112(%r25), %r21
332	ldd		120(%r25), %r22
333	std		%r19, 96(%r26)
334	std		%r20, 104(%r26)
335
336	ldo		128(%r25), %r25
337	std		%r21, 112(%r26)
338	std		%r22, 120(%r26)
339	ldo		128(%r26), %r26
340
341	/* conditional branches nullify on forward taken branch, and on
342	 * non-taken backward branch. Note that .+4 is a backwards branch.
343	 * The ldd should only get executed if the branch is taken.
344	 */
345	ADDIB>,n	-1, %r1, 1b		/* bundle 10 */
346	ldd		0(%r25), %r19		/* start next loads */
347
348#else
349
350	/*
351	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
352	 * bundles (very restricted rules for bundling).
353	 * Note that until (if) we start saving
354	 * the full 64 bit register values on interrupt, we can't
355	 * use ldd/std on a 32 bit kernel.
356	 */
357	ldw		0(%r25), %r19
358	ldi		(PAGE_SIZE / 64), %r1
359
3601:
361	ldw		4(%r25), %r20
362	ldw		8(%r25), %r21
363	ldw		12(%r25), %r22
364	stw		%r19, 0(%r26)
365	stw		%r20, 4(%r26)
366	stw		%r21, 8(%r26)
367	stw		%r22, 12(%r26)
368	ldw		16(%r25), %r19
369	ldw		20(%r25), %r20
370	ldw		24(%r25), %r21
371	ldw		28(%r25), %r22
372	stw		%r19, 16(%r26)
373	stw		%r20, 20(%r26)
374	stw		%r21, 24(%r26)
375	stw		%r22, 28(%r26)
376	ldw		32(%r25), %r19
377	ldw		36(%r25), %r20
378	ldw		40(%r25), %r21
379	ldw		44(%r25), %r22
380	stw		%r19, 32(%r26)
381	stw		%r20, 36(%r26)
382	stw		%r21, 40(%r26)
383	stw		%r22, 44(%r26)
384	ldw		48(%r25), %r19
385	ldw		52(%r25), %r20
386	ldw		56(%r25), %r21
387	ldw		60(%r25), %r22
388	stw		%r19, 48(%r26)
389	stw		%r20, 52(%r26)
390	ldo		64(%r25), %r25
391	stw		%r21, 56(%r26)
392	stw		%r22, 60(%r26)
393	ldo		64(%r26), %r26
394	ADDIB>,n	-1, %r1, 1b
395	ldw		0(%r25), %r19
396#endif
397	bv		%r0(%r2)
398	nop
399	.exit
400
401	.procend
402ENDPROC(copy_user_page_asm)
403
404/*
405 * NOTE: Code in clear_user_page has a hard coded dependency on the
406 *       maximum alias boundary being 4 Mb. We've been assured by the
407 *       parisc chip designers that there will not ever be a parisc
408 *       chip with a larger alias boundary (Never say never :-) ).
409 *
410 *       Subtle: the dtlb miss handlers support the temp alias region by
411 *       "knowing" that if a dtlb miss happens within the temp alias
412 *       region it must have occurred while in clear_user_page. Since
413 *       this routine makes use of processor local translations, we
414 *       don't want to insert them into the kernel page table. Instead,
415 *       we load up some general registers (they need to be registers
416 *       which aren't shadowed) with the physical page numbers (preshifted
417 *       for tlb insertion) needed to insert the translations. When we
418 *       miss on the translation, the dtlb miss handler inserts the
419 *       translation into the tlb using these values:
420 *
421 *          %r26 physical page (shifted for tlb insert) of "to" translation
422 *          %r23 physical page (shifted for tlb insert) of "from" translation
423 */
424
425#if 0
426
427	/*
428	 * We can't do this since copy_user_page is used to bring in
429	 * file data that might have instructions. Since the data would
430	 * then need to be flushed out so the i-fetch can see it, it
431	 * makes more sense to just copy through the kernel translation
432	 * and flush it.
433	 *
434	 * I'm still keeping this around because it may be possible to
435	 * use it if more information is passed into copy_user_page().
436	 * Have to do some measurements to see if it is worthwhile to
437	 * lobby for such a change.
438	 */
439
440ENTRY(copy_user_page_asm)
441	.proc
442	.callinfo NO_CALLS
443	.entry
444
445	ldil		L%(__PAGE_OFFSET), %r1
446	sub		%r26, %r1, %r26
447	sub		%r25, %r1, %r23		/* move physical addr into non shadowed reg */
448
449	ldil		L%(TMPALIAS_MAP_START), %r28
450	/* FIXME for different page sizes != 4k */
451#ifdef CONFIG_64BIT
452	extrd,u		%r26,56,32, %r26		/* convert phys addr to tlb insert format */
453	extrd,u		%r23,56,32, %r23		/* convert phys addr to tlb insert format */
454	depd		%r24,63,22, %r28		/* Form aliased virtual address 'to' */
455	depdi		0, 63,12, %r28		/* Clear any offset bits */
456	copy		%r28, %r29
457	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
458#else
459	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
460	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
461	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
462	depwi		0, 31,12, %r28		/* Clear any offset bits */
463	copy		%r28, %r29
464	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
465#endif
466
467	/* Purge any old translations */
468
469	pdtlb		0(%r28)
470	pdtlb		0(%r29)
471
472	ldi		64, %r1
473
474	/*
475	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
476	 * bundles (very restricted rules for bundling). It probably
477	 * does OK on PCXU and better, but we could do better with
478	 * ldd/std instructions. Note that until (if) we start saving
479	 * the full 64 bit register values on interrupt, we can't
480	 * use ldd/std on a 32 bit kernel.
481	 */
482
483
4841:
485	ldw		0(%r29), %r19
486	ldw		4(%r29), %r20
487	ldw		8(%r29), %r21
488	ldw		12(%r29), %r22
489	stw		%r19, 0(%r28)
490	stw		%r20, 4(%r28)
491	stw		%r21, 8(%r28)
492	stw		%r22, 12(%r28)
493	ldw		16(%r29), %r19
494	ldw		20(%r29), %r20
495	ldw		24(%r29), %r21
496	ldw		28(%r29), %r22
497	stw		%r19, 16(%r28)
498	stw		%r20, 20(%r28)
499	stw		%r21, 24(%r28)
500	stw		%r22, 28(%r28)
501	ldw		32(%r29), %r19
502	ldw		36(%r29), %r20
503	ldw		40(%r29), %r21
504	ldw		44(%r29), %r22
505	stw		%r19, 32(%r28)
506	stw		%r20, 36(%r28)
507	stw		%r21, 40(%r28)
508	stw		%r22, 44(%r28)
509	ldw		48(%r29), %r19
510	ldw		52(%r29), %r20
511	ldw		56(%r29), %r21
512	ldw		60(%r29), %r22
513	stw		%r19, 48(%r28)
514	stw		%r20, 52(%r28)
515	stw		%r21, 56(%r28)
516	stw		%r22, 60(%r28)
517	ldo		64(%r28), %r28
518	ADDIB>		-1, %r1,1b
519	ldo		64(%r29), %r29
520
521	bv		%r0(%r2)
522	nop
523	.exit
524
525	.procend
526ENDPROC(copy_user_page_asm)
527#endif
528
529ENTRY(__clear_user_page_asm)
530	.proc
531	.callinfo NO_CALLS
532	.entry
533
534	tophys_r1	%r26
535
536	ldil		L%(TMPALIAS_MAP_START), %r28
537#ifdef CONFIG_64BIT
538#if (TMPALIAS_MAP_START >= 0x80000000)
539	depdi		0, 31,32, %r28		/* clear any sign extension */
540	/* FIXME: page size dependend */
541#endif
542	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
543	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
544	depdi		0, 63,12, %r28		/* Clear any offset bits */
545#else
546	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
547	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
548	depwi		0, 31,12, %r28		/* Clear any offset bits */
549#endif
550
551	/* Purge any old translation */
552
553	pdtlb		0(%r28)
554
555#ifdef CONFIG_64BIT
556	ldi		(PAGE_SIZE / 128), %r1
557
558	/* PREFETCH (Write) has not (yet) been proven to help here */
559	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
560
5611:	std		%r0, 0(%r28)
562	std		%r0, 8(%r28)
563	std		%r0, 16(%r28)
564	std		%r0, 24(%r28)
565	std		%r0, 32(%r28)
566	std		%r0, 40(%r28)
567	std		%r0, 48(%r28)
568	std		%r0, 56(%r28)
569	std		%r0, 64(%r28)
570	std		%r0, 72(%r28)
571	std		%r0, 80(%r28)
572	std		%r0, 88(%r28)
573	std		%r0, 96(%r28)
574	std		%r0, 104(%r28)
575	std		%r0, 112(%r28)
576	std		%r0, 120(%r28)
577	ADDIB>		-1, %r1, 1b
578	ldo		128(%r28), %r28
579
580#else	/* ! CONFIG_64BIT */
581	ldi		(PAGE_SIZE / 64), %r1
582
5831:
584	stw		%r0, 0(%r28)
585	stw		%r0, 4(%r28)
586	stw		%r0, 8(%r28)
587	stw		%r0, 12(%r28)
588	stw		%r0, 16(%r28)
589	stw		%r0, 20(%r28)
590	stw		%r0, 24(%r28)
591	stw		%r0, 28(%r28)
592	stw		%r0, 32(%r28)
593	stw		%r0, 36(%r28)
594	stw		%r0, 40(%r28)
595	stw		%r0, 44(%r28)
596	stw		%r0, 48(%r28)
597	stw		%r0, 52(%r28)
598	stw		%r0, 56(%r28)
599	stw		%r0, 60(%r28)
600	ADDIB>		-1, %r1, 1b
601	ldo		64(%r28), %r28
602#endif	/* CONFIG_64BIT */
603
604	bv		%r0(%r2)
605	nop
606	.exit
607
608	.procend
609ENDPROC(__clear_user_page_asm)
610
611ENTRY(flush_kernel_dcache_page_asm)
612	.proc
613	.callinfo NO_CALLS
614	.entry
615
616	ldil		L%dcache_stride, %r1
617	ldw		R%dcache_stride(%r1), %r23
618
619#ifdef CONFIG_64BIT
620	depdi,z		1, 63-PAGE_SHIFT,1, %r25
621#else
622	depwi,z		1, 31-PAGE_SHIFT,1, %r25
623#endif
624	add		%r26, %r25, %r25
625	sub		%r25, %r23, %r25
626
627
6281:      fdc,m		%r23(%r26)
629	fdc,m		%r23(%r26)
630	fdc,m		%r23(%r26)
631	fdc,m		%r23(%r26)
632	fdc,m		%r23(%r26)
633	fdc,m		%r23(%r26)
634	fdc,m		%r23(%r26)
635	fdc,m		%r23(%r26)
636	fdc,m		%r23(%r26)
637	fdc,m		%r23(%r26)
638	fdc,m		%r23(%r26)
639	fdc,m		%r23(%r26)
640	fdc,m		%r23(%r26)
641	fdc,m		%r23(%r26)
642	fdc,m		%r23(%r26)
643	CMPB<<		%r26, %r25,1b
644	fdc,m		%r23(%r26)
645
646	sync
647	bv		%r0(%r2)
648	nop
649	.exit
650
651	.procend
652ENDPROC(flush_kernel_dcache_page_asm)
653
654ENTRY(flush_user_dcache_page)
655	.proc
656	.callinfo NO_CALLS
657	.entry
658
659	ldil		L%dcache_stride, %r1
660	ldw		R%dcache_stride(%r1), %r23
661
662#ifdef CONFIG_64BIT
663	depdi,z		1,63-PAGE_SHIFT,1, %r25
664#else
665	depwi,z		1,31-PAGE_SHIFT,1, %r25
666#endif
667	add		%r26, %r25, %r25
668	sub		%r25, %r23, %r25
669
670
6711:      fdc,m		%r23(%sr3, %r26)
672	fdc,m		%r23(%sr3, %r26)
673	fdc,m		%r23(%sr3, %r26)
674	fdc,m		%r23(%sr3, %r26)
675	fdc,m		%r23(%sr3, %r26)
676	fdc,m		%r23(%sr3, %r26)
677	fdc,m		%r23(%sr3, %r26)
678	fdc,m		%r23(%sr3, %r26)
679	fdc,m		%r23(%sr3, %r26)
680	fdc,m		%r23(%sr3, %r26)
681	fdc,m		%r23(%sr3, %r26)
682	fdc,m		%r23(%sr3, %r26)
683	fdc,m		%r23(%sr3, %r26)
684	fdc,m		%r23(%sr3, %r26)
685	fdc,m		%r23(%sr3, %r26)
686	CMPB<<		%r26, %r25,1b
687	fdc,m		%r23(%sr3, %r26)
688
689	sync
690	bv		%r0(%r2)
691	nop
692	.exit
693
694	.procend
695ENDPROC(flush_user_dcache_page)
696
697ENTRY(flush_user_icache_page)
698	.proc
699	.callinfo NO_CALLS
700	.entry
701
702	ldil		L%dcache_stride, %r1
703	ldw		R%dcache_stride(%r1), %r23
704
705#ifdef CONFIG_64BIT
706	depdi,z		1, 63-PAGE_SHIFT,1, %r25
707#else
708	depwi,z		1, 31-PAGE_SHIFT,1, %r25
709#endif
710	add		%r26, %r25, %r25
711	sub		%r25, %r23, %r25
712
713
7141:      fic,m		%r23(%sr3, %r26)
715	fic,m		%r23(%sr3, %r26)
716	fic,m		%r23(%sr3, %r26)
717	fic,m		%r23(%sr3, %r26)
718	fic,m		%r23(%sr3, %r26)
719	fic,m		%r23(%sr3, %r26)
720	fic,m		%r23(%sr3, %r26)
721	fic,m		%r23(%sr3, %r26)
722	fic,m		%r23(%sr3, %r26)
723	fic,m		%r23(%sr3, %r26)
724	fic,m		%r23(%sr3, %r26)
725	fic,m		%r23(%sr3, %r26)
726	fic,m		%r23(%sr3, %r26)
727	fic,m		%r23(%sr3, %r26)
728	fic,m		%r23(%sr3, %r26)
729	CMPB<<		%r26, %r25,1b
730	fic,m		%r23(%sr3, %r26)
731
732	sync
733	bv		%r0(%r2)
734	nop
735	.exit
736
737	.procend
738ENDPROC(flush_user_icache_page)
739
740
741ENTRY(purge_kernel_dcache_page)
742	.proc
743	.callinfo NO_CALLS
744	.entry
745
746	ldil		L%dcache_stride, %r1
747	ldw		R%dcache_stride(%r1), %r23
748
749#ifdef CONFIG_64BIT
750	depdi,z		1, 63-PAGE_SHIFT,1, %r25
751#else
752	depwi,z		1, 31-PAGE_SHIFT,1, %r25
753#endif
754	add		%r26, %r25, %r25
755	sub		%r25, %r23, %r25
756
7571:      pdc,m		%r23(%r26)
758	pdc,m		%r23(%r26)
759	pdc,m		%r23(%r26)
760	pdc,m		%r23(%r26)
761	pdc,m		%r23(%r26)
762	pdc,m		%r23(%r26)
763	pdc,m		%r23(%r26)
764	pdc,m		%r23(%r26)
765	pdc,m		%r23(%r26)
766	pdc,m		%r23(%r26)
767	pdc,m		%r23(%r26)
768	pdc,m		%r23(%r26)
769	pdc,m		%r23(%r26)
770	pdc,m		%r23(%r26)
771	pdc,m		%r23(%r26)
772	CMPB<<		%r26, %r25, 1b
773	pdc,m		%r23(%r26)
774
775	sync
776	bv		%r0(%r2)
777	nop
778	.exit
779
780	.procend
781ENDPROC(purge_kernel_dcache_page)
782
783#if 0
784	/* Currently not used, but it still is a possible alternate
785	 * solution.
786	 */
787
788ENTRY(flush_alias_page)
789	.proc
790	.callinfo NO_CALLS
791	.entry
792
793	tophys_r1		%r26
794
795	ldil		L%(TMPALIAS_MAP_START), %r28
796#ifdef CONFIG_64BIT
797	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
798	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
799	depdi		0, 63,12, %r28		/* Clear any offset bits */
800#else
801	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
802	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
803	depwi		0, 31,12, %r28		/* Clear any offset bits */
804#endif
805
806	/* Purge any old translation */
807
808	pdtlb		0(%r28)
809
810	ldil		L%dcache_stride, %r1
811	ldw		R%dcache_stride(%r1), %r23
812
813#ifdef CONFIG_64BIT
814	depdi,z		1, 63-PAGE_SHIFT,1, %r29
815#else
816	depwi,z		1, 31-PAGE_SHIFT,1, %r29
817#endif
818	add		%r28, %r29, %r29
819	sub		%r29, %r23, %r29
820
8211:      fdc,m		%r23(%r28)
822	fdc,m		%r23(%r28)
823	fdc,m		%r23(%r28)
824	fdc,m		%r23(%r28)
825	fdc,m		%r23(%r28)
826	fdc,m		%r23(%r28)
827	fdc,m		%r23(%r28)
828	fdc,m		%r23(%r28)
829	fdc,m		%r23(%r28)
830	fdc,m		%r23(%r28)
831	fdc,m		%r23(%r28)
832	fdc,m		%r23(%r28)
833	fdc,m		%r23(%r28)
834	fdc,m		%r23(%r28)
835	fdc,m		%r23(%r28)
836	CMPB<<		%r28, %r29, 1b
837	fdc,m		%r23(%r28)
838
839	sync
840	bv		%r0(%r2)
841	nop
842	.exit
843
844	.procend
845#endif
846
847	.export flush_user_dcache_range_asm
848
849flush_user_dcache_range_asm:
850	.proc
851	.callinfo NO_CALLS
852	.entry
853
854	ldil		L%dcache_stride, %r1
855	ldw		R%dcache_stride(%r1), %r23
856	ldo		-1(%r23), %r21
857	ANDCM		%r26, %r21, %r26
858
8591:      CMPB<<,n	%r26, %r25, 1b
860	fdc,m		%r23(%sr3, %r26)
861
862	sync
863	bv		%r0(%r2)
864	nop
865	.exit
866
867	.procend
868ENDPROC(flush_alias_page)
869
870ENTRY(flush_kernel_dcache_range_asm)
871	.proc
872	.callinfo NO_CALLS
873	.entry
874
875	ldil		L%dcache_stride, %r1
876	ldw		R%dcache_stride(%r1), %r23
877	ldo		-1(%r23), %r21
878	ANDCM		%r26, %r21, %r26
879
8801:      CMPB<<,n	%r26, %r25,1b
881	fdc,m		%r23(%r26)
882
883	sync
884	syncdma
885	bv		%r0(%r2)
886	nop
887	.exit
888
889	.procend
890ENDPROC(flush_kernel_dcache_range_asm)
891
892ENTRY(flush_user_icache_range_asm)
893	.proc
894	.callinfo NO_CALLS
895	.entry
896
897	ldil		L%icache_stride, %r1
898	ldw		R%icache_stride(%r1), %r23
899	ldo		-1(%r23), %r21
900	ANDCM		%r26, %r21, %r26
901
9021:      CMPB<<,n	%r26, %r25,1b
903	fic,m		%r23(%sr3, %r26)
904
905	sync
906	bv		%r0(%r2)
907	nop
908	.exit
909
910	.procend
911ENDPROC(flush_user_icache_range_asm)
912
913ENTRY(flush_kernel_icache_page)
914	.proc
915	.callinfo NO_CALLS
916	.entry
917
918	ldil		L%icache_stride, %r1
919	ldw		R%icache_stride(%r1), %r23
920
921#ifdef CONFIG_64BIT
922	depdi,z		1, 63-PAGE_SHIFT,1, %r25
923#else
924	depwi,z		1, 31-PAGE_SHIFT,1, %r25
925#endif
926	add		%r26, %r25, %r25
927	sub		%r25, %r23, %r25
928
929
9301:      fic,m		%r23(%sr4, %r26)
931	fic,m		%r23(%sr4, %r26)
932	fic,m		%r23(%sr4, %r26)
933	fic,m		%r23(%sr4, %r26)
934	fic,m		%r23(%sr4, %r26)
935	fic,m		%r23(%sr4, %r26)
936	fic,m		%r23(%sr4, %r26)
937	fic,m		%r23(%sr4, %r26)
938	fic,m		%r23(%sr4, %r26)
939	fic,m		%r23(%sr4, %r26)
940	fic,m		%r23(%sr4, %r26)
941	fic,m		%r23(%sr4, %r26)
942	fic,m		%r23(%sr4, %r26)
943	fic,m		%r23(%sr4, %r26)
944	fic,m		%r23(%sr4, %r26)
945	CMPB<<		%r26, %r25, 1b
946	fic,m		%r23(%sr4, %r26)
947
948	sync
949	bv		%r0(%r2)
950	nop
951	.exit
952
953	.procend
954ENDPROC(flush_kernel_icache_page)
955
956ENTRY(flush_kernel_icache_range_asm)
957	.proc
958	.callinfo NO_CALLS
959	.entry
960
961	ldil		L%icache_stride, %r1
962	ldw		R%icache_stride(%r1), %r23
963	ldo		-1(%r23), %r21
964	ANDCM		%r26, %r21, %r26
965
9661:      CMPB<<,n	%r26, %r25, 1b
967	fic,m		%r23(%sr4, %r26)
968
969	sync
970	bv		%r0(%r2)
971	nop
972	.exit
973	.procend
974ENDPROC(flush_kernel_icache_range_asm)
975
976	/* align should cover use of rfi in disable_sr_hashing_asm and
977	 * srdis_done.
978	 */
979	.align	256
980ENTRY(disable_sr_hashing_asm)
981	.proc
982	.callinfo NO_CALLS
983	.entry
984
985	/*
986	 * Switch to real mode
987	 */
988	/* pcxt_ssm_bug */
989	rsm		PSW_SM_I, %r0
990	load32		PA(1f), %r1
991	nop
992	nop
993	nop
994	nop
995	nop
996
997	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
998	mtctl		%r0, %cr17		/* Clear IIASQ tail */
999	mtctl		%r0, %cr17		/* Clear IIASQ head */
1000	mtctl		%r1, %cr18		/* IIAOQ head */
1001	ldo		4(%r1), %r1
1002	mtctl		%r1, %cr18		/* IIAOQ tail */
1003	load32		REAL_MODE_PSW, %r1
1004	mtctl		%r1, %ipsw
1005	rfi
1006	nop
1007
10081:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1009	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1010	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1011	b,n		srdis_done
1012
1013srdis_pcxs:
1014
1015	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1016
1017	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1018	.word		0x141c1a00		/* must issue twice */
1019	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1020	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1021	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1022	.word		0x141c1600		/* must issue twice */
1023	b,n		srdis_done
1024
1025srdis_pcxl:
1026
1027	/* Disable Space Register Hashing for PCXL */
1028
1029	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1030	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1031	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1032	b,n		srdis_done
1033
1034srdis_pa20:
1035
1036	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1037
1038	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1039	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1040	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1041
1042
1043srdis_done:
1044	/* Switch back to virtual mode */
1045	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1046	load32 	   	2f, %r1
1047	nop
1048	nop
1049	nop
1050	nop
1051	nop
1052
1053	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1054	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1055	mtctl		%r0, %cr17		/* Clear IIASQ head */
1056	mtctl		%r1, %cr18		/* IIAOQ head */
1057	ldo		4(%r1), %r1
1058	mtctl		%r1, %cr18		/* IIAOQ tail */
1059	load32		KERNEL_PSW, %r1
1060	mtctl		%r1, %ipsw
1061	rfi
1062	nop
1063
10642:      bv		%r0(%r2)
1065	nop
1066	.exit
1067
1068	.procend
1069ENDPROC(disable_sr_hashing_asm)
1070
1071	.end
1072