xref: /openbmc/linux/arch/sparc/mm/ultra.S (revision 82e6fdd6) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * ultra.S: Don't expand these all over the place...
4 *
5 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
6 */
7
8#include <asm/asi.h>
9#include <asm/pgtable.h>
10#include <asm/page.h>
11#include <asm/spitfire.h>
12#include <asm/mmu_context.h>
13#include <asm/mmu.h>
14#include <asm/pil.h>
15#include <asm/head.h>
16#include <asm/thread_info.h>
17#include <asm/cacheflush.h>
18#include <asm/hypervisor.h>
19#include <asm/cpudata.h>
20
21	/* Basically, most of the Spitfire vs. Cheetah madness
22	 * has to do with the fact that Cheetah does not support
23	 * IMMU flushes out of the secondary context.  Someone needs
24	 * to throw a south lake birthday party for the folks
25	 * in Microelectronics who refused to fix this shit.
26	 */
27
28	/* This file is meant to be read efficiently by the CPU, not humans.
29	 * Staraj sie tego nikomu nie pierdolnac...
30	 */
31	.text
32	.align		32
33	.globl		__flush_tlb_mm
34__flush_tlb_mm:		/* 19 insns */
35	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
36	ldxa		[%o1] ASI_DMMU, %g2
37	cmp		%g2, %o0
38	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
39	 mov		0x50, %g3
40	stxa		%g0, [%g3] ASI_DMMU_DEMAP
41	stxa		%g0, [%g3] ASI_IMMU_DEMAP
42	sethi		%hi(KERNBASE), %g3
43	flush		%g3
44	retl
45	 nop
46	nop
47	nop
48	nop
49	nop
50	nop
51	nop
52	nop
53	nop
54	nop
55
56	.align		32
57	.globl		__flush_tlb_page
58__flush_tlb_page:	/* 22 insns */
59	/* %o0 = context, %o1 = vaddr */
60	rdpr		%pstate, %g7
61	andn		%g7, PSTATE_IE, %g2
62	wrpr		%g2, %pstate
63	mov		SECONDARY_CONTEXT, %o4
64	ldxa		[%o4] ASI_DMMU, %g2
65	stxa		%o0, [%o4] ASI_DMMU
66	andcc		%o1, 1, %g0
67	andn		%o1, 1, %o3
68	be,pn		%icc, 1f
69	 or		%o3, 0x10, %o3
70	stxa		%g0, [%o3] ASI_IMMU_DEMAP
711:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
72	membar		#Sync
73	stxa		%g2, [%o4] ASI_DMMU
74	sethi		%hi(KERNBASE), %o4
75	flush		%o4
76	retl
77	 wrpr		%g7, 0x0, %pstate
78	nop
79	nop
80	nop
81	nop
82
83	.align		32
84	.globl		__flush_tlb_pending
85__flush_tlb_pending:	/* 27 insns */
86	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
87	rdpr		%pstate, %g7
88	sllx		%o1, 3, %o1
89	andn		%g7, PSTATE_IE, %g2
90	wrpr		%g2, %pstate
91	mov		SECONDARY_CONTEXT, %o4
92	ldxa		[%o4] ASI_DMMU, %g2
93	stxa		%o0, [%o4] ASI_DMMU
941:	sub		%o1, (1 << 3), %o1
95	ldx		[%o2 + %o1], %o3
96	andcc		%o3, 1, %g0
97	andn		%o3, 1, %o3
98	be,pn		%icc, 2f
99	 or		%o3, 0x10, %o3
100	stxa		%g0, [%o3] ASI_IMMU_DEMAP
1012:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
102	membar		#Sync
103	brnz,pt		%o1, 1b
104	 nop
105	stxa		%g2, [%o4] ASI_DMMU
106	sethi		%hi(KERNBASE), %o4
107	flush		%o4
108	retl
109	 wrpr		%g7, 0x0, %pstate
110	nop
111	nop
112	nop
113	nop
114
115	.align		32
116	.globl		__flush_tlb_kernel_range
117__flush_tlb_kernel_range:	/* 31 insns */
118	/* %o0=start, %o1=end */
119	cmp		%o0, %o1
120	be,pn		%xcc, 2f
121	 sub		%o1, %o0, %o3
122	srlx		%o3, 18, %o4
123	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
124	 sethi		%hi(PAGE_SIZE), %o4
125	sub		%o3, %o4, %o3
126	or		%o0, 0x20, %o0		! Nucleus
1271:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
128	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
129	membar		#Sync
130	brnz,pt		%o3, 1b
131	 sub		%o3, %o4, %o3
1322:	sethi		%hi(KERNBASE), %o3
133	flush		%o3
134	retl
135	 nop
136	nop
137	nop
138	nop
139	nop
140	nop
141	nop
142	nop
143	nop
144	nop
145	nop
146	nop
147	nop
148	nop
149	nop
150
151__spitfire_flush_tlb_kernel_range_slow:
152	mov		63 * 8, %o4
1531:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
154	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
155	bne,pn		%xcc, 2f
156	 mov		TLB_TAG_ACCESS, %o3
157	stxa		%g0, [%o3] ASI_IMMU
158	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
159	membar		#Sync
1602:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
161	andcc		%o3, 0x40, %g0
162	bne,pn		%xcc, 2f
163	 mov		TLB_TAG_ACCESS, %o3
164	stxa		%g0, [%o3] ASI_DMMU
165	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
166	membar		#Sync
1672:	sub		%o4, 8, %o4
168	brgez,pt	%o4, 1b
169	 nop
170	retl
171	 nop
172
173__spitfire_flush_tlb_mm_slow:
174	rdpr		%pstate, %g1
175	wrpr		%g1, PSTATE_IE, %pstate
176	stxa		%o0, [%o1] ASI_DMMU
177	stxa		%g0, [%g3] ASI_DMMU_DEMAP
178	stxa		%g0, [%g3] ASI_IMMU_DEMAP
179	flush		%g6
180	stxa		%g2, [%o1] ASI_DMMU
181	sethi		%hi(KERNBASE), %o1
182	flush		%o1
183	retl
184	 wrpr		%g1, 0, %pstate
185
186/*
187 * The following code flushes one page_size worth.
188 */
189	.section .kprobes.text, "ax"
190	.align		32
191	.globl		__flush_icache_page
192__flush_icache_page:	/* %o0 = phys_page */
193	srlx		%o0, PAGE_SHIFT, %o0
194	sethi		%hi(PAGE_OFFSET), %g1
195	sllx		%o0, PAGE_SHIFT, %o0
196	sethi		%hi(PAGE_SIZE), %g2
197	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
198	add		%o0, %g1, %o0
1991:	subcc		%g2, 32, %g2
200	bne,pt		%icc, 1b
201	 flush		%o0 + %g2
202	retl
203	 nop
204
205#ifdef DCACHE_ALIASING_POSSIBLE
206
207#if (PAGE_SHIFT != 13)
208#error only page shift of 13 is supported by dcache flush
209#endif
210
211#define DTAG_MASK 0x3
212
213	/* This routine is Spitfire specific so the hardcoded
214	 * D-cache size and line-size are OK.
215	 */
216	.align		64
217	.globl		__flush_dcache_page
218__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
219	sethi		%hi(PAGE_OFFSET), %g1
220	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
221	sub		%o0, %g1, %o0			! physical address
222	srlx		%o0, 11, %o0			! make D-cache TAG
223	sethi		%hi(1 << 14), %o2		! D-cache size
224	sub		%o2, (1 << 5), %o2		! D-cache line size
2251:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
226	andcc		%o3, DTAG_MASK, %g0		! Valid?
227	be,pn		%xcc, 2f			! Nope, branch
228	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
229	cmp		%o3, %o0			! TAG match?
230	bne,pt		%xcc, 2f			! Nope, branch
231	 nop
232	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
233	membar		#Sync
2342:	brnz,pt		%o2, 1b
235	 sub		%o2, (1 << 5), %o2		! D-cache line size
236
237	/* The I-cache does not snoop local stores so we
238	 * better flush that too when necessary.
239	 */
240	brnz,pt		%o1, __flush_icache_page
241	 sllx		%o0, 11, %o0
242	retl
243	 nop
244
245#endif /* DCACHE_ALIASING_POSSIBLE */
246
247	.previous
248
249	/* Cheetah specific versions, patched at boot time. */
250__cheetah_flush_tlb_mm: /* 19 insns */
251	rdpr		%pstate, %g7
252	andn		%g7, PSTATE_IE, %g2
253	wrpr		%g2, 0x0, %pstate
254	wrpr		%g0, 1, %tl
255	mov		PRIMARY_CONTEXT, %o2
256	mov		0x40, %g3
257	ldxa		[%o2] ASI_DMMU, %g2
258	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
259	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
260	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
261	stxa		%o0, [%o2] ASI_DMMU
262	stxa		%g0, [%g3] ASI_DMMU_DEMAP
263	stxa		%g0, [%g3] ASI_IMMU_DEMAP
264	stxa		%g2, [%o2] ASI_DMMU
265	sethi		%hi(KERNBASE), %o2
266	flush		%o2
267	wrpr		%g0, 0, %tl
268	retl
269	 wrpr		%g7, 0x0, %pstate
270
271__cheetah_flush_tlb_page:	/* 22 insns */
272	/* %o0 = context, %o1 = vaddr */
273	rdpr		%pstate, %g7
274	andn		%g7, PSTATE_IE, %g2
275	wrpr		%g2, 0x0, %pstate
276	wrpr		%g0, 1, %tl
277	mov		PRIMARY_CONTEXT, %o4
278	ldxa		[%o4] ASI_DMMU, %g2
279	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
280	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
281	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
282	stxa		%o0, [%o4] ASI_DMMU
283	andcc		%o1, 1, %g0
284	be,pn		%icc, 1f
285	 andn		%o1, 1, %o3
286	stxa		%g0, [%o3] ASI_IMMU_DEMAP
2871:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
288	membar		#Sync
289	stxa		%g2, [%o4] ASI_DMMU
290	sethi		%hi(KERNBASE), %o4
291	flush		%o4
292	wrpr		%g0, 0, %tl
293	retl
294	 wrpr		%g7, 0x0, %pstate
295
296__cheetah_flush_tlb_pending:	/* 27 insns */
297	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
298	rdpr		%pstate, %g7
299	sllx		%o1, 3, %o1
300	andn		%g7, PSTATE_IE, %g2
301	wrpr		%g2, 0x0, %pstate
302	wrpr		%g0, 1, %tl
303	mov		PRIMARY_CONTEXT, %o4
304	ldxa		[%o4] ASI_DMMU, %g2
305	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
306	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
307	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
308	stxa		%o0, [%o4] ASI_DMMU
3091:	sub		%o1, (1 << 3), %o1
310	ldx		[%o2 + %o1], %o3
311	andcc		%o3, 1, %g0
312	be,pn		%icc, 2f
313	 andn		%o3, 1, %o3
314	stxa		%g0, [%o3] ASI_IMMU_DEMAP
3152:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
316	membar		#Sync
317	brnz,pt		%o1, 1b
318	 nop
319	stxa		%g2, [%o4] ASI_DMMU
320	sethi		%hi(KERNBASE), %o4
321	flush		%o4
322	wrpr		%g0, 0, %tl
323	retl
324	 wrpr		%g7, 0x0, %pstate
325
326__cheetah_flush_tlb_kernel_range:	/* 31 insns */
327	/* %o0=start, %o1=end */
328	cmp		%o0, %o1
329	be,pn		%xcc, 2f
330	 sub		%o1, %o0, %o3
331	srlx		%o3, 18, %o4
332	brnz,pn		%o4, 3f
333	 sethi		%hi(PAGE_SIZE), %o4
334	sub		%o3, %o4, %o3
335	or		%o0, 0x20, %o0		! Nucleus
3361:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
337	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
338	membar		#Sync
339	brnz,pt		%o3, 1b
340	 sub		%o3, %o4, %o3
3412:	sethi		%hi(KERNBASE), %o3
342	flush		%o3
343	retl
344	 nop
3453:	mov		0x80, %o4
346	stxa		%g0, [%o4] ASI_DMMU_DEMAP
347	membar		#Sync
348	stxa		%g0, [%o4] ASI_IMMU_DEMAP
349	membar		#Sync
350	retl
351	 nop
352	nop
353	nop
354	nop
355	nop
356	nop
357	nop
358	nop
359
360#ifdef DCACHE_ALIASING_POSSIBLE
361__cheetah_flush_dcache_page: /* 11 insns */
362	sethi		%hi(PAGE_OFFSET), %g1
363	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
364	sub		%o0, %g1, %o0
365	sethi		%hi(PAGE_SIZE), %o4
3661:	subcc		%o4, (1 << 5), %o4
367	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
368	membar		#Sync
369	bne,pt		%icc, 1b
370	 nop
371	retl		/* I-cache flush never needed on Cheetah, see callers. */
372	 nop
373#endif /* DCACHE_ALIASING_POSSIBLE */
374
375	/* Hypervisor specific versions, patched at boot time.  */
376__hypervisor_tlb_tl0_error:
377	save		%sp, -192, %sp
378	mov		%i0, %o0
379	call		hypervisor_tlbop_error
380	 mov		%i1, %o1
381	ret
382	 restore
383
384__hypervisor_flush_tlb_mm: /* 19 insns */
385	mov		%o0, %o2	/* ARG2: mmu context */
386	mov		0, %o0		/* ARG0: CPU lists unimplemented */
387	mov		0, %o1		/* ARG1: CPU lists unimplemented */
388	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
389	mov		HV_FAST_MMU_DEMAP_CTX, %o5
390	ta		HV_FAST_TRAP
391	brnz,pn		%o0, 1f
392	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
393	retl
394	 nop
3951:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
396	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
397	 nop
398	nop
399	nop
400	nop
401	nop
402	nop
403	nop
404
405__hypervisor_flush_tlb_page: /* 22 insns */
406	/* %o0 = context, %o1 = vaddr */
407	mov		%o0, %g2
408	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
409	mov		%g2, %o1	      /* ARG1: mmu context */
410	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
411	srlx		%o0, PAGE_SHIFT, %o0
412	sllx		%o0, PAGE_SHIFT, %o0
413	ta		HV_MMU_UNMAP_ADDR_TRAP
414	brnz,pn		%o0, 1f
415	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
416	retl
417	 nop
4181:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
419	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
420	 nop
421	nop
422	nop
423	nop
424	nop
425	nop
426	nop
427	nop
428	nop
429
430__hypervisor_flush_tlb_pending: /* 27 insns */
431	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
432	sllx		%o1, 3, %g1
433	mov		%o2, %g2
434	mov		%o0, %g3
4351:	sub		%g1, (1 << 3), %g1
436	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
437	mov		%g3, %o1	      /* ARG1: mmu context */
438	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
439	srlx		%o0, PAGE_SHIFT, %o0
440	sllx		%o0, PAGE_SHIFT, %o0
441	ta		HV_MMU_UNMAP_ADDR_TRAP
442	brnz,pn		%o0, 1f
443	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
444	brnz,pt		%g1, 1b
445	 nop
446	retl
447	 nop
4481:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
449	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
450	 nop
451	nop
452	nop
453	nop
454	nop
455	nop
456	nop
457	nop
458	nop
459
460__hypervisor_flush_tlb_kernel_range: /* 31 insns */
461	/* %o0=start, %o1=end */
462	cmp		%o0, %o1
463	be,pn		%xcc, 2f
464	 sub		%o1, %o0, %g2
465	srlx		%g2, 18, %g3
466	brnz,pn		%g3, 4f
467	 mov		%o0, %g1
468	sethi		%hi(PAGE_SIZE), %g3
469	sub		%g2, %g3, %g2
4701:	add		%g1, %g2, %o0	/* ARG0: virtual address */
471	mov		0, %o1		/* ARG1: mmu context */
472	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
473	ta		HV_MMU_UNMAP_ADDR_TRAP
474	brnz,pn		%o0, 3f
475	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
476	brnz,pt		%g2, 1b
477	 sub		%g2, %g3, %g2
4782:	retl
479	 nop
4803:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
481	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
482	 nop
4834:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
484	mov		0, %o1		/* ARG1: CPU lists unimplemented */
485	mov		0, %o2		/* ARG2: mmu context == nucleus */
486	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
487	mov		HV_FAST_MMU_DEMAP_CTX, %o5
488	ta		HV_FAST_TRAP
489	brnz,pn		%o0, 3b
490	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
491	retl
492	 nop
493
494#ifdef DCACHE_ALIASING_POSSIBLE
495	/* XXX Niagara and friends have an 8K cache, so no aliasing is
496	 * XXX possible, but nothing explicit in the Hypervisor API
497	 * XXX guarantees this.
498	 */
499__hypervisor_flush_dcache_page:	/* 2 insns */
500	retl
501	 nop
502#endif
503
504tlb_patch_one:
5051:	lduw		[%o1], %g1
506	stw		%g1, [%o0]
507	flush		%o0
508	subcc		%o2, 1, %o2
509	add		%o1, 4, %o1
510	bne,pt		%icc, 1b
511	 add		%o0, 4, %o0
512	retl
513	 nop
514
515#ifdef CONFIG_SMP
516	/* These are all called by the slaves of a cross call, at
517	 * trap level 1, with interrupts fully disabled.
518	 *
519	 * Register usage:
520	 *   %g5	mm->context	(all tlb flushes)
521	 *   %g1	address arg 1	(tlb page and range flushes)
522	 *   %g7	address arg 2	(tlb range flush only)
523	 *
524	 *   %g6	scratch 1
525	 *   %g2	scratch 2
526	 *   %g3	scratch 3
527	 *   %g4	scratch 4
528	 */
529	.align		32
530	.globl		xcall_flush_tlb_mm
531xcall_flush_tlb_mm:	/* 24 insns */
532	mov		PRIMARY_CONTEXT, %g2
533	ldxa		[%g2] ASI_DMMU, %g3
534	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
535	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
536	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
537	stxa		%g5, [%g2] ASI_DMMU
538	mov		0x40, %g4
539	stxa		%g0, [%g4] ASI_DMMU_DEMAP
540	stxa		%g0, [%g4] ASI_IMMU_DEMAP
541	stxa		%g3, [%g2] ASI_DMMU
542	retry
543	nop
544	nop
545	nop
546	nop
547	nop
548	nop
549	nop
550	nop
551	nop
552	nop
553	nop
554	nop
555	nop
556
557	.globl		xcall_flush_tlb_page
558xcall_flush_tlb_page:	/* 20 insns */
559	/* %g5=context, %g1=vaddr */
560	mov		PRIMARY_CONTEXT, %g4
561	ldxa		[%g4] ASI_DMMU, %g2
562	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
563	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
564	or		%g5, %g4, %g5
565	mov		PRIMARY_CONTEXT, %g4
566	stxa		%g5, [%g4] ASI_DMMU
567	andcc		%g1, 0x1, %g0
568	be,pn		%icc, 2f
569	 andn		%g1, 0x1, %g5
570	stxa		%g0, [%g5] ASI_IMMU_DEMAP
5712:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
572	membar		#Sync
573	stxa		%g2, [%g4] ASI_DMMU
574	retry
575	nop
576	nop
577	nop
578	nop
579	nop
580
581	.globl		xcall_flush_tlb_kernel_range
582xcall_flush_tlb_kernel_range:	/* 44 insns */
583	sethi		%hi(PAGE_SIZE - 1), %g2
584	or		%g2, %lo(PAGE_SIZE - 1), %g2
585	andn		%g1, %g2, %g1
586	andn		%g7, %g2, %g7
587	sub		%g7, %g1, %g3
588	srlx		%g3, 18, %g2
589	brnz,pn		%g2, 2f
590	 add		%g2, 1, %g2
591	sub		%g3, %g2, %g3
592	or		%g1, 0x20, %g1		! Nucleus
5931:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
594	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
595	membar		#Sync
596	brnz,pt		%g3, 1b
597	 sub		%g3, %g2, %g3
598	retry
5992:	mov		63 * 8, %g1
6001:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
601	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
602	bne,pn		%xcc, 2f
603	 mov		TLB_TAG_ACCESS, %g2
604	stxa		%g0, [%g2] ASI_IMMU
605	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
606	membar		#Sync
6072:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
608	andcc		%g2, 0x40, %g0
609	bne,pn		%xcc, 2f
610	 mov		TLB_TAG_ACCESS, %g2
611	stxa		%g0, [%g2] ASI_DMMU
612	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
613	membar		#Sync
6142:	sub		%g1, 8, %g1
615	brgez,pt	%g1, 1b
616	 nop
617	retry
618	nop
619	nop
620	nop
621	nop
622	nop
623	nop
624	nop
625	nop
626	nop
627
628	/* This runs in a very controlled environment, so we do
629	 * not need to worry about BH races etc.
630	 */
631	.globl		xcall_sync_tick
632xcall_sync_tick:
633
634661:	rdpr		%pstate, %g2
635	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
636	.section	.sun4v_2insn_patch, "ax"
637	.word		661b
638	nop
639	nop
640	.previous
641
642	rdpr		%pil, %g2
643	wrpr		%g0, PIL_NORMAL_MAX, %pil
644	sethi		%hi(109f), %g7
645	b,pt		%xcc, etrap_irq
646109:	 or		%g7, %lo(109b), %g7
647#ifdef CONFIG_TRACE_IRQFLAGS
648	call		trace_hardirqs_off
649	 nop
650#endif
651	call		smp_synchronize_tick_client
652	 nop
653	b		rtrap_xcall
654	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
655
656	.globl		xcall_fetch_glob_regs
657xcall_fetch_glob_regs:
658	sethi		%hi(global_cpu_snapshot), %g1
659	or		%g1, %lo(global_cpu_snapshot), %g1
660	__GET_CPUID(%g2)
661	sllx		%g2, 6, %g3
662	add		%g1, %g3, %g1
663	rdpr		%tstate, %g7
664	stx		%g7, [%g1 + GR_SNAP_TSTATE]
665	rdpr		%tpc, %g7
666	stx		%g7, [%g1 + GR_SNAP_TPC]
667	rdpr		%tnpc, %g7
668	stx		%g7, [%g1 + GR_SNAP_TNPC]
669	stx		%o7, [%g1 + GR_SNAP_O7]
670	stx		%i7, [%g1 + GR_SNAP_I7]
671	/* Don't try this at home kids... */
672	rdpr		%cwp, %g3
673	sub		%g3, 1, %g7
674	wrpr		%g7, %cwp
675	mov		%i7, %g7
676	wrpr		%g3, %cwp
677	stx		%g7, [%g1 + GR_SNAP_RPC]
678	sethi		%hi(trap_block), %g7
679	or		%g7, %lo(trap_block), %g7
680	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
681	add		%g7, %g2, %g7
682	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
683	stx		%g3, [%g1 + GR_SNAP_THREAD]
684	retry
685
686	.globl		xcall_fetch_glob_pmu
687xcall_fetch_glob_pmu:
688	sethi		%hi(global_cpu_snapshot), %g1
689	or		%g1, %lo(global_cpu_snapshot), %g1
690	__GET_CPUID(%g2)
691	sllx		%g2, 6, %g3
692	add		%g1, %g3, %g1
693	rd		%pic, %g7
694	stx		%g7, [%g1 + (4 * 8)]
695	rd		%pcr, %g7
696	stx		%g7, [%g1 + (0 * 8)]
697	retry
698
699	.globl		xcall_fetch_glob_pmu_n4
700xcall_fetch_glob_pmu_n4:
701	sethi		%hi(global_cpu_snapshot), %g1
702	or		%g1, %lo(global_cpu_snapshot), %g1
703	__GET_CPUID(%g2)
704	sllx		%g2, 6, %g3
705	add		%g1, %g3, %g1
706
707	ldxa		[%g0] ASI_PIC, %g7
708	stx		%g7, [%g1 + (4 * 8)]
709	mov		0x08, %g3
710	ldxa		[%g3] ASI_PIC, %g7
711	stx		%g7, [%g1 + (5 * 8)]
712	mov		0x10, %g3
713	ldxa		[%g3] ASI_PIC, %g7
714	stx		%g7, [%g1 + (6 * 8)]
715	mov		0x18, %g3
716	ldxa		[%g3] ASI_PIC, %g7
717	stx		%g7, [%g1 + (7 * 8)]
718
719	mov		%o0, %g2
720	mov		%o1, %g3
721	mov		%o5, %g7
722
723	mov		HV_FAST_VT_GET_PERFREG, %o5
724	mov		3, %o0
725	ta		HV_FAST_TRAP
726	stx		%o1, [%g1 + (3 * 8)]
727	mov		HV_FAST_VT_GET_PERFREG, %o5
728	mov		2, %o0
729	ta		HV_FAST_TRAP
730	stx		%o1, [%g1 + (2 * 8)]
731	mov		HV_FAST_VT_GET_PERFREG, %o5
732	mov		1, %o0
733	ta		HV_FAST_TRAP
734	stx		%o1, [%g1 + (1 * 8)]
735	mov		HV_FAST_VT_GET_PERFREG, %o5
736	mov		0, %o0
737	ta		HV_FAST_TRAP
738	stx		%o1, [%g1 + (0 * 8)]
739
740	mov		%g2, %o0
741	mov		%g3, %o1
742	mov		%g7, %o5
743
744	retry
745
746__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
747	sethi		%hi(PAGE_SIZE - 1), %g2
748	or		%g2, %lo(PAGE_SIZE - 1), %g2
749	andn		%g1, %g2, %g1
750	andn		%g7, %g2, %g7
751	sub		%g7, %g1, %g3
752	srlx		%g3, 18, %g2
753	brnz,pn		%g2, 2f
754	 add		%g2, 1, %g2
755	sub		%g3, %g2, %g3
756	or		%g1, 0x20, %g1		! Nucleus
7571:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
758	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
759	membar		#Sync
760	brnz,pt		%g3, 1b
761	 sub		%g3, %g2, %g3
762	retry
7632:	mov		0x80, %g2
764	stxa		%g0, [%g2] ASI_DMMU_DEMAP
765	membar		#Sync
766	stxa		%g0, [%g2] ASI_IMMU_DEMAP
767	membar		#Sync
768	retry
769	nop
770	nop
771	nop
772	nop
773	nop
774	nop
775	nop
776	nop
777	nop
778	nop
779	nop
780	nop
781	nop
782	nop
783	nop
784	nop
785	nop
786	nop
787	nop
788	nop
789	nop
790	nop
791
792#ifdef DCACHE_ALIASING_POSSIBLE
793	.align		32
794	.globl		xcall_flush_dcache_page_cheetah
795xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
796	sethi		%hi(PAGE_SIZE), %g3
7971:	subcc		%g3, (1 << 5), %g3
798	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
799	membar		#Sync
800	bne,pt		%icc, 1b
801	 nop
802	retry
803	nop
804#endif /* DCACHE_ALIASING_POSSIBLE */
805
806	.globl		xcall_flush_dcache_page_spitfire
807xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
808				     %g7 == kernel page virtual address
809				     %g5 == (page->mapping != NULL)  */
810#ifdef DCACHE_ALIASING_POSSIBLE
811	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
812	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
813	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
8141:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
815	andcc		%g2, 0x3, %g0
816	be,pn		%xcc, 2f
817	 andn		%g2, 0x3, %g2
818	cmp		%g2, %g1
819
820	bne,pt		%xcc, 2f
821	 nop
822	stxa		%g0, [%g3] ASI_DCACHE_TAG
823	membar		#Sync
8242:	cmp		%g3, 0
825	bne,pt		%xcc, 1b
826	 sub		%g3, (1 << 5), %g3
827
828	brz,pn		%g5, 2f
829#endif /* DCACHE_ALIASING_POSSIBLE */
830	 sethi		%hi(PAGE_SIZE), %g3
831
8321:	flush		%g7
833	subcc		%g3, (1 << 5), %g3
834	bne,pt		%icc, 1b
835	 add		%g7, (1 << 5), %g7
836
8372:	retry
838	nop
839	nop
840
841	/* %g5:	error
842	 * %g6:	tlb op
843	 */
844__hypervisor_tlb_xcall_error:
845	mov	%g5, %g4
846	mov	%g6, %g5
847	ba,pt	%xcc, etrap
848	 rd	%pc, %g7
849	mov	%l4, %o0
850	call	hypervisor_tlbop_error_xcall
851	 mov	%l5, %o1
852	ba,a,pt	%xcc, rtrap
853
854	.globl		__hypervisor_xcall_flush_tlb_mm
855__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
856	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
857	mov		%o0, %g2
858	mov		%o1, %g3
859	mov		%o2, %g4
860	mov		%o3, %g1
861	mov		%o5, %g7
862	clr		%o0		/* ARG0: CPU lists unimplemented */
863	clr		%o1		/* ARG1: CPU lists unimplemented */
864	mov		%g5, %o2	/* ARG2: mmu context */
865	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
866	mov		HV_FAST_MMU_DEMAP_CTX, %o5
867	ta		HV_FAST_TRAP
868	mov		HV_FAST_MMU_DEMAP_CTX, %g6
869	brnz,pn		%o0, 1f
870	 mov		%o0, %g5
871	mov		%g2, %o0
872	mov		%g3, %o1
873	mov		%g4, %o2
874	mov		%g1, %o3
875	mov		%g7, %o5
876	membar		#Sync
877	retry
8781:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
879	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
880	 nop
881
882	.globl		__hypervisor_xcall_flush_tlb_page
883__hypervisor_xcall_flush_tlb_page: /* 20 insns */
884	/* %g5=ctx, %g1=vaddr */
885	mov		%o0, %g2
886	mov		%o1, %g3
887	mov		%o2, %g4
888	mov		%g1, %o0	        /* ARG0: virtual address */
889	mov		%g5, %o1		/* ARG1: mmu context */
890	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
891	srlx		%o0, PAGE_SHIFT, %o0
892	sllx		%o0, PAGE_SHIFT, %o0
893	ta		HV_MMU_UNMAP_ADDR_TRAP
894	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
895	brnz,a,pn	%o0, 1f
896	 mov		%o0, %g5
897	mov		%g2, %o0
898	mov		%g3, %o1
899	mov		%g4, %o2
900	membar		#Sync
901	retry
9021:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
903	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
904	 nop
905
906	.globl		__hypervisor_xcall_flush_tlb_kernel_range
907__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
908	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
909	sethi		%hi(PAGE_SIZE - 1), %g2
910	or		%g2, %lo(PAGE_SIZE - 1), %g2
911	andn		%g1, %g2, %g1
912	andn		%g7, %g2, %g7
913	sub		%g7, %g1, %g3
914	srlx		%g3, 18, %g7
915	add		%g2, 1, %g2
916	sub		%g3, %g2, %g3
917	mov		%o0, %g2
918	mov		%o1, %g4
919	brnz,pn		%g7, 2f
920	 mov		%o2, %g7
9211:	add		%g1, %g3, %o0	/* ARG0: virtual address */
922	mov		0, %o1		/* ARG1: mmu context */
923	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
924	ta		HV_MMU_UNMAP_ADDR_TRAP
925	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
926	brnz,pn		%o0, 1f
927	 mov		%o0, %g5
928	sethi		%hi(PAGE_SIZE), %o2
929	brnz,pt		%g3, 1b
930	 sub		%g3, %o2, %g3
9315:	mov		%g2, %o0
932	mov		%g4, %o1
933	mov		%g7, %o2
934	membar		#Sync
935	retry
9361:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
937	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
938	 nop
9392:	mov		%o3, %g1
940	mov		%o5, %g3
941	mov		0, %o0		/* ARG0: CPU lists unimplemented */
942	mov		0, %o1		/* ARG1: CPU lists unimplemented */
943	mov		0, %o2		/* ARG2: mmu context == nucleus */
944	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
945	mov		HV_FAST_MMU_DEMAP_CTX, %o5
946	ta		HV_FAST_TRAP
947	mov		%g1, %o3
948	brz,pt		%o0, 5b
949	 mov		%g3, %o5
950	mov		HV_FAST_MMU_DEMAP_CTX, %g6
951	ba,pt		%xcc, 1b
952	 clr		%g5
953
954	/* These just get rescheduled to PIL vectors. */
955	.globl		xcall_call_function
956xcall_call_function:
957	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
958	retry
959
960	.globl		xcall_call_function_single
961xcall_call_function_single:
962	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
963	retry
964
965	.globl		xcall_receive_signal
966xcall_receive_signal:
967	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
968	retry
969
970	.globl		xcall_capture
971xcall_capture:
972	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
973	retry
974
975#ifdef CONFIG_KGDB
976	.globl		xcall_kgdb_capture
977xcall_kgdb_capture:
978	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
979	retry
980#endif
981
982#endif /* CONFIG_SMP */
983
984	.globl		cheetah_patch_cachetlbops
985cheetah_patch_cachetlbops:
986	save		%sp, -128, %sp
987
988	sethi		%hi(__flush_tlb_mm), %o0
989	or		%o0, %lo(__flush_tlb_mm), %o0
990	sethi		%hi(__cheetah_flush_tlb_mm), %o1
991	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
992	call		tlb_patch_one
993	 mov		19, %o2
994
995	sethi		%hi(__flush_tlb_page), %o0
996	or		%o0, %lo(__flush_tlb_page), %o0
997	sethi		%hi(__cheetah_flush_tlb_page), %o1
998	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
999	call		tlb_patch_one
1000	 mov		22, %o2
1001
1002	sethi		%hi(__flush_tlb_pending), %o0
1003	or		%o0, %lo(__flush_tlb_pending), %o0
1004	sethi		%hi(__cheetah_flush_tlb_pending), %o1
1005	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
1006	call		tlb_patch_one
1007	 mov		27, %o2
1008
1009	sethi		%hi(__flush_tlb_kernel_range), %o0
1010	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1011	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
1012	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1013	call		tlb_patch_one
1014	 mov		31, %o2
1015
1016#ifdef DCACHE_ALIASING_POSSIBLE
1017	sethi		%hi(__flush_dcache_page), %o0
1018	or		%o0, %lo(__flush_dcache_page), %o0
1019	sethi		%hi(__cheetah_flush_dcache_page), %o1
1020	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
1021	call		tlb_patch_one
1022	 mov		11, %o2
1023#endif /* DCACHE_ALIASING_POSSIBLE */
1024
1025#ifdef CONFIG_SMP
1026	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1027	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1028	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1029	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1030	call		tlb_patch_one
1031	 mov		44, %o2
1032#endif /* CONFIG_SMP */
1033
1034	ret
1035	 restore
1036
1037	.globl		hypervisor_patch_cachetlbops
1038hypervisor_patch_cachetlbops:
1039	save		%sp, -128, %sp
1040
1041	sethi		%hi(__flush_tlb_mm), %o0
1042	or		%o0, %lo(__flush_tlb_mm), %o0
1043	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
1044	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
1045	call		tlb_patch_one
1046	 mov		19, %o2
1047
1048	sethi		%hi(__flush_tlb_page), %o0
1049	or		%o0, %lo(__flush_tlb_page), %o0
1050	sethi		%hi(__hypervisor_flush_tlb_page), %o1
1051	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
1052	call		tlb_patch_one
1053	 mov		22, %o2
1054
1055	sethi		%hi(__flush_tlb_pending), %o0
1056	or		%o0, %lo(__flush_tlb_pending), %o0
1057	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
1058	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
1059	call		tlb_patch_one
1060	 mov		27, %o2
1061
1062	sethi		%hi(__flush_tlb_kernel_range), %o0
1063	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1064	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
1065	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1066	call		tlb_patch_one
1067	 mov		31, %o2
1068
1069#ifdef DCACHE_ALIASING_POSSIBLE
1070	sethi		%hi(__flush_dcache_page), %o0
1071	or		%o0, %lo(__flush_dcache_page), %o0
1072	sethi		%hi(__hypervisor_flush_dcache_page), %o1
1073	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
1074	call		tlb_patch_one
1075	 mov		2, %o2
1076#endif /* DCACHE_ALIASING_POSSIBLE */
1077
1078#ifdef CONFIG_SMP
1079	sethi		%hi(xcall_flush_tlb_mm), %o0
1080	or		%o0, %lo(xcall_flush_tlb_mm), %o0
1081	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
1082	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1083	call		tlb_patch_one
1084	 mov		24, %o2
1085
1086	sethi		%hi(xcall_flush_tlb_page), %o0
1087	or		%o0, %lo(xcall_flush_tlb_page), %o0
1088	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
1089	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1090	call		tlb_patch_one
1091	 mov		20, %o2
1092
1093	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1094	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1095	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1096	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1097	call		tlb_patch_one
1098	 mov		44, %o2
1099#endif /* CONFIG_SMP */
1100
1101	ret
1102	 restore
1103