xref: /openbmc/linux/arch/sparc/mm/ultra.S (revision 4da722ca) 
1/*
2 * ultra.S: Don't expand these all over the place...
3 *
4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
5 */
6
7#include <asm/asi.h>
8#include <asm/pgtable.h>
9#include <asm/page.h>
10#include <asm/spitfire.h>
11#include <asm/mmu_context.h>
12#include <asm/mmu.h>
13#include <asm/pil.h>
14#include <asm/head.h>
15#include <asm/thread_info.h>
16#include <asm/cacheflush.h>
17#include <asm/hypervisor.h>
18#include <asm/cpudata.h>
19
20	/* Basically, most of the Spitfire vs. Cheetah madness
21	 * has to do with the fact that Cheetah does not support
22	 * IMMU flushes out of the secondary context.  Someone needs
23	 * to throw a south lake birthday party for the folks
24	 * in Microelectronics who refused to fix this shit.
25	 */
26
27	/* This file is meant to be read efficiently by the CPU, not humans.
28	 * Staraj sie tego nikomu nie pierdolnac...
29	 */
30	.text
31	.align		32
32	.globl		__flush_tlb_mm
33__flush_tlb_mm:		/* 19 insns */
34	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
35	ldxa		[%o1] ASI_DMMU, %g2
36	cmp		%g2, %o0
37	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
38	 mov		0x50, %g3
39	stxa		%g0, [%g3] ASI_DMMU_DEMAP
40	stxa		%g0, [%g3] ASI_IMMU_DEMAP
41	sethi		%hi(KERNBASE), %g3
42	flush		%g3
43	retl
44	 nop
45	nop
46	nop
47	nop
48	nop
49	nop
50	nop
51	nop
52	nop
53	nop
54
55	.align		32
56	.globl		__flush_tlb_page
57__flush_tlb_page:	/* 22 insns */
58	/* %o0 = context, %o1 = vaddr */
59	rdpr		%pstate, %g7
60	andn		%g7, PSTATE_IE, %g2
61	wrpr		%g2, %pstate
62	mov		SECONDARY_CONTEXT, %o4
63	ldxa		[%o4] ASI_DMMU, %g2
64	stxa		%o0, [%o4] ASI_DMMU
65	andcc		%o1, 1, %g0
66	andn		%o1, 1, %o3
67	be,pn		%icc, 1f
68	 or		%o3, 0x10, %o3
69	stxa		%g0, [%o3] ASI_IMMU_DEMAP
701:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
71	membar		#Sync
72	stxa		%g2, [%o4] ASI_DMMU
73	sethi		%hi(KERNBASE), %o4
74	flush		%o4
75	retl
76	 wrpr		%g7, 0x0, %pstate
77	nop
78	nop
79	nop
80	nop
81
82	.align		32
83	.globl		__flush_tlb_pending
84__flush_tlb_pending:	/* 27 insns */
85	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
86	rdpr		%pstate, %g7
87	sllx		%o1, 3, %o1
88	andn		%g7, PSTATE_IE, %g2
89	wrpr		%g2, %pstate
90	mov		SECONDARY_CONTEXT, %o4
91	ldxa		[%o4] ASI_DMMU, %g2
92	stxa		%o0, [%o4] ASI_DMMU
931:	sub		%o1, (1 << 3), %o1
94	ldx		[%o2 + %o1], %o3
95	andcc		%o3, 1, %g0
96	andn		%o3, 1, %o3
97	be,pn		%icc, 2f
98	 or		%o3, 0x10, %o3
99	stxa		%g0, [%o3] ASI_IMMU_DEMAP
1002:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
101	membar		#Sync
102	brnz,pt		%o1, 1b
103	 nop
104	stxa		%g2, [%o4] ASI_DMMU
105	sethi		%hi(KERNBASE), %o4
106	flush		%o4
107	retl
108	 wrpr		%g7, 0x0, %pstate
109	nop
110	nop
111	nop
112	nop
113
114	.align		32
115	.globl		__flush_tlb_kernel_range
116__flush_tlb_kernel_range:	/* 31 insns */
117	/* %o0=start, %o1=end */
118	cmp		%o0, %o1
119	be,pn		%xcc, 2f
120	 sub		%o1, %o0, %o3
121	srlx		%o3, 18, %o4
122	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
123	 sethi		%hi(PAGE_SIZE), %o4
124	sub		%o3, %o4, %o3
125	or		%o0, 0x20, %o0		! Nucleus
1261:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
127	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
128	membar		#Sync
129	brnz,pt		%o3, 1b
130	 sub		%o3, %o4, %o3
1312:	sethi		%hi(KERNBASE), %o3
132	flush		%o3
133	retl
134	 nop
135	nop
136	nop
137	nop
138	nop
139	nop
140	nop
141	nop
142	nop
143	nop
144	nop
145	nop
146	nop
147	nop
148	nop
149
150__spitfire_flush_tlb_kernel_range_slow:
151	mov		63 * 8, %o4
1521:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
153	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
154	bne,pn		%xcc, 2f
155	 mov		TLB_TAG_ACCESS, %o3
156	stxa		%g0, [%o3] ASI_IMMU
157	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
158	membar		#Sync
1592:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
160	andcc		%o3, 0x40, %g0
161	bne,pn		%xcc, 2f
162	 mov		TLB_TAG_ACCESS, %o3
163	stxa		%g0, [%o3] ASI_DMMU
164	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
165	membar		#Sync
1662:	sub		%o4, 8, %o4
167	brgez,pt	%o4, 1b
168	 nop
169	retl
170	 nop
171
172__spitfire_flush_tlb_mm_slow:
173	rdpr		%pstate, %g1
174	wrpr		%g1, PSTATE_IE, %pstate
175	stxa		%o0, [%o1] ASI_DMMU
176	stxa		%g0, [%g3] ASI_DMMU_DEMAP
177	stxa		%g0, [%g3] ASI_IMMU_DEMAP
178	flush		%g6
179	stxa		%g2, [%o1] ASI_DMMU
180	sethi		%hi(KERNBASE), %o1
181	flush		%o1
182	retl
183	 wrpr		%g1, 0, %pstate
184
185/*
186 * The following code flushes one page_size worth.
187 */
188	.section .kprobes.text, "ax"
189	.align		32
190	.globl		__flush_icache_page
191__flush_icache_page:	/* %o0 = phys_page */
192	srlx		%o0, PAGE_SHIFT, %o0
193	sethi		%hi(PAGE_OFFSET), %g1
194	sllx		%o0, PAGE_SHIFT, %o0
195	sethi		%hi(PAGE_SIZE), %g2
196	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
197	add		%o0, %g1, %o0
1981:	subcc		%g2, 32, %g2
199	bne,pt		%icc, 1b
200	 flush		%o0 + %g2
201	retl
202	 nop
203
204#ifdef DCACHE_ALIASING_POSSIBLE
205
206#if (PAGE_SHIFT != 13)
207#error only page shift of 13 is supported by dcache flush
208#endif
209
210#define DTAG_MASK 0x3
211
212	/* This routine is Spitfire specific so the hardcoded
213	 * D-cache size and line-size are OK.
214	 */
215	.align		64
216	.globl		__flush_dcache_page
217__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
218	sethi		%hi(PAGE_OFFSET), %g1
219	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
220	sub		%o0, %g1, %o0			! physical address
221	srlx		%o0, 11, %o0			! make D-cache TAG
222	sethi		%hi(1 << 14), %o2		! D-cache size
223	sub		%o2, (1 << 5), %o2		! D-cache line size
2241:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
225	andcc		%o3, DTAG_MASK, %g0		! Valid?
226	be,pn		%xcc, 2f			! Nope, branch
227	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
228	cmp		%o3, %o0			! TAG match?
229	bne,pt		%xcc, 2f			! Nope, branch
230	 nop
231	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
232	membar		#Sync
2332:	brnz,pt		%o2, 1b
234	 sub		%o2, (1 << 5), %o2		! D-cache line size
235
236	/* The I-cache does not snoop local stores so we
237	 * better flush that too when necessary.
238	 */
239	brnz,pt		%o1, __flush_icache_page
240	 sllx		%o0, 11, %o0
241	retl
242	 nop
243
244#endif /* DCACHE_ALIASING_POSSIBLE */
245
246	.previous
247
248	/* Cheetah specific versions, patched at boot time. */
249__cheetah_flush_tlb_mm: /* 19 insns */
250	rdpr		%pstate, %g7
251	andn		%g7, PSTATE_IE, %g2
252	wrpr		%g2, 0x0, %pstate
253	wrpr		%g0, 1, %tl
254	mov		PRIMARY_CONTEXT, %o2
255	mov		0x40, %g3
256	ldxa		[%o2] ASI_DMMU, %g2
257	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
258	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
259	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
260	stxa		%o0, [%o2] ASI_DMMU
261	stxa		%g0, [%g3] ASI_DMMU_DEMAP
262	stxa		%g0, [%g3] ASI_IMMU_DEMAP
263	stxa		%g2, [%o2] ASI_DMMU
264	sethi		%hi(KERNBASE), %o2
265	flush		%o2
266	wrpr		%g0, 0, %tl
267	retl
268	 wrpr		%g7, 0x0, %pstate
269
270__cheetah_flush_tlb_page:	/* 22 insns */
271	/* %o0 = context, %o1 = vaddr */
272	rdpr		%pstate, %g7
273	andn		%g7, PSTATE_IE, %g2
274	wrpr		%g2, 0x0, %pstate
275	wrpr		%g0, 1, %tl
276	mov		PRIMARY_CONTEXT, %o4
277	ldxa		[%o4] ASI_DMMU, %g2
278	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
279	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
280	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
281	stxa		%o0, [%o4] ASI_DMMU
282	andcc		%o1, 1, %g0
283	be,pn		%icc, 1f
284	 andn		%o1, 1, %o3
285	stxa		%g0, [%o3] ASI_IMMU_DEMAP
2861:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
287	membar		#Sync
288	stxa		%g2, [%o4] ASI_DMMU
289	sethi		%hi(KERNBASE), %o4
290	flush		%o4
291	wrpr		%g0, 0, %tl
292	retl
293	 wrpr		%g7, 0x0, %pstate
294
295__cheetah_flush_tlb_pending:	/* 27 insns */
296	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
297	rdpr		%pstate, %g7
298	sllx		%o1, 3, %o1
299	andn		%g7, PSTATE_IE, %g2
300	wrpr		%g2, 0x0, %pstate
301	wrpr		%g0, 1, %tl
302	mov		PRIMARY_CONTEXT, %o4
303	ldxa		[%o4] ASI_DMMU, %g2
304	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
305	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
306	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
307	stxa		%o0, [%o4] ASI_DMMU
3081:	sub		%o1, (1 << 3), %o1
309	ldx		[%o2 + %o1], %o3
310	andcc		%o3, 1, %g0
311	be,pn		%icc, 2f
312	 andn		%o3, 1, %o3
313	stxa		%g0, [%o3] ASI_IMMU_DEMAP
3142:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
315	membar		#Sync
316	brnz,pt		%o1, 1b
317	 nop
318	stxa		%g2, [%o4] ASI_DMMU
319	sethi		%hi(KERNBASE), %o4
320	flush		%o4
321	wrpr		%g0, 0, %tl
322	retl
323	 wrpr		%g7, 0x0, %pstate
324
325__cheetah_flush_tlb_kernel_range:	/* 31 insns */
326	/* %o0=start, %o1=end */
327	cmp		%o0, %o1
328	be,pn		%xcc, 2f
329	 sub		%o1, %o0, %o3
330	srlx		%o3, 18, %o4
331	brnz,pn		%o4, 3f
332	 sethi		%hi(PAGE_SIZE), %o4
333	sub		%o3, %o4, %o3
334	or		%o0, 0x20, %o0		! Nucleus
3351:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
336	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
337	membar		#Sync
338	brnz,pt		%o3, 1b
339	 sub		%o3, %o4, %o3
3402:	sethi		%hi(KERNBASE), %o3
341	flush		%o3
342	retl
343	 nop
3443:	mov		0x80, %o4
345	stxa		%g0, [%o4] ASI_DMMU_DEMAP
346	membar		#Sync
347	stxa		%g0, [%o4] ASI_IMMU_DEMAP
348	membar		#Sync
349	retl
350	 nop
351	nop
352	nop
353	nop
354	nop
355	nop
356	nop
357	nop
358
359#ifdef DCACHE_ALIASING_POSSIBLE
360__cheetah_flush_dcache_page: /* 11 insns */
361	sethi		%hi(PAGE_OFFSET), %g1
362	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
363	sub		%o0, %g1, %o0
364	sethi		%hi(PAGE_SIZE), %o4
3651:	subcc		%o4, (1 << 5), %o4
366	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
367	membar		#Sync
368	bne,pt		%icc, 1b
369	 nop
370	retl		/* I-cache flush never needed on Cheetah, see callers. */
371	 nop
372#endif /* DCACHE_ALIASING_POSSIBLE */
373
374	/* Hypervisor specific versions, patched at boot time.  */
375__hypervisor_tlb_tl0_error:
376	save		%sp, -192, %sp
377	mov		%i0, %o0
378	call		hypervisor_tlbop_error
379	 mov		%i1, %o1
380	ret
381	 restore
382
383__hypervisor_flush_tlb_mm: /* 19 insns */
384	mov		%o0, %o2	/* ARG2: mmu context */
385	mov		0, %o0		/* ARG0: CPU lists unimplemented */
386	mov		0, %o1		/* ARG1: CPU lists unimplemented */
387	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
388	mov		HV_FAST_MMU_DEMAP_CTX, %o5
389	ta		HV_FAST_TRAP
390	brnz,pn		%o0, 1f
391	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
392	retl
393	 nop
3941:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
395	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
396	 nop
397	nop
398	nop
399	nop
400	nop
401	nop
402	nop
403
404__hypervisor_flush_tlb_page: /* 22 insns */
405	/* %o0 = context, %o1 = vaddr */
406	mov		%o0, %g2
407	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
408	mov		%g2, %o1	      /* ARG1: mmu context */
409	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
410	srlx		%o0, PAGE_SHIFT, %o0
411	sllx		%o0, PAGE_SHIFT, %o0
412	ta		HV_MMU_UNMAP_ADDR_TRAP
413	brnz,pn		%o0, 1f
414	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
415	retl
416	 nop
4171:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
418	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
419	 nop
420	nop
421	nop
422	nop
423	nop
424	nop
425	nop
426	nop
427	nop
428
429__hypervisor_flush_tlb_pending: /* 27 insns */
430	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
431	sllx		%o1, 3, %g1
432	mov		%o2, %g2
433	mov		%o0, %g3
4341:	sub		%g1, (1 << 3), %g1
435	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
436	mov		%g3, %o1	      /* ARG1: mmu context */
437	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
438	srlx		%o0, PAGE_SHIFT, %o0
439	sllx		%o0, PAGE_SHIFT, %o0
440	ta		HV_MMU_UNMAP_ADDR_TRAP
441	brnz,pn		%o0, 1f
442	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
443	brnz,pt		%g1, 1b
444	 nop
445	retl
446	 nop
4471:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
448	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
449	 nop
450	nop
451	nop
452	nop
453	nop
454	nop
455	nop
456	nop
457	nop
458
459__hypervisor_flush_tlb_kernel_range: /* 31 insns */
460	/* %o0=start, %o1=end */
461	cmp		%o0, %o1
462	be,pn		%xcc, 2f
463	 sub		%o1, %o0, %g2
464	srlx		%g2, 18, %g3
465	brnz,pn		%g3, 4f
466	 mov		%o0, %g1
467	sethi		%hi(PAGE_SIZE), %g3
468	sub		%g2, %g3, %g2
4691:	add		%g1, %g2, %o0	/* ARG0: virtual address */
470	mov		0, %o1		/* ARG1: mmu context */
471	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
472	ta		HV_MMU_UNMAP_ADDR_TRAP
473	brnz,pn		%o0, 3f
474	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
475	brnz,pt		%g2, 1b
476	 sub		%g2, %g3, %g2
4772:	retl
478	 nop
4793:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
480	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
481	 nop
4824:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
483	mov		0, %o1		/* ARG1: CPU lists unimplemented */
484	mov		0, %o2		/* ARG2: mmu context == nucleus */
485	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
486	mov		HV_FAST_MMU_DEMAP_CTX, %o5
487	ta		HV_FAST_TRAP
488	brnz,pn		%o0, 3b
489	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
490	retl
491	 nop
492
493#ifdef DCACHE_ALIASING_POSSIBLE
494	/* XXX Niagara and friends have an 8K cache, so no aliasing is
495	 * XXX possible, but nothing explicit in the Hypervisor API
496	 * XXX guarantees this.
497	 */
498__hypervisor_flush_dcache_page:	/* 2 insns */
499	retl
500	 nop
501#endif
502
503tlb_patch_one:
5041:	lduw		[%o1], %g1
505	stw		%g1, [%o0]
506	flush		%o0
507	subcc		%o2, 1, %o2
508	add		%o1, 4, %o1
509	bne,pt		%icc, 1b
510	 add		%o0, 4, %o0
511	retl
512	 nop
513
514#ifdef CONFIG_SMP
515	/* These are all called by the slaves of a cross call, at
516	 * trap level 1, with interrupts fully disabled.
517	 *
518	 * Register usage:
519	 *   %g5	mm->context	(all tlb flushes)
520	 *   %g1	address arg 1	(tlb page and range flushes)
521	 *   %g7	address arg 2	(tlb range flush only)
522	 *
523	 *   %g6	scratch 1
524	 *   %g2	scratch 2
525	 *   %g3	scratch 3
526	 *   %g4	scratch 4
527	 */
528	.align		32
529	.globl		xcall_flush_tlb_mm
530xcall_flush_tlb_mm:	/* 24 insns */
531	mov		PRIMARY_CONTEXT, %g2
532	ldxa		[%g2] ASI_DMMU, %g3
533	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
534	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
535	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
536	stxa		%g5, [%g2] ASI_DMMU
537	mov		0x40, %g4
538	stxa		%g0, [%g4] ASI_DMMU_DEMAP
539	stxa		%g0, [%g4] ASI_IMMU_DEMAP
540	stxa		%g3, [%g2] ASI_DMMU
541	retry
542	nop
543	nop
544	nop
545	nop
546	nop
547	nop
548	nop
549	nop
550	nop
551	nop
552	nop
553	nop
554	nop
555
556	.globl		xcall_flush_tlb_page
557xcall_flush_tlb_page:	/* 20 insns */
558	/* %g5=context, %g1=vaddr */
559	mov		PRIMARY_CONTEXT, %g4
560	ldxa		[%g4] ASI_DMMU, %g2
561	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
562	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
563	or		%g5, %g4, %g5
564	mov		PRIMARY_CONTEXT, %g4
565	stxa		%g5, [%g4] ASI_DMMU
566	andcc		%g1, 0x1, %g0
567	be,pn		%icc, 2f
568	 andn		%g1, 0x1, %g5
569	stxa		%g0, [%g5] ASI_IMMU_DEMAP
5702:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
571	membar		#Sync
572	stxa		%g2, [%g4] ASI_DMMU
573	retry
574	nop
575	nop
576	nop
577	nop
578	nop
579
580	.globl		xcall_flush_tlb_kernel_range
581xcall_flush_tlb_kernel_range:	/* 44 insns */
582	sethi		%hi(PAGE_SIZE - 1), %g2
583	or		%g2, %lo(PAGE_SIZE - 1), %g2
584	andn		%g1, %g2, %g1
585	andn		%g7, %g2, %g7
586	sub		%g7, %g1, %g3
587	srlx		%g3, 18, %g2
588	brnz,pn		%g2, 2f
589	 add		%g2, 1, %g2
590	sub		%g3, %g2, %g3
591	or		%g1, 0x20, %g1		! Nucleus
5921:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
593	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
594	membar		#Sync
595	brnz,pt		%g3, 1b
596	 sub		%g3, %g2, %g3
597	retry
5982:	mov		63 * 8, %g1
5991:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
600	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
601	bne,pn		%xcc, 2f
602	 mov		TLB_TAG_ACCESS, %g2
603	stxa		%g0, [%g2] ASI_IMMU
604	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
605	membar		#Sync
6062:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
607	andcc		%g2, 0x40, %g0
608	bne,pn		%xcc, 2f
609	 mov		TLB_TAG_ACCESS, %g2
610	stxa		%g0, [%g2] ASI_DMMU
611	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
612	membar		#Sync
6132:	sub		%g1, 8, %g1
614	brgez,pt	%g1, 1b
615	 nop
616	retry
617	nop
618	nop
619	nop
620	nop
621	nop
622	nop
623	nop
624	nop
625	nop
626
627	/* This runs in a very controlled environment, so we do
628	 * not need to worry about BH races etc.
629	 */
630	.globl		xcall_sync_tick
631xcall_sync_tick:
632
633661:	rdpr		%pstate, %g2
634	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
635	.section	.sun4v_2insn_patch, "ax"
636	.word		661b
637	nop
638	nop
639	.previous
640
641	rdpr		%pil, %g2
642	wrpr		%g0, PIL_NORMAL_MAX, %pil
643	sethi		%hi(109f), %g7
644	b,pt		%xcc, etrap_irq
645109:	 or		%g7, %lo(109b), %g7
646#ifdef CONFIG_TRACE_IRQFLAGS
647	call		trace_hardirqs_off
648	 nop
649#endif
650	call		smp_synchronize_tick_client
651	 nop
652	b		rtrap_xcall
653	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
654
655	.globl		xcall_fetch_glob_regs
656xcall_fetch_glob_regs:
657	sethi		%hi(global_cpu_snapshot), %g1
658	or		%g1, %lo(global_cpu_snapshot), %g1
659	__GET_CPUID(%g2)
660	sllx		%g2, 6, %g3
661	add		%g1, %g3, %g1
662	rdpr		%tstate, %g7
663	stx		%g7, [%g1 + GR_SNAP_TSTATE]
664	rdpr		%tpc, %g7
665	stx		%g7, [%g1 + GR_SNAP_TPC]
666	rdpr		%tnpc, %g7
667	stx		%g7, [%g1 + GR_SNAP_TNPC]
668	stx		%o7, [%g1 + GR_SNAP_O7]
669	stx		%i7, [%g1 + GR_SNAP_I7]
670	/* Don't try this at home kids... */
671	rdpr		%cwp, %g3
672	sub		%g3, 1, %g7
673	wrpr		%g7, %cwp
674	mov		%i7, %g7
675	wrpr		%g3, %cwp
676	stx		%g7, [%g1 + GR_SNAP_RPC]
677	sethi		%hi(trap_block), %g7
678	or		%g7, %lo(trap_block), %g7
679	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
680	add		%g7, %g2, %g7
681	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
682	stx		%g3, [%g1 + GR_SNAP_THREAD]
683	retry
684
685	.globl		xcall_fetch_glob_pmu
686xcall_fetch_glob_pmu:
687	sethi		%hi(global_cpu_snapshot), %g1
688	or		%g1, %lo(global_cpu_snapshot), %g1
689	__GET_CPUID(%g2)
690	sllx		%g2, 6, %g3
691	add		%g1, %g3, %g1
692	rd		%pic, %g7
693	stx		%g7, [%g1 + (4 * 8)]
694	rd		%pcr, %g7
695	stx		%g7, [%g1 + (0 * 8)]
696	retry
697
698	.globl		xcall_fetch_glob_pmu_n4
699xcall_fetch_glob_pmu_n4:
700	sethi		%hi(global_cpu_snapshot), %g1
701	or		%g1, %lo(global_cpu_snapshot), %g1
702	__GET_CPUID(%g2)
703	sllx		%g2, 6, %g3
704	add		%g1, %g3, %g1
705
706	ldxa		[%g0] ASI_PIC, %g7
707	stx		%g7, [%g1 + (4 * 8)]
708	mov		0x08, %g3
709	ldxa		[%g3] ASI_PIC, %g7
710	stx		%g7, [%g1 + (5 * 8)]
711	mov		0x10, %g3
712	ldxa		[%g3] ASI_PIC, %g7
713	stx		%g7, [%g1 + (6 * 8)]
714	mov		0x18, %g3
715	ldxa		[%g3] ASI_PIC, %g7
716	stx		%g7, [%g1 + (7 * 8)]
717
718	mov		%o0, %g2
719	mov		%o1, %g3
720	mov		%o5, %g7
721
722	mov		HV_FAST_VT_GET_PERFREG, %o5
723	mov		3, %o0
724	ta		HV_FAST_TRAP
725	stx		%o1, [%g1 + (3 * 8)]
726	mov		HV_FAST_VT_GET_PERFREG, %o5
727	mov		2, %o0
728	ta		HV_FAST_TRAP
729	stx		%o1, [%g1 + (2 * 8)]
730	mov		HV_FAST_VT_GET_PERFREG, %o5
731	mov		1, %o0
732	ta		HV_FAST_TRAP
733	stx		%o1, [%g1 + (1 * 8)]
734	mov		HV_FAST_VT_GET_PERFREG, %o5
735	mov		0, %o0
736	ta		HV_FAST_TRAP
737	stx		%o1, [%g1 + (0 * 8)]
738
739	mov		%g2, %o0
740	mov		%g3, %o1
741	mov		%g7, %o5
742
743	retry
744
745__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
746	sethi		%hi(PAGE_SIZE - 1), %g2
747	or		%g2, %lo(PAGE_SIZE - 1), %g2
748	andn		%g1, %g2, %g1
749	andn		%g7, %g2, %g7
750	sub		%g7, %g1, %g3
751	srlx		%g3, 18, %g2
752	brnz,pn		%g2, 2f
753	 add		%g2, 1, %g2
754	sub		%g3, %g2, %g3
755	or		%g1, 0x20, %g1		! Nucleus
7561:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
757	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
758	membar		#Sync
759	brnz,pt		%g3, 1b
760	 sub		%g3, %g2, %g3
761	retry
7622:	mov		0x80, %g2
763	stxa		%g0, [%g2] ASI_DMMU_DEMAP
764	membar		#Sync
765	stxa		%g0, [%g2] ASI_IMMU_DEMAP
766	membar		#Sync
767	retry
768	nop
769	nop
770	nop
771	nop
772	nop
773	nop
774	nop
775	nop
776	nop
777	nop
778	nop
779	nop
780	nop
781	nop
782	nop
783	nop
784	nop
785	nop
786	nop
787	nop
788	nop
789	nop
790
791#ifdef DCACHE_ALIASING_POSSIBLE
792	.align		32
793	.globl		xcall_flush_dcache_page_cheetah
794xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
795	sethi		%hi(PAGE_SIZE), %g3
7961:	subcc		%g3, (1 << 5), %g3
797	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
798	membar		#Sync
799	bne,pt		%icc, 1b
800	 nop
801	retry
802	nop
803#endif /* DCACHE_ALIASING_POSSIBLE */
804
805	.globl		xcall_flush_dcache_page_spitfire
806xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
807				     %g7 == kernel page virtual address
808				     %g5 == (page->mapping != NULL)  */
809#ifdef DCACHE_ALIASING_POSSIBLE
810	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
811	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
812	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
8131:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
814	andcc		%g2, 0x3, %g0
815	be,pn		%xcc, 2f
816	 andn		%g2, 0x3, %g2
817	cmp		%g2, %g1
818
819	bne,pt		%xcc, 2f
820	 nop
821	stxa		%g0, [%g3] ASI_DCACHE_TAG
822	membar		#Sync
8232:	cmp		%g3, 0
824	bne,pt		%xcc, 1b
825	 sub		%g3, (1 << 5), %g3
826
827	brz,pn		%g5, 2f
828#endif /* DCACHE_ALIASING_POSSIBLE */
829	 sethi		%hi(PAGE_SIZE), %g3
830
8311:	flush		%g7
832	subcc		%g3, (1 << 5), %g3
833	bne,pt		%icc, 1b
834	 add		%g7, (1 << 5), %g7
835
8362:	retry
837	nop
838	nop
839
840	/* %g5:	error
841	 * %g6:	tlb op
842	 */
843__hypervisor_tlb_xcall_error:
844	mov	%g5, %g4
845	mov	%g6, %g5
846	ba,pt	%xcc, etrap
847	 rd	%pc, %g7
848	mov	%l4, %o0
849	call	hypervisor_tlbop_error_xcall
850	 mov	%l5, %o1
851	ba,a,pt	%xcc, rtrap
852
853	.globl		__hypervisor_xcall_flush_tlb_mm
854__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
855	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
856	mov		%o0, %g2
857	mov		%o1, %g3
858	mov		%o2, %g4
859	mov		%o3, %g1
860	mov		%o5, %g7
861	clr		%o0		/* ARG0: CPU lists unimplemented */
862	clr		%o1		/* ARG1: CPU lists unimplemented */
863	mov		%g5, %o2	/* ARG2: mmu context */
864	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
865	mov		HV_FAST_MMU_DEMAP_CTX, %o5
866	ta		HV_FAST_TRAP
867	mov		HV_FAST_MMU_DEMAP_CTX, %g6
868	brnz,pn		%o0, 1f
869	 mov		%o0, %g5
870	mov		%g2, %o0
871	mov		%g3, %o1
872	mov		%g4, %o2
873	mov		%g1, %o3
874	mov		%g7, %o5
875	membar		#Sync
876	retry
8771:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
878	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
879	 nop
880
881	.globl		__hypervisor_xcall_flush_tlb_page
882__hypervisor_xcall_flush_tlb_page: /* 20 insns */
883	/* %g5=ctx, %g1=vaddr */
884	mov		%o0, %g2
885	mov		%o1, %g3
886	mov		%o2, %g4
887	mov		%g1, %o0	        /* ARG0: virtual address */
888	mov		%g5, %o1		/* ARG1: mmu context */
889	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
890	srlx		%o0, PAGE_SHIFT, %o0
891	sllx		%o0, PAGE_SHIFT, %o0
892	ta		HV_MMU_UNMAP_ADDR_TRAP
893	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
894	brnz,a,pn	%o0, 1f
895	 mov		%o0, %g5
896	mov		%g2, %o0
897	mov		%g3, %o1
898	mov		%g4, %o2
899	membar		#Sync
900	retry
9011:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
902	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
903	 nop
904
905	.globl		__hypervisor_xcall_flush_tlb_kernel_range
906__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
907	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
908	sethi		%hi(PAGE_SIZE - 1), %g2
909	or		%g2, %lo(PAGE_SIZE - 1), %g2
910	andn		%g1, %g2, %g1
911	andn		%g7, %g2, %g7
912	sub		%g7, %g1, %g3
913	srlx		%g3, 18, %g7
914	add		%g2, 1, %g2
915	sub		%g3, %g2, %g3
916	mov		%o0, %g2
917	mov		%o1, %g4
918	brnz,pn		%g7, 2f
919	 mov		%o2, %g7
9201:	add		%g1, %g3, %o0	/* ARG0: virtual address */
921	mov		0, %o1		/* ARG1: mmu context */
922	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
923	ta		HV_MMU_UNMAP_ADDR_TRAP
924	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
925	brnz,pn		%o0, 1f
926	 mov		%o0, %g5
927	sethi		%hi(PAGE_SIZE), %o2
928	brnz,pt		%g3, 1b
929	 sub		%g3, %o2, %g3
9305:	mov		%g2, %o0
931	mov		%g4, %o1
932	mov		%g7, %o2
933	membar		#Sync
934	retry
9351:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
936	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
937	 nop
9382:	mov		%o3, %g1
939	mov		%o5, %g3
940	mov		0, %o0		/* ARG0: CPU lists unimplemented */
941	mov		0, %o1		/* ARG1: CPU lists unimplemented */
942	mov		0, %o2		/* ARG2: mmu context == nucleus */
943	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
944	mov		HV_FAST_MMU_DEMAP_CTX, %o5
945	ta		HV_FAST_TRAP
946	mov		%g1, %o3
947	brz,pt		%o0, 5b
948	 mov		%g3, %o5
949	mov		HV_FAST_MMU_DEMAP_CTX, %g6
950	ba,pt		%xcc, 1b
951	 clr		%g5
952
953	/* These just get rescheduled to PIL vectors. */
954	.globl		xcall_call_function
955xcall_call_function:
956	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
957	retry
958
959	.globl		xcall_call_function_single
960xcall_call_function_single:
961	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
962	retry
963
964	.globl		xcall_receive_signal
965xcall_receive_signal:
966	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
967	retry
968
969	.globl		xcall_capture
970xcall_capture:
971	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
972	retry
973
974#ifdef CONFIG_KGDB
975	.globl		xcall_kgdb_capture
976xcall_kgdb_capture:
977	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
978	retry
979#endif
980
981#endif /* CONFIG_SMP */
982
983	.globl		cheetah_patch_cachetlbops
984cheetah_patch_cachetlbops:
985	save		%sp, -128, %sp
986
987	sethi		%hi(__flush_tlb_mm), %o0
988	or		%o0, %lo(__flush_tlb_mm), %o0
989	sethi		%hi(__cheetah_flush_tlb_mm), %o1
990	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
991	call		tlb_patch_one
992	 mov		19, %o2
993
994	sethi		%hi(__flush_tlb_page), %o0
995	or		%o0, %lo(__flush_tlb_page), %o0
996	sethi		%hi(__cheetah_flush_tlb_page), %o1
997	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
998	call		tlb_patch_one
999	 mov		22, %o2
1000
1001	sethi		%hi(__flush_tlb_pending), %o0
1002	or		%o0, %lo(__flush_tlb_pending), %o0
1003	sethi		%hi(__cheetah_flush_tlb_pending), %o1
1004	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
1005	call		tlb_patch_one
1006	 mov		27, %o2
1007
1008	sethi		%hi(__flush_tlb_kernel_range), %o0
1009	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1010	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
1011	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1012	call		tlb_patch_one
1013	 mov		31, %o2
1014
1015#ifdef DCACHE_ALIASING_POSSIBLE
1016	sethi		%hi(__flush_dcache_page), %o0
1017	or		%o0, %lo(__flush_dcache_page), %o0
1018	sethi		%hi(__cheetah_flush_dcache_page), %o1
1019	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
1020	call		tlb_patch_one
1021	 mov		11, %o2
1022#endif /* DCACHE_ALIASING_POSSIBLE */
1023
1024#ifdef CONFIG_SMP
1025	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1026	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1027	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1028	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1029	call		tlb_patch_one
1030	 mov		44, %o2
1031#endif /* CONFIG_SMP */
1032
1033	ret
1034	 restore
1035
1036	.globl		hypervisor_patch_cachetlbops
1037hypervisor_patch_cachetlbops:
1038	save		%sp, -128, %sp
1039
1040	sethi		%hi(__flush_tlb_mm), %o0
1041	or		%o0, %lo(__flush_tlb_mm), %o0
1042	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
1043	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
1044	call		tlb_patch_one
1045	 mov		19, %o2
1046
1047	sethi		%hi(__flush_tlb_page), %o0
1048	or		%o0, %lo(__flush_tlb_page), %o0
1049	sethi		%hi(__hypervisor_flush_tlb_page), %o1
1050	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
1051	call		tlb_patch_one
1052	 mov		22, %o2
1053
1054	sethi		%hi(__flush_tlb_pending), %o0
1055	or		%o0, %lo(__flush_tlb_pending), %o0
1056	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
1057	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
1058	call		tlb_patch_one
1059	 mov		27, %o2
1060
1061	sethi		%hi(__flush_tlb_kernel_range), %o0
1062	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1063	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
1064	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1065	call		tlb_patch_one
1066	 mov		31, %o2
1067
1068#ifdef DCACHE_ALIASING_POSSIBLE
1069	sethi		%hi(__flush_dcache_page), %o0
1070	or		%o0, %lo(__flush_dcache_page), %o0
1071	sethi		%hi(__hypervisor_flush_dcache_page), %o1
1072	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
1073	call		tlb_patch_one
1074	 mov		2, %o2
1075#endif /* DCACHE_ALIASING_POSSIBLE */
1076
1077#ifdef CONFIG_SMP
1078	sethi		%hi(xcall_flush_tlb_mm), %o0
1079	or		%o0, %lo(xcall_flush_tlb_mm), %o0
1080	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
1081	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1082	call		tlb_patch_one
1083	 mov		24, %o2
1084
1085	sethi		%hi(xcall_flush_tlb_page), %o0
1086	or		%o0, %lo(xcall_flush_tlb_page), %o0
1087	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
1088	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1089	call		tlb_patch_one
1090	 mov		20, %o2
1091
1092	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1093	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1094	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1095	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1096	call		tlb_patch_one
1097	 mov		44, %o2
1098#endif /* CONFIG_SMP */
1099
1100	ret
1101	 restore
1102