xref: /openbmc/linux/arch/sparc/kernel/ktlb.S (revision 63dc02bd) 
1/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2 *
3 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
7 */
8
9#include <asm/head.h>
10#include <asm/asi.h>
11#include <asm/page.h>
12#include <asm/pgtable.h>
13#include <asm/tsb.h>
14
15	.text
16	.align		32
17
18kvmap_itlb:
19	/* g6: TAG TARGET */
20	mov		TLB_TAG_ACCESS, %g4
21	ldxa		[%g4] ASI_IMMU, %g4
22
23	/* sun4v_itlb_miss branches here with the missing virtual
24	 * address already loaded into %g4
25	 */
26kvmap_itlb_4v:
27
28kvmap_itlb_nonlinear:
29	/* Catch kernel NULL pointer calls.  */
30	sethi		%hi(PAGE_SIZE), %g5
31	cmp		%g4, %g5
32	bleu,pn		%xcc, kvmap_dtlb_longpath
33	 nop
34
35	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
36
37kvmap_itlb_tsb_miss:
38	sethi		%hi(LOW_OBP_ADDRESS), %g5
39	cmp		%g4, %g5
40	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
41	 mov		0x1, %g5
42	sllx		%g5, 32, %g5
43	cmp		%g4, %g5
44	blu,pn		%xcc, kvmap_itlb_obp
45	 nop
46
47kvmap_itlb_vmalloc_addr:
48	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
49
50	TSB_LOCK_TAG(%g1, %g2, %g7)
51
52	/* Load and check PTE.  */
53	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
54	mov		1, %g7
55	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
56	brgez,a,pn	%g5, kvmap_itlb_longpath
57	 TSB_STORE(%g1, %g7)
58
59	TSB_WRITE(%g1, %g5, %g6)
60
61	/* fallthrough to TLB load */
62
63kvmap_itlb_load:
64
65661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
66	retry
67	.section	.sun4v_2insn_patch, "ax"
68	.word		661b
69	nop
70	nop
71	.previous
72
73	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
74	 * instruction get nop'd out and we get here to branch
75	 * to the sun4v tlb load code.  The registers are setup
76	 * as follows:
77	 *
78	 * %g4: vaddr
79	 * %g5: PTE
80	 * %g6:	TAG
81	 *
82	 * The sun4v TLB load wants the PTE in %g3 so we fix that
83	 * up here.
84	 */
85	ba,pt		%xcc, sun4v_itlb_load
86	 mov		%g5, %g3
87
88kvmap_itlb_longpath:
89
90661:	rdpr	%pstate, %g5
91	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
92	.section .sun4v_2insn_patch, "ax"
93	.word	661b
94	SET_GL(1)
95	nop
96	.previous
97
98	rdpr	%tpc, %g5
99	ba,pt	%xcc, sparc64_realfault_common
100	 mov	FAULT_CODE_ITLB, %g4
101
102kvmap_itlb_obp:
103	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
104
105	TSB_LOCK_TAG(%g1, %g2, %g7)
106
107	TSB_WRITE(%g1, %g5, %g6)
108
109	ba,pt		%xcc, kvmap_itlb_load
110	 nop
111
112kvmap_dtlb_obp:
113	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
114
115	TSB_LOCK_TAG(%g1, %g2, %g7)
116
117	TSB_WRITE(%g1, %g5, %g6)
118
119	ba,pt		%xcc, kvmap_dtlb_load
120	 nop
121
122	.align		32
123kvmap_dtlb_tsb4m_load:
124	TSB_LOCK_TAG(%g1, %g2, %g7)
125	TSB_WRITE(%g1, %g5, %g6)
126	ba,pt		%xcc, kvmap_dtlb_load
127	 nop
128
129kvmap_dtlb:
130	/* %g6: TAG TARGET */
131	mov		TLB_TAG_ACCESS, %g4
132	ldxa		[%g4] ASI_DMMU, %g4
133
134	/* sun4v_dtlb_miss branches here with the missing virtual
135	 * address already loaded into %g4
136	 */
137kvmap_dtlb_4v:
138	brgez,pn	%g4, kvmap_dtlb_nonlinear
139	 nop
140
141#ifdef CONFIG_DEBUG_PAGEALLOC
142	/* Index through the base page size TSB even for linear
143	 * mappings when using page allocation debugging.
144	 */
145	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
146#else
147	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
148	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
149#endif
150	/* TSB entry address left in %g1, lookup linear PTE.
151	 * Must preserve %g1 and %g6 (TAG).
152	 */
153kvmap_dtlb_tsb4m_miss:
154	/* Clear the PAGE_OFFSET top virtual bits, shift
155	 * down to get PFN, and make sure PFN is in range.
156	 */
157	sllx		%g4, 21, %g5
158
159	/* Check to see if we know about valid memory at the 4MB
160	 * chunk this physical address will reside within.
161	 */
162	srlx		%g5, 21 + 41, %g2
163	brnz,pn		%g2, kvmap_dtlb_longpath
164	 nop
165
166	/* This unconditional branch and delay-slot nop gets patched
167	 * by the sethi sequence once the bitmap is properly setup.
168	 */
169	.globl		valid_addr_bitmap_insn
170valid_addr_bitmap_insn:
171	ba,pt		%xcc, 2f
172	 nop
173	.subsection	2
174	.globl		valid_addr_bitmap_patch
175valid_addr_bitmap_patch:
176	sethi		%hi(sparc64_valid_addr_bitmap), %g7
177	or		%g7, %lo(sparc64_valid_addr_bitmap), %g7
178	.previous
179
180	srlx		%g5, 21 + 22, %g2
181	srlx		%g2, 6, %g5
182	and		%g2, 63, %g2
183	sllx		%g5, 3, %g5
184	ldx		[%g7 + %g5], %g5
185	mov		1, %g7
186	sllx		%g7, %g2, %g7
187	andcc		%g5, %g7, %g0
188	be,pn		%xcc, kvmap_dtlb_longpath
189
1902:	 sethi		%hi(kpte_linear_bitmap), %g2
191	or		%g2, %lo(kpte_linear_bitmap), %g2
192
193	/* Get the 256MB physical address index. */
194	sllx		%g4, 21, %g5
195	mov		1, %g7
196	srlx		%g5, 21 + 28, %g5
197
198	/* Don't try this at home kids... this depends upon srlx
199	 * only taking the low 6 bits of the shift count in %g5.
200	 */
201	sllx		%g7, %g5, %g7
202
203	/* Divide by 64 to get the offset into the bitmask.  */
204	srlx		%g5, 6, %g5
205	sllx		%g5, 3, %g5
206
207	/* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
208	ldx		[%g2 + %g5], %g2
209	andcc		%g2, %g7, %g0
210	sethi		%hi(kern_linear_pte_xor), %g5
211	or		%g5, %lo(kern_linear_pte_xor), %g5
212	bne,a,pt	%xcc, 1f
213	 add		%g5, 8, %g5
214
2151:	ldx		[%g5], %g2
216
217	.globl		kvmap_linear_patch
218kvmap_linear_patch:
219	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
220	 xor		%g2, %g4, %g5
221
222kvmap_dtlb_vmalloc_addr:
223	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
224
225	TSB_LOCK_TAG(%g1, %g2, %g7)
226
227	/* Load and check PTE.  */
228	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
229	mov		1, %g7
230	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
231	brgez,a,pn	%g5, kvmap_dtlb_longpath
232	 TSB_STORE(%g1, %g7)
233
234	TSB_WRITE(%g1, %g5, %g6)
235
236	/* fallthrough to TLB load */
237
238kvmap_dtlb_load:
239
240661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
241	retry
242	.section	.sun4v_2insn_patch, "ax"
243	.word		661b
244	nop
245	nop
246	.previous
247
248	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
249	 * instruction get nop'd out and we get here to branch
250	 * to the sun4v tlb load code.  The registers are setup
251	 * as follows:
252	 *
253	 * %g4: vaddr
254	 * %g5: PTE
255	 * %g6:	TAG
256	 *
257	 * The sun4v TLB load wants the PTE in %g3 so we fix that
258	 * up here.
259	 */
260	ba,pt		%xcc, sun4v_dtlb_load
261	 mov		%g5, %g3
262
263#ifdef CONFIG_SPARSEMEM_VMEMMAP
264kvmap_vmemmap:
265	sub		%g4, %g5, %g5
266	srlx		%g5, 22, %g5
267	sethi		%hi(vmemmap_table), %g1
268	sllx		%g5, 3, %g5
269	or		%g1, %lo(vmemmap_table), %g1
270	ba,pt		%xcc, kvmap_dtlb_load
271	 ldx		[%g1 + %g5], %g5
272#endif
273
274kvmap_dtlb_nonlinear:
275	/* Catch kernel NULL pointer derefs.  */
276	sethi		%hi(PAGE_SIZE), %g5
277	cmp		%g4, %g5
278	bleu,pn		%xcc, kvmap_dtlb_longpath
279	 nop
280
281#ifdef CONFIG_SPARSEMEM_VMEMMAP
282	/* Do not use the TSB for vmemmap.  */
283	mov		(VMEMMAP_BASE >> 40), %g5
284	sllx		%g5, 40, %g5
285	cmp		%g4,%g5
286	bgeu,pn		%xcc, kvmap_vmemmap
287	 nop
288#endif
289
290	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
291
292kvmap_dtlb_tsbmiss:
293	sethi		%hi(MODULES_VADDR), %g5
294	cmp		%g4, %g5
295	blu,pn		%xcc, kvmap_dtlb_longpath
296	 mov		(VMALLOC_END >> 40), %g5
297	sllx		%g5, 40, %g5
298	cmp		%g4, %g5
299	bgeu,pn		%xcc, kvmap_dtlb_longpath
300	 nop
301
302kvmap_check_obp:
303	sethi		%hi(LOW_OBP_ADDRESS), %g5
304	cmp		%g4, %g5
305	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
306	 mov		0x1, %g5
307	sllx		%g5, 32, %g5
308	cmp		%g4, %g5
309	blu,pn		%xcc, kvmap_dtlb_obp
310	 nop
311	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
312	 nop
313
314kvmap_dtlb_longpath:
315
316661:	rdpr	%pstate, %g5
317	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
318	.section .sun4v_2insn_patch, "ax"
319	.word	661b
320	SET_GL(1)
321	ldxa		[%g0] ASI_SCRATCHPAD, %g5
322	.previous
323
324	rdpr	%tl, %g3
325	cmp	%g3, 1
326
327661:	mov	TLB_TAG_ACCESS, %g4
328	ldxa	[%g4] ASI_DMMU, %g5
329	.section .sun4v_2insn_patch, "ax"
330	.word	661b
331	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
332	nop
333	.previous
334
335	be,pt	%xcc, sparc64_realfault_common
336	 mov	FAULT_CODE_DTLB, %g4
337	ba,pt	%xcc, winfix_trampoline
338	 nop
339