loongarch/mm/tlbex.S

41 	la_abs		t0, do_page_fault
42 	jirl		ra, t0, 0
47 	csrwr		t0, EXCEPTION_KS0
54 	csrrd		t0, LOONGARCH_CSR_BADV
55 	bltz		t0, vmalloc_load
60 	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
64 	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
69 	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
83 	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
84 	alsl.d		t1, t0, ra, _PTE_T_LOG2
88 	ll.d		t0, t1, 0
90 	ld.d		t0, t1, 0
92 	andi		ra, t0, _PAGE_PRESENT
95 	ori		t0, t0, _PAGE_VALID
97 	sc.d		t0, t1, 0
98 	beqz		t0, smp_pgtable_change_load
100 	st.d		t0, t1, 0
104 	ld.d		t0, t1, 0
106 	csrwr		t0, LOONGARCH_CSR_TLBELO0
110 	csrrd		t0, EXCEPTION_KS0
126 	andi		t0, ra, _PAGE_PRESENT
127 	beqz		t0, nopage_tlb_load
130 	ori		t0, ra, _PAGE_VALID
131 	sc.d		t0, t1, 0
132 	beqz		t0, tlb_huge_update_load
133 	ori		t0, ra, _PAGE_VALID
136 	ori		t0, ra, _PAGE_VALID
137 	st.d		t0, t1, 0
154 	xori		t0, t0, _PAGE_HUGE
156 	and		t1, t0, t1
158 	or		t0, t0, t1
160 	move		ra, t0
166 	add.d		t0, t0, t1
167 	csrwr		t0, LOONGARCH_CSR_TLBELO1
170 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
172 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
176 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
178 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
180 	csrrd		t0, EXCEPTION_KS0
188 	la_abs		t0, tlb_do_page_fault_0
189 	jr		t0
193 	csrwr		t0, LOONGARCH_CSR_KS0
195 	la_abs		t0, tlb_do_page_fault_0
196 	jr		t0
200 	csrwr		t0, EXCEPTION_KS0
207 	csrrd		t0, LOONGARCH_CSR_BADV
208 	bltz		t0, vmalloc_store
213 	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
217 	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
222 	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
236 	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
237 	alsl.d		t1, t0, ra, _PTE_T_LOG2
241 	ll.d		t0, t1, 0
243 	ld.d		t0, t1, 0
245 	andi		ra, t0, _PAGE_PRESENT | _PAGE_WRITE
249 	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
251 	sc.d		t0, t1, 0
252 	beqz		t0, smp_pgtable_change_store
254 	st.d		t0, t1, 0
258 	ld.d		t0, t1, 0
260 	csrwr		t0, LOONGARCH_CSR_TLBELO0
264 	csrrd		t0, EXCEPTION_KS0
280 	andi		t0, ra, _PAGE_PRESENT | _PAGE_WRITE
281 	xori		t0, t0, _PAGE_PRESENT | _PAGE_WRITE
282 	bnez		t0, nopage_tlb_store
285 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
286 	sc.d		t0, t1, 0
287 	beqz		t0, tlb_huge_update_store
288 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
291 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
292 	st.d		t0, t1, 0
309 	xori		t0, t0, _PAGE_HUGE
311 	and		t1, t0, t1
313 	or		t0, t0, t1
315 	move		ra, t0
321 	add.d		t0, t0, t1
322 	csrwr		t0, LOONGARCH_CSR_TLBELO1
325 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
327 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
332 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
334 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
336 	csrrd		t0, EXCEPTION_KS0
344 	la_abs		t0, tlb_do_page_fault_1
345 	jr		t0
349 	csrwr		t0, LOONGARCH_CSR_KS0
351 	la_abs		t0, tlb_do_page_fault_1
352 	jr		t0
356 	csrwr		t0, EXCEPTION_KS0
363 	csrrd		t0, LOONGARCH_CSR_BADV
364 	bltz		t0, vmalloc_modify
369 	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
373 	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
378 	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
392 	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
393 	alsl.d		t1, t0, ra, _PTE_T_LOG2
397 	ll.d		t0, t1, 0
399 	ld.d		t0, t1, 0
401 	andi		ra, t0, _PAGE_WRITE
404 	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
406 	sc.d		t0, t1, 0
407 	beqz		t0, smp_pgtable_change_modify
409 	st.d		t0, t1, 0
413 	ld.d		t0, t1, 0
415 	csrwr		t0, LOONGARCH_CSR_TLBELO0
419 	csrrd		t0, EXCEPTION_KS0
435 	andi		t0, ra, _PAGE_WRITE
436 	beqz		t0, nopage_tlb_modify
439 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
440 	sc.d		t0, t1, 0
441 	beqz		t0, tlb_huge_update_modify
442 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
445 	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
446 	st.d		t0, t1, 0
463 	xori		t0, t0, _PAGE_HUGE
465 	and		t1, t0, t1
467 	or		t0, t0, t1
469 	move		ra, t0
475 	add.d		t0, t0, t1
476 	csrwr		t0, LOONGARCH_CSR_TLBELO1
479 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
481 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
486 	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
488 	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
490 	csrrd		t0, EXCEPTION_KS0
498 	la_abs		t0, tlb_do_page_fault_1
499 	jr		t0
503 	csrwr		t0, LOONGARCH_CSR_KS0
505 	la_abs		t0, tlb_do_page_fault_1
506 	jr		t0
510 	csrwr		t0, LOONGARCH_CSR_TLBRSAVE
511 	csrrd		t0, LOONGARCH_CSR_PGD
512 	lddir		t0, t0, 3
514 	lddir		t0, t0, 2
517 	lddir		t0, t0, 1
519 	ldpte		t0, 0
520 	ldpte		t0, 1
522 	csrrd		t0, LOONGARCH_CSR_TLBRSAVE