109cfefb7SHuacai Chen/* SPDX-License-Identifier: GPL-2.0 */ 209cfefb7SHuacai Chen/* 309cfefb7SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 409cfefb7SHuacai Chen */ 509cfefb7SHuacai Chen#include <asm/asm.h> 609cfefb7SHuacai Chen#include <asm/loongarch.h> 709cfefb7SHuacai Chen#include <asm/page.h> 809cfefb7SHuacai Chen#include <asm/pgtable.h> 909cfefb7SHuacai Chen#include <asm/regdef.h> 1009cfefb7SHuacai Chen#include <asm/stackframe.h> 1109cfefb7SHuacai Chen 12b681604eSHuacai Chen#define INVTLB_ADDR_GFALSE_AND_ASID 5 13b681604eSHuacai Chen 14a2a84e36SRui Wang#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) 15a2a84e36SRui Wang#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) 16a2a84e36SRui Wang#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) 17a2a84e36SRui Wang#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) 18a2a84e36SRui Wang 1909cfefb7SHuacai Chen .macro tlb_do_page_fault, write 20*00c2ca84STiezhu Yang SYM_CODE_START(tlb_do_page_fault_\write) 2109cfefb7SHuacai Chen SAVE_ALL 2209cfefb7SHuacai Chen csrrd a2, LOONGARCH_CSR_BADV 2309cfefb7SHuacai Chen move a0, sp 2409cfefb7SHuacai Chen REG_S a2, sp, PT_BVADDR 2509cfefb7SHuacai Chen li.w a1, \write 26f733f119SXi Ruoyao bl do_page_fault 2709cfefb7SHuacai Chen RESTORE_ALL_AND_RET 28*00c2ca84STiezhu Yang SYM_CODE_END(tlb_do_page_fault_\write) 2909cfefb7SHuacai Chen .endm 3009cfefb7SHuacai Chen 3109cfefb7SHuacai Chen tlb_do_page_fault 0 3209cfefb7SHuacai Chen tlb_do_page_fault 1 3309cfefb7SHuacai Chen 34*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_protect) 3509cfefb7SHuacai Chen BACKUP_T0T1 3609cfefb7SHuacai Chen SAVE_ALL 3709cfefb7SHuacai Chen move a0, sp 3809cfefb7SHuacai Chen move a1, zero 3909cfefb7SHuacai Chen csrrd a2, LOONGARCH_CSR_BADV 4009cfefb7SHuacai Chen REG_S a2, sp, PT_BVADDR 41396233c6SYouling Tang la_abs t0, do_page_fault 4209cfefb7SHuacai Chen jirl ra, t0, 0 4309cfefb7SHuacai Chen RESTORE_ALL_AND_RET 44*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_protect) 4509cfefb7SHuacai Chen 46*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_load) 4709cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 4809cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 4909cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 5009cfefb7SHuacai Chen 5109cfefb7SHuacai Chen /* 5209cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 5309cfefb7SHuacai Chen */ 5409cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 55d1bc75d7SWANG Xuerui bltz t0, vmalloc_load 5609cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 5709cfefb7SHuacai Chen 5809cfefb7SHuacai Chenvmalloc_done_load: 5909cfefb7SHuacai Chen /* Get PGD offset in bytes */ 60a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 61a2a84e36SRui Wang alsl.d t1, ra, t1, 3 6209cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 6309cfefb7SHuacai Chen ld.d t1, t1, 0 64a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 65a2a84e36SRui Wang alsl.d t1, ra, t1, 3 6609cfefb7SHuacai Chen#endif 6709cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 6809cfefb7SHuacai Chen ld.d t1, t1, 0 69a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 70a2a84e36SRui Wang alsl.d t1, ra, t1, 3 7109cfefb7SHuacai Chen#endif 7209cfefb7SHuacai Chen ld.d ra, t1, 0 7309cfefb7SHuacai Chen 7409cfefb7SHuacai Chen /* 7509cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 7609cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 7709cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 7809cfefb7SHuacai Chen */ 79a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 80a2a84e36SRui Wang bltz ra, tlb_huge_update_load 8109cfefb7SHuacai Chen 82a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 83a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 84a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 8509cfefb7SHuacai Chen 8646859ac8SHuacai Chen#ifdef CONFIG_SMP 8746859ac8SHuacai Chensmp_pgtable_change_load: 8846859ac8SHuacai Chen ll.d t0, t1, 0 8946859ac8SHuacai Chen#else 9009cfefb7SHuacai Chen ld.d t0, t1, 0 9146859ac8SHuacai Chen#endif 92a2a84e36SRui Wang andi ra, t0, _PAGE_PRESENT 93d47b2dc8SWANG Xuerui beqz ra, nopage_tlb_load 9409cfefb7SHuacai Chen 9509cfefb7SHuacai Chen ori t0, t0, _PAGE_VALID 9646859ac8SHuacai Chen#ifdef CONFIG_SMP 9746859ac8SHuacai Chen sc.d t0, t1, 0 98d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_load 9946859ac8SHuacai Chen#else 10009cfefb7SHuacai Chen st.d t0, t1, 0 10146859ac8SHuacai Chen#endif 102a2a84e36SRui Wang tlbsrch 103a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 10409cfefb7SHuacai Chen ld.d t0, t1, 0 10509cfefb7SHuacai Chen ld.d t1, t1, 8 10609cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 10709cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 10809cfefb7SHuacai Chen tlbwr 109a2a84e36SRui Wang 11009cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 11109cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 11209cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 11309cfefb7SHuacai Chen ertn 114a2a84e36SRui Wang 11509cfefb7SHuacai Chen#ifdef CONFIG_64BIT 11609cfefb7SHuacai Chenvmalloc_load: 117396233c6SYouling Tang la_abs t1, swapper_pg_dir 11809cfefb7SHuacai Chen b vmalloc_done_load 11909cfefb7SHuacai Chen#endif 12009cfefb7SHuacai Chen 121a2a84e36SRui Wang /* This is the entry point of a huge page. */ 12209cfefb7SHuacai Chentlb_huge_update_load: 12346859ac8SHuacai Chen#ifdef CONFIG_SMP 124a2a84e36SRui Wang ll.d ra, t1, 0 12546859ac8SHuacai Chen#endif 126a2a84e36SRui Wang andi t0, ra, _PAGE_PRESENT 127a2a84e36SRui Wang beqz t0, nopage_tlb_load 12809cfefb7SHuacai Chen 12946859ac8SHuacai Chen#ifdef CONFIG_SMP 130a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 13146859ac8SHuacai Chen sc.d t0, t1, 0 132d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_load 133a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 13446859ac8SHuacai Chen#else 135a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 136a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 13709cfefb7SHuacai Chen st.d t0, t1, 0 13846859ac8SHuacai Chen#endif 139b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 140b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 141b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 142b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 14309cfefb7SHuacai Chen 14409cfefb7SHuacai Chen /* 14509cfefb7SHuacai Chen * A huge PTE describes an area the size of the 14609cfefb7SHuacai Chen * configured huge page size. This is twice the 14709cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 14809cfefb7SHuacai Chen * A TLB entry half the size of the configured 14909cfefb7SHuacai Chen * huge page size is configured into entrylo0 15009cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 15109cfefb7SHuacai Chen * address space. 15209cfefb7SHuacai Chen */ 15309cfefb7SHuacai Chen /* Huge page: Move Global bit */ 15409cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 15509cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 15609cfefb7SHuacai Chen and t1, t0, t1 15709cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 15809cfefb7SHuacai Chen or t0, t0, t1 15909cfefb7SHuacai Chen 160a2a84e36SRui Wang move ra, t0 161a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 16209cfefb7SHuacai Chen 16309cfefb7SHuacai Chen /* Convert to entrylo1 */ 164d8e7f201SWANG Xuerui addi.d t1, zero, 1 16509cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 16609cfefb7SHuacai Chen add.d t0, t0, t1 16709cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 16809cfefb7SHuacai Chen 16909cfefb7SHuacai Chen /* Set huge page tlb entry size */ 170d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 171d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 17209cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 17309cfefb7SHuacai Chen 17409cfefb7SHuacai Chen tlbfill 17509cfefb7SHuacai Chen 176d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 177d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 17809cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 17909cfefb7SHuacai Chen 180a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 181a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 182a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 183a2a84e36SRui Wang ertn 184a2a84e36SRui Wang 18509cfefb7SHuacai Chennopage_tlb_load: 186e031a5f3SHuacai Chen dbar 0x700 18709cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 188396233c6SYouling Tang la_abs t0, tlb_do_page_fault_0 18907b48069SWANG Xuerui jr t0 190*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_load) 19109cfefb7SHuacai Chen 192*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_load_ptw) 19301158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 19401158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 19501158487SHuacai Chen la_abs t0, tlb_do_page_fault_0 19601158487SHuacai Chen jr t0 197*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_load_ptw) 19801158487SHuacai Chen 199*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_store) 20009cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 20109cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 20209cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 20309cfefb7SHuacai Chen 20409cfefb7SHuacai Chen /* 20509cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 20609cfefb7SHuacai Chen */ 20709cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 208d1bc75d7SWANG Xuerui bltz t0, vmalloc_store 20909cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 21009cfefb7SHuacai Chen 21109cfefb7SHuacai Chenvmalloc_done_store: 21209cfefb7SHuacai Chen /* Get PGD offset in bytes */ 213a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 214a2a84e36SRui Wang alsl.d t1, ra, t1, 3 21509cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 21609cfefb7SHuacai Chen ld.d t1, t1, 0 217a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 218a2a84e36SRui Wang alsl.d t1, ra, t1, 3 21909cfefb7SHuacai Chen#endif 22009cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 22109cfefb7SHuacai Chen ld.d t1, t1, 0 222a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 223a2a84e36SRui Wang alsl.d t1, ra, t1, 3 22409cfefb7SHuacai Chen#endif 22509cfefb7SHuacai Chen ld.d ra, t1, 0 22609cfefb7SHuacai Chen 22709cfefb7SHuacai Chen /* 22809cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 22909cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 23009cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 23109cfefb7SHuacai Chen */ 232a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 233a2a84e36SRui Wang bltz ra, tlb_huge_update_store 23409cfefb7SHuacai Chen 235a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 236a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 237a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 23809cfefb7SHuacai Chen 23946859ac8SHuacai Chen#ifdef CONFIG_SMP 24046859ac8SHuacai Chensmp_pgtable_change_store: 24146859ac8SHuacai Chen ll.d t0, t1, 0 24246859ac8SHuacai Chen#else 24309cfefb7SHuacai Chen ld.d t0, t1, 0 24446859ac8SHuacai Chen#endif 245a2a84e36SRui Wang andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 246a2a84e36SRui Wang xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 247d47b2dc8SWANG Xuerui bnez ra, nopage_tlb_store 24809cfefb7SHuacai Chen 24909cfefb7SHuacai Chen ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 25046859ac8SHuacai Chen#ifdef CONFIG_SMP 25146859ac8SHuacai Chen sc.d t0, t1, 0 252d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_store 25346859ac8SHuacai Chen#else 25409cfefb7SHuacai Chen st.d t0, t1, 0 25546859ac8SHuacai Chen#endif 256a2a84e36SRui Wang tlbsrch 257a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 25809cfefb7SHuacai Chen ld.d t0, t1, 0 25909cfefb7SHuacai Chen ld.d t1, t1, 8 26009cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 26109cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 26209cfefb7SHuacai Chen tlbwr 263a2a84e36SRui Wang 26409cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 26509cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 26609cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 26709cfefb7SHuacai Chen ertn 268a2a84e36SRui Wang 26909cfefb7SHuacai Chen#ifdef CONFIG_64BIT 27009cfefb7SHuacai Chenvmalloc_store: 271396233c6SYouling Tang la_abs t1, swapper_pg_dir 27209cfefb7SHuacai Chen b vmalloc_done_store 27309cfefb7SHuacai Chen#endif 27409cfefb7SHuacai Chen 275a2a84e36SRui Wang /* This is the entry point of a huge page. */ 27609cfefb7SHuacai Chentlb_huge_update_store: 27746859ac8SHuacai Chen#ifdef CONFIG_SMP 278a2a84e36SRui Wang ll.d ra, t1, 0 27946859ac8SHuacai Chen#endif 280a2a84e36SRui Wang andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 281a2a84e36SRui Wang xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 282a2a84e36SRui Wang bnez t0, nopage_tlb_store 28309cfefb7SHuacai Chen 28446859ac8SHuacai Chen#ifdef CONFIG_SMP 285a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 28646859ac8SHuacai Chen sc.d t0, t1, 0 287d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_store 288a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 28946859ac8SHuacai Chen#else 290a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 291a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 29209cfefb7SHuacai Chen st.d t0, t1, 0 29346859ac8SHuacai Chen#endif 294b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 295b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 296b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 297b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 29809cfefb7SHuacai Chen 29909cfefb7SHuacai Chen /* 30009cfefb7SHuacai Chen * A huge PTE describes an area the size of the 30109cfefb7SHuacai Chen * configured huge page size. This is twice the 30209cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 30309cfefb7SHuacai Chen * A TLB entry half the size of the configured 30409cfefb7SHuacai Chen * huge page size is configured into entrylo0 30509cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 30609cfefb7SHuacai Chen * address space. 30709cfefb7SHuacai Chen */ 30809cfefb7SHuacai Chen /* Huge page: Move Global bit */ 30909cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 31009cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 31109cfefb7SHuacai Chen and t1, t0, t1 31209cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 31309cfefb7SHuacai Chen or t0, t0, t1 31409cfefb7SHuacai Chen 315a2a84e36SRui Wang move ra, t0 316a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 31709cfefb7SHuacai Chen 31809cfefb7SHuacai Chen /* Convert to entrylo1 */ 319d8e7f201SWANG Xuerui addi.d t1, zero, 1 32009cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 32109cfefb7SHuacai Chen add.d t0, t0, t1 32209cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 32309cfefb7SHuacai Chen 32409cfefb7SHuacai Chen /* Set huge page tlb entry size */ 325d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 326d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 32709cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 32809cfefb7SHuacai Chen 32909cfefb7SHuacai Chen tlbfill 33009cfefb7SHuacai Chen 33109cfefb7SHuacai Chen /* Reset default page size */ 332d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 333d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 33409cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 33509cfefb7SHuacai Chen 336a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 337a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 338a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 339a2a84e36SRui Wang ertn 340a2a84e36SRui Wang 34109cfefb7SHuacai Chennopage_tlb_store: 342e031a5f3SHuacai Chen dbar 0x700 34309cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 344396233c6SYouling Tang la_abs t0, tlb_do_page_fault_1 34507b48069SWANG Xuerui jr t0 346*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_store) 34709cfefb7SHuacai Chen 348*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_store_ptw) 34901158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 35001158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 35101158487SHuacai Chen la_abs t0, tlb_do_page_fault_1 35201158487SHuacai Chen jr t0 353*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_store_ptw) 35401158487SHuacai Chen 355*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_modify) 35609cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 35709cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 35809cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 35909cfefb7SHuacai Chen 36009cfefb7SHuacai Chen /* 36109cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 36209cfefb7SHuacai Chen */ 36309cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 364d1bc75d7SWANG Xuerui bltz t0, vmalloc_modify 36509cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 36609cfefb7SHuacai Chen 36709cfefb7SHuacai Chenvmalloc_done_modify: 36809cfefb7SHuacai Chen /* Get PGD offset in bytes */ 369a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 370a2a84e36SRui Wang alsl.d t1, ra, t1, 3 37109cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 37209cfefb7SHuacai Chen ld.d t1, t1, 0 373a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 374a2a84e36SRui Wang alsl.d t1, ra, t1, 3 37509cfefb7SHuacai Chen#endif 37609cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 37709cfefb7SHuacai Chen ld.d t1, t1, 0 378a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 379a2a84e36SRui Wang alsl.d t1, ra, t1, 3 38009cfefb7SHuacai Chen#endif 38109cfefb7SHuacai Chen ld.d ra, t1, 0 38209cfefb7SHuacai Chen 38309cfefb7SHuacai Chen /* 38409cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 38509cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 38609cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 38709cfefb7SHuacai Chen */ 388a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 389a2a84e36SRui Wang bltz ra, tlb_huge_update_modify 39009cfefb7SHuacai Chen 391a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 392a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 393a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 39409cfefb7SHuacai Chen 39546859ac8SHuacai Chen#ifdef CONFIG_SMP 39646859ac8SHuacai Chensmp_pgtable_change_modify: 39746859ac8SHuacai Chen ll.d t0, t1, 0 39846859ac8SHuacai Chen#else 39909cfefb7SHuacai Chen ld.d t0, t1, 0 40046859ac8SHuacai Chen#endif 401a2a84e36SRui Wang andi ra, t0, _PAGE_WRITE 402d47b2dc8SWANG Xuerui beqz ra, nopage_tlb_modify 40309cfefb7SHuacai Chen 40409cfefb7SHuacai Chen ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 40546859ac8SHuacai Chen#ifdef CONFIG_SMP 40646859ac8SHuacai Chen sc.d t0, t1, 0 407d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_modify 40846859ac8SHuacai Chen#else 40909cfefb7SHuacai Chen st.d t0, t1, 0 41046859ac8SHuacai Chen#endif 411a2a84e36SRui Wang tlbsrch 412a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 41309cfefb7SHuacai Chen ld.d t0, t1, 0 41409cfefb7SHuacai Chen ld.d t1, t1, 8 41509cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 41609cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 41709cfefb7SHuacai Chen tlbwr 418a2a84e36SRui Wang 41909cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 42009cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 42109cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 42209cfefb7SHuacai Chen ertn 423a2a84e36SRui Wang 42409cfefb7SHuacai Chen#ifdef CONFIG_64BIT 42509cfefb7SHuacai Chenvmalloc_modify: 426396233c6SYouling Tang la_abs t1, swapper_pg_dir 42709cfefb7SHuacai Chen b vmalloc_done_modify 42809cfefb7SHuacai Chen#endif 42909cfefb7SHuacai Chen 430a2a84e36SRui Wang /* This is the entry point of a huge page. */ 43109cfefb7SHuacai Chentlb_huge_update_modify: 43246859ac8SHuacai Chen#ifdef CONFIG_SMP 433a2a84e36SRui Wang ll.d ra, t1, 0 43446859ac8SHuacai Chen#endif 435a2a84e36SRui Wang andi t0, ra, _PAGE_WRITE 436a2a84e36SRui Wang beqz t0, nopage_tlb_modify 43709cfefb7SHuacai Chen 43846859ac8SHuacai Chen#ifdef CONFIG_SMP 439a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44046859ac8SHuacai Chen sc.d t0, t1, 0 441d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_modify 442a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44346859ac8SHuacai Chen#else 444a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 445a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44609cfefb7SHuacai Chen st.d t0, t1, 0 44746859ac8SHuacai Chen#endif 448b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 449b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 450b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 451b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 452b681604eSHuacai Chen 45309cfefb7SHuacai Chen /* 45409cfefb7SHuacai Chen * A huge PTE describes an area the size of the 45509cfefb7SHuacai Chen * configured huge page size. This is twice the 45609cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 45709cfefb7SHuacai Chen * A TLB entry half the size of the configured 45809cfefb7SHuacai Chen * huge page size is configured into entrylo0 45909cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 46009cfefb7SHuacai Chen * address space. 46109cfefb7SHuacai Chen */ 46209cfefb7SHuacai Chen /* Huge page: Move Global bit */ 46309cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 46409cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 46509cfefb7SHuacai Chen and t1, t0, t1 46609cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 46709cfefb7SHuacai Chen or t0, t0, t1 46809cfefb7SHuacai Chen 469a2a84e36SRui Wang move ra, t0 470a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 47109cfefb7SHuacai Chen 47209cfefb7SHuacai Chen /* Convert to entrylo1 */ 473d8e7f201SWANG Xuerui addi.d t1, zero, 1 47409cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 47509cfefb7SHuacai Chen add.d t0, t0, t1 47609cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 47709cfefb7SHuacai Chen 47809cfefb7SHuacai Chen /* Set huge page tlb entry size */ 479d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 480d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 48109cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 48209cfefb7SHuacai Chen 483b681604eSHuacai Chen tlbfill 48409cfefb7SHuacai Chen 48509cfefb7SHuacai Chen /* Reset default page size */ 486d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 487d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 48809cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 48909cfefb7SHuacai Chen 490a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 491a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 492a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 493a2a84e36SRui Wang ertn 494a2a84e36SRui Wang 49509cfefb7SHuacai Chennopage_tlb_modify: 496e031a5f3SHuacai Chen dbar 0x700 49709cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 498396233c6SYouling Tang la_abs t0, tlb_do_page_fault_1 49907b48069SWANG Xuerui jr t0 500*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_modify) 50109cfefb7SHuacai Chen 502*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_modify_ptw) 50301158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 50401158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 50501158487SHuacai Chen la_abs t0, tlb_do_page_fault_1 50601158487SHuacai Chen jr t0 507*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_modify_ptw) 50801158487SHuacai Chen 509*00c2ca84STiezhu YangSYM_CODE_START(handle_tlb_refill) 51009cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBRSAVE 51109cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_PGD 51209cfefb7SHuacai Chen lddir t0, t0, 3 51309cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 51409cfefb7SHuacai Chen lddir t0, t0, 2 51509cfefb7SHuacai Chen#endif 51609cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 51709cfefb7SHuacai Chen lddir t0, t0, 1 51809cfefb7SHuacai Chen#endif 51909cfefb7SHuacai Chen ldpte t0, 0 52009cfefb7SHuacai Chen ldpte t0, 1 52109cfefb7SHuacai Chen tlbfill 52209cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_TLBRSAVE 52309cfefb7SHuacai Chen ertn 524*00c2ca84STiezhu YangSYM_CODE_END(handle_tlb_refill) 525