100a9730eSGuo Ren // SPDX-License-Identifier: GPL-2.0 200a9730eSGuo Ren // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. 300a9730eSGuo Ren 400a9730eSGuo Ren #include <linux/init.h> 500a9730eSGuo Ren #include <linux/mm.h> 600a9730eSGuo Ren #include <linux/module.h> 700a9730eSGuo Ren #include <linux/sched.h> 800a9730eSGuo Ren 900a9730eSGuo Ren #include <asm/mmu_context.h> 1000a9730eSGuo Ren #include <asm/pgtable.h> 1100a9730eSGuo Ren #include <asm/setup.h> 1200a9730eSGuo Ren 1300a9730eSGuo Ren #define CSKY_TLB_SIZE CONFIG_CPU_TLB_SIZE 1400a9730eSGuo Ren 1500a9730eSGuo Ren void flush_tlb_all(void) 1600a9730eSGuo Ren { 1700a9730eSGuo Ren tlb_invalid_all(); 1800a9730eSGuo Ren } 1900a9730eSGuo Ren 2000a9730eSGuo Ren void flush_tlb_mm(struct mm_struct *mm) 2100a9730eSGuo Ren { 2200a9730eSGuo Ren int cpu = smp_processor_id(); 2300a9730eSGuo Ren 2400a9730eSGuo Ren if (cpu_context(cpu, mm) != 0) 2500a9730eSGuo Ren drop_mmu_context(mm, cpu); 2600a9730eSGuo Ren 2700a9730eSGuo Ren tlb_invalid_all(); 2800a9730eSGuo Ren } 2900a9730eSGuo Ren 3000a9730eSGuo Ren #define restore_asid_inv_utlb(oldpid, newpid) \ 3100a9730eSGuo Ren do { \ 3200a9730eSGuo Ren if ((oldpid & ASID_MASK) == newpid) \ 3300a9730eSGuo Ren write_mmu_entryhi(oldpid + 1); \ 3400a9730eSGuo Ren write_mmu_entryhi(oldpid); \ 3500a9730eSGuo Ren } while (0) 3600a9730eSGuo Ren 3700a9730eSGuo Ren void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 3800a9730eSGuo Ren unsigned long end) 3900a9730eSGuo Ren { 4000a9730eSGuo Ren struct mm_struct *mm = vma->vm_mm; 4100a9730eSGuo Ren int cpu = smp_processor_id(); 4200a9730eSGuo Ren 4300a9730eSGuo Ren if (cpu_context(cpu, mm) != 0) { 4400a9730eSGuo Ren unsigned long size, flags; 4500a9730eSGuo Ren int newpid = cpu_asid(cpu, mm); 4600a9730eSGuo Ren 4700a9730eSGuo Ren local_irq_save(flags); 4800a9730eSGuo Ren size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 4900a9730eSGuo Ren size = (size + 1) >> 1; 5000a9730eSGuo Ren if (size <= CSKY_TLB_SIZE/2) { 5100a9730eSGuo Ren start &= (PAGE_MASK << 1); 5200a9730eSGuo Ren end += ((PAGE_SIZE << 1) - 1); 5300a9730eSGuo Ren end &= (PAGE_MASK << 1); 5400a9730eSGuo Ren #ifdef CONFIG_CPU_HAS_TLBI 5500a9730eSGuo Ren while (start < end) { 5600a9730eSGuo Ren asm volatile("tlbi.vaas %0" 5700a9730eSGuo Ren ::"r"(start | newpid)); 5800a9730eSGuo Ren start += (PAGE_SIZE << 1); 5900a9730eSGuo Ren } 6000a9730eSGuo Ren sync_is(); 6100a9730eSGuo Ren #else 6200a9730eSGuo Ren { 6300a9730eSGuo Ren int oldpid = read_mmu_entryhi(); 6400a9730eSGuo Ren 6500a9730eSGuo Ren while (start < end) { 6600a9730eSGuo Ren int idx; 6700a9730eSGuo Ren 6800a9730eSGuo Ren write_mmu_entryhi(start | newpid); 6900a9730eSGuo Ren start += (PAGE_SIZE << 1); 7000a9730eSGuo Ren tlb_probe(); 7100a9730eSGuo Ren idx = read_mmu_index(); 7200a9730eSGuo Ren if (idx >= 0) 7300a9730eSGuo Ren tlb_invalid_indexed(); 7400a9730eSGuo Ren } 7500a9730eSGuo Ren restore_asid_inv_utlb(oldpid, newpid); 7600a9730eSGuo Ren } 7700a9730eSGuo Ren #endif 7800a9730eSGuo Ren } else { 7900a9730eSGuo Ren drop_mmu_context(mm, cpu); 8000a9730eSGuo Ren } 8100a9730eSGuo Ren local_irq_restore(flags); 8200a9730eSGuo Ren } 8300a9730eSGuo Ren } 8400a9730eSGuo Ren 8500a9730eSGuo Ren void flush_tlb_kernel_range(unsigned long start, unsigned long end) 8600a9730eSGuo Ren { 8700a9730eSGuo Ren unsigned long size, flags; 8800a9730eSGuo Ren 8900a9730eSGuo Ren local_irq_save(flags); 9000a9730eSGuo Ren size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 9100a9730eSGuo Ren if (size <= CSKY_TLB_SIZE) { 9200a9730eSGuo Ren start &= (PAGE_MASK << 1); 9300a9730eSGuo Ren end += ((PAGE_SIZE << 1) - 1); 9400a9730eSGuo Ren end &= (PAGE_MASK << 1); 9500a9730eSGuo Ren #ifdef CONFIG_CPU_HAS_TLBI 9600a9730eSGuo Ren while (start < end) { 9700a9730eSGuo Ren asm volatile("tlbi.vaas %0"::"r"(start)); 9800a9730eSGuo Ren start += (PAGE_SIZE << 1); 9900a9730eSGuo Ren } 10000a9730eSGuo Ren sync_is(); 10100a9730eSGuo Ren #else 10200a9730eSGuo Ren { 10300a9730eSGuo Ren int oldpid = read_mmu_entryhi(); 10400a9730eSGuo Ren 10500a9730eSGuo Ren while (start < end) { 10600a9730eSGuo Ren int idx; 10700a9730eSGuo Ren 10800a9730eSGuo Ren write_mmu_entryhi(start); 10900a9730eSGuo Ren start += (PAGE_SIZE << 1); 11000a9730eSGuo Ren tlb_probe(); 11100a9730eSGuo Ren idx = read_mmu_index(); 11200a9730eSGuo Ren if (idx >= 0) 11300a9730eSGuo Ren tlb_invalid_indexed(); 11400a9730eSGuo Ren } 11500a9730eSGuo Ren restore_asid_inv_utlb(oldpid, 0); 11600a9730eSGuo Ren } 11700a9730eSGuo Ren #endif 11800a9730eSGuo Ren } else { 11900a9730eSGuo Ren flush_tlb_all(); 12000a9730eSGuo Ren } 12100a9730eSGuo Ren 12200a9730eSGuo Ren local_irq_restore(flags); 12300a9730eSGuo Ren } 12400a9730eSGuo Ren 12500a9730eSGuo Ren void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 12600a9730eSGuo Ren { 12700a9730eSGuo Ren int cpu = smp_processor_id(); 12800a9730eSGuo Ren int newpid = cpu_asid(cpu, vma->vm_mm); 12900a9730eSGuo Ren 13000a9730eSGuo Ren if (!vma || cpu_context(cpu, vma->vm_mm) != 0) { 13100a9730eSGuo Ren page &= (PAGE_MASK << 1); 13200a9730eSGuo Ren 13300a9730eSGuo Ren #ifdef CONFIG_CPU_HAS_TLBI 13400a9730eSGuo Ren asm volatile("tlbi.vaas %0"::"r"(page | newpid)); 13500a9730eSGuo Ren sync_is(); 13600a9730eSGuo Ren #else 13700a9730eSGuo Ren { 13800a9730eSGuo Ren int oldpid, idx; 13900a9730eSGuo Ren unsigned long flags; 14000a9730eSGuo Ren 14100a9730eSGuo Ren local_irq_save(flags); 14200a9730eSGuo Ren oldpid = read_mmu_entryhi(); 14300a9730eSGuo Ren write_mmu_entryhi(page | newpid); 14400a9730eSGuo Ren tlb_probe(); 14500a9730eSGuo Ren idx = read_mmu_index(); 14600a9730eSGuo Ren if (idx >= 0) 14700a9730eSGuo Ren tlb_invalid_indexed(); 14800a9730eSGuo Ren 14900a9730eSGuo Ren restore_asid_inv_utlb(oldpid, newpid); 15000a9730eSGuo Ren local_irq_restore(flags); 15100a9730eSGuo Ren } 15200a9730eSGuo Ren #endif 15300a9730eSGuo Ren } 15400a9730eSGuo Ren } 15500a9730eSGuo Ren 15600a9730eSGuo Ren /* 15700a9730eSGuo Ren * Remove one kernel space TLB entry. This entry is assumed to be marked 15800a9730eSGuo Ren * global so we don't do the ASID thing. 15900a9730eSGuo Ren */ 16000a9730eSGuo Ren void flush_tlb_one(unsigned long page) 16100a9730eSGuo Ren { 16200a9730eSGuo Ren int oldpid; 16300a9730eSGuo Ren 16400a9730eSGuo Ren oldpid = read_mmu_entryhi(); 16500a9730eSGuo Ren page &= (PAGE_MASK << 1); 16600a9730eSGuo Ren 16700a9730eSGuo Ren #ifdef CONFIG_CPU_HAS_TLBI 16800a9730eSGuo Ren page = page | (oldpid & 0xfff); 16900a9730eSGuo Ren asm volatile("tlbi.vaas %0"::"r"(page)); 17000a9730eSGuo Ren sync_is(); 17100a9730eSGuo Ren #else 17200a9730eSGuo Ren { 17300a9730eSGuo Ren int idx; 17400a9730eSGuo Ren unsigned long flags; 17500a9730eSGuo Ren 17600a9730eSGuo Ren page = page | (oldpid & 0xff); 17700a9730eSGuo Ren 17800a9730eSGuo Ren local_irq_save(flags); 17900a9730eSGuo Ren write_mmu_entryhi(page); 18000a9730eSGuo Ren tlb_probe(); 18100a9730eSGuo Ren idx = read_mmu_index(); 18200a9730eSGuo Ren if (idx >= 0) 18300a9730eSGuo Ren tlb_invalid_indexed(); 18400a9730eSGuo Ren restore_asid_inv_utlb(oldpid, oldpid); 18500a9730eSGuo Ren local_irq_restore(flags); 18600a9730eSGuo Ren } 18700a9730eSGuo Ren #endif 18800a9730eSGuo Ren } 18900a9730eSGuo Ren EXPORT_SYMBOL(flush_tlb_one); 19000a9730eSGuo Ren 19100a9730eSGuo Ren /* show current 32 jtlbs */ 19200a9730eSGuo Ren void show_jtlb_table(void) 19300a9730eSGuo Ren { 19400a9730eSGuo Ren unsigned long flags; 19500a9730eSGuo Ren int entryhi, entrylo0, entrylo1; 19600a9730eSGuo Ren int entry; 19700a9730eSGuo Ren int oldpid; 19800a9730eSGuo Ren 19900a9730eSGuo Ren local_irq_save(flags); 20000a9730eSGuo Ren entry = 0; 20100a9730eSGuo Ren pr_info("\n\n\n"); 20200a9730eSGuo Ren 20300a9730eSGuo Ren oldpid = read_mmu_entryhi(); 20400a9730eSGuo Ren while (entry < CSKY_TLB_SIZE) { 20500a9730eSGuo Ren write_mmu_index(entry); 20600a9730eSGuo Ren tlb_read(); 20700a9730eSGuo Ren entryhi = read_mmu_entryhi(); 20800a9730eSGuo Ren entrylo0 = read_mmu_entrylo0(); 20900a9730eSGuo Ren entrylo0 = entrylo0; 21000a9730eSGuo Ren entrylo1 = read_mmu_entrylo1(); 21100a9730eSGuo Ren entrylo1 = entrylo1; 21200a9730eSGuo Ren pr_info("jtlb[%d]: entryhi - 0x%x; entrylo0 - 0x%x;" 21300a9730eSGuo Ren " entrylo1 - 0x%x\n", 21400a9730eSGuo Ren entry, entryhi, entrylo0, entrylo1); 21500a9730eSGuo Ren entry++; 21600a9730eSGuo Ren } 21700a9730eSGuo Ren write_mmu_entryhi(oldpid); 21800a9730eSGuo Ren local_irq_restore(flags); 21900a9730eSGuo Ren } 220