1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
26  * Utility Routine to erase a J-TLB entry
89 	 * with existing location. This will cause Write CMD to over-write  in tlb_entry_insert()
131  * Un-conditionally (without lookup) erase the entire MMU contents
139 	int num_tlb = mmu->sets * mmu->ways;  in local_flush_tlb_all()
175  * Flush the entire MM for userland. The fastest way is to move to Next ASID
185 	if (atomic_read(&mm->mm_users) == 0)  in local_flush_tlb_mm()
189 	 * - Move to a new ASID, but only if the mm is still wired in  in local_flush_tlb_mm()
190 	 *   (Android Binder ended up calling this for vma->mm != tsk->mm,  in local_flush_tlb_mm()
191 	 *    causing h/w - s/w ASID to get out of sync)  in local_flush_tlb_mm()
192 	 * - Also get_new_mmu_context() new implementation allocates a new  in local_flush_tlb_mm()
193 	 *   ASID only if it is not allocated already - so unallocate first  in local_flush_tlb_mm()
196 	if (current->mm == mm)  in local_flush_tlb_mm()
204  *  -Here the fastest way (if range is too large) is to move to next ASID
206  *  -In case of kernel Flush, entry has to be shot down explicitly
221 	if (unlikely((end - start) >= PAGE_SIZE * 32)) {  in local_flush_tlb_range()
222 		local_flush_tlb_mm(vma->vm_mm);  in local_flush_tlb_range()
235 	if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {  in local_flush_tlb_range()
237 			tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu));  in local_flush_tlb_range()
245 /* Flush the kernel TLB entries - vmalloc/modules (Global from MMU perspective)
257 	if (unlikely((end - start) >= PAGE_SIZE * 32)) {  in local_flush_tlb_kernel_range()
288 	if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {  in local_flush_tlb_page()
289 		tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm, cpu));  in local_flush_tlb_page()
307 	local_flush_tlb_page(ta->ta_vma, ta->ta_start);  in ipi_flush_tlb_page()
314 	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);  in ipi_flush_tlb_range()
322 	local_flush_pmd_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);  in ipi_flush_pmd_tlb_range()
330 	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);  in ipi_flush_tlb_kernel_range()
351 	on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, &ta, 1);  in flush_tlb_page()
363 	on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, &ta, 1);  in flush_tlb_range()
376 	on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_pmd_tlb_range, &ta, 1);  in flush_pmd_tlb_range()
402 	 * create_tlb() assumes that current->mm == vma->mm, since  in create_tlb()
403 	 * -it ASID for TLB entry is fetched from MMU ASID reg (valid for curr)  in create_tlb()
404 	 * -completes the lazy write to SASID reg (again valid for curr tsk)  in create_tlb()
407 	 * -Using vma->mm->context{ASID,SASID}, as opposed to MMU reg.  in create_tlb()
408 	 * -More importantly it makes this handler inconsistent with fast-path  in create_tlb()
411 	 * Lets see the use cases when current->mm != vma->mm and we land here  in create_tlb()
412 	 *  1. execve->copy_strings()->__get_user_pages->handle_mm_fault  in create_tlb()
413 	 *     Here VM wants to pre-install a TLB entry for user stack while  in create_tlb()
414 	 *     current->mm still points to pre-execve mm (hence the condition).  in create_tlb()
419 	 *  2. ptrace(POKETEXT) causes a CoW - debugger(current) inserting a  in create_tlb()
425 	if (current->active_mm != vma->vm_mm)  in create_tlb()
444 	 * however Linux only saves 1 set to save PTE real-estate  in create_tlb()
446 	 * -Kernel only entries have Kr Kw Kx 0 0 0  in create_tlb()
447 	 * -User entries have mirrored K and U bits  in create_tlb()
465  *  -pre-install the corresponding TLB entry into MMU
466  *  -Finalize the delayed D-cache flush of kernel mapping of page due to
469  * Note that flush (when done) involves both WBACK - so physical page is
470  * in sync as well as INV - so any non-congruent aliases don't remain
486 	 * Exec page : Independent of aliasing/page-color considerations,  in update_mmu_cache_range()
488 	 *	       K-mapping of a code page needs to be wback+inv so that  in update_mmu_cache_range()
490 	 * !EXEC page: If K-mapping is NOT congruent to U-mapping, flush it  in update_mmu_cache_range()
492 	 *  (Avoids the flush for Non-exec + congruent mapping case)  in update_mmu_cache_range()
494 	if ((vma->vm_flags & VM_EXEC) ||  in update_mmu_cache_range()
497 		int dirty = !test_and_set_bit(PG_dc_clean, &folio->flags);  in update_mmu_cache_range()
501 			paddr -= offset;  in update_mmu_cache_range()
502 			vaddr -= offset;  in update_mmu_cache_range()
503 			/* wback + inv dcache lines (K-mapping) */  in update_mmu_cache_range()
506 			/* invalidate any existing icache lines (U-mapping) */  in update_mmu_cache_range()
507 			if (vma->vm_flags & VM_EXEC)  in update_mmu_cache_range()
519  * Normal and Super pages can co-exist (ofcourse not overlap) in TLB with a
525  *  - MMU page size (typical 8K, RTL fixed)
526  *  - software page walker address split between PGD:PTE:PFN (typical
528  * So for above default, THP size supported is 8K * (2^8) = 2M
530  * Default Page Walker is 2 levels, PGD:PTE:PFN, which in THP regime
553 	if (likely(asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID)) {  in local_flush_pmd_tlb_range()
554 		unsigned int asid = hw_pid(vma->vm_mm, cpu);  in local_flush_pmd_tlb_range()
579 	mmu->ver = (bcr >> 24);  in arc_mmu_mumbojumbo()
581 	if (is_isa_arcompact() && mmu->ver == 3) {  in arc_mmu_mumbojumbo()
583 		mmu->pg_sz_k = 1 << (mmu3->pg_sz - 1);  in arc_mmu_mumbojumbo()
584 		mmu->sets = 1 << mmu3->sets;  in arc_mmu_mumbojumbo()
585 		mmu->ways = 1 << mmu3->ways;  in arc_mmu_mumbojumbo()
586 		u_dtlb = mmu3->u_dtlb;  in arc_mmu_mumbojumbo()
587 		u_itlb = mmu3->u_itlb;  in arc_mmu_mumbojumbo()
588 		sasid = mmu3->sasid;  in arc_mmu_mumbojumbo()
591 		mmu->pg_sz_k = 1 << (mmu4->sz0 - 1);  in arc_mmu_mumbojumbo()
592 		mmu->s_pg_sz_m = 1 << (mmu4->sz1 - 11);  in arc_mmu_mumbojumbo()
593 		mmu->sets = 64 << mmu4->n_entry;  in arc_mmu_mumbojumbo()
594 		mmu->ways = mmu4->n_ways * 2;  in arc_mmu_mumbojumbo()
595 		u_dtlb = mmu4->u_dtlb * 4;  in arc_mmu_mumbojumbo()
596 		u_itlb = mmu4->u_itlb * 4;  in arc_mmu_mumbojumbo()
597 		sasid = mmu4->sasid;  in arc_mmu_mumbojumbo()
598 		mmu->pae = mmu4->pae;  in arc_mmu_mumbojumbo()
601 	if (mmu->s_pg_sz_m)  in arc_mmu_mumbojumbo()
603 			  mmu->s_pg_sz_m,  in arc_mmu_mumbojumbo()
606 	n += scnprintf(buf + n, len - n,  in arc_mmu_mumbojumbo()
608 		       mmu->ver, mmu->pg_sz_k, super_pg, CONFIG_PGTABLE_LEVELS,  in arc_mmu_mumbojumbo()
609 		       mmu->sets, mmu->ways,  in arc_mmu_mumbojumbo()
612 		       IS_AVAIL2(mmu->pae, ", PAE40 ", CONFIG_ARC_HAS_PAE40));  in arc_mmu_mumbojumbo()
640 	 *  - For older ARC700 cpus, only v3 supported  in arc_mmu_init()
641 	 *  - For HS cpus, v4 was baseline and v5 is backwards compatible  in arc_mmu_init()
644 	if (is_isa_arcompact() && mmu->ver == 3)  in arc_mmu_init()
646 	else if (is_isa_arcv2() && mmu->ver >= 4)  in arc_mmu_init()
650 		panic("MMU ver %d doesn't match kernel built for\n", mmu->ver);  in arc_mmu_init()
652 	if (mmu->pg_sz_k != TO_KB(PAGE_SIZE))  in arc_mmu_init()
656 	    mmu->s_pg_sz_m != TO_MB(HPAGE_PMD_SIZE))  in arc_mmu_init()
660 	if (IS_ENABLED(CONFIG_ARC_HAS_PAE40) && !mmu->pae)  in arc_mmu_init()
674  * TLB Programmer's Model uses Linear Indexes: 0 to {255, 511} for 128 x {2,4}
675  * The mapping is Column-first.
676  *		---------------------	-----------
678  *		---------------------	-----------
679  * [set0]	|  0 |  1 |  2 |  3 |	|  0 |  1 |
680  * [set1]	|  4 |  5 |  6 |  7 |	|  2 |  3 |
683  *		---------------------	-----------
688 #define SET_WAY_TO_IDX(mmu, set, way)  ((set) * mmu->ways + (way))  argument
691  * -Could be due to buggy customer tapeouts or obscure kernel bugs
692  * -MMU complaints not at the time of duplicate PD installation, but at the
694  * -Ideally these should never happen - but if they do - workaround by deleting
696  * -Knob to be verbose abt it.(TODO: hook them up to debugfs)
705 	int set, n_ways = mmu->ways;  in do_tlb_overlap_fault()
708 	BUG_ON(mmu->ways > 4);  in do_tlb_overlap_fault()
713 	for (set = 0; set < mmu->sets; set++) {  in do_tlb_overlap_fault()
715 		int is_valid, way;  in do_tlb_overlap_fault()  local
719 		for (way = 0, is_valid = 0; way < n_ways; way++) {  in do_tlb_overlap_fault()
721 					  SET_WAY_TO_IDX(mmu, set, way));  in do_tlb_overlap_fault()
723 			pd0[way] = read_aux_reg(ARC_REG_TLBPD0);  in do_tlb_overlap_fault()
724 			is_valid |= pd0[way] & _PAGE_PRESENT;  in do_tlb_overlap_fault()
725 			pd0[way] &= PAGE_MASK;  in do_tlb_overlap_fault()
733 		for (way = 0; way < n_ways - 1; way++) {  in do_tlb_overlap_fault()
737 			if (!pd0[way])  in do_tlb_overlap_fault()
740 			for (n = way + 1; n < n_ways; n++) {  in do_tlb_overlap_fault()
741 				if (pd0[way] != pd0[n])  in do_tlb_overlap_fault()
746 						pd0[way], set, way, n);  in do_tlb_overlap_fault()
749 				 * clear entry @way and not @n.  in do_tlb_overlap_fault()
752 				pd0[way] = 0;  in do_tlb_overlap_fault()
754 						SET_WAY_TO_IDX(mmu, set, way));  in do_tlb_overlap_fault()