158d0ba57SCatalin Marinas /* 258d0ba57SCatalin Marinas * Based on arch/arm/include/asm/tlb.h 358d0ba57SCatalin Marinas * 458d0ba57SCatalin Marinas * Copyright (C) 2002 Russell King 558d0ba57SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 658d0ba57SCatalin Marinas * 758d0ba57SCatalin Marinas * This program is free software; you can redistribute it and/or modify 858d0ba57SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 958d0ba57SCatalin Marinas * published by the Free Software Foundation. 1058d0ba57SCatalin Marinas * 1158d0ba57SCatalin Marinas * This program is distributed in the hope that it will be useful, 1258d0ba57SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1358d0ba57SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1458d0ba57SCatalin Marinas * GNU General Public License for more details. 1558d0ba57SCatalin Marinas * 1658d0ba57SCatalin Marinas * You should have received a copy of the GNU General Public License 1758d0ba57SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1858d0ba57SCatalin Marinas */ 1958d0ba57SCatalin Marinas #ifndef __ASM_TLB_H 2058d0ba57SCatalin Marinas #define __ASM_TLB_H 2158d0ba57SCatalin Marinas 2258d0ba57SCatalin Marinas #include <linux/pagemap.h> 2358d0ba57SCatalin Marinas #include <linux/swap.h> 2458d0ba57SCatalin Marinas 2558d0ba57SCatalin Marinas #include <asm/pgalloc.h> 2658d0ba57SCatalin Marinas #include <asm/tlbflush.h> 2758d0ba57SCatalin Marinas 2858d0ba57SCatalin Marinas #define MMU_GATHER_BUNDLE 8 2958d0ba57SCatalin Marinas 3058d0ba57SCatalin Marinas /* 3158d0ba57SCatalin Marinas * TLB handling. This allows us to remove pages from the page 3258d0ba57SCatalin Marinas * tables, and efficiently handle the TLB issues. 3358d0ba57SCatalin Marinas */ 3458d0ba57SCatalin Marinas struct mmu_gather { 3558d0ba57SCatalin Marinas struct mm_struct *mm; 3658d0ba57SCatalin Marinas unsigned int fullmm; 3758d0ba57SCatalin Marinas struct vm_area_struct *vma; 3858d0ba57SCatalin Marinas unsigned long range_start; 3958d0ba57SCatalin Marinas unsigned long range_end; 4058d0ba57SCatalin Marinas unsigned int nr; 4158d0ba57SCatalin Marinas unsigned int max; 4258d0ba57SCatalin Marinas struct page **pages; 4358d0ba57SCatalin Marinas struct page *local[MMU_GATHER_BUNDLE]; 4458d0ba57SCatalin Marinas }; 4558d0ba57SCatalin Marinas 4658d0ba57SCatalin Marinas /* 4758d0ba57SCatalin Marinas * This is unnecessarily complex. There's three ways the TLB shootdown 4858d0ba57SCatalin Marinas * code is used: 4958d0ba57SCatalin Marinas * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). 5058d0ba57SCatalin Marinas * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. 5158d0ba57SCatalin Marinas * tlb->vma will be non-NULL. 5258d0ba57SCatalin Marinas * 2. Unmapping all vmas. See exit_mmap(). 5358d0ba57SCatalin Marinas * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. 5458d0ba57SCatalin Marinas * tlb->vma will be non-NULL. Additionally, page tables will be freed. 5558d0ba57SCatalin Marinas * 3. Unmapping argument pages. See shift_arg_pages(). 5658d0ba57SCatalin Marinas * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. 5758d0ba57SCatalin Marinas * tlb->vma will be NULL. 5858d0ba57SCatalin Marinas */ 5958d0ba57SCatalin Marinas static inline void tlb_flush(struct mmu_gather *tlb) 6058d0ba57SCatalin Marinas { 6158d0ba57SCatalin Marinas if (tlb->fullmm || !tlb->vma) 6258d0ba57SCatalin Marinas flush_tlb_mm(tlb->mm); 6358d0ba57SCatalin Marinas else if (tlb->range_end > 0) { 6458d0ba57SCatalin Marinas flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); 6558d0ba57SCatalin Marinas tlb->range_start = TASK_SIZE; 6658d0ba57SCatalin Marinas tlb->range_end = 0; 6758d0ba57SCatalin Marinas } 6858d0ba57SCatalin Marinas } 6958d0ba57SCatalin Marinas 7058d0ba57SCatalin Marinas static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) 7158d0ba57SCatalin Marinas { 7258d0ba57SCatalin Marinas if (!tlb->fullmm) { 7358d0ba57SCatalin Marinas if (addr < tlb->range_start) 7458d0ba57SCatalin Marinas tlb->range_start = addr; 7558d0ba57SCatalin Marinas if (addr + PAGE_SIZE > tlb->range_end) 7658d0ba57SCatalin Marinas tlb->range_end = addr + PAGE_SIZE; 7758d0ba57SCatalin Marinas } 7858d0ba57SCatalin Marinas } 7958d0ba57SCatalin Marinas 8058d0ba57SCatalin Marinas static inline void __tlb_alloc_page(struct mmu_gather *tlb) 8158d0ba57SCatalin Marinas { 8258d0ba57SCatalin Marinas unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 8358d0ba57SCatalin Marinas 8458d0ba57SCatalin Marinas if (addr) { 8558d0ba57SCatalin Marinas tlb->pages = (void *)addr; 8658d0ba57SCatalin Marinas tlb->max = PAGE_SIZE / sizeof(struct page *); 8758d0ba57SCatalin Marinas } 8858d0ba57SCatalin Marinas } 8958d0ba57SCatalin Marinas 9058d0ba57SCatalin Marinas static inline void tlb_flush_mmu(struct mmu_gather *tlb) 9158d0ba57SCatalin Marinas { 9258d0ba57SCatalin Marinas tlb_flush(tlb); 9358d0ba57SCatalin Marinas free_pages_and_swap_cache(tlb->pages, tlb->nr); 9458d0ba57SCatalin Marinas tlb->nr = 0; 9558d0ba57SCatalin Marinas if (tlb->pages == tlb->local) 9658d0ba57SCatalin Marinas __tlb_alloc_page(tlb); 9758d0ba57SCatalin Marinas } 9858d0ba57SCatalin Marinas 9958d0ba57SCatalin Marinas static inline void 10058d0ba57SCatalin Marinas tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int fullmm) 10158d0ba57SCatalin Marinas { 10258d0ba57SCatalin Marinas tlb->mm = mm; 10358d0ba57SCatalin Marinas tlb->fullmm = fullmm; 10458d0ba57SCatalin Marinas tlb->vma = NULL; 10558d0ba57SCatalin Marinas tlb->max = ARRAY_SIZE(tlb->local); 10658d0ba57SCatalin Marinas tlb->pages = tlb->local; 10758d0ba57SCatalin Marinas tlb->nr = 0; 10858d0ba57SCatalin Marinas __tlb_alloc_page(tlb); 10958d0ba57SCatalin Marinas } 11058d0ba57SCatalin Marinas 11158d0ba57SCatalin Marinas static inline void 11258d0ba57SCatalin Marinas tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 11358d0ba57SCatalin Marinas { 11458d0ba57SCatalin Marinas tlb_flush_mmu(tlb); 11558d0ba57SCatalin Marinas 11658d0ba57SCatalin Marinas /* keep the page table cache within bounds */ 11758d0ba57SCatalin Marinas check_pgt_cache(); 11858d0ba57SCatalin Marinas 11958d0ba57SCatalin Marinas if (tlb->pages != tlb->local) 12058d0ba57SCatalin Marinas free_pages((unsigned long)tlb->pages, 0); 12158d0ba57SCatalin Marinas } 12258d0ba57SCatalin Marinas 12358d0ba57SCatalin Marinas /* 12458d0ba57SCatalin Marinas * Memorize the range for the TLB flush. 12558d0ba57SCatalin Marinas */ 12658d0ba57SCatalin Marinas static inline void 12758d0ba57SCatalin Marinas tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) 12858d0ba57SCatalin Marinas { 12958d0ba57SCatalin Marinas tlb_add_flush(tlb, addr); 13058d0ba57SCatalin Marinas } 13158d0ba57SCatalin Marinas 13258d0ba57SCatalin Marinas /* 13358d0ba57SCatalin Marinas * In the case of tlb vma handling, we can optimise these away in the 13458d0ba57SCatalin Marinas * case where we're doing a full MM flush. When we're doing a munmap, 13558d0ba57SCatalin Marinas * the vmas are adjusted to only cover the region to be torn down. 13658d0ba57SCatalin Marinas */ 13758d0ba57SCatalin Marinas static inline void 13858d0ba57SCatalin Marinas tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) 13958d0ba57SCatalin Marinas { 14058d0ba57SCatalin Marinas if (!tlb->fullmm) { 14158d0ba57SCatalin Marinas tlb->vma = vma; 14258d0ba57SCatalin Marinas tlb->range_start = TASK_SIZE; 14358d0ba57SCatalin Marinas tlb->range_end = 0; 14458d0ba57SCatalin Marinas } 14558d0ba57SCatalin Marinas } 14658d0ba57SCatalin Marinas 14758d0ba57SCatalin Marinas static inline void 14858d0ba57SCatalin Marinas tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) 14958d0ba57SCatalin Marinas { 15058d0ba57SCatalin Marinas if (!tlb->fullmm) 15158d0ba57SCatalin Marinas tlb_flush(tlb); 15258d0ba57SCatalin Marinas } 15358d0ba57SCatalin Marinas 15458d0ba57SCatalin Marinas static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 15558d0ba57SCatalin Marinas { 15658d0ba57SCatalin Marinas tlb->pages[tlb->nr++] = page; 15758d0ba57SCatalin Marinas VM_BUG_ON(tlb->nr > tlb->max); 15858d0ba57SCatalin Marinas return tlb->max - tlb->nr; 15958d0ba57SCatalin Marinas } 16058d0ba57SCatalin Marinas 16158d0ba57SCatalin Marinas static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 16258d0ba57SCatalin Marinas { 16358d0ba57SCatalin Marinas if (!__tlb_remove_page(tlb, page)) 16458d0ba57SCatalin Marinas tlb_flush_mmu(tlb); 16558d0ba57SCatalin Marinas } 16658d0ba57SCatalin Marinas 16758d0ba57SCatalin Marinas static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 16858d0ba57SCatalin Marinas unsigned long addr) 16958d0ba57SCatalin Marinas { 17058d0ba57SCatalin Marinas pgtable_page_dtor(pte); 17158d0ba57SCatalin Marinas tlb_add_flush(tlb, addr); 17258d0ba57SCatalin Marinas tlb_remove_page(tlb, pte); 17358d0ba57SCatalin Marinas } 17458d0ba57SCatalin Marinas 17558d0ba57SCatalin Marinas #ifndef CONFIG_ARM64_64K_PAGES 17658d0ba57SCatalin Marinas static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 17758d0ba57SCatalin Marinas unsigned long addr) 17858d0ba57SCatalin Marinas { 17958d0ba57SCatalin Marinas tlb_add_flush(tlb, addr); 18058d0ba57SCatalin Marinas tlb_remove_page(tlb, virt_to_page(pmdp)); 18158d0ba57SCatalin Marinas } 18258d0ba57SCatalin Marinas #endif 18358d0ba57SCatalin Marinas 18458d0ba57SCatalin Marinas #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) 18558d0ba57SCatalin Marinas #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) 18658d0ba57SCatalin Marinas #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) 18758d0ba57SCatalin Marinas 18858d0ba57SCatalin Marinas #define tlb_migrate_finish(mm) do { } while (0) 18958d0ba57SCatalin Marinas 19058d0ba57SCatalin Marinas #endif 191