1f1f3347dSVineet Gupta /* 2f1f3347dSVineet Gupta * TLB Management (flush/create/diagnostics) for ARC700 3f1f3347dSVineet Gupta * 4f1f3347dSVineet Gupta * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 5f1f3347dSVineet Gupta * 6f1f3347dSVineet Gupta * This program is free software; you can redistribute it and/or modify 7f1f3347dSVineet Gupta * it under the terms of the GNU General Public License version 2 as 8f1f3347dSVineet Gupta * published by the Free Software Foundation. 9f1f3347dSVineet Gupta */ 10f1f3347dSVineet Gupta 11f1f3347dSVineet Gupta #include <linux/module.h> 12f1f3347dSVineet Gupta #include <asm/arcregs.h> 13f1f3347dSVineet Gupta #include <asm/mmu_context.h> 14f1f3347dSVineet Gupta #include <asm/tlb.h> 15f1f3347dSVineet Gupta 16f1f3347dSVineet Gupta /* A copy of the ASID from the PID reg is kept in asid_cache */ 17f1f3347dSVineet Gupta int asid_cache = FIRST_ASID; 18f1f3347dSVineet Gupta 19f1f3347dSVineet Gupta /* ASID to mm struct mapping. We have one extra entry corresponding to 20f1f3347dSVineet Gupta * NO_ASID to save us a compare when clearing the mm entry for old asid 21f1f3347dSVineet Gupta * see get_new_mmu_context (asm-arc/mmu_context.h) 22f1f3347dSVineet Gupta */ 23f1f3347dSVineet Gupta struct mm_struct *asid_mm_map[NUM_ASID + 1]; 24cc562d2eSVineet Gupta 25cc562d2eSVineet Gupta 26cc562d2eSVineet Gupta /* 27cc562d2eSVineet Gupta * Routine to create a TLB entry 28cc562d2eSVineet Gupta */ 29cc562d2eSVineet Gupta void create_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) 30cc562d2eSVineet Gupta { 31cc562d2eSVineet Gupta unsigned long flags; 32cc562d2eSVineet Gupta unsigned int idx, asid_or_sasid; 33cc562d2eSVineet Gupta unsigned long pd0_flags; 34cc562d2eSVineet Gupta 35cc562d2eSVineet Gupta /* 36cc562d2eSVineet Gupta * create_tlb() assumes that current->mm == vma->mm, since 37cc562d2eSVineet Gupta * -it ASID for TLB entry is fetched from MMU ASID reg (valid for curr) 38cc562d2eSVineet Gupta * -completes the lazy write to SASID reg (again valid for curr tsk) 39cc562d2eSVineet Gupta * 40cc562d2eSVineet Gupta * Removing the assumption involves 41cc562d2eSVineet Gupta * -Using vma->mm->context{ASID,SASID}, as opposed to MMU reg. 42cc562d2eSVineet Gupta * -Fix the TLB paranoid debug code to not trigger false negatives. 43cc562d2eSVineet Gupta * -More importantly it makes this handler inconsistent with fast-path 44cc562d2eSVineet Gupta * TLB Refill handler which always deals with "current" 45cc562d2eSVineet Gupta * 46cc562d2eSVineet Gupta * Lets see the use cases when current->mm != vma->mm and we land here 47cc562d2eSVineet Gupta * 1. execve->copy_strings()->__get_user_pages->handle_mm_fault 48cc562d2eSVineet Gupta * Here VM wants to pre-install a TLB entry for user stack while 49cc562d2eSVineet Gupta * current->mm still points to pre-execve mm (hence the condition). 50cc562d2eSVineet Gupta * However the stack vaddr is soon relocated (randomization) and 51cc562d2eSVineet Gupta * move_page_tables() tries to undo that TLB entry. 52cc562d2eSVineet Gupta * Thus not creating TLB entry is not any worse. 53cc562d2eSVineet Gupta * 54cc562d2eSVineet Gupta * 2. ptrace(POKETEXT) causes a CoW - debugger(current) inserting a 55cc562d2eSVineet Gupta * breakpoint in debugged task. Not creating a TLB now is not 56cc562d2eSVineet Gupta * performance critical. 57cc562d2eSVineet Gupta * 58cc562d2eSVineet Gupta * Both the cases above are not good enough for code churn. 59cc562d2eSVineet Gupta */ 60cc562d2eSVineet Gupta if (current->active_mm != vma->vm_mm) 61cc562d2eSVineet Gupta return; 62cc562d2eSVineet Gupta 63cc562d2eSVineet Gupta local_irq_save(flags); 64cc562d2eSVineet Gupta 65cc562d2eSVineet Gupta tlb_paranoid_check(vma->vm_mm->context.asid, address); 66cc562d2eSVineet Gupta 67cc562d2eSVineet Gupta address &= PAGE_MASK; 68cc562d2eSVineet Gupta 69cc562d2eSVineet Gupta /* update this PTE credentials */ 70cc562d2eSVineet Gupta pte_val(*ptep) |= (_PAGE_PRESENT | _PAGE_ACCESSED); 71cc562d2eSVineet Gupta 72cc562d2eSVineet Gupta /* Create HW TLB entry Flags (in PD0) from PTE Flags */ 73cc562d2eSVineet Gupta #if (CONFIG_ARC_MMU_VER <= 2) 74cc562d2eSVineet Gupta pd0_flags = ((pte_val(*ptep) & PTE_BITS_IN_PD0) >> 1); 75cc562d2eSVineet Gupta #else 76cc562d2eSVineet Gupta pd0_flags = ((pte_val(*ptep) & PTE_BITS_IN_PD0)); 77cc562d2eSVineet Gupta #endif 78cc562d2eSVineet Gupta 79cc562d2eSVineet Gupta /* ASID for this task */ 80cc562d2eSVineet Gupta asid_or_sasid = read_aux_reg(ARC_REG_PID) & 0xff; 81cc562d2eSVineet Gupta 82cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBPD0, address | pd0_flags | asid_or_sasid); 83cc562d2eSVineet Gupta 84cc562d2eSVineet Gupta /* Load remaining info in PD1 (Page Frame Addr and Kx/Kw/Kr Flags) */ 85cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBPD1, (pte_val(*ptep) & PTE_BITS_IN_PD1)); 86cc562d2eSVineet Gupta 87cc562d2eSVineet Gupta /* First verify if entry for this vaddr+ASID already exists */ 88cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBCOMMAND, TLBProbe); 89cc562d2eSVineet Gupta idx = read_aux_reg(ARC_REG_TLBINDEX); 90cc562d2eSVineet Gupta 91cc562d2eSVineet Gupta /* 92cc562d2eSVineet Gupta * If Not already present get a free slot from MMU. 93cc562d2eSVineet Gupta * Otherwise, Probe would have located the entry and set INDEX Reg 94cc562d2eSVineet Gupta * with existing location. This will cause Write CMD to over-write 95cc562d2eSVineet Gupta * existing entry with new PD0 and PD1 96cc562d2eSVineet Gupta */ 97cc562d2eSVineet Gupta if (likely(idx & TLB_LKUP_ERR)) 98cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBCOMMAND, TLBGetIndex); 99cc562d2eSVineet Gupta 100cc562d2eSVineet Gupta /* 101cc562d2eSVineet Gupta * Commit the Entry to MMU 102cc562d2eSVineet Gupta * It doesnt sound safe to use the TLBWriteNI cmd here 103cc562d2eSVineet Gupta * which doesn't flush uTLBs. I'd rather be safe than sorry. 104cc562d2eSVineet Gupta */ 105cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite); 106cc562d2eSVineet Gupta 107cc562d2eSVineet Gupta local_irq_restore(flags); 108cc562d2eSVineet Gupta } 109cc562d2eSVineet Gupta 110cc562d2eSVineet Gupta /* arch hook called by core VM at the end of handle_mm_fault( ), 111cc562d2eSVineet Gupta * when a new PTE is entered in Page Tables or an existing one 112cc562d2eSVineet Gupta * is modified. We aggresively pre-install a TLB entry 113cc562d2eSVineet Gupta */ 114cc562d2eSVineet Gupta 115cc562d2eSVineet Gupta void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddress, 116cc562d2eSVineet Gupta pte_t *ptep) 117cc562d2eSVineet Gupta { 118cc562d2eSVineet Gupta 119cc562d2eSVineet Gupta create_tlb(vma, vaddress, ptep); 120cc562d2eSVineet Gupta } 121cc562d2eSVineet Gupta 122cc562d2eSVineet Gupta /* Read the Cache Build Confuration Registers, Decode them and save into 123cc562d2eSVineet Gupta * the cpuinfo structure for later use. 124cc562d2eSVineet Gupta * No Validation is done here, simply read/convert the BCRs 125cc562d2eSVineet Gupta */ 126cc562d2eSVineet Gupta void __init read_decode_mmu_bcr(void) 127cc562d2eSVineet Gupta { 128cc562d2eSVineet Gupta unsigned int tmp; 129cc562d2eSVineet Gupta struct bcr_mmu_1_2 *mmu2; /* encoded MMU2 attr */ 130cc562d2eSVineet Gupta struct bcr_mmu_3 *mmu3; /* encoded MMU3 attr */ 131cc562d2eSVineet Gupta struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu; 132cc562d2eSVineet Gupta 133cc562d2eSVineet Gupta tmp = read_aux_reg(ARC_REG_MMU_BCR); 134cc562d2eSVineet Gupta mmu->ver = (tmp >> 24); 135cc562d2eSVineet Gupta 136cc562d2eSVineet Gupta if (mmu->ver <= 2) { 137cc562d2eSVineet Gupta mmu2 = (struct bcr_mmu_1_2 *)&tmp; 138cc562d2eSVineet Gupta mmu->pg_sz = PAGE_SIZE; 139cc562d2eSVineet Gupta mmu->sets = 1 << mmu2->sets; 140cc562d2eSVineet Gupta mmu->ways = 1 << mmu2->ways; 141cc562d2eSVineet Gupta mmu->u_dtlb = mmu2->u_dtlb; 142cc562d2eSVineet Gupta mmu->u_itlb = mmu2->u_itlb; 143cc562d2eSVineet Gupta } else { 144cc562d2eSVineet Gupta mmu3 = (struct bcr_mmu_3 *)&tmp; 145cc562d2eSVineet Gupta mmu->pg_sz = 512 << mmu3->pg_sz; 146cc562d2eSVineet Gupta mmu->sets = 1 << mmu3->sets; 147cc562d2eSVineet Gupta mmu->ways = 1 << mmu3->ways; 148cc562d2eSVineet Gupta mmu->u_dtlb = mmu3->u_dtlb; 149cc562d2eSVineet Gupta mmu->u_itlb = mmu3->u_itlb; 150cc562d2eSVineet Gupta } 151cc562d2eSVineet Gupta 152cc562d2eSVineet Gupta mmu->num_tlb = mmu->sets * mmu->ways; 153cc562d2eSVineet Gupta } 154cc562d2eSVineet Gupta 155cc562d2eSVineet Gupta void __init arc_mmu_init(void) 156cc562d2eSVineet Gupta { 157cc562d2eSVineet Gupta /* 158cc562d2eSVineet Gupta * ASID mgmt data structures are compile time init 159cc562d2eSVineet Gupta * asid_cache = FIRST_ASID and asid_mm_map[] all zeroes 160cc562d2eSVineet Gupta */ 161cc562d2eSVineet Gupta 162cc562d2eSVineet Gupta local_flush_tlb_all(); 163cc562d2eSVineet Gupta 164cc562d2eSVineet Gupta /* Enable the MMU */ 165cc562d2eSVineet Gupta write_aux_reg(ARC_REG_PID, MMU_ENABLE); 166cc562d2eSVineet Gupta } 167cc562d2eSVineet Gupta 168cc562d2eSVineet Gupta /* 169cc562d2eSVineet Gupta * TLB Programmer's Model uses Linear Indexes: 0 to {255, 511} for 128 x {2,4} 170cc562d2eSVineet Gupta * The mapping is Column-first. 171cc562d2eSVineet Gupta * --------------------- ----------- 172cc562d2eSVineet Gupta * |way0|way1|way2|way3| |way0|way1| 173cc562d2eSVineet Gupta * --------------------- ----------- 174cc562d2eSVineet Gupta * [set0] | 0 | 1 | 2 | 3 | | 0 | 1 | 175cc562d2eSVineet Gupta * [set1] | 4 | 5 | 6 | 7 | | 2 | 3 | 176cc562d2eSVineet Gupta * ~ ~ ~ ~ 177cc562d2eSVineet Gupta * [set127] | 508| 509| 510| 511| | 254| 255| 178cc562d2eSVineet Gupta * --------------------- ----------- 179cc562d2eSVineet Gupta * For normal operations we don't(must not) care how above works since 180cc562d2eSVineet Gupta * MMU cmd getIndex(vaddr) abstracts that out. 181cc562d2eSVineet Gupta * However for walking WAYS of a SET, we need to know this 182cc562d2eSVineet Gupta */ 183cc562d2eSVineet Gupta #define SET_WAY_TO_IDX(mmu, set, way) ((set) * mmu->ways + (way)) 184cc562d2eSVineet Gupta 185cc562d2eSVineet Gupta /* Handling of Duplicate PD (TLB entry) in MMU. 186cc562d2eSVineet Gupta * -Could be due to buggy customer tapeouts or obscure kernel bugs 187cc562d2eSVineet Gupta * -MMU complaints not at the time of duplicate PD installation, but at the 188cc562d2eSVineet Gupta * time of lookup matching multiple ways. 189cc562d2eSVineet Gupta * -Ideally these should never happen - but if they do - workaround by deleting 190cc562d2eSVineet Gupta * the duplicate one. 191cc562d2eSVineet Gupta * -Knob to be verbose abt it.(TODO: hook them up to debugfs) 192cc562d2eSVineet Gupta */ 193cc562d2eSVineet Gupta volatile int dup_pd_verbose = 1;/* Be slient abt it or complain (default) */ 194cc562d2eSVineet Gupta 195cc562d2eSVineet Gupta void do_tlb_overlap_fault(unsigned long cause, unsigned long address, 196cc562d2eSVineet Gupta struct pt_regs *regs) 197cc562d2eSVineet Gupta { 198cc562d2eSVineet Gupta int set, way, n; 199cc562d2eSVineet Gupta unsigned int pd0[4], pd1[4]; /* assume max 4 ways */ 200cc562d2eSVineet Gupta unsigned long flags, is_valid; 201cc562d2eSVineet Gupta struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu; 202cc562d2eSVineet Gupta 203cc562d2eSVineet Gupta local_irq_save(flags); 204cc562d2eSVineet Gupta 205cc562d2eSVineet Gupta /* re-enable the MMU */ 206cc562d2eSVineet Gupta write_aux_reg(ARC_REG_PID, MMU_ENABLE | read_aux_reg(ARC_REG_PID)); 207cc562d2eSVineet Gupta 208cc562d2eSVineet Gupta /* loop thru all sets of TLB */ 209cc562d2eSVineet Gupta for (set = 0; set < mmu->sets; set++) { 210cc562d2eSVineet Gupta 211cc562d2eSVineet Gupta /* read out all the ways of current set */ 212cc562d2eSVineet Gupta for (way = 0, is_valid = 0; way < mmu->ways; way++) { 213cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBINDEX, 214cc562d2eSVineet Gupta SET_WAY_TO_IDX(mmu, set, way)); 215cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBCOMMAND, TLBRead); 216cc562d2eSVineet Gupta pd0[way] = read_aux_reg(ARC_REG_TLBPD0); 217cc562d2eSVineet Gupta pd1[way] = read_aux_reg(ARC_REG_TLBPD1); 218cc562d2eSVineet Gupta is_valid |= pd0[way] & _PAGE_PRESENT; 219cc562d2eSVineet Gupta } 220cc562d2eSVineet Gupta 221cc562d2eSVineet Gupta /* If all the WAYS in SET are empty, skip to next SET */ 222cc562d2eSVineet Gupta if (!is_valid) 223cc562d2eSVineet Gupta continue; 224cc562d2eSVineet Gupta 225cc562d2eSVineet Gupta /* Scan the set for duplicate ways: needs a nested loop */ 226cc562d2eSVineet Gupta for (way = 0; way < mmu->ways; way++) { 227cc562d2eSVineet Gupta if (!pd0[way]) 228cc562d2eSVineet Gupta continue; 229cc562d2eSVineet Gupta 230cc562d2eSVineet Gupta for (n = way + 1; n < mmu->ways; n++) { 231cc562d2eSVineet Gupta if ((pd0[way] & PAGE_MASK) == 232cc562d2eSVineet Gupta (pd0[n] & PAGE_MASK)) { 233cc562d2eSVineet Gupta 234cc562d2eSVineet Gupta if (dup_pd_verbose) { 235cc562d2eSVineet Gupta pr_info("Duplicate PD's @" 236cc562d2eSVineet Gupta "[%d:%d]/[%d:%d]\n", 237cc562d2eSVineet Gupta set, way, set, n); 238cc562d2eSVineet Gupta pr_info("TLBPD0[%u]: %08x\n", 239cc562d2eSVineet Gupta way, pd0[way]); 240cc562d2eSVineet Gupta } 241cc562d2eSVineet Gupta 242cc562d2eSVineet Gupta /* 243cc562d2eSVineet Gupta * clear entry @way and not @n. This is 244cc562d2eSVineet Gupta * critical to our optimised loop 245cc562d2eSVineet Gupta */ 246cc562d2eSVineet Gupta pd0[way] = pd1[way] = 0; 247cc562d2eSVineet Gupta write_aux_reg(ARC_REG_TLBINDEX, 248cc562d2eSVineet Gupta SET_WAY_TO_IDX(mmu, set, way)); 249cc562d2eSVineet Gupta __tlb_entry_erase(); 250cc562d2eSVineet Gupta } 251cc562d2eSVineet Gupta } 252cc562d2eSVineet Gupta } 253cc562d2eSVineet Gupta } 254cc562d2eSVineet Gupta 255cc562d2eSVineet Gupta local_irq_restore(flags); 256cc562d2eSVineet Gupta } 257cc562d2eSVineet Gupta 258cc562d2eSVineet Gupta /*********************************************************************** 259cc562d2eSVineet Gupta * Diagnostic Routines 260cc562d2eSVineet Gupta * -Called from Low Level TLB Hanlders if things don;t look good 261cc562d2eSVineet Gupta **********************************************************************/ 262cc562d2eSVineet Gupta 263cc562d2eSVineet Gupta #ifdef CONFIG_ARC_DBG_TLB_PARANOIA 264cc562d2eSVineet Gupta 265cc562d2eSVineet Gupta /* 266cc562d2eSVineet Gupta * Low Level ASM TLB handler calls this if it finds that HW and SW ASIDS 267cc562d2eSVineet Gupta * don't match 268cc562d2eSVineet Gupta */ 269cc562d2eSVineet Gupta void print_asid_mismatch(int is_fast_path) 270cc562d2eSVineet Gupta { 271cc562d2eSVineet Gupta int pid_sw, pid_hw; 272cc562d2eSVineet Gupta pid_sw = current->active_mm->context.asid; 273cc562d2eSVineet Gupta pid_hw = read_aux_reg(ARC_REG_PID) & 0xff; 274cc562d2eSVineet Gupta 275cc562d2eSVineet Gupta pr_emerg("ASID Mismatch in %s Path Handler: sw-pid=0x%x hw-pid=0x%x\n", 276cc562d2eSVineet Gupta is_fast_path ? "Fast" : "Slow", pid_sw, pid_hw); 277cc562d2eSVineet Gupta 278cc562d2eSVineet Gupta __asm__ __volatile__("flag 1"); 279cc562d2eSVineet Gupta } 280cc562d2eSVineet Gupta 281cc562d2eSVineet Gupta void tlb_paranoid_check(unsigned int pid_sw, unsigned long addr) 282cc562d2eSVineet Gupta { 283cc562d2eSVineet Gupta unsigned int pid_hw; 284cc562d2eSVineet Gupta 285cc562d2eSVineet Gupta pid_hw = read_aux_reg(ARC_REG_PID) & 0xff; 286cc562d2eSVineet Gupta 287cc562d2eSVineet Gupta if (addr < 0x70000000 && ((pid_hw != pid_sw) || (pid_sw == NO_ASID))) 288cc562d2eSVineet Gupta print_asid_mismatch(0); 289cc562d2eSVineet Gupta } 290cc562d2eSVineet Gupta #endif 291