1c983e92fSLey Foon Tan /*
2c983e92fSLey Foon Tan * Nios2 TLB handling
3c983e92fSLey Foon Tan *
4c983e92fSLey Foon Tan * Copyright (C) 2009, Wind River Systems Inc
5c983e92fSLey Foon Tan * Implemented by fredrik.markstrom@gmail.com and ivarholmqvist@gmail.com
6c983e92fSLey Foon Tan *
7c983e92fSLey Foon Tan * This file is subject to the terms and conditions of the GNU General Public
8c983e92fSLey Foon Tan * License. See the file "COPYING" in the main directory of this archive
9c983e92fSLey Foon Tan * for more details.
10c983e92fSLey Foon Tan */
11c983e92fSLey Foon Tan
12c983e92fSLey Foon Tan #include <linux/init.h>
13c983e92fSLey Foon Tan #include <linux/sched.h>
14c983e92fSLey Foon Tan #include <linux/mm.h>
15c983e92fSLey Foon Tan #include <linux/pagemap.h>
16c983e92fSLey Foon Tan
17c983e92fSLey Foon Tan #include <asm/tlb.h>
18c983e92fSLey Foon Tan #include <asm/mmu_context.h>
19c983e92fSLey Foon Tan #include <asm/cpuinfo.h>
20c983e92fSLey Foon Tan
21c983e92fSLey Foon Tan #define TLB_INDEX_MASK \
22c983e92fSLey Foon Tan ((((1UL << (cpuinfo.tlb_ptr_sz - cpuinfo.tlb_num_ways_log2))) - 1) \
23c983e92fSLey Foon Tan << PAGE_SHIFT)
24c983e92fSLey Foon Tan
get_misc_and_pid(unsigned long * misc,unsigned long * pid)25c983e92fSLey Foon Tan static void get_misc_and_pid(unsigned long *misc, unsigned long *pid)
26c983e92fSLey Foon Tan {
27c983e92fSLey Foon Tan *misc = RDCTL(CTL_TLBMISC);
28c983e92fSLey Foon Tan *misc &= (TLBMISC_PID | TLBMISC_WAY);
29c983e92fSLey Foon Tan *pid = *misc & TLBMISC_PID;
30c983e92fSLey Foon Tan }
31c983e92fSLey Foon Tan
32c983e92fSLey Foon Tan /*
333437d3c8SNicholas Piggin * This provides a PTEADDR value for addr that will cause a TLB miss
343437d3c8SNicholas Piggin * (fast TLB miss). TLB invalidation replaces entries with this value.
353437d3c8SNicholas Piggin */
pteaddr_invalid(unsigned long addr)363437d3c8SNicholas Piggin static unsigned long pteaddr_invalid(unsigned long addr)
373437d3c8SNicholas Piggin {
383437d3c8SNicholas Piggin return ((addr | 0xC0000000UL) >> PAGE_SHIFT) << 2;
393437d3c8SNicholas Piggin }
403437d3c8SNicholas Piggin
413437d3c8SNicholas Piggin /*
42c983e92fSLey Foon Tan * This one is only used for pages with the global bit set so we don't care
43c983e92fSLey Foon Tan * much about the ASID.
44c983e92fSLey Foon Tan */
replace_tlb_one_pid(unsigned long addr,unsigned long mmu_pid,unsigned long tlbacc)453ac23944SNicholas Piggin static void replace_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, unsigned long tlbacc)
46c983e92fSLey Foon Tan {
47c983e92fSLey Foon Tan unsigned int way;
48c983e92fSLey Foon Tan unsigned long org_misc, pid_misc;
49c983e92fSLey Foon Tan
50c983e92fSLey Foon Tan /* remember pid/way until we return. */
51c983e92fSLey Foon Tan get_misc_and_pid(&org_misc, &pid_misc);
52c983e92fSLey Foon Tan
53c983e92fSLey Foon Tan WRCTL(CTL_PTEADDR, (addr >> PAGE_SHIFT) << 2);
54c983e92fSLey Foon Tan
55c983e92fSLey Foon Tan for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
56c983e92fSLey Foon Tan unsigned long pteaddr;
57c983e92fSLey Foon Tan unsigned long tlbmisc;
58c983e92fSLey Foon Tan unsigned long pid;
59c983e92fSLey Foon Tan
607d173070SNicholas Piggin tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
61c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
62c6b1d363SNicholas Piggin
63c983e92fSLey Foon Tan pteaddr = RDCTL(CTL_PTEADDR);
64c6b1d363SNicholas Piggin if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT))
65c6b1d363SNicholas Piggin continue;
66c6b1d363SNicholas Piggin
67c983e92fSLey Foon Tan tlbmisc = RDCTL(CTL_TLBMISC);
68c983e92fSLey Foon Tan pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK;
69c6b1d363SNicholas Piggin if (pid != mmu_pid)
70c6b1d363SNicholas Piggin continue;
71c983e92fSLey Foon Tan
7221e6bff5SNicholas Piggin tlbmisc = (mmu_pid << TLBMISC_PID_SHIFT) | TLBMISC_WE |
7321e6bff5SNicholas Piggin (way << TLBMISC_WAY_SHIFT);
74c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
753ac23944SNicholas Piggin if (tlbacc == 0)
76c6b1d363SNicholas Piggin WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
773ac23944SNicholas Piggin WRCTL(CTL_TLBACC, tlbacc);
78b6a10463SNicholas Piggin /*
79b6a10463SNicholas Piggin * There should be only a single entry that maps a
80b6a10463SNicholas Piggin * particular {address,pid} so break after a match.
81b6a10463SNicholas Piggin */
82b6a10463SNicholas Piggin break;
83c983e92fSLey Foon Tan }
84c983e92fSLey Foon Tan
85c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, org_misc);
86c983e92fSLey Foon Tan }
87c983e92fSLey Foon Tan
flush_tlb_one_pid(unsigned long addr,unsigned long mmu_pid)883ac23944SNicholas Piggin static void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid)
893ac23944SNicholas Piggin {
903ac23944SNicholas Piggin pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr);
913ac23944SNicholas Piggin
923ac23944SNicholas Piggin replace_tlb_one_pid(addr, mmu_pid, 0);
933ac23944SNicholas Piggin }
943ac23944SNicholas Piggin
reload_tlb_one_pid(unsigned long addr,unsigned long mmu_pid,pte_t pte)953ac23944SNicholas Piggin static void reload_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, pte_t pte)
963ac23944SNicholas Piggin {
973ac23944SNicholas Piggin pr_debug("Reload tlb-entry for vaddr=%#lx\n", addr);
983ac23944SNicholas Piggin
993ac23944SNicholas Piggin replace_tlb_one_pid(addr, mmu_pid, pte_val(pte));
1003ac23944SNicholas Piggin }
1013ac23944SNicholas Piggin
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)102c983e92fSLey Foon Tan void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
103c983e92fSLey Foon Tan unsigned long end)
104c983e92fSLey Foon Tan {
105c983e92fSLey Foon Tan unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context);
106c983e92fSLey Foon Tan
107c983e92fSLey Foon Tan while (start < end) {
108c983e92fSLey Foon Tan flush_tlb_one_pid(start, mmu_pid);
109c983e92fSLey Foon Tan start += PAGE_SIZE;
110c983e92fSLey Foon Tan }
111c983e92fSLey Foon Tan }
112c983e92fSLey Foon Tan
reload_tlb_page(struct vm_area_struct * vma,unsigned long addr,pte_t pte)1133ac23944SNicholas Piggin void reload_tlb_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1143ac23944SNicholas Piggin {
1153ac23944SNicholas Piggin unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context);
1163ac23944SNicholas Piggin
1173ac23944SNicholas Piggin reload_tlb_one_pid(addr, mmu_pid, pte);
1183ac23944SNicholas Piggin }
1193ac23944SNicholas Piggin
120c983e92fSLey Foon Tan /*
121c983e92fSLey Foon Tan * This one is only used for pages with the global bit set so we don't care
122c983e92fSLey Foon Tan * much about the ASID.
123c983e92fSLey Foon Tan */
flush_tlb_one(unsigned long addr)124195568a1SNicholas Piggin static void flush_tlb_one(unsigned long addr)
125c983e92fSLey Foon Tan {
126c983e92fSLey Foon Tan unsigned int way;
127c983e92fSLey Foon Tan unsigned long org_misc, pid_misc;
128c983e92fSLey Foon Tan
129c983e92fSLey Foon Tan pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr);
130c983e92fSLey Foon Tan
131c983e92fSLey Foon Tan /* remember pid/way until we return. */
132c983e92fSLey Foon Tan get_misc_and_pid(&org_misc, &pid_misc);
133c983e92fSLey Foon Tan
134c983e92fSLey Foon Tan WRCTL(CTL_PTEADDR, (addr >> PAGE_SHIFT) << 2);
135c983e92fSLey Foon Tan
136c983e92fSLey Foon Tan for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
137c983e92fSLey Foon Tan unsigned long pteaddr;
138c983e92fSLey Foon Tan unsigned long tlbmisc;
139c983e92fSLey Foon Tan
1407d173070SNicholas Piggin tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
141c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
142c983e92fSLey Foon Tan
143c6b1d363SNicholas Piggin pteaddr = RDCTL(CTL_PTEADDR);
144c6b1d363SNicholas Piggin if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT))
145c6b1d363SNicholas Piggin continue;
146c6b1d363SNicholas Piggin
1473437d3c8SNicholas Piggin pr_debug("Flush entry by writing way=%dl pid=%ld\n",
1483437d3c8SNicholas Piggin way, (pid_misc >> TLBMISC_PID_SHIFT));
149c983e92fSLey Foon Tan
1507d173070SNicholas Piggin tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT);
151c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
152c6b1d363SNicholas Piggin WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
1533437d3c8SNicholas Piggin WRCTL(CTL_TLBACC, 0);
154c983e92fSLey Foon Tan }
155c983e92fSLey Foon Tan
156c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, org_misc);
157c983e92fSLey Foon Tan }
158c983e92fSLey Foon Tan
flush_tlb_kernel_range(unsigned long start,unsigned long end)159195568a1SNicholas Piggin void flush_tlb_kernel_range(unsigned long start, unsigned long end)
160195568a1SNicholas Piggin {
161195568a1SNicholas Piggin while (start < end) {
162195568a1SNicholas Piggin flush_tlb_one(start);
163195568a1SNicholas Piggin start += PAGE_SIZE;
164195568a1SNicholas Piggin }
165195568a1SNicholas Piggin }
166195568a1SNicholas Piggin
dump_tlb_line(unsigned long line)167c983e92fSLey Foon Tan void dump_tlb_line(unsigned long line)
168c983e92fSLey Foon Tan {
169c983e92fSLey Foon Tan unsigned int way;
170c983e92fSLey Foon Tan unsigned long org_misc;
171c983e92fSLey Foon Tan
172c983e92fSLey Foon Tan pr_debug("dump tlb-entries for line=%#lx (addr %08lx)\n", line,
173c983e92fSLey Foon Tan line << (PAGE_SHIFT + cpuinfo.tlb_num_ways_log2));
174c983e92fSLey Foon Tan
175c983e92fSLey Foon Tan /* remember pid/way until we return */
176c983e92fSLey Foon Tan org_misc = (RDCTL(CTL_TLBMISC) & (TLBMISC_PID | TLBMISC_WAY));
177c983e92fSLey Foon Tan
178c983e92fSLey Foon Tan WRCTL(CTL_PTEADDR, line << 2);
179c983e92fSLey Foon Tan
180c983e92fSLey Foon Tan for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
181c983e92fSLey Foon Tan unsigned long pteaddr;
182c983e92fSLey Foon Tan unsigned long tlbmisc;
183c983e92fSLey Foon Tan unsigned long tlbacc;
184c983e92fSLey Foon Tan
185c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, TLBMISC_RD | (way << TLBMISC_WAY_SHIFT));
186c983e92fSLey Foon Tan pteaddr = RDCTL(CTL_PTEADDR);
187c983e92fSLey Foon Tan tlbmisc = RDCTL(CTL_TLBMISC);
188c983e92fSLey Foon Tan tlbacc = RDCTL(CTL_TLBACC);
189c983e92fSLey Foon Tan
1903437d3c8SNicholas Piggin if ((tlbacc << PAGE_SHIFT) != 0) {
191c983e92fSLey Foon Tan pr_debug("-- way:%02x vpn:0x%08lx phys:0x%08lx pid:0x%02lx flags:%c%c%c%c%c\n",
192c983e92fSLey Foon Tan way,
193c983e92fSLey Foon Tan (pteaddr << (PAGE_SHIFT-2)),
194c983e92fSLey Foon Tan (tlbacc << PAGE_SHIFT),
195c983e92fSLey Foon Tan ((tlbmisc >> TLBMISC_PID_SHIFT) &
196c983e92fSLey Foon Tan TLBMISC_PID_MASK),
197c983e92fSLey Foon Tan (tlbacc & _PAGE_READ ? 'r' : '-'),
198c983e92fSLey Foon Tan (tlbacc & _PAGE_WRITE ? 'w' : '-'),
199c983e92fSLey Foon Tan (tlbacc & _PAGE_EXEC ? 'x' : '-'),
200c983e92fSLey Foon Tan (tlbacc & _PAGE_GLOBAL ? 'g' : '-'),
201c983e92fSLey Foon Tan (tlbacc & _PAGE_CACHED ? 'c' : '-'));
202c983e92fSLey Foon Tan }
203c983e92fSLey Foon Tan }
204c983e92fSLey Foon Tan
205c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, org_misc);
206c983e92fSLey Foon Tan }
207c983e92fSLey Foon Tan
dump_tlb(void)208c983e92fSLey Foon Tan void dump_tlb(void)
209c983e92fSLey Foon Tan {
210c983e92fSLey Foon Tan unsigned int i;
211c983e92fSLey Foon Tan
212c983e92fSLey Foon Tan for (i = 0; i < cpuinfo.tlb_num_lines; i++)
213c983e92fSLey Foon Tan dump_tlb_line(i);
214c983e92fSLey Foon Tan }
215c983e92fSLey Foon Tan
flush_tlb_pid(unsigned long mmu_pid)216c6b1d363SNicholas Piggin void flush_tlb_pid(unsigned long mmu_pid)
217c983e92fSLey Foon Tan {
2183437d3c8SNicholas Piggin unsigned long addr = 0;
219c983e92fSLey Foon Tan unsigned int line;
220c983e92fSLey Foon Tan unsigned int way;
221c983e92fSLey Foon Tan unsigned long org_misc, pid_misc;
222c983e92fSLey Foon Tan
223c983e92fSLey Foon Tan /* remember pid/way until we return */
224c983e92fSLey Foon Tan get_misc_and_pid(&org_misc, &pid_misc);
225c983e92fSLey Foon Tan
226c983e92fSLey Foon Tan for (line = 0; line < cpuinfo.tlb_num_lines; line++) {
2273437d3c8SNicholas Piggin WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
228c983e92fSLey Foon Tan
229c983e92fSLey Foon Tan for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
230c983e92fSLey Foon Tan unsigned long tlbmisc;
231c6b1d363SNicholas Piggin unsigned long pid;
232c983e92fSLey Foon Tan
2337d173070SNicholas Piggin tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
234c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
235c983e92fSLey Foon Tan tlbmisc = RDCTL(CTL_TLBMISC);
236c6b1d363SNicholas Piggin pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK;
237c6b1d363SNicholas Piggin if (pid != mmu_pid)
238c6b1d363SNicholas Piggin continue;
239c983e92fSLey Foon Tan
2407d173070SNicholas Piggin tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT);
241c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, tlbmisc);
2423437d3c8SNicholas Piggin WRCTL(CTL_TLBACC, 0);
243c983e92fSLey Foon Tan }
244c983e92fSLey Foon Tan
2453437d3c8SNicholas Piggin addr += PAGE_SIZE;
24658fd4766SNicholas Piggin }
2473437d3c8SNicholas Piggin
248c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, org_misc);
249c983e92fSLey Foon Tan }
250c983e92fSLey Foon Tan
251e71c99feSNicholas Piggin /*
252e71c99feSNicholas Piggin * All entries common to a mm share an asid. To effectively flush these
253e71c99feSNicholas Piggin * entries, we just bump the asid.
254e71c99feSNicholas Piggin */
flush_tlb_mm(struct mm_struct * mm)255e71c99feSNicholas Piggin void flush_tlb_mm(struct mm_struct *mm)
256e71c99feSNicholas Piggin {
257e71c99feSNicholas Piggin if (current->mm == mm) {
258e71c99feSNicholas Piggin unsigned long mmu_pid = get_pid_from_context(&mm->context);
259e71c99feSNicholas Piggin flush_tlb_pid(mmu_pid);
260e71c99feSNicholas Piggin } else {
261e71c99feSNicholas Piggin memset(&mm->context, 0, sizeof(mm_context_t));
262e71c99feSNicholas Piggin }
263e71c99feSNicholas Piggin }
264e71c99feSNicholas Piggin
flush_tlb_all(void)265c983e92fSLey Foon Tan void flush_tlb_all(void)
266c983e92fSLey Foon Tan {
2673437d3c8SNicholas Piggin unsigned long addr = 0;
2683437d3c8SNicholas Piggin unsigned int line;
269c983e92fSLey Foon Tan unsigned int way;
270737a3fa2SNicholas Piggin unsigned long org_misc, pid_misc;
271c983e92fSLey Foon Tan
272c983e92fSLey Foon Tan /* remember pid/way until we return */
273c983e92fSLey Foon Tan get_misc_and_pid(&org_misc, &pid_misc);
274c983e92fSLey Foon Tan
275737a3fa2SNicholas Piggin /* Start at way 0, way is auto-incremented after each TLBACC write */
276737a3fa2SNicholas Piggin WRCTL(CTL_TLBMISC, TLBMISC_WE);
277737a3fa2SNicholas Piggin
278c983e92fSLey Foon Tan /* Map each TLB entry to physcal address 0 with no-access and a
279c983e92fSLey Foon Tan bad ptbase */
2803437d3c8SNicholas Piggin for (line = 0; line < cpuinfo.tlb_num_lines; line++) {
2813437d3c8SNicholas Piggin WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
282737a3fa2SNicholas Piggin for (way = 0; way < cpuinfo.tlb_num_ways; way++)
2833437d3c8SNicholas Piggin WRCTL(CTL_TLBACC, 0);
2843437d3c8SNicholas Piggin
2853437d3c8SNicholas Piggin addr += PAGE_SIZE;
286c983e92fSLey Foon Tan }
287c983e92fSLey Foon Tan
288c983e92fSLey Foon Tan /* restore pid/way */
289c983e92fSLey Foon Tan WRCTL(CTL_TLBMISC, org_misc);
290c983e92fSLey Foon Tan }
291c983e92fSLey Foon Tan
set_mmu_pid(unsigned long pid)292c983e92fSLey Foon Tan void set_mmu_pid(unsigned long pid)
293c983e92fSLey Foon Tan {
294c6b1d363SNicholas Piggin unsigned long tlbmisc;
295c6b1d363SNicholas Piggin
296c6b1d363SNicholas Piggin tlbmisc = RDCTL(CTL_TLBMISC);
297c6b1d363SNicholas Piggin tlbmisc = (tlbmisc & TLBMISC_WAY);
298c6b1d363SNicholas Piggin tlbmisc |= (pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT;
299c6b1d363SNicholas Piggin WRCTL(CTL_TLBMISC, tlbmisc);
300c983e92fSLey Foon Tan }
301