xref: /openbmc/linux/arch/mips/mm/tlb-r4k.c (revision 4800cd83)
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8  * Carsten Langgaard, carstenl@mips.com
9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
10  */
11 #include <linux/init.h>
12 #include <linux/sched.h>
13 #include <linux/smp.h>
14 #include <linux/mm.h>
15 #include <linux/hugetlb.h>
16 
17 #include <asm/cpu.h>
18 #include <asm/bootinfo.h>
19 #include <asm/mmu_context.h>
20 #include <asm/pgtable.h>
21 #include <asm/system.h>
22 
23 extern void build_tlb_refill_handler(void);
24 
25 /*
26  * Make sure all entries differ.  If they're not different
27  * MIPS32 will take revenge ...
28  */
29 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
30 
31 /* Atomicity and interruptability */
32 #ifdef CONFIG_MIPS_MT_SMTC
33 
34 #include <asm/smtc.h>
35 #include <asm/mipsmtregs.h>
36 
37 #define ENTER_CRITICAL(flags) \
38 	{ \
39 	unsigned int mvpflags; \
40 	local_irq_save(flags);\
41 	mvpflags = dvpe()
42 #define EXIT_CRITICAL(flags) \
43 	evpe(mvpflags); \
44 	local_irq_restore(flags); \
45 	}
46 #else
47 
48 #define ENTER_CRITICAL(flags) local_irq_save(flags)
49 #define EXIT_CRITICAL(flags) local_irq_restore(flags)
50 
51 #endif /* CONFIG_MIPS_MT_SMTC */
52 
53 #if defined(CONFIG_CPU_LOONGSON2)
54 /*
55  * LOONGSON2 has a 4 entry itlb which is a subset of dtlb,
56  * unfortrunately, itlb is not totally transparent to software.
57  */
58 #define FLUSH_ITLB write_c0_diag(4);
59 
60 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC)  write_c0_diag(4); }
61 
62 #else
63 
64 #define FLUSH_ITLB
65 #define FLUSH_ITLB_VM(vma)
66 
67 #endif
68 
69 void local_flush_tlb_all(void)
70 {
71 	unsigned long flags;
72 	unsigned long old_ctx;
73 	int entry;
74 
75 	ENTER_CRITICAL(flags);
76 	/* Save old context and create impossible VPN2 value */
77 	old_ctx = read_c0_entryhi();
78 	write_c0_entrylo0(0);
79 	write_c0_entrylo1(0);
80 
81 	entry = read_c0_wired();
82 
83 	/* Blast 'em all away. */
84 	while (entry < current_cpu_data.tlbsize) {
85 		/* Make sure all entries differ. */
86 		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
87 		write_c0_index(entry);
88 		mtc0_tlbw_hazard();
89 		tlb_write_indexed();
90 		entry++;
91 	}
92 	tlbw_use_hazard();
93 	write_c0_entryhi(old_ctx);
94 	FLUSH_ITLB;
95 	EXIT_CRITICAL(flags);
96 }
97 
98 /* All entries common to a mm share an asid.  To effectively flush
99    these entries, we just bump the asid. */
100 void local_flush_tlb_mm(struct mm_struct *mm)
101 {
102 	int cpu;
103 
104 	preempt_disable();
105 
106 	cpu = smp_processor_id();
107 
108 	if (cpu_context(cpu, mm) != 0) {
109 		drop_mmu_context(mm, cpu);
110 	}
111 
112 	preempt_enable();
113 }
114 
115 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
116 	unsigned long end)
117 {
118 	struct mm_struct *mm = vma->vm_mm;
119 	int cpu = smp_processor_id();
120 
121 	if (cpu_context(cpu, mm) != 0) {
122 		unsigned long size, flags;
123 
124 		ENTER_CRITICAL(flags);
125 		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
126 		size = (size + 1) >> 1;
127 		if (size <= current_cpu_data.tlbsize/2) {
128 			int oldpid = read_c0_entryhi();
129 			int newpid = cpu_asid(cpu, mm);
130 
131 			start &= (PAGE_MASK << 1);
132 			end += ((PAGE_SIZE << 1) - 1);
133 			end &= (PAGE_MASK << 1);
134 			while (start < end) {
135 				int idx;
136 
137 				write_c0_entryhi(start | newpid);
138 				start += (PAGE_SIZE << 1);
139 				mtc0_tlbw_hazard();
140 				tlb_probe();
141 				tlb_probe_hazard();
142 				idx = read_c0_index();
143 				write_c0_entrylo0(0);
144 				write_c0_entrylo1(0);
145 				if (idx < 0)
146 					continue;
147 				/* Make sure all entries differ. */
148 				write_c0_entryhi(UNIQUE_ENTRYHI(idx));
149 				mtc0_tlbw_hazard();
150 				tlb_write_indexed();
151 			}
152 			tlbw_use_hazard();
153 			write_c0_entryhi(oldpid);
154 		} else {
155 			drop_mmu_context(mm, cpu);
156 		}
157 		FLUSH_ITLB;
158 		EXIT_CRITICAL(flags);
159 	}
160 }
161 
162 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
163 {
164 	unsigned long size, flags;
165 
166 	ENTER_CRITICAL(flags);
167 	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
168 	size = (size + 1) >> 1;
169 	if (size <= current_cpu_data.tlbsize / 2) {
170 		int pid = read_c0_entryhi();
171 
172 		start &= (PAGE_MASK << 1);
173 		end += ((PAGE_SIZE << 1) - 1);
174 		end &= (PAGE_MASK << 1);
175 
176 		while (start < end) {
177 			int idx;
178 
179 			write_c0_entryhi(start);
180 			start += (PAGE_SIZE << 1);
181 			mtc0_tlbw_hazard();
182 			tlb_probe();
183 			tlb_probe_hazard();
184 			idx = read_c0_index();
185 			write_c0_entrylo0(0);
186 			write_c0_entrylo1(0);
187 			if (idx < 0)
188 				continue;
189 			/* Make sure all entries differ. */
190 			write_c0_entryhi(UNIQUE_ENTRYHI(idx));
191 			mtc0_tlbw_hazard();
192 			tlb_write_indexed();
193 		}
194 		tlbw_use_hazard();
195 		write_c0_entryhi(pid);
196 	} else {
197 		local_flush_tlb_all();
198 	}
199 	FLUSH_ITLB;
200 	EXIT_CRITICAL(flags);
201 }
202 
203 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
204 {
205 	int cpu = smp_processor_id();
206 
207 	if (cpu_context(cpu, vma->vm_mm) != 0) {
208 		unsigned long flags;
209 		int oldpid, newpid, idx;
210 
211 		newpid = cpu_asid(cpu, vma->vm_mm);
212 		page &= (PAGE_MASK << 1);
213 		ENTER_CRITICAL(flags);
214 		oldpid = read_c0_entryhi();
215 		write_c0_entryhi(page | newpid);
216 		mtc0_tlbw_hazard();
217 		tlb_probe();
218 		tlb_probe_hazard();
219 		idx = read_c0_index();
220 		write_c0_entrylo0(0);
221 		write_c0_entrylo1(0);
222 		if (idx < 0)
223 			goto finish;
224 		/* Make sure all entries differ. */
225 		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
226 		mtc0_tlbw_hazard();
227 		tlb_write_indexed();
228 		tlbw_use_hazard();
229 
230 	finish:
231 		write_c0_entryhi(oldpid);
232 		FLUSH_ITLB_VM(vma);
233 		EXIT_CRITICAL(flags);
234 	}
235 }
236 
237 /*
238  * This one is only used for pages with the global bit set so we don't care
239  * much about the ASID.
240  */
241 void local_flush_tlb_one(unsigned long page)
242 {
243 	unsigned long flags;
244 	int oldpid, idx;
245 
246 	ENTER_CRITICAL(flags);
247 	oldpid = read_c0_entryhi();
248 	page &= (PAGE_MASK << 1);
249 	write_c0_entryhi(page);
250 	mtc0_tlbw_hazard();
251 	tlb_probe();
252 	tlb_probe_hazard();
253 	idx = read_c0_index();
254 	write_c0_entrylo0(0);
255 	write_c0_entrylo1(0);
256 	if (idx >= 0) {
257 		/* Make sure all entries differ. */
258 		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
259 		mtc0_tlbw_hazard();
260 		tlb_write_indexed();
261 		tlbw_use_hazard();
262 	}
263 	write_c0_entryhi(oldpid);
264 	FLUSH_ITLB;
265 	EXIT_CRITICAL(flags);
266 }
267 
268 /*
269  * We will need multiple versions of update_mmu_cache(), one that just
270  * updates the TLB with the new pte(s), and another which also checks
271  * for the R4k "end of page" hardware bug and does the needy.
272  */
273 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
274 {
275 	unsigned long flags;
276 	pgd_t *pgdp;
277 	pud_t *pudp;
278 	pmd_t *pmdp;
279 	pte_t *ptep;
280 	int idx, pid;
281 
282 	/*
283 	 * Handle debugger faulting in for debugee.
284 	 */
285 	if (current->active_mm != vma->vm_mm)
286 		return;
287 
288 	ENTER_CRITICAL(flags);
289 
290 	pid = read_c0_entryhi() & ASID_MASK;
291 	address &= (PAGE_MASK << 1);
292 	write_c0_entryhi(address | pid);
293 	pgdp = pgd_offset(vma->vm_mm, address);
294 	mtc0_tlbw_hazard();
295 	tlb_probe();
296 	tlb_probe_hazard();
297 	pudp = pud_offset(pgdp, address);
298 	pmdp = pmd_offset(pudp, address);
299 	idx = read_c0_index();
300 #ifdef CONFIG_HUGETLB_PAGE
301 	/* this could be a huge page  */
302 	if (pmd_huge(*pmdp)) {
303 		unsigned long lo;
304 		write_c0_pagemask(PM_HUGE_MASK);
305 		ptep = (pte_t *)pmdp;
306 		lo = pte_to_entrylo(pte_val(*ptep));
307 		write_c0_entrylo0(lo);
308 		write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
309 
310 		mtc0_tlbw_hazard();
311 		if (idx < 0)
312 			tlb_write_random();
313 		else
314 			tlb_write_indexed();
315 		write_c0_pagemask(PM_DEFAULT_MASK);
316 	} else
317 #endif
318 	{
319 		ptep = pte_offset_map(pmdp, address);
320 
321 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
322 		write_c0_entrylo0(ptep->pte_high);
323 		ptep++;
324 		write_c0_entrylo1(ptep->pte_high);
325 #else
326 		write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
327 		write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
328 #endif
329 		mtc0_tlbw_hazard();
330 		if (idx < 0)
331 			tlb_write_random();
332 		else
333 			tlb_write_indexed();
334 	}
335 	tlbw_use_hazard();
336 	FLUSH_ITLB_VM(vma);
337 	EXIT_CRITICAL(flags);
338 }
339 
340 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
341 	unsigned long entryhi, unsigned long pagemask)
342 {
343 	unsigned long flags;
344 	unsigned long wired;
345 	unsigned long old_pagemask;
346 	unsigned long old_ctx;
347 
348 	ENTER_CRITICAL(flags);
349 	/* Save old context and create impossible VPN2 value */
350 	old_ctx = read_c0_entryhi();
351 	old_pagemask = read_c0_pagemask();
352 	wired = read_c0_wired();
353 	write_c0_wired(wired + 1);
354 	write_c0_index(wired);
355 	tlbw_use_hazard();	/* What is the hazard here? */
356 	write_c0_pagemask(pagemask);
357 	write_c0_entryhi(entryhi);
358 	write_c0_entrylo0(entrylo0);
359 	write_c0_entrylo1(entrylo1);
360 	mtc0_tlbw_hazard();
361 	tlb_write_indexed();
362 	tlbw_use_hazard();
363 
364 	write_c0_entryhi(old_ctx);
365 	tlbw_use_hazard();	/* What is the hazard here? */
366 	write_c0_pagemask(old_pagemask);
367 	local_flush_tlb_all();
368 	EXIT_CRITICAL(flags);
369 }
370 
371 /*
372  * Used for loading TLB entries before trap_init() has started, when we
373  * don't actually want to add a wired entry which remains throughout the
374  * lifetime of the system
375  */
376 
377 static int temp_tlb_entry __cpuinitdata;
378 
379 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
380 			       unsigned long entryhi, unsigned long pagemask)
381 {
382 	int ret = 0;
383 	unsigned long flags;
384 	unsigned long wired;
385 	unsigned long old_pagemask;
386 	unsigned long old_ctx;
387 
388 	ENTER_CRITICAL(flags);
389 	/* Save old context and create impossible VPN2 value */
390 	old_ctx = read_c0_entryhi();
391 	old_pagemask = read_c0_pagemask();
392 	wired = read_c0_wired();
393 	if (--temp_tlb_entry < wired) {
394 		printk(KERN_WARNING
395 		       "No TLB space left for add_temporary_entry\n");
396 		ret = -ENOSPC;
397 		goto out;
398 	}
399 
400 	write_c0_index(temp_tlb_entry);
401 	write_c0_pagemask(pagemask);
402 	write_c0_entryhi(entryhi);
403 	write_c0_entrylo0(entrylo0);
404 	write_c0_entrylo1(entrylo1);
405 	mtc0_tlbw_hazard();
406 	tlb_write_indexed();
407 	tlbw_use_hazard();
408 
409 	write_c0_entryhi(old_ctx);
410 	write_c0_pagemask(old_pagemask);
411 out:
412 	EXIT_CRITICAL(flags);
413 	return ret;
414 }
415 
416 static int __cpuinitdata ntlb;
417 static int __init set_ntlb(char *str)
418 {
419 	get_option(&str, &ntlb);
420 	return 1;
421 }
422 
423 __setup("ntlb=", set_ntlb);
424 
425 void __cpuinit tlb_init(void)
426 {
427 	/*
428 	 * You should never change this register:
429 	 *   - On R4600 1.7 the tlbp never hits for pages smaller than
430 	 *     the value in the c0_pagemask register.
431 	 *   - The entire mm handling assumes the c0_pagemask register to
432 	 *     be set to fixed-size pages.
433 	 */
434 	write_c0_pagemask(PM_DEFAULT_MASK);
435 	write_c0_wired(0);
436 	if (current_cpu_type() == CPU_R10000 ||
437 	    current_cpu_type() == CPU_R12000 ||
438 	    current_cpu_type() == CPU_R14000)
439 		write_c0_framemask(0);
440 
441 	if (kernel_uses_smartmips_rixi) {
442 		/*
443 		 * Enable the no read, no exec bits, and enable large virtual
444 		 * address.
445 		 */
446 		u32 pg = PG_RIE | PG_XIE;
447 #ifdef CONFIG_64BIT
448 		pg |= PG_ELPA;
449 #endif
450 		write_c0_pagegrain(pg);
451 	}
452 
453 	temp_tlb_entry = current_cpu_data.tlbsize - 1;
454 
455         /* From this point on the ARC firmware is dead.  */
456 	local_flush_tlb_all();
457 
458 	/* Did I tell you that ARC SUCKS?  */
459 
460 	if (ntlb) {
461 		if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
462 			int wired = current_cpu_data.tlbsize - ntlb;
463 			write_c0_wired(wired);
464 			write_c0_index(wired-1);
465 			printk("Restricting TLB to %d entries\n", ntlb);
466 		} else
467 			printk("Ignoring invalid argument ntlb=%d\n", ntlb);
468 	}
469 
470 	build_tlb_refill_handler();
471 }
472