xref: /openbmc/linux/arch/um/kernel/tlb.c (revision ba180fd437156f7fd8cfb2fdd021d949eeef08d6) 
1 /*
2  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3  * Licensed under the GPL
4  */
5 
6 #include "linux/mm.h"
7 #include "asm/pgtable.h"
8 #include "asm/tlbflush.h"
9 #include "as-layout.h"
10 #include "mem_user.h"
11 #include "os.h"
12 #include "skas.h"
13 #include "tlb.h"
14 
15 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
16 		    unsigned int prot, struct host_vm_op *ops, int *index,
17 		    int last_filled, union mm_context *mmu, void **flush,
18 		    int (*do_ops)(union mm_context *, struct host_vm_op *,
19 				  int, int, void **))
20 {
21 	__u64 offset;
22 	struct host_vm_op *last;
23 	int fd, ret = 0;
24 
25 	fd = phys_mapping(phys, &offset);
26 	if (*index != -1) {
27 		last = &ops[*index];
28 		if ((last->type == MMAP) &&
29 		   (last->u.mmap.addr + last->u.mmap.len == virt) &&
30 		   (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
31 		   (last->u.mmap.offset + last->u.mmap.len == offset)) {
32 			last->u.mmap.len += len;
33 			return 0;
34 		}
35 	}
36 
37 	if (*index == last_filled) {
38 		ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
39 		*index = -1;
40 	}
41 
42 	ops[++*index] = ((struct host_vm_op) { .type	= MMAP,
43 			     			.u = { .mmap = {
44 						       .addr	= virt,
45 						       .len	= len,
46 						       .prot	= prot,
47 						       .fd	= fd,
48 						       .offset	= offset }
49 			   } });
50 	return ret;
51 }
52 
53 static int add_munmap(unsigned long addr, unsigned long len,
54 		      struct host_vm_op *ops, int *index, int last_filled,
55 		      union mm_context *mmu, void **flush,
56 		      int (*do_ops)(union mm_context *, struct host_vm_op *,
57 				    int, int, void **))
58 {
59 	struct host_vm_op *last;
60 	int ret = 0;
61 
62 	if (*index != -1) {
63 		last = &ops[*index];
64 		if ((last->type == MUNMAP) &&
65 		   (last->u.munmap.addr + last->u.mmap.len == addr)) {
66 			last->u.munmap.len += len;
67 			return 0;
68 		}
69 	}
70 
71 	if (*index == last_filled) {
72 		ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
73 		*index = -1;
74 	}
75 
76 	ops[++*index] = ((struct host_vm_op) { .type	= MUNMAP,
77 			     		       .u = { .munmap = {
78 						        .addr	= addr,
79 							.len	= len } } });
80 	return ret;
81 }
82 
83 static int add_mprotect(unsigned long addr, unsigned long len,
84 			unsigned int prot, struct host_vm_op *ops, int *index,
85 			int last_filled, union mm_context *mmu, void **flush,
86 			int (*do_ops)(union mm_context *, struct host_vm_op *,
87 				      int, int, void **))
88 {
89 	struct host_vm_op *last;
90 	int ret = 0;
91 
92 	if (*index != -1) {
93 		last = &ops[*index];
94 		if ((last->type == MPROTECT) &&
95 		   (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
96 		   (last->u.mprotect.prot == prot)) {
97 			last->u.mprotect.len += len;
98 			return 0;
99 		}
100 	}
101 
102 	if (*index == last_filled) {
103 		ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
104 		*index = -1;
105 	}
106 
107 	ops[++*index] = ((struct host_vm_op) { .type	= MPROTECT,
108 			     		       .u = { .mprotect = {
109 						       .addr	= addr,
110 						       .len	= len,
111 						       .prot	= prot } } });
112 	return ret;
113 }
114 
115 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
116 
117 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
118 				   unsigned long end, struct host_vm_op *ops,
119 				   int last_op, int *op_index, int force,
120 				   union mm_context *mmu, void **flush,
121 				   int (*do_ops)(union mm_context *,
122 						 struct host_vm_op *, int, int,
123 						 void **))
124 {
125 	pte_t *pte;
126 	int r, w, x, prot, ret = 0;
127 
128 	pte = pte_offset_kernel(pmd, addr);
129 	do {
130 		r = pte_read(*pte);
131 		w = pte_write(*pte);
132 		x = pte_exec(*pte);
133 		if (!pte_young(*pte)) {
134 			r = 0;
135 			w = 0;
136 		} else if (!pte_dirty(*pte)) {
137 			w = 0;
138 		}
139 		prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
140 			(x ? UM_PROT_EXEC : 0));
141 		if (force || pte_newpage(*pte)) {
142 			if (pte_present(*pte))
143 				ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
144 					       PAGE_SIZE, prot, ops, op_index,
145 					       last_op, mmu, flush, do_ops);
146 			else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
147 					      last_op, mmu, flush, do_ops);
148 		}
149 		else if (pte_newprot(*pte))
150 			ret = add_mprotect(addr, PAGE_SIZE, prot, ops, op_index,
151 					   last_op, mmu, flush, do_ops);
152 		*pte = pte_mkuptodate(*pte);
153 	} while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
154 	return ret;
155 }
156 
157 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
158 				   unsigned long end, struct host_vm_op *ops,
159 				   int last_op, int *op_index, int force,
160 				   union mm_context *mmu, void **flush,
161 				   int (*do_ops)(union mm_context *,
162 						 struct host_vm_op *, int, int,
163 						 void **))
164 {
165 	pmd_t *pmd;
166 	unsigned long next;
167 	int ret = 0;
168 
169 	pmd = pmd_offset(pud, addr);
170 	do {
171 		next = pmd_addr_end(addr, end);
172 		if (!pmd_present(*pmd)) {
173 			if (force || pmd_newpage(*pmd)) {
174 				ret = add_munmap(addr, next - addr, ops,
175 						 op_index, last_op, mmu,
176 						 flush, do_ops);
177 				pmd_mkuptodate(*pmd);
178 			}
179 		}
180 		else ret = update_pte_range(pmd, addr, next, ops, last_op,
181 					    op_index, force, mmu, flush,
182 					    do_ops);
183 	} while (pmd++, addr = next, ((addr != end) && !ret));
184 	return ret;
185 }
186 
187 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
188 				   unsigned long end, struct host_vm_op *ops,
189 				   int last_op, int *op_index, int force,
190 				   union mm_context *mmu, void **flush,
191 				   int (*do_ops)(union mm_context *,
192 						 struct host_vm_op *, int, int,
193 						 void **))
194 {
195 	pud_t *pud;
196 	unsigned long next;
197 	int ret = 0;
198 
199 	pud = pud_offset(pgd, addr);
200 	do {
201 		next = pud_addr_end(addr, end);
202 		if (!pud_present(*pud)) {
203 			if (force || pud_newpage(*pud)) {
204 				ret = add_munmap(addr, next - addr, ops,
205 						 op_index, last_op, mmu,
206 						 flush, do_ops);
207 				pud_mkuptodate(*pud);
208 			}
209 		}
210 		else ret = update_pmd_range(pud, addr, next, ops, last_op,
211 					    op_index, force, mmu, flush,
212 					    do_ops);
213 	} while (pud++, addr = next, ((addr != end) && !ret));
214 	return ret;
215 }
216 
217 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
218 		      unsigned long end_addr, int force,
219 		      int (*do_ops)(union mm_context *, struct host_vm_op *,
220 				    int, int, void **))
221 {
222 	pgd_t *pgd;
223 	union mm_context *mmu = &mm->context;
224 	struct host_vm_op ops[1];
225 	unsigned long addr = start_addr, next;
226 	int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
227 	void *flush = NULL;
228 
229 	ops[0].type = NONE;
230 	pgd = pgd_offset(mm, addr);
231 	do {
232 		next = pgd_addr_end(addr, end_addr);
233 		if (!pgd_present(*pgd)) {
234 			if (force || pgd_newpage(*pgd)) {
235 				ret = add_munmap(addr, next - addr, ops,
236 						 &op_index, last_op, mmu,
237 						 &flush, do_ops);
238 				pgd_mkuptodate(*pgd);
239 			}
240 		}
241 		else ret = update_pud_range(pgd, addr, next, ops, last_op,
242 					    &op_index, force, mmu, &flush,
243 					    do_ops);
244 	} while (pgd++, addr = next, ((addr != end_addr) && !ret));
245 
246 	if (!ret)
247 		ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
248 
249 	/* This is not an else because ret is modified above */
250 	if (ret) {
251 		printk(KERN_ERR "fix_range_common: failed, killing current "
252 		       "process\n");
253 		force_sig(SIGKILL, current);
254 	}
255 }
256 
257 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
258 {
259 	struct mm_struct *mm;
260 	pgd_t *pgd;
261 	pud_t *pud;
262 	pmd_t *pmd;
263 	pte_t *pte;
264 	unsigned long addr, last;
265 	int updated = 0, err;
266 
267 	mm = &init_mm;
268 	for (addr = start; addr < end;) {
269 		pgd = pgd_offset(mm, addr);
270 		if (!pgd_present(*pgd)) {
271 			last = ADD_ROUND(addr, PGDIR_SIZE);
272 			if (last > end)
273 				last = end;
274 			if (pgd_newpage(*pgd)) {
275 				updated = 1;
276 				err = os_unmap_memory((void *) addr,
277 						      last - addr);
278 				if (err < 0)
279 					panic("munmap failed, errno = %d\n",
280 					      -err);
281 			}
282 			addr = last;
283 			continue;
284 		}
285 
286 		pud = pud_offset(pgd, addr);
287 		if (!pud_present(*pud)) {
288 			last = ADD_ROUND(addr, PUD_SIZE);
289 			if (last > end)
290 				last = end;
291 			if (pud_newpage(*pud)) {
292 				updated = 1;
293 				err = os_unmap_memory((void *) addr,
294 						      last - addr);
295 				if (err < 0)
296 					panic("munmap failed, errno = %d\n",
297 					      -err);
298 			}
299 			addr = last;
300 			continue;
301 		}
302 
303 		pmd = pmd_offset(pud, addr);
304 		if (!pmd_present(*pmd)) {
305 			last = ADD_ROUND(addr, PMD_SIZE);
306 			if (last > end)
307 				last = end;
308 			if (pmd_newpage(*pmd)) {
309 				updated = 1;
310 				err = os_unmap_memory((void *) addr,
311 						      last - addr);
312 				if (err < 0)
313 					panic("munmap failed, errno = %d\n",
314 					      -err);
315 			}
316 			addr = last;
317 			continue;
318 		}
319 
320 		pte = pte_offset_kernel(pmd, addr);
321 		if (!pte_present(*pte) || pte_newpage(*pte)) {
322 			updated = 1;
323 			err = os_unmap_memory((void *) addr,
324 					      PAGE_SIZE);
325 			if (err < 0)
326 				panic("munmap failed, errno = %d\n",
327 				      -err);
328 			if (pte_present(*pte))
329 				map_memory(addr,
330 					   pte_val(*pte) & PAGE_MASK,
331 					   PAGE_SIZE, 1, 1, 1);
332 		}
333 		else if (pte_newprot(*pte)) {
334 			updated = 1;
335 			os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
336 		}
337 		addr += PAGE_SIZE;
338 	}
339 	return updated;
340 }
341 
342 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
343 {
344 	pgd_t *pgd;
345 	pud_t *pud;
346 	pmd_t *pmd;
347 	pte_t *pte;
348 	struct mm_struct *mm = vma->vm_mm;
349 	void *flush = NULL;
350 	int r, w, x, prot, err = 0;
351 	struct mm_id *mm_id;
352 
353 	address &= PAGE_MASK;
354 	pgd = pgd_offset(mm, address);
355 	if (!pgd_present(*pgd))
356 		goto kill;
357 
358 	pud = pud_offset(pgd, address);
359 	if (!pud_present(*pud))
360 		goto kill;
361 
362 	pmd = pmd_offset(pud, address);
363 	if (!pmd_present(*pmd))
364 		goto kill;
365 
366 	pte = pte_offset_kernel(pmd, address);
367 
368 	r = pte_read(*pte);
369 	w = pte_write(*pte);
370 	x = pte_exec(*pte);
371 	if (!pte_young(*pte)) {
372 		r = 0;
373 		w = 0;
374 	} else if (!pte_dirty(*pte)) {
375 		w = 0;
376 	}
377 
378 	mm_id = &mm->context.skas.id;
379 	prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
380 		(x ? UM_PROT_EXEC : 0));
381 	if (pte_newpage(*pte)) {
382 		if (pte_present(*pte)) {
383 			unsigned long long offset;
384 			int fd;
385 
386 			fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
387 			err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
388 				  1, &flush);
389 		}
390 		else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
391 	}
392 	else if (pte_newprot(*pte))
393 		err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
394 
395 	if (err)
396 		goto kill;
397 
398 	*pte = pte_mkuptodate(*pte);
399 
400 	return;
401 
402 kill:
403 	printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
404 	force_sig(SIGKILL, current);
405 }
406 
407 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
408 {
409 	return pgd_offset(mm, address);
410 }
411 
412 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
413 {
414 	return pud_offset(pgd, address);
415 }
416 
417 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
418 {
419 	return pmd_offset(pud, address);
420 }
421 
422 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
423 {
424 	return pte_offset_kernel(pmd, address);
425 }
426 
427 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
428 {
429 	pgd_t *pgd = pgd_offset(task->mm, addr);
430 	pud_t *pud = pud_offset(pgd, addr);
431 	pmd_t *pmd = pmd_offset(pud, addr);
432 
433 	return pte_offset_map(pmd, addr);
434 }
435 
436 void flush_tlb_all(void)
437 {
438 	flush_tlb_mm(current->mm);
439 }
440 
441 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
442 {
443 	flush_tlb_kernel_range_common(start, end);
444 }
445 
446 void flush_tlb_kernel_vm(void)
447 {
448 	flush_tlb_kernel_range_common(start_vm, end_vm);
449 }
450 
451 void __flush_tlb_one(unsigned long addr)
452 {
453 	flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
454 }
455 
456 static int do_ops(union mm_context *mmu, struct host_vm_op *ops, int last,
457 		  int finished, void **flush)
458 {
459 	struct host_vm_op *op;
460 	int i, ret = 0;
461 
462 	for (i = 0; i <= last && !ret; i++) {
463 	op = &ops[i];
464 		switch(op->type) {
465 		case MMAP:
466 			ret = map(&mmu->skas.id, op->u.mmap.addr,
467 				  op->u.mmap.len, op->u.mmap.prot,
468 				  op->u.mmap.fd, op->u.mmap.offset, finished,
469 				  flush);
470 			break;
471 		case MUNMAP:
472 			ret = unmap(&mmu->skas.id, op->u.munmap.addr,
473 				    op->u.munmap.len, finished, flush);
474 			break;
475 		case MPROTECT:
476 			ret = protect(&mmu->skas.id, op->u.mprotect.addr,
477 				      op->u.mprotect.len, op->u.mprotect.prot,
478 				      finished, flush);
479 			break;
480 		default:
481 			printk(KERN_ERR "Unknown op type %d in do_ops\n",
482 			       op->type);
483 			break;
484 		}
485 	}
486 
487 	return ret;
488 }
489 
490 static void fix_range(struct mm_struct *mm, unsigned long start_addr,
491 		      unsigned long end_addr, int force)
492 {
493 	if (!proc_mm && (end_addr > CONFIG_STUB_START))
494 		end_addr = CONFIG_STUB_START;
495 
496 	fix_range_common(mm, start_addr, end_addr, force, do_ops);
497 }
498 
499 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
500 		     unsigned long end)
501 {
502 	if (vma->vm_mm == NULL)
503 		flush_tlb_kernel_range_common(start, end);
504 	else fix_range(vma->vm_mm, start, end, 0);
505 }
506 
507 void flush_tlb_mm(struct mm_struct *mm)
508 {
509 	unsigned long end;
510 
511 	/*
512 	 * Don't bother flushing if this address space is about to be
513 	 * destroyed.
514 	 */
515 	if (atomic_read(&mm->mm_users) == 0)
516 		return;
517 
518 	end = proc_mm ? task_size : CONFIG_STUB_START;
519 	fix_range(mm, 0, end, 0);
520 }
521 
522 void force_flush_all(void)
523 {
524 	struct mm_struct *mm = current->mm;
525 	struct vm_area_struct *vma = mm->mmap;
526 
527 	while (vma != NULL) {
528 		fix_range(mm, vma->vm_start, vma->vm_end, 1);
529 		vma = vma->vm_next;
530 	}
531 }
532