xref: /openbmc/linux/arch/um/kernel/tlb.c (revision 2eb5f31b) 
1 /*
2  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3  * Licensed under the GPL
4  */
5 
6 #include <linux/mm.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <asm/pgtable.h>
10 #include <asm/tlbflush.h>
11 #include <as-layout.h>
12 #include <mem_user.h>
13 #include <os.h>
14 #include <skas.h>
15 #include <kern_util.h>
16 
17 struct host_vm_change {
18 	struct host_vm_op {
19 		enum { NONE, MMAP, MUNMAP, MPROTECT } type;
20 		union {
21 			struct {
22 				unsigned long addr;
23 				unsigned long len;
24 				unsigned int prot;
25 				int fd;
26 				__u64 offset;
27 			} mmap;
28 			struct {
29 				unsigned long addr;
30 				unsigned long len;
31 			} munmap;
32 			struct {
33 				unsigned long addr;
34 				unsigned long len;
35 				unsigned int prot;
36 			} mprotect;
37 		} u;
38 	} ops[1];
39 	int index;
40 	struct mm_id *id;
41 	void *data;
42 	int force;
43 };
44 
45 #define INIT_HVC(mm, force) \
46 	((struct host_vm_change) \
47 	 { .ops		= { { .type = NONE } },	\
48 	   .id		= &mm->context.id, \
49        	   .data	= NULL, \
50 	   .index	= 0, \
51 	   .force	= force })
52 
53 static void report_enomem(void)
54 {
55 	printk(KERN_ERR "UML ran out of memory on the host side! "
56 			"This can happen due to a memory limitation or "
57 			"vm.max_map_count has been reached.\n");
58 }
59 
60 static int do_ops(struct host_vm_change *hvc, int end,
61 		  int finished)
62 {
63 	struct host_vm_op *op;
64 	int i, ret = 0;
65 
66 	for (i = 0; i < end && !ret; i++) {
67 		op = &hvc->ops[i];
68 		switch (op->type) {
69 		case MMAP:
70 			ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len,
71 				  op->u.mmap.prot, op->u.mmap.fd,
72 				  op->u.mmap.offset, finished, &hvc->data);
73 			break;
74 		case MUNMAP:
75 			ret = unmap(hvc->id, op->u.munmap.addr,
76 				    op->u.munmap.len, finished, &hvc->data);
77 			break;
78 		case MPROTECT:
79 			ret = protect(hvc->id, op->u.mprotect.addr,
80 				      op->u.mprotect.len, op->u.mprotect.prot,
81 				      finished, &hvc->data);
82 			break;
83 		default:
84 			printk(KERN_ERR "Unknown op type %d in do_ops\n",
85 			       op->type);
86 			BUG();
87 			break;
88 		}
89 	}
90 
91 	if (ret == -ENOMEM)
92 		report_enomem();
93 
94 	return ret;
95 }
96 
97 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
98 		    unsigned int prot, struct host_vm_change *hvc)
99 {
100 	__u64 offset;
101 	struct host_vm_op *last;
102 	int fd, ret = 0;
103 
104 	fd = phys_mapping(phys, &offset);
105 	if (hvc->index != 0) {
106 		last = &hvc->ops[hvc->index - 1];
107 		if ((last->type == MMAP) &&
108 		   (last->u.mmap.addr + last->u.mmap.len == virt) &&
109 		   (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
110 		   (last->u.mmap.offset + last->u.mmap.len == offset)) {
111 			last->u.mmap.len += len;
112 			return 0;
113 		}
114 	}
115 
116 	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
117 		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
118 		hvc->index = 0;
119 	}
120 
121 	hvc->ops[hvc->index++] = ((struct host_vm_op)
122 				  { .type	= MMAP,
123 				    .u = { .mmap = { .addr	= virt,
124 						     .len	= len,
125 						     .prot	= prot,
126 						     .fd	= fd,
127 						     .offset	= offset }
128 			   } });
129 	return ret;
130 }
131 
132 static int add_munmap(unsigned long addr, unsigned long len,
133 		      struct host_vm_change *hvc)
134 {
135 	struct host_vm_op *last;
136 	int ret = 0;
137 
138 	if ((addr >= STUB_START) && (addr < STUB_END))
139 		return -EINVAL;
140 
141 	if (hvc->index != 0) {
142 		last = &hvc->ops[hvc->index - 1];
143 		if ((last->type == MUNMAP) &&
144 		   (last->u.munmap.addr + last->u.mmap.len == addr)) {
145 			last->u.munmap.len += len;
146 			return 0;
147 		}
148 	}
149 
150 	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
151 		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
152 		hvc->index = 0;
153 	}
154 
155 	hvc->ops[hvc->index++] = ((struct host_vm_op)
156 				  { .type	= MUNMAP,
157 			     	    .u = { .munmap = { .addr	= addr,
158 						       .len	= len } } });
159 	return ret;
160 }
161 
162 static int add_mprotect(unsigned long addr, unsigned long len,
163 			unsigned int prot, struct host_vm_change *hvc)
164 {
165 	struct host_vm_op *last;
166 	int ret = 0;
167 
168 	if (hvc->index != 0) {
169 		last = &hvc->ops[hvc->index - 1];
170 		if ((last->type == MPROTECT) &&
171 		   (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
172 		   (last->u.mprotect.prot == prot)) {
173 			last->u.mprotect.len += len;
174 			return 0;
175 		}
176 	}
177 
178 	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
179 		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
180 		hvc->index = 0;
181 	}
182 
183 	hvc->ops[hvc->index++] = ((struct host_vm_op)
184 				  { .type	= MPROTECT,
185 			     	    .u = { .mprotect = { .addr	= addr,
186 							 .len	= len,
187 							 .prot	= prot } } });
188 	return ret;
189 }
190 
191 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
192 
193 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
194 				   unsigned long end,
195 				   struct host_vm_change *hvc)
196 {
197 	pte_t *pte;
198 	int r, w, x, prot, ret = 0;
199 
200 	pte = pte_offset_kernel(pmd, addr);
201 	do {
202 		if ((addr >= STUB_START) && (addr < STUB_END))
203 			continue;
204 
205 		r = pte_read(*pte);
206 		w = pte_write(*pte);
207 		x = pte_exec(*pte);
208 		if (!pte_young(*pte)) {
209 			r = 0;
210 			w = 0;
211 		} else if (!pte_dirty(*pte))
212 			w = 0;
213 
214 		prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
215 			(x ? UM_PROT_EXEC : 0));
216 		if (hvc->force || pte_newpage(*pte)) {
217 			if (pte_present(*pte))
218 				ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
219 					       PAGE_SIZE, prot, hvc);
220 			else
221 				ret = add_munmap(addr, PAGE_SIZE, hvc);
222 		} else if (pte_newprot(*pte))
223 			ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
224 		*pte = pte_mkuptodate(*pte);
225 	} while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
226 	return ret;
227 }
228 
229 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
230 				   unsigned long end,
231 				   struct host_vm_change *hvc)
232 {
233 	pmd_t *pmd;
234 	unsigned long next;
235 	int ret = 0;
236 
237 	pmd = pmd_offset(pud, addr);
238 	do {
239 		next = pmd_addr_end(addr, end);
240 		if (!pmd_present(*pmd)) {
241 			if (hvc->force || pmd_newpage(*pmd)) {
242 				ret = add_munmap(addr, next - addr, hvc);
243 				pmd_mkuptodate(*pmd);
244 			}
245 		}
246 		else ret = update_pte_range(pmd, addr, next, hvc);
247 	} while (pmd++, addr = next, ((addr < end) && !ret));
248 	return ret;
249 }
250 
251 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
252 				   unsigned long end,
253 				   struct host_vm_change *hvc)
254 {
255 	pud_t *pud;
256 	unsigned long next;
257 	int ret = 0;
258 
259 	pud = pud_offset(pgd, addr);
260 	do {
261 		next = pud_addr_end(addr, end);
262 		if (!pud_present(*pud)) {
263 			if (hvc->force || pud_newpage(*pud)) {
264 				ret = add_munmap(addr, next - addr, hvc);
265 				pud_mkuptodate(*pud);
266 			}
267 		}
268 		else ret = update_pmd_range(pud, addr, next, hvc);
269 	} while (pud++, addr = next, ((addr < end) && !ret));
270 	return ret;
271 }
272 
273 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
274 		      unsigned long end_addr, int force)
275 {
276 	pgd_t *pgd;
277 	struct host_vm_change hvc;
278 	unsigned long addr = start_addr, next;
279 	int ret = 0;
280 
281 	hvc = INIT_HVC(mm, force);
282 	pgd = pgd_offset(mm, addr);
283 	do {
284 		next = pgd_addr_end(addr, end_addr);
285 		if (!pgd_present(*pgd)) {
286 			if (force || pgd_newpage(*pgd)) {
287 				ret = add_munmap(addr, next - addr, &hvc);
288 				pgd_mkuptodate(*pgd);
289 			}
290 		}
291 		else ret = update_pud_range(pgd, addr, next, &hvc);
292 	} while (pgd++, addr = next, ((addr < end_addr) && !ret));
293 
294 	if (!ret)
295 		ret = do_ops(&hvc, hvc.index, 1);
296 
297 	/* This is not an else because ret is modified above */
298 	if (ret) {
299 		printk(KERN_ERR "fix_range_common: failed, killing current "
300 		       "process: %d\n", task_tgid_vnr(current));
301 		/* We are under mmap_sem, release it such that current can terminate */
302 		up_write(&current->mm->mmap_sem);
303 		force_sig(SIGKILL, current);
304 		do_signal(&current->thread.regs);
305 	}
306 }
307 
308 static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
309 {
310 	struct mm_struct *mm;
311 	pgd_t *pgd;
312 	pud_t *pud;
313 	pmd_t *pmd;
314 	pte_t *pte;
315 	unsigned long addr, last;
316 	int updated = 0, err;
317 
318 	mm = &init_mm;
319 	for (addr = start; addr < end;) {
320 		pgd = pgd_offset(mm, addr);
321 		if (!pgd_present(*pgd)) {
322 			last = ADD_ROUND(addr, PGDIR_SIZE);
323 			if (last > end)
324 				last = end;
325 			if (pgd_newpage(*pgd)) {
326 				updated = 1;
327 				err = os_unmap_memory((void *) addr,
328 						      last - addr);
329 				if (err < 0)
330 					panic("munmap failed, errno = %d\n",
331 					      -err);
332 			}
333 			addr = last;
334 			continue;
335 		}
336 
337 		pud = pud_offset(pgd, addr);
338 		if (!pud_present(*pud)) {
339 			last = ADD_ROUND(addr, PUD_SIZE);
340 			if (last > end)
341 				last = end;
342 			if (pud_newpage(*pud)) {
343 				updated = 1;
344 				err = os_unmap_memory((void *) addr,
345 						      last - addr);
346 				if (err < 0)
347 					panic("munmap failed, errno = %d\n",
348 					      -err);
349 			}
350 			addr = last;
351 			continue;
352 		}
353 
354 		pmd = pmd_offset(pud, addr);
355 		if (!pmd_present(*pmd)) {
356 			last = ADD_ROUND(addr, PMD_SIZE);
357 			if (last > end)
358 				last = end;
359 			if (pmd_newpage(*pmd)) {
360 				updated = 1;
361 				err = os_unmap_memory((void *) addr,
362 						      last - addr);
363 				if (err < 0)
364 					panic("munmap failed, errno = %d\n",
365 					      -err);
366 			}
367 			addr = last;
368 			continue;
369 		}
370 
371 		pte = pte_offset_kernel(pmd, addr);
372 		if (!pte_present(*pte) || pte_newpage(*pte)) {
373 			updated = 1;
374 			err = os_unmap_memory((void *) addr,
375 					      PAGE_SIZE);
376 			if (err < 0)
377 				panic("munmap failed, errno = %d\n",
378 				      -err);
379 			if (pte_present(*pte))
380 				map_memory(addr,
381 					   pte_val(*pte) & PAGE_MASK,
382 					   PAGE_SIZE, 1, 1, 1);
383 		}
384 		else if (pte_newprot(*pte)) {
385 			updated = 1;
386 			os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
387 		}
388 		addr += PAGE_SIZE;
389 	}
390 	return updated;
391 }
392 
393 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
394 {
395 	pgd_t *pgd;
396 	pud_t *pud;
397 	pmd_t *pmd;
398 	pte_t *pte;
399 	struct mm_struct *mm = vma->vm_mm;
400 	void *flush = NULL;
401 	int r, w, x, prot, err = 0;
402 	struct mm_id *mm_id;
403 
404 	address &= PAGE_MASK;
405 	pgd = pgd_offset(mm, address);
406 	if (!pgd_present(*pgd))
407 		goto kill;
408 
409 	pud = pud_offset(pgd, address);
410 	if (!pud_present(*pud))
411 		goto kill;
412 
413 	pmd = pmd_offset(pud, address);
414 	if (!pmd_present(*pmd))
415 		goto kill;
416 
417 	pte = pte_offset_kernel(pmd, address);
418 
419 	r = pte_read(*pte);
420 	w = pte_write(*pte);
421 	x = pte_exec(*pte);
422 	if (!pte_young(*pte)) {
423 		r = 0;
424 		w = 0;
425 	} else if (!pte_dirty(*pte)) {
426 		w = 0;
427 	}
428 
429 	mm_id = &mm->context.id;
430 	prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
431 		(x ? UM_PROT_EXEC : 0));
432 	if (pte_newpage(*pte)) {
433 		if (pte_present(*pte)) {
434 			unsigned long long offset;
435 			int fd;
436 
437 			fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
438 			err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
439 				  1, &flush);
440 		}
441 		else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
442 	}
443 	else if (pte_newprot(*pte))
444 		err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
445 
446 	if (err) {
447 		if (err == -ENOMEM)
448 			report_enomem();
449 
450 		goto kill;
451 	}
452 
453 	*pte = pte_mkuptodate(*pte);
454 
455 	return;
456 
457 kill:
458 	printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
459 	force_sig(SIGKILL, current);
460 }
461 
462 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
463 {
464 	return pgd_offset(mm, address);
465 }
466 
467 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
468 {
469 	return pud_offset(pgd, address);
470 }
471 
472 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
473 {
474 	return pmd_offset(pud, address);
475 }
476 
477 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
478 {
479 	return pte_offset_kernel(pmd, address);
480 }
481 
482 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
483 {
484 	pgd_t *pgd = pgd_offset(task->mm, addr);
485 	pud_t *pud = pud_offset(pgd, addr);
486 	pmd_t *pmd = pmd_offset(pud, addr);
487 
488 	return pte_offset_map(pmd, addr);
489 }
490 
491 void flush_tlb_all(void)
492 {
493 	flush_tlb_mm(current->mm);
494 }
495 
496 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
497 {
498 	flush_tlb_kernel_range_common(start, end);
499 }
500 
501 void flush_tlb_kernel_vm(void)
502 {
503 	flush_tlb_kernel_range_common(start_vm, end_vm);
504 }
505 
506 void __flush_tlb_one(unsigned long addr)
507 {
508 	flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
509 }
510 
511 static void fix_range(struct mm_struct *mm, unsigned long start_addr,
512 		      unsigned long end_addr, int force)
513 {
514 	fix_range_common(mm, start_addr, end_addr, force);
515 }
516 
517 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
518 		     unsigned long end)
519 {
520 	if (vma->vm_mm == NULL)
521 		flush_tlb_kernel_range_common(start, end);
522 	else fix_range(vma->vm_mm, start, end, 0);
523 }
524 EXPORT_SYMBOL(flush_tlb_range);
525 
526 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
527 			unsigned long end)
528 {
529 	/*
530 	 * Don't bother flushing if this address space is about to be
531 	 * destroyed.
532 	 */
533 	if (atomic_read(&mm->mm_users) == 0)
534 		return;
535 
536 	fix_range(mm, start, end, 0);
537 }
538 
539 void flush_tlb_mm(struct mm_struct *mm)
540 {
541 	struct vm_area_struct *vma = mm->mmap;
542 
543 	while (vma != NULL) {
544 		fix_range(mm, vma->vm_start, vma->vm_end, 0);
545 		vma = vma->vm_next;
546 	}
547 }
548 
549 void force_flush_all(void)
550 {
551 	struct mm_struct *mm = current->mm;
552 	struct vm_area_struct *vma = mm->mmap;
553 
554 	while (vma != NULL) {
555 		fix_range(mm, vma->vm_start, vma->vm_end, 1);
556 		vma = vma->vm_next;
557 	}
558 }
559