1 /* 2 * mmap support for qemu 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #include "qemu/osdep.h" 20 #include <sys/shm.h> 21 #include "trace.h" 22 #include "exec/log.h" 23 #include "qemu.h" 24 #include "user-internals.h" 25 #include "user-mmap.h" 26 #include "target_mman.h" 27 #include "qemu/interval-tree.h" 28 29 static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER; 30 static __thread int mmap_lock_count; 31 32 void mmap_lock(void) 33 { 34 if (mmap_lock_count++ == 0) { 35 pthread_mutex_lock(&mmap_mutex); 36 } 37 } 38 39 void mmap_unlock(void) 40 { 41 assert(mmap_lock_count > 0); 42 if (--mmap_lock_count == 0) { 43 pthread_mutex_unlock(&mmap_mutex); 44 } 45 } 46 47 bool have_mmap_lock(void) 48 { 49 return mmap_lock_count > 0 ? true : false; 50 } 51 52 /* Grab lock to make sure things are in a consistent state after fork(). */ 53 void mmap_fork_start(void) 54 { 55 if (mmap_lock_count) 56 abort(); 57 pthread_mutex_lock(&mmap_mutex); 58 } 59 60 void mmap_fork_end(int child) 61 { 62 if (child) { 63 pthread_mutex_init(&mmap_mutex, NULL); 64 } else { 65 pthread_mutex_unlock(&mmap_mutex); 66 } 67 } 68 69 /* Protected by mmap_lock. */ 70 static IntervalTreeRoot shm_regions; 71 72 static void shm_region_add(abi_ptr start, abi_ptr last) 73 { 74 IntervalTreeNode *i = g_new0(IntervalTreeNode, 1); 75 76 i->start = start; 77 i->last = last; 78 interval_tree_insert(i, &shm_regions); 79 } 80 81 static abi_ptr shm_region_find(abi_ptr start) 82 { 83 IntervalTreeNode *i; 84 85 for (i = interval_tree_iter_first(&shm_regions, start, start); i; 86 i = interval_tree_iter_next(i, start, start)) { 87 if (i->start == start) { 88 return i->last; 89 } 90 } 91 return 0; 92 } 93 94 static void shm_region_rm_complete(abi_ptr start, abi_ptr last) 95 { 96 IntervalTreeNode *i, *n; 97 98 for (i = interval_tree_iter_first(&shm_regions, start, last); i; i = n) { 99 n = interval_tree_iter_next(i, start, last); 100 if (i->start >= start && i->last <= last) { 101 interval_tree_remove(i, &shm_regions); 102 g_free(i); 103 } 104 } 105 } 106 107 /* 108 * Validate target prot bitmask. 109 * Return the prot bitmask for the host in *HOST_PROT. 110 * Return 0 if the target prot bitmask is invalid, otherwise 111 * the internal qemu page_flags (which will include PAGE_VALID). 112 */ 113 static int validate_prot_to_pageflags(int prot) 114 { 115 int valid = PROT_READ | PROT_WRITE | PROT_EXEC | TARGET_PROT_SEM; 116 int page_flags = (prot & PAGE_BITS) | PAGE_VALID; 117 118 #ifdef TARGET_AARCH64 119 { 120 ARMCPU *cpu = ARM_CPU(thread_cpu); 121 122 /* 123 * The PROT_BTI bit is only accepted if the cpu supports the feature. 124 * Since this is the unusual case, don't bother checking unless 125 * the bit has been requested. If set and valid, record the bit 126 * within QEMU's page_flags. 127 */ 128 if ((prot & TARGET_PROT_BTI) && cpu_isar_feature(aa64_bti, cpu)) { 129 valid |= TARGET_PROT_BTI; 130 page_flags |= PAGE_BTI; 131 } 132 /* Similarly for the PROT_MTE bit. */ 133 if ((prot & TARGET_PROT_MTE) && cpu_isar_feature(aa64_mte, cpu)) { 134 valid |= TARGET_PROT_MTE; 135 page_flags |= PAGE_MTE; 136 } 137 } 138 #elif defined(TARGET_HPPA) 139 valid |= PROT_GROWSDOWN | PROT_GROWSUP; 140 #endif 141 142 return prot & ~valid ? 0 : page_flags; 143 } 144 145 /* 146 * For the host, we need not pass anything except read/write/exec. 147 * While PROT_SEM is allowed by all hosts, it is also ignored, so 148 * don't bother transforming guest bit to host bit. Any other 149 * target-specific prot bits will not be understood by the host 150 * and will need to be encoded into page_flags for qemu emulation. 151 * 152 * Pages that are executable by the guest will never be executed 153 * by the host, but the host will need to be able to read them. 154 */ 155 static int target_to_host_prot(int prot) 156 { 157 return (prot & (PROT_READ | PROT_WRITE)) | 158 (prot & PROT_EXEC ? PROT_READ : 0); 159 } 160 161 /* NOTE: all the constants are the HOST ones, but addresses are target. */ 162 int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) 163 { 164 abi_ulong starts[3]; 165 abi_ulong lens[3]; 166 int prots[3]; 167 abi_ulong host_start, host_last, last; 168 int prot1, ret, page_flags, nranges; 169 170 trace_target_mprotect(start, len, target_prot); 171 172 if ((start & ~TARGET_PAGE_MASK) != 0) { 173 return -TARGET_EINVAL; 174 } 175 page_flags = validate_prot_to_pageflags(target_prot); 176 if (!page_flags) { 177 return -TARGET_EINVAL; 178 } 179 if (len == 0) { 180 return 0; 181 } 182 len = TARGET_PAGE_ALIGN(len); 183 if (!guest_range_valid_untagged(start, len)) { 184 return -TARGET_ENOMEM; 185 } 186 187 last = start + len - 1; 188 host_start = start & qemu_host_page_mask; 189 host_last = HOST_PAGE_ALIGN(last) - 1; 190 nranges = 0; 191 192 mmap_lock(); 193 194 if (host_last - host_start < qemu_host_page_size) { 195 /* Single host page contains all guest pages: sum the prot. */ 196 prot1 = target_prot; 197 for (abi_ulong a = host_start; a < start; a += TARGET_PAGE_SIZE) { 198 prot1 |= page_get_flags(a); 199 } 200 for (abi_ulong a = last; a < host_last; a += TARGET_PAGE_SIZE) { 201 prot1 |= page_get_flags(a + 1); 202 } 203 starts[nranges] = host_start; 204 lens[nranges] = qemu_host_page_size; 205 prots[nranges] = prot1; 206 nranges++; 207 } else { 208 if (host_start < start) { 209 /* Host page contains more than one guest page: sum the prot. */ 210 prot1 = target_prot; 211 for (abi_ulong a = host_start; a < start; a += TARGET_PAGE_SIZE) { 212 prot1 |= page_get_flags(a); 213 } 214 /* If the resulting sum differs, create a new range. */ 215 if (prot1 != target_prot) { 216 starts[nranges] = host_start; 217 lens[nranges] = qemu_host_page_size; 218 prots[nranges] = prot1; 219 nranges++; 220 host_start += qemu_host_page_size; 221 } 222 } 223 224 if (last < host_last) { 225 /* Host page contains more than one guest page: sum the prot. */ 226 prot1 = target_prot; 227 for (abi_ulong a = last; a < host_last; a += TARGET_PAGE_SIZE) { 228 prot1 |= page_get_flags(a + 1); 229 } 230 /* If the resulting sum differs, create a new range. */ 231 if (prot1 != target_prot) { 232 host_last -= qemu_host_page_size; 233 starts[nranges] = host_last + 1; 234 lens[nranges] = qemu_host_page_size; 235 prots[nranges] = prot1; 236 nranges++; 237 } 238 } 239 240 /* Create a range for the middle, if any remains. */ 241 if (host_start < host_last) { 242 starts[nranges] = host_start; 243 lens[nranges] = host_last - host_start + 1; 244 prots[nranges] = target_prot; 245 nranges++; 246 } 247 } 248 249 for (int i = 0; i < nranges; ++i) { 250 ret = mprotect(g2h_untagged(starts[i]), lens[i], 251 target_to_host_prot(prots[i])); 252 if (ret != 0) { 253 goto error; 254 } 255 } 256 257 page_set_flags(start, last, page_flags); 258 ret = 0; 259 260 error: 261 mmap_unlock(); 262 return ret; 263 } 264 265 /* map an incomplete host page */ 266 static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, 267 int prot, int flags, int fd, off_t offset) 268 { 269 abi_ulong real_last; 270 void *host_start; 271 int prot_old, prot_new; 272 int host_prot_old, host_prot_new; 273 274 if (!(flags & MAP_ANONYMOUS) 275 && (flags & MAP_TYPE) == MAP_SHARED 276 && (prot & PROT_WRITE)) { 277 /* 278 * msync() won't work with the partial page, so we return an 279 * error if write is possible while it is a shared mapping. 280 */ 281 errno = EINVAL; 282 return false; 283 } 284 285 real_last = real_start + qemu_host_page_size - 1; 286 host_start = g2h_untagged(real_start); 287 288 /* Get the protection of the target pages outside the mapping. */ 289 prot_old = 0; 290 for (abi_ulong a = real_start; a < start; a += TARGET_PAGE_SIZE) { 291 prot_old |= page_get_flags(a); 292 } 293 for (abi_ulong a = real_last; a > last; a -= TARGET_PAGE_SIZE) { 294 prot_old |= page_get_flags(a); 295 } 296 297 if (prot_old == 0) { 298 /* 299 * Since !(prot_old & PAGE_VALID), there were no guest pages 300 * outside of the fragment we need to map. Allocate a new host 301 * page to cover, discarding whatever else may have been present. 302 */ 303 void *p = mmap(host_start, qemu_host_page_size, 304 target_to_host_prot(prot), 305 flags | MAP_ANONYMOUS, -1, 0); 306 if (p != host_start) { 307 if (p != MAP_FAILED) { 308 munmap(p, qemu_host_page_size); 309 errno = EEXIST; 310 } 311 return false; 312 } 313 prot_old = prot; 314 } 315 prot_new = prot | prot_old; 316 317 host_prot_old = target_to_host_prot(prot_old); 318 host_prot_new = target_to_host_prot(prot_new); 319 320 /* Adjust protection to be able to write. */ 321 if (!(host_prot_old & PROT_WRITE)) { 322 host_prot_old |= PROT_WRITE; 323 mprotect(host_start, qemu_host_page_size, host_prot_old); 324 } 325 326 /* Read or zero the new guest pages. */ 327 if (flags & MAP_ANONYMOUS) { 328 memset(g2h_untagged(start), 0, last - start + 1); 329 } else { 330 if (pread(fd, g2h_untagged(start), last - start + 1, offset) == -1) { 331 return false; 332 } 333 } 334 335 /* Put final protection */ 336 if (host_prot_new != host_prot_old) { 337 mprotect(host_start, qemu_host_page_size, host_prot_new); 338 } 339 return true; 340 } 341 342 abi_ulong task_unmapped_base; 343 abi_ulong elf_et_dyn_base; 344 abi_ulong mmap_next_start; 345 346 /* 347 * Subroutine of mmap_find_vma, used when we have pre-allocated 348 * a chunk of guest address space. 349 */ 350 static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size, 351 abi_ulong align) 352 { 353 target_ulong ret; 354 355 ret = page_find_range_empty(start, reserved_va, size, align); 356 if (ret == -1 && start > mmap_min_addr) { 357 /* Restart at the beginning of the address space. */ 358 ret = page_find_range_empty(mmap_min_addr, start - 1, size, align); 359 } 360 361 return ret; 362 } 363 364 /* 365 * Find and reserve a free memory area of size 'size'. The search 366 * starts at 'start'. 367 * It must be called with mmap_lock() held. 368 * Return -1 if error. 369 */ 370 abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size, abi_ulong align) 371 { 372 void *ptr, *prev; 373 abi_ulong addr; 374 int wrapped, repeat; 375 376 align = MAX(align, qemu_host_page_size); 377 378 /* If 'start' == 0, then a default start address is used. */ 379 if (start == 0) { 380 start = mmap_next_start; 381 } else { 382 start &= qemu_host_page_mask; 383 } 384 start = ROUND_UP(start, align); 385 386 size = HOST_PAGE_ALIGN(size); 387 388 if (reserved_va) { 389 return mmap_find_vma_reserved(start, size, align); 390 } 391 392 addr = start; 393 wrapped = repeat = 0; 394 prev = 0; 395 396 for (;; prev = ptr) { 397 /* 398 * Reserve needed memory area to avoid a race. 399 * It should be discarded using: 400 * - mmap() with MAP_FIXED flag 401 * - mremap() with MREMAP_FIXED flag 402 * - shmat() with SHM_REMAP flag 403 */ 404 ptr = mmap(g2h_untagged(addr), size, PROT_NONE, 405 MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); 406 407 /* ENOMEM, if host address space has no memory */ 408 if (ptr == MAP_FAILED) { 409 return (abi_ulong)-1; 410 } 411 412 /* 413 * Count the number of sequential returns of the same address. 414 * This is used to modify the search algorithm below. 415 */ 416 repeat = (ptr == prev ? repeat + 1 : 0); 417 418 if (h2g_valid(ptr + size - 1)) { 419 addr = h2g(ptr); 420 421 if ((addr & (align - 1)) == 0) { 422 /* Success. */ 423 if (start == mmap_next_start && addr >= task_unmapped_base) { 424 mmap_next_start = addr + size; 425 } 426 return addr; 427 } 428 429 /* The address is not properly aligned for the target. */ 430 switch (repeat) { 431 case 0: 432 /* 433 * Assume the result that the kernel gave us is the 434 * first with enough free space, so start again at the 435 * next higher target page. 436 */ 437 addr = ROUND_UP(addr, align); 438 break; 439 case 1: 440 /* 441 * Sometimes the kernel decides to perform the allocation 442 * at the top end of memory instead. 443 */ 444 addr &= -align; 445 break; 446 case 2: 447 /* Start over at low memory. */ 448 addr = 0; 449 break; 450 default: 451 /* Fail. This unaligned block must the last. */ 452 addr = -1; 453 break; 454 } 455 } else { 456 /* 457 * Since the result the kernel gave didn't fit, start 458 * again at low memory. If any repetition, fail. 459 */ 460 addr = (repeat ? -1 : 0); 461 } 462 463 /* Unmap and try again. */ 464 munmap(ptr, size); 465 466 /* ENOMEM if we checked the whole of the target address space. */ 467 if (addr == (abi_ulong)-1) { 468 return (abi_ulong)-1; 469 } else if (addr == 0) { 470 if (wrapped) { 471 return (abi_ulong)-1; 472 } 473 wrapped = 1; 474 /* 475 * Don't actually use 0 when wrapping, instead indicate 476 * that we'd truly like an allocation in low memory. 477 */ 478 addr = (mmap_min_addr > TARGET_PAGE_SIZE 479 ? TARGET_PAGE_ALIGN(mmap_min_addr) 480 : TARGET_PAGE_SIZE); 481 } else if (wrapped && addr >= start) { 482 return (abi_ulong)-1; 483 } 484 } 485 } 486 487 /* NOTE: all the constants are the HOST ones */ 488 abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot, 489 int flags, int fd, off_t offset) 490 { 491 abi_ulong ret, last, real_start, real_last, retaddr, host_len; 492 abi_ulong passthrough_start = -1, passthrough_last = 0; 493 int page_flags; 494 off_t host_offset; 495 496 mmap_lock(); 497 trace_target_mmap(start, len, target_prot, flags, fd, offset); 498 499 if (!len) { 500 errno = EINVAL; 501 goto fail; 502 } 503 504 page_flags = validate_prot_to_pageflags(target_prot); 505 if (!page_flags) { 506 errno = EINVAL; 507 goto fail; 508 } 509 510 /* Also check for overflows... */ 511 len = TARGET_PAGE_ALIGN(len); 512 if (!len) { 513 errno = ENOMEM; 514 goto fail; 515 } 516 517 if (offset & ~TARGET_PAGE_MASK) { 518 errno = EINVAL; 519 goto fail; 520 } 521 522 /* 523 * If we're mapping shared memory, ensure we generate code for parallel 524 * execution and flush old translations. This will work up to the level 525 * supported by the host -- anything that requires EXCP_ATOMIC will not 526 * be atomic with respect to an external process. 527 */ 528 if (flags & MAP_SHARED) { 529 CPUState *cpu = thread_cpu; 530 if (!(cpu->tcg_cflags & CF_PARALLEL)) { 531 cpu->tcg_cflags |= CF_PARALLEL; 532 tb_flush(cpu); 533 } 534 } 535 536 real_start = start & qemu_host_page_mask; 537 host_offset = offset & qemu_host_page_mask; 538 539 /* 540 * If the user is asking for the kernel to find a location, do that 541 * before we truncate the length for mapping files below. 542 */ 543 if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { 544 host_len = len + offset - host_offset; 545 host_len = HOST_PAGE_ALIGN(host_len); 546 start = mmap_find_vma(real_start, host_len, TARGET_PAGE_SIZE); 547 if (start == (abi_ulong)-1) { 548 errno = ENOMEM; 549 goto fail; 550 } 551 } 552 553 /* 554 * When mapping files into a memory area larger than the file, accesses 555 * to pages beyond the file size will cause a SIGBUS. 556 * 557 * For example, if mmaping a file of 100 bytes on a host with 4K pages 558 * emulating a target with 8K pages, the target expects to be able to 559 * access the first 8K. But the host will trap us on any access beyond 560 * 4K. 561 * 562 * When emulating a target with a larger page-size than the hosts, we 563 * may need to truncate file maps at EOF and add extra anonymous pages 564 * up to the targets page boundary. 565 */ 566 if ((qemu_real_host_page_size() < qemu_host_page_size) && 567 !(flags & MAP_ANONYMOUS)) { 568 struct stat sb; 569 570 if (fstat(fd, &sb) == -1) { 571 goto fail; 572 } 573 574 /* Are we trying to create a map beyond EOF?. */ 575 if (offset + len > sb.st_size) { 576 /* 577 * If so, truncate the file map at eof aligned with 578 * the hosts real pagesize. Additional anonymous maps 579 * will be created beyond EOF. 580 */ 581 len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset); 582 } 583 } 584 585 if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { 586 uintptr_t host_start; 587 int host_prot; 588 void *p; 589 590 host_len = len + offset - host_offset; 591 host_len = HOST_PAGE_ALIGN(host_len); 592 host_prot = target_to_host_prot(target_prot); 593 594 /* 595 * Note: we prefer to control the mapping address. It is 596 * especially important if qemu_host_page_size > 597 * qemu_real_host_page_size. 598 */ 599 p = mmap(g2h_untagged(start), host_len, host_prot, 600 flags | MAP_FIXED | MAP_ANONYMOUS, -1, 0); 601 if (p == MAP_FAILED) { 602 goto fail; 603 } 604 /* update start so that it points to the file position at 'offset' */ 605 host_start = (uintptr_t)p; 606 if (!(flags & MAP_ANONYMOUS)) { 607 p = mmap(g2h_untagged(start), len, host_prot, 608 flags | MAP_FIXED, fd, host_offset); 609 if (p == MAP_FAILED) { 610 munmap(g2h_untagged(start), host_len); 611 goto fail; 612 } 613 host_start += offset - host_offset; 614 } 615 start = h2g(host_start); 616 last = start + len - 1; 617 passthrough_start = start; 618 passthrough_last = last; 619 } else { 620 if (start & ~TARGET_PAGE_MASK) { 621 errno = EINVAL; 622 goto fail; 623 } 624 last = start + len - 1; 625 real_last = HOST_PAGE_ALIGN(last) - 1; 626 627 /* 628 * Test if requested memory area fits target address space 629 * It can fail only on 64-bit host with 32-bit target. 630 * On any other target/host host mmap() handles this error correctly. 631 */ 632 if (last < start || !guest_range_valid_untagged(start, len)) { 633 errno = ENOMEM; 634 goto fail; 635 } 636 637 if (flags & MAP_FIXED_NOREPLACE) { 638 /* Validate that the chosen range is empty. */ 639 if (!page_check_range_empty(start, last)) { 640 errno = EEXIST; 641 goto fail; 642 } 643 644 /* 645 * With reserved_va, the entire address space is mmaped in the 646 * host to ensure it isn't accidentally used for something else. 647 * We have just checked that the guest address is not mapped 648 * within the guest, but need to replace the host reservation. 649 * 650 * Without reserved_va, despite the guest address check above, 651 * keep MAP_FIXED_NOREPLACE so that the guest does not overwrite 652 * any host address mappings. 653 */ 654 if (reserved_va) { 655 flags = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED; 656 } 657 } 658 659 /* 660 * worst case: we cannot map the file because the offset is not 661 * aligned, so we read it 662 */ 663 if (!(flags & MAP_ANONYMOUS) && 664 (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) { 665 /* 666 * msync() won't work here, so we return an error if write is 667 * possible while it is a shared mapping 668 */ 669 if ((flags & MAP_TYPE) == MAP_SHARED 670 && (target_prot & PROT_WRITE)) { 671 errno = EINVAL; 672 goto fail; 673 } 674 retaddr = target_mmap(start, len, target_prot | PROT_WRITE, 675 (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) 676 | MAP_PRIVATE | MAP_ANONYMOUS, 677 -1, 0); 678 if (retaddr == -1) { 679 goto fail; 680 } 681 if (pread(fd, g2h_untagged(start), len, offset) == -1) { 682 goto fail; 683 } 684 if (!(target_prot & PROT_WRITE)) { 685 ret = target_mprotect(start, len, target_prot); 686 assert(ret == 0); 687 } 688 goto the_end; 689 } 690 691 /* handle the start of the mapping */ 692 if (start > real_start) { 693 if (real_last == real_start + qemu_host_page_size - 1) { 694 /* one single host page */ 695 if (!mmap_frag(real_start, start, last, 696 target_prot, flags, fd, offset)) { 697 goto fail; 698 } 699 goto the_end1; 700 } 701 if (!mmap_frag(real_start, start, 702 real_start + qemu_host_page_size - 1, 703 target_prot, flags, fd, offset)) { 704 goto fail; 705 } 706 real_start += qemu_host_page_size; 707 } 708 /* handle the end of the mapping */ 709 if (last < real_last) { 710 abi_ulong real_page = real_last - qemu_host_page_size + 1; 711 if (!mmap_frag(real_page, real_page, last, 712 target_prot, flags, fd, 713 offset + real_page - start)) { 714 goto fail; 715 } 716 real_last -= qemu_host_page_size; 717 } 718 719 /* map the middle (easier) */ 720 if (real_start < real_last) { 721 void *p, *want_p; 722 off_t offset1; 723 size_t len1; 724 725 if (flags & MAP_ANONYMOUS) { 726 offset1 = 0; 727 } else { 728 offset1 = offset + real_start - start; 729 } 730 len1 = real_last - real_start + 1; 731 want_p = g2h_untagged(real_start); 732 733 p = mmap(want_p, len1, target_to_host_prot(target_prot), 734 flags, fd, offset1); 735 if (p != want_p) { 736 if (p != MAP_FAILED) { 737 munmap(p, len1); 738 errno = EEXIST; 739 } 740 goto fail; 741 } 742 passthrough_start = real_start; 743 passthrough_last = real_last; 744 } 745 } 746 the_end1: 747 if (flags & MAP_ANONYMOUS) { 748 page_flags |= PAGE_ANON; 749 } 750 page_flags |= PAGE_RESET; 751 if (passthrough_start > passthrough_last) { 752 page_set_flags(start, last, page_flags); 753 } else { 754 if (start < passthrough_start) { 755 page_set_flags(start, passthrough_start - 1, page_flags); 756 } 757 page_set_flags(passthrough_start, passthrough_last, 758 page_flags | PAGE_PASSTHROUGH); 759 if (passthrough_last < last) { 760 page_set_flags(passthrough_last + 1, last, page_flags); 761 } 762 } 763 shm_region_rm_complete(start, last); 764 the_end: 765 trace_target_mmap_complete(start); 766 if (qemu_loglevel_mask(CPU_LOG_PAGE)) { 767 FILE *f = qemu_log_trylock(); 768 if (f) { 769 fprintf(f, "page layout changed following mmap\n"); 770 page_dump(f); 771 qemu_log_unlock(f); 772 } 773 } 774 mmap_unlock(); 775 return start; 776 fail: 777 mmap_unlock(); 778 return -1; 779 } 780 781 static void mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) 782 { 783 abi_ulong real_start; 784 abi_ulong real_last; 785 abi_ulong real_len; 786 abi_ulong last; 787 abi_ulong a; 788 void *host_start; 789 int prot; 790 791 last = start + len - 1; 792 real_start = start & qemu_host_page_mask; 793 real_last = HOST_PAGE_ALIGN(last) - 1; 794 795 /* 796 * If guest pages remain on the first or last host pages, 797 * adjust the deallocation to retain those guest pages. 798 * The single page special case is required for the last page, 799 * lest real_start overflow to zero. 800 */ 801 if (real_last - real_start < qemu_host_page_size) { 802 prot = 0; 803 for (a = real_start; a < start; a += TARGET_PAGE_SIZE) { 804 prot |= page_get_flags(a); 805 } 806 for (a = last; a < real_last; a += TARGET_PAGE_SIZE) { 807 prot |= page_get_flags(a + 1); 808 } 809 if (prot != 0) { 810 return; 811 } 812 } else { 813 for (prot = 0, a = real_start; a < start; a += TARGET_PAGE_SIZE) { 814 prot |= page_get_flags(a); 815 } 816 if (prot != 0) { 817 real_start += qemu_host_page_size; 818 } 819 820 for (prot = 0, a = last; a < real_last; a += TARGET_PAGE_SIZE) { 821 prot |= page_get_flags(a + 1); 822 } 823 if (prot != 0) { 824 real_last -= qemu_host_page_size; 825 } 826 827 if (real_last < real_start) { 828 return; 829 } 830 } 831 832 real_len = real_last - real_start + 1; 833 host_start = g2h_untagged(real_start); 834 835 if (reserved_va) { 836 void *ptr = mmap(host_start, real_len, PROT_NONE, 837 MAP_FIXED | MAP_ANONYMOUS 838 | MAP_PRIVATE | MAP_NORESERVE, -1, 0); 839 assert(ptr == host_start); 840 } else { 841 int ret = munmap(host_start, real_len); 842 assert(ret == 0); 843 } 844 } 845 846 int target_munmap(abi_ulong start, abi_ulong len) 847 { 848 trace_target_munmap(start, len); 849 850 if (start & ~TARGET_PAGE_MASK) { 851 return -TARGET_EINVAL; 852 } 853 len = TARGET_PAGE_ALIGN(len); 854 if (len == 0 || !guest_range_valid_untagged(start, len)) { 855 return -TARGET_EINVAL; 856 } 857 858 mmap_lock(); 859 mmap_reserve_or_unmap(start, len); 860 page_set_flags(start, start + len - 1, 0); 861 shm_region_rm_complete(start, start + len - 1); 862 mmap_unlock(); 863 864 return 0; 865 } 866 867 abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size, 868 abi_ulong new_size, unsigned long flags, 869 abi_ulong new_addr) 870 { 871 int prot; 872 void *host_addr; 873 874 if (!guest_range_valid_untagged(old_addr, old_size) || 875 ((flags & MREMAP_FIXED) && 876 !guest_range_valid_untagged(new_addr, new_size)) || 877 ((flags & MREMAP_MAYMOVE) == 0 && 878 !guest_range_valid_untagged(old_addr, new_size))) { 879 errno = ENOMEM; 880 return -1; 881 } 882 883 mmap_lock(); 884 885 if (flags & MREMAP_FIXED) { 886 host_addr = mremap(g2h_untagged(old_addr), old_size, new_size, 887 flags, g2h_untagged(new_addr)); 888 889 if (reserved_va && host_addr != MAP_FAILED) { 890 /* 891 * If new and old addresses overlap then the above mremap will 892 * already have failed with EINVAL. 893 */ 894 mmap_reserve_or_unmap(old_addr, old_size); 895 } 896 } else if (flags & MREMAP_MAYMOVE) { 897 abi_ulong mmap_start; 898 899 mmap_start = mmap_find_vma(0, new_size, TARGET_PAGE_SIZE); 900 901 if (mmap_start == -1) { 902 errno = ENOMEM; 903 host_addr = MAP_FAILED; 904 } else { 905 host_addr = mremap(g2h_untagged(old_addr), old_size, new_size, 906 flags | MREMAP_FIXED, 907 g2h_untagged(mmap_start)); 908 if (reserved_va) { 909 mmap_reserve_or_unmap(old_addr, old_size); 910 } 911 } 912 } else { 913 int prot = 0; 914 if (reserved_va && old_size < new_size) { 915 abi_ulong addr; 916 for (addr = old_addr + old_size; 917 addr < old_addr + new_size; 918 addr++) { 919 prot |= page_get_flags(addr); 920 } 921 } 922 if (prot == 0) { 923 host_addr = mremap(g2h_untagged(old_addr), 924 old_size, new_size, flags); 925 926 if (host_addr != MAP_FAILED) { 927 /* Check if address fits target address space */ 928 if (!guest_range_valid_untagged(h2g(host_addr), new_size)) { 929 /* Revert mremap() changes */ 930 host_addr = mremap(g2h_untagged(old_addr), 931 new_size, old_size, flags); 932 errno = ENOMEM; 933 host_addr = MAP_FAILED; 934 } else if (reserved_va && old_size > new_size) { 935 mmap_reserve_or_unmap(old_addr + old_size, 936 old_size - new_size); 937 } 938 } 939 } else { 940 errno = ENOMEM; 941 host_addr = MAP_FAILED; 942 } 943 } 944 945 if (host_addr == MAP_FAILED) { 946 new_addr = -1; 947 } else { 948 new_addr = h2g(host_addr); 949 prot = page_get_flags(old_addr); 950 page_set_flags(old_addr, old_addr + old_size - 1, 0); 951 shm_region_rm_complete(old_addr, old_addr + old_size - 1); 952 page_set_flags(new_addr, new_addr + new_size - 1, 953 prot | PAGE_VALID | PAGE_RESET); 954 shm_region_rm_complete(new_addr, new_addr + new_size - 1); 955 } 956 mmap_unlock(); 957 return new_addr; 958 } 959 960 abi_long target_madvise(abi_ulong start, abi_ulong len_in, int advice) 961 { 962 abi_ulong len; 963 int ret = 0; 964 965 if (start & ~TARGET_PAGE_MASK) { 966 return -TARGET_EINVAL; 967 } 968 if (len_in == 0) { 969 return 0; 970 } 971 len = TARGET_PAGE_ALIGN(len_in); 972 if (len == 0 || !guest_range_valid_untagged(start, len)) { 973 return -TARGET_EINVAL; 974 } 975 976 /* Translate for some architectures which have different MADV_xxx values */ 977 switch (advice) { 978 case TARGET_MADV_DONTNEED: /* alpha */ 979 advice = MADV_DONTNEED; 980 break; 981 case TARGET_MADV_WIPEONFORK: /* parisc */ 982 advice = MADV_WIPEONFORK; 983 break; 984 case TARGET_MADV_KEEPONFORK: /* parisc */ 985 advice = MADV_KEEPONFORK; 986 break; 987 /* we do not care about the other MADV_xxx values yet */ 988 } 989 990 /* 991 * Most advice values are hints, so ignoring and returning success is ok. 992 * 993 * However, some advice values such as MADV_DONTNEED, MADV_WIPEONFORK and 994 * MADV_KEEPONFORK are not hints and need to be emulated. 995 * 996 * A straight passthrough for those may not be safe because qemu sometimes 997 * turns private file-backed mappings into anonymous mappings. 998 * If all guest pages have PAGE_PASSTHROUGH set, mappings have the 999 * same semantics for the host as for the guest. 1000 * 1001 * We pass through MADV_WIPEONFORK and MADV_KEEPONFORK if possible and 1002 * return failure if not. 1003 * 1004 * MADV_DONTNEED is passed through as well, if possible. 1005 * If passthrough isn't possible, we nevertheless (wrongly!) return 1006 * success, which is broken but some userspace programs fail to work 1007 * otherwise. Completely implementing such emulation is quite complicated 1008 * though. 1009 */ 1010 mmap_lock(); 1011 switch (advice) { 1012 case MADV_WIPEONFORK: 1013 case MADV_KEEPONFORK: 1014 ret = -EINVAL; 1015 /* fall through */ 1016 case MADV_DONTNEED: 1017 if (page_check_range(start, len, PAGE_PASSTHROUGH)) { 1018 ret = get_errno(madvise(g2h_untagged(start), len, advice)); 1019 if ((advice == MADV_DONTNEED) && (ret == 0)) { 1020 page_reset_target_data(start, start + len - 1); 1021 } 1022 } 1023 } 1024 mmap_unlock(); 1025 1026 return ret; 1027 } 1028 1029 #ifndef TARGET_FORCE_SHMLBA 1030 /* 1031 * For most architectures, SHMLBA is the same as the page size; 1032 * some architectures have larger values, in which case they should 1033 * define TARGET_FORCE_SHMLBA and provide a target_shmlba() function. 1034 * This corresponds to the kernel arch code defining __ARCH_FORCE_SHMLBA 1035 * and defining its own value for SHMLBA. 1036 * 1037 * The kernel also permits SHMLBA to be set by the architecture to a 1038 * value larger than the page size without setting __ARCH_FORCE_SHMLBA; 1039 * this means that addresses are rounded to the large size if 1040 * SHM_RND is set but addresses not aligned to that size are not rejected 1041 * as long as they are at least page-aligned. Since the only architecture 1042 * which uses this is ia64 this code doesn't provide for that oddity. 1043 */ 1044 static inline abi_ulong target_shmlba(CPUArchState *cpu_env) 1045 { 1046 return TARGET_PAGE_SIZE; 1047 } 1048 #endif 1049 1050 abi_ulong target_shmat(CPUArchState *cpu_env, int shmid, 1051 abi_ulong shmaddr, int shmflg) 1052 { 1053 CPUState *cpu = env_cpu(cpu_env); 1054 abi_ulong raddr; 1055 struct shmid_ds shm_info; 1056 int ret; 1057 abi_ulong shmlba; 1058 1059 /* shmat pointers are always untagged */ 1060 1061 /* find out the length of the shared memory segment */ 1062 ret = get_errno(shmctl(shmid, IPC_STAT, &shm_info)); 1063 if (is_error(ret)) { 1064 /* can't get length, bail out */ 1065 return ret; 1066 } 1067 1068 shmlba = target_shmlba(cpu_env); 1069 1070 if (shmaddr & (shmlba - 1)) { 1071 if (shmflg & SHM_RND) { 1072 shmaddr &= ~(shmlba - 1); 1073 } else { 1074 return -TARGET_EINVAL; 1075 } 1076 } 1077 if (!guest_range_valid_untagged(shmaddr, shm_info.shm_segsz)) { 1078 return -TARGET_EINVAL; 1079 } 1080 1081 WITH_MMAP_LOCK_GUARD() { 1082 void *host_raddr; 1083 abi_ulong last; 1084 1085 if (shmaddr) { 1086 host_raddr = shmat(shmid, (void *)g2h_untagged(shmaddr), shmflg); 1087 } else { 1088 abi_ulong mmap_start; 1089 1090 /* In order to use the host shmat, we need to honor host SHMLBA. */ 1091 mmap_start = mmap_find_vma(0, shm_info.shm_segsz, 1092 MAX(SHMLBA, shmlba)); 1093 1094 if (mmap_start == -1) { 1095 return -TARGET_ENOMEM; 1096 } 1097 host_raddr = shmat(shmid, g2h_untagged(mmap_start), 1098 shmflg | SHM_REMAP); 1099 } 1100 1101 if (host_raddr == (void *)-1) { 1102 return get_errno(-1); 1103 } 1104 raddr = h2g(host_raddr); 1105 last = raddr + shm_info.shm_segsz - 1; 1106 1107 page_set_flags(raddr, last, 1108 PAGE_VALID | PAGE_RESET | PAGE_READ | 1109 (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE)); 1110 1111 shm_region_rm_complete(raddr, last); 1112 shm_region_add(raddr, last); 1113 } 1114 1115 /* 1116 * We're mapping shared memory, so ensure we generate code for parallel 1117 * execution and flush old translations. This will work up to the level 1118 * supported by the host -- anything that requires EXCP_ATOMIC will not 1119 * be atomic with respect to an external process. 1120 */ 1121 if (!(cpu->tcg_cflags & CF_PARALLEL)) { 1122 cpu->tcg_cflags |= CF_PARALLEL; 1123 tb_flush(cpu); 1124 } 1125 1126 return raddr; 1127 } 1128 1129 abi_long target_shmdt(abi_ulong shmaddr) 1130 { 1131 abi_long rv; 1132 1133 /* shmdt pointers are always untagged */ 1134 1135 WITH_MMAP_LOCK_GUARD() { 1136 abi_ulong last = shm_region_find(shmaddr); 1137 if (last == 0) { 1138 return -TARGET_EINVAL; 1139 } 1140 1141 rv = get_errno(shmdt(g2h_untagged(shmaddr))); 1142 if (rv == 0) { 1143 abi_ulong size = last - shmaddr + 1; 1144 1145 page_set_flags(shmaddr, last, 0); 1146 shm_region_rm_complete(shmaddr, last); 1147 mmap_reserve_or_unmap(shmaddr, size); 1148 } 1149 } 1150 return rv; 1151 } 1152