1 /* 2 * mmap support for qemu 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #include "qemu/osdep.h" 20 #include <sys/shm.h> 21 #include "trace.h" 22 #include "exec/log.h" 23 #include "exec/page-protection.h" 24 #include "qemu.h" 25 #include "user-internals.h" 26 #include "user-mmap.h" 27 #include "target_mman.h" 28 #include "qemu/interval-tree.h" 29 30 #ifdef TARGET_ARM 31 #include "target/arm/cpu-features.h" 32 #endif 33 34 static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER; 35 static __thread int mmap_lock_count; 36 37 void mmap_lock(void) 38 { 39 if (mmap_lock_count++ == 0) { 40 pthread_mutex_lock(&mmap_mutex); 41 } 42 } 43 44 void mmap_unlock(void) 45 { 46 assert(mmap_lock_count > 0); 47 if (--mmap_lock_count == 0) { 48 pthread_mutex_unlock(&mmap_mutex); 49 } 50 } 51 52 bool have_mmap_lock(void) 53 { 54 return mmap_lock_count > 0 ? true : false; 55 } 56 57 /* Grab lock to make sure things are in a consistent state after fork(). */ 58 void mmap_fork_start(void) 59 { 60 if (mmap_lock_count) 61 abort(); 62 pthread_mutex_lock(&mmap_mutex); 63 } 64 65 void mmap_fork_end(int child) 66 { 67 if (child) { 68 pthread_mutex_init(&mmap_mutex, NULL); 69 } else { 70 pthread_mutex_unlock(&mmap_mutex); 71 } 72 } 73 74 /* Protected by mmap_lock. */ 75 static IntervalTreeRoot shm_regions; 76 77 static void shm_region_add(abi_ptr start, abi_ptr last) 78 { 79 IntervalTreeNode *i = g_new0(IntervalTreeNode, 1); 80 81 i->start = start; 82 i->last = last; 83 interval_tree_insert(i, &shm_regions); 84 } 85 86 static abi_ptr shm_region_find(abi_ptr start) 87 { 88 IntervalTreeNode *i; 89 90 for (i = interval_tree_iter_first(&shm_regions, start, start); i; 91 i = interval_tree_iter_next(i, start, start)) { 92 if (i->start == start) { 93 return i->last; 94 } 95 } 96 return 0; 97 } 98 99 static void shm_region_rm_complete(abi_ptr start, abi_ptr last) 100 { 101 IntervalTreeNode *i, *n; 102 103 for (i = interval_tree_iter_first(&shm_regions, start, last); i; i = n) { 104 n = interval_tree_iter_next(i, start, last); 105 if (i->start >= start && i->last <= last) { 106 interval_tree_remove(i, &shm_regions); 107 g_free(i); 108 } 109 } 110 } 111 112 /* 113 * Validate target prot bitmask. 114 * Return the prot bitmask for the host in *HOST_PROT. 115 * Return 0 if the target prot bitmask is invalid, otherwise 116 * the internal qemu page_flags (which will include PAGE_VALID). 117 */ 118 static int validate_prot_to_pageflags(int prot) 119 { 120 int valid = PROT_READ | PROT_WRITE | PROT_EXEC | TARGET_PROT_SEM; 121 int page_flags = (prot & PAGE_RWX) | PAGE_VALID; 122 123 #ifdef TARGET_AARCH64 124 { 125 ARMCPU *cpu = ARM_CPU(thread_cpu); 126 127 /* 128 * The PROT_BTI bit is only accepted if the cpu supports the feature. 129 * Since this is the unusual case, don't bother checking unless 130 * the bit has been requested. If set and valid, record the bit 131 * within QEMU's page_flags. 132 */ 133 if ((prot & TARGET_PROT_BTI) && cpu_isar_feature(aa64_bti, cpu)) { 134 valid |= TARGET_PROT_BTI; 135 page_flags |= PAGE_BTI; 136 } 137 /* Similarly for the PROT_MTE bit. */ 138 if ((prot & TARGET_PROT_MTE) && cpu_isar_feature(aa64_mte, cpu)) { 139 valid |= TARGET_PROT_MTE; 140 page_flags |= PAGE_MTE; 141 } 142 } 143 #elif defined(TARGET_HPPA) 144 valid |= PROT_GROWSDOWN | PROT_GROWSUP; 145 #endif 146 147 return prot & ~valid ? 0 : page_flags; 148 } 149 150 /* 151 * For the host, we need not pass anything except read/write/exec. 152 * While PROT_SEM is allowed by all hosts, it is also ignored, so 153 * don't bother transforming guest bit to host bit. Any other 154 * target-specific prot bits will not be understood by the host 155 * and will need to be encoded into page_flags for qemu emulation. 156 * 157 * Pages that are executable by the guest will never be executed 158 * by the host, but the host will need to be able to read them. 159 */ 160 static int target_to_host_prot(int prot) 161 { 162 return (prot & (PROT_READ | PROT_WRITE)) | 163 (prot & PROT_EXEC ? PROT_READ : 0); 164 } 165 166 /* NOTE: all the constants are the HOST ones, but addresses are target. */ 167 int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) 168 { 169 int host_page_size = qemu_real_host_page_size(); 170 abi_ulong starts[3]; 171 abi_ulong lens[3]; 172 int prots[3]; 173 abi_ulong host_start, host_last, last; 174 int prot1, ret, page_flags, nranges; 175 176 trace_target_mprotect(start, len, target_prot); 177 178 if ((start & ~TARGET_PAGE_MASK) != 0) { 179 return -TARGET_EINVAL; 180 } 181 page_flags = validate_prot_to_pageflags(target_prot); 182 if (!page_flags) { 183 return -TARGET_EINVAL; 184 } 185 if (len == 0) { 186 return 0; 187 } 188 len = TARGET_PAGE_ALIGN(len); 189 if (!guest_range_valid_untagged(start, len)) { 190 return -TARGET_ENOMEM; 191 } 192 193 last = start + len - 1; 194 host_start = start & -host_page_size; 195 host_last = ROUND_UP(last, host_page_size) - 1; 196 nranges = 0; 197 198 mmap_lock(); 199 200 if (host_last - host_start < host_page_size) { 201 /* Single host page contains all guest pages: sum the prot. */ 202 prot1 = target_prot; 203 for (abi_ulong a = host_start; a < start; a += TARGET_PAGE_SIZE) { 204 prot1 |= page_get_flags(a); 205 } 206 for (abi_ulong a = last; a < host_last; a += TARGET_PAGE_SIZE) { 207 prot1 |= page_get_flags(a + 1); 208 } 209 starts[nranges] = host_start; 210 lens[nranges] = host_page_size; 211 prots[nranges] = prot1; 212 nranges++; 213 } else { 214 if (host_start < start) { 215 /* Host page contains more than one guest page: sum the prot. */ 216 prot1 = target_prot; 217 for (abi_ulong a = host_start; a < start; a += TARGET_PAGE_SIZE) { 218 prot1 |= page_get_flags(a); 219 } 220 /* If the resulting sum differs, create a new range. */ 221 if (prot1 != target_prot) { 222 starts[nranges] = host_start; 223 lens[nranges] = host_page_size; 224 prots[nranges] = prot1; 225 nranges++; 226 host_start += host_page_size; 227 } 228 } 229 230 if (last < host_last) { 231 /* Host page contains more than one guest page: sum the prot. */ 232 prot1 = target_prot; 233 for (abi_ulong a = last; a < host_last; a += TARGET_PAGE_SIZE) { 234 prot1 |= page_get_flags(a + 1); 235 } 236 /* If the resulting sum differs, create a new range. */ 237 if (prot1 != target_prot) { 238 host_last -= host_page_size; 239 starts[nranges] = host_last + 1; 240 lens[nranges] = host_page_size; 241 prots[nranges] = prot1; 242 nranges++; 243 } 244 } 245 246 /* Create a range for the middle, if any remains. */ 247 if (host_start < host_last) { 248 starts[nranges] = host_start; 249 lens[nranges] = host_last - host_start + 1; 250 prots[nranges] = target_prot; 251 nranges++; 252 } 253 } 254 255 for (int i = 0; i < nranges; ++i) { 256 ret = mprotect(g2h_untagged(starts[i]), lens[i], 257 target_to_host_prot(prots[i])); 258 if (ret != 0) { 259 goto error; 260 } 261 } 262 263 page_set_flags(start, last, page_flags); 264 ret = 0; 265 266 error: 267 mmap_unlock(); 268 return ret; 269 } 270 271 /* 272 * Perform munmap on behalf of the target, with host parameters. 273 * If reserved_va, we must replace the memory reservation. 274 */ 275 static int do_munmap(void *addr, size_t len) 276 { 277 if (reserved_va) { 278 void *ptr = mmap(addr, len, PROT_NONE, 279 MAP_FIXED | MAP_ANONYMOUS 280 | MAP_PRIVATE | MAP_NORESERVE, -1, 0); 281 return ptr == addr ? 0 : -1; 282 } 283 return munmap(addr, len); 284 } 285 286 /* 287 * Map an incomplete host page. 288 * 289 * Here be dragons. This case will not work if there is an existing 290 * overlapping host page, which is file mapped, and for which the mapping 291 * is beyond the end of the file. In that case, we will see SIGBUS when 292 * trying to write a portion of this page. 293 * 294 * FIXME: Work around this with a temporary signal handler and longjmp. 295 */ 296 static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, 297 int prot, int flags, int fd, off_t offset) 298 { 299 int host_page_size = qemu_real_host_page_size(); 300 abi_ulong real_last; 301 void *host_start; 302 int prot_old, prot_new; 303 int host_prot_old, host_prot_new; 304 305 if (!(flags & MAP_ANONYMOUS) 306 && (flags & MAP_TYPE) == MAP_SHARED 307 && (prot & PROT_WRITE)) { 308 /* 309 * msync() won't work with the partial page, so we return an 310 * error if write is possible while it is a shared mapping. 311 */ 312 errno = EINVAL; 313 return false; 314 } 315 316 real_last = real_start + host_page_size - 1; 317 host_start = g2h_untagged(real_start); 318 319 /* Get the protection of the target pages outside the mapping. */ 320 prot_old = 0; 321 for (abi_ulong a = real_start; a < start; a += TARGET_PAGE_SIZE) { 322 prot_old |= page_get_flags(a); 323 } 324 for (abi_ulong a = real_last; a > last; a -= TARGET_PAGE_SIZE) { 325 prot_old |= page_get_flags(a); 326 } 327 328 if (prot_old == 0) { 329 /* 330 * Since !(prot_old & PAGE_VALID), there were no guest pages 331 * outside of the fragment we need to map. Allocate a new host 332 * page to cover, discarding whatever else may have been present. 333 */ 334 void *p = mmap(host_start, host_page_size, 335 target_to_host_prot(prot), 336 flags | MAP_ANONYMOUS, -1, 0); 337 if (p != host_start) { 338 if (p != MAP_FAILED) { 339 do_munmap(p, host_page_size); 340 errno = EEXIST; 341 } 342 return false; 343 } 344 prot_old = prot; 345 } 346 prot_new = prot | prot_old; 347 348 host_prot_old = target_to_host_prot(prot_old); 349 host_prot_new = target_to_host_prot(prot_new); 350 351 /* Adjust protection to be able to write. */ 352 if (!(host_prot_old & PROT_WRITE)) { 353 host_prot_old |= PROT_WRITE; 354 mprotect(host_start, host_page_size, host_prot_old); 355 } 356 357 /* Read or zero the new guest pages. */ 358 if (flags & MAP_ANONYMOUS) { 359 memset(g2h_untagged(start), 0, last - start + 1); 360 } else { 361 if (pread(fd, g2h_untagged(start), last - start + 1, offset) == -1) { 362 return false; 363 } 364 } 365 366 /* Put final protection */ 367 if (host_prot_new != host_prot_old) { 368 mprotect(host_start, host_page_size, host_prot_new); 369 } 370 return true; 371 } 372 373 abi_ulong task_unmapped_base; 374 abi_ulong elf_et_dyn_base; 375 abi_ulong mmap_next_start; 376 377 /* 378 * Subroutine of mmap_find_vma, used when we have pre-allocated 379 * a chunk of guest address space. 380 */ 381 static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size, 382 abi_ulong align) 383 { 384 target_ulong ret; 385 386 ret = page_find_range_empty(start, reserved_va, size, align); 387 if (ret == -1 && start > mmap_min_addr) { 388 /* Restart at the beginning of the address space. */ 389 ret = page_find_range_empty(mmap_min_addr, start - 1, size, align); 390 } 391 392 return ret; 393 } 394 395 /* 396 * Find and reserve a free memory area of size 'size'. The search 397 * starts at 'start'. 398 * It must be called with mmap_lock() held. 399 * Return -1 if error. 400 */ 401 abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size, abi_ulong align) 402 { 403 int host_page_size = qemu_real_host_page_size(); 404 void *ptr, *prev; 405 abi_ulong addr; 406 int wrapped, repeat; 407 408 align = MAX(align, host_page_size); 409 410 /* If 'start' == 0, then a default start address is used. */ 411 if (start == 0) { 412 start = mmap_next_start; 413 } else { 414 start &= -host_page_size; 415 } 416 start = ROUND_UP(start, align); 417 size = ROUND_UP(size, host_page_size); 418 419 if (reserved_va) { 420 return mmap_find_vma_reserved(start, size, align); 421 } 422 423 addr = start; 424 wrapped = repeat = 0; 425 prev = 0; 426 427 for (;; prev = ptr) { 428 /* 429 * Reserve needed memory area to avoid a race. 430 * It should be discarded using: 431 * - mmap() with MAP_FIXED flag 432 * - mremap() with MREMAP_FIXED flag 433 * - shmat() with SHM_REMAP flag 434 */ 435 ptr = mmap(g2h_untagged(addr), size, PROT_NONE, 436 MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); 437 438 /* ENOMEM, if host address space has no memory */ 439 if (ptr == MAP_FAILED) { 440 return (abi_ulong)-1; 441 } 442 443 /* 444 * Count the number of sequential returns of the same address. 445 * This is used to modify the search algorithm below. 446 */ 447 repeat = (ptr == prev ? repeat + 1 : 0); 448 449 if (h2g_valid(ptr + size - 1)) { 450 addr = h2g(ptr); 451 452 if ((addr & (align - 1)) == 0) { 453 /* Success. */ 454 if (start == mmap_next_start && addr >= task_unmapped_base) { 455 mmap_next_start = addr + size; 456 } 457 return addr; 458 } 459 460 /* The address is not properly aligned for the target. */ 461 switch (repeat) { 462 case 0: 463 /* 464 * Assume the result that the kernel gave us is the 465 * first with enough free space, so start again at the 466 * next higher target page. 467 */ 468 addr = ROUND_UP(addr, align); 469 break; 470 case 1: 471 /* 472 * Sometimes the kernel decides to perform the allocation 473 * at the top end of memory instead. 474 */ 475 addr &= -align; 476 break; 477 case 2: 478 /* Start over at low memory. */ 479 addr = 0; 480 break; 481 default: 482 /* Fail. This unaligned block must the last. */ 483 addr = -1; 484 break; 485 } 486 } else { 487 /* 488 * Since the result the kernel gave didn't fit, start 489 * again at low memory. If any repetition, fail. 490 */ 491 addr = (repeat ? -1 : 0); 492 } 493 494 /* Unmap and try again. */ 495 munmap(ptr, size); 496 497 /* ENOMEM if we checked the whole of the target address space. */ 498 if (addr == (abi_ulong)-1) { 499 return (abi_ulong)-1; 500 } else if (addr == 0) { 501 if (wrapped) { 502 return (abi_ulong)-1; 503 } 504 wrapped = 1; 505 /* 506 * Don't actually use 0 when wrapping, instead indicate 507 * that we'd truly like an allocation in low memory. 508 */ 509 addr = (mmap_min_addr > TARGET_PAGE_SIZE 510 ? TARGET_PAGE_ALIGN(mmap_min_addr) 511 : TARGET_PAGE_SIZE); 512 } else if (wrapped && addr >= start) { 513 return (abi_ulong)-1; 514 } 515 } 516 } 517 518 /* 519 * Record a successful mmap within the user-exec interval tree. 520 */ 521 static abi_long mmap_end(abi_ulong start, abi_ulong last, 522 abi_ulong passthrough_start, 523 abi_ulong passthrough_last, 524 int flags, int page_flags) 525 { 526 if (flags & MAP_ANONYMOUS) { 527 page_flags |= PAGE_ANON; 528 } 529 page_flags |= PAGE_RESET; 530 if (passthrough_start > passthrough_last) { 531 page_set_flags(start, last, page_flags); 532 } else { 533 if (start < passthrough_start) { 534 page_set_flags(start, passthrough_start - 1, page_flags); 535 } 536 page_set_flags(passthrough_start, passthrough_last, 537 page_flags | PAGE_PASSTHROUGH); 538 if (passthrough_last < last) { 539 page_set_flags(passthrough_last + 1, last, page_flags); 540 } 541 } 542 shm_region_rm_complete(start, last); 543 trace_target_mmap_complete(start); 544 if (qemu_loglevel_mask(CPU_LOG_PAGE)) { 545 FILE *f = qemu_log_trylock(); 546 if (f) { 547 fprintf(f, "page layout changed following mmap\n"); 548 page_dump(f); 549 qemu_log_unlock(f); 550 } 551 } 552 return start; 553 } 554 555 /* 556 * Special case host page size == target page size, 557 * where there are no edge conditions. 558 */ 559 static abi_long mmap_h_eq_g(abi_ulong start, abi_ulong len, 560 int host_prot, int flags, int page_flags, 561 int fd, off_t offset) 562 { 563 void *p, *want_p = g2h_untagged(start); 564 abi_ulong last; 565 566 p = mmap(want_p, len, host_prot, flags, fd, offset); 567 if (p == MAP_FAILED) { 568 return -1; 569 } 570 /* If the host kernel does not support MAP_FIXED_NOREPLACE, emulate. */ 571 if ((flags & MAP_FIXED_NOREPLACE) && p != want_p) { 572 do_munmap(p, len); 573 errno = EEXIST; 574 return -1; 575 } 576 577 start = h2g(p); 578 last = start + len - 1; 579 return mmap_end(start, last, start, last, flags, page_flags); 580 } 581 582 /* 583 * Special case host page size < target page size. 584 * 585 * The two special cases are increased guest alignment, and mapping 586 * past the end of a file. 587 * 588 * When mapping files into a memory area larger than the file, 589 * accesses to pages beyond the file size will cause a SIGBUS. 590 * 591 * For example, if mmaping a file of 100 bytes on a host with 4K 592 * pages emulating a target with 8K pages, the target expects to 593 * be able to access the first 8K. But the host will trap us on 594 * any access beyond 4K. 595 * 596 * When emulating a target with a larger page-size than the hosts, 597 * we may need to truncate file maps at EOF and add extra anonymous 598 * pages up to the targets page boundary. 599 * 600 * This workaround only works for files that do not change. 601 * If the file is later extended (e.g. ftruncate), the SIGBUS 602 * vanishes and the proper behaviour is that changes within the 603 * anon page should be reflected in the file. 604 * 605 * However, this case is rather common with executable images, 606 * so the workaround is important for even trivial tests, whereas 607 * the mmap of of a file being extended is less common. 608 */ 609 static abi_long mmap_h_lt_g(abi_ulong start, abi_ulong len, int host_prot, 610 int mmap_flags, int page_flags, int fd, 611 off_t offset, int host_page_size) 612 { 613 void *p, *want_p = g2h_untagged(start); 614 off_t fileend_adj = 0; 615 int flags = mmap_flags; 616 abi_ulong last, pass_last; 617 618 if (!(flags & MAP_ANONYMOUS)) { 619 struct stat sb; 620 621 if (fstat(fd, &sb) == -1) { 622 return -1; 623 } 624 if (offset >= sb.st_size) { 625 /* 626 * The entire map is beyond the end of the file. 627 * Transform it to an anonymous mapping. 628 */ 629 flags |= MAP_ANONYMOUS; 630 fd = -1; 631 offset = 0; 632 } else if (offset + len > sb.st_size) { 633 /* 634 * A portion of the map is beyond the end of the file. 635 * Truncate the file portion of the allocation. 636 */ 637 fileend_adj = offset + len - sb.st_size; 638 } 639 } 640 641 if (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) { 642 if (fileend_adj) { 643 p = mmap(want_p, len, host_prot, flags | MAP_ANONYMOUS, -1, 0); 644 } else { 645 p = mmap(want_p, len, host_prot, flags, fd, offset); 646 } 647 if (p != want_p) { 648 if (p != MAP_FAILED) { 649 /* Host does not support MAP_FIXED_NOREPLACE: emulate. */ 650 do_munmap(p, len); 651 errno = EEXIST; 652 } 653 return -1; 654 } 655 656 if (fileend_adj) { 657 void *t = mmap(p, len - fileend_adj, host_prot, 658 (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED, 659 fd, offset); 660 661 if (t == MAP_FAILED) { 662 int save_errno = errno; 663 664 /* 665 * We failed a map over the top of the successful anonymous 666 * mapping above. The only failure mode is running out of VMAs, 667 * and there's nothing that we can do to detect that earlier. 668 * If we have replaced an existing mapping with MAP_FIXED, 669 * then we cannot properly recover. It's a coin toss whether 670 * it would be better to exit or continue here. 671 */ 672 if (!(flags & MAP_FIXED_NOREPLACE) && 673 !page_check_range_empty(start, start + len - 1)) { 674 qemu_log("QEMU target_mmap late failure: %s", 675 strerror(save_errno)); 676 } 677 678 do_munmap(want_p, len); 679 errno = save_errno; 680 return -1; 681 } 682 } 683 } else { 684 size_t host_len, part_len; 685 686 /* 687 * Take care to align the host memory. Perform a larger anonymous 688 * allocation and extract the aligned portion. Remap the file on 689 * top of that. 690 */ 691 host_len = len + TARGET_PAGE_SIZE - host_page_size; 692 p = mmap(want_p, host_len, host_prot, flags | MAP_ANONYMOUS, -1, 0); 693 if (p == MAP_FAILED) { 694 return -1; 695 } 696 697 part_len = (uintptr_t)p & (TARGET_PAGE_SIZE - 1); 698 if (part_len) { 699 part_len = TARGET_PAGE_SIZE - part_len; 700 do_munmap(p, part_len); 701 p += part_len; 702 host_len -= part_len; 703 } 704 if (len < host_len) { 705 do_munmap(p + len, host_len - len); 706 } 707 708 if (!(flags & MAP_ANONYMOUS)) { 709 void *t = mmap(p, len - fileend_adj, host_prot, 710 flags | MAP_FIXED, fd, offset); 711 712 if (t == MAP_FAILED) { 713 int save_errno = errno; 714 do_munmap(p, len); 715 errno = save_errno; 716 return -1; 717 } 718 } 719 720 start = h2g(p); 721 } 722 723 last = start + len - 1; 724 if (fileend_adj) { 725 pass_last = ROUND_UP(last - fileend_adj, host_page_size) - 1; 726 } else { 727 pass_last = last; 728 } 729 return mmap_end(start, last, start, pass_last, mmap_flags, page_flags); 730 } 731 732 /* 733 * Special case host page size > target page size. 734 * 735 * The two special cases are address and file offsets that are valid 736 * for the guest that cannot be directly represented by the host. 737 */ 738 static abi_long mmap_h_gt_g(abi_ulong start, abi_ulong len, 739 int target_prot, int host_prot, 740 int flags, int page_flags, int fd, 741 off_t offset, int host_page_size) 742 { 743 void *p, *want_p = g2h_untagged(start); 744 off_t host_offset = offset & -host_page_size; 745 abi_ulong last, real_start, real_last; 746 bool misaligned_offset = false; 747 size_t host_len; 748 749 if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { 750 /* 751 * Adjust the offset to something representable on the host. 752 */ 753 host_len = len + offset - host_offset; 754 p = mmap(want_p, host_len, host_prot, flags, fd, host_offset); 755 if (p == MAP_FAILED) { 756 return -1; 757 } 758 759 /* Update start to the file position at offset. */ 760 p += offset - host_offset; 761 762 start = h2g(p); 763 last = start + len - 1; 764 return mmap_end(start, last, start, last, flags, page_flags); 765 } 766 767 if (!(flags & MAP_ANONYMOUS)) { 768 misaligned_offset = (start ^ offset) & (host_page_size - 1); 769 770 /* 771 * The fallback for misalignment is a private mapping + read. 772 * This carries none of semantics required of MAP_SHARED. 773 */ 774 if (misaligned_offset && (flags & MAP_TYPE) != MAP_PRIVATE) { 775 errno = EINVAL; 776 return -1; 777 } 778 } 779 780 last = start + len - 1; 781 real_start = start & -host_page_size; 782 real_last = ROUND_UP(last, host_page_size) - 1; 783 784 /* 785 * Handle the start and end of the mapping. 786 */ 787 if (real_start < start) { 788 abi_ulong real_page_last = real_start + host_page_size - 1; 789 if (last <= real_page_last) { 790 /* Entire allocation a subset of one host page. */ 791 if (!mmap_frag(real_start, start, last, target_prot, 792 flags, fd, offset)) { 793 return -1; 794 } 795 return mmap_end(start, last, -1, 0, flags, page_flags); 796 } 797 798 if (!mmap_frag(real_start, start, real_page_last, target_prot, 799 flags, fd, offset)) { 800 return -1; 801 } 802 real_start = real_page_last + 1; 803 } 804 805 if (last < real_last) { 806 abi_ulong real_page_start = real_last - host_page_size + 1; 807 if (!mmap_frag(real_page_start, real_page_start, last, 808 target_prot, flags, fd, 809 offset + real_page_start - start)) { 810 return -1; 811 } 812 real_last = real_page_start - 1; 813 } 814 815 if (real_start > real_last) { 816 return mmap_end(start, last, -1, 0, flags, page_flags); 817 } 818 819 /* 820 * Handle the middle of the mapping. 821 */ 822 823 host_len = real_last - real_start + 1; 824 want_p += real_start - start; 825 826 if (flags & MAP_ANONYMOUS) { 827 p = mmap(want_p, host_len, host_prot, flags, -1, 0); 828 } else if (!misaligned_offset) { 829 p = mmap(want_p, host_len, host_prot, flags, fd, 830 offset + real_start - start); 831 } else { 832 p = mmap(want_p, host_len, host_prot | PROT_WRITE, 833 flags | MAP_ANONYMOUS, -1, 0); 834 } 835 if (p != want_p) { 836 if (p != MAP_FAILED) { 837 do_munmap(p, host_len); 838 errno = EEXIST; 839 } 840 return -1; 841 } 842 843 if (misaligned_offset) { 844 /* TODO: The read could be short. */ 845 if (pread(fd, p, host_len, offset + real_start - start) != host_len) { 846 do_munmap(p, host_len); 847 return -1; 848 } 849 if (!(host_prot & PROT_WRITE)) { 850 mprotect(p, host_len, host_prot); 851 } 852 } 853 854 return mmap_end(start, last, -1, 0, flags, page_flags); 855 } 856 857 static abi_long target_mmap__locked(abi_ulong start, abi_ulong len, 858 int target_prot, int flags, int page_flags, 859 int fd, off_t offset) 860 { 861 int host_page_size = qemu_real_host_page_size(); 862 int host_prot; 863 864 /* 865 * For reserved_va, we are in full control of the allocation. 866 * Find a suitable hole and convert to MAP_FIXED. 867 */ 868 if (reserved_va) { 869 if (flags & MAP_FIXED_NOREPLACE) { 870 /* Validate that the chosen range is empty. */ 871 if (!page_check_range_empty(start, start + len - 1)) { 872 errno = EEXIST; 873 return -1; 874 } 875 flags = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED; 876 } else if (!(flags & MAP_FIXED)) { 877 abi_ulong real_start = start & -host_page_size; 878 off_t host_offset = offset & -host_page_size; 879 size_t real_len = len + offset - host_offset; 880 abi_ulong align = MAX(host_page_size, TARGET_PAGE_SIZE); 881 882 start = mmap_find_vma(real_start, real_len, align); 883 if (start == (abi_ulong)-1) { 884 errno = ENOMEM; 885 return -1; 886 } 887 start += offset - host_offset; 888 flags |= MAP_FIXED; 889 } 890 } 891 892 host_prot = target_to_host_prot(target_prot); 893 894 if (host_page_size == TARGET_PAGE_SIZE) { 895 return mmap_h_eq_g(start, len, host_prot, flags, 896 page_flags, fd, offset); 897 } else if (host_page_size < TARGET_PAGE_SIZE) { 898 return mmap_h_lt_g(start, len, host_prot, flags, 899 page_flags, fd, offset, host_page_size); 900 } else { 901 return mmap_h_gt_g(start, len, target_prot, host_prot, flags, 902 page_flags, fd, offset, host_page_size); 903 } 904 } 905 906 /* NOTE: all the constants are the HOST ones */ 907 abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot, 908 int flags, int fd, off_t offset) 909 { 910 abi_long ret; 911 int page_flags; 912 913 trace_target_mmap(start, len, target_prot, flags, fd, offset); 914 915 if (!len) { 916 errno = EINVAL; 917 return -1; 918 } 919 920 page_flags = validate_prot_to_pageflags(target_prot); 921 if (!page_flags) { 922 errno = EINVAL; 923 return -1; 924 } 925 926 /* Also check for overflows... */ 927 len = TARGET_PAGE_ALIGN(len); 928 if (!len || len != (size_t)len) { 929 errno = ENOMEM; 930 return -1; 931 } 932 933 if (offset & ~TARGET_PAGE_MASK) { 934 errno = EINVAL; 935 return -1; 936 } 937 if (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) { 938 if (start & ~TARGET_PAGE_MASK) { 939 errno = EINVAL; 940 return -1; 941 } 942 if (!guest_range_valid_untagged(start, len)) { 943 errno = ENOMEM; 944 return -1; 945 } 946 } 947 948 mmap_lock(); 949 950 ret = target_mmap__locked(start, len, target_prot, flags, 951 page_flags, fd, offset); 952 953 mmap_unlock(); 954 955 /* 956 * If we're mapping shared memory, ensure we generate code for parallel 957 * execution and flush old translations. This will work up to the level 958 * supported by the host -- anything that requires EXCP_ATOMIC will not 959 * be atomic with respect to an external process. 960 */ 961 if (ret != -1 && (flags & MAP_TYPE) != MAP_PRIVATE) { 962 CPUState *cpu = thread_cpu; 963 if (!tcg_cflags_has(cpu, CF_PARALLEL)) { 964 tcg_cflags_set(cpu, CF_PARALLEL); 965 tb_flush(cpu); 966 } 967 } 968 969 return ret; 970 } 971 972 static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) 973 { 974 int host_page_size = qemu_real_host_page_size(); 975 abi_ulong real_start; 976 abi_ulong real_last; 977 abi_ulong real_len; 978 abi_ulong last; 979 abi_ulong a; 980 void *host_start; 981 int prot; 982 983 last = start + len - 1; 984 real_start = start & -host_page_size; 985 real_last = ROUND_UP(last, host_page_size) - 1; 986 987 /* 988 * If guest pages remain on the first or last host pages, 989 * adjust the deallocation to retain those guest pages. 990 * The single page special case is required for the last page, 991 * lest real_start overflow to zero. 992 */ 993 if (real_last - real_start < host_page_size) { 994 prot = 0; 995 for (a = real_start; a < start; a += TARGET_PAGE_SIZE) { 996 prot |= page_get_flags(a); 997 } 998 for (a = last; a < real_last; a += TARGET_PAGE_SIZE) { 999 prot |= page_get_flags(a + 1); 1000 } 1001 if (prot != 0) { 1002 return 0; 1003 } 1004 } else { 1005 for (prot = 0, a = real_start; a < start; a += TARGET_PAGE_SIZE) { 1006 prot |= page_get_flags(a); 1007 } 1008 if (prot != 0) { 1009 real_start += host_page_size; 1010 } 1011 1012 for (prot = 0, a = last; a < real_last; a += TARGET_PAGE_SIZE) { 1013 prot |= page_get_flags(a + 1); 1014 } 1015 if (prot != 0) { 1016 real_last -= host_page_size; 1017 } 1018 1019 if (real_last < real_start) { 1020 return 0; 1021 } 1022 } 1023 1024 real_len = real_last - real_start + 1; 1025 host_start = g2h_untagged(real_start); 1026 1027 return do_munmap(host_start, real_len); 1028 } 1029 1030 int target_munmap(abi_ulong start, abi_ulong len) 1031 { 1032 int ret; 1033 1034 trace_target_munmap(start, len); 1035 1036 if (start & ~TARGET_PAGE_MASK) { 1037 errno = EINVAL; 1038 return -1; 1039 } 1040 len = TARGET_PAGE_ALIGN(len); 1041 if (len == 0 || !guest_range_valid_untagged(start, len)) { 1042 errno = EINVAL; 1043 return -1; 1044 } 1045 1046 mmap_lock(); 1047 ret = mmap_reserve_or_unmap(start, len); 1048 if (likely(ret == 0)) { 1049 page_set_flags(start, start + len - 1, 0); 1050 shm_region_rm_complete(start, start + len - 1); 1051 } 1052 mmap_unlock(); 1053 1054 return ret; 1055 } 1056 1057 abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size, 1058 abi_ulong new_size, unsigned long flags, 1059 abi_ulong new_addr) 1060 { 1061 int prot; 1062 void *host_addr; 1063 1064 if (!guest_range_valid_untagged(old_addr, old_size) || 1065 ((flags & MREMAP_FIXED) && 1066 !guest_range_valid_untagged(new_addr, new_size)) || 1067 ((flags & MREMAP_MAYMOVE) == 0 && 1068 !guest_range_valid_untagged(old_addr, new_size))) { 1069 errno = ENOMEM; 1070 return -1; 1071 } 1072 1073 mmap_lock(); 1074 1075 if (flags & MREMAP_FIXED) { 1076 host_addr = mremap(g2h_untagged(old_addr), old_size, new_size, 1077 flags, g2h_untagged(new_addr)); 1078 1079 if (reserved_va && host_addr != MAP_FAILED) { 1080 /* 1081 * If new and old addresses overlap then the above mremap will 1082 * already have failed with EINVAL. 1083 */ 1084 mmap_reserve_or_unmap(old_addr, old_size); 1085 } 1086 } else if (flags & MREMAP_MAYMOVE) { 1087 abi_ulong mmap_start; 1088 1089 mmap_start = mmap_find_vma(0, new_size, TARGET_PAGE_SIZE); 1090 1091 if (mmap_start == -1) { 1092 errno = ENOMEM; 1093 host_addr = MAP_FAILED; 1094 } else { 1095 host_addr = mremap(g2h_untagged(old_addr), old_size, new_size, 1096 flags | MREMAP_FIXED, 1097 g2h_untagged(mmap_start)); 1098 if (reserved_va) { 1099 mmap_reserve_or_unmap(old_addr, old_size); 1100 } 1101 } 1102 } else { 1103 int page_flags = 0; 1104 if (reserved_va && old_size < new_size) { 1105 abi_ulong addr; 1106 for (addr = old_addr + old_size; 1107 addr < old_addr + new_size; 1108 addr++) { 1109 page_flags |= page_get_flags(addr); 1110 } 1111 } 1112 if (page_flags == 0) { 1113 host_addr = mremap(g2h_untagged(old_addr), 1114 old_size, new_size, flags); 1115 1116 if (host_addr != MAP_FAILED) { 1117 /* Check if address fits target address space */ 1118 if (!guest_range_valid_untagged(h2g(host_addr), new_size)) { 1119 /* Revert mremap() changes */ 1120 host_addr = mremap(g2h_untagged(old_addr), 1121 new_size, old_size, flags); 1122 errno = ENOMEM; 1123 host_addr = MAP_FAILED; 1124 } else if (reserved_va && old_size > new_size) { 1125 mmap_reserve_or_unmap(old_addr + old_size, 1126 old_size - new_size); 1127 } 1128 } 1129 } else { 1130 errno = ENOMEM; 1131 host_addr = MAP_FAILED; 1132 } 1133 } 1134 1135 if (host_addr == MAP_FAILED) { 1136 new_addr = -1; 1137 } else { 1138 new_addr = h2g(host_addr); 1139 prot = page_get_flags(old_addr); 1140 page_set_flags(old_addr, old_addr + old_size - 1, 0); 1141 shm_region_rm_complete(old_addr, old_addr + old_size - 1); 1142 page_set_flags(new_addr, new_addr + new_size - 1, 1143 prot | PAGE_VALID | PAGE_RESET); 1144 shm_region_rm_complete(new_addr, new_addr + new_size - 1); 1145 } 1146 mmap_unlock(); 1147 return new_addr; 1148 } 1149 1150 abi_long target_madvise(abi_ulong start, abi_ulong len_in, int advice) 1151 { 1152 abi_ulong len; 1153 int ret = 0; 1154 1155 if (start & ~TARGET_PAGE_MASK) { 1156 return -TARGET_EINVAL; 1157 } 1158 if (len_in == 0) { 1159 return 0; 1160 } 1161 len = TARGET_PAGE_ALIGN(len_in); 1162 if (len == 0 || !guest_range_valid_untagged(start, len)) { 1163 return -TARGET_EINVAL; 1164 } 1165 1166 /* Translate for some architectures which have different MADV_xxx values */ 1167 switch (advice) { 1168 case TARGET_MADV_DONTNEED: /* alpha */ 1169 advice = MADV_DONTNEED; 1170 break; 1171 case TARGET_MADV_WIPEONFORK: /* parisc */ 1172 advice = MADV_WIPEONFORK; 1173 break; 1174 case TARGET_MADV_KEEPONFORK: /* parisc */ 1175 advice = MADV_KEEPONFORK; 1176 break; 1177 /* we do not care about the other MADV_xxx values yet */ 1178 } 1179 1180 /* 1181 * Most advice values are hints, so ignoring and returning success is ok. 1182 * 1183 * However, some advice values such as MADV_DONTNEED, MADV_WIPEONFORK and 1184 * MADV_KEEPONFORK are not hints and need to be emulated. 1185 * 1186 * A straight passthrough for those may not be safe because qemu sometimes 1187 * turns private file-backed mappings into anonymous mappings. 1188 * If all guest pages have PAGE_PASSTHROUGH set, mappings have the 1189 * same semantics for the host as for the guest. 1190 * 1191 * We pass through MADV_WIPEONFORK and MADV_KEEPONFORK if possible and 1192 * return failure if not. 1193 * 1194 * MADV_DONTNEED is passed through as well, if possible. 1195 * If passthrough isn't possible, we nevertheless (wrongly!) return 1196 * success, which is broken but some userspace programs fail to work 1197 * otherwise. Completely implementing such emulation is quite complicated 1198 * though. 1199 */ 1200 mmap_lock(); 1201 switch (advice) { 1202 case MADV_WIPEONFORK: 1203 case MADV_KEEPONFORK: 1204 ret = -EINVAL; 1205 /* fall through */ 1206 case MADV_DONTNEED: 1207 if (page_check_range(start, len, PAGE_PASSTHROUGH)) { 1208 ret = get_errno(madvise(g2h_untagged(start), len, advice)); 1209 if ((advice == MADV_DONTNEED) && (ret == 0)) { 1210 page_reset_target_data(start, start + len - 1); 1211 } 1212 } 1213 } 1214 mmap_unlock(); 1215 1216 return ret; 1217 } 1218 1219 #ifndef TARGET_FORCE_SHMLBA 1220 /* 1221 * For most architectures, SHMLBA is the same as the page size; 1222 * some architectures have larger values, in which case they should 1223 * define TARGET_FORCE_SHMLBA and provide a target_shmlba() function. 1224 * This corresponds to the kernel arch code defining __ARCH_FORCE_SHMLBA 1225 * and defining its own value for SHMLBA. 1226 * 1227 * The kernel also permits SHMLBA to be set by the architecture to a 1228 * value larger than the page size without setting __ARCH_FORCE_SHMLBA; 1229 * this means that addresses are rounded to the large size if 1230 * SHM_RND is set but addresses not aligned to that size are not rejected 1231 * as long as they are at least page-aligned. Since the only architecture 1232 * which uses this is ia64 this code doesn't provide for that oddity. 1233 */ 1234 static inline abi_ulong target_shmlba(CPUArchState *cpu_env) 1235 { 1236 return TARGET_PAGE_SIZE; 1237 } 1238 #endif 1239 1240 #if defined(__arm__) || defined(__mips__) || defined(__sparc__) 1241 #define HOST_FORCE_SHMLBA 1 1242 #else 1243 #define HOST_FORCE_SHMLBA 0 1244 #endif 1245 1246 abi_ulong target_shmat(CPUArchState *cpu_env, int shmid, 1247 abi_ulong shmaddr, int shmflg) 1248 { 1249 CPUState *cpu = env_cpu(cpu_env); 1250 struct shmid_ds shm_info; 1251 int ret; 1252 int h_pagesize; 1253 int t_shmlba, h_shmlba, m_shmlba; 1254 size_t t_len, h_len, m_len; 1255 1256 /* shmat pointers are always untagged */ 1257 1258 /* 1259 * Because we can't use host shmat() unless the address is sufficiently 1260 * aligned for the host, we'll need to check both. 1261 * TODO: Could be fixed with softmmu. 1262 */ 1263 t_shmlba = target_shmlba(cpu_env); 1264 h_pagesize = qemu_real_host_page_size(); 1265 h_shmlba = (HOST_FORCE_SHMLBA ? SHMLBA : h_pagesize); 1266 m_shmlba = MAX(t_shmlba, h_shmlba); 1267 1268 if (shmaddr) { 1269 if (shmaddr & (m_shmlba - 1)) { 1270 if (shmflg & SHM_RND) { 1271 /* 1272 * The guest is allowing the kernel to round the address. 1273 * Assume that the guest is ok with us rounding to the 1274 * host required alignment too. Anyway if we don't, we'll 1275 * get an error from the kernel. 1276 */ 1277 shmaddr &= ~(m_shmlba - 1); 1278 if (shmaddr == 0 && (shmflg & SHM_REMAP)) { 1279 return -TARGET_EINVAL; 1280 } 1281 } else { 1282 int require = TARGET_PAGE_SIZE; 1283 #ifdef TARGET_FORCE_SHMLBA 1284 require = t_shmlba; 1285 #endif 1286 /* 1287 * Include host required alignment, as otherwise we cannot 1288 * use host shmat at all. 1289 */ 1290 require = MAX(require, h_shmlba); 1291 if (shmaddr & (require - 1)) { 1292 return -TARGET_EINVAL; 1293 } 1294 } 1295 } 1296 } else { 1297 if (shmflg & SHM_REMAP) { 1298 return -TARGET_EINVAL; 1299 } 1300 } 1301 /* All rounding now manually concluded. */ 1302 shmflg &= ~SHM_RND; 1303 1304 /* Find out the length of the shared memory segment. */ 1305 ret = get_errno(shmctl(shmid, IPC_STAT, &shm_info)); 1306 if (is_error(ret)) { 1307 /* can't get length, bail out */ 1308 return ret; 1309 } 1310 t_len = TARGET_PAGE_ALIGN(shm_info.shm_segsz); 1311 h_len = ROUND_UP(shm_info.shm_segsz, h_pagesize); 1312 m_len = MAX(t_len, h_len); 1313 1314 if (!guest_range_valid_untagged(shmaddr, m_len)) { 1315 return -TARGET_EINVAL; 1316 } 1317 1318 WITH_MMAP_LOCK_GUARD() { 1319 bool mapped = false; 1320 void *want, *test; 1321 abi_ulong last; 1322 1323 if (!shmaddr) { 1324 shmaddr = mmap_find_vma(0, m_len, m_shmlba); 1325 if (shmaddr == -1) { 1326 return -TARGET_ENOMEM; 1327 } 1328 mapped = !reserved_va; 1329 } else if (shmflg & SHM_REMAP) { 1330 /* 1331 * If host page size > target page size, the host shmat may map 1332 * more memory than the guest expects. Reject a mapping that 1333 * would replace memory in the unexpected gap. 1334 * TODO: Could be fixed with softmmu. 1335 */ 1336 if (t_len < h_len && 1337 !page_check_range_empty(shmaddr + t_len, 1338 shmaddr + h_len - 1)) { 1339 return -TARGET_EINVAL; 1340 } 1341 } else { 1342 if (!page_check_range_empty(shmaddr, shmaddr + m_len - 1)) { 1343 return -TARGET_EINVAL; 1344 } 1345 } 1346 1347 /* All placement is now complete. */ 1348 want = (void *)g2h_untagged(shmaddr); 1349 1350 /* 1351 * Map anonymous pages across the entire range, then remap with 1352 * the shared memory. This is required for a number of corner 1353 * cases for which host and guest page sizes differ. 1354 */ 1355 if (h_len != t_len) { 1356 int mmap_p = PROT_READ | (shmflg & SHM_RDONLY ? 0 : PROT_WRITE); 1357 int mmap_f = MAP_PRIVATE | MAP_ANONYMOUS 1358 | (reserved_va || mapped || (shmflg & SHM_REMAP) 1359 ? MAP_FIXED : MAP_FIXED_NOREPLACE); 1360 1361 test = mmap(want, m_len, mmap_p, mmap_f, -1, 0); 1362 if (unlikely(test != want)) { 1363 /* shmat returns EINVAL not EEXIST like mmap. */ 1364 ret = (test == MAP_FAILED && errno != EEXIST 1365 ? get_errno(-1) : -TARGET_EINVAL); 1366 if (mapped) { 1367 do_munmap(want, m_len); 1368 } 1369 return ret; 1370 } 1371 mapped = true; 1372 } 1373 1374 if (reserved_va || mapped) { 1375 shmflg |= SHM_REMAP; 1376 } 1377 test = shmat(shmid, want, shmflg); 1378 if (test == MAP_FAILED) { 1379 ret = get_errno(-1); 1380 if (mapped) { 1381 do_munmap(want, m_len); 1382 } 1383 return ret; 1384 } 1385 assert(test == want); 1386 1387 last = shmaddr + m_len - 1; 1388 page_set_flags(shmaddr, last, 1389 PAGE_VALID | PAGE_RESET | PAGE_READ | 1390 (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE) | 1391 (shmflg & SHM_EXEC ? PAGE_EXEC : 0)); 1392 1393 shm_region_rm_complete(shmaddr, last); 1394 shm_region_add(shmaddr, last); 1395 } 1396 1397 /* 1398 * We're mapping shared memory, so ensure we generate code for parallel 1399 * execution and flush old translations. This will work up to the level 1400 * supported by the host -- anything that requires EXCP_ATOMIC will not 1401 * be atomic with respect to an external process. 1402 */ 1403 if (!tcg_cflags_has(cpu, CF_PARALLEL)) { 1404 tcg_cflags_set(cpu, CF_PARALLEL); 1405 tb_flush(cpu); 1406 } 1407 1408 if (qemu_loglevel_mask(CPU_LOG_PAGE)) { 1409 FILE *f = qemu_log_trylock(); 1410 if (f) { 1411 fprintf(f, "page layout changed following shmat\n"); 1412 page_dump(f); 1413 qemu_log_unlock(f); 1414 } 1415 } 1416 return shmaddr; 1417 } 1418 1419 abi_long target_shmdt(abi_ulong shmaddr) 1420 { 1421 abi_long rv; 1422 1423 /* shmdt pointers are always untagged */ 1424 1425 WITH_MMAP_LOCK_GUARD() { 1426 abi_ulong last = shm_region_find(shmaddr); 1427 if (last == 0) { 1428 return -TARGET_EINVAL; 1429 } 1430 1431 rv = get_errno(shmdt(g2h_untagged(shmaddr))); 1432 if (rv == 0) { 1433 abi_ulong size = last - shmaddr + 1; 1434 1435 page_set_flags(shmaddr, last, 0); 1436 shm_region_rm_complete(shmaddr, last); 1437 mmap_reserve_or_unmap(shmaddr, size); 1438 } 1439 } 1440 return rv; 1441 } 1442