1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. 5 * <benh@kernel.crashing.org> 6 */ 7 8 #include <linux/errno.h> 9 #include <linux/sched.h> 10 #include <linux/kernel.h> 11 #include <linux/mm.h> 12 #include <linux/smp.h> 13 #include <linux/stddef.h> 14 #include <linux/unistd.h> 15 #include <linux/slab.h> 16 #include <linux/user.h> 17 #include <linux/elf.h> 18 #include <linux/security.h> 19 #include <linux/memblock.h> 20 21 #include <asm/pgtable.h> 22 #include <asm/processor.h> 23 #include <asm/mmu.h> 24 #include <asm/mmu_context.h> 25 #include <asm/prom.h> 26 #include <asm/machdep.h> 27 #include <asm/cputable.h> 28 #include <asm/sections.h> 29 #include <asm/firmware.h> 30 #include <asm/vdso.h> 31 #include <asm/vdso_datapage.h> 32 #include <asm/setup.h> 33 34 #undef DEBUG 35 36 #ifdef DEBUG 37 #define DBG(fmt...) printk(fmt) 38 #else 39 #define DBG(fmt...) 40 #endif 41 42 /* Max supported size for symbol names */ 43 #define MAX_SYMNAME 64 44 45 /* The alignment of the vDSO */ 46 #define VDSO_ALIGNMENT (1 << 16) 47 48 static unsigned int vdso32_pages; 49 static void *vdso32_kbase; 50 static struct page **vdso32_pagelist; 51 unsigned long vdso32_sigtramp; 52 unsigned long vdso32_rt_sigtramp; 53 54 #ifdef CONFIG_VDSO32 55 extern char vdso32_start, vdso32_end; 56 #endif 57 58 #ifdef CONFIG_PPC64 59 extern char vdso64_start, vdso64_end; 60 static void *vdso64_kbase = &vdso64_start; 61 static unsigned int vdso64_pages; 62 static struct page **vdso64_pagelist; 63 unsigned long vdso64_rt_sigtramp; 64 #endif /* CONFIG_PPC64 */ 65 66 static int vdso_ready; 67 68 /* 69 * The vdso data page (aka. systemcfg for old ppc64 fans) is here. 70 * Once the early boot kernel code no longer needs to muck around 71 * with it, it will become dynamically allocated 72 */ 73 static union { 74 struct vdso_data data; 75 u8 page[PAGE_SIZE]; 76 } vdso_data_store __page_aligned_data; 77 struct vdso_data *vdso_data = &vdso_data_store.data; 78 79 /* Format of the patch table */ 80 struct vdso_patch_def 81 { 82 unsigned long ftr_mask, ftr_value; 83 const char *gen_name; 84 const char *fix_name; 85 }; 86 87 /* Table of functions to patch based on the CPU type/revision 88 * 89 * Currently, we only change sync_dicache to do nothing on processors 90 * with a coherent icache 91 */ 92 static struct vdso_patch_def vdso_patches[] = { 93 { 94 CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE, 95 "__kernel_sync_dicache", "__kernel_sync_dicache_p5" 96 }, 97 }; 98 99 /* 100 * Some infos carried around for each of them during parsing at 101 * boot time. 102 */ 103 struct lib32_elfinfo 104 { 105 Elf32_Ehdr *hdr; /* ptr to ELF */ 106 Elf32_Sym *dynsym; /* ptr to .dynsym section */ 107 unsigned long dynsymsize; /* size of .dynsym section */ 108 char *dynstr; /* ptr to .dynstr section */ 109 unsigned long text; /* offset of .text section in .so */ 110 }; 111 112 struct lib64_elfinfo 113 { 114 Elf64_Ehdr *hdr; 115 Elf64_Sym *dynsym; 116 unsigned long dynsymsize; 117 char *dynstr; 118 unsigned long text; 119 }; 120 121 122 /* 123 * This is called from binfmt_elf, we create the special vma for the 124 * vDSO and insert it into the mm struct tree 125 */ 126 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 127 { 128 struct mm_struct *mm = current->mm; 129 struct page **vdso_pagelist; 130 unsigned long vdso_pages; 131 unsigned long vdso_base; 132 int rc; 133 134 if (!vdso_ready) 135 return 0; 136 137 #ifdef CONFIG_PPC64 138 if (is_32bit_task()) { 139 vdso_pagelist = vdso32_pagelist; 140 vdso_pages = vdso32_pages; 141 vdso_base = VDSO32_MBASE; 142 } else { 143 vdso_pagelist = vdso64_pagelist; 144 vdso_pages = vdso64_pages; 145 /* 146 * On 64bit we don't have a preferred map address. This 147 * allows get_unmapped_area to find an area near other mmaps 148 * and most likely share a SLB entry. 149 */ 150 vdso_base = 0; 151 } 152 #else 153 vdso_pagelist = vdso32_pagelist; 154 vdso_pages = vdso32_pages; 155 vdso_base = VDSO32_MBASE; 156 #endif 157 158 current->mm->context.vdso_base = 0; 159 160 /* vDSO has a problem and was disabled, just don't "enable" it for the 161 * process 162 */ 163 if (vdso_pages == 0) 164 return 0; 165 /* Add a page to the vdso size for the data page */ 166 vdso_pages ++; 167 168 /* 169 * pick a base address for the vDSO in process space. We try to put it 170 * at vdso_base which is the "natural" base for it, but we might fail 171 * and end up putting it elsewhere. 172 * Add enough to the size so that the result can be aligned. 173 */ 174 if (down_write_killable(&mm->mmap_sem)) 175 return -EINTR; 176 vdso_base = get_unmapped_area(NULL, vdso_base, 177 (vdso_pages << PAGE_SHIFT) + 178 ((VDSO_ALIGNMENT - 1) & PAGE_MASK), 179 0, 0); 180 if (IS_ERR_VALUE(vdso_base)) { 181 rc = vdso_base; 182 goto fail_mmapsem; 183 } 184 185 /* Add required alignment. */ 186 vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT); 187 188 /* 189 * Put vDSO base into mm struct. We need to do this before calling 190 * install_special_mapping or the perf counter mmap tracking code 191 * will fail to recognise it as a vDSO (since arch_vma_name fails). 192 */ 193 current->mm->context.vdso_base = vdso_base; 194 195 /* 196 * our vma flags don't have VM_WRITE so by default, the process isn't 197 * allowed to write those pages. 198 * gdb can break that with ptrace interface, and thus trigger COW on 199 * those pages but it's then your responsibility to never do that on 200 * the "data" page of the vDSO or you'll stop getting kernel updates 201 * and your nice userland gettimeofday will be totally dead. 202 * It's fine to use that for setting breakpoints in the vDSO code 203 * pages though. 204 */ 205 rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, 206 VM_READ|VM_EXEC| 207 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, 208 vdso_pagelist); 209 if (rc) { 210 current->mm->context.vdso_base = 0; 211 goto fail_mmapsem; 212 } 213 214 up_write(&mm->mmap_sem); 215 return 0; 216 217 fail_mmapsem: 218 up_write(&mm->mmap_sem); 219 return rc; 220 } 221 222 const char *arch_vma_name(struct vm_area_struct *vma) 223 { 224 if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base) 225 return "[vdso]"; 226 return NULL; 227 } 228 229 230 231 #ifdef CONFIG_VDSO32 232 static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname, 233 unsigned long *size) 234 { 235 Elf32_Shdr *sechdrs; 236 unsigned int i; 237 char *secnames; 238 239 /* Grab section headers and strings so we can tell who is who */ 240 sechdrs = (void *)ehdr + ehdr->e_shoff; 241 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; 242 243 /* Find the section they want */ 244 for (i = 1; i < ehdr->e_shnum; i++) { 245 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { 246 if (size) 247 *size = sechdrs[i].sh_size; 248 return (void *)ehdr + sechdrs[i].sh_offset; 249 } 250 } 251 *size = 0; 252 return NULL; 253 } 254 255 static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib, 256 const char *symname) 257 { 258 unsigned int i; 259 char name[MAX_SYMNAME], *c; 260 261 for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) { 262 if (lib->dynsym[i].st_name == 0) 263 continue; 264 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, 265 MAX_SYMNAME); 266 c = strchr(name, '@'); 267 if (c) 268 *c = 0; 269 if (strcmp(symname, name) == 0) 270 return &lib->dynsym[i]; 271 } 272 return NULL; 273 } 274 275 /* Note that we assume the section is .text and the symbol is relative to 276 * the library base 277 */ 278 static unsigned long __init find_function32(struct lib32_elfinfo *lib, 279 const char *symname) 280 { 281 Elf32_Sym *sym = find_symbol32(lib, symname); 282 283 if (sym == NULL) { 284 printk(KERN_WARNING "vDSO32: function %s not found !\n", 285 symname); 286 return 0; 287 } 288 return sym->st_value - VDSO32_LBASE; 289 } 290 291 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32, 292 struct lib64_elfinfo *v64, 293 const char *orig, const char *fix) 294 { 295 Elf32_Sym *sym32_gen, *sym32_fix; 296 297 sym32_gen = find_symbol32(v32, orig); 298 if (sym32_gen == NULL) { 299 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig); 300 return -1; 301 } 302 if (fix == NULL) { 303 sym32_gen->st_name = 0; 304 return 0; 305 } 306 sym32_fix = find_symbol32(v32, fix); 307 if (sym32_fix == NULL) { 308 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix); 309 return -1; 310 } 311 sym32_gen->st_value = sym32_fix->st_value; 312 sym32_gen->st_size = sym32_fix->st_size; 313 sym32_gen->st_info = sym32_fix->st_info; 314 sym32_gen->st_other = sym32_fix->st_other; 315 sym32_gen->st_shndx = sym32_fix->st_shndx; 316 317 return 0; 318 } 319 #else /* !CONFIG_VDSO32 */ 320 static unsigned long __init find_function32(struct lib32_elfinfo *lib, 321 const char *symname) 322 { 323 return 0; 324 } 325 326 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32, 327 struct lib64_elfinfo *v64, 328 const char *orig, const char *fix) 329 { 330 return 0; 331 } 332 #endif /* CONFIG_VDSO32 */ 333 334 335 #ifdef CONFIG_PPC64 336 337 static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname, 338 unsigned long *size) 339 { 340 Elf64_Shdr *sechdrs; 341 unsigned int i; 342 char *secnames; 343 344 /* Grab section headers and strings so we can tell who is who */ 345 sechdrs = (void *)ehdr + ehdr->e_shoff; 346 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; 347 348 /* Find the section they want */ 349 for (i = 1; i < ehdr->e_shnum; i++) { 350 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { 351 if (size) 352 *size = sechdrs[i].sh_size; 353 return (void *)ehdr + sechdrs[i].sh_offset; 354 } 355 } 356 if (size) 357 *size = 0; 358 return NULL; 359 } 360 361 static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib, 362 const char *symname) 363 { 364 unsigned int i; 365 char name[MAX_SYMNAME], *c; 366 367 for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) { 368 if (lib->dynsym[i].st_name == 0) 369 continue; 370 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, 371 MAX_SYMNAME); 372 c = strchr(name, '@'); 373 if (c) 374 *c = 0; 375 if (strcmp(symname, name) == 0) 376 return &lib->dynsym[i]; 377 } 378 return NULL; 379 } 380 381 /* Note that we assume the section is .text and the symbol is relative to 382 * the library base 383 */ 384 static unsigned long __init find_function64(struct lib64_elfinfo *lib, 385 const char *symname) 386 { 387 Elf64_Sym *sym = find_symbol64(lib, symname); 388 389 if (sym == NULL) { 390 printk(KERN_WARNING "vDSO64: function %s not found !\n", 391 symname); 392 return 0; 393 } 394 return sym->st_value - VDSO64_LBASE; 395 } 396 397 static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32, 398 struct lib64_elfinfo *v64, 399 const char *orig, const char *fix) 400 { 401 Elf64_Sym *sym64_gen, *sym64_fix; 402 403 sym64_gen = find_symbol64(v64, orig); 404 if (sym64_gen == NULL) { 405 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig); 406 return -1; 407 } 408 if (fix == NULL) { 409 sym64_gen->st_name = 0; 410 return 0; 411 } 412 sym64_fix = find_symbol64(v64, fix); 413 if (sym64_fix == NULL) { 414 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix); 415 return -1; 416 } 417 sym64_gen->st_value = sym64_fix->st_value; 418 sym64_gen->st_size = sym64_fix->st_size; 419 sym64_gen->st_info = sym64_fix->st_info; 420 sym64_gen->st_other = sym64_fix->st_other; 421 sym64_gen->st_shndx = sym64_fix->st_shndx; 422 423 return 0; 424 } 425 426 #endif /* CONFIG_PPC64 */ 427 428 429 static __init int vdso_do_find_sections(struct lib32_elfinfo *v32, 430 struct lib64_elfinfo *v64) 431 { 432 void *sect; 433 434 /* 435 * Locate symbol tables & text section 436 */ 437 438 #ifdef CONFIG_VDSO32 439 v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize); 440 v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL); 441 if (v32->dynsym == NULL || v32->dynstr == NULL) { 442 printk(KERN_ERR "vDSO32: required symbol section not found\n"); 443 return -1; 444 } 445 sect = find_section32(v32->hdr, ".text", NULL); 446 if (sect == NULL) { 447 printk(KERN_ERR "vDSO32: the .text section was not found\n"); 448 return -1; 449 } 450 v32->text = sect - vdso32_kbase; 451 #endif 452 453 #ifdef CONFIG_PPC64 454 v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize); 455 v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL); 456 if (v64->dynsym == NULL || v64->dynstr == NULL) { 457 printk(KERN_ERR "vDSO64: required symbol section not found\n"); 458 return -1; 459 } 460 sect = find_section64(v64->hdr, ".text", NULL); 461 if (sect == NULL) { 462 printk(KERN_ERR "vDSO64: the .text section was not found\n"); 463 return -1; 464 } 465 v64->text = sect - vdso64_kbase; 466 #endif /* CONFIG_PPC64 */ 467 468 return 0; 469 } 470 471 static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32, 472 struct lib64_elfinfo *v64) 473 { 474 /* 475 * Find signal trampolines 476 */ 477 478 #ifdef CONFIG_PPC64 479 vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64"); 480 #endif 481 vdso32_sigtramp = find_function32(v32, "__kernel_sigtramp32"); 482 vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32"); 483 } 484 485 static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32, 486 struct lib64_elfinfo *v64) 487 { 488 #ifdef CONFIG_VDSO32 489 Elf32_Sym *sym32; 490 #endif 491 #ifdef CONFIG_PPC64 492 Elf64_Sym *sym64; 493 494 sym64 = find_symbol64(v64, "__kernel_datapage_offset"); 495 if (sym64 == NULL) { 496 printk(KERN_ERR "vDSO64: Can't find symbol " 497 "__kernel_datapage_offset !\n"); 498 return -1; 499 } 500 *((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) = 501 (vdso64_pages << PAGE_SHIFT) - 502 (sym64->st_value - VDSO64_LBASE); 503 #endif /* CONFIG_PPC64 */ 504 505 #ifdef CONFIG_VDSO32 506 sym32 = find_symbol32(v32, "__kernel_datapage_offset"); 507 if (sym32 == NULL) { 508 printk(KERN_ERR "vDSO32: Can't find symbol " 509 "__kernel_datapage_offset !\n"); 510 return -1; 511 } 512 *((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) = 513 (vdso32_pages << PAGE_SHIFT) - 514 (sym32->st_value - VDSO32_LBASE); 515 #endif 516 517 return 0; 518 } 519 520 521 static __init int vdso_fixup_features(struct lib32_elfinfo *v32, 522 struct lib64_elfinfo *v64) 523 { 524 unsigned long size; 525 void *start; 526 527 #ifdef CONFIG_PPC64 528 start = find_section64(v64->hdr, "__ftr_fixup", &size); 529 if (start) 530 do_feature_fixups(cur_cpu_spec->cpu_features, 531 start, start + size); 532 533 start = find_section64(v64->hdr, "__mmu_ftr_fixup", &size); 534 if (start) 535 do_feature_fixups(cur_cpu_spec->mmu_features, 536 start, start + size); 537 538 start = find_section64(v64->hdr, "__fw_ftr_fixup", &size); 539 if (start) 540 do_feature_fixups(powerpc_firmware_features, 541 start, start + size); 542 543 start = find_section64(v64->hdr, "__lwsync_fixup", &size); 544 if (start) 545 do_lwsync_fixups(cur_cpu_spec->cpu_features, 546 start, start + size); 547 #endif /* CONFIG_PPC64 */ 548 549 #ifdef CONFIG_VDSO32 550 start = find_section32(v32->hdr, "__ftr_fixup", &size); 551 if (start) 552 do_feature_fixups(cur_cpu_spec->cpu_features, 553 start, start + size); 554 555 start = find_section32(v32->hdr, "__mmu_ftr_fixup", &size); 556 if (start) 557 do_feature_fixups(cur_cpu_spec->mmu_features, 558 start, start + size); 559 560 #ifdef CONFIG_PPC64 561 start = find_section32(v32->hdr, "__fw_ftr_fixup", &size); 562 if (start) 563 do_feature_fixups(powerpc_firmware_features, 564 start, start + size); 565 #endif /* CONFIG_PPC64 */ 566 567 start = find_section32(v32->hdr, "__lwsync_fixup", &size); 568 if (start) 569 do_lwsync_fixups(cur_cpu_spec->cpu_features, 570 start, start + size); 571 #endif 572 573 return 0; 574 } 575 576 static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32, 577 struct lib64_elfinfo *v64) 578 { 579 int i; 580 581 for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) { 582 struct vdso_patch_def *patch = &vdso_patches[i]; 583 int match = (cur_cpu_spec->cpu_features & patch->ftr_mask) 584 == patch->ftr_value; 585 if (!match) 586 continue; 587 588 DBG("replacing %s with %s...\n", patch->gen_name, 589 patch->fix_name ? "NONE" : patch->fix_name); 590 591 /* 592 * Patch the 32 bits and 64 bits symbols. Note that we do not 593 * patch the "." symbol on 64 bits. 594 * It would be easy to do, but doesn't seem to be necessary, 595 * patching the OPD symbol is enough. 596 */ 597 vdso_do_func_patch32(v32, v64, patch->gen_name, 598 patch->fix_name); 599 #ifdef CONFIG_PPC64 600 vdso_do_func_patch64(v32, v64, patch->gen_name, 601 patch->fix_name); 602 #endif /* CONFIG_PPC64 */ 603 } 604 605 return 0; 606 } 607 608 609 static __init int vdso_setup(void) 610 { 611 struct lib32_elfinfo v32; 612 struct lib64_elfinfo v64; 613 614 v32.hdr = vdso32_kbase; 615 #ifdef CONFIG_PPC64 616 v64.hdr = vdso64_kbase; 617 #endif 618 if (vdso_do_find_sections(&v32, &v64)) 619 return -1; 620 621 if (vdso_fixup_datapage(&v32, &v64)) 622 return -1; 623 624 if (vdso_fixup_features(&v32, &v64)) 625 return -1; 626 627 if (vdso_fixup_alt_funcs(&v32, &v64)) 628 return -1; 629 630 vdso_setup_trampolines(&v32, &v64); 631 632 return 0; 633 } 634 635 /* 636 * Called from setup_arch to initialize the bitmap of available 637 * syscalls in the systemcfg page 638 */ 639 static void __init vdso_setup_syscall_map(void) 640 { 641 unsigned int i; 642 extern unsigned long *sys_call_table; 643 #ifdef CONFIG_PPC64 644 extern unsigned long *compat_sys_call_table; 645 #endif 646 extern unsigned long sys_ni_syscall; 647 648 649 for (i = 0; i < NR_syscalls; i++) { 650 #ifdef CONFIG_PPC64 651 if (sys_call_table[i] != sys_ni_syscall) 652 vdso_data->syscall_map_64[i >> 5] |= 653 0x80000000UL >> (i & 0x1f); 654 if (IS_ENABLED(CONFIG_COMPAT) && 655 compat_sys_call_table[i] != sys_ni_syscall) 656 vdso_data->syscall_map_32[i >> 5] |= 657 0x80000000UL >> (i & 0x1f); 658 #else /* CONFIG_PPC64 */ 659 if (sys_call_table[i] != sys_ni_syscall) 660 vdso_data->syscall_map_32[i >> 5] |= 661 0x80000000UL >> (i & 0x1f); 662 #endif /* CONFIG_PPC64 */ 663 } 664 } 665 666 #ifdef CONFIG_PPC64 667 int vdso_getcpu_init(void) 668 { 669 unsigned long cpu, node, val; 670 671 /* 672 * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node 673 * in the next 16 bits. The VDSO uses this to implement getcpu(). 674 */ 675 cpu = get_cpu(); 676 WARN_ON_ONCE(cpu > 0xffff); 677 678 node = cpu_to_node(cpu); 679 WARN_ON_ONCE(node > 0xffff); 680 681 val = (cpu & 0xfff) | ((node & 0xffff) << 16); 682 mtspr(SPRN_SPRG_VDSO_WRITE, val); 683 get_paca()->sprg_vdso = val; 684 685 put_cpu(); 686 687 return 0; 688 } 689 /* We need to call this before SMP init */ 690 early_initcall(vdso_getcpu_init); 691 #endif 692 693 static int __init vdso_init(void) 694 { 695 int i; 696 697 #ifdef CONFIG_PPC64 698 /* 699 * Fill up the "systemcfg" stuff for backward compatibility 700 */ 701 strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64"); 702 vdso_data->version.major = SYSTEMCFG_MAJOR; 703 vdso_data->version.minor = SYSTEMCFG_MINOR; 704 vdso_data->processor = mfspr(SPRN_PVR); 705 /* 706 * Fake the old platform number for pSeries and add 707 * in LPAR bit if necessary 708 */ 709 vdso_data->platform = 0x100; 710 if (firmware_has_feature(FW_FEATURE_LPAR)) 711 vdso_data->platform |= 1; 712 vdso_data->physicalMemorySize = memblock_phys_mem_size(); 713 vdso_data->dcache_size = ppc64_caches.l1d.size; 714 vdso_data->dcache_line_size = ppc64_caches.l1d.line_size; 715 vdso_data->icache_size = ppc64_caches.l1i.size; 716 vdso_data->icache_line_size = ppc64_caches.l1i.line_size; 717 vdso_data->dcache_block_size = ppc64_caches.l1d.block_size; 718 vdso_data->icache_block_size = ppc64_caches.l1i.block_size; 719 vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; 720 vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; 721 722 /* 723 * Calculate the size of the 64 bits vDSO 724 */ 725 vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT; 726 DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages); 727 #endif /* CONFIG_PPC64 */ 728 729 730 #ifdef CONFIG_VDSO32 731 vdso32_kbase = &vdso32_start; 732 733 /* 734 * Calculate the size of the 32 bits vDSO 735 */ 736 vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT; 737 DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages); 738 #endif 739 740 741 /* 742 * Setup the syscall map in the vDOS 743 */ 744 vdso_setup_syscall_map(); 745 746 /* 747 * Initialize the vDSO images in memory, that is do necessary 748 * fixups of vDSO symbols, locate trampolines, etc... 749 */ 750 if (vdso_setup()) { 751 printk(KERN_ERR "vDSO setup failure, not enabled !\n"); 752 vdso32_pages = 0; 753 #ifdef CONFIG_PPC64 754 vdso64_pages = 0; 755 #endif 756 return 0; 757 } 758 759 #ifdef CONFIG_VDSO32 760 /* Make sure pages are in the correct state */ 761 vdso32_pagelist = kcalloc(vdso32_pages + 2, sizeof(struct page *), 762 GFP_KERNEL); 763 BUG_ON(vdso32_pagelist == NULL); 764 for (i = 0; i < vdso32_pages; i++) { 765 struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE); 766 get_page(pg); 767 vdso32_pagelist[i] = pg; 768 } 769 vdso32_pagelist[i++] = virt_to_page(vdso_data); 770 vdso32_pagelist[i] = NULL; 771 #endif 772 773 #ifdef CONFIG_PPC64 774 vdso64_pagelist = kcalloc(vdso64_pages + 2, sizeof(struct page *), 775 GFP_KERNEL); 776 BUG_ON(vdso64_pagelist == NULL); 777 for (i = 0; i < vdso64_pages; i++) { 778 struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE); 779 get_page(pg); 780 vdso64_pagelist[i] = pg; 781 } 782 vdso64_pagelist[i++] = virt_to_page(vdso_data); 783 vdso64_pagelist[i] = NULL; 784 #endif /* CONFIG_PPC64 */ 785 786 get_page(virt_to_page(vdso_data)); 787 788 smp_wmb(); 789 vdso_ready = 1; 790 791 return 0; 792 } 793 arch_initcall(vdso_init); 794