1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. 5 * <benh@kernel.crashing.org> 6 */ 7 8 #include <linux/errno.h> 9 #include <linux/sched.h> 10 #include <linux/kernel.h> 11 #include <linux/mm.h> 12 #include <linux/smp.h> 13 #include <linux/stddef.h> 14 #include <linux/unistd.h> 15 #include <linux/slab.h> 16 #include <linux/user.h> 17 #include <linux/elf.h> 18 #include <linux/security.h> 19 #include <linux/memblock.h> 20 21 #include <asm/pgtable.h> 22 #include <asm/processor.h> 23 #include <asm/mmu.h> 24 #include <asm/mmu_context.h> 25 #include <asm/prom.h> 26 #include <asm/machdep.h> 27 #include <asm/cputable.h> 28 #include <asm/sections.h> 29 #include <asm/firmware.h> 30 #include <asm/vdso.h> 31 #include <asm/vdso_datapage.h> 32 #include <asm/setup.h> 33 34 #undef DEBUG 35 36 #ifdef DEBUG 37 #define DBG(fmt...) printk(fmt) 38 #else 39 #define DBG(fmt...) 40 #endif 41 42 /* Max supported size for symbol names */ 43 #define MAX_SYMNAME 64 44 45 /* The alignment of the vDSO */ 46 #define VDSO_ALIGNMENT (1 << 16) 47 48 static unsigned int vdso32_pages; 49 static void *vdso32_kbase; 50 static struct page **vdso32_pagelist; 51 unsigned long vdso32_sigtramp; 52 unsigned long vdso32_rt_sigtramp; 53 54 #ifdef CONFIG_VDSO32 55 extern char vdso32_start, vdso32_end; 56 #endif 57 58 #ifdef CONFIG_PPC64 59 extern char vdso64_start, vdso64_end; 60 static void *vdso64_kbase = &vdso64_start; 61 static unsigned int vdso64_pages; 62 static struct page **vdso64_pagelist; 63 unsigned long vdso64_rt_sigtramp; 64 #endif /* CONFIG_PPC64 */ 65 66 static int vdso_ready; 67 68 /* 69 * The vdso data page (aka. systemcfg for old ppc64 fans) is here. 70 * Once the early boot kernel code no longer needs to muck around 71 * with it, it will become dynamically allocated 72 */ 73 static union { 74 struct vdso_data data; 75 u8 page[PAGE_SIZE]; 76 } vdso_data_store __page_aligned_data; 77 struct vdso_data *vdso_data = &vdso_data_store.data; 78 79 /* Format of the patch table */ 80 struct vdso_patch_def 81 { 82 unsigned long ftr_mask, ftr_value; 83 const char *gen_name; 84 const char *fix_name; 85 }; 86 87 /* Table of functions to patch based on the CPU type/revision 88 * 89 * Currently, we only change sync_dicache to do nothing on processors 90 * with a coherent icache 91 */ 92 static struct vdso_patch_def vdso_patches[] = { 93 { 94 CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE, 95 "__kernel_sync_dicache", "__kernel_sync_dicache_p5" 96 }, 97 #ifdef CONFIG_PPC32 98 { 99 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC, 100 "__kernel_gettimeofday", NULL 101 }, 102 { 103 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC, 104 "__kernel_clock_gettime", NULL 105 }, 106 { 107 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC, 108 "__kernel_clock_getres", NULL 109 }, 110 { 111 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC, 112 "__kernel_get_tbfreq", NULL 113 }, 114 { 115 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC, 116 "__kernel_time", NULL 117 }, 118 #endif 119 }; 120 121 /* 122 * Some infos carried around for each of them during parsing at 123 * boot time. 124 */ 125 struct lib32_elfinfo 126 { 127 Elf32_Ehdr *hdr; /* ptr to ELF */ 128 Elf32_Sym *dynsym; /* ptr to .dynsym section */ 129 unsigned long dynsymsize; /* size of .dynsym section */ 130 char *dynstr; /* ptr to .dynstr section */ 131 unsigned long text; /* offset of .text section in .so */ 132 }; 133 134 struct lib64_elfinfo 135 { 136 Elf64_Ehdr *hdr; 137 Elf64_Sym *dynsym; 138 unsigned long dynsymsize; 139 char *dynstr; 140 unsigned long text; 141 }; 142 143 144 /* 145 * This is called from binfmt_elf, we create the special vma for the 146 * vDSO and insert it into the mm struct tree 147 */ 148 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 149 { 150 struct mm_struct *mm = current->mm; 151 struct page **vdso_pagelist; 152 unsigned long vdso_pages; 153 unsigned long vdso_base; 154 int rc; 155 156 if (!vdso_ready) 157 return 0; 158 159 #ifdef CONFIG_PPC64 160 if (is_32bit_task()) { 161 vdso_pagelist = vdso32_pagelist; 162 vdso_pages = vdso32_pages; 163 vdso_base = VDSO32_MBASE; 164 } else { 165 vdso_pagelist = vdso64_pagelist; 166 vdso_pages = vdso64_pages; 167 /* 168 * On 64bit we don't have a preferred map address. This 169 * allows get_unmapped_area to find an area near other mmaps 170 * and most likely share a SLB entry. 171 */ 172 vdso_base = 0; 173 } 174 #else 175 vdso_pagelist = vdso32_pagelist; 176 vdso_pages = vdso32_pages; 177 vdso_base = VDSO32_MBASE; 178 #endif 179 180 current->mm->context.vdso_base = 0; 181 182 /* vDSO has a problem and was disabled, just don't "enable" it for the 183 * process 184 */ 185 if (vdso_pages == 0) 186 return 0; 187 /* Add a page to the vdso size for the data page */ 188 vdso_pages ++; 189 190 /* 191 * pick a base address for the vDSO in process space. We try to put it 192 * at vdso_base which is the "natural" base for it, but we might fail 193 * and end up putting it elsewhere. 194 * Add enough to the size so that the result can be aligned. 195 */ 196 if (down_write_killable(&mm->mmap_sem)) 197 return -EINTR; 198 vdso_base = get_unmapped_area(NULL, vdso_base, 199 (vdso_pages << PAGE_SHIFT) + 200 ((VDSO_ALIGNMENT - 1) & PAGE_MASK), 201 0, 0); 202 if (IS_ERR_VALUE(vdso_base)) { 203 rc = vdso_base; 204 goto fail_mmapsem; 205 } 206 207 /* Add required alignment. */ 208 vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT); 209 210 /* 211 * Put vDSO base into mm struct. We need to do this before calling 212 * install_special_mapping or the perf counter mmap tracking code 213 * will fail to recognise it as a vDSO (since arch_vma_name fails). 214 */ 215 current->mm->context.vdso_base = vdso_base; 216 217 /* 218 * our vma flags don't have VM_WRITE so by default, the process isn't 219 * allowed to write those pages. 220 * gdb can break that with ptrace interface, and thus trigger COW on 221 * those pages but it's then your responsibility to never do that on 222 * the "data" page of the vDSO or you'll stop getting kernel updates 223 * and your nice userland gettimeofday will be totally dead. 224 * It's fine to use that for setting breakpoints in the vDSO code 225 * pages though. 226 */ 227 rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, 228 VM_READ|VM_EXEC| 229 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, 230 vdso_pagelist); 231 if (rc) { 232 current->mm->context.vdso_base = 0; 233 goto fail_mmapsem; 234 } 235 236 up_write(&mm->mmap_sem); 237 return 0; 238 239 fail_mmapsem: 240 up_write(&mm->mmap_sem); 241 return rc; 242 } 243 244 const char *arch_vma_name(struct vm_area_struct *vma) 245 { 246 if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base) 247 return "[vdso]"; 248 return NULL; 249 } 250 251 252 253 #ifdef CONFIG_VDSO32 254 static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname, 255 unsigned long *size) 256 { 257 Elf32_Shdr *sechdrs; 258 unsigned int i; 259 char *secnames; 260 261 /* Grab section headers and strings so we can tell who is who */ 262 sechdrs = (void *)ehdr + ehdr->e_shoff; 263 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; 264 265 /* Find the section they want */ 266 for (i = 1; i < ehdr->e_shnum; i++) { 267 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { 268 if (size) 269 *size = sechdrs[i].sh_size; 270 return (void *)ehdr + sechdrs[i].sh_offset; 271 } 272 } 273 *size = 0; 274 return NULL; 275 } 276 277 static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib, 278 const char *symname) 279 { 280 unsigned int i; 281 char name[MAX_SYMNAME], *c; 282 283 for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) { 284 if (lib->dynsym[i].st_name == 0) 285 continue; 286 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, 287 MAX_SYMNAME); 288 c = strchr(name, '@'); 289 if (c) 290 *c = 0; 291 if (strcmp(symname, name) == 0) 292 return &lib->dynsym[i]; 293 } 294 return NULL; 295 } 296 297 /* Note that we assume the section is .text and the symbol is relative to 298 * the library base 299 */ 300 static unsigned long __init find_function32(struct lib32_elfinfo *lib, 301 const char *symname) 302 { 303 Elf32_Sym *sym = find_symbol32(lib, symname); 304 305 if (sym == NULL) { 306 printk(KERN_WARNING "vDSO32: function %s not found !\n", 307 symname); 308 return 0; 309 } 310 return sym->st_value - VDSO32_LBASE; 311 } 312 313 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32, 314 struct lib64_elfinfo *v64, 315 const char *orig, const char *fix) 316 { 317 Elf32_Sym *sym32_gen, *sym32_fix; 318 319 sym32_gen = find_symbol32(v32, orig); 320 if (sym32_gen == NULL) { 321 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig); 322 return -1; 323 } 324 if (fix == NULL) { 325 sym32_gen->st_name = 0; 326 return 0; 327 } 328 sym32_fix = find_symbol32(v32, fix); 329 if (sym32_fix == NULL) { 330 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix); 331 return -1; 332 } 333 sym32_gen->st_value = sym32_fix->st_value; 334 sym32_gen->st_size = sym32_fix->st_size; 335 sym32_gen->st_info = sym32_fix->st_info; 336 sym32_gen->st_other = sym32_fix->st_other; 337 sym32_gen->st_shndx = sym32_fix->st_shndx; 338 339 return 0; 340 } 341 #else /* !CONFIG_VDSO32 */ 342 static unsigned long __init find_function32(struct lib32_elfinfo *lib, 343 const char *symname) 344 { 345 return 0; 346 } 347 348 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32, 349 struct lib64_elfinfo *v64, 350 const char *orig, const char *fix) 351 { 352 return 0; 353 } 354 #endif /* CONFIG_VDSO32 */ 355 356 357 #ifdef CONFIG_PPC64 358 359 static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname, 360 unsigned long *size) 361 { 362 Elf64_Shdr *sechdrs; 363 unsigned int i; 364 char *secnames; 365 366 /* Grab section headers and strings so we can tell who is who */ 367 sechdrs = (void *)ehdr + ehdr->e_shoff; 368 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; 369 370 /* Find the section they want */ 371 for (i = 1; i < ehdr->e_shnum; i++) { 372 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { 373 if (size) 374 *size = sechdrs[i].sh_size; 375 return (void *)ehdr + sechdrs[i].sh_offset; 376 } 377 } 378 if (size) 379 *size = 0; 380 return NULL; 381 } 382 383 static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib, 384 const char *symname) 385 { 386 unsigned int i; 387 char name[MAX_SYMNAME], *c; 388 389 for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) { 390 if (lib->dynsym[i].st_name == 0) 391 continue; 392 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, 393 MAX_SYMNAME); 394 c = strchr(name, '@'); 395 if (c) 396 *c = 0; 397 if (strcmp(symname, name) == 0) 398 return &lib->dynsym[i]; 399 } 400 return NULL; 401 } 402 403 /* Note that we assume the section is .text and the symbol is relative to 404 * the library base 405 */ 406 static unsigned long __init find_function64(struct lib64_elfinfo *lib, 407 const char *symname) 408 { 409 Elf64_Sym *sym = find_symbol64(lib, symname); 410 411 if (sym == NULL) { 412 printk(KERN_WARNING "vDSO64: function %s not found !\n", 413 symname); 414 return 0; 415 } 416 #ifdef VDS64_HAS_DESCRIPTORS 417 return *((u64 *)(vdso64_kbase + sym->st_value - VDSO64_LBASE)) - 418 VDSO64_LBASE; 419 #else 420 return sym->st_value - VDSO64_LBASE; 421 #endif 422 } 423 424 static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32, 425 struct lib64_elfinfo *v64, 426 const char *orig, const char *fix) 427 { 428 Elf64_Sym *sym64_gen, *sym64_fix; 429 430 sym64_gen = find_symbol64(v64, orig); 431 if (sym64_gen == NULL) { 432 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig); 433 return -1; 434 } 435 if (fix == NULL) { 436 sym64_gen->st_name = 0; 437 return 0; 438 } 439 sym64_fix = find_symbol64(v64, fix); 440 if (sym64_fix == NULL) { 441 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix); 442 return -1; 443 } 444 sym64_gen->st_value = sym64_fix->st_value; 445 sym64_gen->st_size = sym64_fix->st_size; 446 sym64_gen->st_info = sym64_fix->st_info; 447 sym64_gen->st_other = sym64_fix->st_other; 448 sym64_gen->st_shndx = sym64_fix->st_shndx; 449 450 return 0; 451 } 452 453 #endif /* CONFIG_PPC64 */ 454 455 456 static __init int vdso_do_find_sections(struct lib32_elfinfo *v32, 457 struct lib64_elfinfo *v64) 458 { 459 void *sect; 460 461 /* 462 * Locate symbol tables & text section 463 */ 464 465 #ifdef CONFIG_VDSO32 466 v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize); 467 v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL); 468 if (v32->dynsym == NULL || v32->dynstr == NULL) { 469 printk(KERN_ERR "vDSO32: required symbol section not found\n"); 470 return -1; 471 } 472 sect = find_section32(v32->hdr, ".text", NULL); 473 if (sect == NULL) { 474 printk(KERN_ERR "vDSO32: the .text section was not found\n"); 475 return -1; 476 } 477 v32->text = sect - vdso32_kbase; 478 #endif 479 480 #ifdef CONFIG_PPC64 481 v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize); 482 v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL); 483 if (v64->dynsym == NULL || v64->dynstr == NULL) { 484 printk(KERN_ERR "vDSO64: required symbol section not found\n"); 485 return -1; 486 } 487 sect = find_section64(v64->hdr, ".text", NULL); 488 if (sect == NULL) { 489 printk(KERN_ERR "vDSO64: the .text section was not found\n"); 490 return -1; 491 } 492 v64->text = sect - vdso64_kbase; 493 #endif /* CONFIG_PPC64 */ 494 495 return 0; 496 } 497 498 static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32, 499 struct lib64_elfinfo *v64) 500 { 501 /* 502 * Find signal trampolines 503 */ 504 505 #ifdef CONFIG_PPC64 506 vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64"); 507 #endif 508 vdso32_sigtramp = find_function32(v32, "__kernel_sigtramp32"); 509 vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32"); 510 } 511 512 static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32, 513 struct lib64_elfinfo *v64) 514 { 515 #ifdef CONFIG_VDSO32 516 Elf32_Sym *sym32; 517 #endif 518 #ifdef CONFIG_PPC64 519 Elf64_Sym *sym64; 520 521 sym64 = find_symbol64(v64, "__kernel_datapage_offset"); 522 if (sym64 == NULL) { 523 printk(KERN_ERR "vDSO64: Can't find symbol " 524 "__kernel_datapage_offset !\n"); 525 return -1; 526 } 527 *((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) = 528 (vdso64_pages << PAGE_SHIFT) - 529 (sym64->st_value - VDSO64_LBASE); 530 #endif /* CONFIG_PPC64 */ 531 532 #ifdef CONFIG_VDSO32 533 sym32 = find_symbol32(v32, "__kernel_datapage_offset"); 534 if (sym32 == NULL) { 535 printk(KERN_ERR "vDSO32: Can't find symbol " 536 "__kernel_datapage_offset !\n"); 537 return -1; 538 } 539 *((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) = 540 (vdso32_pages << PAGE_SHIFT) - 541 (sym32->st_value - VDSO32_LBASE); 542 #endif 543 544 return 0; 545 } 546 547 548 static __init int vdso_fixup_features(struct lib32_elfinfo *v32, 549 struct lib64_elfinfo *v64) 550 { 551 unsigned long size; 552 void *start; 553 554 #ifdef CONFIG_PPC64 555 start = find_section64(v64->hdr, "__ftr_fixup", &size); 556 if (start) 557 do_feature_fixups(cur_cpu_spec->cpu_features, 558 start, start + size); 559 560 start = find_section64(v64->hdr, "__mmu_ftr_fixup", &size); 561 if (start) 562 do_feature_fixups(cur_cpu_spec->mmu_features, 563 start, start + size); 564 565 start = find_section64(v64->hdr, "__fw_ftr_fixup", &size); 566 if (start) 567 do_feature_fixups(powerpc_firmware_features, 568 start, start + size); 569 570 start = find_section64(v64->hdr, "__lwsync_fixup", &size); 571 if (start) 572 do_lwsync_fixups(cur_cpu_spec->cpu_features, 573 start, start + size); 574 #endif /* CONFIG_PPC64 */ 575 576 #ifdef CONFIG_VDSO32 577 start = find_section32(v32->hdr, "__ftr_fixup", &size); 578 if (start) 579 do_feature_fixups(cur_cpu_spec->cpu_features, 580 start, start + size); 581 582 start = find_section32(v32->hdr, "__mmu_ftr_fixup", &size); 583 if (start) 584 do_feature_fixups(cur_cpu_spec->mmu_features, 585 start, start + size); 586 587 #ifdef CONFIG_PPC64 588 start = find_section32(v32->hdr, "__fw_ftr_fixup", &size); 589 if (start) 590 do_feature_fixups(powerpc_firmware_features, 591 start, start + size); 592 #endif /* CONFIG_PPC64 */ 593 594 start = find_section32(v32->hdr, "__lwsync_fixup", &size); 595 if (start) 596 do_lwsync_fixups(cur_cpu_spec->cpu_features, 597 start, start + size); 598 #endif 599 600 return 0; 601 } 602 603 static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32, 604 struct lib64_elfinfo *v64) 605 { 606 int i; 607 608 for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) { 609 struct vdso_patch_def *patch = &vdso_patches[i]; 610 int match = (cur_cpu_spec->cpu_features & patch->ftr_mask) 611 == patch->ftr_value; 612 if (!match) 613 continue; 614 615 DBG("replacing %s with %s...\n", patch->gen_name, 616 patch->fix_name ? "NONE" : patch->fix_name); 617 618 /* 619 * Patch the 32 bits and 64 bits symbols. Note that we do not 620 * patch the "." symbol on 64 bits. 621 * It would be easy to do, but doesn't seem to be necessary, 622 * patching the OPD symbol is enough. 623 */ 624 vdso_do_func_patch32(v32, v64, patch->gen_name, 625 patch->fix_name); 626 #ifdef CONFIG_PPC64 627 vdso_do_func_patch64(v32, v64, patch->gen_name, 628 patch->fix_name); 629 #endif /* CONFIG_PPC64 */ 630 } 631 632 return 0; 633 } 634 635 636 static __init int vdso_setup(void) 637 { 638 struct lib32_elfinfo v32; 639 struct lib64_elfinfo v64; 640 641 v32.hdr = vdso32_kbase; 642 #ifdef CONFIG_PPC64 643 v64.hdr = vdso64_kbase; 644 #endif 645 if (vdso_do_find_sections(&v32, &v64)) 646 return -1; 647 648 if (vdso_fixup_datapage(&v32, &v64)) 649 return -1; 650 651 if (vdso_fixup_features(&v32, &v64)) 652 return -1; 653 654 if (vdso_fixup_alt_funcs(&v32, &v64)) 655 return -1; 656 657 vdso_setup_trampolines(&v32, &v64); 658 659 return 0; 660 } 661 662 /* 663 * Called from setup_arch to initialize the bitmap of available 664 * syscalls in the systemcfg page 665 */ 666 static void __init vdso_setup_syscall_map(void) 667 { 668 unsigned int i; 669 extern unsigned long *sys_call_table; 670 #ifdef CONFIG_PPC64 671 extern unsigned long *compat_sys_call_table; 672 #endif 673 extern unsigned long sys_ni_syscall; 674 675 676 for (i = 0; i < NR_syscalls; i++) { 677 #ifdef CONFIG_PPC64 678 if (sys_call_table[i] != sys_ni_syscall) 679 vdso_data->syscall_map_64[i >> 5] |= 680 0x80000000UL >> (i & 0x1f); 681 if (compat_sys_call_table[i] != sys_ni_syscall) 682 vdso_data->syscall_map_32[i >> 5] |= 683 0x80000000UL >> (i & 0x1f); 684 #else /* CONFIG_PPC64 */ 685 if (sys_call_table[i] != sys_ni_syscall) 686 vdso_data->syscall_map_32[i >> 5] |= 687 0x80000000UL >> (i & 0x1f); 688 #endif /* CONFIG_PPC64 */ 689 } 690 } 691 692 #ifdef CONFIG_PPC64 693 int vdso_getcpu_init(void) 694 { 695 unsigned long cpu, node, val; 696 697 /* 698 * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node 699 * in the next 16 bits. The VDSO uses this to implement getcpu(). 700 */ 701 cpu = get_cpu(); 702 WARN_ON_ONCE(cpu > 0xffff); 703 704 node = cpu_to_node(cpu); 705 WARN_ON_ONCE(node > 0xffff); 706 707 val = (cpu & 0xfff) | ((node & 0xffff) << 16); 708 mtspr(SPRN_SPRG_VDSO_WRITE, val); 709 get_paca()->sprg_vdso = val; 710 711 put_cpu(); 712 713 return 0; 714 } 715 /* We need to call this before SMP init */ 716 early_initcall(vdso_getcpu_init); 717 #endif 718 719 static int __init vdso_init(void) 720 { 721 int i; 722 723 #ifdef CONFIG_PPC64 724 /* 725 * Fill up the "systemcfg" stuff for backward compatibility 726 */ 727 strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64"); 728 vdso_data->version.major = SYSTEMCFG_MAJOR; 729 vdso_data->version.minor = SYSTEMCFG_MINOR; 730 vdso_data->processor = mfspr(SPRN_PVR); 731 /* 732 * Fake the old platform number for pSeries and add 733 * in LPAR bit if necessary 734 */ 735 vdso_data->platform = 0x100; 736 if (firmware_has_feature(FW_FEATURE_LPAR)) 737 vdso_data->platform |= 1; 738 vdso_data->physicalMemorySize = memblock_phys_mem_size(); 739 vdso_data->dcache_size = ppc64_caches.l1d.size; 740 vdso_data->dcache_line_size = ppc64_caches.l1d.line_size; 741 vdso_data->icache_size = ppc64_caches.l1i.size; 742 vdso_data->icache_line_size = ppc64_caches.l1i.line_size; 743 vdso_data->dcache_block_size = ppc64_caches.l1d.block_size; 744 vdso_data->icache_block_size = ppc64_caches.l1i.block_size; 745 vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; 746 vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; 747 748 /* 749 * Calculate the size of the 64 bits vDSO 750 */ 751 vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT; 752 DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages); 753 #else 754 vdso_data->dcache_block_size = L1_CACHE_BYTES; 755 vdso_data->dcache_log_block_size = L1_CACHE_SHIFT; 756 vdso_data->icache_block_size = L1_CACHE_BYTES; 757 vdso_data->icache_log_block_size = L1_CACHE_SHIFT; 758 #endif /* CONFIG_PPC64 */ 759 760 761 #ifdef CONFIG_VDSO32 762 vdso32_kbase = &vdso32_start; 763 764 /* 765 * Calculate the size of the 32 bits vDSO 766 */ 767 vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT; 768 DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages); 769 #endif 770 771 772 /* 773 * Setup the syscall map in the vDOS 774 */ 775 vdso_setup_syscall_map(); 776 777 /* 778 * Initialize the vDSO images in memory, that is do necessary 779 * fixups of vDSO symbols, locate trampolines, etc... 780 */ 781 if (vdso_setup()) { 782 printk(KERN_ERR "vDSO setup failure, not enabled !\n"); 783 vdso32_pages = 0; 784 #ifdef CONFIG_PPC64 785 vdso64_pages = 0; 786 #endif 787 return 0; 788 } 789 790 #ifdef CONFIG_VDSO32 791 /* Make sure pages are in the correct state */ 792 vdso32_pagelist = kcalloc(vdso32_pages + 2, sizeof(struct page *), 793 GFP_KERNEL); 794 BUG_ON(vdso32_pagelist == NULL); 795 for (i = 0; i < vdso32_pages; i++) { 796 struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE); 797 get_page(pg); 798 vdso32_pagelist[i] = pg; 799 } 800 vdso32_pagelist[i++] = virt_to_page(vdso_data); 801 vdso32_pagelist[i] = NULL; 802 #endif 803 804 #ifdef CONFIG_PPC64 805 vdso64_pagelist = kcalloc(vdso64_pages + 2, sizeof(struct page *), 806 GFP_KERNEL); 807 BUG_ON(vdso64_pagelist == NULL); 808 for (i = 0; i < vdso64_pages; i++) { 809 struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE); 810 get_page(pg); 811 vdso64_pagelist[i] = pg; 812 } 813 vdso64_pagelist[i++] = virt_to_page(vdso_data); 814 vdso64_pagelist[i] = NULL; 815 #endif /* CONFIG_PPC64 */ 816 817 get_page(virt_to_page(vdso_data)); 818 819 smp_wmb(); 820 vdso_ready = 1; 821 822 return 0; 823 } 824 arch_initcall(vdso_init); 825