1/* 2 * This file contains miscellaneous low-level functions. 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) 6 * and Paul Mackerras. 7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com) 8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 */ 16 17#include <linux/sys.h> 18#include <asm/unistd.h> 19#include <asm/errno.h> 20#include <asm/processor.h> 21#include <asm/page.h> 22#include <asm/cache.h> 23#include <asm/ppc_asm.h> 24#include <asm/asm-offsets.h> 25#include <asm/cputable.h> 26#include <asm/thread_info.h> 27#include <asm/kexec.h> 28#include <asm/ptrace.h> 29#include <asm/mmu.h> 30 31 .text 32 33_GLOBAL(call_do_softirq) 34 mflr r0 35 std r0,16(r1) 36 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 37 mr r1,r3 38 bl __do_softirq 39 ld r1,0(r1) 40 ld r0,16(r1) 41 mtlr r0 42 blr 43 44_GLOBAL(call_do_irq) 45 mflr r0 46 std r0,16(r1) 47 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4) 48 mr r1,r4 49 bl __do_irq 50 ld r1,0(r1) 51 ld r0,16(r1) 52 mtlr r0 53 blr 54 55 .section ".toc","aw" 56PPC64_CACHES: 57 .tc ppc64_caches[TC],ppc64_caches 58 .section ".text" 59 60/* 61 * Write any modified data cache blocks out to memory 62 * and invalidate the corresponding instruction cache blocks. 63 * 64 * flush_icache_range(unsigned long start, unsigned long stop) 65 * 66 * flush all bytes from start through stop-1 inclusive 67 */ 68 69_KPROBE(flush_icache_range) 70BEGIN_FTR_SECTION 71 PURGE_PREFETCHED_INS 72 blr 73END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 74/* 75 * Flush the data cache to memory 76 * 77 * Different systems have different cache line sizes 78 * and in some cases i-cache and d-cache line sizes differ from 79 * each other. 80 */ 81 ld r10,PPC64_CACHES@toc(r2) 82 lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */ 83 addi r5,r7,-1 84 andc r6,r3,r5 /* round low to line bdy */ 85 subf r8,r6,r4 /* compute length */ 86 add r8,r8,r5 /* ensure we get enough */ 87 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */ 88 srw. r8,r8,r9 /* compute line count */ 89 beqlr /* nothing to do? */ 90 mtctr r8 911: dcbst 0,r6 92 add r6,r6,r7 93 bdnz 1b 94 sync 95 96/* Now invalidate the instruction cache */ 97 98 lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */ 99 addi r5,r7,-1 100 andc r6,r3,r5 /* round low to line bdy */ 101 subf r8,r6,r4 /* compute length */ 102 add r8,r8,r5 103 lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */ 104 srw. r8,r8,r9 /* compute line count */ 105 beqlr /* nothing to do? */ 106 mtctr r8 1072: icbi 0,r6 108 add r6,r6,r7 109 bdnz 2b 110 isync 111 blr 112 .previous .text 113/* 114 * Like above, but only do the D-cache. 115 * 116 * flush_dcache_range(unsigned long start, unsigned long stop) 117 * 118 * flush all bytes from start to stop-1 inclusive 119 */ 120_GLOBAL(flush_dcache_range) 121 122/* 123 * Flush the data cache to memory 124 * 125 * Different systems have different cache line sizes 126 */ 127 ld r10,PPC64_CACHES@toc(r2) 128 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 129 addi r5,r7,-1 130 andc r6,r3,r5 /* round low to line bdy */ 131 subf r8,r6,r4 /* compute length */ 132 add r8,r8,r5 /* ensure we get enough */ 133 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 134 srw. r8,r8,r9 /* compute line count */ 135 beqlr /* nothing to do? */ 136 mtctr r8 1370: dcbst 0,r6 138 add r6,r6,r7 139 bdnz 0b 140 sync 141 blr 142 143/* 144 * Like above, but works on non-mapped physical addresses. 145 * Use only for non-LPAR setups ! It also assumes real mode 146 * is cacheable. Used for flushing out the DART before using 147 * it as uncacheable memory 148 * 149 * flush_dcache_phys_range(unsigned long start, unsigned long stop) 150 * 151 * flush all bytes from start to stop-1 inclusive 152 */ 153_GLOBAL(flush_dcache_phys_range) 154 ld r10,PPC64_CACHES@toc(r2) 155 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 156 addi r5,r7,-1 157 andc r6,r3,r5 /* round low to line bdy */ 158 subf r8,r6,r4 /* compute length */ 159 add r8,r8,r5 /* ensure we get enough */ 160 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 161 srw. r8,r8,r9 /* compute line count */ 162 beqlr /* nothing to do? */ 163 mfmsr r5 /* Disable MMU Data Relocation */ 164 ori r0,r5,MSR_DR 165 xori r0,r0,MSR_DR 166 sync 167 mtmsr r0 168 sync 169 isync 170 mtctr r8 1710: dcbst 0,r6 172 add r6,r6,r7 173 bdnz 0b 174 sync 175 isync 176 mtmsr r5 /* Re-enable MMU Data Relocation */ 177 sync 178 isync 179 blr 180 181_GLOBAL(flush_inval_dcache_range) 182 ld r10,PPC64_CACHES@toc(r2) 183 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 184 addi r5,r7,-1 185 andc r6,r3,r5 /* round low to line bdy */ 186 subf r8,r6,r4 /* compute length */ 187 add r8,r8,r5 /* ensure we get enough */ 188 lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */ 189 srw. r8,r8,r9 /* compute line count */ 190 beqlr /* nothing to do? */ 191 sync 192 isync 193 mtctr r8 1940: dcbf 0,r6 195 add r6,r6,r7 196 bdnz 0b 197 sync 198 isync 199 blr 200 201 202/* 203 * Flush a particular page from the data cache to RAM. 204 * Note: this is necessary because the instruction cache does *not* 205 * snoop from the data cache. 206 * 207 * void __flush_dcache_icache(void *page) 208 */ 209_GLOBAL(__flush_dcache_icache) 210/* 211 * Flush the data cache to memory 212 * 213 * Different systems have different cache line sizes 214 */ 215 216BEGIN_FTR_SECTION 217 PURGE_PREFETCHED_INS 218 blr 219END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 220 221/* Flush the dcache */ 222 ld r7,PPC64_CACHES@toc(r2) 223 clrrdi r3,r3,PAGE_SHIFT /* Page align */ 224 lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */ 225 lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */ 226 mr r6,r3 227 mtctr r4 2280: dcbst 0,r6 229 add r6,r6,r5 230 bdnz 0b 231 sync 232 233/* Now invalidate the icache */ 234 235 lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */ 236 lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */ 237 mtctr r4 2381: icbi 0,r3 239 add r3,r3,r5 240 bdnz 1b 241 isync 242 blr 243 244_GLOBAL(__bswapdi2) 245 srdi r8,r3,32 246 rlwinm r7,r3,8,0xffffffff 247 rlwimi r7,r3,24,0,7 248 rlwinm r9,r8,8,0xffffffff 249 rlwimi r7,r3,24,16,23 250 rlwimi r9,r8,24,0,7 251 rlwimi r9,r8,24,16,23 252 sldi r7,r7,32 253 or r3,r7,r9 254 blr 255 256 257#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX 258_GLOBAL(rmci_on) 259 sync 260 isync 261 li r3,0x100 262 rldicl r3,r3,32,0 263 mfspr r5,SPRN_HID4 264 or r5,r5,r3 265 sync 266 mtspr SPRN_HID4,r5 267 isync 268 slbia 269 isync 270 sync 271 blr 272 273_GLOBAL(rmci_off) 274 sync 275 isync 276 li r3,0x100 277 rldicl r3,r3,32,0 278 mfspr r5,SPRN_HID4 279 andc r5,r5,r3 280 sync 281 mtspr SPRN_HID4,r5 282 isync 283 slbia 284 isync 285 sync 286 blr 287#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */ 288 289#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) 290 291/* 292 * Do an IO access in real mode 293 */ 294_GLOBAL(real_readb) 295 mfmsr r7 296 ori r0,r7,MSR_DR 297 xori r0,r0,MSR_DR 298 sync 299 mtmsrd r0 300 sync 301 isync 302 mfspr r6,SPRN_HID4 303 rldicl r5,r6,32,0 304 ori r5,r5,0x100 305 rldicl r5,r5,32,0 306 sync 307 mtspr SPRN_HID4,r5 308 isync 309 slbia 310 isync 311 lbz r3,0(r3) 312 sync 313 mtspr SPRN_HID4,r6 314 isync 315 slbia 316 isync 317 mtmsrd r7 318 sync 319 isync 320 blr 321 322 /* 323 * Do an IO access in real mode 324 */ 325_GLOBAL(real_writeb) 326 mfmsr r7 327 ori r0,r7,MSR_DR 328 xori r0,r0,MSR_DR 329 sync 330 mtmsrd r0 331 sync 332 isync 333 mfspr r6,SPRN_HID4 334 rldicl r5,r6,32,0 335 ori r5,r5,0x100 336 rldicl r5,r5,32,0 337 sync 338 mtspr SPRN_HID4,r5 339 isync 340 slbia 341 isync 342 stb r3,0(r4) 343 sync 344 mtspr SPRN_HID4,r6 345 isync 346 slbia 347 isync 348 mtmsrd r7 349 sync 350 isync 351 blr 352#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */ 353 354#ifdef CONFIG_PPC_PASEMI 355 356_GLOBAL(real_205_readb) 357 mfmsr r7 358 ori r0,r7,MSR_DR 359 xori r0,r0,MSR_DR 360 sync 361 mtmsrd r0 362 sync 363 isync 364 LBZCIX(R3,R0,R3) 365 isync 366 mtmsrd r7 367 sync 368 isync 369 blr 370 371_GLOBAL(real_205_writeb) 372 mfmsr r7 373 ori r0,r7,MSR_DR 374 xori r0,r0,MSR_DR 375 sync 376 mtmsrd r0 377 sync 378 isync 379 STBCIX(R3,R0,R4) 380 isync 381 mtmsrd r7 382 sync 383 isync 384 blr 385 386#endif /* CONFIG_PPC_PASEMI */ 387 388 389#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE) 390/* 391 * SCOM access functions for 970 (FX only for now) 392 * 393 * unsigned long scom970_read(unsigned int address); 394 * void scom970_write(unsigned int address, unsigned long value); 395 * 396 * The address passed in is the 24 bits register address. This code 397 * is 970 specific and will not check the status bits, so you should 398 * know what you are doing. 399 */ 400_GLOBAL(scom970_read) 401 /* interrupts off */ 402 mfmsr r4 403 ori r0,r4,MSR_EE 404 xori r0,r0,MSR_EE 405 mtmsrd r0,1 406 407 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 408 * (including parity). On current CPUs they must be 0'd, 409 * and finally or in RW bit 410 */ 411 rlwinm r3,r3,8,0,15 412 ori r3,r3,0x8000 413 414 /* do the actual scom read */ 415 sync 416 mtspr SPRN_SCOMC,r3 417 isync 418 mfspr r3,SPRN_SCOMD 419 isync 420 mfspr r0,SPRN_SCOMC 421 isync 422 423 /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah 424 * that's the best we can do). Not implemented yet as we don't use 425 * the scom on any of the bogus CPUs yet, but may have to be done 426 * ultimately 427 */ 428 429 /* restore interrupts */ 430 mtmsrd r4,1 431 blr 432 433 434_GLOBAL(scom970_write) 435 /* interrupts off */ 436 mfmsr r5 437 ori r0,r5,MSR_EE 438 xori r0,r0,MSR_EE 439 mtmsrd r0,1 440 441 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 442 * (including parity). On current CPUs they must be 0'd. 443 */ 444 445 rlwinm r3,r3,8,0,15 446 447 sync 448 mtspr SPRN_SCOMD,r4 /* write data */ 449 isync 450 mtspr SPRN_SCOMC,r3 /* write command */ 451 isync 452 mfspr 3,SPRN_SCOMC 453 isync 454 455 /* restore interrupts */ 456 mtmsrd r5,1 457 blr 458#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */ 459 460/* kexec_wait(phys_cpu) 461 * 462 * wait for the flag to change, indicating this kernel is going away but 463 * the slave code for the next one is at addresses 0 to 100. 464 * 465 * This is used by all slaves, even those that did not find a matching 466 * paca in the secondary startup code. 467 * 468 * Physical (hardware) cpu id should be in r3. 469 */ 470_GLOBAL(kexec_wait) 471 bl 1f 4721: mflr r5 473 addi r5,r5,kexec_flag-1b 474 47599: HMT_LOW 476#ifdef CONFIG_KEXEC /* use no memory without kexec */ 477 lwz r4,0(r5) 478 cmpwi 0,r4,0 479 beq 99b 480#ifdef CONFIG_PPC_BOOK3S_64 481 li r10,0x60 482 mfmsr r11 483 clrrdi r11,r11,1 /* Clear MSR_LE */ 484 mtsrr0 r10 485 mtsrr1 r11 486 rfid 487#else 488 /* Create TLB entry in book3e_secondary_core_init */ 489 li r4,0 490 ba 0x60 491#endif 492#endif 493 494/* this can be in text because we won't change it until we are 495 * running in real anyways 496 */ 497kexec_flag: 498 .long 0 499 500 501#ifdef CONFIG_KEXEC 502#ifdef CONFIG_PPC_BOOK3E 503/* 504 * BOOK3E has no real MMU mode, so we have to setup the initial TLB 505 * for a core to identity map v:0 to p:0. This current implementation 506 * assumes that 1G is enough for kexec. 507 */ 508kexec_create_tlb: 509 /* 510 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict. 511 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict. 512 */ 513 PPC_TLBILX_ALL(0,R0) 514 sync 515 isync 516 517 mfspr r10,SPRN_TLB1CFG 518 andi. r10,r10,TLBnCFG_N_ENTRY /* Extract # entries */ 519 subi r10,r10,1 /* Last entry: no conflict with kernel text */ 520 lis r9,MAS0_TLBSEL(1)@h 521 rlwimi r9,r10,16,4,15 /* Setup MAS0 = TLBSEL | ESEL(r9) */ 522 523/* Set up a temp identity mapping v:0 to p:0 and return to it. */ 524#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) 525#define M_IF_NEEDED MAS2_M 526#else 527#define M_IF_NEEDED 0 528#endif 529 mtspr SPRN_MAS0,r9 530 531 lis r9,(MAS1_VALID|MAS1_IPROT)@h 532 ori r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l 533 mtspr SPRN_MAS1,r9 534 535 LOAD_REG_IMMEDIATE(r9, 0x0 | M_IF_NEEDED) 536 mtspr SPRN_MAS2,r9 537 538 LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX) 539 mtspr SPRN_MAS3,r9 540 li r9,0 541 mtspr SPRN_MAS7,r9 542 543 tlbwe 544 isync 545 blr 546#endif 547 548/* kexec_smp_wait(void) 549 * 550 * call with interrupts off 551 * note: this is a terminal routine, it does not save lr 552 * 553 * get phys id from paca 554 * switch to real mode 555 * mark the paca as no longer used 556 * join other cpus in kexec_wait(phys_id) 557 */ 558_GLOBAL(kexec_smp_wait) 559 lhz r3,PACAHWCPUID(r13) 560 bl real_mode 561 562 li r4,KEXEC_STATE_REAL_MODE 563 stb r4,PACAKEXECSTATE(r13) 564 SYNC 565 566 b kexec_wait 567 568/* 569 * switch to real mode (turn mmu off) 570 * we use the early kernel trick that the hardware ignores bits 571 * 0 and 1 (big endian) of the effective address in real mode 572 * 573 * don't overwrite r3 here, it is live for kexec_wait above. 574 */ 575real_mode: /* assume normal blr return */ 576#ifdef CONFIG_PPC_BOOK3E 577 /* Create an identity mapping. */ 578 b kexec_create_tlb 579#else 5801: li r9,MSR_RI 581 li r10,MSR_DR|MSR_IR 582 mflr r11 /* return address to SRR0 */ 583 mfmsr r12 584 andc r9,r12,r9 585 andc r10,r12,r10 586 587 mtmsrd r9,1 588 mtspr SPRN_SRR1,r10 589 mtspr SPRN_SRR0,r11 590 rfid 591#endif 592 593/* 594 * kexec_sequence(newstack, start, image, control, clear_all()) 595 * 596 * does the grungy work with stack switching and real mode switches 597 * also does simple calls to other code 598 */ 599 600_GLOBAL(kexec_sequence) 601 mflr r0 602 std r0,16(r1) 603 604 /* switch stacks to newstack -- &kexec_stack.stack */ 605 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 606 mr r1,r3 607 608 li r0,0 609 std r0,16(r1) 610 611 /* save regs for local vars on new stack. 612 * yes, we won't go back, but ... 613 */ 614 std r31,-8(r1) 615 std r30,-16(r1) 616 std r29,-24(r1) 617 std r28,-32(r1) 618 std r27,-40(r1) 619 std r26,-48(r1) 620 std r25,-56(r1) 621 622 stdu r1,-STACK_FRAME_OVERHEAD-64(r1) 623 624 /* save args into preserved regs */ 625 mr r31,r3 /* newstack (both) */ 626 mr r30,r4 /* start (real) */ 627 mr r29,r5 /* image (virt) */ 628 mr r28,r6 /* control, unused */ 629 mr r27,r7 /* clear_all() fn desc */ 630 mr r26,r8 /* spare */ 631 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ 632 633 /* disable interrupts, we are overwriting kernel data next */ 634#ifdef CONFIG_PPC_BOOK3E 635 wrteei 0 636#else 637 mfmsr r3 638 rlwinm r3,r3,0,17,15 639 mtmsrd r3,1 640#endif 641 642 /* copy dest pages, flush whole dest image */ 643 mr r3,r29 644 bl kexec_copy_flush /* (image) */ 645 646 /* turn off mmu */ 647 bl real_mode 648 649 /* copy 0x100 bytes starting at start to 0 */ 650 li r3,0 651 mr r4,r30 /* start, aka phys mem offset */ 652 li r5,0x100 653 li r6,0 654 bl copy_and_flush /* (dest, src, copy limit, start offset) */ 6551: /* assume normal blr return */ 656 657 /* release other cpus to the new kernel secondary start at 0x60 */ 658 mflr r5 659 li r6,1 660 stw r6,kexec_flag-1b(5) 661 662#ifndef CONFIG_PPC_BOOK3E 663 /* clear out hardware hash page table and tlb */ 664#if !defined(_CALL_ELF) || _CALL_ELF != 2 665 ld r12,0(r27) /* deref function descriptor */ 666#else 667 mr r12,r27 668#endif 669 mtctr r12 670 bctrl /* ppc_md.hpte_clear_all(void); */ 671#endif /* !CONFIG_PPC_BOOK3E */ 672 673/* 674 * kexec image calling is: 675 * the first 0x100 bytes of the entry point are copied to 0 676 * 677 * all slaves branch to slave = 0x60 (absolute) 678 * slave(phys_cpu_id); 679 * 680 * master goes to start = entry point 681 * start(phys_cpu_id, start, 0); 682 * 683 * 684 * a wrapper is needed to call existing kernels, here is an approximate 685 * description of one method: 686 * 687 * v2: (2.6.10) 688 * start will be near the boot_block (maybe 0x100 bytes before it?) 689 * it will have a 0x60, which will b to boot_block, where it will wait 690 * and 0 will store phys into struct boot-block and load r3 from there, 691 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again 692 * 693 * v1: (2.6.9) 694 * boot block will have all cpus scanning device tree to see if they 695 * are the boot cpu ????? 696 * other device tree differences (prop sizes, va vs pa, etc)... 697 */ 698 mr r3,r25 # my phys cpu 699 mr r4,r30 # start, aka phys mem offset 700 mtlr 4 701 li r5,0 702 blr /* image->start(physid, image->start, 0); */ 703#endif /* CONFIG_KEXEC */ 704 705#ifdef CONFIG_MODULES 706#if defined(_CALL_ELF) && _CALL_ELF == 2 707 708#ifdef CONFIG_MODVERSIONS 709.weak __crc_TOC. 710.section "___kcrctab+TOC.","a" 711.globl __kcrctab_TOC. 712__kcrctab_TOC.: 713 .llong __crc_TOC. 714#endif 715 716/* 717 * Export a fake .TOC. since both modpost and depmod will complain otherwise. 718 * Both modpost and depmod strip the leading . so we do the same here. 719 */ 720.section "__ksymtab_strings","a" 721__kstrtab_TOC.: 722 .asciz "TOC." 723 724.section "___ksymtab+TOC.","a" 725/* This symbol name is important: it's used by modpost to find exported syms */ 726.globl __ksymtab_TOC. 727__ksymtab_TOC.: 728 .llong 0 /* .value */ 729 .llong __kstrtab_TOC. 730#endif /* ELFv2 */ 731#endif /* MODULES */ 732