1/* 2 * This file contains miscellaneous low-level functions. 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) 6 * and Paul Mackerras. 7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com) 8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 */ 16 17#include <linux/sys.h> 18#include <asm/unistd.h> 19#include <asm/errno.h> 20#include <asm/processor.h> 21#include <asm/page.h> 22#include <asm/cache.h> 23#include <asm/ppc_asm.h> 24#include <asm/asm-offsets.h> 25#include <asm/cputable.h> 26#include <asm/thread_info.h> 27#include <asm/kexec.h> 28#include <asm/ptrace.h> 29 30 .text 31 32_GLOBAL(call_do_softirq) 33 mflr r0 34 std r0,16(r1) 35 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 36 mr r1,r3 37 bl .__do_softirq 38 ld r1,0(r1) 39 ld r0,16(r1) 40 mtlr r0 41 blr 42 43_GLOBAL(call_do_irq) 44 mflr r0 45 std r0,16(r1) 46 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4) 47 mr r1,r4 48 bl .__do_irq 49 ld r1,0(r1) 50 ld r0,16(r1) 51 mtlr r0 52 blr 53 54 .section ".toc","aw" 55PPC64_CACHES: 56 .tc ppc64_caches[TC],ppc64_caches 57 .section ".text" 58 59/* 60 * Write any modified data cache blocks out to memory 61 * and invalidate the corresponding instruction cache blocks. 62 * 63 * flush_icache_range(unsigned long start, unsigned long stop) 64 * 65 * flush all bytes from start through stop-1 inclusive 66 */ 67 68_KPROBE(flush_icache_range) 69BEGIN_FTR_SECTION 70 blr 71END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 72/* 73 * Flush the data cache to memory 74 * 75 * Different systems have different cache line sizes 76 * and in some cases i-cache and d-cache line sizes differ from 77 * each other. 78 */ 79 ld r10,PPC64_CACHES@toc(r2) 80 lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */ 81 addi r5,r7,-1 82 andc r6,r3,r5 /* round low to line bdy */ 83 subf r8,r6,r4 /* compute length */ 84 add r8,r8,r5 /* ensure we get enough */ 85 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */ 86 srw. r8,r8,r9 /* compute line count */ 87 beqlr /* nothing to do? */ 88 mtctr r8 891: dcbst 0,r6 90 add r6,r6,r7 91 bdnz 1b 92 sync 93 94/* Now invalidate the instruction cache */ 95 96 lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */ 97 addi r5,r7,-1 98 andc r6,r3,r5 /* round low to line bdy */ 99 subf r8,r6,r4 /* compute length */ 100 add r8,r8,r5 101 lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */ 102 srw. r8,r8,r9 /* compute line count */ 103 beqlr /* nothing to do? */ 104 mtctr r8 1052: icbi 0,r6 106 add r6,r6,r7 107 bdnz 2b 108 isync 109 blr 110 .previous .text 111/* 112 * Like above, but only do the D-cache. 113 * 114 * flush_dcache_range(unsigned long start, unsigned long stop) 115 * 116 * flush all bytes from start to stop-1 inclusive 117 */ 118_GLOBAL(flush_dcache_range) 119 120/* 121 * Flush the data cache to memory 122 * 123 * Different systems have different cache line sizes 124 */ 125 ld r10,PPC64_CACHES@toc(r2) 126 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 127 addi r5,r7,-1 128 andc r6,r3,r5 /* round low to line bdy */ 129 subf r8,r6,r4 /* compute length */ 130 add r8,r8,r5 /* ensure we get enough */ 131 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 132 srw. r8,r8,r9 /* compute line count */ 133 beqlr /* nothing to do? */ 134 mtctr r8 1350: dcbst 0,r6 136 add r6,r6,r7 137 bdnz 0b 138 sync 139 blr 140 141/* 142 * Like above, but works on non-mapped physical addresses. 143 * Use only for non-LPAR setups ! It also assumes real mode 144 * is cacheable. Used for flushing out the DART before using 145 * it as uncacheable memory 146 * 147 * flush_dcache_phys_range(unsigned long start, unsigned long stop) 148 * 149 * flush all bytes from start to stop-1 inclusive 150 */ 151_GLOBAL(flush_dcache_phys_range) 152 ld r10,PPC64_CACHES@toc(r2) 153 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 154 addi r5,r7,-1 155 andc r6,r3,r5 /* round low to line bdy */ 156 subf r8,r6,r4 /* compute length */ 157 add r8,r8,r5 /* ensure we get enough */ 158 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 159 srw. r8,r8,r9 /* compute line count */ 160 beqlr /* nothing to do? */ 161 mfmsr r5 /* Disable MMU Data Relocation */ 162 ori r0,r5,MSR_DR 163 xori r0,r0,MSR_DR 164 sync 165 mtmsr r0 166 sync 167 isync 168 mtctr r8 1690: dcbst 0,r6 170 add r6,r6,r7 171 bdnz 0b 172 sync 173 isync 174 mtmsr r5 /* Re-enable MMU Data Relocation */ 175 sync 176 isync 177 blr 178 179_GLOBAL(flush_inval_dcache_range) 180 ld r10,PPC64_CACHES@toc(r2) 181 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 182 addi r5,r7,-1 183 andc r6,r3,r5 /* round low to line bdy */ 184 subf r8,r6,r4 /* compute length */ 185 add r8,r8,r5 /* ensure we get enough */ 186 lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */ 187 srw. r8,r8,r9 /* compute line count */ 188 beqlr /* nothing to do? */ 189 sync 190 isync 191 mtctr r8 1920: dcbf 0,r6 193 add r6,r6,r7 194 bdnz 0b 195 sync 196 isync 197 blr 198 199 200/* 201 * Flush a particular page from the data cache to RAM. 202 * Note: this is necessary because the instruction cache does *not* 203 * snoop from the data cache. 204 * 205 * void __flush_dcache_icache(void *page) 206 */ 207_GLOBAL(__flush_dcache_icache) 208/* 209 * Flush the data cache to memory 210 * 211 * Different systems have different cache line sizes 212 */ 213 214/* Flush the dcache */ 215 ld r7,PPC64_CACHES@toc(r2) 216 clrrdi r3,r3,PAGE_SHIFT /* Page align */ 217 lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */ 218 lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */ 219 mr r6,r3 220 mtctr r4 2210: dcbst 0,r6 222 add r6,r6,r5 223 bdnz 0b 224 sync 225 226/* Now invalidate the icache */ 227 228 lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */ 229 lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */ 230 mtctr r4 2311: icbi 0,r3 232 add r3,r3,r5 233 bdnz 1b 234 isync 235 blr 236 237_GLOBAL(__bswapdi2) 238 srdi r8,r3,32 239 rlwinm r7,r3,8,0xffffffff 240 rlwimi r7,r3,24,0,7 241 rlwinm r9,r8,8,0xffffffff 242 rlwimi r7,r3,24,16,23 243 rlwimi r9,r8,24,0,7 244 rlwimi r9,r8,24,16,23 245 sldi r7,r7,32 246 or r3,r7,r9 247 blr 248 249 250#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX 251_GLOBAL(rmci_on) 252 sync 253 isync 254 li r3,0x100 255 rldicl r3,r3,32,0 256 mfspr r5,SPRN_HID4 257 or r5,r5,r3 258 sync 259 mtspr SPRN_HID4,r5 260 isync 261 slbia 262 isync 263 sync 264 blr 265 266_GLOBAL(rmci_off) 267 sync 268 isync 269 li r3,0x100 270 rldicl r3,r3,32,0 271 mfspr r5,SPRN_HID4 272 andc r5,r5,r3 273 sync 274 mtspr SPRN_HID4,r5 275 isync 276 slbia 277 isync 278 sync 279 blr 280#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */ 281 282#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) 283 284/* 285 * Do an IO access in real mode 286 */ 287_GLOBAL(real_readb) 288 mfmsr r7 289 ori r0,r7,MSR_DR 290 xori r0,r0,MSR_DR 291 sync 292 mtmsrd r0 293 sync 294 isync 295 mfspr r6,SPRN_HID4 296 rldicl r5,r6,32,0 297 ori r5,r5,0x100 298 rldicl r5,r5,32,0 299 sync 300 mtspr SPRN_HID4,r5 301 isync 302 slbia 303 isync 304 lbz r3,0(r3) 305 sync 306 mtspr SPRN_HID4,r6 307 isync 308 slbia 309 isync 310 mtmsrd r7 311 sync 312 isync 313 blr 314 315 /* 316 * Do an IO access in real mode 317 */ 318_GLOBAL(real_writeb) 319 mfmsr r7 320 ori r0,r7,MSR_DR 321 xori r0,r0,MSR_DR 322 sync 323 mtmsrd r0 324 sync 325 isync 326 mfspr r6,SPRN_HID4 327 rldicl r5,r6,32,0 328 ori r5,r5,0x100 329 rldicl r5,r5,32,0 330 sync 331 mtspr SPRN_HID4,r5 332 isync 333 slbia 334 isync 335 stb r3,0(r4) 336 sync 337 mtspr SPRN_HID4,r6 338 isync 339 slbia 340 isync 341 mtmsrd r7 342 sync 343 isync 344 blr 345#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */ 346 347#ifdef CONFIG_PPC_PASEMI 348 349_GLOBAL(real_205_readb) 350 mfmsr r7 351 ori r0,r7,MSR_DR 352 xori r0,r0,MSR_DR 353 sync 354 mtmsrd r0 355 sync 356 isync 357 LBZCIX(R3,R0,R3) 358 isync 359 mtmsrd r7 360 sync 361 isync 362 blr 363 364_GLOBAL(real_205_writeb) 365 mfmsr r7 366 ori r0,r7,MSR_DR 367 xori r0,r0,MSR_DR 368 sync 369 mtmsrd r0 370 sync 371 isync 372 STBCIX(R3,R0,R4) 373 isync 374 mtmsrd r7 375 sync 376 isync 377 blr 378 379#endif /* CONFIG_PPC_PASEMI */ 380 381 382#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE) 383/* 384 * SCOM access functions for 970 (FX only for now) 385 * 386 * unsigned long scom970_read(unsigned int address); 387 * void scom970_write(unsigned int address, unsigned long value); 388 * 389 * The address passed in is the 24 bits register address. This code 390 * is 970 specific and will not check the status bits, so you should 391 * know what you are doing. 392 */ 393_GLOBAL(scom970_read) 394 /* interrupts off */ 395 mfmsr r4 396 ori r0,r4,MSR_EE 397 xori r0,r0,MSR_EE 398 mtmsrd r0,1 399 400 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 401 * (including parity). On current CPUs they must be 0'd, 402 * and finally or in RW bit 403 */ 404 rlwinm r3,r3,8,0,15 405 ori r3,r3,0x8000 406 407 /* do the actual scom read */ 408 sync 409 mtspr SPRN_SCOMC,r3 410 isync 411 mfspr r3,SPRN_SCOMD 412 isync 413 mfspr r0,SPRN_SCOMC 414 isync 415 416 /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah 417 * that's the best we can do). Not implemented yet as we don't use 418 * the scom on any of the bogus CPUs yet, but may have to be done 419 * ultimately 420 */ 421 422 /* restore interrupts */ 423 mtmsrd r4,1 424 blr 425 426 427_GLOBAL(scom970_write) 428 /* interrupts off */ 429 mfmsr r5 430 ori r0,r5,MSR_EE 431 xori r0,r0,MSR_EE 432 mtmsrd r0,1 433 434 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 435 * (including parity). On current CPUs they must be 0'd. 436 */ 437 438 rlwinm r3,r3,8,0,15 439 440 sync 441 mtspr SPRN_SCOMD,r4 /* write data */ 442 isync 443 mtspr SPRN_SCOMC,r3 /* write command */ 444 isync 445 mfspr 3,SPRN_SCOMC 446 isync 447 448 /* restore interrupts */ 449 mtmsrd r5,1 450 blr 451#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */ 452 453/* kexec_wait(phys_cpu) 454 * 455 * wait for the flag to change, indicating this kernel is going away but 456 * the slave code for the next one is at addresses 0 to 100. 457 * 458 * This is used by all slaves, even those that did not find a matching 459 * paca in the secondary startup code. 460 * 461 * Physical (hardware) cpu id should be in r3. 462 */ 463_GLOBAL(kexec_wait) 464 bl 1f 4651: mflr r5 466 addi r5,r5,kexec_flag-1b 467 46899: HMT_LOW 469#ifdef CONFIG_KEXEC /* use no memory without kexec */ 470 lwz r4,0(r5) 471 cmpwi 0,r4,0 472 bnea 0x60 473#endif 474 b 99b 475 476/* this can be in text because we won't change it until we are 477 * running in real anyways 478 */ 479kexec_flag: 480 .long 0 481 482 483#ifdef CONFIG_KEXEC 484 485/* kexec_smp_wait(void) 486 * 487 * call with interrupts off 488 * note: this is a terminal routine, it does not save lr 489 * 490 * get phys id from paca 491 * switch to real mode 492 * mark the paca as no longer used 493 * join other cpus in kexec_wait(phys_id) 494 */ 495_GLOBAL(kexec_smp_wait) 496 lhz r3,PACAHWCPUID(r13) 497 bl real_mode 498 499 li r4,KEXEC_STATE_REAL_MODE 500 stb r4,PACAKEXECSTATE(r13) 501 SYNC 502 503 b .kexec_wait 504 505/* 506 * switch to real mode (turn mmu off) 507 * we use the early kernel trick that the hardware ignores bits 508 * 0 and 1 (big endian) of the effective address in real mode 509 * 510 * don't overwrite r3 here, it is live for kexec_wait above. 511 */ 512real_mode: /* assume normal blr return */ 5131: li r9,MSR_RI 514 li r10,MSR_DR|MSR_IR 515 mflr r11 /* return address to SRR0 */ 516 mfmsr r12 517 andc r9,r12,r9 518 andc r10,r12,r10 519 520 mtmsrd r9,1 521 mtspr SPRN_SRR1,r10 522 mtspr SPRN_SRR0,r11 523 rfid 524 525 526/* 527 * kexec_sequence(newstack, start, image, control, clear_all()) 528 * 529 * does the grungy work with stack switching and real mode switches 530 * also does simple calls to other code 531 */ 532 533_GLOBAL(kexec_sequence) 534 mflr r0 535 std r0,16(r1) 536 537 /* switch stacks to newstack -- &kexec_stack.stack */ 538 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 539 mr r1,r3 540 541 li r0,0 542 std r0,16(r1) 543 544 /* save regs for local vars on new stack. 545 * yes, we won't go back, but ... 546 */ 547 std r31,-8(r1) 548 std r30,-16(r1) 549 std r29,-24(r1) 550 std r28,-32(r1) 551 std r27,-40(r1) 552 std r26,-48(r1) 553 std r25,-56(r1) 554 555 stdu r1,-STACK_FRAME_OVERHEAD-64(r1) 556 557 /* save args into preserved regs */ 558 mr r31,r3 /* newstack (both) */ 559 mr r30,r4 /* start (real) */ 560 mr r29,r5 /* image (virt) */ 561 mr r28,r6 /* control, unused */ 562 mr r27,r7 /* clear_all() fn desc */ 563 mr r26,r8 /* spare */ 564 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ 565 566 /* disable interrupts, we are overwriting kernel data next */ 567 mfmsr r3 568 rlwinm r3,r3,0,17,15 569 mtmsrd r3,1 570 571 /* copy dest pages, flush whole dest image */ 572 mr r3,r29 573 bl .kexec_copy_flush /* (image) */ 574 575 /* turn off mmu */ 576 bl real_mode 577 578 /* copy 0x100 bytes starting at start to 0 */ 579 li r3,0 580 mr r4,r30 /* start, aka phys mem offset */ 581 li r5,0x100 582 li r6,0 583 bl .copy_and_flush /* (dest, src, copy limit, start offset) */ 5841: /* assume normal blr return */ 585 586 /* release other cpus to the new kernel secondary start at 0x60 */ 587 mflr r5 588 li r6,1 589 stw r6,kexec_flag-1b(5) 590 591 /* clear out hardware hash page table and tlb */ 592 ld r5,0(r27) /* deref function descriptor */ 593 mtctr r5 594 bctrl /* ppc_md.hpte_clear_all(void); */ 595 596/* 597 * kexec image calling is: 598 * the first 0x100 bytes of the entry point are copied to 0 599 * 600 * all slaves branch to slave = 0x60 (absolute) 601 * slave(phys_cpu_id); 602 * 603 * master goes to start = entry point 604 * start(phys_cpu_id, start, 0); 605 * 606 * 607 * a wrapper is needed to call existing kernels, here is an approximate 608 * description of one method: 609 * 610 * v2: (2.6.10) 611 * start will be near the boot_block (maybe 0x100 bytes before it?) 612 * it will have a 0x60, which will b to boot_block, where it will wait 613 * and 0 will store phys into struct boot-block and load r3 from there, 614 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again 615 * 616 * v1: (2.6.9) 617 * boot block will have all cpus scanning device tree to see if they 618 * are the boot cpu ????? 619 * other device tree differences (prop sizes, va vs pa, etc)... 620 */ 621 mr r3,r25 # my phys cpu 622 mr r4,r30 # start, aka phys mem offset 623 mtlr 4 624 li r5,0 625 blr /* image->start(physid, image->start, 0); */ 626#endif /* CONFIG_KEXEC */ 627