1/* 2 * This file contains miscellaneous low-level functions. 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) 6 * and Paul Mackerras. 7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com) 8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 */ 16 17#include <linux/sys.h> 18#include <asm/unistd.h> 19#include <asm/errno.h> 20#include <asm/processor.h> 21#include <asm/page.h> 22#include <asm/cache.h> 23#include <asm/ppc_asm.h> 24#include <asm/asm-offsets.h> 25#include <asm/cputable.h> 26#include <asm/thread_info.h> 27#include <asm/kexec.h> 28 29 .text 30 31_GLOBAL(call_do_softirq) 32 mflr r0 33 std r0,16(r1) 34 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 35 mr r1,r3 36 bl .__do_softirq 37 ld r1,0(r1) 38 ld r0,16(r1) 39 mtlr r0 40 blr 41 42_GLOBAL(call_handle_irq) 43 ld r8,0(r6) 44 mflr r0 45 std r0,16(r1) 46 mtctr r8 47 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r5) 48 mr r1,r5 49 bctrl 50 ld r1,0(r1) 51 ld r0,16(r1) 52 mtlr r0 53 blr 54 55 .section ".toc","aw" 56PPC64_CACHES: 57 .tc ppc64_caches[TC],ppc64_caches 58 .section ".text" 59 60/* 61 * Write any modified data cache blocks out to memory 62 * and invalidate the corresponding instruction cache blocks. 63 * 64 * flush_icache_range(unsigned long start, unsigned long stop) 65 * 66 * flush all bytes from start through stop-1 inclusive 67 */ 68 69_KPROBE(__flush_icache_range) 70 71/* 72 * Flush the data cache to memory 73 * 74 * Different systems have different cache line sizes 75 * and in some cases i-cache and d-cache line sizes differ from 76 * each other. 77 */ 78 ld r10,PPC64_CACHES@toc(r2) 79 lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */ 80 addi r5,r7,-1 81 andc r6,r3,r5 /* round low to line bdy */ 82 subf r8,r6,r4 /* compute length */ 83 add r8,r8,r5 /* ensure we get enough */ 84 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */ 85 srw. r8,r8,r9 /* compute line count */ 86 beqlr /* nothing to do? */ 87 mtctr r8 881: dcbst 0,r6 89 add r6,r6,r7 90 bdnz 1b 91 sync 92 93/* Now invalidate the instruction cache */ 94 95 lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */ 96 addi r5,r7,-1 97 andc r6,r3,r5 /* round low to line bdy */ 98 subf r8,r6,r4 /* compute length */ 99 add r8,r8,r5 100 lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */ 101 srw. r8,r8,r9 /* compute line count */ 102 beqlr /* nothing to do? */ 103 mtctr r8 1042: icbi 0,r6 105 add r6,r6,r7 106 bdnz 2b 107 isync 108 blr 109 .previous .text 110/* 111 * Like above, but only do the D-cache. 112 * 113 * flush_dcache_range(unsigned long start, unsigned long stop) 114 * 115 * flush all bytes from start to stop-1 inclusive 116 */ 117_GLOBAL(flush_dcache_range) 118 119/* 120 * Flush the data cache to memory 121 * 122 * Different systems have different cache line sizes 123 */ 124 ld r10,PPC64_CACHES@toc(r2) 125 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 126 addi r5,r7,-1 127 andc r6,r3,r5 /* round low to line bdy */ 128 subf r8,r6,r4 /* compute length */ 129 add r8,r8,r5 /* ensure we get enough */ 130 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 131 srw. r8,r8,r9 /* compute line count */ 132 beqlr /* nothing to do? */ 133 mtctr r8 1340: dcbst 0,r6 135 add r6,r6,r7 136 bdnz 0b 137 sync 138 blr 139 140/* 141 * Like above, but works on non-mapped physical addresses. 142 * Use only for non-LPAR setups ! It also assumes real mode 143 * is cacheable. Used for flushing out the DART before using 144 * it as uncacheable memory 145 * 146 * flush_dcache_phys_range(unsigned long start, unsigned long stop) 147 * 148 * flush all bytes from start to stop-1 inclusive 149 */ 150_GLOBAL(flush_dcache_phys_range) 151 ld r10,PPC64_CACHES@toc(r2) 152 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 153 addi r5,r7,-1 154 andc r6,r3,r5 /* round low to line bdy */ 155 subf r8,r6,r4 /* compute length */ 156 add r8,r8,r5 /* ensure we get enough */ 157 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 158 srw. r8,r8,r9 /* compute line count */ 159 beqlr /* nothing to do? */ 160 mfmsr r5 /* Disable MMU Data Relocation */ 161 ori r0,r5,MSR_DR 162 xori r0,r0,MSR_DR 163 sync 164 mtmsr r0 165 sync 166 isync 167 mtctr r8 1680: dcbst 0,r6 169 add r6,r6,r7 170 bdnz 0b 171 sync 172 isync 173 mtmsr r5 /* Re-enable MMU Data Relocation */ 174 sync 175 isync 176 blr 177 178_GLOBAL(flush_inval_dcache_range) 179 ld r10,PPC64_CACHES@toc(r2) 180 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 181 addi r5,r7,-1 182 andc r6,r3,r5 /* round low to line bdy */ 183 subf r8,r6,r4 /* compute length */ 184 add r8,r8,r5 /* ensure we get enough */ 185 lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */ 186 srw. r8,r8,r9 /* compute line count */ 187 beqlr /* nothing to do? */ 188 sync 189 isync 190 mtctr r8 1910: dcbf 0,r6 192 add r6,r6,r7 193 bdnz 0b 194 sync 195 isync 196 blr 197 198 199/* 200 * Flush a particular page from the data cache to RAM. 201 * Note: this is necessary because the instruction cache does *not* 202 * snoop from the data cache. 203 * 204 * void __flush_dcache_icache(void *page) 205 */ 206_GLOBAL(__flush_dcache_icache) 207/* 208 * Flush the data cache to memory 209 * 210 * Different systems have different cache line sizes 211 */ 212 213/* Flush the dcache */ 214 ld r7,PPC64_CACHES@toc(r2) 215 clrrdi r3,r3,PAGE_SHIFT /* Page align */ 216 lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */ 217 lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */ 218 mr r6,r3 219 mtctr r4 2200: dcbst 0,r6 221 add r6,r6,r5 222 bdnz 0b 223 sync 224 225/* Now invalidate the icache */ 226 227 lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */ 228 lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */ 229 mtctr r4 2301: icbi 0,r3 231 add r3,r3,r5 232 bdnz 1b 233 isync 234 blr 235 236 237#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) 238/* 239 * Do an IO access in real mode 240 */ 241_GLOBAL(real_readb) 242 mfmsr r7 243 ori r0,r7,MSR_DR 244 xori r0,r0,MSR_DR 245 sync 246 mtmsrd r0 247 sync 248 isync 249 mfspr r6,SPRN_HID4 250 rldicl r5,r6,32,0 251 ori r5,r5,0x100 252 rldicl r5,r5,32,0 253 sync 254 mtspr SPRN_HID4,r5 255 isync 256 slbia 257 isync 258 lbz r3,0(r3) 259 sync 260 mtspr SPRN_HID4,r6 261 isync 262 slbia 263 isync 264 mtmsrd r7 265 sync 266 isync 267 blr 268 269 /* 270 * Do an IO access in real mode 271 */ 272_GLOBAL(real_writeb) 273 mfmsr r7 274 ori r0,r7,MSR_DR 275 xori r0,r0,MSR_DR 276 sync 277 mtmsrd r0 278 sync 279 isync 280 mfspr r6,SPRN_HID4 281 rldicl r5,r6,32,0 282 ori r5,r5,0x100 283 rldicl r5,r5,32,0 284 sync 285 mtspr SPRN_HID4,r5 286 isync 287 slbia 288 isync 289 stb r3,0(r4) 290 sync 291 mtspr SPRN_HID4,r6 292 isync 293 slbia 294 isync 295 mtmsrd r7 296 sync 297 isync 298 blr 299#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */ 300 301#ifdef CONFIG_PPC_PASEMI 302 303/* No support in all binutils for these yet, so use defines */ 304#define LBZCIX(RT,RA,RB) .long (0x7c0006aa|(RT<<21)|(RA<<16)|(RB << 11)) 305#define STBCIX(RS,RA,RB) .long (0x7c0007aa|(RS<<21)|(RA<<16)|(RB << 11)) 306 307 308_GLOBAL(real_205_readb) 309 mfmsr r7 310 ori r0,r7,MSR_DR 311 xori r0,r0,MSR_DR 312 sync 313 mtmsrd r0 314 sync 315 isync 316 LBZCIX(r3,0,r3) 317 isync 318 mtmsrd r7 319 sync 320 isync 321 blr 322 323_GLOBAL(real_205_writeb) 324 mfmsr r7 325 ori r0,r7,MSR_DR 326 xori r0,r0,MSR_DR 327 sync 328 mtmsrd r0 329 sync 330 isync 331 STBCIX(r3,0,r4) 332 isync 333 mtmsrd r7 334 sync 335 isync 336 blr 337 338#endif /* CONFIG_PPC_PASEMI */ 339 340 341#ifdef CONFIG_CPU_FREQ_PMAC64 342/* 343 * SCOM access functions for 970 (FX only for now) 344 * 345 * unsigned long scom970_read(unsigned int address); 346 * void scom970_write(unsigned int address, unsigned long value); 347 * 348 * The address passed in is the 24 bits register address. This code 349 * is 970 specific and will not check the status bits, so you should 350 * know what you are doing. 351 */ 352_GLOBAL(scom970_read) 353 /* interrupts off */ 354 mfmsr r4 355 ori r0,r4,MSR_EE 356 xori r0,r0,MSR_EE 357 mtmsrd r0,1 358 359 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 360 * (including parity). On current CPUs they must be 0'd, 361 * and finally or in RW bit 362 */ 363 rlwinm r3,r3,8,0,15 364 ori r3,r3,0x8000 365 366 /* do the actual scom read */ 367 sync 368 mtspr SPRN_SCOMC,r3 369 isync 370 mfspr r3,SPRN_SCOMD 371 isync 372 mfspr r0,SPRN_SCOMC 373 isync 374 375 /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah 376 * that's the best we can do). Not implemented yet as we don't use 377 * the scom on any of the bogus CPUs yet, but may have to be done 378 * ultimately 379 */ 380 381 /* restore interrupts */ 382 mtmsrd r4,1 383 blr 384 385 386_GLOBAL(scom970_write) 387 /* interrupts off */ 388 mfmsr r5 389 ori r0,r5,MSR_EE 390 xori r0,r0,MSR_EE 391 mtmsrd r0,1 392 393 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 394 * (including parity). On current CPUs they must be 0'd. 395 */ 396 397 rlwinm r3,r3,8,0,15 398 399 sync 400 mtspr SPRN_SCOMD,r4 /* write data */ 401 isync 402 mtspr SPRN_SCOMC,r3 /* write command */ 403 isync 404 mfspr 3,SPRN_SCOMC 405 isync 406 407 /* restore interrupts */ 408 mtmsrd r5,1 409 blr 410#endif /* CONFIG_CPU_FREQ_PMAC64 */ 411 412 413/* 414 * Create a kernel thread 415 * kernel_thread(fn, arg, flags) 416 */ 417_GLOBAL(kernel_thread) 418 std r29,-24(r1) 419 std r30,-16(r1) 420 stdu r1,-STACK_FRAME_OVERHEAD(r1) 421 mr r29,r3 422 mr r30,r4 423 ori r3,r5,CLONE_VM /* flags */ 424 oris r3,r3,(CLONE_UNTRACED>>16) 425 li r4,0 /* new sp (unused) */ 426 li r0,__NR_clone 427 sc 428 bns+ 1f /* did system call indicate error? */ 429 neg r3,r3 /* if so, make return code negative */ 4301: cmpdi 0,r3,0 /* parent or child? */ 431 bne 2f /* return if parent */ 432 li r0,0 433 stdu r0,-STACK_FRAME_OVERHEAD(r1) 434 ld r2,8(r29) 435 ld r29,0(r29) 436 mtlr r29 /* fn addr in lr */ 437 mr r3,r30 /* load arg and call fn */ 438 blrl 439 li r0,__NR_exit /* exit after child exits */ 440 li r3,0 441 sc 4422: addi r1,r1,STACK_FRAME_OVERHEAD 443 ld r29,-24(r1) 444 ld r30,-16(r1) 445 blr 446 447/* 448 * disable_kernel_fp() 449 * Disable the FPU. 450 */ 451_GLOBAL(disable_kernel_fp) 452 mfmsr r3 453 rldicl r0,r3,(63-MSR_FP_LG),1 454 rldicl r3,r0,(MSR_FP_LG+1),0 455 mtmsrd r3 /* disable use of fpu now */ 456 isync 457 blr 458 459/* kexec_wait(phys_cpu) 460 * 461 * wait for the flag to change, indicating this kernel is going away but 462 * the slave code for the next one is at addresses 0 to 100. 463 * 464 * This is used by all slaves. 465 * 466 * Physical (hardware) cpu id should be in r3. 467 */ 468_GLOBAL(kexec_wait) 469 bl 1f 4701: mflr r5 471 addi r5,r5,kexec_flag-1b 472 473 li r4,KEXEC_STATE_REAL_MODE 474 stb r4,PACAKEXECSTATE(r13) 475 SYNC 476 47799: HMT_LOW 478#ifdef CONFIG_KEXEC /* use no memory without kexec */ 479 lwz r4,0(r5) 480 cmpwi 0,r4,0 481 bnea 0x60 482#endif 483 b 99b 484 485/* this can be in text because we won't change it until we are 486 * running in real anyways 487 */ 488kexec_flag: 489 .long 0 490 491 492#ifdef CONFIG_KEXEC 493 494/* kexec_smp_wait(void) 495 * 496 * call with interrupts off 497 * note: this is a terminal routine, it does not save lr 498 * 499 * get phys id from paca 500 * switch to real mode 501 * join other cpus in kexec_wait(phys_id) 502 */ 503_GLOBAL(kexec_smp_wait) 504 lhz r3,PACAHWCPUID(r13) 505 bl real_mode 506 b .kexec_wait 507 508/* 509 * switch to real mode (turn mmu off) 510 * we use the early kernel trick that the hardware ignores bits 511 * 0 and 1 (big endian) of the effective address in real mode 512 * 513 * don't overwrite r3 here, it is live for kexec_wait above. 514 */ 515real_mode: /* assume normal blr return */ 5161: li r9,MSR_RI 517 li r10,MSR_DR|MSR_IR 518 mflr r11 /* return address to SRR0 */ 519 mfmsr r12 520 andc r9,r12,r9 521 andc r10,r12,r10 522 523 mtmsrd r9,1 524 mtspr SPRN_SRR1,r10 525 mtspr SPRN_SRR0,r11 526 rfid 527 528 529/* 530 * kexec_sequence(newstack, start, image, control, clear_all()) 531 * 532 * does the grungy work with stack switching and real mode switches 533 * also does simple calls to other code 534 */ 535 536_GLOBAL(kexec_sequence) 537 mflr r0 538 std r0,16(r1) 539 540 /* switch stacks to newstack -- &kexec_stack.stack */ 541 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 542 mr r1,r3 543 544 li r0,0 545 std r0,16(r1) 546 547 /* save regs for local vars on new stack. 548 * yes, we won't go back, but ... 549 */ 550 std r31,-8(r1) 551 std r30,-16(r1) 552 std r29,-24(r1) 553 std r28,-32(r1) 554 std r27,-40(r1) 555 std r26,-48(r1) 556 std r25,-56(r1) 557 558 stdu r1,-STACK_FRAME_OVERHEAD-64(r1) 559 560 /* save args into preserved regs */ 561 mr r31,r3 /* newstack (both) */ 562 mr r30,r4 /* start (real) */ 563 mr r29,r5 /* image (virt) */ 564 mr r28,r6 /* control, unused */ 565 mr r27,r7 /* clear_all() fn desc */ 566 mr r26,r8 /* spare */ 567 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ 568 569 /* disable interrupts, we are overwriting kernel data next */ 570 mfmsr r3 571 rlwinm r3,r3,0,17,15 572 mtmsrd r3,1 573 574 /* copy dest pages, flush whole dest image */ 575 mr r3,r29 576 bl .kexec_copy_flush /* (image) */ 577 578 /* turn off mmu */ 579 bl real_mode 580 581 /* copy 0x100 bytes starting at start to 0 */ 582 li r3,0 583 mr r4,r30 /* start, aka phys mem offset */ 584 li r5,0x100 585 li r6,0 586 bl .copy_and_flush /* (dest, src, copy limit, start offset) */ 5871: /* assume normal blr return */ 588 589 /* release other cpus to the new kernel secondary start at 0x60 */ 590 mflr r5 591 li r6,1 592 stw r6,kexec_flag-1b(5) 593 594 /* clear out hardware hash page table and tlb */ 595 ld r5,0(r27) /* deref function descriptor */ 596 mtctr r5 597 bctrl /* ppc_md.hpte_clear_all(void); */ 598 599/* 600 * kexec image calling is: 601 * the first 0x100 bytes of the entry point are copied to 0 602 * 603 * all slaves branch to slave = 0x60 (absolute) 604 * slave(phys_cpu_id); 605 * 606 * master goes to start = entry point 607 * start(phys_cpu_id, start, 0); 608 * 609 * 610 * a wrapper is needed to call existing kernels, here is an approximate 611 * description of one method: 612 * 613 * v2: (2.6.10) 614 * start will be near the boot_block (maybe 0x100 bytes before it?) 615 * it will have a 0x60, which will b to boot_block, where it will wait 616 * and 0 will store phys into struct boot-block and load r3 from there, 617 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again 618 * 619 * v1: (2.6.9) 620 * boot block will have all cpus scanning device tree to see if they 621 * are the boot cpu ????? 622 * other device tree differences (prop sizes, va vs pa, etc)... 623 */ 624 mr r3,r25 # my phys cpu 625 mr r4,r30 # start, aka phys mem offset 626 mtlr 4 627 li r5,0 628 blr /* image->start(physid, image->start, 0); */ 629#endif /* CONFIG_KEXEC */ 630