1/* 2 * ultra.S: Don't expand these all over the place... 3 * 4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) 5 */ 6 7#include <asm/asi.h> 8#include <asm/pgtable.h> 9#include <asm/page.h> 10#include <asm/spitfire.h> 11#include <asm/mmu_context.h> 12#include <asm/mmu.h> 13#include <asm/pil.h> 14#include <asm/head.h> 15#include <asm/thread_info.h> 16#include <asm/cacheflush.h> 17#include <asm/hypervisor.h> 18#include <asm/cpudata.h> 19 20 /* Basically, most of the Spitfire vs. Cheetah madness 21 * has to do with the fact that Cheetah does not support 22 * IMMU flushes out of the secondary context. Someone needs 23 * to throw a south lake birthday party for the folks 24 * in Microelectronics who refused to fix this shit. 25 */ 26 27 /* This file is meant to be read efficiently by the CPU, not humans. 28 * Staraj sie tego nikomu nie pierdolnac... 29 */ 30 .text 31 .align 32 32 .globl __flush_tlb_mm 33__flush_tlb_mm: /* 19 insns */ 34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ 35 ldxa [%o1] ASI_DMMU, %g2 36 cmp %g2, %o0 37 bne,pn %icc, __spitfire_flush_tlb_mm_slow 38 mov 0x50, %g3 39 stxa %g0, [%g3] ASI_DMMU_DEMAP 40 stxa %g0, [%g3] ASI_IMMU_DEMAP 41 sethi %hi(KERNBASE), %g3 42 flush %g3 43 retl 44 nop 45 nop 46 nop 47 nop 48 nop 49 nop 50 nop 51 nop 52 nop 53 nop 54 55 .align 32 56 .globl __flush_tlb_page 57__flush_tlb_page: /* 22 insns */ 58 /* %o0 = context, %o1 = vaddr */ 59 rdpr %pstate, %g7 60 andn %g7, PSTATE_IE, %g2 61 wrpr %g2, %pstate 62 mov SECONDARY_CONTEXT, %o4 63 ldxa [%o4] ASI_DMMU, %g2 64 stxa %o0, [%o4] ASI_DMMU 65 andcc %o1, 1, %g0 66 andn %o1, 1, %o3 67 be,pn %icc, 1f 68 or %o3, 0x10, %o3 69 stxa %g0, [%o3] ASI_IMMU_DEMAP 701: stxa %g0, [%o3] ASI_DMMU_DEMAP 71 membar #Sync 72 stxa %g2, [%o4] ASI_DMMU 73 sethi %hi(KERNBASE), %o4 74 flush %o4 75 retl 76 wrpr %g7, 0x0, %pstate 77 nop 78 nop 79 nop 80 nop 81 82 .align 32 83 .globl __flush_tlb_pending 84__flush_tlb_pending: /* 27 insns */ 85 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 86 rdpr %pstate, %g7 87 sllx %o1, 3, %o1 88 andn %g7, PSTATE_IE, %g2 89 wrpr %g2, %pstate 90 mov SECONDARY_CONTEXT, %o4 91 ldxa [%o4] ASI_DMMU, %g2 92 stxa %o0, [%o4] ASI_DMMU 931: sub %o1, (1 << 3), %o1 94 ldx [%o2 + %o1], %o3 95 andcc %o3, 1, %g0 96 andn %o3, 1, %o3 97 be,pn %icc, 2f 98 or %o3, 0x10, %o3 99 stxa %g0, [%o3] ASI_IMMU_DEMAP 1002: stxa %g0, [%o3] ASI_DMMU_DEMAP 101 membar #Sync 102 brnz,pt %o1, 1b 103 nop 104 stxa %g2, [%o4] ASI_DMMU 105 sethi %hi(KERNBASE), %o4 106 flush %o4 107 retl 108 wrpr %g7, 0x0, %pstate 109 nop 110 nop 111 nop 112 nop 113 114 .align 32 115 .globl __flush_tlb_kernel_range 116__flush_tlb_kernel_range: /* 19 insns */ 117 /* %o0=start, %o1=end */ 118 cmp %o0, %o1 119 be,pn %xcc, 2f 120 sethi %hi(PAGE_SIZE), %o4 121 sub %o1, %o0, %o3 122 sub %o3, %o4, %o3 123 or %o0, 0x20, %o0 ! Nucleus 1241: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 125 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP 126 membar #Sync 127 brnz,pt %o3, 1b 128 sub %o3, %o4, %o3 1292: sethi %hi(KERNBASE), %o3 130 flush %o3 131 retl 132 nop 133 nop 134 nop 135 nop 136 nop 137 138__spitfire_flush_tlb_mm_slow: 139 rdpr %pstate, %g1 140 wrpr %g1, PSTATE_IE, %pstate 141 stxa %o0, [%o1] ASI_DMMU 142 stxa %g0, [%g3] ASI_DMMU_DEMAP 143 stxa %g0, [%g3] ASI_IMMU_DEMAP 144 flush %g6 145 stxa %g2, [%o1] ASI_DMMU 146 sethi %hi(KERNBASE), %o1 147 flush %o1 148 retl 149 wrpr %g1, 0, %pstate 150 151/* 152 * The following code flushes one page_size worth. 153 */ 154 .section .kprobes.text, "ax" 155 .align 32 156 .globl __flush_icache_page 157__flush_icache_page: /* %o0 = phys_page */ 158 srlx %o0, PAGE_SHIFT, %o0 159 sethi %hi(PAGE_OFFSET), %g1 160 sllx %o0, PAGE_SHIFT, %o0 161 sethi %hi(PAGE_SIZE), %g2 162 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 163 add %o0, %g1, %o0 1641: subcc %g2, 32, %g2 165 bne,pt %icc, 1b 166 flush %o0 + %g2 167 retl 168 nop 169 170#ifdef DCACHE_ALIASING_POSSIBLE 171 172#if (PAGE_SHIFT != 13) 173#error only page shift of 13 is supported by dcache flush 174#endif 175 176#define DTAG_MASK 0x3 177 178 /* This routine is Spitfire specific so the hardcoded 179 * D-cache size and line-size are OK. 180 */ 181 .align 64 182 .globl __flush_dcache_page 183__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ 184 sethi %hi(PAGE_OFFSET), %g1 185 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 186 sub %o0, %g1, %o0 ! physical address 187 srlx %o0, 11, %o0 ! make D-cache TAG 188 sethi %hi(1 << 14), %o2 ! D-cache size 189 sub %o2, (1 << 5), %o2 ! D-cache line size 1901: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG 191 andcc %o3, DTAG_MASK, %g0 ! Valid? 192 be,pn %xcc, 2f ! Nope, branch 193 andn %o3, DTAG_MASK, %o3 ! Clear valid bits 194 cmp %o3, %o0 ! TAG match? 195 bne,pt %xcc, 2f ! Nope, branch 196 nop 197 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG 198 membar #Sync 1992: brnz,pt %o2, 1b 200 sub %o2, (1 << 5), %o2 ! D-cache line size 201 202 /* The I-cache does not snoop local stores so we 203 * better flush that too when necessary. 204 */ 205 brnz,pt %o1, __flush_icache_page 206 sllx %o0, 11, %o0 207 retl 208 nop 209 210#endif /* DCACHE_ALIASING_POSSIBLE */ 211 212 .previous 213 214 /* Cheetah specific versions, patched at boot time. */ 215__cheetah_flush_tlb_mm: /* 19 insns */ 216 rdpr %pstate, %g7 217 andn %g7, PSTATE_IE, %g2 218 wrpr %g2, 0x0, %pstate 219 wrpr %g0, 1, %tl 220 mov PRIMARY_CONTEXT, %o2 221 mov 0x40, %g3 222 ldxa [%o2] ASI_DMMU, %g2 223 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 224 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 225 or %o0, %o1, %o0 /* Preserve nucleus page size fields */ 226 stxa %o0, [%o2] ASI_DMMU 227 stxa %g0, [%g3] ASI_DMMU_DEMAP 228 stxa %g0, [%g3] ASI_IMMU_DEMAP 229 stxa %g2, [%o2] ASI_DMMU 230 sethi %hi(KERNBASE), %o2 231 flush %o2 232 wrpr %g0, 0, %tl 233 retl 234 wrpr %g7, 0x0, %pstate 235 236__cheetah_flush_tlb_page: /* 22 insns */ 237 /* %o0 = context, %o1 = vaddr */ 238 rdpr %pstate, %g7 239 andn %g7, PSTATE_IE, %g2 240 wrpr %g2, 0x0, %pstate 241 wrpr %g0, 1, %tl 242 mov PRIMARY_CONTEXT, %o4 243 ldxa [%o4] ASI_DMMU, %g2 244 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 245 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 246 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 247 stxa %o0, [%o4] ASI_DMMU 248 andcc %o1, 1, %g0 249 be,pn %icc, 1f 250 andn %o1, 1, %o3 251 stxa %g0, [%o3] ASI_IMMU_DEMAP 2521: stxa %g0, [%o3] ASI_DMMU_DEMAP 253 membar #Sync 254 stxa %g2, [%o4] ASI_DMMU 255 sethi %hi(KERNBASE), %o4 256 flush %o4 257 wrpr %g0, 0, %tl 258 retl 259 wrpr %g7, 0x0, %pstate 260 261__cheetah_flush_tlb_pending: /* 27 insns */ 262 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 263 rdpr %pstate, %g7 264 sllx %o1, 3, %o1 265 andn %g7, PSTATE_IE, %g2 266 wrpr %g2, 0x0, %pstate 267 wrpr %g0, 1, %tl 268 mov PRIMARY_CONTEXT, %o4 269 ldxa [%o4] ASI_DMMU, %g2 270 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 271 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 272 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 273 stxa %o0, [%o4] ASI_DMMU 2741: sub %o1, (1 << 3), %o1 275 ldx [%o2 + %o1], %o3 276 andcc %o3, 1, %g0 277 be,pn %icc, 2f 278 andn %o3, 1, %o3 279 stxa %g0, [%o3] ASI_IMMU_DEMAP 2802: stxa %g0, [%o3] ASI_DMMU_DEMAP 281 membar #Sync 282 brnz,pt %o1, 1b 283 nop 284 stxa %g2, [%o4] ASI_DMMU 285 sethi %hi(KERNBASE), %o4 286 flush %o4 287 wrpr %g0, 0, %tl 288 retl 289 wrpr %g7, 0x0, %pstate 290 291#ifdef DCACHE_ALIASING_POSSIBLE 292__cheetah_flush_dcache_page: /* 11 insns */ 293 sethi %hi(PAGE_OFFSET), %g1 294 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 295 sub %o0, %g1, %o0 296 sethi %hi(PAGE_SIZE), %o4 2971: subcc %o4, (1 << 5), %o4 298 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE 299 membar #Sync 300 bne,pt %icc, 1b 301 nop 302 retl /* I-cache flush never needed on Cheetah, see callers. */ 303 nop 304#endif /* DCACHE_ALIASING_POSSIBLE */ 305 306 /* Hypervisor specific versions, patched at boot time. */ 307__hypervisor_tlb_tl0_error: 308 save %sp, -192, %sp 309 mov %i0, %o0 310 call hypervisor_tlbop_error 311 mov %i1, %o1 312 ret 313 restore 314 315__hypervisor_flush_tlb_mm: /* 19 insns */ 316 mov %o0, %o2 /* ARG2: mmu context */ 317 mov 0, %o0 /* ARG0: CPU lists unimplemented */ 318 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 319 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 320 mov HV_FAST_MMU_DEMAP_CTX, %o5 321 ta HV_FAST_TRAP 322 brnz,pn %o0, 1f 323 mov HV_FAST_MMU_DEMAP_CTX, %o1 324 retl 325 nop 3261: sethi %hi(__hypervisor_tlb_tl0_error), %o5 327 jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 328 nop 329 nop 330 nop 331 nop 332 nop 333 nop 334 nop 335 336__hypervisor_flush_tlb_page: /* 22 insns */ 337 /* %o0 = context, %o1 = vaddr */ 338 mov %o0, %g2 339 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ 340 mov %g2, %o1 /* ARG1: mmu context */ 341 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 342 srlx %o0, PAGE_SHIFT, %o0 343 sllx %o0, PAGE_SHIFT, %o0 344 ta HV_MMU_UNMAP_ADDR_TRAP 345 brnz,pn %o0, 1f 346 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 347 retl 348 nop 3491: sethi %hi(__hypervisor_tlb_tl0_error), %o2 350 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 351 nop 352 nop 353 nop 354 nop 355 nop 356 nop 357 nop 358 nop 359 nop 360 361__hypervisor_flush_tlb_pending: /* 27 insns */ 362 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 363 sllx %o1, 3, %g1 364 mov %o2, %g2 365 mov %o0, %g3 3661: sub %g1, (1 << 3), %g1 367 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ 368 mov %g3, %o1 /* ARG1: mmu context */ 369 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 370 srlx %o0, PAGE_SHIFT, %o0 371 sllx %o0, PAGE_SHIFT, %o0 372 ta HV_MMU_UNMAP_ADDR_TRAP 373 brnz,pn %o0, 1f 374 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 375 brnz,pt %g1, 1b 376 nop 377 retl 378 nop 3791: sethi %hi(__hypervisor_tlb_tl0_error), %o2 380 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 381 nop 382 nop 383 nop 384 nop 385 nop 386 nop 387 nop 388 nop 389 nop 390 391__hypervisor_flush_tlb_kernel_range: /* 19 insns */ 392 /* %o0=start, %o1=end */ 393 cmp %o0, %o1 394 be,pn %xcc, 2f 395 sethi %hi(PAGE_SIZE), %g3 396 mov %o0, %g1 397 sub %o1, %g1, %g2 398 sub %g2, %g3, %g2 3991: add %g1, %g2, %o0 /* ARG0: virtual address */ 400 mov 0, %o1 /* ARG1: mmu context */ 401 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 402 ta HV_MMU_UNMAP_ADDR_TRAP 403 brnz,pn %o0, 3f 404 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 405 brnz,pt %g2, 1b 406 sub %g2, %g3, %g2 4072: retl 408 nop 4093: sethi %hi(__hypervisor_tlb_tl0_error), %o2 410 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 411 nop 412 413#ifdef DCACHE_ALIASING_POSSIBLE 414 /* XXX Niagara and friends have an 8K cache, so no aliasing is 415 * XXX possible, but nothing explicit in the Hypervisor API 416 * XXX guarantees this. 417 */ 418__hypervisor_flush_dcache_page: /* 2 insns */ 419 retl 420 nop 421#endif 422 423tlb_patch_one: 4241: lduw [%o1], %g1 425 stw %g1, [%o0] 426 flush %o0 427 subcc %o2, 1, %o2 428 add %o1, 4, %o1 429 bne,pt %icc, 1b 430 add %o0, 4, %o0 431 retl 432 nop 433 434 .globl cheetah_patch_cachetlbops 435cheetah_patch_cachetlbops: 436 save %sp, -128, %sp 437 438 sethi %hi(__flush_tlb_mm), %o0 439 or %o0, %lo(__flush_tlb_mm), %o0 440 sethi %hi(__cheetah_flush_tlb_mm), %o1 441 or %o1, %lo(__cheetah_flush_tlb_mm), %o1 442 call tlb_patch_one 443 mov 19, %o2 444 445 sethi %hi(__flush_tlb_page), %o0 446 or %o0, %lo(__flush_tlb_page), %o0 447 sethi %hi(__cheetah_flush_tlb_page), %o1 448 or %o1, %lo(__cheetah_flush_tlb_page), %o1 449 call tlb_patch_one 450 mov 22, %o2 451 452 sethi %hi(__flush_tlb_pending), %o0 453 or %o0, %lo(__flush_tlb_pending), %o0 454 sethi %hi(__cheetah_flush_tlb_pending), %o1 455 or %o1, %lo(__cheetah_flush_tlb_pending), %o1 456 call tlb_patch_one 457 mov 27, %o2 458 459#ifdef DCACHE_ALIASING_POSSIBLE 460 sethi %hi(__flush_dcache_page), %o0 461 or %o0, %lo(__flush_dcache_page), %o0 462 sethi %hi(__cheetah_flush_dcache_page), %o1 463 or %o1, %lo(__cheetah_flush_dcache_page), %o1 464 call tlb_patch_one 465 mov 11, %o2 466#endif /* DCACHE_ALIASING_POSSIBLE */ 467 468 ret 469 restore 470 471#ifdef CONFIG_SMP 472 /* These are all called by the slaves of a cross call, at 473 * trap level 1, with interrupts fully disabled. 474 * 475 * Register usage: 476 * %g5 mm->context (all tlb flushes) 477 * %g1 address arg 1 (tlb page and range flushes) 478 * %g7 address arg 2 (tlb range flush only) 479 * 480 * %g6 scratch 1 481 * %g2 scratch 2 482 * %g3 scratch 3 483 * %g4 scratch 4 484 */ 485 .align 32 486 .globl xcall_flush_tlb_mm 487xcall_flush_tlb_mm: /* 21 insns */ 488 mov PRIMARY_CONTEXT, %g2 489 ldxa [%g2] ASI_DMMU, %g3 490 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 491 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 492 or %g5, %g4, %g5 /* Preserve nucleus page size fields */ 493 stxa %g5, [%g2] ASI_DMMU 494 mov 0x40, %g4 495 stxa %g0, [%g4] ASI_DMMU_DEMAP 496 stxa %g0, [%g4] ASI_IMMU_DEMAP 497 stxa %g3, [%g2] ASI_DMMU 498 retry 499 nop 500 nop 501 nop 502 nop 503 nop 504 nop 505 nop 506 nop 507 nop 508 nop 509 510 .globl xcall_flush_tlb_page 511xcall_flush_tlb_page: /* 17 insns */ 512 /* %g5=context, %g1=vaddr */ 513 mov PRIMARY_CONTEXT, %g4 514 ldxa [%g4] ASI_DMMU, %g2 515 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 516 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 517 or %g5, %g4, %g5 518 mov PRIMARY_CONTEXT, %g4 519 stxa %g5, [%g4] ASI_DMMU 520 andcc %g1, 0x1, %g0 521 be,pn %icc, 2f 522 andn %g1, 0x1, %g5 523 stxa %g0, [%g5] ASI_IMMU_DEMAP 5242: stxa %g0, [%g5] ASI_DMMU_DEMAP 525 membar #Sync 526 stxa %g2, [%g4] ASI_DMMU 527 retry 528 nop 529 nop 530 531 .globl xcall_flush_tlb_kernel_range 532xcall_flush_tlb_kernel_range: /* 25 insns */ 533 sethi %hi(PAGE_SIZE - 1), %g2 534 or %g2, %lo(PAGE_SIZE - 1), %g2 535 andn %g1, %g2, %g1 536 andn %g7, %g2, %g7 537 sub %g7, %g1, %g3 538 add %g2, 1, %g2 539 sub %g3, %g2, %g3 540 or %g1, 0x20, %g1 ! Nucleus 5411: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 542 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP 543 membar #Sync 544 brnz,pt %g3, 1b 545 sub %g3, %g2, %g3 546 retry 547 nop 548 nop 549 nop 550 nop 551 nop 552 nop 553 nop 554 nop 555 nop 556 nop 557 nop 558 559 /* This runs in a very controlled environment, so we do 560 * not need to worry about BH races etc. 561 */ 562 .globl xcall_sync_tick 563xcall_sync_tick: 564 565661: rdpr %pstate, %g2 566 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate 567 .section .sun4v_2insn_patch, "ax" 568 .word 661b 569 nop 570 nop 571 .previous 572 573 rdpr %pil, %g2 574 wrpr %g0, PIL_NORMAL_MAX, %pil 575 sethi %hi(109f), %g7 576 b,pt %xcc, etrap_irq 577109: or %g7, %lo(109b), %g7 578#ifdef CONFIG_TRACE_IRQFLAGS 579 call trace_hardirqs_off 580 nop 581#endif 582 call smp_synchronize_tick_client 583 nop 584 b rtrap_xcall 585 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 586 587 .globl xcall_fetch_glob_regs 588xcall_fetch_glob_regs: 589 sethi %hi(global_cpu_snapshot), %g1 590 or %g1, %lo(global_cpu_snapshot), %g1 591 __GET_CPUID(%g2) 592 sllx %g2, 6, %g3 593 add %g1, %g3, %g1 594 rdpr %tstate, %g7 595 stx %g7, [%g1 + GR_SNAP_TSTATE] 596 rdpr %tpc, %g7 597 stx %g7, [%g1 + GR_SNAP_TPC] 598 rdpr %tnpc, %g7 599 stx %g7, [%g1 + GR_SNAP_TNPC] 600 stx %o7, [%g1 + GR_SNAP_O7] 601 stx %i7, [%g1 + GR_SNAP_I7] 602 /* Don't try this at home kids... */ 603 rdpr %cwp, %g3 604 sub %g3, 1, %g7 605 wrpr %g7, %cwp 606 mov %i7, %g7 607 wrpr %g3, %cwp 608 stx %g7, [%g1 + GR_SNAP_RPC] 609 sethi %hi(trap_block), %g7 610 or %g7, %lo(trap_block), %g7 611 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 612 add %g7, %g2, %g7 613 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 614 stx %g3, [%g1 + GR_SNAP_THREAD] 615 retry 616 617 .globl xcall_fetch_glob_pmu 618xcall_fetch_glob_pmu: 619 sethi %hi(global_cpu_snapshot), %g1 620 or %g1, %lo(global_cpu_snapshot), %g1 621 __GET_CPUID(%g2) 622 sllx %g2, 6, %g3 623 add %g1, %g3, %g1 624 rd %pic, %g7 625 stx %g7, [%g1 + (4 * 8)] 626 rd %pcr, %g7 627 stx %g7, [%g1 + (0 * 8)] 628 retry 629 630 .globl xcall_fetch_glob_pmu_n4 631xcall_fetch_glob_pmu_n4: 632 sethi %hi(global_cpu_snapshot), %g1 633 or %g1, %lo(global_cpu_snapshot), %g1 634 __GET_CPUID(%g2) 635 sllx %g2, 6, %g3 636 add %g1, %g3, %g1 637 638 ldxa [%g0] ASI_PIC, %g7 639 stx %g7, [%g1 + (4 * 8)] 640 mov 0x08, %g3 641 ldxa [%g3] ASI_PIC, %g7 642 stx %g7, [%g1 + (5 * 8)] 643 mov 0x10, %g3 644 ldxa [%g3] ASI_PIC, %g7 645 stx %g7, [%g1 + (6 * 8)] 646 mov 0x18, %g3 647 ldxa [%g3] ASI_PIC, %g7 648 stx %g7, [%g1 + (7 * 8)] 649 650 mov %o0, %g2 651 mov %o1, %g3 652 mov %o5, %g7 653 654 mov HV_FAST_VT_GET_PERFREG, %o5 655 mov 3, %o0 656 ta HV_FAST_TRAP 657 stx %o1, [%g1 + (3 * 8)] 658 mov HV_FAST_VT_GET_PERFREG, %o5 659 mov 2, %o0 660 ta HV_FAST_TRAP 661 stx %o1, [%g1 + (2 * 8)] 662 mov HV_FAST_VT_GET_PERFREG, %o5 663 mov 1, %o0 664 ta HV_FAST_TRAP 665 stx %o1, [%g1 + (1 * 8)] 666 mov HV_FAST_VT_GET_PERFREG, %o5 667 mov 0, %o0 668 ta HV_FAST_TRAP 669 stx %o1, [%g1 + (0 * 8)] 670 671 mov %g2, %o0 672 mov %g3, %o1 673 mov %g7, %o5 674 675 retry 676 677#ifdef DCACHE_ALIASING_POSSIBLE 678 .align 32 679 .globl xcall_flush_dcache_page_cheetah 680xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ 681 sethi %hi(PAGE_SIZE), %g3 6821: subcc %g3, (1 << 5), %g3 683 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE 684 membar #Sync 685 bne,pt %icc, 1b 686 nop 687 retry 688 nop 689#endif /* DCACHE_ALIASING_POSSIBLE */ 690 691 .globl xcall_flush_dcache_page_spitfire 692xcall_flush_dcache_page_spitfire: /* %g1 == physical page address 693 %g7 == kernel page virtual address 694 %g5 == (page->mapping != NULL) */ 695#ifdef DCACHE_ALIASING_POSSIBLE 696 srlx %g1, (13 - 2), %g1 ! Form tag comparitor 697 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K 698 sub %g3, (1 << 5), %g3 ! D$ linesize == 32 6991: ldxa [%g3] ASI_DCACHE_TAG, %g2 700 andcc %g2, 0x3, %g0 701 be,pn %xcc, 2f 702 andn %g2, 0x3, %g2 703 cmp %g2, %g1 704 705 bne,pt %xcc, 2f 706 nop 707 stxa %g0, [%g3] ASI_DCACHE_TAG 708 membar #Sync 7092: cmp %g3, 0 710 bne,pt %xcc, 1b 711 sub %g3, (1 << 5), %g3 712 713 brz,pn %g5, 2f 714#endif /* DCACHE_ALIASING_POSSIBLE */ 715 sethi %hi(PAGE_SIZE), %g3 716 7171: flush %g7 718 subcc %g3, (1 << 5), %g3 719 bne,pt %icc, 1b 720 add %g7, (1 << 5), %g7 721 7222: retry 723 nop 724 nop 725 726 /* %g5: error 727 * %g6: tlb op 728 */ 729__hypervisor_tlb_xcall_error: 730 mov %g5, %g4 731 mov %g6, %g5 732 ba,pt %xcc, etrap 733 rd %pc, %g7 734 mov %l4, %o0 735 call hypervisor_tlbop_error_xcall 736 mov %l5, %o1 737 ba,a,pt %xcc, rtrap 738 739 .globl __hypervisor_xcall_flush_tlb_mm 740__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ 741 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ 742 mov %o0, %g2 743 mov %o1, %g3 744 mov %o2, %g4 745 mov %o3, %g1 746 mov %o5, %g7 747 clr %o0 /* ARG0: CPU lists unimplemented */ 748 clr %o1 /* ARG1: CPU lists unimplemented */ 749 mov %g5, %o2 /* ARG2: mmu context */ 750 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 751 mov HV_FAST_MMU_DEMAP_CTX, %o5 752 ta HV_FAST_TRAP 753 mov HV_FAST_MMU_DEMAP_CTX, %g6 754 brnz,pn %o0, __hypervisor_tlb_xcall_error 755 mov %o0, %g5 756 mov %g2, %o0 757 mov %g3, %o1 758 mov %g4, %o2 759 mov %g1, %o3 760 mov %g7, %o5 761 membar #Sync 762 retry 763 764 .globl __hypervisor_xcall_flush_tlb_page 765__hypervisor_xcall_flush_tlb_page: /* 17 insns */ 766 /* %g5=ctx, %g1=vaddr */ 767 mov %o0, %g2 768 mov %o1, %g3 769 mov %o2, %g4 770 mov %g1, %o0 /* ARG0: virtual address */ 771 mov %g5, %o1 /* ARG1: mmu context */ 772 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 773 srlx %o0, PAGE_SHIFT, %o0 774 sllx %o0, PAGE_SHIFT, %o0 775 ta HV_MMU_UNMAP_ADDR_TRAP 776 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 777 brnz,a,pn %o0, __hypervisor_tlb_xcall_error 778 mov %o0, %g5 779 mov %g2, %o0 780 mov %g3, %o1 781 mov %g4, %o2 782 membar #Sync 783 retry 784 785 .globl __hypervisor_xcall_flush_tlb_kernel_range 786__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ 787 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ 788 sethi %hi(PAGE_SIZE - 1), %g2 789 or %g2, %lo(PAGE_SIZE - 1), %g2 790 andn %g1, %g2, %g1 791 andn %g7, %g2, %g7 792 sub %g7, %g1, %g3 793 add %g2, 1, %g2 794 sub %g3, %g2, %g3 795 mov %o0, %g2 796 mov %o1, %g4 797 mov %o2, %g7 7981: add %g1, %g3, %o0 /* ARG0: virtual address */ 799 mov 0, %o1 /* ARG1: mmu context */ 800 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 801 ta HV_MMU_UNMAP_ADDR_TRAP 802 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 803 brnz,pn %o0, __hypervisor_tlb_xcall_error 804 mov %o0, %g5 805 sethi %hi(PAGE_SIZE), %o2 806 brnz,pt %g3, 1b 807 sub %g3, %o2, %g3 808 mov %g2, %o0 809 mov %g4, %o1 810 mov %g7, %o2 811 membar #Sync 812 retry 813 814 /* These just get rescheduled to PIL vectors. */ 815 .globl xcall_call_function 816xcall_call_function: 817 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint 818 retry 819 820 .globl xcall_call_function_single 821xcall_call_function_single: 822 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint 823 retry 824 825 .globl xcall_receive_signal 826xcall_receive_signal: 827 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint 828 retry 829 830 .globl xcall_capture 831xcall_capture: 832 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint 833 retry 834 835 .globl xcall_new_mmu_context_version 836xcall_new_mmu_context_version: 837 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint 838 retry 839 840#ifdef CONFIG_KGDB 841 .globl xcall_kgdb_capture 842xcall_kgdb_capture: 843 wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint 844 retry 845#endif 846 847#endif /* CONFIG_SMP */ 848 849 850 .globl hypervisor_patch_cachetlbops 851hypervisor_patch_cachetlbops: 852 save %sp, -128, %sp 853 854 sethi %hi(__flush_tlb_mm), %o0 855 or %o0, %lo(__flush_tlb_mm), %o0 856 sethi %hi(__hypervisor_flush_tlb_mm), %o1 857 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 858 call tlb_patch_one 859 mov 19, %o2 860 861 sethi %hi(__flush_tlb_page), %o0 862 or %o0, %lo(__flush_tlb_page), %o0 863 sethi %hi(__hypervisor_flush_tlb_page), %o1 864 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 865 call tlb_patch_one 866 mov 22, %o2 867 868 sethi %hi(__flush_tlb_pending), %o0 869 or %o0, %lo(__flush_tlb_pending), %o0 870 sethi %hi(__hypervisor_flush_tlb_pending), %o1 871 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 872 call tlb_patch_one 873 mov 27, %o2 874 875 sethi %hi(__flush_tlb_kernel_range), %o0 876 or %o0, %lo(__flush_tlb_kernel_range), %o0 877 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 878 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 879 call tlb_patch_one 880 mov 19, %o2 881 882#ifdef DCACHE_ALIASING_POSSIBLE 883 sethi %hi(__flush_dcache_page), %o0 884 or %o0, %lo(__flush_dcache_page), %o0 885 sethi %hi(__hypervisor_flush_dcache_page), %o1 886 or %o1, %lo(__hypervisor_flush_dcache_page), %o1 887 call tlb_patch_one 888 mov 2, %o2 889#endif /* DCACHE_ALIASING_POSSIBLE */ 890 891#ifdef CONFIG_SMP 892 sethi %hi(xcall_flush_tlb_mm), %o0 893 or %o0, %lo(xcall_flush_tlb_mm), %o0 894 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 895 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 896 call tlb_patch_one 897 mov 21, %o2 898 899 sethi %hi(xcall_flush_tlb_page), %o0 900 or %o0, %lo(xcall_flush_tlb_page), %o0 901 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 902 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 903 call tlb_patch_one 904 mov 17, %o2 905 906 sethi %hi(xcall_flush_tlb_kernel_range), %o0 907 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 908 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 909 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 910 call tlb_patch_one 911 mov 25, %o2 912#endif /* CONFIG_SMP */ 913 914 ret 915 restore 916