1/* 2 * ultra.S: Don't expand these all over the place... 3 * 4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) 5 */ 6 7#include <asm/asi.h> 8#include <asm/pgtable.h> 9#include <asm/page.h> 10#include <asm/spitfire.h> 11#include <asm/mmu_context.h> 12#include <asm/mmu.h> 13#include <asm/pil.h> 14#include <asm/head.h> 15#include <asm/thread_info.h> 16#include <asm/cacheflush.h> 17#include <asm/hypervisor.h> 18#include <asm/cpudata.h> 19 20 /* Basically, most of the Spitfire vs. Cheetah madness 21 * has to do with the fact that Cheetah does not support 22 * IMMU flushes out of the secondary context. Someone needs 23 * to throw a south lake birthday party for the folks 24 * in Microelectronics who refused to fix this shit. 25 */ 26 27 /* This file is meant to be read efficiently by the CPU, not humans. 28 * Staraj sie tego nikomu nie pierdolnac... 29 */ 30 .text 31 .align 32 32 .globl __flush_tlb_mm 33__flush_tlb_mm: /* 18 insns */ 34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ 35 ldxa [%o1] ASI_DMMU, %g2 36 cmp %g2, %o0 37 bne,pn %icc, __spitfire_flush_tlb_mm_slow 38 mov 0x50, %g3 39 stxa %g0, [%g3] ASI_DMMU_DEMAP 40 stxa %g0, [%g3] ASI_IMMU_DEMAP 41 sethi %hi(KERNBASE), %g3 42 flush %g3 43 retl 44 nop 45 nop 46 nop 47 nop 48 nop 49 nop 50 nop 51 nop 52 nop 53 nop 54 55 .align 32 56 .globl __flush_tlb_page 57__flush_tlb_page: /* 22 insns */ 58 /* %o0 = context, %o1 = vaddr */ 59 rdpr %pstate, %g7 60 andn %g7, PSTATE_IE, %g2 61 wrpr %g2, %pstate 62 mov SECONDARY_CONTEXT, %o4 63 ldxa [%o4] ASI_DMMU, %g2 64 stxa %o0, [%o4] ASI_DMMU 65 andcc %o1, 1, %g0 66 andn %o1, 1, %o3 67 be,pn %icc, 1f 68 or %o3, 0x10, %o3 69 stxa %g0, [%o3] ASI_IMMU_DEMAP 701: stxa %g0, [%o3] ASI_DMMU_DEMAP 71 membar #Sync 72 stxa %g2, [%o4] ASI_DMMU 73 sethi %hi(KERNBASE), %o4 74 flush %o4 75 retl 76 wrpr %g7, 0x0, %pstate 77 nop 78 nop 79 nop 80 nop 81 82 .align 32 83 .globl __flush_tlb_pending 84__flush_tlb_pending: /* 26 insns */ 85 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 86 rdpr %pstate, %g7 87 sllx %o1, 3, %o1 88 andn %g7, PSTATE_IE, %g2 89 wrpr %g2, %pstate 90 mov SECONDARY_CONTEXT, %o4 91 ldxa [%o4] ASI_DMMU, %g2 92 stxa %o0, [%o4] ASI_DMMU 931: sub %o1, (1 << 3), %o1 94 ldx [%o2 + %o1], %o3 95 andcc %o3, 1, %g0 96 andn %o3, 1, %o3 97 be,pn %icc, 2f 98 or %o3, 0x10, %o3 99 stxa %g0, [%o3] ASI_IMMU_DEMAP 1002: stxa %g0, [%o3] ASI_DMMU_DEMAP 101 membar #Sync 102 brnz,pt %o1, 1b 103 nop 104 stxa %g2, [%o4] ASI_DMMU 105 sethi %hi(KERNBASE), %o4 106 flush %o4 107 retl 108 wrpr %g7, 0x0, %pstate 109 nop 110 nop 111 nop 112 nop 113 114 .align 32 115 .globl __flush_tlb_kernel_range 116__flush_tlb_kernel_range: /* 16 insns */ 117 /* %o0=start, %o1=end */ 118 cmp %o0, %o1 119 be,pn %xcc, 2f 120 sethi %hi(PAGE_SIZE), %o4 121 sub %o1, %o0, %o3 122 sub %o3, %o4, %o3 123 or %o0, 0x20, %o0 ! Nucleus 1241: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 125 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP 126 membar #Sync 127 brnz,pt %o3, 1b 128 sub %o3, %o4, %o3 1292: sethi %hi(KERNBASE), %o3 130 flush %o3 131 retl 132 nop 133 nop 134 135__spitfire_flush_tlb_mm_slow: 136 rdpr %pstate, %g1 137 wrpr %g1, PSTATE_IE, %pstate 138 stxa %o0, [%o1] ASI_DMMU 139 stxa %g0, [%g3] ASI_DMMU_DEMAP 140 stxa %g0, [%g3] ASI_IMMU_DEMAP 141 flush %g6 142 stxa %g2, [%o1] ASI_DMMU 143 sethi %hi(KERNBASE), %o1 144 flush %o1 145 retl 146 wrpr %g1, 0, %pstate 147 148/* 149 * The following code flushes one page_size worth. 150 */ 151 .section .kprobes.text, "ax" 152 .align 32 153 .globl __flush_icache_page 154__flush_icache_page: /* %o0 = phys_page */ 155 srlx %o0, PAGE_SHIFT, %o0 156 sethi %uhi(PAGE_OFFSET), %g1 157 sllx %o0, PAGE_SHIFT, %o0 158 sethi %hi(PAGE_SIZE), %g2 159 sllx %g1, 32, %g1 160 add %o0, %g1, %o0 1611: subcc %g2, 32, %g2 162 bne,pt %icc, 1b 163 flush %o0 + %g2 164 retl 165 nop 166 167#ifdef DCACHE_ALIASING_POSSIBLE 168 169#if (PAGE_SHIFT != 13) 170#error only page shift of 13 is supported by dcache flush 171#endif 172 173#define DTAG_MASK 0x3 174 175 /* This routine is Spitfire specific so the hardcoded 176 * D-cache size and line-size are OK. 177 */ 178 .align 64 179 .globl __flush_dcache_page 180__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ 181 sethi %uhi(PAGE_OFFSET), %g1 182 sllx %g1, 32, %g1 183 sub %o0, %g1, %o0 ! physical address 184 srlx %o0, 11, %o0 ! make D-cache TAG 185 sethi %hi(1 << 14), %o2 ! D-cache size 186 sub %o2, (1 << 5), %o2 ! D-cache line size 1871: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG 188 andcc %o3, DTAG_MASK, %g0 ! Valid? 189 be,pn %xcc, 2f ! Nope, branch 190 andn %o3, DTAG_MASK, %o3 ! Clear valid bits 191 cmp %o3, %o0 ! TAG match? 192 bne,pt %xcc, 2f ! Nope, branch 193 nop 194 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG 195 membar #Sync 1962: brnz,pt %o2, 1b 197 sub %o2, (1 << 5), %o2 ! D-cache line size 198 199 /* The I-cache does not snoop local stores so we 200 * better flush that too when necessary. 201 */ 202 brnz,pt %o1, __flush_icache_page 203 sllx %o0, 11, %o0 204 retl 205 nop 206 207#endif /* DCACHE_ALIASING_POSSIBLE */ 208 209 .previous 210 211 /* Cheetah specific versions, patched at boot time. */ 212__cheetah_flush_tlb_mm: /* 19 insns */ 213 rdpr %pstate, %g7 214 andn %g7, PSTATE_IE, %g2 215 wrpr %g2, 0x0, %pstate 216 wrpr %g0, 1, %tl 217 mov PRIMARY_CONTEXT, %o2 218 mov 0x40, %g3 219 ldxa [%o2] ASI_DMMU, %g2 220 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 221 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 222 or %o0, %o1, %o0 /* Preserve nucleus page size fields */ 223 stxa %o0, [%o2] ASI_DMMU 224 stxa %g0, [%g3] ASI_DMMU_DEMAP 225 stxa %g0, [%g3] ASI_IMMU_DEMAP 226 stxa %g2, [%o2] ASI_DMMU 227 sethi %hi(KERNBASE), %o2 228 flush %o2 229 wrpr %g0, 0, %tl 230 retl 231 wrpr %g7, 0x0, %pstate 232 233__cheetah_flush_tlb_page: /* 22 insns */ 234 /* %o0 = context, %o1 = vaddr */ 235 rdpr %pstate, %g7 236 andn %g7, PSTATE_IE, %g2 237 wrpr %g2, 0x0, %pstate 238 wrpr %g0, 1, %tl 239 mov PRIMARY_CONTEXT, %o4 240 ldxa [%o4] ASI_DMMU, %g2 241 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 242 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 243 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 244 stxa %o0, [%o4] ASI_DMMU 245 andcc %o1, 1, %g0 246 be,pn %icc, 1f 247 andn %o1, 1, %o3 248 stxa %g0, [%o3] ASI_IMMU_DEMAP 2491: stxa %g0, [%o3] ASI_DMMU_DEMAP 250 membar #Sync 251 stxa %g2, [%o4] ASI_DMMU 252 sethi %hi(KERNBASE), %o4 253 flush %o4 254 wrpr %g0, 0, %tl 255 retl 256 wrpr %g7, 0x0, %pstate 257 258__cheetah_flush_tlb_pending: /* 27 insns */ 259 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 260 rdpr %pstate, %g7 261 sllx %o1, 3, %o1 262 andn %g7, PSTATE_IE, %g2 263 wrpr %g2, 0x0, %pstate 264 wrpr %g0, 1, %tl 265 mov PRIMARY_CONTEXT, %o4 266 ldxa [%o4] ASI_DMMU, %g2 267 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 268 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 269 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 270 stxa %o0, [%o4] ASI_DMMU 2711: sub %o1, (1 << 3), %o1 272 ldx [%o2 + %o1], %o3 273 andcc %o3, 1, %g0 274 be,pn %icc, 2f 275 andn %o3, 1, %o3 276 stxa %g0, [%o3] ASI_IMMU_DEMAP 2772: stxa %g0, [%o3] ASI_DMMU_DEMAP 278 membar #Sync 279 brnz,pt %o1, 1b 280 nop 281 stxa %g2, [%o4] ASI_DMMU 282 sethi %hi(KERNBASE), %o4 283 flush %o4 284 wrpr %g0, 0, %tl 285 retl 286 wrpr %g7, 0x0, %pstate 287 288#ifdef DCACHE_ALIASING_POSSIBLE 289__cheetah_flush_dcache_page: /* 11 insns */ 290 sethi %uhi(PAGE_OFFSET), %g1 291 sllx %g1, 32, %g1 292 sub %o0, %g1, %o0 293 sethi %hi(PAGE_SIZE), %o4 2941: subcc %o4, (1 << 5), %o4 295 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE 296 membar #Sync 297 bne,pt %icc, 1b 298 nop 299 retl /* I-cache flush never needed on Cheetah, see callers. */ 300 nop 301#endif /* DCACHE_ALIASING_POSSIBLE */ 302 303 /* Hypervisor specific versions, patched at boot time. */ 304__hypervisor_tlb_tl0_error: 305 save %sp, -192, %sp 306 mov %i0, %o0 307 call hypervisor_tlbop_error 308 mov %i1, %o1 309 ret 310 restore 311 312__hypervisor_flush_tlb_mm: /* 10 insns */ 313 mov %o0, %o2 /* ARG2: mmu context */ 314 mov 0, %o0 /* ARG0: CPU lists unimplemented */ 315 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 316 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 317 mov HV_FAST_MMU_DEMAP_CTX, %o5 318 ta HV_FAST_TRAP 319 brnz,pn %o0, __hypervisor_tlb_tl0_error 320 mov HV_FAST_MMU_DEMAP_CTX, %o1 321 retl 322 nop 323 324__hypervisor_flush_tlb_page: /* 11 insns */ 325 /* %o0 = context, %o1 = vaddr */ 326 mov %o0, %g2 327 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ 328 mov %g2, %o1 /* ARG1: mmu context */ 329 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 330 srlx %o0, PAGE_SHIFT, %o0 331 sllx %o0, PAGE_SHIFT, %o0 332 ta HV_MMU_UNMAP_ADDR_TRAP 333 brnz,pn %o0, __hypervisor_tlb_tl0_error 334 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 335 retl 336 nop 337 338__hypervisor_flush_tlb_pending: /* 16 insns */ 339 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 340 sllx %o1, 3, %g1 341 mov %o2, %g2 342 mov %o0, %g3 3431: sub %g1, (1 << 3), %g1 344 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ 345 mov %g3, %o1 /* ARG1: mmu context */ 346 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 347 srlx %o0, PAGE_SHIFT, %o0 348 sllx %o0, PAGE_SHIFT, %o0 349 ta HV_MMU_UNMAP_ADDR_TRAP 350 brnz,pn %o0, __hypervisor_tlb_tl0_error 351 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 352 brnz,pt %g1, 1b 353 nop 354 retl 355 nop 356 357__hypervisor_flush_tlb_kernel_range: /* 16 insns */ 358 /* %o0=start, %o1=end */ 359 cmp %o0, %o1 360 be,pn %xcc, 2f 361 sethi %hi(PAGE_SIZE), %g3 362 mov %o0, %g1 363 sub %o1, %g1, %g2 364 sub %g2, %g3, %g2 3651: add %g1, %g2, %o0 /* ARG0: virtual address */ 366 mov 0, %o1 /* ARG1: mmu context */ 367 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 368 ta HV_MMU_UNMAP_ADDR_TRAP 369 brnz,pn %o0, __hypervisor_tlb_tl0_error 370 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 371 brnz,pt %g2, 1b 372 sub %g2, %g3, %g2 3732: retl 374 nop 375 376#ifdef DCACHE_ALIASING_POSSIBLE 377 /* XXX Niagara and friends have an 8K cache, so no aliasing is 378 * XXX possible, but nothing explicit in the Hypervisor API 379 * XXX guarantees this. 380 */ 381__hypervisor_flush_dcache_page: /* 2 insns */ 382 retl 383 nop 384#endif 385 386tlb_patch_one: 3871: lduw [%o1], %g1 388 stw %g1, [%o0] 389 flush %o0 390 subcc %o2, 1, %o2 391 add %o1, 4, %o1 392 bne,pt %icc, 1b 393 add %o0, 4, %o0 394 retl 395 nop 396 397 .globl cheetah_patch_cachetlbops 398cheetah_patch_cachetlbops: 399 save %sp, -128, %sp 400 401 sethi %hi(__flush_tlb_mm), %o0 402 or %o0, %lo(__flush_tlb_mm), %o0 403 sethi %hi(__cheetah_flush_tlb_mm), %o1 404 or %o1, %lo(__cheetah_flush_tlb_mm), %o1 405 call tlb_patch_one 406 mov 19, %o2 407 408 sethi %hi(__flush_tlb_page), %o0 409 or %o0, %lo(__flush_tlb_page), %o0 410 sethi %hi(__cheetah_flush_tlb_page), %o1 411 or %o1, %lo(__cheetah_flush_tlb_page), %o1 412 call tlb_patch_one 413 mov 22, %o2 414 415 sethi %hi(__flush_tlb_pending), %o0 416 or %o0, %lo(__flush_tlb_pending), %o0 417 sethi %hi(__cheetah_flush_tlb_pending), %o1 418 or %o1, %lo(__cheetah_flush_tlb_pending), %o1 419 call tlb_patch_one 420 mov 27, %o2 421 422#ifdef DCACHE_ALIASING_POSSIBLE 423 sethi %hi(__flush_dcache_page), %o0 424 or %o0, %lo(__flush_dcache_page), %o0 425 sethi %hi(__cheetah_flush_dcache_page), %o1 426 or %o1, %lo(__cheetah_flush_dcache_page), %o1 427 call tlb_patch_one 428 mov 11, %o2 429#endif /* DCACHE_ALIASING_POSSIBLE */ 430 431 ret 432 restore 433 434#ifdef CONFIG_SMP 435 /* These are all called by the slaves of a cross call, at 436 * trap level 1, with interrupts fully disabled. 437 * 438 * Register usage: 439 * %g5 mm->context (all tlb flushes) 440 * %g1 address arg 1 (tlb page and range flushes) 441 * %g7 address arg 2 (tlb range flush only) 442 * 443 * %g6 scratch 1 444 * %g2 scratch 2 445 * %g3 scratch 3 446 * %g4 scratch 4 447 */ 448 .align 32 449 .globl xcall_flush_tlb_mm 450xcall_flush_tlb_mm: /* 21 insns */ 451 mov PRIMARY_CONTEXT, %g2 452 ldxa [%g2] ASI_DMMU, %g3 453 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 454 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 455 or %g5, %g4, %g5 /* Preserve nucleus page size fields */ 456 stxa %g5, [%g2] ASI_DMMU 457 mov 0x40, %g4 458 stxa %g0, [%g4] ASI_DMMU_DEMAP 459 stxa %g0, [%g4] ASI_IMMU_DEMAP 460 stxa %g3, [%g2] ASI_DMMU 461 retry 462 nop 463 nop 464 nop 465 nop 466 nop 467 nop 468 nop 469 nop 470 nop 471 nop 472 473 .globl xcall_flush_tlb_page 474xcall_flush_tlb_page: /* 17 insns */ 475 /* %g5=context, %g1=vaddr */ 476 mov PRIMARY_CONTEXT, %g4 477 ldxa [%g4] ASI_DMMU, %g2 478 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 479 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 480 or %g5, %g4, %g5 481 mov PRIMARY_CONTEXT, %g4 482 stxa %g5, [%g4] ASI_DMMU 483 andcc %g1, 0x1, %g0 484 be,pn %icc, 2f 485 andn %g1, 0x1, %g5 486 stxa %g0, [%g5] ASI_IMMU_DEMAP 4872: stxa %g0, [%g5] ASI_DMMU_DEMAP 488 membar #Sync 489 stxa %g2, [%g4] ASI_DMMU 490 retry 491 nop 492 nop 493 494 .globl xcall_flush_tlb_kernel_range 495xcall_flush_tlb_kernel_range: /* 25 insns */ 496 sethi %hi(PAGE_SIZE - 1), %g2 497 or %g2, %lo(PAGE_SIZE - 1), %g2 498 andn %g1, %g2, %g1 499 andn %g7, %g2, %g7 500 sub %g7, %g1, %g3 501 add %g2, 1, %g2 502 sub %g3, %g2, %g3 503 or %g1, 0x20, %g1 ! Nucleus 5041: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 505 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP 506 membar #Sync 507 brnz,pt %g3, 1b 508 sub %g3, %g2, %g3 509 retry 510 nop 511 nop 512 nop 513 nop 514 nop 515 nop 516 nop 517 nop 518 nop 519 nop 520 nop 521 522 /* This runs in a very controlled environment, so we do 523 * not need to worry about BH races etc. 524 */ 525 .globl xcall_sync_tick 526xcall_sync_tick: 527 528661: rdpr %pstate, %g2 529 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate 530 .section .sun4v_2insn_patch, "ax" 531 .word 661b 532 nop 533 nop 534 .previous 535 536 rdpr %pil, %g2 537 wrpr %g0, PIL_NORMAL_MAX, %pil 538 sethi %hi(109f), %g7 539 b,pt %xcc, etrap_irq 540109: or %g7, %lo(109b), %g7 541#ifdef CONFIG_TRACE_IRQFLAGS 542 call trace_hardirqs_off 543 nop 544#endif 545 call smp_synchronize_tick_client 546 nop 547 b rtrap_xcall 548 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 549 550 .globl xcall_fetch_glob_regs 551xcall_fetch_glob_regs: 552 sethi %hi(global_cpu_snapshot), %g1 553 or %g1, %lo(global_cpu_snapshot), %g1 554 __GET_CPUID(%g2) 555 sllx %g2, 6, %g3 556 add %g1, %g3, %g1 557 rdpr %tstate, %g7 558 stx %g7, [%g1 + GR_SNAP_TSTATE] 559 rdpr %tpc, %g7 560 stx %g7, [%g1 + GR_SNAP_TPC] 561 rdpr %tnpc, %g7 562 stx %g7, [%g1 + GR_SNAP_TNPC] 563 stx %o7, [%g1 + GR_SNAP_O7] 564 stx %i7, [%g1 + GR_SNAP_I7] 565 /* Don't try this at home kids... */ 566 rdpr %cwp, %g3 567 sub %g3, 1, %g7 568 wrpr %g7, %cwp 569 mov %i7, %g7 570 wrpr %g3, %cwp 571 stx %g7, [%g1 + GR_SNAP_RPC] 572 sethi %hi(trap_block), %g7 573 or %g7, %lo(trap_block), %g7 574 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 575 add %g7, %g2, %g7 576 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 577 stx %g3, [%g1 + GR_SNAP_THREAD] 578 retry 579 580 .globl xcall_fetch_glob_pmu 581xcall_fetch_glob_pmu: 582 sethi %hi(global_cpu_snapshot), %g1 583 or %g1, %lo(global_cpu_snapshot), %g1 584 __GET_CPUID(%g2) 585 sllx %g2, 6, %g3 586 add %g1, %g3, %g1 587 rd %pic, %g7 588 stx %g7, [%g1 + (4 * 8)] 589 rd %pcr, %g7 590 stx %g7, [%g1 + (0 * 8)] 591 retry 592 593 .globl xcall_fetch_glob_pmu_n4 594xcall_fetch_glob_pmu_n4: 595 sethi %hi(global_cpu_snapshot), %g1 596 or %g1, %lo(global_cpu_snapshot), %g1 597 __GET_CPUID(%g2) 598 sllx %g2, 6, %g3 599 add %g1, %g3, %g1 600 601 ldxa [%g0] ASI_PIC, %g7 602 stx %g7, [%g1 + (4 * 8)] 603 mov 0x08, %g3 604 ldxa [%g3] ASI_PIC, %g7 605 stx %g7, [%g1 + (5 * 8)] 606 mov 0x10, %g3 607 ldxa [%g3] ASI_PIC, %g7 608 stx %g7, [%g1 + (6 * 8)] 609 mov 0x18, %g3 610 ldxa [%g3] ASI_PIC, %g7 611 stx %g7, [%g1 + (7 * 8)] 612 613 mov %o0, %g2 614 mov %o1, %g3 615 mov %o5, %g7 616 617 mov HV_FAST_VT_GET_PERFREG, %o5 618 mov 3, %o0 619 ta HV_FAST_TRAP 620 stx %o1, [%g1 + (3 * 8)] 621 mov HV_FAST_VT_GET_PERFREG, %o5 622 mov 2, %o0 623 ta HV_FAST_TRAP 624 stx %o1, [%g1 + (2 * 8)] 625 mov HV_FAST_VT_GET_PERFREG, %o5 626 mov 1, %o0 627 ta HV_FAST_TRAP 628 stx %o1, [%g1 + (1 * 8)] 629 mov HV_FAST_VT_GET_PERFREG, %o5 630 mov 0, %o0 631 ta HV_FAST_TRAP 632 stx %o1, [%g1 + (0 * 8)] 633 634 mov %g2, %o0 635 mov %g3, %o1 636 mov %g7, %o5 637 638 retry 639 640#ifdef DCACHE_ALIASING_POSSIBLE 641 .align 32 642 .globl xcall_flush_dcache_page_cheetah 643xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ 644 sethi %hi(PAGE_SIZE), %g3 6451: subcc %g3, (1 << 5), %g3 646 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE 647 membar #Sync 648 bne,pt %icc, 1b 649 nop 650 retry 651 nop 652#endif /* DCACHE_ALIASING_POSSIBLE */ 653 654 .globl xcall_flush_dcache_page_spitfire 655xcall_flush_dcache_page_spitfire: /* %g1 == physical page address 656 %g7 == kernel page virtual address 657 %g5 == (page->mapping != NULL) */ 658#ifdef DCACHE_ALIASING_POSSIBLE 659 srlx %g1, (13 - 2), %g1 ! Form tag comparitor 660 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K 661 sub %g3, (1 << 5), %g3 ! D$ linesize == 32 6621: ldxa [%g3] ASI_DCACHE_TAG, %g2 663 andcc %g2, 0x3, %g0 664 be,pn %xcc, 2f 665 andn %g2, 0x3, %g2 666 cmp %g2, %g1 667 668 bne,pt %xcc, 2f 669 nop 670 stxa %g0, [%g3] ASI_DCACHE_TAG 671 membar #Sync 6722: cmp %g3, 0 673 bne,pt %xcc, 1b 674 sub %g3, (1 << 5), %g3 675 676 brz,pn %g5, 2f 677#endif /* DCACHE_ALIASING_POSSIBLE */ 678 sethi %hi(PAGE_SIZE), %g3 679 6801: flush %g7 681 subcc %g3, (1 << 5), %g3 682 bne,pt %icc, 1b 683 add %g7, (1 << 5), %g7 684 6852: retry 686 nop 687 nop 688 689 /* %g5: error 690 * %g6: tlb op 691 */ 692__hypervisor_tlb_xcall_error: 693 mov %g5, %g4 694 mov %g6, %g5 695 ba,pt %xcc, etrap 696 rd %pc, %g7 697 mov %l4, %o0 698 call hypervisor_tlbop_error_xcall 699 mov %l5, %o1 700 ba,a,pt %xcc, rtrap 701 702 .globl __hypervisor_xcall_flush_tlb_mm 703__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ 704 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ 705 mov %o0, %g2 706 mov %o1, %g3 707 mov %o2, %g4 708 mov %o3, %g1 709 mov %o5, %g7 710 clr %o0 /* ARG0: CPU lists unimplemented */ 711 clr %o1 /* ARG1: CPU lists unimplemented */ 712 mov %g5, %o2 /* ARG2: mmu context */ 713 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 714 mov HV_FAST_MMU_DEMAP_CTX, %o5 715 ta HV_FAST_TRAP 716 mov HV_FAST_MMU_DEMAP_CTX, %g6 717 brnz,pn %o0, __hypervisor_tlb_xcall_error 718 mov %o0, %g5 719 mov %g2, %o0 720 mov %g3, %o1 721 mov %g4, %o2 722 mov %g1, %o3 723 mov %g7, %o5 724 membar #Sync 725 retry 726 727 .globl __hypervisor_xcall_flush_tlb_page 728__hypervisor_xcall_flush_tlb_page: /* 17 insns */ 729 /* %g5=ctx, %g1=vaddr */ 730 mov %o0, %g2 731 mov %o1, %g3 732 mov %o2, %g4 733 mov %g1, %o0 /* ARG0: virtual address */ 734 mov %g5, %o1 /* ARG1: mmu context */ 735 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 736 srlx %o0, PAGE_SHIFT, %o0 737 sllx %o0, PAGE_SHIFT, %o0 738 ta HV_MMU_UNMAP_ADDR_TRAP 739 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 740 brnz,a,pn %o0, __hypervisor_tlb_xcall_error 741 mov %o0, %g5 742 mov %g2, %o0 743 mov %g3, %o1 744 mov %g4, %o2 745 membar #Sync 746 retry 747 748 .globl __hypervisor_xcall_flush_tlb_kernel_range 749__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ 750 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ 751 sethi %hi(PAGE_SIZE - 1), %g2 752 or %g2, %lo(PAGE_SIZE - 1), %g2 753 andn %g1, %g2, %g1 754 andn %g7, %g2, %g7 755 sub %g7, %g1, %g3 756 add %g2, 1, %g2 757 sub %g3, %g2, %g3 758 mov %o0, %g2 759 mov %o1, %g4 760 mov %o2, %g7 7611: add %g1, %g3, %o0 /* ARG0: virtual address */ 762 mov 0, %o1 /* ARG1: mmu context */ 763 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 764 ta HV_MMU_UNMAP_ADDR_TRAP 765 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 766 brnz,pn %o0, __hypervisor_tlb_xcall_error 767 mov %o0, %g5 768 sethi %hi(PAGE_SIZE), %o2 769 brnz,pt %g3, 1b 770 sub %g3, %o2, %g3 771 mov %g2, %o0 772 mov %g4, %o1 773 mov %g7, %o2 774 membar #Sync 775 retry 776 777 /* These just get rescheduled to PIL vectors. */ 778 .globl xcall_call_function 779xcall_call_function: 780 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint 781 retry 782 783 .globl xcall_call_function_single 784xcall_call_function_single: 785 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint 786 retry 787 788 .globl xcall_receive_signal 789xcall_receive_signal: 790 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint 791 retry 792 793 .globl xcall_capture 794xcall_capture: 795 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint 796 retry 797 798 .globl xcall_new_mmu_context_version 799xcall_new_mmu_context_version: 800 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint 801 retry 802 803#ifdef CONFIG_KGDB 804 .globl xcall_kgdb_capture 805xcall_kgdb_capture: 806 wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint 807 retry 808#endif 809 810#endif /* CONFIG_SMP */ 811 812 813 .globl hypervisor_patch_cachetlbops 814hypervisor_patch_cachetlbops: 815 save %sp, -128, %sp 816 817 sethi %hi(__flush_tlb_mm), %o0 818 or %o0, %lo(__flush_tlb_mm), %o0 819 sethi %hi(__hypervisor_flush_tlb_mm), %o1 820 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 821 call tlb_patch_one 822 mov 10, %o2 823 824 sethi %hi(__flush_tlb_page), %o0 825 or %o0, %lo(__flush_tlb_page), %o0 826 sethi %hi(__hypervisor_flush_tlb_page), %o1 827 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 828 call tlb_patch_one 829 mov 11, %o2 830 831 sethi %hi(__flush_tlb_pending), %o0 832 or %o0, %lo(__flush_tlb_pending), %o0 833 sethi %hi(__hypervisor_flush_tlb_pending), %o1 834 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 835 call tlb_patch_one 836 mov 16, %o2 837 838 sethi %hi(__flush_tlb_kernel_range), %o0 839 or %o0, %lo(__flush_tlb_kernel_range), %o0 840 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 841 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 842 call tlb_patch_one 843 mov 16, %o2 844 845#ifdef DCACHE_ALIASING_POSSIBLE 846 sethi %hi(__flush_dcache_page), %o0 847 or %o0, %lo(__flush_dcache_page), %o0 848 sethi %hi(__hypervisor_flush_dcache_page), %o1 849 or %o1, %lo(__hypervisor_flush_dcache_page), %o1 850 call tlb_patch_one 851 mov 2, %o2 852#endif /* DCACHE_ALIASING_POSSIBLE */ 853 854#ifdef CONFIG_SMP 855 sethi %hi(xcall_flush_tlb_mm), %o0 856 or %o0, %lo(xcall_flush_tlb_mm), %o0 857 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 858 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 859 call tlb_patch_one 860 mov 21, %o2 861 862 sethi %hi(xcall_flush_tlb_page), %o0 863 or %o0, %lo(xcall_flush_tlb_page), %o0 864 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 865 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 866 call tlb_patch_one 867 mov 17, %o2 868 869 sethi %hi(xcall_flush_tlb_kernel_range), %o0 870 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 871 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 872 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 873 call tlb_patch_one 874 mov 25, %o2 875#endif /* CONFIG_SMP */ 876 877 ret 878 restore 879