1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * ultra.S: Don't expand these all over the place... 4 * 5 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) 6 */ 7 8#include <asm/asi.h> 9#include <asm/pgtable.h> 10#include <asm/page.h> 11#include <asm/spitfire.h> 12#include <asm/mmu_context.h> 13#include <asm/mmu.h> 14#include <asm/pil.h> 15#include <asm/head.h> 16#include <asm/thread_info.h> 17#include <asm/cacheflush.h> 18#include <asm/hypervisor.h> 19#include <asm/cpudata.h> 20 21 /* Basically, most of the Spitfire vs. Cheetah madness 22 * has to do with the fact that Cheetah does not support 23 * IMMU flushes out of the secondary context. Someone needs 24 * to throw a south lake birthday party for the folks 25 * in Microelectronics who refused to fix this shit. 26 */ 27 28 /* This file is meant to be read efficiently by the CPU, not humans. 29 * Staraj sie tego nikomu nie pierdolnac... 30 */ 31 .text 32 .align 32 33 .globl __flush_tlb_mm 34__flush_tlb_mm: /* 19 insns */ 35 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ 36 ldxa [%o1] ASI_DMMU, %g2 37 cmp %g2, %o0 38 bne,pn %icc, __spitfire_flush_tlb_mm_slow 39 mov 0x50, %g3 40 stxa %g0, [%g3] ASI_DMMU_DEMAP 41 stxa %g0, [%g3] ASI_IMMU_DEMAP 42 sethi %hi(KERNBASE), %g3 43 flush %g3 44 retl 45 nop 46 nop 47 nop 48 nop 49 nop 50 nop 51 nop 52 nop 53 nop 54 nop 55 56 .align 32 57 .globl __flush_tlb_page 58__flush_tlb_page: /* 22 insns */ 59 /* %o0 = context, %o1 = vaddr */ 60 rdpr %pstate, %g7 61 andn %g7, PSTATE_IE, %g2 62 wrpr %g2, %pstate 63 mov SECONDARY_CONTEXT, %o4 64 ldxa [%o4] ASI_DMMU, %g2 65 stxa %o0, [%o4] ASI_DMMU 66 andcc %o1, 1, %g0 67 andn %o1, 1, %o3 68 be,pn %icc, 1f 69 or %o3, 0x10, %o3 70 stxa %g0, [%o3] ASI_IMMU_DEMAP 711: stxa %g0, [%o3] ASI_DMMU_DEMAP 72 membar #Sync 73 stxa %g2, [%o4] ASI_DMMU 74 sethi %hi(KERNBASE), %o4 75 flush %o4 76 retl 77 wrpr %g7, 0x0, %pstate 78 nop 79 nop 80 nop 81 nop 82 83 .align 32 84 .globl __flush_tlb_pending 85__flush_tlb_pending: /* 27 insns */ 86 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 87 rdpr %pstate, %g7 88 sllx %o1, 3, %o1 89 andn %g7, PSTATE_IE, %g2 90 wrpr %g2, %pstate 91 mov SECONDARY_CONTEXT, %o4 92 ldxa [%o4] ASI_DMMU, %g2 93 stxa %o0, [%o4] ASI_DMMU 941: sub %o1, (1 << 3), %o1 95 ldx [%o2 + %o1], %o3 96 andcc %o3, 1, %g0 97 andn %o3, 1, %o3 98 be,pn %icc, 2f 99 or %o3, 0x10, %o3 100 stxa %g0, [%o3] ASI_IMMU_DEMAP 1012: stxa %g0, [%o3] ASI_DMMU_DEMAP 102 membar #Sync 103 brnz,pt %o1, 1b 104 nop 105 stxa %g2, [%o4] ASI_DMMU 106 sethi %hi(KERNBASE), %o4 107 flush %o4 108 retl 109 wrpr %g7, 0x0, %pstate 110 nop 111 nop 112 nop 113 nop 114 115 .align 32 116 .globl __flush_tlb_kernel_range 117__flush_tlb_kernel_range: /* 31 insns */ 118 /* %o0=start, %o1=end */ 119 cmp %o0, %o1 120 be,pn %xcc, 2f 121 sub %o1, %o0, %o3 122 srlx %o3, 18, %o4 123 brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow 124 sethi %hi(PAGE_SIZE), %o4 125 sub %o3, %o4, %o3 126 or %o0, 0x20, %o0 ! Nucleus 1271: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 128 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP 129 membar #Sync 130 brnz,pt %o3, 1b 131 sub %o3, %o4, %o3 1322: sethi %hi(KERNBASE), %o3 133 flush %o3 134 retl 135 nop 136 nop 137 nop 138 nop 139 nop 140 nop 141 nop 142 nop 143 nop 144 nop 145 nop 146 nop 147 nop 148 nop 149 nop 150 151__spitfire_flush_tlb_kernel_range_slow: 152 mov 63 * 8, %o4 1531: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 154 andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ 155 bne,pn %xcc, 2f 156 mov TLB_TAG_ACCESS, %o3 157 stxa %g0, [%o3] ASI_IMMU 158 stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS 159 membar #Sync 1602: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 161 andcc %o3, 0x40, %g0 162 bne,pn %xcc, 2f 163 mov TLB_TAG_ACCESS, %o3 164 stxa %g0, [%o3] ASI_DMMU 165 stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS 166 membar #Sync 1672: sub %o4, 8, %o4 168 brgez,pt %o4, 1b 169 nop 170 retl 171 nop 172 173__spitfire_flush_tlb_mm_slow: 174 rdpr %pstate, %g1 175 wrpr %g1, PSTATE_IE, %pstate 176 stxa %o0, [%o1] ASI_DMMU 177 stxa %g0, [%g3] ASI_DMMU_DEMAP 178 stxa %g0, [%g3] ASI_IMMU_DEMAP 179 flush %g6 180 stxa %g2, [%o1] ASI_DMMU 181 sethi %hi(KERNBASE), %o1 182 flush %o1 183 retl 184 wrpr %g1, 0, %pstate 185 186/* 187 * The following code flushes one page_size worth. 188 */ 189 .section .kprobes.text, "ax" 190 .align 32 191 .globl __flush_icache_page 192__flush_icache_page: /* %o0 = phys_page */ 193 srlx %o0, PAGE_SHIFT, %o0 194 sethi %hi(PAGE_OFFSET), %g1 195 sllx %o0, PAGE_SHIFT, %o0 196 sethi %hi(PAGE_SIZE), %g2 197 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 198 add %o0, %g1, %o0 1991: subcc %g2, 32, %g2 200 bne,pt %icc, 1b 201 flush %o0 + %g2 202 retl 203 nop 204 205#ifdef DCACHE_ALIASING_POSSIBLE 206 207#if (PAGE_SHIFT != 13) 208#error only page shift of 13 is supported by dcache flush 209#endif 210 211#define DTAG_MASK 0x3 212 213 /* This routine is Spitfire specific so the hardcoded 214 * D-cache size and line-size are OK. 215 */ 216 .align 64 217 .globl __flush_dcache_page 218__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ 219 sethi %hi(PAGE_OFFSET), %g1 220 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 221 sub %o0, %g1, %o0 ! physical address 222 srlx %o0, 11, %o0 ! make D-cache TAG 223 sethi %hi(1 << 14), %o2 ! D-cache size 224 sub %o2, (1 << 5), %o2 ! D-cache line size 2251: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG 226 andcc %o3, DTAG_MASK, %g0 ! Valid? 227 be,pn %xcc, 2f ! Nope, branch 228 andn %o3, DTAG_MASK, %o3 ! Clear valid bits 229 cmp %o3, %o0 ! TAG match? 230 bne,pt %xcc, 2f ! Nope, branch 231 nop 232 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG 233 membar #Sync 2342: brnz,pt %o2, 1b 235 sub %o2, (1 << 5), %o2 ! D-cache line size 236 237 /* The I-cache does not snoop local stores so we 238 * better flush that too when necessary. 239 */ 240 brnz,pt %o1, __flush_icache_page 241 sllx %o0, 11, %o0 242 retl 243 nop 244 245#endif /* DCACHE_ALIASING_POSSIBLE */ 246 247 .previous 248 249 /* Cheetah specific versions, patched at boot time. */ 250__cheetah_flush_tlb_mm: /* 19 insns */ 251 rdpr %pstate, %g7 252 andn %g7, PSTATE_IE, %g2 253 wrpr %g2, 0x0, %pstate 254 wrpr %g0, 1, %tl 255 mov PRIMARY_CONTEXT, %o2 256 mov 0x40, %g3 257 ldxa [%o2] ASI_DMMU, %g2 258 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 259 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 260 or %o0, %o1, %o0 /* Preserve nucleus page size fields */ 261 stxa %o0, [%o2] ASI_DMMU 262 stxa %g0, [%g3] ASI_DMMU_DEMAP 263 stxa %g0, [%g3] ASI_IMMU_DEMAP 264 stxa %g2, [%o2] ASI_DMMU 265 sethi %hi(KERNBASE), %o2 266 flush %o2 267 wrpr %g0, 0, %tl 268 retl 269 wrpr %g7, 0x0, %pstate 270 271__cheetah_flush_tlb_page: /* 22 insns */ 272 /* %o0 = context, %o1 = vaddr */ 273 rdpr %pstate, %g7 274 andn %g7, PSTATE_IE, %g2 275 wrpr %g2, 0x0, %pstate 276 wrpr %g0, 1, %tl 277 mov PRIMARY_CONTEXT, %o4 278 ldxa [%o4] ASI_DMMU, %g2 279 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 280 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 281 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 282 stxa %o0, [%o4] ASI_DMMU 283 andcc %o1, 1, %g0 284 be,pn %icc, 1f 285 andn %o1, 1, %o3 286 stxa %g0, [%o3] ASI_IMMU_DEMAP 2871: stxa %g0, [%o3] ASI_DMMU_DEMAP 288 membar #Sync 289 stxa %g2, [%o4] ASI_DMMU 290 sethi %hi(KERNBASE), %o4 291 flush %o4 292 wrpr %g0, 0, %tl 293 retl 294 wrpr %g7, 0x0, %pstate 295 296__cheetah_flush_tlb_pending: /* 27 insns */ 297 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 298 rdpr %pstate, %g7 299 sllx %o1, 3, %o1 300 andn %g7, PSTATE_IE, %g2 301 wrpr %g2, 0x0, %pstate 302 wrpr %g0, 1, %tl 303 mov PRIMARY_CONTEXT, %o4 304 ldxa [%o4] ASI_DMMU, %g2 305 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 306 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 307 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 308 stxa %o0, [%o4] ASI_DMMU 3091: sub %o1, (1 << 3), %o1 310 ldx [%o2 + %o1], %o3 311 andcc %o3, 1, %g0 312 be,pn %icc, 2f 313 andn %o3, 1, %o3 314 stxa %g0, [%o3] ASI_IMMU_DEMAP 3152: stxa %g0, [%o3] ASI_DMMU_DEMAP 316 membar #Sync 317 brnz,pt %o1, 1b 318 nop 319 stxa %g2, [%o4] ASI_DMMU 320 sethi %hi(KERNBASE), %o4 321 flush %o4 322 wrpr %g0, 0, %tl 323 retl 324 wrpr %g7, 0x0, %pstate 325 326__cheetah_flush_tlb_kernel_range: /* 31 insns */ 327 /* %o0=start, %o1=end */ 328 cmp %o0, %o1 329 be,pn %xcc, 2f 330 sub %o1, %o0, %o3 331 srlx %o3, 18, %o4 332 brnz,pn %o4, 3f 333 sethi %hi(PAGE_SIZE), %o4 334 sub %o3, %o4, %o3 335 or %o0, 0x20, %o0 ! Nucleus 3361: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 337 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP 338 membar #Sync 339 brnz,pt %o3, 1b 340 sub %o3, %o4, %o3 3412: sethi %hi(KERNBASE), %o3 342 flush %o3 343 retl 344 nop 3453: mov 0x80, %o4 346 stxa %g0, [%o4] ASI_DMMU_DEMAP 347 membar #Sync 348 stxa %g0, [%o4] ASI_IMMU_DEMAP 349 membar #Sync 350 retl 351 nop 352 nop 353 nop 354 nop 355 nop 356 nop 357 nop 358 nop 359 360#ifdef DCACHE_ALIASING_POSSIBLE 361__cheetah_flush_dcache_page: /* 11 insns */ 362 sethi %hi(PAGE_OFFSET), %g1 363 ldx [%g1 + %lo(PAGE_OFFSET)], %g1 364 sub %o0, %g1, %o0 365 sethi %hi(PAGE_SIZE), %o4 3661: subcc %o4, (1 << 5), %o4 367 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE 368 membar #Sync 369 bne,pt %icc, 1b 370 nop 371 retl /* I-cache flush never needed on Cheetah, see callers. */ 372 nop 373#endif /* DCACHE_ALIASING_POSSIBLE */ 374 375 /* Hypervisor specific versions, patched at boot time. */ 376__hypervisor_tlb_tl0_error: 377 save %sp, -192, %sp 378 mov %i0, %o0 379 call hypervisor_tlbop_error 380 mov %i1, %o1 381 ret 382 restore 383 384__hypervisor_flush_tlb_mm: /* 19 insns */ 385 mov %o0, %o2 /* ARG2: mmu context */ 386 mov 0, %o0 /* ARG0: CPU lists unimplemented */ 387 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 388 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 389 mov HV_FAST_MMU_DEMAP_CTX, %o5 390 ta HV_FAST_TRAP 391 brnz,pn %o0, 1f 392 mov HV_FAST_MMU_DEMAP_CTX, %o1 393 retl 394 nop 3951: sethi %hi(__hypervisor_tlb_tl0_error), %o5 396 jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 397 nop 398 nop 399 nop 400 nop 401 nop 402 nop 403 nop 404 405__hypervisor_flush_tlb_page: /* 22 insns */ 406 /* %o0 = context, %o1 = vaddr */ 407 mov %o0, %g2 408 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ 409 mov %g2, %o1 /* ARG1: mmu context */ 410 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 411 srlx %o0, PAGE_SHIFT, %o0 412 sllx %o0, PAGE_SHIFT, %o0 413 ta HV_MMU_UNMAP_ADDR_TRAP 414 brnz,pn %o0, 1f 415 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 416 retl 417 nop 4181: sethi %hi(__hypervisor_tlb_tl0_error), %o2 419 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 420 nop 421 nop 422 nop 423 nop 424 nop 425 nop 426 nop 427 nop 428 nop 429 430__hypervisor_flush_tlb_pending: /* 27 insns */ 431 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 432 sllx %o1, 3, %g1 433 mov %o2, %g2 434 mov %o0, %g3 4351: sub %g1, (1 << 3), %g1 436 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ 437 mov %g3, %o1 /* ARG1: mmu context */ 438 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 439 srlx %o0, PAGE_SHIFT, %o0 440 sllx %o0, PAGE_SHIFT, %o0 441 ta HV_MMU_UNMAP_ADDR_TRAP 442 brnz,pn %o0, 1f 443 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 444 brnz,pt %g1, 1b 445 nop 446 retl 447 nop 4481: sethi %hi(__hypervisor_tlb_tl0_error), %o2 449 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 450 nop 451 nop 452 nop 453 nop 454 nop 455 nop 456 nop 457 nop 458 nop 459 460__hypervisor_flush_tlb_kernel_range: /* 31 insns */ 461 /* %o0=start, %o1=end */ 462 cmp %o0, %o1 463 be,pn %xcc, 2f 464 sub %o1, %o0, %g2 465 srlx %g2, 18, %g3 466 brnz,pn %g3, 4f 467 mov %o0, %g1 468 sethi %hi(PAGE_SIZE), %g3 469 sub %g2, %g3, %g2 4701: add %g1, %g2, %o0 /* ARG0: virtual address */ 471 mov 0, %o1 /* ARG1: mmu context */ 472 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 473 ta HV_MMU_UNMAP_ADDR_TRAP 474 brnz,pn %o0, 3f 475 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 476 brnz,pt %g2, 1b 477 sub %g2, %g3, %g2 4782: retl 479 nop 4803: sethi %hi(__hypervisor_tlb_tl0_error), %o2 481 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 482 nop 4834: mov 0, %o0 /* ARG0: CPU lists unimplemented */ 484 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 485 mov 0, %o2 /* ARG2: mmu context == nucleus */ 486 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 487 mov HV_FAST_MMU_DEMAP_CTX, %o5 488 ta HV_FAST_TRAP 489 brnz,pn %o0, 3b 490 mov HV_FAST_MMU_DEMAP_CTX, %o1 491 retl 492 nop 493 494#ifdef DCACHE_ALIASING_POSSIBLE 495 /* XXX Niagara and friends have an 8K cache, so no aliasing is 496 * XXX possible, but nothing explicit in the Hypervisor API 497 * XXX guarantees this. 498 */ 499__hypervisor_flush_dcache_page: /* 2 insns */ 500 retl 501 nop 502#endif 503 504tlb_patch_one: 5051: lduw [%o1], %g1 506 stw %g1, [%o0] 507 flush %o0 508 subcc %o2, 1, %o2 509 add %o1, 4, %o1 510 bne,pt %icc, 1b 511 add %o0, 4, %o0 512 retl 513 nop 514 515#ifdef CONFIG_SMP 516 /* These are all called by the slaves of a cross call, at 517 * trap level 1, with interrupts fully disabled. 518 * 519 * Register usage: 520 * %g5 mm->context (all tlb flushes) 521 * %g1 address arg 1 (tlb page and range flushes) 522 * %g7 address arg 2 (tlb range flush only) 523 * 524 * %g6 scratch 1 525 * %g2 scratch 2 526 * %g3 scratch 3 527 * %g4 scratch 4 528 */ 529 .align 32 530 .globl xcall_flush_tlb_mm 531xcall_flush_tlb_mm: /* 24 insns */ 532 mov PRIMARY_CONTEXT, %g2 533 ldxa [%g2] ASI_DMMU, %g3 534 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 535 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 536 or %g5, %g4, %g5 /* Preserve nucleus page size fields */ 537 stxa %g5, [%g2] ASI_DMMU 538 mov 0x40, %g4 539 stxa %g0, [%g4] ASI_DMMU_DEMAP 540 stxa %g0, [%g4] ASI_IMMU_DEMAP 541 stxa %g3, [%g2] ASI_DMMU 542 retry 543 nop 544 nop 545 nop 546 nop 547 nop 548 nop 549 nop 550 nop 551 nop 552 nop 553 nop 554 nop 555 nop 556 557 .globl xcall_flush_tlb_page 558xcall_flush_tlb_page: /* 20 insns */ 559 /* %g5=context, %g1=vaddr */ 560 mov PRIMARY_CONTEXT, %g4 561 ldxa [%g4] ASI_DMMU, %g2 562 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 563 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 564 or %g5, %g4, %g5 565 mov PRIMARY_CONTEXT, %g4 566 stxa %g5, [%g4] ASI_DMMU 567 andcc %g1, 0x1, %g0 568 be,pn %icc, 2f 569 andn %g1, 0x1, %g5 570 stxa %g0, [%g5] ASI_IMMU_DEMAP 5712: stxa %g0, [%g5] ASI_DMMU_DEMAP 572 membar #Sync 573 stxa %g2, [%g4] ASI_DMMU 574 retry 575 nop 576 nop 577 nop 578 nop 579 nop 580 581 .globl xcall_flush_tlb_kernel_range 582xcall_flush_tlb_kernel_range: /* 44 insns */ 583 sethi %hi(PAGE_SIZE - 1), %g2 584 or %g2, %lo(PAGE_SIZE - 1), %g2 585 andn %g1, %g2, %g1 586 andn %g7, %g2, %g7 587 sub %g7, %g1, %g3 588 srlx %g3, 18, %g2 589 brnz,pn %g2, 2f 590 sethi %hi(PAGE_SIZE), %g2 591 sub %g3, %g2, %g3 592 or %g1, 0x20, %g1 ! Nucleus 5931: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 594 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP 595 membar #Sync 596 brnz,pt %g3, 1b 597 sub %g3, %g2, %g3 598 retry 5992: mov 63 * 8, %g1 6001: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 601 andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ 602 bne,pn %xcc, 2f 603 mov TLB_TAG_ACCESS, %g2 604 stxa %g0, [%g2] ASI_IMMU 605 stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS 606 membar #Sync 6072: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 608 andcc %g2, 0x40, %g0 609 bne,pn %xcc, 2f 610 mov TLB_TAG_ACCESS, %g2 611 stxa %g0, [%g2] ASI_DMMU 612 stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS 613 membar #Sync 6142: sub %g1, 8, %g1 615 brgez,pt %g1, 1b 616 nop 617 retry 618 nop 619 nop 620 nop 621 nop 622 nop 623 nop 624 nop 625 nop 626 nop 627 628 /* This runs in a very controlled environment, so we do 629 * not need to worry about BH races etc. 630 */ 631 .globl xcall_sync_tick 632xcall_sync_tick: 633 634661: rdpr %pstate, %g2 635 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate 636 .section .sun4v_2insn_patch, "ax" 637 .word 661b 638 nop 639 nop 640 .previous 641 642 rdpr %pil, %g2 643 wrpr %g0, PIL_NORMAL_MAX, %pil 644 sethi %hi(109f), %g7 645 b,pt %xcc, etrap_irq 646109: or %g7, %lo(109b), %g7 647#ifdef CONFIG_TRACE_IRQFLAGS 648 call trace_hardirqs_off 649 nop 650#endif 651 call smp_synchronize_tick_client 652 nop 653 b rtrap_xcall 654 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 655 656 .globl xcall_fetch_glob_regs 657xcall_fetch_glob_regs: 658 sethi %hi(global_cpu_snapshot), %g1 659 or %g1, %lo(global_cpu_snapshot), %g1 660 __GET_CPUID(%g2) 661 sllx %g2, 6, %g3 662 add %g1, %g3, %g1 663 rdpr %tstate, %g7 664 stx %g7, [%g1 + GR_SNAP_TSTATE] 665 rdpr %tpc, %g7 666 stx %g7, [%g1 + GR_SNAP_TPC] 667 rdpr %tnpc, %g7 668 stx %g7, [%g1 + GR_SNAP_TNPC] 669 stx %o7, [%g1 + GR_SNAP_O7] 670 stx %i7, [%g1 + GR_SNAP_I7] 671 /* Don't try this at home kids... */ 672 rdpr %cwp, %g3 673 sub %g3, 1, %g7 674 wrpr %g7, %cwp 675 mov %i7, %g7 676 wrpr %g3, %cwp 677 stx %g7, [%g1 + GR_SNAP_RPC] 678 sethi %hi(trap_block), %g7 679 or %g7, %lo(trap_block), %g7 680 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 681 add %g7, %g2, %g7 682 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 683 stx %g3, [%g1 + GR_SNAP_THREAD] 684 retry 685 686 .globl xcall_fetch_glob_pmu 687xcall_fetch_glob_pmu: 688 sethi %hi(global_cpu_snapshot), %g1 689 or %g1, %lo(global_cpu_snapshot), %g1 690 __GET_CPUID(%g2) 691 sllx %g2, 6, %g3 692 add %g1, %g3, %g1 693 rd %pic, %g7 694 stx %g7, [%g1 + (4 * 8)] 695 rd %pcr, %g7 696 stx %g7, [%g1 + (0 * 8)] 697 retry 698 699 .globl xcall_fetch_glob_pmu_n4 700xcall_fetch_glob_pmu_n4: 701 sethi %hi(global_cpu_snapshot), %g1 702 or %g1, %lo(global_cpu_snapshot), %g1 703 __GET_CPUID(%g2) 704 sllx %g2, 6, %g3 705 add %g1, %g3, %g1 706 707 ldxa [%g0] ASI_PIC, %g7 708 stx %g7, [%g1 + (4 * 8)] 709 mov 0x08, %g3 710 ldxa [%g3] ASI_PIC, %g7 711 stx %g7, [%g1 + (5 * 8)] 712 mov 0x10, %g3 713 ldxa [%g3] ASI_PIC, %g7 714 stx %g7, [%g1 + (6 * 8)] 715 mov 0x18, %g3 716 ldxa [%g3] ASI_PIC, %g7 717 stx %g7, [%g1 + (7 * 8)] 718 719 mov %o0, %g2 720 mov %o1, %g3 721 mov %o5, %g7 722 723 mov HV_FAST_VT_GET_PERFREG, %o5 724 mov 3, %o0 725 ta HV_FAST_TRAP 726 stx %o1, [%g1 + (3 * 8)] 727 mov HV_FAST_VT_GET_PERFREG, %o5 728 mov 2, %o0 729 ta HV_FAST_TRAP 730 stx %o1, [%g1 + (2 * 8)] 731 mov HV_FAST_VT_GET_PERFREG, %o5 732 mov 1, %o0 733 ta HV_FAST_TRAP 734 stx %o1, [%g1 + (1 * 8)] 735 mov HV_FAST_VT_GET_PERFREG, %o5 736 mov 0, %o0 737 ta HV_FAST_TRAP 738 stx %o1, [%g1 + (0 * 8)] 739 740 mov %g2, %o0 741 mov %g3, %o1 742 mov %g7, %o5 743 744 retry 745 746__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ 747 sethi %hi(PAGE_SIZE - 1), %g2 748 or %g2, %lo(PAGE_SIZE - 1), %g2 749 andn %g1, %g2, %g1 750 andn %g7, %g2, %g7 751 sub %g7, %g1, %g3 752 srlx %g3, 18, %g2 753 brnz,pn %g2, 2f 754 sethi %hi(PAGE_SIZE), %g2 755 sub %g3, %g2, %g3 756 or %g1, 0x20, %g1 ! Nucleus 7571: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 758 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP 759 membar #Sync 760 brnz,pt %g3, 1b 761 sub %g3, %g2, %g3 762 retry 7632: mov 0x80, %g2 764 stxa %g0, [%g2] ASI_DMMU_DEMAP 765 membar #Sync 766 stxa %g0, [%g2] ASI_IMMU_DEMAP 767 membar #Sync 768 retry 769 nop 770 nop 771 nop 772 nop 773 nop 774 nop 775 nop 776 nop 777 nop 778 nop 779 nop 780 nop 781 nop 782 nop 783 nop 784 nop 785 nop 786 nop 787 nop 788 nop 789 nop 790 nop 791 792#ifdef DCACHE_ALIASING_POSSIBLE 793 .align 32 794 .globl xcall_flush_dcache_page_cheetah 795xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ 796 sethi %hi(PAGE_SIZE), %g3 7971: subcc %g3, (1 << 5), %g3 798 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE 799 membar #Sync 800 bne,pt %icc, 1b 801 nop 802 retry 803 nop 804#endif /* DCACHE_ALIASING_POSSIBLE */ 805 806 .globl xcall_flush_dcache_page_spitfire 807xcall_flush_dcache_page_spitfire: /* %g1 == physical page address 808 %g7 == kernel page virtual address 809 %g5 == (page->mapping != NULL) */ 810#ifdef DCACHE_ALIASING_POSSIBLE 811 srlx %g1, (13 - 2), %g1 ! Form tag comparitor 812 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K 813 sub %g3, (1 << 5), %g3 ! D$ linesize == 32 8141: ldxa [%g3] ASI_DCACHE_TAG, %g2 815 andcc %g2, 0x3, %g0 816 be,pn %xcc, 2f 817 andn %g2, 0x3, %g2 818 cmp %g2, %g1 819 820 bne,pt %xcc, 2f 821 nop 822 stxa %g0, [%g3] ASI_DCACHE_TAG 823 membar #Sync 8242: cmp %g3, 0 825 bne,pt %xcc, 1b 826 sub %g3, (1 << 5), %g3 827 828 brz,pn %g5, 2f 829#endif /* DCACHE_ALIASING_POSSIBLE */ 830 sethi %hi(PAGE_SIZE), %g3 831 8321: flush %g7 833 subcc %g3, (1 << 5), %g3 834 bne,pt %icc, 1b 835 add %g7, (1 << 5), %g7 836 8372: retry 838 nop 839 nop 840 841 /* %g5: error 842 * %g6: tlb op 843 */ 844__hypervisor_tlb_xcall_error: 845 mov %g5, %g4 846 mov %g6, %g5 847 ba,pt %xcc, etrap 848 rd %pc, %g7 849 mov %l4, %o0 850 call hypervisor_tlbop_error_xcall 851 mov %l5, %o1 852 ba,a,pt %xcc, rtrap 853 854 .globl __hypervisor_xcall_flush_tlb_mm 855__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ 856 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ 857 mov %o0, %g2 858 mov %o1, %g3 859 mov %o2, %g4 860 mov %o3, %g1 861 mov %o5, %g7 862 clr %o0 /* ARG0: CPU lists unimplemented */ 863 clr %o1 /* ARG1: CPU lists unimplemented */ 864 mov %g5, %o2 /* ARG2: mmu context */ 865 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 866 mov HV_FAST_MMU_DEMAP_CTX, %o5 867 ta HV_FAST_TRAP 868 mov HV_FAST_MMU_DEMAP_CTX, %g6 869 brnz,pn %o0, 1f 870 mov %o0, %g5 871 mov %g2, %o0 872 mov %g3, %o1 873 mov %g4, %o2 874 mov %g1, %o3 875 mov %g7, %o5 876 membar #Sync 877 retry 8781: sethi %hi(__hypervisor_tlb_xcall_error), %g4 879 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 880 nop 881 882 .globl __hypervisor_xcall_flush_tlb_page 883__hypervisor_xcall_flush_tlb_page: /* 20 insns */ 884 /* %g5=ctx, %g1=vaddr */ 885 mov %o0, %g2 886 mov %o1, %g3 887 mov %o2, %g4 888 mov %g1, %o0 /* ARG0: virtual address */ 889 mov %g5, %o1 /* ARG1: mmu context */ 890 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 891 srlx %o0, PAGE_SHIFT, %o0 892 sllx %o0, PAGE_SHIFT, %o0 893 ta HV_MMU_UNMAP_ADDR_TRAP 894 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 895 brnz,a,pn %o0, 1f 896 mov %o0, %g5 897 mov %g2, %o0 898 mov %g3, %o1 899 mov %g4, %o2 900 membar #Sync 901 retry 9021: sethi %hi(__hypervisor_tlb_xcall_error), %g4 903 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 904 nop 905 906 .globl __hypervisor_xcall_flush_tlb_kernel_range 907__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ 908 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ 909 sethi %hi(PAGE_SIZE - 1), %g2 910 or %g2, %lo(PAGE_SIZE - 1), %g2 911 andn %g1, %g2, %g1 912 andn %g7, %g2, %g7 913 sub %g7, %g1, %g3 914 srlx %g3, 18, %g7 915 add %g2, 1, %g2 916 sub %g3, %g2, %g3 917 mov %o0, %g2 918 mov %o1, %g4 919 brnz,pn %g7, 2f 920 mov %o2, %g7 9211: add %g1, %g3, %o0 /* ARG0: virtual address */ 922 mov 0, %o1 /* ARG1: mmu context */ 923 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 924 ta HV_MMU_UNMAP_ADDR_TRAP 925 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 926 brnz,pn %o0, 1f 927 mov %o0, %g5 928 sethi %hi(PAGE_SIZE), %o2 929 brnz,pt %g3, 1b 930 sub %g3, %o2, %g3 9315: mov %g2, %o0 932 mov %g4, %o1 933 mov %g7, %o2 934 membar #Sync 935 retry 9361: sethi %hi(__hypervisor_tlb_xcall_error), %g4 937 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 938 nop 9392: mov %o3, %g1 940 mov %o5, %g3 941 mov 0, %o0 /* ARG0: CPU lists unimplemented */ 942 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 943 mov 0, %o2 /* ARG2: mmu context == nucleus */ 944 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 945 mov HV_FAST_MMU_DEMAP_CTX, %o5 946 ta HV_FAST_TRAP 947 mov %g1, %o3 948 brz,pt %o0, 5b 949 mov %g3, %o5 950 mov HV_FAST_MMU_DEMAP_CTX, %g6 951 ba,pt %xcc, 1b 952 clr %g5 953 954 /* These just get rescheduled to PIL vectors. */ 955 .globl xcall_call_function 956xcall_call_function: 957 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint 958 retry 959 960 .globl xcall_call_function_single 961xcall_call_function_single: 962 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint 963 retry 964 965 .globl xcall_receive_signal 966xcall_receive_signal: 967 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint 968 retry 969 970 .globl xcall_capture 971xcall_capture: 972 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint 973 retry 974 975#ifdef CONFIG_KGDB 976 .globl xcall_kgdb_capture 977xcall_kgdb_capture: 978 wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint 979 retry 980#endif 981 982#endif /* CONFIG_SMP */ 983 984 .globl cheetah_patch_cachetlbops 985cheetah_patch_cachetlbops: 986 save %sp, -128, %sp 987 988 sethi %hi(__flush_tlb_mm), %o0 989 or %o0, %lo(__flush_tlb_mm), %o0 990 sethi %hi(__cheetah_flush_tlb_mm), %o1 991 or %o1, %lo(__cheetah_flush_tlb_mm), %o1 992 call tlb_patch_one 993 mov 19, %o2 994 995 sethi %hi(__flush_tlb_page), %o0 996 or %o0, %lo(__flush_tlb_page), %o0 997 sethi %hi(__cheetah_flush_tlb_page), %o1 998 or %o1, %lo(__cheetah_flush_tlb_page), %o1 999 call tlb_patch_one 1000 mov 22, %o2 1001 1002 sethi %hi(__flush_tlb_pending), %o0 1003 or %o0, %lo(__flush_tlb_pending), %o0 1004 sethi %hi(__cheetah_flush_tlb_pending), %o1 1005 or %o1, %lo(__cheetah_flush_tlb_pending), %o1 1006 call tlb_patch_one 1007 mov 27, %o2 1008 1009 sethi %hi(__flush_tlb_kernel_range), %o0 1010 or %o0, %lo(__flush_tlb_kernel_range), %o0 1011 sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 1012 or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 1013 call tlb_patch_one 1014 mov 31, %o2 1015 1016#ifdef DCACHE_ALIASING_POSSIBLE 1017 sethi %hi(__flush_dcache_page), %o0 1018 or %o0, %lo(__flush_dcache_page), %o0 1019 sethi %hi(__cheetah_flush_dcache_page), %o1 1020 or %o1, %lo(__cheetah_flush_dcache_page), %o1 1021 call tlb_patch_one 1022 mov 11, %o2 1023#endif /* DCACHE_ALIASING_POSSIBLE */ 1024 1025#ifdef CONFIG_SMP 1026 sethi %hi(xcall_flush_tlb_kernel_range), %o0 1027 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 1028 sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 1029 or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 1030 call tlb_patch_one 1031 mov 44, %o2 1032#endif /* CONFIG_SMP */ 1033 1034 ret 1035 restore 1036 1037 .globl hypervisor_patch_cachetlbops 1038hypervisor_patch_cachetlbops: 1039 save %sp, -128, %sp 1040 1041 sethi %hi(__flush_tlb_mm), %o0 1042 or %o0, %lo(__flush_tlb_mm), %o0 1043 sethi %hi(__hypervisor_flush_tlb_mm), %o1 1044 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 1045 call tlb_patch_one 1046 mov 19, %o2 1047 1048 sethi %hi(__flush_tlb_page), %o0 1049 or %o0, %lo(__flush_tlb_page), %o0 1050 sethi %hi(__hypervisor_flush_tlb_page), %o1 1051 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 1052 call tlb_patch_one 1053 mov 22, %o2 1054 1055 sethi %hi(__flush_tlb_pending), %o0 1056 or %o0, %lo(__flush_tlb_pending), %o0 1057 sethi %hi(__hypervisor_flush_tlb_pending), %o1 1058 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 1059 call tlb_patch_one 1060 mov 27, %o2 1061 1062 sethi %hi(__flush_tlb_kernel_range), %o0 1063 or %o0, %lo(__flush_tlb_kernel_range), %o0 1064 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 1065 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 1066 call tlb_patch_one 1067 mov 31, %o2 1068 1069#ifdef DCACHE_ALIASING_POSSIBLE 1070 sethi %hi(__flush_dcache_page), %o0 1071 or %o0, %lo(__flush_dcache_page), %o0 1072 sethi %hi(__hypervisor_flush_dcache_page), %o1 1073 or %o1, %lo(__hypervisor_flush_dcache_page), %o1 1074 call tlb_patch_one 1075 mov 2, %o2 1076#endif /* DCACHE_ALIASING_POSSIBLE */ 1077 1078#ifdef CONFIG_SMP 1079 sethi %hi(xcall_flush_tlb_mm), %o0 1080 or %o0, %lo(xcall_flush_tlb_mm), %o0 1081 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 1082 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 1083 call tlb_patch_one 1084 mov 24, %o2 1085 1086 sethi %hi(xcall_flush_tlb_page), %o0 1087 or %o0, %lo(xcall_flush_tlb_page), %o0 1088 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 1089 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 1090 call tlb_patch_one 1091 mov 20, %o2 1092 1093 sethi %hi(xcall_flush_tlb_kernel_range), %o0 1094 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 1095 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 1096 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 1097 call tlb_patch_one 1098 mov 44, %o2 1099#endif /* CONFIG_SMP */ 1100 1101 ret 1102 restore 1103