1 /* 2 * Copyright IBM Corp. 2007, 2011 3 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 4 */ 5 6 #include <linux/sched.h> 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/gfp.h> 10 #include <linux/mm.h> 11 #include <linux/swap.h> 12 #include <linux/smp.h> 13 #include <linux/spinlock.h> 14 #include <linux/rcupdate.h> 15 #include <linux/slab.h> 16 #include <linux/swapops.h> 17 #include <linux/sysctl.h> 18 #include <linux/ksm.h> 19 #include <linux/mman.h> 20 21 #include <asm/pgtable.h> 22 #include <asm/pgalloc.h> 23 #include <asm/tlb.h> 24 #include <asm/tlbflush.h> 25 #include <asm/mmu_context.h> 26 27 static inline pte_t ptep_flush_direct(struct mm_struct *mm, 28 unsigned long addr, pte_t *ptep) 29 { 30 pte_t old; 31 32 old = *ptep; 33 if (unlikely(pte_val(old) & _PAGE_INVALID)) 34 return old; 35 atomic_inc(&mm->context.flush_count); 36 if (MACHINE_HAS_TLB_LC && 37 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 38 __ptep_ipte(addr, ptep, IPTE_LOCAL); 39 else 40 __ptep_ipte(addr, ptep, IPTE_GLOBAL); 41 atomic_dec(&mm->context.flush_count); 42 return old; 43 } 44 45 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, 46 unsigned long addr, pte_t *ptep) 47 { 48 pte_t old; 49 50 old = *ptep; 51 if (unlikely(pte_val(old) & _PAGE_INVALID)) 52 return old; 53 atomic_inc(&mm->context.flush_count); 54 if (cpumask_equal(&mm->context.cpu_attach_mask, 55 cpumask_of(smp_processor_id()))) { 56 pte_val(*ptep) |= _PAGE_INVALID; 57 mm->context.flush_mm = 1; 58 } else 59 __ptep_ipte(addr, ptep, IPTE_GLOBAL); 60 atomic_dec(&mm->context.flush_count); 61 return old; 62 } 63 64 static inline pgste_t pgste_get_lock(pte_t *ptep) 65 { 66 unsigned long new = 0; 67 #ifdef CONFIG_PGSTE 68 unsigned long old; 69 70 asm( 71 " lg %0,%2\n" 72 "0: lgr %1,%0\n" 73 " nihh %0,0xff7f\n" /* clear PCL bit in old */ 74 " oihh %1,0x0080\n" /* set PCL bit in new */ 75 " csg %0,%1,%2\n" 76 " jl 0b\n" 77 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) 78 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); 79 #endif 80 return __pgste(new); 81 } 82 83 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) 84 { 85 #ifdef CONFIG_PGSTE 86 asm( 87 " nihh %1,0xff7f\n" /* clear PCL bit */ 88 " stg %1,%0\n" 89 : "=Q" (ptep[PTRS_PER_PTE]) 90 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) 91 : "cc", "memory"); 92 #endif 93 } 94 95 static inline pgste_t pgste_get(pte_t *ptep) 96 { 97 unsigned long pgste = 0; 98 #ifdef CONFIG_PGSTE 99 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); 100 #endif 101 return __pgste(pgste); 102 } 103 104 static inline void pgste_set(pte_t *ptep, pgste_t pgste) 105 { 106 #ifdef CONFIG_PGSTE 107 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; 108 #endif 109 } 110 111 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, 112 struct mm_struct *mm) 113 { 114 #ifdef CONFIG_PGSTE 115 unsigned long address, bits, skey; 116 117 if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) 118 return pgste; 119 address = pte_val(pte) & PAGE_MASK; 120 skey = (unsigned long) page_get_storage_key(address); 121 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); 122 /* Transfer page changed & referenced bit to guest bits in pgste */ 123 pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ 124 /* Copy page access key and fetch protection bit to pgste */ 125 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); 126 pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; 127 #endif 128 return pgste; 129 130 } 131 132 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, 133 struct mm_struct *mm) 134 { 135 #ifdef CONFIG_PGSTE 136 unsigned long address; 137 unsigned long nkey; 138 139 if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) 140 return; 141 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); 142 address = pte_val(entry) & PAGE_MASK; 143 /* 144 * Set page access key and fetch protection bit from pgste. 145 * The guest C/R information is still in the PGSTE, set real 146 * key C/R to 0. 147 */ 148 nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; 149 nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; 150 page_set_storage_key(address, nkey, 0); 151 #endif 152 } 153 154 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) 155 { 156 #ifdef CONFIG_PGSTE 157 if ((pte_val(entry) & _PAGE_PRESENT) && 158 (pte_val(entry) & _PAGE_WRITE) && 159 !(pte_val(entry) & _PAGE_INVALID)) { 160 if (!MACHINE_HAS_ESOP) { 161 /* 162 * Without enhanced suppression-on-protection force 163 * the dirty bit on for all writable ptes. 164 */ 165 pte_val(entry) |= _PAGE_DIRTY; 166 pte_val(entry) &= ~_PAGE_PROTECT; 167 } 168 if (!(pte_val(entry) & _PAGE_PROTECT)) 169 /* This pte allows write access, set user-dirty */ 170 pgste_val(pgste) |= PGSTE_UC_BIT; 171 } 172 #endif 173 *ptep = entry; 174 return pgste; 175 } 176 177 static inline pgste_t pgste_pte_notify(struct mm_struct *mm, 178 unsigned long addr, 179 pte_t *ptep, pgste_t pgste) 180 { 181 #ifdef CONFIG_PGSTE 182 unsigned long bits; 183 184 bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); 185 if (bits) { 186 pgste_val(pgste) ^= bits; 187 ptep_notify(mm, addr, ptep, bits); 188 } 189 #endif 190 return pgste; 191 } 192 193 static inline pgste_t ptep_xchg_start(struct mm_struct *mm, 194 unsigned long addr, pte_t *ptep) 195 { 196 pgste_t pgste = __pgste(0); 197 198 if (mm_has_pgste(mm)) { 199 pgste = pgste_get_lock(ptep); 200 pgste = pgste_pte_notify(mm, addr, ptep, pgste); 201 } 202 return pgste; 203 } 204 205 static inline pte_t ptep_xchg_commit(struct mm_struct *mm, 206 unsigned long addr, pte_t *ptep, 207 pgste_t pgste, pte_t old, pte_t new) 208 { 209 if (mm_has_pgste(mm)) { 210 if (pte_val(old) & _PAGE_INVALID) 211 pgste_set_key(ptep, pgste, new, mm); 212 if (pte_val(new) & _PAGE_INVALID) { 213 pgste = pgste_update_all(old, pgste, mm); 214 if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == 215 _PGSTE_GPS_USAGE_UNUSED) 216 pte_val(old) |= _PAGE_UNUSED; 217 } 218 pgste = pgste_set_pte(ptep, pgste, new); 219 pgste_set_unlock(ptep, pgste); 220 } else { 221 *ptep = new; 222 } 223 return old; 224 } 225 226 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, 227 pte_t *ptep, pte_t new) 228 { 229 pgste_t pgste; 230 pte_t old; 231 232 preempt_disable(); 233 pgste = ptep_xchg_start(mm, addr, ptep); 234 old = ptep_flush_direct(mm, addr, ptep); 235 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); 236 preempt_enable(); 237 return old; 238 } 239 EXPORT_SYMBOL(ptep_xchg_direct); 240 241 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, 242 pte_t *ptep, pte_t new) 243 { 244 pgste_t pgste; 245 pte_t old; 246 247 preempt_disable(); 248 pgste = ptep_xchg_start(mm, addr, ptep); 249 old = ptep_flush_lazy(mm, addr, ptep); 250 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); 251 preempt_enable(); 252 return old; 253 } 254 EXPORT_SYMBOL(ptep_xchg_lazy); 255 256 pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, 257 pte_t *ptep) 258 { 259 pgste_t pgste; 260 pte_t old; 261 262 preempt_disable(); 263 pgste = ptep_xchg_start(mm, addr, ptep); 264 old = ptep_flush_lazy(mm, addr, ptep); 265 if (mm_has_pgste(mm)) { 266 pgste = pgste_update_all(old, pgste, mm); 267 pgste_set(ptep, pgste); 268 } 269 return old; 270 } 271 EXPORT_SYMBOL(ptep_modify_prot_start); 272 273 void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, 274 pte_t *ptep, pte_t pte) 275 { 276 pgste_t pgste; 277 278 if (!MACHINE_HAS_NX) 279 pte_val(pte) &= ~_PAGE_NOEXEC; 280 if (mm_has_pgste(mm)) { 281 pgste = pgste_get(ptep); 282 pgste_set_key(ptep, pgste, pte, mm); 283 pgste = pgste_set_pte(ptep, pgste, pte); 284 pgste_set_unlock(ptep, pgste); 285 } else { 286 *ptep = pte; 287 } 288 preempt_enable(); 289 } 290 EXPORT_SYMBOL(ptep_modify_prot_commit); 291 292 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, 293 unsigned long addr, pmd_t *pmdp) 294 { 295 pmd_t old; 296 297 old = *pmdp; 298 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) 299 return old; 300 if (!MACHINE_HAS_IDTE) { 301 __pmdp_csp(pmdp); 302 return old; 303 } 304 atomic_inc(&mm->context.flush_count); 305 if (MACHINE_HAS_TLB_LC && 306 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 307 __pmdp_idte(addr, pmdp, IDTE_LOCAL); 308 else 309 __pmdp_idte(addr, pmdp, IDTE_GLOBAL); 310 atomic_dec(&mm->context.flush_count); 311 return old; 312 } 313 314 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, 315 unsigned long addr, pmd_t *pmdp) 316 { 317 pmd_t old; 318 319 old = *pmdp; 320 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) 321 return old; 322 atomic_inc(&mm->context.flush_count); 323 if (cpumask_equal(&mm->context.cpu_attach_mask, 324 cpumask_of(smp_processor_id()))) { 325 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; 326 mm->context.flush_mm = 1; 327 } else if (MACHINE_HAS_IDTE) 328 __pmdp_idte(addr, pmdp, IDTE_GLOBAL); 329 else 330 __pmdp_csp(pmdp); 331 atomic_dec(&mm->context.flush_count); 332 return old; 333 } 334 335 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, 336 pmd_t *pmdp, pmd_t new) 337 { 338 pmd_t old; 339 340 preempt_disable(); 341 old = pmdp_flush_direct(mm, addr, pmdp); 342 *pmdp = new; 343 preempt_enable(); 344 return old; 345 } 346 EXPORT_SYMBOL(pmdp_xchg_direct); 347 348 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, 349 pmd_t *pmdp, pmd_t new) 350 { 351 pmd_t old; 352 353 preempt_disable(); 354 old = pmdp_flush_lazy(mm, addr, pmdp); 355 *pmdp = new; 356 preempt_enable(); 357 return old; 358 } 359 EXPORT_SYMBOL(pmdp_xchg_lazy); 360 361 static inline pud_t pudp_flush_direct(struct mm_struct *mm, 362 unsigned long addr, pud_t *pudp) 363 { 364 pud_t old; 365 366 old = *pudp; 367 if (pud_val(old) & _REGION_ENTRY_INVALID) 368 return old; 369 if (!MACHINE_HAS_IDTE) { 370 /* 371 * Invalid bit position is the same for pmd and pud, so we can 372 * re-use _pmd_csp() here 373 */ 374 __pmdp_csp((pmd_t *) pudp); 375 return old; 376 } 377 atomic_inc(&mm->context.flush_count); 378 if (MACHINE_HAS_TLB_LC && 379 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 380 __pudp_idte(addr, pudp, IDTE_LOCAL); 381 else 382 __pudp_idte(addr, pudp, IDTE_GLOBAL); 383 atomic_dec(&mm->context.flush_count); 384 return old; 385 } 386 387 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, 388 pud_t *pudp, pud_t new) 389 { 390 pud_t old; 391 392 preempt_disable(); 393 old = pudp_flush_direct(mm, addr, pudp); 394 *pudp = new; 395 preempt_enable(); 396 return old; 397 } 398 EXPORT_SYMBOL(pudp_xchg_direct); 399 400 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 401 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 402 pgtable_t pgtable) 403 { 404 struct list_head *lh = (struct list_head *) pgtable; 405 406 assert_spin_locked(pmd_lockptr(mm, pmdp)); 407 408 /* FIFO */ 409 if (!pmd_huge_pte(mm, pmdp)) 410 INIT_LIST_HEAD(lh); 411 else 412 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); 413 pmd_huge_pte(mm, pmdp) = pgtable; 414 } 415 416 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 417 { 418 struct list_head *lh; 419 pgtable_t pgtable; 420 pte_t *ptep; 421 422 assert_spin_locked(pmd_lockptr(mm, pmdp)); 423 424 /* FIFO */ 425 pgtable = pmd_huge_pte(mm, pmdp); 426 lh = (struct list_head *) pgtable; 427 if (list_empty(lh)) 428 pmd_huge_pte(mm, pmdp) = NULL; 429 else { 430 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; 431 list_del(lh); 432 } 433 ptep = (pte_t *) pgtable; 434 pte_val(*ptep) = _PAGE_INVALID; 435 ptep++; 436 pte_val(*ptep) = _PAGE_INVALID; 437 return pgtable; 438 } 439 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 440 441 #ifdef CONFIG_PGSTE 442 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, 443 pte_t *ptep, pte_t entry) 444 { 445 pgste_t pgste; 446 447 /* the mm_has_pgste() check is done in set_pte_at() */ 448 preempt_disable(); 449 pgste = pgste_get_lock(ptep); 450 pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; 451 pgste_set_key(ptep, pgste, entry, mm); 452 pgste = pgste_set_pte(ptep, pgste, entry); 453 pgste_set_unlock(ptep, pgste); 454 preempt_enable(); 455 } 456 457 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 458 { 459 pgste_t pgste; 460 461 preempt_disable(); 462 pgste = pgste_get_lock(ptep); 463 pgste_val(pgste) |= PGSTE_IN_BIT; 464 pgste_set_unlock(ptep, pgste); 465 preempt_enable(); 466 } 467 468 /** 469 * ptep_force_prot - change access rights of a locked pte 470 * @mm: pointer to the process mm_struct 471 * @addr: virtual address in the guest address space 472 * @ptep: pointer to the page table entry 473 * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE 474 * @bit: pgste bit to set (e.g. for notification) 475 * 476 * Returns 0 if the access rights were changed and -EAGAIN if the current 477 * and requested access rights are incompatible. 478 */ 479 int ptep_force_prot(struct mm_struct *mm, unsigned long addr, 480 pte_t *ptep, int prot, unsigned long bit) 481 { 482 pte_t entry; 483 pgste_t pgste; 484 int pte_i, pte_p; 485 486 pgste = pgste_get_lock(ptep); 487 entry = *ptep; 488 /* Check pte entry after all locks have been acquired */ 489 pte_i = pte_val(entry) & _PAGE_INVALID; 490 pte_p = pte_val(entry) & _PAGE_PROTECT; 491 if ((pte_i && (prot != PROT_NONE)) || 492 (pte_p && (prot & PROT_WRITE))) { 493 pgste_set_unlock(ptep, pgste); 494 return -EAGAIN; 495 } 496 /* Change access rights and set pgste bit */ 497 if (prot == PROT_NONE && !pte_i) { 498 ptep_flush_direct(mm, addr, ptep); 499 pgste = pgste_update_all(entry, pgste, mm); 500 pte_val(entry) |= _PAGE_INVALID; 501 } 502 if (prot == PROT_READ && !pte_p) { 503 ptep_flush_direct(mm, addr, ptep); 504 pte_val(entry) &= ~_PAGE_INVALID; 505 pte_val(entry) |= _PAGE_PROTECT; 506 } 507 pgste_val(pgste) |= bit; 508 pgste = pgste_set_pte(ptep, pgste, entry); 509 pgste_set_unlock(ptep, pgste); 510 return 0; 511 } 512 513 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, 514 pte_t *sptep, pte_t *tptep, pte_t pte) 515 { 516 pgste_t spgste, tpgste; 517 pte_t spte, tpte; 518 int rc = -EAGAIN; 519 520 if (!(pte_val(*tptep) & _PAGE_INVALID)) 521 return 0; /* already shadowed */ 522 spgste = pgste_get_lock(sptep); 523 spte = *sptep; 524 if (!(pte_val(spte) & _PAGE_INVALID) && 525 !((pte_val(spte) & _PAGE_PROTECT) && 526 !(pte_val(pte) & _PAGE_PROTECT))) { 527 pgste_val(spgste) |= PGSTE_VSIE_BIT; 528 tpgste = pgste_get_lock(tptep); 529 pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | 530 (pte_val(pte) & _PAGE_PROTECT); 531 /* don't touch the storage key - it belongs to parent pgste */ 532 tpgste = pgste_set_pte(tptep, tpgste, tpte); 533 pgste_set_unlock(tptep, tpgste); 534 rc = 1; 535 } 536 pgste_set_unlock(sptep, spgste); 537 return rc; 538 } 539 540 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) 541 { 542 pgste_t pgste; 543 544 pgste = pgste_get_lock(ptep); 545 /* notifier is called by the caller */ 546 ptep_flush_direct(mm, saddr, ptep); 547 /* don't touch the storage key - it belongs to parent pgste */ 548 pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); 549 pgste_set_unlock(ptep, pgste); 550 } 551 552 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) 553 { 554 if (!non_swap_entry(entry)) 555 dec_mm_counter(mm, MM_SWAPENTS); 556 else if (is_migration_entry(entry)) { 557 struct page *page = migration_entry_to_page(entry); 558 559 dec_mm_counter(mm, mm_counter(page)); 560 } 561 free_swap_and_cache(entry); 562 } 563 564 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, 565 pte_t *ptep, int reset) 566 { 567 unsigned long pgstev; 568 pgste_t pgste; 569 pte_t pte; 570 571 /* Zap unused and logically-zero pages */ 572 preempt_disable(); 573 pgste = pgste_get_lock(ptep); 574 pgstev = pgste_val(pgste); 575 pte = *ptep; 576 if (!reset && pte_swap(pte) && 577 ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || 578 (pgstev & _PGSTE_GPS_ZERO))) { 579 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); 580 pte_clear(mm, addr, ptep); 581 } 582 if (reset) 583 pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; 584 pgste_set_unlock(ptep, pgste); 585 preempt_enable(); 586 } 587 588 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 589 { 590 unsigned long ptev; 591 pgste_t pgste; 592 593 /* Clear storage key */ 594 preempt_disable(); 595 pgste = pgste_get_lock(ptep); 596 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | 597 PGSTE_GR_BIT | PGSTE_GC_BIT); 598 ptev = pte_val(*ptep); 599 if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) 600 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); 601 pgste_set_unlock(ptep, pgste); 602 preempt_enable(); 603 } 604 605 /* 606 * Test and reset if a guest page is dirty 607 */ 608 bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) 609 { 610 spinlock_t *ptl; 611 pgd_t *pgd; 612 pud_t *pud; 613 pmd_t *pmd; 614 pgste_t pgste; 615 pte_t *ptep; 616 pte_t pte; 617 bool dirty; 618 619 pgd = pgd_offset(mm, addr); 620 pud = pud_alloc(mm, pgd, addr); 621 if (!pud) 622 return false; 623 pmd = pmd_alloc(mm, pud, addr); 624 if (!pmd) 625 return false; 626 /* We can't run guests backed by huge pages, but userspace can 627 * still set them up and then try to migrate them without any 628 * migration support. 629 */ 630 if (pmd_large(*pmd)) 631 return true; 632 633 ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl); 634 if (unlikely(!ptep)) 635 return false; 636 637 pgste = pgste_get_lock(ptep); 638 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); 639 pgste_val(pgste) &= ~PGSTE_UC_BIT; 640 pte = *ptep; 641 if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { 642 pgste = pgste_pte_notify(mm, addr, ptep, pgste); 643 __ptep_ipte(addr, ptep, IPTE_GLOBAL); 644 if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) 645 pte_val(pte) |= _PAGE_PROTECT; 646 else 647 pte_val(pte) |= _PAGE_INVALID; 648 *ptep = pte; 649 } 650 pgste_set_unlock(ptep, pgste); 651 652 spin_unlock(ptl); 653 return dirty; 654 } 655 EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); 656 657 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, 658 unsigned char key, bool nq) 659 { 660 unsigned long keyul; 661 spinlock_t *ptl; 662 pgste_t old, new; 663 pte_t *ptep; 664 665 ptep = get_locked_pte(mm, addr, &ptl); 666 if (unlikely(!ptep)) 667 return -EFAULT; 668 669 new = old = pgste_get_lock(ptep); 670 pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | 671 PGSTE_ACC_BITS | PGSTE_FP_BIT); 672 keyul = (unsigned long) key; 673 pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; 674 pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; 675 if (!(pte_val(*ptep) & _PAGE_INVALID)) { 676 unsigned long address, bits, skey; 677 678 address = pte_val(*ptep) & PAGE_MASK; 679 skey = (unsigned long) page_get_storage_key(address); 680 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); 681 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); 682 /* Set storage key ACC and FP */ 683 page_set_storage_key(address, skey, !nq); 684 /* Merge host changed & referenced into pgste */ 685 pgste_val(new) |= bits << 52; 686 } 687 /* changing the guest storage key is considered a change of the page */ 688 if ((pgste_val(new) ^ pgste_val(old)) & 689 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) 690 pgste_val(new) |= PGSTE_UC_BIT; 691 692 pgste_set_unlock(ptep, new); 693 pte_unmap_unlock(ptep, ptl); 694 return 0; 695 } 696 EXPORT_SYMBOL(set_guest_storage_key); 697 698 /** 699 * Conditionally set a guest storage key (handling csske). 700 * oldkey will be updated when either mr or mc is set and a pointer is given. 701 * 702 * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest 703 * storage key was updated and -EFAULT on access errors. 704 */ 705 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, 706 unsigned char key, unsigned char *oldkey, 707 bool nq, bool mr, bool mc) 708 { 709 unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; 710 int rc; 711 712 /* we can drop the pgste lock between getting and setting the key */ 713 if (mr | mc) { 714 rc = get_guest_storage_key(current->mm, addr, &tmp); 715 if (rc) 716 return rc; 717 if (oldkey) 718 *oldkey = tmp; 719 if (!mr) 720 mask |= _PAGE_REFERENCED; 721 if (!mc) 722 mask |= _PAGE_CHANGED; 723 if (!((tmp ^ key) & mask)) 724 return 0; 725 } 726 rc = set_guest_storage_key(current->mm, addr, key, nq); 727 return rc < 0 ? rc : 1; 728 } 729 EXPORT_SYMBOL(cond_set_guest_storage_key); 730 731 /** 732 * Reset a guest reference bit (rrbe), returning the reference and changed bit. 733 * 734 * Returns < 0 in case of error, otherwise the cc to be reported to the guest. 735 */ 736 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) 737 { 738 spinlock_t *ptl; 739 pgste_t old, new; 740 pte_t *ptep; 741 int cc = 0; 742 743 ptep = get_locked_pte(mm, addr, &ptl); 744 if (unlikely(!ptep)) 745 return -EFAULT; 746 747 new = old = pgste_get_lock(ptep); 748 /* Reset guest reference bit only */ 749 pgste_val(new) &= ~PGSTE_GR_BIT; 750 751 if (!(pte_val(*ptep) & _PAGE_INVALID)) { 752 cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); 753 /* Merge real referenced bit into host-set */ 754 pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; 755 } 756 /* Reflect guest's logical view, not physical */ 757 cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; 758 /* Changing the guest storage key is considered a change of the page */ 759 if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) 760 pgste_val(new) |= PGSTE_UC_BIT; 761 762 pgste_set_unlock(ptep, new); 763 pte_unmap_unlock(ptep, ptl); 764 return cc; 765 } 766 EXPORT_SYMBOL(reset_guest_reference_bit); 767 768 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, 769 unsigned char *key) 770 { 771 spinlock_t *ptl; 772 pgste_t pgste; 773 pte_t *ptep; 774 775 ptep = get_locked_pte(mm, addr, &ptl); 776 if (unlikely(!ptep)) 777 return -EFAULT; 778 779 pgste = pgste_get_lock(ptep); 780 *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; 781 if (!(pte_val(*ptep) & _PAGE_INVALID)) 782 *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); 783 /* Reflect guest's logical view, not physical */ 784 *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; 785 pgste_set_unlock(ptep, pgste); 786 pte_unmap_unlock(ptep, ptl); 787 return 0; 788 } 789 EXPORT_SYMBOL(get_guest_storage_key); 790 #endif 791