1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright 2005, Paul Mackerras, IBM Corporation.
4 * Copyright 2009, Benjamin Herrenschmidt, IBM Corporation.
5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
6 */
7
8 #include <linux/sched.h>
9 #include <linux/mm_types.h>
10 #include <linux/mm.h>
11 #include <linux/stop_machine.h>
12
13 #include <asm/sections.h>
14 #include <asm/mmu.h>
15 #include <asm/tlb.h>
16 #include <asm/firmware.h>
17
18 #include <mm/mmu_decl.h>
19
20 #include <trace/events/thp.h>
21
22 #if H_PGTABLE_RANGE > (USER_VSID_RANGE * (TASK_SIZE_USER64 / TASK_CONTEXT_SIZE))
23 #warning Limited user VSID range means pagetable space is wasted
24 #endif
25
26 #ifdef CONFIG_SPARSEMEM_VMEMMAP
27 /*
28 * vmemmap is the starting address of the virtual address space where
29 * struct pages are allocated for all possible PFNs present on the system
30 * including holes and bad memory (hence sparse). These virtual struct
31 * pages are stored in sequence in this virtual address space irrespective
32 * of the fact whether the corresponding PFN is valid or not. This achieves
33 * constant relationship between address of struct page and its PFN.
34 *
35 * During boot or memory hotplug operation when a new memory section is
36 * added, physical memory allocation (including hash table bolting) will
37 * be performed for the set of struct pages which are part of the memory
38 * section. This saves memory by not allocating struct pages for PFNs
39 * which are not valid.
40 *
41 * ----------------------------------------------
42 * | PHYSICAL ALLOCATION OF VIRTUAL STRUCT PAGES|
43 * ----------------------------------------------
44 *
45 * f000000000000000 c000000000000000
46 * vmemmap +--------------+ +--------------+
47 * + | page struct | +--------------> | page struct |
48 * | +--------------+ +--------------+
49 * | | page struct | +--------------> | page struct |
50 * | +--------------+ | +--------------+
51 * | | page struct | + +------> | page struct |
52 * | +--------------+ | +--------------+
53 * | | page struct | | +--> | page struct |
54 * | +--------------+ | | +--------------+
55 * | | page struct | | |
56 * | +--------------+ | |
57 * | | page struct | | |
58 * | +--------------+ | |
59 * | | page struct | | |
60 * | +--------------+ | |
61 * | | page struct | | |
62 * | +--------------+ | |
63 * | | page struct | +-------+ |
64 * | +--------------+ |
65 * | | page struct | +-----------+
66 * | +--------------+
67 * | | page struct | No mapping
68 * | +--------------+
69 * | | page struct | No mapping
70 * v +--------------+
71 *
72 * -----------------------------------------
73 * | RELATION BETWEEN STRUCT PAGES AND PFNS|
74 * -----------------------------------------
75 *
76 * vmemmap +--------------+ +---------------+
77 * + | page struct | +-------------> | PFN |
78 * | +--------------+ +---------------+
79 * | | page struct | +-------------> | PFN |
80 * | +--------------+ +---------------+
81 * | | page struct | +-------------> | PFN |
82 * | +--------------+ +---------------+
83 * | | page struct | +-------------> | PFN |
84 * | +--------------+ +---------------+
85 * | | |
86 * | +--------------+
87 * | | |
88 * | +--------------+
89 * | | |
90 * | +--------------+ +---------------+
91 * | | page struct | +-------------> | PFN |
92 * | +--------------+ +---------------+
93 * | | |
94 * | +--------------+
95 * | | |
96 * | +--------------+ +---------------+
97 * | | page struct | +-------------> | PFN |
98 * | +--------------+ +---------------+
99 * | | page struct | +-------------> | PFN |
100 * v +--------------+ +---------------+
101 */
102 /*
103 * On hash-based CPUs, the vmemmap is bolted in the hash table.
104 *
105 */
hash__vmemmap_create_mapping(unsigned long start,unsigned long page_size,unsigned long phys)106 int __meminit hash__vmemmap_create_mapping(unsigned long start,
107 unsigned long page_size,
108 unsigned long phys)
109 {
110 int rc;
111
112 if ((start + page_size) >= H_VMEMMAP_END) {
113 pr_warn("Outside the supported range\n");
114 return -1;
115 }
116
117 rc = htab_bolt_mapping(start, start + page_size, phys,
118 pgprot_val(PAGE_KERNEL),
119 mmu_vmemmap_psize, mmu_kernel_ssize);
120 if (rc < 0) {
121 int rc2 = htab_remove_mapping(start, start + page_size,
122 mmu_vmemmap_psize,
123 mmu_kernel_ssize);
124 BUG_ON(rc2 && (rc2 != -ENOENT));
125 }
126 return rc;
127 }
128
129 #ifdef CONFIG_MEMORY_HOTPLUG
hash__vmemmap_remove_mapping(unsigned long start,unsigned long page_size)130 void hash__vmemmap_remove_mapping(unsigned long start,
131 unsigned long page_size)
132 {
133 int rc = htab_remove_mapping(start, start + page_size,
134 mmu_vmemmap_psize,
135 mmu_kernel_ssize);
136 BUG_ON((rc < 0) && (rc != -ENOENT));
137 WARN_ON(rc == -ENOENT);
138 }
139 #endif
140 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
141
142 /*
143 * map_kernel_page currently only called by __ioremap
144 * map_kernel_page adds an entry to the ioremap page table
145 * and adds an entry to the HPT, possibly bolting it
146 */
hash__map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t prot)147 int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
148 {
149 pgd_t *pgdp;
150 p4d_t *p4dp;
151 pud_t *pudp;
152 pmd_t *pmdp;
153 pte_t *ptep;
154
155 BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
156 if (slab_is_available()) {
157 pgdp = pgd_offset_k(ea);
158 p4dp = p4d_offset(pgdp, ea);
159 pudp = pud_alloc(&init_mm, p4dp, ea);
160 if (!pudp)
161 return -ENOMEM;
162 pmdp = pmd_alloc(&init_mm, pudp, ea);
163 if (!pmdp)
164 return -ENOMEM;
165 ptep = pte_alloc_kernel(pmdp, ea);
166 if (!ptep)
167 return -ENOMEM;
168 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
169 } else {
170 /*
171 * If the mm subsystem is not fully up, we cannot create a
172 * linux page table entry for this mapping. Simply bolt an
173 * entry in the hardware page table.
174 *
175 */
176 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, pgprot_val(prot),
177 mmu_io_psize, mmu_kernel_ssize)) {
178 printk(KERN_ERR "Failed to do bolted mapping IO "
179 "memory at %016lx !\n", pa);
180 return -ENOMEM;
181 }
182 }
183
184 smp_wmb();
185 return 0;
186 }
187
188 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
189
hash__pmd_hugepage_update(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,unsigned long clr,unsigned long set)190 unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
191 pmd_t *pmdp, unsigned long clr,
192 unsigned long set)
193 {
194 __be64 old_be, tmp;
195 unsigned long old;
196
197 #ifdef CONFIG_DEBUG_VM
198 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
199 assert_spin_locked(pmd_lockptr(mm, pmdp));
200 #endif
201
202 __asm__ __volatile__(
203 "1: ldarx %0,0,%3\n\
204 and. %1,%0,%6\n\
205 bne- 1b \n\
206 andc %1,%0,%4 \n\
207 or %1,%1,%7\n\
208 stdcx. %1,0,%3 \n\
209 bne- 1b"
210 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
211 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
212 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
213 : "cc" );
214
215 old = be64_to_cpu(old_be);
216
217 trace_hugepage_update_pmd(addr, old, clr, set);
218 if (old & H_PAGE_HASHPTE)
219 hpte_do_hugepage_flush(mm, addr, pmdp, old);
220 return old;
221 }
222
hash__pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)223 pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
224 pmd_t *pmdp)
225 {
226 pmd_t pmd;
227
228 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
229 VM_BUG_ON(pmd_trans_huge(*pmdp));
230 VM_BUG_ON(pmd_devmap(*pmdp));
231
232 pmd = *pmdp;
233 pmd_clear(pmdp);
234 /*
235 * Wait for all pending hash_page to finish. This is needed
236 * in case of subpage collapse. When we collapse normal pages
237 * to hugepage, we first clear the pmd, then invalidate all
238 * the PTE entries. The assumption here is that any low level
239 * page fault will see a none pmd and take the slow path that
240 * will wait on mmap_lock. But we could very well be in a
241 * hash_page with local ptep pointer value. Such a hash page
242 * can result in adding new HPTE entries for normal subpages.
243 * That means we could be modifying the page content as we
244 * copy them to a huge page. So wait for parallel hash_page
245 * to finish before invalidating HPTE entries. We can do this
246 * by sending an IPI to all the cpus and executing a dummy
247 * function there.
248 */
249 serialize_against_pte_lookup(vma->vm_mm);
250 /*
251 * Now invalidate the hpte entries in the range
252 * covered by pmd. This make sure we take a
253 * fault and will find the pmd as none, which will
254 * result in a major fault which takes mmap_lock and
255 * hence wait for collapse to complete. Without this
256 * the __collapse_huge_page_copy can result in copying
257 * the old content.
258 */
259 flush_hash_table_pmd_range(vma->vm_mm, &pmd, address);
260 return pmd;
261 }
262
263 /*
264 * We want to put the pgtable in pmd and use pgtable for tracking
265 * the base page size hptes
266 */
hash__pgtable_trans_huge_deposit(struct mm_struct * mm,pmd_t * pmdp,pgtable_t pgtable)267 void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
268 pgtable_t pgtable)
269 {
270 pgtable_t *pgtable_slot;
271
272 assert_spin_locked(pmd_lockptr(mm, pmdp));
273 /*
274 * we store the pgtable in the second half of PMD
275 */
276 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
277 *pgtable_slot = pgtable;
278 /*
279 * expose the deposited pgtable to other cpus.
280 * before we set the hugepage PTE at pmd level
281 * hash fault code looks at the deposted pgtable
282 * to store hash index values.
283 */
284 smp_wmb();
285 }
286
hash__pgtable_trans_huge_withdraw(struct mm_struct * mm,pmd_t * pmdp)287 pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
288 {
289 pgtable_t pgtable;
290 pgtable_t *pgtable_slot;
291
292 assert_spin_locked(pmd_lockptr(mm, pmdp));
293
294 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
295 pgtable = *pgtable_slot;
296 /*
297 * Once we withdraw, mark the entry NULL.
298 */
299 *pgtable_slot = NULL;
300 /*
301 * We store HPTE information in the deposited PTE fragment.
302 * zero out the content on withdraw.
303 */
304 memset(pgtable, 0, PTE_FRAG_SIZE);
305 return pgtable;
306 }
307
308 /*
309 * A linux hugepage PMD was changed and the corresponding hash table entries
310 * neesd to be flushed.
311 */
hpte_do_hugepage_flush(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,unsigned long old_pmd)312 void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
313 pmd_t *pmdp, unsigned long old_pmd)
314 {
315 int ssize;
316 unsigned int psize;
317 unsigned long vsid;
318 unsigned long flags = 0;
319
320 /* get the base page size,vsid and segment size */
321 #ifdef CONFIG_DEBUG_VM
322 psize = get_slice_psize(mm, addr);
323 BUG_ON(psize == MMU_PAGE_16M);
324 #endif
325 if (old_pmd & H_PAGE_COMBO)
326 psize = MMU_PAGE_4K;
327 else
328 psize = MMU_PAGE_64K;
329
330 if (!is_kernel_addr(addr)) {
331 ssize = user_segment_size(addr);
332 vsid = get_user_vsid(&mm->context, addr, ssize);
333 WARN_ON(vsid == 0);
334 } else {
335 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
336 ssize = mmu_kernel_ssize;
337 }
338
339 if (mm_is_thread_local(mm))
340 flags |= HPTE_LOCAL_UPDATE;
341
342 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
343 }
344
hash__pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)345 pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
346 unsigned long addr, pmd_t *pmdp)
347 {
348 pmd_t old_pmd;
349 pgtable_t pgtable;
350 unsigned long old;
351 pgtable_t *pgtable_slot;
352
353 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
354 old_pmd = __pmd(old);
355 /*
356 * We have pmd == none and we are holding page_table_lock.
357 * So we can safely go and clear the pgtable hash
358 * index info.
359 */
360 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
361 pgtable = *pgtable_slot;
362 /*
363 * Let's zero out old valid and hash index details
364 * hash fault look at them.
365 */
366 memset(pgtable, 0, PTE_FRAG_SIZE);
367 return old_pmd;
368 }
369
hash__has_transparent_hugepage(void)370 int hash__has_transparent_hugepage(void)
371 {
372
373 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
374 return 0;
375 /*
376 * We support THP only if PMD_SIZE is 16MB.
377 */
378 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
379 return 0;
380 /*
381 * We need to make sure that we support 16MB hugepage in a segment
382 * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
383 * of 64K.
384 */
385 /*
386 * If we have 64K HPTE, we will be using that by default
387 */
388 if (mmu_psize_defs[MMU_PAGE_64K].shift &&
389 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
390 return 0;
391 /*
392 * Ok we only have 4K HPTE
393 */
394 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
395 return 0;
396
397 return 1;
398 }
399 EXPORT_SYMBOL_GPL(hash__has_transparent_hugepage);
400
401 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
402
403 #ifdef CONFIG_STRICT_KERNEL_RWX
404
405 struct change_memory_parms {
406 unsigned long start, end, newpp;
407 unsigned int step, nr_cpus;
408 atomic_t master_cpu;
409 atomic_t cpu_counter;
410 };
411
412 // We'd rather this was on the stack but it has to be in the RMO
413 static struct change_memory_parms chmem_parms;
414
415 // And therefore we need a lock to protect it from concurrent use
416 static DEFINE_MUTEX(chmem_lock);
417
change_memory_range(unsigned long start,unsigned long end,unsigned int step,unsigned long newpp)418 static void change_memory_range(unsigned long start, unsigned long end,
419 unsigned int step, unsigned long newpp)
420 {
421 unsigned long idx;
422
423 pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
424 start, end, newpp, step);
425
426 for (idx = start; idx < end; idx += step)
427 /* Not sure if we can do much with the return value */
428 mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
429 mmu_kernel_ssize);
430 }
431
chmem_secondary_loop(struct change_memory_parms * parms)432 static int notrace chmem_secondary_loop(struct change_memory_parms *parms)
433 {
434 unsigned long msr, tmp, flags;
435 int *p;
436
437 p = &parms->cpu_counter.counter;
438
439 local_irq_save(flags);
440 hard_irq_disable();
441
442 asm volatile (
443 // Switch to real mode and leave interrupts off
444 "mfmsr %[msr] ;"
445 "li %[tmp], %[MSR_IR_DR] ;"
446 "andc %[tmp], %[msr], %[tmp] ;"
447 "mtmsrd %[tmp] ;"
448
449 // Tell the master we are in real mode
450 "1: "
451 "lwarx %[tmp], 0, %[p] ;"
452 "addic %[tmp], %[tmp], -1 ;"
453 "stwcx. %[tmp], 0, %[p] ;"
454 "bne- 1b ;"
455
456 // Spin until the counter goes to zero
457 "2: ;"
458 "lwz %[tmp], 0(%[p]) ;"
459 "cmpwi %[tmp], 0 ;"
460 "bne- 2b ;"
461
462 // Switch back to virtual mode
463 "mtmsrd %[msr] ;"
464
465 : // outputs
466 [msr] "=&r" (msr), [tmp] "=&b" (tmp), "+m" (*p)
467 : // inputs
468 [p] "b" (p), [MSR_IR_DR] "i" (MSR_IR | MSR_DR)
469 : // clobbers
470 "cc", "xer"
471 );
472
473 local_irq_restore(flags);
474
475 return 0;
476 }
477
change_memory_range_fn(void * data)478 static int change_memory_range_fn(void *data)
479 {
480 struct change_memory_parms *parms = data;
481
482 // First CPU goes through, all others wait.
483 if (atomic_xchg(&parms->master_cpu, 1) == 1)
484 return chmem_secondary_loop(parms);
485
486 // Wait for all but one CPU (this one) to call-in
487 while (atomic_read(&parms->cpu_counter) > 1)
488 barrier();
489
490 change_memory_range(parms->start, parms->end, parms->step, parms->newpp);
491
492 mb();
493
494 // Signal the other CPUs that we're done
495 atomic_dec(&parms->cpu_counter);
496
497 return 0;
498 }
499
hash__change_memory_range(unsigned long start,unsigned long end,unsigned long newpp)500 static bool hash__change_memory_range(unsigned long start, unsigned long end,
501 unsigned long newpp)
502 {
503 unsigned int step, shift;
504
505 shift = mmu_psize_defs[mmu_linear_psize].shift;
506 step = 1 << shift;
507
508 start = ALIGN_DOWN(start, step);
509 end = ALIGN(end, step); // aligns up
510
511 if (start >= end)
512 return false;
513
514 if (firmware_has_feature(FW_FEATURE_LPAR)) {
515 mutex_lock(&chmem_lock);
516
517 chmem_parms.start = start;
518 chmem_parms.end = end;
519 chmem_parms.step = step;
520 chmem_parms.newpp = newpp;
521 atomic_set(&chmem_parms.master_cpu, 0);
522
523 cpus_read_lock();
524
525 atomic_set(&chmem_parms.cpu_counter, num_online_cpus());
526
527 // Ensure state is consistent before we call the other CPUs
528 mb();
529
530 stop_machine_cpuslocked(change_memory_range_fn, &chmem_parms,
531 cpu_online_mask);
532
533 cpus_read_unlock();
534 mutex_unlock(&chmem_lock);
535 } else
536 change_memory_range(start, end, step, newpp);
537
538 return true;
539 }
540
hash__mark_rodata_ro(void)541 void hash__mark_rodata_ro(void)
542 {
543 unsigned long start, end, pp;
544
545 start = (unsigned long)_stext;
546 end = (unsigned long)__end_rodata;
547
548 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY);
549
550 WARN_ON(!hash__change_memory_range(start, end, pp));
551 }
552
hash__mark_initmem_nx(void)553 void hash__mark_initmem_nx(void)
554 {
555 unsigned long start, end, pp;
556
557 start = (unsigned long)__init_begin;
558 end = (unsigned long)__init_end;
559
560 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL), HPTE_USE_KERNEL_KEY);
561
562 WARN_ON(!hash__change_memory_range(start, end, pp));
563 }
564 #endif
565