1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Page table handling routines for radix page table.
4  *
5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
6  */
7 
8 #define pr_fmt(fmt) "radix-mmu: " fmt
9 
10 #include <linux/io.h>
11 #include <linux/kernel.h>
12 #include <linux/sched/mm.h>
13 #include <linux/memblock.h>
14 #include <linux/of.h>
15 #include <linux/of_fdt.h>
16 #include <linux/mm.h>
17 #include <linux/hugetlb.h>
18 #include <linux/string_helpers.h>
19 #include <linux/memory.h>
20 
21 #include <asm/pgalloc.h>
22 #include <asm/mmu_context.h>
23 #include <asm/dma.h>
24 #include <asm/machdep.h>
25 #include <asm/mmu.h>
26 #include <asm/firmware.h>
27 #include <asm/powernv.h>
28 #include <asm/sections.h>
29 #include <asm/smp.h>
30 #include <asm/trace.h>
31 #include <asm/uaccess.h>
32 #include <asm/ultravisor.h>
33 #include <asm/set_memory.h>
34 
35 #include <trace/events/thp.h>
36 
37 #include <mm/mmu_decl.h>
38 
39 unsigned int mmu_base_pid;
40 unsigned long radix_mem_block_size __ro_after_init;
41 
42 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
43 			unsigned long region_start, unsigned long region_end)
44 {
45 	phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
46 	phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
47 	void *ptr;
48 
49 	if (region_start)
50 		min_addr = region_start;
51 	if (region_end)
52 		max_addr = region_end;
53 
54 	ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
55 
56 	if (!ptr)
57 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
58 		      __func__, size, size, nid, &min_addr, &max_addr);
59 
60 	return ptr;
61 }
62 
63 /*
64  * When allocating pud or pmd pointers, we allocate a complete page
65  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
66  * is to ensure that the page obtained from the memblock allocator
67  * can be completely used as page table page and can be freed
68  * correctly when the page table entries are removed.
69  */
70 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
71 			  pgprot_t flags,
72 			  unsigned int map_page_size,
73 			  int nid,
74 			  unsigned long region_start, unsigned long region_end)
75 {
76 	unsigned long pfn = pa >> PAGE_SHIFT;
77 	pgd_t *pgdp;
78 	p4d_t *p4dp;
79 	pud_t *pudp;
80 	pmd_t *pmdp;
81 	pte_t *ptep;
82 
83 	pgdp = pgd_offset_k(ea);
84 	p4dp = p4d_offset(pgdp, ea);
85 	if (p4d_none(*p4dp)) {
86 		pudp = early_alloc_pgtable(PAGE_SIZE, nid,
87 					   region_start, region_end);
88 		p4d_populate(&init_mm, p4dp, pudp);
89 	}
90 	pudp = pud_offset(p4dp, ea);
91 	if (map_page_size == PUD_SIZE) {
92 		ptep = (pte_t *)pudp;
93 		goto set_the_pte;
94 	}
95 	if (pud_none(*pudp)) {
96 		pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
97 					   region_end);
98 		pud_populate(&init_mm, pudp, pmdp);
99 	}
100 	pmdp = pmd_offset(pudp, ea);
101 	if (map_page_size == PMD_SIZE) {
102 		ptep = pmdp_ptep(pmdp);
103 		goto set_the_pte;
104 	}
105 	if (!pmd_present(*pmdp)) {
106 		ptep = early_alloc_pgtable(PAGE_SIZE, nid,
107 						region_start, region_end);
108 		pmd_populate_kernel(&init_mm, pmdp, ptep);
109 	}
110 	ptep = pte_offset_kernel(pmdp, ea);
111 
112 set_the_pte:
113 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
114 	asm volatile("ptesync": : :"memory");
115 	return 0;
116 }
117 
118 /*
119  * nid, region_start, and region_end are hints to try to place the page
120  * table memory in the same node or region.
121  */
122 static int __map_kernel_page(unsigned long ea, unsigned long pa,
123 			  pgprot_t flags,
124 			  unsigned int map_page_size,
125 			  int nid,
126 			  unsigned long region_start, unsigned long region_end)
127 {
128 	unsigned long pfn = pa >> PAGE_SHIFT;
129 	pgd_t *pgdp;
130 	p4d_t *p4dp;
131 	pud_t *pudp;
132 	pmd_t *pmdp;
133 	pte_t *ptep;
134 	/*
135 	 * Make sure task size is correct as per the max adddr
136 	 */
137 	BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
138 
139 #ifdef CONFIG_PPC_64K_PAGES
140 	BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
141 #endif
142 
143 	if (unlikely(!slab_is_available()))
144 		return early_map_kernel_page(ea, pa, flags, map_page_size,
145 						nid, region_start, region_end);
146 
147 	/*
148 	 * Should make page table allocation functions be able to take a
149 	 * node, so we can place kernel page tables on the right nodes after
150 	 * boot.
151 	 */
152 	pgdp = pgd_offset_k(ea);
153 	p4dp = p4d_offset(pgdp, ea);
154 	pudp = pud_alloc(&init_mm, p4dp, ea);
155 	if (!pudp)
156 		return -ENOMEM;
157 	if (map_page_size == PUD_SIZE) {
158 		ptep = (pte_t *)pudp;
159 		goto set_the_pte;
160 	}
161 	pmdp = pmd_alloc(&init_mm, pudp, ea);
162 	if (!pmdp)
163 		return -ENOMEM;
164 	if (map_page_size == PMD_SIZE) {
165 		ptep = pmdp_ptep(pmdp);
166 		goto set_the_pte;
167 	}
168 	ptep = pte_alloc_kernel(pmdp, ea);
169 	if (!ptep)
170 		return -ENOMEM;
171 
172 set_the_pte:
173 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
174 	asm volatile("ptesync": : :"memory");
175 	return 0;
176 }
177 
178 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
179 			  pgprot_t flags,
180 			  unsigned int map_page_size)
181 {
182 	return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
183 }
184 
185 #ifdef CONFIG_STRICT_KERNEL_RWX
186 static void radix__change_memory_range(unsigned long start, unsigned long end,
187 				       unsigned long clear)
188 {
189 	unsigned long idx;
190 	pgd_t *pgdp;
191 	p4d_t *p4dp;
192 	pud_t *pudp;
193 	pmd_t *pmdp;
194 	pte_t *ptep;
195 
196 	start = ALIGN_DOWN(start, PAGE_SIZE);
197 	end = PAGE_ALIGN(end); // aligns up
198 
199 	pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
200 		 start, end, clear);
201 
202 	for (idx = start; idx < end; idx += PAGE_SIZE) {
203 		pgdp = pgd_offset_k(idx);
204 		p4dp = p4d_offset(pgdp, idx);
205 		pudp = pud_alloc(&init_mm, p4dp, idx);
206 		if (!pudp)
207 			continue;
208 		if (pud_is_leaf(*pudp)) {
209 			ptep = (pte_t *)pudp;
210 			goto update_the_pte;
211 		}
212 		pmdp = pmd_alloc(&init_mm, pudp, idx);
213 		if (!pmdp)
214 			continue;
215 		if (pmd_is_leaf(*pmdp)) {
216 			ptep = pmdp_ptep(pmdp);
217 			goto update_the_pte;
218 		}
219 		ptep = pte_alloc_kernel(pmdp, idx);
220 		if (!ptep)
221 			continue;
222 update_the_pte:
223 		radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
224 	}
225 
226 	radix__flush_tlb_kernel_range(start, end);
227 }
228 
229 void radix__mark_rodata_ro(void)
230 {
231 	unsigned long start, end;
232 
233 	start = (unsigned long)_stext;
234 	end = (unsigned long)__end_rodata;
235 
236 	radix__change_memory_range(start, end, _PAGE_WRITE);
237 }
238 
239 void radix__mark_initmem_nx(void)
240 {
241 	unsigned long start = (unsigned long)__init_begin;
242 	unsigned long end = (unsigned long)__init_end;
243 
244 	radix__change_memory_range(start, end, _PAGE_EXEC);
245 }
246 #endif /* CONFIG_STRICT_KERNEL_RWX */
247 
248 static inline void __meminit
249 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
250 {
251 	char buf[10];
252 
253 	if (end <= start)
254 		return;
255 
256 	string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
257 
258 	pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
259 		exec ? " (exec)" : "");
260 }
261 
262 static unsigned long next_boundary(unsigned long addr, unsigned long end)
263 {
264 #ifdef CONFIG_STRICT_KERNEL_RWX
265 	if (addr < __pa_symbol(__srwx_boundary))
266 		return __pa_symbol(__srwx_boundary);
267 #endif
268 	return end;
269 }
270 
271 static int __meminit create_physical_mapping(unsigned long start,
272 					     unsigned long end,
273 					     int nid, pgprot_t _prot)
274 {
275 	unsigned long vaddr, addr, mapping_size = 0;
276 	bool prev_exec, exec = false;
277 	pgprot_t prot;
278 	int psize;
279 	unsigned long max_mapping_size = radix_mem_block_size;
280 
281 	if (debug_pagealloc_enabled_or_kfence())
282 		max_mapping_size = PAGE_SIZE;
283 
284 	start = ALIGN(start, PAGE_SIZE);
285 	end   = ALIGN_DOWN(end, PAGE_SIZE);
286 	for (addr = start; addr < end; addr += mapping_size) {
287 		unsigned long gap, previous_size;
288 		int rc;
289 
290 		gap = next_boundary(addr, end) - addr;
291 		if (gap > max_mapping_size)
292 			gap = max_mapping_size;
293 		previous_size = mapping_size;
294 		prev_exec = exec;
295 
296 		if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
297 		    mmu_psize_defs[MMU_PAGE_1G].shift) {
298 			mapping_size = PUD_SIZE;
299 			psize = MMU_PAGE_1G;
300 		} else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
301 			   mmu_psize_defs[MMU_PAGE_2M].shift) {
302 			mapping_size = PMD_SIZE;
303 			psize = MMU_PAGE_2M;
304 		} else {
305 			mapping_size = PAGE_SIZE;
306 			psize = mmu_virtual_psize;
307 		}
308 
309 		vaddr = (unsigned long)__va(addr);
310 
311 		if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
312 		    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
313 			prot = PAGE_KERNEL_X;
314 			exec = true;
315 		} else {
316 			prot = _prot;
317 			exec = false;
318 		}
319 
320 		if (mapping_size != previous_size || exec != prev_exec) {
321 			print_mapping(start, addr, previous_size, prev_exec);
322 			start = addr;
323 		}
324 
325 		rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
326 		if (rc)
327 			return rc;
328 
329 		update_page_count(psize, 1);
330 	}
331 
332 	print_mapping(start, addr, mapping_size, exec);
333 	return 0;
334 }
335 
336 static void __init radix_init_pgtable(void)
337 {
338 	unsigned long rts_field;
339 	phys_addr_t start, end;
340 	u64 i;
341 
342 	/* We don't support slb for radix */
343 	slb_set_size(0);
344 
345 	/*
346 	 * Create the linear mapping
347 	 */
348 	for_each_mem_range(i, &start, &end) {
349 		/*
350 		 * The memblock allocator  is up at this point, so the
351 		 * page tables will be allocated within the range. No
352 		 * need or a node (which we don't have yet).
353 		 */
354 
355 		if (end >= RADIX_VMALLOC_START) {
356 			pr_warn("Outside the supported range\n");
357 			continue;
358 		}
359 
360 		WARN_ON(create_physical_mapping(start, end,
361 						-1, PAGE_KERNEL));
362 	}
363 
364 	if (!cpu_has_feature(CPU_FTR_HVMODE) &&
365 			cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
366 		/*
367 		 * Older versions of KVM on these machines prefer if the
368 		 * guest only uses the low 19 PID bits.
369 		 */
370 		mmu_pid_bits = 19;
371 	}
372 	mmu_base_pid = 1;
373 
374 	/*
375 	 * Allocate Partition table and process table for the
376 	 * host.
377 	 */
378 	BUG_ON(PRTB_SIZE_SHIFT > 36);
379 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
380 	/*
381 	 * Fill in the process table.
382 	 */
383 	rts_field = radix__get_tree_size();
384 	process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
385 
386 	/*
387 	 * The init_mm context is given the first available (non-zero) PID,
388 	 * which is the "guard PID" and contains no page table. PIDR should
389 	 * never be set to zero because that duplicates the kernel address
390 	 * space at the 0x0... offset (quadrant 0)!
391 	 *
392 	 * An arbitrary PID that may later be allocated by the PID allocator
393 	 * for userspace processes must not be used either, because that
394 	 * would cause stale user mappings for that PID on CPUs outside of
395 	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
396 	 *
397 	 * So permanently carve out one PID for the purpose of a guard PID.
398 	 */
399 	init_mm.context.id = mmu_base_pid;
400 	mmu_base_pid++;
401 }
402 
403 static void __init radix_init_partition_table(void)
404 {
405 	unsigned long rts_field, dw0, dw1;
406 
407 	mmu_partition_table_init();
408 	rts_field = radix__get_tree_size();
409 	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
410 	dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
411 	mmu_partition_table_set_entry(0, dw0, dw1, false);
412 
413 	pr_info("Initializing Radix MMU\n");
414 }
415 
416 static int __init get_idx_from_shift(unsigned int shift)
417 {
418 	int idx = -1;
419 
420 	switch (shift) {
421 	case 0xc:
422 		idx = MMU_PAGE_4K;
423 		break;
424 	case 0x10:
425 		idx = MMU_PAGE_64K;
426 		break;
427 	case 0x15:
428 		idx = MMU_PAGE_2M;
429 		break;
430 	case 0x1e:
431 		idx = MMU_PAGE_1G;
432 		break;
433 	}
434 	return idx;
435 }
436 
437 static int __init radix_dt_scan_page_sizes(unsigned long node,
438 					   const char *uname, int depth,
439 					   void *data)
440 {
441 	int size = 0;
442 	int shift, idx;
443 	unsigned int ap;
444 	const __be32 *prop;
445 	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
446 
447 	/* We are scanning "cpu" nodes only */
448 	if (type == NULL || strcmp(type, "cpu") != 0)
449 		return 0;
450 
451 	/* Grab page size encodings */
452 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
453 	if (!prop)
454 		return 0;
455 
456 	pr_info("Page sizes from device-tree:\n");
457 	for (; size >= 4; size -= 4, ++prop) {
458 
459 		struct mmu_psize_def *def;
460 
461 		/* top 3 bit is AP encoding */
462 		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
463 		ap = be32_to_cpu(prop[0]) >> 29;
464 		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
465 
466 		idx = get_idx_from_shift(shift);
467 		if (idx < 0)
468 			continue;
469 
470 		def = &mmu_psize_defs[idx];
471 		def->shift = shift;
472 		def->ap  = ap;
473 		def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
474 	}
475 
476 	/* needed ? */
477 	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
478 	return 1;
479 }
480 
481 #ifdef CONFIG_MEMORY_HOTPLUG
482 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
483 					  depth, void *data)
484 {
485 	unsigned long *mem_block_size = (unsigned long *)data;
486 	const __be32 *prop;
487 	int len;
488 
489 	if (depth != 1)
490 		return 0;
491 
492 	if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
493 		return 0;
494 
495 	prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
496 
497 	if (!prop || len < dt_root_size_cells * sizeof(__be32))
498 		/*
499 		 * Nothing in the device tree
500 		 */
501 		*mem_block_size = MIN_MEMORY_BLOCK_SIZE;
502 	else
503 		*mem_block_size = of_read_number(prop, dt_root_size_cells);
504 	return 1;
505 }
506 
507 static unsigned long __init radix_memory_block_size(void)
508 {
509 	unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
510 
511 	/*
512 	 * OPAL firmware feature is set by now. Hence we are ok
513 	 * to test OPAL feature.
514 	 */
515 	if (firmware_has_feature(FW_FEATURE_OPAL))
516 		mem_block_size = 1UL * 1024 * 1024 * 1024;
517 	else
518 		of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
519 
520 	return mem_block_size;
521 }
522 
523 #else   /* CONFIG_MEMORY_HOTPLUG */
524 
525 static unsigned long __init radix_memory_block_size(void)
526 {
527 	return 1UL * 1024 * 1024 * 1024;
528 }
529 
530 #endif /* CONFIG_MEMORY_HOTPLUG */
531 
532 
533 void __init radix__early_init_devtree(void)
534 {
535 	int rc;
536 
537 	/*
538 	 * Try to find the available page sizes in the device-tree
539 	 */
540 	rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
541 	if (!rc) {
542 		/*
543 		 * No page size details found in device tree.
544 		 * Let's assume we have page 4k and 64k support
545 		 */
546 		mmu_psize_defs[MMU_PAGE_4K].shift = 12;
547 		mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
548 		mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
549 			psize_to_rpti_pgsize(MMU_PAGE_4K);
550 
551 		mmu_psize_defs[MMU_PAGE_64K].shift = 16;
552 		mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
553 		mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
554 			psize_to_rpti_pgsize(MMU_PAGE_64K);
555 	}
556 
557 	/*
558 	 * Max mapping size used when mapping pages. We don't use
559 	 * ppc_md.memory_block_size() here because this get called
560 	 * early and we don't have machine probe called yet. Also
561 	 * the pseries implementation only check for ibm,lmb-size.
562 	 * All hypervisor supporting radix do expose that device
563 	 * tree node.
564 	 */
565 	radix_mem_block_size = radix_memory_block_size();
566 	return;
567 }
568 
569 void __init radix__early_init_mmu(void)
570 {
571 	unsigned long lpcr;
572 
573 #ifdef CONFIG_PPC_64S_HASH_MMU
574 #ifdef CONFIG_PPC_64K_PAGES
575 	/* PAGE_SIZE mappings */
576 	mmu_virtual_psize = MMU_PAGE_64K;
577 #else
578 	mmu_virtual_psize = MMU_PAGE_4K;
579 #endif
580 
581 #ifdef CONFIG_SPARSEMEM_VMEMMAP
582 	/* vmemmap mapping */
583 	if (mmu_psize_defs[MMU_PAGE_2M].shift) {
584 		/*
585 		 * map vmemmap using 2M if available
586 		 */
587 		mmu_vmemmap_psize = MMU_PAGE_2M;
588 	} else
589 		mmu_vmemmap_psize = mmu_virtual_psize;
590 #endif
591 #endif
592 	/*
593 	 * initialize page table size
594 	 */
595 	__pte_index_size = RADIX_PTE_INDEX_SIZE;
596 	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
597 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
598 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
599 	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
600 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
601 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
602 	__pud_table_size = RADIX_PUD_TABLE_SIZE;
603 	__pgd_table_size = RADIX_PGD_TABLE_SIZE;
604 
605 	__pmd_val_bits = RADIX_PMD_VAL_BITS;
606 	__pud_val_bits = RADIX_PUD_VAL_BITS;
607 	__pgd_val_bits = RADIX_PGD_VAL_BITS;
608 
609 	__kernel_virt_start = RADIX_KERN_VIRT_START;
610 	__vmalloc_start = RADIX_VMALLOC_START;
611 	__vmalloc_end = RADIX_VMALLOC_END;
612 	__kernel_io_start = RADIX_KERN_IO_START;
613 	__kernel_io_end = RADIX_KERN_IO_END;
614 	vmemmap = (struct page *)RADIX_VMEMMAP_START;
615 	ioremap_bot = IOREMAP_BASE;
616 
617 #ifdef CONFIG_PCI
618 	pci_io_base = ISA_IO_BASE;
619 #endif
620 	__pte_frag_nr = RADIX_PTE_FRAG_NR;
621 	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
622 	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
623 	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
624 
625 	radix_init_pgtable();
626 
627 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
628 		lpcr = mfspr(SPRN_LPCR);
629 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
630 		radix_init_partition_table();
631 	} else {
632 		radix_init_pseries();
633 	}
634 
635 	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
636 
637 	/* Switch to the guard PID before turning on MMU */
638 	radix__switch_mmu_context(NULL, &init_mm);
639 	tlbiel_all();
640 }
641 
642 void radix__early_init_mmu_secondary(void)
643 {
644 	unsigned long lpcr;
645 	/*
646 	 * update partition table control register and UPRT
647 	 */
648 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
649 		lpcr = mfspr(SPRN_LPCR);
650 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
651 
652 		set_ptcr_when_no_uv(__pa(partition_tb) |
653 				    (PATB_SIZE_SHIFT - 12));
654 	}
655 
656 	radix__switch_mmu_context(NULL, &init_mm);
657 	tlbiel_all();
658 
659 	/* Make sure userspace can't change the AMR */
660 	mtspr(SPRN_UAMOR, 0);
661 }
662 
663 /* Called during kexec sequence with MMU off */
664 notrace void radix__mmu_cleanup_all(void)
665 {
666 	unsigned long lpcr;
667 
668 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
669 		lpcr = mfspr(SPRN_LPCR);
670 		mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
671 		set_ptcr_when_no_uv(0);
672 		powernv_set_nmmu_ptcr(0);
673 		radix__flush_tlb_all();
674 	}
675 }
676 
677 #ifdef CONFIG_MEMORY_HOTPLUG
678 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
679 {
680 	pte_t *pte;
681 	int i;
682 
683 	for (i = 0; i < PTRS_PER_PTE; i++) {
684 		pte = pte_start + i;
685 		if (!pte_none(*pte))
686 			return;
687 	}
688 
689 	pte_free_kernel(&init_mm, pte_start);
690 	pmd_clear(pmd);
691 }
692 
693 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
694 {
695 	pmd_t *pmd;
696 	int i;
697 
698 	for (i = 0; i < PTRS_PER_PMD; i++) {
699 		pmd = pmd_start + i;
700 		if (!pmd_none(*pmd))
701 			return;
702 	}
703 
704 	pmd_free(&init_mm, pmd_start);
705 	pud_clear(pud);
706 }
707 
708 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
709 {
710 	pud_t *pud;
711 	int i;
712 
713 	for (i = 0; i < PTRS_PER_PUD; i++) {
714 		pud = pud_start + i;
715 		if (!pud_none(*pud))
716 			return;
717 	}
718 
719 	pud_free(&init_mm, pud_start);
720 	p4d_clear(p4d);
721 }
722 
723 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
724 			     unsigned long end)
725 {
726 	unsigned long next;
727 	pte_t *pte;
728 
729 	pte = pte_start + pte_index(addr);
730 	for (; addr < end; addr = next, pte++) {
731 		next = (addr + PAGE_SIZE) & PAGE_MASK;
732 		if (next > end)
733 			next = end;
734 
735 		if (!pte_present(*pte))
736 			continue;
737 
738 		if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
739 			/*
740 			 * The vmemmap_free() and remove_section_mapping()
741 			 * codepaths call us with aligned addresses.
742 			 */
743 			WARN_ONCE(1, "%s: unaligned range\n", __func__);
744 			continue;
745 		}
746 
747 		pte_clear(&init_mm, addr, pte);
748 	}
749 }
750 
751 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
752 			     unsigned long end)
753 {
754 	unsigned long next;
755 	pte_t *pte_base;
756 	pmd_t *pmd;
757 
758 	pmd = pmd_start + pmd_index(addr);
759 	for (; addr < end; addr = next, pmd++) {
760 		next = pmd_addr_end(addr, end);
761 
762 		if (!pmd_present(*pmd))
763 			continue;
764 
765 		if (pmd_is_leaf(*pmd)) {
766 			if (!IS_ALIGNED(addr, PMD_SIZE) ||
767 			    !IS_ALIGNED(next, PMD_SIZE)) {
768 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
769 				continue;
770 			}
771 			pte_clear(&init_mm, addr, (pte_t *)pmd);
772 			continue;
773 		}
774 
775 		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
776 		remove_pte_table(pte_base, addr, next);
777 		free_pte_table(pte_base, pmd);
778 	}
779 }
780 
781 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
782 			     unsigned long end)
783 {
784 	unsigned long next;
785 	pmd_t *pmd_base;
786 	pud_t *pud;
787 
788 	pud = pud_start + pud_index(addr);
789 	for (; addr < end; addr = next, pud++) {
790 		next = pud_addr_end(addr, end);
791 
792 		if (!pud_present(*pud))
793 			continue;
794 
795 		if (pud_is_leaf(*pud)) {
796 			if (!IS_ALIGNED(addr, PUD_SIZE) ||
797 			    !IS_ALIGNED(next, PUD_SIZE)) {
798 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
799 				continue;
800 			}
801 			pte_clear(&init_mm, addr, (pte_t *)pud);
802 			continue;
803 		}
804 
805 		pmd_base = pud_pgtable(*pud);
806 		remove_pmd_table(pmd_base, addr, next);
807 		free_pmd_table(pmd_base, pud);
808 	}
809 }
810 
811 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
812 {
813 	unsigned long addr, next;
814 	pud_t *pud_base;
815 	pgd_t *pgd;
816 	p4d_t *p4d;
817 
818 	spin_lock(&init_mm.page_table_lock);
819 
820 	for (addr = start; addr < end; addr = next) {
821 		next = pgd_addr_end(addr, end);
822 
823 		pgd = pgd_offset_k(addr);
824 		p4d = p4d_offset(pgd, addr);
825 		if (!p4d_present(*p4d))
826 			continue;
827 
828 		if (p4d_is_leaf(*p4d)) {
829 			if (!IS_ALIGNED(addr, P4D_SIZE) ||
830 			    !IS_ALIGNED(next, P4D_SIZE)) {
831 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
832 				continue;
833 			}
834 
835 			pte_clear(&init_mm, addr, (pte_t *)pgd);
836 			continue;
837 		}
838 
839 		pud_base = p4d_pgtable(*p4d);
840 		remove_pud_table(pud_base, addr, next);
841 		free_pud_table(pud_base, p4d);
842 	}
843 
844 	spin_unlock(&init_mm.page_table_lock);
845 	radix__flush_tlb_kernel_range(start, end);
846 }
847 
848 int __meminit radix__create_section_mapping(unsigned long start,
849 					    unsigned long end, int nid,
850 					    pgprot_t prot)
851 {
852 	if (end >= RADIX_VMALLOC_START) {
853 		pr_warn("Outside the supported range\n");
854 		return -1;
855 	}
856 
857 	return create_physical_mapping(__pa(start), __pa(end),
858 				       nid, prot);
859 }
860 
861 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
862 {
863 	remove_pagetable(start, end);
864 	return 0;
865 }
866 #endif /* CONFIG_MEMORY_HOTPLUG */
867 
868 #ifdef CONFIG_SPARSEMEM_VMEMMAP
869 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
870 				 pgprot_t flags, unsigned int map_page_size,
871 				 int nid)
872 {
873 	return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
874 }
875 
876 int __meminit radix__vmemmap_create_mapping(unsigned long start,
877 				      unsigned long page_size,
878 				      unsigned long phys)
879 {
880 	/* Create a PTE encoding */
881 	unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
882 	int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
883 	int ret;
884 
885 	if ((start + page_size) >= RADIX_VMEMMAP_END) {
886 		pr_warn("Outside the supported range\n");
887 		return -1;
888 	}
889 
890 	ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
891 	BUG_ON(ret);
892 
893 	return 0;
894 }
895 
896 #ifdef CONFIG_MEMORY_HOTPLUG
897 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
898 {
899 	remove_pagetable(start, start + page_size);
900 }
901 #endif
902 #endif
903 
904 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
905 void radix__kernel_map_pages(struct page *page, int numpages, int enable)
906 {
907 	unsigned long addr;
908 
909 	addr = (unsigned long)page_address(page);
910 
911 	if (enable)
912 		set_memory_p(addr, numpages);
913 	else
914 		set_memory_np(addr, numpages);
915 }
916 #endif
917 
918 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
919 
920 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
921 				  pmd_t *pmdp, unsigned long clr,
922 				  unsigned long set)
923 {
924 	unsigned long old;
925 
926 #ifdef CONFIG_DEBUG_VM
927 	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
928 	assert_spin_locked(pmd_lockptr(mm, pmdp));
929 #endif
930 
931 	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
932 	trace_hugepage_update(addr, old, clr, set);
933 
934 	return old;
935 }
936 
937 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
938 			pmd_t *pmdp)
939 
940 {
941 	pmd_t pmd;
942 
943 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
944 	VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
945 	VM_BUG_ON(pmd_devmap(*pmdp));
946 	/*
947 	 * khugepaged calls this for normal pmd
948 	 */
949 	pmd = *pmdp;
950 	pmd_clear(pmdp);
951 
952 	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
953 
954 	return pmd;
955 }
956 
957 /*
958  * For us pgtable_t is pte_t *. Inorder to save the deposisted
959  * page table, we consider the allocated page table as a list
960  * head. On withdraw we need to make sure we zero out the used
961  * list_head memory area.
962  */
963 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
964 				 pgtable_t pgtable)
965 {
966 	struct list_head *lh = (struct list_head *) pgtable;
967 
968 	assert_spin_locked(pmd_lockptr(mm, pmdp));
969 
970 	/* FIFO */
971 	if (!pmd_huge_pte(mm, pmdp))
972 		INIT_LIST_HEAD(lh);
973 	else
974 		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
975 	pmd_huge_pte(mm, pmdp) = pgtable;
976 }
977 
978 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
979 {
980 	pte_t *ptep;
981 	pgtable_t pgtable;
982 	struct list_head *lh;
983 
984 	assert_spin_locked(pmd_lockptr(mm, pmdp));
985 
986 	/* FIFO */
987 	pgtable = pmd_huge_pte(mm, pmdp);
988 	lh = (struct list_head *) pgtable;
989 	if (list_empty(lh))
990 		pmd_huge_pte(mm, pmdp) = NULL;
991 	else {
992 		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
993 		list_del(lh);
994 	}
995 	ptep = (pte_t *) pgtable;
996 	*ptep = __pte(0);
997 	ptep++;
998 	*ptep = __pte(0);
999 	return pgtable;
1000 }
1001 
1002 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1003 				     unsigned long addr, pmd_t *pmdp)
1004 {
1005 	pmd_t old_pmd;
1006 	unsigned long old;
1007 
1008 	old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1009 	old_pmd = __pmd(old);
1010 	return old_pmd;
1011 }
1012 
1013 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1014 
1015 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1016 				  pte_t entry, unsigned long address, int psize)
1017 {
1018 	struct mm_struct *mm = vma->vm_mm;
1019 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1020 					      _PAGE_RW | _PAGE_EXEC);
1021 
1022 	unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1023 	/*
1024 	 * On POWER9, the NMMU is not able to relax PTE access permissions
1025 	 * for a translation with a TLB. The PTE must be invalidated, TLB
1026 	 * flushed before the new PTE is installed.
1027 	 *
1028 	 * This only needs to be done for radix, because hash translation does
1029 	 * flush when updating the linux pte (and we don't support NMMU
1030 	 * accelerators on HPT on POWER9 anyway XXX: do we?).
1031 	 *
1032 	 * POWER10 (and P9P) NMMU does behave as per ISA.
1033 	 */
1034 	if (!cpu_has_feature(CPU_FTR_ARCH_31) && (change & _PAGE_RW) &&
1035 	    atomic_read(&mm->context.copros) > 0) {
1036 		unsigned long old_pte, new_pte;
1037 
1038 		old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1039 		new_pte = old_pte | set;
1040 		radix__flush_tlb_page_psize(mm, address, psize);
1041 		__radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1042 	} else {
1043 		__radix_pte_update(ptep, 0, set);
1044 		/*
1045 		 * Book3S does not require a TLB flush when relaxing access
1046 		 * restrictions when the address space (modulo the POWER9 nest
1047 		 * MMU issue above) because the MMU will reload the PTE after
1048 		 * taking an access fault, as defined by the architecture. See
1049 		 * "Setting a Reference or Change Bit or Upgrading Access
1050 		 *  Authority (PTE Subject to Atomic Hardware Updates)" in
1051 		 *  Power ISA Version 3.1B.
1052 		 */
1053 	}
1054 	/* See ptesync comment in radix__set_pte_at */
1055 }
1056 
1057 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1058 				    unsigned long addr, pte_t *ptep,
1059 				    pte_t old_pte, pte_t pte)
1060 {
1061 	struct mm_struct *mm = vma->vm_mm;
1062 
1063 	/*
1064 	 * POWER9 NMMU must flush the TLB after clearing the PTE before
1065 	 * installing a PTE with more relaxed access permissions, see
1066 	 * radix__ptep_set_access_flags.
1067 	 */
1068 	if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
1069 	    is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1070 	    (atomic_read(&mm->context.copros) > 0))
1071 		radix__flush_tlb_page(vma, addr);
1072 
1073 	set_pte_at(mm, addr, ptep, pte);
1074 }
1075 
1076 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1077 {
1078 	pte_t *ptep = (pte_t *)pud;
1079 	pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1080 
1081 	if (!radix_enabled())
1082 		return 0;
1083 
1084 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1085 
1086 	return 1;
1087 }
1088 
1089 int pud_clear_huge(pud_t *pud)
1090 {
1091 	if (pud_is_leaf(*pud)) {
1092 		pud_clear(pud);
1093 		return 1;
1094 	}
1095 
1096 	return 0;
1097 }
1098 
1099 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1100 {
1101 	pmd_t *pmd;
1102 	int i;
1103 
1104 	pmd = pud_pgtable(*pud);
1105 	pud_clear(pud);
1106 
1107 	flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1108 
1109 	for (i = 0; i < PTRS_PER_PMD; i++) {
1110 		if (!pmd_none(pmd[i])) {
1111 			pte_t *pte;
1112 			pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1113 
1114 			pte_free_kernel(&init_mm, pte);
1115 		}
1116 	}
1117 
1118 	pmd_free(&init_mm, pmd);
1119 
1120 	return 1;
1121 }
1122 
1123 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1124 {
1125 	pte_t *ptep = (pte_t *)pmd;
1126 	pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1127 
1128 	if (!radix_enabled())
1129 		return 0;
1130 
1131 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1132 
1133 	return 1;
1134 }
1135 
1136 int pmd_clear_huge(pmd_t *pmd)
1137 {
1138 	if (pmd_is_leaf(*pmd)) {
1139 		pmd_clear(pmd);
1140 		return 1;
1141 	}
1142 
1143 	return 0;
1144 }
1145 
1146 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1147 {
1148 	pte_t *pte;
1149 
1150 	pte = (pte_t *)pmd_page_vaddr(*pmd);
1151 	pmd_clear(pmd);
1152 
1153 	flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1154 
1155 	pte_free_kernel(&init_mm, pte);
1156 
1157 	return 1;
1158 }
1159