1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * S390 version 4a53c8fabSHeiko Carstens * Copyright IBM Corp. 1999 51da177e4SLinus Torvalds * Author(s): Hartmut Penner (hp@de.ibm.com) 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Derived from "arch/i386/mm/init.c" 81da177e4SLinus Torvalds * Copyright (C) 1995 Linus Torvalds 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 111da177e4SLinus Torvalds #include <linux/signal.h> 121da177e4SLinus Torvalds #include <linux/sched.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/errno.h> 151da177e4SLinus Torvalds #include <linux/string.h> 161da177e4SLinus Torvalds #include <linux/types.h> 171da177e4SLinus Torvalds #include <linux/ptrace.h> 181da177e4SLinus Torvalds #include <linux/mman.h> 191da177e4SLinus Torvalds #include <linux/mm.h> 201da177e4SLinus Torvalds #include <linux/swap.h> 2164e1f0c5SHalil Pasic #include <linux/swiotlb.h> 221da177e4SLinus Torvalds #include <linux/smp.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/pagemap.h> 2557c8a661SMike Rapoport #include <linux/memblock.h> 26e5d709bbSHeiko Carstens #include <linux/memory.h> 27d882b172SHeiko Carstens #include <linux/pfn.h> 28028d9b3cSHeiko Carstens #include <linux/poison.h> 292b67fc46SHeiko Carstens #include <linux/initrd.h> 303a4c5d59SHeiko Carstens #include <linux/export.h> 3134ad7cdcSHeiko Carstens #include <linux/cma.h> 325a0e3ad6STejun Heo #include <linux/gfp.h> 339087c375STom Lendacky #include <linux/dma-direct.h> 341da177e4SLinus Torvalds #include <asm/processor.h> 357c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 361da177e4SLinus Torvalds #include <asm/pgtable.h> 371da177e4SLinus Torvalds #include <asm/pgalloc.h> 381da177e4SLinus Torvalds #include <asm/dma.h> 391da177e4SLinus Torvalds #include <asm/lowcore.h> 401da177e4SLinus Torvalds #include <asm/tlb.h> 411da177e4SLinus Torvalds #include <asm/tlbflush.h> 42d882b172SHeiko Carstens #include <asm/sections.h> 43a0616cdeSDavid Howells #include <asm/ctl_reg.h> 44e5d709bbSHeiko Carstens #include <asm/sclp.h> 45e6c7c630SLaura Abbott #include <asm/set_memory.h> 4642db5ed8SVasily Gorbik #include <asm/kasan.h> 4764e1f0c5SHalil Pasic #include <asm/dma-mapping.h> 4864e1f0c5SHalil Pasic #include <asm/uv.h> 491da177e4SLinus Torvalds 500ccb32c9SHeiko Carstens pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir); 511485c5c8SHeiko Carstens 52238ec4efSMartin Schwidefsky unsigned long empty_zero_page, zero_page_mask; 531485c5c8SHeiko Carstens EXPORT_SYMBOL(empty_zero_page); 540b70068eSArd Biesheuvel EXPORT_SYMBOL(zero_page_mask); 551da177e4SLinus Torvalds 567a5da02dSGerald Schaefer bool initmem_freed; 577a5da02dSGerald Schaefer 580999f119SJiang Liu static void __init setup_zero_pages(void) 59238ec4efSMartin Schwidefsky { 60238ec4efSMartin Schwidefsky unsigned int order; 61238ec4efSMartin Schwidefsky struct page *page; 62238ec4efSMartin Schwidefsky int i; 63238ec4efSMartin Schwidefsky 64c7e8b2c2SMartin Schwidefsky /* Latest machines require a mapping granularity of 512KB */ 651f6b83e5SMartin Schwidefsky order = 7; 66c7e8b2c2SMartin Schwidefsky 677919e91bSMartin Schwidefsky /* Limit number of empty zero pages for small memory sizes */ 68ca79b0c2SArun KS while (order > 2 && (totalram_pages() >> 10) < (1UL << order)) 691f6b83e5SMartin Schwidefsky order--; 70238ec4efSMartin Schwidefsky 71238ec4efSMartin Schwidefsky empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 72238ec4efSMartin Schwidefsky if (!empty_zero_page) 73238ec4efSMartin Schwidefsky panic("Out of memory in setup_zero_pages"); 74238ec4efSMartin Schwidefsky 75238ec4efSMartin Schwidefsky page = virt_to_page((void *) empty_zero_page); 76238ec4efSMartin Schwidefsky split_page(page, order); 77238ec4efSMartin Schwidefsky for (i = 1 << order; i > 0; i--) { 780999f119SJiang Liu mark_page_reserved(page); 79238ec4efSMartin Schwidefsky page++; 80238ec4efSMartin Schwidefsky } 81238ec4efSMartin Schwidefsky 820999f119SJiang Liu zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; 83238ec4efSMartin Schwidefsky } 84238ec4efSMartin Schwidefsky 851da177e4SLinus Torvalds /* 861da177e4SLinus Torvalds * paging_init() sets up the page tables 871da177e4SLinus Torvalds */ 881da177e4SLinus Torvalds void __init paging_init(void) 891da177e4SLinus Torvalds { 9039b742f9SHeiko Carstens unsigned long max_zone_pfns[MAX_NR_ZONES]; 9114045ebfSMartin Schwidefsky unsigned long pgd_type, asce_bits; 9260c49701SHeiko Carstens psw_t psw; 931da177e4SLinus Torvalds 943610cce8SMartin Schwidefsky init_mm.pgd = swapper_pg_dir; 95f1c1174fSHeiko Carstens if (VMALLOC_END > _REGION2_SIZE) { 9614045ebfSMartin Schwidefsky asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; 9714045ebfSMartin Schwidefsky pgd_type = _REGION2_ENTRY_EMPTY; 9814045ebfSMartin Schwidefsky } else { 9914045ebfSMartin Schwidefsky asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 1006252d702SMartin Schwidefsky pgd_type = _REGION3_ENTRY_EMPTY; 10114045ebfSMartin Schwidefsky } 102723cacbdSGerald Schaefer init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; 103723cacbdSGerald Schaefer S390_lowcore.kernel_asce = init_mm.context.asce; 1040aaba41bSMartin Schwidefsky S390_lowcore.user_asce = S390_lowcore.kernel_asce; 105f1c1174fSHeiko Carstens crst_table_init((unsigned long *) init_mm.pgd, pgd_type); 106f4eb07c1SHeiko Carstens vmem_map_init(); 10742db5ed8SVasily Gorbik kasan_copy_shadow(init_mm.pgd); 1081da177e4SLinus Torvalds 1091da177e4SLinus Torvalds /* enable virtual mapping in kernel mode */ 1103610cce8SMartin Schwidefsky __ctl_load(S390_lowcore.kernel_asce, 1, 1); 1113610cce8SMartin Schwidefsky __ctl_load(S390_lowcore.kernel_asce, 7, 7); 1123610cce8SMartin Schwidefsky __ctl_load(S390_lowcore.kernel_asce, 13, 13); 11360c49701SHeiko Carstens psw.mask = __extract_psw(); 114a7525982SHeiko Carstens psw_bits(psw).dat = 1; 1158bb3fdd6SHeiko Carstens psw_bits(psw).as = PSW_BITS_AS_HOME; 11660c49701SHeiko Carstens __load_psw_mask(psw.mask); 117135ff163SVasily Gorbik kasan_free_early_identity(); 1181da177e4SLinus Torvalds 11917f34580SHeiko Carstens sparse_memory_present_with_active_regions(MAX_NUMNODES); 12017f34580SHeiko Carstens sparse_init(); 1218b5369eaSNicolas Saenz Julienne zone_dma_bits = 31; 12239b742f9SHeiko Carstens memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 12339b742f9SHeiko Carstens max_zone_pfns[ZONE_DMA] = PFN_DOWN(MAX_DMA_ADDRESS); 12439b742f9SHeiko Carstens max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 12539b742f9SHeiko Carstens free_area_init_nodes(max_zone_pfns); 1261da177e4SLinus Torvalds } 1271da177e4SLinus Torvalds 12891d37211SHeiko Carstens void mark_rodata_ro(void) 12991d37211SHeiko Carstens { 130d07a980cSHeiko Carstens unsigned long size = __end_ro_after_init - __start_ro_after_init; 131d07a980cSHeiko Carstens 132d07a980cSHeiko Carstens set_memory_ro((unsigned long)__start_ro_after_init, size >> PAGE_SHIFT); 133d07a980cSHeiko Carstens pr_info("Write protected read-only-after-init data: %luk\n", size >> 10); 13491d37211SHeiko Carstens } 13591d37211SHeiko Carstens 13664e1f0c5SHalil Pasic int set_memory_encrypted(unsigned long addr, int numpages) 13764e1f0c5SHalil Pasic { 13864e1f0c5SHalil Pasic int i; 13964e1f0c5SHalil Pasic 14064e1f0c5SHalil Pasic /* make specified pages unshared, (swiotlb, dma_free) */ 14164e1f0c5SHalil Pasic for (i = 0; i < numpages; ++i) { 14264e1f0c5SHalil Pasic uv_remove_shared(addr); 14364e1f0c5SHalil Pasic addr += PAGE_SIZE; 14464e1f0c5SHalil Pasic } 14564e1f0c5SHalil Pasic return 0; 14664e1f0c5SHalil Pasic } 14764e1f0c5SHalil Pasic 14864e1f0c5SHalil Pasic int set_memory_decrypted(unsigned long addr, int numpages) 14964e1f0c5SHalil Pasic { 15064e1f0c5SHalil Pasic int i; 15164e1f0c5SHalil Pasic /* make specified pages shared (swiotlb, dma_alloca) */ 15264e1f0c5SHalil Pasic for (i = 0; i < numpages; ++i) { 15364e1f0c5SHalil Pasic uv_set_shared(addr); 15464e1f0c5SHalil Pasic addr += PAGE_SIZE; 15564e1f0c5SHalil Pasic } 15664e1f0c5SHalil Pasic return 0; 15764e1f0c5SHalil Pasic } 15864e1f0c5SHalil Pasic 15964e1f0c5SHalil Pasic /* are we a protected virtualization guest? */ 1609087c375STom Lendacky bool force_dma_unencrypted(struct device *dev) 1619087c375STom Lendacky { 1625cbdaeefSThiago Jung Bauermann return is_prot_virt_guest(); 1639087c375STom Lendacky } 1649087c375STom Lendacky 16564e1f0c5SHalil Pasic /* protected virtualization */ 16664e1f0c5SHalil Pasic static void pv_init(void) 16764e1f0c5SHalil Pasic { 16864e1f0c5SHalil Pasic if (!is_prot_virt_guest()) 16964e1f0c5SHalil Pasic return; 17064e1f0c5SHalil Pasic 17164e1f0c5SHalil Pasic /* make sure bounce buffers are shared */ 17264e1f0c5SHalil Pasic swiotlb_init(1); 17364e1f0c5SHalil Pasic swiotlb_update_mem_attributes(); 17464e1f0c5SHalil Pasic swiotlb_force = SWIOTLB_FORCE; 17564e1f0c5SHalil Pasic } 17664e1f0c5SHalil Pasic 1771da177e4SLinus Torvalds void __init mem_init(void) 1781da177e4SLinus Torvalds { 1791b948d6cSMartin Schwidefsky cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask); 1801b948d6cSMartin Schwidefsky cpumask_set_cpu(0, mm_cpumask(&init_mm)); 1811b948d6cSMartin Schwidefsky 1823a368f74SPhilipp Hachtmann set_max_mapnr(max_low_pfn); 1831da177e4SLinus Torvalds high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); 1841da177e4SLinus Torvalds 18564e1f0c5SHalil Pasic pv_init(); 18664e1f0c5SHalil Pasic 18745e576b1SMartin Schwidefsky /* Setup guest page hinting */ 18845e576b1SMartin Schwidefsky cmma_init(); 18945e576b1SMartin Schwidefsky 1901da177e4SLinus Torvalds /* this will put all low memory onto the freelists */ 191c6ffc5caSMike Rapoport memblock_free_all(); 1920999f119SJiang Liu setup_zero_pages(); /* Setup zeroed pages. */ 1931da177e4SLinus Torvalds 194c9b5ad54SMartin Schwidefsky cmma_init_nodat(); 195c9b5ad54SMartin Schwidefsky 196a18d0e2dSJiang Liu mem_init_print_info(NULL); 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds 199d96221abSHeiko Carstens void free_initmem(void) 200d96221abSHeiko Carstens { 2017a5da02dSGerald Schaefer initmem_freed = true; 20257d7f939SMartin Schwidefsky __set_memory((unsigned long)_sinittext, 203ead7a22eSHeiko Carstens (unsigned long)(_einittext - _sinittext) >> PAGE_SHIFT, 20457d7f939SMartin Schwidefsky SET_MEMORY_RW | SET_MEMORY_NX); 205dbe67df4SJiang Liu free_initmem_default(POISON_FREE_INITMEM); 2061da177e4SLinus Torvalds } 2071da177e4SLinus Torvalds 208604ddad0SHeiko Carstens unsigned long memory_block_size_bytes(void) 209604ddad0SHeiko Carstens { 210604ddad0SHeiko Carstens /* 211604ddad0SHeiko Carstens * Make sure the memory block size is always greater 212604ddad0SHeiko Carstens * or equal than the memory increment size. 213604ddad0SHeiko Carstens */ 214604ddad0SHeiko Carstens return max_t(unsigned long, MIN_MEMORY_BLOCK_SIZE, sclp.rzm); 215604ddad0SHeiko Carstens } 216604ddad0SHeiko Carstens 217421c175cSHeiko Carstens #ifdef CONFIG_MEMORY_HOTPLUG 21834ad7cdcSHeiko Carstens 21934ad7cdcSHeiko Carstens #ifdef CONFIG_CMA 22034ad7cdcSHeiko Carstens 22134ad7cdcSHeiko Carstens /* Prevent memory blocks which contain cma regions from going offline */ 22234ad7cdcSHeiko Carstens 22334ad7cdcSHeiko Carstens struct s390_cma_mem_data { 22434ad7cdcSHeiko Carstens unsigned long start; 22534ad7cdcSHeiko Carstens unsigned long end; 22634ad7cdcSHeiko Carstens }; 22734ad7cdcSHeiko Carstens 22834ad7cdcSHeiko Carstens static int s390_cma_check_range(struct cma *cma, void *data) 22934ad7cdcSHeiko Carstens { 23034ad7cdcSHeiko Carstens struct s390_cma_mem_data *mem_data; 23134ad7cdcSHeiko Carstens unsigned long start, end; 23234ad7cdcSHeiko Carstens 23334ad7cdcSHeiko Carstens mem_data = data; 23434ad7cdcSHeiko Carstens start = cma_get_base(cma); 23534ad7cdcSHeiko Carstens end = start + cma_get_size(cma); 23634ad7cdcSHeiko Carstens if (end < mem_data->start) 23734ad7cdcSHeiko Carstens return 0; 23834ad7cdcSHeiko Carstens if (start >= mem_data->end) 23934ad7cdcSHeiko Carstens return 0; 24034ad7cdcSHeiko Carstens return -EBUSY; 24134ad7cdcSHeiko Carstens } 24234ad7cdcSHeiko Carstens 24334ad7cdcSHeiko Carstens static int s390_cma_mem_notifier(struct notifier_block *nb, 24434ad7cdcSHeiko Carstens unsigned long action, void *data) 24534ad7cdcSHeiko Carstens { 24634ad7cdcSHeiko Carstens struct s390_cma_mem_data mem_data; 24734ad7cdcSHeiko Carstens struct memory_notify *arg; 24834ad7cdcSHeiko Carstens int rc = 0; 24934ad7cdcSHeiko Carstens 25034ad7cdcSHeiko Carstens arg = data; 25134ad7cdcSHeiko Carstens mem_data.start = arg->start_pfn << PAGE_SHIFT; 25234ad7cdcSHeiko Carstens mem_data.end = mem_data.start + (arg->nr_pages << PAGE_SHIFT); 25334ad7cdcSHeiko Carstens if (action == MEM_GOING_OFFLINE) 25434ad7cdcSHeiko Carstens rc = cma_for_each_area(s390_cma_check_range, &mem_data); 25534ad7cdcSHeiko Carstens return notifier_from_errno(rc); 25634ad7cdcSHeiko Carstens } 25734ad7cdcSHeiko Carstens 25834ad7cdcSHeiko Carstens static struct notifier_block s390_cma_mem_nb = { 25934ad7cdcSHeiko Carstens .notifier_call = s390_cma_mem_notifier, 26034ad7cdcSHeiko Carstens }; 26134ad7cdcSHeiko Carstens 26234ad7cdcSHeiko Carstens static int __init s390_cma_mem_init(void) 26334ad7cdcSHeiko Carstens { 26434ad7cdcSHeiko Carstens return register_memory_notifier(&s390_cma_mem_nb); 26534ad7cdcSHeiko Carstens } 26634ad7cdcSHeiko Carstens device_initcall(s390_cma_mem_init); 26734ad7cdcSHeiko Carstens 26834ad7cdcSHeiko Carstens #endif /* CONFIG_CMA */ 26934ad7cdcSHeiko Carstens 270940519f0SMichal Hocko int arch_add_memory(int nid, u64 start, u64 size, 271940519f0SMichal Hocko struct mhp_restrictions *restrictions) 272421c175cSHeiko Carstens { 273892365abSGerald Schaefer unsigned long start_pfn = PFN_DOWN(start); 274892365abSGerald Schaefer unsigned long size_pages = PFN_DOWN(size); 275f1dd2cd1SMichal Hocko int rc; 276421c175cSHeiko Carstens 277973de24aSDavid Hildenbrand if (WARN_ON_ONCE(restrictions->altmap)) 278973de24aSDavid Hildenbrand return -EINVAL; 279973de24aSDavid Hildenbrand 280421c175cSHeiko Carstens rc = vmem_add_mapping(start, size); 281421c175cSHeiko Carstens if (rc) 282421c175cSHeiko Carstens return rc; 283199071f1SGerald Schaefer 284940519f0SMichal Hocko rc = __add_pages(nid, start_pfn, size_pages, restrictions); 285421c175cSHeiko Carstens if (rc) 286421c175cSHeiko Carstens vmem_remove_mapping(start, size); 287421c175cSHeiko Carstens return rc; 288421c175cSHeiko Carstens } 28924d335caSWen Congyang 290ac5c9426SDavid Hildenbrand void arch_remove_memory(int nid, u64 start, u64 size, 291ac5c9426SDavid Hildenbrand struct vmem_altmap *altmap) 29224d335caSWen Congyang { 29318c86506SDavid Hildenbrand unsigned long start_pfn = start >> PAGE_SHIFT; 29418c86506SDavid Hildenbrand unsigned long nr_pages = size >> PAGE_SHIFT; 29518c86506SDavid Hildenbrand struct zone *zone; 29618c86506SDavid Hildenbrand 29718c86506SDavid Hildenbrand zone = page_zone(pfn_to_page(start_pfn)); 29818c86506SDavid Hildenbrand __remove_pages(zone, start_pfn, nr_pages, altmap); 29918c86506SDavid Hildenbrand vmem_remove_mapping(start, size); 30024d335caSWen Congyang } 301421c175cSHeiko Carstens #endif /* CONFIG_MEMORY_HOTPLUG */ 302