1c221c0b0SDave Hansen // SPDX-License-Identifier: GPL-2.0 2c221c0b0SDave Hansen /* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ 3c221c0b0SDave Hansen #include <linux/memremap.h> 4c221c0b0SDave Hansen #include <linux/pagemap.h> 5c221c0b0SDave Hansen #include <linux/memory.h> 6c221c0b0SDave Hansen #include <linux/module.h> 7c221c0b0SDave Hansen #include <linux/device.h> 8c221c0b0SDave Hansen #include <linux/pfn_t.h> 9c221c0b0SDave Hansen #include <linux/slab.h> 10c221c0b0SDave Hansen #include <linux/dax.h> 11c221c0b0SDave Hansen #include <linux/fs.h> 12c221c0b0SDave Hansen #include <linux/mm.h> 13c221c0b0SDave Hansen #include <linux/mman.h> 14c221c0b0SDave Hansen #include "dax-private.h" 15c221c0b0SDave Hansen #include "bus.h" 16c221c0b0SDave Hansen 178a725e46SDavid Hildenbrand /* Memory resource name used for add_memory_driver_managed(). */ 188a725e46SDavid Hildenbrand static const char *kmem_name; 198a725e46SDavid Hildenbrand /* Set if any memory will remain added when the driver will be unloaded. */ 208a725e46SDavid Hildenbrand static bool any_hotremove_failed; 218a725e46SDavid Hildenbrand 22c221c0b0SDave Hansen int dev_dax_kmem_probe(struct device *dev) 23c221c0b0SDave Hansen { 24c221c0b0SDave Hansen struct dev_dax *dev_dax = to_dev_dax(dev); 25c221c0b0SDave Hansen struct resource *res = &dev_dax->region->res; 26c221c0b0SDave Hansen resource_size_t kmem_start; 27c221c0b0SDave Hansen resource_size_t kmem_size; 28c221c0b0SDave Hansen resource_size_t kmem_end; 29c221c0b0SDave Hansen struct resource *new_res; 3060858c00SDavid Hildenbrand const char *new_res_name; 31c221c0b0SDave Hansen int numa_node; 32c221c0b0SDave Hansen int rc; 33c221c0b0SDave Hansen 34c221c0b0SDave Hansen /* 35c221c0b0SDave Hansen * Ensure good NUMA information for the persistent memory. 36c221c0b0SDave Hansen * Without this check, there is a risk that slow memory 37c221c0b0SDave Hansen * could be mixed in a node with faster memory, causing 38c221c0b0SDave Hansen * unavoidable performance issues. 39c221c0b0SDave Hansen */ 40c221c0b0SDave Hansen numa_node = dev_dax->target_node; 41c221c0b0SDave Hansen if (numa_node < 0) { 42c221c0b0SDave Hansen dev_warn(dev, "rejecting DAX region %pR with invalid node: %d\n", 43c221c0b0SDave Hansen res, numa_node); 44c221c0b0SDave Hansen return -EINVAL; 45c221c0b0SDave Hansen } 46c221c0b0SDave Hansen 47c221c0b0SDave Hansen /* Hotplug starting at the beginning of the next block: */ 48c221c0b0SDave Hansen kmem_start = ALIGN(res->start, memory_block_size_bytes()); 49c221c0b0SDave Hansen 50c221c0b0SDave Hansen kmem_size = resource_size(res); 51c221c0b0SDave Hansen /* Adjust the size down to compensate for moving up kmem_start: */ 52c221c0b0SDave Hansen kmem_size -= kmem_start - res->start; 53c221c0b0SDave Hansen /* Align the size down to cover only complete blocks: */ 54c221c0b0SDave Hansen kmem_size &= ~(memory_block_size_bytes() - 1); 55c221c0b0SDave Hansen kmem_end = kmem_start + kmem_size; 56c221c0b0SDave Hansen 5760858c00SDavid Hildenbrand new_res_name = kstrdup(dev_name(dev), GFP_KERNEL); 5860858c00SDavid Hildenbrand if (!new_res_name) 5960858c00SDavid Hildenbrand return -ENOMEM; 6060858c00SDavid Hildenbrand 6160858c00SDavid Hildenbrand /* Region is permanently reserved if hotremove fails. */ 6260858c00SDavid Hildenbrand new_res = request_mem_region(kmem_start, kmem_size, new_res_name); 63c221c0b0SDave Hansen if (!new_res) { 64c221c0b0SDave Hansen dev_warn(dev, "could not reserve region [%pa-%pa]\n", 65c221c0b0SDave Hansen &kmem_start, &kmem_end); 6660858c00SDavid Hildenbrand kfree(new_res_name); 67c221c0b0SDave Hansen return -EBUSY; 68c221c0b0SDave Hansen } 69c221c0b0SDave Hansen 70c221c0b0SDave Hansen /* 71c221c0b0SDave Hansen * Set flags appropriate for System RAM. Leave ..._BUSY clear 72c221c0b0SDave Hansen * so that add_memory() can add a child resource. Do not 73c221c0b0SDave Hansen * inherit flags from the parent since it may set new flags 74c221c0b0SDave Hansen * unknown to us that will break add_memory() below. 75c221c0b0SDave Hansen */ 76c221c0b0SDave Hansen new_res->flags = IORESOURCE_SYSTEM_RAM; 77c221c0b0SDave Hansen 788a725e46SDavid Hildenbrand /* 798a725e46SDavid Hildenbrand * Ensure that future kexec'd kernels will not treat this as RAM 808a725e46SDavid Hildenbrand * automatically. 818a725e46SDavid Hildenbrand */ 828a725e46SDavid Hildenbrand rc = add_memory_driver_managed(numa_node, new_res->start, 838a725e46SDavid Hildenbrand resource_size(new_res), kmem_name); 8431e4ca92SPavel Tatashin if (rc) { 8531e4ca92SPavel Tatashin release_resource(new_res); 8631e4ca92SPavel Tatashin kfree(new_res); 8760858c00SDavid Hildenbrand kfree(new_res_name); 88c221c0b0SDave Hansen return rc; 8931e4ca92SPavel Tatashin } 909f960da7SPavel Tatashin dev_dax->dax_kmem_res = new_res; 91c221c0b0SDave Hansen 92c221c0b0SDave Hansen return 0; 93c221c0b0SDave Hansen } 94c221c0b0SDave Hansen 959f960da7SPavel Tatashin #ifdef CONFIG_MEMORY_HOTREMOVE 969f960da7SPavel Tatashin static int dev_dax_kmem_remove(struct device *dev) 979f960da7SPavel Tatashin { 989f960da7SPavel Tatashin struct dev_dax *dev_dax = to_dev_dax(dev); 999f960da7SPavel Tatashin struct resource *res = dev_dax->dax_kmem_res; 1009f960da7SPavel Tatashin resource_size_t kmem_start = res->start; 1019f960da7SPavel Tatashin resource_size_t kmem_size = resource_size(res); 10260858c00SDavid Hildenbrand const char *res_name = res->name; 1039f960da7SPavel Tatashin int rc; 1049f960da7SPavel Tatashin 1059f960da7SPavel Tatashin /* 1069f960da7SPavel Tatashin * We have one shot for removing memory, if some memory blocks were not 1079f960da7SPavel Tatashin * offline prior to calling this function remove_memory() will fail, and 1089f960da7SPavel Tatashin * there is no way to hotremove this memory until reboot because device 1099f960da7SPavel Tatashin * unbind will succeed even if we return failure. 1109f960da7SPavel Tatashin */ 1119f960da7SPavel Tatashin rc = remove_memory(dev_dax->target_node, kmem_start, kmem_size); 1129f960da7SPavel Tatashin if (rc) { 1138a725e46SDavid Hildenbrand any_hotremove_failed = true; 1149f960da7SPavel Tatashin dev_err(dev, 1159f960da7SPavel Tatashin "DAX region %pR cannot be hotremoved until the next reboot\n", 1169f960da7SPavel Tatashin res); 1179f960da7SPavel Tatashin return rc; 1189f960da7SPavel Tatashin } 1199f960da7SPavel Tatashin 1209f960da7SPavel Tatashin /* Release and free dax resources */ 1219f960da7SPavel Tatashin release_resource(res); 1229f960da7SPavel Tatashin kfree(res); 12360858c00SDavid Hildenbrand kfree(res_name); 1249f960da7SPavel Tatashin dev_dax->dax_kmem_res = NULL; 1259f960da7SPavel Tatashin 1269f960da7SPavel Tatashin return 0; 1279f960da7SPavel Tatashin } 1289f960da7SPavel Tatashin #else 129c221c0b0SDave Hansen static int dev_dax_kmem_remove(struct device *dev) 130c221c0b0SDave Hansen { 131c221c0b0SDave Hansen /* 1329f960da7SPavel Tatashin * Without hotremove purposely leak the request_mem_region() for the 1339f960da7SPavel Tatashin * device-dax range and return '0' to ->remove() attempts. The removal 1349f960da7SPavel Tatashin * of the device from the driver always succeeds, but the region is 1359f960da7SPavel Tatashin * permanently pinned as reserved by the unreleased 136c221c0b0SDave Hansen * request_mem_region(). 137c221c0b0SDave Hansen */ 1388a725e46SDavid Hildenbrand any_hotremove_failed = true; 139c221c0b0SDave Hansen return 0; 140c221c0b0SDave Hansen } 1419f960da7SPavel Tatashin #endif /* CONFIG_MEMORY_HOTREMOVE */ 142c221c0b0SDave Hansen 143c221c0b0SDave Hansen static struct dax_device_driver device_dax_kmem_driver = { 144c221c0b0SDave Hansen .drv = { 145c221c0b0SDave Hansen .probe = dev_dax_kmem_probe, 146c221c0b0SDave Hansen .remove = dev_dax_kmem_remove, 147c221c0b0SDave Hansen }, 148c221c0b0SDave Hansen }; 149c221c0b0SDave Hansen 150c221c0b0SDave Hansen static int __init dax_kmem_init(void) 151c221c0b0SDave Hansen { 1528a725e46SDavid Hildenbrand int rc; 1538a725e46SDavid Hildenbrand 1548a725e46SDavid Hildenbrand /* Resource name is permanently allocated if any hotremove fails. */ 1558a725e46SDavid Hildenbrand kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); 1568a725e46SDavid Hildenbrand if (!kmem_name) 1578a725e46SDavid Hildenbrand return -ENOMEM; 1588a725e46SDavid Hildenbrand 1598a725e46SDavid Hildenbrand rc = dax_driver_register(&device_dax_kmem_driver); 1608a725e46SDavid Hildenbrand if (rc) 1618a725e46SDavid Hildenbrand kfree_const(kmem_name); 1628a725e46SDavid Hildenbrand return rc; 163c221c0b0SDave Hansen } 164c221c0b0SDave Hansen 165c221c0b0SDave Hansen static void __exit dax_kmem_exit(void) 166c221c0b0SDave Hansen { 167c221c0b0SDave Hansen dax_driver_unregister(&device_dax_kmem_driver); 1688a725e46SDavid Hildenbrand if (!any_hotremove_failed) 1698a725e46SDavid Hildenbrand kfree_const(kmem_name); 170c221c0b0SDave Hansen } 171c221c0b0SDave Hansen 172c221c0b0SDave Hansen MODULE_AUTHOR("Intel Corporation"); 173c221c0b0SDave Hansen MODULE_LICENSE("GPL v2"); 174c221c0b0SDave Hansen module_init(dax_kmem_init); 175c221c0b0SDave Hansen module_exit(dax_kmem_exit); 176c221c0b0SDave Hansen MODULE_ALIAS_DAX_DEVICE(0); 177