1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 310fbcf4cSKay Sievers * Basic Node interface support 41da177e4SLinus Torvalds */ 51da177e4SLinus Torvalds 61da177e4SLinus Torvalds #include <linux/module.h> 71da177e4SLinus Torvalds #include <linux/init.h> 81da177e4SLinus Torvalds #include <linux/mm.h> 9c04fc586SGary Hade #include <linux/memory.h> 10fa25c503SKOSAKI Motohiro #include <linux/vmstat.h> 116e259e7dSAndrew Morton #include <linux/notifier.h> 121da177e4SLinus Torvalds #include <linux/node.h> 131da177e4SLinus Torvalds #include <linux/hugetlb.h> 14ed4a6d7fSMel Gorman #include <linux/compaction.h> 151da177e4SLinus Torvalds #include <linux/cpumask.h> 161da177e4SLinus Torvalds #include <linux/topology.h> 171da177e4SLinus Torvalds #include <linux/nodemask.h> 1876b67ed9SKAMEZAWA Hiroyuki #include <linux/cpu.h> 19bde631a5SLee Schermerhorn #include <linux/device.h> 2008d9dbe7SKeith Busch #include <linux/pm_runtime.h> 21af936a16SLee Schermerhorn #include <linux/swap.h> 2218e5b539STejun Heo #include <linux/slab.h> 231da177e4SLinus Torvalds 2410fbcf4cSKay Sievers static struct bus_type node_subsys = { 25af5ca3f4SKay Sievers .name = "node", 2610fbcf4cSKay Sievers .dev_name = "node", 271da177e4SLinus Torvalds }; 281da177e4SLinus Torvalds 2975bd50faSTian Tao static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj, 3075bd50faSTian Tao struct bin_attribute *attr, char *buf, 3175bd50faSTian Tao loff_t off, size_t count) 321da177e4SLinus Torvalds { 3375bd50faSTian Tao struct device *dev = kobj_to_dev(kobj); 341da177e4SLinus Torvalds struct node *node_dev = to_node(dev); 3575bd50faSTian Tao cpumask_var_t mask; 3675bd50faSTian Tao ssize_t n; 371da177e4SLinus Torvalds 38064f0e93SZhen Lei if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 39064f0e93SZhen Lei return 0; 40064f0e93SZhen Lei 41064f0e93SZhen Lei cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); 4275bd50faSTian Tao n = cpumap_print_bitmask_to_buf(buf, mask, off, count); 43064f0e93SZhen Lei free_cpumask_var(mask); 44064f0e93SZhen Lei 45064f0e93SZhen Lei return n; 461da177e4SLinus Torvalds } 471da177e4SLinus Torvalds 4875bd50faSTian Tao static BIN_ATTR_RO(cpumap, 0); 4975bd50faSTian Tao 5075bd50faSTian Tao static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj, 5175bd50faSTian Tao struct bin_attribute *attr, char *buf, 5275bd50faSTian Tao loff_t off, size_t count) 5339106dcfSMike Travis { 5475bd50faSTian Tao struct device *dev = kobj_to_dev(kobj); 5575bd50faSTian Tao struct node *node_dev = to_node(dev); 5675bd50faSTian Tao cpumask_var_t mask; 5775bd50faSTian Tao ssize_t n; 5875bd50faSTian Tao 5975bd50faSTian Tao if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 6075bd50faSTian Tao return 0; 6175bd50faSTian Tao 6275bd50faSTian Tao cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); 6375bd50faSTian Tao n = cpumap_print_list_to_buf(buf, mask, off, count); 6475bd50faSTian Tao free_cpumask_var(mask); 6575bd50faSTian Tao 6675bd50faSTian Tao return n; 6739106dcfSMike Travis } 68948b3edbSJoe Perches 6975bd50faSTian Tao static BIN_ATTR_RO(cpulist, 0); 701da177e4SLinus Torvalds 7108d9dbe7SKeith Busch /** 7208d9dbe7SKeith Busch * struct node_access_nodes - Access class device to hold user visible 7308d9dbe7SKeith Busch * relationships to other nodes. 7408d9dbe7SKeith Busch * @dev: Device for this memory access class 7508d9dbe7SKeith Busch * @list_node: List element in the node's access list 7608d9dbe7SKeith Busch * @access: The access class rank 7758cb346cSMauro Carvalho Chehab * @hmem_attrs: Heterogeneous memory performance attributes 7808d9dbe7SKeith Busch */ 7908d9dbe7SKeith Busch struct node_access_nodes { 8008d9dbe7SKeith Busch struct device dev; 8108d9dbe7SKeith Busch struct list_head list_node; 82e7deeb9dSJinchao Wang unsigned int access; 83e1cf33aaSKeith Busch #ifdef CONFIG_HMEM_REPORTING 84e1cf33aaSKeith Busch struct node_hmem_attrs hmem_attrs; 85e1cf33aaSKeith Busch #endif 8608d9dbe7SKeith Busch }; 8708d9dbe7SKeith Busch #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev) 8808d9dbe7SKeith Busch 8908d9dbe7SKeith Busch static struct attribute *node_init_access_node_attrs[] = { 9008d9dbe7SKeith Busch NULL, 9108d9dbe7SKeith Busch }; 9208d9dbe7SKeith Busch 9308d9dbe7SKeith Busch static struct attribute *node_targ_access_node_attrs[] = { 9408d9dbe7SKeith Busch NULL, 9508d9dbe7SKeith Busch }; 9608d9dbe7SKeith Busch 9708d9dbe7SKeith Busch static const struct attribute_group initiators = { 9808d9dbe7SKeith Busch .name = "initiators", 9908d9dbe7SKeith Busch .attrs = node_init_access_node_attrs, 10008d9dbe7SKeith Busch }; 10108d9dbe7SKeith Busch 10208d9dbe7SKeith Busch static const struct attribute_group targets = { 10308d9dbe7SKeith Busch .name = "targets", 10408d9dbe7SKeith Busch .attrs = node_targ_access_node_attrs, 10508d9dbe7SKeith Busch }; 10608d9dbe7SKeith Busch 10708d9dbe7SKeith Busch static const struct attribute_group *node_access_node_groups[] = { 10808d9dbe7SKeith Busch &initiators, 10908d9dbe7SKeith Busch &targets, 11008d9dbe7SKeith Busch NULL, 11108d9dbe7SKeith Busch }; 11208d9dbe7SKeith Busch 11308d9dbe7SKeith Busch static void node_remove_accesses(struct node *node) 11408d9dbe7SKeith Busch { 11508d9dbe7SKeith Busch struct node_access_nodes *c, *cnext; 11608d9dbe7SKeith Busch 11708d9dbe7SKeith Busch list_for_each_entry_safe(c, cnext, &node->access_list, list_node) { 11808d9dbe7SKeith Busch list_del(&c->list_node); 11908d9dbe7SKeith Busch device_unregister(&c->dev); 12008d9dbe7SKeith Busch } 12108d9dbe7SKeith Busch } 12208d9dbe7SKeith Busch 12308d9dbe7SKeith Busch static void node_access_release(struct device *dev) 12408d9dbe7SKeith Busch { 12508d9dbe7SKeith Busch kfree(to_access_nodes(dev)); 12608d9dbe7SKeith Busch } 12708d9dbe7SKeith Busch 12808d9dbe7SKeith Busch static struct node_access_nodes *node_init_node_access(struct node *node, 129e7deeb9dSJinchao Wang unsigned int access) 13008d9dbe7SKeith Busch { 13108d9dbe7SKeith Busch struct node_access_nodes *access_node; 13208d9dbe7SKeith Busch struct device *dev; 13308d9dbe7SKeith Busch 13408d9dbe7SKeith Busch list_for_each_entry(access_node, &node->access_list, list_node) 13508d9dbe7SKeith Busch if (access_node->access == access) 13608d9dbe7SKeith Busch return access_node; 13708d9dbe7SKeith Busch 13808d9dbe7SKeith Busch access_node = kzalloc(sizeof(*access_node), GFP_KERNEL); 13908d9dbe7SKeith Busch if (!access_node) 14008d9dbe7SKeith Busch return NULL; 14108d9dbe7SKeith Busch 14208d9dbe7SKeith Busch access_node->access = access; 14308d9dbe7SKeith Busch dev = &access_node->dev; 14408d9dbe7SKeith Busch dev->parent = &node->dev; 14508d9dbe7SKeith Busch dev->release = node_access_release; 14608d9dbe7SKeith Busch dev->groups = node_access_node_groups; 14708d9dbe7SKeith Busch if (dev_set_name(dev, "access%u", access)) 14808d9dbe7SKeith Busch goto free; 14908d9dbe7SKeith Busch 15008d9dbe7SKeith Busch if (device_register(dev)) 15108d9dbe7SKeith Busch goto free_name; 15208d9dbe7SKeith Busch 15308d9dbe7SKeith Busch pm_runtime_no_callbacks(dev); 15408d9dbe7SKeith Busch list_add_tail(&access_node->list_node, &node->access_list); 15508d9dbe7SKeith Busch return access_node; 15608d9dbe7SKeith Busch free_name: 15708d9dbe7SKeith Busch kfree_const(dev->kobj.name); 15808d9dbe7SKeith Busch free: 15908d9dbe7SKeith Busch kfree(access_node); 16008d9dbe7SKeith Busch return NULL; 16108d9dbe7SKeith Busch } 16208d9dbe7SKeith Busch 163e1cf33aaSKeith Busch #ifdef CONFIG_HMEM_REPORTING 164e1cf33aaSKeith Busch #define ACCESS_ATTR(name) \ 165e1cf33aaSKeith Busch static ssize_t name##_show(struct device *dev, \ 166e1cf33aaSKeith Busch struct device_attribute *attr, \ 167e1cf33aaSKeith Busch char *buf) \ 168e1cf33aaSKeith Busch { \ 169948b3edbSJoe Perches return sysfs_emit(buf, "%u\n", \ 170948b3edbSJoe Perches to_access_nodes(dev)->hmem_attrs.name); \ 171e1cf33aaSKeith Busch } \ 1726284a6e8SJoe Perches static DEVICE_ATTR_RO(name) 173e1cf33aaSKeith Busch 1746284a6e8SJoe Perches ACCESS_ATTR(read_bandwidth); 1756284a6e8SJoe Perches ACCESS_ATTR(read_latency); 1766284a6e8SJoe Perches ACCESS_ATTR(write_bandwidth); 1776284a6e8SJoe Perches ACCESS_ATTR(write_latency); 178e1cf33aaSKeith Busch 179e1cf33aaSKeith Busch static struct attribute *access_attrs[] = { 180e1cf33aaSKeith Busch &dev_attr_read_bandwidth.attr, 181e1cf33aaSKeith Busch &dev_attr_read_latency.attr, 182e1cf33aaSKeith Busch &dev_attr_write_bandwidth.attr, 183e1cf33aaSKeith Busch &dev_attr_write_latency.attr, 184e1cf33aaSKeith Busch NULL, 185e1cf33aaSKeith Busch }; 186e1cf33aaSKeith Busch 187e1cf33aaSKeith Busch /** 188e1cf33aaSKeith Busch * node_set_perf_attrs - Set the performance values for given access class 189e1cf33aaSKeith Busch * @nid: Node identifier to be set 190e1cf33aaSKeith Busch * @hmem_attrs: Heterogeneous memory performance attributes 191e1cf33aaSKeith Busch * @access: The access class the for the given attributes 192e1cf33aaSKeith Busch */ 193e1cf33aaSKeith Busch void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, 194e7deeb9dSJinchao Wang unsigned int access) 195e1cf33aaSKeith Busch { 196e1cf33aaSKeith Busch struct node_access_nodes *c; 197e1cf33aaSKeith Busch struct node *node; 198e1cf33aaSKeith Busch int i; 199e1cf33aaSKeith Busch 200e1cf33aaSKeith Busch if (WARN_ON_ONCE(!node_online(nid))) 201e1cf33aaSKeith Busch return; 202e1cf33aaSKeith Busch 203e1cf33aaSKeith Busch node = node_devices[nid]; 204e1cf33aaSKeith Busch c = node_init_node_access(node, access); 205e1cf33aaSKeith Busch if (!c) 206e1cf33aaSKeith Busch return; 207e1cf33aaSKeith Busch 208e1cf33aaSKeith Busch c->hmem_attrs = *hmem_attrs; 209e1cf33aaSKeith Busch for (i = 0; access_attrs[i] != NULL; i++) { 210e1cf33aaSKeith Busch if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i], 211e1cf33aaSKeith Busch "initiators")) { 212e1cf33aaSKeith Busch pr_info("failed to add performance attribute to node %d\n", 213e1cf33aaSKeith Busch nid); 214e1cf33aaSKeith Busch break; 215e1cf33aaSKeith Busch } 216e1cf33aaSKeith Busch } 217e1cf33aaSKeith Busch } 218acc02a10SKeith Busch 219acc02a10SKeith Busch /** 220acc02a10SKeith Busch * struct node_cache_info - Internal tracking for memory node caches 221acc02a10SKeith Busch * @dev: Device represeting the cache level 222acc02a10SKeith Busch * @node: List element for tracking in the node 223acc02a10SKeith Busch * @cache_attrs:Attributes for this cache level 224acc02a10SKeith Busch */ 225acc02a10SKeith Busch struct node_cache_info { 226acc02a10SKeith Busch struct device dev; 227acc02a10SKeith Busch struct list_head node; 228acc02a10SKeith Busch struct node_cache_attrs cache_attrs; 229acc02a10SKeith Busch }; 230acc02a10SKeith Busch #define to_cache_info(device) container_of(device, struct node_cache_info, dev) 231acc02a10SKeith Busch 232acc02a10SKeith Busch #define CACHE_ATTR(name, fmt) \ 233acc02a10SKeith Busch static ssize_t name##_show(struct device *dev, \ 234acc02a10SKeith Busch struct device_attribute *attr, \ 235acc02a10SKeith Busch char *buf) \ 236acc02a10SKeith Busch { \ 237948b3edbSJoe Perches return sysfs_emit(buf, fmt "\n", \ 238948b3edbSJoe Perches to_cache_info(dev)->cache_attrs.name); \ 239acc02a10SKeith Busch } \ 240fd03c075SRuiqi Gong static DEVICE_ATTR_RO(name); 241acc02a10SKeith Busch 242acc02a10SKeith Busch CACHE_ATTR(size, "%llu") 243acc02a10SKeith Busch CACHE_ATTR(line_size, "%u") 244acc02a10SKeith Busch CACHE_ATTR(indexing, "%u") 245acc02a10SKeith Busch CACHE_ATTR(write_policy, "%u") 246acc02a10SKeith Busch 247acc02a10SKeith Busch static struct attribute *cache_attrs[] = { 248acc02a10SKeith Busch &dev_attr_indexing.attr, 249acc02a10SKeith Busch &dev_attr_size.attr, 250acc02a10SKeith Busch &dev_attr_line_size.attr, 251acc02a10SKeith Busch &dev_attr_write_policy.attr, 252acc02a10SKeith Busch NULL, 253acc02a10SKeith Busch }; 254acc02a10SKeith Busch ATTRIBUTE_GROUPS(cache); 255acc02a10SKeith Busch 256acc02a10SKeith Busch static void node_cache_release(struct device *dev) 257acc02a10SKeith Busch { 258acc02a10SKeith Busch kfree(dev); 259acc02a10SKeith Busch } 260acc02a10SKeith Busch 261acc02a10SKeith Busch static void node_cacheinfo_release(struct device *dev) 262acc02a10SKeith Busch { 263acc02a10SKeith Busch struct node_cache_info *info = to_cache_info(dev); 264acc02a10SKeith Busch kfree(info); 265acc02a10SKeith Busch } 266acc02a10SKeith Busch 267acc02a10SKeith Busch static void node_init_cache_dev(struct node *node) 268acc02a10SKeith Busch { 269acc02a10SKeith Busch struct device *dev; 270acc02a10SKeith Busch 271acc02a10SKeith Busch dev = kzalloc(sizeof(*dev), GFP_KERNEL); 272acc02a10SKeith Busch if (!dev) 273acc02a10SKeith Busch return; 274acc02a10SKeith Busch 2754ce535ecSDan Carpenter device_initialize(dev); 276acc02a10SKeith Busch dev->parent = &node->dev; 277acc02a10SKeith Busch dev->release = node_cache_release; 278acc02a10SKeith Busch if (dev_set_name(dev, "memory_side_cache")) 2794ce535ecSDan Carpenter goto put_device; 280acc02a10SKeith Busch 2814ce535ecSDan Carpenter if (device_add(dev)) 2824ce535ecSDan Carpenter goto put_device; 283acc02a10SKeith Busch 284acc02a10SKeith Busch pm_runtime_no_callbacks(dev); 285acc02a10SKeith Busch node->cache_dev = dev; 286acc02a10SKeith Busch return; 2874ce535ecSDan Carpenter put_device: 2884ce535ecSDan Carpenter put_device(dev); 289acc02a10SKeith Busch } 290acc02a10SKeith Busch 291acc02a10SKeith Busch /** 292acc02a10SKeith Busch * node_add_cache() - add cache attribute to a memory node 293acc02a10SKeith Busch * @nid: Node identifier that has new cache attributes 294acc02a10SKeith Busch * @cache_attrs: Attributes for the cache being added 295acc02a10SKeith Busch */ 296acc02a10SKeith Busch void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) 297acc02a10SKeith Busch { 298acc02a10SKeith Busch struct node_cache_info *info; 299acc02a10SKeith Busch struct device *dev; 300acc02a10SKeith Busch struct node *node; 301acc02a10SKeith Busch 302acc02a10SKeith Busch if (!node_online(nid) || !node_devices[nid]) 303acc02a10SKeith Busch return; 304acc02a10SKeith Busch 305acc02a10SKeith Busch node = node_devices[nid]; 306acc02a10SKeith Busch list_for_each_entry(info, &node->cache_attrs, node) { 307acc02a10SKeith Busch if (info->cache_attrs.level == cache_attrs->level) { 308acc02a10SKeith Busch dev_warn(&node->dev, 309acc02a10SKeith Busch "attempt to add duplicate cache level:%d\n", 310acc02a10SKeith Busch cache_attrs->level); 311acc02a10SKeith Busch return; 312acc02a10SKeith Busch } 313acc02a10SKeith Busch } 314acc02a10SKeith Busch 315acc02a10SKeith Busch if (!node->cache_dev) 316acc02a10SKeith Busch node_init_cache_dev(node); 317acc02a10SKeith Busch if (!node->cache_dev) 318acc02a10SKeith Busch return; 319acc02a10SKeith Busch 320acc02a10SKeith Busch info = kzalloc(sizeof(*info), GFP_KERNEL); 321acc02a10SKeith Busch if (!info) 322acc02a10SKeith Busch return; 323acc02a10SKeith Busch 324acc02a10SKeith Busch dev = &info->dev; 3254ce535ecSDan Carpenter device_initialize(dev); 326acc02a10SKeith Busch dev->parent = node->cache_dev; 327acc02a10SKeith Busch dev->release = node_cacheinfo_release; 328acc02a10SKeith Busch dev->groups = cache_groups; 329acc02a10SKeith Busch if (dev_set_name(dev, "index%d", cache_attrs->level)) 3304ce535ecSDan Carpenter goto put_device; 331acc02a10SKeith Busch 332acc02a10SKeith Busch info->cache_attrs = *cache_attrs; 3334ce535ecSDan Carpenter if (device_add(dev)) { 334acc02a10SKeith Busch dev_warn(&node->dev, "failed to add cache level:%d\n", 335acc02a10SKeith Busch cache_attrs->level); 3364ce535ecSDan Carpenter goto put_device; 337acc02a10SKeith Busch } 338acc02a10SKeith Busch pm_runtime_no_callbacks(dev); 339acc02a10SKeith Busch list_add_tail(&info->node, &node->cache_attrs); 340acc02a10SKeith Busch return; 3414ce535ecSDan Carpenter put_device: 3424ce535ecSDan Carpenter put_device(dev); 343acc02a10SKeith Busch } 344acc02a10SKeith Busch 345acc02a10SKeith Busch static void node_remove_caches(struct node *node) 346acc02a10SKeith Busch { 347acc02a10SKeith Busch struct node_cache_info *info, *next; 348acc02a10SKeith Busch 349acc02a10SKeith Busch if (!node->cache_dev) 350acc02a10SKeith Busch return; 351acc02a10SKeith Busch 352acc02a10SKeith Busch list_for_each_entry_safe(info, next, &node->cache_attrs, node) { 353acc02a10SKeith Busch list_del(&info->node); 354acc02a10SKeith Busch device_unregister(&info->dev); 355acc02a10SKeith Busch } 356acc02a10SKeith Busch device_unregister(node->cache_dev); 357acc02a10SKeith Busch } 358acc02a10SKeith Busch 359acc02a10SKeith Busch static void node_init_caches(unsigned int nid) 360acc02a10SKeith Busch { 361acc02a10SKeith Busch INIT_LIST_HEAD(&node_devices[nid]->cache_attrs); 362acc02a10SKeith Busch } 363acc02a10SKeith Busch #else 364acc02a10SKeith Busch static void node_init_caches(unsigned int nid) { } 365acc02a10SKeith Busch static void node_remove_caches(struct node *node) { } 366e1cf33aaSKeith Busch #endif 367e1cf33aaSKeith Busch 3681da177e4SLinus Torvalds #define K(x) ((x) << (PAGE_SHIFT - 10)) 36910fbcf4cSKay Sievers static ssize_t node_read_meminfo(struct device *dev, 37010fbcf4cSKay Sievers struct device_attribute *attr, char *buf) 3711da177e4SLinus Torvalds { 372948b3edbSJoe Perches int len = 0; 3731da177e4SLinus Torvalds int nid = dev->id; 374599d0c95SMel Gorman struct pglist_data *pgdat = NODE_DATA(nid); 3751da177e4SLinus Torvalds struct sysinfo i; 37661f94e18SVlastimil Babka unsigned long sreclaimable, sunreclaimable; 377b6038942SShakeel Butt unsigned long swapcached = 0; 3781da177e4SLinus Torvalds 3791da177e4SLinus Torvalds si_meminfo_node(&i, nid); 380d42f3245SRoman Gushchin sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); 381d42f3245SRoman Gushchin sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); 382b6038942SShakeel Butt #ifdef CONFIG_SWAP 383b6038942SShakeel Butt swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); 384b6038942SShakeel Butt #endif 385948b3edbSJoe Perches len = sysfs_emit_at(buf, len, 3861da177e4SLinus Torvalds "Node %d MemTotal: %8lu kB\n" 3871da177e4SLinus Torvalds "Node %d MemFree: %8lu kB\n" 3881da177e4SLinus Torvalds "Node %d MemUsed: %8lu kB\n" 389b6038942SShakeel Butt "Node %d SwapCached: %8lu kB\n" 3901da177e4SLinus Torvalds "Node %d Active: %8lu kB\n" 3911da177e4SLinus Torvalds "Node %d Inactive: %8lu kB\n" 3924f98a2feSRik van Riel "Node %d Active(anon): %8lu kB\n" 3934f98a2feSRik van Riel "Node %d Inactive(anon): %8lu kB\n" 3944f98a2feSRik van Riel "Node %d Active(file): %8lu kB\n" 3954f98a2feSRik van Riel "Node %d Inactive(file): %8lu kB\n" 3965344b7e6SNick Piggin "Node %d Unevictable: %8lu kB\n" 3977ee92255SKOSAKI Motohiro "Node %d Mlocked: %8lu kB\n", 3987ee92255SKOSAKI Motohiro nid, K(i.totalram), 3997ee92255SKOSAKI Motohiro nid, K(i.freeram), 4007ee92255SKOSAKI Motohiro nid, K(i.totalram - i.freeram), 401b6038942SShakeel Butt nid, K(swapcached), 402599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + 403599d0c95SMel Gorman node_page_state(pgdat, NR_ACTIVE_FILE)), 404599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + 405599d0c95SMel Gorman node_page_state(pgdat, NR_INACTIVE_FILE)), 406599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)), 407599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)), 408599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)), 409599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)), 410599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_UNEVICTABLE)), 41175ef7184SMel Gorman nid, K(sum_zone_node_page_state(nid, NR_MLOCK))); 4127ee92255SKOSAKI Motohiro 413182e8e23SChristoph Lameter #ifdef CONFIG_HIGHMEM 414948b3edbSJoe Perches len += sysfs_emit_at(buf, len, 4151da177e4SLinus Torvalds "Node %d HighTotal: %8lu kB\n" 4161da177e4SLinus Torvalds "Node %d HighFree: %8lu kB\n" 4171da177e4SLinus Torvalds "Node %d LowTotal: %8lu kB\n" 4187ee92255SKOSAKI Motohiro "Node %d LowFree: %8lu kB\n", 4197ee92255SKOSAKI Motohiro nid, K(i.totalhigh), 4207ee92255SKOSAKI Motohiro nid, K(i.freehigh), 4217ee92255SKOSAKI Motohiro nid, K(i.totalram - i.totalhigh), 4227ee92255SKOSAKI Motohiro nid, K(i.freeram - i.freehigh)); 423182e8e23SChristoph Lameter #endif 424948b3edbSJoe Perches len += sysfs_emit_at(buf, len, 425c07e02dbSMartin Hicks "Node %d Dirty: %8lu kB\n" 426c07e02dbSMartin Hicks "Node %d Writeback: %8lu kB\n" 427347ce434SChristoph Lameter "Node %d FilePages: %8lu kB\n" 428c07e02dbSMartin Hicks "Node %d Mapped: %8lu kB\n" 429f3dbd344SChristoph Lameter "Node %d AnonPages: %8lu kB\n" 4304b02108aSKOSAKI Motohiro "Node %d Shmem: %8lu kB\n" 431c6a7f572SKOSAKI Motohiro "Node %d KernelStack: %8lu kB\n" 432628d06a4SSami Tolvanen #ifdef CONFIG_SHADOW_CALL_STACK 433628d06a4SSami Tolvanen "Node %d ShadowCallStack:%8lu kB\n" 434628d06a4SSami Tolvanen #endif 435df849a15SChristoph Lameter "Node %d PageTables: %8lu kB\n" 436f5ef68daSAndrew Morton "Node %d NFS_Unstable: %8lu kB\n" 437d2c5e30cSChristoph Lameter "Node %d Bounce: %8lu kB\n" 438fc3ba692SMiklos Szeredi "Node %d WritebackTmp: %8lu kB\n" 43961f94e18SVlastimil Babka "Node %d KReclaimable: %8lu kB\n" 440972d1a7bSChristoph Lameter "Node %d Slab: %8lu kB\n" 441972d1a7bSChristoph Lameter "Node %d SReclaimable: %8lu kB\n" 44205b258e9SDavid Rientjes "Node %d SUnreclaim: %8lu kB\n" 44305b258e9SDavid Rientjes #ifdef CONFIG_TRANSPARENT_HUGEPAGE 44405b258e9SDavid Rientjes "Node %d AnonHugePages: %8lu kB\n" 44565c45377SKirill A. Shutemov "Node %d ShmemHugePages: %8lu kB\n" 44665c45377SKirill A. Shutemov "Node %d ShmemPmdMapped: %8lu kB\n" 44760fbf0abSSong Liu "Node %d FileHugePages: %8lu kB\n" 44860fbf0abSSong Liu "Node %d FilePmdMapped: %8lu kB\n" 44905b258e9SDavid Rientjes #endif 45005b258e9SDavid Rientjes , 45111fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_FILE_DIRTY)), 45211fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_WRITEBACK)), 45311fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_FILE_PAGES)), 45450658e2eSMel Gorman nid, K(node_page_state(pgdat, NR_FILE_MAPPED)), 4554b9d0fabSMel Gorman nid, K(node_page_state(pgdat, NR_ANON_MAPPED)), 456cc7452b6SRafael Aquini nid, K(i.sharedram), 457991e7673SShakeel Butt nid, node_page_state(pgdat, NR_KERNEL_STACK_KB), 458628d06a4SSami Tolvanen #ifdef CONFIG_SHADOW_CALL_STACK 459991e7673SShakeel Butt nid, node_page_state(pgdat, NR_KERNEL_SCS_KB), 460628d06a4SSami Tolvanen #endif 461f0c0c115SShakeel Butt nid, K(node_page_state(pgdat, NR_PAGETABLE)), 4628d92890bSNeilBrown nid, 0UL, 46375ef7184SMel Gorman nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)), 46411fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 46561f94e18SVlastimil Babka nid, K(sreclaimable + 46661f94e18SVlastimil Babka node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)), 46761f94e18SVlastimil Babka nid, K(sreclaimable + sunreclaimable), 46861f94e18SVlastimil Babka nid, K(sreclaimable), 46961f94e18SVlastimil Babka nid, K(sunreclaimable) 47005b258e9SDavid Rientjes #ifdef CONFIG_TRANSPARENT_HUGEPAGE 47161f94e18SVlastimil Babka , 47269473e5dSMuchun Song nid, K(node_page_state(pgdat, NR_ANON_THPS)), 47357b2847dSMuchun Song nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), 474a1528e21SMuchun Song nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 475bf9eceadSMuchun Song nid, K(node_page_state(pgdat, NR_FILE_THPS)), 476380780e7SMuchun Song nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) 47705b258e9SDavid Rientjes #endif 47861f94e18SVlastimil Babka ); 4797981593bSJoe Perches len += hugetlb_report_node_meminfo(buf, len, nid); 480948b3edbSJoe Perches return len; 4811da177e4SLinus Torvalds } 4821da177e4SLinus Torvalds 4831da177e4SLinus Torvalds #undef K 484948b3edbSJoe Perches static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL); 4851da177e4SLinus Torvalds 48610fbcf4cSKay Sievers static ssize_t node_read_numastat(struct device *dev, 48710fbcf4cSKay Sievers struct device_attribute *attr, char *buf) 4881da177e4SLinus Torvalds { 489f19298b9SMel Gorman fold_vm_numa_events(); 490aa838896SJoe Perches return sysfs_emit(buf, 4911da177e4SLinus Torvalds "numa_hit %lu\n" 4921da177e4SLinus Torvalds "numa_miss %lu\n" 4931da177e4SLinus Torvalds "numa_foreign %lu\n" 4941da177e4SLinus Torvalds "interleave_hit %lu\n" 4951da177e4SLinus Torvalds "local_node %lu\n" 4961da177e4SLinus Torvalds "other_node %lu\n", 497f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_HIT), 498f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_MISS), 499f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_FOREIGN), 500f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT), 501f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_LOCAL), 502f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_OTHER)); 5031da177e4SLinus Torvalds } 504948b3edbSJoe Perches static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL); 5051da177e4SLinus Torvalds 50610fbcf4cSKay Sievers static ssize_t node_read_vmstat(struct device *dev, 50710fbcf4cSKay Sievers struct device_attribute *attr, char *buf) 5082ac39037SMichael Rubin { 5092ac39037SMichael Rubin int nid = dev->id; 51075ef7184SMel Gorman struct pglist_data *pgdat = NODE_DATA(nid); 511fa25c503SKOSAKI Motohiro int i; 512948b3edbSJoe Perches int len = 0; 513fa25c503SKOSAKI Motohiro 514fa25c503SKOSAKI Motohiro for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 515948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s %lu\n", 516948b3edbSJoe Perches zone_stat_name(i), 51775ef7184SMel Gorman sum_zone_node_page_state(nid, i)); 51875ef7184SMel Gorman 5193a321d2aSKemi Wang #ifdef CONFIG_NUMA 520f19298b9SMel Gorman fold_vm_numa_events(); 521f19298b9SMel Gorman for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 522948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s %lu\n", 523948b3edbSJoe Perches numa_stat_name(i), 524f19298b9SMel Gorman sum_zone_numa_event_state(nid, i)); 5253a321d2aSKemi Wang 526948b3edbSJoe Perches #endif 52769473e5dSMuchun Song for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 52869473e5dSMuchun Song unsigned long pages = node_page_state_pages(pgdat, i); 52969473e5dSMuchun Song 53069473e5dSMuchun Song if (vmstat_item_print_in_thp(i)) 53169473e5dSMuchun Song pages /= HPAGE_PMD_NR; 53269473e5dSMuchun Song len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), 53369473e5dSMuchun Song pages); 53469473e5dSMuchun Song } 535fa25c503SKOSAKI Motohiro 536948b3edbSJoe Perches return len; 5372ac39037SMichael Rubin } 538948b3edbSJoe Perches static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL); 5392ac39037SMichael Rubin 54010fbcf4cSKay Sievers static ssize_t node_read_distance(struct device *dev, 54110fbcf4cSKay Sievers struct device_attribute *attr, char *buf) 5421da177e4SLinus Torvalds { 5431da177e4SLinus Torvalds int nid = dev->id; 5441da177e4SLinus Torvalds int len = 0; 5451da177e4SLinus Torvalds int i; 5461da177e4SLinus Torvalds 54712ee3c0aSDavid Rientjes /* 54812ee3c0aSDavid Rientjes * buf is currently PAGE_SIZE in length and each node needs 4 chars 54912ee3c0aSDavid Rientjes * at the most (distance + space or newline). 55012ee3c0aSDavid Rientjes */ 55112ee3c0aSDavid Rientjes BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE); 5521da177e4SLinus Torvalds 553948b3edbSJoe Perches for_each_online_node(i) { 554948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s%d", 555948b3edbSJoe Perches i ? " " : "", node_distance(nid, i)); 556948b3edbSJoe Perches } 5571da177e4SLinus Torvalds 558948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "\n"); 5591da177e4SLinus Torvalds return len; 5601da177e4SLinus Torvalds } 561948b3edbSJoe Perches static DEVICE_ATTR(distance, 0444, node_read_distance, NULL); 5621da177e4SLinus Torvalds 5633c9b8aafSTakashi Iwai static struct attribute *node_dev_attrs[] = { 5643c9b8aafSTakashi Iwai &dev_attr_meminfo.attr, 5653c9b8aafSTakashi Iwai &dev_attr_numastat.attr, 5663c9b8aafSTakashi Iwai &dev_attr_distance.attr, 5673c9b8aafSTakashi Iwai &dev_attr_vmstat.attr, 5683c9b8aafSTakashi Iwai NULL 5693c9b8aafSTakashi Iwai }; 57075bd50faSTian Tao 57175bd50faSTian Tao static struct bin_attribute *node_dev_bin_attrs[] = { 57275bd50faSTian Tao &bin_attr_cpumap, 57375bd50faSTian Tao &bin_attr_cpulist, 57475bd50faSTian Tao NULL 57575bd50faSTian Tao }; 57675bd50faSTian Tao 57775bd50faSTian Tao static const struct attribute_group node_dev_group = { 57875bd50faSTian Tao .attrs = node_dev_attrs, 57975bd50faSTian Tao .bin_attrs = node_dev_bin_attrs 58075bd50faSTian Tao }; 58175bd50faSTian Tao 58275bd50faSTian Tao static const struct attribute_group *node_dev_groups[] = { 58375bd50faSTian Tao &node_dev_group, 584*50468e43SJarkko Sakkinen #ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP 585*50468e43SJarkko Sakkinen &arch_node_dev_group, 586*50468e43SJarkko Sakkinen #endif 58775bd50faSTian Tao NULL 58875bd50faSTian Tao }; 5893c9b8aafSTakashi Iwai 5909a305230SLee Schermerhorn #ifdef CONFIG_HUGETLBFS 5919a305230SLee Schermerhorn /* 5929a305230SLee Schermerhorn * hugetlbfs per node attributes registration interface: 5939a305230SLee Schermerhorn * When/if hugetlb[fs] subsystem initializes [sometime after this module], 5944faf8d95SLee Schermerhorn * it will register its per node attributes for all online nodes with 5954faf8d95SLee Schermerhorn * memory. It will also call register_hugetlbfs_with_node(), below, to 5969a305230SLee Schermerhorn * register its attribute registration functions with this node driver. 5979a305230SLee Schermerhorn * Once these hooks have been initialized, the node driver will call into 5989a305230SLee Schermerhorn * the hugetlb module to [un]register attributes for hot-plugged nodes. 5999a305230SLee Schermerhorn */ 6009a305230SLee Schermerhorn static node_registration_func_t __hugetlb_register_node; 6019a305230SLee Schermerhorn static node_registration_func_t __hugetlb_unregister_node; 6029a305230SLee Schermerhorn 60339da08cbSLee Schermerhorn static inline bool hugetlb_register_node(struct node *node) 6049a305230SLee Schermerhorn { 6054faf8d95SLee Schermerhorn if (__hugetlb_register_node && 6068cebfcd0SLai Jiangshan node_state(node->dev.id, N_MEMORY)) { 6079a305230SLee Schermerhorn __hugetlb_register_node(node); 60839da08cbSLee Schermerhorn return true; 60939da08cbSLee Schermerhorn } 61039da08cbSLee Schermerhorn return false; 6119a305230SLee Schermerhorn } 6129a305230SLee Schermerhorn 6139a305230SLee Schermerhorn static inline void hugetlb_unregister_node(struct node *node) 6149a305230SLee Schermerhorn { 6159a305230SLee Schermerhorn if (__hugetlb_unregister_node) 6169a305230SLee Schermerhorn __hugetlb_unregister_node(node); 6179a305230SLee Schermerhorn } 6189a305230SLee Schermerhorn 6199a305230SLee Schermerhorn void register_hugetlbfs_with_node(node_registration_func_t doregister, 6209a305230SLee Schermerhorn node_registration_func_t unregister) 6219a305230SLee Schermerhorn { 6229a305230SLee Schermerhorn __hugetlb_register_node = doregister; 6239a305230SLee Schermerhorn __hugetlb_unregister_node = unregister; 6249a305230SLee Schermerhorn } 6259a305230SLee Schermerhorn #else 6269a305230SLee Schermerhorn static inline void hugetlb_register_node(struct node *node) {} 6279a305230SLee Schermerhorn 6289a305230SLee Schermerhorn static inline void hugetlb_unregister_node(struct node *node) {} 6299a305230SLee Schermerhorn #endif 6309a305230SLee Schermerhorn 6318c7b5b4eSYasuaki Ishimatsu static void node_device_release(struct device *dev) 6328c7b5b4eSYasuaki Ishimatsu { 6338c7b5b4eSYasuaki Ishimatsu struct node *node = to_node(dev); 6348c7b5b4eSYasuaki Ishimatsu 63550f9481eSDavid Hildenbrand #if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS) 6368c7b5b4eSYasuaki Ishimatsu /* 6378c7b5b4eSYasuaki Ishimatsu * We schedule the work only when a memory section is 6388c7b5b4eSYasuaki Ishimatsu * onlined/offlined on this node. When we come here, 6398c7b5b4eSYasuaki Ishimatsu * all the memory on this node has been offlined, 6408c7b5b4eSYasuaki Ishimatsu * so we won't enqueue new work to this work. 6418c7b5b4eSYasuaki Ishimatsu * 6428c7b5b4eSYasuaki Ishimatsu * The work is using node->node_work, so we should 6438c7b5b4eSYasuaki Ishimatsu * flush work before freeing the memory. 6448c7b5b4eSYasuaki Ishimatsu */ 6458c7b5b4eSYasuaki Ishimatsu flush_work(&node->node_work); 6468c7b5b4eSYasuaki Ishimatsu #endif 6478c7b5b4eSYasuaki Ishimatsu kfree(node); 6488c7b5b4eSYasuaki Ishimatsu } 6491da177e4SLinus Torvalds 6501da177e4SLinus Torvalds /* 651405ae7d3SRobert P. J. Day * register_node - Setup a sysfs device for a node. 6521da177e4SLinus Torvalds * @num - Node number to use when creating the device. 6531da177e4SLinus Torvalds * 6541da177e4SLinus Torvalds * Initialize and register the node device. 6551da177e4SLinus Torvalds */ 656a7be6e5aSDou Liyang static int register_node(struct node *node, int num) 6571da177e4SLinus Torvalds { 6581da177e4SLinus Torvalds int error; 6591da177e4SLinus Torvalds 66010fbcf4cSKay Sievers node->dev.id = num; 66110fbcf4cSKay Sievers node->dev.bus = &node_subsys; 6628c7b5b4eSYasuaki Ishimatsu node->dev.release = node_device_release; 6637ca7ec40SGreg Kroah-Hartman node->dev.groups = node_dev_groups; 66410fbcf4cSKay Sievers error = device_register(&node->dev); 6651da177e4SLinus Torvalds 666c1cc0d51SArvind Yadav if (error) 667c1cc0d51SArvind Yadav put_device(&node->dev); 668c1cc0d51SArvind Yadav else { 6699a305230SLee Schermerhorn hugetlb_register_node(node); 670ed4a6d7fSMel Gorman 671ed4a6d7fSMel Gorman compaction_register_node(node); 6721da177e4SLinus Torvalds } 6731da177e4SLinus Torvalds return error; 6741da177e4SLinus Torvalds } 6751da177e4SLinus Torvalds 6764b45099bSKeiichiro Tokunaga /** 6774b45099bSKeiichiro Tokunaga * unregister_node - unregister a node device 6784b45099bSKeiichiro Tokunaga * @node: node going away 6794b45099bSKeiichiro Tokunaga * 6804b45099bSKeiichiro Tokunaga * Unregisters a node device @node. All the devices on the node must be 6814b45099bSKeiichiro Tokunaga * unregistered before calling this function. 6824b45099bSKeiichiro Tokunaga */ 6834b45099bSKeiichiro Tokunaga void unregister_node(struct node *node) 6844b45099bSKeiichiro Tokunaga { 6854faf8d95SLee Schermerhorn hugetlb_unregister_node(node); /* no-op, if memoryless node */ 68608d9dbe7SKeith Busch node_remove_accesses(node); 687acc02a10SKeith Busch node_remove_caches(node); 68810fbcf4cSKay Sievers device_unregister(&node->dev); 6894b45099bSKeiichiro Tokunaga } 6904b45099bSKeiichiro Tokunaga 6918732794bSWen Congyang struct node *node_devices[MAX_NUMNODES]; 6920fc44159SYasunori Goto 69376b67ed9SKAMEZAWA Hiroyuki /* 69476b67ed9SKAMEZAWA Hiroyuki * register cpu under node 69576b67ed9SKAMEZAWA Hiroyuki */ 69676b67ed9SKAMEZAWA Hiroyuki int register_cpu_under_node(unsigned int cpu, unsigned int nid) 69776b67ed9SKAMEZAWA Hiroyuki { 6981830794aSAlex Chiang int ret; 6998a25a2fdSKay Sievers struct device *obj; 700f8246f31SAlex Chiang 701f8246f31SAlex Chiang if (!node_online(nid)) 702f8246f31SAlex Chiang return 0; 703f8246f31SAlex Chiang 7048a25a2fdSKay Sievers obj = get_cpu_device(cpu); 70576b67ed9SKAMEZAWA Hiroyuki if (!obj) 70676b67ed9SKAMEZAWA Hiroyuki return 0; 707f8246f31SAlex Chiang 7088732794bSWen Congyang ret = sysfs_create_link(&node_devices[nid]->dev.kobj, 70976b67ed9SKAMEZAWA Hiroyuki &obj->kobj, 71076b67ed9SKAMEZAWA Hiroyuki kobject_name(&obj->kobj)); 7111830794aSAlex Chiang if (ret) 7121830794aSAlex Chiang return ret; 7131830794aSAlex Chiang 7141830794aSAlex Chiang return sysfs_create_link(&obj->kobj, 7158732794bSWen Congyang &node_devices[nid]->dev.kobj, 7168732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj)); 71776b67ed9SKAMEZAWA Hiroyuki } 71876b67ed9SKAMEZAWA Hiroyuki 71908d9dbe7SKeith Busch /** 72008d9dbe7SKeith Busch * register_memory_node_under_compute_node - link memory node to its compute 72108d9dbe7SKeith Busch * node for a given access class. 72258cb346cSMauro Carvalho Chehab * @mem_nid: Memory node number 72358cb346cSMauro Carvalho Chehab * @cpu_nid: Cpu node number 72408d9dbe7SKeith Busch * @access: Access class to register 72508d9dbe7SKeith Busch * 72608d9dbe7SKeith Busch * Description: 72708d9dbe7SKeith Busch * For use with platforms that may have separate memory and compute nodes. 72808d9dbe7SKeith Busch * This function will export node relationships linking which memory 72908d9dbe7SKeith Busch * initiator nodes can access memory targets at a given ranked access 73008d9dbe7SKeith Busch * class. 73108d9dbe7SKeith Busch */ 73208d9dbe7SKeith Busch int register_memory_node_under_compute_node(unsigned int mem_nid, 73308d9dbe7SKeith Busch unsigned int cpu_nid, 734e7deeb9dSJinchao Wang unsigned int access) 73508d9dbe7SKeith Busch { 73608d9dbe7SKeith Busch struct node *init_node, *targ_node; 73708d9dbe7SKeith Busch struct node_access_nodes *initiator, *target; 73808d9dbe7SKeith Busch int ret; 73908d9dbe7SKeith Busch 74008d9dbe7SKeith Busch if (!node_online(cpu_nid) || !node_online(mem_nid)) 74108d9dbe7SKeith Busch return -ENODEV; 74208d9dbe7SKeith Busch 74308d9dbe7SKeith Busch init_node = node_devices[cpu_nid]; 74408d9dbe7SKeith Busch targ_node = node_devices[mem_nid]; 74508d9dbe7SKeith Busch initiator = node_init_node_access(init_node, access); 74608d9dbe7SKeith Busch target = node_init_node_access(targ_node, access); 74708d9dbe7SKeith Busch if (!initiator || !target) 74808d9dbe7SKeith Busch return -ENOMEM; 74908d9dbe7SKeith Busch 75008d9dbe7SKeith Busch ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets", 75108d9dbe7SKeith Busch &targ_node->dev.kobj, 75208d9dbe7SKeith Busch dev_name(&targ_node->dev)); 75308d9dbe7SKeith Busch if (ret) 75408d9dbe7SKeith Busch return ret; 75508d9dbe7SKeith Busch 75608d9dbe7SKeith Busch ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators", 75708d9dbe7SKeith Busch &init_node->dev.kobj, 75808d9dbe7SKeith Busch dev_name(&init_node->dev)); 75908d9dbe7SKeith Busch if (ret) 76008d9dbe7SKeith Busch goto err; 76108d9dbe7SKeith Busch 76208d9dbe7SKeith Busch return 0; 76308d9dbe7SKeith Busch err: 76408d9dbe7SKeith Busch sysfs_remove_link_from_group(&initiator->dev.kobj, "targets", 76508d9dbe7SKeith Busch dev_name(&targ_node->dev)); 76608d9dbe7SKeith Busch return ret; 76708d9dbe7SKeith Busch } 76808d9dbe7SKeith Busch 76976b67ed9SKAMEZAWA Hiroyuki int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) 77076b67ed9SKAMEZAWA Hiroyuki { 7718a25a2fdSKay Sievers struct device *obj; 772b9d52dadSAlex Chiang 773b9d52dadSAlex Chiang if (!node_online(nid)) 774b9d52dadSAlex Chiang return 0; 775b9d52dadSAlex Chiang 7768a25a2fdSKay Sievers obj = get_cpu_device(cpu); 777b9d52dadSAlex Chiang if (!obj) 778b9d52dadSAlex Chiang return 0; 779b9d52dadSAlex Chiang 7808732794bSWen Congyang sysfs_remove_link(&node_devices[nid]->dev.kobj, 78176b67ed9SKAMEZAWA Hiroyuki kobject_name(&obj->kobj)); 7821830794aSAlex Chiang sysfs_remove_link(&obj->kobj, 7838732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj)); 784b9d52dadSAlex Chiang 78576b67ed9SKAMEZAWA Hiroyuki return 0; 78676b67ed9SKAMEZAWA Hiroyuki } 78776b67ed9SKAMEZAWA Hiroyuki 78850f9481eSDavid Hildenbrand #ifdef CONFIG_MEMORY_HOTPLUG 789bd721ea7SFabian Frederick static int __ref get_nid_for_pfn(unsigned long pfn) 790c04fc586SGary Hade { 7913a80a7faSMel Gorman #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7928cdde385SThomas Gleixner if (system_state < SYSTEM_RUNNING) 7933a80a7faSMel Gorman return early_pfn_to_nid(pfn); 7943a80a7faSMel Gorman #endif 795c04fc586SGary Hade return pfn_to_nid(pfn); 796c04fc586SGary Hade } 797c04fc586SGary Hade 79890c7eaebSLaurent Dufour static void do_register_memory_block_under_node(int nid, 799f85086f9SLaurent Dufour struct memory_block *mem_blk) 800c04fc586SGary Hade { 801f85086f9SLaurent Dufour int ret; 802d84f2f5aSDavid Hildenbrand 803d84f2f5aSDavid Hildenbrand /* 804d84f2f5aSDavid Hildenbrand * If this memory block spans multiple nodes, we only indicate 805d84f2f5aSDavid Hildenbrand * the last processed node. 806d84f2f5aSDavid Hildenbrand */ 807d84f2f5aSDavid Hildenbrand mem_blk->nid = nid; 808d84f2f5aSDavid Hildenbrand 8098732794bSWen Congyang ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, 81010fbcf4cSKay Sievers &mem_blk->dev.kobj, 81110fbcf4cSKay Sievers kobject_name(&mem_blk->dev.kobj)); 81290c7eaebSLaurent Dufour if (ret && ret != -EEXIST) 81390c7eaebSLaurent Dufour dev_err_ratelimited(&node_devices[nid]->dev, 81490c7eaebSLaurent Dufour "can't create link to %s in sysfs (%d)\n", 81590c7eaebSLaurent Dufour kobject_name(&mem_blk->dev.kobj), ret); 816dee5d0d5SAlex Chiang 81790c7eaebSLaurent Dufour ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj, 8188732794bSWen Congyang &node_devices[nid]->dev.kobj, 8198732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj)); 82090c7eaebSLaurent Dufour if (ret && ret != -EEXIST) 82190c7eaebSLaurent Dufour dev_err_ratelimited(&mem_blk->dev, 82290c7eaebSLaurent Dufour "can't create link to %s in sysfs (%d)\n", 82390c7eaebSLaurent Dufour kobject_name(&node_devices[nid]->dev.kobj), 82490c7eaebSLaurent Dufour ret); 825c04fc586SGary Hade } 826f85086f9SLaurent Dufour 827f85086f9SLaurent Dufour /* register memory section under specified node if it spans that node */ 828f85086f9SLaurent Dufour static int register_mem_block_under_node_early(struct memory_block *mem_blk, 829f85086f9SLaurent Dufour void *arg) 830f85086f9SLaurent Dufour { 831f85086f9SLaurent Dufour unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE; 832f85086f9SLaurent Dufour unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr); 833f85086f9SLaurent Dufour unsigned long end_pfn = start_pfn + memory_block_pfns - 1; 834f85086f9SLaurent Dufour int nid = *(int *)arg; 835f85086f9SLaurent Dufour unsigned long pfn; 836f85086f9SLaurent Dufour 837f85086f9SLaurent Dufour for (pfn = start_pfn; pfn <= end_pfn; pfn++) { 838f85086f9SLaurent Dufour int page_nid; 839f85086f9SLaurent Dufour 840f85086f9SLaurent Dufour /* 841f85086f9SLaurent Dufour * memory block could have several absent sections from start. 842f85086f9SLaurent Dufour * skip pfn range from absent section 843f85086f9SLaurent Dufour */ 844f85086f9SLaurent Dufour if (!pfn_in_present_section(pfn)) { 845f85086f9SLaurent Dufour pfn = round_down(pfn + PAGES_PER_SECTION, 846f85086f9SLaurent Dufour PAGES_PER_SECTION) - 1; 847f85086f9SLaurent Dufour continue; 848f85086f9SLaurent Dufour } 849f85086f9SLaurent Dufour 850f85086f9SLaurent Dufour /* 851f85086f9SLaurent Dufour * We need to check if page belongs to nid only at the boot 852f85086f9SLaurent Dufour * case because node's ranges can be interleaved. 853f85086f9SLaurent Dufour */ 854f85086f9SLaurent Dufour page_nid = get_nid_for_pfn(pfn); 855f85086f9SLaurent Dufour if (page_nid < 0) 856f85086f9SLaurent Dufour continue; 857f85086f9SLaurent Dufour if (page_nid != nid) 858f85086f9SLaurent Dufour continue; 859f85086f9SLaurent Dufour 86090c7eaebSLaurent Dufour do_register_memory_block_under_node(nid, mem_blk); 86190c7eaebSLaurent Dufour return 0; 862f85086f9SLaurent Dufour } 863c04fc586SGary Hade /* mem section does not span the specified node */ 864c04fc586SGary Hade return 0; 865c04fc586SGary Hade } 866c04fc586SGary Hade 8674c4b7f9bSDavid Hildenbrand /* 868f85086f9SLaurent Dufour * During hotplug we know that all pages in the memory block belong to the same 869f85086f9SLaurent Dufour * node. 870f85086f9SLaurent Dufour */ 871f85086f9SLaurent Dufour static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk, 872f85086f9SLaurent Dufour void *arg) 873f85086f9SLaurent Dufour { 874f85086f9SLaurent Dufour int nid = *(int *)arg; 875f85086f9SLaurent Dufour 87690c7eaebSLaurent Dufour do_register_memory_block_under_node(nid, mem_blk); 87790c7eaebSLaurent Dufour return 0; 878f85086f9SLaurent Dufour } 879f85086f9SLaurent Dufour 880f85086f9SLaurent Dufour /* 881d84f2f5aSDavid Hildenbrand * Unregister a memory block device under the node it spans. Memory blocks 882d84f2f5aSDavid Hildenbrand * with multiple nodes cannot be offlined and therefore also never be removed. 8834c4b7f9bSDavid Hildenbrand */ 884a31b264cSDavid Hildenbrand void unregister_memory_block_under_nodes(struct memory_block *mem_blk) 885c04fc586SGary Hade { 886d84f2f5aSDavid Hildenbrand if (mem_blk->nid == NUMA_NO_NODE) 887d84f2f5aSDavid Hildenbrand return; 888c04fc586SGary Hade 889d84f2f5aSDavid Hildenbrand sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj, 89010fbcf4cSKay Sievers kobject_name(&mem_blk->dev.kobj)); 89110fbcf4cSKay Sievers sysfs_remove_link(&mem_blk->dev.kobj, 892d84f2f5aSDavid Hildenbrand kobject_name(&node_devices[mem_blk->nid]->dev.kobj)); 893c04fc586SGary Hade } 894c04fc586SGary Hade 89590c7eaebSLaurent Dufour void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, 896f85086f9SLaurent Dufour enum meminit_context context) 897c04fc586SGary Hade { 898f85086f9SLaurent Dufour walk_memory_blocks_func_t func; 899f85086f9SLaurent Dufour 900f85086f9SLaurent Dufour if (context == MEMINIT_HOTPLUG) 901f85086f9SLaurent Dufour func = register_mem_block_under_node_hotplug; 902f85086f9SLaurent Dufour else 903f85086f9SLaurent Dufour func = register_mem_block_under_node_early; 904f85086f9SLaurent Dufour 90590c7eaebSLaurent Dufour walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), 90690c7eaebSLaurent Dufour (void *)&nid, func); 90790c7eaebSLaurent Dufour return; 908c04fc586SGary Hade } 9094faf8d95SLee Schermerhorn 91039da08cbSLee Schermerhorn #ifdef CONFIG_HUGETLBFS 9114faf8d95SLee Schermerhorn /* 9124faf8d95SLee Schermerhorn * Handle per node hstate attribute [un]registration on transistions 9134faf8d95SLee Schermerhorn * to/from memoryless state. 9144faf8d95SLee Schermerhorn */ 91539da08cbSLee Schermerhorn static void node_hugetlb_work(struct work_struct *work) 91639da08cbSLee Schermerhorn { 91739da08cbSLee Schermerhorn struct node *node = container_of(work, struct node, node_work); 91839da08cbSLee Schermerhorn 91939da08cbSLee Schermerhorn /* 92039da08cbSLee Schermerhorn * We only get here when a node transitions to/from memoryless state. 92139da08cbSLee Schermerhorn * We can detect which transition occurred by examining whether the 92239da08cbSLee Schermerhorn * node has memory now. hugetlb_register_node() already check this 92339da08cbSLee Schermerhorn * so we try to register the attributes. If that fails, then the 92439da08cbSLee Schermerhorn * node has transitioned to memoryless, try to unregister the 92539da08cbSLee Schermerhorn * attributes. 92639da08cbSLee Schermerhorn */ 92739da08cbSLee Schermerhorn if (!hugetlb_register_node(node)) 92839da08cbSLee Schermerhorn hugetlb_unregister_node(node); 92939da08cbSLee Schermerhorn } 93039da08cbSLee Schermerhorn 93139da08cbSLee Schermerhorn static void init_node_hugetlb_work(int nid) 93239da08cbSLee Schermerhorn { 9338732794bSWen Congyang INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work); 93439da08cbSLee Schermerhorn } 9354faf8d95SLee Schermerhorn 9364faf8d95SLee Schermerhorn static int node_memory_callback(struct notifier_block *self, 9374faf8d95SLee Schermerhorn unsigned long action, void *arg) 9384faf8d95SLee Schermerhorn { 9394faf8d95SLee Schermerhorn struct memory_notify *mnb = arg; 9404faf8d95SLee Schermerhorn int nid = mnb->status_change_nid; 9414faf8d95SLee Schermerhorn 9424faf8d95SLee Schermerhorn switch (action) { 94339da08cbSLee Schermerhorn case MEM_ONLINE: 94439da08cbSLee Schermerhorn case MEM_OFFLINE: 94539da08cbSLee Schermerhorn /* 94639da08cbSLee Schermerhorn * offload per node hstate [un]registration to a work thread 94739da08cbSLee Schermerhorn * when transitioning to/from memoryless state. 94839da08cbSLee Schermerhorn */ 9494faf8d95SLee Schermerhorn if (nid != NUMA_NO_NODE) 9508732794bSWen Congyang schedule_work(&node_devices[nid]->node_work); 9514faf8d95SLee Schermerhorn break; 95239da08cbSLee Schermerhorn 9534faf8d95SLee Schermerhorn case MEM_GOING_ONLINE: 9544faf8d95SLee Schermerhorn case MEM_GOING_OFFLINE: 9554faf8d95SLee Schermerhorn case MEM_CANCEL_ONLINE: 9564faf8d95SLee Schermerhorn case MEM_CANCEL_OFFLINE: 9574faf8d95SLee Schermerhorn default: 9584faf8d95SLee Schermerhorn break; 9594faf8d95SLee Schermerhorn } 9604faf8d95SLee Schermerhorn 9614faf8d95SLee Schermerhorn return NOTIFY_OK; 9624faf8d95SLee Schermerhorn } 96339da08cbSLee Schermerhorn #endif /* CONFIG_HUGETLBFS */ 96450f9481eSDavid Hildenbrand #endif /* CONFIG_MEMORY_HOTPLUG */ 96539da08cbSLee Schermerhorn 96650f9481eSDavid Hildenbrand #if !defined(CONFIG_MEMORY_HOTPLUG) || !defined(CONFIG_HUGETLBFS) 9674faf8d95SLee Schermerhorn static inline int node_memory_callback(struct notifier_block *self, 9684faf8d95SLee Schermerhorn unsigned long action, void *arg) 9694faf8d95SLee Schermerhorn { 9704faf8d95SLee Schermerhorn return NOTIFY_OK; 9714faf8d95SLee Schermerhorn } 97239da08cbSLee Schermerhorn 97339da08cbSLee Schermerhorn static void init_node_hugetlb_work(int nid) { } 97439da08cbSLee Schermerhorn 97539da08cbSLee Schermerhorn #endif 976c04fc586SGary Hade 9779037a993SMichal Hocko int __register_one_node(int nid) 9780fc44159SYasunori Goto { 9799037a993SMichal Hocko int error; 9809037a993SMichal Hocko int cpu; 9810fc44159SYasunori Goto 9828732794bSWen Congyang node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL); 9838732794bSWen Congyang if (!node_devices[nid]) 9848732794bSWen Congyang return -ENOMEM; 9858732794bSWen Congyang 986a7be6e5aSDou Liyang error = register_node(node_devices[nid], nid); 98776b67ed9SKAMEZAWA Hiroyuki 98876b67ed9SKAMEZAWA Hiroyuki /* link cpu under this node */ 98976b67ed9SKAMEZAWA Hiroyuki for_each_present_cpu(cpu) { 99076b67ed9SKAMEZAWA Hiroyuki if (cpu_to_node(cpu) == nid) 99176b67ed9SKAMEZAWA Hiroyuki register_cpu_under_node(cpu, nid); 99276b67ed9SKAMEZAWA Hiroyuki } 993c04fc586SGary Hade 99408d9dbe7SKeith Busch INIT_LIST_HEAD(&node_devices[nid]->access_list); 99539da08cbSLee Schermerhorn /* initialize work queue for memory hot plug */ 99639da08cbSLee Schermerhorn init_node_hugetlb_work(nid); 997acc02a10SKeith Busch node_init_caches(nid); 9980fc44159SYasunori Goto 9990fc44159SYasunori Goto return error; 10000fc44159SYasunori Goto } 10010fc44159SYasunori Goto 10020fc44159SYasunori Goto void unregister_one_node(int nid) 10030fc44159SYasunori Goto { 100492d585efSXishi Qiu if (!node_devices[nid]) 100592d585efSXishi Qiu return; 100692d585efSXishi Qiu 10078732794bSWen Congyang unregister_node(node_devices[nid]); 10088732794bSWen Congyang node_devices[nid] = NULL; 10090fc44159SYasunori Goto } 10100fc44159SYasunori Goto 1011bde631a5SLee Schermerhorn /* 1012bde631a5SLee Schermerhorn * node states attributes 1013bde631a5SLee Schermerhorn */ 1014bde631a5SLee Schermerhorn 1015b15f562fSAndi Kleen struct node_attr { 101610fbcf4cSKay Sievers struct device_attribute attr; 1017b15f562fSAndi Kleen enum node_states state; 1018b15f562fSAndi Kleen }; 1019b15f562fSAndi Kleen 102010fbcf4cSKay Sievers static ssize_t show_node_state(struct device *dev, 102110fbcf4cSKay Sievers struct device_attribute *attr, char *buf) 1022bde631a5SLee Schermerhorn { 1023b15f562fSAndi Kleen struct node_attr *na = container_of(attr, struct node_attr, attr); 1024948b3edbSJoe Perches 1025948b3edbSJoe Perches return sysfs_emit(buf, "%*pbl\n", 1026948b3edbSJoe Perches nodemask_pr_args(&node_states[na->state])); 1027bde631a5SLee Schermerhorn } 1028bde631a5SLee Schermerhorn 1029b15f562fSAndi Kleen #define _NODE_ATTR(name, state) \ 103010fbcf4cSKay Sievers { __ATTR(name, 0444, show_node_state, NULL), state } 1031bde631a5SLee Schermerhorn 1032b15f562fSAndi Kleen static struct node_attr node_state_attr[] = { 1033fcf07d22SLai Jiangshan [N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE), 1034fcf07d22SLai Jiangshan [N_ONLINE] = _NODE_ATTR(online, N_ONLINE), 1035fcf07d22SLai Jiangshan [N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY), 1036bde631a5SLee Schermerhorn #ifdef CONFIG_HIGHMEM 1037fcf07d22SLai Jiangshan [N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY), 1038bde631a5SLee Schermerhorn #endif 103920b2f52bSLai Jiangshan [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY), 1040fcf07d22SLai Jiangshan [N_CPU] = _NODE_ATTR(has_cpu, N_CPU), 1041894c26a1SJonathan Cameron [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator, 1042894c26a1SJonathan Cameron N_GENERIC_INITIATOR), 1043bde631a5SLee Schermerhorn }; 1044bde631a5SLee Schermerhorn 104510fbcf4cSKay Sievers static struct attribute *node_state_attrs[] = { 1046fcf07d22SLai Jiangshan &node_state_attr[N_POSSIBLE].attr.attr, 1047fcf07d22SLai Jiangshan &node_state_attr[N_ONLINE].attr.attr, 1048fcf07d22SLai Jiangshan &node_state_attr[N_NORMAL_MEMORY].attr.attr, 10493701cde6SAndi Kleen #ifdef CONFIG_HIGHMEM 1050fcf07d22SLai Jiangshan &node_state_attr[N_HIGH_MEMORY].attr.attr, 10513701cde6SAndi Kleen #endif 105220b2f52bSLai Jiangshan &node_state_attr[N_MEMORY].attr.attr, 1053fcf07d22SLai Jiangshan &node_state_attr[N_CPU].attr.attr, 1054894c26a1SJonathan Cameron &node_state_attr[N_GENERIC_INITIATOR].attr.attr, 10553701cde6SAndi Kleen NULL 10563701cde6SAndi Kleen }; 1057bde631a5SLee Schermerhorn 10585a576764SRikard Falkeborn static const struct attribute_group memory_root_attr_group = { 105910fbcf4cSKay Sievers .attrs = node_state_attrs, 106010fbcf4cSKay Sievers }; 106110fbcf4cSKay Sievers 106210fbcf4cSKay Sievers static const struct attribute_group *cpu_root_attr_groups[] = { 106310fbcf4cSKay Sievers &memory_root_attr_group, 106410fbcf4cSKay Sievers NULL, 106510fbcf4cSKay Sievers }; 106610fbcf4cSKay Sievers 10674faf8d95SLee Schermerhorn #define NODE_CALLBACK_PRI 2 /* lower than SLAB */ 10684b45099bSKeiichiro Tokunaga static int __init register_node_type(void) 10691da177e4SLinus Torvalds { 1070bde631a5SLee Schermerhorn int ret; 1071bde631a5SLee Schermerhorn 10723701cde6SAndi Kleen BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES); 10733701cde6SAndi Kleen BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES); 10743701cde6SAndi Kleen 107510fbcf4cSKay Sievers ret = subsys_system_register(&node_subsys, cpu_root_attr_groups); 10764faf8d95SLee Schermerhorn if (!ret) { 10776e259e7dSAndrew Morton static struct notifier_block node_memory_callback_nb = { 10786e259e7dSAndrew Morton .notifier_call = node_memory_callback, 10796e259e7dSAndrew Morton .priority = NODE_CALLBACK_PRI, 10806e259e7dSAndrew Morton }; 10816e259e7dSAndrew Morton register_hotmemory_notifier(&node_memory_callback_nb); 10824faf8d95SLee Schermerhorn } 1083bde631a5SLee Schermerhorn 1084bde631a5SLee Schermerhorn /* 1085bde631a5SLee Schermerhorn * Note: we're not going to unregister the node class if we fail 1086bde631a5SLee Schermerhorn * to register the node state class attribute files. 1087bde631a5SLee Schermerhorn */ 1088bde631a5SLee Schermerhorn return ret; 10891da177e4SLinus Torvalds } 10901da177e4SLinus Torvalds postcore_initcall(register_node_type); 1091