1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds /*
310fbcf4cSKay Sievers * Basic Node interface support
41da177e4SLinus Torvalds */
51da177e4SLinus Torvalds
61da177e4SLinus Torvalds #include <linux/module.h>
71da177e4SLinus Torvalds #include <linux/init.h>
81da177e4SLinus Torvalds #include <linux/mm.h>
9c04fc586SGary Hade #include <linux/memory.h>
10fa25c503SKOSAKI Motohiro #include <linux/vmstat.h>
116e259e7dSAndrew Morton #include <linux/notifier.h>
121da177e4SLinus Torvalds #include <linux/node.h>
131da177e4SLinus Torvalds #include <linux/hugetlb.h>
14ed4a6d7fSMel Gorman #include <linux/compaction.h>
151da177e4SLinus Torvalds #include <linux/cpumask.h>
161da177e4SLinus Torvalds #include <linux/topology.h>
171da177e4SLinus Torvalds #include <linux/nodemask.h>
1876b67ed9SKAMEZAWA Hiroyuki #include <linux/cpu.h>
19bde631a5SLee Schermerhorn #include <linux/device.h>
2008d9dbe7SKeith Busch #include <linux/pm_runtime.h>
21af936a16SLee Schermerhorn #include <linux/swap.h>
2218e5b539STejun Heo #include <linux/slab.h>
231da177e4SLinus Torvalds
2410fbcf4cSKay Sievers static struct bus_type node_subsys = {
25af5ca3f4SKay Sievers .name = "node",
2610fbcf4cSKay Sievers .dev_name = "node",
271da177e4SLinus Torvalds };
281da177e4SLinus Torvalds
cpumap_read(struct file * file,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t off,size_t count)2975bd50faSTian Tao static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj,
3075bd50faSTian Tao struct bin_attribute *attr, char *buf,
3175bd50faSTian Tao loff_t off, size_t count)
321da177e4SLinus Torvalds {
3375bd50faSTian Tao struct device *dev = kobj_to_dev(kobj);
341da177e4SLinus Torvalds struct node *node_dev = to_node(dev);
3575bd50faSTian Tao cpumask_var_t mask;
3675bd50faSTian Tao ssize_t n;
371da177e4SLinus Torvalds
38064f0e93SZhen Lei if (!alloc_cpumask_var(&mask, GFP_KERNEL))
39064f0e93SZhen Lei return 0;
40064f0e93SZhen Lei
41064f0e93SZhen Lei cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
4275bd50faSTian Tao n = cpumap_print_bitmask_to_buf(buf, mask, off, count);
43064f0e93SZhen Lei free_cpumask_var(mask);
44064f0e93SZhen Lei
45064f0e93SZhen Lei return n;
461da177e4SLinus Torvalds }
471da177e4SLinus Torvalds
487ee951acSPhil Auld static BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES);
4975bd50faSTian Tao
cpulist_read(struct file * file,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t off,size_t count)5075bd50faSTian Tao static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj,
5175bd50faSTian Tao struct bin_attribute *attr, char *buf,
5275bd50faSTian Tao loff_t off, size_t count)
5339106dcfSMike Travis {
5475bd50faSTian Tao struct device *dev = kobj_to_dev(kobj);
5575bd50faSTian Tao struct node *node_dev = to_node(dev);
5675bd50faSTian Tao cpumask_var_t mask;
5775bd50faSTian Tao ssize_t n;
5875bd50faSTian Tao
5975bd50faSTian Tao if (!alloc_cpumask_var(&mask, GFP_KERNEL))
6075bd50faSTian Tao return 0;
6175bd50faSTian Tao
6275bd50faSTian Tao cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
6375bd50faSTian Tao n = cpumap_print_list_to_buf(buf, mask, off, count);
6475bd50faSTian Tao free_cpumask_var(mask);
6575bd50faSTian Tao
6675bd50faSTian Tao return n;
6739106dcfSMike Travis }
68948b3edbSJoe Perches
697ee951acSPhil Auld static BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES);
701da177e4SLinus Torvalds
7108d9dbe7SKeith Busch /**
7208d9dbe7SKeith Busch * struct node_access_nodes - Access class device to hold user visible
7308d9dbe7SKeith Busch * relationships to other nodes.
7408d9dbe7SKeith Busch * @dev: Device for this memory access class
7508d9dbe7SKeith Busch * @list_node: List element in the node's access list
7608d9dbe7SKeith Busch * @access: The access class rank
7758cb346cSMauro Carvalho Chehab * @hmem_attrs: Heterogeneous memory performance attributes
7808d9dbe7SKeith Busch */
7908d9dbe7SKeith Busch struct node_access_nodes {
8008d9dbe7SKeith Busch struct device dev;
8108d9dbe7SKeith Busch struct list_head list_node;
82e7deeb9dSJinchao Wang unsigned int access;
83e1cf33aaSKeith Busch #ifdef CONFIG_HMEM_REPORTING
84e1cf33aaSKeith Busch struct node_hmem_attrs hmem_attrs;
85e1cf33aaSKeith Busch #endif
8608d9dbe7SKeith Busch };
8708d9dbe7SKeith Busch #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
8808d9dbe7SKeith Busch
8908d9dbe7SKeith Busch static struct attribute *node_init_access_node_attrs[] = {
9008d9dbe7SKeith Busch NULL,
9108d9dbe7SKeith Busch };
9208d9dbe7SKeith Busch
9308d9dbe7SKeith Busch static struct attribute *node_targ_access_node_attrs[] = {
9408d9dbe7SKeith Busch NULL,
9508d9dbe7SKeith Busch };
9608d9dbe7SKeith Busch
9708d9dbe7SKeith Busch static const struct attribute_group initiators = {
9808d9dbe7SKeith Busch .name = "initiators",
9908d9dbe7SKeith Busch .attrs = node_init_access_node_attrs,
10008d9dbe7SKeith Busch };
10108d9dbe7SKeith Busch
10208d9dbe7SKeith Busch static const struct attribute_group targets = {
10308d9dbe7SKeith Busch .name = "targets",
10408d9dbe7SKeith Busch .attrs = node_targ_access_node_attrs,
10508d9dbe7SKeith Busch };
10608d9dbe7SKeith Busch
10708d9dbe7SKeith Busch static const struct attribute_group *node_access_node_groups[] = {
10808d9dbe7SKeith Busch &initiators,
10908d9dbe7SKeith Busch &targets,
11008d9dbe7SKeith Busch NULL,
11108d9dbe7SKeith Busch };
11208d9dbe7SKeith Busch
node_remove_accesses(struct node * node)11308d9dbe7SKeith Busch static void node_remove_accesses(struct node *node)
11408d9dbe7SKeith Busch {
11508d9dbe7SKeith Busch struct node_access_nodes *c, *cnext;
11608d9dbe7SKeith Busch
11708d9dbe7SKeith Busch list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
11808d9dbe7SKeith Busch list_del(&c->list_node);
11908d9dbe7SKeith Busch device_unregister(&c->dev);
12008d9dbe7SKeith Busch }
12108d9dbe7SKeith Busch }
12208d9dbe7SKeith Busch
node_access_release(struct device * dev)12308d9dbe7SKeith Busch static void node_access_release(struct device *dev)
12408d9dbe7SKeith Busch {
12508d9dbe7SKeith Busch kfree(to_access_nodes(dev));
12608d9dbe7SKeith Busch }
12708d9dbe7SKeith Busch
node_init_node_access(struct node * node,unsigned int access)12808d9dbe7SKeith Busch static struct node_access_nodes *node_init_node_access(struct node *node,
129e7deeb9dSJinchao Wang unsigned int access)
13008d9dbe7SKeith Busch {
13108d9dbe7SKeith Busch struct node_access_nodes *access_node;
13208d9dbe7SKeith Busch struct device *dev;
13308d9dbe7SKeith Busch
13408d9dbe7SKeith Busch list_for_each_entry(access_node, &node->access_list, list_node)
13508d9dbe7SKeith Busch if (access_node->access == access)
13608d9dbe7SKeith Busch return access_node;
13708d9dbe7SKeith Busch
13808d9dbe7SKeith Busch access_node = kzalloc(sizeof(*access_node), GFP_KERNEL);
13908d9dbe7SKeith Busch if (!access_node)
14008d9dbe7SKeith Busch return NULL;
14108d9dbe7SKeith Busch
14208d9dbe7SKeith Busch access_node->access = access;
14308d9dbe7SKeith Busch dev = &access_node->dev;
14408d9dbe7SKeith Busch dev->parent = &node->dev;
14508d9dbe7SKeith Busch dev->release = node_access_release;
14608d9dbe7SKeith Busch dev->groups = node_access_node_groups;
14708d9dbe7SKeith Busch if (dev_set_name(dev, "access%u", access))
14808d9dbe7SKeith Busch goto free;
14908d9dbe7SKeith Busch
15008d9dbe7SKeith Busch if (device_register(dev))
15108d9dbe7SKeith Busch goto free_name;
15208d9dbe7SKeith Busch
15308d9dbe7SKeith Busch pm_runtime_no_callbacks(dev);
15408d9dbe7SKeith Busch list_add_tail(&access_node->list_node, &node->access_list);
15508d9dbe7SKeith Busch return access_node;
15608d9dbe7SKeith Busch free_name:
15708d9dbe7SKeith Busch kfree_const(dev->kobj.name);
15808d9dbe7SKeith Busch free:
15908d9dbe7SKeith Busch kfree(access_node);
16008d9dbe7SKeith Busch return NULL;
16108d9dbe7SKeith Busch }
16208d9dbe7SKeith Busch
163e1cf33aaSKeith Busch #ifdef CONFIG_HMEM_REPORTING
1647810f4dcSDave Jiang #define ACCESS_ATTR(property) \
1657810f4dcSDave Jiang static ssize_t property##_show(struct device *dev, \
166e1cf33aaSKeith Busch struct device_attribute *attr, \
167e1cf33aaSKeith Busch char *buf) \
168e1cf33aaSKeith Busch { \
169948b3edbSJoe Perches return sysfs_emit(buf, "%u\n", \
1707810f4dcSDave Jiang to_access_nodes(dev)->hmem_attrs.property); \
171e1cf33aaSKeith Busch } \
1727810f4dcSDave Jiang static DEVICE_ATTR_RO(property)
173e1cf33aaSKeith Busch
1746284a6e8SJoe Perches ACCESS_ATTR(read_bandwidth);
1756284a6e8SJoe Perches ACCESS_ATTR(read_latency);
1766284a6e8SJoe Perches ACCESS_ATTR(write_bandwidth);
1776284a6e8SJoe Perches ACCESS_ATTR(write_latency);
178e1cf33aaSKeith Busch
179e1cf33aaSKeith Busch static struct attribute *access_attrs[] = {
180e1cf33aaSKeith Busch &dev_attr_read_bandwidth.attr,
181e1cf33aaSKeith Busch &dev_attr_read_latency.attr,
182e1cf33aaSKeith Busch &dev_attr_write_bandwidth.attr,
183e1cf33aaSKeith Busch &dev_attr_write_latency.attr,
184e1cf33aaSKeith Busch NULL,
185e1cf33aaSKeith Busch };
186e1cf33aaSKeith Busch
187e1cf33aaSKeith Busch /**
188e1cf33aaSKeith Busch * node_set_perf_attrs - Set the performance values for given access class
189e1cf33aaSKeith Busch * @nid: Node identifier to be set
190e1cf33aaSKeith Busch * @hmem_attrs: Heterogeneous memory performance attributes
191e1cf33aaSKeith Busch * @access: The access class the for the given attributes
192e1cf33aaSKeith Busch */
node_set_perf_attrs(unsigned int nid,struct node_hmem_attrs * hmem_attrs,unsigned int access)193e1cf33aaSKeith Busch void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs,
194e7deeb9dSJinchao Wang unsigned int access)
195e1cf33aaSKeith Busch {
196e1cf33aaSKeith Busch struct node_access_nodes *c;
197e1cf33aaSKeith Busch struct node *node;
198e1cf33aaSKeith Busch int i;
199e1cf33aaSKeith Busch
200e1cf33aaSKeith Busch if (WARN_ON_ONCE(!node_online(nid)))
201e1cf33aaSKeith Busch return;
202e1cf33aaSKeith Busch
203e1cf33aaSKeith Busch node = node_devices[nid];
204e1cf33aaSKeith Busch c = node_init_node_access(node, access);
205e1cf33aaSKeith Busch if (!c)
206e1cf33aaSKeith Busch return;
207e1cf33aaSKeith Busch
208e1cf33aaSKeith Busch c->hmem_attrs = *hmem_attrs;
209e1cf33aaSKeith Busch for (i = 0; access_attrs[i] != NULL; i++) {
210e1cf33aaSKeith Busch if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
211e1cf33aaSKeith Busch "initiators")) {
212e1cf33aaSKeith Busch pr_info("failed to add performance attribute to node %d\n",
213e1cf33aaSKeith Busch nid);
214e1cf33aaSKeith Busch break;
215e1cf33aaSKeith Busch }
216e1cf33aaSKeith Busch }
217e1cf33aaSKeith Busch }
218acc02a10SKeith Busch
219acc02a10SKeith Busch /**
220acc02a10SKeith Busch * struct node_cache_info - Internal tracking for memory node caches
221acc02a10SKeith Busch * @dev: Device represeting the cache level
222acc02a10SKeith Busch * @node: List element for tracking in the node
223acc02a10SKeith Busch * @cache_attrs:Attributes for this cache level
224acc02a10SKeith Busch */
225acc02a10SKeith Busch struct node_cache_info {
226acc02a10SKeith Busch struct device dev;
227acc02a10SKeith Busch struct list_head node;
228acc02a10SKeith Busch struct node_cache_attrs cache_attrs;
229acc02a10SKeith Busch };
230acc02a10SKeith Busch #define to_cache_info(device) container_of(device, struct node_cache_info, dev)
231acc02a10SKeith Busch
232acc02a10SKeith Busch #define CACHE_ATTR(name, fmt) \
233acc02a10SKeith Busch static ssize_t name##_show(struct device *dev, \
234acc02a10SKeith Busch struct device_attribute *attr, \
235acc02a10SKeith Busch char *buf) \
236acc02a10SKeith Busch { \
237948b3edbSJoe Perches return sysfs_emit(buf, fmt "\n", \
238948b3edbSJoe Perches to_cache_info(dev)->cache_attrs.name); \
239acc02a10SKeith Busch } \
240fd03c075SRuiqi Gong static DEVICE_ATTR_RO(name);
241acc02a10SKeith Busch
242acc02a10SKeith Busch CACHE_ATTR(size, "%llu")
243acc02a10SKeith Busch CACHE_ATTR(line_size, "%u")
244acc02a10SKeith Busch CACHE_ATTR(indexing, "%u")
245acc02a10SKeith Busch CACHE_ATTR(write_policy, "%u")
246acc02a10SKeith Busch
247acc02a10SKeith Busch static struct attribute *cache_attrs[] = {
248acc02a10SKeith Busch &dev_attr_indexing.attr,
249acc02a10SKeith Busch &dev_attr_size.attr,
250acc02a10SKeith Busch &dev_attr_line_size.attr,
251acc02a10SKeith Busch &dev_attr_write_policy.attr,
252acc02a10SKeith Busch NULL,
253acc02a10SKeith Busch };
254acc02a10SKeith Busch ATTRIBUTE_GROUPS(cache);
255acc02a10SKeith Busch
node_cache_release(struct device * dev)256acc02a10SKeith Busch static void node_cache_release(struct device *dev)
257acc02a10SKeith Busch {
258acc02a10SKeith Busch kfree(dev);
259acc02a10SKeith Busch }
260acc02a10SKeith Busch
node_cacheinfo_release(struct device * dev)261acc02a10SKeith Busch static void node_cacheinfo_release(struct device *dev)
262acc02a10SKeith Busch {
263acc02a10SKeith Busch struct node_cache_info *info = to_cache_info(dev);
264acc02a10SKeith Busch kfree(info);
265acc02a10SKeith Busch }
266acc02a10SKeith Busch
node_init_cache_dev(struct node * node)267acc02a10SKeith Busch static void node_init_cache_dev(struct node *node)
268acc02a10SKeith Busch {
269acc02a10SKeith Busch struct device *dev;
270acc02a10SKeith Busch
271acc02a10SKeith Busch dev = kzalloc(sizeof(*dev), GFP_KERNEL);
272acc02a10SKeith Busch if (!dev)
273acc02a10SKeith Busch return;
274acc02a10SKeith Busch
2754ce535ecSDan Carpenter device_initialize(dev);
276acc02a10SKeith Busch dev->parent = &node->dev;
277acc02a10SKeith Busch dev->release = node_cache_release;
278acc02a10SKeith Busch if (dev_set_name(dev, "memory_side_cache"))
2794ce535ecSDan Carpenter goto put_device;
280acc02a10SKeith Busch
2814ce535ecSDan Carpenter if (device_add(dev))
2824ce535ecSDan Carpenter goto put_device;
283acc02a10SKeith Busch
284acc02a10SKeith Busch pm_runtime_no_callbacks(dev);
285acc02a10SKeith Busch node->cache_dev = dev;
286acc02a10SKeith Busch return;
2874ce535ecSDan Carpenter put_device:
2884ce535ecSDan Carpenter put_device(dev);
289acc02a10SKeith Busch }
290acc02a10SKeith Busch
291acc02a10SKeith Busch /**
292acc02a10SKeith Busch * node_add_cache() - add cache attribute to a memory node
293acc02a10SKeith Busch * @nid: Node identifier that has new cache attributes
294acc02a10SKeith Busch * @cache_attrs: Attributes for the cache being added
295acc02a10SKeith Busch */
node_add_cache(unsigned int nid,struct node_cache_attrs * cache_attrs)296acc02a10SKeith Busch void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
297acc02a10SKeith Busch {
298acc02a10SKeith Busch struct node_cache_info *info;
299acc02a10SKeith Busch struct device *dev;
300acc02a10SKeith Busch struct node *node;
301acc02a10SKeith Busch
302acc02a10SKeith Busch if (!node_online(nid) || !node_devices[nid])
303acc02a10SKeith Busch return;
304acc02a10SKeith Busch
305acc02a10SKeith Busch node = node_devices[nid];
306acc02a10SKeith Busch list_for_each_entry(info, &node->cache_attrs, node) {
307acc02a10SKeith Busch if (info->cache_attrs.level == cache_attrs->level) {
308acc02a10SKeith Busch dev_warn(&node->dev,
309acc02a10SKeith Busch "attempt to add duplicate cache level:%d\n",
310acc02a10SKeith Busch cache_attrs->level);
311acc02a10SKeith Busch return;
312acc02a10SKeith Busch }
313acc02a10SKeith Busch }
314acc02a10SKeith Busch
315acc02a10SKeith Busch if (!node->cache_dev)
316acc02a10SKeith Busch node_init_cache_dev(node);
317acc02a10SKeith Busch if (!node->cache_dev)
318acc02a10SKeith Busch return;
319acc02a10SKeith Busch
320acc02a10SKeith Busch info = kzalloc(sizeof(*info), GFP_KERNEL);
321acc02a10SKeith Busch if (!info)
322acc02a10SKeith Busch return;
323acc02a10SKeith Busch
324acc02a10SKeith Busch dev = &info->dev;
3254ce535ecSDan Carpenter device_initialize(dev);
326acc02a10SKeith Busch dev->parent = node->cache_dev;
327acc02a10SKeith Busch dev->release = node_cacheinfo_release;
328acc02a10SKeith Busch dev->groups = cache_groups;
329acc02a10SKeith Busch if (dev_set_name(dev, "index%d", cache_attrs->level))
3304ce535ecSDan Carpenter goto put_device;
331acc02a10SKeith Busch
332acc02a10SKeith Busch info->cache_attrs = *cache_attrs;
3334ce535ecSDan Carpenter if (device_add(dev)) {
334acc02a10SKeith Busch dev_warn(&node->dev, "failed to add cache level:%d\n",
335acc02a10SKeith Busch cache_attrs->level);
3364ce535ecSDan Carpenter goto put_device;
337acc02a10SKeith Busch }
338acc02a10SKeith Busch pm_runtime_no_callbacks(dev);
339acc02a10SKeith Busch list_add_tail(&info->node, &node->cache_attrs);
340acc02a10SKeith Busch return;
3414ce535ecSDan Carpenter put_device:
3424ce535ecSDan Carpenter put_device(dev);
343acc02a10SKeith Busch }
344acc02a10SKeith Busch
node_remove_caches(struct node * node)345acc02a10SKeith Busch static void node_remove_caches(struct node *node)
346acc02a10SKeith Busch {
347acc02a10SKeith Busch struct node_cache_info *info, *next;
348acc02a10SKeith Busch
349acc02a10SKeith Busch if (!node->cache_dev)
350acc02a10SKeith Busch return;
351acc02a10SKeith Busch
352acc02a10SKeith Busch list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
353acc02a10SKeith Busch list_del(&info->node);
354acc02a10SKeith Busch device_unregister(&info->dev);
355acc02a10SKeith Busch }
356acc02a10SKeith Busch device_unregister(node->cache_dev);
357acc02a10SKeith Busch }
358acc02a10SKeith Busch
node_init_caches(unsigned int nid)359acc02a10SKeith Busch static void node_init_caches(unsigned int nid)
360acc02a10SKeith Busch {
361acc02a10SKeith Busch INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
362acc02a10SKeith Busch }
363acc02a10SKeith Busch #else
node_init_caches(unsigned int nid)364acc02a10SKeith Busch static void node_init_caches(unsigned int nid) { }
node_remove_caches(struct node * node)365acc02a10SKeith Busch static void node_remove_caches(struct node *node) { }
366e1cf33aaSKeith Busch #endif
367e1cf33aaSKeith Busch
3681da177e4SLinus Torvalds #define K(x) ((x) << (PAGE_SHIFT - 10))
node_read_meminfo(struct device * dev,struct device_attribute * attr,char * buf)36910fbcf4cSKay Sievers static ssize_t node_read_meminfo(struct device *dev,
37010fbcf4cSKay Sievers struct device_attribute *attr, char *buf)
3711da177e4SLinus Torvalds {
372948b3edbSJoe Perches int len = 0;
3731da177e4SLinus Torvalds int nid = dev->id;
374599d0c95SMel Gorman struct pglist_data *pgdat = NODE_DATA(nid);
3751da177e4SLinus Torvalds struct sysinfo i;
37661f94e18SVlastimil Babka unsigned long sreclaimable, sunreclaimable;
377b6038942SShakeel Butt unsigned long swapcached = 0;
3781da177e4SLinus Torvalds
3791da177e4SLinus Torvalds si_meminfo_node(&i, nid);
380d42f3245SRoman Gushchin sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
381d42f3245SRoman Gushchin sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
382b6038942SShakeel Butt #ifdef CONFIG_SWAP
383b6038942SShakeel Butt swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE);
384b6038942SShakeel Butt #endif
385948b3edbSJoe Perches len = sysfs_emit_at(buf, len,
3861da177e4SLinus Torvalds "Node %d MemTotal: %8lu kB\n"
3871da177e4SLinus Torvalds "Node %d MemFree: %8lu kB\n"
3881da177e4SLinus Torvalds "Node %d MemUsed: %8lu kB\n"
389b6038942SShakeel Butt "Node %d SwapCached: %8lu kB\n"
3901da177e4SLinus Torvalds "Node %d Active: %8lu kB\n"
3911da177e4SLinus Torvalds "Node %d Inactive: %8lu kB\n"
3924f98a2feSRik van Riel "Node %d Active(anon): %8lu kB\n"
3934f98a2feSRik van Riel "Node %d Inactive(anon): %8lu kB\n"
3944f98a2feSRik van Riel "Node %d Active(file): %8lu kB\n"
3954f98a2feSRik van Riel "Node %d Inactive(file): %8lu kB\n"
3965344b7e6SNick Piggin "Node %d Unevictable: %8lu kB\n"
3977ee92255SKOSAKI Motohiro "Node %d Mlocked: %8lu kB\n",
3987ee92255SKOSAKI Motohiro nid, K(i.totalram),
3997ee92255SKOSAKI Motohiro nid, K(i.freeram),
4007ee92255SKOSAKI Motohiro nid, K(i.totalram - i.freeram),
401b6038942SShakeel Butt nid, K(swapcached),
402599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
403599d0c95SMel Gorman node_page_state(pgdat, NR_ACTIVE_FILE)),
404599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
405599d0c95SMel Gorman node_page_state(pgdat, NR_INACTIVE_FILE)),
406599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
407599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
408599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
409599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
410599d0c95SMel Gorman nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
41175ef7184SMel Gorman nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
4127ee92255SKOSAKI Motohiro
413182e8e23SChristoph Lameter #ifdef CONFIG_HIGHMEM
414948b3edbSJoe Perches len += sysfs_emit_at(buf, len,
4151da177e4SLinus Torvalds "Node %d HighTotal: %8lu kB\n"
4161da177e4SLinus Torvalds "Node %d HighFree: %8lu kB\n"
4171da177e4SLinus Torvalds "Node %d LowTotal: %8lu kB\n"
4187ee92255SKOSAKI Motohiro "Node %d LowFree: %8lu kB\n",
4197ee92255SKOSAKI Motohiro nid, K(i.totalhigh),
4207ee92255SKOSAKI Motohiro nid, K(i.freehigh),
4217ee92255SKOSAKI Motohiro nid, K(i.totalram - i.totalhigh),
4227ee92255SKOSAKI Motohiro nid, K(i.freeram - i.freehigh));
423182e8e23SChristoph Lameter #endif
424948b3edbSJoe Perches len += sysfs_emit_at(buf, len,
425c07e02dbSMartin Hicks "Node %d Dirty: %8lu kB\n"
426c07e02dbSMartin Hicks "Node %d Writeback: %8lu kB\n"
427347ce434SChristoph Lameter "Node %d FilePages: %8lu kB\n"
428c07e02dbSMartin Hicks "Node %d Mapped: %8lu kB\n"
429f3dbd344SChristoph Lameter "Node %d AnonPages: %8lu kB\n"
4304b02108aSKOSAKI Motohiro "Node %d Shmem: %8lu kB\n"
431c6a7f572SKOSAKI Motohiro "Node %d KernelStack: %8lu kB\n"
432628d06a4SSami Tolvanen #ifdef CONFIG_SHADOW_CALL_STACK
433628d06a4SSami Tolvanen "Node %d ShadowCallStack:%8lu kB\n"
434628d06a4SSami Tolvanen #endif
435df849a15SChristoph Lameter "Node %d PageTables: %8lu kB\n"
436ebc97a52SYosry Ahmed "Node %d SecPageTables: %8lu kB\n"
437f5ef68daSAndrew Morton "Node %d NFS_Unstable: %8lu kB\n"
438d2c5e30cSChristoph Lameter "Node %d Bounce: %8lu kB\n"
439fc3ba692SMiklos Szeredi "Node %d WritebackTmp: %8lu kB\n"
44061f94e18SVlastimil Babka "Node %d KReclaimable: %8lu kB\n"
441972d1a7bSChristoph Lameter "Node %d Slab: %8lu kB\n"
442972d1a7bSChristoph Lameter "Node %d SReclaimable: %8lu kB\n"
44305b258e9SDavid Rientjes "Node %d SUnreclaim: %8lu kB\n"
44405b258e9SDavid Rientjes #ifdef CONFIG_TRANSPARENT_HUGEPAGE
44505b258e9SDavid Rientjes "Node %d AnonHugePages: %8lu kB\n"
44665c45377SKirill A. Shutemov "Node %d ShmemHugePages: %8lu kB\n"
44765c45377SKirill A. Shutemov "Node %d ShmemPmdMapped: %8lu kB\n"
44860fbf0abSSong Liu "Node %d FileHugePages: %8lu kB\n"
44960fbf0abSSong Liu "Node %d FilePmdMapped: %8lu kB\n"
45005b258e9SDavid Rientjes #endif
451dcdfdd40SKirill A. Shutemov #ifdef CONFIG_UNACCEPTED_MEMORY
452dcdfdd40SKirill A. Shutemov "Node %d Unaccepted: %8lu kB\n"
453dcdfdd40SKirill A. Shutemov #endif
45405b258e9SDavid Rientjes ,
45511fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
45611fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_WRITEBACK)),
45711fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
45850658e2eSMel Gorman nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
4594b9d0fabSMel Gorman nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
460cc7452b6SRafael Aquini nid, K(i.sharedram),
461991e7673SShakeel Butt nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
462628d06a4SSami Tolvanen #ifdef CONFIG_SHADOW_CALL_STACK
463991e7673SShakeel Butt nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
464628d06a4SSami Tolvanen #endif
465f0c0c115SShakeel Butt nid, K(node_page_state(pgdat, NR_PAGETABLE)),
466ebc97a52SYosry Ahmed nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)),
4678d92890bSNeilBrown nid, 0UL,
46875ef7184SMel Gorman nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
46911fb9989SMel Gorman nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
47061f94e18SVlastimil Babka nid, K(sreclaimable +
47161f94e18SVlastimil Babka node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
47261f94e18SVlastimil Babka nid, K(sreclaimable + sunreclaimable),
47361f94e18SVlastimil Babka nid, K(sreclaimable),
47461f94e18SVlastimil Babka nid, K(sunreclaimable)
47505b258e9SDavid Rientjes #ifdef CONFIG_TRANSPARENT_HUGEPAGE
47661f94e18SVlastimil Babka ,
47769473e5dSMuchun Song nid, K(node_page_state(pgdat, NR_ANON_THPS)),
47857b2847dSMuchun Song nid, K(node_page_state(pgdat, NR_SHMEM_THPS)),
479a1528e21SMuchun Song nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
480bf9eceadSMuchun Song nid, K(node_page_state(pgdat, NR_FILE_THPS)),
481380780e7SMuchun Song nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED))
48205b258e9SDavid Rientjes #endif
483dcdfdd40SKirill A. Shutemov #ifdef CONFIG_UNACCEPTED_MEMORY
484dcdfdd40SKirill A. Shutemov ,
485dcdfdd40SKirill A. Shutemov nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED))
486dcdfdd40SKirill A. Shutemov #endif
48761f94e18SVlastimil Babka );
4887981593bSJoe Perches len += hugetlb_report_node_meminfo(buf, len, nid);
489948b3edbSJoe Perches return len;
4901da177e4SLinus Torvalds }
4911da177e4SLinus Torvalds
4921da177e4SLinus Torvalds #undef K
493948b3edbSJoe Perches static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
4941da177e4SLinus Torvalds
node_read_numastat(struct device * dev,struct device_attribute * attr,char * buf)49510fbcf4cSKay Sievers static ssize_t node_read_numastat(struct device *dev,
49610fbcf4cSKay Sievers struct device_attribute *attr, char *buf)
4971da177e4SLinus Torvalds {
498f19298b9SMel Gorman fold_vm_numa_events();
499aa838896SJoe Perches return sysfs_emit(buf,
5001da177e4SLinus Torvalds "numa_hit %lu\n"
5011da177e4SLinus Torvalds "numa_miss %lu\n"
5021da177e4SLinus Torvalds "numa_foreign %lu\n"
5031da177e4SLinus Torvalds "interleave_hit %lu\n"
5041da177e4SLinus Torvalds "local_node %lu\n"
5051da177e4SLinus Torvalds "other_node %lu\n",
506f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_HIT),
507f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_MISS),
508f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_FOREIGN),
509f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT),
510f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_LOCAL),
511f19298b9SMel Gorman sum_zone_numa_event_state(dev->id, NUMA_OTHER));
5121da177e4SLinus Torvalds }
513948b3edbSJoe Perches static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
5141da177e4SLinus Torvalds
node_read_vmstat(struct device * dev,struct device_attribute * attr,char * buf)51510fbcf4cSKay Sievers static ssize_t node_read_vmstat(struct device *dev,
51610fbcf4cSKay Sievers struct device_attribute *attr, char *buf)
5172ac39037SMichael Rubin {
5182ac39037SMichael Rubin int nid = dev->id;
51975ef7184SMel Gorman struct pglist_data *pgdat = NODE_DATA(nid);
520fa25c503SKOSAKI Motohiro int i;
521948b3edbSJoe Perches int len = 0;
522fa25c503SKOSAKI Motohiro
523fa25c503SKOSAKI Motohiro for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
524948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s %lu\n",
525948b3edbSJoe Perches zone_stat_name(i),
52675ef7184SMel Gorman sum_zone_node_page_state(nid, i));
52775ef7184SMel Gorman
5283a321d2aSKemi Wang #ifdef CONFIG_NUMA
529f19298b9SMel Gorman fold_vm_numa_events();
530f19298b9SMel Gorman for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
531948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s %lu\n",
532948b3edbSJoe Perches numa_stat_name(i),
533f19298b9SMel Gorman sum_zone_numa_event_state(nid, i));
5343a321d2aSKemi Wang
535948b3edbSJoe Perches #endif
53669473e5dSMuchun Song for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
53769473e5dSMuchun Song unsigned long pages = node_page_state_pages(pgdat, i);
53869473e5dSMuchun Song
53969473e5dSMuchun Song if (vmstat_item_print_in_thp(i))
54069473e5dSMuchun Song pages /= HPAGE_PMD_NR;
54169473e5dSMuchun Song len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i),
54269473e5dSMuchun Song pages);
54369473e5dSMuchun Song }
544fa25c503SKOSAKI Motohiro
545948b3edbSJoe Perches return len;
5462ac39037SMichael Rubin }
547948b3edbSJoe Perches static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
5482ac39037SMichael Rubin
node_read_distance(struct device * dev,struct device_attribute * attr,char * buf)54910fbcf4cSKay Sievers static ssize_t node_read_distance(struct device *dev,
55010fbcf4cSKay Sievers struct device_attribute *attr, char *buf)
5511da177e4SLinus Torvalds {
5521da177e4SLinus Torvalds int nid = dev->id;
5531da177e4SLinus Torvalds int len = 0;
5541da177e4SLinus Torvalds int i;
5551da177e4SLinus Torvalds
55612ee3c0aSDavid Rientjes /*
55712ee3c0aSDavid Rientjes * buf is currently PAGE_SIZE in length and each node needs 4 chars
55812ee3c0aSDavid Rientjes * at the most (distance + space or newline).
55912ee3c0aSDavid Rientjes */
56012ee3c0aSDavid Rientjes BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
5611da177e4SLinus Torvalds
562948b3edbSJoe Perches for_each_online_node(i) {
563948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "%s%d",
564948b3edbSJoe Perches i ? " " : "", node_distance(nid, i));
565948b3edbSJoe Perches }
5661da177e4SLinus Torvalds
567948b3edbSJoe Perches len += sysfs_emit_at(buf, len, "\n");
5681da177e4SLinus Torvalds return len;
5691da177e4SLinus Torvalds }
570948b3edbSJoe Perches static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
5711da177e4SLinus Torvalds
5723c9b8aafSTakashi Iwai static struct attribute *node_dev_attrs[] = {
5733c9b8aafSTakashi Iwai &dev_attr_meminfo.attr,
5743c9b8aafSTakashi Iwai &dev_attr_numastat.attr,
5753c9b8aafSTakashi Iwai &dev_attr_distance.attr,
5763c9b8aafSTakashi Iwai &dev_attr_vmstat.attr,
5773c9b8aafSTakashi Iwai NULL
5783c9b8aafSTakashi Iwai };
57975bd50faSTian Tao
58075bd50faSTian Tao static struct bin_attribute *node_dev_bin_attrs[] = {
58175bd50faSTian Tao &bin_attr_cpumap,
58275bd50faSTian Tao &bin_attr_cpulist,
58375bd50faSTian Tao NULL
58475bd50faSTian Tao };
58575bd50faSTian Tao
58675bd50faSTian Tao static const struct attribute_group node_dev_group = {
58775bd50faSTian Tao .attrs = node_dev_attrs,
58875bd50faSTian Tao .bin_attrs = node_dev_bin_attrs
58975bd50faSTian Tao };
59075bd50faSTian Tao
59175bd50faSTian Tao static const struct attribute_group *node_dev_groups[] = {
59275bd50faSTian Tao &node_dev_group,
59350468e43SJarkko Sakkinen #ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP
59450468e43SJarkko Sakkinen &arch_node_dev_group,
59550468e43SJarkko Sakkinen #endif
59644b8f8bfSJiaqi Yan #ifdef CONFIG_MEMORY_FAILURE
59744b8f8bfSJiaqi Yan &memory_failure_attr_group,
59844b8f8bfSJiaqi Yan #endif
59975bd50faSTian Tao NULL
60075bd50faSTian Tao };
6013c9b8aafSTakashi Iwai
node_device_release(struct device * dev)6028c7b5b4eSYasuaki Ishimatsu static void node_device_release(struct device *dev)
6038c7b5b4eSYasuaki Ishimatsu {
604b958d4d0SMuchun Song kfree(to_node(dev));
6058c7b5b4eSYasuaki Ishimatsu }
6061da177e4SLinus Torvalds
6071da177e4SLinus Torvalds /*
608405ae7d3SRobert P. J. Day * register_node - Setup a sysfs device for a node.
6091da177e4SLinus Torvalds * @num - Node number to use when creating the device.
6101da177e4SLinus Torvalds *
6111da177e4SLinus Torvalds * Initialize and register the node device.
6121da177e4SLinus Torvalds */
register_node(struct node * node,int num)613a7be6e5aSDou Liyang static int register_node(struct node *node, int num)
6141da177e4SLinus Torvalds {
6151da177e4SLinus Torvalds int error;
6161da177e4SLinus Torvalds
61710fbcf4cSKay Sievers node->dev.id = num;
61810fbcf4cSKay Sievers node->dev.bus = &node_subsys;
6198c7b5b4eSYasuaki Ishimatsu node->dev.release = node_device_release;
6207ca7ec40SGreg Kroah-Hartman node->dev.groups = node_dev_groups;
62110fbcf4cSKay Sievers error = device_register(&node->dev);
6221da177e4SLinus Torvalds
623a4a00b45SMuchun Song if (error) {
624c1cc0d51SArvind Yadav put_device(&node->dev);
625a4a00b45SMuchun Song } else {
6269a305230SLee Schermerhorn hugetlb_register_node(node);
627ed4a6d7fSMel Gorman compaction_register_node(node);
6281da177e4SLinus Torvalds }
629b958d4d0SMuchun Song
6301da177e4SLinus Torvalds return error;
6311da177e4SLinus Torvalds }
6321da177e4SLinus Torvalds
6334b45099bSKeiichiro Tokunaga /**
6344b45099bSKeiichiro Tokunaga * unregister_node - unregister a node device
6354b45099bSKeiichiro Tokunaga * @node: node going away
6364b45099bSKeiichiro Tokunaga *
6374b45099bSKeiichiro Tokunaga * Unregisters a node device @node. All the devices on the node must be
6384b45099bSKeiichiro Tokunaga * unregistered before calling this function.
6394b45099bSKeiichiro Tokunaga */
unregister_node(struct node * node)6404b45099bSKeiichiro Tokunaga void unregister_node(struct node *node)
6414b45099bSKeiichiro Tokunaga {
642a4a00b45SMuchun Song hugetlb_unregister_node(node);
643da63dc84SMiaohe Lin compaction_unregister_node(node);
64408d9dbe7SKeith Busch node_remove_accesses(node);
645acc02a10SKeith Busch node_remove_caches(node);
64610fbcf4cSKay Sievers device_unregister(&node->dev);
6474b45099bSKeiichiro Tokunaga }
6484b45099bSKeiichiro Tokunaga
6498732794bSWen Congyang struct node *node_devices[MAX_NUMNODES];
6500fc44159SYasunori Goto
65176b67ed9SKAMEZAWA Hiroyuki /*
65276b67ed9SKAMEZAWA Hiroyuki * register cpu under node
65376b67ed9SKAMEZAWA Hiroyuki */
register_cpu_under_node(unsigned int cpu,unsigned int nid)65476b67ed9SKAMEZAWA Hiroyuki int register_cpu_under_node(unsigned int cpu, unsigned int nid)
65576b67ed9SKAMEZAWA Hiroyuki {
6561830794aSAlex Chiang int ret;
6578a25a2fdSKay Sievers struct device *obj;
658f8246f31SAlex Chiang
659f8246f31SAlex Chiang if (!node_online(nid))
660f8246f31SAlex Chiang return 0;
661f8246f31SAlex Chiang
6628a25a2fdSKay Sievers obj = get_cpu_device(cpu);
66376b67ed9SKAMEZAWA Hiroyuki if (!obj)
66476b67ed9SKAMEZAWA Hiroyuki return 0;
665f8246f31SAlex Chiang
6668732794bSWen Congyang ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
66776b67ed9SKAMEZAWA Hiroyuki &obj->kobj,
66876b67ed9SKAMEZAWA Hiroyuki kobject_name(&obj->kobj));
6691830794aSAlex Chiang if (ret)
6701830794aSAlex Chiang return ret;
6711830794aSAlex Chiang
6721830794aSAlex Chiang return sysfs_create_link(&obj->kobj,
6738732794bSWen Congyang &node_devices[nid]->dev.kobj,
6748732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj));
67576b67ed9SKAMEZAWA Hiroyuki }
67676b67ed9SKAMEZAWA Hiroyuki
67708d9dbe7SKeith Busch /**
67808d9dbe7SKeith Busch * register_memory_node_under_compute_node - link memory node to its compute
67908d9dbe7SKeith Busch * node for a given access class.
68058cb346cSMauro Carvalho Chehab * @mem_nid: Memory node number
68158cb346cSMauro Carvalho Chehab * @cpu_nid: Cpu node number
68208d9dbe7SKeith Busch * @access: Access class to register
68308d9dbe7SKeith Busch *
68408d9dbe7SKeith Busch * Description:
68508d9dbe7SKeith Busch * For use with platforms that may have separate memory and compute nodes.
68608d9dbe7SKeith Busch * This function will export node relationships linking which memory
68708d9dbe7SKeith Busch * initiator nodes can access memory targets at a given ranked access
68808d9dbe7SKeith Busch * class.
68908d9dbe7SKeith Busch */
register_memory_node_under_compute_node(unsigned int mem_nid,unsigned int cpu_nid,unsigned int access)69008d9dbe7SKeith Busch int register_memory_node_under_compute_node(unsigned int mem_nid,
69108d9dbe7SKeith Busch unsigned int cpu_nid,
692e7deeb9dSJinchao Wang unsigned int access)
69308d9dbe7SKeith Busch {
69408d9dbe7SKeith Busch struct node *init_node, *targ_node;
69508d9dbe7SKeith Busch struct node_access_nodes *initiator, *target;
69608d9dbe7SKeith Busch int ret;
69708d9dbe7SKeith Busch
69808d9dbe7SKeith Busch if (!node_online(cpu_nid) || !node_online(mem_nid))
69908d9dbe7SKeith Busch return -ENODEV;
70008d9dbe7SKeith Busch
70108d9dbe7SKeith Busch init_node = node_devices[cpu_nid];
70208d9dbe7SKeith Busch targ_node = node_devices[mem_nid];
70308d9dbe7SKeith Busch initiator = node_init_node_access(init_node, access);
70408d9dbe7SKeith Busch target = node_init_node_access(targ_node, access);
70508d9dbe7SKeith Busch if (!initiator || !target)
70608d9dbe7SKeith Busch return -ENOMEM;
70708d9dbe7SKeith Busch
70808d9dbe7SKeith Busch ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
70908d9dbe7SKeith Busch &targ_node->dev.kobj,
71008d9dbe7SKeith Busch dev_name(&targ_node->dev));
71108d9dbe7SKeith Busch if (ret)
71208d9dbe7SKeith Busch return ret;
71308d9dbe7SKeith Busch
71408d9dbe7SKeith Busch ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
71508d9dbe7SKeith Busch &init_node->dev.kobj,
71608d9dbe7SKeith Busch dev_name(&init_node->dev));
71708d9dbe7SKeith Busch if (ret)
71808d9dbe7SKeith Busch goto err;
71908d9dbe7SKeith Busch
72008d9dbe7SKeith Busch return 0;
72108d9dbe7SKeith Busch err:
72208d9dbe7SKeith Busch sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
72308d9dbe7SKeith Busch dev_name(&targ_node->dev));
72408d9dbe7SKeith Busch return ret;
72508d9dbe7SKeith Busch }
72608d9dbe7SKeith Busch
unregister_cpu_under_node(unsigned int cpu,unsigned int nid)72776b67ed9SKAMEZAWA Hiroyuki int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
72876b67ed9SKAMEZAWA Hiroyuki {
7298a25a2fdSKay Sievers struct device *obj;
730b9d52dadSAlex Chiang
731b9d52dadSAlex Chiang if (!node_online(nid))
732b9d52dadSAlex Chiang return 0;
733b9d52dadSAlex Chiang
7348a25a2fdSKay Sievers obj = get_cpu_device(cpu);
735b9d52dadSAlex Chiang if (!obj)
736b9d52dadSAlex Chiang return 0;
737b9d52dadSAlex Chiang
7388732794bSWen Congyang sysfs_remove_link(&node_devices[nid]->dev.kobj,
73976b67ed9SKAMEZAWA Hiroyuki kobject_name(&obj->kobj));
7401830794aSAlex Chiang sysfs_remove_link(&obj->kobj,
7418732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj));
742b9d52dadSAlex Chiang
74376b67ed9SKAMEZAWA Hiroyuki return 0;
74476b67ed9SKAMEZAWA Hiroyuki }
74576b67ed9SKAMEZAWA Hiroyuki
74650f9481eSDavid Hildenbrand #ifdef CONFIG_MEMORY_HOTPLUG
get_nid_for_pfn(unsigned long pfn)747bd721ea7SFabian Frederick static int __ref get_nid_for_pfn(unsigned long pfn)
748c04fc586SGary Hade {
7493a80a7faSMel Gorman #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
7508cdde385SThomas Gleixner if (system_state < SYSTEM_RUNNING)
7513a80a7faSMel Gorman return early_pfn_to_nid(pfn);
7523a80a7faSMel Gorman #endif
753c04fc586SGary Hade return pfn_to_nid(pfn);
754c04fc586SGary Hade }
755c04fc586SGary Hade
do_register_memory_block_under_node(int nid,struct memory_block * mem_blk,enum meminit_context context)75690c7eaebSLaurent Dufour static void do_register_memory_block_under_node(int nid,
757395f6081SDavid Hildenbrand struct memory_block *mem_blk,
758395f6081SDavid Hildenbrand enum meminit_context context)
759c04fc586SGary Hade {
760f85086f9SLaurent Dufour int ret;
761d84f2f5aSDavid Hildenbrand
762395f6081SDavid Hildenbrand memory_block_add_nid(mem_blk, nid, context);
763d84f2f5aSDavid Hildenbrand
7648732794bSWen Congyang ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
76510fbcf4cSKay Sievers &mem_blk->dev.kobj,
76610fbcf4cSKay Sievers kobject_name(&mem_blk->dev.kobj));
76790c7eaebSLaurent Dufour if (ret && ret != -EEXIST)
76890c7eaebSLaurent Dufour dev_err_ratelimited(&node_devices[nid]->dev,
76990c7eaebSLaurent Dufour "can't create link to %s in sysfs (%d)\n",
77090c7eaebSLaurent Dufour kobject_name(&mem_blk->dev.kobj), ret);
771dee5d0d5SAlex Chiang
77290c7eaebSLaurent Dufour ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
7738732794bSWen Congyang &node_devices[nid]->dev.kobj,
7748732794bSWen Congyang kobject_name(&node_devices[nid]->dev.kobj));
77590c7eaebSLaurent Dufour if (ret && ret != -EEXIST)
77690c7eaebSLaurent Dufour dev_err_ratelimited(&mem_blk->dev,
77790c7eaebSLaurent Dufour "can't create link to %s in sysfs (%d)\n",
77890c7eaebSLaurent Dufour kobject_name(&node_devices[nid]->dev.kobj),
77990c7eaebSLaurent Dufour ret);
780c04fc586SGary Hade }
781f85086f9SLaurent Dufour
782f85086f9SLaurent Dufour /* register memory section under specified node if it spans that node */
register_mem_block_under_node_early(struct memory_block * mem_blk,void * arg)783f85086f9SLaurent Dufour static int register_mem_block_under_node_early(struct memory_block *mem_blk,
784f85086f9SLaurent Dufour void *arg)
785f85086f9SLaurent Dufour {
786f85086f9SLaurent Dufour unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE;
787f85086f9SLaurent Dufour unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
788f85086f9SLaurent Dufour unsigned long end_pfn = start_pfn + memory_block_pfns - 1;
789f85086f9SLaurent Dufour int nid = *(int *)arg;
790f85086f9SLaurent Dufour unsigned long pfn;
791f85086f9SLaurent Dufour
792f85086f9SLaurent Dufour for (pfn = start_pfn; pfn <= end_pfn; pfn++) {
793f85086f9SLaurent Dufour int page_nid;
794f85086f9SLaurent Dufour
795f85086f9SLaurent Dufour /*
796f85086f9SLaurent Dufour * memory block could have several absent sections from start.
797f85086f9SLaurent Dufour * skip pfn range from absent section
798f85086f9SLaurent Dufour */
799f85086f9SLaurent Dufour if (!pfn_in_present_section(pfn)) {
800f85086f9SLaurent Dufour pfn = round_down(pfn + PAGES_PER_SECTION,
801f85086f9SLaurent Dufour PAGES_PER_SECTION) - 1;
802f85086f9SLaurent Dufour continue;
803f85086f9SLaurent Dufour }
804f85086f9SLaurent Dufour
805f85086f9SLaurent Dufour /*
806f85086f9SLaurent Dufour * We need to check if page belongs to nid only at the boot
807f85086f9SLaurent Dufour * case because node's ranges can be interleaved.
808f85086f9SLaurent Dufour */
809f85086f9SLaurent Dufour page_nid = get_nid_for_pfn(pfn);
810f85086f9SLaurent Dufour if (page_nid < 0)
811f85086f9SLaurent Dufour continue;
812f85086f9SLaurent Dufour if (page_nid != nid)
813f85086f9SLaurent Dufour continue;
814f85086f9SLaurent Dufour
815395f6081SDavid Hildenbrand do_register_memory_block_under_node(nid, mem_blk, MEMINIT_EARLY);
81690c7eaebSLaurent Dufour return 0;
817f85086f9SLaurent Dufour }
818c04fc586SGary Hade /* mem section does not span the specified node */
819c04fc586SGary Hade return 0;
820c04fc586SGary Hade }
821c04fc586SGary Hade
8224c4b7f9bSDavid Hildenbrand /*
823f85086f9SLaurent Dufour * During hotplug we know that all pages in the memory block belong to the same
824f85086f9SLaurent Dufour * node.
825f85086f9SLaurent Dufour */
register_mem_block_under_node_hotplug(struct memory_block * mem_blk,void * arg)826f85086f9SLaurent Dufour static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
827f85086f9SLaurent Dufour void *arg)
828f85086f9SLaurent Dufour {
829f85086f9SLaurent Dufour int nid = *(int *)arg;
830f85086f9SLaurent Dufour
831395f6081SDavid Hildenbrand do_register_memory_block_under_node(nid, mem_blk, MEMINIT_HOTPLUG);
83290c7eaebSLaurent Dufour return 0;
833f85086f9SLaurent Dufour }
834f85086f9SLaurent Dufour
835f85086f9SLaurent Dufour /*
836d84f2f5aSDavid Hildenbrand * Unregister a memory block device under the node it spans. Memory blocks
837d84f2f5aSDavid Hildenbrand * with multiple nodes cannot be offlined and therefore also never be removed.
8384c4b7f9bSDavid Hildenbrand */
unregister_memory_block_under_nodes(struct memory_block * mem_blk)839a31b264cSDavid Hildenbrand void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
840c04fc586SGary Hade {
841d84f2f5aSDavid Hildenbrand if (mem_blk->nid == NUMA_NO_NODE)
842d84f2f5aSDavid Hildenbrand return;
843c04fc586SGary Hade
844d84f2f5aSDavid Hildenbrand sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
84510fbcf4cSKay Sievers kobject_name(&mem_blk->dev.kobj));
84610fbcf4cSKay Sievers sysfs_remove_link(&mem_blk->dev.kobj,
847d84f2f5aSDavid Hildenbrand kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
848c04fc586SGary Hade }
849c04fc586SGary Hade
register_memory_blocks_under_node(int nid,unsigned long start_pfn,unsigned long end_pfn,enum meminit_context context)850cc651559SDavid Hildenbrand void register_memory_blocks_under_node(int nid, unsigned long start_pfn,
851cc651559SDavid Hildenbrand unsigned long end_pfn,
852f85086f9SLaurent Dufour enum meminit_context context)
853c04fc586SGary Hade {
854f85086f9SLaurent Dufour walk_memory_blocks_func_t func;
855f85086f9SLaurent Dufour
856f85086f9SLaurent Dufour if (context == MEMINIT_HOTPLUG)
857f85086f9SLaurent Dufour func = register_mem_block_under_node_hotplug;
858f85086f9SLaurent Dufour else
859f85086f9SLaurent Dufour func = register_mem_block_under_node_early;
860f85086f9SLaurent Dufour
86190c7eaebSLaurent Dufour walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
86290c7eaebSLaurent Dufour (void *)&nid, func);
86390c7eaebSLaurent Dufour return;
864c04fc586SGary Hade }
86550f9481eSDavid Hildenbrand #endif /* CONFIG_MEMORY_HOTPLUG */
86639da08cbSLee Schermerhorn
__register_one_node(int nid)8679037a993SMichal Hocko int __register_one_node(int nid)
8680fc44159SYasunori Goto {
8699037a993SMichal Hocko int error;
8709037a993SMichal Hocko int cpu;
871*a5e1c3feSGregory Price struct node *node;
8720fc44159SYasunori Goto
873*a5e1c3feSGregory Price node = kzalloc(sizeof(struct node), GFP_KERNEL);
874*a5e1c3feSGregory Price if (!node)
8758732794bSWen Congyang return -ENOMEM;
8768732794bSWen Congyang
877*a5e1c3feSGregory Price INIT_LIST_HEAD(&node->access_list);
878*a5e1c3feSGregory Price node_devices[nid] = node;
879*a5e1c3feSGregory Price
880a7be6e5aSDou Liyang error = register_node(node_devices[nid], nid);
88176b67ed9SKAMEZAWA Hiroyuki
88276b67ed9SKAMEZAWA Hiroyuki /* link cpu under this node */
88376b67ed9SKAMEZAWA Hiroyuki for_each_present_cpu(cpu) {
88476b67ed9SKAMEZAWA Hiroyuki if (cpu_to_node(cpu) == nid)
88576b67ed9SKAMEZAWA Hiroyuki register_cpu_under_node(cpu, nid);
88676b67ed9SKAMEZAWA Hiroyuki }
887c04fc586SGary Hade
888acc02a10SKeith Busch node_init_caches(nid);
8890fc44159SYasunori Goto
8900fc44159SYasunori Goto return error;
8910fc44159SYasunori Goto }
8920fc44159SYasunori Goto
unregister_one_node(int nid)8930fc44159SYasunori Goto void unregister_one_node(int nid)
8940fc44159SYasunori Goto {
89592d585efSXishi Qiu if (!node_devices[nid])
89692d585efSXishi Qiu return;
89792d585efSXishi Qiu
8988732794bSWen Congyang unregister_node(node_devices[nid]);
8998732794bSWen Congyang node_devices[nid] = NULL;
9000fc44159SYasunori Goto }
9010fc44159SYasunori Goto
902bde631a5SLee Schermerhorn /*
903bde631a5SLee Schermerhorn * node states attributes
904bde631a5SLee Schermerhorn */
905bde631a5SLee Schermerhorn
906b15f562fSAndi Kleen struct node_attr {
90710fbcf4cSKay Sievers struct device_attribute attr;
908b15f562fSAndi Kleen enum node_states state;
909b15f562fSAndi Kleen };
910b15f562fSAndi Kleen
show_node_state(struct device * dev,struct device_attribute * attr,char * buf)91110fbcf4cSKay Sievers static ssize_t show_node_state(struct device *dev,
91210fbcf4cSKay Sievers struct device_attribute *attr, char *buf)
913bde631a5SLee Schermerhorn {
914b15f562fSAndi Kleen struct node_attr *na = container_of(attr, struct node_attr, attr);
915948b3edbSJoe Perches
916948b3edbSJoe Perches return sysfs_emit(buf, "%*pbl\n",
917948b3edbSJoe Perches nodemask_pr_args(&node_states[na->state]));
918bde631a5SLee Schermerhorn }
919bde631a5SLee Schermerhorn
920b15f562fSAndi Kleen #define _NODE_ATTR(name, state) \
92110fbcf4cSKay Sievers { __ATTR(name, 0444, show_node_state, NULL), state }
922bde631a5SLee Schermerhorn
923b15f562fSAndi Kleen static struct node_attr node_state_attr[] = {
924fcf07d22SLai Jiangshan [N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
925fcf07d22SLai Jiangshan [N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
926fcf07d22SLai Jiangshan [N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
927bde631a5SLee Schermerhorn #ifdef CONFIG_HIGHMEM
928fcf07d22SLai Jiangshan [N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
929bde631a5SLee Schermerhorn #endif
93020b2f52bSLai Jiangshan [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
931fcf07d22SLai Jiangshan [N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
932894c26a1SJonathan Cameron [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
933894c26a1SJonathan Cameron N_GENERIC_INITIATOR),
934bde631a5SLee Schermerhorn };
935bde631a5SLee Schermerhorn
93610fbcf4cSKay Sievers static struct attribute *node_state_attrs[] = {
937fcf07d22SLai Jiangshan &node_state_attr[N_POSSIBLE].attr.attr,
938fcf07d22SLai Jiangshan &node_state_attr[N_ONLINE].attr.attr,
939fcf07d22SLai Jiangshan &node_state_attr[N_NORMAL_MEMORY].attr.attr,
9403701cde6SAndi Kleen #ifdef CONFIG_HIGHMEM
941fcf07d22SLai Jiangshan &node_state_attr[N_HIGH_MEMORY].attr.attr,
9423701cde6SAndi Kleen #endif
94320b2f52bSLai Jiangshan &node_state_attr[N_MEMORY].attr.attr,
944fcf07d22SLai Jiangshan &node_state_attr[N_CPU].attr.attr,
945894c26a1SJonathan Cameron &node_state_attr[N_GENERIC_INITIATOR].attr.attr,
9463701cde6SAndi Kleen NULL
9473701cde6SAndi Kleen };
948bde631a5SLee Schermerhorn
9495a576764SRikard Falkeborn static const struct attribute_group memory_root_attr_group = {
95010fbcf4cSKay Sievers .attrs = node_state_attrs,
95110fbcf4cSKay Sievers };
95210fbcf4cSKay Sievers
95310fbcf4cSKay Sievers static const struct attribute_group *cpu_root_attr_groups[] = {
95410fbcf4cSKay Sievers &memory_root_attr_group,
95510fbcf4cSKay Sievers NULL,
95610fbcf4cSKay Sievers };
95710fbcf4cSKay Sievers
node_dev_init(void)9582848a28bSDavid Hildenbrand void __init node_dev_init(void)
9591da177e4SLinus Torvalds {
9602848a28bSDavid Hildenbrand int ret, i;
961bde631a5SLee Schermerhorn
9623701cde6SAndi Kleen BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
9633701cde6SAndi Kleen BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
9643701cde6SAndi Kleen
96510fbcf4cSKay Sievers ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
9662848a28bSDavid Hildenbrand if (ret)
9672848a28bSDavid Hildenbrand panic("%s() failed to register subsystem: %d\n", __func__, ret);
9682848a28bSDavid Hildenbrand
969bde631a5SLee Schermerhorn /*
9702848a28bSDavid Hildenbrand * Create all node devices, which will properly link the node
9712848a28bSDavid Hildenbrand * to applicable memory block devices and already created cpu devices.
972bde631a5SLee Schermerhorn */
9732848a28bSDavid Hildenbrand for_each_online_node(i) {
9742848a28bSDavid Hildenbrand ret = register_one_node(i);
9752848a28bSDavid Hildenbrand if (ret)
9762848a28bSDavid Hildenbrand panic("%s() failed to add node: %d\n", __func__, ret);
9771da177e4SLinus Torvalds }
9782848a28bSDavid Hildenbrand }
979