topology.c (c87866ede44ad7da6b296d732221dc34ce1b154d) topology.c (0083242c93759dde353a963a90cb351c5c283379)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Scheduler topology setup/handling methods
4 */
5#include "sched.h"
6
7DEFINE_MUTEX(sched_domains_mutex);
8

--- 1468 unchanged lines hidden (view full) ---

1477
1478static int sched_domains_numa_levels;
1479static int sched_domains_curr_level;
1480
1481int sched_max_numa_distance;
1482static int *sched_domains_numa_distance;
1483static struct cpumask ***sched_domains_numa_masks;
1484int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Scheduler topology setup/handling methods
4 */
5#include "sched.h"
6
7DEFINE_MUTEX(sched_domains_mutex);
8

--- 1468 unchanged lines hidden (view full) ---

1477
1478static int sched_domains_numa_levels;
1479static int sched_domains_curr_level;
1480
1481int sched_max_numa_distance;
1482static int *sched_domains_numa_distance;
1483static struct cpumask ***sched_domains_numa_masks;
1484int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
1485
1486static unsigned long __read_mostly *sched_numa_onlined_nodes;
1485#endif
1486
1487/*
1488 * SD_flags allowed in topology descriptions.
1489 *
1490 * These flags are purely descriptive of the topology and do not prescribe
1491 * behaviour. Behaviour is artificial and mapped in the below sd_init()
1492 * function:

--- 335 unchanged lines hidden (view full) ---

1828 int k;
1829
1830 if (!mask)
1831 return;
1832
1833 sched_domains_numa_masks[i][j] = mask;
1834
1835 for_each_node(k) {
1487#endif
1488
1489/*
1490 * SD_flags allowed in topology descriptions.
1491 *
1492 * These flags are purely descriptive of the topology and do not prescribe
1493 * behaviour. Behaviour is artificial and mapped in the below sd_init()
1494 * function:

--- 335 unchanged lines hidden (view full) ---

1830 int k;
1831
1832 if (!mask)
1833 return;
1834
1835 sched_domains_numa_masks[i][j] = mask;
1836
1837 for_each_node(k) {
1838 /*
1839 * Distance information can be unreliable for
1840 * offline nodes, defer building the node
1841 * masks to its bringup.
1842 * This relies on all unique distance values
1843 * still being visible at init time.
1844 */
1845 if (!node_online(j))
1846 continue;
1847
1836 if (sched_debug() && (node_distance(j, k) != node_distance(k, j)))
1837 sched_numa_warn("Node-distance not symmetric");
1838
1839 if (node_distance(j, k) > sched_domains_numa_distance[i])
1840 continue;
1841
1842 cpumask_or(mask, mask, cpumask_of_node(k));
1843 }

--- 37 unchanged lines hidden (view full) ---

1881 }
1882
1883 sched_domain_topology = tl;
1884
1885 sched_domains_numa_levels = nr_levels;
1886 sched_max_numa_distance = sched_domains_numa_distance[nr_levels - 1];
1887
1888 init_numa_topology_type();
1848 if (sched_debug() && (node_distance(j, k) != node_distance(k, j)))
1849 sched_numa_warn("Node-distance not symmetric");
1850
1851 if (node_distance(j, k) > sched_domains_numa_distance[i])
1852 continue;
1853
1854 cpumask_or(mask, mask, cpumask_of_node(k));
1855 }

--- 37 unchanged lines hidden (view full) ---

1893 }
1894
1895 sched_domain_topology = tl;
1896
1897 sched_domains_numa_levels = nr_levels;
1898 sched_max_numa_distance = sched_domains_numa_distance[nr_levels - 1];
1899
1900 init_numa_topology_type();
1901
1902 sched_numa_onlined_nodes = bitmap_alloc(nr_node_ids, GFP_KERNEL);
1903 if (!sched_numa_onlined_nodes)
1904 return;
1905
1906 bitmap_zero(sched_numa_onlined_nodes, nr_node_ids);
1907 for_each_online_node(i)
1908 bitmap_set(sched_numa_onlined_nodes, i, 1);
1889}
1890
1909}
1910
1911static void __sched_domains_numa_masks_set(unsigned int node)
1912{
1913 int i, j;
1914
1915 /*
1916 * NUMA masks are not built for offline nodes in sched_init_numa().
1917 * Thus, when a CPU of a never-onlined-before node gets plugged in,
1918 * adding that new CPU to the right NUMA masks is not sufficient: the
1919 * masks of that CPU's node must also be updated.
1920 */
1921 if (test_bit(node, sched_numa_onlined_nodes))
1922 return;
1923
1924 bitmap_set(sched_numa_onlined_nodes, node, 1);
1925
1926 for (i = 0; i < sched_domains_numa_levels; i++) {
1927 for (j = 0; j < nr_node_ids; j++) {
1928 if (!node_online(j) || node == j)
1929 continue;
1930
1931 if (node_distance(j, node) > sched_domains_numa_distance[i])
1932 continue;
1933
1934 /* Add remote nodes in our masks */
1935 cpumask_or(sched_domains_numa_masks[i][node],
1936 sched_domains_numa_masks[i][node],
1937 sched_domains_numa_masks[0][j]);
1938 }
1939 }
1940
1941 /*
1942 * A new node has been brought up, potentially changing the topology
1943 * classification.
1944 *
1945 * Note that this is racy vs any use of sched_numa_topology_type :/
1946 */
1947 init_numa_topology_type();
1948}
1949
1891void sched_domains_numa_masks_set(unsigned int cpu)
1892{
1893 int node = cpu_to_node(cpu);
1894 int i, j;
1895
1950void sched_domains_numa_masks_set(unsigned int cpu)
1951{
1952 int node = cpu_to_node(cpu);
1953 int i, j;
1954
1955 __sched_domains_numa_masks_set(node);
1956
1896 for (i = 0; i < sched_domains_numa_levels; i++) {
1897 for (j = 0; j < nr_node_ids; j++) {
1957 for (i = 0; i < sched_domains_numa_levels; i++) {
1958 for (j = 0; j < nr_node_ids; j++) {
1959 if (!node_online(j))
1960 continue;
1961
1962 /* Set ourselves in the remote node's masks */
1898 if (node_distance(j, node) <= sched_domains_numa_distance[i])
1899 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
1900 }
1901 }
1902}
1903
1904void sched_domains_numa_masks_clear(unsigned int cpu)
1905{

--- 548 unchanged lines hidden --- 		1963 if (node_distance(j, node) <= sched_domains_numa_distance[i])
1964 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
1965 }
1966 }
1967}
1968
1969void sched_domains_numa_masks_clear(unsigned int cpu)
1970{

--- 548 unchanged lines hidden ---