1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCHED_TOPOLOGY_H 3 #define _LINUX_SCHED_TOPOLOGY_H 4 5 #include <linux/topology.h> 6 7 #include <linux/sched/idle.h> 8 9 /* 10 * sched-domains (multiprocessor balancing) declarations: 11 */ 12 #ifdef CONFIG_SMP 13 14 #define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ 15 #define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ 16 #define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ 17 #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ 18 #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ 19 #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ 20 #define SD_ASYM_CPUCAPACITY 0x0040 /* Domain members have different CPU capacities */ 21 #define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share CPU capacity */ 22 #define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ 23 #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share CPU pkg resources */ 24 #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ 25 #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ 26 #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ 27 #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ 28 #define SD_NUMA 0x4000 /* cross-node balancing */ 29 30 #ifdef CONFIG_SCHED_SMT 31 static inline int cpu_smt_flags(void) 32 { 33 return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; 34 } 35 #endif 36 37 #ifdef CONFIG_SCHED_MC 38 static inline int cpu_core_flags(void) 39 { 40 return SD_SHARE_PKG_RESOURCES; 41 } 42 #endif 43 44 #ifdef CONFIG_NUMA 45 static inline int cpu_numa_flags(void) 46 { 47 return SD_NUMA; 48 } 49 #endif 50 51 extern int arch_asym_cpu_priority(int cpu); 52 53 struct sched_domain_attr { 54 int relax_domain_level; 55 }; 56 57 #define SD_ATTR_INIT (struct sched_domain_attr) { \ 58 .relax_domain_level = -1, \ 59 } 60 61 extern int sched_domain_level_max; 62 63 struct sched_group; 64 65 struct sched_domain_shared { 66 atomic_t ref; 67 atomic_t nr_busy_cpus; 68 int has_idle_cores; 69 }; 70 71 struct sched_domain { 72 /* These fields must be setup */ 73 struct sched_domain __rcu *parent; /* top domain must be null terminated */ 74 struct sched_domain __rcu *child; /* bottom domain must be null terminated */ 75 struct sched_group *groups; /* the balancing groups of the domain */ 76 unsigned long min_interval; /* Minimum balance interval ms */ 77 unsigned long max_interval; /* Maximum balance interval ms */ 78 unsigned int busy_factor; /* less balancing by factor if busy */ 79 unsigned int imbalance_pct; /* No balance until over watermark */ 80 unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */ 81 82 int nohz_idle; /* NOHZ IDLE status */ 83 int flags; /* See SD_* */ 84 int level; 85 86 /* Runtime fields. */ 87 unsigned long last_balance; /* init to jiffies. units in jiffies */ 88 unsigned int balance_interval; /* initialise to 1. units in ms. */ 89 unsigned int nr_balance_failed; /* initialise to 0 */ 90 91 /* idle_balance() stats */ 92 u64 max_newidle_lb_cost; 93 unsigned long next_decay_max_lb_cost; 94 95 u64 avg_scan_cost; /* select_idle_sibling */ 96 97 #ifdef CONFIG_SCHEDSTATS 98 /* load_balance() stats */ 99 unsigned int lb_count[CPU_MAX_IDLE_TYPES]; 100 unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; 101 unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; 102 unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; 103 unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; 104 unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; 105 unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; 106 unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; 107 108 /* Active load balancing */ 109 unsigned int alb_count; 110 unsigned int alb_failed; 111 unsigned int alb_pushed; 112 113 /* SD_BALANCE_EXEC stats */ 114 unsigned int sbe_count; 115 unsigned int sbe_balanced; 116 unsigned int sbe_pushed; 117 118 /* SD_BALANCE_FORK stats */ 119 unsigned int sbf_count; 120 unsigned int sbf_balanced; 121 unsigned int sbf_pushed; 122 123 /* try_to_wake_up() stats */ 124 unsigned int ttwu_wake_remote; 125 unsigned int ttwu_move_affine; 126 unsigned int ttwu_move_balance; 127 #endif 128 #ifdef CONFIG_SCHED_DEBUG 129 char *name; 130 #endif 131 union { 132 void *private; /* used during construction */ 133 struct rcu_head rcu; /* used during destruction */ 134 }; 135 struct sched_domain_shared *shared; 136 137 unsigned int span_weight; 138 /* 139 * Span of all CPUs in this domain. 140 * 141 * NOTE: this field is variable length. (Allocated dynamically 142 * by attaching extra space to the end of the structure, 143 * depending on how many CPUs the kernel has booted up with) 144 */ 145 unsigned long span[]; 146 }; 147 148 static inline struct cpumask *sched_domain_span(struct sched_domain *sd) 149 { 150 return to_cpumask(sd->span); 151 } 152 153 extern void partition_sched_domains_locked(int ndoms_new, 154 cpumask_var_t doms_new[], 155 struct sched_domain_attr *dattr_new); 156 157 extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 158 struct sched_domain_attr *dattr_new); 159 160 /* Allocate an array of sched domains, for partition_sched_domains(). */ 161 cpumask_var_t *alloc_sched_domains(unsigned int ndoms); 162 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); 163 164 bool cpus_share_cache(int this_cpu, int that_cpu); 165 166 typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); 167 typedef int (*sched_domain_flags_f)(void); 168 169 #define SDTL_OVERLAP 0x01 170 171 struct sd_data { 172 struct sched_domain *__percpu *sd; 173 struct sched_domain_shared *__percpu *sds; 174 struct sched_group *__percpu *sg; 175 struct sched_group_capacity *__percpu *sgc; 176 }; 177 178 struct sched_domain_topology_level { 179 sched_domain_mask_f mask; 180 sched_domain_flags_f sd_flags; 181 int flags; 182 int numa_level; 183 struct sd_data data; 184 #ifdef CONFIG_SCHED_DEBUG 185 char *name; 186 #endif 187 }; 188 189 extern void set_sched_topology(struct sched_domain_topology_level *tl); 190 191 #ifdef CONFIG_SCHED_DEBUG 192 # define SD_INIT_NAME(type) .name = #type 193 #else 194 # define SD_INIT_NAME(type) 195 #endif 196 197 #else /* CONFIG_SMP */ 198 199 struct sched_domain_attr; 200 201 static inline void 202 partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[], 203 struct sched_domain_attr *dattr_new) 204 { 205 } 206 207 static inline void 208 partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 209 struct sched_domain_attr *dattr_new) 210 { 211 } 212 213 static inline bool cpus_share_cache(int this_cpu, int that_cpu) 214 { 215 return true; 216 } 217 218 #endif /* !CONFIG_SMP */ 219 220 #ifndef arch_scale_cpu_capacity 221 static __always_inline 222 unsigned long arch_scale_cpu_capacity(int cpu) 223 { 224 return SCHED_CAPACITY_SCALE; 225 } 226 #endif 227 228 #ifndef arch_scale_thermal_pressure 229 static __always_inline 230 unsigned long arch_scale_thermal_pressure(int cpu) 231 { 232 return 0; 233 } 234 #endif 235 236 static inline int task_node(const struct task_struct *p) 237 { 238 return cpu_to_node(task_cpu(p)); 239 } 240 241 #endif /* _LINUX_SCHED_TOPOLOGY_H */ 242