1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 11 enum hk_flags { 12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), 13 HK_FLAG_RCU = BIT(HK_TYPE_RCU), 14 HK_FLAG_MISC = BIT(HK_TYPE_MISC), 15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), 16 HK_FLAG_TICK = BIT(HK_TYPE_TICK), 17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 18 HK_FLAG_WQ = BIT(HK_TYPE_WQ), 19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), 21 }; 22 23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 24 EXPORT_SYMBOL_GPL(housekeeping_overridden); 25 26 struct housekeeping { 27 cpumask_var_t cpumasks[HK_TYPE_MAX]; 28 unsigned long flags; 29 }; 30 31 static struct housekeeping housekeeping; 32 33 bool housekeeping_enabled(enum hk_type type) 34 { 35 return !!(housekeeping.flags & BIT(type)); 36 } 37 EXPORT_SYMBOL_GPL(housekeeping_enabled); 38 39 int housekeeping_any_cpu(enum hk_type type) 40 { 41 int cpu; 42 43 if (static_branch_unlikely(&housekeeping_overridden)) { 44 if (housekeeping.flags & BIT(type)) { 45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); 46 if (cpu < nr_cpu_ids) 47 return cpu; 48 49 return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); 50 } 51 } 52 return smp_processor_id(); 53 } 54 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 55 56 const struct cpumask *housekeeping_cpumask(enum hk_type type) 57 { 58 if (static_branch_unlikely(&housekeeping_overridden)) 59 if (housekeeping.flags & BIT(type)) 60 return housekeeping.cpumasks[type]; 61 return cpu_possible_mask; 62 } 63 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 64 65 void housekeeping_affine(struct task_struct *t, enum hk_type type) 66 { 67 if (static_branch_unlikely(&housekeeping_overridden)) 68 if (housekeeping.flags & BIT(type)) 69 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); 70 } 71 EXPORT_SYMBOL_GPL(housekeeping_affine); 72 73 bool housekeeping_test_cpu(int cpu, enum hk_type type) 74 { 75 if (static_branch_unlikely(&housekeeping_overridden)) 76 if (housekeeping.flags & BIT(type)) 77 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); 78 return true; 79 } 80 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 81 82 void __init housekeeping_init(void) 83 { 84 enum hk_type type; 85 86 if (!housekeeping.flags) 87 return; 88 89 static_branch_enable(&housekeeping_overridden); 90 91 if (housekeeping.flags & HK_FLAG_TICK) 92 sched_tick_offload_init(); 93 94 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 95 /* We need at least one CPU to handle housekeeping work */ 96 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); 97 } 98 } 99 100 static void __init housekeeping_setup_type(enum hk_type type, 101 cpumask_var_t housekeeping_staging) 102 { 103 104 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); 105 cpumask_copy(housekeeping.cpumasks[type], 106 housekeeping_staging); 107 } 108 109 static int __init housekeeping_setup(char *str, unsigned long flags) 110 { 111 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 112 unsigned int first_cpu; 113 int err = 0; 114 115 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { 116 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 117 pr_warn("Housekeeping: nohz unsupported." 118 " Build with CONFIG_NO_HZ_FULL\n"); 119 return 0; 120 } 121 } 122 123 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 124 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 125 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 126 goto free_non_housekeeping_mask; 127 } 128 129 alloc_bootmem_cpumask_var(&housekeeping_staging); 130 cpumask_andnot(housekeeping_staging, 131 cpu_possible_mask, non_housekeeping_mask); 132 133 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); 134 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { 135 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 136 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 137 if (!housekeeping.flags) { 138 pr_warn("Housekeeping: must include one present CPU, " 139 "using boot CPU:%d\n", smp_processor_id()); 140 } 141 } 142 143 if (cpumask_empty(non_housekeeping_mask)) 144 goto free_housekeeping_staging; 145 146 if (!housekeeping.flags) { 147 /* First setup call ("nohz_full=" or "isolcpus=") */ 148 enum hk_type type; 149 150 for_each_set_bit(type, &flags, HK_TYPE_MAX) 151 housekeeping_setup_type(type, housekeeping_staging); 152 } else { 153 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 154 enum hk_type type; 155 unsigned long iter_flags = flags & housekeeping.flags; 156 157 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 158 if (!cpumask_equal(housekeeping_staging, 159 housekeeping.cpumasks[type])) { 160 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 161 goto free_housekeeping_staging; 162 } 163 } 164 165 iter_flags = flags & ~housekeeping.flags; 166 167 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 168 housekeeping_setup_type(type, housekeeping_staging); 169 } 170 171 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) 172 tick_nohz_full_setup(non_housekeeping_mask); 173 174 housekeeping.flags |= flags; 175 err = 1; 176 177 free_housekeeping_staging: 178 free_bootmem_cpumask_var(housekeeping_staging); 179 free_non_housekeeping_mask: 180 free_bootmem_cpumask_var(non_housekeeping_mask); 181 182 return err; 183 } 184 185 static int __init housekeeping_nohz_full_setup(char *str) 186 { 187 unsigned long flags; 188 189 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | 190 HK_FLAG_MISC | HK_FLAG_KTHREAD; 191 192 return housekeeping_setup(str, flags); 193 } 194 __setup("nohz_full=", housekeeping_nohz_full_setup); 195 196 static int __init housekeeping_isolcpus_setup(char *str) 197 { 198 unsigned long flags = 0; 199 bool illegal = false; 200 char *par; 201 int len; 202 203 while (isalpha(*str)) { 204 if (!strncmp(str, "nohz,", 5)) { 205 str += 5; 206 flags |= HK_FLAG_TICK; 207 continue; 208 } 209 210 if (!strncmp(str, "domain,", 7)) { 211 str += 7; 212 flags |= HK_FLAG_DOMAIN; 213 continue; 214 } 215 216 if (!strncmp(str, "managed_irq,", 12)) { 217 str += 12; 218 flags |= HK_FLAG_MANAGED_IRQ; 219 continue; 220 } 221 222 /* 223 * Skip unknown sub-parameter and validate that it is not 224 * containing an invalid character. 225 */ 226 for (par = str, len = 0; *str && *str != ','; str++, len++) { 227 if (!isalpha(*str) && *str != '_') 228 illegal = true; 229 } 230 231 if (illegal) { 232 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 233 return 0; 234 } 235 236 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 237 str++; 238 } 239 240 /* Default behaviour for isolcpus without flags */ 241 if (!flags) 242 flags |= HK_FLAG_DOMAIN; 243 244 return housekeeping_setup(str, flags); 245 } 246 __setup("isolcpus=", housekeeping_isolcpus_setup); 247