1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 11 enum hk_flags { 12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), 13 HK_FLAG_RCU = BIT(HK_TYPE_RCU), 14 HK_FLAG_MISC = BIT(HK_TYPE_MISC), 15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), 16 HK_FLAG_TICK = BIT(HK_TYPE_TICK), 17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 18 HK_FLAG_WQ = BIT(HK_TYPE_WQ), 19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), 21 }; 22 23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 24 EXPORT_SYMBOL_GPL(housekeeping_overridden); 25 26 struct housekeeping { 27 cpumask_var_t cpumasks[HK_TYPE_MAX]; 28 unsigned long flags; 29 }; 30 31 static struct housekeeping housekeeping; 32 33 bool housekeeping_enabled(enum hk_type type) 34 { 35 return !!(housekeeping.flags & BIT(type)); 36 } 37 EXPORT_SYMBOL_GPL(housekeeping_enabled); 38 39 int housekeeping_any_cpu(enum hk_type type) 40 { 41 int cpu; 42 43 if (static_branch_unlikely(&housekeeping_overridden)) { 44 if (housekeeping.flags & BIT(type)) { 45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); 46 if (cpu < nr_cpu_ids) 47 return cpu; 48 49 return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); 50 } 51 } 52 return smp_processor_id(); 53 } 54 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 55 56 const struct cpumask *housekeeping_cpumask(enum hk_type type) 57 { 58 if (static_branch_unlikely(&housekeeping_overridden)) 59 if (housekeeping.flags & BIT(type)) 60 return housekeeping.cpumasks[type]; 61 return cpu_possible_mask; 62 } 63 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 64 65 void housekeeping_affine(struct task_struct *t, enum hk_type type) 66 { 67 if (static_branch_unlikely(&housekeeping_overridden)) 68 if (housekeeping.flags & BIT(type)) 69 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); 70 } 71 EXPORT_SYMBOL_GPL(housekeeping_affine); 72 73 bool housekeeping_test_cpu(int cpu, enum hk_type type) 74 { 75 if (static_branch_unlikely(&housekeeping_overridden)) 76 if (housekeeping.flags & BIT(type)) 77 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); 78 return true; 79 } 80 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 81 82 void __init housekeeping_init(void) 83 { 84 enum hk_type type; 85 86 if (!housekeeping.flags) 87 return; 88 89 static_branch_enable(&housekeeping_overridden); 90 91 if (housekeeping.flags & HK_FLAG_TICK) 92 sched_tick_offload_init(); 93 94 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 95 /* We need at least one CPU to handle housekeeping work */ 96 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); 97 } 98 } 99 100 static void __init housekeeping_setup_type(enum hk_type type, 101 cpumask_var_t housekeeping_staging) 102 { 103 104 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); 105 cpumask_copy(housekeeping.cpumasks[type], 106 housekeeping_staging); 107 } 108 109 static int __init housekeeping_setup(char *str, unsigned long flags) 110 { 111 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 112 int err = 0; 113 114 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { 115 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 116 pr_warn("Housekeeping: nohz unsupported." 117 " Build with CONFIG_NO_HZ_FULL\n"); 118 return 0; 119 } 120 } 121 122 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 123 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 124 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 125 goto free_non_housekeeping_mask; 126 } 127 128 alloc_bootmem_cpumask_var(&housekeeping_staging); 129 cpumask_andnot(housekeeping_staging, 130 cpu_possible_mask, non_housekeeping_mask); 131 132 if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) { 133 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 134 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 135 if (!housekeeping.flags) { 136 pr_warn("Housekeeping: must include one present CPU, " 137 "using boot CPU:%d\n", smp_processor_id()); 138 } 139 } 140 141 if (!housekeeping.flags) { 142 /* First setup call ("nohz_full=" or "isolcpus=") */ 143 enum hk_type type; 144 145 for_each_set_bit(type, &flags, HK_TYPE_MAX) 146 housekeeping_setup_type(type, housekeeping_staging); 147 } else { 148 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 149 enum hk_type type; 150 unsigned long iter_flags = flags & housekeeping.flags; 151 152 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 153 if (!cpumask_equal(housekeeping_staging, 154 housekeeping.cpumasks[type])) { 155 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 156 goto free_housekeeping_staging; 157 } 158 } 159 160 iter_flags = flags & ~housekeeping.flags; 161 162 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 163 housekeeping_setup_type(type, housekeeping_staging); 164 } 165 166 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) 167 tick_nohz_full_setup(non_housekeeping_mask); 168 169 housekeeping.flags |= flags; 170 err = 1; 171 172 free_housekeeping_staging: 173 free_bootmem_cpumask_var(housekeeping_staging); 174 free_non_housekeeping_mask: 175 free_bootmem_cpumask_var(non_housekeeping_mask); 176 177 return err; 178 } 179 180 static int __init housekeeping_nohz_full_setup(char *str) 181 { 182 unsigned long flags; 183 184 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | 185 HK_FLAG_MISC | HK_FLAG_KTHREAD; 186 187 return housekeeping_setup(str, flags); 188 } 189 __setup("nohz_full=", housekeeping_nohz_full_setup); 190 191 static int __init housekeeping_isolcpus_setup(char *str) 192 { 193 unsigned long flags = 0; 194 bool illegal = false; 195 char *par; 196 int len; 197 198 while (isalpha(*str)) { 199 if (!strncmp(str, "nohz,", 5)) { 200 str += 5; 201 flags |= HK_FLAG_TICK; 202 continue; 203 } 204 205 if (!strncmp(str, "domain,", 7)) { 206 str += 7; 207 flags |= HK_FLAG_DOMAIN; 208 continue; 209 } 210 211 if (!strncmp(str, "managed_irq,", 12)) { 212 str += 12; 213 flags |= HK_FLAG_MANAGED_IRQ; 214 continue; 215 } 216 217 /* 218 * Skip unknown sub-parameter and validate that it is not 219 * containing an invalid character. 220 */ 221 for (par = str, len = 0; *str && *str != ','; str++, len++) { 222 if (!isalpha(*str) && *str != '_') 223 illegal = true; 224 } 225 226 if (illegal) { 227 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 228 return 0; 229 } 230 231 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 232 str++; 233 } 234 235 /* Default behaviour for isolcpus without flags */ 236 if (!flags) 237 flags |= HK_FLAG_DOMAIN; 238 239 return housekeeping_setup(str, flags); 240 } 241 __setup("isolcpus=", housekeeping_isolcpus_setup); 242