1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include "sched.h" 11 12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 13 EXPORT_SYMBOL_GPL(housekeeping_overridden); 14 static cpumask_var_t housekeeping_mask; 15 static unsigned int housekeeping_flags; 16 17 bool housekeeping_enabled(enum hk_flags flags) 18 { 19 return !!(housekeeping_flags & flags); 20 } 21 EXPORT_SYMBOL_GPL(housekeeping_enabled); 22 23 int housekeeping_any_cpu(enum hk_flags flags) 24 { 25 int cpu; 26 27 if (static_branch_unlikely(&housekeeping_overridden)) { 28 if (housekeeping_flags & flags) { 29 cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id()); 30 if (cpu < nr_cpu_ids) 31 return cpu; 32 33 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 34 } 35 } 36 return smp_processor_id(); 37 } 38 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 39 40 const struct cpumask *housekeeping_cpumask(enum hk_flags flags) 41 { 42 if (static_branch_unlikely(&housekeeping_overridden)) 43 if (housekeeping_flags & flags) 44 return housekeeping_mask; 45 return cpu_possible_mask; 46 } 47 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 48 49 void housekeeping_affine(struct task_struct *t, enum hk_flags flags) 50 { 51 if (static_branch_unlikely(&housekeeping_overridden)) 52 if (housekeeping_flags & flags) 53 set_cpus_allowed_ptr(t, housekeeping_mask); 54 } 55 EXPORT_SYMBOL_GPL(housekeeping_affine); 56 57 bool housekeeping_test_cpu(int cpu, enum hk_flags flags) 58 { 59 if (static_branch_unlikely(&housekeeping_overridden)) 60 if (housekeeping_flags & flags) 61 return cpumask_test_cpu(cpu, housekeeping_mask); 62 return true; 63 } 64 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 65 66 void __init housekeeping_init(void) 67 { 68 if (!housekeeping_flags) 69 return; 70 71 static_branch_enable(&housekeeping_overridden); 72 73 if (housekeeping_flags & HK_FLAG_TICK) 74 sched_tick_offload_init(); 75 76 /* We need at least one CPU to handle housekeeping work */ 77 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 78 } 79 80 static int __init housekeeping_setup(char *str, enum hk_flags flags) 81 { 82 cpumask_var_t non_housekeeping_mask; 83 cpumask_var_t tmp; 84 85 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 86 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 87 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 88 free_bootmem_cpumask_var(non_housekeeping_mask); 89 return 0; 90 } 91 92 alloc_bootmem_cpumask_var(&tmp); 93 if (!housekeeping_flags) { 94 alloc_bootmem_cpumask_var(&housekeeping_mask); 95 cpumask_andnot(housekeeping_mask, 96 cpu_possible_mask, non_housekeeping_mask); 97 98 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 99 if (cpumask_empty(tmp)) { 100 pr_warn("Housekeeping: must include one present CPU, " 101 "using boot CPU:%d\n", smp_processor_id()); 102 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); 103 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 104 } 105 } else { 106 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 107 if (cpumask_empty(tmp)) 108 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 109 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); 110 if (!cpumask_equal(tmp, housekeeping_mask)) { 111 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 112 free_bootmem_cpumask_var(tmp); 113 free_bootmem_cpumask_var(non_housekeeping_mask); 114 return 0; 115 } 116 } 117 free_bootmem_cpumask_var(tmp); 118 119 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { 120 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { 121 tick_nohz_full_setup(non_housekeeping_mask); 122 } else { 123 pr_warn("Housekeeping: nohz unsupported." 124 " Build with CONFIG_NO_HZ_FULL\n"); 125 free_bootmem_cpumask_var(non_housekeeping_mask); 126 return 0; 127 } 128 } 129 130 housekeeping_flags |= flags; 131 132 free_bootmem_cpumask_var(non_housekeeping_mask); 133 134 return 1; 135 } 136 137 static int __init housekeeping_nohz_full_setup(char *str) 138 { 139 unsigned int flags; 140 141 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | 142 HK_FLAG_MISC | HK_FLAG_KTHREAD; 143 144 return housekeeping_setup(str, flags); 145 } 146 __setup("nohz_full=", housekeeping_nohz_full_setup); 147 148 static int __init housekeeping_isolcpus_setup(char *str) 149 { 150 unsigned int flags = 0; 151 bool illegal = false; 152 char *par; 153 int len; 154 155 while (isalpha(*str)) { 156 if (!strncmp(str, "nohz,", 5)) { 157 str += 5; 158 flags |= HK_FLAG_TICK; 159 continue; 160 } 161 162 if (!strncmp(str, "domain,", 7)) { 163 str += 7; 164 flags |= HK_FLAG_DOMAIN; 165 continue; 166 } 167 168 if (!strncmp(str, "managed_irq,", 12)) { 169 str += 12; 170 flags |= HK_FLAG_MANAGED_IRQ; 171 continue; 172 } 173 174 /* 175 * Skip unknown sub-parameter and validate that it is not 176 * containing an invalid character. 177 */ 178 for (par = str, len = 0; *str && *str != ','; str++, len++) { 179 if (!isalpha(*str) && *str != '_') 180 illegal = true; 181 } 182 183 if (illegal) { 184 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 185 return 0; 186 } 187 188 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 189 str++; 190 } 191 192 /* Default behaviour for isolcpus without flags */ 193 if (!flags) 194 flags |= HK_FLAG_DOMAIN; 195 196 return housekeeping_setup(str, flags); 197 } 198 __setup("isolcpus=", housekeeping_isolcpus_setup); 199