1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include "sched.h" 11 12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 13 EXPORT_SYMBOL_GPL(housekeeping_overridden); 14 static cpumask_var_t housekeeping_mask; 15 static unsigned int housekeeping_flags; 16 17 bool housekeeping_enabled(enum hk_flags flags) 18 { 19 return !!(housekeeping_flags & flags); 20 } 21 EXPORT_SYMBOL_GPL(housekeeping_enabled); 22 23 int housekeeping_any_cpu(enum hk_flags flags) 24 { 25 int cpu; 26 27 if (static_branch_unlikely(&housekeeping_overridden)) { 28 if (housekeeping_flags & flags) { 29 cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id()); 30 if (cpu < nr_cpu_ids) 31 return cpu; 32 33 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 34 } 35 } 36 return smp_processor_id(); 37 } 38 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 39 40 const struct cpumask *housekeeping_cpumask(enum hk_flags flags) 41 { 42 if (static_branch_unlikely(&housekeeping_overridden)) 43 if (housekeeping_flags & flags) 44 return housekeeping_mask; 45 return cpu_possible_mask; 46 } 47 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 48 49 void housekeeping_affine(struct task_struct *t, enum hk_flags flags) 50 { 51 if (static_branch_unlikely(&housekeeping_overridden)) 52 if (housekeeping_flags & flags) 53 set_cpus_allowed_ptr(t, housekeeping_mask); 54 } 55 EXPORT_SYMBOL_GPL(housekeeping_affine); 56 57 bool housekeeping_test_cpu(int cpu, enum hk_flags flags) 58 { 59 if (static_branch_unlikely(&housekeeping_overridden)) 60 if (housekeeping_flags & flags) 61 return cpumask_test_cpu(cpu, housekeeping_mask); 62 return true; 63 } 64 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 65 66 void __init housekeeping_init(void) 67 { 68 if (!housekeeping_flags) 69 return; 70 71 static_branch_enable(&housekeeping_overridden); 72 73 if (housekeeping_flags & HK_FLAG_TICK) 74 sched_tick_offload_init(); 75 76 /* We need at least one CPU to handle housekeeping work */ 77 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 78 } 79 80 static int __init housekeeping_setup(char *str, enum hk_flags flags) 81 { 82 cpumask_var_t non_housekeeping_mask; 83 cpumask_var_t tmp; 84 int err; 85 86 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 87 err = cpulist_parse(str, non_housekeeping_mask); 88 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { 89 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 90 free_bootmem_cpumask_var(non_housekeeping_mask); 91 return 0; 92 } 93 94 alloc_bootmem_cpumask_var(&tmp); 95 if (!housekeeping_flags) { 96 alloc_bootmem_cpumask_var(&housekeeping_mask); 97 cpumask_andnot(housekeeping_mask, 98 cpu_possible_mask, non_housekeeping_mask); 99 100 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 101 if (cpumask_empty(tmp)) { 102 pr_warn("Housekeeping: must include one present CPU, " 103 "using boot CPU:%d\n", smp_processor_id()); 104 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); 105 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 106 } 107 } else { 108 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 109 if (cpumask_empty(tmp)) 110 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 111 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); 112 if (!cpumask_equal(tmp, housekeeping_mask)) { 113 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 114 free_bootmem_cpumask_var(tmp); 115 free_bootmem_cpumask_var(non_housekeeping_mask); 116 return 0; 117 } 118 } 119 free_bootmem_cpumask_var(tmp); 120 121 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { 122 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { 123 tick_nohz_full_setup(non_housekeeping_mask); 124 } else { 125 pr_warn("Housekeeping: nohz unsupported." 126 " Build with CONFIG_NO_HZ_FULL\n"); 127 free_bootmem_cpumask_var(non_housekeeping_mask); 128 return 0; 129 } 130 } 131 132 housekeeping_flags |= flags; 133 134 free_bootmem_cpumask_var(non_housekeeping_mask); 135 136 return 1; 137 } 138 139 static int __init housekeeping_nohz_full_setup(char *str) 140 { 141 unsigned int flags; 142 143 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; 144 145 return housekeeping_setup(str, flags); 146 } 147 __setup("nohz_full=", housekeeping_nohz_full_setup); 148 149 static int __init housekeeping_isolcpus_setup(char *str) 150 { 151 unsigned int flags = 0; 152 153 while (isalpha(*str)) { 154 if (!strncmp(str, "nohz,", 5)) { 155 str += 5; 156 flags |= HK_FLAG_TICK; 157 continue; 158 } 159 160 if (!strncmp(str, "domain,", 7)) { 161 str += 7; 162 flags |= HK_FLAG_DOMAIN; 163 continue; 164 } 165 166 pr_warn("isolcpus: Error, unknown flag\n"); 167 return 0; 168 } 169 170 /* Default behaviour for isolcpus without flags */ 171 if (!flags) 172 flags |= HK_FLAG_DOMAIN; 173 174 return housekeeping_setup(str, flags); 175 } 176 __setup("isolcpus=", housekeeping_isolcpus_setup); 177