1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include "sched.h" 11 12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 13 EXPORT_SYMBOL_GPL(housekeeping_overridden); 14 static cpumask_var_t housekeeping_mask; 15 static unsigned int housekeeping_flags; 16 17 bool housekeeping_enabled(enum hk_flags flags) 18 { 19 return !!(housekeeping_flags & flags); 20 } 21 EXPORT_SYMBOL_GPL(housekeeping_enabled); 22 23 int housekeeping_any_cpu(enum hk_flags flags) 24 { 25 if (static_branch_unlikely(&housekeeping_overridden)) 26 if (housekeeping_flags & flags) 27 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 28 return smp_processor_id(); 29 } 30 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 31 32 const struct cpumask *housekeeping_cpumask(enum hk_flags flags) 33 { 34 if (static_branch_unlikely(&housekeeping_overridden)) 35 if (housekeeping_flags & flags) 36 return housekeeping_mask; 37 return cpu_possible_mask; 38 } 39 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 40 41 void housekeeping_affine(struct task_struct *t, enum hk_flags flags) 42 { 43 if (static_branch_unlikely(&housekeeping_overridden)) 44 if (housekeeping_flags & flags) 45 set_cpus_allowed_ptr(t, housekeeping_mask); 46 } 47 EXPORT_SYMBOL_GPL(housekeeping_affine); 48 49 bool housekeeping_test_cpu(int cpu, enum hk_flags flags) 50 { 51 if (static_branch_unlikely(&housekeeping_overridden)) 52 if (housekeeping_flags & flags) 53 return cpumask_test_cpu(cpu, housekeeping_mask); 54 return true; 55 } 56 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 57 58 void __init housekeeping_init(void) 59 { 60 if (!housekeeping_flags) 61 return; 62 63 static_branch_enable(&housekeeping_overridden); 64 65 if (housekeeping_flags & HK_FLAG_TICK) 66 sched_tick_offload_init(); 67 68 /* We need at least one CPU to handle housekeeping work */ 69 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 70 } 71 72 static int __init housekeeping_setup(char *str, enum hk_flags flags) 73 { 74 cpumask_var_t non_housekeeping_mask; 75 cpumask_var_t tmp; 76 int err; 77 78 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 79 err = cpulist_parse(str, non_housekeeping_mask); 80 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { 81 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 82 free_bootmem_cpumask_var(non_housekeeping_mask); 83 return 0; 84 } 85 86 alloc_bootmem_cpumask_var(&tmp); 87 if (!housekeeping_flags) { 88 alloc_bootmem_cpumask_var(&housekeeping_mask); 89 cpumask_andnot(housekeeping_mask, 90 cpu_possible_mask, non_housekeeping_mask); 91 92 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 93 if (cpumask_empty(tmp)) { 94 pr_warn("Housekeeping: must include one present CPU, " 95 "using boot CPU:%d\n", smp_processor_id()); 96 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); 97 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 98 } 99 } else { 100 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 101 if (cpumask_empty(tmp)) 102 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 103 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); 104 if (!cpumask_equal(tmp, housekeeping_mask)) { 105 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 106 free_bootmem_cpumask_var(tmp); 107 free_bootmem_cpumask_var(non_housekeeping_mask); 108 return 0; 109 } 110 } 111 free_bootmem_cpumask_var(tmp); 112 113 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { 114 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { 115 tick_nohz_full_setup(non_housekeeping_mask); 116 } else { 117 pr_warn("Housekeeping: nohz unsupported." 118 " Build with CONFIG_NO_HZ_FULL\n"); 119 free_bootmem_cpumask_var(non_housekeeping_mask); 120 return 0; 121 } 122 } 123 124 housekeeping_flags |= flags; 125 126 free_bootmem_cpumask_var(non_housekeeping_mask); 127 128 return 1; 129 } 130 131 static int __init housekeeping_nohz_full_setup(char *str) 132 { 133 unsigned int flags; 134 135 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; 136 137 return housekeeping_setup(str, flags); 138 } 139 __setup("nohz_full=", housekeeping_nohz_full_setup); 140 141 static int __init housekeeping_isolcpus_setup(char *str) 142 { 143 unsigned int flags = 0; 144 145 while (isalpha(*str)) { 146 if (!strncmp(str, "nohz,", 5)) { 147 str += 5; 148 flags |= HK_FLAG_TICK; 149 continue; 150 } 151 152 if (!strncmp(str, "domain,", 7)) { 153 str += 7; 154 flags |= HK_FLAG_DOMAIN; 155 continue; 156 } 157 158 pr_warn("isolcpus: Error, unknown flag\n"); 159 return 0; 160 } 161 162 /* Default behaviour for isolcpus without flags */ 163 if (!flags) 164 flags |= HK_FLAG_DOMAIN; 165 166 return housekeeping_setup(str, flags); 167 } 168 __setup("isolcpus=", housekeeping_isolcpus_setup); 169