1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include "sched.h" 11 12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 13 EXPORT_SYMBOL_GPL(housekeeping_overridden); 14 static cpumask_var_t housekeeping_mask; 15 static unsigned int housekeeping_flags; 16 17 int housekeeping_any_cpu(enum hk_flags flags) 18 { 19 if (static_branch_unlikely(&housekeeping_overridden)) 20 if (housekeeping_flags & flags) 21 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 22 return smp_processor_id(); 23 } 24 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 25 26 const struct cpumask *housekeeping_cpumask(enum hk_flags flags) 27 { 28 if (static_branch_unlikely(&housekeeping_overridden)) 29 if (housekeeping_flags & flags) 30 return housekeeping_mask; 31 return cpu_possible_mask; 32 } 33 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 34 35 void housekeeping_affine(struct task_struct *t, enum hk_flags flags) 36 { 37 if (static_branch_unlikely(&housekeeping_overridden)) 38 if (housekeeping_flags & flags) 39 set_cpus_allowed_ptr(t, housekeeping_mask); 40 } 41 EXPORT_SYMBOL_GPL(housekeeping_affine); 42 43 bool housekeeping_test_cpu(int cpu, enum hk_flags flags) 44 { 45 if (static_branch_unlikely(&housekeeping_overridden)) 46 if (housekeeping_flags & flags) 47 return cpumask_test_cpu(cpu, housekeeping_mask); 48 return true; 49 } 50 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 51 52 void __init housekeeping_init(void) 53 { 54 if (!housekeeping_flags) 55 return; 56 57 static_branch_enable(&housekeeping_overridden); 58 59 if (housekeeping_flags & HK_FLAG_TICK) 60 sched_tick_offload_init(); 61 62 /* We need at least one CPU to handle housekeeping work */ 63 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 64 } 65 66 static int __init housekeeping_setup(char *str, enum hk_flags flags) 67 { 68 cpumask_var_t non_housekeeping_mask; 69 cpumask_var_t tmp; 70 int err; 71 72 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 73 err = cpulist_parse(str, non_housekeeping_mask); 74 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { 75 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 76 free_bootmem_cpumask_var(non_housekeeping_mask); 77 return 0; 78 } 79 80 alloc_bootmem_cpumask_var(&tmp); 81 if (!housekeeping_flags) { 82 alloc_bootmem_cpumask_var(&housekeeping_mask); 83 cpumask_andnot(housekeeping_mask, 84 cpu_possible_mask, non_housekeeping_mask); 85 86 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 87 if (cpumask_empty(tmp)) { 88 pr_warn("Housekeeping: must include one present CPU, " 89 "using boot CPU:%d\n", smp_processor_id()); 90 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); 91 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 92 } 93 } else { 94 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 95 if (cpumask_empty(tmp)) 96 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 97 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); 98 if (!cpumask_equal(tmp, housekeeping_mask)) { 99 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 100 free_bootmem_cpumask_var(tmp); 101 free_bootmem_cpumask_var(non_housekeeping_mask); 102 return 0; 103 } 104 } 105 free_bootmem_cpumask_var(tmp); 106 107 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { 108 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { 109 tick_nohz_full_setup(non_housekeeping_mask); 110 } else { 111 pr_warn("Housekeeping: nohz unsupported." 112 " Build with CONFIG_NO_HZ_FULL\n"); 113 free_bootmem_cpumask_var(non_housekeeping_mask); 114 return 0; 115 } 116 } 117 118 housekeeping_flags |= flags; 119 120 free_bootmem_cpumask_var(non_housekeeping_mask); 121 122 return 1; 123 } 124 125 static int __init housekeeping_nohz_full_setup(char *str) 126 { 127 unsigned int flags; 128 129 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; 130 131 return housekeeping_setup(str, flags); 132 } 133 __setup("nohz_full=", housekeeping_nohz_full_setup); 134 135 static int __init housekeeping_isolcpus_setup(char *str) 136 { 137 unsigned int flags = 0; 138 139 while (isalpha(*str)) { 140 if (!strncmp(str, "nohz,", 5)) { 141 str += 5; 142 flags |= HK_FLAG_TICK; 143 continue; 144 } 145 146 if (!strncmp(str, "domain,", 7)) { 147 str += 7; 148 flags |= HK_FLAG_DOMAIN; 149 continue; 150 } 151 152 pr_warn("isolcpus: Error, unknown flag\n"); 153 return 0; 154 } 155 156 /* Default behaviour for isolcpus without flags */ 157 if (!flags) 158 flags |= HK_FLAG_DOMAIN; 159 160 return housekeeping_setup(str, flags); 161 } 162 __setup("isolcpus=", housekeeping_isolcpus_setup); 163