1 /* 2 * Housekeeping management. Manage the targets for routine code that can run on 3 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 4 * 5 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 6 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 7 * 8 */ 9 #include "sched.h" 10 11 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 12 EXPORT_SYMBOL_GPL(housekeeping_overridden); 13 static cpumask_var_t housekeeping_mask; 14 static unsigned int housekeeping_flags; 15 16 int housekeeping_any_cpu(enum hk_flags flags) 17 { 18 if (static_branch_unlikely(&housekeeping_overridden)) 19 if (housekeeping_flags & flags) 20 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 21 return smp_processor_id(); 22 } 23 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 24 25 const struct cpumask *housekeeping_cpumask(enum hk_flags flags) 26 { 27 if (static_branch_unlikely(&housekeeping_overridden)) 28 if (housekeeping_flags & flags) 29 return housekeeping_mask; 30 return cpu_possible_mask; 31 } 32 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 33 34 void housekeeping_affine(struct task_struct *t, enum hk_flags flags) 35 { 36 if (static_branch_unlikely(&housekeeping_overridden)) 37 if (housekeeping_flags & flags) 38 set_cpus_allowed_ptr(t, housekeeping_mask); 39 } 40 EXPORT_SYMBOL_GPL(housekeeping_affine); 41 42 bool housekeeping_test_cpu(int cpu, enum hk_flags flags) 43 { 44 if (static_branch_unlikely(&housekeeping_overridden)) 45 if (housekeeping_flags & flags) 46 return cpumask_test_cpu(cpu, housekeeping_mask); 47 return true; 48 } 49 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 50 51 void __init housekeeping_init(void) 52 { 53 if (!housekeeping_flags) 54 return; 55 56 static_branch_enable(&housekeeping_overridden); 57 58 if (housekeeping_flags & HK_FLAG_TICK) 59 sched_tick_offload_init(); 60 61 /* We need at least one CPU to handle housekeeping work */ 62 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 63 } 64 65 static int __init housekeeping_setup(char *str, enum hk_flags flags) 66 { 67 cpumask_var_t non_housekeeping_mask; 68 cpumask_var_t tmp; 69 int err; 70 71 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 72 err = cpulist_parse(str, non_housekeeping_mask); 73 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { 74 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 75 free_bootmem_cpumask_var(non_housekeeping_mask); 76 return 0; 77 } 78 79 alloc_bootmem_cpumask_var(&tmp); 80 if (!housekeeping_flags) { 81 alloc_bootmem_cpumask_var(&housekeeping_mask); 82 cpumask_andnot(housekeeping_mask, 83 cpu_possible_mask, non_housekeeping_mask); 84 85 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 86 if (cpumask_empty(tmp)) { 87 pr_warn("Housekeeping: must include one present CPU, " 88 "using boot CPU:%d\n", smp_processor_id()); 89 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); 90 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 91 } 92 } else { 93 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); 94 if (cpumask_empty(tmp)) 95 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 96 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); 97 if (!cpumask_equal(tmp, housekeeping_mask)) { 98 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 99 free_bootmem_cpumask_var(tmp); 100 free_bootmem_cpumask_var(non_housekeeping_mask); 101 return 0; 102 } 103 } 104 free_bootmem_cpumask_var(tmp); 105 106 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { 107 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { 108 tick_nohz_full_setup(non_housekeeping_mask); 109 } else { 110 pr_warn("Housekeeping: nohz unsupported." 111 " Build with CONFIG_NO_HZ_FULL\n"); 112 free_bootmem_cpumask_var(non_housekeeping_mask); 113 return 0; 114 } 115 } 116 117 housekeeping_flags |= flags; 118 119 free_bootmem_cpumask_var(non_housekeeping_mask); 120 121 return 1; 122 } 123 124 static int __init housekeeping_nohz_full_setup(char *str) 125 { 126 unsigned int flags; 127 128 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; 129 130 return housekeeping_setup(str, flags); 131 } 132 __setup("nohz_full=", housekeeping_nohz_full_setup); 133 134 static int __init housekeeping_isolcpus_setup(char *str) 135 { 136 unsigned int flags = 0; 137 138 while (isalpha(*str)) { 139 if (!strncmp(str, "nohz,", 5)) { 140 str += 5; 141 flags |= HK_FLAG_TICK; 142 continue; 143 } 144 145 if (!strncmp(str, "domain,", 7)) { 146 str += 7; 147 flags |= HK_FLAG_DOMAIN; 148 continue; 149 } 150 151 pr_warn("isolcpus: Error, unknown flag\n"); 152 return 0; 153 } 154 155 /* Default behaviour for isolcpus without flags */ 156 if (!flags) 157 flags |= HK_FLAG_DOMAIN; 158 159 return housekeeping_setup(str, flags); 160 } 161 __setup("isolcpus=", housekeeping_isolcpus_setup); 162