1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10
11 enum hk_flags {
12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
13 HK_FLAG_RCU = BIT(HK_TYPE_RCU),
14 HK_FLAG_MISC = BIT(HK_TYPE_MISC),
15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED),
16 HK_FLAG_TICK = BIT(HK_TYPE_TICK),
17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
18 HK_FLAG_WQ = BIT(HK_TYPE_WQ),
19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD),
21 };
22
23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
24 EXPORT_SYMBOL_GPL(housekeeping_overridden);
25
26 struct housekeeping {
27 cpumask_var_t cpumasks[HK_TYPE_MAX];
28 unsigned long flags;
29 };
30
31 static struct housekeeping housekeeping;
32
housekeeping_enabled(enum hk_type type)33 bool housekeeping_enabled(enum hk_type type)
34 {
35 return !!(housekeeping.flags & BIT(type));
36 }
37 EXPORT_SYMBOL_GPL(housekeeping_enabled);
38
housekeeping_any_cpu(enum hk_type type)39 int housekeeping_any_cpu(enum hk_type type)
40 {
41 int cpu;
42
43 if (static_branch_unlikely(&housekeeping_overridden)) {
44 if (housekeeping.flags & BIT(type)) {
45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
46 if (cpu < nr_cpu_ids)
47 return cpu;
48
49 return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
50 }
51 }
52 return smp_processor_id();
53 }
54 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
55
housekeeping_cpumask(enum hk_type type)56 const struct cpumask *housekeeping_cpumask(enum hk_type type)
57 {
58 if (static_branch_unlikely(&housekeeping_overridden))
59 if (housekeeping.flags & BIT(type))
60 return housekeeping.cpumasks[type];
61 return cpu_possible_mask;
62 }
63 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
64
housekeeping_affine(struct task_struct * t,enum hk_type type)65 void housekeeping_affine(struct task_struct *t, enum hk_type type)
66 {
67 if (static_branch_unlikely(&housekeeping_overridden))
68 if (housekeeping.flags & BIT(type))
69 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
70 }
71 EXPORT_SYMBOL_GPL(housekeeping_affine);
72
housekeeping_test_cpu(int cpu,enum hk_type type)73 bool housekeeping_test_cpu(int cpu, enum hk_type type)
74 {
75 if (static_branch_unlikely(&housekeeping_overridden))
76 if (housekeeping.flags & BIT(type))
77 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
78 return true;
79 }
80 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
81
housekeeping_init(void)82 void __init housekeeping_init(void)
83 {
84 enum hk_type type;
85
86 if (!housekeeping.flags)
87 return;
88
89 static_branch_enable(&housekeeping_overridden);
90
91 if (housekeeping.flags & HK_FLAG_TICK)
92 sched_tick_offload_init();
93
94 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
95 /* We need at least one CPU to handle housekeeping work */
96 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
97 }
98 }
99
housekeeping_setup_type(enum hk_type type,cpumask_var_t housekeeping_staging)100 static void __init housekeeping_setup_type(enum hk_type type,
101 cpumask_var_t housekeeping_staging)
102 {
103
104 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
105 cpumask_copy(housekeeping.cpumasks[type],
106 housekeeping_staging);
107 }
108
housekeeping_setup(char * str,unsigned long flags)109 static int __init housekeeping_setup(char *str, unsigned long flags)
110 {
111 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
112 unsigned int first_cpu;
113 int err = 0;
114
115 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
116 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
117 pr_warn("Housekeeping: nohz unsupported."
118 " Build with CONFIG_NO_HZ_FULL\n");
119 return 0;
120 }
121 }
122
123 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
124 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
125 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
126 goto free_non_housekeeping_mask;
127 }
128
129 alloc_bootmem_cpumask_var(&housekeeping_staging);
130 cpumask_andnot(housekeeping_staging,
131 cpu_possible_mask, non_housekeeping_mask);
132
133 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
134 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
135 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
136 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
137 if (!housekeeping.flags) {
138 pr_warn("Housekeeping: must include one present CPU, "
139 "using boot CPU:%d\n", smp_processor_id());
140 }
141 }
142
143 if (cpumask_empty(non_housekeeping_mask))
144 goto free_housekeeping_staging;
145
146 if (!housekeeping.flags) {
147 /* First setup call ("nohz_full=" or "isolcpus=") */
148 enum hk_type type;
149
150 for_each_set_bit(type, &flags, HK_TYPE_MAX)
151 housekeeping_setup_type(type, housekeeping_staging);
152 } else {
153 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
154 enum hk_type type;
155 unsigned long iter_flags = flags & housekeeping.flags;
156
157 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
158 if (!cpumask_equal(housekeeping_staging,
159 housekeeping.cpumasks[type])) {
160 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
161 goto free_housekeeping_staging;
162 }
163 }
164
165 iter_flags = flags & ~housekeeping.flags;
166
167 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
168 housekeeping_setup_type(type, housekeeping_staging);
169 }
170
171 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
172 tick_nohz_full_setup(non_housekeeping_mask);
173
174 housekeeping.flags |= flags;
175 err = 1;
176
177 free_housekeeping_staging:
178 free_bootmem_cpumask_var(housekeeping_staging);
179 free_non_housekeeping_mask:
180 free_bootmem_cpumask_var(non_housekeeping_mask);
181
182 return err;
183 }
184
housekeeping_nohz_full_setup(char * str)185 static int __init housekeeping_nohz_full_setup(char *str)
186 {
187 unsigned long flags;
188
189 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
190 HK_FLAG_MISC | HK_FLAG_KTHREAD;
191
192 return housekeeping_setup(str, flags);
193 }
194 __setup("nohz_full=", housekeeping_nohz_full_setup);
195
housekeeping_isolcpus_setup(char * str)196 static int __init housekeeping_isolcpus_setup(char *str)
197 {
198 unsigned long flags = 0;
199 bool illegal = false;
200 char *par;
201 int len;
202
203 while (isalpha(*str)) {
204 if (!strncmp(str, "nohz,", 5)) {
205 str += 5;
206 flags |= HK_FLAG_TICK;
207 continue;
208 }
209
210 if (!strncmp(str, "domain,", 7)) {
211 str += 7;
212 flags |= HK_FLAG_DOMAIN;
213 continue;
214 }
215
216 if (!strncmp(str, "managed_irq,", 12)) {
217 str += 12;
218 flags |= HK_FLAG_MANAGED_IRQ;
219 continue;
220 }
221
222 /*
223 * Skip unknown sub-parameter and validate that it is not
224 * containing an invalid character.
225 */
226 for (par = str, len = 0; *str && *str != ','; str++, len++) {
227 if (!isalpha(*str) && *str != '_')
228 illegal = true;
229 }
230
231 if (illegal) {
232 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
233 return 0;
234 }
235
236 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
237 str++;
238 }
239
240 /* Default behaviour for isolcpus without flags */
241 if (!flags)
242 flags |= HK_FLAG_DOMAIN;
243
244 return housekeeping_setup(str, flags);
245 }
246 __setup("isolcpus=", housekeeping_isolcpus_setup);
247