1 /*
2  * Copyright(c) 2015 - 2018 Intel Corporation.
3  *
4  * This file is provided under a dual BSD/GPLv2 license.  When using or
5  * redistributing this file, you may do so under either license.
6  *
7  * GPL LICENSE SUMMARY
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of version 2 of the GNU General Public License as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  * BSD LICENSE
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions
22  * are met:
23  *
24  *  - Redistributions of source code must retain the above copyright
25  *    notice, this list of conditions and the following disclaimer.
26  *  - Redistributions in binary form must reproduce the above copyright
27  *    notice, this list of conditions and the following disclaimer in
28  *    the documentation and/or other materials provided with the
29  *    distribution.
30  *  - Neither the name of Intel Corporation nor the names of its
31  *    contributors may be used to endorse or promote products derived
32  *    from this software without specific prior written permission.
33  *
34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  *
46  */
47 #include <linux/topology.h>
48 #include <linux/cpumask.h>
49 #include <linux/module.h>
50 #include <linux/interrupt.h>
51 #include <linux/numa.h>
52 
53 #include "hfi.h"
54 #include "affinity.h"
55 #include "sdma.h"
56 #include "trace.h"
57 
58 struct hfi1_affinity_node_list node_affinity = {
59 	.list = LIST_HEAD_INIT(node_affinity.list),
60 	.lock = __MUTEX_INITIALIZER(node_affinity.lock)
61 };
62 
63 /* Name of IRQ types, indexed by enum irq_type */
64 static const char * const irq_type_names[] = {
65 	"SDMA",
66 	"RCVCTXT",
67 	"GENERAL",
68 	"OTHER",
69 };
70 
71 /* Per NUMA node count of HFI devices */
72 static unsigned int *hfi1_per_node_cntr;
73 
74 static inline void init_cpu_mask_set(struct cpu_mask_set *set)
75 {
76 	cpumask_clear(&set->mask);
77 	cpumask_clear(&set->used);
78 	set->gen = 0;
79 }
80 
81 /* Increment generation of CPU set if needed */
82 static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
83 {
84 	if (cpumask_equal(&set->mask, &set->used)) {
85 		/*
86 		 * We've used up all the CPUs, bump up the generation
87 		 * and reset the 'used' map
88 		 */
89 		set->gen++;
90 		cpumask_clear(&set->used);
91 	}
92 }
93 
94 static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
95 {
96 	if (cpumask_empty(&set->used) && set->gen) {
97 		set->gen--;
98 		cpumask_copy(&set->used, &set->mask);
99 	}
100 }
101 
102 /* Get the first CPU from the list of unused CPUs in a CPU set data structure */
103 static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
104 {
105 	int cpu;
106 
107 	if (!diff || !set)
108 		return -EINVAL;
109 
110 	_cpu_mask_set_gen_inc(set);
111 
112 	/* Find out CPUs left in CPU mask */
113 	cpumask_andnot(diff, &set->mask, &set->used);
114 
115 	cpu = cpumask_first(diff);
116 	if (cpu >= nr_cpu_ids) /* empty */
117 		cpu = -EINVAL;
118 	else
119 		cpumask_set_cpu(cpu, &set->used);
120 
121 	return cpu;
122 }
123 
124 static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
125 {
126 	if (!set)
127 		return;
128 
129 	cpumask_clear_cpu(cpu, &set->used);
130 	_cpu_mask_set_gen_dec(set);
131 }
132 
133 /* Initialize non-HT cpu cores mask */
134 void init_real_cpu_mask(void)
135 {
136 	int possible, curr_cpu, i, ht;
137 
138 	cpumask_clear(&node_affinity.real_cpu_mask);
139 
140 	/* Start with cpu online mask as the real cpu mask */
141 	cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
142 
143 	/*
144 	 * Remove HT cores from the real cpu mask.  Do this in two steps below.
145 	 */
146 	possible = cpumask_weight(&node_affinity.real_cpu_mask);
147 	ht = cpumask_weight(topology_sibling_cpumask(
148 				cpumask_first(&node_affinity.real_cpu_mask)));
149 	/*
150 	 * Step 1.  Skip over the first N HT siblings and use them as the
151 	 * "real" cores.  Assumes that HT cores are not enumerated in
152 	 * succession (except in the single core case).
153 	 */
154 	curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
155 	for (i = 0; i < possible / ht; i++)
156 		curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
157 	/*
158 	 * Step 2.  Remove the remaining HT siblings.  Use cpumask_next() to
159 	 * skip any gaps.
160 	 */
161 	for (; i < possible; i++) {
162 		cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
163 		curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
164 	}
165 }
166 
167 int node_affinity_init(void)
168 {
169 	int node;
170 	struct pci_dev *dev = NULL;
171 	const struct pci_device_id *ids = hfi1_pci_tbl;
172 
173 	cpumask_clear(&node_affinity.proc.used);
174 	cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
175 
176 	node_affinity.proc.gen = 0;
177 	node_affinity.num_core_siblings =
178 				cpumask_weight(topology_sibling_cpumask(
179 					cpumask_first(&node_affinity.proc.mask)
180 					));
181 	node_affinity.num_possible_nodes = num_possible_nodes();
182 	node_affinity.num_online_nodes = num_online_nodes();
183 	node_affinity.num_online_cpus = num_online_cpus();
184 
185 	/*
186 	 * The real cpu mask is part of the affinity struct but it has to be
187 	 * initialized early. It is needed to calculate the number of user
188 	 * contexts in set_up_context_variables().
189 	 */
190 	init_real_cpu_mask();
191 
192 	hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
193 				     sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
194 	if (!hfi1_per_node_cntr)
195 		return -ENOMEM;
196 
197 	while (ids->vendor) {
198 		dev = NULL;
199 		while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
200 			node = pcibus_to_node(dev->bus);
201 			if (node < 0)
202 				goto out;
203 
204 			hfi1_per_node_cntr[node]++;
205 		}
206 		ids++;
207 	}
208 
209 	return 0;
210 
211 out:
212 	/*
213 	 * Invalid PCI NUMA node information found, note it, and populate
214 	 * our database 1:1.
215 	 */
216 	pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
217 	pr_err("HFI: System BIOS may need to be upgraded\n");
218 	for (node = 0; node < node_affinity.num_possible_nodes; node++)
219 		hfi1_per_node_cntr[node] = 1;
220 
221 	return 0;
222 }
223 
224 static void node_affinity_destroy(struct hfi1_affinity_node *entry)
225 {
226 	free_percpu(entry->comp_vect_affinity);
227 	kfree(entry);
228 }
229 
230 void node_affinity_destroy_all(void)
231 {
232 	struct list_head *pos, *q;
233 	struct hfi1_affinity_node *entry;
234 
235 	mutex_lock(&node_affinity.lock);
236 	list_for_each_safe(pos, q, &node_affinity.list) {
237 		entry = list_entry(pos, struct hfi1_affinity_node,
238 				   list);
239 		list_del(pos);
240 		node_affinity_destroy(entry);
241 	}
242 	mutex_unlock(&node_affinity.lock);
243 	kfree(hfi1_per_node_cntr);
244 }
245 
246 static struct hfi1_affinity_node *node_affinity_allocate(int node)
247 {
248 	struct hfi1_affinity_node *entry;
249 
250 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
251 	if (!entry)
252 		return NULL;
253 	entry->node = node;
254 	entry->comp_vect_affinity = alloc_percpu(u16);
255 	INIT_LIST_HEAD(&entry->list);
256 
257 	return entry;
258 }
259 
260 /*
261  * It appends an entry to the list.
262  * It *must* be called with node_affinity.lock held.
263  */
264 static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
265 {
266 	list_add_tail(&entry->list, &node_affinity.list);
267 }
268 
269 /* It must be called with node_affinity.lock held */
270 static struct hfi1_affinity_node *node_affinity_lookup(int node)
271 {
272 	struct list_head *pos;
273 	struct hfi1_affinity_node *entry;
274 
275 	list_for_each(pos, &node_affinity.list) {
276 		entry = list_entry(pos, struct hfi1_affinity_node, list);
277 		if (entry->node == node)
278 			return entry;
279 	}
280 
281 	return NULL;
282 }
283 
284 static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
285 				u16 __percpu *comp_vect_affinity)
286 {
287 	int curr_cpu;
288 	u16 cntr;
289 	u16 prev_cntr;
290 	int ret_cpu;
291 
292 	if (!possible_cpumask) {
293 		ret_cpu = -EINVAL;
294 		goto fail;
295 	}
296 
297 	if (!comp_vect_affinity) {
298 		ret_cpu = -EINVAL;
299 		goto fail;
300 	}
301 
302 	ret_cpu = cpumask_first(possible_cpumask);
303 	if (ret_cpu >= nr_cpu_ids) {
304 		ret_cpu = -EINVAL;
305 		goto fail;
306 	}
307 
308 	prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
309 	for_each_cpu(curr_cpu, possible_cpumask) {
310 		cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
311 
312 		if (cntr < prev_cntr) {
313 			ret_cpu = curr_cpu;
314 			prev_cntr = cntr;
315 		}
316 	}
317 
318 	*per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
319 
320 fail:
321 	return ret_cpu;
322 }
323 
324 static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
325 				    u16 __percpu *comp_vect_affinity)
326 {
327 	int curr_cpu;
328 	int max_cpu;
329 	u16 cntr;
330 	u16 prev_cntr;
331 
332 	if (!possible_cpumask)
333 		return -EINVAL;
334 
335 	if (!comp_vect_affinity)
336 		return -EINVAL;
337 
338 	max_cpu = cpumask_first(possible_cpumask);
339 	if (max_cpu >= nr_cpu_ids)
340 		return -EINVAL;
341 
342 	prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
343 	for_each_cpu(curr_cpu, possible_cpumask) {
344 		cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
345 
346 		if (cntr > prev_cntr) {
347 			max_cpu = curr_cpu;
348 			prev_cntr = cntr;
349 		}
350 	}
351 
352 	*per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
353 
354 	return max_cpu;
355 }
356 
357 /*
358  * Non-interrupt CPUs are used first, then interrupt CPUs.
359  * Two already allocated cpu masks must be passed.
360  */
361 static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
362 				  struct hfi1_affinity_node *entry,
363 				  cpumask_var_t non_intr_cpus,
364 				  cpumask_var_t available_cpus)
365 	__must_hold(&node_affinity.lock)
366 {
367 	int cpu;
368 	struct cpu_mask_set *set = dd->comp_vect;
369 
370 	lockdep_assert_held(&node_affinity.lock);
371 	if (!non_intr_cpus) {
372 		cpu = -1;
373 		goto fail;
374 	}
375 
376 	if (!available_cpus) {
377 		cpu = -1;
378 		goto fail;
379 	}
380 
381 	/* Available CPUs for pinning completion vectors */
382 	_cpu_mask_set_gen_inc(set);
383 	cpumask_andnot(available_cpus, &set->mask, &set->used);
384 
385 	/* Available CPUs without SDMA engine interrupts */
386 	cpumask_andnot(non_intr_cpus, available_cpus,
387 		       &entry->def_intr.used);
388 
389 	/* If there are non-interrupt CPUs available, use them first */
390 	if (!cpumask_empty(non_intr_cpus))
391 		cpu = cpumask_first(non_intr_cpus);
392 	else /* Otherwise, use interrupt CPUs */
393 		cpu = cpumask_first(available_cpus);
394 
395 	if (cpu >= nr_cpu_ids) { /* empty */
396 		cpu = -1;
397 		goto fail;
398 	}
399 	cpumask_set_cpu(cpu, &set->used);
400 
401 fail:
402 	return cpu;
403 }
404 
405 static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
406 {
407 	struct cpu_mask_set *set = dd->comp_vect;
408 
409 	if (cpu < 0)
410 		return;
411 
412 	cpu_mask_set_put(set, cpu);
413 }
414 
415 /* _dev_comp_vect_mappings_destroy() is reentrant */
416 static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
417 {
418 	int i, cpu;
419 
420 	if (!dd->comp_vect_mappings)
421 		return;
422 
423 	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
424 		cpu = dd->comp_vect_mappings[i];
425 		_dev_comp_vect_cpu_put(dd, cpu);
426 		dd->comp_vect_mappings[i] = -1;
427 		hfi1_cdbg(AFFINITY,
428 			  "[%s] Release CPU %d from completion vector %d",
429 			  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
430 	}
431 
432 	kfree(dd->comp_vect_mappings);
433 	dd->comp_vect_mappings = NULL;
434 }
435 
436 /*
437  * This function creates the table for looking up CPUs for completion vectors.
438  * num_comp_vectors needs to have been initilized before calling this function.
439  */
440 static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
441 					  struct hfi1_affinity_node *entry)
442 	__must_hold(&node_affinity.lock)
443 {
444 	int i, cpu, ret;
445 	cpumask_var_t non_intr_cpus;
446 	cpumask_var_t available_cpus;
447 
448 	lockdep_assert_held(&node_affinity.lock);
449 
450 	if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
451 		return -ENOMEM;
452 
453 	if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
454 		free_cpumask_var(non_intr_cpus);
455 		return -ENOMEM;
456 	}
457 
458 	dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
459 					 sizeof(*dd->comp_vect_mappings),
460 					 GFP_KERNEL);
461 	if (!dd->comp_vect_mappings) {
462 		ret = -ENOMEM;
463 		goto fail;
464 	}
465 	for (i = 0; i < dd->comp_vect_possible_cpus; i++)
466 		dd->comp_vect_mappings[i] = -1;
467 
468 	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
469 		cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
470 					     available_cpus);
471 		if (cpu < 0) {
472 			ret = -EINVAL;
473 			goto fail;
474 		}
475 
476 		dd->comp_vect_mappings[i] = cpu;
477 		hfi1_cdbg(AFFINITY,
478 			  "[%s] Completion Vector %d -> CPU %d",
479 			  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
480 	}
481 
482 	return 0;
483 
484 fail:
485 	free_cpumask_var(available_cpus);
486 	free_cpumask_var(non_intr_cpus);
487 	_dev_comp_vect_mappings_destroy(dd);
488 
489 	return ret;
490 }
491 
492 int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
493 {
494 	int ret;
495 	struct hfi1_affinity_node *entry;
496 
497 	mutex_lock(&node_affinity.lock);
498 	entry = node_affinity_lookup(dd->node);
499 	if (!entry) {
500 		ret = -EINVAL;
501 		goto unlock;
502 	}
503 	ret = _dev_comp_vect_mappings_create(dd, entry);
504 unlock:
505 	mutex_unlock(&node_affinity.lock);
506 
507 	return ret;
508 }
509 
510 void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
511 {
512 	_dev_comp_vect_mappings_destroy(dd);
513 }
514 
515 int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
516 {
517 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
518 	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
519 
520 	if (!dd->comp_vect_mappings)
521 		return -EINVAL;
522 	if (comp_vect >= dd->comp_vect_possible_cpus)
523 		return -EINVAL;
524 
525 	return dd->comp_vect_mappings[comp_vect];
526 }
527 
528 /*
529  * It assumes dd->comp_vect_possible_cpus is available.
530  */
531 static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
532 					struct hfi1_affinity_node *entry,
533 					bool first_dev_init)
534 	__must_hold(&node_affinity.lock)
535 {
536 	int i, j, curr_cpu;
537 	int possible_cpus_comp_vect = 0;
538 	struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
539 
540 	lockdep_assert_held(&node_affinity.lock);
541 	/*
542 	 * If there's only one CPU available for completion vectors, then
543 	 * there will only be one completion vector available. Othewise,
544 	 * the number of completion vector available will be the number of
545 	 * available CPUs divide it by the number of devices in the
546 	 * local NUMA node.
547 	 */
548 	if (cpumask_weight(&entry->comp_vect_mask) == 1) {
549 		possible_cpus_comp_vect = 1;
550 		dd_dev_warn(dd,
551 			    "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
552 	} else {
553 		possible_cpus_comp_vect +=
554 			cpumask_weight(&entry->comp_vect_mask) /
555 				       hfi1_per_node_cntr[dd->node];
556 
557 		/*
558 		 * If the completion vector CPUs available doesn't divide
559 		 * evenly among devices, then the first device device to be
560 		 * initialized gets an extra CPU.
561 		 */
562 		if (first_dev_init &&
563 		    cpumask_weight(&entry->comp_vect_mask) %
564 		    hfi1_per_node_cntr[dd->node] != 0)
565 			possible_cpus_comp_vect++;
566 	}
567 
568 	dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
569 
570 	/* Reserving CPUs for device completion vector */
571 	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
572 		curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
573 						entry->comp_vect_affinity);
574 		if (curr_cpu < 0)
575 			goto fail;
576 
577 		cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
578 	}
579 
580 	hfi1_cdbg(AFFINITY,
581 		  "[%s] Completion vector affinity CPU set(s) %*pbl",
582 		  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
583 		  cpumask_pr_args(dev_comp_vect_mask));
584 
585 	return 0;
586 
587 fail:
588 	for (j = 0; j < i; j++)
589 		per_cpu_affinity_put_max(&entry->comp_vect_mask,
590 					 entry->comp_vect_affinity);
591 
592 	return curr_cpu;
593 }
594 
595 /*
596  * It assumes dd->comp_vect_possible_cpus is available.
597  */
598 static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
599 					     struct hfi1_affinity_node *entry)
600 	__must_hold(&node_affinity.lock)
601 {
602 	int i, cpu;
603 
604 	lockdep_assert_held(&node_affinity.lock);
605 	if (!dd->comp_vect_possible_cpus)
606 		return;
607 
608 	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
609 		cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
610 					       entry->comp_vect_affinity);
611 		/* Clearing CPU in device completion vector cpu mask */
612 		if (cpu >= 0)
613 			cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
614 	}
615 
616 	dd->comp_vect_possible_cpus = 0;
617 }
618 
619 /*
620  * Interrupt affinity.
621  *
622  * non-rcv avail gets a default mask that
623  * starts as possible cpus with threads reset
624  * and each rcv avail reset.
625  *
626  * rcv avail gets node relative 1 wrapping back
627  * to the node relative 1 as necessary.
628  *
629  */
630 int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
631 {
632 	int node = pcibus_to_node(dd->pcidev->bus);
633 	struct hfi1_affinity_node *entry;
634 	const struct cpumask *local_mask;
635 	int curr_cpu, possible, i, ret;
636 	bool new_entry = false;
637 
638 	/*
639 	 * If the BIOS does not have the NUMA node information set, select
640 	 * NUMA 0 so we get consistent performance.
641 	 */
642 	if (node < 0) {
643 		dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
644 		node = 0;
645 	}
646 	dd->node = node;
647 
648 	local_mask = cpumask_of_node(dd->node);
649 	if (cpumask_first(local_mask) >= nr_cpu_ids)
650 		local_mask = topology_core_cpumask(0);
651 
652 	mutex_lock(&node_affinity.lock);
653 	entry = node_affinity_lookup(dd->node);
654 
655 	/*
656 	 * If this is the first time this NUMA node's affinity is used,
657 	 * create an entry in the global affinity structure and initialize it.
658 	 */
659 	if (!entry) {
660 		entry = node_affinity_allocate(node);
661 		if (!entry) {
662 			dd_dev_err(dd,
663 				   "Unable to allocate global affinity node\n");
664 			ret = -ENOMEM;
665 			goto fail;
666 		}
667 		new_entry = true;
668 
669 		init_cpu_mask_set(&entry->def_intr);
670 		init_cpu_mask_set(&entry->rcv_intr);
671 		cpumask_clear(&entry->comp_vect_mask);
672 		cpumask_clear(&entry->general_intr_mask);
673 		/* Use the "real" cpu mask of this node as the default */
674 		cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
675 			    local_mask);
676 
677 		/* fill in the receive list */
678 		possible = cpumask_weight(&entry->def_intr.mask);
679 		curr_cpu = cpumask_first(&entry->def_intr.mask);
680 
681 		if (possible == 1) {
682 			/* only one CPU, everyone will use it */
683 			cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
684 			cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
685 		} else {
686 			/*
687 			 * The general/control context will be the first CPU in
688 			 * the default list, so it is removed from the default
689 			 * list and added to the general interrupt list.
690 			 */
691 			cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
692 			cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
693 			curr_cpu = cpumask_next(curr_cpu,
694 						&entry->def_intr.mask);
695 
696 			/*
697 			 * Remove the remaining kernel receive queues from
698 			 * the default list and add them to the receive list.
699 			 */
700 			for (i = 0;
701 			     i < (dd->n_krcv_queues - 1) *
702 				  hfi1_per_node_cntr[dd->node];
703 			     i++) {
704 				cpumask_clear_cpu(curr_cpu,
705 						  &entry->def_intr.mask);
706 				cpumask_set_cpu(curr_cpu,
707 						&entry->rcv_intr.mask);
708 				curr_cpu = cpumask_next(curr_cpu,
709 							&entry->def_intr.mask);
710 				if (curr_cpu >= nr_cpu_ids)
711 					break;
712 			}
713 
714 			/*
715 			 * If there ends up being 0 CPU cores leftover for SDMA
716 			 * engines, use the same CPU cores as general/control
717 			 * context.
718 			 */
719 			if (cpumask_weight(&entry->def_intr.mask) == 0)
720 				cpumask_copy(&entry->def_intr.mask,
721 					     &entry->general_intr_mask);
722 		}
723 
724 		/* Determine completion vector CPUs for the entire node */
725 		cpumask_and(&entry->comp_vect_mask,
726 			    &node_affinity.real_cpu_mask, local_mask);
727 		cpumask_andnot(&entry->comp_vect_mask,
728 			       &entry->comp_vect_mask,
729 			       &entry->rcv_intr.mask);
730 		cpumask_andnot(&entry->comp_vect_mask,
731 			       &entry->comp_vect_mask,
732 			       &entry->general_intr_mask);
733 
734 		/*
735 		 * If there ends up being 0 CPU cores leftover for completion
736 		 * vectors, use the same CPU core as the general/control
737 		 * context.
738 		 */
739 		if (cpumask_weight(&entry->comp_vect_mask) == 0)
740 			cpumask_copy(&entry->comp_vect_mask,
741 				     &entry->general_intr_mask);
742 	}
743 
744 	ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
745 	if (ret < 0)
746 		goto fail;
747 
748 	if (new_entry)
749 		node_affinity_add_tail(entry);
750 
751 	mutex_unlock(&node_affinity.lock);
752 
753 	return 0;
754 
755 fail:
756 	if (new_entry)
757 		node_affinity_destroy(entry);
758 	mutex_unlock(&node_affinity.lock);
759 	return ret;
760 }
761 
762 void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
763 {
764 	struct hfi1_affinity_node *entry;
765 
766 	if (dd->node < 0)
767 		return;
768 
769 	mutex_lock(&node_affinity.lock);
770 	entry = node_affinity_lookup(dd->node);
771 	if (!entry)
772 		goto unlock;
773 
774 	/*
775 	 * Free device completion vector CPUs to be used by future
776 	 * completion vectors
777 	 */
778 	_dev_comp_vect_cpu_mask_clean_up(dd, entry);
779 unlock:
780 	mutex_unlock(&node_affinity.lock);
781 	dd->node = NUMA_NO_NODE;
782 }
783 
784 /*
785  * Function updates the irq affinity hint for msix after it has been changed
786  * by the user using the /proc/irq interface. This function only accepts
787  * one cpu in the mask.
788  */
789 static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
790 {
791 	struct sdma_engine *sde = msix->arg;
792 	struct hfi1_devdata *dd = sde->dd;
793 	struct hfi1_affinity_node *entry;
794 	struct cpu_mask_set *set;
795 	int i, old_cpu;
796 
797 	if (cpu > num_online_cpus() || cpu == sde->cpu)
798 		return;
799 
800 	mutex_lock(&node_affinity.lock);
801 	entry = node_affinity_lookup(dd->node);
802 	if (!entry)
803 		goto unlock;
804 
805 	old_cpu = sde->cpu;
806 	sde->cpu = cpu;
807 	cpumask_clear(&msix->mask);
808 	cpumask_set_cpu(cpu, &msix->mask);
809 	dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
810 		   msix->irq, irq_type_names[msix->type],
811 		   sde->this_idx, cpu);
812 	irq_set_affinity_hint(msix->irq, &msix->mask);
813 
814 	/*
815 	 * Set the new cpu in the hfi1_affinity_node and clean
816 	 * the old cpu if it is not used by any other IRQ
817 	 */
818 	set = &entry->def_intr;
819 	cpumask_set_cpu(cpu, &set->mask);
820 	cpumask_set_cpu(cpu, &set->used);
821 	for (i = 0; i < dd->msix_info.max_requested; i++) {
822 		struct hfi1_msix_entry *other_msix;
823 
824 		other_msix = &dd->msix_info.msix_entries[i];
825 		if (other_msix->type != IRQ_SDMA || other_msix == msix)
826 			continue;
827 
828 		if (cpumask_test_cpu(old_cpu, &other_msix->mask))
829 			goto unlock;
830 	}
831 	cpumask_clear_cpu(old_cpu, &set->mask);
832 	cpumask_clear_cpu(old_cpu, &set->used);
833 unlock:
834 	mutex_unlock(&node_affinity.lock);
835 }
836 
837 static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
838 				     const cpumask_t *mask)
839 {
840 	int cpu = cpumask_first(mask);
841 	struct hfi1_msix_entry *msix = container_of(notify,
842 						    struct hfi1_msix_entry,
843 						    notify);
844 
845 	/* Only one CPU configuration supported currently */
846 	hfi1_update_sdma_affinity(msix, cpu);
847 }
848 
849 static void hfi1_irq_notifier_release(struct kref *ref)
850 {
851 	/*
852 	 * This is required by affinity notifier. We don't have anything to
853 	 * free here.
854 	 */
855 }
856 
857 static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
858 {
859 	struct irq_affinity_notify *notify = &msix->notify;
860 
861 	notify->irq = msix->irq;
862 	notify->notify = hfi1_irq_notifier_notify;
863 	notify->release = hfi1_irq_notifier_release;
864 
865 	if (irq_set_affinity_notifier(notify->irq, notify))
866 		pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
867 		       notify->irq);
868 }
869 
870 static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
871 {
872 	struct irq_affinity_notify *notify = &msix->notify;
873 
874 	if (irq_set_affinity_notifier(notify->irq, NULL))
875 		pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
876 		       notify->irq);
877 }
878 
879 /*
880  * Function sets the irq affinity for msix.
881  * It *must* be called with node_affinity.lock held.
882  */
883 static int get_irq_affinity(struct hfi1_devdata *dd,
884 			    struct hfi1_msix_entry *msix)
885 {
886 	cpumask_var_t diff;
887 	struct hfi1_affinity_node *entry;
888 	struct cpu_mask_set *set = NULL;
889 	struct sdma_engine *sde = NULL;
890 	struct hfi1_ctxtdata *rcd = NULL;
891 	char extra[64];
892 	int cpu = -1;
893 
894 	extra[0] = '\0';
895 	cpumask_clear(&msix->mask);
896 
897 	entry = node_affinity_lookup(dd->node);
898 
899 	switch (msix->type) {
900 	case IRQ_SDMA:
901 		sde = (struct sdma_engine *)msix->arg;
902 		scnprintf(extra, 64, "engine %u", sde->this_idx);
903 		set = &entry->def_intr;
904 		break;
905 	case IRQ_GENERAL:
906 		cpu = cpumask_first(&entry->general_intr_mask);
907 		break;
908 	case IRQ_RCVCTXT:
909 		rcd = (struct hfi1_ctxtdata *)msix->arg;
910 		if (rcd->ctxt == HFI1_CTRL_CTXT)
911 			cpu = cpumask_first(&entry->general_intr_mask);
912 		else
913 			set = &entry->rcv_intr;
914 		scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
915 		break;
916 	default:
917 		dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
918 		return -EINVAL;
919 	}
920 
921 	/*
922 	 * The general and control contexts are placed on a particular
923 	 * CPU, which is set above. Skip accounting for it. Everything else
924 	 * finds its CPU here.
925 	 */
926 	if (cpu == -1 && set) {
927 		if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
928 			return -ENOMEM;
929 
930 		cpu = cpu_mask_set_get_first(set, diff);
931 		if (cpu < 0) {
932 			free_cpumask_var(diff);
933 			dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
934 			return cpu;
935 		}
936 
937 		free_cpumask_var(diff);
938 	}
939 
940 	cpumask_set_cpu(cpu, &msix->mask);
941 	dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
942 		    msix->irq, irq_type_names[msix->type],
943 		    extra, cpu);
944 	irq_set_affinity_hint(msix->irq, &msix->mask);
945 
946 	if (msix->type == IRQ_SDMA) {
947 		sde->cpu = cpu;
948 		hfi1_setup_sdma_notifier(msix);
949 	}
950 
951 	return 0;
952 }
953 
954 int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
955 {
956 	int ret;
957 
958 	mutex_lock(&node_affinity.lock);
959 	ret = get_irq_affinity(dd, msix);
960 	mutex_unlock(&node_affinity.lock);
961 	return ret;
962 }
963 
964 void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
965 			   struct hfi1_msix_entry *msix)
966 {
967 	struct cpu_mask_set *set = NULL;
968 	struct hfi1_ctxtdata *rcd;
969 	struct hfi1_affinity_node *entry;
970 
971 	mutex_lock(&node_affinity.lock);
972 	entry = node_affinity_lookup(dd->node);
973 
974 	switch (msix->type) {
975 	case IRQ_SDMA:
976 		set = &entry->def_intr;
977 		hfi1_cleanup_sdma_notifier(msix);
978 		break;
979 	case IRQ_GENERAL:
980 		/* Don't do accounting for general contexts */
981 		break;
982 	case IRQ_RCVCTXT:
983 		rcd = (struct hfi1_ctxtdata *)msix->arg;
984 		/* Don't do accounting for control contexts */
985 		if (rcd->ctxt != HFI1_CTRL_CTXT)
986 			set = &entry->rcv_intr;
987 		break;
988 	default:
989 		mutex_unlock(&node_affinity.lock);
990 		return;
991 	}
992 
993 	if (set) {
994 		cpumask_andnot(&set->used, &set->used, &msix->mask);
995 		_cpu_mask_set_gen_dec(set);
996 	}
997 
998 	irq_set_affinity_hint(msix->irq, NULL);
999 	cpumask_clear(&msix->mask);
1000 	mutex_unlock(&node_affinity.lock);
1001 }
1002 
1003 /* This should be called with node_affinity.lock held */
1004 static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
1005 				struct hfi1_affinity_node_list *affinity)
1006 {
1007 	int possible, curr_cpu, i;
1008 	uint num_cores_per_socket = node_affinity.num_online_cpus /
1009 					affinity->num_core_siblings /
1010 						node_affinity.num_online_nodes;
1011 
1012 	cpumask_copy(hw_thread_mask, &affinity->proc.mask);
1013 	if (affinity->num_core_siblings > 0) {
1014 		/* Removing other siblings not needed for now */
1015 		possible = cpumask_weight(hw_thread_mask);
1016 		curr_cpu = cpumask_first(hw_thread_mask);
1017 		for (i = 0;
1018 		     i < num_cores_per_socket * node_affinity.num_online_nodes;
1019 		     i++)
1020 			curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1021 
1022 		for (; i < possible; i++) {
1023 			cpumask_clear_cpu(curr_cpu, hw_thread_mask);
1024 			curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1025 		}
1026 
1027 		/* Identifying correct HW threads within physical cores */
1028 		cpumask_shift_left(hw_thread_mask, hw_thread_mask,
1029 				   num_cores_per_socket *
1030 				   node_affinity.num_online_nodes *
1031 				   hw_thread_no);
1032 	}
1033 }
1034 
1035 int hfi1_get_proc_affinity(int node)
1036 {
1037 	int cpu = -1, ret, i;
1038 	struct hfi1_affinity_node *entry;
1039 	cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
1040 	const struct cpumask *node_mask,
1041 		*proc_mask = &current->cpus_allowed;
1042 	struct hfi1_affinity_node_list *affinity = &node_affinity;
1043 	struct cpu_mask_set *set = &affinity->proc;
1044 
1045 	/*
1046 	 * check whether process/context affinity has already
1047 	 * been set
1048 	 */
1049 	if (cpumask_weight(proc_mask) == 1) {
1050 		hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
1051 			  current->pid, current->comm,
1052 			  cpumask_pr_args(proc_mask));
1053 		/*
1054 		 * Mark the pre-set CPU as used. This is atomic so we don't
1055 		 * need the lock
1056 		 */
1057 		cpu = cpumask_first(proc_mask);
1058 		cpumask_set_cpu(cpu, &set->used);
1059 		goto done;
1060 	} else if (cpumask_weight(proc_mask) < cpumask_weight(&set->mask)) {
1061 		hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
1062 			  current->pid, current->comm,
1063 			  cpumask_pr_args(proc_mask));
1064 		goto done;
1065 	}
1066 
1067 	/*
1068 	 * The process does not have a preset CPU affinity so find one to
1069 	 * recommend using the following algorithm:
1070 	 *
1071 	 * For each user process that is opening a context on HFI Y:
1072 	 *  a) If all cores are filled, reinitialize the bitmask
1073 	 *  b) Fill real cores first, then HT cores (First set of HT
1074 	 *     cores on all physical cores, then second set of HT core,
1075 	 *     and, so on) in the following order:
1076 	 *
1077 	 *     1. Same NUMA node as HFI Y and not running an IRQ
1078 	 *        handler
1079 	 *     2. Same NUMA node as HFI Y and running an IRQ handler
1080 	 *     3. Different NUMA node to HFI Y and not running an IRQ
1081 	 *        handler
1082 	 *     4. Different NUMA node to HFI Y and running an IRQ
1083 	 *        handler
1084 	 *  c) Mark core as filled in the bitmask. As user processes are
1085 	 *     done, clear cores from the bitmask.
1086 	 */
1087 
1088 	ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
1089 	if (!ret)
1090 		goto done;
1091 	ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
1092 	if (!ret)
1093 		goto free_diff;
1094 	ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
1095 	if (!ret)
1096 		goto free_hw_thread_mask;
1097 	ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
1098 	if (!ret)
1099 		goto free_available_mask;
1100 
1101 	mutex_lock(&affinity->lock);
1102 	/*
1103 	 * If we've used all available HW threads, clear the mask and start
1104 	 * overloading.
1105 	 */
1106 	_cpu_mask_set_gen_inc(set);
1107 
1108 	/*
1109 	 * If NUMA node has CPUs used by interrupt handlers, include them in the
1110 	 * interrupt handler mask.
1111 	 */
1112 	entry = node_affinity_lookup(node);
1113 	if (entry) {
1114 		cpumask_copy(intrs_mask, (entry->def_intr.gen ?
1115 					  &entry->def_intr.mask :
1116 					  &entry->def_intr.used));
1117 		cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
1118 						    &entry->rcv_intr.mask :
1119 						    &entry->rcv_intr.used));
1120 		cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
1121 	}
1122 	hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
1123 		  cpumask_pr_args(intrs_mask));
1124 
1125 	cpumask_copy(hw_thread_mask, &set->mask);
1126 
1127 	/*
1128 	 * If HT cores are enabled, identify which HW threads within the
1129 	 * physical cores should be used.
1130 	 */
1131 	if (affinity->num_core_siblings > 0) {
1132 		for (i = 0; i < affinity->num_core_siblings; i++) {
1133 			find_hw_thread_mask(i, hw_thread_mask, affinity);
1134 
1135 			/*
1136 			 * If there's at least one available core for this HW
1137 			 * thread number, stop looking for a core.
1138 			 *
1139 			 * diff will always be not empty at least once in this
1140 			 * loop as the used mask gets reset when
1141 			 * (set->mask == set->used) before this loop.
1142 			 */
1143 			cpumask_andnot(diff, hw_thread_mask, &set->used);
1144 			if (!cpumask_empty(diff))
1145 				break;
1146 		}
1147 	}
1148 	hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
1149 		  cpumask_pr_args(hw_thread_mask));
1150 
1151 	node_mask = cpumask_of_node(node);
1152 	hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
1153 		  cpumask_pr_args(node_mask));
1154 
1155 	/* Get cpumask of available CPUs on preferred NUMA */
1156 	cpumask_and(available_mask, hw_thread_mask, node_mask);
1157 	cpumask_andnot(available_mask, available_mask, &set->used);
1158 	hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
1159 		  cpumask_pr_args(available_mask));
1160 
1161 	/*
1162 	 * At first, we don't want to place processes on the same
1163 	 * CPUs as interrupt handlers. Then, CPUs running interrupt
1164 	 * handlers are used.
1165 	 *
1166 	 * 1) If diff is not empty, then there are CPUs not running
1167 	 *    non-interrupt handlers available, so diff gets copied
1168 	 *    over to available_mask.
1169 	 * 2) If diff is empty, then all CPUs not running interrupt
1170 	 *    handlers are taken, so available_mask contains all
1171 	 *    available CPUs running interrupt handlers.
1172 	 * 3) If available_mask is empty, then all CPUs on the
1173 	 *    preferred NUMA node are taken, so other NUMA nodes are
1174 	 *    used for process assignments using the same method as
1175 	 *    the preferred NUMA node.
1176 	 */
1177 	cpumask_andnot(diff, available_mask, intrs_mask);
1178 	if (!cpumask_empty(diff))
1179 		cpumask_copy(available_mask, diff);
1180 
1181 	/* If we don't have CPUs on the preferred node, use other NUMA nodes */
1182 	if (cpumask_empty(available_mask)) {
1183 		cpumask_andnot(available_mask, hw_thread_mask, &set->used);
1184 		/* Excluding preferred NUMA cores */
1185 		cpumask_andnot(available_mask, available_mask, node_mask);
1186 		hfi1_cdbg(PROC,
1187 			  "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
1188 			  cpumask_pr_args(available_mask));
1189 
1190 		/*
1191 		 * At first, we don't want to place processes on the same
1192 		 * CPUs as interrupt handlers.
1193 		 */
1194 		cpumask_andnot(diff, available_mask, intrs_mask);
1195 		if (!cpumask_empty(diff))
1196 			cpumask_copy(available_mask, diff);
1197 	}
1198 	hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
1199 		  cpumask_pr_args(available_mask));
1200 
1201 	cpu = cpumask_first(available_mask);
1202 	if (cpu >= nr_cpu_ids) /* empty */
1203 		cpu = -1;
1204 	else
1205 		cpumask_set_cpu(cpu, &set->used);
1206 
1207 	mutex_unlock(&affinity->lock);
1208 	hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
1209 
1210 	free_cpumask_var(intrs_mask);
1211 free_available_mask:
1212 	free_cpumask_var(available_mask);
1213 free_hw_thread_mask:
1214 	free_cpumask_var(hw_thread_mask);
1215 free_diff:
1216 	free_cpumask_var(diff);
1217 done:
1218 	return cpu;
1219 }
1220 
1221 void hfi1_put_proc_affinity(int cpu)
1222 {
1223 	struct hfi1_affinity_node_list *affinity = &node_affinity;
1224 	struct cpu_mask_set *set = &affinity->proc;
1225 
1226 	if (cpu < 0)
1227 		return;
1228 
1229 	mutex_lock(&affinity->lock);
1230 	cpu_mask_set_put(set, cpu);
1231 	hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
1232 	mutex_unlock(&affinity->lock);
1233 }
1234