1145eba1aSCai Huoqing // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2f48ad614SDennis Dalessandro /* 30bae02d5SGrzegorz Andrejczuk * Copyright(c) 2015 - 2020 Intel Corporation. 4f48ad614SDennis Dalessandro */ 5145eba1aSCai Huoqing 6f48ad614SDennis Dalessandro #include <linux/topology.h> 7f48ad614SDennis Dalessandro #include <linux/cpumask.h> 82d01c37dSTadeusz Struk #include <linux/interrupt.h> 998fa15f3SAnshuman Khandual #include <linux/numa.h> 10f48ad614SDennis Dalessandro 11f48ad614SDennis Dalessandro #include "hfi.h" 12f48ad614SDennis Dalessandro #include "affinity.h" 13f48ad614SDennis Dalessandro #include "sdma.h" 14f48ad614SDennis Dalessandro #include "trace.h" 15f48ad614SDennis Dalessandro 164197344bSDennis Dalessandro struct hfi1_affinity_node_list node_affinity = { 174197344bSDennis Dalessandro .list = LIST_HEAD_INIT(node_affinity.list), 18584d9577STadeusz Struk .lock = __MUTEX_INITIALIZER(node_affinity.lock) 194197344bSDennis Dalessandro }; 204197344bSDennis Dalessandro 21f48ad614SDennis Dalessandro /* Name of IRQ types, indexed by enum irq_type */ 22f48ad614SDennis Dalessandro static const char * const irq_type_names[] = { 23f48ad614SDennis Dalessandro "SDMA", 24f48ad614SDennis Dalessandro "RCVCTXT", 250bae02d5SGrzegorz Andrejczuk "NETDEVCTXT", 26f48ad614SDennis Dalessandro "GENERAL", 27f48ad614SDennis Dalessandro "OTHER", 28f48ad614SDennis Dalessandro }; 29f48ad614SDennis Dalessandro 30d6373019SSebastian Sanchez /* Per NUMA node count of HFI devices */ 31d6373019SSebastian Sanchez static unsigned int *hfi1_per_node_cntr; 32d6373019SSebastian Sanchez 33f48ad614SDennis Dalessandro static inline void init_cpu_mask_set(struct cpu_mask_set *set) 34f48ad614SDennis Dalessandro { 35f48ad614SDennis Dalessandro cpumask_clear(&set->mask); 36f48ad614SDennis Dalessandro cpumask_clear(&set->used); 37f48ad614SDennis Dalessandro set->gen = 0; 38f48ad614SDennis Dalessandro } 39f48ad614SDennis Dalessandro 40cf38ea10SSebastian Sanchez /* Increment generation of CPU set if needed */ 41cf38ea10SSebastian Sanchez static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set) 42cf38ea10SSebastian Sanchez { 43cf38ea10SSebastian Sanchez if (cpumask_equal(&set->mask, &set->used)) { 44cf38ea10SSebastian Sanchez /* 45cf38ea10SSebastian Sanchez * We've used up all the CPUs, bump up the generation 46cf38ea10SSebastian Sanchez * and reset the 'used' map 47cf38ea10SSebastian Sanchez */ 48cf38ea10SSebastian Sanchez set->gen++; 49cf38ea10SSebastian Sanchez cpumask_clear(&set->used); 50cf38ea10SSebastian Sanchez } 51cf38ea10SSebastian Sanchez } 52cf38ea10SSebastian Sanchez 53cf38ea10SSebastian Sanchez static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set) 54cf38ea10SSebastian Sanchez { 55cf38ea10SSebastian Sanchez if (cpumask_empty(&set->used) && set->gen) { 56cf38ea10SSebastian Sanchez set->gen--; 57cf38ea10SSebastian Sanchez cpumask_copy(&set->used, &set->mask); 58cf38ea10SSebastian Sanchez } 59cf38ea10SSebastian Sanchez } 60cf38ea10SSebastian Sanchez 61cf38ea10SSebastian Sanchez /* Get the first CPU from the list of unused CPUs in a CPU set data structure */ 62cf38ea10SSebastian Sanchez static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff) 63cf38ea10SSebastian Sanchez { 64cf38ea10SSebastian Sanchez int cpu; 65cf38ea10SSebastian Sanchez 66cf38ea10SSebastian Sanchez if (!diff || !set) 67cf38ea10SSebastian Sanchez return -EINVAL; 68cf38ea10SSebastian Sanchez 69cf38ea10SSebastian Sanchez _cpu_mask_set_gen_inc(set); 70cf38ea10SSebastian Sanchez 71cf38ea10SSebastian Sanchez /* Find out CPUs left in CPU mask */ 72cf38ea10SSebastian Sanchez cpumask_andnot(diff, &set->mask, &set->used); 73cf38ea10SSebastian Sanchez 74cf38ea10SSebastian Sanchez cpu = cpumask_first(diff); 75cf38ea10SSebastian Sanchez if (cpu >= nr_cpu_ids) /* empty */ 76cf38ea10SSebastian Sanchez cpu = -EINVAL; 77cf38ea10SSebastian Sanchez else 78cf38ea10SSebastian Sanchez cpumask_set_cpu(cpu, &set->used); 79cf38ea10SSebastian Sanchez 80cf38ea10SSebastian Sanchez return cpu; 81cf38ea10SSebastian Sanchez } 82cf38ea10SSebastian Sanchez 83cf38ea10SSebastian Sanchez static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu) 84cf38ea10SSebastian Sanchez { 85cf38ea10SSebastian Sanchez if (!set) 86cf38ea10SSebastian Sanchez return; 87cf38ea10SSebastian Sanchez 88cf38ea10SSebastian Sanchez cpumask_clear_cpu(cpu, &set->used); 89cf38ea10SSebastian Sanchez _cpu_mask_set_gen_dec(set); 90cf38ea10SSebastian Sanchez } 91cf38ea10SSebastian Sanchez 92f48ad614SDennis Dalessandro /* Initialize non-HT cpu cores mask */ 934197344bSDennis Dalessandro void init_real_cpu_mask(void) 94f48ad614SDennis Dalessandro { 95f48ad614SDennis Dalessandro int possible, curr_cpu, i, ht; 96f48ad614SDennis Dalessandro 974197344bSDennis Dalessandro cpumask_clear(&node_affinity.real_cpu_mask); 98f48ad614SDennis Dalessandro 99f48ad614SDennis Dalessandro /* Start with cpu online mask as the real cpu mask */ 1004197344bSDennis Dalessandro cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask); 101f48ad614SDennis Dalessandro 102f48ad614SDennis Dalessandro /* 103f48ad614SDennis Dalessandro * Remove HT cores from the real cpu mask. Do this in two steps below. 104f48ad614SDennis Dalessandro */ 1054197344bSDennis Dalessandro possible = cpumask_weight(&node_affinity.real_cpu_mask); 106f48ad614SDennis Dalessandro ht = cpumask_weight(topology_sibling_cpumask( 1074197344bSDennis Dalessandro cpumask_first(&node_affinity.real_cpu_mask))); 108f48ad614SDennis Dalessandro /* 109f48ad614SDennis Dalessandro * Step 1. Skip over the first N HT siblings and use them as the 110f48ad614SDennis Dalessandro * "real" cores. Assumes that HT cores are not enumerated in 111f48ad614SDennis Dalessandro * succession (except in the single core case). 112f48ad614SDennis Dalessandro */ 1134197344bSDennis Dalessandro curr_cpu = cpumask_first(&node_affinity.real_cpu_mask); 114f48ad614SDennis Dalessandro for (i = 0; i < possible / ht; i++) 1154197344bSDennis Dalessandro curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); 116f48ad614SDennis Dalessandro /* 117f48ad614SDennis Dalessandro * Step 2. Remove the remaining HT siblings. Use cpumask_next() to 118f48ad614SDennis Dalessandro * skip any gaps. 119f48ad614SDennis Dalessandro */ 120f48ad614SDennis Dalessandro for (; i < possible; i++) { 1214197344bSDennis Dalessandro cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask); 1224197344bSDennis Dalessandro curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); 1234197344bSDennis Dalessandro } 124f48ad614SDennis Dalessandro } 125f48ad614SDennis Dalessandro 126d6373019SSebastian Sanchez int node_affinity_init(void) 1274197344bSDennis Dalessandro { 128d6373019SSebastian Sanchez int node; 129d6373019SSebastian Sanchez struct pci_dev *dev = NULL; 130d6373019SSebastian Sanchez const struct pci_device_id *ids = hfi1_pci_tbl; 131d6373019SSebastian Sanchez 132b094a36fSSebastian Sanchez cpumask_clear(&node_affinity.proc.used); 1334197344bSDennis Dalessandro cpumask_copy(&node_affinity.proc.mask, cpu_online_mask); 134b094a36fSSebastian Sanchez 135b094a36fSSebastian Sanchez node_affinity.proc.gen = 0; 136b094a36fSSebastian Sanchez node_affinity.num_core_siblings = 137b094a36fSSebastian Sanchez cpumask_weight(topology_sibling_cpumask( 138b094a36fSSebastian Sanchez cpumask_first(&node_affinity.proc.mask) 139b094a36fSSebastian Sanchez )); 1409d8145a6SHarish Chegondi node_affinity.num_possible_nodes = num_possible_nodes(); 141b094a36fSSebastian Sanchez node_affinity.num_online_nodes = num_online_nodes(); 142b094a36fSSebastian Sanchez node_affinity.num_online_cpus = num_online_cpus(); 143b094a36fSSebastian Sanchez 1444197344bSDennis Dalessandro /* 1454197344bSDennis Dalessandro * The real cpu mask is part of the affinity struct but it has to be 1464197344bSDennis Dalessandro * initialized early. It is needed to calculate the number of user 1474197344bSDennis Dalessandro * contexts in set_up_context_variables(). 1484197344bSDennis Dalessandro */ 1494197344bSDennis Dalessandro init_real_cpu_mask(); 150d6373019SSebastian Sanchez 1519d8145a6SHarish Chegondi hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes, 152d6373019SSebastian Sanchez sizeof(*hfi1_per_node_cntr), GFP_KERNEL); 153d6373019SSebastian Sanchez if (!hfi1_per_node_cntr) 154d6373019SSebastian Sanchez return -ENOMEM; 155d6373019SSebastian Sanchez 156d6373019SSebastian Sanchez while (ids->vendor) { 157d6373019SSebastian Sanchez dev = NULL; 158d6373019SSebastian Sanchez while ((dev = pci_get_device(ids->vendor, ids->device, dev))) { 159d6373019SSebastian Sanchez node = pcibus_to_node(dev->bus); 160d6373019SSebastian Sanchez if (node < 0) 161c513de49SMichael J. Ruhl goto out; 162d6373019SSebastian Sanchez 163d6373019SSebastian Sanchez hfi1_per_node_cntr[node]++; 164d6373019SSebastian Sanchez } 165d6373019SSebastian Sanchez ids++; 166d6373019SSebastian Sanchez } 167d6373019SSebastian Sanchez 168d6373019SSebastian Sanchez return 0; 169c513de49SMichael J. Ruhl 170c513de49SMichael J. Ruhl out: 171c513de49SMichael J. Ruhl /* 172c513de49SMichael J. Ruhl * Invalid PCI NUMA node information found, note it, and populate 173c513de49SMichael J. Ruhl * our database 1:1. 174c513de49SMichael J. Ruhl */ 175c513de49SMichael J. Ruhl pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n"); 176c513de49SMichael J. Ruhl pr_err("HFI: System BIOS may need to be upgraded\n"); 177c513de49SMichael J. Ruhl for (node = 0; node < node_affinity.num_possible_nodes; node++) 178c513de49SMichael J. Ruhl hfi1_per_node_cntr[node] = 1; 179c513de49SMichael J. Ruhl 180*9b51d072SXiongfeng Wang pci_dev_put(dev); 181*9b51d072SXiongfeng Wang 182c513de49SMichael J. Ruhl return 0; 1834197344bSDennis Dalessandro } 1844197344bSDennis Dalessandro 1855d18ee67SSebastian Sanchez static void node_affinity_destroy(struct hfi1_affinity_node *entry) 1865d18ee67SSebastian Sanchez { 1875d18ee67SSebastian Sanchez free_percpu(entry->comp_vect_affinity); 1885d18ee67SSebastian Sanchez kfree(entry); 1895d18ee67SSebastian Sanchez } 1905d18ee67SSebastian Sanchez 1915d18ee67SSebastian Sanchez void node_affinity_destroy_all(void) 1924197344bSDennis Dalessandro { 1934197344bSDennis Dalessandro struct list_head *pos, *q; 1944197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 1954197344bSDennis Dalessandro 196584d9577STadeusz Struk mutex_lock(&node_affinity.lock); 1974197344bSDennis Dalessandro list_for_each_safe(pos, q, &node_affinity.list) { 1984197344bSDennis Dalessandro entry = list_entry(pos, struct hfi1_affinity_node, 1994197344bSDennis Dalessandro list); 2004197344bSDennis Dalessandro list_del(pos); 2015d18ee67SSebastian Sanchez node_affinity_destroy(entry); 2024197344bSDennis Dalessandro } 203584d9577STadeusz Struk mutex_unlock(&node_affinity.lock); 204d6373019SSebastian Sanchez kfree(hfi1_per_node_cntr); 2054197344bSDennis Dalessandro } 2064197344bSDennis Dalessandro 2074197344bSDennis Dalessandro static struct hfi1_affinity_node *node_affinity_allocate(int node) 2084197344bSDennis Dalessandro { 2094197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 2104197344bSDennis Dalessandro 2114197344bSDennis Dalessandro entry = kzalloc(sizeof(*entry), GFP_KERNEL); 2124197344bSDennis Dalessandro if (!entry) 2134197344bSDennis Dalessandro return NULL; 2144197344bSDennis Dalessandro entry->node = node; 2155d18ee67SSebastian Sanchez entry->comp_vect_affinity = alloc_percpu(u16); 2164197344bSDennis Dalessandro INIT_LIST_HEAD(&entry->list); 2174197344bSDennis Dalessandro 2184197344bSDennis Dalessandro return entry; 2194197344bSDennis Dalessandro } 2204197344bSDennis Dalessandro 2214197344bSDennis Dalessandro /* 2224197344bSDennis Dalessandro * It appends an entry to the list. 2234197344bSDennis Dalessandro * It *must* be called with node_affinity.lock held. 2244197344bSDennis Dalessandro */ 2254197344bSDennis Dalessandro static void node_affinity_add_tail(struct hfi1_affinity_node *entry) 2264197344bSDennis Dalessandro { 2274197344bSDennis Dalessandro list_add_tail(&entry->list, &node_affinity.list); 2284197344bSDennis Dalessandro } 2294197344bSDennis Dalessandro 2304197344bSDennis Dalessandro /* It must be called with node_affinity.lock held */ 2314197344bSDennis Dalessandro static struct hfi1_affinity_node *node_affinity_lookup(int node) 2324197344bSDennis Dalessandro { 2334197344bSDennis Dalessandro struct list_head *pos; 2344197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 2354197344bSDennis Dalessandro 2364197344bSDennis Dalessandro list_for_each(pos, &node_affinity.list) { 2374197344bSDennis Dalessandro entry = list_entry(pos, struct hfi1_affinity_node, list); 2384197344bSDennis Dalessandro if (entry->node == node) 2394197344bSDennis Dalessandro return entry; 2404197344bSDennis Dalessandro } 2414197344bSDennis Dalessandro 2424197344bSDennis Dalessandro return NULL; 243f48ad614SDennis Dalessandro } 244f48ad614SDennis Dalessandro 2455d18ee67SSebastian Sanchez static int per_cpu_affinity_get(cpumask_var_t possible_cpumask, 2465d18ee67SSebastian Sanchez u16 __percpu *comp_vect_affinity) 2475d18ee67SSebastian Sanchez { 2485d18ee67SSebastian Sanchez int curr_cpu; 2495d18ee67SSebastian Sanchez u16 cntr; 2505d18ee67SSebastian Sanchez u16 prev_cntr; 2515d18ee67SSebastian Sanchez int ret_cpu; 2525d18ee67SSebastian Sanchez 2535d18ee67SSebastian Sanchez if (!possible_cpumask) { 2545d18ee67SSebastian Sanchez ret_cpu = -EINVAL; 2555d18ee67SSebastian Sanchez goto fail; 2565d18ee67SSebastian Sanchez } 2575d18ee67SSebastian Sanchez 2585d18ee67SSebastian Sanchez if (!comp_vect_affinity) { 2595d18ee67SSebastian Sanchez ret_cpu = -EINVAL; 2605d18ee67SSebastian Sanchez goto fail; 2615d18ee67SSebastian Sanchez } 2625d18ee67SSebastian Sanchez 2635d18ee67SSebastian Sanchez ret_cpu = cpumask_first(possible_cpumask); 2645d18ee67SSebastian Sanchez if (ret_cpu >= nr_cpu_ids) { 2655d18ee67SSebastian Sanchez ret_cpu = -EINVAL; 2665d18ee67SSebastian Sanchez goto fail; 2675d18ee67SSebastian Sanchez } 2685d18ee67SSebastian Sanchez 2695d18ee67SSebastian Sanchez prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu); 2705d18ee67SSebastian Sanchez for_each_cpu(curr_cpu, possible_cpumask) { 2715d18ee67SSebastian Sanchez cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu); 2725d18ee67SSebastian Sanchez 2735d18ee67SSebastian Sanchez if (cntr < prev_cntr) { 2745d18ee67SSebastian Sanchez ret_cpu = curr_cpu; 2755d18ee67SSebastian Sanchez prev_cntr = cntr; 2765d18ee67SSebastian Sanchez } 2775d18ee67SSebastian Sanchez } 2785d18ee67SSebastian Sanchez 2795d18ee67SSebastian Sanchez *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1; 2805d18ee67SSebastian Sanchez 2815d18ee67SSebastian Sanchez fail: 2825d18ee67SSebastian Sanchez return ret_cpu; 2835d18ee67SSebastian Sanchez } 2845d18ee67SSebastian Sanchez 2855d18ee67SSebastian Sanchez static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask, 2865d18ee67SSebastian Sanchez u16 __percpu *comp_vect_affinity) 2875d18ee67SSebastian Sanchez { 2885d18ee67SSebastian Sanchez int curr_cpu; 2895d18ee67SSebastian Sanchez int max_cpu; 2905d18ee67SSebastian Sanchez u16 cntr; 2915d18ee67SSebastian Sanchez u16 prev_cntr; 2925d18ee67SSebastian Sanchez 2935d18ee67SSebastian Sanchez if (!possible_cpumask) 2945d18ee67SSebastian Sanchez return -EINVAL; 2955d18ee67SSebastian Sanchez 2965d18ee67SSebastian Sanchez if (!comp_vect_affinity) 2975d18ee67SSebastian Sanchez return -EINVAL; 2985d18ee67SSebastian Sanchez 2995d18ee67SSebastian Sanchez max_cpu = cpumask_first(possible_cpumask); 3005d18ee67SSebastian Sanchez if (max_cpu >= nr_cpu_ids) 3015d18ee67SSebastian Sanchez return -EINVAL; 3025d18ee67SSebastian Sanchez 3035d18ee67SSebastian Sanchez prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu); 3045d18ee67SSebastian Sanchez for_each_cpu(curr_cpu, possible_cpumask) { 3055d18ee67SSebastian Sanchez cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu); 3065d18ee67SSebastian Sanchez 3075d18ee67SSebastian Sanchez if (cntr > prev_cntr) { 3085d18ee67SSebastian Sanchez max_cpu = curr_cpu; 3095d18ee67SSebastian Sanchez prev_cntr = cntr; 3105d18ee67SSebastian Sanchez } 3115d18ee67SSebastian Sanchez } 3125d18ee67SSebastian Sanchez 3135d18ee67SSebastian Sanchez *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1; 3145d18ee67SSebastian Sanchez 3155d18ee67SSebastian Sanchez return max_cpu; 3165d18ee67SSebastian Sanchez } 3175d18ee67SSebastian Sanchez 3185d18ee67SSebastian Sanchez /* 3195d18ee67SSebastian Sanchez * Non-interrupt CPUs are used first, then interrupt CPUs. 3205d18ee67SSebastian Sanchez * Two already allocated cpu masks must be passed. 3215d18ee67SSebastian Sanchez */ 3225d18ee67SSebastian Sanchez static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd, 3235d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry, 3245d18ee67SSebastian Sanchez cpumask_var_t non_intr_cpus, 3255d18ee67SSebastian Sanchez cpumask_var_t available_cpus) 3265d18ee67SSebastian Sanchez __must_hold(&node_affinity.lock) 3275d18ee67SSebastian Sanchez { 3285d18ee67SSebastian Sanchez int cpu; 3295d18ee67SSebastian Sanchez struct cpu_mask_set *set = dd->comp_vect; 3305d18ee67SSebastian Sanchez 3315d18ee67SSebastian Sanchez lockdep_assert_held(&node_affinity.lock); 3325d18ee67SSebastian Sanchez if (!non_intr_cpus) { 3335d18ee67SSebastian Sanchez cpu = -1; 3345d18ee67SSebastian Sanchez goto fail; 3355d18ee67SSebastian Sanchez } 3365d18ee67SSebastian Sanchez 3375d18ee67SSebastian Sanchez if (!available_cpus) { 3385d18ee67SSebastian Sanchez cpu = -1; 3395d18ee67SSebastian Sanchez goto fail; 3405d18ee67SSebastian Sanchez } 3415d18ee67SSebastian Sanchez 3425d18ee67SSebastian Sanchez /* Available CPUs for pinning completion vectors */ 3435d18ee67SSebastian Sanchez _cpu_mask_set_gen_inc(set); 3445d18ee67SSebastian Sanchez cpumask_andnot(available_cpus, &set->mask, &set->used); 3455d18ee67SSebastian Sanchez 3465d18ee67SSebastian Sanchez /* Available CPUs without SDMA engine interrupts */ 3475d18ee67SSebastian Sanchez cpumask_andnot(non_intr_cpus, available_cpus, 3485d18ee67SSebastian Sanchez &entry->def_intr.used); 3495d18ee67SSebastian Sanchez 3505d18ee67SSebastian Sanchez /* If there are non-interrupt CPUs available, use them first */ 3515d18ee67SSebastian Sanchez if (!cpumask_empty(non_intr_cpus)) 3525d18ee67SSebastian Sanchez cpu = cpumask_first(non_intr_cpus); 3535d18ee67SSebastian Sanchez else /* Otherwise, use interrupt CPUs */ 3545d18ee67SSebastian Sanchez cpu = cpumask_first(available_cpus); 3555d18ee67SSebastian Sanchez 3565d18ee67SSebastian Sanchez if (cpu >= nr_cpu_ids) { /* empty */ 3575d18ee67SSebastian Sanchez cpu = -1; 3585d18ee67SSebastian Sanchez goto fail; 3595d18ee67SSebastian Sanchez } 3605d18ee67SSebastian Sanchez cpumask_set_cpu(cpu, &set->used); 3615d18ee67SSebastian Sanchez 3625d18ee67SSebastian Sanchez fail: 3635d18ee67SSebastian Sanchez return cpu; 3645d18ee67SSebastian Sanchez } 3655d18ee67SSebastian Sanchez 3665d18ee67SSebastian Sanchez static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu) 3675d18ee67SSebastian Sanchez { 3685d18ee67SSebastian Sanchez struct cpu_mask_set *set = dd->comp_vect; 3695d18ee67SSebastian Sanchez 3705d18ee67SSebastian Sanchez if (cpu < 0) 3715d18ee67SSebastian Sanchez return; 3725d18ee67SSebastian Sanchez 3735d18ee67SSebastian Sanchez cpu_mask_set_put(set, cpu); 3745d18ee67SSebastian Sanchez } 3755d18ee67SSebastian Sanchez 3765d18ee67SSebastian Sanchez /* _dev_comp_vect_mappings_destroy() is reentrant */ 3775d18ee67SSebastian Sanchez static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd) 3785d18ee67SSebastian Sanchez { 3795d18ee67SSebastian Sanchez int i, cpu; 3805d18ee67SSebastian Sanchez 3815d18ee67SSebastian Sanchez if (!dd->comp_vect_mappings) 3825d18ee67SSebastian Sanchez return; 3835d18ee67SSebastian Sanchez 3845d18ee67SSebastian Sanchez for (i = 0; i < dd->comp_vect_possible_cpus; i++) { 3855d18ee67SSebastian Sanchez cpu = dd->comp_vect_mappings[i]; 3865d18ee67SSebastian Sanchez _dev_comp_vect_cpu_put(dd, cpu); 3875d18ee67SSebastian Sanchez dd->comp_vect_mappings[i] = -1; 3885d18ee67SSebastian Sanchez hfi1_cdbg(AFFINITY, 3895d18ee67SSebastian Sanchez "[%s] Release CPU %d from completion vector %d", 3905d18ee67SSebastian Sanchez rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i); 3915d18ee67SSebastian Sanchez } 3925d18ee67SSebastian Sanchez 3935d18ee67SSebastian Sanchez kfree(dd->comp_vect_mappings); 3945d18ee67SSebastian Sanchez dd->comp_vect_mappings = NULL; 3955d18ee67SSebastian Sanchez } 3965d18ee67SSebastian Sanchez 3975d18ee67SSebastian Sanchez /* 3985d18ee67SSebastian Sanchez * This function creates the table for looking up CPUs for completion vectors. 3995d18ee67SSebastian Sanchez * num_comp_vectors needs to have been initilized before calling this function. 4005d18ee67SSebastian Sanchez */ 4015d18ee67SSebastian Sanchez static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd, 4025d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry) 4035d18ee67SSebastian Sanchez __must_hold(&node_affinity.lock) 4045d18ee67SSebastian Sanchez { 4055d18ee67SSebastian Sanchez int i, cpu, ret; 4065d18ee67SSebastian Sanchez cpumask_var_t non_intr_cpus; 4075d18ee67SSebastian Sanchez cpumask_var_t available_cpus; 4085d18ee67SSebastian Sanchez 4095d18ee67SSebastian Sanchez lockdep_assert_held(&node_affinity.lock); 4105d18ee67SSebastian Sanchez 4115d18ee67SSebastian Sanchez if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL)) 4125d18ee67SSebastian Sanchez return -ENOMEM; 4135d18ee67SSebastian Sanchez 4145d18ee67SSebastian Sanchez if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) { 4155d18ee67SSebastian Sanchez free_cpumask_var(non_intr_cpus); 4165d18ee67SSebastian Sanchez return -ENOMEM; 4175d18ee67SSebastian Sanchez } 4185d18ee67SSebastian Sanchez 4195d18ee67SSebastian Sanchez dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus, 4205d18ee67SSebastian Sanchez sizeof(*dd->comp_vect_mappings), 4215d18ee67SSebastian Sanchez GFP_KERNEL); 4225d18ee67SSebastian Sanchez if (!dd->comp_vect_mappings) { 4235d18ee67SSebastian Sanchez ret = -ENOMEM; 4245d18ee67SSebastian Sanchez goto fail; 4255d18ee67SSebastian Sanchez } 4265d18ee67SSebastian Sanchez for (i = 0; i < dd->comp_vect_possible_cpus; i++) 4275d18ee67SSebastian Sanchez dd->comp_vect_mappings[i] = -1; 4285d18ee67SSebastian Sanchez 4295d18ee67SSebastian Sanchez for (i = 0; i < dd->comp_vect_possible_cpus; i++) { 4305d18ee67SSebastian Sanchez cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus, 4315d18ee67SSebastian Sanchez available_cpus); 4325d18ee67SSebastian Sanchez if (cpu < 0) { 4335d18ee67SSebastian Sanchez ret = -EINVAL; 4345d18ee67SSebastian Sanchez goto fail; 4355d18ee67SSebastian Sanchez } 4365d18ee67SSebastian Sanchez 4375d18ee67SSebastian Sanchez dd->comp_vect_mappings[i] = cpu; 4385d18ee67SSebastian Sanchez hfi1_cdbg(AFFINITY, 4395d18ee67SSebastian Sanchez "[%s] Completion Vector %d -> CPU %d", 4405d18ee67SSebastian Sanchez rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu); 4415d18ee67SSebastian Sanchez } 4425d18ee67SSebastian Sanchez 4438a4f300bSKamal Heib free_cpumask_var(available_cpus); 4448a4f300bSKamal Heib free_cpumask_var(non_intr_cpus); 4455d18ee67SSebastian Sanchez return 0; 4465d18ee67SSebastian Sanchez 4475d18ee67SSebastian Sanchez fail: 4485d18ee67SSebastian Sanchez free_cpumask_var(available_cpus); 4495d18ee67SSebastian Sanchez free_cpumask_var(non_intr_cpus); 4505d18ee67SSebastian Sanchez _dev_comp_vect_mappings_destroy(dd); 4515d18ee67SSebastian Sanchez 4525d18ee67SSebastian Sanchez return ret; 4535d18ee67SSebastian Sanchez } 4545d18ee67SSebastian Sanchez 4555d18ee67SSebastian Sanchez int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd) 4565d18ee67SSebastian Sanchez { 4575d18ee67SSebastian Sanchez int ret; 4585d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry; 4595d18ee67SSebastian Sanchez 4605d18ee67SSebastian Sanchez mutex_lock(&node_affinity.lock); 4615d18ee67SSebastian Sanchez entry = node_affinity_lookup(dd->node); 4625d18ee67SSebastian Sanchez if (!entry) { 4635d18ee67SSebastian Sanchez ret = -EINVAL; 4645d18ee67SSebastian Sanchez goto unlock; 4655d18ee67SSebastian Sanchez } 4665d18ee67SSebastian Sanchez ret = _dev_comp_vect_mappings_create(dd, entry); 4675d18ee67SSebastian Sanchez unlock: 4685d18ee67SSebastian Sanchez mutex_unlock(&node_affinity.lock); 4695d18ee67SSebastian Sanchez 4705d18ee67SSebastian Sanchez return ret; 4715d18ee67SSebastian Sanchez } 4725d18ee67SSebastian Sanchez 4735d18ee67SSebastian Sanchez void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd) 4745d18ee67SSebastian Sanchez { 4755d18ee67SSebastian Sanchez _dev_comp_vect_mappings_destroy(dd); 4765d18ee67SSebastian Sanchez } 4775d18ee67SSebastian Sanchez 4785d18ee67SSebastian Sanchez int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect) 4795d18ee67SSebastian Sanchez { 4805d18ee67SSebastian Sanchez struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); 4815d18ee67SSebastian Sanchez struct hfi1_devdata *dd = dd_from_dev(verbs_dev); 4825d18ee67SSebastian Sanchez 4835d18ee67SSebastian Sanchez if (!dd->comp_vect_mappings) 4845d18ee67SSebastian Sanchez return -EINVAL; 4855d18ee67SSebastian Sanchez if (comp_vect >= dd->comp_vect_possible_cpus) 4865d18ee67SSebastian Sanchez return -EINVAL; 4875d18ee67SSebastian Sanchez 4885d18ee67SSebastian Sanchez return dd->comp_vect_mappings[comp_vect]; 4895d18ee67SSebastian Sanchez } 4905d18ee67SSebastian Sanchez 4915d18ee67SSebastian Sanchez /* 4925d18ee67SSebastian Sanchez * It assumes dd->comp_vect_possible_cpus is available. 4935d18ee67SSebastian Sanchez */ 4945d18ee67SSebastian Sanchez static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd, 4955d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry, 4965d18ee67SSebastian Sanchez bool first_dev_init) 4975d18ee67SSebastian Sanchez __must_hold(&node_affinity.lock) 4985d18ee67SSebastian Sanchez { 4995d18ee67SSebastian Sanchez int i, j, curr_cpu; 5005d18ee67SSebastian Sanchez int possible_cpus_comp_vect = 0; 5015d18ee67SSebastian Sanchez struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask; 5025d18ee67SSebastian Sanchez 5035d18ee67SSebastian Sanchez lockdep_assert_held(&node_affinity.lock); 5045d18ee67SSebastian Sanchez /* 5055d18ee67SSebastian Sanchez * If there's only one CPU available for completion vectors, then 5065d18ee67SSebastian Sanchez * there will only be one completion vector available. Othewise, 5075d18ee67SSebastian Sanchez * the number of completion vector available will be the number of 5085d18ee67SSebastian Sanchez * available CPUs divide it by the number of devices in the 5095d18ee67SSebastian Sanchez * local NUMA node. 5105d18ee67SSebastian Sanchez */ 5115d18ee67SSebastian Sanchez if (cpumask_weight(&entry->comp_vect_mask) == 1) { 5125d18ee67SSebastian Sanchez possible_cpus_comp_vect = 1; 5135d18ee67SSebastian Sanchez dd_dev_warn(dd, 5145d18ee67SSebastian Sanchez "Number of kernel receive queues is too large for completion vector affinity to be effective\n"); 5155d18ee67SSebastian Sanchez } else { 5165d18ee67SSebastian Sanchez possible_cpus_comp_vect += 5175d18ee67SSebastian Sanchez cpumask_weight(&entry->comp_vect_mask) / 5185d18ee67SSebastian Sanchez hfi1_per_node_cntr[dd->node]; 5195d18ee67SSebastian Sanchez 5205d18ee67SSebastian Sanchez /* 5215d18ee67SSebastian Sanchez * If the completion vector CPUs available doesn't divide 5225d18ee67SSebastian Sanchez * evenly among devices, then the first device device to be 5235d18ee67SSebastian Sanchez * initialized gets an extra CPU. 5245d18ee67SSebastian Sanchez */ 5255d18ee67SSebastian Sanchez if (first_dev_init && 5265d18ee67SSebastian Sanchez cpumask_weight(&entry->comp_vect_mask) % 5275d18ee67SSebastian Sanchez hfi1_per_node_cntr[dd->node] != 0) 5285d18ee67SSebastian Sanchez possible_cpus_comp_vect++; 5295d18ee67SSebastian Sanchez } 5305d18ee67SSebastian Sanchez 5315d18ee67SSebastian Sanchez dd->comp_vect_possible_cpus = possible_cpus_comp_vect; 5325d18ee67SSebastian Sanchez 5335d18ee67SSebastian Sanchez /* Reserving CPUs for device completion vector */ 5345d18ee67SSebastian Sanchez for (i = 0; i < dd->comp_vect_possible_cpus; i++) { 5355d18ee67SSebastian Sanchez curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask, 5365d18ee67SSebastian Sanchez entry->comp_vect_affinity); 5375d18ee67SSebastian Sanchez if (curr_cpu < 0) 5385d18ee67SSebastian Sanchez goto fail; 5395d18ee67SSebastian Sanchez 5405d18ee67SSebastian Sanchez cpumask_set_cpu(curr_cpu, dev_comp_vect_mask); 5415d18ee67SSebastian Sanchez } 5425d18ee67SSebastian Sanchez 5435d18ee67SSebastian Sanchez hfi1_cdbg(AFFINITY, 5445d18ee67SSebastian Sanchez "[%s] Completion vector affinity CPU set(s) %*pbl", 5455d18ee67SSebastian Sanchez rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), 5465d18ee67SSebastian Sanchez cpumask_pr_args(dev_comp_vect_mask)); 5475d18ee67SSebastian Sanchez 5485d18ee67SSebastian Sanchez return 0; 5495d18ee67SSebastian Sanchez 5505d18ee67SSebastian Sanchez fail: 5515d18ee67SSebastian Sanchez for (j = 0; j < i; j++) 5525d18ee67SSebastian Sanchez per_cpu_affinity_put_max(&entry->comp_vect_mask, 5535d18ee67SSebastian Sanchez entry->comp_vect_affinity); 5545d18ee67SSebastian Sanchez 5555d18ee67SSebastian Sanchez return curr_cpu; 5565d18ee67SSebastian Sanchez } 5575d18ee67SSebastian Sanchez 5585d18ee67SSebastian Sanchez /* 5595d18ee67SSebastian Sanchez * It assumes dd->comp_vect_possible_cpus is available. 5605d18ee67SSebastian Sanchez */ 5615d18ee67SSebastian Sanchez static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd, 5625d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry) 5635d18ee67SSebastian Sanchez __must_hold(&node_affinity.lock) 5645d18ee67SSebastian Sanchez { 5655d18ee67SSebastian Sanchez int i, cpu; 5665d18ee67SSebastian Sanchez 5675d18ee67SSebastian Sanchez lockdep_assert_held(&node_affinity.lock); 5685d18ee67SSebastian Sanchez if (!dd->comp_vect_possible_cpus) 5695d18ee67SSebastian Sanchez return; 5705d18ee67SSebastian Sanchez 5715d18ee67SSebastian Sanchez for (i = 0; i < dd->comp_vect_possible_cpus; i++) { 5725d18ee67SSebastian Sanchez cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask, 5735d18ee67SSebastian Sanchez entry->comp_vect_affinity); 5745d18ee67SSebastian Sanchez /* Clearing CPU in device completion vector cpu mask */ 5755d18ee67SSebastian Sanchez if (cpu >= 0) 5765d18ee67SSebastian Sanchez cpumask_clear_cpu(cpu, &dd->comp_vect->mask); 5775d18ee67SSebastian Sanchez } 5785d18ee67SSebastian Sanchez 5795d18ee67SSebastian Sanchez dd->comp_vect_possible_cpus = 0; 5805d18ee67SSebastian Sanchez } 5815d18ee67SSebastian Sanchez 582f48ad614SDennis Dalessandro /* 583f48ad614SDennis Dalessandro * Interrupt affinity. 584f48ad614SDennis Dalessandro * 585f48ad614SDennis Dalessandro * non-rcv avail gets a default mask that 586f48ad614SDennis Dalessandro * starts as possible cpus with threads reset 587f48ad614SDennis Dalessandro * and each rcv avail reset. 588f48ad614SDennis Dalessandro * 589f48ad614SDennis Dalessandro * rcv avail gets node relative 1 wrapping back 590f48ad614SDennis Dalessandro * to the node relative 1 as necessary. 591f48ad614SDennis Dalessandro * 592f48ad614SDennis Dalessandro */ 5934197344bSDennis Dalessandro int hfi1_dev_affinity_init(struct hfi1_devdata *dd) 594f48ad614SDennis Dalessandro { 5954197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 596f48ad614SDennis Dalessandro const struct cpumask *local_mask; 5975d18ee67SSebastian Sanchez int curr_cpu, possible, i, ret; 5985d18ee67SSebastian Sanchez bool new_entry = false; 599f48ad614SDennis Dalessandro 600f48ad614SDennis Dalessandro local_mask = cpumask_of_node(dd->node); 601f48ad614SDennis Dalessandro if (cpumask_first(local_mask) >= nr_cpu_ids) 602f48ad614SDennis Dalessandro local_mask = topology_core_cpumask(0); 6034197344bSDennis Dalessandro 604584d9577STadeusz Struk mutex_lock(&node_affinity.lock); 6054197344bSDennis Dalessandro entry = node_affinity_lookup(dd->node); 6064197344bSDennis Dalessandro 6074197344bSDennis Dalessandro /* 6084197344bSDennis Dalessandro * If this is the first time this NUMA node's affinity is used, 6094197344bSDennis Dalessandro * create an entry in the global affinity structure and initialize it. 6104197344bSDennis Dalessandro */ 6114197344bSDennis Dalessandro if (!entry) { 6125de61a47SMike Marciniszyn entry = node_affinity_allocate(dd->node); 6134197344bSDennis Dalessandro if (!entry) { 6144197344bSDennis Dalessandro dd_dev_err(dd, 6154197344bSDennis Dalessandro "Unable to allocate global affinity node\n"); 6165d18ee67SSebastian Sanchez ret = -ENOMEM; 6175d18ee67SSebastian Sanchez goto fail; 6184197344bSDennis Dalessandro } 6195d18ee67SSebastian Sanchez new_entry = true; 6205d18ee67SSebastian Sanchez 6214197344bSDennis Dalessandro init_cpu_mask_set(&entry->def_intr); 6224197344bSDennis Dalessandro init_cpu_mask_set(&entry->rcv_intr); 6235d18ee67SSebastian Sanchez cpumask_clear(&entry->comp_vect_mask); 624d6373019SSebastian Sanchez cpumask_clear(&entry->general_intr_mask); 625f48ad614SDennis Dalessandro /* Use the "real" cpu mask of this node as the default */ 6264197344bSDennis Dalessandro cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask, 6274197344bSDennis Dalessandro local_mask); 628f48ad614SDennis Dalessandro 629f48ad614SDennis Dalessandro /* fill in the receive list */ 6304197344bSDennis Dalessandro possible = cpumask_weight(&entry->def_intr.mask); 6314197344bSDennis Dalessandro curr_cpu = cpumask_first(&entry->def_intr.mask); 6324197344bSDennis Dalessandro 633f48ad614SDennis Dalessandro if (possible == 1) { 634f48ad614SDennis Dalessandro /* only one CPU, everyone will use it */ 6354197344bSDennis Dalessandro cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask); 636d6373019SSebastian Sanchez cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); 637f48ad614SDennis Dalessandro } else { 638f48ad614SDennis Dalessandro /* 639d6373019SSebastian Sanchez * The general/control context will be the first CPU in 640d6373019SSebastian Sanchez * the default list, so it is removed from the default 641d6373019SSebastian Sanchez * list and added to the general interrupt list. 642f48ad614SDennis Dalessandro */ 643d6373019SSebastian Sanchez cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask); 644d6373019SSebastian Sanchez cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); 6454197344bSDennis Dalessandro curr_cpu = cpumask_next(curr_cpu, 6464197344bSDennis Dalessandro &entry->def_intr.mask); 6474197344bSDennis Dalessandro 648f48ad614SDennis Dalessandro /* 649f48ad614SDennis Dalessandro * Remove the remaining kernel receive queues from 650f48ad614SDennis Dalessandro * the default list and add them to the receive list. 651f48ad614SDennis Dalessandro */ 652d6373019SSebastian Sanchez for (i = 0; 653d6373019SSebastian Sanchez i < (dd->n_krcv_queues - 1) * 654d6373019SSebastian Sanchez hfi1_per_node_cntr[dd->node]; 655d6373019SSebastian Sanchez i++) { 6564197344bSDennis Dalessandro cpumask_clear_cpu(curr_cpu, 6574197344bSDennis Dalessandro &entry->def_intr.mask); 6584197344bSDennis Dalessandro cpumask_set_cpu(curr_cpu, 6594197344bSDennis Dalessandro &entry->rcv_intr.mask); 6604197344bSDennis Dalessandro curr_cpu = cpumask_next(curr_cpu, 6614197344bSDennis Dalessandro &entry->def_intr.mask); 662f48ad614SDennis Dalessandro if (curr_cpu >= nr_cpu_ids) 663f48ad614SDennis Dalessandro break; 664f48ad614SDennis Dalessandro } 665d6373019SSebastian Sanchez 666d6373019SSebastian Sanchez /* 667d6373019SSebastian Sanchez * If there ends up being 0 CPU cores leftover for SDMA 668d6373019SSebastian Sanchez * engines, use the same CPU cores as general/control 669d6373019SSebastian Sanchez * context. 670d6373019SSebastian Sanchez */ 6713c8bc395SYury Norov if (cpumask_empty(&entry->def_intr.mask)) 672d6373019SSebastian Sanchez cpumask_copy(&entry->def_intr.mask, 673d6373019SSebastian Sanchez &entry->general_intr_mask); 674f48ad614SDennis Dalessandro } 675f48ad614SDennis Dalessandro 6765d18ee67SSebastian Sanchez /* Determine completion vector CPUs for the entire node */ 6775d18ee67SSebastian Sanchez cpumask_and(&entry->comp_vect_mask, 6785d18ee67SSebastian Sanchez &node_affinity.real_cpu_mask, local_mask); 6795d18ee67SSebastian Sanchez cpumask_andnot(&entry->comp_vect_mask, 6805d18ee67SSebastian Sanchez &entry->comp_vect_mask, 6815d18ee67SSebastian Sanchez &entry->rcv_intr.mask); 6825d18ee67SSebastian Sanchez cpumask_andnot(&entry->comp_vect_mask, 6835d18ee67SSebastian Sanchez &entry->comp_vect_mask, 6845d18ee67SSebastian Sanchez &entry->general_intr_mask); 6855d18ee67SSebastian Sanchez 6865d18ee67SSebastian Sanchez /* 6875d18ee67SSebastian Sanchez * If there ends up being 0 CPU cores leftover for completion 6885d18ee67SSebastian Sanchez * vectors, use the same CPU core as the general/control 6895d18ee67SSebastian Sanchez * context. 6905d18ee67SSebastian Sanchez */ 6913c8bc395SYury Norov if (cpumask_empty(&entry->comp_vect_mask)) 6925d18ee67SSebastian Sanchez cpumask_copy(&entry->comp_vect_mask, 6935d18ee67SSebastian Sanchez &entry->general_intr_mask); 694f48ad614SDennis Dalessandro } 6955d18ee67SSebastian Sanchez 6965d18ee67SSebastian Sanchez ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry); 6975d18ee67SSebastian Sanchez if (ret < 0) 6985d18ee67SSebastian Sanchez goto fail; 6995d18ee67SSebastian Sanchez 7005d18ee67SSebastian Sanchez if (new_entry) 7015d18ee67SSebastian Sanchez node_affinity_add_tail(entry); 7025d18ee67SSebastian Sanchez 7035de61a47SMike Marciniszyn dd->affinity_entry = entry; 704584d9577STadeusz Struk mutex_unlock(&node_affinity.lock); 7055d18ee67SSebastian Sanchez 7064197344bSDennis Dalessandro return 0; 7075d18ee67SSebastian Sanchez 7085d18ee67SSebastian Sanchez fail: 7095d18ee67SSebastian Sanchez if (new_entry) 7105d18ee67SSebastian Sanchez node_affinity_destroy(entry); 7115d18ee67SSebastian Sanchez mutex_unlock(&node_affinity.lock); 7125d18ee67SSebastian Sanchez return ret; 7135d18ee67SSebastian Sanchez } 7145d18ee67SSebastian Sanchez 7155d18ee67SSebastian Sanchez void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd) 7165d18ee67SSebastian Sanchez { 7175d18ee67SSebastian Sanchez struct hfi1_affinity_node *entry; 7185d18ee67SSebastian Sanchez 7195d18ee67SSebastian Sanchez mutex_lock(&node_affinity.lock); 7205de61a47SMike Marciniszyn if (!dd->affinity_entry) 7215de61a47SMike Marciniszyn goto unlock; 7225d18ee67SSebastian Sanchez entry = node_affinity_lookup(dd->node); 7235d18ee67SSebastian Sanchez if (!entry) 7245d18ee67SSebastian Sanchez goto unlock; 7255d18ee67SSebastian Sanchez 7265d18ee67SSebastian Sanchez /* 7275d18ee67SSebastian Sanchez * Free device completion vector CPUs to be used by future 7285d18ee67SSebastian Sanchez * completion vectors 7295d18ee67SSebastian Sanchez */ 7305d18ee67SSebastian Sanchez _dev_comp_vect_cpu_mask_clean_up(dd, entry); 7315d18ee67SSebastian Sanchez unlock: 7325de61a47SMike Marciniszyn dd->affinity_entry = NULL; 7335d18ee67SSebastian Sanchez mutex_unlock(&node_affinity.lock); 734f48ad614SDennis Dalessandro } 735f48ad614SDennis Dalessandro 736584d9577STadeusz Struk /* 7372d01c37dSTadeusz Struk * Function updates the irq affinity hint for msix after it has been changed 7382d01c37dSTadeusz Struk * by the user using the /proc/irq interface. This function only accepts 7392d01c37dSTadeusz Struk * one cpu in the mask. 7402d01c37dSTadeusz Struk */ 7412d01c37dSTadeusz Struk static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu) 7422d01c37dSTadeusz Struk { 7432d01c37dSTadeusz Struk struct sdma_engine *sde = msix->arg; 7442d01c37dSTadeusz Struk struct hfi1_devdata *dd = sde->dd; 7452d01c37dSTadeusz Struk struct hfi1_affinity_node *entry; 7462d01c37dSTadeusz Struk struct cpu_mask_set *set; 7472d01c37dSTadeusz Struk int i, old_cpu; 7482d01c37dSTadeusz Struk 7492d01c37dSTadeusz Struk if (cpu > num_online_cpus() || cpu == sde->cpu) 7502d01c37dSTadeusz Struk return; 7512d01c37dSTadeusz Struk 7522d01c37dSTadeusz Struk mutex_lock(&node_affinity.lock); 7532d01c37dSTadeusz Struk entry = node_affinity_lookup(dd->node); 7542d01c37dSTadeusz Struk if (!entry) 7552d01c37dSTadeusz Struk goto unlock; 7562d01c37dSTadeusz Struk 7572d01c37dSTadeusz Struk old_cpu = sde->cpu; 7582d01c37dSTadeusz Struk sde->cpu = cpu; 7592d01c37dSTadeusz Struk cpumask_clear(&msix->mask); 7602d01c37dSTadeusz Struk cpumask_set_cpu(cpu, &msix->mask); 761bb7dde87SMichael J. Ruhl dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n", 762bb7dde87SMichael J. Ruhl msix->irq, irq_type_names[msix->type], 7632d01c37dSTadeusz Struk sde->this_idx, cpu); 764bb7dde87SMichael J. Ruhl irq_set_affinity_hint(msix->irq, &msix->mask); 7652d01c37dSTadeusz Struk 7662d01c37dSTadeusz Struk /* 7672d01c37dSTadeusz Struk * Set the new cpu in the hfi1_affinity_node and clean 7682d01c37dSTadeusz Struk * the old cpu if it is not used by any other IRQ 7692d01c37dSTadeusz Struk */ 7702d01c37dSTadeusz Struk set = &entry->def_intr; 7712d01c37dSTadeusz Struk cpumask_set_cpu(cpu, &set->mask); 7722d01c37dSTadeusz Struk cpumask_set_cpu(cpu, &set->used); 7736eb4eb10SMichael J. Ruhl for (i = 0; i < dd->msix_info.max_requested; i++) { 7742d01c37dSTadeusz Struk struct hfi1_msix_entry *other_msix; 7752d01c37dSTadeusz Struk 7766eb4eb10SMichael J. Ruhl other_msix = &dd->msix_info.msix_entries[i]; 7772d01c37dSTadeusz Struk if (other_msix->type != IRQ_SDMA || other_msix == msix) 7782d01c37dSTadeusz Struk continue; 7792d01c37dSTadeusz Struk 7802d01c37dSTadeusz Struk if (cpumask_test_cpu(old_cpu, &other_msix->mask)) 7812d01c37dSTadeusz Struk goto unlock; 7822d01c37dSTadeusz Struk } 7832d01c37dSTadeusz Struk cpumask_clear_cpu(old_cpu, &set->mask); 7842d01c37dSTadeusz Struk cpumask_clear_cpu(old_cpu, &set->used); 7852d01c37dSTadeusz Struk unlock: 7862d01c37dSTadeusz Struk mutex_unlock(&node_affinity.lock); 7872d01c37dSTadeusz Struk } 7882d01c37dSTadeusz Struk 7892d01c37dSTadeusz Struk static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify, 7902d01c37dSTadeusz Struk const cpumask_t *mask) 7912d01c37dSTadeusz Struk { 7922d01c37dSTadeusz Struk int cpu = cpumask_first(mask); 7932d01c37dSTadeusz Struk struct hfi1_msix_entry *msix = container_of(notify, 7942d01c37dSTadeusz Struk struct hfi1_msix_entry, 7952d01c37dSTadeusz Struk notify); 7962d01c37dSTadeusz Struk 7972d01c37dSTadeusz Struk /* Only one CPU configuration supported currently */ 7982d01c37dSTadeusz Struk hfi1_update_sdma_affinity(msix, cpu); 7992d01c37dSTadeusz Struk } 8002d01c37dSTadeusz Struk 8012d01c37dSTadeusz Struk static void hfi1_irq_notifier_release(struct kref *ref) 8022d01c37dSTadeusz Struk { 8032d01c37dSTadeusz Struk /* 8042d01c37dSTadeusz Struk * This is required by affinity notifier. We don't have anything to 8052d01c37dSTadeusz Struk * free here. 8062d01c37dSTadeusz Struk */ 8072d01c37dSTadeusz Struk } 8082d01c37dSTadeusz Struk 8092d01c37dSTadeusz Struk static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix) 8102d01c37dSTadeusz Struk { 8112d01c37dSTadeusz Struk struct irq_affinity_notify *notify = &msix->notify; 8122d01c37dSTadeusz Struk 813bb7dde87SMichael J. Ruhl notify->irq = msix->irq; 8142d01c37dSTadeusz Struk notify->notify = hfi1_irq_notifier_notify; 8152d01c37dSTadeusz Struk notify->release = hfi1_irq_notifier_release; 8162d01c37dSTadeusz Struk 8172d01c37dSTadeusz Struk if (irq_set_affinity_notifier(notify->irq, notify)) 8182d01c37dSTadeusz Struk pr_err("Failed to register sdma irq affinity notifier for irq %d\n", 8192d01c37dSTadeusz Struk notify->irq); 8202d01c37dSTadeusz Struk } 8212d01c37dSTadeusz Struk 8222d01c37dSTadeusz Struk static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix) 8232d01c37dSTadeusz Struk { 8242d01c37dSTadeusz Struk struct irq_affinity_notify *notify = &msix->notify; 8252d01c37dSTadeusz Struk 8262d01c37dSTadeusz Struk if (irq_set_affinity_notifier(notify->irq, NULL)) 8272d01c37dSTadeusz Struk pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n", 8282d01c37dSTadeusz Struk notify->irq); 8292d01c37dSTadeusz Struk } 8302d01c37dSTadeusz Struk 8312d01c37dSTadeusz Struk /* 832584d9577STadeusz Struk * Function sets the irq affinity for msix. 833584d9577STadeusz Struk * It *must* be called with node_affinity.lock held. 834584d9577STadeusz Struk */ 835584d9577STadeusz Struk static int get_irq_affinity(struct hfi1_devdata *dd, 836584d9577STadeusz Struk struct hfi1_msix_entry *msix) 837f48ad614SDennis Dalessandro { 838f48ad614SDennis Dalessandro cpumask_var_t diff; 8394197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 840d6373019SSebastian Sanchez struct cpu_mask_set *set = NULL; 841f48ad614SDennis Dalessandro struct sdma_engine *sde = NULL; 842f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = NULL; 843f48ad614SDennis Dalessandro char extra[64]; 844f48ad614SDennis Dalessandro int cpu = -1; 845f48ad614SDennis Dalessandro 846f48ad614SDennis Dalessandro extra[0] = '\0'; 847f48ad614SDennis Dalessandro cpumask_clear(&msix->mask); 848f48ad614SDennis Dalessandro 8494197344bSDennis Dalessandro entry = node_affinity_lookup(dd->node); 8504197344bSDennis Dalessandro 851f48ad614SDennis Dalessandro switch (msix->type) { 852f48ad614SDennis Dalessandro case IRQ_SDMA: 853f48ad614SDennis Dalessandro sde = (struct sdma_engine *)msix->arg; 854f48ad614SDennis Dalessandro scnprintf(extra, 64, "engine %u", sde->this_idx); 8554197344bSDennis Dalessandro set = &entry->def_intr; 856f48ad614SDennis Dalessandro break; 857d6373019SSebastian Sanchez case IRQ_GENERAL: 858d6373019SSebastian Sanchez cpu = cpumask_first(&entry->general_intr_mask); 859d6373019SSebastian Sanchez break; 860f48ad614SDennis Dalessandro case IRQ_RCVCTXT: 861f48ad614SDennis Dalessandro rcd = (struct hfi1_ctxtdata *)msix->arg; 862d6373019SSebastian Sanchez if (rcd->ctxt == HFI1_CTRL_CTXT) 863d6373019SSebastian Sanchez cpu = cpumask_first(&entry->general_intr_mask); 864d6373019SSebastian Sanchez else 8654197344bSDennis Dalessandro set = &entry->rcv_intr; 866f48ad614SDennis Dalessandro scnprintf(extra, 64, "ctxt %u", rcd->ctxt); 867f48ad614SDennis Dalessandro break; 8680bae02d5SGrzegorz Andrejczuk case IRQ_NETDEVCTXT: 8690bae02d5SGrzegorz Andrejczuk rcd = (struct hfi1_ctxtdata *)msix->arg; 8700bae02d5SGrzegorz Andrejczuk set = &entry->def_intr; 8710bae02d5SGrzegorz Andrejczuk scnprintf(extra, 64, "ctxt %u", rcd->ctxt); 8720bae02d5SGrzegorz Andrejczuk break; 873f48ad614SDennis Dalessandro default: 874f48ad614SDennis Dalessandro dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type); 875f48ad614SDennis Dalessandro return -EINVAL; 876f48ad614SDennis Dalessandro } 877f48ad614SDennis Dalessandro 878f48ad614SDennis Dalessandro /* 879d6373019SSebastian Sanchez * The general and control contexts are placed on a particular 880d6373019SSebastian Sanchez * CPU, which is set above. Skip accounting for it. Everything else 881d6373019SSebastian Sanchez * finds its CPU here. 882f48ad614SDennis Dalessandro */ 8834197344bSDennis Dalessandro if (cpu == -1 && set) { 88459482a14SSebastian Sanchez if (!zalloc_cpumask_var(&diff, GFP_KERNEL)) 88559482a14SSebastian Sanchez return -ENOMEM; 88659482a14SSebastian Sanchez 887cf38ea10SSebastian Sanchez cpu = cpu_mask_set_get_first(set, diff); 888cf38ea10SSebastian Sanchez if (cpu < 0) { 889cf38ea10SSebastian Sanchez free_cpumask_var(diff); 890cf38ea10SSebastian Sanchez dd_dev_err(dd, "Failure to obtain CPU for IRQ\n"); 891cf38ea10SSebastian Sanchez return cpu; 892f48ad614SDennis Dalessandro } 89359482a14SSebastian Sanchez 89459482a14SSebastian Sanchez free_cpumask_var(diff); 895f48ad614SDennis Dalessandro } 896f48ad614SDennis Dalessandro 897f48ad614SDennis Dalessandro cpumask_set_cpu(cpu, &msix->mask); 898bb7dde87SMichael J. Ruhl dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n", 899bb7dde87SMichael J. Ruhl msix->irq, irq_type_names[msix->type], 900f48ad614SDennis Dalessandro extra, cpu); 901bb7dde87SMichael J. Ruhl irq_set_affinity_hint(msix->irq, &msix->mask); 902f48ad614SDennis Dalessandro 9032d01c37dSTadeusz Struk if (msix->type == IRQ_SDMA) { 9042d01c37dSTadeusz Struk sde->cpu = cpu; 9052d01c37dSTadeusz Struk hfi1_setup_sdma_notifier(msix); 9062d01c37dSTadeusz Struk } 9072d01c37dSTadeusz Struk 908f48ad614SDennis Dalessandro return 0; 909f48ad614SDennis Dalessandro } 910f48ad614SDennis Dalessandro 911584d9577STadeusz Struk int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix) 912584d9577STadeusz Struk { 913584d9577STadeusz Struk int ret; 914584d9577STadeusz Struk 915584d9577STadeusz Struk mutex_lock(&node_affinity.lock); 916584d9577STadeusz Struk ret = get_irq_affinity(dd, msix); 917584d9577STadeusz Struk mutex_unlock(&node_affinity.lock); 918584d9577STadeusz Struk return ret; 919584d9577STadeusz Struk } 920584d9577STadeusz Struk 921f48ad614SDennis Dalessandro void hfi1_put_irq_affinity(struct hfi1_devdata *dd, 922f48ad614SDennis Dalessandro struct hfi1_msix_entry *msix) 923f48ad614SDennis Dalessandro { 924f48ad614SDennis Dalessandro struct cpu_mask_set *set = NULL; 9254197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 9264197344bSDennis Dalessandro 927584d9577STadeusz Struk mutex_lock(&node_affinity.lock); 9284197344bSDennis Dalessandro entry = node_affinity_lookup(dd->node); 929f48ad614SDennis Dalessandro 930f48ad614SDennis Dalessandro switch (msix->type) { 931f48ad614SDennis Dalessandro case IRQ_SDMA: 9324197344bSDennis Dalessandro set = &entry->def_intr; 9332d01c37dSTadeusz Struk hfi1_cleanup_sdma_notifier(msix); 934f48ad614SDennis Dalessandro break; 935d6373019SSebastian Sanchez case IRQ_GENERAL: 936b094a36fSSebastian Sanchez /* Don't do accounting for general contexts */ 937d6373019SSebastian Sanchez break; 9382d085ee1SJiapeng Chong case IRQ_RCVCTXT: { 9392d085ee1SJiapeng Chong struct hfi1_ctxtdata *rcd = msix->arg; 9402d085ee1SJiapeng Chong 941d6373019SSebastian Sanchez /* Don't do accounting for control contexts */ 942f48ad614SDennis Dalessandro if (rcd->ctxt != HFI1_CTRL_CTXT) 9434197344bSDennis Dalessandro set = &entry->rcv_intr; 944f48ad614SDennis Dalessandro break; 9452d085ee1SJiapeng Chong } 9460bae02d5SGrzegorz Andrejczuk case IRQ_NETDEVCTXT: 9470bae02d5SGrzegorz Andrejczuk set = &entry->def_intr; 9480bae02d5SGrzegorz Andrejczuk break; 949f48ad614SDennis Dalessandro default: 950584d9577STadeusz Struk mutex_unlock(&node_affinity.lock); 951f48ad614SDennis Dalessandro return; 952f48ad614SDennis Dalessandro } 953f48ad614SDennis Dalessandro 954f48ad614SDennis Dalessandro if (set) { 955f48ad614SDennis Dalessandro cpumask_andnot(&set->used, &set->used, &msix->mask); 956cf38ea10SSebastian Sanchez _cpu_mask_set_gen_dec(set); 957f48ad614SDennis Dalessandro } 958f48ad614SDennis Dalessandro 959bb7dde87SMichael J. Ruhl irq_set_affinity_hint(msix->irq, NULL); 960f48ad614SDennis Dalessandro cpumask_clear(&msix->mask); 961584d9577STadeusz Struk mutex_unlock(&node_affinity.lock); 962f48ad614SDennis Dalessandro } 963f48ad614SDennis Dalessandro 964b094a36fSSebastian Sanchez /* This should be called with node_affinity.lock held */ 965b094a36fSSebastian Sanchez static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask, 966b094a36fSSebastian Sanchez struct hfi1_affinity_node_list *affinity) 967f48ad614SDennis Dalessandro { 968b094a36fSSebastian Sanchez int possible, curr_cpu, i; 969b094a36fSSebastian Sanchez uint num_cores_per_socket = node_affinity.num_online_cpus / 970b094a36fSSebastian Sanchez affinity->num_core_siblings / 971b094a36fSSebastian Sanchez node_affinity.num_online_nodes; 972b094a36fSSebastian Sanchez 973b094a36fSSebastian Sanchez cpumask_copy(hw_thread_mask, &affinity->proc.mask); 974b094a36fSSebastian Sanchez if (affinity->num_core_siblings > 0) { 975b094a36fSSebastian Sanchez /* Removing other siblings not needed for now */ 976b094a36fSSebastian Sanchez possible = cpumask_weight(hw_thread_mask); 977b094a36fSSebastian Sanchez curr_cpu = cpumask_first(hw_thread_mask); 978b094a36fSSebastian Sanchez for (i = 0; 979b094a36fSSebastian Sanchez i < num_cores_per_socket * node_affinity.num_online_nodes; 980b094a36fSSebastian Sanchez i++) 981b094a36fSSebastian Sanchez curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); 982b094a36fSSebastian Sanchez 983b094a36fSSebastian Sanchez for (; i < possible; i++) { 984b094a36fSSebastian Sanchez cpumask_clear_cpu(curr_cpu, hw_thread_mask); 985b094a36fSSebastian Sanchez curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); 986b094a36fSSebastian Sanchez } 987b094a36fSSebastian Sanchez 988b094a36fSSebastian Sanchez /* Identifying correct HW threads within physical cores */ 989b094a36fSSebastian Sanchez cpumask_shift_left(hw_thread_mask, hw_thread_mask, 990b094a36fSSebastian Sanchez num_cores_per_socket * 991b094a36fSSebastian Sanchez node_affinity.num_online_nodes * 992b094a36fSSebastian Sanchez hw_thread_no); 993b094a36fSSebastian Sanchez } 994b094a36fSSebastian Sanchez } 995b094a36fSSebastian Sanchez 996b094a36fSSebastian Sanchez int hfi1_get_proc_affinity(int node) 997b094a36fSSebastian Sanchez { 998b094a36fSSebastian Sanchez int cpu = -1, ret, i; 9994197344bSDennis Dalessandro struct hfi1_affinity_node *entry; 1000b094a36fSSebastian Sanchez cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask; 1001f48ad614SDennis Dalessandro const struct cpumask *node_mask, 10023bd37062SSebastian Andrzej Siewior *proc_mask = current->cpus_ptr; 1003b094a36fSSebastian Sanchez struct hfi1_affinity_node_list *affinity = &node_affinity; 1004b094a36fSSebastian Sanchez struct cpu_mask_set *set = &affinity->proc; 1005f48ad614SDennis Dalessandro 1006f48ad614SDennis Dalessandro /* 1007f48ad614SDennis Dalessandro * check whether process/context affinity has already 1008f48ad614SDennis Dalessandro * been set 1009f48ad614SDennis Dalessandro */ 10103bd37062SSebastian Andrzej Siewior if (current->nr_cpus_allowed == 1) { 1011f242d93aSLeon Romanovsky hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl", 1012f242d93aSLeon Romanovsky current->pid, current->comm, 1013f242d93aSLeon Romanovsky cpumask_pr_args(proc_mask)); 1014f48ad614SDennis Dalessandro /* 1015f48ad614SDennis Dalessandro * Mark the pre-set CPU as used. This is atomic so we don't 1016f48ad614SDennis Dalessandro * need the lock 1017f48ad614SDennis Dalessandro */ 1018f48ad614SDennis Dalessandro cpu = cpumask_first(proc_mask); 1019f48ad614SDennis Dalessandro cpumask_set_cpu(cpu, &set->used); 1020f48ad614SDennis Dalessandro goto done; 10213bd37062SSebastian Andrzej Siewior } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) { 1022f242d93aSLeon Romanovsky hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl", 1023f242d93aSLeon Romanovsky current->pid, current->comm, 1024f242d93aSLeon Romanovsky cpumask_pr_args(proc_mask)); 1025f48ad614SDennis Dalessandro goto done; 1026f48ad614SDennis Dalessandro } 1027f48ad614SDennis Dalessandro 1028f48ad614SDennis Dalessandro /* 1029f48ad614SDennis Dalessandro * The process does not have a preset CPU affinity so find one to 1030b094a36fSSebastian Sanchez * recommend using the following algorithm: 1031b094a36fSSebastian Sanchez * 1032b094a36fSSebastian Sanchez * For each user process that is opening a context on HFI Y: 1033b094a36fSSebastian Sanchez * a) If all cores are filled, reinitialize the bitmask 1034b094a36fSSebastian Sanchez * b) Fill real cores first, then HT cores (First set of HT 1035b094a36fSSebastian Sanchez * cores on all physical cores, then second set of HT core, 1036b094a36fSSebastian Sanchez * and, so on) in the following order: 1037b094a36fSSebastian Sanchez * 1038b094a36fSSebastian Sanchez * 1. Same NUMA node as HFI Y and not running an IRQ 1039b094a36fSSebastian Sanchez * handler 1040b094a36fSSebastian Sanchez * 2. Same NUMA node as HFI Y and running an IRQ handler 1041b094a36fSSebastian Sanchez * 3. Different NUMA node to HFI Y and not running an IRQ 1042b094a36fSSebastian Sanchez * handler 1043b094a36fSSebastian Sanchez * 4. Different NUMA node to HFI Y and running an IRQ 1044b094a36fSSebastian Sanchez * handler 1045b094a36fSSebastian Sanchez * c) Mark core as filled in the bitmask. As user processes are 1046b094a36fSSebastian Sanchez * done, clear cores from the bitmask. 1047f48ad614SDennis Dalessandro */ 1048f48ad614SDennis Dalessandro 1049f48ad614SDennis Dalessandro ret = zalloc_cpumask_var(&diff, GFP_KERNEL); 1050f48ad614SDennis Dalessandro if (!ret) 1051f48ad614SDennis Dalessandro goto done; 1052b094a36fSSebastian Sanchez ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL); 1053f48ad614SDennis Dalessandro if (!ret) 1054f48ad614SDennis Dalessandro goto free_diff; 1055b094a36fSSebastian Sanchez ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL); 1056f48ad614SDennis Dalessandro if (!ret) 1057b094a36fSSebastian Sanchez goto free_hw_thread_mask; 1058b094a36fSSebastian Sanchez ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL); 1059b094a36fSSebastian Sanchez if (!ret) 1060b094a36fSSebastian Sanchez goto free_available_mask; 1061f48ad614SDennis Dalessandro 1062584d9577STadeusz Struk mutex_lock(&affinity->lock); 1063f48ad614SDennis Dalessandro /* 1064b094a36fSSebastian Sanchez * If we've used all available HW threads, clear the mask and start 1065f48ad614SDennis Dalessandro * overloading. 1066f48ad614SDennis Dalessandro */ 1067cf38ea10SSebastian Sanchez _cpu_mask_set_gen_inc(set); 1068f48ad614SDennis Dalessandro 1069d6373019SSebastian Sanchez /* 1070d6373019SSebastian Sanchez * If NUMA node has CPUs used by interrupt handlers, include them in the 1071d6373019SSebastian Sanchez * interrupt handler mask. 1072d6373019SSebastian Sanchez */ 1073d6373019SSebastian Sanchez entry = node_affinity_lookup(node); 1074d6373019SSebastian Sanchez if (entry) { 1075b094a36fSSebastian Sanchez cpumask_copy(intrs_mask, (entry->def_intr.gen ? 10764197344bSDennis Dalessandro &entry->def_intr.mask : 10774197344bSDennis Dalessandro &entry->def_intr.used)); 1078b094a36fSSebastian Sanchez cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ? 10794197344bSDennis Dalessandro &entry->rcv_intr.mask : 10804197344bSDennis Dalessandro &entry->rcv_intr.used)); 1081b094a36fSSebastian Sanchez cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask); 1082d6373019SSebastian Sanchez } 1083f242d93aSLeon Romanovsky hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl", 1084b094a36fSSebastian Sanchez cpumask_pr_args(intrs_mask)); 1085b094a36fSSebastian Sanchez 1086b094a36fSSebastian Sanchez cpumask_copy(hw_thread_mask, &set->mask); 1087f48ad614SDennis Dalessandro 1088f48ad614SDennis Dalessandro /* 1089b094a36fSSebastian Sanchez * If HT cores are enabled, identify which HW threads within the 1090b094a36fSSebastian Sanchez * physical cores should be used. 1091f48ad614SDennis Dalessandro */ 1092b094a36fSSebastian Sanchez if (affinity->num_core_siblings > 0) { 1093b094a36fSSebastian Sanchez for (i = 0; i < affinity->num_core_siblings; i++) { 1094b094a36fSSebastian Sanchez find_hw_thread_mask(i, hw_thread_mask, affinity); 1095b094a36fSSebastian Sanchez 1096b094a36fSSebastian Sanchez /* 1097b094a36fSSebastian Sanchez * If there's at least one available core for this HW 1098b094a36fSSebastian Sanchez * thread number, stop looking for a core. 1099b094a36fSSebastian Sanchez * 1100b094a36fSSebastian Sanchez * diff will always be not empty at least once in this 1101b094a36fSSebastian Sanchez * loop as the used mask gets reset when 1102b094a36fSSebastian Sanchez * (set->mask == set->used) before this loop. 1103b094a36fSSebastian Sanchez */ 1104b094a36fSSebastian Sanchez cpumask_andnot(diff, hw_thread_mask, &set->used); 1105b094a36fSSebastian Sanchez if (!cpumask_empty(diff)) 1106b094a36fSSebastian Sanchez break; 1107b094a36fSSebastian Sanchez } 1108b094a36fSSebastian Sanchez } 1109b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl", 1110b094a36fSSebastian Sanchez cpumask_pr_args(hw_thread_mask)); 1111b094a36fSSebastian Sanchez 1112f48ad614SDennis Dalessandro node_mask = cpumask_of_node(node); 1113b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node, 1114f242d93aSLeon Romanovsky cpumask_pr_args(node_mask)); 1115f48ad614SDennis Dalessandro 1116b094a36fSSebastian Sanchez /* Get cpumask of available CPUs on preferred NUMA */ 1117b094a36fSSebastian Sanchez cpumask_and(available_mask, hw_thread_mask, node_mask); 1118b094a36fSSebastian Sanchez cpumask_andnot(available_mask, available_mask, &set->used); 1119b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node, 1120b094a36fSSebastian Sanchez cpumask_pr_args(available_mask)); 1121f48ad614SDennis Dalessandro 1122b094a36fSSebastian Sanchez /* 1123b094a36fSSebastian Sanchez * At first, we don't want to place processes on the same 1124b094a36fSSebastian Sanchez * CPUs as interrupt handlers. Then, CPUs running interrupt 1125b094a36fSSebastian Sanchez * handlers are used. 1126b094a36fSSebastian Sanchez * 1127b094a36fSSebastian Sanchez * 1) If diff is not empty, then there are CPUs not running 1128b094a36fSSebastian Sanchez * non-interrupt handlers available, so diff gets copied 1129b094a36fSSebastian Sanchez * over to available_mask. 1130b094a36fSSebastian Sanchez * 2) If diff is empty, then all CPUs not running interrupt 1131b094a36fSSebastian Sanchez * handlers are taken, so available_mask contains all 1132b094a36fSSebastian Sanchez * available CPUs running interrupt handlers. 1133b094a36fSSebastian Sanchez * 3) If available_mask is empty, then all CPUs on the 1134b094a36fSSebastian Sanchez * preferred NUMA node are taken, so other NUMA nodes are 1135b094a36fSSebastian Sanchez * used for process assignments using the same method as 1136b094a36fSSebastian Sanchez * the preferred NUMA node. 1137b094a36fSSebastian Sanchez */ 1138b094a36fSSebastian Sanchez cpumask_andnot(diff, available_mask, intrs_mask); 1139b094a36fSSebastian Sanchez if (!cpumask_empty(diff)) 1140b094a36fSSebastian Sanchez cpumask_copy(available_mask, diff); 1141b094a36fSSebastian Sanchez 1142b094a36fSSebastian Sanchez /* If we don't have CPUs on the preferred node, use other NUMA nodes */ 1143b094a36fSSebastian Sanchez if (cpumask_empty(available_mask)) { 1144b094a36fSSebastian Sanchez cpumask_andnot(available_mask, hw_thread_mask, &set->used); 1145b094a36fSSebastian Sanchez /* Excluding preferred NUMA cores */ 1146b094a36fSSebastian Sanchez cpumask_andnot(available_mask, available_mask, node_mask); 1147b094a36fSSebastian Sanchez hfi1_cdbg(PROC, 1148b094a36fSSebastian Sanchez "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl", 1149b094a36fSSebastian Sanchez cpumask_pr_args(available_mask)); 1150f48ad614SDennis Dalessandro 1151f48ad614SDennis Dalessandro /* 1152f48ad614SDennis Dalessandro * At first, we don't want to place processes on the same 1153f48ad614SDennis Dalessandro * CPUs as interrupt handlers. 1154f48ad614SDennis Dalessandro */ 1155b094a36fSSebastian Sanchez cpumask_andnot(diff, available_mask, intrs_mask); 1156f48ad614SDennis Dalessandro if (!cpumask_empty(diff)) 1157b094a36fSSebastian Sanchez cpumask_copy(available_mask, diff); 1158f48ad614SDennis Dalessandro } 1159b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl", 1160b094a36fSSebastian Sanchez cpumask_pr_args(available_mask)); 1161f48ad614SDennis Dalessandro 1162b094a36fSSebastian Sanchez cpu = cpumask_first(available_mask); 1163f48ad614SDennis Dalessandro if (cpu >= nr_cpu_ids) /* empty */ 1164f48ad614SDennis Dalessandro cpu = -1; 1165f48ad614SDennis Dalessandro else 1166f48ad614SDennis Dalessandro cpumask_set_cpu(cpu, &set->used); 1167584d9577STadeusz Struk 1168584d9577STadeusz Struk mutex_unlock(&affinity->lock); 1169b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu); 1170f48ad614SDennis Dalessandro 1171b094a36fSSebastian Sanchez free_cpumask_var(intrs_mask); 1172b094a36fSSebastian Sanchez free_available_mask: 1173b094a36fSSebastian Sanchez free_cpumask_var(available_mask); 1174b094a36fSSebastian Sanchez free_hw_thread_mask: 1175b094a36fSSebastian Sanchez free_cpumask_var(hw_thread_mask); 1176f48ad614SDennis Dalessandro free_diff: 1177f48ad614SDennis Dalessandro free_cpumask_var(diff); 1178f48ad614SDennis Dalessandro done: 1179f48ad614SDennis Dalessandro return cpu; 1180f48ad614SDennis Dalessandro } 1181f48ad614SDennis Dalessandro 1182b094a36fSSebastian Sanchez void hfi1_put_proc_affinity(int cpu) 1183f48ad614SDennis Dalessandro { 1184b094a36fSSebastian Sanchez struct hfi1_affinity_node_list *affinity = &node_affinity; 1185b094a36fSSebastian Sanchez struct cpu_mask_set *set = &affinity->proc; 1186f48ad614SDennis Dalessandro 1187f48ad614SDennis Dalessandro if (cpu < 0) 1188f48ad614SDennis Dalessandro return; 1189584d9577STadeusz Struk 1190584d9577STadeusz Struk mutex_lock(&affinity->lock); 1191cf38ea10SSebastian Sanchez cpu_mask_set_put(set, cpu); 1192b094a36fSSebastian Sanchez hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu); 1193584d9577STadeusz Struk mutex_unlock(&affinity->lock); 1194f48ad614SDennis Dalessandro } 1195