1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 - 2020 Intel Corporation. 4 */ 5 6 #include "hfi.h" 7 #include "affinity.h" 8 #include "sdma.h" 9 #include "netdev.h" 10 11 /** 12 * msix_initialize() - Calculate, request and configure MSIx IRQs 13 * @dd: valid hfi1 devdata 14 * 15 */ 16 int msix_initialize(struct hfi1_devdata *dd) 17 { 18 u32 total; 19 int ret; 20 struct hfi1_msix_entry *entries; 21 22 /* 23 * MSIx interrupt count: 24 * one for the general, "slow path" interrupt 25 * one per used SDMA engine 26 * one per kernel receive context 27 * one for each VNIC context 28 * ...any new IRQs should be added here. 29 */ 30 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts; 31 32 if (total >= CCE_NUM_MSIX_VECTORS) 33 return -EINVAL; 34 35 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX); 36 if (ret < 0) { 37 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret); 38 return ret; 39 } 40 41 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries), 42 GFP_KERNEL); 43 if (!entries) { 44 pci_free_irq_vectors(dd->pcidev); 45 return -ENOMEM; 46 } 47 48 dd->msix_info.msix_entries = entries; 49 spin_lock_init(&dd->msix_info.msix_lock); 50 bitmap_zero(dd->msix_info.in_use_msix, total); 51 dd->msix_info.max_requested = total; 52 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 53 54 return 0; 55 } 56 57 /** 58 * msix_request_irq() - Allocate a free MSIx IRQ 59 * @dd: valid devdata 60 * @arg: context information for the IRQ 61 * @handler: IRQ handler 62 * @thread: IRQ thread handler (could be NULL) 63 * @type: affinty IRQ type 64 * @name: IRQ name 65 * 66 * Allocated an MSIx vector if available, and then create the appropriate 67 * meta data needed to keep track of the pci IRQ request. 68 * 69 * Return: 70 * < 0 Error 71 * >= 0 MSIx vector 72 * 73 */ 74 static int msix_request_irq(struct hfi1_devdata *dd, void *arg, 75 irq_handler_t handler, irq_handler_t thread, 76 enum irq_type type, const char *name) 77 { 78 unsigned long nr; 79 int irq; 80 int ret; 81 struct hfi1_msix_entry *me; 82 83 /* Allocate an MSIx vector */ 84 spin_lock(&dd->msix_info.msix_lock); 85 nr = find_first_zero_bit(dd->msix_info.in_use_msix, 86 dd->msix_info.max_requested); 87 if (nr < dd->msix_info.max_requested) 88 __set_bit(nr, dd->msix_info.in_use_msix); 89 spin_unlock(&dd->msix_info.msix_lock); 90 91 if (nr == dd->msix_info.max_requested) 92 return -ENOSPC; 93 94 if (type < IRQ_SDMA || type >= IRQ_OTHER) 95 return -EINVAL; 96 97 irq = pci_irq_vector(dd->pcidev, nr); 98 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name); 99 if (ret) { 100 dd_dev_err(dd, 101 "%s: request for IRQ %d failed, MSIx %lx, err %d\n", 102 name, irq, nr, ret); 103 spin_lock(&dd->msix_info.msix_lock); 104 __clear_bit(nr, dd->msix_info.in_use_msix); 105 spin_unlock(&dd->msix_info.msix_lock); 106 return ret; 107 } 108 109 /* 110 * assign arg after pci_request_irq call, so it will be 111 * cleaned up 112 */ 113 me = &dd->msix_info.msix_entries[nr]; 114 me->irq = irq; 115 me->arg = arg; 116 me->type = type; 117 118 /* This is a request, so a failure is not fatal */ 119 ret = hfi1_get_irq_affinity(dd, me); 120 if (ret) 121 dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret); 122 123 return nr; 124 } 125 126 static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd, 127 irq_handler_t handler, 128 irq_handler_t thread, 129 const char *name) 130 { 131 int nr = msix_request_irq(rcd->dd, rcd, handler, thread, 132 rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT, 133 name); 134 if (nr < 0) 135 return nr; 136 137 /* 138 * Set the interrupt register and mask for this 139 * context's interrupt. 140 */ 141 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64; 142 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64); 143 rcd->msix_intr = nr; 144 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr); 145 146 return 0; 147 } 148 149 /** 150 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs 151 * @rcd: valid rcd context 152 * 153 */ 154 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) 155 { 156 char name[MAX_NAME_SIZE]; 157 158 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", 159 rcd->dd->unit, rcd->ctxt); 160 161 return msix_request_rcd_irq_common(rcd, receive_context_interrupt, 162 receive_context_thread, name); 163 } 164 165 /** 166 * msix_netdev_request_rcd_irq - Helper function for RCVAVAIL IRQs 167 * for netdev context 168 * @rcd: valid netdev contexti 169 */ 170 int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd) 171 { 172 char name[MAX_NAME_SIZE]; 173 174 snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d", 175 rcd->dd->unit, rcd->ctxt); 176 return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi, 177 NULL, name); 178 } 179 180 /** 181 * msix_request_sdma_irq - Helper for getting SDMA IRQ resources 182 * @sde: valid sdma engine 183 * 184 */ 185 int msix_request_sdma_irq(struct sdma_engine *sde) 186 { 187 int nr; 188 char name[MAX_NAME_SIZE]; 189 190 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", 191 sde->dd->unit, sde->this_idx); 192 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL, 193 IRQ_SDMA, name); 194 if (nr < 0) 195 return nr; 196 sde->msix_intr = nr; 197 remap_sdma_interrupts(sde->dd, sde->this_idx, nr); 198 199 return 0; 200 } 201 202 /** 203 * msix_request_general_irq - Helper for getting general IRQ 204 * resources 205 * @dd: valid device data 206 */ 207 int msix_request_general_irq(struct hfi1_devdata *dd) 208 { 209 int nr; 210 char name[MAX_NAME_SIZE]; 211 212 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); 213 nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL, 214 name); 215 if (nr < 0) 216 return nr; 217 218 /* general interrupt must be MSIx vector 0 */ 219 if (nr) { 220 msix_free_irq(dd, (u8)nr); 221 dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr); 222 return -EINVAL; 223 } 224 225 return 0; 226 } 227 228 /** 229 * enable_sdma_srcs - Helper to enable SDMA IRQ srcs 230 * @dd: valid devdata structure 231 * @i: index of SDMA engine 232 */ 233 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) 234 { 235 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true); 236 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i, 237 IS_SDMA_PROGRESS_START + i, true); 238 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true); 239 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i, 240 true); 241 } 242 243 /** 244 * msix_request_irqs() - Allocate all MSIx IRQs 245 * @dd: valid devdata structure 246 * 247 * Helper function to request the used MSIx IRQs. 248 * 249 */ 250 int msix_request_irqs(struct hfi1_devdata *dd) 251 { 252 int i; 253 int ret = msix_request_general_irq(dd); 254 255 if (ret) 256 return ret; 257 258 for (i = 0; i < dd->num_sdma; i++) { 259 struct sdma_engine *sde = &dd->per_sdma[i]; 260 261 ret = msix_request_sdma_irq(sde); 262 if (ret) 263 return ret; 264 enable_sdma_srcs(sde->dd, i); 265 } 266 267 for (i = 0; i < dd->n_krcv_queues; i++) { 268 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i); 269 270 if (rcd) 271 ret = msix_request_rcd_irq(rcd); 272 hfi1_rcd_put(rcd); 273 if (ret) 274 return ret; 275 } 276 277 return 0; 278 } 279 280 /** 281 * msix_free_irq() - Free the specified MSIx resources and IRQ 282 * @dd: valid devdata 283 * @msix_intr: MSIx vector to free. 284 * 285 */ 286 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) 287 { 288 struct hfi1_msix_entry *me; 289 290 if (msix_intr >= dd->msix_info.max_requested) 291 return; 292 293 me = &dd->msix_info.msix_entries[msix_intr]; 294 295 if (!me->arg) /* => no irq, no affinity */ 296 return; 297 298 hfi1_put_irq_affinity(dd, me); 299 pci_free_irq(dd->pcidev, msix_intr, me->arg); 300 301 me->arg = NULL; 302 303 spin_lock(&dd->msix_info.msix_lock); 304 __clear_bit(msix_intr, dd->msix_info.in_use_msix); 305 spin_unlock(&dd->msix_info.msix_lock); 306 } 307 308 /** 309 * msix_clean_up_interrupts - Free all MSIx IRQ resources 310 * @dd: valid device data data structure 311 * 312 * Free the MSIx and associated PCI resources, if they have been allocated. 313 */ 314 void msix_clean_up_interrupts(struct hfi1_devdata *dd) 315 { 316 int i; 317 struct hfi1_msix_entry *me = dd->msix_info.msix_entries; 318 319 /* remove irqs - must happen before disabling/turning off */ 320 for (i = 0; i < dd->msix_info.max_requested; i++, me++) 321 msix_free_irq(dd, i); 322 323 /* clean structures */ 324 kfree(dd->msix_info.msix_entries); 325 dd->msix_info.msix_entries = NULL; 326 dd->msix_info.max_requested = 0; 327 328 pci_free_irq_vectors(dd->pcidev); 329 } 330 331 /** 332 * msix_netdev_synchronize_irq - netdev IRQ synchronize 333 * @dd: valid devdata 334 */ 335 void msix_netdev_synchronize_irq(struct hfi1_devdata *dd) 336 { 337 int i; 338 int ctxt_count = hfi1_netdev_ctxt_count(dd); 339 340 for (i = 0; i < ctxt_count; i++) { 341 struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i); 342 struct hfi1_msix_entry *me; 343 344 me = &dd->msix_info.msix_entries[rcd->msix_intr]; 345 346 synchronize_irq(me->irq); 347 } 348 } 349