1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 * This file is provided under a dual BSD/GPLv2 license. When using or 6 * redistributing this file, you may do so under either license. 7 * 8 * GPL LICENSE SUMMARY 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * BSD LICENSE 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 25 * - Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * - Redistributions in binary form must reproduce the above copyright 28 * notice, this list of conditions and the following disclaimer in 29 * the documentation and/or other materials provided with the 30 * distribution. 31 * - Neither the name of Intel Corporation nor the names of its 32 * contributors may be used to endorse or promote products derived 33 * from this software without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 */ 48 49 #include "hfi.h" 50 #include "affinity.h" 51 #include "sdma.h" 52 53 /** 54 * msix_initialize() - Calculate, request and configure MSIx IRQs 55 * @dd: valid hfi1 devdata 56 * 57 */ 58 int msix_initialize(struct hfi1_devdata *dd) 59 { 60 u32 total; 61 int ret; 62 struct hfi1_msix_entry *entries; 63 64 /* 65 * MSIx interrupt count: 66 * one for the general, "slow path" interrupt 67 * one per used SDMA engine 68 * one per kernel receive context 69 * one for each VNIC context 70 * ...any new IRQs should be added here. 71 */ 72 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_vnic_contexts; 73 74 if (total >= CCE_NUM_MSIX_VECTORS) 75 return -EINVAL; 76 77 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX); 78 if (ret < 0) { 79 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret); 80 return ret; 81 } 82 83 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries), 84 GFP_KERNEL); 85 if (!entries) { 86 pci_free_irq_vectors(dd->pcidev); 87 return -ENOMEM; 88 } 89 90 dd->msix_info.msix_entries = entries; 91 spin_lock_init(&dd->msix_info.msix_lock); 92 bitmap_zero(dd->msix_info.in_use_msix, total); 93 dd->msix_info.max_requested = total; 94 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 95 96 return 0; 97 } 98 99 /** 100 * msix_request_irq() - Allocate a free MSIx IRQ 101 * @dd: valid devdata 102 * @arg: context information for the IRQ 103 * @handler: IRQ handler 104 * @thread: IRQ thread handler (could be NULL) 105 * @idx: zero base idx if multiple devices are needed 106 * @type: affinty IRQ type 107 * 108 * Allocated an MSIx vector if available, and then create the appropriate 109 * meta data needed to keep track of the pci IRQ request. 110 * 111 * Return: 112 * < 0 Error 113 * >= 0 MSIx vector 114 * 115 */ 116 static int msix_request_irq(struct hfi1_devdata *dd, void *arg, 117 irq_handler_t handler, irq_handler_t thread, 118 u32 idx, enum irq_type type) 119 { 120 unsigned long nr; 121 int irq; 122 int ret; 123 const char *err_info; 124 char name[MAX_NAME_SIZE]; 125 struct hfi1_msix_entry *me; 126 127 /* Allocate an MSIx vector */ 128 spin_lock(&dd->msix_info.msix_lock); 129 nr = find_first_zero_bit(dd->msix_info.in_use_msix, 130 dd->msix_info.max_requested); 131 if (nr < dd->msix_info.max_requested) 132 __set_bit(nr, dd->msix_info.in_use_msix); 133 spin_unlock(&dd->msix_info.msix_lock); 134 135 if (nr == dd->msix_info.max_requested) 136 return -ENOSPC; 137 138 /* Specific verification and determine the name */ 139 switch (type) { 140 case IRQ_GENERAL: 141 /* general interrupt must be MSIx vector 0 */ 142 if (nr) { 143 spin_lock(&dd->msix_info.msix_lock); 144 __clear_bit(nr, dd->msix_info.in_use_msix); 145 spin_unlock(&dd->msix_info.msix_lock); 146 dd_dev_err(dd, "Invalid index %lu for GENERAL IRQ\n", 147 nr); 148 return -EINVAL; 149 } 150 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); 151 err_info = "general"; 152 break; 153 case IRQ_SDMA: 154 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", 155 dd->unit, idx); 156 err_info = "sdma"; 157 break; 158 case IRQ_RCVCTXT: 159 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", 160 dd->unit, idx); 161 err_info = "receive context"; 162 break; 163 case IRQ_OTHER: 164 default: 165 return -EINVAL; 166 } 167 name[sizeof(name) - 1] = 0; 168 169 irq = pci_irq_vector(dd->pcidev, nr); 170 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name); 171 if (ret) { 172 dd_dev_err(dd, 173 "%s: request for IRQ %d failed, MSIx %d, err %d\n", 174 err_info, irq, idx, ret); 175 spin_lock(&dd->msix_info.msix_lock); 176 __clear_bit(nr, dd->msix_info.in_use_msix); 177 spin_unlock(&dd->msix_info.msix_lock); 178 return ret; 179 } 180 181 /* 182 * assign arg after pci_request_irq call, so it will be 183 * cleaned up 184 */ 185 me = &dd->msix_info.msix_entries[nr]; 186 me->irq = irq; 187 me->arg = arg; 188 me->type = type; 189 190 /* This is a request, so a failure is not fatal */ 191 ret = hfi1_get_irq_affinity(dd, me); 192 if (ret) 193 dd_dev_err(dd, "unable to pin IRQ %d\n", ret); 194 195 return nr; 196 } 197 198 /** 199 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs 200 * @rcd: valid rcd context 201 * 202 */ 203 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) 204 { 205 int nr; 206 207 nr = msix_request_irq(rcd->dd, rcd, receive_context_interrupt, 208 receive_context_thread, rcd->ctxt, IRQ_RCVCTXT); 209 if (nr < 0) 210 return nr; 211 212 /* 213 * Set the interrupt register and mask for this 214 * context's interrupt. 215 */ 216 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64; 217 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64); 218 rcd->msix_intr = nr; 219 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr); 220 221 return 0; 222 } 223 224 /** 225 * msix_request_smda_ira() - Helper for getting SDMA IRQ resources 226 * @sde: valid sdma engine 227 * 228 */ 229 int msix_request_sdma_irq(struct sdma_engine *sde) 230 { 231 int nr; 232 233 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL, 234 sde->this_idx, IRQ_SDMA); 235 if (nr < 0) 236 return nr; 237 sde->msix_intr = nr; 238 remap_sdma_interrupts(sde->dd, sde->this_idx, nr); 239 240 return 0; 241 } 242 243 /** 244 * enable_sdma_src() - Helper to enable SDMA IRQ srcs 245 * @dd: valid devdata structure 246 * @i: index of SDMA engine 247 */ 248 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) 249 { 250 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true); 251 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i, 252 IS_SDMA_PROGRESS_START + i, true); 253 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true); 254 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i, 255 true); 256 } 257 258 /** 259 * msix_request_irqs() - Allocate all MSIx IRQs 260 * @dd: valid devdata structure 261 * 262 * Helper function to request the used MSIx IRQs. 263 * 264 */ 265 int msix_request_irqs(struct hfi1_devdata *dd) 266 { 267 int i; 268 int ret; 269 270 ret = msix_request_irq(dd, dd, general_interrupt, NULL, 0, IRQ_GENERAL); 271 if (ret < 0) 272 return ret; 273 274 for (i = 0; i < dd->num_sdma; i++) { 275 struct sdma_engine *sde = &dd->per_sdma[i]; 276 277 ret = msix_request_sdma_irq(sde); 278 if (ret) 279 return ret; 280 enable_sdma_srcs(sde->dd, i); 281 } 282 283 for (i = 0; i < dd->n_krcv_queues; i++) { 284 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i); 285 286 if (rcd) 287 ret = msix_request_rcd_irq(rcd); 288 hfi1_rcd_put(rcd); 289 if (ret) 290 return ret; 291 } 292 293 return 0; 294 } 295 296 /** 297 * msix_free_irq() - Free the specified MSIx resources and IRQ 298 * @dd: valid devdata 299 * @msix_intr: MSIx vector to free. 300 * 301 */ 302 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) 303 { 304 struct hfi1_msix_entry *me; 305 306 if (msix_intr >= dd->msix_info.max_requested) 307 return; 308 309 me = &dd->msix_info.msix_entries[msix_intr]; 310 311 if (!me->arg) /* => no irq, no affinity */ 312 return; 313 314 hfi1_put_irq_affinity(dd, me); 315 pci_free_irq(dd->pcidev, msix_intr, me->arg); 316 317 me->arg = NULL; 318 319 spin_lock(&dd->msix_info.msix_lock); 320 __clear_bit(msix_intr, dd->msix_info.in_use_msix); 321 spin_unlock(&dd->msix_info.msix_lock); 322 } 323 324 /** 325 * hfi1_clean_up_msix_interrupts() - Free all MSIx IRQ resources 326 * @dd: valid device data data structure 327 * 328 * Free the MSIx and associated PCI resources, if they have been allocated. 329 */ 330 void msix_clean_up_interrupts(struct hfi1_devdata *dd) 331 { 332 int i; 333 struct hfi1_msix_entry *me = dd->msix_info.msix_entries; 334 335 /* remove irqs - must happen before disabling/turning off */ 336 for (i = 0; i < dd->msix_info.max_requested; i++, me++) 337 msix_free_irq(dd, i); 338 339 /* clean structures */ 340 kfree(dd->msix_info.msix_entries); 341 dd->msix_info.msix_entries = NULL; 342 dd->msix_info.max_requested = 0; 343 344 pci_free_irq_vectors(dd->pcidev); 345 } 346 347 /** 348 * msix_vnic_syncrhonize_irq() - Vnic IRQ synchronize 349 * @dd: valid devdata 350 */ 351 void msix_vnic_synchronize_irq(struct hfi1_devdata *dd) 352 { 353 int i; 354 355 for (i = 0; i < dd->vnic.num_ctxt; i++) { 356 struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i]; 357 struct hfi1_msix_entry *me; 358 359 me = &dd->msix_info.msix_entries[rcd->msix_intr]; 360 361 synchronize_irq(me->irq); 362 } 363 } 364