1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <uapi/linux/idxd.h> 10 #include "../dmaengine.h" 11 #include "idxd.h" 12 #include "registers.h" 13 14 enum irq_work_type { 15 IRQ_WORK_NORMAL = 0, 16 IRQ_WORK_PROCESS_FAULT, 17 }; 18 19 struct idxd_fault { 20 struct work_struct work; 21 u64 addr; 22 struct idxd_device *idxd; 23 }; 24 25 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 26 enum irq_work_type wtype, 27 int *processed, u64 data); 28 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 29 enum irq_work_type wtype, 30 int *processed, u64 data); 31 32 static void idxd_device_reinit(struct work_struct *work) 33 { 34 struct idxd_device *idxd = container_of(work, struct idxd_device, work); 35 struct device *dev = &idxd->pdev->dev; 36 int rc, i; 37 38 idxd_device_reset(idxd); 39 rc = idxd_device_config(idxd); 40 if (rc < 0) 41 goto out; 42 43 rc = idxd_device_enable(idxd); 44 if (rc < 0) 45 goto out; 46 47 for (i = 0; i < idxd->max_wqs; i++) { 48 struct idxd_wq *wq = &idxd->wqs[i]; 49 50 if (wq->state == IDXD_WQ_ENABLED) { 51 rc = idxd_wq_enable(wq); 52 if (rc < 0) { 53 dev_warn(dev, "Unable to re-enable wq %s\n", 54 dev_name(&wq->conf_dev)); 55 } 56 } 57 } 58 59 return; 60 61 out: 62 idxd_device_wqs_clear_state(idxd); 63 } 64 65 static void idxd_device_fault_work(struct work_struct *work) 66 { 67 struct idxd_fault *fault = container_of(work, struct idxd_fault, work); 68 struct idxd_irq_entry *ie; 69 int i; 70 int processed; 71 int irqcnt = fault->idxd->num_wq_irqs + 1; 72 73 for (i = 1; i < irqcnt; i++) { 74 ie = &fault->idxd->irq_entries[i]; 75 irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT, 76 &processed, fault->addr); 77 if (processed) 78 break; 79 80 irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT, 81 &processed, fault->addr); 82 if (processed) 83 break; 84 } 85 86 kfree(fault); 87 } 88 89 static int idxd_device_schedule_fault_process(struct idxd_device *idxd, 90 u64 fault_addr) 91 { 92 struct idxd_fault *fault; 93 94 fault = kmalloc(sizeof(*fault), GFP_ATOMIC); 95 if (!fault) 96 return -ENOMEM; 97 98 fault->addr = fault_addr; 99 fault->idxd = idxd; 100 INIT_WORK(&fault->work, idxd_device_fault_work); 101 queue_work(idxd->wq, &fault->work); 102 return 0; 103 } 104 105 irqreturn_t idxd_irq_handler(int vec, void *data) 106 { 107 struct idxd_irq_entry *irq_entry = data; 108 struct idxd_device *idxd = irq_entry->idxd; 109 110 idxd_mask_msix_vector(idxd, irq_entry->id); 111 return IRQ_WAKE_THREAD; 112 } 113 114 irqreturn_t idxd_misc_thread(int vec, void *data) 115 { 116 struct idxd_irq_entry *irq_entry = data; 117 struct idxd_device *idxd = irq_entry->idxd; 118 struct device *dev = &idxd->pdev->dev; 119 union gensts_reg gensts; 120 u32 cause, val = 0; 121 int i; 122 bool err = false; 123 124 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); 125 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); 126 127 if (cause & IDXD_INTC_ERR) { 128 spin_lock_bh(&idxd->dev_lock); 129 for (i = 0; i < 4; i++) 130 idxd->sw_err.bits[i] = ioread64(idxd->reg_base + 131 IDXD_SWERR_OFFSET + i * sizeof(u64)); 132 iowrite64(IDXD_SWERR_ACK, idxd->reg_base + IDXD_SWERR_OFFSET); 133 134 if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { 135 int id = idxd->sw_err.wq_idx; 136 struct idxd_wq *wq = &idxd->wqs[id]; 137 138 if (wq->type == IDXD_WQT_USER) 139 wake_up_interruptible(&wq->idxd_cdev.err_queue); 140 } else { 141 int i; 142 143 for (i = 0; i < idxd->max_wqs; i++) { 144 struct idxd_wq *wq = &idxd->wqs[i]; 145 146 if (wq->type == IDXD_WQT_USER) 147 wake_up_interruptible(&wq->idxd_cdev.err_queue); 148 } 149 } 150 151 spin_unlock_bh(&idxd->dev_lock); 152 val |= IDXD_INTC_ERR; 153 154 for (i = 0; i < 4; i++) 155 dev_warn(dev, "err[%d]: %#16.16llx\n", 156 i, idxd->sw_err.bits[i]); 157 err = true; 158 } 159 160 if (cause & IDXD_INTC_CMD) { 161 val |= IDXD_INTC_CMD; 162 complete(idxd->cmd_done); 163 } 164 165 if (cause & IDXD_INTC_OCCUPY) { 166 /* Driver does not utilize occupancy interrupt */ 167 val |= IDXD_INTC_OCCUPY; 168 } 169 170 if (cause & IDXD_INTC_PERFMON_OVFL) { 171 /* 172 * Driver does not utilize perfmon counter overflow interrupt 173 * yet. 174 */ 175 val |= IDXD_INTC_PERFMON_OVFL; 176 } 177 178 val ^= cause; 179 if (val) 180 dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", 181 val); 182 183 if (!err) 184 goto out; 185 186 /* 187 * This case should rarely happen and typically is due to software 188 * programming error by the driver. 189 */ 190 if (idxd->sw_err.valid && 191 idxd->sw_err.desc_valid && 192 idxd->sw_err.fault_addr) 193 idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr); 194 195 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 196 if (gensts.state == IDXD_DEVICE_STATE_HALT) { 197 idxd->state = IDXD_DEV_HALTED; 198 if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { 199 /* 200 * If we need a software reset, we will throw the work 201 * on a system workqueue in order to allow interrupts 202 * for the device command completions. 203 */ 204 INIT_WORK(&idxd->work, idxd_device_reinit); 205 queue_work(idxd->wq, &idxd->work); 206 } else { 207 spin_lock_bh(&idxd->dev_lock); 208 idxd_device_wqs_clear_state(idxd); 209 dev_err(&idxd->pdev->dev, 210 "idxd halted, need %s.\n", 211 gensts.reset_type == IDXD_DEVICE_RESET_FLR ? 212 "FLR" : "system reset"); 213 spin_unlock_bh(&idxd->dev_lock); 214 } 215 } 216 217 out: 218 idxd_unmask_msix_vector(idxd, irq_entry->id); 219 return IRQ_HANDLED; 220 } 221 222 static bool process_fault(struct idxd_desc *desc, u64 fault_addr) 223 { 224 /* 225 * Completion address can be bad as well. Check fault address match for descriptor 226 * and completion address. 227 */ 228 if ((u64)desc->hw == fault_addr || 229 (u64)desc->completion == fault_addr) { 230 idxd_dma_complete_txd(desc, IDXD_COMPLETE_DEV_FAIL); 231 return true; 232 } 233 234 return false; 235 } 236 237 static bool complete_desc(struct idxd_desc *desc) 238 { 239 if (desc->completion->status) { 240 idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); 241 return true; 242 } 243 244 return false; 245 } 246 247 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 248 enum irq_work_type wtype, 249 int *processed, u64 data) 250 { 251 struct idxd_desc *desc, *t; 252 struct llist_node *head; 253 int queued = 0; 254 bool completed = false; 255 unsigned long flags; 256 257 *processed = 0; 258 head = llist_del_all(&irq_entry->pending_llist); 259 if (!head) 260 goto out; 261 262 llist_for_each_entry_safe(desc, t, head, llnode) { 263 if (wtype == IRQ_WORK_NORMAL) 264 completed = complete_desc(desc); 265 else if (wtype == IRQ_WORK_PROCESS_FAULT) 266 completed = process_fault(desc, data); 267 268 if (completed) { 269 idxd_free_desc(desc->wq, desc); 270 (*processed)++; 271 if (wtype == IRQ_WORK_PROCESS_FAULT) 272 break; 273 } else { 274 spin_lock_irqsave(&irq_entry->list_lock, flags); 275 list_add_tail(&desc->list, 276 &irq_entry->work_list); 277 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 278 queued++; 279 } 280 } 281 282 out: 283 return queued; 284 } 285 286 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 287 enum irq_work_type wtype, 288 int *processed, u64 data) 289 { 290 struct list_head *node, *next; 291 int queued = 0; 292 bool completed = false; 293 unsigned long flags; 294 295 *processed = 0; 296 spin_lock_irqsave(&irq_entry->list_lock, flags); 297 if (list_empty(&irq_entry->work_list)) 298 goto out; 299 300 list_for_each_safe(node, next, &irq_entry->work_list) { 301 struct idxd_desc *desc = 302 container_of(node, struct idxd_desc, list); 303 304 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 305 if (wtype == IRQ_WORK_NORMAL) 306 completed = complete_desc(desc); 307 else if (wtype == IRQ_WORK_PROCESS_FAULT) 308 completed = process_fault(desc, data); 309 310 if (completed) { 311 spin_lock_irqsave(&irq_entry->list_lock, flags); 312 list_del(&desc->list); 313 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 314 idxd_free_desc(desc->wq, desc); 315 (*processed)++; 316 if (wtype == IRQ_WORK_PROCESS_FAULT) 317 return queued; 318 } else { 319 queued++; 320 } 321 spin_lock_irqsave(&irq_entry->list_lock, flags); 322 } 323 324 out: 325 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 326 return queued; 327 } 328 329 static int idxd_desc_process(struct idxd_irq_entry *irq_entry) 330 { 331 int rc, processed, total = 0; 332 333 /* 334 * There are two lists we are processing. The pending_llist is where 335 * submmiter adds all the submitted descriptor after sending it to 336 * the workqueue. It's a lockless singly linked list. The work_list 337 * is the common linux double linked list. We are in a scenario of 338 * multiple producers and a single consumer. The producers are all 339 * the kernel submitters of descriptors, and the consumer is the 340 * kernel irq handler thread for the msix vector when using threaded 341 * irq. To work with the restrictions of llist to remain lockless, 342 * we are doing the following steps: 343 * 1. Iterate through the work_list and process any completed 344 * descriptor. Delete the completed entries during iteration. 345 * 2. llist_del_all() from the pending list. 346 * 3. Iterate through the llist that was deleted from the pending list 347 * and process the completed entries. 348 * 4. If the entry is still waiting on hardware, list_add_tail() to 349 * the work_list. 350 * 5. Repeat until no more descriptors. 351 */ 352 do { 353 rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL, 354 &processed, 0); 355 total += processed; 356 if (rc != 0) 357 continue; 358 359 rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL, 360 &processed, 0); 361 total += processed; 362 } while (rc != 0); 363 364 return total; 365 } 366 367 irqreturn_t idxd_wq_thread(int irq, void *data) 368 { 369 struct idxd_irq_entry *irq_entry = data; 370 int processed; 371 372 processed = idxd_desc_process(irq_entry); 373 idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); 374 375 if (processed == 0) 376 return IRQ_NONE; 377 378 return IRQ_HANDLED; 379 } 380