1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <uapi/linux/idxd.h> 10 #include "../dmaengine.h" 11 #include "idxd.h" 12 #include "registers.h" 13 14 enum irq_work_type { 15 IRQ_WORK_NORMAL = 0, 16 IRQ_WORK_PROCESS_FAULT, 17 }; 18 19 struct idxd_fault { 20 struct work_struct work; 21 u64 addr; 22 struct idxd_device *idxd; 23 }; 24 25 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 26 int *processed, u64 data); 27 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 28 int *processed, u64 data); 29 30 static void idxd_device_reinit(struct work_struct *work) 31 { 32 struct idxd_device *idxd = container_of(work, struct idxd_device, work); 33 struct device *dev = &idxd->pdev->dev; 34 int rc, i; 35 36 idxd_device_reset(idxd); 37 rc = idxd_device_config(idxd); 38 if (rc < 0) 39 goto out; 40 41 rc = idxd_device_enable(idxd); 42 if (rc < 0) 43 goto out; 44 45 for (i = 0; i < idxd->max_wqs; i++) { 46 struct idxd_wq *wq = idxd->wqs[i]; 47 48 if (wq->state == IDXD_WQ_ENABLED) { 49 rc = idxd_wq_enable(wq); 50 if (rc < 0) { 51 dev_warn(dev, "Unable to re-enable wq %s\n", 52 dev_name(wq_confdev(wq))); 53 } 54 } 55 } 56 57 return; 58 59 out: 60 idxd_device_clear_state(idxd); 61 } 62 63 static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) 64 { 65 struct device *dev = &idxd->pdev->dev; 66 union gensts_reg gensts; 67 u32 val = 0; 68 int i; 69 bool err = false; 70 71 if (cause & IDXD_INTC_ERR) { 72 spin_lock_bh(&idxd->dev_lock); 73 for (i = 0; i < 4; i++) 74 idxd->sw_err.bits[i] = ioread64(idxd->reg_base + 75 IDXD_SWERR_OFFSET + i * sizeof(u64)); 76 77 iowrite64(idxd->sw_err.bits[0] & IDXD_SWERR_ACK, 78 idxd->reg_base + IDXD_SWERR_OFFSET); 79 80 if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { 81 int id = idxd->sw_err.wq_idx; 82 struct idxd_wq *wq = idxd->wqs[id]; 83 84 if (wq->type == IDXD_WQT_USER) 85 wake_up_interruptible(&wq->err_queue); 86 } else { 87 int i; 88 89 for (i = 0; i < idxd->max_wqs; i++) { 90 struct idxd_wq *wq = idxd->wqs[i]; 91 92 if (wq->type == IDXD_WQT_USER) 93 wake_up_interruptible(&wq->err_queue); 94 } 95 } 96 97 spin_unlock_bh(&idxd->dev_lock); 98 val |= IDXD_INTC_ERR; 99 100 for (i = 0; i < 4; i++) 101 dev_warn(dev, "err[%d]: %#16.16llx\n", 102 i, idxd->sw_err.bits[i]); 103 err = true; 104 } 105 106 if (cause & IDXD_INTC_CMD) { 107 val |= IDXD_INTC_CMD; 108 complete(idxd->cmd_done); 109 } 110 111 if (cause & IDXD_INTC_OCCUPY) { 112 /* Driver does not utilize occupancy interrupt */ 113 val |= IDXD_INTC_OCCUPY; 114 } 115 116 if (cause & IDXD_INTC_PERFMON_OVFL) { 117 val |= IDXD_INTC_PERFMON_OVFL; 118 perfmon_counter_overflow(idxd); 119 } 120 121 val ^= cause; 122 if (val) 123 dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", 124 val); 125 126 if (!err) 127 return 0; 128 129 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 130 if (gensts.state == IDXD_DEVICE_STATE_HALT) { 131 idxd->state = IDXD_DEV_HALTED; 132 if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { 133 /* 134 * If we need a software reset, we will throw the work 135 * on a system workqueue in order to allow interrupts 136 * for the device command completions. 137 */ 138 INIT_WORK(&idxd->work, idxd_device_reinit); 139 queue_work(idxd->wq, &idxd->work); 140 } else { 141 spin_lock_bh(&idxd->dev_lock); 142 idxd_wqs_quiesce(idxd); 143 idxd_wqs_unmap_portal(idxd); 144 idxd_device_clear_state(idxd); 145 dev_err(&idxd->pdev->dev, 146 "idxd halted, need %s.\n", 147 gensts.reset_type == IDXD_DEVICE_RESET_FLR ? 148 "FLR" : "system reset"); 149 spin_unlock_bh(&idxd->dev_lock); 150 return -ENXIO; 151 } 152 } 153 154 return 0; 155 } 156 157 irqreturn_t idxd_misc_thread(int vec, void *data) 158 { 159 struct idxd_irq_entry *irq_entry = data; 160 struct idxd_device *idxd = irq_entry->idxd; 161 int rc; 162 u32 cause; 163 164 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); 165 if (cause) 166 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); 167 168 while (cause) { 169 rc = process_misc_interrupts(idxd, cause); 170 if (rc < 0) 171 break; 172 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); 173 if (cause) 174 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); 175 } 176 177 return IRQ_HANDLED; 178 } 179 180 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 181 int *processed, u64 data) 182 { 183 struct idxd_desc *desc, *t; 184 struct llist_node *head; 185 int queued = 0; 186 unsigned long flags; 187 188 *processed = 0; 189 head = llist_del_all(&irq_entry->pending_llist); 190 if (!head) 191 goto out; 192 193 llist_for_each_entry_safe(desc, t, head, llnode) { 194 u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; 195 196 if (status) { 197 /* 198 * Check against the original status as ABORT is software defined 199 * and 0xff, which DSA_COMP_STATUS_MASK can mask out. 200 */ 201 if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { 202 complete_desc(desc, IDXD_COMPLETE_ABORT); 203 (*processed)++; 204 continue; 205 } 206 207 complete_desc(desc, IDXD_COMPLETE_NORMAL); 208 (*processed)++; 209 } else { 210 spin_lock_irqsave(&irq_entry->list_lock, flags); 211 list_add_tail(&desc->list, 212 &irq_entry->work_list); 213 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 214 queued++; 215 } 216 } 217 218 out: 219 return queued; 220 } 221 222 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 223 int *processed, u64 data) 224 { 225 int queued = 0; 226 unsigned long flags; 227 LIST_HEAD(flist); 228 struct idxd_desc *desc, *n; 229 230 *processed = 0; 231 232 /* 233 * This lock protects list corruption from access of list outside of the irq handler 234 * thread. 235 */ 236 spin_lock_irqsave(&irq_entry->list_lock, flags); 237 if (list_empty(&irq_entry->work_list)) { 238 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 239 return 0; 240 } 241 242 list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) { 243 if (desc->completion->status) { 244 list_del(&desc->list); 245 (*processed)++; 246 list_add_tail(&desc->list, &flist); 247 } else { 248 queued++; 249 } 250 } 251 252 spin_unlock_irqrestore(&irq_entry->list_lock, flags); 253 254 list_for_each_entry(desc, &flist, list) { 255 /* 256 * Check against the original status as ABORT is software defined 257 * and 0xff, which DSA_COMP_STATUS_MASK can mask out. 258 */ 259 if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { 260 complete_desc(desc, IDXD_COMPLETE_ABORT); 261 continue; 262 } 263 264 complete_desc(desc, IDXD_COMPLETE_NORMAL); 265 } 266 267 return queued; 268 } 269 270 static int idxd_desc_process(struct idxd_irq_entry *irq_entry) 271 { 272 int rc, processed, total = 0; 273 274 /* 275 * There are two lists we are processing. The pending_llist is where 276 * submmiter adds all the submitted descriptor after sending it to 277 * the workqueue. It's a lockless singly linked list. The work_list 278 * is the common linux double linked list. We are in a scenario of 279 * multiple producers and a single consumer. The producers are all 280 * the kernel submitters of descriptors, and the consumer is the 281 * kernel irq handler thread for the msix vector when using threaded 282 * irq. To work with the restrictions of llist to remain lockless, 283 * we are doing the following steps: 284 * 1. Iterate through the work_list and process any completed 285 * descriptor. Delete the completed entries during iteration. 286 * 2. llist_del_all() from the pending list. 287 * 3. Iterate through the llist that was deleted from the pending list 288 * and process the completed entries. 289 * 4. If the entry is still waiting on hardware, list_add_tail() to 290 * the work_list. 291 * 5. Repeat until no more descriptors. 292 */ 293 do { 294 rc = irq_process_work_list(irq_entry, &processed, 0); 295 total += processed; 296 if (rc != 0) 297 continue; 298 299 rc = irq_process_pending_llist(irq_entry, &processed, 0); 300 total += processed; 301 } while (rc != 0); 302 303 return total; 304 } 305 306 irqreturn_t idxd_wq_thread(int irq, void *data) 307 { 308 struct idxd_irq_entry *irq_entry = data; 309 int processed; 310 311 processed = idxd_desc_process(irq_entry); 312 if (processed == 0) 313 return IRQ_NONE; 314 315 return IRQ_HANDLED; 316 } 317