1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <uapi/linux/idxd.h> 10 #include "../dmaengine.h" 11 #include "idxd.h" 12 #include "registers.h" 13 14 void idxd_device_wqs_clear_state(struct idxd_device *idxd) 15 { 16 int i; 17 18 lockdep_assert_held(&idxd->dev_lock); 19 for (i = 0; i < idxd->max_wqs; i++) { 20 struct idxd_wq *wq = &idxd->wqs[i]; 21 22 wq->state = IDXD_WQ_DISABLED; 23 } 24 } 25 26 static int idxd_restart(struct idxd_device *idxd) 27 { 28 int i, rc; 29 30 lockdep_assert_held(&idxd->dev_lock); 31 32 rc = __idxd_device_reset(idxd); 33 if (rc < 0) 34 goto out; 35 36 rc = idxd_device_config(idxd); 37 if (rc < 0) 38 goto out; 39 40 rc = idxd_device_enable(idxd); 41 if (rc < 0) 42 goto out; 43 44 for (i = 0; i < idxd->max_wqs; i++) { 45 struct idxd_wq *wq = &idxd->wqs[i]; 46 47 if (wq->state == IDXD_WQ_ENABLED) { 48 rc = idxd_wq_enable(wq); 49 if (rc < 0) { 50 dev_warn(&idxd->pdev->dev, 51 "Unable to re-enable wq %s\n", 52 dev_name(&wq->conf_dev)); 53 } 54 } 55 } 56 57 return 0; 58 59 out: 60 idxd_device_wqs_clear_state(idxd); 61 idxd->state = IDXD_DEV_HALTED; 62 return rc; 63 } 64 65 irqreturn_t idxd_irq_handler(int vec, void *data) 66 { 67 struct idxd_irq_entry *irq_entry = data; 68 struct idxd_device *idxd = irq_entry->idxd; 69 70 idxd_mask_msix_vector(idxd, irq_entry->id); 71 return IRQ_WAKE_THREAD; 72 } 73 74 irqreturn_t idxd_misc_thread(int vec, void *data) 75 { 76 struct idxd_irq_entry *irq_entry = data; 77 struct idxd_device *idxd = irq_entry->idxd; 78 struct device *dev = &idxd->pdev->dev; 79 union gensts_reg gensts; 80 u32 cause, val = 0; 81 int i, rc; 82 bool err = false; 83 84 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); 85 86 if (cause & IDXD_INTC_ERR) { 87 spin_lock_bh(&idxd->dev_lock); 88 for (i = 0; i < 4; i++) 89 idxd->sw_err.bits[i] = ioread64(idxd->reg_base + 90 IDXD_SWERR_OFFSET + i * sizeof(u64)); 91 iowrite64(IDXD_SWERR_ACK, idxd->reg_base + IDXD_SWERR_OFFSET); 92 93 if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { 94 int id = idxd->sw_err.wq_idx; 95 struct idxd_wq *wq = &idxd->wqs[id]; 96 97 if (wq->type == IDXD_WQT_USER) 98 wake_up_interruptible(&wq->idxd_cdev.err_queue); 99 } else { 100 int i; 101 102 for (i = 0; i < idxd->max_wqs; i++) { 103 struct idxd_wq *wq = &idxd->wqs[i]; 104 105 if (wq->type == IDXD_WQT_USER) 106 wake_up_interruptible(&wq->idxd_cdev.err_queue); 107 } 108 } 109 110 spin_unlock_bh(&idxd->dev_lock); 111 val |= IDXD_INTC_ERR; 112 113 for (i = 0; i < 4; i++) 114 dev_warn(dev, "err[%d]: %#16.16llx\n", 115 i, idxd->sw_err.bits[i]); 116 err = true; 117 } 118 119 if (cause & IDXD_INTC_CMD) { 120 /* Driver does use command interrupts */ 121 val |= IDXD_INTC_CMD; 122 } 123 124 if (cause & IDXD_INTC_OCCUPY) { 125 /* Driver does not utilize occupancy interrupt */ 126 val |= IDXD_INTC_OCCUPY; 127 } 128 129 if (cause & IDXD_INTC_PERFMON_OVFL) { 130 /* 131 * Driver does not utilize perfmon counter overflow interrupt 132 * yet. 133 */ 134 val |= IDXD_INTC_PERFMON_OVFL; 135 } 136 137 val ^= cause; 138 if (val) 139 dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", 140 val); 141 142 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); 143 if (!err) 144 goto out; 145 146 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 147 if (gensts.state == IDXD_DEVICE_STATE_HALT) { 148 spin_lock_bh(&idxd->dev_lock); 149 if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { 150 rc = idxd_restart(idxd); 151 if (rc < 0) 152 dev_err(&idxd->pdev->dev, 153 "idxd restart failed, device halt."); 154 } else { 155 idxd_device_wqs_clear_state(idxd); 156 idxd->state = IDXD_DEV_HALTED; 157 dev_err(&idxd->pdev->dev, 158 "idxd halted, need %s.\n", 159 gensts.reset_type == IDXD_DEVICE_RESET_FLR ? 160 "FLR" : "system reset"); 161 } 162 spin_unlock_bh(&idxd->dev_lock); 163 } 164 165 out: 166 idxd_unmask_msix_vector(idxd, irq_entry->id); 167 return IRQ_HANDLED; 168 } 169 170 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 171 int *processed) 172 { 173 struct idxd_desc *desc, *t; 174 struct llist_node *head; 175 int queued = 0; 176 177 *processed = 0; 178 head = llist_del_all(&irq_entry->pending_llist); 179 if (!head) 180 return 0; 181 182 llist_for_each_entry_safe(desc, t, head, llnode) { 183 if (desc->completion->status) { 184 idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); 185 idxd_free_desc(desc->wq, desc); 186 (*processed)++; 187 } else { 188 list_add_tail(&desc->list, &irq_entry->work_list); 189 queued++; 190 } 191 } 192 193 return queued; 194 } 195 196 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 197 int *processed) 198 { 199 struct list_head *node, *next; 200 int queued = 0; 201 202 *processed = 0; 203 if (list_empty(&irq_entry->work_list)) 204 return 0; 205 206 list_for_each_safe(node, next, &irq_entry->work_list) { 207 struct idxd_desc *desc = 208 container_of(node, struct idxd_desc, list); 209 210 if (desc->completion->status) { 211 list_del(&desc->list); 212 /* process and callback */ 213 idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); 214 idxd_free_desc(desc->wq, desc); 215 (*processed)++; 216 } else { 217 queued++; 218 } 219 } 220 221 return queued; 222 } 223 224 static int idxd_desc_process(struct idxd_irq_entry *irq_entry) 225 { 226 int rc, processed, total = 0; 227 228 /* 229 * There are two lists we are processing. The pending_llist is where 230 * submmiter adds all the submitted descriptor after sending it to 231 * the workqueue. It's a lockless singly linked list. The work_list 232 * is the common linux double linked list. We are in a scenario of 233 * multiple producers and a single consumer. The producers are all 234 * the kernel submitters of descriptors, and the consumer is the 235 * kernel irq handler thread for the msix vector when using threaded 236 * irq. To work with the restrictions of llist to remain lockless, 237 * we are doing the following steps: 238 * 1. Iterate through the work_list and process any completed 239 * descriptor. Delete the completed entries during iteration. 240 * 2. llist_del_all() from the pending list. 241 * 3. Iterate through the llist that was deleted from the pending list 242 * and process the completed entries. 243 * 4. If the entry is still waiting on hardware, list_add_tail() to 244 * the work_list. 245 * 5. Repeat until no more descriptors. 246 */ 247 do { 248 rc = irq_process_work_list(irq_entry, &processed); 249 total += processed; 250 if (rc != 0) 251 continue; 252 253 rc = irq_process_pending_llist(irq_entry, &processed); 254 total += processed; 255 } while (rc != 0); 256 257 return total; 258 } 259 260 irqreturn_t idxd_wq_thread(int irq, void *data) 261 { 262 struct idxd_irq_entry *irq_entry = data; 263 int processed; 264 265 processed = idxd_desc_process(irq_entry); 266 idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); 267 /* catch anything unprocessed after unmasking */ 268 processed += idxd_desc_process(irq_entry); 269 270 if (processed == 0) 271 return IRQ_NONE; 272 273 return IRQ_HANDLED; 274 } 275