1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 #include <linux/dmaengine.h> 9 #include <uapi/linux/idxd.h> 10 #include "../dmaengine.h" 11 #include "idxd.h" 12 #include "registers.h" 13 14 void idxd_device_wqs_clear_state(struct idxd_device *idxd) 15 { 16 int i; 17 18 lockdep_assert_held(&idxd->dev_lock); 19 for (i = 0; i < idxd->max_wqs; i++) { 20 struct idxd_wq *wq = &idxd->wqs[i]; 21 22 wq->state = IDXD_WQ_DISABLED; 23 } 24 } 25 26 static void idxd_device_reinit(struct work_struct *work) 27 { 28 struct idxd_device *idxd = container_of(work, struct idxd_device, work); 29 struct device *dev = &idxd->pdev->dev; 30 int rc, i; 31 32 idxd_device_reset(idxd); 33 rc = idxd_device_config(idxd); 34 if (rc < 0) 35 goto out; 36 37 rc = idxd_device_enable(idxd); 38 if (rc < 0) 39 goto out; 40 41 for (i = 0; i < idxd->max_wqs; i++) { 42 struct idxd_wq *wq = &idxd->wqs[i]; 43 44 if (wq->state == IDXD_WQ_ENABLED) { 45 rc = idxd_wq_enable(wq); 46 if (rc < 0) { 47 dev_warn(dev, "Unable to re-enable wq %s\n", 48 dev_name(&wq->conf_dev)); 49 } 50 } 51 } 52 53 return; 54 55 out: 56 idxd_device_wqs_clear_state(idxd); 57 } 58 59 irqreturn_t idxd_irq_handler(int vec, void *data) 60 { 61 struct idxd_irq_entry *irq_entry = data; 62 struct idxd_device *idxd = irq_entry->idxd; 63 64 idxd_mask_msix_vector(idxd, irq_entry->id); 65 return IRQ_WAKE_THREAD; 66 } 67 68 irqreturn_t idxd_misc_thread(int vec, void *data) 69 { 70 struct idxd_irq_entry *irq_entry = data; 71 struct idxd_device *idxd = irq_entry->idxd; 72 struct device *dev = &idxd->pdev->dev; 73 union gensts_reg gensts; 74 u32 cause, val = 0; 75 int i; 76 bool err = false; 77 78 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); 79 80 if (cause & IDXD_INTC_ERR) { 81 spin_lock_bh(&idxd->dev_lock); 82 for (i = 0; i < 4; i++) 83 idxd->sw_err.bits[i] = ioread64(idxd->reg_base + 84 IDXD_SWERR_OFFSET + i * sizeof(u64)); 85 iowrite64(IDXD_SWERR_ACK, idxd->reg_base + IDXD_SWERR_OFFSET); 86 87 if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { 88 int id = idxd->sw_err.wq_idx; 89 struct idxd_wq *wq = &idxd->wqs[id]; 90 91 if (wq->type == IDXD_WQT_USER) 92 wake_up_interruptible(&wq->idxd_cdev.err_queue); 93 } else { 94 int i; 95 96 for (i = 0; i < idxd->max_wqs; i++) { 97 struct idxd_wq *wq = &idxd->wqs[i]; 98 99 if (wq->type == IDXD_WQT_USER) 100 wake_up_interruptible(&wq->idxd_cdev.err_queue); 101 } 102 } 103 104 spin_unlock_bh(&idxd->dev_lock); 105 val |= IDXD_INTC_ERR; 106 107 for (i = 0; i < 4; i++) 108 dev_warn(dev, "err[%d]: %#16.16llx\n", 109 i, idxd->sw_err.bits[i]); 110 err = true; 111 } 112 113 if (cause & IDXD_INTC_CMD) { 114 val |= IDXD_INTC_CMD; 115 complete(idxd->cmd_done); 116 } 117 118 if (cause & IDXD_INTC_OCCUPY) { 119 /* Driver does not utilize occupancy interrupt */ 120 val |= IDXD_INTC_OCCUPY; 121 } 122 123 if (cause & IDXD_INTC_PERFMON_OVFL) { 124 /* 125 * Driver does not utilize perfmon counter overflow interrupt 126 * yet. 127 */ 128 val |= IDXD_INTC_PERFMON_OVFL; 129 } 130 131 val ^= cause; 132 if (val) 133 dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", 134 val); 135 136 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); 137 if (!err) 138 goto out; 139 140 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); 141 if (gensts.state == IDXD_DEVICE_STATE_HALT) { 142 idxd->state = IDXD_DEV_HALTED; 143 if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { 144 /* 145 * If we need a software reset, we will throw the work 146 * on a system workqueue in order to allow interrupts 147 * for the device command completions. 148 */ 149 INIT_WORK(&idxd->work, idxd_device_reinit); 150 queue_work(idxd->wq, &idxd->work); 151 } else { 152 spin_lock_bh(&idxd->dev_lock); 153 idxd_device_wqs_clear_state(idxd); 154 dev_err(&idxd->pdev->dev, 155 "idxd halted, need %s.\n", 156 gensts.reset_type == IDXD_DEVICE_RESET_FLR ? 157 "FLR" : "system reset"); 158 spin_unlock_bh(&idxd->dev_lock); 159 } 160 } 161 162 out: 163 idxd_unmask_msix_vector(idxd, irq_entry->id); 164 return IRQ_HANDLED; 165 } 166 167 static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 168 int *processed) 169 { 170 struct idxd_desc *desc, *t; 171 struct llist_node *head; 172 int queued = 0; 173 174 *processed = 0; 175 head = llist_del_all(&irq_entry->pending_llist); 176 if (!head) 177 return 0; 178 179 llist_for_each_entry_safe(desc, t, head, llnode) { 180 if (desc->completion->status) { 181 idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); 182 idxd_free_desc(desc->wq, desc); 183 (*processed)++; 184 } else { 185 list_add_tail(&desc->list, &irq_entry->work_list); 186 queued++; 187 } 188 } 189 190 return queued; 191 } 192 193 static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 194 int *processed) 195 { 196 struct list_head *node, *next; 197 int queued = 0; 198 199 *processed = 0; 200 if (list_empty(&irq_entry->work_list)) 201 return 0; 202 203 list_for_each_safe(node, next, &irq_entry->work_list) { 204 struct idxd_desc *desc = 205 container_of(node, struct idxd_desc, list); 206 207 if (desc->completion->status) { 208 list_del(&desc->list); 209 /* process and callback */ 210 idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); 211 idxd_free_desc(desc->wq, desc); 212 (*processed)++; 213 } else { 214 queued++; 215 } 216 } 217 218 return queued; 219 } 220 221 static int idxd_desc_process(struct idxd_irq_entry *irq_entry) 222 { 223 int rc, processed, total = 0; 224 225 /* 226 * There are two lists we are processing. The pending_llist is where 227 * submmiter adds all the submitted descriptor after sending it to 228 * the workqueue. It's a lockless singly linked list. The work_list 229 * is the common linux double linked list. We are in a scenario of 230 * multiple producers and a single consumer. The producers are all 231 * the kernel submitters of descriptors, and the consumer is the 232 * kernel irq handler thread for the msix vector when using threaded 233 * irq. To work with the restrictions of llist to remain lockless, 234 * we are doing the following steps: 235 * 1. Iterate through the work_list and process any completed 236 * descriptor. Delete the completed entries during iteration. 237 * 2. llist_del_all() from the pending list. 238 * 3. Iterate through the llist that was deleted from the pending list 239 * and process the completed entries. 240 * 4. If the entry is still waiting on hardware, list_add_tail() to 241 * the work_list. 242 * 5. Repeat until no more descriptors. 243 */ 244 do { 245 rc = irq_process_work_list(irq_entry, &processed); 246 total += processed; 247 if (rc != 0) 248 continue; 249 250 rc = irq_process_pending_llist(irq_entry, &processed); 251 total += processed; 252 } while (rc != 0); 253 254 return total; 255 } 256 257 irqreturn_t idxd_wq_thread(int irq, void *data) 258 { 259 struct idxd_irq_entry *irq_entry = data; 260 int processed; 261 262 processed = idxd_desc_process(irq_entry); 263 idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); 264 265 if (processed == 0) 266 return IRQ_NONE; 267 268 return IRQ_HANDLED; 269 } 270