1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/io-64-nonatomic-lo-hi.h> 9 #include <linux/dmaengine.h> 10 #include <uapi/linux/idxd.h> 11 #include "../dmaengine.h" 12 #include "registers.h" 13 #include "idxd.h" 14 15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c) 16 { 17 struct idxd_dma_chan *idxd_chan; 18 19 idxd_chan = container_of(c, struct idxd_dma_chan, chan); 20 return idxd_chan->wq; 21 } 22 23 void idxd_dma_complete_txd(struct idxd_desc *desc, 24 enum idxd_complete_type comp_type, 25 bool free_desc) 26 { 27 struct idxd_device *idxd = desc->wq->idxd; 28 struct dma_async_tx_descriptor *tx; 29 struct dmaengine_result res; 30 int complete = 1; 31 32 if (desc->completion->status == DSA_COMP_SUCCESS) { 33 res.result = DMA_TRANS_NOERROR; 34 } else if (desc->completion->status) { 35 if (idxd->request_int_handles && comp_type != IDXD_COMPLETE_ABORT && 36 desc->completion->status == DSA_COMP_INT_HANDLE_INVAL && 37 idxd_queue_int_handle_resubmit(desc)) 38 return; 39 res.result = DMA_TRANS_WRITE_FAILED; 40 } else if (comp_type == IDXD_COMPLETE_ABORT) { 41 res.result = DMA_TRANS_ABORTED; 42 } else { 43 complete = 0; 44 } 45 46 tx = &desc->txd; 47 if (complete && tx->cookie) { 48 dma_cookie_complete(tx); 49 dma_descriptor_unmap(tx); 50 dmaengine_desc_get_callback_invoke(tx, &res); 51 tx->callback = NULL; 52 tx->callback_result = NULL; 53 } 54 55 if (free_desc) 56 idxd_free_desc(desc->wq, desc); 57 } 58 59 static void op_flag_setup(unsigned long flags, u32 *desc_flags) 60 { 61 *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; 62 if (flags & DMA_PREP_INTERRUPT) 63 *desc_flags |= IDXD_OP_FLAG_RCI; 64 } 65 66 static inline void set_completion_address(struct idxd_desc *desc, 67 u64 *compl_addr) 68 { 69 *compl_addr = desc->compl_dma; 70 } 71 72 static inline void idxd_prep_desc_common(struct idxd_wq *wq, 73 struct dsa_hw_desc *hw, char opcode, 74 u64 addr_f1, u64 addr_f2, u64 len, 75 u64 compl, u32 flags) 76 { 77 hw->flags = flags; 78 hw->opcode = opcode; 79 hw->src_addr = addr_f1; 80 hw->dst_addr = addr_f2; 81 hw->xfer_size = len; 82 /* 83 * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv 84 * field instead. This field should be set to 1 for kernel descriptors. 85 */ 86 hw->priv = 1; 87 hw->completion_addr = compl; 88 } 89 90 static struct dma_async_tx_descriptor * 91 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest, 92 dma_addr_t dma_src, size_t len, unsigned long flags) 93 { 94 struct idxd_wq *wq = to_idxd_wq(c); 95 u32 desc_flags; 96 struct idxd_device *idxd = wq->idxd; 97 struct idxd_desc *desc; 98 99 if (wq->state != IDXD_WQ_ENABLED) 100 return NULL; 101 102 if (len > idxd->max_xfer_bytes) 103 return NULL; 104 105 op_flag_setup(flags, &desc_flags); 106 desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); 107 if (IS_ERR(desc)) 108 return NULL; 109 110 idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE, 111 dma_src, dma_dest, len, desc->compl_dma, 112 desc_flags); 113 114 desc->txd.flags = flags; 115 116 return &desc->txd; 117 } 118 119 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan) 120 { 121 struct idxd_wq *wq = to_idxd_wq(chan); 122 struct device *dev = &wq->idxd->pdev->dev; 123 124 idxd_wq_get(wq); 125 dev_dbg(dev, "%s: client_count: %d\n", __func__, 126 idxd_wq_refcount(wq)); 127 return 0; 128 } 129 130 static void idxd_dma_free_chan_resources(struct dma_chan *chan) 131 { 132 struct idxd_wq *wq = to_idxd_wq(chan); 133 struct device *dev = &wq->idxd->pdev->dev; 134 135 idxd_wq_put(wq); 136 dev_dbg(dev, "%s: client_count: %d\n", __func__, 137 idxd_wq_refcount(wq)); 138 } 139 140 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan, 141 dma_cookie_t cookie, 142 struct dma_tx_state *txstate) 143 { 144 return DMA_OUT_OF_ORDER; 145 } 146 147 /* 148 * issue_pending() does not need to do anything since tx_submit() does the job 149 * already. 150 */ 151 static void idxd_dma_issue_pending(struct dma_chan *dma_chan) 152 { 153 } 154 155 static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx) 156 { 157 struct dma_chan *c = tx->chan; 158 struct idxd_wq *wq = to_idxd_wq(c); 159 dma_cookie_t cookie; 160 int rc; 161 struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd); 162 163 cookie = dma_cookie_assign(tx); 164 165 rc = idxd_submit_desc(wq, desc); 166 if (rc < 0) { 167 idxd_free_desc(wq, desc); 168 return rc; 169 } 170 171 return cookie; 172 } 173 174 static void idxd_dma_release(struct dma_device *device) 175 { 176 struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma); 177 178 kfree(idxd_dma); 179 } 180 181 int idxd_register_dma_device(struct idxd_device *idxd) 182 { 183 struct idxd_dma_dev *idxd_dma; 184 struct dma_device *dma; 185 struct device *dev = &idxd->pdev->dev; 186 int rc; 187 188 idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev)); 189 if (!idxd_dma) 190 return -ENOMEM; 191 192 dma = &idxd_dma->dma; 193 INIT_LIST_HEAD(&dma->channels); 194 dma->dev = dev; 195 196 dma_cap_set(DMA_INTERRUPT, dma->cap_mask); 197 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 198 dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); 199 dma->device_release = idxd_dma_release; 200 201 if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) { 202 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 203 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy; 204 } 205 206 dma->device_tx_status = idxd_dma_tx_status; 207 dma->device_issue_pending = idxd_dma_issue_pending; 208 dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources; 209 dma->device_free_chan_resources = idxd_dma_free_chan_resources; 210 211 rc = dma_async_device_register(dma); 212 if (rc < 0) { 213 kfree(idxd_dma); 214 return rc; 215 } 216 217 idxd_dma->idxd = idxd; 218 /* 219 * This pointer is protected by the refs taken by the dma_chan. It will remain valid 220 * as long as there are outstanding channels. 221 */ 222 idxd->idxd_dma = idxd_dma; 223 return 0; 224 } 225 226 void idxd_unregister_dma_device(struct idxd_device *idxd) 227 { 228 dma_async_device_unregister(&idxd->idxd_dma->dma); 229 } 230 231 int idxd_register_dma_channel(struct idxd_wq *wq) 232 { 233 struct idxd_device *idxd = wq->idxd; 234 struct dma_device *dma = &idxd->idxd_dma->dma; 235 struct device *dev = &idxd->pdev->dev; 236 struct idxd_dma_chan *idxd_chan; 237 struct dma_chan *chan; 238 int rc, i; 239 240 idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev)); 241 if (!idxd_chan) 242 return -ENOMEM; 243 244 chan = &idxd_chan->chan; 245 chan->device = dma; 246 list_add_tail(&chan->device_node, &dma->channels); 247 248 for (i = 0; i < wq->num_descs; i++) { 249 struct idxd_desc *desc = wq->descs[i]; 250 251 dma_async_tx_descriptor_init(&desc->txd, chan); 252 desc->txd.tx_submit = idxd_dma_tx_submit; 253 } 254 255 rc = dma_async_device_channel_register(dma, chan); 256 if (rc < 0) { 257 kfree(idxd_chan); 258 return rc; 259 } 260 261 wq->idxd_chan = idxd_chan; 262 idxd_chan->wq = wq; 263 get_device(wq_confdev(wq)); 264 265 return 0; 266 } 267 268 void idxd_unregister_dma_channel(struct idxd_wq *wq) 269 { 270 struct idxd_dma_chan *idxd_chan = wq->idxd_chan; 271 struct dma_chan *chan = &idxd_chan->chan; 272 struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma; 273 274 dma_async_device_channel_unregister(&idxd_dma->dma, chan); 275 list_del(&chan->device_node); 276 kfree(wq->idxd_chan); 277 wq->idxd_chan = NULL; 278 put_device(wq_confdev(wq)); 279 } 280 281 static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) 282 { 283 struct device *dev = &idxd_dev->conf_dev; 284 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 285 struct idxd_device *idxd = wq->idxd; 286 int rc; 287 288 if (idxd->state != IDXD_DEV_ENABLED) 289 return -ENXIO; 290 291 mutex_lock(&wq->wq_lock); 292 wq->type = IDXD_WQT_KERNEL; 293 294 rc = __drv_enable_wq(wq); 295 if (rc < 0) { 296 dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); 297 rc = -ENXIO; 298 goto err; 299 } 300 301 rc = idxd_wq_request_irq(wq); 302 if (rc < 0) { 303 idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR; 304 dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc); 305 goto err_irq; 306 } 307 308 rc = idxd_wq_alloc_resources(wq); 309 if (rc < 0) { 310 idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR; 311 dev_dbg(dev, "WQ resource alloc failed\n"); 312 goto err_res_alloc; 313 } 314 315 rc = idxd_wq_init_percpu_ref(wq); 316 if (rc < 0) { 317 idxd->cmd_status = IDXD_SCMD_PERCPU_ERR; 318 dev_dbg(dev, "percpu_ref setup failed\n"); 319 goto err_ref; 320 } 321 322 rc = idxd_register_dma_channel(wq); 323 if (rc < 0) { 324 idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR; 325 dev_dbg(dev, "Failed to register dma channel\n"); 326 goto err_dma; 327 } 328 329 idxd->cmd_status = 0; 330 mutex_unlock(&wq->wq_lock); 331 return 0; 332 333 err_dma: 334 __idxd_wq_quiesce(wq); 335 percpu_ref_exit(&wq->wq_active); 336 err_ref: 337 idxd_wq_free_resources(wq); 338 err_res_alloc: 339 idxd_wq_free_irq(wq); 340 err_irq: 341 __drv_disable_wq(wq); 342 err: 343 wq->type = IDXD_WQT_NONE; 344 mutex_unlock(&wq->wq_lock); 345 return rc; 346 } 347 348 static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) 349 { 350 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 351 352 mutex_lock(&wq->wq_lock); 353 __idxd_wq_quiesce(wq); 354 idxd_unregister_dma_channel(wq); 355 idxd_wq_free_resources(wq); 356 __drv_disable_wq(wq); 357 percpu_ref_exit(&wq->wq_active); 358 idxd_wq_free_irq(wq); 359 wq->type = IDXD_WQT_NONE; 360 mutex_unlock(&wq->wq_lock); 361 } 362 363 static enum idxd_dev_type dev_types[] = { 364 IDXD_DEV_WQ, 365 IDXD_DEV_NONE, 366 }; 367 368 struct idxd_device_driver idxd_dmaengine_drv = { 369 .probe = idxd_dmaengine_drv_probe, 370 .remove = idxd_dmaengine_drv_remove, 371 .name = "dmaengine", 372 .type = dev_types, 373 }; 374 EXPORT_SYMBOL_GPL(idxd_dmaengine_drv); 375