1d1dfe5b8SDave Jiang // SPDX-License-Identifier: GPL-2.0 2d1dfe5b8SDave Jiang /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3d1dfe5b8SDave Jiang #include <linux/init.h> 4d1dfe5b8SDave Jiang #include <linux/kernel.h> 5d1dfe5b8SDave Jiang #include <linux/module.h> 6d1dfe5b8SDave Jiang #include <linux/pci.h> 7d1dfe5b8SDave Jiang #include <uapi/linux/idxd.h> 8d1dfe5b8SDave Jiang #include "idxd.h" 9d1dfe5b8SDave Jiang #include "registers.h" 10d1dfe5b8SDave Jiang 110705107fSDave Jiang static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) 12d1dfe5b8SDave Jiang { 13d1dfe5b8SDave Jiang struct idxd_desc *desc; 14d1dfe5b8SDave Jiang 15d1dfe5b8SDave Jiang desc = wq->descs[idx]; 16d1dfe5b8SDave Jiang memset(desc->hw, 0, sizeof(struct dsa_hw_desc)); 17d1dfe5b8SDave Jiang memset(desc->completion, 0, sizeof(struct dsa_completion_record)); 180705107fSDave Jiang desc->cpu = cpu; 19d1dfe5b8SDave Jiang return desc; 20d1dfe5b8SDave Jiang } 21d1dfe5b8SDave Jiang 220705107fSDave Jiang struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) 230705107fSDave Jiang { 240705107fSDave Jiang int cpu, idx; 250705107fSDave Jiang struct idxd_device *idxd = wq->idxd; 260705107fSDave Jiang DEFINE_SBQ_WAIT(wait); 270705107fSDave Jiang struct sbq_wait_state *ws; 280705107fSDave Jiang struct sbitmap_queue *sbq; 290705107fSDave Jiang 300705107fSDave Jiang if (idxd->state != IDXD_DEV_ENABLED) 310705107fSDave Jiang return ERR_PTR(-EIO); 320705107fSDave Jiang 330705107fSDave Jiang sbq = &wq->sbq; 340705107fSDave Jiang idx = sbitmap_queue_get(sbq, &cpu); 350705107fSDave Jiang if (idx < 0) { 360705107fSDave Jiang if (optype == IDXD_OP_NONBLOCK) 370705107fSDave Jiang return ERR_PTR(-EAGAIN); 380705107fSDave Jiang } else { 390705107fSDave Jiang return __get_desc(wq, idx, cpu); 400705107fSDave Jiang } 410705107fSDave Jiang 420705107fSDave Jiang ws = &sbq->ws[0]; 430705107fSDave Jiang for (;;) { 440705107fSDave Jiang sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE); 450705107fSDave Jiang if (signal_pending_state(TASK_INTERRUPTIBLE, current)) 460705107fSDave Jiang break; 470705107fSDave Jiang idx = sbitmap_queue_get(sbq, &cpu); 480705107fSDave Jiang if (idx > 0) 490705107fSDave Jiang break; 500705107fSDave Jiang schedule(); 510705107fSDave Jiang } 520705107fSDave Jiang 530705107fSDave Jiang sbitmap_finish_wait(sbq, ws, &wait); 540705107fSDave Jiang if (idx < 0) 550705107fSDave Jiang return ERR_PTR(-EAGAIN); 560705107fSDave Jiang 570705107fSDave Jiang return __get_desc(wq, idx, cpu); 580705107fSDave Jiang } 590705107fSDave Jiang 60d1dfe5b8SDave Jiang void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) 61d1dfe5b8SDave Jiang { 620705107fSDave Jiang int cpu = desc->cpu; 63d1dfe5b8SDave Jiang 640705107fSDave Jiang desc->cpu = -1; 650705107fSDave Jiang sbitmap_queue_clear(&wq->sbq, desc->id, cpu); 66d1dfe5b8SDave Jiang } 67d1dfe5b8SDave Jiang 68d1dfe5b8SDave Jiang int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) 69d1dfe5b8SDave Jiang { 70d1dfe5b8SDave Jiang struct idxd_device *idxd = wq->idxd; 71d1dfe5b8SDave Jiang int vec = desc->hw->int_handle; 7242d279f9SDave Jiang void __iomem *portal; 73d1dfe5b8SDave Jiang 74d1dfe5b8SDave Jiang if (idxd->state != IDXD_DEV_ENABLED) 75d1dfe5b8SDave Jiang return -EIO; 76d1dfe5b8SDave Jiang 7742d279f9SDave Jiang portal = wq->dportal + idxd_get_wq_portal_offset(IDXD_PORTAL_UNLIMITED); 78d1dfe5b8SDave Jiang /* 79d1dfe5b8SDave Jiang * The wmb() flushes writes to coherent DMA data before possibly 80d1dfe5b8SDave Jiang * triggering a DMA read. The wmb() is necessary even on UP because 81d1dfe5b8SDave Jiang * the recipient is a device. 82d1dfe5b8SDave Jiang */ 83d1dfe5b8SDave Jiang wmb(); 8442d279f9SDave Jiang iosubmit_cmds512(portal, desc->hw, 1); 85d1dfe5b8SDave Jiang 86d1dfe5b8SDave Jiang /* 87d1dfe5b8SDave Jiang * Pending the descriptor to the lockless list for the irq_entry 88d1dfe5b8SDave Jiang * that we designated the descriptor to. 89d1dfe5b8SDave Jiang */ 908f47d1a5SDave Jiang if (desc->hw->flags & IDXD_OP_FLAG_RCI) 918f47d1a5SDave Jiang llist_add(&desc->llnode, 928f47d1a5SDave Jiang &idxd->irq_entries[vec].pending_llist); 93d1dfe5b8SDave Jiang 94d1dfe5b8SDave Jiang return 0; 95d1dfe5b8SDave Jiang } 96