1e1ffcc66SDimitris Michailidis // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2e1ffcc66SDimitris Michailidis
3e1ffcc66SDimitris Michailidis #include <linux/bitmap.h>
4e1ffcc66SDimitris Michailidis #include <linux/delay.h>
5e1ffcc66SDimitris Michailidis #include <linux/interrupt.h>
6e1ffcc66SDimitris Michailidis #include <linux/io.h>
7e1ffcc66SDimitris Michailidis #include <linux/io-64-nonatomic-lo-hi.h>
8e1ffcc66SDimitris Michailidis #include <linux/mm.h>
9e1ffcc66SDimitris Michailidis #include <linux/module.h>
10e1ffcc66SDimitris Michailidis #include <linux/nvme.h>
11e1ffcc66SDimitris Michailidis #include <linux/pci.h>
12e1ffcc66SDimitris Michailidis #include <linux/wait.h>
13e1ffcc66SDimitris Michailidis #include <linux/sched/signal.h>
14e1ffcc66SDimitris Michailidis
15e1ffcc66SDimitris Michailidis #include "fun_queue.h"
16e1ffcc66SDimitris Michailidis #include "fun_dev.h"
17e1ffcc66SDimitris Michailidis
18e1ffcc66SDimitris Michailidis #define FUN_ADMIN_CMD_TO_MS 3000
19e1ffcc66SDimitris Michailidis
20e1ffcc66SDimitris Michailidis enum {
21e1ffcc66SDimitris Michailidis AQA_ASQS_SHIFT = 0,
22e1ffcc66SDimitris Michailidis AQA_ACQS_SHIFT = 16,
23e1ffcc66SDimitris Michailidis AQA_MIN_QUEUE_SIZE = 2,
24e1ffcc66SDimitris Michailidis AQA_MAX_QUEUE_SIZE = 4096
25e1ffcc66SDimitris Michailidis };
26e1ffcc66SDimitris Michailidis
27e1ffcc66SDimitris Michailidis /* context for admin commands */
28e1ffcc66SDimitris Michailidis struct fun_cmd_ctx {
29e1ffcc66SDimitris Michailidis fun_admin_callback_t cb; /* callback to invoke on completion */
30e1ffcc66SDimitris Michailidis void *cb_data; /* user data provided to callback */
31e1ffcc66SDimitris Michailidis int cpu; /* CPU where the cmd's tag was allocated */
32e1ffcc66SDimitris Michailidis };
33e1ffcc66SDimitris Michailidis
34e1ffcc66SDimitris Michailidis /* Context for synchronous admin commands. */
35e1ffcc66SDimitris Michailidis struct fun_sync_cmd_ctx {
36e1ffcc66SDimitris Michailidis struct completion compl;
37e1ffcc66SDimitris Michailidis u8 *rsp_buf; /* caller provided response buffer */
38e1ffcc66SDimitris Michailidis unsigned int rsp_len; /* response buffer size */
39e1ffcc66SDimitris Michailidis u8 rsp_status; /* command response status */
40e1ffcc66SDimitris Michailidis };
41e1ffcc66SDimitris Michailidis
42e1ffcc66SDimitris Michailidis /* Wait for the CSTS.RDY bit to match @enabled. */
fun_wait_ready(struct fun_dev * fdev,bool enabled)43e1ffcc66SDimitris Michailidis static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
44e1ffcc66SDimitris Michailidis {
45e1ffcc66SDimitris Michailidis unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
46e1ffcc66SDimitris Michailidis u32 bit = enabled ? NVME_CSTS_RDY : 0;
47e1ffcc66SDimitris Michailidis unsigned long deadline;
48e1ffcc66SDimitris Michailidis
49e1ffcc66SDimitris Michailidis deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
50e1ffcc66SDimitris Michailidis
51e1ffcc66SDimitris Michailidis for (;;) {
52e1ffcc66SDimitris Michailidis u32 csts = readl(fdev->bar + NVME_REG_CSTS);
53e1ffcc66SDimitris Michailidis
54e1ffcc66SDimitris Michailidis if (csts == ~0) {
55e1ffcc66SDimitris Michailidis dev_err(fdev->dev, "CSTS register read %#x\n", csts);
56e1ffcc66SDimitris Michailidis return -EIO;
57e1ffcc66SDimitris Michailidis }
58e1ffcc66SDimitris Michailidis
59e1ffcc66SDimitris Michailidis if ((csts & NVME_CSTS_RDY) == bit)
60e1ffcc66SDimitris Michailidis return 0;
61e1ffcc66SDimitris Michailidis
62e1ffcc66SDimitris Michailidis if (time_is_before_jiffies(deadline))
63e1ffcc66SDimitris Michailidis break;
64e1ffcc66SDimitris Michailidis
65e1ffcc66SDimitris Michailidis msleep(100);
66e1ffcc66SDimitris Michailidis }
67e1ffcc66SDimitris Michailidis
68e1ffcc66SDimitris Michailidis dev_err(fdev->dev,
69e1ffcc66SDimitris Michailidis "Timed out waiting for device to indicate RDY %u; aborting %s\n",
70e1ffcc66SDimitris Michailidis enabled, enabled ? "initialization" : "reset");
71e1ffcc66SDimitris Michailidis return -ETIMEDOUT;
72e1ffcc66SDimitris Michailidis }
73e1ffcc66SDimitris Michailidis
74e1ffcc66SDimitris Michailidis /* Check CSTS and return an error if it is unreadable or has unexpected
75e1ffcc66SDimitris Michailidis * RDY value.
76e1ffcc66SDimitris Michailidis */
fun_check_csts_rdy(struct fun_dev * fdev,unsigned int expected_rdy)77e1ffcc66SDimitris Michailidis static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
78e1ffcc66SDimitris Michailidis {
79e1ffcc66SDimitris Michailidis u32 csts = readl(fdev->bar + NVME_REG_CSTS);
80e1ffcc66SDimitris Michailidis u32 actual_rdy = csts & NVME_CSTS_RDY;
81e1ffcc66SDimitris Michailidis
82e1ffcc66SDimitris Michailidis if (csts == ~0) {
83e1ffcc66SDimitris Michailidis dev_err(fdev->dev, "CSTS register read %#x\n", csts);
84e1ffcc66SDimitris Michailidis return -EIO;
85e1ffcc66SDimitris Michailidis }
86e1ffcc66SDimitris Michailidis if (actual_rdy != expected_rdy) {
87e1ffcc66SDimitris Michailidis dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
88e1ffcc66SDimitris Michailidis return -EINVAL;
89e1ffcc66SDimitris Michailidis }
90e1ffcc66SDimitris Michailidis return 0;
91e1ffcc66SDimitris Michailidis }
92e1ffcc66SDimitris Michailidis
93e1ffcc66SDimitris Michailidis /* Check that CSTS RDY has the expected value. Then write a new value to the CC
94e1ffcc66SDimitris Michailidis * register and wait for CSTS RDY to match the new CC ENABLE state.
95e1ffcc66SDimitris Michailidis */
fun_update_cc_enable(struct fun_dev * fdev,unsigned int initial_rdy)96e1ffcc66SDimitris Michailidis static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
97e1ffcc66SDimitris Michailidis {
98e1ffcc66SDimitris Michailidis int rc = fun_check_csts_rdy(fdev, initial_rdy);
99e1ffcc66SDimitris Michailidis
100e1ffcc66SDimitris Michailidis if (rc)
101e1ffcc66SDimitris Michailidis return rc;
102e1ffcc66SDimitris Michailidis writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
103e1ffcc66SDimitris Michailidis return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
104e1ffcc66SDimitris Michailidis }
105e1ffcc66SDimitris Michailidis
fun_disable_ctrl(struct fun_dev * fdev)106e1ffcc66SDimitris Michailidis static int fun_disable_ctrl(struct fun_dev *fdev)
107e1ffcc66SDimitris Michailidis {
108e1ffcc66SDimitris Michailidis fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);
109e1ffcc66SDimitris Michailidis return fun_update_cc_enable(fdev, 1);
110e1ffcc66SDimitris Michailidis }
111e1ffcc66SDimitris Michailidis
fun_enable_ctrl(struct fun_dev * fdev,u32 admin_cqesz_log2,u32 admin_sqesz_log2)112e1ffcc66SDimitris Michailidis static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
113e1ffcc66SDimitris Michailidis u32 admin_sqesz_log2)
114e1ffcc66SDimitris Michailidis {
115e1ffcc66SDimitris Michailidis fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |
116e1ffcc66SDimitris Michailidis (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |
117e1ffcc66SDimitris Michailidis ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |
118e1ffcc66SDimitris Michailidis NVME_CC_ENABLE;
119e1ffcc66SDimitris Michailidis
120e1ffcc66SDimitris Michailidis return fun_update_cc_enable(fdev, 0);
121e1ffcc66SDimitris Michailidis }
122e1ffcc66SDimitris Michailidis
fun_map_bars(struct fun_dev * fdev,const char * name)123e1ffcc66SDimitris Michailidis static int fun_map_bars(struct fun_dev *fdev, const char *name)
124e1ffcc66SDimitris Michailidis {
125e1ffcc66SDimitris Michailidis struct pci_dev *pdev = to_pci_dev(fdev->dev);
126e1ffcc66SDimitris Michailidis int err;
127e1ffcc66SDimitris Michailidis
128e1ffcc66SDimitris Michailidis err = pci_request_mem_regions(pdev, name);
129e1ffcc66SDimitris Michailidis if (err) {
130e1ffcc66SDimitris Michailidis dev_err(&pdev->dev,
131e1ffcc66SDimitris Michailidis "Couldn't get PCI memory resources, err %d\n", err);
132e1ffcc66SDimitris Michailidis return err;
133e1ffcc66SDimitris Michailidis }
134e1ffcc66SDimitris Michailidis
135e1ffcc66SDimitris Michailidis fdev->bar = pci_ioremap_bar(pdev, 0);
136e1ffcc66SDimitris Michailidis if (!fdev->bar) {
137e1ffcc66SDimitris Michailidis dev_err(&pdev->dev, "Couldn't map BAR 0\n");
138e1ffcc66SDimitris Michailidis pci_release_mem_regions(pdev);
139e1ffcc66SDimitris Michailidis return -ENOMEM;
140e1ffcc66SDimitris Michailidis }
141e1ffcc66SDimitris Michailidis
142e1ffcc66SDimitris Michailidis return 0;
143e1ffcc66SDimitris Michailidis }
144e1ffcc66SDimitris Michailidis
fun_unmap_bars(struct fun_dev * fdev)145e1ffcc66SDimitris Michailidis static void fun_unmap_bars(struct fun_dev *fdev)
146e1ffcc66SDimitris Michailidis {
147e1ffcc66SDimitris Michailidis struct pci_dev *pdev = to_pci_dev(fdev->dev);
148e1ffcc66SDimitris Michailidis
149e1ffcc66SDimitris Michailidis if (fdev->bar) {
150e1ffcc66SDimitris Michailidis iounmap(fdev->bar);
151e1ffcc66SDimitris Michailidis fdev->bar = NULL;
152e1ffcc66SDimitris Michailidis pci_release_mem_regions(pdev);
153e1ffcc66SDimitris Michailidis }
154e1ffcc66SDimitris Michailidis }
155e1ffcc66SDimitris Michailidis
fun_set_dma_masks(struct device * dev)156e1ffcc66SDimitris Michailidis static int fun_set_dma_masks(struct device *dev)
157e1ffcc66SDimitris Michailidis {
158e1ffcc66SDimitris Michailidis int err;
159e1ffcc66SDimitris Michailidis
160e1ffcc66SDimitris Michailidis err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
161e1ffcc66SDimitris Michailidis if (err)
162e1ffcc66SDimitris Michailidis dev_err(dev, "DMA mask configuration failed, err %d\n", err);
163e1ffcc66SDimitris Michailidis return err;
164e1ffcc66SDimitris Michailidis }
165e1ffcc66SDimitris Michailidis
fun_admin_irq(int irq,void * data)166e1ffcc66SDimitris Michailidis static irqreturn_t fun_admin_irq(int irq, void *data)
167e1ffcc66SDimitris Michailidis {
168e1ffcc66SDimitris Michailidis struct fun_queue *funq = data;
169e1ffcc66SDimitris Michailidis
170e1ffcc66SDimitris Michailidis return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
171e1ffcc66SDimitris Michailidis }
172e1ffcc66SDimitris Michailidis
fun_complete_admin_cmd(struct fun_queue * funq,void * data,void * entry,const struct fun_cqe_info * info)173e1ffcc66SDimitris Michailidis static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,
174e1ffcc66SDimitris Michailidis void *entry, const struct fun_cqe_info *info)
175e1ffcc66SDimitris Michailidis {
176e1ffcc66SDimitris Michailidis const struct fun_admin_rsp_common *rsp_common = entry;
177e1ffcc66SDimitris Michailidis struct fun_dev *fdev = funq->fdev;
178e1ffcc66SDimitris Michailidis struct fun_cmd_ctx *cmd_ctx;
179e1ffcc66SDimitris Michailidis int cpu;
180e1ffcc66SDimitris Michailidis u16 cid;
181e1ffcc66SDimitris Michailidis
182e1ffcc66SDimitris Michailidis if (info->sqhd == cpu_to_be16(0xffff)) {
183e1ffcc66SDimitris Michailidis dev_dbg(fdev->dev, "adminq event");
184e1ffcc66SDimitris Michailidis if (fdev->adminq_cb)
185e1ffcc66SDimitris Michailidis fdev->adminq_cb(fdev, entry);
186e1ffcc66SDimitris Michailidis return;
187e1ffcc66SDimitris Michailidis }
188e1ffcc66SDimitris Michailidis
189e1ffcc66SDimitris Michailidis cid = be16_to_cpu(rsp_common->cid);
190e1ffcc66SDimitris Michailidis dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
191e1ffcc66SDimitris Michailidis rsp_common->op, rsp_common->ret);
192e1ffcc66SDimitris Michailidis
193e1ffcc66SDimitris Michailidis cmd_ctx = &fdev->cmd_ctx[cid];
194e1ffcc66SDimitris Michailidis if (cmd_ctx->cpu < 0) {
195e1ffcc66SDimitris Michailidis dev_err(fdev->dev,
196e1ffcc66SDimitris Michailidis "admin CQE with CID=%u, op=%u does not match a pending command\n",
197e1ffcc66SDimitris Michailidis cid, rsp_common->op);
198e1ffcc66SDimitris Michailidis return;
199e1ffcc66SDimitris Michailidis }
200e1ffcc66SDimitris Michailidis
201e1ffcc66SDimitris Michailidis if (cmd_ctx->cb)
202e1ffcc66SDimitris Michailidis cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
203e1ffcc66SDimitris Michailidis
204e1ffcc66SDimitris Michailidis cpu = cmd_ctx->cpu;
205e1ffcc66SDimitris Michailidis cmd_ctx->cpu = -1;
206e1ffcc66SDimitris Michailidis sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
207e1ffcc66SDimitris Michailidis }
208e1ffcc66SDimitris Michailidis
fun_init_cmd_ctx(struct fun_dev * fdev,unsigned int ntags)209e1ffcc66SDimitris Michailidis static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
210e1ffcc66SDimitris Michailidis {
211e1ffcc66SDimitris Michailidis unsigned int i;
212e1ffcc66SDimitris Michailidis
213e1ffcc66SDimitris Michailidis fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
214e1ffcc66SDimitris Michailidis if (!fdev->cmd_ctx)
215e1ffcc66SDimitris Michailidis return -ENOMEM;
216e1ffcc66SDimitris Michailidis
217e1ffcc66SDimitris Michailidis for (i = 0; i < ntags; i++)
218e1ffcc66SDimitris Michailidis fdev->cmd_ctx[i].cpu = -1;
219e1ffcc66SDimitris Michailidis
220e1ffcc66SDimitris Michailidis return 0;
221e1ffcc66SDimitris Michailidis }
222e1ffcc66SDimitris Michailidis
223e1ffcc66SDimitris Michailidis /* Allocate and enable an admin queue and assign it the first IRQ vector. */
fun_enable_admin_queue(struct fun_dev * fdev,const struct fun_dev_params * areq)224e1ffcc66SDimitris Michailidis static int fun_enable_admin_queue(struct fun_dev *fdev,
225e1ffcc66SDimitris Michailidis const struct fun_dev_params *areq)
226e1ffcc66SDimitris Michailidis {
227e1ffcc66SDimitris Michailidis struct fun_queue_alloc_req qreq = {
228e1ffcc66SDimitris Michailidis .cqe_size_log2 = areq->cqe_size_log2,
229e1ffcc66SDimitris Michailidis .sqe_size_log2 = areq->sqe_size_log2,
230e1ffcc66SDimitris Michailidis .cq_depth = areq->cq_depth,
231e1ffcc66SDimitris Michailidis .sq_depth = areq->sq_depth,
232e1ffcc66SDimitris Michailidis .rq_depth = areq->rq_depth,
233e1ffcc66SDimitris Michailidis };
234e1ffcc66SDimitris Michailidis unsigned int ntags = areq->sq_depth - 1;
235e1ffcc66SDimitris Michailidis struct fun_queue *funq;
236e1ffcc66SDimitris Michailidis int rc;
237e1ffcc66SDimitris Michailidis
238e1ffcc66SDimitris Michailidis if (fdev->admin_q)
239e1ffcc66SDimitris Michailidis return -EEXIST;
240e1ffcc66SDimitris Michailidis
241e1ffcc66SDimitris Michailidis if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||
242e1ffcc66SDimitris Michailidis areq->sq_depth > AQA_MAX_QUEUE_SIZE ||
243e1ffcc66SDimitris Michailidis areq->cq_depth < AQA_MIN_QUEUE_SIZE ||
244e1ffcc66SDimitris Michailidis areq->cq_depth > AQA_MAX_QUEUE_SIZE)
245e1ffcc66SDimitris Michailidis return -EINVAL;
246e1ffcc66SDimitris Michailidis
247e1ffcc66SDimitris Michailidis fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
248e1ffcc66SDimitris Michailidis if (!fdev->admin_q)
249e1ffcc66SDimitris Michailidis return -ENOMEM;
250e1ffcc66SDimitris Michailidis
251e1ffcc66SDimitris Michailidis rc = fun_init_cmd_ctx(fdev, ntags);
252e1ffcc66SDimitris Michailidis if (rc)
253e1ffcc66SDimitris Michailidis goto free_q;
254e1ffcc66SDimitris Michailidis
255e1ffcc66SDimitris Michailidis rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
256e1ffcc66SDimitris Michailidis GFP_KERNEL, dev_to_node(fdev->dev));
257e1ffcc66SDimitris Michailidis if (rc)
258e1ffcc66SDimitris Michailidis goto free_cmd_ctx;
259e1ffcc66SDimitris Michailidis
260e1ffcc66SDimitris Michailidis funq = fdev->admin_q;
261e1ffcc66SDimitris Michailidis funq->cq_vector = 0;
262e1ffcc66SDimitris Michailidis rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
263e1ffcc66SDimitris Michailidis if (rc)
264e1ffcc66SDimitris Michailidis goto free_sbq;
265e1ffcc66SDimitris Michailidis
266e1ffcc66SDimitris Michailidis fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
267e1ffcc66SDimitris Michailidis fdev->adminq_cb = areq->event_cb;
268e1ffcc66SDimitris Michailidis
269e1ffcc66SDimitris Michailidis writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |
270e1ffcc66SDimitris Michailidis (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
271e1ffcc66SDimitris Michailidis fdev->bar + NVME_REG_AQA);
272e1ffcc66SDimitris Michailidis
273e1ffcc66SDimitris Michailidis writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
274e1ffcc66SDimitris Michailidis writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);
275e1ffcc66SDimitris Michailidis
276e1ffcc66SDimitris Michailidis rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
277e1ffcc66SDimitris Michailidis if (rc)
278e1ffcc66SDimitris Michailidis goto free_irq;
279e1ffcc66SDimitris Michailidis
280e1ffcc66SDimitris Michailidis if (areq->rq_depth) {
281e1ffcc66SDimitris Michailidis rc = fun_create_rq(funq);
282e1ffcc66SDimitris Michailidis if (rc)
283e1ffcc66SDimitris Michailidis goto disable_ctrl;
284e1ffcc66SDimitris Michailidis
285e1ffcc66SDimitris Michailidis funq_rq_post(funq);
286e1ffcc66SDimitris Michailidis }
287e1ffcc66SDimitris Michailidis
288e1ffcc66SDimitris Michailidis return 0;
289e1ffcc66SDimitris Michailidis
290e1ffcc66SDimitris Michailidis disable_ctrl:
291e1ffcc66SDimitris Michailidis fun_disable_ctrl(fdev);
292e1ffcc66SDimitris Michailidis free_irq:
293e1ffcc66SDimitris Michailidis fun_free_irq(funq);
294e1ffcc66SDimitris Michailidis free_sbq:
295e1ffcc66SDimitris Michailidis sbitmap_queue_free(&fdev->admin_sbq);
296e1ffcc66SDimitris Michailidis free_cmd_ctx:
297e1ffcc66SDimitris Michailidis kvfree(fdev->cmd_ctx);
298e1ffcc66SDimitris Michailidis fdev->cmd_ctx = NULL;
299e1ffcc66SDimitris Michailidis free_q:
300e1ffcc66SDimitris Michailidis fun_free_queue(fdev->admin_q);
301e1ffcc66SDimitris Michailidis fdev->admin_q = NULL;
302e1ffcc66SDimitris Michailidis return rc;
303e1ffcc66SDimitris Michailidis }
304e1ffcc66SDimitris Michailidis
fun_disable_admin_queue(struct fun_dev * fdev)305e1ffcc66SDimitris Michailidis static void fun_disable_admin_queue(struct fun_dev *fdev)
306e1ffcc66SDimitris Michailidis {
307e1ffcc66SDimitris Michailidis struct fun_queue *admq = fdev->admin_q;
308e1ffcc66SDimitris Michailidis
309e1ffcc66SDimitris Michailidis if (!admq)
310e1ffcc66SDimitris Michailidis return;
311e1ffcc66SDimitris Michailidis
312e1ffcc66SDimitris Michailidis fun_disable_ctrl(fdev);
313e1ffcc66SDimitris Michailidis
314e1ffcc66SDimitris Michailidis fun_free_irq(admq);
315e1ffcc66SDimitris Michailidis __fun_process_cq(admq, 0);
316e1ffcc66SDimitris Michailidis
317e1ffcc66SDimitris Michailidis sbitmap_queue_free(&fdev->admin_sbq);
318e1ffcc66SDimitris Michailidis
319e1ffcc66SDimitris Michailidis kvfree(fdev->cmd_ctx);
320e1ffcc66SDimitris Michailidis fdev->cmd_ctx = NULL;
321e1ffcc66SDimitris Michailidis
322e1ffcc66SDimitris Michailidis fun_free_queue(admq);
323e1ffcc66SDimitris Michailidis fdev->admin_q = NULL;
324e1ffcc66SDimitris Michailidis }
325e1ffcc66SDimitris Michailidis
326e1ffcc66SDimitris Michailidis /* Return %true if the admin queue has stopped servicing commands as can be
327e1ffcc66SDimitris Michailidis * detected through registers. This isn't exhaustive and may provide false
328e1ffcc66SDimitris Michailidis * negatives.
329e1ffcc66SDimitris Michailidis */
fun_adminq_stopped(struct fun_dev * fdev)330e1ffcc66SDimitris Michailidis static bool fun_adminq_stopped(struct fun_dev *fdev)
331e1ffcc66SDimitris Michailidis {
332e1ffcc66SDimitris Michailidis u32 csts = readl(fdev->bar + NVME_REG_CSTS);
333e1ffcc66SDimitris Michailidis
334e1ffcc66SDimitris Michailidis return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;
335e1ffcc66SDimitris Michailidis }
336e1ffcc66SDimitris Michailidis
fun_wait_for_tag(struct fun_dev * fdev,int * cpup)337e1ffcc66SDimitris Michailidis static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)
338e1ffcc66SDimitris Michailidis {
339e1ffcc66SDimitris Michailidis struct sbitmap_queue *sbq = &fdev->admin_sbq;
340e1ffcc66SDimitris Michailidis struct sbq_wait_state *ws = &sbq->ws[0];
341e1ffcc66SDimitris Michailidis DEFINE_SBQ_WAIT(wait);
342e1ffcc66SDimitris Michailidis int tag;
343e1ffcc66SDimitris Michailidis
344e1ffcc66SDimitris Michailidis for (;;) {
345e1ffcc66SDimitris Michailidis sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
346e1ffcc66SDimitris Michailidis if (fdev->suppress_cmds) {
347e1ffcc66SDimitris Michailidis tag = -ESHUTDOWN;
348e1ffcc66SDimitris Michailidis break;
349e1ffcc66SDimitris Michailidis }
350e1ffcc66SDimitris Michailidis tag = sbitmap_queue_get(sbq, cpup);
351e1ffcc66SDimitris Michailidis if (tag >= 0)
352e1ffcc66SDimitris Michailidis break;
353e1ffcc66SDimitris Michailidis schedule();
354e1ffcc66SDimitris Michailidis }
355e1ffcc66SDimitris Michailidis
356e1ffcc66SDimitris Michailidis sbitmap_finish_wait(sbq, ws, &wait);
357e1ffcc66SDimitris Michailidis return tag;
358e1ffcc66SDimitris Michailidis }
359e1ffcc66SDimitris Michailidis
360e1ffcc66SDimitris Michailidis /* Submit an asynchronous admin command. Caller is responsible for implementing
361e1ffcc66SDimitris Michailidis * any waiting or timeout. Upon command completion the callback @cb is called.
362e1ffcc66SDimitris Michailidis */
fun_submit_admin_cmd(struct fun_dev * fdev,struct fun_admin_req_common * cmd,fun_admin_callback_t cb,void * cb_data,bool wait_ok)363e1ffcc66SDimitris Michailidis int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,
364e1ffcc66SDimitris Michailidis fun_admin_callback_t cb, void *cb_data, bool wait_ok)
365e1ffcc66SDimitris Michailidis {
366e1ffcc66SDimitris Michailidis struct fun_queue *funq = fdev->admin_q;
367e1ffcc66SDimitris Michailidis unsigned int cmdsize = cmd->len8 * 8;
368e1ffcc66SDimitris Michailidis struct fun_cmd_ctx *cmd_ctx;
369e1ffcc66SDimitris Michailidis int tag, cpu, rc = 0;
370e1ffcc66SDimitris Michailidis
371e1ffcc66SDimitris Michailidis if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
372e1ffcc66SDimitris Michailidis return -EMSGSIZE;
373e1ffcc66SDimitris Michailidis
374e1ffcc66SDimitris Michailidis tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
375e1ffcc66SDimitris Michailidis if (tag < 0) {
376e1ffcc66SDimitris Michailidis if (!wait_ok)
377e1ffcc66SDimitris Michailidis return -EAGAIN;
378e1ffcc66SDimitris Michailidis tag = fun_wait_for_tag(fdev, &cpu);
379e1ffcc66SDimitris Michailidis if (tag < 0)
380e1ffcc66SDimitris Michailidis return tag;
381e1ffcc66SDimitris Michailidis }
382e1ffcc66SDimitris Michailidis
383e1ffcc66SDimitris Michailidis cmd->cid = cpu_to_be16(tag);
384e1ffcc66SDimitris Michailidis
385e1ffcc66SDimitris Michailidis cmd_ctx = &fdev->cmd_ctx[tag];
386e1ffcc66SDimitris Michailidis cmd_ctx->cb = cb;
387e1ffcc66SDimitris Michailidis cmd_ctx->cb_data = cb_data;
388e1ffcc66SDimitris Michailidis
389e1ffcc66SDimitris Michailidis spin_lock(&funq->sq_lock);
390e1ffcc66SDimitris Michailidis
391e1ffcc66SDimitris Michailidis if (unlikely(fdev->suppress_cmds)) {
392e1ffcc66SDimitris Michailidis rc = -ESHUTDOWN;
393e1ffcc66SDimitris Michailidis sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
394e1ffcc66SDimitris Michailidis } else {
395e1ffcc66SDimitris Michailidis cmd_ctx->cpu = cpu;
396e1ffcc66SDimitris Michailidis memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
397e1ffcc66SDimitris Michailidis
398e1ffcc66SDimitris Michailidis dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
399e1ffcc66SDimitris Michailidis cmd);
400e1ffcc66SDimitris Michailidis
401e1ffcc66SDimitris Michailidis if (++funq->sq_tail == funq->sq_depth)
402e1ffcc66SDimitris Michailidis funq->sq_tail = 0;
403e1ffcc66SDimitris Michailidis writel(funq->sq_tail, funq->sq_db);
404e1ffcc66SDimitris Michailidis }
405e1ffcc66SDimitris Michailidis spin_unlock(&funq->sq_lock);
406e1ffcc66SDimitris Michailidis return rc;
407e1ffcc66SDimitris Michailidis }
408e1ffcc66SDimitris Michailidis
409e1ffcc66SDimitris Michailidis /* Abandon a pending admin command by clearing the issuer's callback data.
410e1ffcc66SDimitris Michailidis * Failure indicates that the command either has already completed or its
411e1ffcc66SDimitris Michailidis * completion is racing with this call.
412e1ffcc66SDimitris Michailidis */
fun_abandon_admin_cmd(struct fun_dev * fd,const struct fun_admin_req_common * cmd,void * cb_data)413e1ffcc66SDimitris Michailidis static bool fun_abandon_admin_cmd(struct fun_dev *fd,
414e1ffcc66SDimitris Michailidis const struct fun_admin_req_common *cmd,
415e1ffcc66SDimitris Michailidis void *cb_data)
416e1ffcc66SDimitris Michailidis {
417e1ffcc66SDimitris Michailidis u16 cid = be16_to_cpu(cmd->cid);
418e1ffcc66SDimitris Michailidis struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
419e1ffcc66SDimitris Michailidis
420e1ffcc66SDimitris Michailidis return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
421e1ffcc66SDimitris Michailidis }
422e1ffcc66SDimitris Michailidis
423e1ffcc66SDimitris Michailidis /* Stop submission of new admin commands and wake up any processes waiting for
424e1ffcc66SDimitris Michailidis * tags. Already submitted commands are left to complete or time out.
425e1ffcc66SDimitris Michailidis */
fun_admin_stop(struct fun_dev * fdev)426e1ffcc66SDimitris Michailidis static void fun_admin_stop(struct fun_dev *fdev)
427e1ffcc66SDimitris Michailidis {
428e1ffcc66SDimitris Michailidis spin_lock(&fdev->admin_q->sq_lock);
429e1ffcc66SDimitris Michailidis fdev->suppress_cmds = true;
430e1ffcc66SDimitris Michailidis spin_unlock(&fdev->admin_q->sq_lock);
431e1ffcc66SDimitris Michailidis sbitmap_queue_wake_all(&fdev->admin_sbq);
432e1ffcc66SDimitris Michailidis }
433e1ffcc66SDimitris Michailidis
434e1ffcc66SDimitris Michailidis /* The callback for synchronous execution of admin commands. It copies the
435e1ffcc66SDimitris Michailidis * command response to the caller's buffer and signals completion.
436e1ffcc66SDimitris Michailidis */
fun_admin_cmd_sync_cb(struct fun_dev * fd,void * rsp,void * cb_data)437e1ffcc66SDimitris Michailidis static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)
438e1ffcc66SDimitris Michailidis {
439e1ffcc66SDimitris Michailidis const struct fun_admin_rsp_common *rsp_common = rsp;
440e1ffcc66SDimitris Michailidis struct fun_sync_cmd_ctx *ctx = cb_data;
441e1ffcc66SDimitris Michailidis
442e1ffcc66SDimitris Michailidis if (!ctx)
443e1ffcc66SDimitris Michailidis return; /* command issuer timed out and left */
444e1ffcc66SDimitris Michailidis if (ctx->rsp_buf) {
445e1ffcc66SDimitris Michailidis unsigned int rsp_len = rsp_common->len8 * 8;
446e1ffcc66SDimitris Michailidis
447e1ffcc66SDimitris Michailidis if (unlikely(rsp_len > ctx->rsp_len)) {
448e1ffcc66SDimitris Michailidis dev_err(fd->dev,
449e1ffcc66SDimitris Michailidis "response for op %u is %uB > response buffer %uB\n",
450e1ffcc66SDimitris Michailidis rsp_common->op, rsp_len, ctx->rsp_len);
451e1ffcc66SDimitris Michailidis rsp_len = ctx->rsp_len;
452e1ffcc66SDimitris Michailidis }
453e1ffcc66SDimitris Michailidis memcpy(ctx->rsp_buf, rsp, rsp_len);
454e1ffcc66SDimitris Michailidis }
455e1ffcc66SDimitris Michailidis ctx->rsp_status = rsp_common->ret;
456e1ffcc66SDimitris Michailidis complete(&ctx->compl);
457e1ffcc66SDimitris Michailidis }
458e1ffcc66SDimitris Michailidis
459e1ffcc66SDimitris Michailidis /* Submit a synchronous admin command. */
fun_submit_admin_sync_cmd(struct fun_dev * fdev,struct fun_admin_req_common * cmd,void * rsp,size_t rspsize,unsigned int timeout)460e1ffcc66SDimitris Michailidis int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
461e1ffcc66SDimitris Michailidis struct fun_admin_req_common *cmd, void *rsp,
462e1ffcc66SDimitris Michailidis size_t rspsize, unsigned int timeout)
463e1ffcc66SDimitris Michailidis {
464e1ffcc66SDimitris Michailidis struct fun_sync_cmd_ctx ctx = {
465e1ffcc66SDimitris Michailidis .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
466e1ffcc66SDimitris Michailidis .rsp_buf = rsp,
467e1ffcc66SDimitris Michailidis .rsp_len = rspsize,
468e1ffcc66SDimitris Michailidis };
469e1ffcc66SDimitris Michailidis unsigned int cmdlen = cmd->len8 * 8;
470e1ffcc66SDimitris Michailidis unsigned long jiffies_left;
471e1ffcc66SDimitris Michailidis int ret;
472e1ffcc66SDimitris Michailidis
473e1ffcc66SDimitris Michailidis ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
474e1ffcc66SDimitris Michailidis true);
475e1ffcc66SDimitris Michailidis if (ret)
476e1ffcc66SDimitris Michailidis return ret;
477e1ffcc66SDimitris Michailidis
478e1ffcc66SDimitris Michailidis if (!timeout)
479e1ffcc66SDimitris Michailidis timeout = FUN_ADMIN_CMD_TO_MS;
480e1ffcc66SDimitris Michailidis
481e1ffcc66SDimitris Michailidis jiffies_left = wait_for_completion_timeout(&ctx.compl,
482e1ffcc66SDimitris Michailidis msecs_to_jiffies(timeout));
483e1ffcc66SDimitris Michailidis if (!jiffies_left) {
484e1ffcc66SDimitris Michailidis /* The command timed out. Attempt to cancel it so we can return.
485e1ffcc66SDimitris Michailidis * But if the command is in the process of completing we'll
486e1ffcc66SDimitris Michailidis * wait for it.
487e1ffcc66SDimitris Michailidis */
488e1ffcc66SDimitris Michailidis if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
489e1ffcc66SDimitris Michailidis dev_err(fdev->dev, "admin command timed out: %*ph\n",
490e1ffcc66SDimitris Michailidis cmdlen, cmd);
491e1ffcc66SDimitris Michailidis fun_admin_stop(fdev);
492e1ffcc66SDimitris Michailidis /* see if the timeout was due to a queue failure */
493e1ffcc66SDimitris Michailidis if (fun_adminq_stopped(fdev))
494e1ffcc66SDimitris Michailidis dev_err(fdev->dev,
495e1ffcc66SDimitris Michailidis "device does not accept admin commands\n");
496e1ffcc66SDimitris Michailidis
497e1ffcc66SDimitris Michailidis return -ETIMEDOUT;
498e1ffcc66SDimitris Michailidis }
499e1ffcc66SDimitris Michailidis wait_for_completion(&ctx.compl);
500e1ffcc66SDimitris Michailidis }
501e1ffcc66SDimitris Michailidis
502e1ffcc66SDimitris Michailidis if (ctx.rsp_status) {
503e1ffcc66SDimitris Michailidis dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
504e1ffcc66SDimitris Michailidis ctx.rsp_status, cmdlen, cmd);
505e1ffcc66SDimitris Michailidis }
506e1ffcc66SDimitris Michailidis
507e1ffcc66SDimitris Michailidis return -ctx.rsp_status;
508e1ffcc66SDimitris Michailidis }
509e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
510e1ffcc66SDimitris Michailidis
511e1ffcc66SDimitris Michailidis /* Return the number of device resources of the requested type. */
fun_get_res_count(struct fun_dev * fdev,enum fun_admin_op res)512e1ffcc66SDimitris Michailidis int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
513e1ffcc66SDimitris Michailidis {
514e1ffcc66SDimitris Michailidis union {
515e1ffcc66SDimitris Michailidis struct fun_admin_res_count_req req;
516e1ffcc66SDimitris Michailidis struct fun_admin_res_count_rsp rsp;
517e1ffcc66SDimitris Michailidis } cmd;
518e1ffcc66SDimitris Michailidis int rc;
519e1ffcc66SDimitris Michailidis
520e1ffcc66SDimitris Michailidis cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
521e1ffcc66SDimitris Michailidis cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
522e1ffcc66SDimitris Michailidis 0, 0);
523e1ffcc66SDimitris Michailidis
524e1ffcc66SDimitris Michailidis rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
525e1ffcc66SDimitris Michailidis sizeof(cmd), 0);
526e1ffcc66SDimitris Michailidis return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
527e1ffcc66SDimitris Michailidis }
528e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_get_res_count);
529e1ffcc66SDimitris Michailidis
530e1ffcc66SDimitris Michailidis /* Request that the instance of resource @res with the given id be deleted. */
fun_res_destroy(struct fun_dev * fdev,enum fun_admin_op res,unsigned int flags,u32 id)531e1ffcc66SDimitris Michailidis int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
532e1ffcc66SDimitris Michailidis unsigned int flags, u32 id)
533e1ffcc66SDimitris Michailidis {
534e1ffcc66SDimitris Michailidis struct fun_admin_generic_destroy_req req = {
535e1ffcc66SDimitris Michailidis .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
536e1ffcc66SDimitris Michailidis .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
537e1ffcc66SDimitris Michailidis flags, id)
538e1ffcc66SDimitris Michailidis };
539e1ffcc66SDimitris Michailidis
540e1ffcc66SDimitris Michailidis return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
541e1ffcc66SDimitris Michailidis }
542e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_res_destroy);
543e1ffcc66SDimitris Michailidis
544e1ffcc66SDimitris Michailidis /* Bind two entities of the given types and IDs. */
fun_bind(struct fun_dev * fdev,enum fun_admin_bind_type type0,unsigned int id0,enum fun_admin_bind_type type1,unsigned int id1)545e1ffcc66SDimitris Michailidis int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
546e1ffcc66SDimitris Michailidis unsigned int id0, enum fun_admin_bind_type type1,
547e1ffcc66SDimitris Michailidis unsigned int id1)
548e1ffcc66SDimitris Michailidis {
549e1ffcc66SDimitris Michailidis struct {
550e1ffcc66SDimitris Michailidis struct fun_admin_bind_req req;
551e1ffcc66SDimitris Michailidis struct fun_admin_bind_entry entry[2];
552e1ffcc66SDimitris Michailidis } cmd = {
553e1ffcc66SDimitris Michailidis .req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
554e1ffcc66SDimitris Michailidis sizeof(cmd)),
555e1ffcc66SDimitris Michailidis .entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),
556e1ffcc66SDimitris Michailidis .entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),
557e1ffcc66SDimitris Michailidis };
558e1ffcc66SDimitris Michailidis
559e1ffcc66SDimitris Michailidis return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);
560e1ffcc66SDimitris Michailidis }
561e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_bind);
562e1ffcc66SDimitris Michailidis
fun_get_dev_limits(struct fun_dev * fdev)563e1ffcc66SDimitris Michailidis static int fun_get_dev_limits(struct fun_dev *fdev)
564e1ffcc66SDimitris Michailidis {
565e1ffcc66SDimitris Michailidis struct pci_dev *pdev = to_pci_dev(fdev->dev);
566e1ffcc66SDimitris Michailidis unsigned int cq_count, sq_count, num_dbs;
567e1ffcc66SDimitris Michailidis int rc;
568e1ffcc66SDimitris Michailidis
569e1ffcc66SDimitris Michailidis rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
570e1ffcc66SDimitris Michailidis if (rc < 0)
571e1ffcc66SDimitris Michailidis return rc;
572e1ffcc66SDimitris Michailidis cq_count = rc;
573e1ffcc66SDimitris Michailidis
574e1ffcc66SDimitris Michailidis rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
575e1ffcc66SDimitris Michailidis if (rc < 0)
576e1ffcc66SDimitris Michailidis return rc;
577e1ffcc66SDimitris Michailidis sq_count = rc;
578e1ffcc66SDimitris Michailidis
579e1ffcc66SDimitris Michailidis /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
580e1ffcc66SDimitris Michailidis * device must provide additional queues.
581e1ffcc66SDimitris Michailidis */
582e1ffcc66SDimitris Michailidis if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)
583e1ffcc66SDimitris Michailidis return -EINVAL;
584e1ffcc66SDimitris Michailidis
585e1ffcc66SDimitris Michailidis /* Calculate the max QID based on SQ/CQ/doorbell counts.
586e1ffcc66SDimitris Michailidis * SQ/CQ doorbells alternate.
587e1ffcc66SDimitris Michailidis */
588*31ac3bceSDimitris Michailidis num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >>
589*31ac3bceSDimitris Michailidis (2 + NVME_CAP_STRIDE(fdev->cap_reg));
590e1ffcc66SDimitris Michailidis fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
591e1ffcc66SDimitris Michailidis fdev->kern_end_qid = fdev->max_qid + 1;
592e1ffcc66SDimitris Michailidis return 0;
593e1ffcc66SDimitris Michailidis }
594e1ffcc66SDimitris Michailidis
595e1ffcc66SDimitris Michailidis /* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
fun_alloc_irqs(struct pci_dev * pdev,unsigned int min_vecs)596e1ffcc66SDimitris Michailidis static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
597e1ffcc66SDimitris Michailidis {
598e1ffcc66SDimitris Michailidis int vecs, num_msix = pci_msix_vec_count(pdev);
599e1ffcc66SDimitris Michailidis
600e1ffcc66SDimitris Michailidis if (num_msix < 0)
601e1ffcc66SDimitris Michailidis return num_msix;
602e1ffcc66SDimitris Michailidis if (min_vecs > num_msix)
603e1ffcc66SDimitris Michailidis return -ERANGE;
604e1ffcc66SDimitris Michailidis
605e1ffcc66SDimitris Michailidis vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
606e1ffcc66SDimitris Michailidis if (vecs > 0) {
607e1ffcc66SDimitris Michailidis dev_info(&pdev->dev,
608e1ffcc66SDimitris Michailidis "Allocated %d IRQ vectors of %d requested\n",
609e1ffcc66SDimitris Michailidis vecs, num_msix);
610e1ffcc66SDimitris Michailidis } else {
611e1ffcc66SDimitris Michailidis dev_err(&pdev->dev,
612e1ffcc66SDimitris Michailidis "Unable to allocate at least %u IRQ vectors\n",
613e1ffcc66SDimitris Michailidis min_vecs);
614e1ffcc66SDimitris Michailidis }
615e1ffcc66SDimitris Michailidis return vecs;
616e1ffcc66SDimitris Michailidis }
617e1ffcc66SDimitris Michailidis
618e1ffcc66SDimitris Michailidis /* Allocate and initialize the IRQ manager state. */
fun_alloc_irq_mgr(struct fun_dev * fdev)619e1ffcc66SDimitris Michailidis static int fun_alloc_irq_mgr(struct fun_dev *fdev)
620e1ffcc66SDimitris Michailidis {
621e1ffcc66SDimitris Michailidis fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
622e1ffcc66SDimitris Michailidis if (!fdev->irq_map)
623e1ffcc66SDimitris Michailidis return -ENOMEM;
624e1ffcc66SDimitris Michailidis
625e1ffcc66SDimitris Michailidis spin_lock_init(&fdev->irqmgr_lock);
626e1ffcc66SDimitris Michailidis /* mark IRQ 0 allocated, it is used by the admin queue */
627e1ffcc66SDimitris Michailidis __set_bit(0, fdev->irq_map);
628e1ffcc66SDimitris Michailidis fdev->irqs_avail = fdev->num_irqs - 1;
629e1ffcc66SDimitris Michailidis return 0;
630e1ffcc66SDimitris Michailidis }
631e1ffcc66SDimitris Michailidis
632e1ffcc66SDimitris Michailidis /* Reserve @nirqs of the currently available IRQs and return their indices. */
fun_reserve_irqs(struct fun_dev * fdev,unsigned int nirqs,u16 * irq_indices)633e1ffcc66SDimitris Michailidis int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)
634e1ffcc66SDimitris Michailidis {
635e1ffcc66SDimitris Michailidis unsigned int b, n = 0;
636e1ffcc66SDimitris Michailidis int err = -ENOSPC;
637e1ffcc66SDimitris Michailidis
638e1ffcc66SDimitris Michailidis if (!nirqs)
639e1ffcc66SDimitris Michailidis return 0;
640e1ffcc66SDimitris Michailidis
641e1ffcc66SDimitris Michailidis spin_lock(&fdev->irqmgr_lock);
642e1ffcc66SDimitris Michailidis if (nirqs > fdev->irqs_avail)
643e1ffcc66SDimitris Michailidis goto unlock;
644e1ffcc66SDimitris Michailidis
645e1ffcc66SDimitris Michailidis for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
646e1ffcc66SDimitris Michailidis __set_bit(b, fdev->irq_map);
647e1ffcc66SDimitris Michailidis irq_indices[n++] = b;
648e1ffcc66SDimitris Michailidis if (n >= nirqs)
649e1ffcc66SDimitris Michailidis break;
650e1ffcc66SDimitris Michailidis }
651e1ffcc66SDimitris Michailidis
652e1ffcc66SDimitris Michailidis WARN_ON(n < nirqs);
653e1ffcc66SDimitris Michailidis fdev->irqs_avail -= n;
654e1ffcc66SDimitris Michailidis err = n;
655e1ffcc66SDimitris Michailidis unlock:
656e1ffcc66SDimitris Michailidis spin_unlock(&fdev->irqmgr_lock);
657e1ffcc66SDimitris Michailidis return err;
658e1ffcc66SDimitris Michailidis }
659e1ffcc66SDimitris Michailidis EXPORT_SYMBOL(fun_reserve_irqs);
660e1ffcc66SDimitris Michailidis
661e1ffcc66SDimitris Michailidis /* Release @nirqs previously allocated IRQS with the supplied indices. */
fun_release_irqs(struct fun_dev * fdev,unsigned int nirqs,u16 * irq_indices)662e1ffcc66SDimitris Michailidis void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
663e1ffcc66SDimitris Michailidis u16 *irq_indices)
664e1ffcc66SDimitris Michailidis {
665e1ffcc66SDimitris Michailidis unsigned int i;
666e1ffcc66SDimitris Michailidis
667e1ffcc66SDimitris Michailidis spin_lock(&fdev->irqmgr_lock);
668e1ffcc66SDimitris Michailidis for (i = 0; i < nirqs; i++)
669e1ffcc66SDimitris Michailidis __clear_bit(irq_indices[i], fdev->irq_map);
670e1ffcc66SDimitris Michailidis fdev->irqs_avail += nirqs;
671e1ffcc66SDimitris Michailidis spin_unlock(&fdev->irqmgr_lock);
672e1ffcc66SDimitris Michailidis }
673e1ffcc66SDimitris Michailidis EXPORT_SYMBOL(fun_release_irqs);
674e1ffcc66SDimitris Michailidis
fun_serv_handler(struct work_struct * work)675e1ffcc66SDimitris Michailidis static void fun_serv_handler(struct work_struct *work)
676e1ffcc66SDimitris Michailidis {
677e1ffcc66SDimitris Michailidis struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
678e1ffcc66SDimitris Michailidis
679e1ffcc66SDimitris Michailidis if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
680e1ffcc66SDimitris Michailidis return;
681e1ffcc66SDimitris Michailidis if (fd->serv_cb)
682e1ffcc66SDimitris Michailidis fd->serv_cb(fd);
683e1ffcc66SDimitris Michailidis }
684e1ffcc66SDimitris Michailidis
fun_serv_stop(struct fun_dev * fd)685e1ffcc66SDimitris Michailidis void fun_serv_stop(struct fun_dev *fd)
686e1ffcc66SDimitris Michailidis {
687e1ffcc66SDimitris Michailidis set_bit(FUN_SERV_DISABLED, &fd->service_flags);
688e1ffcc66SDimitris Michailidis cancel_work_sync(&fd->service_task);
689e1ffcc66SDimitris Michailidis }
690e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_serv_stop);
691e1ffcc66SDimitris Michailidis
fun_serv_restart(struct fun_dev * fd)692e1ffcc66SDimitris Michailidis void fun_serv_restart(struct fun_dev *fd)
693e1ffcc66SDimitris Michailidis {
694e1ffcc66SDimitris Michailidis clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
695e1ffcc66SDimitris Michailidis if (fd->service_flags)
696e1ffcc66SDimitris Michailidis schedule_work(&fd->service_task);
697e1ffcc66SDimitris Michailidis }
698e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_serv_restart);
699e1ffcc66SDimitris Michailidis
fun_serv_sched(struct fun_dev * fd)700e1ffcc66SDimitris Michailidis void fun_serv_sched(struct fun_dev *fd)
701e1ffcc66SDimitris Michailidis {
702e1ffcc66SDimitris Michailidis if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
703e1ffcc66SDimitris Michailidis schedule_work(&fd->service_task);
704e1ffcc66SDimitris Michailidis }
705e1ffcc66SDimitris Michailidis EXPORT_SYMBOL_GPL(fun_serv_sched);
706e1ffcc66SDimitris Michailidis
707e1ffcc66SDimitris Michailidis /* Check and try to get the device into a proper state for initialization,
708e1ffcc66SDimitris Michailidis * i.e., CSTS.RDY = CC.EN = 0.
709e1ffcc66SDimitris Michailidis */
sanitize_dev(struct fun_dev * fdev)710e1ffcc66SDimitris Michailidis static int sanitize_dev(struct fun_dev *fdev)
711e1ffcc66SDimitris Michailidis {
712e1ffcc66SDimitris Michailidis int rc;
713e1ffcc66SDimitris Michailidis
714e1ffcc66SDimitris Michailidis fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
715e1ffcc66SDimitris Michailidis fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);
716e1ffcc66SDimitris Michailidis
717e1ffcc66SDimitris Michailidis /* First get RDY to agree with the current EN. Give RDY the opportunity
718e1ffcc66SDimitris Michailidis * to complete a potential recent EN change.
719e1ffcc66SDimitris Michailidis */
720e1ffcc66SDimitris Michailidis rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
721e1ffcc66SDimitris Michailidis if (rc)
722e1ffcc66SDimitris Michailidis return rc;
723e1ffcc66SDimitris Michailidis
724e1ffcc66SDimitris Michailidis /* Next, reset the device if EN is currently 1. */
725e1ffcc66SDimitris Michailidis if (fdev->cc_reg & NVME_CC_ENABLE)
726e1ffcc66SDimitris Michailidis rc = fun_disable_ctrl(fdev);
727e1ffcc66SDimitris Michailidis
728e1ffcc66SDimitris Michailidis return rc;
729e1ffcc66SDimitris Michailidis }
730e1ffcc66SDimitris Michailidis
731e1ffcc66SDimitris Michailidis /* Undo the device initialization of fun_dev_enable(). */
fun_dev_disable(struct fun_dev * fdev)732e1ffcc66SDimitris Michailidis void fun_dev_disable(struct fun_dev *fdev)
733e1ffcc66SDimitris Michailidis {
734e1ffcc66SDimitris Michailidis struct pci_dev *pdev = to_pci_dev(fdev->dev);
735e1ffcc66SDimitris Michailidis
736e1ffcc66SDimitris Michailidis pci_set_drvdata(pdev, NULL);
737e1ffcc66SDimitris Michailidis
738e1ffcc66SDimitris Michailidis if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
739e1ffcc66SDimitris Michailidis fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
740e1ffcc66SDimitris Michailidis fdev->fw_handle);
741e1ffcc66SDimitris Michailidis fdev->fw_handle = FUN_HCI_ID_INVALID;
742e1ffcc66SDimitris Michailidis }
743e1ffcc66SDimitris Michailidis
744e1ffcc66SDimitris Michailidis fun_disable_admin_queue(fdev);
745e1ffcc66SDimitris Michailidis
746e1ffcc66SDimitris Michailidis bitmap_free(fdev->irq_map);
747e1ffcc66SDimitris Michailidis pci_free_irq_vectors(pdev);
748e1ffcc66SDimitris Michailidis
749e1ffcc66SDimitris Michailidis pci_disable_device(pdev);
750e1ffcc66SDimitris Michailidis
751e1ffcc66SDimitris Michailidis fun_unmap_bars(fdev);
752e1ffcc66SDimitris Michailidis }
753e1ffcc66SDimitris Michailidis EXPORT_SYMBOL(fun_dev_disable);
754e1ffcc66SDimitris Michailidis
755e1ffcc66SDimitris Michailidis /* Perform basic initialization of a device, including
756e1ffcc66SDimitris Michailidis * - PCI config space setup and BAR0 mapping
757e1ffcc66SDimitris Michailidis * - interrupt management initialization
758e1ffcc66SDimitris Michailidis * - 1 admin queue setup
759e1ffcc66SDimitris Michailidis * - determination of some device limits, such as number of queues.
760e1ffcc66SDimitris Michailidis */
fun_dev_enable(struct fun_dev * fdev,struct pci_dev * pdev,const struct fun_dev_params * areq,const char * name)761e1ffcc66SDimitris Michailidis int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
762e1ffcc66SDimitris Michailidis const struct fun_dev_params *areq, const char *name)
763e1ffcc66SDimitris Michailidis {
764e1ffcc66SDimitris Michailidis int rc;
765e1ffcc66SDimitris Michailidis
766e1ffcc66SDimitris Michailidis fdev->dev = &pdev->dev;
767e1ffcc66SDimitris Michailidis rc = fun_map_bars(fdev, name);
768e1ffcc66SDimitris Michailidis if (rc)
769e1ffcc66SDimitris Michailidis return rc;
770e1ffcc66SDimitris Michailidis
771e1ffcc66SDimitris Michailidis rc = fun_set_dma_masks(fdev->dev);
772e1ffcc66SDimitris Michailidis if (rc)
773e1ffcc66SDimitris Michailidis goto unmap;
774e1ffcc66SDimitris Michailidis
775e1ffcc66SDimitris Michailidis rc = pci_enable_device_mem(pdev);
776e1ffcc66SDimitris Michailidis if (rc) {
777e1ffcc66SDimitris Michailidis dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
778e1ffcc66SDimitris Michailidis goto unmap;
779e1ffcc66SDimitris Michailidis }
780e1ffcc66SDimitris Michailidis
781e1ffcc66SDimitris Michailidis rc = sanitize_dev(fdev);
782e1ffcc66SDimitris Michailidis if (rc)
783e1ffcc66SDimitris Michailidis goto disable_dev;
784e1ffcc66SDimitris Michailidis
785e1ffcc66SDimitris Michailidis fdev->fw_handle = FUN_HCI_ID_INVALID;
786e1ffcc66SDimitris Michailidis fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
787e1ffcc66SDimitris Michailidis fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
788e1ffcc66SDimitris Michailidis fdev->dbs = fdev->bar + NVME_REG_DBS;
789e1ffcc66SDimitris Michailidis
790e1ffcc66SDimitris Michailidis INIT_WORK(&fdev->service_task, fun_serv_handler);
791e1ffcc66SDimitris Michailidis fdev->service_flags = FUN_SERV_DISABLED;
792e1ffcc66SDimitris Michailidis fdev->serv_cb = areq->serv_cb;
793e1ffcc66SDimitris Michailidis
794e1ffcc66SDimitris Michailidis rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
795e1ffcc66SDimitris Michailidis if (rc < 0)
796e1ffcc66SDimitris Michailidis goto disable_dev;
797e1ffcc66SDimitris Michailidis fdev->num_irqs = rc;
798e1ffcc66SDimitris Michailidis
799e1ffcc66SDimitris Michailidis rc = fun_alloc_irq_mgr(fdev);
800e1ffcc66SDimitris Michailidis if (rc)
801e1ffcc66SDimitris Michailidis goto free_irqs;
802e1ffcc66SDimitris Michailidis
803e1ffcc66SDimitris Michailidis pci_set_master(pdev);
804e1ffcc66SDimitris Michailidis rc = fun_enable_admin_queue(fdev, areq);
805e1ffcc66SDimitris Michailidis if (rc)
806e1ffcc66SDimitris Michailidis goto free_irq_mgr;
807e1ffcc66SDimitris Michailidis
808e1ffcc66SDimitris Michailidis rc = fun_get_dev_limits(fdev);
809e1ffcc66SDimitris Michailidis if (rc < 0)
810e1ffcc66SDimitris Michailidis goto disable_admin;
811e1ffcc66SDimitris Michailidis
812e1ffcc66SDimitris Michailidis pci_save_state(pdev);
813e1ffcc66SDimitris Michailidis pci_set_drvdata(pdev, fdev);
814e1ffcc66SDimitris Michailidis pcie_print_link_status(pdev);
815e1ffcc66SDimitris Michailidis dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
816e1ffcc66SDimitris Michailidis fdev->q_depth, fdev->db_stride, fdev->max_qid,
817e1ffcc66SDimitris Michailidis fdev->kern_end_qid);
818e1ffcc66SDimitris Michailidis return 0;
819e1ffcc66SDimitris Michailidis
820e1ffcc66SDimitris Michailidis disable_admin:
821e1ffcc66SDimitris Michailidis fun_disable_admin_queue(fdev);
822e1ffcc66SDimitris Michailidis free_irq_mgr:
823e1ffcc66SDimitris Michailidis bitmap_free(fdev->irq_map);
824e1ffcc66SDimitris Michailidis free_irqs:
825e1ffcc66SDimitris Michailidis pci_free_irq_vectors(pdev);
826e1ffcc66SDimitris Michailidis disable_dev:
827e1ffcc66SDimitris Michailidis pci_disable_device(pdev);
828e1ffcc66SDimitris Michailidis unmap:
829e1ffcc66SDimitris Michailidis fun_unmap_bars(fdev);
830e1ffcc66SDimitris Michailidis return rc;
831e1ffcc66SDimitris Michailidis }
832e1ffcc66SDimitris Michailidis EXPORT_SYMBOL(fun_dev_enable);
833e1ffcc66SDimitris Michailidis
834e1ffcc66SDimitris Michailidis MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
835e1ffcc66SDimitris Michailidis MODULE_DESCRIPTION("Core services driver for Fungible devices");
836e1ffcc66SDimitris Michailidis MODULE_LICENSE("Dual BSD/GPL");
837