15f37396dSChristoph Hellwig // SPDX-License-Identifier: GPL-2.0
257dacad5SJay Sternberg /*
357dacad5SJay Sternberg * NVM Express device driver
457dacad5SJay Sternberg * Copyright (c) 2011-2014, Intel Corporation.
557dacad5SJay Sternberg */
657dacad5SJay Sternberg
7df4f9bc4SDavid E. Box #include <linux/acpi.h>
818119775SKeith Busch #include <linux/async.h>
957dacad5SJay Sternberg #include <linux/blkdev.h>
1057dacad5SJay Sternberg #include <linux/blk-mq.h>
11dca51e78SChristoph Hellwig #include <linux/blk-mq-pci.h>
12fe45e630SChristoph Hellwig #include <linux/blk-integrity.h>
13ff5350a8SAndy Lutomirski #include <linux/dmi.h>
1457dacad5SJay Sternberg #include <linux/init.h>
1557dacad5SJay Sternberg #include <linux/interrupt.h>
1657dacad5SJay Sternberg #include <linux/io.h>
1799722c8aSChristophe JAILLET #include <linux/kstrtox.h>
18dc90f084SChristoph Hellwig #include <linux/memremap.h>
1957dacad5SJay Sternberg #include <linux/mm.h>
2057dacad5SJay Sternberg #include <linux/module.h>
2177bf25eaSKeith Busch #include <linux/mutex.h>
22d0877473SKeith Busch #include <linux/once.h>
2357dacad5SJay Sternberg #include <linux/pci.h>
24d916b1beSKeith Busch #include <linux/suspend.h>
2557dacad5SJay Sternberg #include <linux/t10-pi.h>
2657dacad5SJay Sternberg #include <linux/types.h>
279cf5c095SLinus Torvalds #include <linux/io-64-nonatomic-lo-hi.h>
2820d3bb92SKlaus Jensen #include <linux/io-64-nonatomic-hi-lo.h>
29a98e58e5SScott Bauer #include <linux/sed-opal.h>
300f238ff5SLogan Gunthorpe #include <linux/pci-p2pdma.h>
3157dacad5SJay Sternberg
32604c01d5Syupeng #include "trace.h"
3357dacad5SJay Sternberg #include "nvme.h"
3457dacad5SJay Sternberg
35c1e0cc7eSBenjamin Herrenschmidt #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes)
368a1d09a6SBenjamin Herrenschmidt #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion))
3757dacad5SJay Sternberg
3884173423SKeith Busch #define SGES_PER_PAGE (NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc))
39adf68f21SChristoph Hellwig
40943e942eSJens Axboe /*
41943e942eSJens Axboe * These can be higher, but we need to ensure that any command doesn't
42943e942eSJens Axboe * require an sg allocation that needs more than a page of data.
43943e942eSJens Axboe */
447846c1b5SKeith Busch #define NVME_MAX_KB_SZ 8192
457846c1b5SKeith Busch #define NVME_MAX_SEGS 128
467846c1b5SKeith Busch #define NVME_MAX_NR_ALLOCATIONS 5
47943e942eSJens Axboe
4857dacad5SJay Sternberg static int use_threaded_interrupts;
492e21e445SXin Hao module_param(use_threaded_interrupts, int, 0444);
5057dacad5SJay Sternberg
5157dacad5SJay Sternberg static bool use_cmb_sqes = true;
5269f4eb9fSKeith Busch module_param(use_cmb_sqes, bool, 0444);
5357dacad5SJay Sternberg MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
5457dacad5SJay Sternberg
5587ad72a5SChristoph Hellwig static unsigned int max_host_mem_size_mb = 128;
5687ad72a5SChristoph Hellwig module_param(max_host_mem_size_mb, uint, 0444);
5787ad72a5SChristoph Hellwig MODULE_PARM_DESC(max_host_mem_size_mb,
5887ad72a5SChristoph Hellwig "Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
5957dacad5SJay Sternberg
60a7a7cbe3SChaitanya Kulkarni static unsigned int sgl_threshold = SZ_32K;
61a7a7cbe3SChaitanya Kulkarni module_param(sgl_threshold, uint, 0644);
62a7a7cbe3SChaitanya Kulkarni MODULE_PARM_DESC(sgl_threshold,
63a7a7cbe3SChaitanya Kulkarni "Use SGLs when average request segment size is larger or equal to "
64a7a7cbe3SChaitanya Kulkarni "this size. Use 0 to disable SGLs.");
65a7a7cbe3SChaitanya Kulkarni
6627453b45SSagi Grimberg #define NVME_PCI_MIN_QUEUE_SIZE 2
6727453b45SSagi Grimberg #define NVME_PCI_MAX_QUEUE_SIZE 4095
68b27c1e68Sweiping zhang static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
69b27c1e68Sweiping zhang static const struct kernel_param_ops io_queue_depth_ops = {
70b27c1e68Sweiping zhang .set = io_queue_depth_set,
7161f3b896SChaitanya Kulkarni .get = param_get_uint,
72b27c1e68Sweiping zhang };
73b27c1e68Sweiping zhang
7461f3b896SChaitanya Kulkarni static unsigned int io_queue_depth = 1024;
75b27c1e68Sweiping zhang module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
7627453b45SSagi Grimberg MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2 and < 4096");
77b27c1e68Sweiping zhang
io_queue_count_set(const char * val,const struct kernel_param * kp)789c9e76d5SWeiping Zhang static int io_queue_count_set(const char *val, const struct kernel_param *kp)
799c9e76d5SWeiping Zhang {
809c9e76d5SWeiping Zhang unsigned int n;
819c9e76d5SWeiping Zhang int ret;
829c9e76d5SWeiping Zhang
839c9e76d5SWeiping Zhang ret = kstrtouint(val, 10, &n);
849c9e76d5SWeiping Zhang if (ret != 0 || n > num_possible_cpus())
859c9e76d5SWeiping Zhang return -EINVAL;
869c9e76d5SWeiping Zhang return param_set_uint(val, kp);
879c9e76d5SWeiping Zhang }
889c9e76d5SWeiping Zhang
899c9e76d5SWeiping Zhang static const struct kernel_param_ops io_queue_count_ops = {
909c9e76d5SWeiping Zhang .set = io_queue_count_set,
919c9e76d5SWeiping Zhang .get = param_get_uint,
929c9e76d5SWeiping Zhang };
939c9e76d5SWeiping Zhang
943f68baf7SKeith Busch static unsigned int write_queues;
959c9e76d5SWeiping Zhang module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
963b6592f7SJens Axboe MODULE_PARM_DESC(write_queues,
973b6592f7SJens Axboe "Number of queues to use for writes. If not set, reads and writes "
983b6592f7SJens Axboe "will share a queue set.");
993b6592f7SJens Axboe
1003f68baf7SKeith Busch static unsigned int poll_queues;
1019c9e76d5SWeiping Zhang module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
1024b04cc6aSJens Axboe MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
1034b04cc6aSJens Axboe
104df4f9bc4SDavid E. Box static bool noacpi;
105df4f9bc4SDavid E. Box module_param(noacpi, bool, 0444);
106df4f9bc4SDavid E. Box MODULE_PARM_DESC(noacpi, "disable acpi bios quirks");
107df4f9bc4SDavid E. Box
1081c63dc66SChristoph Hellwig struct nvme_dev;
1091c63dc66SChristoph Hellwig struct nvme_queue;
11057dacad5SJay Sternberg
111a5cdb68cSKeith Busch static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
1127d879c90SChristoph Hellwig static void nvme_delete_io_queues(struct nvme_dev *dev);
113e917a849SKeith Busch static void nvme_update_attrs(struct nvme_dev *dev);
11457dacad5SJay Sternberg
11557dacad5SJay Sternberg /*
1161c63dc66SChristoph Hellwig * Represents an NVM Express device. Each nvme_dev is a PCI function.
1171c63dc66SChristoph Hellwig */
1181c63dc66SChristoph Hellwig struct nvme_dev {
119147b27e4SSagi Grimberg struct nvme_queue *queues;
1201c63dc66SChristoph Hellwig struct blk_mq_tag_set tagset;
1211c63dc66SChristoph Hellwig struct blk_mq_tag_set admin_tagset;
1221c63dc66SChristoph Hellwig u32 __iomem *dbs;
1231c63dc66SChristoph Hellwig struct device *dev;
1241c63dc66SChristoph Hellwig struct dma_pool *prp_page_pool;
1251c63dc66SChristoph Hellwig struct dma_pool *prp_small_pool;
1261c63dc66SChristoph Hellwig unsigned online_queues;
1271c63dc66SChristoph Hellwig unsigned max_qid;
128e20ba6e1SChristoph Hellwig unsigned io_queues[HCTX_MAX_TYPES];
12922b55601SKeith Busch unsigned int num_vecs;
1307442ddceSJohn Garry u32 q_depth;
131c1e0cc7eSBenjamin Herrenschmidt int io_sqes;
1321c63dc66SChristoph Hellwig u32 db_stride;
1331c63dc66SChristoph Hellwig void __iomem *bar;
13497f6ef64SXu Yu unsigned long bar_mapped_size;
13577bf25eaSKeith Busch struct mutex shutdown_lock;
1361c63dc66SChristoph Hellwig bool subsystem;
1371c63dc66SChristoph Hellwig u64 cmb_size;
1380f238ff5SLogan Gunthorpe bool cmb_use_sqes;
1391c63dc66SChristoph Hellwig u32 cmbsz;
140202021c1SStephen Bates u32 cmbloc;
1411c63dc66SChristoph Hellwig struct nvme_ctrl ctrl;
142d916b1beSKeith Busch u32 last_ps;
143a5df5e79SKeith Busch bool hmb;
14487ad72a5SChristoph Hellwig
145943e942eSJens Axboe mempool_t *iod_mempool;
146943e942eSJens Axboe
14787ad72a5SChristoph Hellwig /* shadow doorbell buffer support: */
148b5f96cb7SKlaus Jensen __le32 *dbbuf_dbs;
149f9f38e33SHelen Koike dma_addr_t dbbuf_dbs_dma_addr;
150b5f96cb7SKlaus Jensen __le32 *dbbuf_eis;
151f9f38e33SHelen Koike dma_addr_t dbbuf_eis_dma_addr;
15287ad72a5SChristoph Hellwig
15387ad72a5SChristoph Hellwig /* host memory buffer support: */
15487ad72a5SChristoph Hellwig u64 host_mem_size;
15587ad72a5SChristoph Hellwig u32 nr_host_mem_descs;
1564033f35dSChristoph Hellwig dma_addr_t host_mem_descs_dma;
15787ad72a5SChristoph Hellwig struct nvme_host_mem_buf_desc *host_mem_descs;
15887ad72a5SChristoph Hellwig void **host_mem_desc_bufs;
1592a5bcfddSWeiping Zhang unsigned int nr_allocated_queues;
1602a5bcfddSWeiping Zhang unsigned int nr_write_queues;
1612a5bcfddSWeiping Zhang unsigned int nr_poll_queues;
16257dacad5SJay Sternberg };
16357dacad5SJay Sternberg
io_queue_depth_set(const char * val,const struct kernel_param * kp)164b27c1e68Sweiping zhang static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
165b27c1e68Sweiping zhang {
16627453b45SSagi Grimberg return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE,
16727453b45SSagi Grimberg NVME_PCI_MAX_QUEUE_SIZE);
168b27c1e68Sweiping zhang }
169b27c1e68Sweiping zhang
sq_idx(unsigned int qid,u32 stride)170f9f38e33SHelen Koike static inline unsigned int sq_idx(unsigned int qid, u32 stride)
171f9f38e33SHelen Koike {
172f9f38e33SHelen Koike return qid * 2 * stride;
173f9f38e33SHelen Koike }
174f9f38e33SHelen Koike
cq_idx(unsigned int qid,u32 stride)175f9f38e33SHelen Koike static inline unsigned int cq_idx(unsigned int qid, u32 stride)
176f9f38e33SHelen Koike {
177f9f38e33SHelen Koike return (qid * 2 + 1) * stride;
178f9f38e33SHelen Koike }
179f9f38e33SHelen Koike
to_nvme_dev(struct nvme_ctrl * ctrl)1801c63dc66SChristoph Hellwig static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
1811c63dc66SChristoph Hellwig {
1821c63dc66SChristoph Hellwig return container_of(ctrl, struct nvme_dev, ctrl);
1831c63dc66SChristoph Hellwig }
1841c63dc66SChristoph Hellwig
18557dacad5SJay Sternberg /*
18657dacad5SJay Sternberg * An NVM Express queue. Each device has at least two (one for admin
18757dacad5SJay Sternberg * commands and one for I/O commands).
18857dacad5SJay Sternberg */
18957dacad5SJay Sternberg struct nvme_queue {
19057dacad5SJay Sternberg struct nvme_dev *dev;
1911ab0cd69SJens Axboe spinlock_t sq_lock;
192c1e0cc7eSBenjamin Herrenschmidt void *sq_cmds;
1933a7afd8eSChristoph Hellwig /* only used for poll queues: */
1943a7afd8eSChristoph Hellwig spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
19574943d45SKeith Busch struct nvme_completion *cqes;
19657dacad5SJay Sternberg dma_addr_t sq_dma_addr;
19757dacad5SJay Sternberg dma_addr_t cq_dma_addr;
19857dacad5SJay Sternberg u32 __iomem *q_db;
1997442ddceSJohn Garry u32 q_depth;
2007c349ddeSKeith Busch u16 cq_vector;
20157dacad5SJay Sternberg u16 sq_tail;
20238210800SKeith Busch u16 last_sq_tail;
20357dacad5SJay Sternberg u16 cq_head;
20457dacad5SJay Sternberg u16 qid;
20557dacad5SJay Sternberg u8 cq_phase;
206c1e0cc7eSBenjamin Herrenschmidt u8 sqes;
2074e224106SChristoph Hellwig unsigned long flags;
2084e224106SChristoph Hellwig #define NVMEQ_ENABLED 0
20963223078SChristoph Hellwig #define NVMEQ_SQ_CMB 1
210d1ed6aa1SChristoph Hellwig #define NVMEQ_DELETE_ERROR 2
2117c349ddeSKeith Busch #define NVMEQ_POLLED 3
212b5f96cb7SKlaus Jensen __le32 *dbbuf_sq_db;
213b5f96cb7SKlaus Jensen __le32 *dbbuf_cq_db;
214b5f96cb7SKlaus Jensen __le32 *dbbuf_sq_ei;
215b5f96cb7SKlaus Jensen __le32 *dbbuf_cq_ei;
216d1ed6aa1SChristoph Hellwig struct completion delete_done;
21757dacad5SJay Sternberg };
21857dacad5SJay Sternberg
2197846c1b5SKeith Busch union nvme_descriptor {
2207846c1b5SKeith Busch struct nvme_sgl_desc *sg_list;
2217846c1b5SKeith Busch __le64 *prp_list;
2227846c1b5SKeith Busch };
2237846c1b5SKeith Busch
22457dacad5SJay Sternberg /*
2259b048119SChristoph Hellwig * The nvme_iod describes the data in an I/O.
2269b048119SChristoph Hellwig *
2279b048119SChristoph Hellwig * The sg pointer contains the list of PRP/SGL chunk allocations in addition
2289b048119SChristoph Hellwig * to the actual struct scatterlist.
22971bd150cSChristoph Hellwig */
23071bd150cSChristoph Hellwig struct nvme_iod {
231d49187e9SChristoph Hellwig struct nvme_request req;
232af7fae85SKeith Busch struct nvme_command cmd;
23352da4f3fSKeith Busch bool aborted;
234c372cdd1SKeith Busch s8 nr_allocations; /* PRP list pool allocations. 0 means small
235c372cdd1SKeith Busch pool in use */
236dff824b2SChristoph Hellwig unsigned int dma_len; /* length of single DMA segment mapping */
237c4c22c52SKeith Busch dma_addr_t first_dma;
238783b94bdSChristoph Hellwig dma_addr_t meta_dma;
23991fb2b60SLogan Gunthorpe struct sg_table sgt;
2407846c1b5SKeith Busch union nvme_descriptor list[NVME_MAX_NR_ALLOCATIONS];
24157dacad5SJay Sternberg };
24257dacad5SJay Sternberg
nvme_dbbuf_size(struct nvme_dev * dev)2432a5bcfddSWeiping Zhang static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
2443b6592f7SJens Axboe {
2452a5bcfddSWeiping Zhang return dev->nr_allocated_queues * 8 * dev->db_stride;
246f9f38e33SHelen Koike }
247f9f38e33SHelen Koike
nvme_dbbuf_dma_alloc(struct nvme_dev * dev)24865a54646SChristoph Hellwig static void nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
249f9f38e33SHelen Koike {
2502a5bcfddSWeiping Zhang unsigned int mem_size = nvme_dbbuf_size(dev);
251f9f38e33SHelen Koike
25265a54646SChristoph Hellwig if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP))
25365a54646SChristoph Hellwig return;
25465a54646SChristoph Hellwig
25558847f12SKeith Busch if (dev->dbbuf_dbs) {
25658847f12SKeith Busch /*
25758847f12SKeith Busch * Clear the dbbuf memory so the driver doesn't observe stale
25858847f12SKeith Busch * values from the previous instantiation.
25958847f12SKeith Busch */
26058847f12SKeith Busch memset(dev->dbbuf_dbs, 0, mem_size);
26158847f12SKeith Busch memset(dev->dbbuf_eis, 0, mem_size);
26265a54646SChristoph Hellwig return;
26358847f12SKeith Busch }
264f9f38e33SHelen Koike
265f9f38e33SHelen Koike dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
266f9f38e33SHelen Koike &dev->dbbuf_dbs_dma_addr,
267f9f38e33SHelen Koike GFP_KERNEL);
268f9f38e33SHelen Koike if (!dev->dbbuf_dbs)
26965a54646SChristoph Hellwig goto fail;
270f9f38e33SHelen Koike dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
271f9f38e33SHelen Koike &dev->dbbuf_eis_dma_addr,
272f9f38e33SHelen Koike GFP_KERNEL);
27365a54646SChristoph Hellwig if (!dev->dbbuf_eis)
27465a54646SChristoph Hellwig goto fail_free_dbbuf_dbs;
27565a54646SChristoph Hellwig return;
276f9f38e33SHelen Koike
27765a54646SChristoph Hellwig fail_free_dbbuf_dbs:
27865a54646SChristoph Hellwig dma_free_coherent(dev->dev, mem_size, dev->dbbuf_dbs,
27965a54646SChristoph Hellwig dev->dbbuf_dbs_dma_addr);
28065a54646SChristoph Hellwig dev->dbbuf_dbs = NULL;
28165a54646SChristoph Hellwig fail:
28265a54646SChristoph Hellwig dev_warn(dev->dev, "unable to allocate dma for dbbuf\n");
283f9f38e33SHelen Koike }
284f9f38e33SHelen Koike
nvme_dbbuf_dma_free(struct nvme_dev * dev)285f9f38e33SHelen Koike static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
286f9f38e33SHelen Koike {
2872a5bcfddSWeiping Zhang unsigned int mem_size = nvme_dbbuf_size(dev);
288f9f38e33SHelen Koike
289f9f38e33SHelen Koike if (dev->dbbuf_dbs) {
290f9f38e33SHelen Koike dma_free_coherent(dev->dev, mem_size,
291f9f38e33SHelen Koike dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
292f9f38e33SHelen Koike dev->dbbuf_dbs = NULL;
293f9f38e33SHelen Koike }
294f9f38e33SHelen Koike if (dev->dbbuf_eis) {
295f9f38e33SHelen Koike dma_free_coherent(dev->dev, mem_size,
296f9f38e33SHelen Koike dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
297f9f38e33SHelen Koike dev->dbbuf_eis = NULL;
298f9f38e33SHelen Koike }
299f9f38e33SHelen Koike }
300f9f38e33SHelen Koike
nvme_dbbuf_init(struct nvme_dev * dev,struct nvme_queue * nvmeq,int qid)301f9f38e33SHelen Koike static void nvme_dbbuf_init(struct nvme_dev *dev,
302f9f38e33SHelen Koike struct nvme_queue *nvmeq, int qid)
303f9f38e33SHelen Koike {
304f9f38e33SHelen Koike if (!dev->dbbuf_dbs || !qid)
305f9f38e33SHelen Koike return;
306f9f38e33SHelen Koike
307f9f38e33SHelen Koike nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
308f9f38e33SHelen Koike nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
309f9f38e33SHelen Koike nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
310f9f38e33SHelen Koike nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
311f9f38e33SHelen Koike }
312f9f38e33SHelen Koike
nvme_dbbuf_free(struct nvme_queue * nvmeq)3130f0d2c87SMinwoo Im static void nvme_dbbuf_free(struct nvme_queue *nvmeq)
3140f0d2c87SMinwoo Im {
3150f0d2c87SMinwoo Im if (!nvmeq->qid)
3160f0d2c87SMinwoo Im return;
3170f0d2c87SMinwoo Im
3180f0d2c87SMinwoo Im nvmeq->dbbuf_sq_db = NULL;
3190f0d2c87SMinwoo Im nvmeq->dbbuf_cq_db = NULL;
3200f0d2c87SMinwoo Im nvmeq->dbbuf_sq_ei = NULL;
3210f0d2c87SMinwoo Im nvmeq->dbbuf_cq_ei = NULL;
3220f0d2c87SMinwoo Im }
3230f0d2c87SMinwoo Im
nvme_dbbuf_set(struct nvme_dev * dev)324f9f38e33SHelen Koike static void nvme_dbbuf_set(struct nvme_dev *dev)
325f9f38e33SHelen Koike {
326f66e2804SChaitanya Kulkarni struct nvme_command c = { };
3270f0d2c87SMinwoo Im unsigned int i;
328f9f38e33SHelen Koike
329f9f38e33SHelen Koike if (!dev->dbbuf_dbs)
330f9f38e33SHelen Koike return;
331f9f38e33SHelen Koike
332f9f38e33SHelen Koike c.dbbuf.opcode = nvme_admin_dbbuf;
333f9f38e33SHelen Koike c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
334f9f38e33SHelen Koike c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
335f9f38e33SHelen Koike
336f9f38e33SHelen Koike if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
3379bdcfb10SChristoph Hellwig dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
338f9f38e33SHelen Koike /* Free memory and continue on */
339f9f38e33SHelen Koike nvme_dbbuf_dma_free(dev);
3400f0d2c87SMinwoo Im
3410f0d2c87SMinwoo Im for (i = 1; i <= dev->online_queues; i++)
3420f0d2c87SMinwoo Im nvme_dbbuf_free(&dev->queues[i]);
343f9f38e33SHelen Koike }
344f9f38e33SHelen Koike }
345f9f38e33SHelen Koike
nvme_dbbuf_need_event(u16 event_idx,u16 new_idx,u16 old)346f9f38e33SHelen Koike static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
347f9f38e33SHelen Koike {
348f9f38e33SHelen Koike return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
349f9f38e33SHelen Koike }
350f9f38e33SHelen Koike
351f9f38e33SHelen Koike /* Update dbbuf and return true if an MMIO is required */
nvme_dbbuf_update_and_check_event(u16 value,__le32 * dbbuf_db,volatile __le32 * dbbuf_ei)352b5f96cb7SKlaus Jensen static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db,
353b5f96cb7SKlaus Jensen volatile __le32 *dbbuf_ei)
354f9f38e33SHelen Koike {
355f9f38e33SHelen Koike if (dbbuf_db) {
356b5f96cb7SKlaus Jensen u16 old_value, event_idx;
357f9f38e33SHelen Koike
358f9f38e33SHelen Koike /*
359f9f38e33SHelen Koike * Ensure that the queue is written before updating
360f9f38e33SHelen Koike * the doorbell in memory
361f9f38e33SHelen Koike */
362f9f38e33SHelen Koike wmb();
363f9f38e33SHelen Koike
364b5f96cb7SKlaus Jensen old_value = le32_to_cpu(*dbbuf_db);
365b5f96cb7SKlaus Jensen *dbbuf_db = cpu_to_le32(value);
366f9f38e33SHelen Koike
367f1ed3df2SMichal Wnukowski /*
368f1ed3df2SMichal Wnukowski * Ensure that the doorbell is updated before reading the event
369f1ed3df2SMichal Wnukowski * index from memory. The controller needs to provide similar
370f1ed3df2SMichal Wnukowski * ordering to ensure the envent index is updated before reading
371f1ed3df2SMichal Wnukowski * the doorbell.
372f1ed3df2SMichal Wnukowski */
373f1ed3df2SMichal Wnukowski mb();
374f1ed3df2SMichal Wnukowski
375b5f96cb7SKlaus Jensen event_idx = le32_to_cpu(*dbbuf_ei);
376b5f96cb7SKlaus Jensen if (!nvme_dbbuf_need_event(event_idx, value, old_value))
377f9f38e33SHelen Koike return false;
378f9f38e33SHelen Koike }
379f9f38e33SHelen Koike
380f9f38e33SHelen Koike return true;
38157dacad5SJay Sternberg }
38257dacad5SJay Sternberg
38357dacad5SJay Sternberg /*
38457dacad5SJay Sternberg * Will slightly overestimate the number of pages needed. This is OK
38557dacad5SJay Sternberg * as it only leads to a small amount of wasted memory for the lifetime of
38657dacad5SJay Sternberg * the I/O.
38757dacad5SJay Sternberg */
nvme_pci_npages_prp(void)388b13c6393SChaitanya Kulkarni static int nvme_pci_npages_prp(void)
38957dacad5SJay Sternberg {
390c89a529eSKeith Busch unsigned max_bytes = (NVME_MAX_KB_SZ * 1024) + NVME_CTRL_PAGE_SIZE;
391c89a529eSKeith Busch unsigned nprps = DIV_ROUND_UP(max_bytes, NVME_CTRL_PAGE_SIZE);
39284173423SKeith Busch return DIV_ROUND_UP(8 * nprps, NVME_CTRL_PAGE_SIZE - 8);
39357dacad5SJay Sternberg }
39457dacad5SJay Sternberg
nvme_admin_init_hctx(struct blk_mq_hw_ctx * hctx,void * data,unsigned int hctx_idx)39557dacad5SJay Sternberg static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
39657dacad5SJay Sternberg unsigned int hctx_idx)
39757dacad5SJay Sternberg {
3980da7feaaSChristoph Hellwig struct nvme_dev *dev = to_nvme_dev(data);
399147b27e4SSagi Grimberg struct nvme_queue *nvmeq = &dev->queues[0];
40057dacad5SJay Sternberg
40157dacad5SJay Sternberg WARN_ON(hctx_idx != 0);
40257dacad5SJay Sternberg WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
40357dacad5SJay Sternberg
40457dacad5SJay Sternberg hctx->driver_data = nvmeq;
40557dacad5SJay Sternberg return 0;
40657dacad5SJay Sternberg }
40757dacad5SJay Sternberg
nvme_init_hctx(struct blk_mq_hw_ctx * hctx,void * data,unsigned int hctx_idx)40857dacad5SJay Sternberg static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
40957dacad5SJay Sternberg unsigned int hctx_idx)
41057dacad5SJay Sternberg {
4110da7feaaSChristoph Hellwig struct nvme_dev *dev = to_nvme_dev(data);
412147b27e4SSagi Grimberg struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
41357dacad5SJay Sternberg
41457dacad5SJay Sternberg WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
41557dacad5SJay Sternberg hctx->driver_data = nvmeq;
41657dacad5SJay Sternberg return 0;
41757dacad5SJay Sternberg }
41857dacad5SJay Sternberg
nvme_pci_init_request(struct blk_mq_tag_set * set,struct request * req,unsigned int hctx_idx,unsigned int numa_node)419e559398fSChristoph Hellwig static int nvme_pci_init_request(struct blk_mq_tag_set *set,
420e559398fSChristoph Hellwig struct request *req, unsigned int hctx_idx,
421e559398fSChristoph Hellwig unsigned int numa_node)
42257dacad5SJay Sternberg {
423f4800d6dSChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
42459e29ce6SSagi Grimberg
4254a4d9bc0SIrvin Cote nvme_req(req)->ctrl = set->driver_data;
426f4b9e6c9SKeith Busch nvme_req(req)->cmd = &iod->cmd;
42757dacad5SJay Sternberg return 0;
42857dacad5SJay Sternberg }
42957dacad5SJay Sternberg
queue_irq_offset(struct nvme_dev * dev)4303b6592f7SJens Axboe static int queue_irq_offset(struct nvme_dev *dev)
4313b6592f7SJens Axboe {
4323b6592f7SJens Axboe /* if we have more than 1 vec, admin queue offsets us by 1 */
4333b6592f7SJens Axboe if (dev->num_vecs > 1)
4343b6592f7SJens Axboe return 1;
4353b6592f7SJens Axboe
4363b6592f7SJens Axboe return 0;
4373b6592f7SJens Axboe }
4383b6592f7SJens Axboe
nvme_pci_map_queues(struct blk_mq_tag_set * set)439a4e1d0b7SBart Van Assche static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
440dca51e78SChristoph Hellwig {
4410da7feaaSChristoph Hellwig struct nvme_dev *dev = to_nvme_dev(set->driver_data);
4423b6592f7SJens Axboe int i, qoff, offset;
443dca51e78SChristoph Hellwig
4443b6592f7SJens Axboe offset = queue_irq_offset(dev);
4453b6592f7SJens Axboe for (i = 0, qoff = 0; i < set->nr_maps; i++) {
4463b6592f7SJens Axboe struct blk_mq_queue_map *map = &set->map[i];
4473b6592f7SJens Axboe
4483b6592f7SJens Axboe map->nr_queues = dev->io_queues[i];
4493b6592f7SJens Axboe if (!map->nr_queues) {
450e20ba6e1SChristoph Hellwig BUG_ON(i == HCTX_TYPE_DEFAULT);
4517e849dd9SChristoph Hellwig continue;
4523b6592f7SJens Axboe }
4533b6592f7SJens Axboe
4544b04cc6aSJens Axboe /*
4554b04cc6aSJens Axboe * The poll queue(s) doesn't have an IRQ (and hence IRQ
4564b04cc6aSJens Axboe * affinity), so use the regular blk-mq cpu mapping
4574b04cc6aSJens Axboe */
4583b6592f7SJens Axboe map->queue_offset = qoff;
459cb9e0e50SKeith Busch if (i != HCTX_TYPE_POLL && offset)
4603b6592f7SJens Axboe blk_mq_pci_map_queues(map, to_pci_dev(dev->dev), offset);
4614b04cc6aSJens Axboe else
4624b04cc6aSJens Axboe blk_mq_map_queues(map);
4633b6592f7SJens Axboe qoff += map->nr_queues;
4643b6592f7SJens Axboe offset += map->nr_queues;
4653b6592f7SJens Axboe }
466dca51e78SChristoph Hellwig }
467dca51e78SChristoph Hellwig
46838210800SKeith Busch /*
46938210800SKeith Busch * Write sq tail if we are asked to, or if the next command would wrap.
47038210800SKeith Busch */
nvme_write_sq_db(struct nvme_queue * nvmeq,bool write_sq)47138210800SKeith Busch static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)
47204f3eafdSJens Axboe {
47338210800SKeith Busch if (!write_sq) {
47438210800SKeith Busch u16 next_tail = nvmeq->sq_tail + 1;
47538210800SKeith Busch
47638210800SKeith Busch if (next_tail == nvmeq->q_depth)
47738210800SKeith Busch next_tail = 0;
47838210800SKeith Busch if (next_tail != nvmeq->last_sq_tail)
47938210800SKeith Busch return;
48038210800SKeith Busch }
48138210800SKeith Busch
48204f3eafdSJens Axboe if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
48304f3eafdSJens Axboe nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
48404f3eafdSJens Axboe writel(nvmeq->sq_tail, nvmeq->q_db);
48538210800SKeith Busch nvmeq->last_sq_tail = nvmeq->sq_tail;
48604f3eafdSJens Axboe }
48704f3eafdSJens Axboe
nvme_sq_copy_cmd(struct nvme_queue * nvmeq,struct nvme_command * cmd)4883233b94cSJens Axboe static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq,
4893233b94cSJens Axboe struct nvme_command *cmd)
49057dacad5SJay Sternberg {
491c1e0cc7eSBenjamin Herrenschmidt memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes),
4923233b94cSJens Axboe absolute_pointer(cmd), sizeof(*cmd));
49390ea5ca4SChristoph Hellwig if (++nvmeq->sq_tail == nvmeq->q_depth)
49490ea5ca4SChristoph Hellwig nvmeq->sq_tail = 0;
49504f3eafdSJens Axboe }
49604f3eafdSJens Axboe
nvme_commit_rqs(struct blk_mq_hw_ctx * hctx)49704f3eafdSJens Axboe static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
49804f3eafdSJens Axboe {
49904f3eafdSJens Axboe struct nvme_queue *nvmeq = hctx->driver_data;
50004f3eafdSJens Axboe
50104f3eafdSJens Axboe spin_lock(&nvmeq->sq_lock);
50238210800SKeith Busch if (nvmeq->sq_tail != nvmeq->last_sq_tail)
50338210800SKeith Busch nvme_write_sq_db(nvmeq, true);
50490ea5ca4SChristoph Hellwig spin_unlock(&nvmeq->sq_lock);
50557dacad5SJay Sternberg }
50657dacad5SJay Sternberg
nvme_pci_use_sgls(struct nvme_dev * dev,struct request * req,int nseg)507ae582935SKeith Busch static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req,
508ae582935SKeith Busch int nseg)
509955b1b5aSMinwoo Im {
510a53232cbSKeith Busch struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
511955b1b5aSMinwoo Im unsigned int avg_seg_size;
512955b1b5aSMinwoo Im
51320469a37SKeith Busch avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
514955b1b5aSMinwoo Im
515253a0b76SChaitanya Kulkarni if (!nvme_ctrl_sgl_supported(&dev->ctrl))
516955b1b5aSMinwoo Im return false;
517a53232cbSKeith Busch if (!nvmeq->qid)
518955b1b5aSMinwoo Im return false;
519955b1b5aSMinwoo Im if (!sgl_threshold || avg_seg_size < sgl_threshold)
520955b1b5aSMinwoo Im return false;
521955b1b5aSMinwoo Im return true;
522955b1b5aSMinwoo Im }
523955b1b5aSMinwoo Im
nvme_free_prps(struct nvme_dev * dev,struct request * req)5249275c206SChristoph Hellwig static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
52557dacad5SJay Sternberg {
5266c3c05b0SChaitanya Kulkarni const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
5279275c206SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
5289275c206SChristoph Hellwig dma_addr_t dma_addr = iod->first_dma;
52957dacad5SJay Sternberg int i;
53057dacad5SJay Sternberg
531c372cdd1SKeith Busch for (i = 0; i < iod->nr_allocations; i++) {
5327846c1b5SKeith Busch __le64 *prp_list = iod->list[i].prp_list;
5339275c206SChristoph Hellwig dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
5349275c206SChristoph Hellwig
5359275c206SChristoph Hellwig dma_pool_free(dev->prp_page_pool, prp_list, dma_addr);
5369275c206SChristoph Hellwig dma_addr = next_dma_addr;
537dff824b2SChristoph Hellwig }
5389275c206SChristoph Hellwig }
5399275c206SChristoph Hellwig
nvme_unmap_data(struct nvme_dev * dev,struct request * req)5409275c206SChristoph Hellwig static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
5419275c206SChristoph Hellwig {
5429275c206SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
5437fe07d14SChristoph Hellwig
5449275c206SChristoph Hellwig if (iod->dma_len) {
5459275c206SChristoph Hellwig dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
5469275c206SChristoph Hellwig rq_dma_dir(req));
5479275c206SChristoph Hellwig return;
5489275c206SChristoph Hellwig }
5499275c206SChristoph Hellwig
55091fb2b60SLogan Gunthorpe WARN_ON_ONCE(!iod->sgt.nents);
5519275c206SChristoph Hellwig
55291fb2b60SLogan Gunthorpe dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
55391fb2b60SLogan Gunthorpe
554c372cdd1SKeith Busch if (iod->nr_allocations == 0)
5557846c1b5SKeith Busch dma_pool_free(dev->prp_small_pool, iod->list[0].sg_list,
5569275c206SChristoph Hellwig iod->first_dma);
5578f0edf45SKeith Busch else if (iod->nr_allocations == 1)
5587846c1b5SKeith Busch dma_pool_free(dev->prp_page_pool, iod->list[0].sg_list,
55901df742dSKeith Busch iod->first_dma);
5609275c206SChristoph Hellwig else
5619275c206SChristoph Hellwig nvme_free_prps(dev, req);
56291fb2b60SLogan Gunthorpe mempool_free(iod->sgt.sgl, dev->iod_mempool);
56357dacad5SJay Sternberg }
56457dacad5SJay Sternberg
nvme_print_sgl(struct scatterlist * sgl,int nents)565d0877473SKeith Busch static void nvme_print_sgl(struct scatterlist *sgl, int nents)
566d0877473SKeith Busch {
567d0877473SKeith Busch int i;
568d0877473SKeith Busch struct scatterlist *sg;
569d0877473SKeith Busch
570d0877473SKeith Busch for_each_sg(sgl, sg, nents, i) {
571d0877473SKeith Busch dma_addr_t phys = sg_phys(sg);
572d0877473SKeith Busch pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
573d0877473SKeith Busch "dma_address:%pad dma_length:%d\n",
574d0877473SKeith Busch i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
575d0877473SKeith Busch sg_dma_len(sg));
576d0877473SKeith Busch }
577d0877473SKeith Busch }
578d0877473SKeith Busch
nvme_pci_setup_prps(struct nvme_dev * dev,struct request * req,struct nvme_rw_command * cmnd)579a7a7cbe3SChaitanya Kulkarni static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
580a7a7cbe3SChaitanya Kulkarni struct request *req, struct nvme_rw_command *cmnd)
58157dacad5SJay Sternberg {
582f4800d6dSChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
58357dacad5SJay Sternberg struct dma_pool *pool;
584b131c61dSChristoph Hellwig int length = blk_rq_payload_bytes(req);
58591fb2b60SLogan Gunthorpe struct scatterlist *sg = iod->sgt.sgl;
58657dacad5SJay Sternberg int dma_len = sg_dma_len(sg);
58757dacad5SJay Sternberg u64 dma_addr = sg_dma_address(sg);
5886c3c05b0SChaitanya Kulkarni int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
58957dacad5SJay Sternberg __le64 *prp_list;
59057dacad5SJay Sternberg dma_addr_t prp_dma;
59157dacad5SJay Sternberg int nprps, i;
59257dacad5SJay Sternberg
5936c3c05b0SChaitanya Kulkarni length -= (NVME_CTRL_PAGE_SIZE - offset);
5945228b328SJan H. Schönherr if (length <= 0) {
5955228b328SJan H. Schönherr iod->first_dma = 0;
596a7a7cbe3SChaitanya Kulkarni goto done;
5975228b328SJan H. Schönherr }
59857dacad5SJay Sternberg
5996c3c05b0SChaitanya Kulkarni dma_len -= (NVME_CTRL_PAGE_SIZE - offset);
60057dacad5SJay Sternberg if (dma_len) {
6016c3c05b0SChaitanya Kulkarni dma_addr += (NVME_CTRL_PAGE_SIZE - offset);
60257dacad5SJay Sternberg } else {
60357dacad5SJay Sternberg sg = sg_next(sg);
60457dacad5SJay Sternberg dma_addr = sg_dma_address(sg);
60557dacad5SJay Sternberg dma_len = sg_dma_len(sg);
60657dacad5SJay Sternberg }
60757dacad5SJay Sternberg
6086c3c05b0SChaitanya Kulkarni if (length <= NVME_CTRL_PAGE_SIZE) {
60957dacad5SJay Sternberg iod->first_dma = dma_addr;
610a7a7cbe3SChaitanya Kulkarni goto done;
61157dacad5SJay Sternberg }
61257dacad5SJay Sternberg
6136c3c05b0SChaitanya Kulkarni nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
61457dacad5SJay Sternberg if (nprps <= (256 / 8)) {
61557dacad5SJay Sternberg pool = dev->prp_small_pool;
616c372cdd1SKeith Busch iod->nr_allocations = 0;
61757dacad5SJay Sternberg } else {
61857dacad5SJay Sternberg pool = dev->prp_page_pool;
619c372cdd1SKeith Busch iod->nr_allocations = 1;
62057dacad5SJay Sternberg }
62157dacad5SJay Sternberg
62269d2b571SChristoph Hellwig prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
62357dacad5SJay Sternberg if (!prp_list) {
624c372cdd1SKeith Busch iod->nr_allocations = -1;
62586eea289SKeith Busch return BLK_STS_RESOURCE;
62657dacad5SJay Sternberg }
6277846c1b5SKeith Busch iod->list[0].prp_list = prp_list;
62857dacad5SJay Sternberg iod->first_dma = prp_dma;
62957dacad5SJay Sternberg i = 0;
63057dacad5SJay Sternberg for (;;) {
6316c3c05b0SChaitanya Kulkarni if (i == NVME_CTRL_PAGE_SIZE >> 3) {
63257dacad5SJay Sternberg __le64 *old_prp_list = prp_list;
63369d2b571SChristoph Hellwig prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
63457dacad5SJay Sternberg if (!prp_list)
635fa073216SChristoph Hellwig goto free_prps;
6367846c1b5SKeith Busch iod->list[iod->nr_allocations++].prp_list = prp_list;
63757dacad5SJay Sternberg prp_list[0] = old_prp_list[i - 1];
63857dacad5SJay Sternberg old_prp_list[i - 1] = cpu_to_le64(prp_dma);
63957dacad5SJay Sternberg i = 1;
64057dacad5SJay Sternberg }
64157dacad5SJay Sternberg prp_list[i++] = cpu_to_le64(dma_addr);
6426c3c05b0SChaitanya Kulkarni dma_len -= NVME_CTRL_PAGE_SIZE;
6436c3c05b0SChaitanya Kulkarni dma_addr += NVME_CTRL_PAGE_SIZE;
6446c3c05b0SChaitanya Kulkarni length -= NVME_CTRL_PAGE_SIZE;
64557dacad5SJay Sternberg if (length <= 0)
64657dacad5SJay Sternberg break;
64757dacad5SJay Sternberg if (dma_len > 0)
64857dacad5SJay Sternberg continue;
64986eea289SKeith Busch if (unlikely(dma_len < 0))
65086eea289SKeith Busch goto bad_sgl;
65157dacad5SJay Sternberg sg = sg_next(sg);
65257dacad5SJay Sternberg dma_addr = sg_dma_address(sg);
65357dacad5SJay Sternberg dma_len = sg_dma_len(sg);
65457dacad5SJay Sternberg }
655a7a7cbe3SChaitanya Kulkarni done:
65691fb2b60SLogan Gunthorpe cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl));
657a7a7cbe3SChaitanya Kulkarni cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
65886eea289SKeith Busch return BLK_STS_OK;
659fa073216SChristoph Hellwig free_prps:
660fa073216SChristoph Hellwig nvme_free_prps(dev, req);
661fa073216SChristoph Hellwig return BLK_STS_RESOURCE;
66286eea289SKeith Busch bad_sgl:
66391fb2b60SLogan Gunthorpe WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents),
664d0877473SKeith Busch "Invalid SGL for payload:%d nents:%d\n",
66591fb2b60SLogan Gunthorpe blk_rq_payload_bytes(req), iod->sgt.nents);
66686eea289SKeith Busch return BLK_STS_IOERR;
66757dacad5SJay Sternberg }
66857dacad5SJay Sternberg
nvme_pci_sgl_set_data(struct nvme_sgl_desc * sge,struct scatterlist * sg)669a7a7cbe3SChaitanya Kulkarni static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
670a7a7cbe3SChaitanya Kulkarni struct scatterlist *sg)
671a7a7cbe3SChaitanya Kulkarni {
672a7a7cbe3SChaitanya Kulkarni sge->addr = cpu_to_le64(sg_dma_address(sg));
673a7a7cbe3SChaitanya Kulkarni sge->length = cpu_to_le32(sg_dma_len(sg));
674a7a7cbe3SChaitanya Kulkarni sge->type = NVME_SGL_FMT_DATA_DESC << 4;
675a7a7cbe3SChaitanya Kulkarni }
676a7a7cbe3SChaitanya Kulkarni
nvme_pci_sgl_set_seg(struct nvme_sgl_desc * sge,dma_addr_t dma_addr,int entries)677a7a7cbe3SChaitanya Kulkarni static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
678a7a7cbe3SChaitanya Kulkarni dma_addr_t dma_addr, int entries)
679a7a7cbe3SChaitanya Kulkarni {
680a7a7cbe3SChaitanya Kulkarni sge->addr = cpu_to_le64(dma_addr);
681a7a7cbe3SChaitanya Kulkarni sge->length = cpu_to_le32(entries * sizeof(*sge));
682a7a7cbe3SChaitanya Kulkarni sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
683a7a7cbe3SChaitanya Kulkarni }
684a7a7cbe3SChaitanya Kulkarni
nvme_pci_setup_sgls(struct nvme_dev * dev,struct request * req,struct nvme_rw_command * cmd)685a7a7cbe3SChaitanya Kulkarni static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
68691fb2b60SLogan Gunthorpe struct request *req, struct nvme_rw_command *cmd)
687a7a7cbe3SChaitanya Kulkarni {
688a7a7cbe3SChaitanya Kulkarni struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
689a7a7cbe3SChaitanya Kulkarni struct dma_pool *pool;
690a7a7cbe3SChaitanya Kulkarni struct nvme_sgl_desc *sg_list;
69191fb2b60SLogan Gunthorpe struct scatterlist *sg = iod->sgt.sgl;
69291fb2b60SLogan Gunthorpe unsigned int entries = iod->sgt.nents;
693a7a7cbe3SChaitanya Kulkarni dma_addr_t sgl_dma;
694b0f2853bSChristoph Hellwig int i = 0;
695a7a7cbe3SChaitanya Kulkarni
696a7a7cbe3SChaitanya Kulkarni /* setting the transfer type as SGL */
697a7a7cbe3SChaitanya Kulkarni cmd->flags = NVME_CMD_SGL_METABUF;
698a7a7cbe3SChaitanya Kulkarni
699b0f2853bSChristoph Hellwig if (entries == 1) {
700a7a7cbe3SChaitanya Kulkarni nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
701a7a7cbe3SChaitanya Kulkarni return BLK_STS_OK;
702a7a7cbe3SChaitanya Kulkarni }
703a7a7cbe3SChaitanya Kulkarni
704a7a7cbe3SChaitanya Kulkarni if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
705a7a7cbe3SChaitanya Kulkarni pool = dev->prp_small_pool;
706c372cdd1SKeith Busch iod->nr_allocations = 0;
707a7a7cbe3SChaitanya Kulkarni } else {
708a7a7cbe3SChaitanya Kulkarni pool = dev->prp_page_pool;
709c372cdd1SKeith Busch iod->nr_allocations = 1;
710a7a7cbe3SChaitanya Kulkarni }
711a7a7cbe3SChaitanya Kulkarni
712a7a7cbe3SChaitanya Kulkarni sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
713a7a7cbe3SChaitanya Kulkarni if (!sg_list) {
714c372cdd1SKeith Busch iod->nr_allocations = -1;
715a7a7cbe3SChaitanya Kulkarni return BLK_STS_RESOURCE;
716a7a7cbe3SChaitanya Kulkarni }
717a7a7cbe3SChaitanya Kulkarni
7187846c1b5SKeith Busch iod->list[0].sg_list = sg_list;
719a7a7cbe3SChaitanya Kulkarni iod->first_dma = sgl_dma;
720a7a7cbe3SChaitanya Kulkarni
721a7a7cbe3SChaitanya Kulkarni nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
722a7a7cbe3SChaitanya Kulkarni do {
723a7a7cbe3SChaitanya Kulkarni nvme_pci_sgl_set_data(&sg_list[i++], sg);
724a7a7cbe3SChaitanya Kulkarni sg = sg_next(sg);
725b0f2853bSChristoph Hellwig } while (--entries > 0);
726a7a7cbe3SChaitanya Kulkarni
727a7a7cbe3SChaitanya Kulkarni return BLK_STS_OK;
728a7a7cbe3SChaitanya Kulkarni }
729a7a7cbe3SChaitanya Kulkarni
nvme_setup_prp_simple(struct nvme_dev * dev,struct request * req,struct nvme_rw_command * cmnd,struct bio_vec * bv)730dff824b2SChristoph Hellwig static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
731dff824b2SChristoph Hellwig struct request *req, struct nvme_rw_command *cmnd,
732dff824b2SChristoph Hellwig struct bio_vec *bv)
733dff824b2SChristoph Hellwig {
734dff824b2SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
7356c3c05b0SChaitanya Kulkarni unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
7366c3c05b0SChaitanya Kulkarni unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;
737dff824b2SChristoph Hellwig
738dff824b2SChristoph Hellwig iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
739dff824b2SChristoph Hellwig if (dma_mapping_error(dev->dev, iod->first_dma))
740dff824b2SChristoph Hellwig return BLK_STS_RESOURCE;
741dff824b2SChristoph Hellwig iod->dma_len = bv->bv_len;
742dff824b2SChristoph Hellwig
743dff824b2SChristoph Hellwig cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
744dff824b2SChristoph Hellwig if (bv->bv_len > first_prp_len)
745dff824b2SChristoph Hellwig cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
746a56ea614SLei Rao else
747a56ea614SLei Rao cmnd->dptr.prp2 = 0;
748359c1f88SBaolin Wang return BLK_STS_OK;
749dff824b2SChristoph Hellwig }
750dff824b2SChristoph Hellwig
nvme_setup_sgl_simple(struct nvme_dev * dev,struct request * req,struct nvme_rw_command * cmnd,struct bio_vec * bv)75129791057SChristoph Hellwig static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev,
75229791057SChristoph Hellwig struct request *req, struct nvme_rw_command *cmnd,
75329791057SChristoph Hellwig struct bio_vec *bv)
75429791057SChristoph Hellwig {
75529791057SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
75629791057SChristoph Hellwig
75729791057SChristoph Hellwig iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
75829791057SChristoph Hellwig if (dma_mapping_error(dev->dev, iod->first_dma))
75929791057SChristoph Hellwig return BLK_STS_RESOURCE;
76029791057SChristoph Hellwig iod->dma_len = bv->bv_len;
76129791057SChristoph Hellwig
762049bf372SKlaus Birkelund Jensen cmnd->flags = NVME_CMD_SGL_METABUF;
76329791057SChristoph Hellwig cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma);
76429791057SChristoph Hellwig cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len);
76529791057SChristoph Hellwig cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
766359c1f88SBaolin Wang return BLK_STS_OK;
76729791057SChristoph Hellwig }
76829791057SChristoph Hellwig
nvme_map_data(struct nvme_dev * dev,struct request * req,struct nvme_command * cmnd)769fc17b653SChristoph Hellwig static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
770b131c61dSChristoph Hellwig struct nvme_command *cmnd)
77157dacad5SJay Sternberg {
772f4800d6dSChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
77370479b71SChristoph Hellwig blk_status_t ret = BLK_STS_RESOURCE;
77491fb2b60SLogan Gunthorpe int rc;
77557dacad5SJay Sternberg
776dff824b2SChristoph Hellwig if (blk_rq_nr_phys_segments(req) == 1) {
777a53232cbSKeith Busch struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
778dff824b2SChristoph Hellwig struct bio_vec bv = req_bvec(req);
779dff824b2SChristoph Hellwig
780dff824b2SChristoph Hellwig if (!is_pci_p2pdma_page(bv.bv_page)) {
7816c3c05b0SChaitanya Kulkarni if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)
782dff824b2SChristoph Hellwig return nvme_setup_prp_simple(dev, req,
783dff824b2SChristoph Hellwig &cmnd->rw, &bv);
78429791057SChristoph Hellwig
785a53232cbSKeith Busch if (nvmeq->qid && sgl_threshold &&
786253a0b76SChaitanya Kulkarni nvme_ctrl_sgl_supported(&dev->ctrl))
78729791057SChristoph Hellwig return nvme_setup_sgl_simple(dev, req,
78829791057SChristoph Hellwig &cmnd->rw, &bv);
789dff824b2SChristoph Hellwig }
790dff824b2SChristoph Hellwig }
791dff824b2SChristoph Hellwig
792dff824b2SChristoph Hellwig iod->dma_len = 0;
79391fb2b60SLogan Gunthorpe iod->sgt.sgl = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
79491fb2b60SLogan Gunthorpe if (!iod->sgt.sgl)
7959b048119SChristoph Hellwig return BLK_STS_RESOURCE;
79691fb2b60SLogan Gunthorpe sg_init_table(iod->sgt.sgl, blk_rq_nr_phys_segments(req));
79791fb2b60SLogan Gunthorpe iod->sgt.orig_nents = blk_rq_map_sg(req->q, req, iod->sgt.sgl);
79891fb2b60SLogan Gunthorpe if (!iod->sgt.orig_nents)
799fa073216SChristoph Hellwig goto out_free_sg;
800ba1ca37eSChristoph Hellwig
80191fb2b60SLogan Gunthorpe rc = dma_map_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req),
80291fb2b60SLogan Gunthorpe DMA_ATTR_NO_WARN);
80391fb2b60SLogan Gunthorpe if (rc) {
80491fb2b60SLogan Gunthorpe if (rc == -EREMOTEIO)
80591fb2b60SLogan Gunthorpe ret = BLK_STS_TARGET;
806fa073216SChristoph Hellwig goto out_free_sg;
80791fb2b60SLogan Gunthorpe }
808ba1ca37eSChristoph Hellwig
809b6c0c237SKeith Busch if (nvme_pci_use_sgls(dev, req, iod->sgt.nents))
81091fb2b60SLogan Gunthorpe ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw);
811a7a7cbe3SChaitanya Kulkarni else
812a7a7cbe3SChaitanya Kulkarni ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
8134aedb705SChristoph Hellwig if (ret != BLK_STS_OK)
814fa073216SChristoph Hellwig goto out_unmap_sg;
815fa073216SChristoph Hellwig return BLK_STS_OK;
816fa073216SChristoph Hellwig
817fa073216SChristoph Hellwig out_unmap_sg:
81891fb2b60SLogan Gunthorpe dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
819fa073216SChristoph Hellwig out_free_sg:
82091fb2b60SLogan Gunthorpe mempool_free(iod->sgt.sgl, dev->iod_mempool);
821ba1ca37eSChristoph Hellwig return ret;
82257dacad5SJay Sternberg }
82357dacad5SJay Sternberg
nvme_map_metadata(struct nvme_dev * dev,struct request * req,struct nvme_command * cmnd)8244aedb705SChristoph Hellwig static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
8254aedb705SChristoph Hellwig struct nvme_command *cmnd)
8264aedb705SChristoph Hellwig {
8274aedb705SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
8284aedb705SChristoph Hellwig
8294aedb705SChristoph Hellwig iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
8304aedb705SChristoph Hellwig rq_dma_dir(req), 0);
8314aedb705SChristoph Hellwig if (dma_mapping_error(dev->dev, iod->meta_dma))
8324aedb705SChristoph Hellwig return BLK_STS_IOERR;
8334aedb705SChristoph Hellwig cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
834359c1f88SBaolin Wang return BLK_STS_OK;
8354aedb705SChristoph Hellwig }
8364aedb705SChristoph Hellwig
nvme_prep_rq(struct nvme_dev * dev,struct request * req)83762451a2bSJens Axboe static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
83862451a2bSJens Axboe {
83962451a2bSJens Axboe struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
84062451a2bSJens Axboe blk_status_t ret;
84162451a2bSJens Axboe
84252da4f3fSKeith Busch iod->aborted = false;
843c372cdd1SKeith Busch iod->nr_allocations = -1;
84491fb2b60SLogan Gunthorpe iod->sgt.nents = 0;
84562451a2bSJens Axboe
84662451a2bSJens Axboe ret = nvme_setup_cmd(req->q->queuedata, req);
84762451a2bSJens Axboe if (ret)
84862451a2bSJens Axboe return ret;
84962451a2bSJens Axboe
85062451a2bSJens Axboe if (blk_rq_nr_phys_segments(req)) {
85162451a2bSJens Axboe ret = nvme_map_data(dev, req, &iod->cmd);
85262451a2bSJens Axboe if (ret)
85362451a2bSJens Axboe goto out_free_cmd;
85462451a2bSJens Axboe }
85562451a2bSJens Axboe
85662451a2bSJens Axboe if (blk_integrity_rq(req)) {
85762451a2bSJens Axboe ret = nvme_map_metadata(dev, req, &iod->cmd);
85862451a2bSJens Axboe if (ret)
85962451a2bSJens Axboe goto out_unmap_data;
86062451a2bSJens Axboe }
86162451a2bSJens Axboe
8626887fc64SSagi Grimberg nvme_start_request(req);
86362451a2bSJens Axboe return BLK_STS_OK;
86462451a2bSJens Axboe out_unmap_data:
86562451a2bSJens Axboe nvme_unmap_data(dev, req);
86662451a2bSJens Axboe out_free_cmd:
86762451a2bSJens Axboe nvme_cleanup_cmd(req);
86862451a2bSJens Axboe return ret;
86962451a2bSJens Axboe }
87062451a2bSJens Axboe
87157dacad5SJay Sternberg /*
87257dacad5SJay Sternberg * NOTE: ns is NULL when called on the admin queue.
87357dacad5SJay Sternberg */
nvme_queue_rq(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * bd)874fc17b653SChristoph Hellwig static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
87557dacad5SJay Sternberg const struct blk_mq_queue_data *bd)
87657dacad5SJay Sternberg {
87757dacad5SJay Sternberg struct nvme_queue *nvmeq = hctx->driver_data;
87857dacad5SJay Sternberg struct nvme_dev *dev = nvmeq->dev;
87957dacad5SJay Sternberg struct request *req = bd->rq;
8809b048119SChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
881ebe6d874SChristoph Hellwig blk_status_t ret;
88257dacad5SJay Sternberg
883d1f06f4aSJens Axboe /*
884d1f06f4aSJens Axboe * We should not need to do this, but we're still using this to
885d1f06f4aSJens Axboe * ensure we can drain requests on a dying queue.
886d1f06f4aSJens Axboe */
8874e224106SChristoph Hellwig if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
888d1f06f4aSJens Axboe return BLK_STS_IOERR;
889d1f06f4aSJens Axboe
89062451a2bSJens Axboe if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
891d4060d2bSTao Chiu return nvme_fail_nonready_command(&dev->ctrl, req);
892d4060d2bSTao Chiu
89362451a2bSJens Axboe ret = nvme_prep_rq(dev, req);
89462451a2bSJens Axboe if (unlikely(ret))
895f4800d6dSChristoph Hellwig return ret;
8963233b94cSJens Axboe spin_lock(&nvmeq->sq_lock);
8973233b94cSJens Axboe nvme_sq_copy_cmd(nvmeq, &iod->cmd);
8983233b94cSJens Axboe nvme_write_sq_db(nvmeq, bd->last);
8993233b94cSJens Axboe spin_unlock(&nvmeq->sq_lock);
900fc17b653SChristoph Hellwig return BLK_STS_OK;
90157dacad5SJay Sternberg }
90257dacad5SJay Sternberg
nvme_submit_cmds(struct nvme_queue * nvmeq,struct request ** rqlist)903d62cbcf6SJens Axboe static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
904d62cbcf6SJens Axboe {
905d62cbcf6SJens Axboe spin_lock(&nvmeq->sq_lock);
906d62cbcf6SJens Axboe while (!rq_list_empty(*rqlist)) {
907d62cbcf6SJens Axboe struct request *req = rq_list_pop(rqlist);
908d62cbcf6SJens Axboe struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
909d62cbcf6SJens Axboe
910d62cbcf6SJens Axboe nvme_sq_copy_cmd(nvmeq, &iod->cmd);
911d62cbcf6SJens Axboe }
912d62cbcf6SJens Axboe nvme_write_sq_db(nvmeq, true);
913d62cbcf6SJens Axboe spin_unlock(&nvmeq->sq_lock);
914d62cbcf6SJens Axboe }
915d62cbcf6SJens Axboe
nvme_prep_rq_batch(struct nvme_queue * nvmeq,struct request * req)916d62cbcf6SJens Axboe static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
917d62cbcf6SJens Axboe {
918d62cbcf6SJens Axboe /*
919d62cbcf6SJens Axboe * We should not need to do this, but we're still using this to
920d62cbcf6SJens Axboe * ensure we can drain requests on a dying queue.
921d62cbcf6SJens Axboe */
922d62cbcf6SJens Axboe if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
923d62cbcf6SJens Axboe return false;
924d62cbcf6SJens Axboe if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
925d62cbcf6SJens Axboe return false;
926d62cbcf6SJens Axboe
927d62cbcf6SJens Axboe req->mq_hctx->tags->rqs[req->tag] = req;
928d62cbcf6SJens Axboe return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
929d62cbcf6SJens Axboe }
930d62cbcf6SJens Axboe
nvme_queue_rqs(struct request ** rqlist)931d62cbcf6SJens Axboe static void nvme_queue_rqs(struct request **rqlist)
932d62cbcf6SJens Axboe {
9336bfec799SKeith Busch struct request *req, *next, *prev = NULL;
934d62cbcf6SJens Axboe struct request *requeue_list = NULL;
935d62cbcf6SJens Axboe
9366bfec799SKeith Busch rq_list_for_each_safe(rqlist, req, next) {
937d62cbcf6SJens Axboe struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
938d62cbcf6SJens Axboe
939d62cbcf6SJens Axboe if (!nvme_prep_rq_batch(nvmeq, req)) {
940d62cbcf6SJens Axboe /* detach 'req' and add to remainder list */
9416bfec799SKeith Busch rq_list_move(rqlist, &requeue_list, req, prev);
9426bfec799SKeith Busch
9436bfec799SKeith Busch req = prev;
9446bfec799SKeith Busch if (!req)
9456bfec799SKeith Busch continue;
946d62cbcf6SJens Axboe }
947d62cbcf6SJens Axboe
9486bfec799SKeith Busch if (!next || req->mq_hctx != next->mq_hctx) {
949d62cbcf6SJens Axboe /* detach rest of list, and submit */
9506bfec799SKeith Busch req->rq_next = NULL;
951d62cbcf6SJens Axboe nvme_submit_cmds(nvmeq, rqlist);
9526bfec799SKeith Busch *rqlist = next;
9536bfec799SKeith Busch prev = NULL;
9546bfec799SKeith Busch } else
9556bfec799SKeith Busch prev = req;
956d62cbcf6SJens Axboe }
957d62cbcf6SJens Axboe
958d62cbcf6SJens Axboe *rqlist = requeue_list;
959d62cbcf6SJens Axboe }
960d62cbcf6SJens Axboe
nvme_pci_unmap_rq(struct request * req)961c234a653SJens Axboe static __always_inline void nvme_pci_unmap_rq(struct request *req)
962eee417b0SChristoph Hellwig {
963a53232cbSKeith Busch struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
964a53232cbSKeith Busch struct nvme_dev *dev = nvmeq->dev;
965eee417b0SChristoph Hellwig
966a53232cbSKeith Busch if (blk_integrity_rq(req)) {
967a53232cbSKeith Busch struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
968a53232cbSKeith Busch
9694aedb705SChristoph Hellwig dma_unmap_page(dev->dev, iod->meta_dma,
970b8f6446bSMing Lei rq_integrity_vec(req)->bv_len, rq_dma_dir(req));
971a53232cbSKeith Busch }
972a53232cbSKeith Busch
973b15c592dSChristoph Hellwig if (blk_rq_nr_phys_segments(req))
9744aedb705SChristoph Hellwig nvme_unmap_data(dev, req);
975c234a653SJens Axboe }
976c234a653SJens Axboe
nvme_pci_complete_rq(struct request * req)977c234a653SJens Axboe static void nvme_pci_complete_rq(struct request *req)
978c234a653SJens Axboe {
979c234a653SJens Axboe nvme_pci_unmap_rq(req);
98077f02a7aSChristoph Hellwig nvme_complete_rq(req);
98157dacad5SJay Sternberg }
98257dacad5SJay Sternberg
nvme_pci_complete_batch(struct io_comp_batch * iob)983c234a653SJens Axboe static void nvme_pci_complete_batch(struct io_comp_batch *iob)
984c234a653SJens Axboe {
985c234a653SJens Axboe nvme_complete_batch(iob, nvme_pci_unmap_rq);
986c234a653SJens Axboe }
987c234a653SJens Axboe
988d783e0bdSMarta Rybczynska /* We read the CQE phase first to check if the rest of the entry is valid */
nvme_cqe_pending(struct nvme_queue * nvmeq)989750dde44SChristoph Hellwig static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
990d783e0bdSMarta Rybczynska {
99174943d45SKeith Busch struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
99274943d45SKeith Busch
99374943d45SKeith Busch return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
994d783e0bdSMarta Rybczynska }
995d783e0bdSMarta Rybczynska
nvme_ring_cq_doorbell(struct nvme_queue * nvmeq)996eb281c82SSagi Grimberg static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
99757dacad5SJay Sternberg {
998eb281c82SSagi Grimberg u16 head = nvmeq->cq_head;
99957dacad5SJay Sternberg
1000eb281c82SSagi Grimberg if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
1001eb281c82SSagi Grimberg nvmeq->dbbuf_cq_ei))
1002eb281c82SSagi Grimberg writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
1003eb281c82SSagi Grimberg }
1004adf68f21SChristoph Hellwig
nvme_queue_tagset(struct nvme_queue * nvmeq)1005cfa27356SChristoph Hellwig static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
1006cfa27356SChristoph Hellwig {
1007cfa27356SChristoph Hellwig if (!nvmeq->qid)
1008cfa27356SChristoph Hellwig return nvmeq->dev->admin_tagset.tags[0];
1009cfa27356SChristoph Hellwig return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
1010cfa27356SChristoph Hellwig }
1011cfa27356SChristoph Hellwig
nvme_handle_cqe(struct nvme_queue * nvmeq,struct io_comp_batch * iob,u16 idx)1012c234a653SJens Axboe static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
1013c234a653SJens Axboe struct io_comp_batch *iob, u16 idx)
101457dacad5SJay Sternberg {
101574943d45SKeith Busch struct nvme_completion *cqe = &nvmeq->cqes[idx];
101662df8016SLalithambika Krishnakumar __u16 command_id = READ_ONCE(cqe->command_id);
101757dacad5SJay Sternberg struct request *req;
1018adf68f21SChristoph Hellwig
1019adf68f21SChristoph Hellwig /*
1020adf68f21SChristoph Hellwig * AEN requests are special as they don't time out and can
1021adf68f21SChristoph Hellwig * survive any kind of queue freeze and often don't respond to
1022adf68f21SChristoph Hellwig * aborts. We don't even bother to allocate a struct request
1023adf68f21SChristoph Hellwig * for them but rather special case them here.
1024adf68f21SChristoph Hellwig */
102562df8016SLalithambika Krishnakumar if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) {
10267bf58533SChristoph Hellwig nvme_complete_async_event(&nvmeq->dev->ctrl,
102783a12fb7SSagi Grimberg cqe->status, &cqe->result);
1028a0fa9647SJens Axboe return;
102957dacad5SJay Sternberg }
103057dacad5SJay Sternberg
1031e7006de6SSagi Grimberg req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id);
103250b7c243SXianting Tian if (unlikely(!req)) {
103350b7c243SXianting Tian dev_warn(nvmeq->dev->ctrl.device,
103450b7c243SXianting Tian "invalid id %d completed on queue %d\n",
103562df8016SLalithambika Krishnakumar command_id, le16_to_cpu(cqe->sq_id));
103650b7c243SXianting Tian return;
103750b7c243SXianting Tian }
103850b7c243SXianting Tian
1039604c01d5Syupeng trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
1040c234a653SJens Axboe if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
1041c234a653SJens Axboe !blk_mq_add_to_batch(req, iob, nvme_req(req)->status,
1042c234a653SJens Axboe nvme_pci_complete_batch))
1043ff029451SChristoph Hellwig nvme_pci_complete_rq(req);
104483a12fb7SSagi Grimberg }
104557dacad5SJay Sternberg
nvme_update_cq_head(struct nvme_queue * nvmeq)10465cb525c8SJens Axboe static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
10475cb525c8SJens Axboe {
1048a0aac973SJK Kim u32 tmp = nvmeq->cq_head + 1;
1049a8de6639SAlexey Dobriyan
1050a8de6639SAlexey Dobriyan if (tmp == nvmeq->q_depth) {
1051920d13a8SSagi Grimberg nvmeq->cq_head = 0;
1052e2a366a4SAlexey Dobriyan nvmeq->cq_phase ^= 1;
1053a8de6639SAlexey Dobriyan } else {
1054a8de6639SAlexey Dobriyan nvmeq->cq_head = tmp;
1055920d13a8SSagi Grimberg }
1056a0fa9647SJens Axboe }
1057a0fa9647SJens Axboe
nvme_poll_cq(struct nvme_queue * nvmeq,struct io_comp_batch * iob)1058c234a653SJens Axboe static inline int nvme_poll_cq(struct nvme_queue *nvmeq,
1059c234a653SJens Axboe struct io_comp_batch *iob)
1060a0fa9647SJens Axboe {
10611052b8acSJens Axboe int found = 0;
106283a12fb7SSagi Grimberg
10631052b8acSJens Axboe while (nvme_cqe_pending(nvmeq)) {
10641052b8acSJens Axboe found++;
1065b69e2ef2SKeith Busch /*
1066b69e2ef2SKeith Busch * load-load control dependency between phase and the rest of
1067b69e2ef2SKeith Busch * the cqe requires a full read memory barrier
1068b69e2ef2SKeith Busch */
1069b69e2ef2SKeith Busch dma_rmb();
1070c234a653SJens Axboe nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
10715cb525c8SJens Axboe nvme_update_cq_head(nvmeq);
107257dacad5SJay Sternberg }
107357dacad5SJay Sternberg
1074324b494cSKeith Busch if (found)
1075eb281c82SSagi Grimberg nvme_ring_cq_doorbell(nvmeq);
10765cb525c8SJens Axboe return found;
107757dacad5SJay Sternberg }
107857dacad5SJay Sternberg
nvme_irq(int irq,void * data)107957dacad5SJay Sternberg static irqreturn_t nvme_irq(int irq, void *data)
108057dacad5SJay Sternberg {
108157dacad5SJay Sternberg struct nvme_queue *nvmeq = data;
10824f502245SJens Axboe DEFINE_IO_COMP_BATCH(iob);
10835cb525c8SJens Axboe
10844f502245SJens Axboe if (nvme_poll_cq(nvmeq, &iob)) {
10854f502245SJens Axboe if (!rq_list_empty(iob.req_list))
10864f502245SJens Axboe nvme_pci_complete_batch(&iob);
108705fae499SChaitanya Kulkarni return IRQ_HANDLED;
10884f502245SJens Axboe }
108905fae499SChaitanya Kulkarni return IRQ_NONE;
109057dacad5SJay Sternberg }
109157dacad5SJay Sternberg
nvme_irq_check(int irq,void * data)109257dacad5SJay Sternberg static irqreturn_t nvme_irq_check(int irq, void *data)
109357dacad5SJay Sternberg {
109457dacad5SJay Sternberg struct nvme_queue *nvmeq = data;
10954e523547SBaolin Wang
1096750dde44SChristoph Hellwig if (nvme_cqe_pending(nvmeq))
109757dacad5SJay Sternberg return IRQ_WAKE_THREAD;
1098d783e0bdSMarta Rybczynska return IRQ_NONE;
109957dacad5SJay Sternberg }
110057dacad5SJay Sternberg
11010b2a8a9fSChristoph Hellwig /*
1102fa059b85SKeith Busch * Poll for completions for any interrupt driven queue
11030b2a8a9fSChristoph Hellwig * Can be called from any context.
11040b2a8a9fSChristoph Hellwig */
nvme_poll_irqdisable(struct nvme_queue * nvmeq)1105fa059b85SKeith Busch static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
1106a0fa9647SJens Axboe {
11073a7afd8eSChristoph Hellwig struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1108a0fa9647SJens Axboe
1109fa059b85SKeith Busch WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
1110fa059b85SKeith Busch
11113a7afd8eSChristoph Hellwig disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
1112c234a653SJens Axboe nvme_poll_cq(nvmeq, NULL);
11133a7afd8eSChristoph Hellwig enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
111491a509f8SChristoph Hellwig }
1115442e19b7SSagi Grimberg
nvme_poll(struct blk_mq_hw_ctx * hctx,struct io_comp_batch * iob)11165a72e899SJens Axboe static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
11177776db1cSKeith Busch {
11187776db1cSKeith Busch struct nvme_queue *nvmeq = hctx->driver_data;
1119dabcefabSJens Axboe bool found;
1120dabcefabSJens Axboe
1121dabcefabSJens Axboe if (!nvme_cqe_pending(nvmeq))
1122dabcefabSJens Axboe return 0;
1123dabcefabSJens Axboe
11243a7afd8eSChristoph Hellwig spin_lock(&nvmeq->cq_poll_lock);
1125c234a653SJens Axboe found = nvme_poll_cq(nvmeq, iob);
11263a7afd8eSChristoph Hellwig spin_unlock(&nvmeq->cq_poll_lock);
1127dabcefabSJens Axboe
1128dabcefabSJens Axboe return found;
1129dabcefabSJens Axboe }
1130dabcefabSJens Axboe
nvme_pci_submit_async_event(struct nvme_ctrl * ctrl)1131ad22c355SKeith Busch static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
113257dacad5SJay Sternberg {
1133f866fc42SChristoph Hellwig struct nvme_dev *dev = to_nvme_dev(ctrl);
1134147b27e4SSagi Grimberg struct nvme_queue *nvmeq = &dev->queues[0];
1135f66e2804SChaitanya Kulkarni struct nvme_command c = { };
113657dacad5SJay Sternberg
113757dacad5SJay Sternberg c.common.opcode = nvme_admin_async_event;
1138ad22c355SKeith Busch c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
11393233b94cSJens Axboe
11403233b94cSJens Axboe spin_lock(&nvmeq->sq_lock);
11413233b94cSJens Axboe nvme_sq_copy_cmd(nvmeq, &c);
11423233b94cSJens Axboe nvme_write_sq_db(nvmeq, true);
11433233b94cSJens Axboe spin_unlock(&nvmeq->sq_lock);
114457dacad5SJay Sternberg }
114557dacad5SJay Sternberg
adapter_delete_queue(struct nvme_dev * dev,u8 opcode,u16 id)114657dacad5SJay Sternberg static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
114757dacad5SJay Sternberg {
1148f66e2804SChaitanya Kulkarni struct nvme_command c = { };
114957dacad5SJay Sternberg
115057dacad5SJay Sternberg c.delete_queue.opcode = opcode;
115157dacad5SJay Sternberg c.delete_queue.qid = cpu_to_le16(id);
115257dacad5SJay Sternberg
11531c63dc66SChristoph Hellwig return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
115457dacad5SJay Sternberg }
115557dacad5SJay Sternberg
adapter_alloc_cq(struct nvme_dev * dev,u16 qid,struct nvme_queue * nvmeq,s16 vector)115657dacad5SJay Sternberg static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1157a8e3e0bbSJianchao Wang struct nvme_queue *nvmeq, s16 vector)
115857dacad5SJay Sternberg {
1159f66e2804SChaitanya Kulkarni struct nvme_command c = { };
11604b04cc6aSJens Axboe int flags = NVME_QUEUE_PHYS_CONTIG;
11614b04cc6aSJens Axboe
11627c349ddeSKeith Busch if (!test_bit(NVMEQ_POLLED, &nvmeq->flags))
11634b04cc6aSJens Axboe flags |= NVME_CQ_IRQ_ENABLED;
116457dacad5SJay Sternberg
116557dacad5SJay Sternberg /*
116616772ae6SMinwoo Im * Note: we (ab)use the fact that the prp fields survive if no data
116757dacad5SJay Sternberg * is attached to the request.
116857dacad5SJay Sternberg */
116957dacad5SJay Sternberg c.create_cq.opcode = nvme_admin_create_cq;
117057dacad5SJay Sternberg c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
117157dacad5SJay Sternberg c.create_cq.cqid = cpu_to_le16(qid);
117257dacad5SJay Sternberg c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
117357dacad5SJay Sternberg c.create_cq.cq_flags = cpu_to_le16(flags);
1174a8e3e0bbSJianchao Wang c.create_cq.irq_vector = cpu_to_le16(vector);
117557dacad5SJay Sternberg
11761c63dc66SChristoph Hellwig return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
117757dacad5SJay Sternberg }
117857dacad5SJay Sternberg
adapter_alloc_sq(struct nvme_dev * dev,u16 qid,struct nvme_queue * nvmeq)117957dacad5SJay Sternberg static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
118057dacad5SJay Sternberg struct nvme_queue *nvmeq)
118157dacad5SJay Sternberg {
11829abd68efSJens Axboe struct nvme_ctrl *ctrl = &dev->ctrl;
1183f66e2804SChaitanya Kulkarni struct nvme_command c = { };
118481c1cd98SKeith Busch int flags = NVME_QUEUE_PHYS_CONTIG;
118557dacad5SJay Sternberg
118657dacad5SJay Sternberg /*
11879abd68efSJens Axboe * Some drives have a bug that auto-enables WRRU if MEDIUM isn't
11889abd68efSJens Axboe * set. Since URGENT priority is zeroes, it makes all queues
11899abd68efSJens Axboe * URGENT.
11909abd68efSJens Axboe */
11919abd68efSJens Axboe if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
11929abd68efSJens Axboe flags |= NVME_SQ_PRIO_MEDIUM;
11939abd68efSJens Axboe
11949abd68efSJens Axboe /*
119516772ae6SMinwoo Im * Note: we (ab)use the fact that the prp fields survive if no data
119657dacad5SJay Sternberg * is attached to the request.
119757dacad5SJay Sternberg */
119857dacad5SJay Sternberg c.create_sq.opcode = nvme_admin_create_sq;
119957dacad5SJay Sternberg c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
120057dacad5SJay Sternberg c.create_sq.sqid = cpu_to_le16(qid);
120157dacad5SJay Sternberg c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
120257dacad5SJay Sternberg c.create_sq.sq_flags = cpu_to_le16(flags);
120357dacad5SJay Sternberg c.create_sq.cqid = cpu_to_le16(qid);
120457dacad5SJay Sternberg
12051c63dc66SChristoph Hellwig return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
120657dacad5SJay Sternberg }
120757dacad5SJay Sternberg
adapter_delete_cq(struct nvme_dev * dev,u16 cqid)120857dacad5SJay Sternberg static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
120957dacad5SJay Sternberg {
121057dacad5SJay Sternberg return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
121157dacad5SJay Sternberg }
121257dacad5SJay Sternberg
adapter_delete_sq(struct nvme_dev * dev,u16 sqid)121357dacad5SJay Sternberg static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
121457dacad5SJay Sternberg {
121557dacad5SJay Sternberg return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
121657dacad5SJay Sternberg }
121757dacad5SJay Sternberg
abort_endio(struct request * req,blk_status_t error)1218de671d61SJens Axboe static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error)
121957dacad5SJay Sternberg {
1220a53232cbSKeith Busch struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
122157dacad5SJay Sternberg
122227fa9bc5SChristoph Hellwig dev_warn(nvmeq->dev->ctrl.device,
122327fa9bc5SChristoph Hellwig "Abort status: 0x%x", nvme_req(req)->status);
1224e7a2a87dSChristoph Hellwig atomic_inc(&nvmeq->dev->ctrl.abort_limit);
1225e7a2a87dSChristoph Hellwig blk_mq_free_request(req);
1226de671d61SJens Axboe return RQ_END_IO_NONE;
122757dacad5SJay Sternberg }
122857dacad5SJay Sternberg
nvme_should_reset(struct nvme_dev * dev,u32 csts)1229b2a0eb1aSKeith Busch static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1230b2a0eb1aSKeith Busch {
1231b2a0eb1aSKeith Busch /* If true, indicates loss of adapter communication, possibly by a
1232b2a0eb1aSKeith Busch * NVMe Subsystem reset.
1233b2a0eb1aSKeith Busch */
1234b2a0eb1aSKeith Busch bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1235b2a0eb1aSKeith Busch
1236ad70062cSJianchao Wang /* If there is a reset/reinit ongoing, we shouldn't reset again. */
12378884a56dSKeith Busch switch (nvme_ctrl_state(&dev->ctrl)) {
1238ad70062cSJianchao Wang case NVME_CTRL_RESETTING:
1239ad6a0a52SMax Gurtovoy case NVME_CTRL_CONNECTING:
1240b2a0eb1aSKeith Busch return false;
1241ad70062cSJianchao Wang default:
1242ad70062cSJianchao Wang break;
1243ad70062cSJianchao Wang }
1244b2a0eb1aSKeith Busch
1245b2a0eb1aSKeith Busch /* We shouldn't reset unless the controller is on fatal error state
1246b2a0eb1aSKeith Busch * _or_ if we lost the communication with it.
1247b2a0eb1aSKeith Busch */
1248b2a0eb1aSKeith Busch if (!(csts & NVME_CSTS_CFS) && !nssro)
1249b2a0eb1aSKeith Busch return false;
1250b2a0eb1aSKeith Busch
1251b2a0eb1aSKeith Busch return true;
1252b2a0eb1aSKeith Busch }
1253b2a0eb1aSKeith Busch
nvme_warn_reset(struct nvme_dev * dev,u32 csts)1254b2a0eb1aSKeith Busch static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1255b2a0eb1aSKeith Busch {
1256b2a0eb1aSKeith Busch /* Read a config register to help see what died. */
1257b2a0eb1aSKeith Busch u16 pci_status;
1258b2a0eb1aSKeith Busch int result;
1259b2a0eb1aSKeith Busch
1260b2a0eb1aSKeith Busch result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1261b2a0eb1aSKeith Busch &pci_status);
1262b2a0eb1aSKeith Busch if (result == PCIBIOS_SUCCESSFUL)
1263b2a0eb1aSKeith Busch dev_warn(dev->ctrl.device,
1264b2a0eb1aSKeith Busch "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1265b2a0eb1aSKeith Busch csts, pci_status);
1266b2a0eb1aSKeith Busch else
1267b2a0eb1aSKeith Busch dev_warn(dev->ctrl.device,
1268b2a0eb1aSKeith Busch "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1269b2a0eb1aSKeith Busch csts, result);
12704641a8e6SKeith Busch
12714641a8e6SKeith Busch if (csts != ~0)
12724641a8e6SKeith Busch return;
12734641a8e6SKeith Busch
12744641a8e6SKeith Busch dev_warn(dev->ctrl.device,
12754641a8e6SKeith Busch "Does your device have a faulty power saving mode enabled?\n");
12764641a8e6SKeith Busch dev_warn(dev->ctrl.device,
12774641a8e6SKeith Busch "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off\" and report a bug\n");
1278b2a0eb1aSKeith Busch }
1279b2a0eb1aSKeith Busch
nvme_timeout(struct request * req)12809bdb4833SJohn Garry static enum blk_eh_timer_return nvme_timeout(struct request *req)
128157dacad5SJay Sternberg {
1282f4800d6dSChristoph Hellwig struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1283a53232cbSKeith Busch struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
128457dacad5SJay Sternberg struct nvme_dev *dev = nvmeq->dev;
128557dacad5SJay Sternberg struct request *abort_req;
1286f66e2804SChaitanya Kulkarni struct nvme_command cmd = { };
1287b2a0eb1aSKeith Busch u32 csts = readl(dev->bar + NVME_REG_CSTS);
1288b2a0eb1aSKeith Busch
1289b6eaa53fSNilay Shroff if (nvme_state_terminal(&dev->ctrl))
1290b6eaa53fSNilay Shroff goto disable;
1291b6eaa53fSNilay Shroff
1292651438bbSWen Xiong /* If PCI error recovery process is happening, we cannot reset or
1293651438bbSWen Xiong * the recovery mechanism will surely fail.
1294651438bbSWen Xiong */
1295651438bbSWen Xiong mb();
1296651438bbSWen Xiong if (pci_channel_offline(to_pci_dev(dev->dev)))
1297651438bbSWen Xiong return BLK_EH_RESET_TIMER;
1298651438bbSWen Xiong
1299b2a0eb1aSKeith Busch /*
1300b2a0eb1aSKeith Busch * Reset immediately if the controller is failed
1301b2a0eb1aSKeith Busch */
1302b2a0eb1aSKeith Busch if (nvme_should_reset(dev, csts)) {
1303b2a0eb1aSKeith Busch nvme_warn_reset(dev, csts);
130471a5bb15SKeith Busch goto disable;
1305b2a0eb1aSKeith Busch }
130657dacad5SJay Sternberg
130731c7c7d2SChristoph Hellwig /*
13087776db1cSKeith Busch * Did we miss an interrupt?
13097776db1cSKeith Busch */
1310fa059b85SKeith Busch if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
13115a72e899SJens Axboe nvme_poll(req->mq_hctx, NULL);
1312fa059b85SKeith Busch else
1313bf392a5dSKeith Busch nvme_poll_irqdisable(nvmeq);
1314fa059b85SKeith Busch
13151c584208SKeith Busch if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT) {
13167776db1cSKeith Busch dev_warn(dev->ctrl.device,
13177776db1cSKeith Busch "I/O %d QID %d timeout, completion polled\n",
13187776db1cSKeith Busch req->tag, nvmeq->qid);
1319db8c48e4SChristoph Hellwig return BLK_EH_DONE;
13207776db1cSKeith Busch }
13217776db1cSKeith Busch
13227776db1cSKeith Busch /*
1323fd634f41SChristoph Hellwig * Shutdown immediately if controller times out while starting. The
1324fd634f41SChristoph Hellwig * reset work will see the pci device disabled when it gets the forced
1325fd634f41SChristoph Hellwig * cancellation error. All outstanding requests are completed on
1326db8c48e4SChristoph Hellwig * shutdown, so we return BLK_EH_DONE.
1327fd634f41SChristoph Hellwig */
13288884a56dSKeith Busch switch (nvme_ctrl_state(&dev->ctrl)) {
13294244140dSKeith Busch case NVME_CTRL_CONNECTING:
13302036f726SKeith Busch nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1331df561f66SGustavo A. R. Silva fallthrough;
13322036f726SKeith Busch case NVME_CTRL_DELETING:
1333b9cac43cSKeith Busch dev_warn_ratelimited(dev->ctrl.device,
1334fd634f41SChristoph Hellwig "I/O %d QID %d timeout, disable controller\n",
1335fd634f41SChristoph Hellwig req->tag, nvmeq->qid);
133627fa9bc5SChristoph Hellwig nvme_req(req)->flags |= NVME_REQ_CANCELLED;
13377ad92f65STong Zhang nvme_dev_disable(dev, true);
1338db8c48e4SChristoph Hellwig return BLK_EH_DONE;
133939a9dd81SKeith Busch case NVME_CTRL_RESETTING:
134039a9dd81SKeith Busch return BLK_EH_RESET_TIMER;
13414244140dSKeith Busch default:
13424244140dSKeith Busch break;
1343fd634f41SChristoph Hellwig }
1344fd634f41SChristoph Hellwig
1345fd634f41SChristoph Hellwig /*
1346e1569a16SKeith Busch * Shutdown the controller immediately and schedule a reset if the
1347e1569a16SKeith Busch * command was already aborted once before and still hasn't been
1348e1569a16SKeith Busch * returned to the driver, or if this is the admin queue.
134931c7c7d2SChristoph Hellwig */
1350f4800d6dSChristoph Hellwig if (!nvmeq->qid || iod->aborted) {
13511b3c47c1SSagi Grimberg dev_warn(dev->ctrl.device,
135257dacad5SJay Sternberg "I/O %d QID %d timeout, reset controller\n",
135357dacad5SJay Sternberg req->tag, nvmeq->qid);
13547ad92f65STong Zhang nvme_req(req)->flags |= NVME_REQ_CANCELLED;
135571a5bb15SKeith Busch goto disable;
135657dacad5SJay Sternberg }
135757dacad5SJay Sternberg
1358e7a2a87dSChristoph Hellwig if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1359e7a2a87dSChristoph Hellwig atomic_inc(&dev->ctrl.abort_limit);
1360e7a2a87dSChristoph Hellwig return BLK_EH_RESET_TIMER;
1361e7a2a87dSChristoph Hellwig }
136252da4f3fSKeith Busch iod->aborted = true;
136357dacad5SJay Sternberg
136457dacad5SJay Sternberg cmd.abort.opcode = nvme_admin_abort_cmd;
136585f74acfSKeith Busch cmd.abort.cid = nvme_cid(req);
136657dacad5SJay Sternberg cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
136757dacad5SJay Sternberg
13681b3c47c1SSagi Grimberg dev_warn(nvmeq->dev->ctrl.device,
136986141440SChristoph Hellwig "I/O %d (%s) QID %d timeout, aborting\n",
137086141440SChristoph Hellwig req->tag,
137186141440SChristoph Hellwig nvme_get_opcode_str(nvme_req(req)->cmd->common.opcode),
137286141440SChristoph Hellwig nvmeq->qid);
1373e7a2a87dSChristoph Hellwig
1374e559398fSChristoph Hellwig abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd),
137539dfe844SChaitanya Kulkarni BLK_MQ_REQ_NOWAIT);
13766bf25d16SChristoph Hellwig if (IS_ERR(abort_req)) {
13776bf25d16SChristoph Hellwig atomic_inc(&dev->ctrl.abort_limit);
137831c7c7d2SChristoph Hellwig return BLK_EH_RESET_TIMER;
137957dacad5SJay Sternberg }
1380e559398fSChristoph Hellwig nvme_init_request(abort_req, &cmd);
138157dacad5SJay Sternberg
1382e2e53086SChristoph Hellwig abort_req->end_io = abort_endio;
1383e7a2a87dSChristoph Hellwig abort_req->end_io_data = NULL;
1384e2e53086SChristoph Hellwig blk_execute_rq_nowait(abort_req, false);
138557dacad5SJay Sternberg
138657dacad5SJay Sternberg /*
138757dacad5SJay Sternberg * The aborted req will be completed on receiving the abort req.
138857dacad5SJay Sternberg * We enable the timer again. If hit twice, it'll cause a device reset,
138957dacad5SJay Sternberg * as the device then is in a faulty state.
139057dacad5SJay Sternberg */
139157dacad5SJay Sternberg return BLK_EH_RESET_TIMER;
139271a5bb15SKeith Busch
139371a5bb15SKeith Busch disable:
1394b6eaa53fSNilay Shroff if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
1395b6eaa53fSNilay Shroff if (nvme_state_terminal(&dev->ctrl))
1396b6eaa53fSNilay Shroff nvme_dev_disable(dev, true);
139771a5bb15SKeith Busch return BLK_EH_DONE;
1398b6eaa53fSNilay Shroff }
139971a5bb15SKeith Busch
140071a5bb15SKeith Busch nvme_dev_disable(dev, false);
140171a5bb15SKeith Busch if (nvme_try_sched_reset(&dev->ctrl))
140271a5bb15SKeith Busch nvme_unquiesce_io_queues(&dev->ctrl);
140371a5bb15SKeith Busch return BLK_EH_DONE;
140457dacad5SJay Sternberg }
140557dacad5SJay Sternberg
nvme_free_queue(struct nvme_queue * nvmeq)140657dacad5SJay Sternberg static void nvme_free_queue(struct nvme_queue *nvmeq)
140757dacad5SJay Sternberg {
14088a1d09a6SBenjamin Herrenschmidt dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
140957dacad5SJay Sternberg (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
141063223078SChristoph Hellwig if (!nvmeq->sq_cmds)
141163223078SChristoph Hellwig return;
14120f238ff5SLogan Gunthorpe
141363223078SChristoph Hellwig if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
141488a041f4SKeith Busch pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
14158a1d09a6SBenjamin Herrenschmidt nvmeq->sq_cmds, SQ_SIZE(nvmeq));
141663223078SChristoph Hellwig } else {
14178a1d09a6SBenjamin Herrenschmidt dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
141863223078SChristoph Hellwig nvmeq->sq_cmds, nvmeq->sq_dma_addr);
14190f238ff5SLogan Gunthorpe }
142057dacad5SJay Sternberg }
142157dacad5SJay Sternberg
nvme_free_queues(struct nvme_dev * dev,int lowest)142257dacad5SJay Sternberg static void nvme_free_queues(struct nvme_dev *dev, int lowest)
142357dacad5SJay Sternberg {
142457dacad5SJay Sternberg int i;
142557dacad5SJay Sternberg
1426d858e5f0SSagi Grimberg for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1427d858e5f0SSagi Grimberg dev->ctrl.queue_count--;
1428147b27e4SSagi Grimberg nvme_free_queue(&dev->queues[i]);
142957dacad5SJay Sternberg }
143057dacad5SJay Sternberg }
143157dacad5SJay Sternberg
nvme_suspend_queue(struct nvme_dev * dev,unsigned int qid)143210981f23SChristoph Hellwig static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
143357dacad5SJay Sternberg {
143410981f23SChristoph Hellwig struct nvme_queue *nvmeq = &dev->queues[qid];
143510981f23SChristoph Hellwig
14364e224106SChristoph Hellwig if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
143710981f23SChristoph Hellwig return;
143857dacad5SJay Sternberg
14394e224106SChristoph Hellwig /* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
1440d1f06f4aSJens Axboe mb();
144157dacad5SJay Sternberg
14424e224106SChristoph Hellwig nvmeq->dev->online_queues--;
14431c63dc66SChristoph Hellwig if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
14449f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
14457c349ddeSKeith Busch if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
144610981f23SChristoph Hellwig pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
144757dacad5SJay Sternberg }
144857dacad5SJay Sternberg
nvme_suspend_io_queues(struct nvme_dev * dev)14498fae268bSKeith Busch static void nvme_suspend_io_queues(struct nvme_dev *dev)
14508fae268bSKeith Busch {
14518fae268bSKeith Busch int i;
14528fae268bSKeith Busch
14538fae268bSKeith Busch for (i = dev->ctrl.queue_count - 1; i > 0; i--)
145410981f23SChristoph Hellwig nvme_suspend_queue(dev, i);
145557dacad5SJay Sternberg }
145657dacad5SJay Sternberg
1457fa46c6fbSKeith Busch /*
1458fa46c6fbSKeith Busch * Called only on a device that has been disabled and after all other threads
14599210c075SDongli Zhang * that can check this device's completion queues have synced, except
14609210c075SDongli Zhang * nvme_poll(). This is the last chance for the driver to see a natural
14619210c075SDongli Zhang * completion before nvme_cancel_request() terminates all incomplete requests.
1462fa46c6fbSKeith Busch */
nvme_reap_pending_cqes(struct nvme_dev * dev)1463fa46c6fbSKeith Busch static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1464fa46c6fbSKeith Busch {
1465fa46c6fbSKeith Busch int i;
1466fa46c6fbSKeith Busch
14679210c075SDongli Zhang for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
14689210c075SDongli Zhang spin_lock(&dev->queues[i].cq_poll_lock);
1469c234a653SJens Axboe nvme_poll_cq(&dev->queues[i], NULL);
14709210c075SDongli Zhang spin_unlock(&dev->queues[i].cq_poll_lock);
14719210c075SDongli Zhang }
1472fa46c6fbSKeith Busch }
1473fa46c6fbSKeith Busch
nvme_cmb_qdepth(struct nvme_dev * dev,int nr_io_queues,int entry_size)147457dacad5SJay Sternberg static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
147557dacad5SJay Sternberg int entry_size)
147657dacad5SJay Sternberg {
147757dacad5SJay Sternberg int q_depth = dev->q_depth;
14785fd4ce1bSChristoph Hellwig unsigned q_size_aligned = roundup(q_depth * entry_size,
14796c3c05b0SChaitanya Kulkarni NVME_CTRL_PAGE_SIZE);
148057dacad5SJay Sternberg
148157dacad5SJay Sternberg if (q_size_aligned * nr_io_queues > dev->cmb_size) {
148257dacad5SJay Sternberg u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
14834e523547SBaolin Wang
14846c3c05b0SChaitanya Kulkarni mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);
148557dacad5SJay Sternberg q_depth = div_u64(mem_per_q, entry_size);
148657dacad5SJay Sternberg
148757dacad5SJay Sternberg /*
148857dacad5SJay Sternberg * Ensure the reduced q_depth is above some threshold where it
148957dacad5SJay Sternberg * would be better to map queues in system memory with the
149057dacad5SJay Sternberg * original depth
149157dacad5SJay Sternberg */
149257dacad5SJay Sternberg if (q_depth < 64)
149357dacad5SJay Sternberg return -ENOMEM;
149457dacad5SJay Sternberg }
149557dacad5SJay Sternberg
149657dacad5SJay Sternberg return q_depth;
149757dacad5SJay Sternberg }
149857dacad5SJay Sternberg
nvme_alloc_sq_cmds(struct nvme_dev * dev,struct nvme_queue * nvmeq,int qid)149957dacad5SJay Sternberg static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
15008a1d09a6SBenjamin Herrenschmidt int qid)
150157dacad5SJay Sternberg {
15020f238ff5SLogan Gunthorpe struct pci_dev *pdev = to_pci_dev(dev->dev);
1503815c6704SKeith Busch
15040f238ff5SLogan Gunthorpe if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
15058a1d09a6SBenjamin Herrenschmidt nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq));
1506bfac8e9fSAlan Mikhak if (nvmeq->sq_cmds) {
15070f238ff5SLogan Gunthorpe nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev,
15080f238ff5SLogan Gunthorpe nvmeq->sq_cmds);
150963223078SChristoph Hellwig if (nvmeq->sq_dma_addr) {
151063223078SChristoph Hellwig set_bit(NVMEQ_SQ_CMB, &nvmeq->flags);
151163223078SChristoph Hellwig return 0;
151263223078SChristoph Hellwig }
1513bfac8e9fSAlan Mikhak
15148a1d09a6SBenjamin Herrenschmidt pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1515bfac8e9fSAlan Mikhak }
15160f238ff5SLogan Gunthorpe }
15170f238ff5SLogan Gunthorpe
15188a1d09a6SBenjamin Herrenschmidt nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq),
151957dacad5SJay Sternberg &nvmeq->sq_dma_addr, GFP_KERNEL);
152057dacad5SJay Sternberg if (!nvmeq->sq_cmds)
152157dacad5SJay Sternberg return -ENOMEM;
152257dacad5SJay Sternberg return 0;
152357dacad5SJay Sternberg }
152457dacad5SJay Sternberg
nvme_alloc_queue(struct nvme_dev * dev,int qid,int depth)1525a6ff7262SKeith Busch static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
152657dacad5SJay Sternberg {
1527147b27e4SSagi Grimberg struct nvme_queue *nvmeq = &dev->queues[qid];
152857dacad5SJay Sternberg
152962314e40SKeith Busch if (dev->ctrl.queue_count > qid)
153062314e40SKeith Busch return 0;
153157dacad5SJay Sternberg
1532c1e0cc7eSBenjamin Herrenschmidt nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
15338a1d09a6SBenjamin Herrenschmidt nvmeq->q_depth = depth;
15348a1d09a6SBenjamin Herrenschmidt nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq),
153557dacad5SJay Sternberg &nvmeq->cq_dma_addr, GFP_KERNEL);
153657dacad5SJay Sternberg if (!nvmeq->cqes)
153757dacad5SJay Sternberg goto free_nvmeq;
153857dacad5SJay Sternberg
15398a1d09a6SBenjamin Herrenschmidt if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
154057dacad5SJay Sternberg goto free_cqdma;
154157dacad5SJay Sternberg
154257dacad5SJay Sternberg nvmeq->dev = dev;
15431ab0cd69SJens Axboe spin_lock_init(&nvmeq->sq_lock);
15443a7afd8eSChristoph Hellwig spin_lock_init(&nvmeq->cq_poll_lock);
154557dacad5SJay Sternberg nvmeq->cq_head = 0;
154657dacad5SJay Sternberg nvmeq->cq_phase = 1;
154757dacad5SJay Sternberg nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
154857dacad5SJay Sternberg nvmeq->qid = qid;
1549d858e5f0SSagi Grimberg dev->ctrl.queue_count++;
155057dacad5SJay Sternberg
1551147b27e4SSagi Grimberg return 0;
155257dacad5SJay Sternberg
155357dacad5SJay Sternberg free_cqdma:
15548a1d09a6SBenjamin Herrenschmidt dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes,
155557dacad5SJay Sternberg nvmeq->cq_dma_addr);
155657dacad5SJay Sternberg free_nvmeq:
1557147b27e4SSagi Grimberg return -ENOMEM;
155857dacad5SJay Sternberg }
155957dacad5SJay Sternberg
queue_request_irq(struct nvme_queue * nvmeq)1560dca51e78SChristoph Hellwig static int queue_request_irq(struct nvme_queue *nvmeq)
156157dacad5SJay Sternberg {
15620ff199cbSChristoph Hellwig struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
15630ff199cbSChristoph Hellwig int nr = nvmeq->dev->ctrl.instance;
15640ff199cbSChristoph Hellwig
15650ff199cbSChristoph Hellwig if (use_threaded_interrupts) {
15660ff199cbSChristoph Hellwig return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
15670ff199cbSChristoph Hellwig nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
15680ff199cbSChristoph Hellwig } else {
15690ff199cbSChristoph Hellwig return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
15700ff199cbSChristoph Hellwig NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
15710ff199cbSChristoph Hellwig }
157257dacad5SJay Sternberg }
157357dacad5SJay Sternberg
nvme_init_queue(struct nvme_queue * nvmeq,u16 qid)157457dacad5SJay Sternberg static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
157557dacad5SJay Sternberg {
157657dacad5SJay Sternberg struct nvme_dev *dev = nvmeq->dev;
157757dacad5SJay Sternberg
157857dacad5SJay Sternberg nvmeq->sq_tail = 0;
157938210800SKeith Busch nvmeq->last_sq_tail = 0;
158057dacad5SJay Sternberg nvmeq->cq_head = 0;
158157dacad5SJay Sternberg nvmeq->cq_phase = 1;
158257dacad5SJay Sternberg nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
15838a1d09a6SBenjamin Herrenschmidt memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
1584f9f38e33SHelen Koike nvme_dbbuf_init(dev, nvmeq, qid);
158557dacad5SJay Sternberg dev->online_queues++;
15863a7afd8eSChristoph Hellwig wmb(); /* ensure the first interrupt sees the initialization */
158757dacad5SJay Sternberg }
158857dacad5SJay Sternberg
1589e4b9852aSCasey Chen /*
1590e4b9852aSCasey Chen * Try getting shutdown_lock while setting up IO queues.
1591e4b9852aSCasey Chen */
nvme_setup_io_queues_trylock(struct nvme_dev * dev)1592e4b9852aSCasey Chen static int nvme_setup_io_queues_trylock(struct nvme_dev *dev)
1593e4b9852aSCasey Chen {
1594e4b9852aSCasey Chen /*
1595e4b9852aSCasey Chen * Give up if the lock is being held by nvme_dev_disable.
1596e4b9852aSCasey Chen */
1597e4b9852aSCasey Chen if (!mutex_trylock(&dev->shutdown_lock))
1598e4b9852aSCasey Chen return -ENODEV;
1599e4b9852aSCasey Chen
1600e4b9852aSCasey Chen /*
1601e4b9852aSCasey Chen * Controller is in wrong state, fail early.
1602e4b9852aSCasey Chen */
16038884a56dSKeith Busch if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_CONNECTING) {
1604e4b9852aSCasey Chen mutex_unlock(&dev->shutdown_lock);
1605e4b9852aSCasey Chen return -ENODEV;
1606e4b9852aSCasey Chen }
1607e4b9852aSCasey Chen
1608e4b9852aSCasey Chen return 0;
1609e4b9852aSCasey Chen }
1610e4b9852aSCasey Chen
nvme_create_queue(struct nvme_queue * nvmeq,int qid,bool polled)16114b04cc6aSJens Axboe static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
161257dacad5SJay Sternberg {
161357dacad5SJay Sternberg struct nvme_dev *dev = nvmeq->dev;
161457dacad5SJay Sternberg int result;
16157c349ddeSKeith Busch u16 vector = 0;
161657dacad5SJay Sternberg
1617d1ed6aa1SChristoph Hellwig clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
1618d1ed6aa1SChristoph Hellwig
161922b55601SKeith Busch /*
162022b55601SKeith Busch * A queue's vector matches the queue identifier unless the controller
162122b55601SKeith Busch * has only one vector available.
162222b55601SKeith Busch */
16234b04cc6aSJens Axboe if (!polled)
1624a8e3e0bbSJianchao Wang vector = dev->num_vecs == 1 ? 0 : qid;
16254b04cc6aSJens Axboe else
16267c349ddeSKeith Busch set_bit(NVMEQ_POLLED, &nvmeq->flags);
16274b04cc6aSJens Axboe
1628a8e3e0bbSJianchao Wang result = adapter_alloc_cq(dev, qid, nvmeq, vector);
1629ded45505SKeith Busch if (result)
1630ded45505SKeith Busch return result;
163157dacad5SJay Sternberg
163257dacad5SJay Sternberg result = adapter_alloc_sq(dev, qid, nvmeq);
163357dacad5SJay Sternberg if (result < 0)
1634ded45505SKeith Busch return result;
1635c80b36cdSEdmund Nadolski if (result)
163657dacad5SJay Sternberg goto release_cq;
163757dacad5SJay Sternberg
1638a8e3e0bbSJianchao Wang nvmeq->cq_vector = vector;
16394b04cc6aSJens Axboe
1640e4b9852aSCasey Chen result = nvme_setup_io_queues_trylock(dev);
1641e4b9852aSCasey Chen if (result)
1642e4b9852aSCasey Chen return result;
1643e4b9852aSCasey Chen nvme_init_queue(nvmeq, qid);
16447c349ddeSKeith Busch if (!polled) {
1645dca51e78SChristoph Hellwig result = queue_request_irq(nvmeq);
164657dacad5SJay Sternberg if (result < 0)
164757dacad5SJay Sternberg goto release_sq;
16484b04cc6aSJens Axboe }
164957dacad5SJay Sternberg
16504e224106SChristoph Hellwig set_bit(NVMEQ_ENABLED, &nvmeq->flags);
1651e4b9852aSCasey Chen mutex_unlock(&dev->shutdown_lock);
165257dacad5SJay Sternberg return result;
165357dacad5SJay Sternberg
165457dacad5SJay Sternberg release_sq:
1655f25a2dfcSJianchao Wang dev->online_queues--;
1656e4b9852aSCasey Chen mutex_unlock(&dev->shutdown_lock);
165757dacad5SJay Sternberg adapter_delete_sq(dev, qid);
165857dacad5SJay Sternberg release_cq:
165957dacad5SJay Sternberg adapter_delete_cq(dev, qid);
166057dacad5SJay Sternberg return result;
166157dacad5SJay Sternberg }
166257dacad5SJay Sternberg
1663f363b089SEric Biggers static const struct blk_mq_ops nvme_mq_admin_ops = {
166457dacad5SJay Sternberg .queue_rq = nvme_queue_rq,
166577f02a7aSChristoph Hellwig .complete = nvme_pci_complete_rq,
166657dacad5SJay Sternberg .init_hctx = nvme_admin_init_hctx,
1667e559398fSChristoph Hellwig .init_request = nvme_pci_init_request,
166857dacad5SJay Sternberg .timeout = nvme_timeout,
166957dacad5SJay Sternberg };
167057dacad5SJay Sternberg
1671f363b089SEric Biggers static const struct blk_mq_ops nvme_mq_ops = {
1672376f7ef8SChristoph Hellwig .queue_rq = nvme_queue_rq,
1673d62cbcf6SJens Axboe .queue_rqs = nvme_queue_rqs,
1674376f7ef8SChristoph Hellwig .complete = nvme_pci_complete_rq,
1675376f7ef8SChristoph Hellwig .commit_rqs = nvme_commit_rqs,
1676376f7ef8SChristoph Hellwig .init_hctx = nvme_init_hctx,
1677e559398fSChristoph Hellwig .init_request = nvme_pci_init_request,
1678376f7ef8SChristoph Hellwig .map_queues = nvme_pci_map_queues,
1679376f7ef8SChristoph Hellwig .timeout = nvme_timeout,
1680c6d962aeSChristoph Hellwig .poll = nvme_poll,
1681dabcefabSJens Axboe };
1682dabcefabSJens Axboe
nvme_dev_remove_admin(struct nvme_dev * dev)168357dacad5SJay Sternberg static void nvme_dev_remove_admin(struct nvme_dev *dev)
168457dacad5SJay Sternberg {
16851c63dc66SChristoph Hellwig if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
168669d9a99cSKeith Busch /*
168769d9a99cSKeith Busch * If the controller was reset during removal, it's possible
168869d9a99cSKeith Busch * user requests may be waiting on a stopped queue. Start the
168969d9a99cSKeith Busch * queue to flush these to completion.
169069d9a99cSKeith Busch */
16919f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(&dev->ctrl);
16920da7feaaSChristoph Hellwig nvme_remove_admin_tag_set(&dev->ctrl);
169357dacad5SJay Sternberg }
169457dacad5SJay Sternberg }
169557dacad5SJay Sternberg
db_bar_size(struct nvme_dev * dev,unsigned nr_io_queues)169697f6ef64SXu Yu static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
169797f6ef64SXu Yu {
169897f6ef64SXu Yu return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
169997f6ef64SXu Yu }
170097f6ef64SXu Yu
nvme_remap_bar(struct nvme_dev * dev,unsigned long size)170197f6ef64SXu Yu static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
170297f6ef64SXu Yu {
170397f6ef64SXu Yu struct pci_dev *pdev = to_pci_dev(dev->dev);
170497f6ef64SXu Yu
170597f6ef64SXu Yu if (size <= dev->bar_mapped_size)
170697f6ef64SXu Yu return 0;
170797f6ef64SXu Yu if (size > pci_resource_len(pdev, 0))
170897f6ef64SXu Yu return -ENOMEM;
170997f6ef64SXu Yu if (dev->bar)
171097f6ef64SXu Yu iounmap(dev->bar);
171197f6ef64SXu Yu dev->bar = ioremap(pci_resource_start(pdev, 0), size);
171297f6ef64SXu Yu if (!dev->bar) {
171397f6ef64SXu Yu dev->bar_mapped_size = 0;
171497f6ef64SXu Yu return -ENOMEM;
171597f6ef64SXu Yu }
171697f6ef64SXu Yu dev->bar_mapped_size = size;
171797f6ef64SXu Yu dev->dbs = dev->bar + NVME_REG_DBS;
171897f6ef64SXu Yu
171997f6ef64SXu Yu return 0;
172097f6ef64SXu Yu }
172197f6ef64SXu Yu
nvme_pci_configure_admin_queue(struct nvme_dev * dev)172201ad0990SSagi Grimberg static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
172357dacad5SJay Sternberg {
172457dacad5SJay Sternberg int result;
172557dacad5SJay Sternberg u32 aqa;
172657dacad5SJay Sternberg struct nvme_queue *nvmeq;
172757dacad5SJay Sternberg
172897f6ef64SXu Yu result = nvme_remap_bar(dev, db_bar_size(dev, 0));
172997f6ef64SXu Yu if (result < 0)
173097f6ef64SXu Yu return result;
173197f6ef64SXu Yu
17328ef2074dSGabriel Krisman Bertazi dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
173320d0dfe6SSagi Grimberg NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
173457dacad5SJay Sternberg
17357a67cbeaSChristoph Hellwig if (dev->subsystem &&
17367a67cbeaSChristoph Hellwig (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
17377a67cbeaSChristoph Hellwig writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
173857dacad5SJay Sternberg
1739285b6e9bSChristoph Hellwig /*
1740285b6e9bSChristoph Hellwig * If the device has been passed off to us in an enabled state, just
1741285b6e9bSChristoph Hellwig * clear the enabled bit. The spec says we should set the 'shutdown
1742285b6e9bSChristoph Hellwig * notification bits', but doing so may cause the device to complete
1743285b6e9bSChristoph Hellwig * commands to the admin queue ... and we don't know what memory that
1744285b6e9bSChristoph Hellwig * might be pointing at!
1745285b6e9bSChristoph Hellwig */
1746285b6e9bSChristoph Hellwig result = nvme_disable_ctrl(&dev->ctrl, false);
174757dacad5SJay Sternberg if (result < 0)
174857dacad5SJay Sternberg return result;
174957dacad5SJay Sternberg
1750a6ff7262SKeith Busch result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1751147b27e4SSagi Grimberg if (result)
1752147b27e4SSagi Grimberg return result;
175357dacad5SJay Sternberg
1754635333e4SMax Gurtovoy dev->ctrl.numa_node = dev_to_node(dev->dev);
1755635333e4SMax Gurtovoy
1756147b27e4SSagi Grimberg nvmeq = &dev->queues[0];
175757dacad5SJay Sternberg aqa = nvmeq->q_depth - 1;
175857dacad5SJay Sternberg aqa |= aqa << 16;
175957dacad5SJay Sternberg
17607a67cbeaSChristoph Hellwig writel(aqa, dev->bar + NVME_REG_AQA);
17617a67cbeaSChristoph Hellwig lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
17627a67cbeaSChristoph Hellwig lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
176357dacad5SJay Sternberg
1764c0f2f45bSSagi Grimberg result = nvme_enable_ctrl(&dev->ctrl);
176557dacad5SJay Sternberg if (result)
1766d4875622SKeith Busch return result;
176757dacad5SJay Sternberg
176857dacad5SJay Sternberg nvmeq->cq_vector = 0;
1769161b8be2SKeith Busch nvme_init_queue(nvmeq, 0);
1770dca51e78SChristoph Hellwig result = queue_request_irq(nvmeq);
177157dacad5SJay Sternberg if (result) {
17727c349ddeSKeith Busch dev->online_queues--;
1773d4875622SKeith Busch return result;
177457dacad5SJay Sternberg }
177557dacad5SJay Sternberg
17764e224106SChristoph Hellwig set_bit(NVMEQ_ENABLED, &nvmeq->flags);
177757dacad5SJay Sternberg return result;
177857dacad5SJay Sternberg }
177957dacad5SJay Sternberg
nvme_create_io_queues(struct nvme_dev * dev)1780749941f2SChristoph Hellwig static int nvme_create_io_queues(struct nvme_dev *dev)
178157dacad5SJay Sternberg {
17824b04cc6aSJens Axboe unsigned i, max, rw_queues;
1783749941f2SChristoph Hellwig int ret = 0;
178457dacad5SJay Sternberg
1785d858e5f0SSagi Grimberg for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1786a6ff7262SKeith Busch if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1787749941f2SChristoph Hellwig ret = -ENOMEM;
178857dacad5SJay Sternberg break;
1789749941f2SChristoph Hellwig }
1790749941f2SChristoph Hellwig }
179157dacad5SJay Sternberg
1792d858e5f0SSagi Grimberg max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1793e20ba6e1SChristoph Hellwig if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
1794e20ba6e1SChristoph Hellwig rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
1795e20ba6e1SChristoph Hellwig dev->io_queues[HCTX_TYPE_READ];
17964b04cc6aSJens Axboe } else {
17974b04cc6aSJens Axboe rw_queues = max;
17984b04cc6aSJens Axboe }
17994b04cc6aSJens Axboe
1800949928c1SKeith Busch for (i = dev->online_queues; i <= max; i++) {
18014b04cc6aSJens Axboe bool polled = i > rw_queues;
18024b04cc6aSJens Axboe
18034b04cc6aSJens Axboe ret = nvme_create_queue(&dev->queues[i], i, polled);
1804d4875622SKeith Busch if (ret)
180557dacad5SJay Sternberg break;
180657dacad5SJay Sternberg }
180757dacad5SJay Sternberg
1808749941f2SChristoph Hellwig /*
1809749941f2SChristoph Hellwig * Ignore failing Create SQ/CQ commands, we can continue with less
18108adb8c14SMinwoo Im * than the desired amount of queues, and even a controller without
18118adb8c14SMinwoo Im * I/O queues can still be used to issue admin commands. This might
1812749941f2SChristoph Hellwig * be useful to upgrade a buggy firmware for example.
1813749941f2SChristoph Hellwig */
1814749941f2SChristoph Hellwig return ret >= 0 ? 0 : ret;
181557dacad5SJay Sternberg }
181657dacad5SJay Sternberg
nvme_cmb_size_unit(struct nvme_dev * dev)181788de4598SChristoph Hellwig static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
181857dacad5SJay Sternberg {
181988de4598SChristoph Hellwig u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
182088de4598SChristoph Hellwig
182188de4598SChristoph Hellwig return 1ULL << (12 + 4 * szu);
182288de4598SChristoph Hellwig }
182388de4598SChristoph Hellwig
nvme_cmb_size(struct nvme_dev * dev)182488de4598SChristoph Hellwig static u32 nvme_cmb_size(struct nvme_dev *dev)
182588de4598SChristoph Hellwig {
182688de4598SChristoph Hellwig return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
182788de4598SChristoph Hellwig }
182888de4598SChristoph Hellwig
nvme_map_cmb(struct nvme_dev * dev)1829f65efd6dSChristoph Hellwig static void nvme_map_cmb(struct nvme_dev *dev)
183057dacad5SJay Sternberg {
183188de4598SChristoph Hellwig u64 size, offset;
183257dacad5SJay Sternberg resource_size_t bar_size;
183357dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev->dev);
18348969f1f8SChristoph Hellwig int bar;
183557dacad5SJay Sternberg
18369fe5c59fSKeith Busch if (dev->cmb_size)
18379fe5c59fSKeith Busch return;
18389fe5c59fSKeith Busch
183920d3bb92SKlaus Jensen if (NVME_CAP_CMBS(dev->ctrl.cap))
184020d3bb92SKlaus Jensen writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC);
184120d3bb92SKlaus Jensen
18427a67cbeaSChristoph Hellwig dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1843f65efd6dSChristoph Hellwig if (!dev->cmbsz)
1844f65efd6dSChristoph Hellwig return;
1845202021c1SStephen Bates dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
184657dacad5SJay Sternberg
184788de4598SChristoph Hellwig size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
184888de4598SChristoph Hellwig offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
18498969f1f8SChristoph Hellwig bar = NVME_CMB_BIR(dev->cmbloc);
18508969f1f8SChristoph Hellwig bar_size = pci_resource_len(pdev, bar);
185157dacad5SJay Sternberg
185257dacad5SJay Sternberg if (offset > bar_size)
1853f65efd6dSChristoph Hellwig return;
185457dacad5SJay Sternberg
185557dacad5SJay Sternberg /*
185620d3bb92SKlaus Jensen * Tell the controller about the host side address mapping the CMB,
185720d3bb92SKlaus Jensen * and enable CMB decoding for the NVMe 1.4+ scheme:
185820d3bb92SKlaus Jensen */
185920d3bb92SKlaus Jensen if (NVME_CAP_CMBS(dev->ctrl.cap)) {
186020d3bb92SKlaus Jensen hi_lo_writeq(NVME_CMBMSC_CRE | NVME_CMBMSC_CMSE |
186120d3bb92SKlaus Jensen (pci_bus_address(pdev, bar) + offset),
186220d3bb92SKlaus Jensen dev->bar + NVME_REG_CMBMSC);
186320d3bb92SKlaus Jensen }
186420d3bb92SKlaus Jensen
186520d3bb92SKlaus Jensen /*
186657dacad5SJay Sternberg * Controllers may support a CMB size larger than their BAR,
186757dacad5SJay Sternberg * for example, due to being behind a bridge. Reduce the CMB to
186857dacad5SJay Sternberg * the reported size of the BAR
186957dacad5SJay Sternberg */
187057dacad5SJay Sternberg if (size > bar_size - offset)
187157dacad5SJay Sternberg size = bar_size - offset;
187257dacad5SJay Sternberg
18730f238ff5SLogan Gunthorpe if (pci_p2pdma_add_resource(pdev, bar, size, offset)) {
18740f238ff5SLogan Gunthorpe dev_warn(dev->ctrl.device,
18750f238ff5SLogan Gunthorpe "failed to register the CMB\n");
1876f65efd6dSChristoph Hellwig return;
18770f238ff5SLogan Gunthorpe }
18780f238ff5SLogan Gunthorpe
187957dacad5SJay Sternberg dev->cmb_size = size;
18800f238ff5SLogan Gunthorpe dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
18810f238ff5SLogan Gunthorpe
18820f238ff5SLogan Gunthorpe if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
18830f238ff5SLogan Gunthorpe (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS))
18840f238ff5SLogan Gunthorpe pci_p2pmem_publish(pdev, true);
1885e917a849SKeith Busch
1886e917a849SKeith Busch nvme_update_attrs(dev);
188757dacad5SJay Sternberg }
188857dacad5SJay Sternberg
nvme_set_host_mem(struct nvme_dev * dev,u32 bits)188987ad72a5SChristoph Hellwig static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
189057dacad5SJay Sternberg {
18916c3c05b0SChaitanya Kulkarni u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
18924033f35dSChristoph Hellwig u64 dma_addr = dev->host_mem_descs_dma;
1893f66e2804SChaitanya Kulkarni struct nvme_command c = { };
189487ad72a5SChristoph Hellwig int ret;
189587ad72a5SChristoph Hellwig
189687ad72a5SChristoph Hellwig c.features.opcode = nvme_admin_set_features;
189787ad72a5SChristoph Hellwig c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
189887ad72a5SChristoph Hellwig c.features.dword11 = cpu_to_le32(bits);
18996c3c05b0SChaitanya Kulkarni c.features.dword12 = cpu_to_le32(host_mem_size);
190087ad72a5SChristoph Hellwig c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr));
190187ad72a5SChristoph Hellwig c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr));
190287ad72a5SChristoph Hellwig c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs);
190387ad72a5SChristoph Hellwig
190487ad72a5SChristoph Hellwig ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
190587ad72a5SChristoph Hellwig if (ret) {
190687ad72a5SChristoph Hellwig dev_warn(dev->ctrl.device,
190787ad72a5SChristoph Hellwig "failed to set host mem (err %d, flags %#x).\n",
190887ad72a5SChristoph Hellwig ret, bits);
1909a5df5e79SKeith Busch } else
1910a5df5e79SKeith Busch dev->hmb = bits & NVME_HOST_MEM_ENABLE;
1911a5df5e79SKeith Busch
191287ad72a5SChristoph Hellwig return ret;
191387ad72a5SChristoph Hellwig }
191487ad72a5SChristoph Hellwig
nvme_free_host_mem(struct nvme_dev * dev)191587ad72a5SChristoph Hellwig static void nvme_free_host_mem(struct nvme_dev *dev)
191687ad72a5SChristoph Hellwig {
191787ad72a5SChristoph Hellwig int i;
191887ad72a5SChristoph Hellwig
191987ad72a5SChristoph Hellwig for (i = 0; i < dev->nr_host_mem_descs; i++) {
192087ad72a5SChristoph Hellwig struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
19216c3c05b0SChaitanya Kulkarni size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;
192287ad72a5SChristoph Hellwig
1923cc667f6dSLiviu Dudau dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
1924cc667f6dSLiviu Dudau le64_to_cpu(desc->addr),
1925cc667f6dSLiviu Dudau DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
192687ad72a5SChristoph Hellwig }
192787ad72a5SChristoph Hellwig
192887ad72a5SChristoph Hellwig kfree(dev->host_mem_desc_bufs);
192987ad72a5SChristoph Hellwig dev->host_mem_desc_bufs = NULL;
19304033f35dSChristoph Hellwig dma_free_coherent(dev->dev,
19314033f35dSChristoph Hellwig dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
19324033f35dSChristoph Hellwig dev->host_mem_descs, dev->host_mem_descs_dma);
193387ad72a5SChristoph Hellwig dev->host_mem_descs = NULL;
19347e5dd57eSMinwoo Im dev->nr_host_mem_descs = 0;
193587ad72a5SChristoph Hellwig }
193687ad72a5SChristoph Hellwig
__nvme_alloc_host_mem(struct nvme_dev * dev,u64 preferred,u32 chunk_size)193792dc6895SChristoph Hellwig static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
193892dc6895SChristoph Hellwig u32 chunk_size)
193987ad72a5SChristoph Hellwig {
194087ad72a5SChristoph Hellwig struct nvme_host_mem_buf_desc *descs;
194192dc6895SChristoph Hellwig u32 max_entries, len;
19424033f35dSChristoph Hellwig dma_addr_t descs_dma;
19432ee0e4edSDan Carpenter int i = 0;
194487ad72a5SChristoph Hellwig void **bufs;
19456fbcde66SMinwoo Im u64 size, tmp;
194687ad72a5SChristoph Hellwig
194787ad72a5SChristoph Hellwig tmp = (preferred + chunk_size - 1);
194887ad72a5SChristoph Hellwig do_div(tmp, chunk_size);
194987ad72a5SChristoph Hellwig max_entries = tmp;
1950044a9df1SChristoph Hellwig
1951044a9df1SChristoph Hellwig if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
1952044a9df1SChristoph Hellwig max_entries = dev->ctrl.hmmaxd;
1953044a9df1SChristoph Hellwig
1954750afb08SLuis Chamberlain descs = dma_alloc_coherent(dev->dev, max_entries * sizeof(*descs),
19554033f35dSChristoph Hellwig &descs_dma, GFP_KERNEL);
195687ad72a5SChristoph Hellwig if (!descs)
195787ad72a5SChristoph Hellwig goto out;
195887ad72a5SChristoph Hellwig
195987ad72a5SChristoph Hellwig bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
196087ad72a5SChristoph Hellwig if (!bufs)
196187ad72a5SChristoph Hellwig goto out_free_descs;
196287ad72a5SChristoph Hellwig
1963244a8fe4SMinwoo Im for (size = 0; size < preferred && i < max_entries; size += len) {
196487ad72a5SChristoph Hellwig dma_addr_t dma_addr;
196587ad72a5SChristoph Hellwig
196650cdb7c6SChristoph Hellwig len = min_t(u64, chunk_size, preferred - size);
196787ad72a5SChristoph Hellwig bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
196887ad72a5SChristoph Hellwig DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
196987ad72a5SChristoph Hellwig if (!bufs[i])
197087ad72a5SChristoph Hellwig break;
197187ad72a5SChristoph Hellwig
197287ad72a5SChristoph Hellwig descs[i].addr = cpu_to_le64(dma_addr);
19736c3c05b0SChaitanya Kulkarni descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);
197487ad72a5SChristoph Hellwig i++;
197587ad72a5SChristoph Hellwig }
197687ad72a5SChristoph Hellwig
197792dc6895SChristoph Hellwig if (!size)
197887ad72a5SChristoph Hellwig goto out_free_bufs;
197987ad72a5SChristoph Hellwig
198087ad72a5SChristoph Hellwig dev->nr_host_mem_descs = i;
198187ad72a5SChristoph Hellwig dev->host_mem_size = size;
198287ad72a5SChristoph Hellwig dev->host_mem_descs = descs;
19834033f35dSChristoph Hellwig dev->host_mem_descs_dma = descs_dma;
198487ad72a5SChristoph Hellwig dev->host_mem_desc_bufs = bufs;
198587ad72a5SChristoph Hellwig return 0;
198687ad72a5SChristoph Hellwig
198787ad72a5SChristoph Hellwig out_free_bufs:
198887ad72a5SChristoph Hellwig while (--i >= 0) {
19896c3c05b0SChaitanya Kulkarni size_t size = le32_to_cpu(descs[i].size) * NVME_CTRL_PAGE_SIZE;
199087ad72a5SChristoph Hellwig
1991cc667f6dSLiviu Dudau dma_free_attrs(dev->dev, size, bufs[i],
1992cc667f6dSLiviu Dudau le64_to_cpu(descs[i].addr),
1993cc667f6dSLiviu Dudau DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
199487ad72a5SChristoph Hellwig }
199587ad72a5SChristoph Hellwig
199687ad72a5SChristoph Hellwig kfree(bufs);
199787ad72a5SChristoph Hellwig out_free_descs:
19984033f35dSChristoph Hellwig dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
19994033f35dSChristoph Hellwig descs_dma);
200087ad72a5SChristoph Hellwig out:
200187ad72a5SChristoph Hellwig dev->host_mem_descs = NULL;
200287ad72a5SChristoph Hellwig return -ENOMEM;
200387ad72a5SChristoph Hellwig }
200487ad72a5SChristoph Hellwig
nvme_alloc_host_mem(struct nvme_dev * dev,u64 min,u64 preferred)200592dc6895SChristoph Hellwig static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
200692dc6895SChristoph Hellwig {
20079dc54a0dSChaitanya Kulkarni u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
20089dc54a0dSChaitanya Kulkarni u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
20099dc54a0dSChaitanya Kulkarni u64 chunk_size;
201092dc6895SChristoph Hellwig
201192dc6895SChristoph Hellwig /* start big and work our way down */
20129dc54a0dSChaitanya Kulkarni for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {
201392dc6895SChristoph Hellwig if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
201492dc6895SChristoph Hellwig if (!min || dev->host_mem_size >= min)
201592dc6895SChristoph Hellwig return 0;
201692dc6895SChristoph Hellwig nvme_free_host_mem(dev);
201792dc6895SChristoph Hellwig }
201892dc6895SChristoph Hellwig }
201992dc6895SChristoph Hellwig
202092dc6895SChristoph Hellwig return -ENOMEM;
202192dc6895SChristoph Hellwig }
202292dc6895SChristoph Hellwig
nvme_setup_host_mem(struct nvme_dev * dev)20239620cfbaSChristoph Hellwig static int nvme_setup_host_mem(struct nvme_dev *dev)
202487ad72a5SChristoph Hellwig {
202587ad72a5SChristoph Hellwig u64 max = (u64)max_host_mem_size_mb * SZ_1M;
202687ad72a5SChristoph Hellwig u64 preferred = (u64)dev->ctrl.hmpre * 4096;
202787ad72a5SChristoph Hellwig u64 min = (u64)dev->ctrl.hmmin * 4096;
202887ad72a5SChristoph Hellwig u32 enable_bits = NVME_HOST_MEM_ENABLE;
20296fbcde66SMinwoo Im int ret;
203087ad72a5SChristoph Hellwig
2031acb71e53SChristoph Hellwig if (!dev->ctrl.hmpre)
2032acb71e53SChristoph Hellwig return 0;
2033acb71e53SChristoph Hellwig
203487ad72a5SChristoph Hellwig preferred = min(preferred, max);
203587ad72a5SChristoph Hellwig if (min > max) {
203687ad72a5SChristoph Hellwig dev_warn(dev->ctrl.device,
203787ad72a5SChristoph Hellwig "min host memory (%lld MiB) above limit (%d MiB).\n",
203887ad72a5SChristoph Hellwig min >> ilog2(SZ_1M), max_host_mem_size_mb);
203987ad72a5SChristoph Hellwig nvme_free_host_mem(dev);
20409620cfbaSChristoph Hellwig return 0;
204187ad72a5SChristoph Hellwig }
204287ad72a5SChristoph Hellwig
204387ad72a5SChristoph Hellwig /*
204487ad72a5SChristoph Hellwig * If we already have a buffer allocated check if we can reuse it.
204587ad72a5SChristoph Hellwig */
204687ad72a5SChristoph Hellwig if (dev->host_mem_descs) {
204787ad72a5SChristoph Hellwig if (dev->host_mem_size >= min)
204887ad72a5SChristoph Hellwig enable_bits |= NVME_HOST_MEM_RETURN;
204987ad72a5SChristoph Hellwig else
205087ad72a5SChristoph Hellwig nvme_free_host_mem(dev);
205187ad72a5SChristoph Hellwig }
205287ad72a5SChristoph Hellwig
205387ad72a5SChristoph Hellwig if (!dev->host_mem_descs) {
205492dc6895SChristoph Hellwig if (nvme_alloc_host_mem(dev, min, preferred)) {
205592dc6895SChristoph Hellwig dev_warn(dev->ctrl.device,
205692dc6895SChristoph Hellwig "failed to allocate host memory buffer.\n");
20579620cfbaSChristoph Hellwig return 0; /* controller must work without HMB */
205887ad72a5SChristoph Hellwig }
205987ad72a5SChristoph Hellwig
206092dc6895SChristoph Hellwig dev_info(dev->ctrl.device,
206192dc6895SChristoph Hellwig "allocated %lld MiB host memory buffer.\n",
206292dc6895SChristoph Hellwig dev->host_mem_size >> ilog2(SZ_1M));
206392dc6895SChristoph Hellwig }
206492dc6895SChristoph Hellwig
20659620cfbaSChristoph Hellwig ret = nvme_set_host_mem(dev, enable_bits);
20669620cfbaSChristoph Hellwig if (ret)
206787ad72a5SChristoph Hellwig nvme_free_host_mem(dev);
20689620cfbaSChristoph Hellwig return ret;
206957dacad5SJay Sternberg }
207057dacad5SJay Sternberg
cmb_show(struct device * dev,struct device_attribute * attr,char * buf)20710521905eSKeith Busch static ssize_t cmb_show(struct device *dev, struct device_attribute *attr,
20720521905eSKeith Busch char *buf)
20730521905eSKeith Busch {
20740521905eSKeith Busch struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
20750521905eSKeith Busch
20760521905eSKeith Busch return sysfs_emit(buf, "cmbloc : x%08x\ncmbsz : x%08x\n",
20770521905eSKeith Busch ndev->cmbloc, ndev->cmbsz);
20780521905eSKeith Busch }
20790521905eSKeith Busch static DEVICE_ATTR_RO(cmb);
20800521905eSKeith Busch
cmbloc_show(struct device * dev,struct device_attribute * attr,char * buf)20811751e97aSKeith Busch static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr,
20821751e97aSKeith Busch char *buf)
20831751e97aSKeith Busch {
20841751e97aSKeith Busch struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
20851751e97aSKeith Busch
20861751e97aSKeith Busch return sysfs_emit(buf, "%u\n", ndev->cmbloc);
20871751e97aSKeith Busch }
20881751e97aSKeith Busch static DEVICE_ATTR_RO(cmbloc);
20891751e97aSKeith Busch
cmbsz_show(struct device * dev,struct device_attribute * attr,char * buf)20901751e97aSKeith Busch static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr,
20911751e97aSKeith Busch char *buf)
20921751e97aSKeith Busch {
20931751e97aSKeith Busch struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
20941751e97aSKeith Busch
20951751e97aSKeith Busch return sysfs_emit(buf, "%u\n", ndev->cmbsz);
20961751e97aSKeith Busch }
20971751e97aSKeith Busch static DEVICE_ATTR_RO(cmbsz);
20981751e97aSKeith Busch
hmb_show(struct device * dev,struct device_attribute * attr,char * buf)2099a5df5e79SKeith Busch static ssize_t hmb_show(struct device *dev, struct device_attribute *attr,
2100a5df5e79SKeith Busch char *buf)
2101a5df5e79SKeith Busch {
2102a5df5e79SKeith Busch struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2103a5df5e79SKeith Busch
2104a5df5e79SKeith Busch return sysfs_emit(buf, "%d\n", ndev->hmb);
2105a5df5e79SKeith Busch }
2106a5df5e79SKeith Busch
hmb_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2107a5df5e79SKeith Busch static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
2108a5df5e79SKeith Busch const char *buf, size_t count)
2109a5df5e79SKeith Busch {
2110a5df5e79SKeith Busch struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2111a5df5e79SKeith Busch bool new;
2112a5df5e79SKeith Busch int ret;
2113a5df5e79SKeith Busch
211499722c8aSChristophe JAILLET if (kstrtobool(buf, &new) < 0)
2115a5df5e79SKeith Busch return -EINVAL;
2116a5df5e79SKeith Busch
2117a5df5e79SKeith Busch if (new == ndev->hmb)
2118a5df5e79SKeith Busch return count;
2119a5df5e79SKeith Busch
2120a5df5e79SKeith Busch if (new) {
2121a5df5e79SKeith Busch ret = nvme_setup_host_mem(ndev);
2122a5df5e79SKeith Busch } else {
2123a5df5e79SKeith Busch ret = nvme_set_host_mem(ndev, 0);
2124a5df5e79SKeith Busch if (!ret)
2125a5df5e79SKeith Busch nvme_free_host_mem(ndev);
2126a5df5e79SKeith Busch }
2127a5df5e79SKeith Busch
2128a5df5e79SKeith Busch if (ret < 0)
2129a5df5e79SKeith Busch return ret;
2130a5df5e79SKeith Busch
2131a5df5e79SKeith Busch return count;
2132a5df5e79SKeith Busch }
2133a5df5e79SKeith Busch static DEVICE_ATTR_RW(hmb);
2134a5df5e79SKeith Busch
nvme_pci_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)21350521905eSKeith Busch static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
21360521905eSKeith Busch struct attribute *a, int n)
21370521905eSKeith Busch {
21380521905eSKeith Busch struct nvme_ctrl *ctrl =
21390521905eSKeith Busch dev_get_drvdata(container_of(kobj, struct device, kobj));
21400521905eSKeith Busch struct nvme_dev *dev = to_nvme_dev(ctrl);
21410521905eSKeith Busch
21421751e97aSKeith Busch if (a == &dev_attr_cmb.attr ||
21431751e97aSKeith Busch a == &dev_attr_cmbloc.attr ||
21441751e97aSKeith Busch a == &dev_attr_cmbsz.attr) {
21451751e97aSKeith Busch if (!dev->cmbsz)
21460521905eSKeith Busch return 0;
21471751e97aSKeith Busch }
2148a5df5e79SKeith Busch if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
2149a5df5e79SKeith Busch return 0;
2150a5df5e79SKeith Busch
21510521905eSKeith Busch return a->mode;
21520521905eSKeith Busch }
21530521905eSKeith Busch
21540521905eSKeith Busch static struct attribute *nvme_pci_attrs[] = {
21550521905eSKeith Busch &dev_attr_cmb.attr,
21561751e97aSKeith Busch &dev_attr_cmbloc.attr,
21571751e97aSKeith Busch &dev_attr_cmbsz.attr,
2158a5df5e79SKeith Busch &dev_attr_hmb.attr,
21590521905eSKeith Busch NULL,
21600521905eSKeith Busch };
21610521905eSKeith Busch
216286adbf0cSChristoph Hellwig static const struct attribute_group nvme_pci_dev_attrs_group = {
21630521905eSKeith Busch .attrs = nvme_pci_attrs,
21640521905eSKeith Busch .is_visible = nvme_pci_attrs_are_visible,
21650521905eSKeith Busch };
21660521905eSKeith Busch
216786adbf0cSChristoph Hellwig static const struct attribute_group *nvme_pci_dev_attr_groups[] = {
216886adbf0cSChristoph Hellwig &nvme_dev_attrs_group,
216986adbf0cSChristoph Hellwig &nvme_pci_dev_attrs_group,
217086adbf0cSChristoph Hellwig NULL,
217186adbf0cSChristoph Hellwig };
217286adbf0cSChristoph Hellwig
nvme_update_attrs(struct nvme_dev * dev)2173e917a849SKeith Busch static void nvme_update_attrs(struct nvme_dev *dev)
2174e917a849SKeith Busch {
2175e917a849SKeith Busch sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group);
2176e917a849SKeith Busch }
2177e917a849SKeith Busch
2178612b7286SMing Lei /*
2179612b7286SMing Lei * nirqs is the number of interrupts available for write and read
2180612b7286SMing Lei * queues. The core already reserved an interrupt for the admin queue.
2181612b7286SMing Lei */
nvme_calc_irq_sets(struct irq_affinity * affd,unsigned int nrirqs)2182612b7286SMing Lei static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs)
21833b6592f7SJens Axboe {
2184612b7286SMing Lei struct nvme_dev *dev = affd->priv;
21852a5bcfddSWeiping Zhang unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2186c45b1fa2SMing Lei
21873b6592f7SJens Axboe /*
2188ee0d96d3SBaolin Wang * If there is no interrupt available for queues, ensure that
2189612b7286SMing Lei * the default queue is set to 1. The affinity set size is
2190612b7286SMing Lei * also set to one, but the irq core ignores it for this case.
2191612b7286SMing Lei *
2192612b7286SMing Lei * If only one interrupt is available or 'write_queue' == 0, combine
2193612b7286SMing Lei * write and read queues.
2194612b7286SMing Lei *
2195612b7286SMing Lei * If 'write_queues' > 0, ensure it leaves room for at least one read
2196612b7286SMing Lei * queue.
21973b6592f7SJens Axboe */
2198612b7286SMing Lei if (!nrirqs) {
2199612b7286SMing Lei nrirqs = 1;
2200612b7286SMing Lei nr_read_queues = 0;
22012a5bcfddSWeiping Zhang } else if (nrirqs == 1 || !nr_write_queues) {
2202612b7286SMing Lei nr_read_queues = 0;
22032a5bcfddSWeiping Zhang } else if (nr_write_queues >= nrirqs) {
2204612b7286SMing Lei nr_read_queues = 1;
22053b6592f7SJens Axboe } else {
22062a5bcfddSWeiping Zhang nr_read_queues = nrirqs - nr_write_queues;
22073b6592f7SJens Axboe }
2208612b7286SMing Lei
2209612b7286SMing Lei dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2210612b7286SMing Lei affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2211612b7286SMing Lei dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2212612b7286SMing Lei affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
2213612b7286SMing Lei affd->nr_sets = nr_read_queues ? 2 : 1;
22143b6592f7SJens Axboe }
22153b6592f7SJens Axboe
nvme_setup_irqs(struct nvme_dev * dev,unsigned int nr_io_queues)22166451fe73SJens Axboe static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
22173b6592f7SJens Axboe {
22183b6592f7SJens Axboe struct pci_dev *pdev = to_pci_dev(dev->dev);
22193b6592f7SJens Axboe struct irq_affinity affd = {
22203b6592f7SJens Axboe .pre_vectors = 1,
2221612b7286SMing Lei .calc_sets = nvme_calc_irq_sets,
2222612b7286SMing Lei .priv = dev,
22233b6592f7SJens Axboe };
222421cc2f3fSJeffle Xu unsigned int irq_queues, poll_queues;
2225a2da0e5cSSean Anderson unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY;
22266451fe73SJens Axboe
22276451fe73SJens Axboe /*
222821cc2f3fSJeffle Xu * Poll queues don't need interrupts, but we need at least one I/O queue
222921cc2f3fSJeffle Xu * left over for non-polled I/O.
22306451fe73SJens Axboe */
223121cc2f3fSJeffle Xu poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
223221cc2f3fSJeffle Xu dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
22333b6592f7SJens Axboe
223421cc2f3fSJeffle Xu /*
223521cc2f3fSJeffle Xu * Initialize for the single interrupt case, will be updated in
223621cc2f3fSJeffle Xu * nvme_calc_irq_sets().
223721cc2f3fSJeffle Xu */
2238612b7286SMing Lei dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2239612b7286SMing Lei dev->io_queues[HCTX_TYPE_READ] = 0;
22403b6592f7SJens Axboe
224166341331SBenjamin Herrenschmidt /*
224221cc2f3fSJeffle Xu * We need interrupts for the admin queue and each non-polled I/O queue,
224321cc2f3fSJeffle Xu * but some Apple controllers require all queues to use the first
224421cc2f3fSJeffle Xu * vector.
224566341331SBenjamin Herrenschmidt */
224666341331SBenjamin Herrenschmidt irq_queues = 1;
224721cc2f3fSJeffle Xu if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
224821cc2f3fSJeffle Xu irq_queues += (nr_io_queues - poll_queues);
2249a2da0e5cSSean Anderson if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
2250a2da0e5cSSean Anderson flags &= ~PCI_IRQ_MSI;
2251a2da0e5cSSean Anderson return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues, flags,
2252a2da0e5cSSean Anderson &affd);
22533b6592f7SJens Axboe }
22543b6592f7SJens Axboe
nvme_max_io_queues(struct nvme_dev * dev)22552a5bcfddSWeiping Zhang static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
22562a5bcfddSWeiping Zhang {
2257e3aef095SNiklas Schnelle /*
2258e3aef095SNiklas Schnelle * If tags are shared with admin queue (Apple bug), then
2259e3aef095SNiklas Schnelle * make sure we only use one IO queue.
2260e3aef095SNiklas Schnelle */
2261e3aef095SNiklas Schnelle if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2262e3aef095SNiklas Schnelle return 1;
22632a5bcfddSWeiping Zhang return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues;
22642a5bcfddSWeiping Zhang }
22652a5bcfddSWeiping Zhang
nvme_setup_io_queues(struct nvme_dev * dev)226657dacad5SJay Sternberg static int nvme_setup_io_queues(struct nvme_dev *dev)
226757dacad5SJay Sternberg {
2268147b27e4SSagi Grimberg struct nvme_queue *adminq = &dev->queues[0];
226957dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev->dev);
22702a5bcfddSWeiping Zhang unsigned int nr_io_queues;
227197f6ef64SXu Yu unsigned long size;
22722a5bcfddSWeiping Zhang int result;
227357dacad5SJay Sternberg
22742a5bcfddSWeiping Zhang /*
22752a5bcfddSWeiping Zhang * Sample the module parameters once at reset time so that we have
22762a5bcfddSWeiping Zhang * stable values to work with.
22772a5bcfddSWeiping Zhang */
22782a5bcfddSWeiping Zhang dev->nr_write_queues = write_queues;
22792a5bcfddSWeiping Zhang dev->nr_poll_queues = poll_queues;
2280d38e9f04SBenjamin Herrenschmidt
2281ff4e5fbaSNiklas Schnelle nr_io_queues = dev->nr_allocated_queues - 1;
22829a0be7abSChristoph Hellwig result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
22839a0be7abSChristoph Hellwig if (result < 0)
228457dacad5SJay Sternberg return result;
22859a0be7abSChristoph Hellwig
2286f5fa90dcSChristoph Hellwig if (nr_io_queues == 0)
2287a5229050SKeith Busch return 0;
228857dacad5SJay Sternberg
2289e4b9852aSCasey Chen /*
2290e4b9852aSCasey Chen * Free IRQ resources as soon as NVMEQ_ENABLED bit transitions
2291e4b9852aSCasey Chen * from set to unset. If there is a window to it is truely freed,
2292e4b9852aSCasey Chen * pci_free_irq_vectors() jumping into this window will crash.
2293e4b9852aSCasey Chen * And take lock to avoid racing with pci_free_irq_vectors() in
2294e4b9852aSCasey Chen * nvme_dev_disable() path.
2295e4b9852aSCasey Chen */
2296e4b9852aSCasey Chen result = nvme_setup_io_queues_trylock(dev);
2297e4b9852aSCasey Chen if (result)
2298e4b9852aSCasey Chen return result;
2299e4b9852aSCasey Chen if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
2300e4b9852aSCasey Chen pci_free_irq(pdev, 0, adminq);
23014e224106SChristoph Hellwig
23020f238ff5SLogan Gunthorpe if (dev->cmb_use_sqes) {
230357dacad5SJay Sternberg result = nvme_cmb_qdepth(dev, nr_io_queues,
230457dacad5SJay Sternberg sizeof(struct nvme_command));
230588d356caSChristoph Hellwig if (result > 0) {
230657dacad5SJay Sternberg dev->q_depth = result;
230788d356caSChristoph Hellwig dev->ctrl.sqsize = result - 1;
230888d356caSChristoph Hellwig } else {
23090f238ff5SLogan Gunthorpe dev->cmb_use_sqes = false;
231057dacad5SJay Sternberg }
231188d356caSChristoph Hellwig }
231257dacad5SJay Sternberg
231357dacad5SJay Sternberg do {
231497f6ef64SXu Yu size = db_bar_size(dev, nr_io_queues);
231597f6ef64SXu Yu result = nvme_remap_bar(dev, size);
231697f6ef64SXu Yu if (!result)
231757dacad5SJay Sternberg break;
2318e4b9852aSCasey Chen if (!--nr_io_queues) {
2319e4b9852aSCasey Chen result = -ENOMEM;
2320e4b9852aSCasey Chen goto out_unlock;
2321e4b9852aSCasey Chen }
232257dacad5SJay Sternberg } while (1);
232357dacad5SJay Sternberg adminq->q_db = dev->dbs;
232457dacad5SJay Sternberg
23258fae268bSKeith Busch retry:
232657dacad5SJay Sternberg /* Deregister the admin queue's interrupt */
2327e4b9852aSCasey Chen if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
23280ff199cbSChristoph Hellwig pci_free_irq(pdev, 0, adminq);
232957dacad5SJay Sternberg
233057dacad5SJay Sternberg /*
233157dacad5SJay Sternberg * If we enable msix early due to not intx, disable it again before
233257dacad5SJay Sternberg * setting up the full range we need.
233357dacad5SJay Sternberg */
2334dca51e78SChristoph Hellwig pci_free_irq_vectors(pdev);
23353b6592f7SJens Axboe
23363b6592f7SJens Axboe result = nvme_setup_irqs(dev, nr_io_queues);
2337e4b9852aSCasey Chen if (result <= 0) {
2338e4b9852aSCasey Chen result = -EIO;
2339e4b9852aSCasey Chen goto out_unlock;
2340e4b9852aSCasey Chen }
23413b6592f7SJens Axboe
234222b55601SKeith Busch dev->num_vecs = result;
23434b04cc6aSJens Axboe result = max(result - 1, 1);
2344e20ba6e1SChristoph Hellwig dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
234557dacad5SJay Sternberg
234657dacad5SJay Sternberg /*
234757dacad5SJay Sternberg * Should investigate if there's a performance win from allocating
234857dacad5SJay Sternberg * more queues than interrupt vectors; it might allow the submission
234957dacad5SJay Sternberg * path to scale better, even if the receive path is limited by the
235057dacad5SJay Sternberg * number of interrupts.
235157dacad5SJay Sternberg */
2352dca51e78SChristoph Hellwig result = queue_request_irq(adminq);
23537c349ddeSKeith Busch if (result)
2354e4b9852aSCasey Chen goto out_unlock;
23554e224106SChristoph Hellwig set_bit(NVMEQ_ENABLED, &adminq->flags);
2356e4b9852aSCasey Chen mutex_unlock(&dev->shutdown_lock);
23578fae268bSKeith Busch
23588fae268bSKeith Busch result = nvme_create_io_queues(dev);
23598fae268bSKeith Busch if (result || dev->online_queues < 2)
23608fae268bSKeith Busch return result;
23618fae268bSKeith Busch
23628fae268bSKeith Busch if (dev->online_queues - 1 < dev->max_qid) {
23638fae268bSKeith Busch nr_io_queues = dev->online_queues - 1;
23647d879c90SChristoph Hellwig nvme_delete_io_queues(dev);
2365e4b9852aSCasey Chen result = nvme_setup_io_queues_trylock(dev);
2366e4b9852aSCasey Chen if (result)
2367e4b9852aSCasey Chen return result;
23688fae268bSKeith Busch nvme_suspend_io_queues(dev);
23698fae268bSKeith Busch goto retry;
23708fae268bSKeith Busch }
23718fae268bSKeith Busch dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
23728fae268bSKeith Busch dev->io_queues[HCTX_TYPE_DEFAULT],
23738fae268bSKeith Busch dev->io_queues[HCTX_TYPE_READ],
23748fae268bSKeith Busch dev->io_queues[HCTX_TYPE_POLL]);
23758fae268bSKeith Busch return 0;
2376e4b9852aSCasey Chen out_unlock:
2377e4b9852aSCasey Chen mutex_unlock(&dev->shutdown_lock);
2378e4b9852aSCasey Chen return result;
237957dacad5SJay Sternberg }
238057dacad5SJay Sternberg
nvme_del_queue_end(struct request * req,blk_status_t error)2381de671d61SJens Axboe static enum rq_end_io_ret nvme_del_queue_end(struct request *req,
2382de671d61SJens Axboe blk_status_t error)
2383db3cbfffSKeith Busch {
2384db3cbfffSKeith Busch struct nvme_queue *nvmeq = req->end_io_data;
2385db3cbfffSKeith Busch
2386db3cbfffSKeith Busch blk_mq_free_request(req);
2387d1ed6aa1SChristoph Hellwig complete(&nvmeq->delete_done);
2388de671d61SJens Axboe return RQ_END_IO_NONE;
2389db3cbfffSKeith Busch }
2390db3cbfffSKeith Busch
nvme_del_cq_end(struct request * req,blk_status_t error)2391de671d61SJens Axboe static enum rq_end_io_ret nvme_del_cq_end(struct request *req,
2392de671d61SJens Axboe blk_status_t error)
2393db3cbfffSKeith Busch {
2394db3cbfffSKeith Busch struct nvme_queue *nvmeq = req->end_io_data;
2395db3cbfffSKeith Busch
2396d1ed6aa1SChristoph Hellwig if (error)
2397d1ed6aa1SChristoph Hellwig set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
2398db3cbfffSKeith Busch
2399de671d61SJens Axboe return nvme_del_queue_end(req, error);
2400db3cbfffSKeith Busch }
2401db3cbfffSKeith Busch
nvme_delete_queue(struct nvme_queue * nvmeq,u8 opcode)2402db3cbfffSKeith Busch static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2403db3cbfffSKeith Busch {
2404db3cbfffSKeith Busch struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2405db3cbfffSKeith Busch struct request *req;
2406f66e2804SChaitanya Kulkarni struct nvme_command cmd = { };
2407db3cbfffSKeith Busch
2408db3cbfffSKeith Busch cmd.delete_queue.opcode = opcode;
2409db3cbfffSKeith Busch cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2410db3cbfffSKeith Busch
2411e559398fSChristoph Hellwig req = blk_mq_alloc_request(q, nvme_req_op(&cmd), BLK_MQ_REQ_NOWAIT);
2412db3cbfffSKeith Busch if (IS_ERR(req))
2413db3cbfffSKeith Busch return PTR_ERR(req);
2414e559398fSChristoph Hellwig nvme_init_request(req, &cmd);
2415db3cbfffSKeith Busch
2416e2e53086SChristoph Hellwig if (opcode == nvme_admin_delete_cq)
2417e2e53086SChristoph Hellwig req->end_io = nvme_del_cq_end;
2418e2e53086SChristoph Hellwig else
2419e2e53086SChristoph Hellwig req->end_io = nvme_del_queue_end;
2420db3cbfffSKeith Busch req->end_io_data = nvmeq;
2421db3cbfffSKeith Busch
2422d1ed6aa1SChristoph Hellwig init_completion(&nvmeq->delete_done);
2423e2e53086SChristoph Hellwig blk_execute_rq_nowait(req, false);
2424db3cbfffSKeith Busch return 0;
2425db3cbfffSKeith Busch }
2426db3cbfffSKeith Busch
__nvme_delete_io_queues(struct nvme_dev * dev,u8 opcode)24277d879c90SChristoph Hellwig static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode)
2428db3cbfffSKeith Busch {
24295271edd4SChristoph Hellwig int nr_queues = dev->online_queues - 1, sent = 0;
2430db3cbfffSKeith Busch unsigned long timeout;
2431db3cbfffSKeith Busch
2432db3cbfffSKeith Busch retry:
2433dc96f938SChaitanya Kulkarni timeout = NVME_ADMIN_TIMEOUT;
24345271edd4SChristoph Hellwig while (nr_queues > 0) {
24355271edd4SChristoph Hellwig if (nvme_delete_queue(&dev->queues[nr_queues], opcode))
2436db3cbfffSKeith Busch break;
24375271edd4SChristoph Hellwig nr_queues--;
24385271edd4SChristoph Hellwig sent++;
24395271edd4SChristoph Hellwig }
2440d1ed6aa1SChristoph Hellwig while (sent) {
2441d1ed6aa1SChristoph Hellwig struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2442d1ed6aa1SChristoph Hellwig
2443d1ed6aa1SChristoph Hellwig timeout = wait_for_completion_io_timeout(&nvmeq->delete_done,
24445271edd4SChristoph Hellwig timeout);
2445db3cbfffSKeith Busch if (timeout == 0)
24465271edd4SChristoph Hellwig return false;
2447d1ed6aa1SChristoph Hellwig
2448d1ed6aa1SChristoph Hellwig sent--;
24495271edd4SChristoph Hellwig if (nr_queues)
2450db3cbfffSKeith Busch goto retry;
2451db3cbfffSKeith Busch }
24525271edd4SChristoph Hellwig return true;
2453db3cbfffSKeith Busch }
2454db3cbfffSKeith Busch
nvme_delete_io_queues(struct nvme_dev * dev)24557d879c90SChristoph Hellwig static void nvme_delete_io_queues(struct nvme_dev *dev)
245657dacad5SJay Sternberg {
24577d879c90SChristoph Hellwig if (__nvme_delete_io_queues(dev, nvme_admin_delete_sq))
24587d879c90SChristoph Hellwig __nvme_delete_io_queues(dev, nvme_admin_delete_cq);
24592b1b7e78SJianchao Wang }
24607d879c90SChristoph Hellwig
nvme_pci_nr_maps(struct nvme_dev * dev)24610da7feaaSChristoph Hellwig static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
246257dacad5SJay Sternberg {
246357dacad5SJay Sternberg if (dev->io_queues[HCTX_TYPE_POLL])
24640da7feaaSChristoph Hellwig return 3;
24650da7feaaSChristoph Hellwig if (dev->io_queues[HCTX_TYPE_READ])
24660da7feaaSChristoph Hellwig return 2;
24670da7feaaSChristoph Hellwig return 1;
246857dacad5SJay Sternberg }
2469949928c1SKeith Busch
nvme_pci_update_nr_queues(struct nvme_dev * dev)24702455a4b7SChristoph Hellwig static void nvme_pci_update_nr_queues(struct nvme_dev *dev)
24712455a4b7SChristoph Hellwig {
24722455a4b7SChristoph Hellwig blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
24732455a4b7SChristoph Hellwig /* free previously allocated queues that are no longer usable */
24742455a4b7SChristoph Hellwig nvme_free_queues(dev, dev->online_queues);
247557dacad5SJay Sternberg }
247657dacad5SJay Sternberg
nvme_pci_enable(struct nvme_dev * dev)2477b00a726aSKeith Busch static int nvme_pci_enable(struct nvme_dev *dev)
247857dacad5SJay Sternberg {
2479b00a726aSKeith Busch int result = -ENOMEM;
248057dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev->dev);
2481a2da0e5cSSean Anderson unsigned int flags = PCI_IRQ_ALL_TYPES;
248257dacad5SJay Sternberg
248357dacad5SJay Sternberg if (pci_enable_device_mem(pdev))
248457dacad5SJay Sternberg return result;
248557dacad5SJay Sternberg
248657dacad5SJay Sternberg pci_set_master(pdev);
248757dacad5SJay Sternberg
24887a67cbeaSChristoph Hellwig if (readl(dev->bar + NVME_REG_CSTS) == -1) {
248957dacad5SJay Sternberg result = -ENODEV;
2490b00a726aSKeith Busch goto disable;
249157dacad5SJay Sternberg }
249257dacad5SJay Sternberg
249357dacad5SJay Sternberg /*
2494a5229050SKeith Busch * Some devices and/or platforms don't advertise or work with INTx
2495a5229050SKeith Busch * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
2496a5229050SKeith Busch * adjust this later.
249757dacad5SJay Sternberg */
2498a2da0e5cSSean Anderson if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
2499a2da0e5cSSean Anderson flags &= ~PCI_IRQ_MSI;
2500a2da0e5cSSean Anderson result = pci_alloc_irq_vectors(pdev, 1, 1, flags);
2501dca51e78SChristoph Hellwig if (result < 0)
250209113abfSTong Zhang goto disable;
250357dacad5SJay Sternberg
250420d0dfe6SSagi Grimberg dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
25057a67cbeaSChristoph Hellwig
25067442ddceSJohn Garry dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2507b27c1e68Sweiping zhang io_queue_depth);
250820d0dfe6SSagi Grimberg dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
25097a67cbeaSChristoph Hellwig dev->dbs = dev->bar + 4096;
25101f390c1fSStephan Günther
25111f390c1fSStephan Günther /*
251266341331SBenjamin Herrenschmidt * Some Apple controllers require a non-standard SQE size.
251366341331SBenjamin Herrenschmidt * Interestingly they also seem to ignore the CC:IOSQES register
251466341331SBenjamin Herrenschmidt * so we don't bother updating it here.
251566341331SBenjamin Herrenschmidt */
251666341331SBenjamin Herrenschmidt if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
251766341331SBenjamin Herrenschmidt dev->io_sqes = 7;
251866341331SBenjamin Herrenschmidt else
2519c1e0cc7eSBenjamin Herrenschmidt dev->io_sqes = NVME_NVM_IOSQES;
25201f390c1fSStephan Günther
25211f390c1fSStephan Günther /*
25221f390c1fSStephan Günther * Temporary fix for the Apple controller found in the MacBook8,1 and
25231f390c1fSStephan Günther * some MacBook7,1 to avoid controller resets and data loss.
25241f390c1fSStephan Günther */
25251f390c1fSStephan Günther if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
25261f390c1fSStephan Günther dev->q_depth = 2;
25279bdcfb10SChristoph Hellwig dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
25289bdcfb10SChristoph Hellwig "set queue depth=%u to work around controller resets\n",
25291f390c1fSStephan Günther dev->q_depth);
2530d554b5e1SMartin K. Petersen } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
2531d554b5e1SMartin K. Petersen (pdev->device == 0xa821 || pdev->device == 0xa822) &&
253220d0dfe6SSagi Grimberg NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2533d554b5e1SMartin K. Petersen dev->q_depth = 64;
2534d554b5e1SMartin K. Petersen dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
2535d554b5e1SMartin K. Petersen "set queue depth=%u\n", dev->q_depth);
25361f390c1fSStephan Günther }
25371f390c1fSStephan Günther
2538d38e9f04SBenjamin Herrenschmidt /*
2539d38e9f04SBenjamin Herrenschmidt * Controllers with the shared tags quirk need the IO queue to be
2540d38e9f04SBenjamin Herrenschmidt * big enough so that we get 32 tags for the admin queue
2541d38e9f04SBenjamin Herrenschmidt */
2542d38e9f04SBenjamin Herrenschmidt if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
2543d38e9f04SBenjamin Herrenschmidt (dev->q_depth < (NVME_AQ_DEPTH + 2))) {
2544d38e9f04SBenjamin Herrenschmidt dev->q_depth = NVME_AQ_DEPTH + 2;
2545d38e9f04SBenjamin Herrenschmidt dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
2546d38e9f04SBenjamin Herrenschmidt dev->q_depth);
2547d38e9f04SBenjamin Herrenschmidt }
254888d356caSChristoph Hellwig dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
2549d38e9f04SBenjamin Herrenschmidt
2550f65efd6dSChristoph Hellwig nvme_map_cmb(dev);
2551202021c1SStephen Bates
2552a0a3408eSKeith Busch pci_save_state(pdev);
2553a6ee7f19SChristoph Hellwig
255409113abfSTong Zhang result = nvme_pci_configure_admin_queue(dev);
255509113abfSTong Zhang if (result)
255609113abfSTong Zhang goto free_irq;
255709113abfSTong Zhang return result;
255857dacad5SJay Sternberg
255909113abfSTong Zhang free_irq:
256009113abfSTong Zhang pci_free_irq_vectors(pdev);
256157dacad5SJay Sternberg disable:
256257dacad5SJay Sternberg pci_disable_device(pdev);
256357dacad5SJay Sternberg return result;
256457dacad5SJay Sternberg }
256557dacad5SJay Sternberg
nvme_dev_unmap(struct nvme_dev * dev)256657dacad5SJay Sternberg static void nvme_dev_unmap(struct nvme_dev *dev)
256757dacad5SJay Sternberg {
2568b00a726aSKeith Busch if (dev->bar)
2569b00a726aSKeith Busch iounmap(dev->bar);
2570a1f447b3SJohannes Thumshirn pci_release_mem_regions(to_pci_dev(dev->dev));
2571b00a726aSKeith Busch }
2572b00a726aSKeith Busch
nvme_pci_ctrl_is_dead(struct nvme_dev * dev)257368e81ebaSChristoph Hellwig static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev)
2574b00a726aSKeith Busch {
257557dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev->dev);
2576081f5e75SKeith Busch u32 csts;
257757dacad5SJay Sternberg
257868e81ebaSChristoph Hellwig if (!pci_is_enabled(pdev) || !pci_device_is_present(pdev))
257968e81ebaSChristoph Hellwig return true;
258068e81ebaSChristoph Hellwig if (pdev->error_state != pci_channel_io_normal)
258168e81ebaSChristoph Hellwig return true;
258257dacad5SJay Sternberg
258368e81ebaSChristoph Hellwig csts = readl(dev->bar + NVME_REG_CSTS);
258468e81ebaSChristoph Hellwig return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY);
2585a0a3408eSKeith Busch }
258657dacad5SJay Sternberg
nvme_dev_disable(struct nvme_dev * dev,bool shutdown)2587a5cdb68cSKeith Busch static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
258857dacad5SJay Sternberg {
25898884a56dSKeith Busch enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl);
2590302ad8ccSKeith Busch struct pci_dev *pdev = to_pci_dev(dev->dev);
259168e81ebaSChristoph Hellwig bool dead;
259257dacad5SJay Sternberg
259377bf25eaSKeith Busch mutex_lock(&dev->shutdown_lock);
259468e81ebaSChristoph Hellwig dead = nvme_pci_ctrl_is_dead(dev);
25958884a56dSKeith Busch if (state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) {
259668e81ebaSChristoph Hellwig if (pci_is_enabled(pdev))
2597302ad8ccSKeith Busch nvme_start_freeze(&dev->ctrl);
2598302ad8ccSKeith Busch /*
259968e81ebaSChristoph Hellwig * Give the controller a chance to complete all entered requests
260068e81ebaSChristoph Hellwig * if doing a safe shutdown.
2601302ad8ccSKeith Busch */
260268e81ebaSChristoph Hellwig if (!dead && shutdown)
2603302ad8ccSKeith Busch nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
260468e81ebaSChristoph Hellwig }
260587ad72a5SChristoph Hellwig
26069f27bd70SChristoph Hellwig nvme_quiesce_io_queues(&dev->ctrl);
26079a915a5bSJianchao Wang
260864ee0ac0SKeith Busch if (!dead && dev->ctrl.queue_count > 0) {
26097d879c90SChristoph Hellwig nvme_delete_io_queues(dev);
261047d42d22SChristoph Hellwig nvme_disable_ctrl(&dev->ctrl, shutdown);
261147d42d22SChristoph Hellwig nvme_poll_irqdisable(&dev->queues[0]);
261257dacad5SJay Sternberg }
26138fae268bSKeith Busch nvme_suspend_io_queues(dev);
261410981f23SChristoph Hellwig nvme_suspend_queue(dev, 0);
2615c80767f7SChristoph Hellwig pci_free_irq_vectors(pdev);
26161ad11eafSBjorn Helgaas if (pci_is_enabled(pdev))
2617c80767f7SChristoph Hellwig pci_disable_device(pdev);
2618fa46c6fbSKeith Busch nvme_reap_pending_cqes(dev);
261957dacad5SJay Sternberg
26201fcfca78SGuixin Liu nvme_cancel_tagset(&dev->ctrl);
26211fcfca78SGuixin Liu nvme_cancel_admin_tagset(&dev->ctrl);
2622302ad8ccSKeith Busch
2623302ad8ccSKeith Busch /*
2624302ad8ccSKeith Busch * The driver will not be starting up queues again if shutting down so
2625302ad8ccSKeith Busch * must flush all entered requests to their failed completion to avoid
2626302ad8ccSKeith Busch * deadlocking blk-mq hot-cpu notifier.
2627302ad8ccSKeith Busch */
2628c8e9e9b7SKeith Busch if (shutdown) {
26299f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(&dev->ctrl);
2630c8e9e9b7SKeith Busch if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
26319f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(&dev->ctrl);
2632c8e9e9b7SKeith Busch }
263377bf25eaSKeith Busch mutex_unlock(&dev->shutdown_lock);
263457dacad5SJay Sternberg }
263557dacad5SJay Sternberg
nvme_disable_prepare_reset(struct nvme_dev * dev,bool shutdown)2636c1ac9a4bSKeith Busch static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
2637c1ac9a4bSKeith Busch {
2638c1ac9a4bSKeith Busch if (!nvme_wait_reset(&dev->ctrl))
2639c1ac9a4bSKeith Busch return -EBUSY;
2640c1ac9a4bSKeith Busch nvme_dev_disable(dev, shutdown);
2641c1ac9a4bSKeith Busch return 0;
2642c1ac9a4bSKeith Busch }
2643c1ac9a4bSKeith Busch
nvme_setup_prp_pools(struct nvme_dev * dev)264457dacad5SJay Sternberg static int nvme_setup_prp_pools(struct nvme_dev *dev)
264557dacad5SJay Sternberg {
264657dacad5SJay Sternberg dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2647c61b82c7SChristoph Hellwig NVME_CTRL_PAGE_SIZE,
2648c61b82c7SChristoph Hellwig NVME_CTRL_PAGE_SIZE, 0);
264957dacad5SJay Sternberg if (!dev->prp_page_pool)
265057dacad5SJay Sternberg return -ENOMEM;
265157dacad5SJay Sternberg
265257dacad5SJay Sternberg /* Optimisation for I/Os between 4k and 128k */
265357dacad5SJay Sternberg dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
265457dacad5SJay Sternberg 256, 256, 0);
265557dacad5SJay Sternberg if (!dev->prp_small_pool) {
265657dacad5SJay Sternberg dma_pool_destroy(dev->prp_page_pool);
265757dacad5SJay Sternberg return -ENOMEM;
265857dacad5SJay Sternberg }
265957dacad5SJay Sternberg return 0;
266057dacad5SJay Sternberg }
266157dacad5SJay Sternberg
nvme_release_prp_pools(struct nvme_dev * dev)266257dacad5SJay Sternberg static void nvme_release_prp_pools(struct nvme_dev *dev)
266357dacad5SJay Sternberg {
266457dacad5SJay Sternberg dma_pool_destroy(dev->prp_page_pool);
266557dacad5SJay Sternberg dma_pool_destroy(dev->prp_small_pool);
266657dacad5SJay Sternberg }
266757dacad5SJay Sternberg
nvme_pci_alloc_iod_mempool(struct nvme_dev * dev)2668081a7d95SChristoph Hellwig static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)
2669081a7d95SChristoph Hellwig {
26707846c1b5SKeith Busch size_t alloc_size = sizeof(struct scatterlist) * NVME_MAX_SEGS;
2671081a7d95SChristoph Hellwig
2672081a7d95SChristoph Hellwig dev->iod_mempool = mempool_create_node(1,
2673081a7d95SChristoph Hellwig mempool_kmalloc, mempool_kfree,
2674081a7d95SChristoph Hellwig (void *)alloc_size, GFP_KERNEL,
2675081a7d95SChristoph Hellwig dev_to_node(dev->dev));
2676081a7d95SChristoph Hellwig if (!dev->iod_mempool)
2677081a7d95SChristoph Hellwig return -ENOMEM;
2678081a7d95SChristoph Hellwig return 0;
2679081a7d95SChristoph Hellwig }
2680081a7d95SChristoph Hellwig
nvme_free_tagset(struct nvme_dev * dev)2681770597ecSKeith Busch static void nvme_free_tagset(struct nvme_dev *dev)
2682770597ecSKeith Busch {
2683770597ecSKeith Busch if (dev->tagset.tags)
26840da7feaaSChristoph Hellwig nvme_remove_io_tag_set(&dev->ctrl);
2685770597ecSKeith Busch dev->ctrl.tagset = NULL;
2686770597ecSKeith Busch }
2687770597ecSKeith Busch
26882e87570bSChristoph Hellwig /* pairs with nvme_pci_alloc_dev */
nvme_pci_free_ctrl(struct nvme_ctrl * ctrl)26891673f1f0SChristoph Hellwig static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
269057dacad5SJay Sternberg {
26911673f1f0SChristoph Hellwig struct nvme_dev *dev = to_nvme_dev(ctrl);
269257dacad5SJay Sternberg
2693770597ecSKeith Busch nvme_free_tagset(dev);
2694253fd4acSIsrael Rukshin put_device(dev->dev);
2695253fd4acSIsrael Rukshin kfree(dev->queues);
269657dacad5SJay Sternberg kfree(dev);
269757dacad5SJay Sternberg }
269857dacad5SJay Sternberg
nvme_reset_work(struct work_struct * work)2699fd634f41SChristoph Hellwig static void nvme_reset_work(struct work_struct *work)
270057dacad5SJay Sternberg {
2701d86c4d8eSChristoph Hellwig struct nvme_dev *dev =
2702d86c4d8eSChristoph Hellwig container_of(work, struct nvme_dev, ctrl.reset_work);
2703a98e58e5SScott Bauer bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2704e71afda4SChaitanya Kulkarni int result;
270557dacad5SJay Sternberg
27068884a56dSKeith Busch if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_RESETTING) {
27077764656bSZhihao Cheng dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
27087764656bSZhihao Cheng dev->ctrl.state);
27094e69d4daSKeith Busch result = -ENODEV;
27104e69d4daSKeith Busch goto out;
2711e71afda4SChaitanya Kulkarni }
2712fd634f41SChristoph Hellwig
2713fd634f41SChristoph Hellwig /*
2714fd634f41SChristoph Hellwig * If we're called to reset a live controller first shut it down before
2715fd634f41SChristoph Hellwig * moving on.
2716fd634f41SChristoph Hellwig */
2717b00a726aSKeith Busch if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2718a5cdb68cSKeith Busch nvme_dev_disable(dev, false);
2719d6135c3aSKeith Busch nvme_sync_queues(&dev->ctrl);
2720fd634f41SChristoph Hellwig
27215c959d73SKeith Busch mutex_lock(&dev->shutdown_lock);
2722b00a726aSKeith Busch result = nvme_pci_enable(dev);
272357dacad5SJay Sternberg if (result)
27244726bcf3SKeith Busch goto out_unlock;
27259f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(&dev->ctrl);
27265c959d73SKeith Busch mutex_unlock(&dev->shutdown_lock);
27275c959d73SKeith Busch
27285c959d73SKeith Busch /*
27295c959d73SKeith Busch * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
27305c959d73SKeith Busch * initializing procedure here.
27315c959d73SKeith Busch */
27325c959d73SKeith Busch if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
27335c959d73SKeith Busch dev_warn(dev->ctrl.device,
27345c959d73SKeith Busch "failed to mark controller CONNECTING\n");
2735cee6c269SMinwoo Im result = -EBUSY;
27365c959d73SKeith Busch goto out;
27375c959d73SKeith Busch }
2738943e942eSJens Axboe
273994cc781fSChristoph Hellwig result = nvme_init_ctrl_finish(&dev->ctrl, was_suspend);
2740ce4541f4SChristoph Hellwig if (result)
2741f58944e2SKeith Busch goto out;
2742ce4541f4SChristoph Hellwig
274365a54646SChristoph Hellwig nvme_dbbuf_dma_alloc(dev);
2744a98e58e5SScott Bauer
27459620cfbaSChristoph Hellwig result = nvme_setup_host_mem(dev);
27469620cfbaSChristoph Hellwig if (result < 0)
27479620cfbaSChristoph Hellwig goto out;
274887ad72a5SChristoph Hellwig
274957dacad5SJay Sternberg result = nvme_setup_io_queues(dev);
275057dacad5SJay Sternberg if (result)
2751f58944e2SKeith Busch goto out;
275257dacad5SJay Sternberg
275321f033f7SKeith Busch /*
27540ffc7e98SChristoph Hellwig * Freeze and update the number of I/O queues as thos might have
2755eac3ef26SChristoph Hellwig * changed. If there are no I/O queues left after this reset, keep the
2756eac3ef26SChristoph Hellwig * controller around but remove all namespaces.
275757dacad5SJay Sternberg */
27580ffc7e98SChristoph Hellwig if (dev->online_queues > 1) {
27599f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(&dev->ctrl);
2760302ad8ccSKeith Busch nvme_wait_freeze(&dev->ctrl);
27612455a4b7SChristoph Hellwig nvme_pci_update_nr_queues(dev);
27622455a4b7SChristoph Hellwig nvme_dbbuf_set(dev);
2763302ad8ccSKeith Busch nvme_unfreeze(&dev->ctrl);
27640ffc7e98SChristoph Hellwig } else {
27650ffc7e98SChristoph Hellwig dev_warn(dev->ctrl.device, "IO queues lost\n");
2766cd50f9b2SChristoph Hellwig nvme_mark_namespaces_dead(&dev->ctrl);
27679f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(&dev->ctrl);
27680ffc7e98SChristoph Hellwig nvme_remove_namespaces(&dev->ctrl);
27690ffc7e98SChristoph Hellwig nvme_free_tagset(dev);
277057dacad5SJay Sternberg }
277157dacad5SJay Sternberg
27722b1b7e78SJianchao Wang /*
27732b1b7e78SJianchao Wang * If only admin queue live, keep it to do further investigation or
27742b1b7e78SJianchao Wang * recovery.
27752b1b7e78SJianchao Wang */
27765d02a5c1SKeith Busch if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
27772b1b7e78SJianchao Wang dev_warn(dev->ctrl.device,
27785d02a5c1SKeith Busch "failed to mark controller live state\n");
2779e71afda4SChaitanya Kulkarni result = -ENODEV;
2780bb8d261eSChristoph Hellwig goto out;
2781bb8d261eSChristoph Hellwig }
278292911a55SChristoph Hellwig
2783d09f2b45SSagi Grimberg nvme_start_ctrl(&dev->ctrl);
278457dacad5SJay Sternberg return;
278557dacad5SJay Sternberg
27864726bcf3SKeith Busch out_unlock:
27874726bcf3SKeith Busch mutex_unlock(&dev->shutdown_lock);
278857dacad5SJay Sternberg out:
2789c7c16c5bSChristoph Hellwig /*
2790c7c16c5bSChristoph Hellwig * Set state to deleting now to avoid blocking nvme_wait_reset(), which
2791c7c16c5bSChristoph Hellwig * may be holding this pci_dev's device lock.
2792c7c16c5bSChristoph Hellwig */
2793c7c16c5bSChristoph Hellwig dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n",
2794c7c16c5bSChristoph Hellwig result);
2795c7c16c5bSChristoph Hellwig nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2796c7c16c5bSChristoph Hellwig nvme_dev_disable(dev, true);
2797a2b5d544SKeith Busch nvme_sync_queues(&dev->ctrl);
2798c7c16c5bSChristoph Hellwig nvme_mark_namespaces_dead(&dev->ctrl);
27992ab4e5f4SKeith Busch nvme_unquiesce_io_queues(&dev->ctrl);
2800c7c16c5bSChristoph Hellwig nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
280157dacad5SJay Sternberg }
280257dacad5SJay Sternberg
nvme_pci_reg_read32(struct nvme_ctrl * ctrl,u32 off,u32 * val)28031c63dc66SChristoph Hellwig static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
280457dacad5SJay Sternberg {
28051c63dc66SChristoph Hellwig *val = readl(to_nvme_dev(ctrl)->bar + off);
28061c63dc66SChristoph Hellwig return 0;
280757dacad5SJay Sternberg }
28081c63dc66SChristoph Hellwig
nvme_pci_reg_write32(struct nvme_ctrl * ctrl,u32 off,u32 val)28095fd4ce1bSChristoph Hellwig static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
28105fd4ce1bSChristoph Hellwig {
28115fd4ce1bSChristoph Hellwig writel(val, to_nvme_dev(ctrl)->bar + off);
28125fd4ce1bSChristoph Hellwig return 0;
28135fd4ce1bSChristoph Hellwig }
28145fd4ce1bSChristoph Hellwig
nvme_pci_reg_read64(struct nvme_ctrl * ctrl,u32 off,u64 * val)28157fd8930fSChristoph Hellwig static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
28167fd8930fSChristoph Hellwig {
28173a8ecc93SArd Biesheuvel *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off);
28187fd8930fSChristoph Hellwig return 0;
28197fd8930fSChristoph Hellwig }
28207fd8930fSChristoph Hellwig
nvme_pci_get_address(struct nvme_ctrl * ctrl,char * buf,int size)282197c12223SKeith Busch static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
282297c12223SKeith Busch {
282397c12223SKeith Busch struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
282497c12223SKeith Busch
28252db24e4aSMax Gurtovoy return snprintf(buf, size, "%s\n", dev_name(&pdev->dev));
282697c12223SKeith Busch }
282797c12223SKeith Busch
nvme_pci_print_device_info(struct nvme_ctrl * ctrl)28282f0dad17SKeith Busch static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl)
28292f0dad17SKeith Busch {
28302f0dad17SKeith Busch struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
28312f0dad17SKeith Busch struct nvme_subsystem *subsys = ctrl->subsys;
28322f0dad17SKeith Busch
28332f0dad17SKeith Busch dev_err(ctrl->device,
28342f0dad17SKeith Busch "VID:DID %04x:%04x model:%.*s firmware:%.*s\n",
28352f0dad17SKeith Busch pdev->vendor, pdev->device,
28362f0dad17SKeith Busch nvme_strlen(subsys->model, sizeof(subsys->model)),
28372f0dad17SKeith Busch subsys->model, nvme_strlen(subsys->firmware_rev,
28382f0dad17SKeith Busch sizeof(subsys->firmware_rev)),
28392f0dad17SKeith Busch subsys->firmware_rev);
28402f0dad17SKeith Busch }
28412f0dad17SKeith Busch
nvme_pci_supports_pci_p2pdma(struct nvme_ctrl * ctrl)28422f859441SLogan Gunthorpe static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
28432f859441SLogan Gunthorpe {
28442f859441SLogan Gunthorpe struct nvme_dev *dev = to_nvme_dev(ctrl);
28452f859441SLogan Gunthorpe
28462f859441SLogan Gunthorpe return dma_pci_p2pdma_supported(dev->dev);
28472f859441SLogan Gunthorpe }
28482f859441SLogan Gunthorpe
28491c63dc66SChristoph Hellwig static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
28501a353d85SMing Lin .name = "pcie",
2851e439bb12SSagi Grimberg .module = THIS_MODULE,
28522f859441SLogan Gunthorpe .flags = NVME_F_METADATA_SUPPORTED,
285386adbf0cSChristoph Hellwig .dev_attr_groups = nvme_pci_dev_attr_groups,
28541c63dc66SChristoph Hellwig .reg_read32 = nvme_pci_reg_read32,
28555fd4ce1bSChristoph Hellwig .reg_write32 = nvme_pci_reg_write32,
28567fd8930fSChristoph Hellwig .reg_read64 = nvme_pci_reg_read64,
28571673f1f0SChristoph Hellwig .free_ctrl = nvme_pci_free_ctrl,
2858f866fc42SChristoph Hellwig .submit_async_event = nvme_pci_submit_async_event,
285997c12223SKeith Busch .get_address = nvme_pci_get_address,
28602f0dad17SKeith Busch .print_device_info = nvme_pci_print_device_info,
28612f859441SLogan Gunthorpe .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma,
28621c63dc66SChristoph Hellwig };
286357dacad5SJay Sternberg
nvme_dev_map(struct nvme_dev * dev)2864b00a726aSKeith Busch static int nvme_dev_map(struct nvme_dev *dev)
2865b00a726aSKeith Busch {
2866b00a726aSKeith Busch struct pci_dev *pdev = to_pci_dev(dev->dev);
2867b00a726aSKeith Busch
2868a1f447b3SJohannes Thumshirn if (pci_request_mem_regions(pdev, "nvme"))
2869b00a726aSKeith Busch return -ENODEV;
2870b00a726aSKeith Busch
287197f6ef64SXu Yu if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2872b00a726aSKeith Busch goto release;
2873b00a726aSKeith Busch
2874b00a726aSKeith Busch return 0;
2875b00a726aSKeith Busch release:
2876a1f447b3SJohannes Thumshirn pci_release_mem_regions(pdev);
2877b00a726aSKeith Busch return -ENODEV;
2878b00a726aSKeith Busch }
2879b00a726aSKeith Busch
check_vendor_combination_bug(struct pci_dev * pdev)28808427bbc2SKai-Heng Feng static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
2881ff5350a8SAndy Lutomirski {
2882ff5350a8SAndy Lutomirski if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
2883ff5350a8SAndy Lutomirski /*
2884ff5350a8SAndy Lutomirski * Several Samsung devices seem to drop off the PCIe bus
2885ff5350a8SAndy Lutomirski * randomly when APST is on and uses the deepest sleep state.
2886ff5350a8SAndy Lutomirski * This has been observed on a Samsung "SM951 NVMe SAMSUNG
2887ff5350a8SAndy Lutomirski * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
2888ff5350a8SAndy Lutomirski * 950 PRO 256GB", but it seems to be restricted to two Dell
2889ff5350a8SAndy Lutomirski * laptops.
2890ff5350a8SAndy Lutomirski */
2891ff5350a8SAndy Lutomirski if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
2892ff5350a8SAndy Lutomirski (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
2893ff5350a8SAndy Lutomirski dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
2894ff5350a8SAndy Lutomirski return NVME_QUIRK_NO_DEEPEST_PS;
28958427bbc2SKai-Heng Feng } else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
28968427bbc2SKai-Heng Feng /*
28978427bbc2SKai-Heng Feng * Samsung SSD 960 EVO drops off the PCIe bus after system
2898467c77d4SJarosław Janik * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
2899467c77d4SJarosław Janik * within few minutes after bootup on a Coffee Lake board -
2900467c77d4SJarosław Janik * ASUS PRIME Z370-A
29018427bbc2SKai-Heng Feng */
29028427bbc2SKai-Heng Feng if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
2903467c77d4SJarosław Janik (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
2904467c77d4SJarosław Janik dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
29058427bbc2SKai-Heng Feng return NVME_QUIRK_NO_APST;
29061fae37acSShyjumon N } else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 ||
29071fae37acSShyjumon N pdev->device == 0xa808 || pdev->device == 0xa809)) ||
29081fae37acSShyjumon N (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) {
29091fae37acSShyjumon N /*
29101fae37acSShyjumon N * Forcing to use host managed nvme power settings for
29111fae37acSShyjumon N * lowest idle power with quick resume latency on
29121fae37acSShyjumon N * Samsung and Toshiba SSDs based on suspend behavior
29131fae37acSShyjumon N * on Coffee Lake board for LENOVO C640
29141fae37acSShyjumon N */
29151fae37acSShyjumon N if ((dmi_match(DMI_BOARD_VENDOR, "LENOVO")) &&
29161fae37acSShyjumon N dmi_match(DMI_BOARD_NAME, "LNVNB161216"))
29171fae37acSShyjumon N return NVME_QUIRK_SIMPLE_SUSPEND;
2918dd864f6eSGeorg Gottleuber } else if (pdev->vendor == 0x2646 && (pdev->device == 0x2263 ||
2919dd864f6eSGeorg Gottleuber pdev->device == 0x500f)) {
2920dd864f6eSGeorg Gottleuber /*
2921dd864f6eSGeorg Gottleuber * Exclude some Kingston NV1 and A2000 devices from
2922dd864f6eSGeorg Gottleuber * NVME_QUIRK_SIMPLE_SUSPEND. Do a full suspend to save a
2923dd864f6eSGeorg Gottleuber * lot fo energy with s2idle sleep on some TUXEDO platforms.
2924dd864f6eSGeorg Gottleuber */
2925dd864f6eSGeorg Gottleuber if (dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") ||
2926dd864f6eSGeorg Gottleuber dmi_match(DMI_BOARD_NAME, "NS5x_7xAU") ||
2927dd864f6eSGeorg Gottleuber dmi_match(DMI_BOARD_NAME, "NS5x_7xPU") ||
2928dd864f6eSGeorg Gottleuber dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1"))
2929dd864f6eSGeorg Gottleuber return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND;
2930ff5350a8SAndy Lutomirski }
2931ff5350a8SAndy Lutomirski
2932ff5350a8SAndy Lutomirski return 0;
2933ff5350a8SAndy Lutomirski }
2934ff5350a8SAndy Lutomirski
nvme_pci_alloc_dev(struct pci_dev * pdev,const struct pci_device_id * id)29352e87570bSChristoph Hellwig static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
29362e87570bSChristoph Hellwig const struct pci_device_id *id)
293718119775SKeith Busch {
2938ff5350a8SAndy Lutomirski unsigned long quirks = id->driver_data;
29392e87570bSChristoph Hellwig int node = dev_to_node(&pdev->dev);
29402e87570bSChristoph Hellwig struct nvme_dev *dev;
29412e87570bSChristoph Hellwig int ret = -ENOMEM;
294257dacad5SJay Sternberg
294357dacad5SJay Sternberg dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
294457dacad5SJay Sternberg if (!dev)
2945dc785d69SIrvin Cote return ERR_PTR(-ENOMEM);
29462e87570bSChristoph Hellwig INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
29472e87570bSChristoph Hellwig mutex_init(&dev->shutdown_lock);
2948147b27e4SSagi Grimberg
29492a5bcfddSWeiping Zhang dev->nr_write_queues = write_queues;
29502a5bcfddSWeiping Zhang dev->nr_poll_queues = poll_queues;
29512a5bcfddSWeiping Zhang dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
29522a5bcfddSWeiping Zhang dev->queues = kcalloc_node(dev->nr_allocated_queues,
29532a5bcfddSWeiping Zhang sizeof(struct nvme_queue), GFP_KERNEL, node);
295457dacad5SJay Sternberg if (!dev->queues)
29552e87570bSChristoph Hellwig goto out_free_dev;
295657dacad5SJay Sternberg
295757dacad5SJay Sternberg dev->dev = get_device(&pdev->dev);
2958f3ca80fcSChristoph Hellwig
29598427bbc2SKai-Heng Feng quirks |= check_vendor_combination_bug(pdev);
2960dd864f6eSGeorg Gottleuber if (!noacpi &&
2961dd864f6eSGeorg Gottleuber !(quirks & NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND) &&
2962dd864f6eSGeorg Gottleuber acpi_storage_d3(&pdev->dev)) {
2963df4f9bc4SDavid E. Box /*
2964df4f9bc4SDavid E. Box * Some systems use a bios work around to ask for D3 on
2965df4f9bc4SDavid E. Box * platforms that support kernel managed suspend.
2966df4f9bc4SDavid E. Box */
2967df4f9bc4SDavid E. Box dev_info(&pdev->dev,
2968df4f9bc4SDavid E. Box "platform quirk: setting simple suspend\n");
2969df4f9bc4SDavid E. Box quirks |= NVME_QUIRK_SIMPLE_SUSPEND;
2970df4f9bc4SDavid E. Box }
29712e87570bSChristoph Hellwig ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
29722e87570bSChristoph Hellwig quirks);
29732e87570bSChristoph Hellwig if (ret)
29742e87570bSChristoph Hellwig goto out_put_device;
29753f30a79cSChristoph Hellwig
2976924bd96eSChristoph Hellwig if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48)
2977924bd96eSChristoph Hellwig dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
2978924bd96eSChristoph Hellwig else
2979924bd96eSChristoph Hellwig dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
29803f30a79cSChristoph Hellwig dma_set_min_align_mask(&pdev->dev, NVME_CTRL_PAGE_SIZE - 1);
29813f30a79cSChristoph Hellwig dma_set_max_seg_size(&pdev->dev, 0xffffffff);
2982df4f9bc4SDavid E. Box
2983943e942eSJens Axboe /*
29843f30a79cSChristoph Hellwig * Limit the max command size to prevent iod->sg allocations going
29853f30a79cSChristoph Hellwig * over a single page.
2986943e942eSJens Axboe */
29873f30a79cSChristoph Hellwig dev->ctrl.max_hw_sectors = min_t(u32,
29883710e2b0SAdrian Huang NVME_MAX_KB_SZ << 1, dma_opt_mapping_size(&pdev->dev) >> 9);
29893f30a79cSChristoph Hellwig dev->ctrl.max_segments = NVME_MAX_SEGS;
2990943e942eSJens Axboe
29913f30a79cSChristoph Hellwig /*
29923f30a79cSChristoph Hellwig * There is no support for SGLs for metadata (yet), so we are limited to
29933f30a79cSChristoph Hellwig * a single integrity segment for the separate metadata pointer.
29943f30a79cSChristoph Hellwig */
29953f30a79cSChristoph Hellwig dev->ctrl.max_integrity_segments = 1;
29962e87570bSChristoph Hellwig return dev;
29972e87570bSChristoph Hellwig
29982e87570bSChristoph Hellwig out_put_device:
29992e87570bSChristoph Hellwig put_device(dev->dev);
30002e87570bSChristoph Hellwig kfree(dev->queues);
30012e87570bSChristoph Hellwig out_free_dev:
30022e87570bSChristoph Hellwig kfree(dev);
30032e87570bSChristoph Hellwig return ERR_PTR(ret);
3004943e942eSJens Axboe }
3005943e942eSJens Axboe
nvme_probe(struct pci_dev * pdev,const struct pci_device_id * id)30062e87570bSChristoph Hellwig static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
30072e87570bSChristoph Hellwig {
30082e87570bSChristoph Hellwig struct nvme_dev *dev;
30092e87570bSChristoph Hellwig int result = -ENOMEM;
30102e87570bSChristoph Hellwig
30112e87570bSChristoph Hellwig dev = nvme_pci_alloc_dev(pdev, id);
3012dc785d69SIrvin Cote if (IS_ERR(dev))
3013dc785d69SIrvin Cote return PTR_ERR(dev);
30142e87570bSChristoph Hellwig
30152e87570bSChristoph Hellwig result = nvme_dev_map(dev);
3016b6e44b4cSKeith Busch if (result)
30172e87570bSChristoph Hellwig goto out_uninit_ctrl;
30182e87570bSChristoph Hellwig
30192e87570bSChristoph Hellwig result = nvme_setup_prp_pools(dev);
30202e87570bSChristoph Hellwig if (result)
30212e87570bSChristoph Hellwig goto out_dev_unmap;
302257dacad5SJay Sternberg
3023081a7d95SChristoph Hellwig result = nvme_pci_alloc_iod_mempool(dev);
3024081a7d95SChristoph Hellwig if (result)
30252e87570bSChristoph Hellwig goto out_release_prp_pools;
3026b6e44b4cSKeith Busch
302757dacad5SJay Sternberg dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
302857dacad5SJay Sternberg
3029eac3ef26SChristoph Hellwig result = nvme_pci_enable(dev);
3030eac3ef26SChristoph Hellwig if (result)
3031eac3ef26SChristoph Hellwig goto out_release_iod_mempool;
303257dacad5SJay Sternberg
30330da7feaaSChristoph Hellwig result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset,
30340da7feaaSChristoph Hellwig &nvme_mq_admin_ops, sizeof(struct nvme_iod));
3035eac3ef26SChristoph Hellwig if (result)
3036eac3ef26SChristoph Hellwig goto out_disable;
3037eac3ef26SChristoph Hellwig
3038eac3ef26SChristoph Hellwig /*
3039eac3ef26SChristoph Hellwig * Mark the controller as connecting before sending admin commands to
3040eac3ef26SChristoph Hellwig * allow the timeout handler to do the right thing.
3041eac3ef26SChristoph Hellwig */
3042eac3ef26SChristoph Hellwig if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
3043eac3ef26SChristoph Hellwig dev_warn(dev->ctrl.device,
3044eac3ef26SChristoph Hellwig "failed to mark controller CONNECTING\n");
3045eac3ef26SChristoph Hellwig result = -EBUSY;
3046eac3ef26SChristoph Hellwig goto out_disable;
3047eac3ef26SChristoph Hellwig }
3048eac3ef26SChristoph Hellwig
3049eac3ef26SChristoph Hellwig result = nvme_init_ctrl_finish(&dev->ctrl, false);
3050eac3ef26SChristoph Hellwig if (result)
3051eac3ef26SChristoph Hellwig goto out_disable;
3052eac3ef26SChristoph Hellwig
3053eac3ef26SChristoph Hellwig nvme_dbbuf_dma_alloc(dev);
3054eac3ef26SChristoph Hellwig
3055eac3ef26SChristoph Hellwig result = nvme_setup_host_mem(dev);
3056eac3ef26SChristoph Hellwig if (result < 0)
3057eac3ef26SChristoph Hellwig goto out_disable;
3058eac3ef26SChristoph Hellwig
3059eac3ef26SChristoph Hellwig result = nvme_setup_io_queues(dev);
3060eac3ef26SChristoph Hellwig if (result)
3061eac3ef26SChristoph Hellwig goto out_disable;
3062eac3ef26SChristoph Hellwig
3063eac3ef26SChristoph Hellwig if (dev->online_queues > 1) {
30640da7feaaSChristoph Hellwig nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops,
30650da7feaaSChristoph Hellwig nvme_pci_nr_maps(dev), sizeof(struct nvme_iod));
3066eac3ef26SChristoph Hellwig nvme_dbbuf_set(dev);
3067eac3ef26SChristoph Hellwig }
3068eac3ef26SChristoph Hellwig
30690da7feaaSChristoph Hellwig if (!dev->ctrl.tagset)
30700da7feaaSChristoph Hellwig dev_warn(dev->ctrl.device, "IO queues not created\n");
30710da7feaaSChristoph Hellwig
3072eac3ef26SChristoph Hellwig if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
3073eac3ef26SChristoph Hellwig dev_warn(dev->ctrl.device,
3074eac3ef26SChristoph Hellwig "failed to mark controller live state\n");
3075eac3ef26SChristoph Hellwig result = -ENODEV;
3076eac3ef26SChristoph Hellwig goto out_disable;
3077eac3ef26SChristoph Hellwig }
3078eac3ef26SChristoph Hellwig
30792e87570bSChristoph Hellwig pci_set_drvdata(pdev, dev);
308057dacad5SJay Sternberg
3081eac3ef26SChristoph Hellwig nvme_start_ctrl(&dev->ctrl);
3082eac3ef26SChristoph Hellwig nvme_put_ctrl(&dev->ctrl);
30835a5754a4SKeith Busch flush_work(&dev->ctrl.scan_work);
308457dacad5SJay Sternberg return 0;
308557dacad5SJay Sternberg
3086eac3ef26SChristoph Hellwig out_disable:
3087eac3ef26SChristoph Hellwig nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3088eac3ef26SChristoph Hellwig nvme_dev_disable(dev, true);
3089eac3ef26SChristoph Hellwig nvme_free_host_mem(dev);
3090eac3ef26SChristoph Hellwig nvme_dev_remove_admin(dev);
3091eac3ef26SChristoph Hellwig nvme_dbbuf_dma_free(dev);
3092eac3ef26SChristoph Hellwig nvme_free_queues(dev, 0);
3093eac3ef26SChristoph Hellwig out_release_iod_mempool:
3094b6e44b4cSKeith Busch mempool_destroy(dev->iod_mempool);
30952e87570bSChristoph Hellwig out_release_prp_pools:
309657dacad5SJay Sternberg nvme_release_prp_pools(dev);
30972e87570bSChristoph Hellwig out_dev_unmap:
309857dacad5SJay Sternberg nvme_dev_unmap(dev);
30992e87570bSChristoph Hellwig out_uninit_ctrl:
31002e87570bSChristoph Hellwig nvme_uninit_ctrl(&dev->ctrl);
3101a61d2655SIrvin Cote nvme_put_ctrl(&dev->ctrl);
310257dacad5SJay Sternberg return result;
310357dacad5SJay Sternberg }
310457dacad5SJay Sternberg
nvme_reset_prepare(struct pci_dev * pdev)3105775755edSChristoph Hellwig static void nvme_reset_prepare(struct pci_dev *pdev)
310657dacad5SJay Sternberg {
310757dacad5SJay Sternberg struct nvme_dev *dev = pci_get_drvdata(pdev);
3108c1ac9a4bSKeith Busch
3109c1ac9a4bSKeith Busch /*
3110c1ac9a4bSKeith Busch * We don't need to check the return value from waiting for the reset
3111c1ac9a4bSKeith Busch * state as pci_dev device lock is held, making it impossible to race
3112c1ac9a4bSKeith Busch * with ->remove().
3113c1ac9a4bSKeith Busch */
3114c1ac9a4bSKeith Busch nvme_disable_prepare_reset(dev, false);
3115c1ac9a4bSKeith Busch nvme_sync_queues(&dev->ctrl);
3116775755edSChristoph Hellwig }
311757dacad5SJay Sternberg
nvme_reset_done(struct pci_dev * pdev)3118775755edSChristoph Hellwig static void nvme_reset_done(struct pci_dev *pdev)
3119775755edSChristoph Hellwig {
3120f263fbb8SLinus Torvalds struct nvme_dev *dev = pci_get_drvdata(pdev);
3121c1ac9a4bSKeith Busch
3122c1ac9a4bSKeith Busch if (!nvme_try_sched_reset(&dev->ctrl))
3123c1ac9a4bSKeith Busch flush_work(&dev->ctrl.reset_work);
312457dacad5SJay Sternberg }
312557dacad5SJay Sternberg
nvme_shutdown(struct pci_dev * pdev)312657dacad5SJay Sternberg static void nvme_shutdown(struct pci_dev *pdev)
312757dacad5SJay Sternberg {
312857dacad5SJay Sternberg struct nvme_dev *dev = pci_get_drvdata(pdev);
31294e523547SBaolin Wang
3130c1ac9a4bSKeith Busch nvme_disable_prepare_reset(dev, true);
313157dacad5SJay Sternberg }
313257dacad5SJay Sternberg
3133f58944e2SKeith Busch /*
3134f58944e2SKeith Busch * The driver's remove may be called on a device in a partially initialized
3135f58944e2SKeith Busch * state. This function must not have any dependencies on the device state in
3136f58944e2SKeith Busch * order to proceed.
3137f58944e2SKeith Busch */
nvme_remove(struct pci_dev * pdev)313857dacad5SJay Sternberg static void nvme_remove(struct pci_dev *pdev)
313957dacad5SJay Sternberg {
314057dacad5SJay Sternberg struct nvme_dev *dev = pci_get_drvdata(pdev);
314157dacad5SJay Sternberg
3142bb8d261eSChristoph Hellwig nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
314357dacad5SJay Sternberg pci_set_drvdata(pdev, NULL);
31440ff9d4e1SKeith Busch
31456db28edaSKeith Busch if (!pci_device_is_present(pdev)) {
31460ff9d4e1SKeith Busch nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
31471d39e692SKeith Busch nvme_dev_disable(dev, true);
31486db28edaSKeith Busch }
31490ff9d4e1SKeith Busch
3150d86c4d8eSChristoph Hellwig flush_work(&dev->ctrl.reset_work);
3151d09f2b45SSagi Grimberg nvme_stop_ctrl(&dev->ctrl);
3152d09f2b45SSagi Grimberg nvme_remove_namespaces(&dev->ctrl);
3153a5cdb68cSKeith Busch nvme_dev_disable(dev, true);
315487ad72a5SChristoph Hellwig nvme_free_host_mem(dev);
315557dacad5SJay Sternberg nvme_dev_remove_admin(dev);
3156c11b7716SChristoph Hellwig nvme_dbbuf_dma_free(dev);
315757dacad5SJay Sternberg nvme_free_queues(dev, 0);
3158c11b7716SChristoph Hellwig mempool_destroy(dev->iod_mempool);
315957dacad5SJay Sternberg nvme_release_prp_pools(dev);
3160b00a726aSKeith Busch nvme_dev_unmap(dev);
3161726612b6SIsrael Rukshin nvme_uninit_ctrl(&dev->ctrl);
316257dacad5SJay Sternberg }
316357dacad5SJay Sternberg
316457dacad5SJay Sternberg #ifdef CONFIG_PM_SLEEP
nvme_get_power_state(struct nvme_ctrl * ctrl,u32 * ps)3165d916b1beSKeith Busch static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
3166d916b1beSKeith Busch {
3167d916b1beSKeith Busch return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps);
3168d916b1beSKeith Busch }
3169d916b1beSKeith Busch
nvme_set_power_state(struct nvme_ctrl * ctrl,u32 ps)3170d916b1beSKeith Busch static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
3171d916b1beSKeith Busch {
3172d916b1beSKeith Busch return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL);
3173d916b1beSKeith Busch }
3174d916b1beSKeith Busch
nvme_resume(struct device * dev)3175d916b1beSKeith Busch static int nvme_resume(struct device *dev)
3176d916b1beSKeith Busch {
3177d916b1beSKeith Busch struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3178d916b1beSKeith Busch struct nvme_ctrl *ctrl = &ndev->ctrl;
3179d916b1beSKeith Busch
31804eaefe8cSRafael J. Wysocki if (ndev->last_ps == U32_MAX ||
3181d916b1beSKeith Busch nvme_set_power_state(ctrl, ndev->last_ps) != 0)
3182e5ad96f3SKeith Busch goto reset;
3183e5ad96f3SKeith Busch if (ctrl->hmpre && nvme_setup_host_mem(ndev))
3184e5ad96f3SKeith Busch goto reset;
3185e5ad96f3SKeith Busch
3186d916b1beSKeith Busch return 0;
3187e5ad96f3SKeith Busch reset:
3188e5ad96f3SKeith Busch return nvme_try_sched_reset(ctrl);
3189d916b1beSKeith Busch }
3190d916b1beSKeith Busch
nvme_suspend(struct device * dev)319157dacad5SJay Sternberg static int nvme_suspend(struct device *dev)
319257dacad5SJay Sternberg {
319357dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev);
319457dacad5SJay Sternberg struct nvme_dev *ndev = pci_get_drvdata(pdev);
3195d916b1beSKeith Busch struct nvme_ctrl *ctrl = &ndev->ctrl;
3196d916b1beSKeith Busch int ret = -EBUSY;
3197d916b1beSKeith Busch
31984eaefe8cSRafael J. Wysocki ndev->last_ps = U32_MAX;
31994eaefe8cSRafael J. Wysocki
3200d916b1beSKeith Busch /*
3201d916b1beSKeith Busch * The platform does not remove power for a kernel managed suspend so
3202d916b1beSKeith Busch * use host managed nvme power settings for lowest idle power if
3203d916b1beSKeith Busch * possible. This should have quicker resume latency than a full device
3204d916b1beSKeith Busch * shutdown. But if the firmware is involved after the suspend or the
3205d916b1beSKeith Busch * device does not support any non-default power states, shut down the
3206d916b1beSKeith Busch * device fully.
32074eaefe8cSRafael J. Wysocki *
32084eaefe8cSRafael J. Wysocki * If ASPM is not enabled for the device, shut down the device and allow
32094eaefe8cSRafael J. Wysocki * the PCI bus layer to put it into D3 in order to take the PCIe link
32104eaefe8cSRafael J. Wysocki * down, so as to allow the platform to achieve its minimum low-power
32114eaefe8cSRafael J. Wysocki * state (which may not be possible if the link is up).
3212d916b1beSKeith Busch */
32134eaefe8cSRafael J. Wysocki if (pm_suspend_via_firmware() || !ctrl->npss ||
3214cb32de1bSMario Limonciello !pcie_aspm_enabled(pdev) ||
3215c1ac9a4bSKeith Busch (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
3216c1ac9a4bSKeith Busch return nvme_disable_prepare_reset(ndev, true);
3217d916b1beSKeith Busch
3218d916b1beSKeith Busch nvme_start_freeze(ctrl);
3219d916b1beSKeith Busch nvme_wait_freeze(ctrl);
3220d916b1beSKeith Busch nvme_sync_queues(ctrl);
3221d916b1beSKeith Busch
32228884a56dSKeith Busch if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
3223d916b1beSKeith Busch goto unfreeze;
3224d916b1beSKeith Busch
3225e5ad96f3SKeith Busch /*
3226e5ad96f3SKeith Busch * Host memory access may not be successful in a system suspend state,
3227e5ad96f3SKeith Busch * but the specification allows the controller to access memory in a
3228e5ad96f3SKeith Busch * non-operational power state.
3229e5ad96f3SKeith Busch */
3230e5ad96f3SKeith Busch if (ndev->hmb) {
3231e5ad96f3SKeith Busch ret = nvme_set_host_mem(ndev, 0);
3232e5ad96f3SKeith Busch if (ret < 0)
3233e5ad96f3SKeith Busch goto unfreeze;
3234e5ad96f3SKeith Busch }
3235e5ad96f3SKeith Busch
3236d916b1beSKeith Busch ret = nvme_get_power_state(ctrl, &ndev->last_ps);
3237d916b1beSKeith Busch if (ret < 0)
3238d916b1beSKeith Busch goto unfreeze;
3239d916b1beSKeith Busch
32407cbb5c6fSMario Limonciello /*
32417cbb5c6fSMario Limonciello * A saved state prevents pci pm from generically controlling the
32427cbb5c6fSMario Limonciello * device's power. If we're using protocol specific settings, we don't
32437cbb5c6fSMario Limonciello * want pci interfering.
32447cbb5c6fSMario Limonciello */
32457cbb5c6fSMario Limonciello pci_save_state(pdev);
32467cbb5c6fSMario Limonciello
3247d916b1beSKeith Busch ret = nvme_set_power_state(ctrl, ctrl->npss);
3248d916b1beSKeith Busch if (ret < 0)
3249d916b1beSKeith Busch goto unfreeze;
3250d916b1beSKeith Busch
3251d916b1beSKeith Busch if (ret) {
32527cbb5c6fSMario Limonciello /* discard the saved state */
32537cbb5c6fSMario Limonciello pci_load_saved_state(pdev, NULL);
32547cbb5c6fSMario Limonciello
3255d916b1beSKeith Busch /*
3256d916b1beSKeith Busch * Clearing npss forces a controller reset on resume. The
325705d3046fSGeert Uytterhoeven * correct value will be rediscovered then.
3258d916b1beSKeith Busch */
3259c1ac9a4bSKeith Busch ret = nvme_disable_prepare_reset(ndev, true);
3260d916b1beSKeith Busch ctrl->npss = 0;
3261d916b1beSKeith Busch }
3262d916b1beSKeith Busch unfreeze:
3263d916b1beSKeith Busch nvme_unfreeze(ctrl);
3264d916b1beSKeith Busch return ret;
3265d916b1beSKeith Busch }
3266d916b1beSKeith Busch
nvme_simple_suspend(struct device * dev)3267d916b1beSKeith Busch static int nvme_simple_suspend(struct device *dev)
3268d916b1beSKeith Busch {
3269d916b1beSKeith Busch struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
32704e523547SBaolin Wang
3271c1ac9a4bSKeith Busch return nvme_disable_prepare_reset(ndev, true);
327257dacad5SJay Sternberg }
327357dacad5SJay Sternberg
nvme_simple_resume(struct device * dev)3274d916b1beSKeith Busch static int nvme_simple_resume(struct device *dev)
327557dacad5SJay Sternberg {
327657dacad5SJay Sternberg struct pci_dev *pdev = to_pci_dev(dev);
327757dacad5SJay Sternberg struct nvme_dev *ndev = pci_get_drvdata(pdev);
327857dacad5SJay Sternberg
3279c1ac9a4bSKeith Busch return nvme_try_sched_reset(&ndev->ctrl);
328057dacad5SJay Sternberg }
328157dacad5SJay Sternberg
328221774222SYueHaibing static const struct dev_pm_ops nvme_dev_pm_ops = {
3283d916b1beSKeith Busch .suspend = nvme_suspend,
3284d916b1beSKeith Busch .resume = nvme_resume,
3285d916b1beSKeith Busch .freeze = nvme_simple_suspend,
3286d916b1beSKeith Busch .thaw = nvme_simple_resume,
3287d916b1beSKeith Busch .poweroff = nvme_simple_suspend,
3288d916b1beSKeith Busch .restore = nvme_simple_resume,
3289d916b1beSKeith Busch };
3290d916b1beSKeith Busch #endif /* CONFIG_PM_SLEEP */
329157dacad5SJay Sternberg
nvme_error_detected(struct pci_dev * pdev,pci_channel_state_t state)3292a0a3408eSKeith Busch static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
3293a0a3408eSKeith Busch pci_channel_state_t state)
3294a0a3408eSKeith Busch {
3295a0a3408eSKeith Busch struct nvme_dev *dev = pci_get_drvdata(pdev);
3296a0a3408eSKeith Busch
3297a0a3408eSKeith Busch /*
3298a0a3408eSKeith Busch * A frozen channel requires a reset. When detected, this method will
3299a0a3408eSKeith Busch * shutdown the controller to quiesce. The controller will be restarted
3300a0a3408eSKeith Busch * after the slot reset through driver's slot_reset callback.
3301a0a3408eSKeith Busch */
3302a0a3408eSKeith Busch switch (state) {
3303a0a3408eSKeith Busch case pci_channel_io_normal:
3304a0a3408eSKeith Busch return PCI_ERS_RESULT_CAN_RECOVER;
3305a0a3408eSKeith Busch case pci_channel_io_frozen:
3306d011fb31SKeith Busch dev_warn(dev->ctrl.device,
3307d011fb31SKeith Busch "frozen state error detected, reset controller\n");
330871a5bb15SKeith Busch if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
330971a5bb15SKeith Busch nvme_dev_disable(dev, true);
331071a5bb15SKeith Busch return PCI_ERS_RESULT_DISCONNECT;
331171a5bb15SKeith Busch }
3312a5cdb68cSKeith Busch nvme_dev_disable(dev, false);
3313a0a3408eSKeith Busch return PCI_ERS_RESULT_NEED_RESET;
3314a0a3408eSKeith Busch case pci_channel_io_perm_failure:
3315d011fb31SKeith Busch dev_warn(dev->ctrl.device,
3316d011fb31SKeith Busch "failure state error detected, request disconnect\n");
3317a0a3408eSKeith Busch return PCI_ERS_RESULT_DISCONNECT;
3318a0a3408eSKeith Busch }
3319a0a3408eSKeith Busch return PCI_ERS_RESULT_NEED_RESET;
3320a0a3408eSKeith Busch }
3321a0a3408eSKeith Busch
nvme_slot_reset(struct pci_dev * pdev)3322a0a3408eSKeith Busch static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
3323a0a3408eSKeith Busch {
3324a0a3408eSKeith Busch struct nvme_dev *dev = pci_get_drvdata(pdev);
3325a0a3408eSKeith Busch
33261b3c47c1SSagi Grimberg dev_info(dev->ctrl.device, "restart after slot reset\n");
3327a0a3408eSKeith Busch pci_restore_state(pdev);
332871a5bb15SKeith Busch if (!nvme_try_sched_reset(&dev->ctrl))
332971a5bb15SKeith Busch nvme_unquiesce_io_queues(&dev->ctrl);
3330a0a3408eSKeith Busch return PCI_ERS_RESULT_RECOVERED;
3331a0a3408eSKeith Busch }
3332a0a3408eSKeith Busch
nvme_error_resume(struct pci_dev * pdev)3333a0a3408eSKeith Busch static void nvme_error_resume(struct pci_dev *pdev)
3334a0a3408eSKeith Busch {
333572cd4cc2SKeith Busch struct nvme_dev *dev = pci_get_drvdata(pdev);
333672cd4cc2SKeith Busch
333772cd4cc2SKeith Busch flush_work(&dev->ctrl.reset_work);
3338a0a3408eSKeith Busch }
3339a0a3408eSKeith Busch
334057dacad5SJay Sternberg static const struct pci_error_handlers nvme_err_handler = {
334157dacad5SJay Sternberg .error_detected = nvme_error_detected,
334257dacad5SJay Sternberg .slot_reset = nvme_slot_reset,
334357dacad5SJay Sternberg .resume = nvme_error_resume,
3344775755edSChristoph Hellwig .reset_prepare = nvme_reset_prepare,
3345775755edSChristoph Hellwig .reset_done = nvme_reset_done,
334657dacad5SJay Sternberg };
334757dacad5SJay Sternberg
334857dacad5SJay Sternberg static const struct pci_device_id nvme_id_table[] = {
3349972b13e2SDavid Fugate { PCI_VDEVICE(INTEL, 0x0953), /* Intel 750/P3500/P3600/P3700 */
335008095e70SKeith Busch .driver_data = NVME_QUIRK_STRIPE_SIZE |
3351e850fd16SChristoph Hellwig NVME_QUIRK_DEALLOCATE_ZEROES, },
3352972b13e2SDavid Fugate { PCI_VDEVICE(INTEL, 0x0a53), /* Intel P3520 */
335399466e70SKeith Busch .driver_data = NVME_QUIRK_STRIPE_SIZE |
3354e850fd16SChristoph Hellwig NVME_QUIRK_DEALLOCATE_ZEROES, },
3355972b13e2SDavid Fugate { PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */
335699466e70SKeith Busch .driver_data = NVME_QUIRK_STRIPE_SIZE |
335725e58af4SWu Zheng NVME_QUIRK_DEALLOCATE_ZEROES |
33585c3f4066SKeith Busch NVME_QUIRK_IGNORE_DEV_SUBNQN |
33595c3f4066SKeith Busch NVME_QUIRK_BOGUS_NID, },
3360972b13e2SDavid Fugate { PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */
3361f99cb7afSDavid Wayne Fugate .driver_data = NVME_QUIRK_STRIPE_SIZE |
3362f99cb7afSDavid Wayne Fugate NVME_QUIRK_DEALLOCATE_ZEROES, },
336350af47d0SAndy Lutomirski { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */
33649abd68efSJens Axboe .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
33656c6aa2f2SAkinobu Mita NVME_QUIRK_MEDIUM_PRIO_SQ |
3366ce4cc313SDavid Milburn NVME_QUIRK_NO_TEMP_THRESH_CHANGE |
3367ce4cc313SDavid Milburn NVME_QUIRK_DISABLE_WRITE_ZEROES, },
33686299358dSJames Dingwall { PCI_VDEVICE(INTEL, 0xf1a6), /* Intel 760p/Pro 7600p */
33696299358dSJames Dingwall .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3370540c801cSKeith Busch { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */
33717b210e4eSChristoph Hellwig .driver_data = NVME_QUIRK_IDENTIFY_CNS |
337266dd346bSChristoph Hellwig NVME_QUIRK_DISABLE_WRITE_ZEROES |
337366dd346bSChristoph Hellwig NVME_QUIRK_BOGUS_NID, },
337466dd346bSChristoph Hellwig { PCI_VDEVICE(REDHAT, 0x0010), /* Qemu emulated controller */
337566dd346bSChristoph Hellwig .driver_data = NVME_QUIRK_BOGUS_NID, },
33765e11bacfSJiawei Fu (iBug) { PCI_DEVICE(0x126f, 0x2262), /* Silicon Motion generic */
33775e11bacfSJiawei Fu (iBug) .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
33785e11bacfSJiawei Fu (iBug) NVME_QUIRK_BOGUS_NID, },
33795bedd3afSChristoph Hellwig { PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */
3380c98a8793SKeith Busch .driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3381c98a8793SKeith Busch NVME_QUIRK_BOGUS_NID, },
33820302ae60SMicah Parrish { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */
33835e112d3fSJulian Einwag .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
33845e112d3fSJulian Einwag NVME_QUIRK_NO_NS_DESC_LIST, },
338554adc010SGuilherme G. Piccoli { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */
338654adc010SGuilherme G. Piccoli .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
33878c97eeccSJeff Lien { PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */
33888c97eeccSJeff Lien .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3389015282c9SWenbo Wang { PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */
3390015282c9SWenbo Wang .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3391d554b5e1SMartin K. Petersen { PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */
3392d554b5e1SMartin K. Petersen .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3393d554b5e1SMartin K. Petersen { PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */
33947ee5c78cSGopal Tiwari .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3395abbb5f59SDmitry Monakhov NVME_QUIRK_DISABLE_WRITE_ZEROES|
33967ee5c78cSGopal Tiwari NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3397a2da0e5cSSean Anderson { PCI_DEVICE(0x15b7, 0x5008), /* Sandisk SN530 */
3398a2da0e5cSSean Anderson .driver_data = NVME_QUIRK_BROKEN_MSI },
33992cf7a77eSKeith Busch { PCI_DEVICE(0x1987, 0x5012), /* Phison E12 */
34002cf7a77eSKeith Busch .driver_data = NVME_QUIRK_BOGUS_NID, },
3401c9e95c39SClaus Stovgaard { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */
340273029c9bSKeith Busch .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
340373029c9bSKeith Busch NVME_QUIRK_BOGUS_NID, },
3404d14c2731STina Hsu { PCI_DEVICE(0x1987, 0x5019), /* phison E19 */
3405d14c2731STina Hsu .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3406d14c2731STina Hsu { PCI_DEVICE(0x1987, 0x5021), /* Phison E21 */
3407d14c2731STina Hsu .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
34086e6a6828SPascal Terjan { PCI_DEVICE(0x1b4b, 0x1092), /* Lexar 256 GB SSD */
34096e6a6828SPascal Terjan .driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
34106e6a6828SPascal Terjan NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3411e1c70d79SLamarque Vieira Souza { PCI_DEVICE(0x1cc1, 0x33f8), /* ADATA IM2P33F8ABR1 1 TB */
3412e1c70d79SLamarque Vieira Souza .driver_data = NVME_QUIRK_BOGUS_NID, },
341308b903b5SMisha Nasledov { PCI_DEVICE(0x10ec, 0x5762), /* ADATA SX6000LNP */
34141629de0eSPablo Greco .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
34151629de0eSPablo Greco NVME_QUIRK_BOGUS_NID, },
34165f69f009SDaniel Wagner { PCI_DEVICE(0x10ec, 0x5763), /* ADATA SX6000PNP */
34175f69f009SDaniel Wagner .driver_data = NVME_QUIRK_BOGUS_NID, },
3418f03e42c6SGabriel Craciunescu { PCI_DEVICE(0x1cc1, 0x8201), /* ADATA SX8200PNP 512GB */
3419f03e42c6SGabriel Craciunescu .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3420f03e42c6SGabriel Craciunescu NVME_QUIRK_IGNORE_DEV_SUBNQN, },
342141f38043SLeo Savernik { PCI_DEVICE(0x1344, 0x5407), /* Micron Technology Inc NVMe SSD */
342241f38043SLeo Savernik .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN },
3423d5ceb4d1SBean Huo { PCI_DEVICE(0x1344, 0x6001), /* Micron Nitro NVMe */
3424d5ceb4d1SBean Huo .driver_data = NVME_QUIRK_BOGUS_NID, },
34255611ec2bSKai-Heng Feng { PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */
34265611ec2bSKai-Heng Feng .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3427c4f01a77SKeith Busch { PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */
3428c4f01a77SKeith Busch .driver_data = NVME_QUIRK_BOGUS_NID, },
342902ca079cSKai-Heng Feng { PCI_DEVICE(0x15b7, 0x2001), /* Sandisk Skyhawk */
343002ca079cSKai-Heng Feng .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
343189919929SChaitanya Kulkarni { PCI_DEVICE(0x1d97, 0x2263), /* SPCC */
343289919929SChaitanya Kulkarni .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
343343047e08Srasheed.hsueh { PCI_DEVICE(0x144d, 0xa80b), /* Samsung PM9B1 256G and 512G */
3434688b419cSAugust Wikerfors .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES |
3435688b419cSAugust Wikerfors NVME_QUIRK_BOGUS_NID, },
343643047e08Srasheed.hsueh { PCI_DEVICE(0x144d, 0xa809), /* Samsung MZALQ256HBJD 256G */
343743047e08Srasheed.hsueh .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3438e5bb0988SPankaj Raghav { PCI_DEVICE(0x144d, 0xa802), /* Samsung SM953 */
3439e5bb0988SPankaj Raghav .driver_data = NVME_QUIRK_BOGUS_NID, },
344043047e08Srasheed.hsueh { PCI_DEVICE(0x1cc4, 0x6303), /* UMIS RPJTJ512MGE1QDY 512G */
344143047e08Srasheed.hsueh .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
344243047e08Srasheed.hsueh { PCI_DEVICE(0x1cc4, 0x6302), /* UMIS RPJTJ256MGE1QDY 256G */
344343047e08Srasheed.hsueh .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3444dc22c1c0SZoltán Böszörményi { PCI_DEVICE(0x2646, 0x2262), /* KINGSTON SKC2000 NVMe SSD */
3445dc22c1c0SZoltán Böszörményi .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3446538e4a8cSThorsten Leemhuis { PCI_DEVICE(0x2646, 0x2263), /* KINGSTON A2000 NVMe SSD */
3447538e4a8cSThorsten Leemhuis .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3448bd375feeSHristo Venev { PCI_DEVICE(0x2646, 0x5013), /* Kingston KC3000, Kingston FURY Renegade */
3449bd375feeSHristo Venev .driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, },
3450ac9b57d4SXander Li { PCI_DEVICE(0x2646, 0x5018), /* KINGSTON OM8SFP4xxxxP OS21012 NVMe SSD */
3451ac9b57d4SXander Li .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3452ac9b57d4SXander Li { PCI_DEVICE(0x2646, 0x5016), /* KINGSTON OM3PGP4xxxxP OS21011 NVMe SSD */
3453ac9b57d4SXander Li .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3454ac9b57d4SXander Li { PCI_DEVICE(0x2646, 0x501A), /* KINGSTON OM8PGP4xxxxP OS21005 NVMe SSD */
3455ac9b57d4SXander Li .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3456ac9b57d4SXander Li { PCI_DEVICE(0x2646, 0x501B), /* KINGSTON OM8PGP4xxxxQ OS21005 NVMe SSD */
3457ac9b57d4SXander Li .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3458ac9b57d4SXander Li { PCI_DEVICE(0x2646, 0x501E), /* KINGSTON OM3PGP4xxxxQ OS21011 NVMe SSD */
3459ac9b57d4SXander Li .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
34609630d806SElmer Miroslav Mosher Golovin { PCI_DEVICE(0x1f40, 0x1202), /* Netac Technologies Co. NV3000 NVMe SSD */
34619630d806SElmer Miroslav Mosher Golovin .driver_data = NVME_QUIRK_BOGUS_NID, },
34628d6e38f6STiago Dias Ferreira { PCI_DEVICE(0x1f40, 0x5236), /* Netac Technologies Co. NV7000 NVMe SSD */
34638d6e38f6STiago Dias Ferreira .driver_data = NVME_QUIRK_BOGUS_NID, },
346470ce3455SChristoph Hellwig { PCI_DEVICE(0x1e4B, 0x1001), /* MAXIO MAP1001 */
346570ce3455SChristoph Hellwig .driver_data = NVME_QUIRK_BOGUS_NID, },
3466a98a945bSChristoph Hellwig { PCI_DEVICE(0x1e4B, 0x1002), /* MAXIO MAP1002 */
3467a98a945bSChristoph Hellwig .driver_data = NVME_QUIRK_BOGUS_NID, },
3468a98a945bSChristoph Hellwig { PCI_DEVICE(0x1e4B, 0x1202), /* MAXIO MAP1202 */
3469a98a945bSChristoph Hellwig .driver_data = NVME_QUIRK_BOGUS_NID, },
3470a3a9d63dSTatsuki Sugiura { PCI_DEVICE(0x1e4B, 0x1602), /* MAXIO MAP1602 */
3471a3a9d63dSTatsuki Sugiura .driver_data = NVME_QUIRK_BOGUS_NID, },
34723765fad5SStefan Reiter { PCI_DEVICE(0x1cc1, 0x5350), /* ADATA XPG GAMMIX S50 */
34733765fad5SStefan Reiter .driver_data = NVME_QUIRK_BOGUS_NID, },
3474f37527a0SDennis P. Kliem { PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */
3475f37527a0SDennis P. Kliem .driver_data = NVME_QUIRK_BOGUS_NID, },
3476d5d3c100SXi Ruoyao { PCI_DEVICE(0x1e49, 0x0021), /* ZHITAI TiPro5000 NVMe SSD */
3477d5d3c100SXi Ruoyao .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
34786b961bceSNing Wang { PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */
34796b961bceSNing Wang .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3480d6c52fa3STobias Gruetzmacher { PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */
3481d6c52fa3STobias Gruetzmacher .driver_data = NVME_QUIRK_BOGUS_NID, },
3482200dccd0SShyamin Ayesh { PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */
3483200dccd0SShyamin Ayesh .driver_data = NVME_QUIRK_BOGUS_NID, },
3484b65d44faSPhilipp Geulen { PCI_DEVICE(0x1d97, 0x1d97), /* Lexar NM620 */
3485b65d44faSPhilipp Geulen .driver_data = NVME_QUIRK_BOGUS_NID, },
348680b26240SAbhijit { PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */
34871231363aSJuraj Pecigos .driver_data = NVME_QUIRK_BOGUS_NID |
34881231363aSJuraj Pecigos NVME_QUIRK_IGNORE_DEV_SUBNQN, },
348974391b3eSDuy Truong { PCI_DEVICE(0x10ec, 0x5763), /* TEAMGROUP T-FORCE CARDEA ZERO Z330 SSD */
349074391b3eSDuy Truong .driver_data = NVME_QUIRK_BOGUS_NID, },
34911616d6c3SSagi Grimberg { PCI_DEVICE(0x1e4b, 0x1602), /* HS-SSD-FUTURE 2048G */
34921616d6c3SSagi Grimberg .driver_data = NVME_QUIRK_BOGUS_NID, },
349306497281SDaniel Smith { PCI_DEVICE(0x10ec, 0x5765), /* TEAMGROUP MP33 2TB SSD */
349406497281SDaniel Smith .driver_data = NVME_QUIRK_BOGUS_NID, },
34954bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
34964bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
34974bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
34984bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
34994bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x8061),
35004bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
35014bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd00),
35024bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
35034bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd01),
35044bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
35054bdf2603SFilippo Sironi { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd02),
35064bdf2603SFilippo Sironi .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
350798f7b86aSAndy Shevchenko { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001),
350898f7b86aSAndy Shevchenko .driver_data = NVME_QUIRK_SINGLE_VECTOR },
3509124298bdSDaniel Roschka { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
351066341331SBenjamin Herrenschmidt { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005),
351166341331SBenjamin Herrenschmidt .driver_data = NVME_QUIRK_SINGLE_VECTOR |
3512d38e9f04SBenjamin Herrenschmidt NVME_QUIRK_128_BYTES_SQES |
3513a2941f6aSKeith Busch NVME_QUIRK_SHARED_TAGS |
3514453116a4SHector Martin NVME_QUIRK_SKIP_CID_GEN |
3515453116a4SHector Martin NVME_QUIRK_IDENTIFY_CNS },
35160b85f59dSAndy Shevchenko { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
351757dacad5SJay Sternberg { 0, }
351857dacad5SJay Sternberg };
351957dacad5SJay Sternberg MODULE_DEVICE_TABLE(pci, nvme_id_table);
352057dacad5SJay Sternberg
352157dacad5SJay Sternberg static struct pci_driver nvme_driver = {
352257dacad5SJay Sternberg .name = "nvme",
352357dacad5SJay Sternberg .id_table = nvme_id_table,
352457dacad5SJay Sternberg .probe = nvme_probe,
352557dacad5SJay Sternberg .remove = nvme_remove,
352657dacad5SJay Sternberg .shutdown = nvme_shutdown,
352757dacad5SJay Sternberg .driver = {
3528eac3ef26SChristoph Hellwig .probe_type = PROBE_PREFER_ASYNCHRONOUS,
3529eac3ef26SChristoph Hellwig #ifdef CONFIG_PM_SLEEP
353057dacad5SJay Sternberg .pm = &nvme_dev_pm_ops,
3531d916b1beSKeith Busch #endif
3532eac3ef26SChristoph Hellwig },
353374d986abSAlexander Duyck .sriov_configure = pci_sriov_configure_simple,
353457dacad5SJay Sternberg .err_handler = &nvme_err_handler,
353557dacad5SJay Sternberg };
353657dacad5SJay Sternberg
nvme_init(void)353757dacad5SJay Sternberg static int __init nvme_init(void)
353857dacad5SJay Sternberg {
353981101540SChristoph Hellwig BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
354081101540SChristoph Hellwig BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
354181101540SChristoph Hellwig BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
3542612b7286SMing Lei BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
354301df742dSKeith Busch BUILD_BUG_ON(NVME_MAX_SEGS > SGES_PER_PAGE);
35447846c1b5SKeith Busch BUILD_BUG_ON(sizeof(struct scatterlist) * NVME_MAX_SEGS > PAGE_SIZE);
35457846c1b5SKeith Busch BUILD_BUG_ON(nvme_pci_npages_prp() > NVME_MAX_NR_ALLOCATIONS);
354617c33167SKeith Busch
35479a6327d2SSagi Grimberg return pci_register_driver(&nvme_driver);
354857dacad5SJay Sternberg }
354957dacad5SJay Sternberg
nvme_exit(void)355057dacad5SJay Sternberg static void __exit nvme_exit(void)
355157dacad5SJay Sternberg {
355257dacad5SJay Sternberg pci_unregister_driver(&nvme_driver);
355303e0f3a6SMing Lei flush_workqueue(nvme_wq);
355457dacad5SJay Sternberg }
355557dacad5SJay Sternberg
355657dacad5SJay Sternberg MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
355757dacad5SJay Sternberg MODULE_LICENSE("GPL");
355857dacad5SJay Sternberg MODULE_VERSION("1.0");
355957dacad5SJay Sternberg module_init(nvme_init);
356057dacad5SJay Sternberg module_exit(nvme_exit);
3561