1 /* 2 * NVM Express device driver 3 * Copyright (c) 2011-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 15 #include <linux/aer.h> 16 #include <linux/bitops.h> 17 #include <linux/blkdev.h> 18 #include <linux/blk-mq.h> 19 #include <linux/blk-mq-pci.h> 20 #include <linux/dmi.h> 21 #include <linux/init.h> 22 #include <linux/interrupt.h> 23 #include <linux/io.h> 24 #include <linux/mm.h> 25 #include <linux/module.h> 26 #include <linux/mutex.h> 27 #include <linux/pci.h> 28 #include <linux/poison.h> 29 #include <linux/t10-pi.h> 30 #include <linux/timer.h> 31 #include <linux/types.h> 32 #include <linux/io-64-nonatomic-lo-hi.h> 33 #include <asm/unaligned.h> 34 #include <linux/sed-opal.h> 35 36 #include "nvme.h" 37 38 #define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 39 #define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 40 41 /* 42 * We handle AEN commands ourselves and don't even let the 43 * block layer know about them. 44 */ 45 #define NVME_AQ_BLKMQ_DEPTH (NVME_AQ_DEPTH - NVME_NR_AERS) 46 47 static int use_threaded_interrupts; 48 module_param(use_threaded_interrupts, int, 0); 49 50 static bool use_cmb_sqes = true; 51 module_param(use_cmb_sqes, bool, 0644); 52 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes"); 53 54 static unsigned int max_host_mem_size_mb = 128; 55 module_param(max_host_mem_size_mb, uint, 0444); 56 MODULE_PARM_DESC(max_host_mem_size_mb, 57 "Maximum Host Memory Buffer (HMB) size per controller (in MiB)"); 58 59 static int io_queue_depth_set(const char *val, const struct kernel_param *kp); 60 static const struct kernel_param_ops io_queue_depth_ops = { 61 .set = io_queue_depth_set, 62 .get = param_get_int, 63 }; 64 65 static int io_queue_depth = 1024; 66 module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); 67 MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); 68 69 struct nvme_dev; 70 struct nvme_queue; 71 72 static void nvme_process_cq(struct nvme_queue *nvmeq); 73 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown); 74 75 /* 76 * Represents an NVM Express device. Each nvme_dev is a PCI function. 77 */ 78 struct nvme_dev { 79 struct nvme_queue **queues; 80 struct blk_mq_tag_set tagset; 81 struct blk_mq_tag_set admin_tagset; 82 u32 __iomem *dbs; 83 struct device *dev; 84 struct dma_pool *prp_page_pool; 85 struct dma_pool *prp_small_pool; 86 unsigned online_queues; 87 unsigned max_qid; 88 int q_depth; 89 u32 db_stride; 90 void __iomem *bar; 91 unsigned long bar_mapped_size; 92 struct work_struct remove_work; 93 struct mutex shutdown_lock; 94 bool subsystem; 95 void __iomem *cmb; 96 dma_addr_t cmb_dma_addr; 97 u64 cmb_size; 98 u32 cmbsz; 99 u32 cmbloc; 100 struct nvme_ctrl ctrl; 101 struct completion ioq_wait; 102 103 /* shadow doorbell buffer support: */ 104 u32 *dbbuf_dbs; 105 dma_addr_t dbbuf_dbs_dma_addr; 106 u32 *dbbuf_eis; 107 dma_addr_t dbbuf_eis_dma_addr; 108 109 /* host memory buffer support: */ 110 u64 host_mem_size; 111 u32 nr_host_mem_descs; 112 struct nvme_host_mem_buf_desc *host_mem_descs; 113 void **host_mem_desc_bufs; 114 }; 115 116 static int io_queue_depth_set(const char *val, const struct kernel_param *kp) 117 { 118 int n = 0, ret; 119 120 ret = kstrtoint(val, 10, &n); 121 if (ret != 0 || n < 2) 122 return -EINVAL; 123 124 return param_set_int(val, kp); 125 } 126 127 static inline unsigned int sq_idx(unsigned int qid, u32 stride) 128 { 129 return qid * 2 * stride; 130 } 131 132 static inline unsigned int cq_idx(unsigned int qid, u32 stride) 133 { 134 return (qid * 2 + 1) * stride; 135 } 136 137 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) 138 { 139 return container_of(ctrl, struct nvme_dev, ctrl); 140 } 141 142 /* 143 * An NVM Express queue. Each device has at least two (one for admin 144 * commands and one for I/O commands). 145 */ 146 struct nvme_queue { 147 struct device *q_dmadev; 148 struct nvme_dev *dev; 149 spinlock_t q_lock; 150 struct nvme_command *sq_cmds; 151 struct nvme_command __iomem *sq_cmds_io; 152 volatile struct nvme_completion *cqes; 153 struct blk_mq_tags **tags; 154 dma_addr_t sq_dma_addr; 155 dma_addr_t cq_dma_addr; 156 u32 __iomem *q_db; 157 u16 q_depth; 158 s16 cq_vector; 159 u16 sq_tail; 160 u16 cq_head; 161 u16 qid; 162 u8 cq_phase; 163 u8 cqe_seen; 164 u32 *dbbuf_sq_db; 165 u32 *dbbuf_cq_db; 166 u32 *dbbuf_sq_ei; 167 u32 *dbbuf_cq_ei; 168 }; 169 170 /* 171 * The nvme_iod describes the data in an I/O, including the list of PRP 172 * entries. You can't see it in this data structure because C doesn't let 173 * me express that. Use nvme_init_iod to ensure there's enough space 174 * allocated to store the PRP list. 175 */ 176 struct nvme_iod { 177 struct nvme_request req; 178 struct nvme_queue *nvmeq; 179 int aborted; 180 int npages; /* In the PRP list. 0 means small pool in use */ 181 int nents; /* Used in scatterlist */ 182 int length; /* Of data, in bytes */ 183 dma_addr_t first_dma; 184 struct scatterlist meta_sg; /* metadata requires single contiguous buffer */ 185 struct scatterlist *sg; 186 struct scatterlist inline_sg[0]; 187 }; 188 189 /* 190 * Check we didin't inadvertently grow the command struct 191 */ 192 static inline void _nvme_check_size(void) 193 { 194 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 195 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 196 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 197 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 198 BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 199 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 200 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); 201 BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 202 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE); 203 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE); 204 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 205 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 206 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64); 207 } 208 209 static inline unsigned int nvme_dbbuf_size(u32 stride) 210 { 211 return ((num_possible_cpus() + 1) * 8 * stride); 212 } 213 214 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) 215 { 216 unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); 217 218 if (dev->dbbuf_dbs) 219 return 0; 220 221 dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size, 222 &dev->dbbuf_dbs_dma_addr, 223 GFP_KERNEL); 224 if (!dev->dbbuf_dbs) 225 return -ENOMEM; 226 dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size, 227 &dev->dbbuf_eis_dma_addr, 228 GFP_KERNEL); 229 if (!dev->dbbuf_eis) { 230 dma_free_coherent(dev->dev, mem_size, 231 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr); 232 dev->dbbuf_dbs = NULL; 233 return -ENOMEM; 234 } 235 236 return 0; 237 } 238 239 static void nvme_dbbuf_dma_free(struct nvme_dev *dev) 240 { 241 unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); 242 243 if (dev->dbbuf_dbs) { 244 dma_free_coherent(dev->dev, mem_size, 245 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr); 246 dev->dbbuf_dbs = NULL; 247 } 248 if (dev->dbbuf_eis) { 249 dma_free_coherent(dev->dev, mem_size, 250 dev->dbbuf_eis, dev->dbbuf_eis_dma_addr); 251 dev->dbbuf_eis = NULL; 252 } 253 } 254 255 static void nvme_dbbuf_init(struct nvme_dev *dev, 256 struct nvme_queue *nvmeq, int qid) 257 { 258 if (!dev->dbbuf_dbs || !qid) 259 return; 260 261 nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)]; 262 nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)]; 263 nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)]; 264 nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)]; 265 } 266 267 static void nvme_dbbuf_set(struct nvme_dev *dev) 268 { 269 struct nvme_command c; 270 271 if (!dev->dbbuf_dbs) 272 return; 273 274 memset(&c, 0, sizeof(c)); 275 c.dbbuf.opcode = nvme_admin_dbbuf; 276 c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr); 277 c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr); 278 279 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) { 280 dev_warn(dev->ctrl.device, "unable to set dbbuf\n"); 281 /* Free memory and continue on */ 282 nvme_dbbuf_dma_free(dev); 283 } 284 } 285 286 static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old) 287 { 288 return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 289 } 290 291 /* Update dbbuf and return true if an MMIO is required */ 292 static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db, 293 volatile u32 *dbbuf_ei) 294 { 295 if (dbbuf_db) { 296 u16 old_value; 297 298 /* 299 * Ensure that the queue is written before updating 300 * the doorbell in memory 301 */ 302 wmb(); 303 304 old_value = *dbbuf_db; 305 *dbbuf_db = value; 306 307 if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 308 return false; 309 } 310 311 return true; 312 } 313 314 /* 315 * Max size of iod being embedded in the request payload 316 */ 317 #define NVME_INT_PAGES 2 318 #define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->ctrl.page_size) 319 320 /* 321 * Will slightly overestimate the number of pages needed. This is OK 322 * as it only leads to a small amount of wasted memory for the lifetime of 323 * the I/O. 324 */ 325 static int nvme_npages(unsigned size, struct nvme_dev *dev) 326 { 327 unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size, 328 dev->ctrl.page_size); 329 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 330 } 331 332 static unsigned int nvme_iod_alloc_size(struct nvme_dev *dev, 333 unsigned int size, unsigned int nseg) 334 { 335 return sizeof(__le64 *) * nvme_npages(size, dev) + 336 sizeof(struct scatterlist) * nseg; 337 } 338 339 static unsigned int nvme_cmd_size(struct nvme_dev *dev) 340 { 341 return sizeof(struct nvme_iod) + 342 nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES); 343 } 344 345 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 346 unsigned int hctx_idx) 347 { 348 struct nvme_dev *dev = data; 349 struct nvme_queue *nvmeq = dev->queues[0]; 350 351 WARN_ON(hctx_idx != 0); 352 WARN_ON(dev->admin_tagset.tags[0] != hctx->tags); 353 WARN_ON(nvmeq->tags); 354 355 hctx->driver_data = nvmeq; 356 nvmeq->tags = &dev->admin_tagset.tags[0]; 357 return 0; 358 } 359 360 static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 361 { 362 struct nvme_queue *nvmeq = hctx->driver_data; 363 364 nvmeq->tags = NULL; 365 } 366 367 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 368 unsigned int hctx_idx) 369 { 370 struct nvme_dev *dev = data; 371 struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1]; 372 373 if (!nvmeq->tags) 374 nvmeq->tags = &dev->tagset.tags[hctx_idx]; 375 376 WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags); 377 hctx->driver_data = nvmeq; 378 return 0; 379 } 380 381 static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req, 382 unsigned int hctx_idx, unsigned int numa_node) 383 { 384 struct nvme_dev *dev = set->driver_data; 385 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 386 int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0; 387 struct nvme_queue *nvmeq = dev->queues[queue_idx]; 388 389 BUG_ON(!nvmeq); 390 iod->nvmeq = nvmeq; 391 return 0; 392 } 393 394 static int nvme_pci_map_queues(struct blk_mq_tag_set *set) 395 { 396 struct nvme_dev *dev = set->driver_data; 397 398 return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev)); 399 } 400 401 /** 402 * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 403 * @nvmeq: The queue to use 404 * @cmd: The command to send 405 * 406 * Safe to use from interrupt context 407 */ 408 static void __nvme_submit_cmd(struct nvme_queue *nvmeq, 409 struct nvme_command *cmd) 410 { 411 u16 tail = nvmeq->sq_tail; 412 413 if (nvmeq->sq_cmds_io) 414 memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd)); 415 else 416 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 417 418 if (++tail == nvmeq->q_depth) 419 tail = 0; 420 if (nvme_dbbuf_update_and_check_event(tail, nvmeq->dbbuf_sq_db, 421 nvmeq->dbbuf_sq_ei)) 422 writel(tail, nvmeq->q_db); 423 nvmeq->sq_tail = tail; 424 } 425 426 static __le64 **iod_list(struct request *req) 427 { 428 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 429 return (__le64 **)(iod->sg + blk_rq_nr_phys_segments(req)); 430 } 431 432 static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev) 433 { 434 struct nvme_iod *iod = blk_mq_rq_to_pdu(rq); 435 int nseg = blk_rq_nr_phys_segments(rq); 436 unsigned int size = blk_rq_payload_bytes(rq); 437 438 if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) { 439 iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC); 440 if (!iod->sg) 441 return BLK_STS_RESOURCE; 442 } else { 443 iod->sg = iod->inline_sg; 444 } 445 446 iod->aborted = 0; 447 iod->npages = -1; 448 iod->nents = 0; 449 iod->length = size; 450 451 return BLK_STS_OK; 452 } 453 454 static void nvme_free_iod(struct nvme_dev *dev, struct request *req) 455 { 456 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 457 const int last_prp = dev->ctrl.page_size / 8 - 1; 458 int i; 459 __le64 **list = iod_list(req); 460 dma_addr_t prp_dma = iod->first_dma; 461 462 if (iod->npages == 0) 463 dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 464 for (i = 0; i < iod->npages; i++) { 465 __le64 *prp_list = list[i]; 466 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 467 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 468 prp_dma = next_prp_dma; 469 } 470 471 if (iod->sg != iod->inline_sg) 472 kfree(iod->sg); 473 } 474 475 #ifdef CONFIG_BLK_DEV_INTEGRITY 476 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi) 477 { 478 if (be32_to_cpu(pi->ref_tag) == v) 479 pi->ref_tag = cpu_to_be32(p); 480 } 481 482 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi) 483 { 484 if (be32_to_cpu(pi->ref_tag) == p) 485 pi->ref_tag = cpu_to_be32(v); 486 } 487 488 /** 489 * nvme_dif_remap - remaps ref tags to bip seed and physical lba 490 * 491 * The virtual start sector is the one that was originally submitted by the 492 * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical 493 * start sector may be different. Remap protection information to match the 494 * physical LBA on writes, and back to the original seed on reads. 495 * 496 * Type 0 and 3 do not have a ref tag, so no remapping required. 497 */ 498 static void nvme_dif_remap(struct request *req, 499 void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi)) 500 { 501 struct nvme_ns *ns = req->rq_disk->private_data; 502 struct bio_integrity_payload *bip; 503 struct t10_pi_tuple *pi; 504 void *p, *pmap; 505 u32 i, nlb, ts, phys, virt; 506 507 if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3) 508 return; 509 510 bip = bio_integrity(req->bio); 511 if (!bip) 512 return; 513 514 pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; 515 516 p = pmap; 517 virt = bip_get_seed(bip); 518 phys = nvme_block_nr(ns, blk_rq_pos(req)); 519 nlb = (blk_rq_bytes(req) >> ns->lba_shift); 520 ts = ns->disk->queue->integrity.tuple_size; 521 522 for (i = 0; i < nlb; i++, virt++, phys++) { 523 pi = (struct t10_pi_tuple *)p; 524 dif_swap(phys, virt, pi); 525 p += ts; 526 } 527 kunmap_atomic(pmap); 528 } 529 #else /* CONFIG_BLK_DEV_INTEGRITY */ 530 static void nvme_dif_remap(struct request *req, 531 void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi)) 532 { 533 } 534 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi) 535 { 536 } 537 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi) 538 { 539 } 540 #endif 541 542 static bool nvme_setup_prps(struct nvme_dev *dev, struct request *req) 543 { 544 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 545 struct dma_pool *pool; 546 int length = blk_rq_payload_bytes(req); 547 struct scatterlist *sg = iod->sg; 548 int dma_len = sg_dma_len(sg); 549 u64 dma_addr = sg_dma_address(sg); 550 u32 page_size = dev->ctrl.page_size; 551 int offset = dma_addr & (page_size - 1); 552 __le64 *prp_list; 553 __le64 **list = iod_list(req); 554 dma_addr_t prp_dma; 555 int nprps, i; 556 557 length -= (page_size - offset); 558 if (length <= 0) 559 return true; 560 561 dma_len -= (page_size - offset); 562 if (dma_len) { 563 dma_addr += (page_size - offset); 564 } else { 565 sg = sg_next(sg); 566 dma_addr = sg_dma_address(sg); 567 dma_len = sg_dma_len(sg); 568 } 569 570 if (length <= page_size) { 571 iod->first_dma = dma_addr; 572 return true; 573 } 574 575 nprps = DIV_ROUND_UP(length, page_size); 576 if (nprps <= (256 / 8)) { 577 pool = dev->prp_small_pool; 578 iod->npages = 0; 579 } else { 580 pool = dev->prp_page_pool; 581 iod->npages = 1; 582 } 583 584 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 585 if (!prp_list) { 586 iod->first_dma = dma_addr; 587 iod->npages = -1; 588 return false; 589 } 590 list[0] = prp_list; 591 iod->first_dma = prp_dma; 592 i = 0; 593 for (;;) { 594 if (i == page_size >> 3) { 595 __le64 *old_prp_list = prp_list; 596 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 597 if (!prp_list) 598 return false; 599 list[iod->npages++] = prp_list; 600 prp_list[0] = old_prp_list[i - 1]; 601 old_prp_list[i - 1] = cpu_to_le64(prp_dma); 602 i = 1; 603 } 604 prp_list[i++] = cpu_to_le64(dma_addr); 605 dma_len -= page_size; 606 dma_addr += page_size; 607 length -= page_size; 608 if (length <= 0) 609 break; 610 if (dma_len > 0) 611 continue; 612 BUG_ON(dma_len < 0); 613 sg = sg_next(sg); 614 dma_addr = sg_dma_address(sg); 615 dma_len = sg_dma_len(sg); 616 } 617 618 return true; 619 } 620 621 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, 622 struct nvme_command *cmnd) 623 { 624 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 625 struct request_queue *q = req->q; 626 enum dma_data_direction dma_dir = rq_data_dir(req) ? 627 DMA_TO_DEVICE : DMA_FROM_DEVICE; 628 blk_status_t ret = BLK_STS_IOERR; 629 630 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); 631 iod->nents = blk_rq_map_sg(q, req, iod->sg); 632 if (!iod->nents) 633 goto out; 634 635 ret = BLK_STS_RESOURCE; 636 if (!dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir, 637 DMA_ATTR_NO_WARN)) 638 goto out; 639 640 if (!nvme_setup_prps(dev, req)) 641 goto out_unmap; 642 643 ret = BLK_STS_IOERR; 644 if (blk_integrity_rq(req)) { 645 if (blk_rq_count_integrity_sg(q, req->bio) != 1) 646 goto out_unmap; 647 648 sg_init_table(&iod->meta_sg, 1); 649 if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1) 650 goto out_unmap; 651 652 if (rq_data_dir(req)) 653 nvme_dif_remap(req, nvme_dif_prep); 654 655 if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir)) 656 goto out_unmap; 657 } 658 659 cmnd->rw.dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); 660 cmnd->rw.dptr.prp2 = cpu_to_le64(iod->first_dma); 661 if (blk_integrity_rq(req)) 662 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg)); 663 return BLK_STS_OK; 664 665 out_unmap: 666 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir); 667 out: 668 return ret; 669 } 670 671 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req) 672 { 673 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 674 enum dma_data_direction dma_dir = rq_data_dir(req) ? 675 DMA_TO_DEVICE : DMA_FROM_DEVICE; 676 677 if (iod->nents) { 678 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir); 679 if (blk_integrity_rq(req)) { 680 if (!rq_data_dir(req)) 681 nvme_dif_remap(req, nvme_dif_complete); 682 dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir); 683 } 684 } 685 686 nvme_cleanup_cmd(req); 687 nvme_free_iod(dev, req); 688 } 689 690 /* 691 * NOTE: ns is NULL when called on the admin queue. 692 */ 693 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, 694 const struct blk_mq_queue_data *bd) 695 { 696 struct nvme_ns *ns = hctx->queue->queuedata; 697 struct nvme_queue *nvmeq = hctx->driver_data; 698 struct nvme_dev *dev = nvmeq->dev; 699 struct request *req = bd->rq; 700 struct nvme_command cmnd; 701 blk_status_t ret; 702 703 ret = nvme_setup_cmd(ns, req, &cmnd); 704 if (ret) 705 return ret; 706 707 ret = nvme_init_iod(req, dev); 708 if (ret) 709 goto out_free_cmd; 710 711 if (blk_rq_nr_phys_segments(req)) { 712 ret = nvme_map_data(dev, req, &cmnd); 713 if (ret) 714 goto out_cleanup_iod; 715 } 716 717 blk_mq_start_request(req); 718 719 spin_lock_irq(&nvmeq->q_lock); 720 if (unlikely(nvmeq->cq_vector < 0)) { 721 ret = BLK_STS_IOERR; 722 spin_unlock_irq(&nvmeq->q_lock); 723 goto out_cleanup_iod; 724 } 725 __nvme_submit_cmd(nvmeq, &cmnd); 726 nvme_process_cq(nvmeq); 727 spin_unlock_irq(&nvmeq->q_lock); 728 return BLK_STS_OK; 729 out_cleanup_iod: 730 nvme_free_iod(dev, req); 731 out_free_cmd: 732 nvme_cleanup_cmd(req); 733 return ret; 734 } 735 736 static void nvme_pci_complete_rq(struct request *req) 737 { 738 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 739 740 nvme_unmap_data(iod->nvmeq->dev, req); 741 nvme_complete_rq(req); 742 } 743 744 /* We read the CQE phase first to check if the rest of the entry is valid */ 745 static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head, 746 u16 phase) 747 { 748 return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase; 749 } 750 751 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) 752 { 753 u16 head = nvmeq->cq_head; 754 755 if (likely(nvmeq->cq_vector >= 0)) { 756 if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db, 757 nvmeq->dbbuf_cq_ei)) 758 writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 759 } 760 } 761 762 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, 763 struct nvme_completion *cqe) 764 { 765 struct request *req; 766 767 if (unlikely(cqe->command_id >= nvmeq->q_depth)) { 768 dev_warn(nvmeq->dev->ctrl.device, 769 "invalid id %d completed on queue %d\n", 770 cqe->command_id, le16_to_cpu(cqe->sq_id)); 771 return; 772 } 773 774 /* 775 * AEN requests are special as they don't time out and can 776 * survive any kind of queue freeze and often don't respond to 777 * aborts. We don't even bother to allocate a struct request 778 * for them but rather special case them here. 779 */ 780 if (unlikely(nvmeq->qid == 0 && 781 cqe->command_id >= NVME_AQ_BLKMQ_DEPTH)) { 782 nvme_complete_async_event(&nvmeq->dev->ctrl, 783 cqe->status, &cqe->result); 784 return; 785 } 786 787 req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id); 788 nvme_end_request(req, cqe->status, cqe->result); 789 } 790 791 static inline bool nvme_read_cqe(struct nvme_queue *nvmeq, 792 struct nvme_completion *cqe) 793 { 794 if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) { 795 *cqe = nvmeq->cqes[nvmeq->cq_head]; 796 797 if (++nvmeq->cq_head == nvmeq->q_depth) { 798 nvmeq->cq_head = 0; 799 nvmeq->cq_phase = !nvmeq->cq_phase; 800 } 801 return true; 802 } 803 return false; 804 } 805 806 static void nvme_process_cq(struct nvme_queue *nvmeq) 807 { 808 struct nvme_completion cqe; 809 int consumed = 0; 810 811 while (nvme_read_cqe(nvmeq, &cqe)) { 812 nvme_handle_cqe(nvmeq, &cqe); 813 consumed++; 814 } 815 816 if (consumed) { 817 nvme_ring_cq_doorbell(nvmeq); 818 nvmeq->cqe_seen = 1; 819 } 820 } 821 822 static irqreturn_t nvme_irq(int irq, void *data) 823 { 824 irqreturn_t result; 825 struct nvme_queue *nvmeq = data; 826 spin_lock(&nvmeq->q_lock); 827 nvme_process_cq(nvmeq); 828 result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE; 829 nvmeq->cqe_seen = 0; 830 spin_unlock(&nvmeq->q_lock); 831 return result; 832 } 833 834 static irqreturn_t nvme_irq_check(int irq, void *data) 835 { 836 struct nvme_queue *nvmeq = data; 837 if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) 838 return IRQ_WAKE_THREAD; 839 return IRQ_NONE; 840 } 841 842 static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag) 843 { 844 struct nvme_completion cqe; 845 int found = 0, consumed = 0; 846 847 if (!nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) 848 return 0; 849 850 spin_lock_irq(&nvmeq->q_lock); 851 while (nvme_read_cqe(nvmeq, &cqe)) { 852 nvme_handle_cqe(nvmeq, &cqe); 853 consumed++; 854 855 if (tag == cqe.command_id) { 856 found = 1; 857 break; 858 } 859 } 860 861 if (consumed) 862 nvme_ring_cq_doorbell(nvmeq); 863 spin_unlock_irq(&nvmeq->q_lock); 864 865 return found; 866 } 867 868 static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) 869 { 870 struct nvme_queue *nvmeq = hctx->driver_data; 871 872 return __nvme_poll(nvmeq, tag); 873 } 874 875 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl, int aer_idx) 876 { 877 struct nvme_dev *dev = to_nvme_dev(ctrl); 878 struct nvme_queue *nvmeq = dev->queues[0]; 879 struct nvme_command c; 880 881 memset(&c, 0, sizeof(c)); 882 c.common.opcode = nvme_admin_async_event; 883 c.common.command_id = NVME_AQ_BLKMQ_DEPTH + aer_idx; 884 885 spin_lock_irq(&nvmeq->q_lock); 886 __nvme_submit_cmd(nvmeq, &c); 887 spin_unlock_irq(&nvmeq->q_lock); 888 } 889 890 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 891 { 892 struct nvme_command c; 893 894 memset(&c, 0, sizeof(c)); 895 c.delete_queue.opcode = opcode; 896 c.delete_queue.qid = cpu_to_le16(id); 897 898 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 899 } 900 901 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 902 struct nvme_queue *nvmeq) 903 { 904 struct nvme_command c; 905 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 906 907 /* 908 * Note: we (ab)use the fact the the prp fields survive if no data 909 * is attached to the request. 910 */ 911 memset(&c, 0, sizeof(c)); 912 c.create_cq.opcode = nvme_admin_create_cq; 913 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 914 c.create_cq.cqid = cpu_to_le16(qid); 915 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 916 c.create_cq.cq_flags = cpu_to_le16(flags); 917 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 918 919 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 920 } 921 922 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 923 struct nvme_queue *nvmeq) 924 { 925 struct nvme_command c; 926 int flags = NVME_QUEUE_PHYS_CONTIG; 927 928 /* 929 * Note: we (ab)use the fact the the prp fields survive if no data 930 * is attached to the request. 931 */ 932 memset(&c, 0, sizeof(c)); 933 c.create_sq.opcode = nvme_admin_create_sq; 934 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 935 c.create_sq.sqid = cpu_to_le16(qid); 936 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 937 c.create_sq.sq_flags = cpu_to_le16(flags); 938 c.create_sq.cqid = cpu_to_le16(qid); 939 940 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 941 } 942 943 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 944 { 945 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 946 } 947 948 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 949 { 950 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 951 } 952 953 static void abort_endio(struct request *req, blk_status_t error) 954 { 955 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 956 struct nvme_queue *nvmeq = iod->nvmeq; 957 958 dev_warn(nvmeq->dev->ctrl.device, 959 "Abort status: 0x%x", nvme_req(req)->status); 960 atomic_inc(&nvmeq->dev->ctrl.abort_limit); 961 blk_mq_free_request(req); 962 } 963 964 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) 965 { 966 967 /* If true, indicates loss of adapter communication, possibly by a 968 * NVMe Subsystem reset. 969 */ 970 bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO); 971 972 /* If there is a reset ongoing, we shouldn't reset again. */ 973 if (dev->ctrl.state == NVME_CTRL_RESETTING) 974 return false; 975 976 /* We shouldn't reset unless the controller is on fatal error state 977 * _or_ if we lost the communication with it. 978 */ 979 if (!(csts & NVME_CSTS_CFS) && !nssro) 980 return false; 981 982 /* If PCI error recovery process is happening, we cannot reset or 983 * the recovery mechanism will surely fail. 984 */ 985 if (pci_channel_offline(to_pci_dev(dev->dev))) 986 return false; 987 988 return true; 989 } 990 991 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) 992 { 993 /* Read a config register to help see what died. */ 994 u16 pci_status; 995 int result; 996 997 result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS, 998 &pci_status); 999 if (result == PCIBIOS_SUCCESSFUL) 1000 dev_warn(dev->ctrl.device, 1001 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n", 1002 csts, pci_status); 1003 else 1004 dev_warn(dev->ctrl.device, 1005 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n", 1006 csts, result); 1007 } 1008 1009 static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) 1010 { 1011 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1012 struct nvme_queue *nvmeq = iod->nvmeq; 1013 struct nvme_dev *dev = nvmeq->dev; 1014 struct request *abort_req; 1015 struct nvme_command cmd; 1016 u32 csts = readl(dev->bar + NVME_REG_CSTS); 1017 1018 /* 1019 * Reset immediately if the controller is failed 1020 */ 1021 if (nvme_should_reset(dev, csts)) { 1022 nvme_warn_reset(dev, csts); 1023 nvme_dev_disable(dev, false); 1024 nvme_reset_ctrl(&dev->ctrl); 1025 return BLK_EH_HANDLED; 1026 } 1027 1028 /* 1029 * Did we miss an interrupt? 1030 */ 1031 if (__nvme_poll(nvmeq, req->tag)) { 1032 dev_warn(dev->ctrl.device, 1033 "I/O %d QID %d timeout, completion polled\n", 1034 req->tag, nvmeq->qid); 1035 return BLK_EH_HANDLED; 1036 } 1037 1038 /* 1039 * Shutdown immediately if controller times out while starting. The 1040 * reset work will see the pci device disabled when it gets the forced 1041 * cancellation error. All outstanding requests are completed on 1042 * shutdown, so we return BLK_EH_HANDLED. 1043 */ 1044 if (dev->ctrl.state == NVME_CTRL_RESETTING) { 1045 dev_warn(dev->ctrl.device, 1046 "I/O %d QID %d timeout, disable controller\n", 1047 req->tag, nvmeq->qid); 1048 nvme_dev_disable(dev, false); 1049 nvme_req(req)->flags |= NVME_REQ_CANCELLED; 1050 return BLK_EH_HANDLED; 1051 } 1052 1053 /* 1054 * Shutdown the controller immediately and schedule a reset if the 1055 * command was already aborted once before and still hasn't been 1056 * returned to the driver, or if this is the admin queue. 1057 */ 1058 if (!nvmeq->qid || iod->aborted) { 1059 dev_warn(dev->ctrl.device, 1060 "I/O %d QID %d timeout, reset controller\n", 1061 req->tag, nvmeq->qid); 1062 nvme_dev_disable(dev, false); 1063 nvme_reset_ctrl(&dev->ctrl); 1064 1065 /* 1066 * Mark the request as handled, since the inline shutdown 1067 * forces all outstanding requests to complete. 1068 */ 1069 nvme_req(req)->flags |= NVME_REQ_CANCELLED; 1070 return BLK_EH_HANDLED; 1071 } 1072 1073 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) { 1074 atomic_inc(&dev->ctrl.abort_limit); 1075 return BLK_EH_RESET_TIMER; 1076 } 1077 iod->aborted = 1; 1078 1079 memset(&cmd, 0, sizeof(cmd)); 1080 cmd.abort.opcode = nvme_admin_abort_cmd; 1081 cmd.abort.cid = req->tag; 1082 cmd.abort.sqid = cpu_to_le16(nvmeq->qid); 1083 1084 dev_warn(nvmeq->dev->ctrl.device, 1085 "I/O %d QID %d timeout, aborting\n", 1086 req->tag, nvmeq->qid); 1087 1088 abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd, 1089 BLK_MQ_REQ_NOWAIT, NVME_QID_ANY); 1090 if (IS_ERR(abort_req)) { 1091 atomic_inc(&dev->ctrl.abort_limit); 1092 return BLK_EH_RESET_TIMER; 1093 } 1094 1095 abort_req->timeout = ADMIN_TIMEOUT; 1096 abort_req->end_io_data = NULL; 1097 blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio); 1098 1099 /* 1100 * The aborted req will be completed on receiving the abort req. 1101 * We enable the timer again. If hit twice, it'll cause a device reset, 1102 * as the device then is in a faulty state. 1103 */ 1104 return BLK_EH_RESET_TIMER; 1105 } 1106 1107 static void nvme_free_queue(struct nvme_queue *nvmeq) 1108 { 1109 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 1110 (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 1111 if (nvmeq->sq_cmds) 1112 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 1113 nvmeq->sq_cmds, nvmeq->sq_dma_addr); 1114 kfree(nvmeq); 1115 } 1116 1117 static void nvme_free_queues(struct nvme_dev *dev, int lowest) 1118 { 1119 int i; 1120 1121 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) { 1122 struct nvme_queue *nvmeq = dev->queues[i]; 1123 dev->ctrl.queue_count--; 1124 dev->queues[i] = NULL; 1125 nvme_free_queue(nvmeq); 1126 } 1127 } 1128 1129 /** 1130 * nvme_suspend_queue - put queue into suspended state 1131 * @nvmeq - queue to suspend 1132 */ 1133 static int nvme_suspend_queue(struct nvme_queue *nvmeq) 1134 { 1135 int vector; 1136 1137 spin_lock_irq(&nvmeq->q_lock); 1138 if (nvmeq->cq_vector == -1) { 1139 spin_unlock_irq(&nvmeq->q_lock); 1140 return 1; 1141 } 1142 vector = nvmeq->cq_vector; 1143 nvmeq->dev->online_queues--; 1144 nvmeq->cq_vector = -1; 1145 spin_unlock_irq(&nvmeq->q_lock); 1146 1147 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q) 1148 blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q); 1149 1150 pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq); 1151 1152 return 0; 1153 } 1154 1155 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown) 1156 { 1157 struct nvme_queue *nvmeq = dev->queues[0]; 1158 1159 if (!nvmeq) 1160 return; 1161 if (nvme_suspend_queue(nvmeq)) 1162 return; 1163 1164 if (shutdown) 1165 nvme_shutdown_ctrl(&dev->ctrl); 1166 else 1167 nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap); 1168 1169 spin_lock_irq(&nvmeq->q_lock); 1170 nvme_process_cq(nvmeq); 1171 spin_unlock_irq(&nvmeq->q_lock); 1172 } 1173 1174 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, 1175 int entry_size) 1176 { 1177 int q_depth = dev->q_depth; 1178 unsigned q_size_aligned = roundup(q_depth * entry_size, 1179 dev->ctrl.page_size); 1180 1181 if (q_size_aligned * nr_io_queues > dev->cmb_size) { 1182 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); 1183 mem_per_q = round_down(mem_per_q, dev->ctrl.page_size); 1184 q_depth = div_u64(mem_per_q, entry_size); 1185 1186 /* 1187 * Ensure the reduced q_depth is above some threshold where it 1188 * would be better to map queues in system memory with the 1189 * original depth 1190 */ 1191 if (q_depth < 64) 1192 return -ENOMEM; 1193 } 1194 1195 return q_depth; 1196 } 1197 1198 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, 1199 int qid, int depth) 1200 { 1201 if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) { 1202 unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth), 1203 dev->ctrl.page_size); 1204 nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset; 1205 nvmeq->sq_cmds_io = dev->cmb + offset; 1206 } else { 1207 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth), 1208 &nvmeq->sq_dma_addr, GFP_KERNEL); 1209 if (!nvmeq->sq_cmds) 1210 return -ENOMEM; 1211 } 1212 1213 return 0; 1214 } 1215 1216 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1217 int depth, int node) 1218 { 1219 struct nvme_queue *nvmeq = kzalloc_node(sizeof(*nvmeq), GFP_KERNEL, 1220 node); 1221 if (!nvmeq) 1222 return NULL; 1223 1224 nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth), 1225 &nvmeq->cq_dma_addr, GFP_KERNEL); 1226 if (!nvmeq->cqes) 1227 goto free_nvmeq; 1228 1229 if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth)) 1230 goto free_cqdma; 1231 1232 nvmeq->q_dmadev = dev->dev; 1233 nvmeq->dev = dev; 1234 spin_lock_init(&nvmeq->q_lock); 1235 nvmeq->cq_head = 0; 1236 nvmeq->cq_phase = 1; 1237 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1238 nvmeq->q_depth = depth; 1239 nvmeq->qid = qid; 1240 nvmeq->cq_vector = -1; 1241 dev->queues[qid] = nvmeq; 1242 dev->ctrl.queue_count++; 1243 1244 return nvmeq; 1245 1246 free_cqdma: 1247 dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1248 nvmeq->cq_dma_addr); 1249 free_nvmeq: 1250 kfree(nvmeq); 1251 return NULL; 1252 } 1253 1254 static int queue_request_irq(struct nvme_queue *nvmeq) 1255 { 1256 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); 1257 int nr = nvmeq->dev->ctrl.instance; 1258 1259 if (use_threaded_interrupts) { 1260 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check, 1261 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid); 1262 } else { 1263 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq, 1264 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid); 1265 } 1266 } 1267 1268 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 1269 { 1270 struct nvme_dev *dev = nvmeq->dev; 1271 1272 spin_lock_irq(&nvmeq->q_lock); 1273 nvmeq->sq_tail = 0; 1274 nvmeq->cq_head = 0; 1275 nvmeq->cq_phase = 1; 1276 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1277 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); 1278 nvme_dbbuf_init(dev, nvmeq, qid); 1279 dev->online_queues++; 1280 spin_unlock_irq(&nvmeq->q_lock); 1281 } 1282 1283 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 1284 { 1285 struct nvme_dev *dev = nvmeq->dev; 1286 int result; 1287 1288 nvmeq->cq_vector = qid - 1; 1289 result = adapter_alloc_cq(dev, qid, nvmeq); 1290 if (result < 0) 1291 return result; 1292 1293 result = adapter_alloc_sq(dev, qid, nvmeq); 1294 if (result < 0) 1295 goto release_cq; 1296 1297 result = queue_request_irq(nvmeq); 1298 if (result < 0) 1299 goto release_sq; 1300 1301 nvme_init_queue(nvmeq, qid); 1302 return result; 1303 1304 release_sq: 1305 adapter_delete_sq(dev, qid); 1306 release_cq: 1307 adapter_delete_cq(dev, qid); 1308 return result; 1309 } 1310 1311 static const struct blk_mq_ops nvme_mq_admin_ops = { 1312 .queue_rq = nvme_queue_rq, 1313 .complete = nvme_pci_complete_rq, 1314 .init_hctx = nvme_admin_init_hctx, 1315 .exit_hctx = nvme_admin_exit_hctx, 1316 .init_request = nvme_init_request, 1317 .timeout = nvme_timeout, 1318 }; 1319 1320 static const struct blk_mq_ops nvme_mq_ops = { 1321 .queue_rq = nvme_queue_rq, 1322 .complete = nvme_pci_complete_rq, 1323 .init_hctx = nvme_init_hctx, 1324 .init_request = nvme_init_request, 1325 .map_queues = nvme_pci_map_queues, 1326 .timeout = nvme_timeout, 1327 .poll = nvme_poll, 1328 }; 1329 1330 static void nvme_dev_remove_admin(struct nvme_dev *dev) 1331 { 1332 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) { 1333 /* 1334 * If the controller was reset during removal, it's possible 1335 * user requests may be waiting on a stopped queue. Start the 1336 * queue to flush these to completion. 1337 */ 1338 blk_mq_unquiesce_queue(dev->ctrl.admin_q); 1339 blk_cleanup_queue(dev->ctrl.admin_q); 1340 blk_mq_free_tag_set(&dev->admin_tagset); 1341 } 1342 } 1343 1344 static int nvme_alloc_admin_tags(struct nvme_dev *dev) 1345 { 1346 if (!dev->ctrl.admin_q) { 1347 dev->admin_tagset.ops = &nvme_mq_admin_ops; 1348 dev->admin_tagset.nr_hw_queues = 1; 1349 1350 /* 1351 * Subtract one to leave an empty queue entry for 'Full Queue' 1352 * condition. See NVM-Express 1.2 specification, section 4.1.2. 1353 */ 1354 dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH - 1; 1355 dev->admin_tagset.timeout = ADMIN_TIMEOUT; 1356 dev->admin_tagset.numa_node = dev_to_node(dev->dev); 1357 dev->admin_tagset.cmd_size = nvme_cmd_size(dev); 1358 dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED; 1359 dev->admin_tagset.driver_data = dev; 1360 1361 if (blk_mq_alloc_tag_set(&dev->admin_tagset)) 1362 return -ENOMEM; 1363 1364 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset); 1365 if (IS_ERR(dev->ctrl.admin_q)) { 1366 blk_mq_free_tag_set(&dev->admin_tagset); 1367 return -ENOMEM; 1368 } 1369 if (!blk_get_queue(dev->ctrl.admin_q)) { 1370 nvme_dev_remove_admin(dev); 1371 dev->ctrl.admin_q = NULL; 1372 return -ENODEV; 1373 } 1374 } else 1375 blk_mq_unquiesce_queue(dev->ctrl.admin_q); 1376 1377 return 0; 1378 } 1379 1380 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 1381 { 1382 return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride); 1383 } 1384 1385 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size) 1386 { 1387 struct pci_dev *pdev = to_pci_dev(dev->dev); 1388 1389 if (size <= dev->bar_mapped_size) 1390 return 0; 1391 if (size > pci_resource_len(pdev, 0)) 1392 return -ENOMEM; 1393 if (dev->bar) 1394 iounmap(dev->bar); 1395 dev->bar = ioremap(pci_resource_start(pdev, 0), size); 1396 if (!dev->bar) { 1397 dev->bar_mapped_size = 0; 1398 return -ENOMEM; 1399 } 1400 dev->bar_mapped_size = size; 1401 dev->dbs = dev->bar + NVME_REG_DBS; 1402 1403 return 0; 1404 } 1405 1406 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) 1407 { 1408 int result; 1409 u32 aqa; 1410 struct nvme_queue *nvmeq; 1411 1412 result = nvme_remap_bar(dev, db_bar_size(dev, 0)); 1413 if (result < 0) 1414 return result; 1415 1416 dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ? 1417 NVME_CAP_NSSRC(dev->ctrl.cap) : 0; 1418 1419 if (dev->subsystem && 1420 (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO)) 1421 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS); 1422 1423 result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap); 1424 if (result < 0) 1425 return result; 1426 1427 nvmeq = dev->queues[0]; 1428 if (!nvmeq) { 1429 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH, 1430 dev_to_node(dev->dev)); 1431 if (!nvmeq) 1432 return -ENOMEM; 1433 } 1434 1435 aqa = nvmeq->q_depth - 1; 1436 aqa |= aqa << 16; 1437 1438 writel(aqa, dev->bar + NVME_REG_AQA); 1439 lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ); 1440 lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ); 1441 1442 result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap); 1443 if (result) 1444 return result; 1445 1446 nvmeq->cq_vector = 0; 1447 result = queue_request_irq(nvmeq); 1448 if (result) { 1449 nvmeq->cq_vector = -1; 1450 return result; 1451 } 1452 1453 return result; 1454 } 1455 1456 static int nvme_create_io_queues(struct nvme_dev *dev) 1457 { 1458 unsigned i, max; 1459 int ret = 0; 1460 1461 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) { 1462 /* vector == qid - 1, match nvme_create_queue */ 1463 if (!nvme_alloc_queue(dev, i, dev->q_depth, 1464 pci_irq_get_node(to_pci_dev(dev->dev), i - 1))) { 1465 ret = -ENOMEM; 1466 break; 1467 } 1468 } 1469 1470 max = min(dev->max_qid, dev->ctrl.queue_count - 1); 1471 for (i = dev->online_queues; i <= max; i++) { 1472 ret = nvme_create_queue(dev->queues[i], i); 1473 if (ret) 1474 break; 1475 } 1476 1477 /* 1478 * Ignore failing Create SQ/CQ commands, we can continue with less 1479 * than the desired aount of queues, and even a controller without 1480 * I/O queues an still be used to issue admin commands. This might 1481 * be useful to upgrade a buggy firmware for example. 1482 */ 1483 return ret >= 0 ? 0 : ret; 1484 } 1485 1486 static ssize_t nvme_cmb_show(struct device *dev, 1487 struct device_attribute *attr, 1488 char *buf) 1489 { 1490 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 1491 1492 return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz : x%08x\n", 1493 ndev->cmbloc, ndev->cmbsz); 1494 } 1495 static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL); 1496 1497 static void __iomem *nvme_map_cmb(struct nvme_dev *dev) 1498 { 1499 u64 szu, size, offset; 1500 resource_size_t bar_size; 1501 struct pci_dev *pdev = to_pci_dev(dev->dev); 1502 void __iomem *cmb; 1503 dma_addr_t dma_addr; 1504 1505 dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); 1506 if (!(NVME_CMB_SZ(dev->cmbsz))) 1507 return NULL; 1508 dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); 1509 1510 if (!use_cmb_sqes) 1511 return NULL; 1512 1513 szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz)); 1514 size = szu * NVME_CMB_SZ(dev->cmbsz); 1515 offset = szu * NVME_CMB_OFST(dev->cmbloc); 1516 bar_size = pci_resource_len(pdev, NVME_CMB_BIR(dev->cmbloc)); 1517 1518 if (offset > bar_size) 1519 return NULL; 1520 1521 /* 1522 * Controllers may support a CMB size larger than their BAR, 1523 * for example, due to being behind a bridge. Reduce the CMB to 1524 * the reported size of the BAR 1525 */ 1526 if (size > bar_size - offset) 1527 size = bar_size - offset; 1528 1529 dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(dev->cmbloc)) + offset; 1530 cmb = ioremap_wc(dma_addr, size); 1531 if (!cmb) 1532 return NULL; 1533 1534 dev->cmb_dma_addr = dma_addr; 1535 dev->cmb_size = size; 1536 return cmb; 1537 } 1538 1539 static inline void nvme_release_cmb(struct nvme_dev *dev) 1540 { 1541 if (dev->cmb) { 1542 iounmap(dev->cmb); 1543 dev->cmb = NULL; 1544 if (dev->cmbsz) { 1545 sysfs_remove_file_from_group(&dev->ctrl.device->kobj, 1546 &dev_attr_cmb.attr, NULL); 1547 dev->cmbsz = 0; 1548 } 1549 } 1550 } 1551 1552 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) 1553 { 1554 size_t len = dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs); 1555 struct nvme_command c; 1556 u64 dma_addr; 1557 int ret; 1558 1559 dma_addr = dma_map_single(dev->dev, dev->host_mem_descs, len, 1560 DMA_TO_DEVICE); 1561 if (dma_mapping_error(dev->dev, dma_addr)) 1562 return -ENOMEM; 1563 1564 memset(&c, 0, sizeof(c)); 1565 c.features.opcode = nvme_admin_set_features; 1566 c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF); 1567 c.features.dword11 = cpu_to_le32(bits); 1568 c.features.dword12 = cpu_to_le32(dev->host_mem_size >> 1569 ilog2(dev->ctrl.page_size)); 1570 c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr)); 1571 c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr)); 1572 c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs); 1573 1574 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 1575 if (ret) { 1576 dev_warn(dev->ctrl.device, 1577 "failed to set host mem (err %d, flags %#x).\n", 1578 ret, bits); 1579 } 1580 dma_unmap_single(dev->dev, dma_addr, len, DMA_TO_DEVICE); 1581 return ret; 1582 } 1583 1584 static void nvme_free_host_mem(struct nvme_dev *dev) 1585 { 1586 int i; 1587 1588 for (i = 0; i < dev->nr_host_mem_descs; i++) { 1589 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i]; 1590 size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size; 1591 1592 dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i], 1593 le64_to_cpu(desc->addr)); 1594 } 1595 1596 kfree(dev->host_mem_desc_bufs); 1597 dev->host_mem_desc_bufs = NULL; 1598 kfree(dev->host_mem_descs); 1599 dev->host_mem_descs = NULL; 1600 } 1601 1602 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) 1603 { 1604 struct nvme_host_mem_buf_desc *descs; 1605 u32 chunk_size, max_entries; 1606 int i = 0; 1607 void **bufs; 1608 u64 size = 0, tmp; 1609 1610 /* start big and work our way down */ 1611 chunk_size = min(preferred, (u64)PAGE_SIZE << MAX_ORDER); 1612 retry: 1613 tmp = (preferred + chunk_size - 1); 1614 do_div(tmp, chunk_size); 1615 max_entries = tmp; 1616 descs = kcalloc(max_entries, sizeof(*descs), GFP_KERNEL); 1617 if (!descs) 1618 goto out; 1619 1620 bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL); 1621 if (!bufs) 1622 goto out_free_descs; 1623 1624 for (size = 0; size < preferred; size += chunk_size) { 1625 u32 len = min_t(u64, chunk_size, preferred - size); 1626 dma_addr_t dma_addr; 1627 1628 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL, 1629 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN); 1630 if (!bufs[i]) 1631 break; 1632 1633 descs[i].addr = cpu_to_le64(dma_addr); 1634 descs[i].size = cpu_to_le32(len / dev->ctrl.page_size); 1635 i++; 1636 } 1637 1638 if (!size || (min && size < min)) { 1639 dev_warn(dev->ctrl.device, 1640 "failed to allocate host memory buffer.\n"); 1641 goto out_free_bufs; 1642 } 1643 1644 dev_info(dev->ctrl.device, 1645 "allocated %lld MiB host memory buffer.\n", 1646 size >> ilog2(SZ_1M)); 1647 dev->nr_host_mem_descs = i; 1648 dev->host_mem_size = size; 1649 dev->host_mem_descs = descs; 1650 dev->host_mem_desc_bufs = bufs; 1651 return 0; 1652 1653 out_free_bufs: 1654 while (--i >= 0) { 1655 size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size; 1656 1657 dma_free_coherent(dev->dev, size, bufs[i], 1658 le64_to_cpu(descs[i].addr)); 1659 } 1660 1661 kfree(bufs); 1662 out_free_descs: 1663 kfree(descs); 1664 out: 1665 /* try a smaller chunk size if we failed early */ 1666 if (chunk_size >= PAGE_SIZE * 2 && (i == 0 || size < min)) { 1667 chunk_size /= 2; 1668 goto retry; 1669 } 1670 dev->host_mem_descs = NULL; 1671 return -ENOMEM; 1672 } 1673 1674 static void nvme_setup_host_mem(struct nvme_dev *dev) 1675 { 1676 u64 max = (u64)max_host_mem_size_mb * SZ_1M; 1677 u64 preferred = (u64)dev->ctrl.hmpre * 4096; 1678 u64 min = (u64)dev->ctrl.hmmin * 4096; 1679 u32 enable_bits = NVME_HOST_MEM_ENABLE; 1680 1681 preferred = min(preferred, max); 1682 if (min > max) { 1683 dev_warn(dev->ctrl.device, 1684 "min host memory (%lld MiB) above limit (%d MiB).\n", 1685 min >> ilog2(SZ_1M), max_host_mem_size_mb); 1686 nvme_free_host_mem(dev); 1687 return; 1688 } 1689 1690 /* 1691 * If we already have a buffer allocated check if we can reuse it. 1692 */ 1693 if (dev->host_mem_descs) { 1694 if (dev->host_mem_size >= min) 1695 enable_bits |= NVME_HOST_MEM_RETURN; 1696 else 1697 nvme_free_host_mem(dev); 1698 } 1699 1700 if (!dev->host_mem_descs) { 1701 if (nvme_alloc_host_mem(dev, min, preferred)) 1702 return; 1703 } 1704 1705 if (nvme_set_host_mem(dev, enable_bits)) 1706 nvme_free_host_mem(dev); 1707 } 1708 1709 static int nvme_setup_io_queues(struct nvme_dev *dev) 1710 { 1711 struct nvme_queue *adminq = dev->queues[0]; 1712 struct pci_dev *pdev = to_pci_dev(dev->dev); 1713 int result, nr_io_queues; 1714 unsigned long size; 1715 1716 nr_io_queues = num_present_cpus(); 1717 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); 1718 if (result < 0) 1719 return result; 1720 1721 if (nr_io_queues == 0) 1722 return 0; 1723 1724 if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) { 1725 result = nvme_cmb_qdepth(dev, nr_io_queues, 1726 sizeof(struct nvme_command)); 1727 if (result > 0) 1728 dev->q_depth = result; 1729 else 1730 nvme_release_cmb(dev); 1731 } 1732 1733 do { 1734 size = db_bar_size(dev, nr_io_queues); 1735 result = nvme_remap_bar(dev, size); 1736 if (!result) 1737 break; 1738 if (!--nr_io_queues) 1739 return -ENOMEM; 1740 } while (1); 1741 adminq->q_db = dev->dbs; 1742 1743 /* Deregister the admin queue's interrupt */ 1744 pci_free_irq(pdev, 0, adminq); 1745 1746 /* 1747 * If we enable msix early due to not intx, disable it again before 1748 * setting up the full range we need. 1749 */ 1750 pci_free_irq_vectors(pdev); 1751 nr_io_queues = pci_alloc_irq_vectors(pdev, 1, nr_io_queues, 1752 PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY); 1753 if (nr_io_queues <= 0) 1754 return -EIO; 1755 dev->max_qid = nr_io_queues; 1756 1757 /* 1758 * Should investigate if there's a performance win from allocating 1759 * more queues than interrupt vectors; it might allow the submission 1760 * path to scale better, even if the receive path is limited by the 1761 * number of interrupts. 1762 */ 1763 1764 result = queue_request_irq(adminq); 1765 if (result) { 1766 adminq->cq_vector = -1; 1767 return result; 1768 } 1769 return nvme_create_io_queues(dev); 1770 } 1771 1772 static void nvme_del_queue_end(struct request *req, blk_status_t error) 1773 { 1774 struct nvme_queue *nvmeq = req->end_io_data; 1775 1776 blk_mq_free_request(req); 1777 complete(&nvmeq->dev->ioq_wait); 1778 } 1779 1780 static void nvme_del_cq_end(struct request *req, blk_status_t error) 1781 { 1782 struct nvme_queue *nvmeq = req->end_io_data; 1783 1784 if (!error) { 1785 unsigned long flags; 1786 1787 /* 1788 * We might be called with the AQ q_lock held 1789 * and the I/O queue q_lock should always 1790 * nest inside the AQ one. 1791 */ 1792 spin_lock_irqsave_nested(&nvmeq->q_lock, flags, 1793 SINGLE_DEPTH_NESTING); 1794 nvme_process_cq(nvmeq); 1795 spin_unlock_irqrestore(&nvmeq->q_lock, flags); 1796 } 1797 1798 nvme_del_queue_end(req, error); 1799 } 1800 1801 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) 1802 { 1803 struct request_queue *q = nvmeq->dev->ctrl.admin_q; 1804 struct request *req; 1805 struct nvme_command cmd; 1806 1807 memset(&cmd, 0, sizeof(cmd)); 1808 cmd.delete_queue.opcode = opcode; 1809 cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid); 1810 1811 req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY); 1812 if (IS_ERR(req)) 1813 return PTR_ERR(req); 1814 1815 req->timeout = ADMIN_TIMEOUT; 1816 req->end_io_data = nvmeq; 1817 1818 blk_execute_rq_nowait(q, NULL, req, false, 1819 opcode == nvme_admin_delete_cq ? 1820 nvme_del_cq_end : nvme_del_queue_end); 1821 return 0; 1822 } 1823 1824 static void nvme_disable_io_queues(struct nvme_dev *dev, int queues) 1825 { 1826 int pass; 1827 unsigned long timeout; 1828 u8 opcode = nvme_admin_delete_sq; 1829 1830 for (pass = 0; pass < 2; pass++) { 1831 int sent = 0, i = queues; 1832 1833 reinit_completion(&dev->ioq_wait); 1834 retry: 1835 timeout = ADMIN_TIMEOUT; 1836 for (; i > 0; i--, sent++) 1837 if (nvme_delete_queue(dev->queues[i], opcode)) 1838 break; 1839 1840 while (sent--) { 1841 timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout); 1842 if (timeout == 0) 1843 return; 1844 if (i) 1845 goto retry; 1846 } 1847 opcode = nvme_admin_delete_cq; 1848 } 1849 } 1850 1851 /* 1852 * Return: error value if an error occurred setting up the queues or calling 1853 * Identify Device. 0 if these succeeded, even if adding some of the 1854 * namespaces failed. At the moment, these failures are silent. TBD which 1855 * failures should be reported. 1856 */ 1857 static int nvme_dev_add(struct nvme_dev *dev) 1858 { 1859 if (!dev->ctrl.tagset) { 1860 dev->tagset.ops = &nvme_mq_ops; 1861 dev->tagset.nr_hw_queues = dev->online_queues - 1; 1862 dev->tagset.timeout = NVME_IO_TIMEOUT; 1863 dev->tagset.numa_node = dev_to_node(dev->dev); 1864 dev->tagset.queue_depth = 1865 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1; 1866 dev->tagset.cmd_size = nvme_cmd_size(dev); 1867 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE; 1868 dev->tagset.driver_data = dev; 1869 1870 if (blk_mq_alloc_tag_set(&dev->tagset)) 1871 return 0; 1872 dev->ctrl.tagset = &dev->tagset; 1873 1874 nvme_dbbuf_set(dev); 1875 } else { 1876 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1); 1877 1878 /* Free previously allocated queues that are no longer usable */ 1879 nvme_free_queues(dev, dev->online_queues); 1880 } 1881 1882 return 0; 1883 } 1884 1885 static int nvme_pci_enable(struct nvme_dev *dev) 1886 { 1887 int result = -ENOMEM; 1888 struct pci_dev *pdev = to_pci_dev(dev->dev); 1889 1890 if (pci_enable_device_mem(pdev)) 1891 return result; 1892 1893 pci_set_master(pdev); 1894 1895 if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) && 1896 dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32))) 1897 goto disable; 1898 1899 if (readl(dev->bar + NVME_REG_CSTS) == -1) { 1900 result = -ENODEV; 1901 goto disable; 1902 } 1903 1904 /* 1905 * Some devices and/or platforms don't advertise or work with INTx 1906 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll 1907 * adjust this later. 1908 */ 1909 result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); 1910 if (result < 0) 1911 return result; 1912 1913 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); 1914 1915 dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1, 1916 io_queue_depth); 1917 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); 1918 dev->dbs = dev->bar + 4096; 1919 1920 /* 1921 * Temporary fix for the Apple controller found in the MacBook8,1 and 1922 * some MacBook7,1 to avoid controller resets and data loss. 1923 */ 1924 if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) { 1925 dev->q_depth = 2; 1926 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, " 1927 "set queue depth=%u to work around controller resets\n", 1928 dev->q_depth); 1929 } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG && 1930 (pdev->device == 0xa821 || pdev->device == 0xa822) && 1931 NVME_CAP_MQES(dev->ctrl.cap) == 0) { 1932 dev->q_depth = 64; 1933 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, " 1934 "set queue depth=%u\n", dev->q_depth); 1935 } 1936 1937 /* 1938 * CMBs can currently only exist on >=1.2 PCIe devices. We only 1939 * populate sysfs if a CMB is implemented. Note that we add the 1940 * CMB attribute to the nvme_ctrl kobj which removes the need to remove 1941 * it on exit. Since nvme_dev_attrs_group has no name we can pass 1942 * NULL as final argument to sysfs_add_file_to_group. 1943 */ 1944 1945 if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2, 0)) { 1946 dev->cmb = nvme_map_cmb(dev); 1947 1948 if (dev->cmbsz) { 1949 if (sysfs_add_file_to_group(&dev->ctrl.device->kobj, 1950 &dev_attr_cmb.attr, NULL)) 1951 dev_warn(dev->ctrl.device, 1952 "failed to add sysfs attribute for CMB\n"); 1953 } 1954 } 1955 1956 pci_enable_pcie_error_reporting(pdev); 1957 pci_save_state(pdev); 1958 return 0; 1959 1960 disable: 1961 pci_disable_device(pdev); 1962 return result; 1963 } 1964 1965 static void nvme_dev_unmap(struct nvme_dev *dev) 1966 { 1967 if (dev->bar) 1968 iounmap(dev->bar); 1969 pci_release_mem_regions(to_pci_dev(dev->dev)); 1970 } 1971 1972 static void nvme_pci_disable(struct nvme_dev *dev) 1973 { 1974 struct pci_dev *pdev = to_pci_dev(dev->dev); 1975 1976 nvme_release_cmb(dev); 1977 pci_free_irq_vectors(pdev); 1978 1979 if (pci_is_enabled(pdev)) { 1980 pci_disable_pcie_error_reporting(pdev); 1981 pci_disable_device(pdev); 1982 } 1983 } 1984 1985 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) 1986 { 1987 int i, queues; 1988 bool dead = true; 1989 struct pci_dev *pdev = to_pci_dev(dev->dev); 1990 1991 mutex_lock(&dev->shutdown_lock); 1992 if (pci_is_enabled(pdev)) { 1993 u32 csts = readl(dev->bar + NVME_REG_CSTS); 1994 1995 if (dev->ctrl.state == NVME_CTRL_LIVE || 1996 dev->ctrl.state == NVME_CTRL_RESETTING) 1997 nvme_start_freeze(&dev->ctrl); 1998 dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) || 1999 pdev->error_state != pci_channel_io_normal); 2000 } 2001 2002 /* 2003 * Give the controller a chance to complete all entered requests if 2004 * doing a safe shutdown. 2005 */ 2006 if (!dead) { 2007 if (shutdown) 2008 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT); 2009 2010 /* 2011 * If the controller is still alive tell it to stop using the 2012 * host memory buffer. In theory the shutdown / reset should 2013 * make sure that it doesn't access the host memoery anymore, 2014 * but I'd rather be safe than sorry.. 2015 */ 2016 if (dev->host_mem_descs) 2017 nvme_set_host_mem(dev, 0); 2018 2019 } 2020 nvme_stop_queues(&dev->ctrl); 2021 2022 queues = dev->online_queues - 1; 2023 for (i = dev->ctrl.queue_count - 1; i > 0; i--) 2024 nvme_suspend_queue(dev->queues[i]); 2025 2026 if (dead) { 2027 /* A device might become IO incapable very soon during 2028 * probe, before the admin queue is configured. Thus, 2029 * queue_count can be 0 here. 2030 */ 2031 if (dev->ctrl.queue_count) 2032 nvme_suspend_queue(dev->queues[0]); 2033 } else { 2034 nvme_disable_io_queues(dev, queues); 2035 nvme_disable_admin_queue(dev, shutdown); 2036 } 2037 nvme_pci_disable(dev); 2038 2039 blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl); 2040 blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl); 2041 2042 /* 2043 * The driver will not be starting up queues again if shutting down so 2044 * must flush all entered requests to their failed completion to avoid 2045 * deadlocking blk-mq hot-cpu notifier. 2046 */ 2047 if (shutdown) 2048 nvme_start_queues(&dev->ctrl); 2049 mutex_unlock(&dev->shutdown_lock); 2050 } 2051 2052 static int nvme_setup_prp_pools(struct nvme_dev *dev) 2053 { 2054 dev->prp_page_pool = dma_pool_create("prp list page", dev->dev, 2055 PAGE_SIZE, PAGE_SIZE, 0); 2056 if (!dev->prp_page_pool) 2057 return -ENOMEM; 2058 2059 /* Optimisation for I/Os between 4k and 128k */ 2060 dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev, 2061 256, 256, 0); 2062 if (!dev->prp_small_pool) { 2063 dma_pool_destroy(dev->prp_page_pool); 2064 return -ENOMEM; 2065 } 2066 return 0; 2067 } 2068 2069 static void nvme_release_prp_pools(struct nvme_dev *dev) 2070 { 2071 dma_pool_destroy(dev->prp_page_pool); 2072 dma_pool_destroy(dev->prp_small_pool); 2073 } 2074 2075 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) 2076 { 2077 struct nvme_dev *dev = to_nvme_dev(ctrl); 2078 2079 nvme_dbbuf_dma_free(dev); 2080 put_device(dev->dev); 2081 if (dev->tagset.tags) 2082 blk_mq_free_tag_set(&dev->tagset); 2083 if (dev->ctrl.admin_q) 2084 blk_put_queue(dev->ctrl.admin_q); 2085 kfree(dev->queues); 2086 free_opal_dev(dev->ctrl.opal_dev); 2087 kfree(dev); 2088 } 2089 2090 static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status) 2091 { 2092 dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status); 2093 2094 kref_get(&dev->ctrl.kref); 2095 nvme_dev_disable(dev, false); 2096 if (!schedule_work(&dev->remove_work)) 2097 nvme_put_ctrl(&dev->ctrl); 2098 } 2099 2100 static void nvme_reset_work(struct work_struct *work) 2101 { 2102 struct nvme_dev *dev = 2103 container_of(work, struct nvme_dev, ctrl.reset_work); 2104 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL); 2105 int result = -ENODEV; 2106 2107 if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING)) 2108 goto out; 2109 2110 /* 2111 * If we're called to reset a live controller first shut it down before 2112 * moving on. 2113 */ 2114 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE) 2115 nvme_dev_disable(dev, false); 2116 2117 result = nvme_pci_enable(dev); 2118 if (result) 2119 goto out; 2120 2121 result = nvme_pci_configure_admin_queue(dev); 2122 if (result) 2123 goto out; 2124 2125 nvme_init_queue(dev->queues[0], 0); 2126 result = nvme_alloc_admin_tags(dev); 2127 if (result) 2128 goto out; 2129 2130 result = nvme_init_identify(&dev->ctrl); 2131 if (result) 2132 goto out; 2133 2134 if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) { 2135 if (!dev->ctrl.opal_dev) 2136 dev->ctrl.opal_dev = 2137 init_opal_dev(&dev->ctrl, &nvme_sec_submit); 2138 else if (was_suspend) 2139 opal_unlock_from_suspend(dev->ctrl.opal_dev); 2140 } else { 2141 free_opal_dev(dev->ctrl.opal_dev); 2142 dev->ctrl.opal_dev = NULL; 2143 } 2144 2145 if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) { 2146 result = nvme_dbbuf_dma_alloc(dev); 2147 if (result) 2148 dev_warn(dev->dev, 2149 "unable to allocate dma for dbbuf\n"); 2150 } 2151 2152 if (dev->ctrl.hmpre) 2153 nvme_setup_host_mem(dev); 2154 2155 result = nvme_setup_io_queues(dev); 2156 if (result) 2157 goto out; 2158 2159 /* 2160 * Keep the controller around but remove all namespaces if we don't have 2161 * any working I/O queue. 2162 */ 2163 if (dev->online_queues < 2) { 2164 dev_warn(dev->ctrl.device, "IO queues not created\n"); 2165 nvme_kill_queues(&dev->ctrl); 2166 nvme_remove_namespaces(&dev->ctrl); 2167 } else { 2168 nvme_start_queues(&dev->ctrl); 2169 nvme_wait_freeze(&dev->ctrl); 2170 nvme_dev_add(dev); 2171 nvme_unfreeze(&dev->ctrl); 2172 } 2173 2174 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { 2175 dev_warn(dev->ctrl.device, "failed to mark controller live\n"); 2176 goto out; 2177 } 2178 2179 nvme_start_ctrl(&dev->ctrl); 2180 return; 2181 2182 out: 2183 nvme_remove_dead_ctrl(dev, result); 2184 } 2185 2186 static void nvme_remove_dead_ctrl_work(struct work_struct *work) 2187 { 2188 struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work); 2189 struct pci_dev *pdev = to_pci_dev(dev->dev); 2190 2191 nvme_kill_queues(&dev->ctrl); 2192 if (pci_get_drvdata(pdev)) 2193 device_release_driver(&pdev->dev); 2194 nvme_put_ctrl(&dev->ctrl); 2195 } 2196 2197 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) 2198 { 2199 *val = readl(to_nvme_dev(ctrl)->bar + off); 2200 return 0; 2201 } 2202 2203 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) 2204 { 2205 writel(val, to_nvme_dev(ctrl)->bar + off); 2206 return 0; 2207 } 2208 2209 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) 2210 { 2211 *val = readq(to_nvme_dev(ctrl)->bar + off); 2212 return 0; 2213 } 2214 2215 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { 2216 .name = "pcie", 2217 .module = THIS_MODULE, 2218 .flags = NVME_F_METADATA_SUPPORTED, 2219 .reg_read32 = nvme_pci_reg_read32, 2220 .reg_write32 = nvme_pci_reg_write32, 2221 .reg_read64 = nvme_pci_reg_read64, 2222 .free_ctrl = nvme_pci_free_ctrl, 2223 .submit_async_event = nvme_pci_submit_async_event, 2224 }; 2225 2226 static int nvme_dev_map(struct nvme_dev *dev) 2227 { 2228 struct pci_dev *pdev = to_pci_dev(dev->dev); 2229 2230 if (pci_request_mem_regions(pdev, "nvme")) 2231 return -ENODEV; 2232 2233 if (nvme_remap_bar(dev, NVME_REG_DBS + 4096)) 2234 goto release; 2235 2236 return 0; 2237 release: 2238 pci_release_mem_regions(pdev); 2239 return -ENODEV; 2240 } 2241 2242 static unsigned long check_dell_samsung_bug(struct pci_dev *pdev) 2243 { 2244 if (pdev->vendor == 0x144d && pdev->device == 0xa802) { 2245 /* 2246 * Several Samsung devices seem to drop off the PCIe bus 2247 * randomly when APST is on and uses the deepest sleep state. 2248 * This has been observed on a Samsung "SM951 NVMe SAMSUNG 2249 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD 2250 * 950 PRO 256GB", but it seems to be restricted to two Dell 2251 * laptops. 2252 */ 2253 if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") && 2254 (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") || 2255 dmi_match(DMI_PRODUCT_NAME, "Precision 5510"))) 2256 return NVME_QUIRK_NO_DEEPEST_PS; 2257 } 2258 2259 return 0; 2260 } 2261 2262 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2263 { 2264 int node, result = -ENOMEM; 2265 struct nvme_dev *dev; 2266 unsigned long quirks = id->driver_data; 2267 2268 node = dev_to_node(&pdev->dev); 2269 if (node == NUMA_NO_NODE) 2270 set_dev_node(&pdev->dev, first_memory_node); 2271 2272 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node); 2273 if (!dev) 2274 return -ENOMEM; 2275 dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *), 2276 GFP_KERNEL, node); 2277 if (!dev->queues) 2278 goto free; 2279 2280 dev->dev = get_device(&pdev->dev); 2281 pci_set_drvdata(pdev, dev); 2282 2283 result = nvme_dev_map(dev); 2284 if (result) 2285 goto free; 2286 2287 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); 2288 INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work); 2289 mutex_init(&dev->shutdown_lock); 2290 init_completion(&dev->ioq_wait); 2291 2292 result = nvme_setup_prp_pools(dev); 2293 if (result) 2294 goto put_pci; 2295 2296 quirks |= check_dell_samsung_bug(pdev); 2297 2298 result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops, 2299 quirks); 2300 if (result) 2301 goto release_pools; 2302 2303 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING); 2304 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev)); 2305 2306 queue_work(nvme_wq, &dev->ctrl.reset_work); 2307 return 0; 2308 2309 release_pools: 2310 nvme_release_prp_pools(dev); 2311 put_pci: 2312 put_device(dev->dev); 2313 nvme_dev_unmap(dev); 2314 free: 2315 kfree(dev->queues); 2316 kfree(dev); 2317 return result; 2318 } 2319 2320 static void nvme_reset_prepare(struct pci_dev *pdev) 2321 { 2322 struct nvme_dev *dev = pci_get_drvdata(pdev); 2323 nvme_dev_disable(dev, false); 2324 } 2325 2326 static void nvme_reset_done(struct pci_dev *pdev) 2327 { 2328 struct nvme_dev *dev = pci_get_drvdata(pdev); 2329 nvme_reset_ctrl(&dev->ctrl); 2330 } 2331 2332 static void nvme_shutdown(struct pci_dev *pdev) 2333 { 2334 struct nvme_dev *dev = pci_get_drvdata(pdev); 2335 nvme_dev_disable(dev, true); 2336 } 2337 2338 /* 2339 * The driver's remove may be called on a device in a partially initialized 2340 * state. This function must not have any dependencies on the device state in 2341 * order to proceed. 2342 */ 2343 static void nvme_remove(struct pci_dev *pdev) 2344 { 2345 struct nvme_dev *dev = pci_get_drvdata(pdev); 2346 2347 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 2348 2349 cancel_work_sync(&dev->ctrl.reset_work); 2350 pci_set_drvdata(pdev, NULL); 2351 2352 if (!pci_device_is_present(pdev)) { 2353 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); 2354 nvme_dev_disable(dev, false); 2355 } 2356 2357 flush_work(&dev->ctrl.reset_work); 2358 nvme_stop_ctrl(&dev->ctrl); 2359 nvme_remove_namespaces(&dev->ctrl); 2360 nvme_dev_disable(dev, true); 2361 nvme_free_host_mem(dev); 2362 nvme_dev_remove_admin(dev); 2363 nvme_free_queues(dev, 0); 2364 nvme_uninit_ctrl(&dev->ctrl); 2365 nvme_release_prp_pools(dev); 2366 nvme_dev_unmap(dev); 2367 nvme_put_ctrl(&dev->ctrl); 2368 } 2369 2370 static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs) 2371 { 2372 int ret = 0; 2373 2374 if (numvfs == 0) { 2375 if (pci_vfs_assigned(pdev)) { 2376 dev_warn(&pdev->dev, 2377 "Cannot disable SR-IOV VFs while assigned\n"); 2378 return -EPERM; 2379 } 2380 pci_disable_sriov(pdev); 2381 return 0; 2382 } 2383 2384 ret = pci_enable_sriov(pdev, numvfs); 2385 return ret ? ret : numvfs; 2386 } 2387 2388 #ifdef CONFIG_PM_SLEEP 2389 static int nvme_suspend(struct device *dev) 2390 { 2391 struct pci_dev *pdev = to_pci_dev(dev); 2392 struct nvme_dev *ndev = pci_get_drvdata(pdev); 2393 2394 nvme_dev_disable(ndev, true); 2395 return 0; 2396 } 2397 2398 static int nvme_resume(struct device *dev) 2399 { 2400 struct pci_dev *pdev = to_pci_dev(dev); 2401 struct nvme_dev *ndev = pci_get_drvdata(pdev); 2402 2403 nvme_reset_ctrl(&ndev->ctrl); 2404 return 0; 2405 } 2406 #endif 2407 2408 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); 2409 2410 static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev, 2411 pci_channel_state_t state) 2412 { 2413 struct nvme_dev *dev = pci_get_drvdata(pdev); 2414 2415 /* 2416 * A frozen channel requires a reset. When detected, this method will 2417 * shutdown the controller to quiesce. The controller will be restarted 2418 * after the slot reset through driver's slot_reset callback. 2419 */ 2420 switch (state) { 2421 case pci_channel_io_normal: 2422 return PCI_ERS_RESULT_CAN_RECOVER; 2423 case pci_channel_io_frozen: 2424 dev_warn(dev->ctrl.device, 2425 "frozen state error detected, reset controller\n"); 2426 nvme_dev_disable(dev, false); 2427 return PCI_ERS_RESULT_NEED_RESET; 2428 case pci_channel_io_perm_failure: 2429 dev_warn(dev->ctrl.device, 2430 "failure state error detected, request disconnect\n"); 2431 return PCI_ERS_RESULT_DISCONNECT; 2432 } 2433 return PCI_ERS_RESULT_NEED_RESET; 2434 } 2435 2436 static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev) 2437 { 2438 struct nvme_dev *dev = pci_get_drvdata(pdev); 2439 2440 dev_info(dev->ctrl.device, "restart after slot reset\n"); 2441 pci_restore_state(pdev); 2442 nvme_reset_ctrl(&dev->ctrl); 2443 return PCI_ERS_RESULT_RECOVERED; 2444 } 2445 2446 static void nvme_error_resume(struct pci_dev *pdev) 2447 { 2448 pci_cleanup_aer_uncorrect_error_status(pdev); 2449 } 2450 2451 static const struct pci_error_handlers nvme_err_handler = { 2452 .error_detected = nvme_error_detected, 2453 .slot_reset = nvme_slot_reset, 2454 .resume = nvme_error_resume, 2455 .reset_prepare = nvme_reset_prepare, 2456 .reset_done = nvme_reset_done, 2457 }; 2458 2459 static const struct pci_device_id nvme_id_table[] = { 2460 { PCI_VDEVICE(INTEL, 0x0953), 2461 .driver_data = NVME_QUIRK_STRIPE_SIZE | 2462 NVME_QUIRK_DEALLOCATE_ZEROES, }, 2463 { PCI_VDEVICE(INTEL, 0x0a53), 2464 .driver_data = NVME_QUIRK_STRIPE_SIZE | 2465 NVME_QUIRK_DEALLOCATE_ZEROES, }, 2466 { PCI_VDEVICE(INTEL, 0x0a54), 2467 .driver_data = NVME_QUIRK_STRIPE_SIZE | 2468 NVME_QUIRK_DEALLOCATE_ZEROES, }, 2469 { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */ 2470 .driver_data = NVME_QUIRK_NO_DEEPEST_PS }, 2471 { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */ 2472 .driver_data = NVME_QUIRK_IDENTIFY_CNS, }, 2473 { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */ 2474 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 2475 { PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */ 2476 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 2477 { PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */ 2478 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 2479 { PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */ 2480 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 2481 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 2482 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) }, 2483 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, 2484 { 0, } 2485 }; 2486 MODULE_DEVICE_TABLE(pci, nvme_id_table); 2487 2488 static struct pci_driver nvme_driver = { 2489 .name = "nvme", 2490 .id_table = nvme_id_table, 2491 .probe = nvme_probe, 2492 .remove = nvme_remove, 2493 .shutdown = nvme_shutdown, 2494 .driver = { 2495 .pm = &nvme_dev_pm_ops, 2496 }, 2497 .sriov_configure = nvme_pci_sriov_configure, 2498 .err_handler = &nvme_err_handler, 2499 }; 2500 2501 static int __init nvme_init(void) 2502 { 2503 return pci_register_driver(&nvme_driver); 2504 } 2505 2506 static void __exit nvme_exit(void) 2507 { 2508 pci_unregister_driver(&nvme_driver); 2509 _nvme_check_size(); 2510 } 2511 2512 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2513 MODULE_LICENSE("GPL"); 2514 MODULE_VERSION("1.0"); 2515 module_init(nvme_init); 2516 module_exit(nvme_exit); 2517