157dacad5SJay Sternberg /* 257dacad5SJay Sternberg * Copyright (c) 2011-2014, Intel Corporation. 357dacad5SJay Sternberg * 457dacad5SJay Sternberg * This program is free software; you can redistribute it and/or modify it 557dacad5SJay Sternberg * under the terms and conditions of the GNU General Public License, 657dacad5SJay Sternberg * version 2, as published by the Free Software Foundation. 757dacad5SJay Sternberg * 857dacad5SJay Sternberg * This program is distributed in the hope it will be useful, but WITHOUT 957dacad5SJay Sternberg * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1057dacad5SJay Sternberg * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1157dacad5SJay Sternberg * more details. 1257dacad5SJay Sternberg */ 1357dacad5SJay Sternberg 1457dacad5SJay Sternberg #ifndef _NVME_H 1557dacad5SJay Sternberg #define _NVME_H 1657dacad5SJay Sternberg 1757dacad5SJay Sternberg #include <linux/nvme.h> 1857dacad5SJay Sternberg #include <linux/pci.h> 1957dacad5SJay Sternberg #include <linux/kref.h> 2057dacad5SJay Sternberg #include <linux/blk-mq.h> 21b0b4e09cSMatias Bjørling #include <linux/lightnvm.h> 2257dacad5SJay Sternberg 23297465c8SChristoph Hellwig enum { 24297465c8SChristoph Hellwig /* 25297465c8SChristoph Hellwig * Driver internal status code for commands that were cancelled due 26297465c8SChristoph Hellwig * to timeouts or controller shutdown. The value is negative so 27297465c8SChristoph Hellwig * that it a) doesn't overlap with the unsigned hardware error codes, 28297465c8SChristoph Hellwig * and b) can easily be tested for. 29297465c8SChristoph Hellwig */ 30297465c8SChristoph Hellwig NVME_SC_CANCELLED = -EINTR, 31297465c8SChristoph Hellwig }; 32297465c8SChristoph Hellwig 3357dacad5SJay Sternberg extern unsigned char nvme_io_timeout; 3457dacad5SJay Sternberg #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) 3557dacad5SJay Sternberg 3621d34711SChristoph Hellwig extern unsigned char admin_timeout; 3721d34711SChristoph Hellwig #define ADMIN_TIMEOUT (admin_timeout * HZ) 3821d34711SChristoph Hellwig 395fd4ce1bSChristoph Hellwig extern unsigned char shutdown_timeout; 405fd4ce1bSChristoph Hellwig #define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ) 415fd4ce1bSChristoph Hellwig 42038bd4cbSSagi Grimberg #define NVME_DEFAULT_KATO 5 43038bd4cbSSagi Grimberg #define NVME_KATO_GRACE 10 44038bd4cbSSagi Grimberg 45f80ec966SKeith Busch extern unsigned int nvme_max_retries; 46f80ec966SKeith Busch 47ca064085SMatias Bjørling enum { 48ca064085SMatias Bjørling NVME_NS_LBA = 0, 49ca064085SMatias Bjørling NVME_NS_LIGHTNVM = 1, 50ca064085SMatias Bjørling }; 51ca064085SMatias Bjørling 5257dacad5SJay Sternberg /* 53106198edSChristoph Hellwig * List of workarounds for devices that required behavior not specified in 54106198edSChristoph Hellwig * the standard. 5557dacad5SJay Sternberg */ 56106198edSChristoph Hellwig enum nvme_quirks { 57106198edSChristoph Hellwig /* 58106198edSChristoph Hellwig * Prefers I/O aligned to a stripe size specified in a vendor 59106198edSChristoph Hellwig * specific Identify field. 60106198edSChristoph Hellwig */ 61106198edSChristoph Hellwig NVME_QUIRK_STRIPE_SIZE = (1 << 0), 62540c801cSKeith Busch 63540c801cSKeith Busch /* 64540c801cSKeith Busch * The controller doesn't handle Identify value others than 0 or 1 65540c801cSKeith Busch * correctly. 66540c801cSKeith Busch */ 67540c801cSKeith Busch NVME_QUIRK_IDENTIFY_CNS = (1 << 1), 6808095e70SKeith Busch 6908095e70SKeith Busch /* 7008095e70SKeith Busch * The controller deterministically returns O's on reads to discarded 7108095e70SKeith Busch * logical blocks. 7208095e70SKeith Busch */ 7308095e70SKeith Busch NVME_QUIRK_DISCARD_ZEROES = (1 << 2), 7454adc010SGuilherme G. Piccoli 7554adc010SGuilherme G. Piccoli /* 7654adc010SGuilherme G. Piccoli * The controller needs a delay before starts checking the device 7754adc010SGuilherme G. Piccoli * readiness, which is done by reading the NVME_CSTS_RDY bit. 7854adc010SGuilherme G. Piccoli */ 7954adc010SGuilherme G. Piccoli NVME_QUIRK_DELAY_BEFORE_CHK_RDY = (1 << 3), 80106198edSChristoph Hellwig }; 81106198edSChristoph Hellwig 8254adc010SGuilherme G. Piccoli /* The below value is the specific amount of delay needed before checking 8354adc010SGuilherme G. Piccoli * readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the 8454adc010SGuilherme G. Piccoli * NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was 8554adc010SGuilherme G. Piccoli * found empirically. 8654adc010SGuilherme G. Piccoli */ 8754adc010SGuilherme G. Piccoli #define NVME_QUIRK_DELAY_AMOUNT 2000 8854adc010SGuilherme G. Piccoli 89bb8d261eSChristoph Hellwig enum nvme_ctrl_state { 90bb8d261eSChristoph Hellwig NVME_CTRL_NEW, 91bb8d261eSChristoph Hellwig NVME_CTRL_LIVE, 92bb8d261eSChristoph Hellwig NVME_CTRL_RESETTING, 93def61ecaSChristoph Hellwig NVME_CTRL_RECONNECTING, 94bb8d261eSChristoph Hellwig NVME_CTRL_DELETING, 950ff9d4e1SKeith Busch NVME_CTRL_DEAD, 96bb8d261eSChristoph Hellwig }; 97bb8d261eSChristoph Hellwig 981c63dc66SChristoph Hellwig struct nvme_ctrl { 99bb8d261eSChristoph Hellwig enum nvme_ctrl_state state; 100bb8d261eSChristoph Hellwig spinlock_t lock; 1011c63dc66SChristoph Hellwig const struct nvme_ctrl_ops *ops; 10257dacad5SJay Sternberg struct request_queue *admin_q; 10307bfcd09SChristoph Hellwig struct request_queue *connect_q; 10457dacad5SJay Sternberg struct device *dev; 10557dacad5SJay Sternberg struct kref kref; 10657dacad5SJay Sternberg int instance; 1075bae7f73SChristoph Hellwig struct blk_mq_tag_set *tagset; 1085bae7f73SChristoph Hellwig struct list_head namespaces; 10969d3b8acSChristoph Hellwig struct mutex namespaces_mutex; 1105bae7f73SChristoph Hellwig struct device *device; /* char device */ 111f3ca80fcSChristoph Hellwig struct list_head node; 112075790ebSKeith Busch struct ida ns_ida; 1131c63dc66SChristoph Hellwig 11457dacad5SJay Sternberg char name[12]; 11557dacad5SJay Sternberg char serial[20]; 11657dacad5SJay Sternberg char model[40]; 11757dacad5SJay Sternberg char firmware_rev[8]; 11876e3914aSChristoph Hellwig u16 cntlid; 1195fd4ce1bSChristoph Hellwig 1205fd4ce1bSChristoph Hellwig u32 ctrl_config; 1215fd4ce1bSChristoph Hellwig 1225fd4ce1bSChristoph Hellwig u32 page_size; 12357dacad5SJay Sternberg u32 max_hw_sectors; 12457dacad5SJay Sternberg u32 stripe_size; 12557dacad5SJay Sternberg u16 oncs; 126118472abSKeith Busch u16 vid; 1276bf25d16SChristoph Hellwig atomic_t abort_limit; 12857dacad5SJay Sternberg u8 event_limit; 12957dacad5SJay Sternberg u8 vwc; 130f3ca80fcSChristoph Hellwig u32 vs; 13107bfcd09SChristoph Hellwig u32 sgls; 132038bd4cbSSagi Grimberg u16 kas; 133038bd4cbSSagi Grimberg unsigned int kato; 134f3ca80fcSChristoph Hellwig bool subsystem; 135106198edSChristoph Hellwig unsigned long quirks; 1365955be21SChristoph Hellwig struct work_struct scan_work; 137f866fc42SChristoph Hellwig struct work_struct async_event_work; 138038bd4cbSSagi Grimberg struct delayed_work ka_work; 13907bfcd09SChristoph Hellwig 14007bfcd09SChristoph Hellwig /* Fabrics only */ 14107bfcd09SChristoph Hellwig u16 sqsize; 14207bfcd09SChristoph Hellwig u32 ioccsz; 14307bfcd09SChristoph Hellwig u32 iorcsz; 14407bfcd09SChristoph Hellwig u16 icdoff; 14507bfcd09SChristoph Hellwig u16 maxcmd; 14607bfcd09SChristoph Hellwig struct nvmf_ctrl_options *opts; 14757dacad5SJay Sternberg }; 14857dacad5SJay Sternberg 14957dacad5SJay Sternberg /* 15057dacad5SJay Sternberg * An NVM Express namespace is equivalent to a SCSI LUN 15157dacad5SJay Sternberg */ 15257dacad5SJay Sternberg struct nvme_ns { 15357dacad5SJay Sternberg struct list_head list; 15457dacad5SJay Sternberg 1551c63dc66SChristoph Hellwig struct nvme_ctrl *ctrl; 15657dacad5SJay Sternberg struct request_queue *queue; 15757dacad5SJay Sternberg struct gendisk *disk; 158b0b4e09cSMatias Bjørling struct nvm_dev *ndev; 15957dacad5SJay Sternberg struct kref kref; 160075790ebSKeith Busch int instance; 16157dacad5SJay Sternberg 1622b9b6e86SKeith Busch u8 eui[8]; 1632b9b6e86SKeith Busch u8 uuid[16]; 1642b9b6e86SKeith Busch 16557dacad5SJay Sternberg unsigned ns_id; 16657dacad5SJay Sternberg int lba_shift; 16757dacad5SJay Sternberg u16 ms; 16857dacad5SJay Sternberg bool ext; 16957dacad5SJay Sternberg u8 pi_type; 170646017a6SKeith Busch unsigned long flags; 171646017a6SKeith Busch 172646017a6SKeith Busch #define NVME_NS_REMOVING 0 17369d9a99cSKeith Busch #define NVME_NS_DEAD 1 174646017a6SKeith Busch 17557dacad5SJay Sternberg u64 mode_select_num_blocks; 17657dacad5SJay Sternberg u32 mode_select_block_len; 17757dacad5SJay Sternberg }; 17857dacad5SJay Sternberg 1791c63dc66SChristoph Hellwig struct nvme_ctrl_ops { 1801a353d85SMing Lin const char *name; 181e439bb12SSagi Grimberg struct module *module; 18207bfcd09SChristoph Hellwig bool is_fabrics; 1831c63dc66SChristoph Hellwig int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); 1845fd4ce1bSChristoph Hellwig int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); 1857fd8930fSChristoph Hellwig int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); 186f3ca80fcSChristoph Hellwig int (*reset_ctrl)(struct nvme_ctrl *ctrl); 1871673f1f0SChristoph Hellwig void (*free_ctrl)(struct nvme_ctrl *ctrl); 1885955be21SChristoph Hellwig void (*post_scan)(struct nvme_ctrl *ctrl); 189f866fc42SChristoph Hellwig void (*submit_async_event)(struct nvme_ctrl *ctrl, int aer_idx); 1901a353d85SMing Lin int (*delete_ctrl)(struct nvme_ctrl *ctrl); 1911a353d85SMing Lin const char *(*get_subsysnqn)(struct nvme_ctrl *ctrl); 1921a353d85SMing Lin int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); 19357dacad5SJay Sternberg }; 19457dacad5SJay Sternberg 1951c63dc66SChristoph Hellwig static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl) 1961c63dc66SChristoph Hellwig { 1971c63dc66SChristoph Hellwig u32 val = 0; 1981c63dc66SChristoph Hellwig 1991c63dc66SChristoph Hellwig if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val)) 2001c63dc66SChristoph Hellwig return false; 2011c63dc66SChristoph Hellwig return val & NVME_CSTS_RDY; 2021c63dc66SChristoph Hellwig } 2031c63dc66SChristoph Hellwig 204f3ca80fcSChristoph Hellwig static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) 205f3ca80fcSChristoph Hellwig { 206f3ca80fcSChristoph Hellwig if (!ctrl->subsystem) 207f3ca80fcSChristoph Hellwig return -ENOTTY; 208f3ca80fcSChristoph Hellwig return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); 209f3ca80fcSChristoph Hellwig } 210f3ca80fcSChristoph Hellwig 21157dacad5SJay Sternberg static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector) 21257dacad5SJay Sternberg { 21357dacad5SJay Sternberg return (sector >> (ns->lba_shift - 9)); 21457dacad5SJay Sternberg } 21557dacad5SJay Sternberg 21658b45602SMing Lin static inline unsigned nvme_map_len(struct request *rq) 21758b45602SMing Lin { 218c2df40dfSMike Christie if (req_op(rq) == REQ_OP_DISCARD) 21958b45602SMing Lin return sizeof(struct nvme_dsm_range); 22058b45602SMing Lin else 22158b45602SMing Lin return blk_rq_bytes(rq); 22258b45602SMing Lin } 22358b45602SMing Lin 2246904242dSMing Lin static inline void nvme_cleanup_cmd(struct request *req) 2256904242dSMing Lin { 226c2df40dfSMike Christie if (req_op(req) == REQ_OP_DISCARD) 2276904242dSMing Lin kfree(req->completion_data); 2286904242dSMing Lin } 2296904242dSMing Lin 23015a190f7SChristoph Hellwig static inline int nvme_error_status(u16 status) 23115a190f7SChristoph Hellwig { 23215a190f7SChristoph Hellwig switch (status & 0x7ff) { 23315a190f7SChristoph Hellwig case NVME_SC_SUCCESS: 23415a190f7SChristoph Hellwig return 0; 23515a190f7SChristoph Hellwig case NVME_SC_CAP_EXCEEDED: 23615a190f7SChristoph Hellwig return -ENOSPC; 23715a190f7SChristoph Hellwig default: 23815a190f7SChristoph Hellwig return -EIO; 23915a190f7SChristoph Hellwig } 24015a190f7SChristoph Hellwig } 24115a190f7SChristoph Hellwig 2427688faa6SChristoph Hellwig static inline bool nvme_req_needs_retry(struct request *req, u16 status) 2437688faa6SChristoph Hellwig { 2447688faa6SChristoph Hellwig return !(status & NVME_SC_DNR || blk_noretry_request(req)) && 245f80ec966SKeith Busch (jiffies - req->start_time) < req->timeout && 246f80ec966SKeith Busch req->retries < nvme_max_retries; 2477688faa6SChristoph Hellwig } 2487688faa6SChristoph Hellwig 249c55a2fd4SMing Lin void nvme_cancel_request(struct request *req, void *data, bool reserved); 250bb8d261eSChristoph Hellwig bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, 251bb8d261eSChristoph Hellwig enum nvme_ctrl_state new_state); 2525fd4ce1bSChristoph Hellwig int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2535fd4ce1bSChristoph Hellwig int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2545fd4ce1bSChristoph Hellwig int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); 255f3ca80fcSChristoph Hellwig int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 256f3ca80fcSChristoph Hellwig const struct nvme_ctrl_ops *ops, unsigned long quirks); 25753029b04SKeith Busch void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); 2581673f1f0SChristoph Hellwig void nvme_put_ctrl(struct nvme_ctrl *ctrl); 2597fd8930fSChristoph Hellwig int nvme_init_identify(struct nvme_ctrl *ctrl); 2605bae7f73SChristoph Hellwig 2615955be21SChristoph Hellwig void nvme_queue_scan(struct nvme_ctrl *ctrl); 2625bae7f73SChristoph Hellwig void nvme_remove_namespaces(struct nvme_ctrl *ctrl); 2631673f1f0SChristoph Hellwig 264f866fc42SChristoph Hellwig #define NVME_NR_AERS 1 265f866fc42SChristoph Hellwig void nvme_complete_async_event(struct nvme_ctrl *ctrl, 266f866fc42SChristoph Hellwig struct nvme_completion *cqe); 267f866fc42SChristoph Hellwig void nvme_queue_async_events(struct nvme_ctrl *ctrl); 268f866fc42SChristoph Hellwig 26925646264SKeith Busch void nvme_stop_queues(struct nvme_ctrl *ctrl); 27025646264SKeith Busch void nvme_start_queues(struct nvme_ctrl *ctrl); 27169d9a99cSKeith Busch void nvme_kill_queues(struct nvme_ctrl *ctrl); 272363c9aacSSagi Grimberg 273eb71f435SChristoph Hellwig #define NVME_QID_ANY -1 2744160982eSChristoph Hellwig struct request *nvme_alloc_request(struct request_queue *q, 275eb71f435SChristoph Hellwig struct nvme_command *cmd, unsigned int flags, int qid); 2767688faa6SChristoph Hellwig void nvme_requeue_req(struct request *req); 2778093f7caSMing Lin int nvme_setup_cmd(struct nvme_ns *ns, struct request *req, 2788093f7caSMing Lin struct nvme_command *cmd); 27957dacad5SJay Sternberg int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 28057dacad5SJay Sternberg void *buf, unsigned bufflen); 28157dacad5SJay Sternberg int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 2821cb3cce5SChristoph Hellwig struct nvme_completion *cqe, void *buffer, unsigned bufflen, 283eb71f435SChristoph Hellwig unsigned timeout, int qid, int at_head, int flags); 2844160982eSChristoph Hellwig int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2854160982eSChristoph Hellwig void __user *ubuffer, unsigned bufflen, u32 *result, 2864160982eSChristoph Hellwig unsigned timeout); 2870b7f1f26SKeith Busch int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2880b7f1f26SKeith Busch void __user *ubuffer, unsigned bufflen, 2890b7f1f26SKeith Busch void __user *meta_buffer, unsigned meta_len, u32 meta_seed, 29057dacad5SJay Sternberg u32 *result, unsigned timeout); 2911c63dc66SChristoph Hellwig int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id); 2921c63dc66SChristoph Hellwig int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, 29357dacad5SJay Sternberg struct nvme_id_ns **id); 2941c63dc66SChristoph Hellwig int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log); 2951c63dc66SChristoph Hellwig int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, 29657dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2971c63dc66SChristoph Hellwig int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, 29857dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2999a0be7abSChristoph Hellwig int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); 300038bd4cbSSagi Grimberg void nvme_start_keep_alive(struct nvme_ctrl *ctrl); 301038bd4cbSSagi Grimberg void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); 30257dacad5SJay Sternberg 30357dacad5SJay Sternberg struct sg_io_hdr; 30457dacad5SJay Sternberg 30557dacad5SJay Sternberg int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr); 30657dacad5SJay Sternberg int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); 30757dacad5SJay Sternberg int nvme_sg_get_version_num(int __user *ip); 30857dacad5SJay Sternberg 309c4699e70SKeith Busch #ifdef CONFIG_NVM 310ca064085SMatias Bjørling int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id); 311*40267efdSSimon A. F. Lund int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node, 312*40267efdSSimon A. F. Lund const struct attribute_group *attrs); 313b0b4e09cSMatias Bjørling void nvme_nvm_unregister(struct nvme_ns *ns); 314*40267efdSSimon A. F. Lund 315*40267efdSSimon A. F. Lund static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev) 316*40267efdSSimon A. F. Lund { 317*40267efdSSimon A. F. Lund if (dev->type->devnode) 318*40267efdSSimon A. F. Lund return dev_to_disk(dev)->private_data; 319*40267efdSSimon A. F. Lund 320*40267efdSSimon A. F. Lund return (container_of(dev, struct nvm_dev, dev))->private_data; 321*40267efdSSimon A. F. Lund } 322c4699e70SKeith Busch #else 323b0b4e09cSMatias Bjørling static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, 324*40267efdSSimon A. F. Lund int node, 325*40267efdSSimon A. F. Lund const struct attribute_group *attrs) 326c4699e70SKeith Busch { 327c4699e70SKeith Busch return 0; 328c4699e70SKeith Busch } 329c4699e70SKeith Busch 330b0b4e09cSMatias Bjørling static inline void nvme_nvm_unregister(struct nvme_ns *ns) {}; 331c4699e70SKeith Busch 332c4699e70SKeith Busch static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id) 333c4699e70SKeith Busch { 334c4699e70SKeith Busch return 0; 335c4699e70SKeith Busch } 336*40267efdSSimon A. F. Lund static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev) 337*40267efdSSimon A. F. Lund { 338*40267efdSSimon A. F. Lund return dev_to_disk(dev)->private_data; 339*40267efdSSimon A. F. Lund } 340c4699e70SKeith Busch #endif /* CONFIG_NVM */ 341ca064085SMatias Bjørling 3425bae7f73SChristoph Hellwig int __init nvme_core_init(void); 3435bae7f73SChristoph Hellwig void nvme_core_exit(void); 3445bae7f73SChristoph Hellwig 34557dacad5SJay Sternberg #endif /* _NVME_H */ 346