157dacad5SJay Sternberg /* 257dacad5SJay Sternberg * Copyright (c) 2011-2014, Intel Corporation. 357dacad5SJay Sternberg * 457dacad5SJay Sternberg * This program is free software; you can redistribute it and/or modify it 557dacad5SJay Sternberg * under the terms and conditions of the GNU General Public License, 657dacad5SJay Sternberg * version 2, as published by the Free Software Foundation. 757dacad5SJay Sternberg * 857dacad5SJay Sternberg * This program is distributed in the hope it will be useful, but WITHOUT 957dacad5SJay Sternberg * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1057dacad5SJay Sternberg * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1157dacad5SJay Sternberg * more details. 1257dacad5SJay Sternberg */ 1357dacad5SJay Sternberg 1457dacad5SJay Sternberg #ifndef _NVME_H 1557dacad5SJay Sternberg #define _NVME_H 1657dacad5SJay Sternberg 1757dacad5SJay Sternberg #include <linux/nvme.h> 1857dacad5SJay Sternberg #include <linux/pci.h> 1957dacad5SJay Sternberg #include <linux/kref.h> 2057dacad5SJay Sternberg #include <linux/blk-mq.h> 2157dacad5SJay Sternberg 22297465c8SChristoph Hellwig enum { 23297465c8SChristoph Hellwig /* 24297465c8SChristoph Hellwig * Driver internal status code for commands that were cancelled due 25297465c8SChristoph Hellwig * to timeouts or controller shutdown. The value is negative so 26297465c8SChristoph Hellwig * that it a) doesn't overlap with the unsigned hardware error codes, 27297465c8SChristoph Hellwig * and b) can easily be tested for. 28297465c8SChristoph Hellwig */ 29297465c8SChristoph Hellwig NVME_SC_CANCELLED = -EINTR, 30297465c8SChristoph Hellwig }; 31297465c8SChristoph Hellwig 3257dacad5SJay Sternberg extern unsigned char nvme_io_timeout; 3357dacad5SJay Sternberg #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) 3457dacad5SJay Sternberg 3521d34711SChristoph Hellwig extern unsigned char admin_timeout; 3621d34711SChristoph Hellwig #define ADMIN_TIMEOUT (admin_timeout * HZ) 3721d34711SChristoph Hellwig 385fd4ce1bSChristoph Hellwig extern unsigned char shutdown_timeout; 395fd4ce1bSChristoph Hellwig #define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ) 405fd4ce1bSChristoph Hellwig 41ca064085SMatias Bjørling enum { 42ca064085SMatias Bjørling NVME_NS_LBA = 0, 43ca064085SMatias Bjørling NVME_NS_LIGHTNVM = 1, 44ca064085SMatias Bjørling }; 45ca064085SMatias Bjørling 4657dacad5SJay Sternberg /* 47106198edSChristoph Hellwig * List of workarounds for devices that required behavior not specified in 48106198edSChristoph Hellwig * the standard. 4957dacad5SJay Sternberg */ 50106198edSChristoph Hellwig enum nvme_quirks { 51106198edSChristoph Hellwig /* 52106198edSChristoph Hellwig * Prefers I/O aligned to a stripe size specified in a vendor 53106198edSChristoph Hellwig * specific Identify field. 54106198edSChristoph Hellwig */ 55106198edSChristoph Hellwig NVME_QUIRK_STRIPE_SIZE = (1 << 0), 56540c801cSKeith Busch 57540c801cSKeith Busch /* 58540c801cSKeith Busch * The controller doesn't handle Identify value others than 0 or 1 59540c801cSKeith Busch * correctly. 60540c801cSKeith Busch */ 61540c801cSKeith Busch NVME_QUIRK_IDENTIFY_CNS = (1 << 1), 6208095e70SKeith Busch 6308095e70SKeith Busch /* 6408095e70SKeith Busch * The controller deterministically returns O's on reads to discarded 6508095e70SKeith Busch * logical blocks. 6608095e70SKeith Busch */ 6708095e70SKeith Busch NVME_QUIRK_DISCARD_ZEROES = (1 << 2), 68106198edSChristoph Hellwig }; 69106198edSChristoph Hellwig 70bb8d261eSChristoph Hellwig enum nvme_ctrl_state { 71bb8d261eSChristoph Hellwig NVME_CTRL_NEW, 72bb8d261eSChristoph Hellwig NVME_CTRL_LIVE, 73bb8d261eSChristoph Hellwig NVME_CTRL_RESETTING, 74bb8d261eSChristoph Hellwig NVME_CTRL_DELETING, 750ff9d4e1SKeith Busch NVME_CTRL_DEAD, 76bb8d261eSChristoph Hellwig }; 77bb8d261eSChristoph Hellwig 781c63dc66SChristoph Hellwig struct nvme_ctrl { 79bb8d261eSChristoph Hellwig enum nvme_ctrl_state state; 80bb8d261eSChristoph Hellwig spinlock_t lock; 811c63dc66SChristoph Hellwig const struct nvme_ctrl_ops *ops; 8257dacad5SJay Sternberg struct request_queue *admin_q; 8357dacad5SJay Sternberg struct device *dev; 8457dacad5SJay Sternberg struct kref kref; 8557dacad5SJay Sternberg int instance; 865bae7f73SChristoph Hellwig struct blk_mq_tag_set *tagset; 875bae7f73SChristoph Hellwig struct list_head namespaces; 8869d3b8acSChristoph Hellwig struct mutex namespaces_mutex; 895bae7f73SChristoph Hellwig struct device *device; /* char device */ 90f3ca80fcSChristoph Hellwig struct list_head node; 91075790ebSKeith Busch struct ida ns_ida; 921c63dc66SChristoph Hellwig 9357dacad5SJay Sternberg char name[12]; 9457dacad5SJay Sternberg char serial[20]; 9557dacad5SJay Sternberg char model[40]; 9657dacad5SJay Sternberg char firmware_rev[8]; 9776e3914aSChristoph Hellwig u16 cntlid; 985fd4ce1bSChristoph Hellwig 995fd4ce1bSChristoph Hellwig u32 ctrl_config; 1005fd4ce1bSChristoph Hellwig 1015fd4ce1bSChristoph Hellwig u32 page_size; 10257dacad5SJay Sternberg u32 max_hw_sectors; 10357dacad5SJay Sternberg u32 stripe_size; 10457dacad5SJay Sternberg u16 oncs; 105118472abSKeith Busch u16 vid; 1066bf25d16SChristoph Hellwig atomic_t abort_limit; 10757dacad5SJay Sternberg u8 event_limit; 10857dacad5SJay Sternberg u8 vwc; 109f3ca80fcSChristoph Hellwig u32 vs; 110f3ca80fcSChristoph Hellwig bool subsystem; 111106198edSChristoph Hellwig unsigned long quirks; 1125955be21SChristoph Hellwig struct work_struct scan_work; 113f866fc42SChristoph Hellwig struct work_struct async_event_work; 11457dacad5SJay Sternberg }; 11557dacad5SJay Sternberg 11657dacad5SJay Sternberg /* 11757dacad5SJay Sternberg * An NVM Express namespace is equivalent to a SCSI LUN 11857dacad5SJay Sternberg */ 11957dacad5SJay Sternberg struct nvme_ns { 12057dacad5SJay Sternberg struct list_head list; 12157dacad5SJay Sternberg 1221c63dc66SChristoph Hellwig struct nvme_ctrl *ctrl; 12357dacad5SJay Sternberg struct request_queue *queue; 12457dacad5SJay Sternberg struct gendisk *disk; 12557dacad5SJay Sternberg struct kref kref; 126075790ebSKeith Busch int instance; 12757dacad5SJay Sternberg 1282b9b6e86SKeith Busch u8 eui[8]; 1292b9b6e86SKeith Busch u8 uuid[16]; 1302b9b6e86SKeith Busch 13157dacad5SJay Sternberg unsigned ns_id; 13257dacad5SJay Sternberg int lba_shift; 13357dacad5SJay Sternberg u16 ms; 13457dacad5SJay Sternberg bool ext; 13557dacad5SJay Sternberg u8 pi_type; 136ca064085SMatias Bjørling int type; 137646017a6SKeith Busch unsigned long flags; 138646017a6SKeith Busch 139646017a6SKeith Busch #define NVME_NS_REMOVING 0 14069d9a99cSKeith Busch #define NVME_NS_DEAD 1 141646017a6SKeith Busch 14257dacad5SJay Sternberg u64 mode_select_num_blocks; 14357dacad5SJay Sternberg u32 mode_select_block_len; 14457dacad5SJay Sternberg }; 14557dacad5SJay Sternberg 1461c63dc66SChristoph Hellwig struct nvme_ctrl_ops { 1471a353d85SMing Lin const char *name; 148e439bb12SSagi Grimberg struct module *module; 1491c63dc66SChristoph Hellwig int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); 1505fd4ce1bSChristoph Hellwig int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); 1517fd8930fSChristoph Hellwig int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); 152f3ca80fcSChristoph Hellwig int (*reset_ctrl)(struct nvme_ctrl *ctrl); 1531673f1f0SChristoph Hellwig void (*free_ctrl)(struct nvme_ctrl *ctrl); 1545955be21SChristoph Hellwig void (*post_scan)(struct nvme_ctrl *ctrl); 155f866fc42SChristoph Hellwig void (*submit_async_event)(struct nvme_ctrl *ctrl, int aer_idx); 1561a353d85SMing Lin int (*delete_ctrl)(struct nvme_ctrl *ctrl); 1571a353d85SMing Lin const char *(*get_subsysnqn)(struct nvme_ctrl *ctrl); 1581a353d85SMing Lin int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); 15957dacad5SJay Sternberg }; 16057dacad5SJay Sternberg 1611c63dc66SChristoph Hellwig static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl) 1621c63dc66SChristoph Hellwig { 1631c63dc66SChristoph Hellwig u32 val = 0; 1641c63dc66SChristoph Hellwig 1651c63dc66SChristoph Hellwig if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val)) 1661c63dc66SChristoph Hellwig return false; 1671c63dc66SChristoph Hellwig return val & NVME_CSTS_RDY; 1681c63dc66SChristoph Hellwig } 1691c63dc66SChristoph Hellwig 170f3ca80fcSChristoph Hellwig static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) 171f3ca80fcSChristoph Hellwig { 172f3ca80fcSChristoph Hellwig if (!ctrl->subsystem) 173f3ca80fcSChristoph Hellwig return -ENOTTY; 174f3ca80fcSChristoph Hellwig return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); 175f3ca80fcSChristoph Hellwig } 176f3ca80fcSChristoph Hellwig 17757dacad5SJay Sternberg static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector) 17857dacad5SJay Sternberg { 17957dacad5SJay Sternberg return (sector >> (ns->lba_shift - 9)); 18057dacad5SJay Sternberg } 18157dacad5SJay Sternberg 18258b45602SMing Lin static inline unsigned nvme_map_len(struct request *rq) 18358b45602SMing Lin { 184c2df40dfSMike Christie if (req_op(rq) == REQ_OP_DISCARD) 18558b45602SMing Lin return sizeof(struct nvme_dsm_range); 18658b45602SMing Lin else 18758b45602SMing Lin return blk_rq_bytes(rq); 18858b45602SMing Lin } 18958b45602SMing Lin 1906904242dSMing Lin static inline void nvme_cleanup_cmd(struct request *req) 1916904242dSMing Lin { 192c2df40dfSMike Christie if (req_op(req) == REQ_OP_DISCARD) 1936904242dSMing Lin kfree(req->completion_data); 1946904242dSMing Lin } 1956904242dSMing Lin 19615a190f7SChristoph Hellwig static inline int nvme_error_status(u16 status) 19715a190f7SChristoph Hellwig { 19815a190f7SChristoph Hellwig switch (status & 0x7ff) { 19915a190f7SChristoph Hellwig case NVME_SC_SUCCESS: 20015a190f7SChristoph Hellwig return 0; 20115a190f7SChristoph Hellwig case NVME_SC_CAP_EXCEEDED: 20215a190f7SChristoph Hellwig return -ENOSPC; 20315a190f7SChristoph Hellwig default: 20415a190f7SChristoph Hellwig return -EIO; 20515a190f7SChristoph Hellwig } 20615a190f7SChristoph Hellwig } 20715a190f7SChristoph Hellwig 2087688faa6SChristoph Hellwig static inline bool nvme_req_needs_retry(struct request *req, u16 status) 2097688faa6SChristoph Hellwig { 2107688faa6SChristoph Hellwig return !(status & NVME_SC_DNR || blk_noretry_request(req)) && 2117688faa6SChristoph Hellwig (jiffies - req->start_time) < req->timeout; 2127688faa6SChristoph Hellwig } 2137688faa6SChristoph Hellwig 214c55a2fd4SMing Lin void nvme_cancel_request(struct request *req, void *data, bool reserved); 215bb8d261eSChristoph Hellwig bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, 216bb8d261eSChristoph Hellwig enum nvme_ctrl_state new_state); 2175fd4ce1bSChristoph Hellwig int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2185fd4ce1bSChristoph Hellwig int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2195fd4ce1bSChristoph Hellwig int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); 220f3ca80fcSChristoph Hellwig int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 221f3ca80fcSChristoph Hellwig const struct nvme_ctrl_ops *ops, unsigned long quirks); 22253029b04SKeith Busch void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); 2231673f1f0SChristoph Hellwig void nvme_put_ctrl(struct nvme_ctrl *ctrl); 2247fd8930fSChristoph Hellwig int nvme_init_identify(struct nvme_ctrl *ctrl); 2255bae7f73SChristoph Hellwig 2265955be21SChristoph Hellwig void nvme_queue_scan(struct nvme_ctrl *ctrl); 2275bae7f73SChristoph Hellwig void nvme_remove_namespaces(struct nvme_ctrl *ctrl); 2281673f1f0SChristoph Hellwig 229f866fc42SChristoph Hellwig #define NVME_NR_AERS 1 230f866fc42SChristoph Hellwig void nvme_complete_async_event(struct nvme_ctrl *ctrl, 231f866fc42SChristoph Hellwig struct nvme_completion *cqe); 232f866fc42SChristoph Hellwig void nvme_queue_async_events(struct nvme_ctrl *ctrl); 233f866fc42SChristoph Hellwig 23425646264SKeith Busch void nvme_stop_queues(struct nvme_ctrl *ctrl); 23525646264SKeith Busch void nvme_start_queues(struct nvme_ctrl *ctrl); 23669d9a99cSKeith Busch void nvme_kill_queues(struct nvme_ctrl *ctrl); 237363c9aacSSagi Grimberg 238eb71f435SChristoph Hellwig #define NVME_QID_ANY -1 2394160982eSChristoph Hellwig struct request *nvme_alloc_request(struct request_queue *q, 240eb71f435SChristoph Hellwig struct nvme_command *cmd, unsigned int flags, int qid); 2417688faa6SChristoph Hellwig void nvme_requeue_req(struct request *req); 2428093f7caSMing Lin int nvme_setup_cmd(struct nvme_ns *ns, struct request *req, 2438093f7caSMing Lin struct nvme_command *cmd); 24457dacad5SJay Sternberg int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 24557dacad5SJay Sternberg void *buf, unsigned bufflen); 24657dacad5SJay Sternberg int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 2471cb3cce5SChristoph Hellwig struct nvme_completion *cqe, void *buffer, unsigned bufflen, 248eb71f435SChristoph Hellwig unsigned timeout, int qid, int at_head, int flags); 2494160982eSChristoph Hellwig int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2504160982eSChristoph Hellwig void __user *ubuffer, unsigned bufflen, u32 *result, 2514160982eSChristoph Hellwig unsigned timeout); 2520b7f1f26SKeith Busch int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2530b7f1f26SKeith Busch void __user *ubuffer, unsigned bufflen, 2540b7f1f26SKeith Busch void __user *meta_buffer, unsigned meta_len, u32 meta_seed, 25557dacad5SJay Sternberg u32 *result, unsigned timeout); 2561c63dc66SChristoph Hellwig int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id); 2571c63dc66SChristoph Hellwig int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, 25857dacad5SJay Sternberg struct nvme_id_ns **id); 2591c63dc66SChristoph Hellwig int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log); 2601c63dc66SChristoph Hellwig int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, 26157dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2621c63dc66SChristoph Hellwig int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, 26357dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2649a0be7abSChristoph Hellwig int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); 26557dacad5SJay Sternberg 26657dacad5SJay Sternberg struct sg_io_hdr; 26757dacad5SJay Sternberg 26857dacad5SJay Sternberg int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr); 26957dacad5SJay Sternberg int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); 27057dacad5SJay Sternberg int nvme_sg_get_version_num(int __user *ip); 27157dacad5SJay Sternberg 272c4699e70SKeith Busch #ifdef CONFIG_NVM 273ca064085SMatias Bjørling int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id); 274ca064085SMatias Bjørling int nvme_nvm_register(struct request_queue *q, char *disk_name); 275ca064085SMatias Bjørling void nvme_nvm_unregister(struct request_queue *q, char *disk_name); 276c4699e70SKeith Busch #else 277c4699e70SKeith Busch static inline int nvme_nvm_register(struct request_queue *q, char *disk_name) 278c4699e70SKeith Busch { 279c4699e70SKeith Busch return 0; 280c4699e70SKeith Busch } 281c4699e70SKeith Busch 282c4699e70SKeith Busch static inline void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {}; 283c4699e70SKeith Busch 284c4699e70SKeith Busch static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id) 285c4699e70SKeith Busch { 286c4699e70SKeith Busch return 0; 287c4699e70SKeith Busch } 288c4699e70SKeith Busch #endif /* CONFIG_NVM */ 289ca064085SMatias Bjørling 2905bae7f73SChristoph Hellwig int __init nvme_core_init(void); 2915bae7f73SChristoph Hellwig void nvme_core_exit(void); 2925bae7f73SChristoph Hellwig 29357dacad5SJay Sternberg #endif /* _NVME_H */ 294