157dacad5SJay Sternberg /* 257dacad5SJay Sternberg * Copyright (c) 2011-2014, Intel Corporation. 357dacad5SJay Sternberg * 457dacad5SJay Sternberg * This program is free software; you can redistribute it and/or modify it 557dacad5SJay Sternberg * under the terms and conditions of the GNU General Public License, 657dacad5SJay Sternberg * version 2, as published by the Free Software Foundation. 757dacad5SJay Sternberg * 857dacad5SJay Sternberg * This program is distributed in the hope it will be useful, but WITHOUT 957dacad5SJay Sternberg * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1057dacad5SJay Sternberg * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1157dacad5SJay Sternberg * more details. 1257dacad5SJay Sternberg */ 1357dacad5SJay Sternberg 1457dacad5SJay Sternberg #ifndef _NVME_H 1557dacad5SJay Sternberg #define _NVME_H 1657dacad5SJay Sternberg 1757dacad5SJay Sternberg #include <linux/nvme.h> 1857dacad5SJay Sternberg #include <linux/pci.h> 1957dacad5SJay Sternberg #include <linux/kref.h> 2057dacad5SJay Sternberg #include <linux/blk-mq.h> 2157dacad5SJay Sternberg 22297465c8SChristoph Hellwig enum { 23297465c8SChristoph Hellwig /* 24297465c8SChristoph Hellwig * Driver internal status code for commands that were cancelled due 25297465c8SChristoph Hellwig * to timeouts or controller shutdown. The value is negative so 26297465c8SChristoph Hellwig * that it a) doesn't overlap with the unsigned hardware error codes, 27297465c8SChristoph Hellwig * and b) can easily be tested for. 28297465c8SChristoph Hellwig */ 29297465c8SChristoph Hellwig NVME_SC_CANCELLED = -EINTR, 30297465c8SChristoph Hellwig }; 31297465c8SChristoph Hellwig 3257dacad5SJay Sternberg extern unsigned char nvme_io_timeout; 3357dacad5SJay Sternberg #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) 3457dacad5SJay Sternberg 3521d34711SChristoph Hellwig extern unsigned char admin_timeout; 3621d34711SChristoph Hellwig #define ADMIN_TIMEOUT (admin_timeout * HZ) 3721d34711SChristoph Hellwig 385fd4ce1bSChristoph Hellwig extern unsigned char shutdown_timeout; 395fd4ce1bSChristoph Hellwig #define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ) 405fd4ce1bSChristoph Hellwig 41ca064085SMatias Bjørling enum { 42ca064085SMatias Bjørling NVME_NS_LBA = 0, 43ca064085SMatias Bjørling NVME_NS_LIGHTNVM = 1, 44ca064085SMatias Bjørling }; 45ca064085SMatias Bjørling 4657dacad5SJay Sternberg /* 47106198edSChristoph Hellwig * List of workarounds for devices that required behavior not specified in 48106198edSChristoph Hellwig * the standard. 4957dacad5SJay Sternberg */ 50106198edSChristoph Hellwig enum nvme_quirks { 51106198edSChristoph Hellwig /* 52106198edSChristoph Hellwig * Prefers I/O aligned to a stripe size specified in a vendor 53106198edSChristoph Hellwig * specific Identify field. 54106198edSChristoph Hellwig */ 55106198edSChristoph Hellwig NVME_QUIRK_STRIPE_SIZE = (1 << 0), 56540c801cSKeith Busch 57540c801cSKeith Busch /* 58540c801cSKeith Busch * The controller doesn't handle Identify value others than 0 or 1 59540c801cSKeith Busch * correctly. 60540c801cSKeith Busch */ 61540c801cSKeith Busch NVME_QUIRK_IDENTIFY_CNS = (1 << 1), 62106198edSChristoph Hellwig }; 63106198edSChristoph Hellwig 641c63dc66SChristoph Hellwig struct nvme_ctrl { 651c63dc66SChristoph Hellwig const struct nvme_ctrl_ops *ops; 6657dacad5SJay Sternberg struct request_queue *admin_q; 6757dacad5SJay Sternberg struct device *dev; 6857dacad5SJay Sternberg struct kref kref; 6957dacad5SJay Sternberg int instance; 705bae7f73SChristoph Hellwig struct blk_mq_tag_set *tagset; 715bae7f73SChristoph Hellwig struct list_head namespaces; 7269d3b8acSChristoph Hellwig struct mutex namespaces_mutex; 735bae7f73SChristoph Hellwig struct device *device; /* char device */ 74f3ca80fcSChristoph Hellwig struct list_head node; 75075790ebSKeith Busch struct ida ns_ida; 761c63dc66SChristoph Hellwig 7757dacad5SJay Sternberg char name[12]; 7857dacad5SJay Sternberg char serial[20]; 7957dacad5SJay Sternberg char model[40]; 8057dacad5SJay Sternberg char firmware_rev[8]; 815fd4ce1bSChristoph Hellwig 825fd4ce1bSChristoph Hellwig u32 ctrl_config; 835fd4ce1bSChristoph Hellwig 845fd4ce1bSChristoph Hellwig u32 page_size; 8557dacad5SJay Sternberg u32 max_hw_sectors; 8657dacad5SJay Sternberg u32 stripe_size; 8757dacad5SJay Sternberg u16 oncs; 886bf25d16SChristoph Hellwig atomic_t abort_limit; 8957dacad5SJay Sternberg u8 event_limit; 9057dacad5SJay Sternberg u8 vwc; 91f3ca80fcSChristoph Hellwig u32 vs; 92f3ca80fcSChristoph Hellwig bool subsystem; 93106198edSChristoph Hellwig unsigned long quirks; 9457dacad5SJay Sternberg }; 9557dacad5SJay Sternberg 9657dacad5SJay Sternberg /* 9757dacad5SJay Sternberg * An NVM Express namespace is equivalent to a SCSI LUN 9857dacad5SJay Sternberg */ 9957dacad5SJay Sternberg struct nvme_ns { 10057dacad5SJay Sternberg struct list_head list; 10157dacad5SJay Sternberg 1021c63dc66SChristoph Hellwig struct nvme_ctrl *ctrl; 10357dacad5SJay Sternberg struct request_queue *queue; 10457dacad5SJay Sternberg struct gendisk *disk; 10557dacad5SJay Sternberg struct kref kref; 106075790ebSKeith Busch int instance; 10757dacad5SJay Sternberg 1082b9b6e86SKeith Busch u8 eui[8]; 1092b9b6e86SKeith Busch u8 uuid[16]; 1102b9b6e86SKeith Busch 11157dacad5SJay Sternberg unsigned ns_id; 11257dacad5SJay Sternberg int lba_shift; 11357dacad5SJay Sternberg u16 ms; 11457dacad5SJay Sternberg bool ext; 11557dacad5SJay Sternberg u8 pi_type; 116ca064085SMatias Bjørling int type; 11757dacad5SJay Sternberg u64 mode_select_num_blocks; 11857dacad5SJay Sternberg u32 mode_select_block_len; 11957dacad5SJay Sternberg }; 12057dacad5SJay Sternberg 1211c63dc66SChristoph Hellwig struct nvme_ctrl_ops { 1221c63dc66SChristoph Hellwig int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); 1235fd4ce1bSChristoph Hellwig int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); 1247fd8930fSChristoph Hellwig int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); 1255bae7f73SChristoph Hellwig bool (*io_incapable)(struct nvme_ctrl *ctrl); 126f3ca80fcSChristoph Hellwig int (*reset_ctrl)(struct nvme_ctrl *ctrl); 1271673f1f0SChristoph Hellwig void (*free_ctrl)(struct nvme_ctrl *ctrl); 12857dacad5SJay Sternberg }; 12957dacad5SJay Sternberg 1301c63dc66SChristoph Hellwig static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl) 1311c63dc66SChristoph Hellwig { 1321c63dc66SChristoph Hellwig u32 val = 0; 1331c63dc66SChristoph Hellwig 1341c63dc66SChristoph Hellwig if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val)) 1351c63dc66SChristoph Hellwig return false; 1361c63dc66SChristoph Hellwig return val & NVME_CSTS_RDY; 1371c63dc66SChristoph Hellwig } 1381c63dc66SChristoph Hellwig 1395bae7f73SChristoph Hellwig static inline bool nvme_io_incapable(struct nvme_ctrl *ctrl) 1405bae7f73SChristoph Hellwig { 1415bae7f73SChristoph Hellwig u32 val = 0; 1425bae7f73SChristoph Hellwig 1435bae7f73SChristoph Hellwig if (ctrl->ops->io_incapable(ctrl)) 1444f76d0e4SKeith Busch return true; 1455bae7f73SChristoph Hellwig if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val)) 1464f76d0e4SKeith Busch return true; 1475bae7f73SChristoph Hellwig return val & NVME_CSTS_CFS; 1485bae7f73SChristoph Hellwig } 1495bae7f73SChristoph Hellwig 150f3ca80fcSChristoph Hellwig static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) 151f3ca80fcSChristoph Hellwig { 152f3ca80fcSChristoph Hellwig if (!ctrl->subsystem) 153f3ca80fcSChristoph Hellwig return -ENOTTY; 154f3ca80fcSChristoph Hellwig return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); 155f3ca80fcSChristoph Hellwig } 156f3ca80fcSChristoph Hellwig 15757dacad5SJay Sternberg static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector) 15857dacad5SJay Sternberg { 15957dacad5SJay Sternberg return (sector >> (ns->lba_shift - 9)); 16057dacad5SJay Sternberg } 16157dacad5SJay Sternberg 16222944e99SChristoph Hellwig static inline void nvme_setup_flush(struct nvme_ns *ns, 16322944e99SChristoph Hellwig struct nvme_command *cmnd) 16422944e99SChristoph Hellwig { 16522944e99SChristoph Hellwig memset(cmnd, 0, sizeof(*cmnd)); 16622944e99SChristoph Hellwig cmnd->common.opcode = nvme_cmd_flush; 16722944e99SChristoph Hellwig cmnd->common.nsid = cpu_to_le32(ns->ns_id); 16822944e99SChristoph Hellwig } 16922944e99SChristoph Hellwig 17022944e99SChristoph Hellwig static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req, 17122944e99SChristoph Hellwig struct nvme_command *cmnd) 17222944e99SChristoph Hellwig { 17322944e99SChristoph Hellwig u16 control = 0; 17422944e99SChristoph Hellwig u32 dsmgmt = 0; 17522944e99SChristoph Hellwig 17622944e99SChristoph Hellwig if (req->cmd_flags & REQ_FUA) 17722944e99SChristoph Hellwig control |= NVME_RW_FUA; 17822944e99SChristoph Hellwig if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 17922944e99SChristoph Hellwig control |= NVME_RW_LR; 18022944e99SChristoph Hellwig 18122944e99SChristoph Hellwig if (req->cmd_flags & REQ_RAHEAD) 18222944e99SChristoph Hellwig dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 18322944e99SChristoph Hellwig 18422944e99SChristoph Hellwig memset(cmnd, 0, sizeof(*cmnd)); 18522944e99SChristoph Hellwig cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); 18622944e99SChristoph Hellwig cmnd->rw.command_id = req->tag; 18722944e99SChristoph Hellwig cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 18822944e99SChristoph Hellwig cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); 18922944e99SChristoph Hellwig cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 19022944e99SChristoph Hellwig 19122944e99SChristoph Hellwig if (ns->ms) { 19222944e99SChristoph Hellwig switch (ns->pi_type) { 19322944e99SChristoph Hellwig case NVME_NS_DPS_PI_TYPE3: 19422944e99SChristoph Hellwig control |= NVME_RW_PRINFO_PRCHK_GUARD; 19522944e99SChristoph Hellwig break; 19622944e99SChristoph Hellwig case NVME_NS_DPS_PI_TYPE1: 19722944e99SChristoph Hellwig case NVME_NS_DPS_PI_TYPE2: 19822944e99SChristoph Hellwig control |= NVME_RW_PRINFO_PRCHK_GUARD | 19922944e99SChristoph Hellwig NVME_RW_PRINFO_PRCHK_REF; 20022944e99SChristoph Hellwig cmnd->rw.reftag = cpu_to_le32( 20122944e99SChristoph Hellwig nvme_block_nr(ns, blk_rq_pos(req))); 20222944e99SChristoph Hellwig break; 20322944e99SChristoph Hellwig } 20422944e99SChristoph Hellwig if (!blk_integrity_rq(req)) 20522944e99SChristoph Hellwig control |= NVME_RW_PRINFO_PRACT; 20622944e99SChristoph Hellwig } 20722944e99SChristoph Hellwig 20822944e99SChristoph Hellwig cmnd->rw.control = cpu_to_le16(control); 20922944e99SChristoph Hellwig cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 21022944e99SChristoph Hellwig } 21122944e99SChristoph Hellwig 21222944e99SChristoph Hellwig 21315a190f7SChristoph Hellwig static inline int nvme_error_status(u16 status) 21415a190f7SChristoph Hellwig { 21515a190f7SChristoph Hellwig switch (status & 0x7ff) { 21615a190f7SChristoph Hellwig case NVME_SC_SUCCESS: 21715a190f7SChristoph Hellwig return 0; 21815a190f7SChristoph Hellwig case NVME_SC_CAP_EXCEEDED: 21915a190f7SChristoph Hellwig return -ENOSPC; 22015a190f7SChristoph Hellwig default: 22115a190f7SChristoph Hellwig return -EIO; 22215a190f7SChristoph Hellwig } 22315a190f7SChristoph Hellwig } 22415a190f7SChristoph Hellwig 2257688faa6SChristoph Hellwig static inline bool nvme_req_needs_retry(struct request *req, u16 status) 2267688faa6SChristoph Hellwig { 2277688faa6SChristoph Hellwig return !(status & NVME_SC_DNR || blk_noretry_request(req)) && 2287688faa6SChristoph Hellwig (jiffies - req->start_time) < req->timeout; 2297688faa6SChristoph Hellwig } 2307688faa6SChristoph Hellwig 2315fd4ce1bSChristoph Hellwig int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2325fd4ce1bSChristoph Hellwig int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap); 2335fd4ce1bSChristoph Hellwig int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); 234f3ca80fcSChristoph Hellwig int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 235f3ca80fcSChristoph Hellwig const struct nvme_ctrl_ops *ops, unsigned long quirks); 23653029b04SKeith Busch void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); 2371673f1f0SChristoph Hellwig void nvme_put_ctrl(struct nvme_ctrl *ctrl); 2387fd8930fSChristoph Hellwig int nvme_init_identify(struct nvme_ctrl *ctrl); 2395bae7f73SChristoph Hellwig 2405bae7f73SChristoph Hellwig void nvme_scan_namespaces(struct nvme_ctrl *ctrl); 2415bae7f73SChristoph Hellwig void nvme_remove_namespaces(struct nvme_ctrl *ctrl); 2421673f1f0SChristoph Hellwig 24325646264SKeith Busch void nvme_stop_queues(struct nvme_ctrl *ctrl); 24425646264SKeith Busch void nvme_start_queues(struct nvme_ctrl *ctrl); 245363c9aacSSagi Grimberg 2464160982eSChristoph Hellwig struct request *nvme_alloc_request(struct request_queue *q, 2474160982eSChristoph Hellwig struct nvme_command *cmd, unsigned int flags); 2487688faa6SChristoph Hellwig void nvme_requeue_req(struct request *req); 24957dacad5SJay Sternberg int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 25057dacad5SJay Sternberg void *buf, unsigned bufflen); 25157dacad5SJay Sternberg int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 2524160982eSChristoph Hellwig void *buffer, unsigned bufflen, u32 *result, unsigned timeout); 2534160982eSChristoph Hellwig int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2544160982eSChristoph Hellwig void __user *ubuffer, unsigned bufflen, u32 *result, 2554160982eSChristoph Hellwig unsigned timeout); 2560b7f1f26SKeith Busch int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd, 2570b7f1f26SKeith Busch void __user *ubuffer, unsigned bufflen, 2580b7f1f26SKeith Busch void __user *meta_buffer, unsigned meta_len, u32 meta_seed, 25957dacad5SJay Sternberg u32 *result, unsigned timeout); 2601c63dc66SChristoph Hellwig int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id); 2611c63dc66SChristoph Hellwig int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, 26257dacad5SJay Sternberg struct nvme_id_ns **id); 2631c63dc66SChristoph Hellwig int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log); 2641c63dc66SChristoph Hellwig int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, 26557dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2661c63dc66SChristoph Hellwig int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, 26757dacad5SJay Sternberg dma_addr_t dma_addr, u32 *result); 2689a0be7abSChristoph Hellwig int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); 26957dacad5SJay Sternberg 2701673f1f0SChristoph Hellwig extern spinlock_t dev_list_lock; 27157dacad5SJay Sternberg 27257dacad5SJay Sternberg struct sg_io_hdr; 27357dacad5SJay Sternberg 27457dacad5SJay Sternberg int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr); 27557dacad5SJay Sternberg int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); 27657dacad5SJay Sternberg int nvme_sg_get_version_num(int __user *ip); 27757dacad5SJay Sternberg 278c4699e70SKeith Busch #ifdef CONFIG_NVM 279ca064085SMatias Bjørling int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id); 280ca064085SMatias Bjørling int nvme_nvm_register(struct request_queue *q, char *disk_name); 281ca064085SMatias Bjørling void nvme_nvm_unregister(struct request_queue *q, char *disk_name); 282c4699e70SKeith Busch #else 283c4699e70SKeith Busch static inline int nvme_nvm_register(struct request_queue *q, char *disk_name) 284c4699e70SKeith Busch { 285c4699e70SKeith Busch return 0; 286c4699e70SKeith Busch } 287c4699e70SKeith Busch 288c4699e70SKeith Busch static inline void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {}; 289c4699e70SKeith Busch 290c4699e70SKeith Busch static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id) 291c4699e70SKeith Busch { 292c4699e70SKeith Busch return 0; 293c4699e70SKeith Busch } 294c4699e70SKeith Busch #endif /* CONFIG_NVM */ 295ca064085SMatias Bjørling 2965bae7f73SChristoph Hellwig int __init nvme_core_init(void); 2975bae7f73SChristoph Hellwig void nvme_core_exit(void); 2985bae7f73SChristoph Hellwig 29957dacad5SJay Sternberg #endif /* _NVME_H */ 300