177141dc6SChristoph Hellwig /* SPDX-License-Identifier: GPL-2.0 */ 2a07b4970SChristoph Hellwig /* 3a07b4970SChristoph Hellwig * Copyright (c) 2015-2016 HGST, a Western Digital Company. 4a07b4970SChristoph Hellwig */ 5a07b4970SChristoph Hellwig 6a07b4970SChristoph Hellwig #ifndef _NVMET_H 7a07b4970SChristoph Hellwig #define _NVMET_H 8a07b4970SChristoph Hellwig 9a07b4970SChristoph Hellwig #include <linux/dma-mapping.h> 10a07b4970SChristoph Hellwig #include <linux/types.h> 11a07b4970SChristoph Hellwig #include <linux/device.h> 12a07b4970SChristoph Hellwig #include <linux/kref.h> 13a07b4970SChristoph Hellwig #include <linux/percpu-refcount.h> 14a07b4970SChristoph Hellwig #include <linux/list.h> 15a07b4970SChristoph Hellwig #include <linux/mutex.h> 168e412263SChristoph Hellwig #include <linux/uuid.h> 17a07b4970SChristoph Hellwig #include <linux/nvme.h> 18a07b4970SChristoph Hellwig #include <linux/configfs.h> 19a07b4970SChristoph Hellwig #include <linux/rcupdate.h> 20a07b4970SChristoph Hellwig #include <linux/blkdev.h> 21c6925093SLogan Gunthorpe #include <linux/radix-tree.h> 22d2d1c454SIsrael Rukshin #include <linux/t10-pi.h> 23a07b4970SChristoph Hellwig 24ba76af67SLogan Gunthorpe #define NVMET_DEFAULT_VS NVME_VS(1, 3, 0) 25ba76af67SLogan Gunthorpe 26a07b4970SChristoph Hellwig #define NVMET_ASYNC_EVENTS 4 27a07b4970SChristoph Hellwig #define NVMET_ERROR_LOG_SLOTS 128 285698b805SChaitanya Kulkarni #define NVMET_NO_ERROR_LOC ((u16)-1) 29013b7ebeSMark Ruijter #define NVMET_DEFAULT_CTRL_MODEL "Linux" 3048b4c010SNoam Gottlieb #define NVMET_MN_MAX_SIZE 40 31e13b0615SNoam Gottlieb #define NVMET_SN_MAX_SIZE 20 32a07b4970SChristoph Hellwig 33c86b8f7bSChristoph Hellwig /* 34c86b8f7bSChristoph Hellwig * Supported optional AENs: 35c86b8f7bSChristoph Hellwig */ 36c86b8f7bSChristoph Hellwig #define NVMET_AEN_CFG_OPTIONAL \ 3762ac0d32SChristoph Hellwig (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_ANA_CHANGE) 38f301c2b1SJay Sternberg #define NVMET_DISC_AEN_CFG_OPTIONAL \ 39f301c2b1SJay Sternberg (NVME_AEN_CFG_DISC_CHANGE) 40c86b8f7bSChristoph Hellwig 41c86b8f7bSChristoph Hellwig /* 42c86b8f7bSChristoph Hellwig * Plus mandatory SMART AENs (we'll never send them, but allow enabling them): 43c86b8f7bSChristoph Hellwig */ 44c86b8f7bSChristoph Hellwig #define NVMET_AEN_CFG_ALL \ 45c86b8f7bSChristoph Hellwig (NVME_SMART_CRIT_SPARE | NVME_SMART_CRIT_TEMPERATURE | \ 46c86b8f7bSChristoph Hellwig NVME_SMART_CRIT_RELIABILITY | NVME_SMART_CRIT_MEDIA | \ 47c86b8f7bSChristoph Hellwig NVME_SMART_CRIT_VOLATILE_MEMORY | NVMET_AEN_CFG_OPTIONAL) 48c86b8f7bSChristoph Hellwig 49a07b4970SChristoph Hellwig /* Helper Macros when NVMe error is NVME_SC_CONNECT_INVALID_PARAM 50a07b4970SChristoph Hellwig * The 16 bit shift is to set IATTR bit to 1, which means offending 51a07b4970SChristoph Hellwig * offset starts in the data section of connect() 52a07b4970SChristoph Hellwig */ 53a07b4970SChristoph Hellwig #define IPO_IATTR_CONNECT_DATA(x) \ 54a07b4970SChristoph Hellwig (cpu_to_le32((1 << 16) | (offsetof(struct nvmf_connect_data, x)))) 55a07b4970SChristoph Hellwig #define IPO_IATTR_CONNECT_SQE(x) \ 56a07b4970SChristoph Hellwig (cpu_to_le32(offsetof(struct nvmf_connect_command, x))) 57a07b4970SChristoph Hellwig 58a07b4970SChristoph Hellwig struct nvmet_ns { 59a07b4970SChristoph Hellwig struct percpu_ref ref; 60a07b4970SChristoph Hellwig struct block_device *bdev; 61d5eff33eSChaitanya Kulkarni struct file *file; 62dedf0be5SChaitanya Kulkarni bool readonly; 63a07b4970SChristoph Hellwig u32 nsid; 64a07b4970SChristoph Hellwig u32 blksize_shift; 65a07b4970SChristoph Hellwig loff_t size; 66a07b4970SChristoph Hellwig u8 nguid[16]; 67637dc0f3SJohannes Thumshirn uuid_t uuid; 6872efd25dSChristoph Hellwig u32 anagrpid; 69a07b4970SChristoph Hellwig 7055eb942eSChaitanya Kulkarni bool buffered_io; 71e4fcf07cSSolganik Alexander bool enabled; 72a07b4970SChristoph Hellwig struct nvmet_subsys *subsys; 73a07b4970SChristoph Hellwig const char *device_path; 74a07b4970SChristoph Hellwig 75a07b4970SChristoph Hellwig struct config_group device_group; 76a07b4970SChristoph Hellwig struct config_group group; 77a07b4970SChristoph Hellwig 78a07b4970SChristoph Hellwig struct completion disable_done; 79d5eff33eSChaitanya Kulkarni mempool_t *bvec_pool; 80d5eff33eSChaitanya Kulkarni struct kmem_cache *bvec_cache; 81c6925093SLogan Gunthorpe 82c6925093SLogan Gunthorpe int use_p2pmem; 83c6925093SLogan Gunthorpe struct pci_dev *p2p_dev; 84d2d1c454SIsrael Rukshin int pi_type; 85d2d1c454SIsrael Rukshin int metadata_size; 86ab5d0b38SChaitanya Kulkarni u8 csi; 87a07b4970SChristoph Hellwig }; 88a07b4970SChristoph Hellwig 89a07b4970SChristoph Hellwig static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) 90a07b4970SChristoph Hellwig { 91a07b4970SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_ns, group); 92a07b4970SChristoph Hellwig } 93a07b4970SChristoph Hellwig 94c6925093SLogan Gunthorpe static inline struct device *nvmet_ns_dev(struct nvmet_ns *ns) 95c6925093SLogan Gunthorpe { 96c6925093SLogan Gunthorpe return ns->bdev ? disk_to_dev(ns->bdev->bd_disk) : NULL; 97c6925093SLogan Gunthorpe } 98c6925093SLogan Gunthorpe 99a07b4970SChristoph Hellwig struct nvmet_cq { 100a07b4970SChristoph Hellwig u16 qid; 101a07b4970SChristoph Hellwig u16 size; 102a07b4970SChristoph Hellwig }; 103a07b4970SChristoph Hellwig 104a07b4970SChristoph Hellwig struct nvmet_sq { 105a07b4970SChristoph Hellwig struct nvmet_ctrl *ctrl; 106a07b4970SChristoph Hellwig struct percpu_ref ref; 107a07b4970SChristoph Hellwig u16 qid; 108a07b4970SChristoph Hellwig u16 size; 109f9cf2a64SJames Smart u32 sqhd; 110e6a622fdSSagi Grimberg bool sqhd_disabled; 111db1312ddSHannes Reinecke #ifdef CONFIG_NVME_TARGET_AUTH 112db1312ddSHannes Reinecke bool authenticated; 113db1312ddSHannes Reinecke u16 dhchap_tid; 114db1312ddSHannes Reinecke u16 dhchap_status; 115db1312ddSHannes Reinecke int dhchap_step; 116db1312ddSHannes Reinecke u8 *dhchap_c1; 117db1312ddSHannes Reinecke u8 *dhchap_c2; 118db1312ddSHannes Reinecke u32 dhchap_s1; 119db1312ddSHannes Reinecke u32 dhchap_s2; 120db1312ddSHannes Reinecke u8 *dhchap_skey; 121db1312ddSHannes Reinecke int dhchap_skey_len; 122db1312ddSHannes Reinecke #endif 123a07b4970SChristoph Hellwig struct completion free_done; 124427242ceSSagi Grimberg struct completion confirm_done; 125a07b4970SChristoph Hellwig }; 126a07b4970SChristoph Hellwig 12762ac0d32SChristoph Hellwig struct nvmet_ana_group { 12862ac0d32SChristoph Hellwig struct config_group group; 12962ac0d32SChristoph Hellwig struct nvmet_port *port; 13062ac0d32SChristoph Hellwig u32 grpid; 13162ac0d32SChristoph Hellwig }; 13262ac0d32SChristoph Hellwig 13362ac0d32SChristoph Hellwig static inline struct nvmet_ana_group *to_ana_group(struct config_item *item) 13462ac0d32SChristoph Hellwig { 13562ac0d32SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_ana_group, 13662ac0d32SChristoph Hellwig group); 13762ac0d32SChristoph Hellwig } 13862ac0d32SChristoph Hellwig 139a07b4970SChristoph Hellwig /** 140a07b4970SChristoph Hellwig * struct nvmet_port - Common structure to keep port 141a07b4970SChristoph Hellwig * information for the target. 142fe4a9791SChristoph Hellwig * @entry: Entry into referrals or transport list. 143a07b4970SChristoph Hellwig * @disc_addr: Address information is stored in a format defined 144a07b4970SChristoph Hellwig * for a discovery log page entry. 145a07b4970SChristoph Hellwig * @group: ConfigFS group for this element's folder. 146a07b4970SChristoph Hellwig * @priv: Private data for the transport. 147a07b4970SChristoph Hellwig */ 148a07b4970SChristoph Hellwig struct nvmet_port { 149a07b4970SChristoph Hellwig struct list_head entry; 150a07b4970SChristoph Hellwig struct nvmf_disc_rsp_page_entry disc_addr; 151a07b4970SChristoph Hellwig struct config_group group; 152a07b4970SChristoph Hellwig struct config_group subsys_group; 153a07b4970SChristoph Hellwig struct list_head subsystems; 154a07b4970SChristoph Hellwig struct config_group referrals_group; 155a07b4970SChristoph Hellwig struct list_head referrals; 156b662a078SJay Sternberg struct list_head global_entry; 15762ac0d32SChristoph Hellwig struct config_group ana_groups_group; 15862ac0d32SChristoph Hellwig struct nvmet_ana_group ana_default_group; 15972efd25dSChristoph Hellwig enum nvme_ana_state *ana_state; 160a07b4970SChristoph Hellwig void *priv; 161a07b4970SChristoph Hellwig bool enabled; 1620d5ee2b2SSteve Wise int inline_data_size; 1639d09dd8dSJames Smart const struct nvmet_fabrics_ops *tr_ops; 164ea52ac1cSIsrael Rukshin bool pi_enable; 165a07b4970SChristoph Hellwig }; 166a07b4970SChristoph Hellwig 167a07b4970SChristoph Hellwig static inline struct nvmet_port *to_nvmet_port(struct config_item *item) 168a07b4970SChristoph Hellwig { 169a07b4970SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_port, 170a07b4970SChristoph Hellwig group); 171a07b4970SChristoph Hellwig } 172a07b4970SChristoph Hellwig 17362ac0d32SChristoph Hellwig static inline struct nvmet_port *ana_groups_to_port( 17462ac0d32SChristoph Hellwig struct config_item *item) 17562ac0d32SChristoph Hellwig { 17662ac0d32SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_port, 17762ac0d32SChristoph Hellwig ana_groups_group); 17862ac0d32SChristoph Hellwig } 17962ac0d32SChristoph Hellwig 180a07b4970SChristoph Hellwig struct nvmet_ctrl { 181a07b4970SChristoph Hellwig struct nvmet_subsys *subsys; 182a07b4970SChristoph Hellwig struct nvmet_sq **sqs; 183a07b4970SChristoph Hellwig 184aaeadd70SSagi Grimberg bool reset_tbkas; 185c09305aeSSagi Grimberg 186a07b4970SChristoph Hellwig struct mutex lock; 187a07b4970SChristoph Hellwig u64 cap; 188a07b4970SChristoph Hellwig u32 cc; 189a07b4970SChristoph Hellwig u32 csts; 190a07b4970SChristoph Hellwig 19128dd5cf7SOmri Mann uuid_t hostid; 192a07b4970SChristoph Hellwig u16 cntlid; 193a07b4970SChristoph Hellwig u32 kato; 194a07b4970SChristoph Hellwig 1954ee43280SChristoph Hellwig struct nvmet_port *port; 1964ee43280SChristoph Hellwig 197c86b8f7bSChristoph Hellwig u32 aen_enabled; 19855fdd6b6SChristoph Hellwig unsigned long aen_masked; 199a07b4970SChristoph Hellwig struct nvmet_req *async_event_cmds[NVMET_ASYNC_EVENTS]; 200a07b4970SChristoph Hellwig unsigned int nr_async_event_cmds; 201a07b4970SChristoph Hellwig struct list_head async_events; 202a07b4970SChristoph Hellwig struct work_struct async_event_work; 203a07b4970SChristoph Hellwig 204a07b4970SChristoph Hellwig struct list_head subsys_entry; 205a07b4970SChristoph Hellwig struct kref ref; 206a07b4970SChristoph Hellwig struct delayed_work ka_work; 207a07b4970SChristoph Hellwig struct work_struct fatal_err_work; 208a07b4970SChristoph Hellwig 209e929f06dSChristoph Hellwig const struct nvmet_fabrics_ops *ops; 210a07b4970SChristoph Hellwig 211c16734eaSChristoph Hellwig __le32 *changed_ns_list; 212c16734eaSChristoph Hellwig u32 nr_changed_ns; 213c16734eaSChristoph Hellwig 214a07b4970SChristoph Hellwig char subsysnqn[NVMF_NQN_FIELD_LEN]; 215a07b4970SChristoph Hellwig char hostnqn[NVMF_NQN_FIELD_LEN]; 216c6925093SLogan Gunthorpe 217c6925093SLogan Gunthorpe struct device *p2p_client; 218c6925093SLogan Gunthorpe struct radix_tree_root p2p_ns_map; 219e4a97625SChaitanya Kulkarni 220e4a97625SChaitanya Kulkarni spinlock_t error_lock; 221e4a97625SChaitanya Kulkarni u64 err_counter; 222e4a97625SChaitanya Kulkarni struct nvme_error_slot slots[NVMET_ERROR_LOG_SLOTS]; 223ea52ac1cSIsrael Rukshin bool pi_support; 224db1312ddSHannes Reinecke #ifdef CONFIG_NVME_TARGET_AUTH 225db1312ddSHannes Reinecke struct nvme_dhchap_key *host_key; 226db1312ddSHannes Reinecke struct nvme_dhchap_key *ctrl_key; 227db1312ddSHannes Reinecke u8 shash_id; 228*7a277c37SHannes Reinecke struct crypto_kpp *dh_tfm; 229*7a277c37SHannes Reinecke u8 dh_gid; 230*7a277c37SHannes Reinecke u8 *dh_key; 231*7a277c37SHannes Reinecke size_t dh_keysize; 232db1312ddSHannes Reinecke #endif 233a07b4970SChristoph Hellwig }; 234a07b4970SChristoph Hellwig 235a07b4970SChristoph Hellwig struct nvmet_subsys { 236a07b4970SChristoph Hellwig enum nvme_subsys_type type; 237a07b4970SChristoph Hellwig 238a07b4970SChristoph Hellwig struct mutex lock; 239a07b4970SChristoph Hellwig struct kref ref; 240a07b4970SChristoph Hellwig 2417774e77eSChaitanya Kulkarni struct xarray namespaces; 242793c7cfcSChristoph Hellwig unsigned int nr_namespaces; 24386693c43SChaitanya Kulkarni u32 max_nsid; 24494a39d61SChaitanya Kulkarni u16 cntlid_min; 24594a39d61SChaitanya Kulkarni u16 cntlid_max; 246a07b4970SChristoph Hellwig 247a07b4970SChristoph Hellwig struct list_head ctrls; 248a07b4970SChristoph Hellwig 249a07b4970SChristoph Hellwig struct list_head hosts; 250a07b4970SChristoph Hellwig bool allow_any_host; 251a07b4970SChristoph Hellwig 252a07b4970SChristoph Hellwig u16 max_qid; 253a07b4970SChristoph Hellwig 254a07b4970SChristoph Hellwig u64 ver; 255e13b0615SNoam Gottlieb char serial[NVMET_SN_MAX_SIZE]; 2567ae023c5SNoam Gottlieb bool subsys_discovered; 257a07b4970SChristoph Hellwig char *subsysnqn; 258ea52ac1cSIsrael Rukshin bool pi_support; 259a07b4970SChristoph Hellwig 260a07b4970SChristoph Hellwig struct config_group group; 261a07b4970SChristoph Hellwig 262a07b4970SChristoph Hellwig struct config_group namespaces_group; 263a07b4970SChristoph Hellwig struct config_group allowed_hosts_group; 264013b7ebeSMark Ruijter 265d9f273b7SMax Gurtovoy char *model_number; 266c1fef73fSLogan Gunthorpe 267c1fef73fSLogan Gunthorpe #ifdef CONFIG_NVME_TARGET_PASSTHRU 268c1fef73fSLogan Gunthorpe struct nvme_ctrl *passthru_ctrl; 269ba76af67SLogan Gunthorpe char *passthru_ctrl_path; 270cae5b01aSLogan Gunthorpe struct config_group passthru_group; 271a2f6a2b8SChaitanya Kulkarni unsigned int admin_timeout; 27247e9730cSChaitanya Kulkarni unsigned int io_timeout; 27334ad6151SAlan Adamson unsigned int clear_ids; 274c1fef73fSLogan Gunthorpe #endif /* CONFIG_NVME_TARGET_PASSTHRU */ 275aaf2e048SChaitanya Kulkarni 276aaf2e048SChaitanya Kulkarni #ifdef CONFIG_BLK_DEV_ZONED 277aaf2e048SChaitanya Kulkarni u8 zasl; 278aaf2e048SChaitanya Kulkarni #endif /* CONFIG_BLK_DEV_ZONED */ 279a07b4970SChristoph Hellwig }; 280a07b4970SChristoph Hellwig 281a07b4970SChristoph Hellwig static inline struct nvmet_subsys *to_subsys(struct config_item *item) 282a07b4970SChristoph Hellwig { 283a07b4970SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_subsys, group); 284a07b4970SChristoph Hellwig } 285a07b4970SChristoph Hellwig 286a07b4970SChristoph Hellwig static inline struct nvmet_subsys *namespaces_to_subsys( 287a07b4970SChristoph Hellwig struct config_item *item) 288a07b4970SChristoph Hellwig { 289a07b4970SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_subsys, 290a07b4970SChristoph Hellwig namespaces_group); 291a07b4970SChristoph Hellwig } 292a07b4970SChristoph Hellwig 293a07b4970SChristoph Hellwig struct nvmet_host { 294a07b4970SChristoph Hellwig struct config_group group; 295db1312ddSHannes Reinecke u8 *dhchap_secret; 296db1312ddSHannes Reinecke u8 *dhchap_ctrl_secret; 297db1312ddSHannes Reinecke u8 dhchap_key_hash; 298db1312ddSHannes Reinecke u8 dhchap_ctrl_key_hash; 299db1312ddSHannes Reinecke u8 dhchap_hash_id; 300db1312ddSHannes Reinecke u8 dhchap_dhgroup_id; 301a07b4970SChristoph Hellwig }; 302a07b4970SChristoph Hellwig 303a07b4970SChristoph Hellwig static inline struct nvmet_host *to_host(struct config_item *item) 304a07b4970SChristoph Hellwig { 305a07b4970SChristoph Hellwig return container_of(to_config_group(item), struct nvmet_host, group); 306a07b4970SChristoph Hellwig } 307a07b4970SChristoph Hellwig 308a07b4970SChristoph Hellwig static inline char *nvmet_host_name(struct nvmet_host *host) 309a07b4970SChristoph Hellwig { 310a07b4970SChristoph Hellwig return config_item_name(&host->group.cg_item); 311a07b4970SChristoph Hellwig } 312a07b4970SChristoph Hellwig 313a07b4970SChristoph Hellwig struct nvmet_host_link { 314a07b4970SChristoph Hellwig struct list_head entry; 315a07b4970SChristoph Hellwig struct nvmet_host *host; 316a07b4970SChristoph Hellwig }; 317a07b4970SChristoph Hellwig 318a07b4970SChristoph Hellwig struct nvmet_subsys_link { 319a07b4970SChristoph Hellwig struct list_head entry; 320a07b4970SChristoph Hellwig struct nvmet_subsys *subsys; 321a07b4970SChristoph Hellwig }; 322a07b4970SChristoph Hellwig 323a07b4970SChristoph Hellwig struct nvmet_req; 324a07b4970SChristoph Hellwig struct nvmet_fabrics_ops { 325a07b4970SChristoph Hellwig struct module *owner; 326a07b4970SChristoph Hellwig unsigned int type; 327a07b4970SChristoph Hellwig unsigned int msdbd; 3286fa350f7SMax Gurtovoy unsigned int flags; 3296fa350f7SMax Gurtovoy #define NVMF_KEYED_SGLS (1 << 0) 3306fa350f7SMax Gurtovoy #define NVMF_METADATA_SUPPORTED (1 << 1) 331a07b4970SChristoph Hellwig void (*queue_response)(struct nvmet_req *req); 332a07b4970SChristoph Hellwig int (*add_port)(struct nvmet_port *port); 333a07b4970SChristoph Hellwig void (*remove_port)(struct nvmet_port *port); 334a07b4970SChristoph Hellwig void (*delete_ctrl)(struct nvmet_ctrl *ctrl); 3354c652685SSagi Grimberg void (*disc_traddr)(struct nvmet_req *req, 3364c652685SSagi Grimberg struct nvmet_port *port, char *traddr); 3371672ddb8SSagi Grimberg u16 (*install_queue)(struct nvmet_sq *nvme_sq); 3389d09dd8dSJames Smart void (*discovery_chg)(struct nvmet_port *port); 33902cb00e2SMax Gurtovoy u8 (*get_mdts)(const struct nvmet_ctrl *ctrl); 3406d1555ccSMax Gurtovoy u16 (*get_max_queue_size)(const struct nvmet_ctrl *ctrl); 341a07b4970SChristoph Hellwig }; 342a07b4970SChristoph Hellwig 343a07b4970SChristoph Hellwig #define NVMET_MAX_INLINE_BIOVEC 8 34473383adfSSagi Grimberg #define NVMET_MAX_INLINE_DATA_LEN NVMET_MAX_INLINE_BIOVEC * PAGE_SIZE 345a07b4970SChristoph Hellwig 346a07b4970SChristoph Hellwig struct nvmet_req { 347a07b4970SChristoph Hellwig struct nvme_command *cmd; 348fc6c9730SMax Gurtovoy struct nvme_completion *cqe; 349a07b4970SChristoph Hellwig struct nvmet_sq *sq; 350a07b4970SChristoph Hellwig struct nvmet_cq *cq; 351a07b4970SChristoph Hellwig struct nvmet_ns *ns; 352a07b4970SChristoph Hellwig struct scatterlist *sg; 353c6e3f133SIsrael Rukshin struct scatterlist *metadata_sg; 354a07b4970SChristoph Hellwig struct bio_vec inline_bvec[NVMET_MAX_INLINE_BIOVEC]; 355d5eff33eSChaitanya Kulkarni union { 356d5eff33eSChaitanya Kulkarni struct { 357d5eff33eSChaitanya Kulkarni struct bio inline_bio; 358d5eff33eSChaitanya Kulkarni } b; 359d5eff33eSChaitanya Kulkarni struct { 360d5eff33eSChaitanya Kulkarni bool mpool_alloc; 361d5eff33eSChaitanya Kulkarni struct kiocb iocb; 362d5eff33eSChaitanya Kulkarni struct bio_vec *bvec; 363d5eff33eSChaitanya Kulkarni struct work_struct work; 364d5eff33eSChaitanya Kulkarni } f; 365c1fef73fSLogan Gunthorpe struct { 366dab3902bSChaitanya Kulkarni struct bio inline_bio; 367c1fef73fSLogan Gunthorpe struct request *rq; 368c1fef73fSLogan Gunthorpe struct work_struct work; 369c1fef73fSLogan Gunthorpe bool use_workqueue; 370c1fef73fSLogan Gunthorpe } p; 371aaf2e048SChaitanya Kulkarni #ifdef CONFIG_BLK_DEV_ZONED 372aaf2e048SChaitanya Kulkarni struct { 373aaf2e048SChaitanya Kulkarni struct bio inline_bio; 374aaf2e048SChaitanya Kulkarni struct work_struct zmgmt_work; 375aaf2e048SChaitanya Kulkarni } z; 376aaf2e048SChaitanya Kulkarni #endif /* CONFIG_BLK_DEV_ZONED */ 377d5eff33eSChaitanya Kulkarni }; 378a07b4970SChristoph Hellwig int sg_cnt; 379c6e3f133SIsrael Rukshin int metadata_sg_cnt; 3805e62d5c9SChristoph Hellwig /* data length as parsed from the SGL descriptor: */ 3815e62d5c9SChristoph Hellwig size_t transfer_len; 382c6e3f133SIsrael Rukshin size_t metadata_len; 383a07b4970SChristoph Hellwig 384a07b4970SChristoph Hellwig struct nvmet_port *port; 385a07b4970SChristoph Hellwig 386a07b4970SChristoph Hellwig void (*execute)(struct nvmet_req *req); 387e929f06dSChristoph Hellwig const struct nvmet_fabrics_ops *ops; 388c6925093SLogan Gunthorpe 389c6925093SLogan Gunthorpe struct pci_dev *p2p_dev; 390c6925093SLogan Gunthorpe struct device *p2p_client; 391e4a97625SChaitanya Kulkarni u16 error_loc; 392e4a97625SChaitanya Kulkarni u64 error_slba; 393a07b4970SChristoph Hellwig }; 394a07b4970SChristoph Hellwig 39555eb942eSChaitanya Kulkarni extern struct workqueue_struct *buffered_io_wq; 396aaf2e048SChaitanya Kulkarni extern struct workqueue_struct *zbd_wq; 3978832cf92SSagi Grimberg extern struct workqueue_struct *nvmet_wq; 39855eb942eSChaitanya Kulkarni 399a07b4970SChristoph Hellwig static inline void nvmet_set_result(struct nvmet_req *req, u32 result) 400a07b4970SChristoph Hellwig { 401fc6c9730SMax Gurtovoy req->cqe->result.u32 = cpu_to_le32(result); 402a07b4970SChristoph Hellwig } 403a07b4970SChristoph Hellwig 404a07b4970SChristoph Hellwig /* 405a07b4970SChristoph Hellwig * NVMe command writes actually are DMA reads for us on the target side. 406a07b4970SChristoph Hellwig */ 407a07b4970SChristoph Hellwig static inline enum dma_data_direction 408a07b4970SChristoph Hellwig nvmet_data_dir(struct nvmet_req *req) 409a07b4970SChristoph Hellwig { 410a07b4970SChristoph Hellwig return nvme_is_write(req->cmd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 411a07b4970SChristoph Hellwig } 412a07b4970SChristoph Hellwig 413a07b4970SChristoph Hellwig struct nvmet_async_event { 414a07b4970SChristoph Hellwig struct list_head entry; 415a07b4970SChristoph Hellwig u8 event_type; 416a07b4970SChristoph Hellwig u8 event_info; 417a07b4970SChristoph Hellwig u8 log_page; 418a07b4970SChristoph Hellwig }; 419a07b4970SChristoph Hellwig 4207114ddebSJay Sternberg static inline void nvmet_clear_aen_bit(struct nvmet_req *req, u32 bn) 4216c8312adSJay Sternberg { 422b7c8f366SChaitanya Kulkarni int rae = le32_to_cpu(req->cmd->common.cdw10) & 1 << 15; 4236c8312adSJay Sternberg 4246c8312adSJay Sternberg if (!rae) 4257114ddebSJay Sternberg clear_bit(bn, &req->sq->ctrl->aen_masked); 4266c8312adSJay Sternberg } 4276c8312adSJay Sternberg 4287114ddebSJay Sternberg static inline bool nvmet_aen_bit_disabled(struct nvmet_ctrl *ctrl, u32 bn) 4296c8312adSJay Sternberg { 4307114ddebSJay Sternberg if (!(READ_ONCE(ctrl->aen_enabled) & (1 << bn))) 4316c8312adSJay Sternberg return true; 4327114ddebSJay Sternberg return test_and_set_bit(bn, &ctrl->aen_masked); 4336c8312adSJay Sternberg } 4346c8312adSJay Sternberg 43590107455SJay Sternberg void nvmet_get_feat_kato(struct nvmet_req *req); 43690107455SJay Sternberg void nvmet_get_feat_async_event(struct nvmet_req *req); 43790107455SJay Sternberg u16 nvmet_set_feat_kato(struct nvmet_req *req); 43890107455SJay Sternberg u16 nvmet_set_feat_async_event(struct nvmet_req *req, u32 mask); 43990107455SJay Sternberg void nvmet_execute_async_event(struct nvmet_req *req); 4404e683c48SAmit Engel void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl); 4414e683c48SAmit Engel void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl); 44290107455SJay Sternberg 44364a0ca88SParav Pandit u16 nvmet_parse_connect_cmd(struct nvmet_req *req); 4449d05a96eSBart Van Assche void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id); 445d5eff33eSChaitanya Kulkarni u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req); 446d5eff33eSChaitanya Kulkarni u16 nvmet_file_parse_io_cmd(struct nvmet_req *req); 447aaf2e048SChaitanya Kulkarni u16 nvmet_bdev_zns_parse_io_cmd(struct nvmet_req *req); 44864a0ca88SParav Pandit u16 nvmet_parse_admin_cmd(struct nvmet_req *req); 44964a0ca88SParav Pandit u16 nvmet_parse_discovery_cmd(struct nvmet_req *req); 4506490c9edSHannes Reinecke u16 nvmet_parse_fabrics_admin_cmd(struct nvmet_req *req); 4516490c9edSHannes Reinecke u16 nvmet_parse_fabrics_io_cmd(struct nvmet_req *req); 452a07b4970SChristoph Hellwig 453a07b4970SChristoph Hellwig bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, 454e929f06dSChristoph Hellwig struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops); 455549f01aeSVijay Immanuel void nvmet_req_uninit(struct nvmet_req *req); 456136cc1ffSIsrael Rukshin bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); 457b716e688SSagi Grimberg bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); 458a07b4970SChristoph Hellwig void nvmet_req_complete(struct nvmet_req *req, u16 status); 459c6e3f133SIsrael Rukshin int nvmet_req_alloc_sgls(struct nvmet_req *req); 460c6e3f133SIsrael Rukshin void nvmet_req_free_sgls(struct nvmet_req *req); 461a07b4970SChristoph Hellwig 462c1fef73fSLogan Gunthorpe void nvmet_execute_set_features(struct nvmet_req *req); 463c1fef73fSLogan Gunthorpe void nvmet_execute_get_features(struct nvmet_req *req); 464f9362ac1SJay Sternberg void nvmet_execute_keep_alive(struct nvmet_req *req); 465f9362ac1SJay Sternberg 466a07b4970SChristoph Hellwig void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, u16 qid, 467a07b4970SChristoph Hellwig u16 size); 468a07b4970SChristoph Hellwig void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, u16 qid, 469a07b4970SChristoph Hellwig u16 size); 470a07b4970SChristoph Hellwig void nvmet_sq_destroy(struct nvmet_sq *sq); 471a07b4970SChristoph Hellwig int nvmet_sq_init(struct nvmet_sq *sq); 472a07b4970SChristoph Hellwig 473a07b4970SChristoph Hellwig void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl); 474a07b4970SChristoph Hellwig 475a07b4970SChristoph Hellwig void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new); 476a07b4970SChristoph Hellwig u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, 477a07b4970SChristoph Hellwig struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp); 478de587804SChaitanya Kulkarni struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, 479de587804SChaitanya Kulkarni const char *hostnqn, u16 cntlid, 480de587804SChaitanya Kulkarni struct nvmet_req *req); 481a07b4970SChristoph Hellwig void nvmet_ctrl_put(struct nvmet_ctrl *ctrl); 4827798df6fSChaitanya Kulkarni u16 nvmet_check_ctrl_status(struct nvmet_req *req); 483a07b4970SChristoph Hellwig 484a07b4970SChristoph Hellwig struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, 485a07b4970SChristoph Hellwig enum nvme_subsys_type type); 486a07b4970SChristoph Hellwig void nvmet_subsys_put(struct nvmet_subsys *subsys); 487344770b0SSagi Grimberg void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys); 488a07b4970SChristoph Hellwig 4893a1f7c79SChaitanya Kulkarni u16 nvmet_req_find_ns(struct nvmet_req *req); 490a07b4970SChristoph Hellwig void nvmet_put_namespace(struct nvmet_ns *ns); 491a07b4970SChristoph Hellwig int nvmet_ns_enable(struct nvmet_ns *ns); 492a07b4970SChristoph Hellwig void nvmet_ns_disable(struct nvmet_ns *ns); 493a07b4970SChristoph Hellwig struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid); 494a07b4970SChristoph Hellwig void nvmet_ns_free(struct nvmet_ns *ns); 495a07b4970SChristoph Hellwig 49662ac0d32SChristoph Hellwig void nvmet_send_ana_event(struct nvmet_subsys *subsys, 49762ac0d32SChristoph Hellwig struct nvmet_port *port); 49862ac0d32SChristoph Hellwig void nvmet_port_send_ana_event(struct nvmet_port *port); 49962ac0d32SChristoph Hellwig 500e929f06dSChristoph Hellwig int nvmet_register_transport(const struct nvmet_fabrics_ops *ops); 501e929f06dSChristoph Hellwig void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops); 502a07b4970SChristoph Hellwig 5033aed8673SLogan Gunthorpe void nvmet_port_del_ctrls(struct nvmet_port *port, 5043aed8673SLogan Gunthorpe struct nvmet_subsys *subsys); 5053aed8673SLogan Gunthorpe 506a07b4970SChristoph Hellwig int nvmet_enable_port(struct nvmet_port *port); 507a07b4970SChristoph Hellwig void nvmet_disable_port(struct nvmet_port *port); 508a07b4970SChristoph Hellwig 509a07b4970SChristoph Hellwig void nvmet_referral_enable(struct nvmet_port *parent, struct nvmet_port *port); 510b662a078SJay Sternberg void nvmet_referral_disable(struct nvmet_port *parent, struct nvmet_port *port); 511a07b4970SChristoph Hellwig 512a07b4970SChristoph Hellwig u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf, 513a07b4970SChristoph Hellwig size_t len); 514a07b4970SChristoph Hellwig u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, 515a07b4970SChristoph Hellwig size_t len); 516c7759fffSChristoph Hellwig u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len); 517a07b4970SChristoph Hellwig 518a07b4970SChristoph Hellwig u32 nvmet_get_log_page_len(struct nvme_command *cmd); 519d808b7f7SKeith Busch u64 nvmet_get_log_page_offset(struct nvme_command *cmd); 520a07b4970SChristoph Hellwig 521b662a078SJay Sternberg extern struct list_head *nvmet_ports; 522b662a078SJay Sternberg void nvmet_port_disc_changed(struct nvmet_port *port, 523b662a078SJay Sternberg struct nvmet_subsys *subsys); 524b662a078SJay Sternberg void nvmet_subsys_disc_changed(struct nvmet_subsys *subsys, 525b662a078SJay Sternberg struct nvmet_host *host); 526b662a078SJay Sternberg void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, 527b662a078SJay Sternberg u8 event_info, u8 log_page); 528b662a078SJay Sternberg 529a07b4970SChristoph Hellwig #define NVMET_QUEUE_SIZE 1024 5303375e29cSJames Smart #define NVMET_NR_QUEUES 128 531a07b4970SChristoph Hellwig #define NVMET_MAX_CMD NVMET_QUEUE_SIZE 532793c7cfcSChristoph Hellwig 533793c7cfcSChristoph Hellwig /* 534793c7cfcSChristoph Hellwig * Nice round number that makes a list of nsids fit into a page. 535793c7cfcSChristoph Hellwig * Should become tunable at some point in the future. 536793c7cfcSChristoph Hellwig */ 537793c7cfcSChristoph Hellwig #define NVMET_MAX_NAMESPACES 1024 538793c7cfcSChristoph Hellwig 53972efd25dSChristoph Hellwig /* 54072efd25dSChristoph Hellwig * 0 is not a valid ANA group ID, so we start numbering at 1. 54172efd25dSChristoph Hellwig * 54272efd25dSChristoph Hellwig * ANA Group 1 exists without manual intervention, has namespaces assigned to it 54372efd25dSChristoph Hellwig * by default, and is available in an optimized state through all ports. 54472efd25dSChristoph Hellwig */ 54562ac0d32SChristoph Hellwig #define NVMET_MAX_ANAGRPS 128 54672efd25dSChristoph Hellwig #define NVMET_DEFAULT_ANA_GRPID 1 54772efd25dSChristoph Hellwig 548a07b4970SChristoph Hellwig #define NVMET_KAS 10 549f9362ac1SJay Sternberg #define NVMET_DISC_KATO_MS 120000 550a07b4970SChristoph Hellwig 551a07b4970SChristoph Hellwig int __init nvmet_init_configfs(void); 552a07b4970SChristoph Hellwig void __exit nvmet_exit_configfs(void); 553a07b4970SChristoph Hellwig 554a07b4970SChristoph Hellwig int __init nvmet_init_discovery(void); 555a07b4970SChristoph Hellwig void nvmet_exit_discovery(void); 556a07b4970SChristoph Hellwig 557a07b4970SChristoph Hellwig extern struct nvmet_subsys *nvmet_disc_subsys; 558a07b4970SChristoph Hellwig extern struct rw_semaphore nvmet_config_sem; 559a07b4970SChristoph Hellwig 56072efd25dSChristoph Hellwig extern u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1]; 56172efd25dSChristoph Hellwig extern u64 nvmet_ana_chgcnt; 56272efd25dSChristoph Hellwig extern struct rw_semaphore nvmet_ana_sem; 56372efd25dSChristoph Hellwig 564253928eeSSagi Grimberg bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn); 565a07b4970SChristoph Hellwig 566d5eff33eSChaitanya Kulkarni int nvmet_bdev_ns_enable(struct nvmet_ns *ns); 567d5eff33eSChaitanya Kulkarni int nvmet_file_ns_enable(struct nvmet_ns *ns); 568d5eff33eSChaitanya Kulkarni void nvmet_bdev_ns_disable(struct nvmet_ns *ns); 569d5eff33eSChaitanya Kulkarni void nvmet_file_ns_disable(struct nvmet_ns *ns); 570dedf0be5SChaitanya Kulkarni u16 nvmet_bdev_flush(struct nvmet_req *req); 571dedf0be5SChaitanya Kulkarni u16 nvmet_file_flush(struct nvmet_req *req); 572dedf0be5SChaitanya Kulkarni void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid); 573e8cd1ff1SAnthony Iliopoulos void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns); 5742caecd62SChaitanya Kulkarni void nvmet_file_ns_revalidate(struct nvmet_ns *ns); 575da783733SChristoph Hellwig bool nvmet_ns_revalidate(struct nvmet_ns *ns); 576aaf2e048SChaitanya Kulkarni u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts); 577aaf2e048SChaitanya Kulkarni 578aaf2e048SChaitanya Kulkarni bool nvmet_bdev_zns_enable(struct nvmet_ns *ns); 579aaf2e048SChaitanya Kulkarni void nvmet_execute_identify_cns_cs_ctrl(struct nvmet_req *req); 580aaf2e048SChaitanya Kulkarni void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req); 581aaf2e048SChaitanya Kulkarni void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req); 582aaf2e048SChaitanya Kulkarni void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req); 583aaf2e048SChaitanya Kulkarni void nvmet_bdev_execute_zone_append(struct nvmet_req *req); 584d5eff33eSChaitanya Kulkarni 58526af180cSIsrael Rukshin static inline u32 nvmet_rw_data_len(struct nvmet_req *req) 586d5eff33eSChaitanya Kulkarni { 587d5eff33eSChaitanya Kulkarni return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) << 588d5eff33eSChaitanya Kulkarni req->ns->blksize_shift; 589d5eff33eSChaitanya Kulkarni } 590c6aa3542SChaitanya Kulkarni 591ea52ac1cSIsrael Rukshin static inline u32 nvmet_rw_metadata_len(struct nvmet_req *req) 592ea52ac1cSIsrael Rukshin { 593ea52ac1cSIsrael Rukshin if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 594ea52ac1cSIsrael Rukshin return 0; 595ea52ac1cSIsrael Rukshin return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) * 596ea52ac1cSIsrael Rukshin req->ns->metadata_size; 597ea52ac1cSIsrael Rukshin } 598ea52ac1cSIsrael Rukshin 59959ef0eaaSChristoph Hellwig static inline u32 nvmet_dsm_len(struct nvmet_req *req) 60059ef0eaaSChristoph Hellwig { 60159ef0eaaSChristoph Hellwig return (le32_to_cpu(req->cmd->dsm.nr) + 1) * 60259ef0eaaSChristoph Hellwig sizeof(struct nvme_dsm_range); 60359ef0eaaSChristoph Hellwig } 60459ef0eaaSChristoph Hellwig 60520c2c3bbSChaitanya Kulkarni static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) 60620c2c3bbSChaitanya Kulkarni { 60720c2c3bbSChaitanya Kulkarni return req->sq->ctrl->subsys; 60820c2c3bbSChaitanya Kulkarni } 60920c2c3bbSChaitanya Kulkarni 610a294711eSHannes Reinecke static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) 611a294711eSHannes Reinecke { 612598e7593SHannes Reinecke return subsys->type != NVME_NQN_NVME; 613a294711eSHannes Reinecke } 614a294711eSHannes Reinecke 615c1fef73fSLogan Gunthorpe #ifdef CONFIG_NVME_TARGET_PASSTHRU 616ba76af67SLogan Gunthorpe void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys); 617ba76af67SLogan Gunthorpe int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys); 618ba76af67SLogan Gunthorpe void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys); 619c1fef73fSLogan Gunthorpe u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req); 620c1fef73fSLogan Gunthorpe u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req); 621ab7a2737SChristoph Hellwig static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys) 622c1fef73fSLogan Gunthorpe { 623c1fef73fSLogan Gunthorpe return subsys->passthru_ctrl; 624c1fef73fSLogan Gunthorpe } 625c1fef73fSLogan Gunthorpe #else /* CONFIG_NVME_TARGET_PASSTHRU */ 626ba76af67SLogan Gunthorpe static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys) 627ba76af67SLogan Gunthorpe { 628ba76af67SLogan Gunthorpe } 629ba76af67SLogan Gunthorpe static inline void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys) 630ba76af67SLogan Gunthorpe { 631ba76af67SLogan Gunthorpe } 632c1fef73fSLogan Gunthorpe static inline u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req) 633c1fef73fSLogan Gunthorpe { 634c1fef73fSLogan Gunthorpe return 0; 635c1fef73fSLogan Gunthorpe } 636c1fef73fSLogan Gunthorpe static inline u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req) 637c1fef73fSLogan Gunthorpe { 638c1fef73fSLogan Gunthorpe return 0; 639c1fef73fSLogan Gunthorpe } 640ab7a2737SChristoph Hellwig static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys) 641c1fef73fSLogan Gunthorpe { 642c1fef73fSLogan Gunthorpe return NULL; 643c1fef73fSLogan Gunthorpe } 644c1fef73fSLogan Gunthorpe #endif /* CONFIG_NVME_TARGET_PASSTHRU */ 645c1fef73fSLogan Gunthorpe 646ab7a2737SChristoph Hellwig static inline bool nvmet_is_passthru_req(struct nvmet_req *req) 647c1fef73fSLogan Gunthorpe { 648ab7a2737SChristoph Hellwig return nvmet_is_passthru_subsys(nvmet_req_subsys(req)); 649c1fef73fSLogan Gunthorpe } 650c1fef73fSLogan Gunthorpe 65177d651a6SAdam Manzanares void nvmet_passthrough_override_cap(struct nvmet_ctrl *ctrl); 65277d651a6SAdam Manzanares 653c6aa3542SChaitanya Kulkarni u16 errno_to_nvme_status(struct nvmet_req *req, int errno); 654d81d57cfSChaitanya Kulkarni u16 nvmet_report_invalid_opcode(struct nvmet_req *req); 6559d05a96eSBart Van Assche 6569d05a96eSBart Van Assche /* Convert a 32-bit number to a 16-bit 0's based number */ 6579d05a96eSBart Van Assche static inline __le16 to0based(u32 a) 6589d05a96eSBart Van Assche { 6599d05a96eSBart Van Assche return cpu_to_le16(max(1U, min(1U << 16, a)) - 1); 6609d05a96eSBart Van Assche } 6619d05a96eSBart Van Assche 662ea52ac1cSIsrael Rukshin static inline bool nvmet_ns_has_pi(struct nvmet_ns *ns) 663ea52ac1cSIsrael Rukshin { 664ea52ac1cSIsrael Rukshin if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 665ea52ac1cSIsrael Rukshin return false; 666ea52ac1cSIsrael Rukshin return ns->pi_type && ns->metadata_size == sizeof(struct t10_pi_tuple); 667ea52ac1cSIsrael Rukshin } 668ea52ac1cSIsrael Rukshin 669193fcf37SChaitanya Kulkarni static inline __le64 nvmet_sect_to_lba(struct nvmet_ns *ns, sector_t sect) 670193fcf37SChaitanya Kulkarni { 671193fcf37SChaitanya Kulkarni return cpu_to_le64(sect >> (ns->blksize_shift - SECTOR_SHIFT)); 672193fcf37SChaitanya Kulkarni } 673193fcf37SChaitanya Kulkarni 674193fcf37SChaitanya Kulkarni static inline sector_t nvmet_lba_to_sect(struct nvmet_ns *ns, __le64 lba) 675193fcf37SChaitanya Kulkarni { 676193fcf37SChaitanya Kulkarni return le64_to_cpu(lba) << (ns->blksize_shift - SECTOR_SHIFT); 677193fcf37SChaitanya Kulkarni } 678193fcf37SChaitanya Kulkarni 679608a9690SChaitanya Kulkarni static inline bool nvmet_use_inline_bvec(struct nvmet_req *req) 680608a9690SChaitanya Kulkarni { 681608a9690SChaitanya Kulkarni return req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN && 682608a9690SChaitanya Kulkarni req->sg_cnt <= NVMET_MAX_INLINE_BIOVEC; 683608a9690SChaitanya Kulkarni } 684608a9690SChaitanya Kulkarni 6856e597263SChaitanya Kulkarni static inline void nvmet_req_cns_error_complete(struct nvmet_req *req) 6866e597263SChaitanya Kulkarni { 6876e597263SChaitanya Kulkarni pr_debug("unhandled identify cns %d on qid %d\n", 6886e597263SChaitanya Kulkarni req->cmd->identify.cns, req->sq->qid); 6896e597263SChaitanya Kulkarni req->error_loc = offsetof(struct nvme_identify, cns); 6906e597263SChaitanya Kulkarni nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR); 6916e597263SChaitanya Kulkarni } 6926e597263SChaitanya Kulkarni 6939a01b58cSChaitanya Kulkarni static inline void nvmet_req_bio_put(struct nvmet_req *req, struct bio *bio) 6949a01b58cSChaitanya Kulkarni { 6959a01b58cSChaitanya Kulkarni if (bio != &req->b.inline_bio) 6969a01b58cSChaitanya Kulkarni bio_put(bio); 6979a01b58cSChaitanya Kulkarni } 6989a01b58cSChaitanya Kulkarni 699db1312ddSHannes Reinecke #ifdef CONFIG_NVME_TARGET_AUTH 700db1312ddSHannes Reinecke void nvmet_execute_auth_send(struct nvmet_req *req); 701db1312ddSHannes Reinecke void nvmet_execute_auth_receive(struct nvmet_req *req); 702db1312ddSHannes Reinecke int nvmet_auth_set_key(struct nvmet_host *host, const char *secret, 703db1312ddSHannes Reinecke bool set_ctrl); 704db1312ddSHannes Reinecke int nvmet_auth_set_host_hash(struct nvmet_host *host, const char *hash); 705db1312ddSHannes Reinecke int nvmet_setup_auth(struct nvmet_ctrl *ctrl); 706db1312ddSHannes Reinecke void nvmet_init_auth(struct nvmet_ctrl *ctrl, struct nvmet_req *req); 707db1312ddSHannes Reinecke void nvmet_destroy_auth(struct nvmet_ctrl *ctrl); 708db1312ddSHannes Reinecke void nvmet_auth_sq_free(struct nvmet_sq *sq); 709*7a277c37SHannes Reinecke int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id); 710db1312ddSHannes Reinecke bool nvmet_check_auth_status(struct nvmet_req *req); 711db1312ddSHannes Reinecke int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, 712db1312ddSHannes Reinecke unsigned int hash_len); 713db1312ddSHannes Reinecke int nvmet_auth_ctrl_hash(struct nvmet_req *req, u8 *response, 714db1312ddSHannes Reinecke unsigned int hash_len); 715db1312ddSHannes Reinecke static inline bool nvmet_has_auth(struct nvmet_ctrl *ctrl) 716db1312ddSHannes Reinecke { 717db1312ddSHannes Reinecke return ctrl->host_key != NULL; 718db1312ddSHannes Reinecke } 719*7a277c37SHannes Reinecke int nvmet_auth_ctrl_exponential(struct nvmet_req *req, 720*7a277c37SHannes Reinecke u8 *buf, int buf_size); 721*7a277c37SHannes Reinecke int nvmet_auth_ctrl_sesskey(struct nvmet_req *req, 722*7a277c37SHannes Reinecke u8 *buf, int buf_size); 723db1312ddSHannes Reinecke #else 724db1312ddSHannes Reinecke static inline int nvmet_setup_auth(struct nvmet_ctrl *ctrl) 725db1312ddSHannes Reinecke { 726db1312ddSHannes Reinecke return 0; 727db1312ddSHannes Reinecke } 728db1312ddSHannes Reinecke static inline void nvmet_init_auth(struct nvmet_ctrl *ctrl, 729db1312ddSHannes Reinecke struct nvmet_req *req) {}; 730db1312ddSHannes Reinecke static inline void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) {}; 731db1312ddSHannes Reinecke static inline void nvmet_auth_sq_free(struct nvmet_sq *sq) {}; 732db1312ddSHannes Reinecke static inline bool nvmet_check_auth_status(struct nvmet_req *req) 733db1312ddSHannes Reinecke { 734db1312ddSHannes Reinecke return true; 735db1312ddSHannes Reinecke } 736db1312ddSHannes Reinecke static inline bool nvmet_has_auth(struct nvmet_ctrl *ctrl) 737db1312ddSHannes Reinecke { 738db1312ddSHannes Reinecke return false; 739db1312ddSHannes Reinecke } 740db1312ddSHannes Reinecke static inline const char *nvmet_dhchap_dhgroup_name(u8 dhgid) { return NULL; } 741db1312ddSHannes Reinecke #endif 742db1312ddSHannes Reinecke 743a07b4970SChristoph Hellwig #endif /* _NVMET_H */ 744