1b668f2f5SMike Christie // SPDX-License-Identifier: GPL-2.0 2b668f2f5SMike Christie /* 3b668f2f5SMike Christie * Copyright (c) 2015 Intel Corporation 4b668f2f5SMike Christie * Keith Busch <kbusch@kernel.org> 5b668f2f5SMike Christie */ 6b668f2f5SMike Christie #include <linux/blkdev.h> 7b668f2f5SMike Christie #include <linux/pr.h> 8b668f2f5SMike Christie #include <asm/unaligned.h> 9b668f2f5SMike Christie 10b668f2f5SMike Christie #include "nvme.h" 11b668f2f5SMike Christie 12be1a7cd2SMike Christie static enum nvme_pr_type nvme_pr_type_from_blk(enum pr_type type) 13b668f2f5SMike Christie { 14b668f2f5SMike Christie switch (type) { 15b668f2f5SMike Christie case PR_WRITE_EXCLUSIVE: 16be1a7cd2SMike Christie return NVME_PR_WRITE_EXCLUSIVE; 17b668f2f5SMike Christie case PR_EXCLUSIVE_ACCESS: 18be1a7cd2SMike Christie return NVME_PR_EXCLUSIVE_ACCESS; 19b668f2f5SMike Christie case PR_WRITE_EXCLUSIVE_REG_ONLY: 20be1a7cd2SMike Christie return NVME_PR_WRITE_EXCLUSIVE_REG_ONLY; 21b668f2f5SMike Christie case PR_EXCLUSIVE_ACCESS_REG_ONLY: 22be1a7cd2SMike Christie return NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY; 23b668f2f5SMike Christie case PR_WRITE_EXCLUSIVE_ALL_REGS: 24be1a7cd2SMike Christie return NVME_PR_WRITE_EXCLUSIVE_ALL_REGS; 25b668f2f5SMike Christie case PR_EXCLUSIVE_ACCESS_ALL_REGS: 26be1a7cd2SMike Christie return NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS; 27b668f2f5SMike Christie } 28be1a7cd2SMike Christie 29be1a7cd2SMike Christie return 0; 30b668f2f5SMike Christie } 31b668f2f5SMike Christie 32*28c97ba3SMike Christie static enum pr_type block_pr_type_from_nvme(enum nvme_pr_type type) 33*28c97ba3SMike Christie { 34*28c97ba3SMike Christie switch (type) { 35*28c97ba3SMike Christie case NVME_PR_WRITE_EXCLUSIVE: 36*28c97ba3SMike Christie return PR_WRITE_EXCLUSIVE; 37*28c97ba3SMike Christie case NVME_PR_EXCLUSIVE_ACCESS: 38*28c97ba3SMike Christie return PR_EXCLUSIVE_ACCESS; 39*28c97ba3SMike Christie case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY: 40*28c97ba3SMike Christie return PR_WRITE_EXCLUSIVE_REG_ONLY; 41*28c97ba3SMike Christie case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY: 42*28c97ba3SMike Christie return PR_EXCLUSIVE_ACCESS_REG_ONLY; 43*28c97ba3SMike Christie case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS: 44*28c97ba3SMike Christie return PR_WRITE_EXCLUSIVE_ALL_REGS; 45*28c97ba3SMike Christie case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS: 46*28c97ba3SMike Christie return PR_EXCLUSIVE_ACCESS_ALL_REGS; 47*28c97ba3SMike Christie } 48*28c97ba3SMike Christie 49*28c97ba3SMike Christie return 0; 50*28c97ba3SMike Christie } 51*28c97ba3SMike Christie 52b668f2f5SMike Christie static int nvme_send_ns_head_pr_command(struct block_device *bdev, 53f0614790SMike Christie struct nvme_command *c, void *data, unsigned int data_len) 54b668f2f5SMike Christie { 55b668f2f5SMike Christie struct nvme_ns_head *head = bdev->bd_disk->private_data; 56b668f2f5SMike Christie int srcu_idx = srcu_read_lock(&head->srcu); 57b668f2f5SMike Christie struct nvme_ns *ns = nvme_find_path(head); 58b668f2f5SMike Christie int ret = -EWOULDBLOCK; 59b668f2f5SMike Christie 60b668f2f5SMike Christie if (ns) { 61b668f2f5SMike Christie c->common.nsid = cpu_to_le32(ns->head->ns_id); 62b668f2f5SMike Christie ret = nvme_submit_sync_cmd(ns->queue, c, data, data_len); 63b668f2f5SMike Christie } 64b668f2f5SMike Christie srcu_read_unlock(&head->srcu, srcu_idx); 65b668f2f5SMike Christie return ret; 66b668f2f5SMike Christie } 67b668f2f5SMike Christie 68b668f2f5SMike Christie static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c, 69f0614790SMike Christie void *data, unsigned int data_len) 70b668f2f5SMike Christie { 71b668f2f5SMike Christie c->common.nsid = cpu_to_le32(ns->head->ns_id); 72b668f2f5SMike Christie return nvme_submit_sync_cmd(ns->queue, c, data, data_len); 73b668f2f5SMike Christie } 74b668f2f5SMike Christie 75b668f2f5SMike Christie static int nvme_sc_to_pr_err(int nvme_sc) 76b668f2f5SMike Christie { 77b668f2f5SMike Christie if (nvme_is_path_error(nvme_sc)) 78b668f2f5SMike Christie return PR_STS_PATH_FAILED; 79b668f2f5SMike Christie 80b668f2f5SMike Christie switch (nvme_sc) { 81b668f2f5SMike Christie case NVME_SC_SUCCESS: 82b668f2f5SMike Christie return PR_STS_SUCCESS; 83b668f2f5SMike Christie case NVME_SC_RESERVATION_CONFLICT: 84b668f2f5SMike Christie return PR_STS_RESERVATION_CONFLICT; 85b668f2f5SMike Christie case NVME_SC_ONCS_NOT_SUPPORTED: 86b668f2f5SMike Christie return -EOPNOTSUPP; 87b668f2f5SMike Christie case NVME_SC_BAD_ATTRIBUTES: 88b668f2f5SMike Christie case NVME_SC_INVALID_OPCODE: 89b668f2f5SMike Christie case NVME_SC_INVALID_FIELD: 90b668f2f5SMike Christie case NVME_SC_INVALID_NS: 91b668f2f5SMike Christie return -EINVAL; 92b668f2f5SMike Christie default: 93b668f2f5SMike Christie return PR_STS_IOERR; 94b668f2f5SMike Christie } 95b668f2f5SMike Christie } 96b668f2f5SMike Christie 97f0614790SMike Christie static int nvme_send_pr_command(struct block_device *bdev, 98f0614790SMike Christie struct nvme_command *c, void *data, unsigned int data_len) 99f0614790SMike Christie { 100f0614790SMike Christie if (IS_ENABLED(CONFIG_NVME_MULTIPATH) && 101f0614790SMike Christie bdev->bd_disk->fops == &nvme_ns_head_ops) 102f0614790SMike Christie return nvme_send_ns_head_pr_command(bdev, c, data, data_len); 103f0614790SMike Christie 104f0614790SMike Christie return nvme_send_ns_pr_command(bdev->bd_disk->private_data, c, data, 105f0614790SMike Christie data_len); 106f0614790SMike Christie } 107f0614790SMike Christie 108b668f2f5SMike Christie static int nvme_pr_command(struct block_device *bdev, u32 cdw10, 109b668f2f5SMike Christie u64 key, u64 sa_key, u8 op) 110b668f2f5SMike Christie { 111b668f2f5SMike Christie struct nvme_command c = { }; 112b668f2f5SMike Christie u8 data[16] = { 0, }; 113b668f2f5SMike Christie int ret; 114b668f2f5SMike Christie 115b668f2f5SMike Christie put_unaligned_le64(key, &data[0]); 116b668f2f5SMike Christie put_unaligned_le64(sa_key, &data[8]); 117b668f2f5SMike Christie 118b668f2f5SMike Christie c.common.opcode = op; 119b668f2f5SMike Christie c.common.cdw10 = cpu_to_le32(cdw10); 120b668f2f5SMike Christie 121f0614790SMike Christie ret = nvme_send_pr_command(bdev, &c, data, sizeof(data)); 122b668f2f5SMike Christie if (ret < 0) 123b668f2f5SMike Christie return ret; 124b668f2f5SMike Christie 125b668f2f5SMike Christie return nvme_sc_to_pr_err(ret); 126b668f2f5SMike Christie } 127b668f2f5SMike Christie 128b668f2f5SMike Christie static int nvme_pr_register(struct block_device *bdev, u64 old, 129b668f2f5SMike Christie u64 new, unsigned flags) 130b668f2f5SMike Christie { 131b668f2f5SMike Christie u32 cdw10; 132b668f2f5SMike Christie 133b668f2f5SMike Christie if (flags & ~PR_FL_IGNORE_KEY) 134b668f2f5SMike Christie return -EOPNOTSUPP; 135b668f2f5SMike Christie 136b668f2f5SMike Christie cdw10 = old ? 2 : 0; 137b668f2f5SMike Christie cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0; 138b668f2f5SMike Christie cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */ 139b668f2f5SMike Christie return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register); 140b668f2f5SMike Christie } 141b668f2f5SMike Christie 142b668f2f5SMike Christie static int nvme_pr_reserve(struct block_device *bdev, u64 key, 143b668f2f5SMike Christie enum pr_type type, unsigned flags) 144b668f2f5SMike Christie { 145b668f2f5SMike Christie u32 cdw10; 146b668f2f5SMike Christie 147b668f2f5SMike Christie if (flags & ~PR_FL_IGNORE_KEY) 148b668f2f5SMike Christie return -EOPNOTSUPP; 149b668f2f5SMike Christie 150be1a7cd2SMike Christie cdw10 = nvme_pr_type_from_blk(type) << 8; 151b668f2f5SMike Christie cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0); 152b668f2f5SMike Christie return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire); 153b668f2f5SMike Christie } 154b668f2f5SMike Christie 155b668f2f5SMike Christie static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new, 156b668f2f5SMike Christie enum pr_type type, bool abort) 157b668f2f5SMike Christie { 158be1a7cd2SMike Christie u32 cdw10 = nvme_pr_type_from_blk(type) << 8 | (abort ? 2 : 1); 159b668f2f5SMike Christie 160b668f2f5SMike Christie return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire); 161b668f2f5SMike Christie } 162b668f2f5SMike Christie 163b668f2f5SMike Christie static int nvme_pr_clear(struct block_device *bdev, u64 key) 164b668f2f5SMike Christie { 165b668f2f5SMike Christie u32 cdw10 = 1 | (key ? 0 : 1 << 3); 166b668f2f5SMike Christie 167b668f2f5SMike Christie return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); 168b668f2f5SMike Christie } 169b668f2f5SMike Christie 170b668f2f5SMike Christie static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 171b668f2f5SMike Christie { 172be1a7cd2SMike Christie u32 cdw10 = nvme_pr_type_from_blk(type) << 8 | (key ? 0 : 1 << 3); 173b668f2f5SMike Christie 174b668f2f5SMike Christie return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); 175b668f2f5SMike Christie } 176b668f2f5SMike Christie 1775fd96a4eSMike Christie static int nvme_pr_resv_report(struct block_device *bdev, void *data, 1785fd96a4eSMike Christie u32 data_len, bool *eds) 1795fd96a4eSMike Christie { 1805fd96a4eSMike Christie struct nvme_command c = { }; 1815fd96a4eSMike Christie int ret; 1825fd96a4eSMike Christie 1835fd96a4eSMike Christie c.common.opcode = nvme_cmd_resv_report; 1845fd96a4eSMike Christie c.common.cdw10 = cpu_to_le32(nvme_bytes_to_numd(data_len)); 1855fd96a4eSMike Christie c.common.cdw11 = cpu_to_le32(NVME_EXTENDED_DATA_STRUCT); 1865fd96a4eSMike Christie *eds = true; 1875fd96a4eSMike Christie 1885fd96a4eSMike Christie retry: 1895fd96a4eSMike Christie ret = nvme_send_pr_command(bdev, &c, data, data_len); 1905fd96a4eSMike Christie if (ret == NVME_SC_HOST_ID_INCONSIST && 1915fd96a4eSMike Christie c.common.cdw11 == cpu_to_le32(NVME_EXTENDED_DATA_STRUCT)) { 1925fd96a4eSMike Christie c.common.cdw11 = 0; 1935fd96a4eSMike Christie *eds = false; 1945fd96a4eSMike Christie goto retry; 1955fd96a4eSMike Christie } 1965fd96a4eSMike Christie 1975fd96a4eSMike Christie if (ret < 0) 1985fd96a4eSMike Christie return ret; 1995fd96a4eSMike Christie 2005fd96a4eSMike Christie return nvme_sc_to_pr_err(ret); 2015fd96a4eSMike Christie } 2025fd96a4eSMike Christie 2035fd96a4eSMike Christie static int nvme_pr_read_keys(struct block_device *bdev, 2045fd96a4eSMike Christie struct pr_keys *keys_info) 2055fd96a4eSMike Christie { 2065fd96a4eSMike Christie u32 rse_len, num_keys = keys_info->num_keys; 2075fd96a4eSMike Christie struct nvme_reservation_status_ext *rse; 2085fd96a4eSMike Christie int ret, i; 2095fd96a4eSMike Christie bool eds; 2105fd96a4eSMike Christie 2115fd96a4eSMike Christie /* 2125fd96a4eSMike Christie * Assume we are using 128-bit host IDs and allocate a buffer large 2135fd96a4eSMike Christie * enough to get enough keys to fill the return keys buffer. 2145fd96a4eSMike Christie */ 2155fd96a4eSMike Christie rse_len = struct_size(rse, regctl_eds, num_keys); 2165fd96a4eSMike Christie rse = kzalloc(rse_len, GFP_KERNEL); 2175fd96a4eSMike Christie if (!rse) 2185fd96a4eSMike Christie return -ENOMEM; 2195fd96a4eSMike Christie 2205fd96a4eSMike Christie ret = nvme_pr_resv_report(bdev, rse, rse_len, &eds); 2215fd96a4eSMike Christie if (ret) 2225fd96a4eSMike Christie goto free_rse; 2235fd96a4eSMike Christie 2245fd96a4eSMike Christie keys_info->generation = le32_to_cpu(rse->gen); 2255fd96a4eSMike Christie keys_info->num_keys = get_unaligned_le16(&rse->regctl); 2265fd96a4eSMike Christie 2275fd96a4eSMike Christie num_keys = min(num_keys, keys_info->num_keys); 2285fd96a4eSMike Christie for (i = 0; i < num_keys; i++) { 2295fd96a4eSMike Christie if (eds) { 2305fd96a4eSMike Christie keys_info->keys[i] = 2315fd96a4eSMike Christie le64_to_cpu(rse->regctl_eds[i].rkey); 2325fd96a4eSMike Christie } else { 2335fd96a4eSMike Christie struct nvme_reservation_status *rs; 2345fd96a4eSMike Christie 2355fd96a4eSMike Christie rs = (struct nvme_reservation_status *)rse; 2365fd96a4eSMike Christie keys_info->keys[i] = le64_to_cpu(rs->regctl_ds[i].rkey); 2375fd96a4eSMike Christie } 2385fd96a4eSMike Christie } 2395fd96a4eSMike Christie 2405fd96a4eSMike Christie free_rse: 2415fd96a4eSMike Christie kfree(rse); 2425fd96a4eSMike Christie return ret; 2435fd96a4eSMike Christie } 2445fd96a4eSMike Christie 245*28c97ba3SMike Christie static int nvme_pr_read_reservation(struct block_device *bdev, 246*28c97ba3SMike Christie struct pr_held_reservation *resv) 247*28c97ba3SMike Christie { 248*28c97ba3SMike Christie struct nvme_reservation_status_ext tmp_rse, *rse; 249*28c97ba3SMike Christie int ret, i, num_regs; 250*28c97ba3SMike Christie u32 rse_len; 251*28c97ba3SMike Christie bool eds; 252*28c97ba3SMike Christie 253*28c97ba3SMike Christie get_num_regs: 254*28c97ba3SMike Christie /* 255*28c97ba3SMike Christie * Get the number of registrations so we know how big to allocate 256*28c97ba3SMike Christie * the response buffer. 257*28c97ba3SMike Christie */ 258*28c97ba3SMike Christie ret = nvme_pr_resv_report(bdev, &tmp_rse, sizeof(tmp_rse), &eds); 259*28c97ba3SMike Christie if (ret) 260*28c97ba3SMike Christie return ret; 261*28c97ba3SMike Christie 262*28c97ba3SMike Christie num_regs = get_unaligned_le16(&tmp_rse.regctl); 263*28c97ba3SMike Christie if (!num_regs) { 264*28c97ba3SMike Christie resv->generation = le32_to_cpu(tmp_rse.gen); 265*28c97ba3SMike Christie return 0; 266*28c97ba3SMike Christie } 267*28c97ba3SMike Christie 268*28c97ba3SMike Christie rse_len = struct_size(rse, regctl_eds, num_regs); 269*28c97ba3SMike Christie rse = kzalloc(rse_len, GFP_KERNEL); 270*28c97ba3SMike Christie if (!rse) 271*28c97ba3SMike Christie return -ENOMEM; 272*28c97ba3SMike Christie 273*28c97ba3SMike Christie ret = nvme_pr_resv_report(bdev, rse, rse_len, &eds); 274*28c97ba3SMike Christie if (ret) 275*28c97ba3SMike Christie goto free_rse; 276*28c97ba3SMike Christie 277*28c97ba3SMike Christie if (num_regs != get_unaligned_le16(&rse->regctl)) { 278*28c97ba3SMike Christie kfree(rse); 279*28c97ba3SMike Christie goto get_num_regs; 280*28c97ba3SMike Christie } 281*28c97ba3SMike Christie 282*28c97ba3SMike Christie resv->generation = le32_to_cpu(rse->gen); 283*28c97ba3SMike Christie resv->type = block_pr_type_from_nvme(rse->rtype); 284*28c97ba3SMike Christie 285*28c97ba3SMike Christie for (i = 0; i < num_regs; i++) { 286*28c97ba3SMike Christie if (eds) { 287*28c97ba3SMike Christie if (rse->regctl_eds[i].rcsts) { 288*28c97ba3SMike Christie resv->key = le64_to_cpu(rse->regctl_eds[i].rkey); 289*28c97ba3SMike Christie break; 290*28c97ba3SMike Christie } 291*28c97ba3SMike Christie } else { 292*28c97ba3SMike Christie struct nvme_reservation_status *rs; 293*28c97ba3SMike Christie 294*28c97ba3SMike Christie rs = (struct nvme_reservation_status *)rse; 295*28c97ba3SMike Christie if (rs->regctl_ds[i].rcsts) { 296*28c97ba3SMike Christie resv->key = le64_to_cpu(rs->regctl_ds[i].rkey); 297*28c97ba3SMike Christie break; 298*28c97ba3SMike Christie } 299*28c97ba3SMike Christie } 300*28c97ba3SMike Christie } 301*28c97ba3SMike Christie 302*28c97ba3SMike Christie free_rse: 303*28c97ba3SMike Christie kfree(rse); 304*28c97ba3SMike Christie return ret; 305*28c97ba3SMike Christie } 306*28c97ba3SMike Christie 307b668f2f5SMike Christie const struct pr_ops nvme_pr_ops = { 308b668f2f5SMike Christie .pr_register = nvme_pr_register, 309b668f2f5SMike Christie .pr_reserve = nvme_pr_reserve, 310b668f2f5SMike Christie .pr_release = nvme_pr_release, 311b668f2f5SMike Christie .pr_preempt = nvme_pr_preempt, 312b668f2f5SMike Christie .pr_clear = nvme_pr_clear, 3135fd96a4eSMike Christie .pr_read_keys = nvme_pr_read_keys, 314*28c97ba3SMike Christie .pr_read_reservation = nvme_pr_read_reservation, 315b668f2f5SMike Christie }; 316