1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common code for the NVMe target.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/module.h>
8 #include <linux/random.h>
9 #include <linux/rculist.h>
10 #include <linux/pci-p2pdma.h>
11 #include <linux/scatterlist.h>
12
13 #include <generated/utsrelease.h>
14
15 #define CREATE_TRACE_POINTS
16 #include "trace.h"
17
18 #include "nvmet.h"
19
20 struct kmem_cache *nvmet_bvec_cache;
21 struct workqueue_struct *buffered_io_wq;
22 struct workqueue_struct *zbd_wq;
23 static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
24 static DEFINE_IDA(cntlid_ida);
25
26 struct workqueue_struct *nvmet_wq;
27 EXPORT_SYMBOL_GPL(nvmet_wq);
28
29 /*
30 * This read/write semaphore is used to synchronize access to configuration
31 * information on a target system that will result in discovery log page
32 * information change for at least one host.
33 * The full list of resources to protected by this semaphore is:
34 *
35 * - subsystems list
36 * - per-subsystem allowed hosts list
37 * - allow_any_host subsystem attribute
38 * - nvmet_genctr
39 * - the nvmet_transports array
40 *
41 * When updating any of those lists/structures write lock should be obtained,
42 * while when reading (popolating discovery log page or checking host-subsystem
43 * link) read lock is obtained to allow concurrent reads.
44 */
45 DECLARE_RWSEM(nvmet_config_sem);
46
47 u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1];
48 u64 nvmet_ana_chgcnt;
49 DECLARE_RWSEM(nvmet_ana_sem);
50
errno_to_nvme_status(struct nvmet_req * req,int errno)51 inline u16 errno_to_nvme_status(struct nvmet_req *req, int errno)
52 {
53 switch (errno) {
54 case 0:
55 return NVME_SC_SUCCESS;
56 case -ENOSPC:
57 req->error_loc = offsetof(struct nvme_rw_command, length);
58 return NVME_SC_CAP_EXCEEDED | NVME_SC_DNR;
59 case -EREMOTEIO:
60 req->error_loc = offsetof(struct nvme_rw_command, slba);
61 return NVME_SC_LBA_RANGE | NVME_SC_DNR;
62 case -EOPNOTSUPP:
63 req->error_loc = offsetof(struct nvme_common_command, opcode);
64 switch (req->cmd->common.opcode) {
65 case nvme_cmd_dsm:
66 case nvme_cmd_write_zeroes:
67 return NVME_SC_ONCS_NOT_SUPPORTED | NVME_SC_DNR;
68 default:
69 return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
70 }
71 break;
72 case -ENODATA:
73 req->error_loc = offsetof(struct nvme_rw_command, nsid);
74 return NVME_SC_ACCESS_DENIED;
75 case -EIO:
76 fallthrough;
77 default:
78 req->error_loc = offsetof(struct nvme_common_command, opcode);
79 return NVME_SC_INTERNAL | NVME_SC_DNR;
80 }
81 }
82
nvmet_report_invalid_opcode(struct nvmet_req * req)83 u16 nvmet_report_invalid_opcode(struct nvmet_req *req)
84 {
85 pr_debug("unhandled cmd %d on qid %d\n", req->cmd->common.opcode,
86 req->sq->qid);
87
88 req->error_loc = offsetof(struct nvme_common_command, opcode);
89 return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
90 }
91
92 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
93 const char *subsysnqn);
94
nvmet_copy_to_sgl(struct nvmet_req * req,off_t off,const void * buf,size_t len)95 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf,
96 size_t len)
97 {
98 if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len) {
99 req->error_loc = offsetof(struct nvme_common_command, dptr);
100 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
101 }
102 return 0;
103 }
104
nvmet_copy_from_sgl(struct nvmet_req * req,off_t off,void * buf,size_t len)105 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len)
106 {
107 if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len) {
108 req->error_loc = offsetof(struct nvme_common_command, dptr);
109 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
110 }
111 return 0;
112 }
113
nvmet_zero_sgl(struct nvmet_req * req,off_t off,size_t len)114 u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len)
115 {
116 if (sg_zero_buffer(req->sg, req->sg_cnt, len, off) != len) {
117 req->error_loc = offsetof(struct nvme_common_command, dptr);
118 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
119 }
120 return 0;
121 }
122
nvmet_max_nsid(struct nvmet_subsys * subsys)123 static u32 nvmet_max_nsid(struct nvmet_subsys *subsys)
124 {
125 struct nvmet_ns *cur;
126 unsigned long idx;
127 u32 nsid = 0;
128
129 xa_for_each(&subsys->namespaces, idx, cur)
130 nsid = cur->nsid;
131
132 return nsid;
133 }
134
nvmet_async_event_result(struct nvmet_async_event * aen)135 static u32 nvmet_async_event_result(struct nvmet_async_event *aen)
136 {
137 return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16);
138 }
139
nvmet_async_events_failall(struct nvmet_ctrl * ctrl)140 static void nvmet_async_events_failall(struct nvmet_ctrl *ctrl)
141 {
142 struct nvmet_req *req;
143
144 mutex_lock(&ctrl->lock);
145 while (ctrl->nr_async_event_cmds) {
146 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
147 mutex_unlock(&ctrl->lock);
148 nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR);
149 mutex_lock(&ctrl->lock);
150 }
151 mutex_unlock(&ctrl->lock);
152 }
153
nvmet_async_events_process(struct nvmet_ctrl * ctrl)154 static void nvmet_async_events_process(struct nvmet_ctrl *ctrl)
155 {
156 struct nvmet_async_event *aen;
157 struct nvmet_req *req;
158
159 mutex_lock(&ctrl->lock);
160 while (ctrl->nr_async_event_cmds && !list_empty(&ctrl->async_events)) {
161 aen = list_first_entry(&ctrl->async_events,
162 struct nvmet_async_event, entry);
163 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
164 nvmet_set_result(req, nvmet_async_event_result(aen));
165
166 list_del(&aen->entry);
167 kfree(aen);
168
169 mutex_unlock(&ctrl->lock);
170 trace_nvmet_async_event(ctrl, req->cqe->result.u32);
171 nvmet_req_complete(req, 0);
172 mutex_lock(&ctrl->lock);
173 }
174 mutex_unlock(&ctrl->lock);
175 }
176
nvmet_async_events_free(struct nvmet_ctrl * ctrl)177 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)
178 {
179 struct nvmet_async_event *aen, *tmp;
180
181 mutex_lock(&ctrl->lock);
182 list_for_each_entry_safe(aen, tmp, &ctrl->async_events, entry) {
183 list_del(&aen->entry);
184 kfree(aen);
185 }
186 mutex_unlock(&ctrl->lock);
187 }
188
nvmet_async_event_work(struct work_struct * work)189 static void nvmet_async_event_work(struct work_struct *work)
190 {
191 struct nvmet_ctrl *ctrl =
192 container_of(work, struct nvmet_ctrl, async_event_work);
193
194 nvmet_async_events_process(ctrl);
195 }
196
nvmet_add_async_event(struct nvmet_ctrl * ctrl,u8 event_type,u8 event_info,u8 log_page)197 void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
198 u8 event_info, u8 log_page)
199 {
200 struct nvmet_async_event *aen;
201
202 aen = kmalloc(sizeof(*aen), GFP_KERNEL);
203 if (!aen)
204 return;
205
206 aen->event_type = event_type;
207 aen->event_info = event_info;
208 aen->log_page = log_page;
209
210 mutex_lock(&ctrl->lock);
211 list_add_tail(&aen->entry, &ctrl->async_events);
212 mutex_unlock(&ctrl->lock);
213
214 queue_work(nvmet_wq, &ctrl->async_event_work);
215 }
216
nvmet_add_to_changed_ns_log(struct nvmet_ctrl * ctrl,__le32 nsid)217 static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid)
218 {
219 u32 i;
220
221 mutex_lock(&ctrl->lock);
222 if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES)
223 goto out_unlock;
224
225 for (i = 0; i < ctrl->nr_changed_ns; i++) {
226 if (ctrl->changed_ns_list[i] == nsid)
227 goto out_unlock;
228 }
229
230 if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) {
231 ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff);
232 ctrl->nr_changed_ns = U32_MAX;
233 goto out_unlock;
234 }
235
236 ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid;
237 out_unlock:
238 mutex_unlock(&ctrl->lock);
239 }
240
nvmet_ns_changed(struct nvmet_subsys * subsys,u32 nsid)241 void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid)
242 {
243 struct nvmet_ctrl *ctrl;
244
245 lockdep_assert_held(&subsys->lock);
246
247 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
248 nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid));
249 if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_NS_ATTR))
250 continue;
251 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
252 NVME_AER_NOTICE_NS_CHANGED,
253 NVME_LOG_CHANGED_NS);
254 }
255 }
256
nvmet_send_ana_event(struct nvmet_subsys * subsys,struct nvmet_port * port)257 void nvmet_send_ana_event(struct nvmet_subsys *subsys,
258 struct nvmet_port *port)
259 {
260 struct nvmet_ctrl *ctrl;
261
262 mutex_lock(&subsys->lock);
263 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
264 if (port && ctrl->port != port)
265 continue;
266 if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_ANA_CHANGE))
267 continue;
268 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
269 NVME_AER_NOTICE_ANA, NVME_LOG_ANA);
270 }
271 mutex_unlock(&subsys->lock);
272 }
273
nvmet_port_send_ana_event(struct nvmet_port * port)274 void nvmet_port_send_ana_event(struct nvmet_port *port)
275 {
276 struct nvmet_subsys_link *p;
277
278 down_read(&nvmet_config_sem);
279 list_for_each_entry(p, &port->subsystems, entry)
280 nvmet_send_ana_event(p->subsys, port);
281 up_read(&nvmet_config_sem);
282 }
283
nvmet_register_transport(const struct nvmet_fabrics_ops * ops)284 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops)
285 {
286 int ret = 0;
287
288 down_write(&nvmet_config_sem);
289 if (nvmet_transports[ops->type])
290 ret = -EINVAL;
291 else
292 nvmet_transports[ops->type] = ops;
293 up_write(&nvmet_config_sem);
294
295 return ret;
296 }
297 EXPORT_SYMBOL_GPL(nvmet_register_transport);
298
nvmet_unregister_transport(const struct nvmet_fabrics_ops * ops)299 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops)
300 {
301 down_write(&nvmet_config_sem);
302 nvmet_transports[ops->type] = NULL;
303 up_write(&nvmet_config_sem);
304 }
305 EXPORT_SYMBOL_GPL(nvmet_unregister_transport);
306
nvmet_port_del_ctrls(struct nvmet_port * port,struct nvmet_subsys * subsys)307 void nvmet_port_del_ctrls(struct nvmet_port *port, struct nvmet_subsys *subsys)
308 {
309 struct nvmet_ctrl *ctrl;
310
311 mutex_lock(&subsys->lock);
312 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
313 if (ctrl->port == port)
314 ctrl->ops->delete_ctrl(ctrl);
315 }
316 mutex_unlock(&subsys->lock);
317 }
318
nvmet_enable_port(struct nvmet_port * port)319 int nvmet_enable_port(struct nvmet_port *port)
320 {
321 const struct nvmet_fabrics_ops *ops;
322 int ret;
323
324 lockdep_assert_held(&nvmet_config_sem);
325
326 ops = nvmet_transports[port->disc_addr.trtype];
327 if (!ops) {
328 up_write(&nvmet_config_sem);
329 request_module("nvmet-transport-%d", port->disc_addr.trtype);
330 down_write(&nvmet_config_sem);
331 ops = nvmet_transports[port->disc_addr.trtype];
332 if (!ops) {
333 pr_err("transport type %d not supported\n",
334 port->disc_addr.trtype);
335 return -EINVAL;
336 }
337 }
338
339 if (!try_module_get(ops->owner))
340 return -EINVAL;
341
342 /*
343 * If the user requested PI support and the transport isn't pi capable,
344 * don't enable the port.
345 */
346 if (port->pi_enable && !(ops->flags & NVMF_METADATA_SUPPORTED)) {
347 pr_err("T10-PI is not supported by transport type %d\n",
348 port->disc_addr.trtype);
349 ret = -EINVAL;
350 goto out_put;
351 }
352
353 ret = ops->add_port(port);
354 if (ret)
355 goto out_put;
356
357 /* If the transport didn't set inline_data_size, then disable it. */
358 if (port->inline_data_size < 0)
359 port->inline_data_size = 0;
360
361 port->enabled = true;
362 port->tr_ops = ops;
363 return 0;
364
365 out_put:
366 module_put(ops->owner);
367 return ret;
368 }
369
nvmet_disable_port(struct nvmet_port * port)370 void nvmet_disable_port(struct nvmet_port *port)
371 {
372 const struct nvmet_fabrics_ops *ops;
373
374 lockdep_assert_held(&nvmet_config_sem);
375
376 port->enabled = false;
377 port->tr_ops = NULL;
378
379 ops = nvmet_transports[port->disc_addr.trtype];
380 ops->remove_port(port);
381 module_put(ops->owner);
382 }
383
nvmet_keep_alive_timer(struct work_struct * work)384 static void nvmet_keep_alive_timer(struct work_struct *work)
385 {
386 struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),
387 struct nvmet_ctrl, ka_work);
388 bool reset_tbkas = ctrl->reset_tbkas;
389
390 ctrl->reset_tbkas = false;
391 if (reset_tbkas) {
392 pr_debug("ctrl %d reschedule traffic based keep-alive timer\n",
393 ctrl->cntlid);
394 queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ);
395 return;
396 }
397
398 pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n",
399 ctrl->cntlid, ctrl->kato);
400
401 nvmet_ctrl_fatal_error(ctrl);
402 }
403
nvmet_start_keep_alive_timer(struct nvmet_ctrl * ctrl)404 void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)
405 {
406 if (unlikely(ctrl->kato == 0))
407 return;
408
409 pr_debug("ctrl %d start keep-alive timer for %d secs\n",
410 ctrl->cntlid, ctrl->kato);
411
412 queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ);
413 }
414
nvmet_stop_keep_alive_timer(struct nvmet_ctrl * ctrl)415 void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)
416 {
417 if (unlikely(ctrl->kato == 0))
418 return;
419
420 pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid);
421
422 cancel_delayed_work_sync(&ctrl->ka_work);
423 }
424
nvmet_req_find_ns(struct nvmet_req * req)425 u16 nvmet_req_find_ns(struct nvmet_req *req)
426 {
427 u32 nsid = le32_to_cpu(req->cmd->common.nsid);
428 struct nvmet_subsys *subsys = nvmet_req_subsys(req);
429
430 req->ns = xa_load(&subsys->namespaces, nsid);
431 if (unlikely(!req->ns)) {
432 req->error_loc = offsetof(struct nvme_common_command, nsid);
433 if (nvmet_subsys_nsid_exists(subsys, nsid))
434 return NVME_SC_INTERNAL_PATH_ERROR;
435 return NVME_SC_INVALID_NS | NVME_SC_DNR;
436 }
437
438 percpu_ref_get(&req->ns->ref);
439 return NVME_SC_SUCCESS;
440 }
441
nvmet_destroy_namespace(struct percpu_ref * ref)442 static void nvmet_destroy_namespace(struct percpu_ref *ref)
443 {
444 struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref);
445
446 complete(&ns->disable_done);
447 }
448
nvmet_put_namespace(struct nvmet_ns * ns)449 void nvmet_put_namespace(struct nvmet_ns *ns)
450 {
451 percpu_ref_put(&ns->ref);
452 }
453
nvmet_ns_dev_disable(struct nvmet_ns * ns)454 static void nvmet_ns_dev_disable(struct nvmet_ns *ns)
455 {
456 nvmet_bdev_ns_disable(ns);
457 nvmet_file_ns_disable(ns);
458 }
459
nvmet_p2pmem_ns_enable(struct nvmet_ns * ns)460 static int nvmet_p2pmem_ns_enable(struct nvmet_ns *ns)
461 {
462 int ret;
463 struct pci_dev *p2p_dev;
464
465 if (!ns->use_p2pmem)
466 return 0;
467
468 if (!ns->bdev) {
469 pr_err("peer-to-peer DMA is not supported by non-block device namespaces\n");
470 return -EINVAL;
471 }
472
473 if (!blk_queue_pci_p2pdma(ns->bdev->bd_disk->queue)) {
474 pr_err("peer-to-peer DMA is not supported by the driver of %s\n",
475 ns->device_path);
476 return -EINVAL;
477 }
478
479 if (ns->p2p_dev) {
480 ret = pci_p2pdma_distance(ns->p2p_dev, nvmet_ns_dev(ns), true);
481 if (ret < 0)
482 return -EINVAL;
483 } else {
484 /*
485 * Right now we just check that there is p2pmem available so
486 * we can report an error to the user right away if there
487 * is not. We'll find the actual device to use once we
488 * setup the controller when the port's device is available.
489 */
490
491 p2p_dev = pci_p2pmem_find(nvmet_ns_dev(ns));
492 if (!p2p_dev) {
493 pr_err("no peer-to-peer memory is available for %s\n",
494 ns->device_path);
495 return -EINVAL;
496 }
497
498 pci_dev_put(p2p_dev);
499 }
500
501 return 0;
502 }
503
504 /*
505 * Note: ctrl->subsys->lock should be held when calling this function
506 */
nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl * ctrl,struct nvmet_ns * ns)507 static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl,
508 struct nvmet_ns *ns)
509 {
510 struct device *clients[2];
511 struct pci_dev *p2p_dev;
512 int ret;
513
514 if (!ctrl->p2p_client || !ns->use_p2pmem)
515 return;
516
517 if (ns->p2p_dev) {
518 ret = pci_p2pdma_distance(ns->p2p_dev, ctrl->p2p_client, true);
519 if (ret < 0)
520 return;
521
522 p2p_dev = pci_dev_get(ns->p2p_dev);
523 } else {
524 clients[0] = ctrl->p2p_client;
525 clients[1] = nvmet_ns_dev(ns);
526
527 p2p_dev = pci_p2pmem_find_many(clients, ARRAY_SIZE(clients));
528 if (!p2p_dev) {
529 pr_err("no peer-to-peer memory is available that's supported by %s and %s\n",
530 dev_name(ctrl->p2p_client), ns->device_path);
531 return;
532 }
533 }
534
535 ret = radix_tree_insert(&ctrl->p2p_ns_map, ns->nsid, p2p_dev);
536 if (ret < 0)
537 pci_dev_put(p2p_dev);
538
539 pr_info("using p2pmem on %s for nsid %d\n", pci_name(p2p_dev),
540 ns->nsid);
541 }
542
nvmet_ns_revalidate(struct nvmet_ns * ns)543 bool nvmet_ns_revalidate(struct nvmet_ns *ns)
544 {
545 loff_t oldsize = ns->size;
546
547 if (ns->bdev)
548 nvmet_bdev_ns_revalidate(ns);
549 else
550 nvmet_file_ns_revalidate(ns);
551
552 return oldsize != ns->size;
553 }
554
nvmet_ns_enable(struct nvmet_ns * ns)555 int nvmet_ns_enable(struct nvmet_ns *ns)
556 {
557 struct nvmet_subsys *subsys = ns->subsys;
558 struct nvmet_ctrl *ctrl;
559 int ret;
560
561 mutex_lock(&subsys->lock);
562 ret = 0;
563
564 if (nvmet_is_passthru_subsys(subsys)) {
565 pr_info("cannot enable both passthru and regular namespaces for a single subsystem");
566 goto out_unlock;
567 }
568
569 if (ns->enabled)
570 goto out_unlock;
571
572 ret = -EMFILE;
573 if (subsys->nr_namespaces == NVMET_MAX_NAMESPACES)
574 goto out_unlock;
575
576 ret = nvmet_bdev_ns_enable(ns);
577 if (ret == -ENOTBLK)
578 ret = nvmet_file_ns_enable(ns);
579 if (ret)
580 goto out_unlock;
581
582 ret = nvmet_p2pmem_ns_enable(ns);
583 if (ret)
584 goto out_dev_disable;
585
586 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
587 nvmet_p2pmem_ns_add_p2p(ctrl, ns);
588
589 ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace,
590 0, GFP_KERNEL);
591 if (ret)
592 goto out_dev_put;
593
594 if (ns->nsid > subsys->max_nsid)
595 subsys->max_nsid = ns->nsid;
596
597 ret = xa_insert(&subsys->namespaces, ns->nsid, ns, GFP_KERNEL);
598 if (ret)
599 goto out_restore_subsys_maxnsid;
600
601 subsys->nr_namespaces++;
602
603 nvmet_ns_changed(subsys, ns->nsid);
604 ns->enabled = true;
605 ret = 0;
606 out_unlock:
607 mutex_unlock(&subsys->lock);
608 return ret;
609
610 out_restore_subsys_maxnsid:
611 subsys->max_nsid = nvmet_max_nsid(subsys);
612 percpu_ref_exit(&ns->ref);
613 out_dev_put:
614 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
615 pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid));
616 out_dev_disable:
617 nvmet_ns_dev_disable(ns);
618 goto out_unlock;
619 }
620
nvmet_ns_disable(struct nvmet_ns * ns)621 void nvmet_ns_disable(struct nvmet_ns *ns)
622 {
623 struct nvmet_subsys *subsys = ns->subsys;
624 struct nvmet_ctrl *ctrl;
625
626 mutex_lock(&subsys->lock);
627 if (!ns->enabled)
628 goto out_unlock;
629
630 ns->enabled = false;
631 xa_erase(&ns->subsys->namespaces, ns->nsid);
632 if (ns->nsid == subsys->max_nsid)
633 subsys->max_nsid = nvmet_max_nsid(subsys);
634
635 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
636 pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid));
637
638 mutex_unlock(&subsys->lock);
639
640 /*
641 * Now that we removed the namespaces from the lookup list, we
642 * can kill the per_cpu ref and wait for any remaining references
643 * to be dropped, as well as a RCU grace period for anyone only
644 * using the namepace under rcu_read_lock(). Note that we can't
645 * use call_rcu here as we need to ensure the namespaces have
646 * been fully destroyed before unloading the module.
647 */
648 percpu_ref_kill(&ns->ref);
649 synchronize_rcu();
650 wait_for_completion(&ns->disable_done);
651 percpu_ref_exit(&ns->ref);
652
653 mutex_lock(&subsys->lock);
654
655 subsys->nr_namespaces--;
656 nvmet_ns_changed(subsys, ns->nsid);
657 nvmet_ns_dev_disable(ns);
658 out_unlock:
659 mutex_unlock(&subsys->lock);
660 }
661
nvmet_ns_free(struct nvmet_ns * ns)662 void nvmet_ns_free(struct nvmet_ns *ns)
663 {
664 nvmet_ns_disable(ns);
665
666 down_write(&nvmet_ana_sem);
667 nvmet_ana_group_enabled[ns->anagrpid]--;
668 up_write(&nvmet_ana_sem);
669
670 kfree(ns->device_path);
671 kfree(ns);
672 }
673
nvmet_ns_alloc(struct nvmet_subsys * subsys,u32 nsid)674 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid)
675 {
676 struct nvmet_ns *ns;
677
678 ns = kzalloc(sizeof(*ns), GFP_KERNEL);
679 if (!ns)
680 return NULL;
681
682 init_completion(&ns->disable_done);
683
684 ns->nsid = nsid;
685 ns->subsys = subsys;
686
687 down_write(&nvmet_ana_sem);
688 ns->anagrpid = NVMET_DEFAULT_ANA_GRPID;
689 nvmet_ana_group_enabled[ns->anagrpid]++;
690 up_write(&nvmet_ana_sem);
691
692 uuid_gen(&ns->uuid);
693 ns->buffered_io = false;
694 ns->csi = NVME_CSI_NVM;
695
696 return ns;
697 }
698
nvmet_update_sq_head(struct nvmet_req * req)699 static void nvmet_update_sq_head(struct nvmet_req *req)
700 {
701 if (req->sq->size) {
702 u32 old_sqhd, new_sqhd;
703
704 old_sqhd = READ_ONCE(req->sq->sqhd);
705 do {
706 new_sqhd = (old_sqhd + 1) % req->sq->size;
707 } while (!try_cmpxchg(&req->sq->sqhd, &old_sqhd, new_sqhd));
708 }
709 req->cqe->sq_head = cpu_to_le16(req->sq->sqhd & 0x0000FFFF);
710 }
711
nvmet_set_error(struct nvmet_req * req,u16 status)712 static void nvmet_set_error(struct nvmet_req *req, u16 status)
713 {
714 struct nvmet_ctrl *ctrl = req->sq->ctrl;
715 struct nvme_error_slot *new_error_slot;
716 unsigned long flags;
717
718 req->cqe->status = cpu_to_le16(status << 1);
719
720 if (!ctrl || req->error_loc == NVMET_NO_ERROR_LOC)
721 return;
722
723 spin_lock_irqsave(&ctrl->error_lock, flags);
724 ctrl->err_counter++;
725 new_error_slot =
726 &ctrl->slots[ctrl->err_counter % NVMET_ERROR_LOG_SLOTS];
727
728 new_error_slot->error_count = cpu_to_le64(ctrl->err_counter);
729 new_error_slot->sqid = cpu_to_le16(req->sq->qid);
730 new_error_slot->cmdid = cpu_to_le16(req->cmd->common.command_id);
731 new_error_slot->status_field = cpu_to_le16(status << 1);
732 new_error_slot->param_error_location = cpu_to_le16(req->error_loc);
733 new_error_slot->lba = cpu_to_le64(req->error_slba);
734 new_error_slot->nsid = req->cmd->common.nsid;
735 spin_unlock_irqrestore(&ctrl->error_lock, flags);
736
737 /* set the more bit for this request */
738 req->cqe->status |= cpu_to_le16(1 << 14);
739 }
740
__nvmet_req_complete(struct nvmet_req * req,u16 status)741 static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
742 {
743 struct nvmet_ns *ns = req->ns;
744
745 if (!req->sq->sqhd_disabled)
746 nvmet_update_sq_head(req);
747 req->cqe->sq_id = cpu_to_le16(req->sq->qid);
748 req->cqe->command_id = req->cmd->common.command_id;
749
750 if (unlikely(status))
751 nvmet_set_error(req, status);
752
753 trace_nvmet_req_complete(req);
754
755 req->ops->queue_response(req);
756 if (ns)
757 nvmet_put_namespace(ns);
758 }
759
nvmet_req_complete(struct nvmet_req * req,u16 status)760 void nvmet_req_complete(struct nvmet_req *req, u16 status)
761 {
762 struct nvmet_sq *sq = req->sq;
763
764 __nvmet_req_complete(req, status);
765 percpu_ref_put(&sq->ref);
766 }
767 EXPORT_SYMBOL_GPL(nvmet_req_complete);
768
nvmet_cq_setup(struct nvmet_ctrl * ctrl,struct nvmet_cq * cq,u16 qid,u16 size)769 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq,
770 u16 qid, u16 size)
771 {
772 cq->qid = qid;
773 cq->size = size;
774 }
775
nvmet_sq_setup(struct nvmet_ctrl * ctrl,struct nvmet_sq * sq,u16 qid,u16 size)776 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,
777 u16 qid, u16 size)
778 {
779 sq->sqhd = 0;
780 sq->qid = qid;
781 sq->size = size;
782
783 ctrl->sqs[qid] = sq;
784 }
785
nvmet_confirm_sq(struct percpu_ref * ref)786 static void nvmet_confirm_sq(struct percpu_ref *ref)
787 {
788 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);
789
790 complete(&sq->confirm_done);
791 }
792
nvmet_sq_destroy(struct nvmet_sq * sq)793 void nvmet_sq_destroy(struct nvmet_sq *sq)
794 {
795 struct nvmet_ctrl *ctrl = sq->ctrl;
796
797 /*
798 * If this is the admin queue, complete all AERs so that our
799 * queue doesn't have outstanding requests on it.
800 */
801 if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq)
802 nvmet_async_events_failall(ctrl);
803 percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq);
804 wait_for_completion(&sq->confirm_done);
805 wait_for_completion(&sq->free_done);
806 percpu_ref_exit(&sq->ref);
807 nvmet_auth_sq_free(sq);
808
809 if (ctrl) {
810 /*
811 * The teardown flow may take some time, and the host may not
812 * send us keep-alive during this period, hence reset the
813 * traffic based keep-alive timer so we don't trigger a
814 * controller teardown as a result of a keep-alive expiration.
815 */
816 ctrl->reset_tbkas = true;
817 sq->ctrl->sqs[sq->qid] = NULL;
818 nvmet_ctrl_put(ctrl);
819 sq->ctrl = NULL; /* allows reusing the queue later */
820 }
821 }
822 EXPORT_SYMBOL_GPL(nvmet_sq_destroy);
823
nvmet_sq_free(struct percpu_ref * ref)824 static void nvmet_sq_free(struct percpu_ref *ref)
825 {
826 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);
827
828 complete(&sq->free_done);
829 }
830
nvmet_sq_init(struct nvmet_sq * sq)831 int nvmet_sq_init(struct nvmet_sq *sq)
832 {
833 int ret;
834
835 ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL);
836 if (ret) {
837 pr_err("percpu_ref init failed!\n");
838 return ret;
839 }
840 init_completion(&sq->free_done);
841 init_completion(&sq->confirm_done);
842 nvmet_auth_sq_init(sq);
843
844 return 0;
845 }
846 EXPORT_SYMBOL_GPL(nvmet_sq_init);
847
nvmet_check_ana_state(struct nvmet_port * port,struct nvmet_ns * ns)848 static inline u16 nvmet_check_ana_state(struct nvmet_port *port,
849 struct nvmet_ns *ns)
850 {
851 enum nvme_ana_state state = port->ana_state[ns->anagrpid];
852
853 if (unlikely(state == NVME_ANA_INACCESSIBLE))
854 return NVME_SC_ANA_INACCESSIBLE;
855 if (unlikely(state == NVME_ANA_PERSISTENT_LOSS))
856 return NVME_SC_ANA_PERSISTENT_LOSS;
857 if (unlikely(state == NVME_ANA_CHANGE))
858 return NVME_SC_ANA_TRANSITION;
859 return 0;
860 }
861
nvmet_io_cmd_check_access(struct nvmet_req * req)862 static inline u16 nvmet_io_cmd_check_access(struct nvmet_req *req)
863 {
864 if (unlikely(req->ns->readonly)) {
865 switch (req->cmd->common.opcode) {
866 case nvme_cmd_read:
867 case nvme_cmd_flush:
868 break;
869 default:
870 return NVME_SC_NS_WRITE_PROTECTED;
871 }
872 }
873
874 return 0;
875 }
876
nvmet_parse_io_cmd(struct nvmet_req * req)877 static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
878 {
879 struct nvme_command *cmd = req->cmd;
880 u16 ret;
881
882 if (nvme_is_fabrics(cmd))
883 return nvmet_parse_fabrics_io_cmd(req);
884
885 if (unlikely(!nvmet_check_auth_status(req)))
886 return NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
887
888 ret = nvmet_check_ctrl_status(req);
889 if (unlikely(ret))
890 return ret;
891
892 if (nvmet_is_passthru_req(req))
893 return nvmet_parse_passthru_io_cmd(req);
894
895 ret = nvmet_req_find_ns(req);
896 if (unlikely(ret))
897 return ret;
898
899 ret = nvmet_check_ana_state(req->port, req->ns);
900 if (unlikely(ret)) {
901 req->error_loc = offsetof(struct nvme_common_command, nsid);
902 return ret;
903 }
904 ret = nvmet_io_cmd_check_access(req);
905 if (unlikely(ret)) {
906 req->error_loc = offsetof(struct nvme_common_command, nsid);
907 return ret;
908 }
909
910 switch (req->ns->csi) {
911 case NVME_CSI_NVM:
912 if (req->ns->file)
913 return nvmet_file_parse_io_cmd(req);
914 return nvmet_bdev_parse_io_cmd(req);
915 case NVME_CSI_ZNS:
916 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
917 return nvmet_bdev_zns_parse_io_cmd(req);
918 return NVME_SC_INVALID_IO_CMD_SET;
919 default:
920 return NVME_SC_INVALID_IO_CMD_SET;
921 }
922 }
923
nvmet_req_init(struct nvmet_req * req,struct nvmet_cq * cq,struct nvmet_sq * sq,const struct nvmet_fabrics_ops * ops)924 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
925 struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops)
926 {
927 u8 flags = req->cmd->common.flags;
928 u16 status;
929
930 req->cq = cq;
931 req->sq = sq;
932 req->ops = ops;
933 req->sg = NULL;
934 req->metadata_sg = NULL;
935 req->sg_cnt = 0;
936 req->metadata_sg_cnt = 0;
937 req->transfer_len = 0;
938 req->metadata_len = 0;
939 req->cqe->status = 0;
940 req->cqe->sq_head = 0;
941 req->ns = NULL;
942 req->error_loc = NVMET_NO_ERROR_LOC;
943 req->error_slba = 0;
944
945 /* no support for fused commands yet */
946 if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) {
947 req->error_loc = offsetof(struct nvme_common_command, flags);
948 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
949 goto fail;
950 }
951
952 /*
953 * For fabrics, PSDT field shall describe metadata pointer (MPTR) that
954 * contains an address of a single contiguous physical buffer that is
955 * byte aligned.
956 */
957 if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF)) {
958 req->error_loc = offsetof(struct nvme_common_command, flags);
959 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
960 goto fail;
961 }
962
963 if (unlikely(!req->sq->ctrl))
964 /* will return an error for any non-connect command: */
965 status = nvmet_parse_connect_cmd(req);
966 else if (likely(req->sq->qid != 0))
967 status = nvmet_parse_io_cmd(req);
968 else
969 status = nvmet_parse_admin_cmd(req);
970
971 if (status)
972 goto fail;
973
974 trace_nvmet_req_init(req, req->cmd);
975
976 if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {
977 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
978 goto fail;
979 }
980
981 if (sq->ctrl)
982 sq->ctrl->reset_tbkas = true;
983
984 return true;
985
986 fail:
987 __nvmet_req_complete(req, status);
988 return false;
989 }
990 EXPORT_SYMBOL_GPL(nvmet_req_init);
991
nvmet_req_uninit(struct nvmet_req * req)992 void nvmet_req_uninit(struct nvmet_req *req)
993 {
994 percpu_ref_put(&req->sq->ref);
995 if (req->ns)
996 nvmet_put_namespace(req->ns);
997 }
998 EXPORT_SYMBOL_GPL(nvmet_req_uninit);
999
nvmet_check_transfer_len(struct nvmet_req * req,size_t len)1000 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len)
1001 {
1002 if (unlikely(len != req->transfer_len)) {
1003 req->error_loc = offsetof(struct nvme_common_command, dptr);
1004 nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR);
1005 return false;
1006 }
1007
1008 return true;
1009 }
1010 EXPORT_SYMBOL_GPL(nvmet_check_transfer_len);
1011
nvmet_check_data_len_lte(struct nvmet_req * req,size_t data_len)1012 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len)
1013 {
1014 if (unlikely(data_len > req->transfer_len)) {
1015 req->error_loc = offsetof(struct nvme_common_command, dptr);
1016 nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR);
1017 return false;
1018 }
1019
1020 return true;
1021 }
1022
nvmet_data_transfer_len(struct nvmet_req * req)1023 static unsigned int nvmet_data_transfer_len(struct nvmet_req *req)
1024 {
1025 return req->transfer_len - req->metadata_len;
1026 }
1027
nvmet_req_alloc_p2pmem_sgls(struct pci_dev * p2p_dev,struct nvmet_req * req)1028 static int nvmet_req_alloc_p2pmem_sgls(struct pci_dev *p2p_dev,
1029 struct nvmet_req *req)
1030 {
1031 req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt,
1032 nvmet_data_transfer_len(req));
1033 if (!req->sg)
1034 goto out_err;
1035
1036 if (req->metadata_len) {
1037 req->metadata_sg = pci_p2pmem_alloc_sgl(p2p_dev,
1038 &req->metadata_sg_cnt, req->metadata_len);
1039 if (!req->metadata_sg)
1040 goto out_free_sg;
1041 }
1042
1043 req->p2p_dev = p2p_dev;
1044
1045 return 0;
1046 out_free_sg:
1047 pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
1048 out_err:
1049 return -ENOMEM;
1050 }
1051
nvmet_req_find_p2p_dev(struct nvmet_req * req)1052 static struct pci_dev *nvmet_req_find_p2p_dev(struct nvmet_req *req)
1053 {
1054 if (!IS_ENABLED(CONFIG_PCI_P2PDMA) ||
1055 !req->sq->ctrl || !req->sq->qid || !req->ns)
1056 return NULL;
1057 return radix_tree_lookup(&req->sq->ctrl->p2p_ns_map, req->ns->nsid);
1058 }
1059
nvmet_req_alloc_sgls(struct nvmet_req * req)1060 int nvmet_req_alloc_sgls(struct nvmet_req *req)
1061 {
1062 struct pci_dev *p2p_dev = nvmet_req_find_p2p_dev(req);
1063
1064 if (p2p_dev && !nvmet_req_alloc_p2pmem_sgls(p2p_dev, req))
1065 return 0;
1066
1067 req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL,
1068 &req->sg_cnt);
1069 if (unlikely(!req->sg))
1070 goto out;
1071
1072 if (req->metadata_len) {
1073 req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL,
1074 &req->metadata_sg_cnt);
1075 if (unlikely(!req->metadata_sg))
1076 goto out_free;
1077 }
1078
1079 return 0;
1080 out_free:
1081 sgl_free(req->sg);
1082 out:
1083 return -ENOMEM;
1084 }
1085 EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls);
1086
nvmet_req_free_sgls(struct nvmet_req * req)1087 void nvmet_req_free_sgls(struct nvmet_req *req)
1088 {
1089 if (req->p2p_dev) {
1090 pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
1091 if (req->metadata_sg)
1092 pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg);
1093 req->p2p_dev = NULL;
1094 } else {
1095 sgl_free(req->sg);
1096 if (req->metadata_sg)
1097 sgl_free(req->metadata_sg);
1098 }
1099
1100 req->sg = NULL;
1101 req->metadata_sg = NULL;
1102 req->sg_cnt = 0;
1103 req->metadata_sg_cnt = 0;
1104 }
1105 EXPORT_SYMBOL_GPL(nvmet_req_free_sgls);
1106
nvmet_cc_en(u32 cc)1107 static inline bool nvmet_cc_en(u32 cc)
1108 {
1109 return (cc >> NVME_CC_EN_SHIFT) & 0x1;
1110 }
1111
nvmet_cc_css(u32 cc)1112 static inline u8 nvmet_cc_css(u32 cc)
1113 {
1114 return (cc >> NVME_CC_CSS_SHIFT) & 0x7;
1115 }
1116
nvmet_cc_mps(u32 cc)1117 static inline u8 nvmet_cc_mps(u32 cc)
1118 {
1119 return (cc >> NVME_CC_MPS_SHIFT) & 0xf;
1120 }
1121
nvmet_cc_ams(u32 cc)1122 static inline u8 nvmet_cc_ams(u32 cc)
1123 {
1124 return (cc >> NVME_CC_AMS_SHIFT) & 0x7;
1125 }
1126
nvmet_cc_shn(u32 cc)1127 static inline u8 nvmet_cc_shn(u32 cc)
1128 {
1129 return (cc >> NVME_CC_SHN_SHIFT) & 0x3;
1130 }
1131
nvmet_cc_iosqes(u32 cc)1132 static inline u8 nvmet_cc_iosqes(u32 cc)
1133 {
1134 return (cc >> NVME_CC_IOSQES_SHIFT) & 0xf;
1135 }
1136
nvmet_cc_iocqes(u32 cc)1137 static inline u8 nvmet_cc_iocqes(u32 cc)
1138 {
1139 return (cc >> NVME_CC_IOCQES_SHIFT) & 0xf;
1140 }
1141
nvmet_css_supported(u8 cc_css)1142 static inline bool nvmet_css_supported(u8 cc_css)
1143 {
1144 switch (cc_css << NVME_CC_CSS_SHIFT) {
1145 case NVME_CC_CSS_NVM:
1146 case NVME_CC_CSS_CSI:
1147 return true;
1148 default:
1149 return false;
1150 }
1151 }
1152
nvmet_start_ctrl(struct nvmet_ctrl * ctrl)1153 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)
1154 {
1155 lockdep_assert_held(&ctrl->lock);
1156
1157 /*
1158 * Only I/O controllers should verify iosqes,iocqes.
1159 * Strictly speaking, the spec says a discovery controller
1160 * should verify iosqes,iocqes are zeroed, however that
1161 * would break backwards compatibility, so don't enforce it.
1162 */
1163 if (!nvmet_is_disc_subsys(ctrl->subsys) &&
1164 (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES ||
1165 nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES)) {
1166 ctrl->csts = NVME_CSTS_CFS;
1167 return;
1168 }
1169
1170 if (nvmet_cc_mps(ctrl->cc) != 0 ||
1171 nvmet_cc_ams(ctrl->cc) != 0 ||
1172 !nvmet_css_supported(nvmet_cc_css(ctrl->cc))) {
1173 ctrl->csts = NVME_CSTS_CFS;
1174 return;
1175 }
1176
1177 ctrl->csts = NVME_CSTS_RDY;
1178
1179 /*
1180 * Controllers that are not yet enabled should not really enforce the
1181 * keep alive timeout, but we still want to track a timeout and cleanup
1182 * in case a host died before it enabled the controller. Hence, simply
1183 * reset the keep alive timer when the controller is enabled.
1184 */
1185 if (ctrl->kato)
1186 mod_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ);
1187 }
1188
nvmet_clear_ctrl(struct nvmet_ctrl * ctrl)1189 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl)
1190 {
1191 lockdep_assert_held(&ctrl->lock);
1192
1193 /* XXX: tear down queues? */
1194 ctrl->csts &= ~NVME_CSTS_RDY;
1195 ctrl->cc = 0;
1196 }
1197
nvmet_update_cc(struct nvmet_ctrl * ctrl,u32 new)1198 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new)
1199 {
1200 u32 old;
1201
1202 mutex_lock(&ctrl->lock);
1203 old = ctrl->cc;
1204 ctrl->cc = new;
1205
1206 if (nvmet_cc_en(new) && !nvmet_cc_en(old))
1207 nvmet_start_ctrl(ctrl);
1208 if (!nvmet_cc_en(new) && nvmet_cc_en(old))
1209 nvmet_clear_ctrl(ctrl);
1210 if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) {
1211 nvmet_clear_ctrl(ctrl);
1212 ctrl->csts |= NVME_CSTS_SHST_CMPLT;
1213 }
1214 if (!nvmet_cc_shn(new) && nvmet_cc_shn(old))
1215 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;
1216 mutex_unlock(&ctrl->lock);
1217 }
1218
nvmet_init_cap(struct nvmet_ctrl * ctrl)1219 static void nvmet_init_cap(struct nvmet_ctrl *ctrl)
1220 {
1221 /* command sets supported: NVMe command set: */
1222 ctrl->cap = (1ULL << 37);
1223 /* Controller supports one or more I/O Command Sets */
1224 ctrl->cap |= (1ULL << 43);
1225 /* CC.EN timeout in 500msec units: */
1226 ctrl->cap |= (15ULL << 24);
1227 /* maximum queue entries supported: */
1228 if (ctrl->ops->get_max_queue_size)
1229 ctrl->cap |= ctrl->ops->get_max_queue_size(ctrl) - 1;
1230 else
1231 ctrl->cap |= NVMET_QUEUE_SIZE - 1;
1232
1233 if (nvmet_is_passthru_subsys(ctrl->subsys))
1234 nvmet_passthrough_override_cap(ctrl);
1235 }
1236
nvmet_ctrl_find_get(const char * subsysnqn,const char * hostnqn,u16 cntlid,struct nvmet_req * req)1237 struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn,
1238 const char *hostnqn, u16 cntlid,
1239 struct nvmet_req *req)
1240 {
1241 struct nvmet_ctrl *ctrl = NULL;
1242 struct nvmet_subsys *subsys;
1243
1244 subsys = nvmet_find_get_subsys(req->port, subsysnqn);
1245 if (!subsys) {
1246 pr_warn("connect request for invalid subsystem %s!\n",
1247 subsysnqn);
1248 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
1249 goto out;
1250 }
1251
1252 mutex_lock(&subsys->lock);
1253 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
1254 if (ctrl->cntlid == cntlid) {
1255 if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) {
1256 pr_warn("hostnqn mismatch.\n");
1257 continue;
1258 }
1259 if (!kref_get_unless_zero(&ctrl->ref))
1260 continue;
1261
1262 /* ctrl found */
1263 goto found;
1264 }
1265 }
1266
1267 ctrl = NULL; /* ctrl not found */
1268 pr_warn("could not find controller %d for subsys %s / host %s\n",
1269 cntlid, subsysnqn, hostnqn);
1270 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid);
1271
1272 found:
1273 mutex_unlock(&subsys->lock);
1274 nvmet_subsys_put(subsys);
1275 out:
1276 return ctrl;
1277 }
1278
nvmet_check_ctrl_status(struct nvmet_req * req)1279 u16 nvmet_check_ctrl_status(struct nvmet_req *req)
1280 {
1281 if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {
1282 pr_err("got cmd %d while CC.EN == 0 on qid = %d\n",
1283 req->cmd->common.opcode, req->sq->qid);
1284 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
1285 }
1286
1287 if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {
1288 pr_err("got cmd %d while CSTS.RDY == 0 on qid = %d\n",
1289 req->cmd->common.opcode, req->sq->qid);
1290 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
1291 }
1292
1293 if (unlikely(!nvmet_check_auth_status(req))) {
1294 pr_warn("qid %d not authenticated\n", req->sq->qid);
1295 return NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
1296 }
1297 return 0;
1298 }
1299
nvmet_host_allowed(struct nvmet_subsys * subsys,const char * hostnqn)1300 bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn)
1301 {
1302 struct nvmet_host_link *p;
1303
1304 lockdep_assert_held(&nvmet_config_sem);
1305
1306 if (subsys->allow_any_host)
1307 return true;
1308
1309 if (nvmet_is_disc_subsys(subsys)) /* allow all access to disc subsys */
1310 return true;
1311
1312 list_for_each_entry(p, &subsys->hosts, entry) {
1313 if (!strcmp(nvmet_host_name(p->host), hostnqn))
1314 return true;
1315 }
1316
1317 return false;
1318 }
1319
1320 /*
1321 * Note: ctrl->subsys->lock should be held when calling this function
1322 */
nvmet_setup_p2p_ns_map(struct nvmet_ctrl * ctrl,struct nvmet_req * req)1323 static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl,
1324 struct nvmet_req *req)
1325 {
1326 struct nvmet_ns *ns;
1327 unsigned long idx;
1328
1329 if (!req->p2p_client)
1330 return;
1331
1332 ctrl->p2p_client = get_device(req->p2p_client);
1333
1334 xa_for_each(&ctrl->subsys->namespaces, idx, ns)
1335 nvmet_p2pmem_ns_add_p2p(ctrl, ns);
1336 }
1337
1338 /*
1339 * Note: ctrl->subsys->lock should be held when calling this function
1340 */
nvmet_release_p2p_ns_map(struct nvmet_ctrl * ctrl)1341 static void nvmet_release_p2p_ns_map(struct nvmet_ctrl *ctrl)
1342 {
1343 struct radix_tree_iter iter;
1344 void __rcu **slot;
1345
1346 radix_tree_for_each_slot(slot, &ctrl->p2p_ns_map, &iter, 0)
1347 pci_dev_put(radix_tree_deref_slot(slot));
1348
1349 put_device(ctrl->p2p_client);
1350 }
1351
nvmet_fatal_error_handler(struct work_struct * work)1352 static void nvmet_fatal_error_handler(struct work_struct *work)
1353 {
1354 struct nvmet_ctrl *ctrl =
1355 container_of(work, struct nvmet_ctrl, fatal_err_work);
1356
1357 pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid);
1358 ctrl->ops->delete_ctrl(ctrl);
1359 }
1360
nvmet_alloc_ctrl(const char * subsysnqn,const char * hostnqn,struct nvmet_req * req,u32 kato,struct nvmet_ctrl ** ctrlp)1361 u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
1362 struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp)
1363 {
1364 struct nvmet_subsys *subsys;
1365 struct nvmet_ctrl *ctrl;
1366 int ret;
1367 u16 status;
1368
1369 status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
1370 subsys = nvmet_find_get_subsys(req->port, subsysnqn);
1371 if (!subsys) {
1372 pr_warn("connect request for invalid subsystem %s!\n",
1373 subsysnqn);
1374 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
1375 req->error_loc = offsetof(struct nvme_common_command, dptr);
1376 goto out;
1377 }
1378
1379 down_read(&nvmet_config_sem);
1380 if (!nvmet_host_allowed(subsys, hostnqn)) {
1381 pr_info("connect by host %s for subsystem %s not allowed\n",
1382 hostnqn, subsysnqn);
1383 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn);
1384 up_read(&nvmet_config_sem);
1385 status = NVME_SC_CONNECT_INVALID_HOST | NVME_SC_DNR;
1386 req->error_loc = offsetof(struct nvme_common_command, dptr);
1387 goto out_put_subsystem;
1388 }
1389 up_read(&nvmet_config_sem);
1390
1391 status = NVME_SC_INTERNAL;
1392 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1393 if (!ctrl)
1394 goto out_put_subsystem;
1395 mutex_init(&ctrl->lock);
1396
1397 ctrl->port = req->port;
1398 ctrl->ops = req->ops;
1399
1400 #ifdef CONFIG_NVME_TARGET_PASSTHRU
1401 /* By default, set loop targets to clear IDS by default */
1402 if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP)
1403 subsys->clear_ids = 1;
1404 #endif
1405
1406 INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);
1407 INIT_LIST_HEAD(&ctrl->async_events);
1408 INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL);
1409 INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler);
1410 INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);
1411
1412 memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);
1413 memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);
1414
1415 kref_init(&ctrl->ref);
1416 ctrl->subsys = subsys;
1417 nvmet_init_cap(ctrl);
1418 WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL);
1419
1420 ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES,
1421 sizeof(__le32), GFP_KERNEL);
1422 if (!ctrl->changed_ns_list)
1423 goto out_free_ctrl;
1424
1425 ctrl->sqs = kcalloc(subsys->max_qid + 1,
1426 sizeof(struct nvmet_sq *),
1427 GFP_KERNEL);
1428 if (!ctrl->sqs)
1429 goto out_free_changed_ns_list;
1430
1431 if (subsys->cntlid_min > subsys->cntlid_max)
1432 goto out_free_sqs;
1433
1434 ret = ida_alloc_range(&cntlid_ida,
1435 subsys->cntlid_min, subsys->cntlid_max,
1436 GFP_KERNEL);
1437 if (ret < 0) {
1438 status = NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR;
1439 goto out_free_sqs;
1440 }
1441 ctrl->cntlid = ret;
1442
1443 /*
1444 * Discovery controllers may use some arbitrary high value
1445 * in order to cleanup stale discovery sessions
1446 */
1447 if (nvmet_is_disc_subsys(ctrl->subsys) && !kato)
1448 kato = NVMET_DISC_KATO_MS;
1449
1450 /* keep-alive timeout in seconds */
1451 ctrl->kato = DIV_ROUND_UP(kato, 1000);
1452
1453 ctrl->err_counter = 0;
1454 spin_lock_init(&ctrl->error_lock);
1455
1456 nvmet_start_keep_alive_timer(ctrl);
1457
1458 mutex_lock(&subsys->lock);
1459 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
1460 nvmet_setup_p2p_ns_map(ctrl, req);
1461 mutex_unlock(&subsys->lock);
1462
1463 *ctrlp = ctrl;
1464 return 0;
1465
1466 out_free_sqs:
1467 kfree(ctrl->sqs);
1468 out_free_changed_ns_list:
1469 kfree(ctrl->changed_ns_list);
1470 out_free_ctrl:
1471 kfree(ctrl);
1472 out_put_subsystem:
1473 nvmet_subsys_put(subsys);
1474 out:
1475 return status;
1476 }
1477
nvmet_ctrl_free(struct kref * ref)1478 static void nvmet_ctrl_free(struct kref *ref)
1479 {
1480 struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref);
1481 struct nvmet_subsys *subsys = ctrl->subsys;
1482
1483 mutex_lock(&subsys->lock);
1484 nvmet_release_p2p_ns_map(ctrl);
1485 list_del(&ctrl->subsys_entry);
1486 mutex_unlock(&subsys->lock);
1487
1488 nvmet_stop_keep_alive_timer(ctrl);
1489
1490 flush_work(&ctrl->async_event_work);
1491 cancel_work_sync(&ctrl->fatal_err_work);
1492
1493 nvmet_destroy_auth(ctrl);
1494
1495 ida_free(&cntlid_ida, ctrl->cntlid);
1496
1497 nvmet_async_events_free(ctrl);
1498 kfree(ctrl->sqs);
1499 kfree(ctrl->changed_ns_list);
1500 kfree(ctrl);
1501
1502 nvmet_subsys_put(subsys);
1503 }
1504
nvmet_ctrl_put(struct nvmet_ctrl * ctrl)1505 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl)
1506 {
1507 kref_put(&ctrl->ref, nvmet_ctrl_free);
1508 }
1509
nvmet_ctrl_fatal_error(struct nvmet_ctrl * ctrl)1510 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)
1511 {
1512 mutex_lock(&ctrl->lock);
1513 if (!(ctrl->csts & NVME_CSTS_CFS)) {
1514 ctrl->csts |= NVME_CSTS_CFS;
1515 queue_work(nvmet_wq, &ctrl->fatal_err_work);
1516 }
1517 mutex_unlock(&ctrl->lock);
1518 }
1519 EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error);
1520
nvmet_find_get_subsys(struct nvmet_port * port,const char * subsysnqn)1521 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
1522 const char *subsysnqn)
1523 {
1524 struct nvmet_subsys_link *p;
1525
1526 if (!port)
1527 return NULL;
1528
1529 if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn)) {
1530 if (!kref_get_unless_zero(&nvmet_disc_subsys->ref))
1531 return NULL;
1532 return nvmet_disc_subsys;
1533 }
1534
1535 down_read(&nvmet_config_sem);
1536 list_for_each_entry(p, &port->subsystems, entry) {
1537 if (!strncmp(p->subsys->subsysnqn, subsysnqn,
1538 NVMF_NQN_SIZE)) {
1539 if (!kref_get_unless_zero(&p->subsys->ref))
1540 break;
1541 up_read(&nvmet_config_sem);
1542 return p->subsys;
1543 }
1544 }
1545 up_read(&nvmet_config_sem);
1546 return NULL;
1547 }
1548
nvmet_subsys_alloc(const char * subsysnqn,enum nvme_subsys_type type)1549 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn,
1550 enum nvme_subsys_type type)
1551 {
1552 struct nvmet_subsys *subsys;
1553 char serial[NVMET_SN_MAX_SIZE / 2];
1554 int ret;
1555
1556 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
1557 if (!subsys)
1558 return ERR_PTR(-ENOMEM);
1559
1560 subsys->ver = NVMET_DEFAULT_VS;
1561 /* generate a random serial number as our controllers are ephemeral: */
1562 get_random_bytes(&serial, sizeof(serial));
1563 bin2hex(subsys->serial, &serial, sizeof(serial));
1564
1565 subsys->model_number = kstrdup(NVMET_DEFAULT_CTRL_MODEL, GFP_KERNEL);
1566 if (!subsys->model_number) {
1567 ret = -ENOMEM;
1568 goto free_subsys;
1569 }
1570
1571 subsys->ieee_oui = 0;
1572
1573 subsys->firmware_rev = kstrndup(UTS_RELEASE, NVMET_FR_MAX_SIZE, GFP_KERNEL);
1574 if (!subsys->firmware_rev) {
1575 ret = -ENOMEM;
1576 goto free_mn;
1577 }
1578
1579 switch (type) {
1580 case NVME_NQN_NVME:
1581 subsys->max_qid = NVMET_NR_QUEUES;
1582 break;
1583 case NVME_NQN_DISC:
1584 case NVME_NQN_CURR:
1585 subsys->max_qid = 0;
1586 break;
1587 default:
1588 pr_err("%s: Unknown Subsystem type - %d\n", __func__, type);
1589 ret = -EINVAL;
1590 goto free_fr;
1591 }
1592 subsys->type = type;
1593 subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE,
1594 GFP_KERNEL);
1595 if (!subsys->subsysnqn) {
1596 ret = -ENOMEM;
1597 goto free_fr;
1598 }
1599 subsys->cntlid_min = NVME_CNTLID_MIN;
1600 subsys->cntlid_max = NVME_CNTLID_MAX;
1601 kref_init(&subsys->ref);
1602
1603 mutex_init(&subsys->lock);
1604 xa_init(&subsys->namespaces);
1605 INIT_LIST_HEAD(&subsys->ctrls);
1606 INIT_LIST_HEAD(&subsys->hosts);
1607
1608 return subsys;
1609
1610 free_fr:
1611 kfree(subsys->firmware_rev);
1612 free_mn:
1613 kfree(subsys->model_number);
1614 free_subsys:
1615 kfree(subsys);
1616 return ERR_PTR(ret);
1617 }
1618
nvmet_subsys_free(struct kref * ref)1619 static void nvmet_subsys_free(struct kref *ref)
1620 {
1621 struct nvmet_subsys *subsys =
1622 container_of(ref, struct nvmet_subsys, ref);
1623
1624 WARN_ON_ONCE(!xa_empty(&subsys->namespaces));
1625
1626 xa_destroy(&subsys->namespaces);
1627 nvmet_passthru_subsys_free(subsys);
1628
1629 kfree(subsys->subsysnqn);
1630 kfree(subsys->model_number);
1631 kfree(subsys->firmware_rev);
1632 kfree(subsys);
1633 }
1634
nvmet_subsys_del_ctrls(struct nvmet_subsys * subsys)1635 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys)
1636 {
1637 struct nvmet_ctrl *ctrl;
1638
1639 mutex_lock(&subsys->lock);
1640 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
1641 ctrl->ops->delete_ctrl(ctrl);
1642 mutex_unlock(&subsys->lock);
1643 }
1644
nvmet_subsys_put(struct nvmet_subsys * subsys)1645 void nvmet_subsys_put(struct nvmet_subsys *subsys)
1646 {
1647 kref_put(&subsys->ref, nvmet_subsys_free);
1648 }
1649
nvmet_init(void)1650 static int __init nvmet_init(void)
1651 {
1652 int error = -ENOMEM;
1653
1654 nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID] = 1;
1655
1656 nvmet_bvec_cache = kmem_cache_create("nvmet-bvec",
1657 NVMET_MAX_MPOOL_BVEC * sizeof(struct bio_vec), 0,
1658 SLAB_HWCACHE_ALIGN, NULL);
1659 if (!nvmet_bvec_cache)
1660 return -ENOMEM;
1661
1662 zbd_wq = alloc_workqueue("nvmet-zbd-wq", WQ_MEM_RECLAIM, 0);
1663 if (!zbd_wq)
1664 goto out_destroy_bvec_cache;
1665
1666 buffered_io_wq = alloc_workqueue("nvmet-buffered-io-wq",
1667 WQ_MEM_RECLAIM, 0);
1668 if (!buffered_io_wq)
1669 goto out_free_zbd_work_queue;
1670
1671 nvmet_wq = alloc_workqueue("nvmet-wq", WQ_MEM_RECLAIM, 0);
1672 if (!nvmet_wq)
1673 goto out_free_buffered_work_queue;
1674
1675 error = nvmet_init_discovery();
1676 if (error)
1677 goto out_free_nvmet_work_queue;
1678
1679 error = nvmet_init_configfs();
1680 if (error)
1681 goto out_exit_discovery;
1682 return 0;
1683
1684 out_exit_discovery:
1685 nvmet_exit_discovery();
1686 out_free_nvmet_work_queue:
1687 destroy_workqueue(nvmet_wq);
1688 out_free_buffered_work_queue:
1689 destroy_workqueue(buffered_io_wq);
1690 out_free_zbd_work_queue:
1691 destroy_workqueue(zbd_wq);
1692 out_destroy_bvec_cache:
1693 kmem_cache_destroy(nvmet_bvec_cache);
1694 return error;
1695 }
1696
nvmet_exit(void)1697 static void __exit nvmet_exit(void)
1698 {
1699 nvmet_exit_configfs();
1700 nvmet_exit_discovery();
1701 ida_destroy(&cntlid_ida);
1702 destroy_workqueue(nvmet_wq);
1703 destroy_workqueue(buffered_io_wq);
1704 destroy_workqueue(zbd_wq);
1705 kmem_cache_destroy(nvmet_bvec_cache);
1706
1707 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024);
1708 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024);
1709 }
1710
1711 module_init(nvmet_init);
1712 module_exit(nvmet_exit);
1713
1714 MODULE_LICENSE("GPL v2");
1715