xref: /openbmc/linux/drivers/nvme/host/core.c (revision d3741027)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVM Express device driver
4  * Copyright (c) 2011-2014, Intel Corporation.
5  */
6 
7 #include <linux/blkdev.h>
8 #include <linux/blk-mq.h>
9 #include <linux/blk-integrity.h>
10 #include <linux/compat.h>
11 #include <linux/delay.h>
12 #include <linux/errno.h>
13 #include <linux/hdreg.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/slab.h>
18 #include <linux/types.h>
19 #include <linux/pr.h>
20 #include <linux/ptrace.h>
21 #include <linux/nvme_ioctl.h>
22 #include <linux/pm_qos.h>
23 #include <asm/unaligned.h>
24 
25 #include "nvme.h"
26 #include "fabrics.h"
27 
28 #define CREATE_TRACE_POINTS
29 #include "trace.h"
30 
31 #define NVME_MINORS		(1U << MINORBITS)
32 
33 unsigned int admin_timeout = 60;
34 module_param(admin_timeout, uint, 0644);
35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
36 EXPORT_SYMBOL_GPL(admin_timeout);
37 
38 unsigned int nvme_io_timeout = 30;
39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
42 
43 static unsigned char shutdown_timeout = 5;
44 module_param(shutdown_timeout, byte, 0644);
45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46 
47 static u8 nvme_max_retries = 5;
48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
50 
51 static unsigned long default_ps_max_latency_us = 100000;
52 module_param(default_ps_max_latency_us, ulong, 0644);
53 MODULE_PARM_DESC(default_ps_max_latency_us,
54 		 "max power saving latency for new devices; use PM QOS to change per device");
55 
56 static bool force_apst;
57 module_param(force_apst, bool, 0644);
58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59 
60 static unsigned long apst_primary_timeout_ms = 100;
61 module_param(apst_primary_timeout_ms, ulong, 0644);
62 MODULE_PARM_DESC(apst_primary_timeout_ms,
63 	"primary APST timeout in ms");
64 
65 static unsigned long apst_secondary_timeout_ms = 2000;
66 module_param(apst_secondary_timeout_ms, ulong, 0644);
67 MODULE_PARM_DESC(apst_secondary_timeout_ms,
68 	"secondary APST timeout in ms");
69 
70 static unsigned long apst_primary_latency_tol_us = 15000;
71 module_param(apst_primary_latency_tol_us, ulong, 0644);
72 MODULE_PARM_DESC(apst_primary_latency_tol_us,
73 	"primary APST latency tolerance in us");
74 
75 static unsigned long apst_secondary_latency_tol_us = 100000;
76 module_param(apst_secondary_latency_tol_us, ulong, 0644);
77 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
78 	"secondary APST latency tolerance in us");
79 
80 /*
81  * nvme_wq - hosts nvme related works that are not reset or delete
82  * nvme_reset_wq - hosts nvme reset works
83  * nvme_delete_wq - hosts nvme delete works
84  *
85  * nvme_wq will host works such as scan, aen handling, fw activation,
86  * keep-alive, periodic reconnects etc. nvme_reset_wq
87  * runs reset works which also flush works hosted on nvme_wq for
88  * serialization purposes. nvme_delete_wq host controller deletion
89  * works which flush reset works for serialization.
90  */
91 struct workqueue_struct *nvme_wq;
92 EXPORT_SYMBOL_GPL(nvme_wq);
93 
94 struct workqueue_struct *nvme_reset_wq;
95 EXPORT_SYMBOL_GPL(nvme_reset_wq);
96 
97 struct workqueue_struct *nvme_delete_wq;
98 EXPORT_SYMBOL_GPL(nvme_delete_wq);
99 
100 static LIST_HEAD(nvme_subsystems);
101 static DEFINE_MUTEX(nvme_subsystems_lock);
102 
103 static DEFINE_IDA(nvme_instance_ida);
104 static dev_t nvme_ctrl_base_chr_devt;
105 static struct class *nvme_class;
106 static struct class *nvme_subsys_class;
107 
108 static DEFINE_IDA(nvme_ns_chr_minor_ida);
109 static dev_t nvme_ns_chr_devt;
110 static struct class *nvme_ns_chr_class;
111 
112 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
113 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
114 					   unsigned nsid);
115 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
116 				   struct nvme_command *cmd);
117 
118 void nvme_queue_scan(struct nvme_ctrl *ctrl)
119 {
120 	/*
121 	 * Only new queue scan work when admin and IO queues are both alive
122 	 */
123 	if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
124 		queue_work(nvme_wq, &ctrl->scan_work);
125 }
126 
127 /*
128  * Use this function to proceed with scheduling reset_work for a controller
129  * that had previously been set to the resetting state. This is intended for
130  * code paths that can't be interrupted by other reset attempts. A hot removal
131  * may prevent this from succeeding.
132  */
133 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
134 {
135 	if (ctrl->state != NVME_CTRL_RESETTING)
136 		return -EBUSY;
137 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
138 		return -EBUSY;
139 	return 0;
140 }
141 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
142 
143 static void nvme_failfast_work(struct work_struct *work)
144 {
145 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
146 			struct nvme_ctrl, failfast_work);
147 
148 	if (ctrl->state != NVME_CTRL_CONNECTING)
149 		return;
150 
151 	set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
152 	dev_info(ctrl->device, "failfast expired\n");
153 	nvme_kick_requeue_lists(ctrl);
154 }
155 
156 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
157 {
158 	if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
159 		return;
160 
161 	schedule_delayed_work(&ctrl->failfast_work,
162 			      ctrl->opts->fast_io_fail_tmo * HZ);
163 }
164 
165 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
166 {
167 	if (!ctrl->opts)
168 		return;
169 
170 	cancel_delayed_work_sync(&ctrl->failfast_work);
171 	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
172 }
173 
174 
175 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
176 {
177 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
178 		return -EBUSY;
179 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
180 		return -EBUSY;
181 	return 0;
182 }
183 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
184 
185 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
186 {
187 	int ret;
188 
189 	ret = nvme_reset_ctrl(ctrl);
190 	if (!ret) {
191 		flush_work(&ctrl->reset_work);
192 		if (ctrl->state != NVME_CTRL_LIVE)
193 			ret = -ENETRESET;
194 	}
195 
196 	return ret;
197 }
198 
199 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
200 {
201 	dev_info(ctrl->device,
202 		 "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
203 
204 	flush_work(&ctrl->reset_work);
205 	nvme_stop_ctrl(ctrl);
206 	nvme_remove_namespaces(ctrl);
207 	ctrl->ops->delete_ctrl(ctrl);
208 	nvme_uninit_ctrl(ctrl);
209 }
210 
211 static void nvme_delete_ctrl_work(struct work_struct *work)
212 {
213 	struct nvme_ctrl *ctrl =
214 		container_of(work, struct nvme_ctrl, delete_work);
215 
216 	nvme_do_delete_ctrl(ctrl);
217 }
218 
219 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
220 {
221 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
222 		return -EBUSY;
223 	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
224 		return -EBUSY;
225 	return 0;
226 }
227 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
228 
229 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
230 {
231 	/*
232 	 * Keep a reference until nvme_do_delete_ctrl() complete,
233 	 * since ->delete_ctrl can free the controller.
234 	 */
235 	nvme_get_ctrl(ctrl);
236 	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
237 		nvme_do_delete_ctrl(ctrl);
238 	nvme_put_ctrl(ctrl);
239 }
240 
241 static blk_status_t nvme_error_status(u16 status)
242 {
243 	switch (status & 0x7ff) {
244 	case NVME_SC_SUCCESS:
245 		return BLK_STS_OK;
246 	case NVME_SC_CAP_EXCEEDED:
247 		return BLK_STS_NOSPC;
248 	case NVME_SC_LBA_RANGE:
249 	case NVME_SC_CMD_INTERRUPTED:
250 	case NVME_SC_NS_NOT_READY:
251 		return BLK_STS_TARGET;
252 	case NVME_SC_BAD_ATTRIBUTES:
253 	case NVME_SC_ONCS_NOT_SUPPORTED:
254 	case NVME_SC_INVALID_OPCODE:
255 	case NVME_SC_INVALID_FIELD:
256 	case NVME_SC_INVALID_NS:
257 		return BLK_STS_NOTSUPP;
258 	case NVME_SC_WRITE_FAULT:
259 	case NVME_SC_READ_ERROR:
260 	case NVME_SC_UNWRITTEN_BLOCK:
261 	case NVME_SC_ACCESS_DENIED:
262 	case NVME_SC_READ_ONLY:
263 	case NVME_SC_COMPARE_FAILED:
264 		return BLK_STS_MEDIUM;
265 	case NVME_SC_GUARD_CHECK:
266 	case NVME_SC_APPTAG_CHECK:
267 	case NVME_SC_REFTAG_CHECK:
268 	case NVME_SC_INVALID_PI:
269 		return BLK_STS_PROTECTION;
270 	case NVME_SC_RESERVATION_CONFLICT:
271 		return BLK_STS_NEXUS;
272 	case NVME_SC_HOST_PATH_ERROR:
273 		return BLK_STS_TRANSPORT;
274 	case NVME_SC_ZONE_TOO_MANY_ACTIVE:
275 		return BLK_STS_ZONE_ACTIVE_RESOURCE;
276 	case NVME_SC_ZONE_TOO_MANY_OPEN:
277 		return BLK_STS_ZONE_OPEN_RESOURCE;
278 	default:
279 		return BLK_STS_IOERR;
280 	}
281 }
282 
283 static void nvme_retry_req(struct request *req)
284 {
285 	unsigned long delay = 0;
286 	u16 crd;
287 
288 	/* The mask and shift result must be <= 3 */
289 	crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
290 	if (crd)
291 		delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
292 
293 	nvme_req(req)->retries++;
294 	blk_mq_requeue_request(req, false);
295 	blk_mq_delay_kick_requeue_list(req->q, delay);
296 }
297 
298 static void nvme_log_error(struct request *req)
299 {
300 	struct nvme_ns *ns = req->q->queuedata;
301 	struct nvme_request *nr = nvme_req(req);
302 
303 	if (ns) {
304 		pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
305 		       ns->disk ? ns->disk->disk_name : "?",
306 		       nvme_get_opcode_str(nr->cmd->common.opcode),
307 		       nr->cmd->common.opcode,
308 		       (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
309 		       (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
310 		       nvme_get_error_status_str(nr->status),
311 		       nr->status >> 8 & 7,	/* Status Code Type */
312 		       nr->status & 0xff,	/* Status Code */
313 		       nr->status & NVME_SC_MORE ? "MORE " : "",
314 		       nr->status & NVME_SC_DNR  ? "DNR "  : "");
315 		return;
316 	}
317 
318 	pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
319 			   dev_name(nr->ctrl->device),
320 			   nvme_get_admin_opcode_str(nr->cmd->common.opcode),
321 			   nr->cmd->common.opcode,
322 			   nvme_get_error_status_str(nr->status),
323 			   nr->status >> 8 & 7,	/* Status Code Type */
324 			   nr->status & 0xff,	/* Status Code */
325 			   nr->status & NVME_SC_MORE ? "MORE " : "",
326 			   nr->status & NVME_SC_DNR  ? "DNR "  : "");
327 }
328 
329 enum nvme_disposition {
330 	COMPLETE,
331 	RETRY,
332 	FAILOVER,
333 };
334 
335 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
336 {
337 	if (likely(nvme_req(req)->status == 0))
338 		return COMPLETE;
339 
340 	if (blk_noretry_request(req) ||
341 	    (nvme_req(req)->status & NVME_SC_DNR) ||
342 	    nvme_req(req)->retries >= nvme_max_retries)
343 		return COMPLETE;
344 
345 	if (req->cmd_flags & REQ_NVME_MPATH) {
346 		if (nvme_is_path_error(nvme_req(req)->status) ||
347 		    blk_queue_dying(req->q))
348 			return FAILOVER;
349 	} else {
350 		if (blk_queue_dying(req->q))
351 			return COMPLETE;
352 	}
353 
354 	return RETRY;
355 }
356 
357 static inline void nvme_end_req_zoned(struct request *req)
358 {
359 	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
360 	    req_op(req) == REQ_OP_ZONE_APPEND)
361 		req->__sector = nvme_lba_to_sect(req->q->queuedata,
362 			le64_to_cpu(nvme_req(req)->result.u64));
363 }
364 
365 static inline void nvme_end_req(struct request *req)
366 {
367 	blk_status_t status = nvme_error_status(nvme_req(req)->status);
368 
369 	if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
370 		nvme_log_error(req);
371 	nvme_end_req_zoned(req);
372 	nvme_trace_bio_complete(req);
373 	blk_mq_end_request(req, status);
374 }
375 
376 void nvme_complete_rq(struct request *req)
377 {
378 	trace_nvme_complete_rq(req);
379 	nvme_cleanup_cmd(req);
380 
381 	if (nvme_req(req)->ctrl->kas)
382 		nvme_req(req)->ctrl->comp_seen = true;
383 
384 	switch (nvme_decide_disposition(req)) {
385 	case COMPLETE:
386 		nvme_end_req(req);
387 		return;
388 	case RETRY:
389 		nvme_retry_req(req);
390 		return;
391 	case FAILOVER:
392 		nvme_failover_req(req);
393 		return;
394 	}
395 }
396 EXPORT_SYMBOL_GPL(nvme_complete_rq);
397 
398 void nvme_complete_batch_req(struct request *req)
399 {
400 	trace_nvme_complete_rq(req);
401 	nvme_cleanup_cmd(req);
402 	nvme_end_req_zoned(req);
403 }
404 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
405 
406 /*
407  * Called to unwind from ->queue_rq on a failed command submission so that the
408  * multipathing code gets called to potentially failover to another path.
409  * The caller needs to unwind all transport specific resource allocations and
410  * must return propagate the return value.
411  */
412 blk_status_t nvme_host_path_error(struct request *req)
413 {
414 	nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
415 	blk_mq_set_request_complete(req);
416 	nvme_complete_rq(req);
417 	return BLK_STS_OK;
418 }
419 EXPORT_SYMBOL_GPL(nvme_host_path_error);
420 
421 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
422 {
423 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
424 				"Cancelling I/O %d", req->tag);
425 
426 	/* don't abort one completed request */
427 	if (blk_mq_request_completed(req))
428 		return true;
429 
430 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
431 	nvme_req(req)->flags |= NVME_REQ_CANCELLED;
432 	blk_mq_complete_request(req);
433 	return true;
434 }
435 EXPORT_SYMBOL_GPL(nvme_cancel_request);
436 
437 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
438 {
439 	if (ctrl->tagset) {
440 		blk_mq_tagset_busy_iter(ctrl->tagset,
441 				nvme_cancel_request, ctrl);
442 		blk_mq_tagset_wait_completed_request(ctrl->tagset);
443 	}
444 }
445 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
446 
447 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
448 {
449 	if (ctrl->admin_tagset) {
450 		blk_mq_tagset_busy_iter(ctrl->admin_tagset,
451 				nvme_cancel_request, ctrl);
452 		blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
453 	}
454 }
455 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
456 
457 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
458 		enum nvme_ctrl_state new_state)
459 {
460 	enum nvme_ctrl_state old_state;
461 	unsigned long flags;
462 	bool changed = false;
463 
464 	spin_lock_irqsave(&ctrl->lock, flags);
465 
466 	old_state = ctrl->state;
467 	switch (new_state) {
468 	case NVME_CTRL_LIVE:
469 		switch (old_state) {
470 		case NVME_CTRL_NEW:
471 		case NVME_CTRL_RESETTING:
472 		case NVME_CTRL_CONNECTING:
473 			changed = true;
474 			fallthrough;
475 		default:
476 			break;
477 		}
478 		break;
479 	case NVME_CTRL_RESETTING:
480 		switch (old_state) {
481 		case NVME_CTRL_NEW:
482 		case NVME_CTRL_LIVE:
483 			changed = true;
484 			fallthrough;
485 		default:
486 			break;
487 		}
488 		break;
489 	case NVME_CTRL_CONNECTING:
490 		switch (old_state) {
491 		case NVME_CTRL_NEW:
492 		case NVME_CTRL_RESETTING:
493 			changed = true;
494 			fallthrough;
495 		default:
496 			break;
497 		}
498 		break;
499 	case NVME_CTRL_DELETING:
500 		switch (old_state) {
501 		case NVME_CTRL_LIVE:
502 		case NVME_CTRL_RESETTING:
503 		case NVME_CTRL_CONNECTING:
504 			changed = true;
505 			fallthrough;
506 		default:
507 			break;
508 		}
509 		break;
510 	case NVME_CTRL_DELETING_NOIO:
511 		switch (old_state) {
512 		case NVME_CTRL_DELETING:
513 		case NVME_CTRL_DEAD:
514 			changed = true;
515 			fallthrough;
516 		default:
517 			break;
518 		}
519 		break;
520 	case NVME_CTRL_DEAD:
521 		switch (old_state) {
522 		case NVME_CTRL_DELETING:
523 			changed = true;
524 			fallthrough;
525 		default:
526 			break;
527 		}
528 		break;
529 	default:
530 		break;
531 	}
532 
533 	if (changed) {
534 		ctrl->state = new_state;
535 		wake_up_all(&ctrl->state_wq);
536 	}
537 
538 	spin_unlock_irqrestore(&ctrl->lock, flags);
539 	if (!changed)
540 		return false;
541 
542 	if (ctrl->state == NVME_CTRL_LIVE) {
543 		if (old_state == NVME_CTRL_CONNECTING)
544 			nvme_stop_failfast_work(ctrl);
545 		nvme_kick_requeue_lists(ctrl);
546 	} else if (ctrl->state == NVME_CTRL_CONNECTING &&
547 		old_state == NVME_CTRL_RESETTING) {
548 		nvme_start_failfast_work(ctrl);
549 	}
550 	return changed;
551 }
552 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
553 
554 /*
555  * Returns true for sink states that can't ever transition back to live.
556  */
557 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
558 {
559 	switch (ctrl->state) {
560 	case NVME_CTRL_NEW:
561 	case NVME_CTRL_LIVE:
562 	case NVME_CTRL_RESETTING:
563 	case NVME_CTRL_CONNECTING:
564 		return false;
565 	case NVME_CTRL_DELETING:
566 	case NVME_CTRL_DELETING_NOIO:
567 	case NVME_CTRL_DEAD:
568 		return true;
569 	default:
570 		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
571 		return true;
572 	}
573 }
574 
575 /*
576  * Waits for the controller state to be resetting, or returns false if it is
577  * not possible to ever transition to that state.
578  */
579 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
580 {
581 	wait_event(ctrl->state_wq,
582 		   nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
583 		   nvme_state_terminal(ctrl));
584 	return ctrl->state == NVME_CTRL_RESETTING;
585 }
586 EXPORT_SYMBOL_GPL(nvme_wait_reset);
587 
588 static void nvme_free_ns_head(struct kref *ref)
589 {
590 	struct nvme_ns_head *head =
591 		container_of(ref, struct nvme_ns_head, ref);
592 
593 	nvme_mpath_remove_disk(head);
594 	ida_free(&head->subsys->ns_ida, head->instance);
595 	cleanup_srcu_struct(&head->srcu);
596 	nvme_put_subsystem(head->subsys);
597 	kfree(head);
598 }
599 
600 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
601 {
602 	return kref_get_unless_zero(&head->ref);
603 }
604 
605 void nvme_put_ns_head(struct nvme_ns_head *head)
606 {
607 	kref_put(&head->ref, nvme_free_ns_head);
608 }
609 
610 static void nvme_free_ns(struct kref *kref)
611 {
612 	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
613 
614 	put_disk(ns->disk);
615 	nvme_put_ns_head(ns->head);
616 	nvme_put_ctrl(ns->ctrl);
617 	kfree(ns);
618 }
619 
620 static inline bool nvme_get_ns(struct nvme_ns *ns)
621 {
622 	return kref_get_unless_zero(&ns->kref);
623 }
624 
625 void nvme_put_ns(struct nvme_ns *ns)
626 {
627 	kref_put(&ns->kref, nvme_free_ns);
628 }
629 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
630 
631 static inline void nvme_clear_nvme_request(struct request *req)
632 {
633 	nvme_req(req)->status = 0;
634 	nvme_req(req)->retries = 0;
635 	nvme_req(req)->flags = 0;
636 	req->rq_flags |= RQF_DONTPREP;
637 }
638 
639 /* initialize a passthrough request */
640 void nvme_init_request(struct request *req, struct nvme_command *cmd)
641 {
642 	if (req->q->queuedata)
643 		req->timeout = NVME_IO_TIMEOUT;
644 	else /* no queuedata implies admin queue */
645 		req->timeout = NVME_ADMIN_TIMEOUT;
646 
647 	/* passthru commands should let the driver set the SGL flags */
648 	cmd->common.flags &= ~NVME_CMD_SGL_ALL;
649 
650 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
651 	if (req->mq_hctx->type == HCTX_TYPE_POLL)
652 		req->cmd_flags |= REQ_POLLED;
653 	nvme_clear_nvme_request(req);
654 	memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
655 }
656 EXPORT_SYMBOL_GPL(nvme_init_request);
657 
658 /*
659  * For something we're not in a state to send to the device the default action
660  * is to busy it and retry it after the controller state is recovered.  However,
661  * if the controller is deleting or if anything is marked for failfast or
662  * nvme multipath it is immediately failed.
663  *
664  * Note: commands used to initialize the controller will be marked for failfast.
665  * Note: nvme cli/ioctl commands are marked for failfast.
666  */
667 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
668 		struct request *rq)
669 {
670 	if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
671 	    ctrl->state != NVME_CTRL_DELETING &&
672 	    ctrl->state != NVME_CTRL_DEAD &&
673 	    !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
674 	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
675 		return BLK_STS_RESOURCE;
676 	return nvme_host_path_error(rq);
677 }
678 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
679 
680 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
681 		bool queue_live)
682 {
683 	struct nvme_request *req = nvme_req(rq);
684 
685 	/*
686 	 * currently we have a problem sending passthru commands
687 	 * on the admin_q if the controller is not LIVE because we can't
688 	 * make sure that they are going out after the admin connect,
689 	 * controller enable and/or other commands in the initialization
690 	 * sequence. until the controller will be LIVE, fail with
691 	 * BLK_STS_RESOURCE so that they will be rescheduled.
692 	 */
693 	if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
694 		return false;
695 
696 	if (ctrl->ops->flags & NVME_F_FABRICS) {
697 		/*
698 		 * Only allow commands on a live queue, except for the connect
699 		 * command, which is require to set the queue live in the
700 		 * appropinquate states.
701 		 */
702 		switch (ctrl->state) {
703 		case NVME_CTRL_CONNECTING:
704 			if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
705 			    req->cmd->fabrics.fctype == nvme_fabrics_type_connect)
706 				return true;
707 			break;
708 		default:
709 			break;
710 		case NVME_CTRL_DEAD:
711 			return false;
712 		}
713 	}
714 
715 	return queue_live;
716 }
717 EXPORT_SYMBOL_GPL(__nvme_check_ready);
718 
719 static inline void nvme_setup_flush(struct nvme_ns *ns,
720 		struct nvme_command *cmnd)
721 {
722 	memset(cmnd, 0, sizeof(*cmnd));
723 	cmnd->common.opcode = nvme_cmd_flush;
724 	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
725 }
726 
727 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
728 		struct nvme_command *cmnd)
729 {
730 	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
731 	struct nvme_dsm_range *range;
732 	struct bio *bio;
733 
734 	/*
735 	 * Some devices do not consider the DSM 'Number of Ranges' field when
736 	 * determining how much data to DMA. Always allocate memory for maximum
737 	 * number of segments to prevent device reading beyond end of buffer.
738 	 */
739 	static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
740 
741 	range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
742 	if (!range) {
743 		/*
744 		 * If we fail allocation our range, fallback to the controller
745 		 * discard page. If that's also busy, it's safe to return
746 		 * busy, as we know we can make progress once that's freed.
747 		 */
748 		if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
749 			return BLK_STS_RESOURCE;
750 
751 		range = page_address(ns->ctrl->discard_page);
752 	}
753 
754 	__rq_for_each_bio(bio, req) {
755 		u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
756 		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
757 
758 		if (n < segments) {
759 			range[n].cattr = cpu_to_le32(0);
760 			range[n].nlb = cpu_to_le32(nlb);
761 			range[n].slba = cpu_to_le64(slba);
762 		}
763 		n++;
764 	}
765 
766 	if (WARN_ON_ONCE(n != segments)) {
767 		if (virt_to_page(range) == ns->ctrl->discard_page)
768 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
769 		else
770 			kfree(range);
771 		return BLK_STS_IOERR;
772 	}
773 
774 	memset(cmnd, 0, sizeof(*cmnd));
775 	cmnd->dsm.opcode = nvme_cmd_dsm;
776 	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
777 	cmnd->dsm.nr = cpu_to_le32(segments - 1);
778 	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
779 
780 	req->special_vec.bv_page = virt_to_page(range);
781 	req->special_vec.bv_offset = offset_in_page(range);
782 	req->special_vec.bv_len = alloc_size;
783 	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
784 
785 	return BLK_STS_OK;
786 }
787 
788 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
789 			      struct request *req)
790 {
791 	u32 upper, lower;
792 	u64 ref48;
793 
794 	/* both rw and write zeroes share the same reftag format */
795 	switch (ns->guard_type) {
796 	case NVME_NVM_NS_16B_GUARD:
797 		cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
798 		break;
799 	case NVME_NVM_NS_64B_GUARD:
800 		ref48 = ext_pi_ref_tag(req);
801 		lower = lower_32_bits(ref48);
802 		upper = upper_32_bits(ref48);
803 
804 		cmnd->rw.reftag = cpu_to_le32(lower);
805 		cmnd->rw.cdw3 = cpu_to_le32(upper);
806 		break;
807 	default:
808 		break;
809 	}
810 }
811 
812 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
813 		struct request *req, struct nvme_command *cmnd)
814 {
815 	memset(cmnd, 0, sizeof(*cmnd));
816 
817 	if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
818 		return nvme_setup_discard(ns, req, cmnd);
819 
820 	cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
821 	cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
822 	cmnd->write_zeroes.slba =
823 		cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
824 	cmnd->write_zeroes.length =
825 		cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
826 
827 	if (nvme_ns_has_pi(ns)) {
828 		cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);
829 
830 		switch (ns->pi_type) {
831 		case NVME_NS_DPS_PI_TYPE1:
832 		case NVME_NS_DPS_PI_TYPE2:
833 			nvme_set_ref_tag(ns, cmnd, req);
834 			break;
835 		}
836 	}
837 
838 	return BLK_STS_OK;
839 }
840 
841 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
842 		struct request *req, struct nvme_command *cmnd,
843 		enum nvme_opcode op)
844 {
845 	u16 control = 0;
846 	u32 dsmgmt = 0;
847 
848 	if (req->cmd_flags & REQ_FUA)
849 		control |= NVME_RW_FUA;
850 	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
851 		control |= NVME_RW_LR;
852 
853 	if (req->cmd_flags & REQ_RAHEAD)
854 		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
855 
856 	cmnd->rw.opcode = op;
857 	cmnd->rw.flags = 0;
858 	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
859 	cmnd->rw.cdw2 = 0;
860 	cmnd->rw.cdw3 = 0;
861 	cmnd->rw.metadata = 0;
862 	cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
863 	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
864 	cmnd->rw.reftag = 0;
865 	cmnd->rw.apptag = 0;
866 	cmnd->rw.appmask = 0;
867 
868 	if (ns->ms) {
869 		/*
870 		 * If formated with metadata, the block layer always provides a
871 		 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
872 		 * we enable the PRACT bit for protection information or set the
873 		 * namespace capacity to zero to prevent any I/O.
874 		 */
875 		if (!blk_integrity_rq(req)) {
876 			if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
877 				return BLK_STS_NOTSUPP;
878 			control |= NVME_RW_PRINFO_PRACT;
879 		}
880 
881 		switch (ns->pi_type) {
882 		case NVME_NS_DPS_PI_TYPE3:
883 			control |= NVME_RW_PRINFO_PRCHK_GUARD;
884 			break;
885 		case NVME_NS_DPS_PI_TYPE1:
886 		case NVME_NS_DPS_PI_TYPE2:
887 			control |= NVME_RW_PRINFO_PRCHK_GUARD |
888 					NVME_RW_PRINFO_PRCHK_REF;
889 			if (op == nvme_cmd_zone_append)
890 				control |= NVME_RW_APPEND_PIREMAP;
891 			nvme_set_ref_tag(ns, cmnd, req);
892 			break;
893 		}
894 	}
895 
896 	cmnd->rw.control = cpu_to_le16(control);
897 	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
898 	return 0;
899 }
900 
901 void nvme_cleanup_cmd(struct request *req)
902 {
903 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
904 		struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
905 
906 		if (req->special_vec.bv_page == ctrl->discard_page)
907 			clear_bit_unlock(0, &ctrl->discard_page_busy);
908 		else
909 			kfree(bvec_virt(&req->special_vec));
910 	}
911 }
912 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
913 
914 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
915 {
916 	struct nvme_command *cmd = nvme_req(req)->cmd;
917 	blk_status_t ret = BLK_STS_OK;
918 
919 	if (!(req->rq_flags & RQF_DONTPREP))
920 		nvme_clear_nvme_request(req);
921 
922 	switch (req_op(req)) {
923 	case REQ_OP_DRV_IN:
924 	case REQ_OP_DRV_OUT:
925 		/* these are setup prior to execution in nvme_init_request() */
926 		break;
927 	case REQ_OP_FLUSH:
928 		nvme_setup_flush(ns, cmd);
929 		break;
930 	case REQ_OP_ZONE_RESET_ALL:
931 	case REQ_OP_ZONE_RESET:
932 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
933 		break;
934 	case REQ_OP_ZONE_OPEN:
935 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
936 		break;
937 	case REQ_OP_ZONE_CLOSE:
938 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
939 		break;
940 	case REQ_OP_ZONE_FINISH:
941 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
942 		break;
943 	case REQ_OP_WRITE_ZEROES:
944 		ret = nvme_setup_write_zeroes(ns, req, cmd);
945 		break;
946 	case REQ_OP_DISCARD:
947 		ret = nvme_setup_discard(ns, req, cmd);
948 		break;
949 	case REQ_OP_READ:
950 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
951 		break;
952 	case REQ_OP_WRITE:
953 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
954 		break;
955 	case REQ_OP_ZONE_APPEND:
956 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
957 		break;
958 	default:
959 		WARN_ON_ONCE(1);
960 		return BLK_STS_IOERR;
961 	}
962 
963 	cmd->common.command_id = nvme_cid(req);
964 	trace_nvme_setup_cmd(req, cmd);
965 	return ret;
966 }
967 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
968 
969 /*
970  * Return values:
971  * 0:  success
972  * >0: nvme controller's cqe status response
973  * <0: kernel error in lieu of controller response
974  */
975 static int nvme_execute_rq(struct request *rq, bool at_head)
976 {
977 	blk_status_t status;
978 
979 	status = blk_execute_rq(rq, at_head);
980 	if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
981 		return -EINTR;
982 	if (nvme_req(rq)->status)
983 		return nvme_req(rq)->status;
984 	return blk_status_to_errno(status);
985 }
986 
987 /*
988  * Returns 0 on success.  If the result is negative, it's a Linux error code;
989  * if the result is positive, it's an NVM Express status code
990  */
991 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
992 		union nvme_result *result, void *buffer, unsigned bufflen,
993 		unsigned timeout, int qid, int at_head,
994 		blk_mq_req_flags_t flags)
995 {
996 	struct request *req;
997 	int ret;
998 
999 	if (qid == NVME_QID_ANY)
1000 		req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
1001 	else
1002 		req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1003 						qid ? qid - 1 : 0);
1004 
1005 	if (IS_ERR(req))
1006 		return PTR_ERR(req);
1007 	nvme_init_request(req, cmd);
1008 
1009 	if (timeout)
1010 		req->timeout = timeout;
1011 
1012 	if (buffer && bufflen) {
1013 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1014 		if (ret)
1015 			goto out;
1016 	}
1017 
1018 	req->rq_flags |= RQF_QUIET;
1019 	ret = nvme_execute_rq(req, at_head);
1020 	if (result && ret >= 0)
1021 		*result = nvme_req(req)->result;
1022  out:
1023 	blk_mq_free_request(req);
1024 	return ret;
1025 }
1026 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1027 
1028 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1029 		void *buffer, unsigned bufflen)
1030 {
1031 	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
1032 			NVME_QID_ANY, 0, 0);
1033 }
1034 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1035 
1036 static u32 nvme_known_admin_effects(u8 opcode)
1037 {
1038 	switch (opcode) {
1039 	case nvme_admin_format_nvm:
1040 		return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
1041 			NVME_CMD_EFFECTS_CSE_MASK;
1042 	case nvme_admin_sanitize_nvm:
1043 		return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
1044 	default:
1045 		break;
1046 	}
1047 	return 0;
1048 }
1049 
1050 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1051 {
1052 	u32 effects = 0;
1053 
1054 	if (ns) {
1055 		if (ns->head->effects)
1056 			effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1057 		if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1058 			dev_warn_once(ctrl->device,
1059 				"IO command:%02x has unhandled effects:%08x\n",
1060 				opcode, effects);
1061 		return 0;
1062 	}
1063 
1064 	if (ctrl->effects)
1065 		effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1066 	effects |= nvme_known_admin_effects(opcode);
1067 
1068 	return effects;
1069 }
1070 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1071 
1072 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1073 			       u8 opcode)
1074 {
1075 	u32 effects = nvme_command_effects(ctrl, ns, opcode);
1076 
1077 	/*
1078 	 * For simplicity, IO to all namespaces is quiesced even if the command
1079 	 * effects say only one namespace is affected.
1080 	 */
1081 	if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1082 		mutex_lock(&ctrl->scan_lock);
1083 		mutex_lock(&ctrl->subsys->lock);
1084 		nvme_mpath_start_freeze(ctrl->subsys);
1085 		nvme_mpath_wait_freeze(ctrl->subsys);
1086 		nvme_start_freeze(ctrl);
1087 		nvme_wait_freeze(ctrl);
1088 	}
1089 	return effects;
1090 }
1091 
1092 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1093 			      struct nvme_command *cmd, int status)
1094 {
1095 	if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1096 		nvme_unfreeze(ctrl);
1097 		nvme_mpath_unfreeze(ctrl->subsys);
1098 		mutex_unlock(&ctrl->subsys->lock);
1099 		nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1100 		mutex_unlock(&ctrl->scan_lock);
1101 	}
1102 	if (effects & NVME_CMD_EFFECTS_CCC)
1103 		nvme_init_ctrl_finish(ctrl);
1104 	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1105 		nvme_queue_scan(ctrl);
1106 		flush_work(&ctrl->scan_work);
1107 	}
1108 
1109 	switch (cmd->common.opcode) {
1110 	case nvme_admin_set_features:
1111 		switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1112 		case NVME_FEAT_KATO:
1113 			/*
1114 			 * Keep alive commands interval on the host should be
1115 			 * updated when KATO is modified by Set Features
1116 			 * commands.
1117 			 */
1118 			if (!status)
1119 				nvme_update_keep_alive(ctrl, cmd);
1120 			break;
1121 		default:
1122 			break;
1123 		}
1124 		break;
1125 	default:
1126 		break;
1127 	}
1128 }
1129 
1130 int nvme_execute_passthru_rq(struct request *rq)
1131 {
1132 	struct nvme_command *cmd = nvme_req(rq)->cmd;
1133 	struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1134 	struct nvme_ns *ns = rq->q->queuedata;
1135 	u32 effects;
1136 	int  ret;
1137 
1138 	effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1139 	ret = nvme_execute_rq(rq, false);
1140 	if (effects) /* nothing to be done for zero cmd effects */
1141 		nvme_passthru_end(ctrl, effects, cmd, ret);
1142 
1143 	return ret;
1144 }
1145 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1146 
1147 /*
1148  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1149  *
1150  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1151  *   accounting for transport roundtrip times [..].
1152  */
1153 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1154 {
1155 	queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
1156 }
1157 
1158 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1159 {
1160 	struct nvme_ctrl *ctrl = rq->end_io_data;
1161 	unsigned long flags;
1162 	bool startka = false;
1163 
1164 	blk_mq_free_request(rq);
1165 
1166 	if (status) {
1167 		dev_err(ctrl->device,
1168 			"failed nvme_keep_alive_end_io error=%d\n",
1169 				status);
1170 		return;
1171 	}
1172 
1173 	ctrl->comp_seen = false;
1174 	spin_lock_irqsave(&ctrl->lock, flags);
1175 	if (ctrl->state == NVME_CTRL_LIVE ||
1176 	    ctrl->state == NVME_CTRL_CONNECTING)
1177 		startka = true;
1178 	spin_unlock_irqrestore(&ctrl->lock, flags);
1179 	if (startka)
1180 		nvme_queue_keep_alive_work(ctrl);
1181 }
1182 
1183 static void nvme_keep_alive_work(struct work_struct *work)
1184 {
1185 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1186 			struct nvme_ctrl, ka_work);
1187 	bool comp_seen = ctrl->comp_seen;
1188 	struct request *rq;
1189 
1190 	if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1191 		dev_dbg(ctrl->device,
1192 			"reschedule traffic based keep-alive timer\n");
1193 		ctrl->comp_seen = false;
1194 		nvme_queue_keep_alive_work(ctrl);
1195 		return;
1196 	}
1197 
1198 	rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1199 				  BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1200 	if (IS_ERR(rq)) {
1201 		/* allocation failure, reset the controller */
1202 		dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1203 		nvme_reset_ctrl(ctrl);
1204 		return;
1205 	}
1206 	nvme_init_request(rq, &ctrl->ka_cmd);
1207 
1208 	rq->timeout = ctrl->kato * HZ;
1209 	rq->end_io = nvme_keep_alive_end_io;
1210 	rq->end_io_data = ctrl;
1211 	rq->rq_flags |= RQF_QUIET;
1212 	blk_execute_rq_nowait(rq, false);
1213 }
1214 
1215 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1216 {
1217 	if (unlikely(ctrl->kato == 0))
1218 		return;
1219 
1220 	nvme_queue_keep_alive_work(ctrl);
1221 }
1222 
1223 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1224 {
1225 	if (unlikely(ctrl->kato == 0))
1226 		return;
1227 
1228 	cancel_delayed_work_sync(&ctrl->ka_work);
1229 }
1230 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1231 
1232 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1233 				   struct nvme_command *cmd)
1234 {
1235 	unsigned int new_kato =
1236 		DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1237 
1238 	dev_info(ctrl->device,
1239 		 "keep alive interval updated from %u ms to %u ms\n",
1240 		 ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1241 
1242 	nvme_stop_keep_alive(ctrl);
1243 	ctrl->kato = new_kato;
1244 	nvme_start_keep_alive(ctrl);
1245 }
1246 
1247 /*
1248  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1249  * flag, thus sending any new CNS opcodes has a big chance of not working.
1250  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1251  * (but not for any later version).
1252  */
1253 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1254 {
1255 	if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1256 		return ctrl->vs < NVME_VS(1, 2, 0);
1257 	return ctrl->vs < NVME_VS(1, 1, 0);
1258 }
1259 
1260 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1261 {
1262 	struct nvme_command c = { };
1263 	int error;
1264 
1265 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1266 	c.identify.opcode = nvme_admin_identify;
1267 	c.identify.cns = NVME_ID_CNS_CTRL;
1268 
1269 	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1270 	if (!*id)
1271 		return -ENOMEM;
1272 
1273 	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1274 			sizeof(struct nvme_id_ctrl));
1275 	if (error)
1276 		kfree(*id);
1277 	return error;
1278 }
1279 
1280 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1281 		struct nvme_ns_id_desc *cur, bool *csi_seen)
1282 {
1283 	const char *warn_str = "ctrl returned bogus length:";
1284 	void *data = cur;
1285 
1286 	switch (cur->nidt) {
1287 	case NVME_NIDT_EUI64:
1288 		if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1289 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1290 				 warn_str, cur->nidl);
1291 			return -1;
1292 		}
1293 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1294 			return NVME_NIDT_EUI64_LEN;
1295 		memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1296 		return NVME_NIDT_EUI64_LEN;
1297 	case NVME_NIDT_NGUID:
1298 		if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1299 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1300 				 warn_str, cur->nidl);
1301 			return -1;
1302 		}
1303 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1304 			return NVME_NIDT_NGUID_LEN;
1305 		memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1306 		return NVME_NIDT_NGUID_LEN;
1307 	case NVME_NIDT_UUID:
1308 		if (cur->nidl != NVME_NIDT_UUID_LEN) {
1309 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1310 				 warn_str, cur->nidl);
1311 			return -1;
1312 		}
1313 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1314 			return NVME_NIDT_UUID_LEN;
1315 		uuid_copy(&ids->uuid, data + sizeof(*cur));
1316 		return NVME_NIDT_UUID_LEN;
1317 	case NVME_NIDT_CSI:
1318 		if (cur->nidl != NVME_NIDT_CSI_LEN) {
1319 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1320 				 warn_str, cur->nidl);
1321 			return -1;
1322 		}
1323 		memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1324 		*csi_seen = true;
1325 		return NVME_NIDT_CSI_LEN;
1326 	default:
1327 		/* Skip unknown types */
1328 		return cur->nidl;
1329 	}
1330 }
1331 
1332 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1333 		struct nvme_ns_ids *ids)
1334 {
1335 	struct nvme_command c = { };
1336 	bool csi_seen = false;
1337 	int status, pos, len;
1338 	void *data;
1339 
1340 	if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1341 		return 0;
1342 	if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1343 		return 0;
1344 
1345 	c.identify.opcode = nvme_admin_identify;
1346 	c.identify.nsid = cpu_to_le32(nsid);
1347 	c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1348 
1349 	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1350 	if (!data)
1351 		return -ENOMEM;
1352 
1353 	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1354 				      NVME_IDENTIFY_DATA_SIZE);
1355 	if (status) {
1356 		dev_warn(ctrl->device,
1357 			"Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1358 			nsid, status);
1359 		goto free_data;
1360 	}
1361 
1362 	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1363 		struct nvme_ns_id_desc *cur = data + pos;
1364 
1365 		if (cur->nidl == 0)
1366 			break;
1367 
1368 		len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1369 		if (len < 0)
1370 			break;
1371 
1372 		len += sizeof(*cur);
1373 	}
1374 
1375 	if (nvme_multi_css(ctrl) && !csi_seen) {
1376 		dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1377 			 nsid);
1378 		status = -EINVAL;
1379 	}
1380 
1381 free_data:
1382 	kfree(data);
1383 	return status;
1384 }
1385 
1386 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1387 			struct nvme_ns_ids *ids, struct nvme_id_ns **id)
1388 {
1389 	struct nvme_command c = { };
1390 	int error;
1391 
1392 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1393 	c.identify.opcode = nvme_admin_identify;
1394 	c.identify.nsid = cpu_to_le32(nsid);
1395 	c.identify.cns = NVME_ID_CNS_NS;
1396 
1397 	*id = kmalloc(sizeof(**id), GFP_KERNEL);
1398 	if (!*id)
1399 		return -ENOMEM;
1400 
1401 	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1402 	if (error) {
1403 		dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1404 		goto out_free_id;
1405 	}
1406 
1407 	error = NVME_SC_INVALID_NS | NVME_SC_DNR;
1408 	if ((*id)->ncap == 0) /* namespace not allocated or attached */
1409 		goto out_free_id;
1410 
1411 
1412 	if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1413 		dev_info(ctrl->device,
1414 			 "Ignoring bogus Namespace Identifiers\n");
1415 	} else {
1416 		if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1417 		    !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1418 			memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
1419 		if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1420 		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1421 			memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
1422 	}
1423 
1424 	return 0;
1425 
1426 out_free_id:
1427 	kfree(*id);
1428 	return error;
1429 }
1430 
1431 static int nvme_identify_ns_cs_indep(struct nvme_ctrl *ctrl, unsigned nsid,
1432 			struct nvme_id_ns_cs_indep **id)
1433 {
1434 	struct nvme_command c = {
1435 		.identify.opcode	= nvme_admin_identify,
1436 		.identify.nsid		= cpu_to_le32(nsid),
1437 		.identify.cns		= NVME_ID_CNS_NS_CS_INDEP,
1438 	};
1439 	int ret;
1440 
1441 	*id = kmalloc(sizeof(**id), GFP_KERNEL);
1442 	if (!*id)
1443 		return -ENOMEM;
1444 
1445 	ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1446 	if (ret) {
1447 		dev_warn(ctrl->device,
1448 			 "Identify namespace (CS independent) failed (%d)\n",
1449 			 ret);
1450 		kfree(*id);
1451 		return ret;
1452 	}
1453 
1454 	return 0;
1455 }
1456 
1457 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1458 		unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1459 {
1460 	union nvme_result res = { 0 };
1461 	struct nvme_command c = { };
1462 	int ret;
1463 
1464 	c.features.opcode = op;
1465 	c.features.fid = cpu_to_le32(fid);
1466 	c.features.dword11 = cpu_to_le32(dword11);
1467 
1468 	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1469 			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
1470 	if (ret >= 0 && result)
1471 		*result = le32_to_cpu(res.u32);
1472 	return ret;
1473 }
1474 
1475 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1476 		      unsigned int dword11, void *buffer, size_t buflen,
1477 		      u32 *result)
1478 {
1479 	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1480 			     buflen, result);
1481 }
1482 EXPORT_SYMBOL_GPL(nvme_set_features);
1483 
1484 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1485 		      unsigned int dword11, void *buffer, size_t buflen,
1486 		      u32 *result)
1487 {
1488 	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1489 			     buflen, result);
1490 }
1491 EXPORT_SYMBOL_GPL(nvme_get_features);
1492 
1493 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1494 {
1495 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
1496 	u32 result;
1497 	int status, nr_io_queues;
1498 
1499 	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1500 			&result);
1501 	if (status < 0)
1502 		return status;
1503 
1504 	/*
1505 	 * Degraded controllers might return an error when setting the queue
1506 	 * count.  We still want to be able to bring them online and offer
1507 	 * access to the admin queue, as that might be only way to fix them up.
1508 	 */
1509 	if (status > 0) {
1510 		dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1511 		*count = 0;
1512 	} else {
1513 		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1514 		*count = min(*count, nr_io_queues);
1515 	}
1516 
1517 	return 0;
1518 }
1519 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1520 
1521 #define NVME_AEN_SUPPORTED \
1522 	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1523 	 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1524 
1525 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1526 {
1527 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1528 	int status;
1529 
1530 	if (!supported_aens)
1531 		return;
1532 
1533 	status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1534 			NULL, 0, &result);
1535 	if (status)
1536 		dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1537 			 supported_aens);
1538 
1539 	queue_work(nvme_wq, &ctrl->async_event_work);
1540 }
1541 
1542 static int nvme_ns_open(struct nvme_ns *ns)
1543 {
1544 
1545 	/* should never be called due to GENHD_FL_HIDDEN */
1546 	if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1547 		goto fail;
1548 	if (!nvme_get_ns(ns))
1549 		goto fail;
1550 	if (!try_module_get(ns->ctrl->ops->module))
1551 		goto fail_put_ns;
1552 
1553 	return 0;
1554 
1555 fail_put_ns:
1556 	nvme_put_ns(ns);
1557 fail:
1558 	return -ENXIO;
1559 }
1560 
1561 static void nvme_ns_release(struct nvme_ns *ns)
1562 {
1563 
1564 	module_put(ns->ctrl->ops->module);
1565 	nvme_put_ns(ns);
1566 }
1567 
1568 static int nvme_open(struct block_device *bdev, fmode_t mode)
1569 {
1570 	return nvme_ns_open(bdev->bd_disk->private_data);
1571 }
1572 
1573 static void nvme_release(struct gendisk *disk, fmode_t mode)
1574 {
1575 	nvme_ns_release(disk->private_data);
1576 }
1577 
1578 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1579 {
1580 	/* some standard values */
1581 	geo->heads = 1 << 6;
1582 	geo->sectors = 1 << 5;
1583 	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1584 	return 0;
1585 }
1586 
1587 #ifdef CONFIG_BLK_DEV_INTEGRITY
1588 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1589 				u32 max_integrity_segments)
1590 {
1591 	struct blk_integrity integrity = { };
1592 
1593 	switch (ns->pi_type) {
1594 	case NVME_NS_DPS_PI_TYPE3:
1595 		switch (ns->guard_type) {
1596 		case NVME_NVM_NS_16B_GUARD:
1597 			integrity.profile = &t10_pi_type3_crc;
1598 			integrity.tag_size = sizeof(u16) + sizeof(u32);
1599 			integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1600 			break;
1601 		case NVME_NVM_NS_64B_GUARD:
1602 			integrity.profile = &ext_pi_type3_crc64;
1603 			integrity.tag_size = sizeof(u16) + 6;
1604 			integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1605 			break;
1606 		default:
1607 			integrity.profile = NULL;
1608 			break;
1609 		}
1610 		break;
1611 	case NVME_NS_DPS_PI_TYPE1:
1612 	case NVME_NS_DPS_PI_TYPE2:
1613 		switch (ns->guard_type) {
1614 		case NVME_NVM_NS_16B_GUARD:
1615 			integrity.profile = &t10_pi_type1_crc;
1616 			integrity.tag_size = sizeof(u16);
1617 			integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1618 			break;
1619 		case NVME_NVM_NS_64B_GUARD:
1620 			integrity.profile = &ext_pi_type1_crc64;
1621 			integrity.tag_size = sizeof(u16);
1622 			integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1623 			break;
1624 		default:
1625 			integrity.profile = NULL;
1626 			break;
1627 		}
1628 		break;
1629 	default:
1630 		integrity.profile = NULL;
1631 		break;
1632 	}
1633 
1634 	integrity.tuple_size = ns->ms;
1635 	blk_integrity_register(disk, &integrity);
1636 	blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1637 }
1638 #else
1639 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1640 				u32 max_integrity_segments)
1641 {
1642 }
1643 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1644 
1645 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1646 {
1647 	struct nvme_ctrl *ctrl = ns->ctrl;
1648 	struct request_queue *queue = disk->queue;
1649 	u32 size = queue_logical_block_size(queue);
1650 
1651 	if (ctrl->max_discard_sectors == 0) {
1652 		blk_queue_max_discard_sectors(queue, 0);
1653 		return;
1654 	}
1655 
1656 	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1657 			NVME_DSM_MAX_RANGES);
1658 
1659 	queue->limits.discard_granularity = size;
1660 
1661 	/* If discard is already enabled, don't reset queue limits */
1662 	if (queue->limits.max_discard_sectors)
1663 		return;
1664 
1665 	if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX))
1666 		ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl);
1667 
1668 	blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1669 	blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
1670 
1671 	if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1672 		blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1673 }
1674 
1675 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1676 {
1677 	return uuid_equal(&a->uuid, &b->uuid) &&
1678 		memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1679 		memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1680 		a->csi == b->csi;
1681 }
1682 
1683 static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
1684 {
1685 	bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1686 	unsigned lbaf = nvme_lbaf_index(id->flbas);
1687 	struct nvme_ctrl *ctrl = ns->ctrl;
1688 	struct nvme_command c = { };
1689 	struct nvme_id_ns_nvm *nvm;
1690 	int ret = 0;
1691 	u32 elbaf;
1692 
1693 	ns->pi_size = 0;
1694 	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1695 	if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1696 		ns->pi_size = sizeof(struct t10_pi_tuple);
1697 		ns->guard_type = NVME_NVM_NS_16B_GUARD;
1698 		goto set_pi;
1699 	}
1700 
1701 	nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1702 	if (!nvm)
1703 		return -ENOMEM;
1704 
1705 	c.identify.opcode = nvme_admin_identify;
1706 	c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1707 	c.identify.cns = NVME_ID_CNS_CS_NS;
1708 	c.identify.csi = NVME_CSI_NVM;
1709 
1710 	ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1711 	if (ret)
1712 		goto free_data;
1713 
1714 	elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1715 
1716 	/* no support for storage tag formats right now */
1717 	if (nvme_elbaf_sts(elbaf))
1718 		goto free_data;
1719 
1720 	ns->guard_type = nvme_elbaf_guard_type(elbaf);
1721 	switch (ns->guard_type) {
1722 	case NVME_NVM_NS_64B_GUARD:
1723 		ns->pi_size = sizeof(struct crc64_pi_tuple);
1724 		break;
1725 	case NVME_NVM_NS_16B_GUARD:
1726 		ns->pi_size = sizeof(struct t10_pi_tuple);
1727 		break;
1728 	default:
1729 		break;
1730 	}
1731 
1732 free_data:
1733 	kfree(nvm);
1734 set_pi:
1735 	if (ns->pi_size && (first || ns->ms == ns->pi_size))
1736 		ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1737 	else
1738 		ns->pi_type = 0;
1739 
1740 	return ret;
1741 }
1742 
1743 static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1744 {
1745 	struct nvme_ctrl *ctrl = ns->ctrl;
1746 
1747 	if (nvme_init_ms(ns, id))
1748 		return;
1749 
1750 	ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1751 	if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1752 		return;
1753 
1754 	if (ctrl->ops->flags & NVME_F_FABRICS) {
1755 		/*
1756 		 * The NVMe over Fabrics specification only supports metadata as
1757 		 * part of the extended data LBA.  We rely on HCA/HBA support to
1758 		 * remap the separate metadata buffer from the block layer.
1759 		 */
1760 		if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1761 			return;
1762 
1763 		ns->features |= NVME_NS_EXT_LBAS;
1764 
1765 		/*
1766 		 * The current fabrics transport drivers support namespace
1767 		 * metadata formats only if nvme_ns_has_pi() returns true.
1768 		 * Suppress support for all other formats so the namespace will
1769 		 * have a 0 capacity and not be usable through the block stack.
1770 		 *
1771 		 * Note, this check will need to be modified if any drivers
1772 		 * gain the ability to use other metadata formats.
1773 		 */
1774 		if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1775 			ns->features |= NVME_NS_METADATA_SUPPORTED;
1776 	} else {
1777 		/*
1778 		 * For PCIe controllers, we can't easily remap the separate
1779 		 * metadata buffer from the block layer and thus require a
1780 		 * separate metadata buffer for block layer metadata/PI support.
1781 		 * We allow extended LBAs for the passthrough interface, though.
1782 		 */
1783 		if (id->flbas & NVME_NS_FLBAS_META_EXT)
1784 			ns->features |= NVME_NS_EXT_LBAS;
1785 		else
1786 			ns->features |= NVME_NS_METADATA_SUPPORTED;
1787 	}
1788 }
1789 
1790 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1791 		struct request_queue *q)
1792 {
1793 	bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1794 
1795 	if (ctrl->max_hw_sectors) {
1796 		u32 max_segments =
1797 			(ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1798 
1799 		max_segments = min_not_zero(max_segments, ctrl->max_segments);
1800 		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1801 		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1802 	}
1803 	blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1804 	blk_queue_dma_alignment(q, 3);
1805 	blk_queue_write_cache(q, vwc, vwc);
1806 }
1807 
1808 static void nvme_update_disk_info(struct gendisk *disk,
1809 		struct nvme_ns *ns, struct nvme_id_ns *id)
1810 {
1811 	sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1812 	unsigned short bs = 1 << ns->lba_shift;
1813 	u32 atomic_bs, phys_bs, io_opt = 0;
1814 
1815 	/*
1816 	 * The block layer can't support LBA sizes larger than the page size
1817 	 * yet, so catch this early and don't allow block I/O.
1818 	 */
1819 	if (ns->lba_shift > PAGE_SHIFT) {
1820 		capacity = 0;
1821 		bs = (1 << 9);
1822 	}
1823 
1824 	blk_integrity_unregister(disk);
1825 
1826 	atomic_bs = phys_bs = bs;
1827 	if (id->nabo == 0) {
1828 		/*
1829 		 * Bit 1 indicates whether NAWUPF is defined for this namespace
1830 		 * and whether it should be used instead of AWUPF. If NAWUPF ==
1831 		 * 0 then AWUPF must be used instead.
1832 		 */
1833 		if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1834 			atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1835 		else
1836 			atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1837 	}
1838 
1839 	if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1840 		/* NPWG = Namespace Preferred Write Granularity */
1841 		phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1842 		/* NOWS = Namespace Optimal Write Size */
1843 		io_opt = bs * (1 + le16_to_cpu(id->nows));
1844 	}
1845 
1846 	blk_queue_logical_block_size(disk->queue, bs);
1847 	/*
1848 	 * Linux filesystems assume writing a single physical block is
1849 	 * an atomic operation. Hence limit the physical block size to the
1850 	 * value of the Atomic Write Unit Power Fail parameter.
1851 	 */
1852 	blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1853 	blk_queue_io_min(disk->queue, phys_bs);
1854 	blk_queue_io_opt(disk->queue, io_opt);
1855 
1856 	/*
1857 	 * Register a metadata profile for PI, or the plain non-integrity NVMe
1858 	 * metadata masquerading as Type 0 if supported, otherwise reject block
1859 	 * I/O to namespaces with metadata except when the namespace supports
1860 	 * PI, as it can strip/insert in that case.
1861 	 */
1862 	if (ns->ms) {
1863 		if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1864 		    (ns->features & NVME_NS_METADATA_SUPPORTED))
1865 			nvme_init_integrity(disk, ns,
1866 					    ns->ctrl->max_integrity_segments);
1867 		else if (!nvme_ns_has_pi(ns))
1868 			capacity = 0;
1869 	}
1870 
1871 	set_capacity_and_notify(disk, capacity);
1872 
1873 	nvme_config_discard(disk, ns);
1874 	blk_queue_max_write_zeroes_sectors(disk->queue,
1875 					   ns->ctrl->max_zeroes_sectors);
1876 }
1877 
1878 static inline bool nvme_first_scan(struct gendisk *disk)
1879 {
1880 	/* nvme_alloc_ns() scans the disk prior to adding it */
1881 	return !disk_live(disk);
1882 }
1883 
1884 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1885 {
1886 	struct nvme_ctrl *ctrl = ns->ctrl;
1887 	u32 iob;
1888 
1889 	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1890 	    is_power_of_2(ctrl->max_hw_sectors))
1891 		iob = ctrl->max_hw_sectors;
1892 	else
1893 		iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1894 
1895 	if (!iob)
1896 		return;
1897 
1898 	if (!is_power_of_2(iob)) {
1899 		if (nvme_first_scan(ns->disk))
1900 			pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1901 				ns->disk->disk_name, iob);
1902 		return;
1903 	}
1904 
1905 	if (blk_queue_is_zoned(ns->disk->queue)) {
1906 		if (nvme_first_scan(ns->disk))
1907 			pr_warn("%s: ignoring zoned namespace IO boundary\n",
1908 				ns->disk->disk_name);
1909 		return;
1910 	}
1911 
1912 	blk_queue_chunk_sectors(ns->queue, iob);
1913 }
1914 
1915 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
1916 {
1917 	unsigned lbaf = nvme_lbaf_index(id->flbas);
1918 	int ret;
1919 
1920 	blk_mq_freeze_queue(ns->disk->queue);
1921 	ns->lba_shift = id->lbaf[lbaf].ds;
1922 	nvme_set_queue_limits(ns->ctrl, ns->queue);
1923 
1924 	nvme_configure_metadata(ns, id);
1925 	nvme_set_chunk_sectors(ns, id);
1926 	nvme_update_disk_info(ns->disk, ns, id);
1927 
1928 	if (ns->head->ids.csi == NVME_CSI_ZNS) {
1929 		ret = nvme_update_zone_info(ns, lbaf);
1930 		if (ret)
1931 			goto out_unfreeze;
1932 	}
1933 
1934 	set_disk_ro(ns->disk, (id->nsattr & NVME_NS_ATTR_RO) ||
1935 		test_bit(NVME_NS_FORCE_RO, &ns->flags));
1936 	set_bit(NVME_NS_READY, &ns->flags);
1937 	blk_mq_unfreeze_queue(ns->disk->queue);
1938 
1939 	if (blk_queue_is_zoned(ns->queue)) {
1940 		ret = nvme_revalidate_zones(ns);
1941 		if (ret && !nvme_first_scan(ns->disk))
1942 			return ret;
1943 	}
1944 
1945 	if (nvme_ns_head_multipath(ns->head)) {
1946 		blk_mq_freeze_queue(ns->head->disk->queue);
1947 		nvme_update_disk_info(ns->head->disk, ns, id);
1948 		set_disk_ro(ns->head->disk,
1949 			    (id->nsattr & NVME_NS_ATTR_RO) ||
1950 				    test_bit(NVME_NS_FORCE_RO, &ns->flags));
1951 		nvme_mpath_revalidate_paths(ns);
1952 		blk_stack_limits(&ns->head->disk->queue->limits,
1953 				 &ns->queue->limits, 0);
1954 		disk_update_readahead(ns->head->disk);
1955 		blk_mq_unfreeze_queue(ns->head->disk->queue);
1956 	}
1957 	return 0;
1958 
1959 out_unfreeze:
1960 	/*
1961 	 * If probing fails due an unsupported feature, hide the block device,
1962 	 * but still allow other access.
1963 	 */
1964 	if (ret == -ENODEV) {
1965 		ns->disk->flags |= GENHD_FL_HIDDEN;
1966 		set_bit(NVME_NS_READY, &ns->flags);
1967 		ret = 0;
1968 	}
1969 	blk_mq_unfreeze_queue(ns->disk->queue);
1970 	return ret;
1971 }
1972 
1973 static char nvme_pr_type(enum pr_type type)
1974 {
1975 	switch (type) {
1976 	case PR_WRITE_EXCLUSIVE:
1977 		return 1;
1978 	case PR_EXCLUSIVE_ACCESS:
1979 		return 2;
1980 	case PR_WRITE_EXCLUSIVE_REG_ONLY:
1981 		return 3;
1982 	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1983 		return 4;
1984 	case PR_WRITE_EXCLUSIVE_ALL_REGS:
1985 		return 5;
1986 	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1987 		return 6;
1988 	default:
1989 		return 0;
1990 	}
1991 }
1992 
1993 static int nvme_send_ns_head_pr_command(struct block_device *bdev,
1994 		struct nvme_command *c, u8 data[16])
1995 {
1996 	struct nvme_ns_head *head = bdev->bd_disk->private_data;
1997 	int srcu_idx = srcu_read_lock(&head->srcu);
1998 	struct nvme_ns *ns = nvme_find_path(head);
1999 	int ret = -EWOULDBLOCK;
2000 
2001 	if (ns) {
2002 		c->common.nsid = cpu_to_le32(ns->head->ns_id);
2003 		ret = nvme_submit_sync_cmd(ns->queue, c, data, 16);
2004 	}
2005 	srcu_read_unlock(&head->srcu, srcu_idx);
2006 	return ret;
2007 }
2008 
2009 static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
2010 		u8 data[16])
2011 {
2012 	c->common.nsid = cpu_to_le32(ns->head->ns_id);
2013 	return nvme_submit_sync_cmd(ns->queue, c, data, 16);
2014 }
2015 
2016 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2017 				u64 key, u64 sa_key, u8 op)
2018 {
2019 	struct nvme_command c = { };
2020 	u8 data[16] = { 0, };
2021 
2022 	put_unaligned_le64(key, &data[0]);
2023 	put_unaligned_le64(sa_key, &data[8]);
2024 
2025 	c.common.opcode = op;
2026 	c.common.cdw10 = cpu_to_le32(cdw10);
2027 
2028 	if (IS_ENABLED(CONFIG_NVME_MULTIPATH) &&
2029 	    bdev->bd_disk->fops == &nvme_ns_head_ops)
2030 		return nvme_send_ns_head_pr_command(bdev, &c, data);
2031 	return nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c, data);
2032 }
2033 
2034 static int nvme_pr_register(struct block_device *bdev, u64 old,
2035 		u64 new, unsigned flags)
2036 {
2037 	u32 cdw10;
2038 
2039 	if (flags & ~PR_FL_IGNORE_KEY)
2040 		return -EOPNOTSUPP;
2041 
2042 	cdw10 = old ? 2 : 0;
2043 	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2044 	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2045 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2046 }
2047 
2048 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2049 		enum pr_type type, unsigned flags)
2050 {
2051 	u32 cdw10;
2052 
2053 	if (flags & ~PR_FL_IGNORE_KEY)
2054 		return -EOPNOTSUPP;
2055 
2056 	cdw10 = nvme_pr_type(type) << 8;
2057 	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2058 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2059 }
2060 
2061 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2062 		enum pr_type type, bool abort)
2063 {
2064 	u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2065 
2066 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2067 }
2068 
2069 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2070 {
2071 	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
2072 
2073 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2074 }
2075 
2076 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2077 {
2078 	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
2079 
2080 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2081 }
2082 
2083 const struct pr_ops nvme_pr_ops = {
2084 	.pr_register	= nvme_pr_register,
2085 	.pr_reserve	= nvme_pr_reserve,
2086 	.pr_release	= nvme_pr_release,
2087 	.pr_preempt	= nvme_pr_preempt,
2088 	.pr_clear	= nvme_pr_clear,
2089 };
2090 
2091 #ifdef CONFIG_BLK_SED_OPAL
2092 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2093 		bool send)
2094 {
2095 	struct nvme_ctrl *ctrl = data;
2096 	struct nvme_command cmd = { };
2097 
2098 	if (send)
2099 		cmd.common.opcode = nvme_admin_security_send;
2100 	else
2101 		cmd.common.opcode = nvme_admin_security_recv;
2102 	cmd.common.nsid = 0;
2103 	cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2104 	cmd.common.cdw11 = cpu_to_le32(len);
2105 
2106 	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len, 0,
2107 			NVME_QID_ANY, 1, 0);
2108 }
2109 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2110 #endif /* CONFIG_BLK_SED_OPAL */
2111 
2112 #ifdef CONFIG_BLK_DEV_ZONED
2113 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2114 		unsigned int nr_zones, report_zones_cb cb, void *data)
2115 {
2116 	return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2117 			data);
2118 }
2119 #else
2120 #define nvme_report_zones	NULL
2121 #endif /* CONFIG_BLK_DEV_ZONED */
2122 
2123 static const struct block_device_operations nvme_bdev_ops = {
2124 	.owner		= THIS_MODULE,
2125 	.ioctl		= nvme_ioctl,
2126 	.open		= nvme_open,
2127 	.release	= nvme_release,
2128 	.getgeo		= nvme_getgeo,
2129 	.report_zones	= nvme_report_zones,
2130 	.pr_ops		= &nvme_pr_ops,
2131 };
2132 
2133 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)
2134 {
2135 	unsigned long timeout_jiffies = ((timeout + 1) * HZ / 2) + jiffies;
2136 	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2137 	int ret;
2138 
2139 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2140 		if (csts == ~0)
2141 			return -ENODEV;
2142 		if ((csts & NVME_CSTS_RDY) == bit)
2143 			break;
2144 
2145 		usleep_range(1000, 2000);
2146 		if (fatal_signal_pending(current))
2147 			return -EINTR;
2148 		if (time_after(jiffies, timeout_jiffies)) {
2149 			dev_err(ctrl->device,
2150 				"Device not ready; aborting %s, CSTS=0x%x\n",
2151 				enabled ? "initialisation" : "reset", csts);
2152 			return -ENODEV;
2153 		}
2154 	}
2155 
2156 	return ret;
2157 }
2158 
2159 /*
2160  * If the device has been passed off to us in an enabled state, just clear
2161  * the enabled bit.  The spec says we should set the 'shutdown notification
2162  * bits', but doing so may cause the device to complete commands to the
2163  * admin queue ... and we don't know what memory that might be pointing at!
2164  */
2165 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2166 {
2167 	int ret;
2168 
2169 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2170 	ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2171 
2172 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2173 	if (ret)
2174 		return ret;
2175 
2176 	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2177 		msleep(NVME_QUIRK_DELAY_AMOUNT);
2178 
2179 	return nvme_wait_ready(ctrl, NVME_CAP_TIMEOUT(ctrl->cap), false);
2180 }
2181 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2182 
2183 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2184 {
2185 	unsigned dev_page_min;
2186 	u32 timeout;
2187 	int ret;
2188 
2189 	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2190 	if (ret) {
2191 		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2192 		return ret;
2193 	}
2194 	dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2195 
2196 	if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2197 		dev_err(ctrl->device,
2198 			"Minimum device page size %u too large for host (%u)\n",
2199 			1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2200 		return -ENODEV;
2201 	}
2202 
2203 	if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2204 		ctrl->ctrl_config = NVME_CC_CSS_CSI;
2205 	else
2206 		ctrl->ctrl_config = NVME_CC_CSS_NVM;
2207 
2208 	if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2209 		u32 crto;
2210 
2211 		ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2212 		if (ret) {
2213 			dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2214 				ret);
2215 			return ret;
2216 		}
2217 
2218 		if (ctrl->cap & NVME_CAP_CRMS_CRIMS) {
2219 			ctrl->ctrl_config |= NVME_CC_CRIME;
2220 			timeout = NVME_CRTO_CRIMT(crto);
2221 		} else {
2222 			timeout = NVME_CRTO_CRWMT(crto);
2223 		}
2224 	} else {
2225 		timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2226 	}
2227 
2228 	ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2229 	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2230 	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2231 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2232 	if (ret)
2233 		return ret;
2234 
2235 	/* Flush write to device (required if transport is PCI) */
2236 	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2237 	if (ret)
2238 		return ret;
2239 
2240 	ctrl->ctrl_config |= NVME_CC_ENABLE;
2241 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2242 	if (ret)
2243 		return ret;
2244 	return nvme_wait_ready(ctrl, timeout, true);
2245 }
2246 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2247 
2248 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2249 {
2250 	unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2251 	u32 csts;
2252 	int ret;
2253 
2254 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2255 	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2256 
2257 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2258 	if (ret)
2259 		return ret;
2260 
2261 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2262 		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2263 			break;
2264 
2265 		msleep(100);
2266 		if (fatal_signal_pending(current))
2267 			return -EINTR;
2268 		if (time_after(jiffies, timeout)) {
2269 			dev_err(ctrl->device,
2270 				"Device shutdown incomplete; abort shutdown\n");
2271 			return -ENODEV;
2272 		}
2273 	}
2274 
2275 	return ret;
2276 }
2277 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2278 
2279 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2280 {
2281 	__le64 ts;
2282 	int ret;
2283 
2284 	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2285 		return 0;
2286 
2287 	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2288 	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2289 			NULL);
2290 	if (ret)
2291 		dev_warn_once(ctrl->device,
2292 			"could not set timestamp (%d)\n", ret);
2293 	return ret;
2294 }
2295 
2296 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2297 {
2298 	struct nvme_feat_host_behavior *host;
2299 	u8 acre = 0, lbafee = 0;
2300 	int ret;
2301 
2302 	/* Don't bother enabling the feature if retry delay is not reported */
2303 	if (ctrl->crdt[0])
2304 		acre = NVME_ENABLE_ACRE;
2305 	if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2306 		lbafee = NVME_ENABLE_LBAFEE;
2307 
2308 	if (!acre && !lbafee)
2309 		return 0;
2310 
2311 	host = kzalloc(sizeof(*host), GFP_KERNEL);
2312 	if (!host)
2313 		return 0;
2314 
2315 	host->acre = acre;
2316 	host->lbafee = lbafee;
2317 	ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2318 				host, sizeof(*host), NULL);
2319 	kfree(host);
2320 	return ret;
2321 }
2322 
2323 /*
2324  * The function checks whether the given total (exlat + enlat) latency of
2325  * a power state allows the latter to be used as an APST transition target.
2326  * It does so by comparing the latency to the primary and secondary latency
2327  * tolerances defined by module params. If there's a match, the corresponding
2328  * timeout value is returned and the matching tolerance index (1 or 2) is
2329  * reported.
2330  */
2331 static bool nvme_apst_get_transition_time(u64 total_latency,
2332 		u64 *transition_time, unsigned *last_index)
2333 {
2334 	if (total_latency <= apst_primary_latency_tol_us) {
2335 		if (*last_index == 1)
2336 			return false;
2337 		*last_index = 1;
2338 		*transition_time = apst_primary_timeout_ms;
2339 		return true;
2340 	}
2341 	if (apst_secondary_timeout_ms &&
2342 		total_latency <= apst_secondary_latency_tol_us) {
2343 		if (*last_index <= 2)
2344 			return false;
2345 		*last_index = 2;
2346 		*transition_time = apst_secondary_timeout_ms;
2347 		return true;
2348 	}
2349 	return false;
2350 }
2351 
2352 /*
2353  * APST (Autonomous Power State Transition) lets us program a table of power
2354  * state transitions that the controller will perform automatically.
2355  *
2356  * Depending on module params, one of the two supported techniques will be used:
2357  *
2358  * - If the parameters provide explicit timeouts and tolerances, they will be
2359  *   used to build a table with up to 2 non-operational states to transition to.
2360  *   The default parameter values were selected based on the values used by
2361  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2362  *   regeneration of the APST table in the event of switching between external
2363  *   and battery power, the timeouts and tolerances reflect a compromise
2364  *   between values used by Microsoft for AC and battery scenarios.
2365  * - If not, we'll configure the table with a simple heuristic: we are willing
2366  *   to spend at most 2% of the time transitioning between power states.
2367  *   Therefore, when running in any given state, we will enter the next
2368  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2369  *   microseconds, as long as that state's exit latency is under the requested
2370  *   maximum latency.
2371  *
2372  * We will not autonomously enter any non-operational state for which the total
2373  * latency exceeds ps_max_latency_us.
2374  *
2375  * Users can set ps_max_latency_us to zero to turn off APST.
2376  */
2377 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2378 {
2379 	struct nvme_feat_auto_pst *table;
2380 	unsigned apste = 0;
2381 	u64 max_lat_us = 0;
2382 	__le64 target = 0;
2383 	int max_ps = -1;
2384 	int state;
2385 	int ret;
2386 	unsigned last_lt_index = UINT_MAX;
2387 
2388 	/*
2389 	 * If APST isn't supported or if we haven't been initialized yet,
2390 	 * then don't do anything.
2391 	 */
2392 	if (!ctrl->apsta)
2393 		return 0;
2394 
2395 	if (ctrl->npss > 31) {
2396 		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2397 		return 0;
2398 	}
2399 
2400 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2401 	if (!table)
2402 		return 0;
2403 
2404 	if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2405 		/* Turn off APST. */
2406 		dev_dbg(ctrl->device, "APST disabled\n");
2407 		goto done;
2408 	}
2409 
2410 	/*
2411 	 * Walk through all states from lowest- to highest-power.
2412 	 * According to the spec, lower-numbered states use more power.  NPSS,
2413 	 * despite the name, is the index of the lowest-power state, not the
2414 	 * number of states.
2415 	 */
2416 	for (state = (int)ctrl->npss; state >= 0; state--) {
2417 		u64 total_latency_us, exit_latency_us, transition_ms;
2418 
2419 		if (target)
2420 			table->entries[state] = target;
2421 
2422 		/*
2423 		 * Don't allow transitions to the deepest state if it's quirked
2424 		 * off.
2425 		 */
2426 		if (state == ctrl->npss &&
2427 		    (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2428 			continue;
2429 
2430 		/*
2431 		 * Is this state a useful non-operational state for higher-power
2432 		 * states to autonomously transition to?
2433 		 */
2434 		if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2435 			continue;
2436 
2437 		exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2438 		if (exit_latency_us > ctrl->ps_max_latency_us)
2439 			continue;
2440 
2441 		total_latency_us = exit_latency_us +
2442 			le32_to_cpu(ctrl->psd[state].entry_lat);
2443 
2444 		/*
2445 		 * This state is good. It can be used as the APST idle target
2446 		 * for higher power states.
2447 		 */
2448 		if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2449 			if (!nvme_apst_get_transition_time(total_latency_us,
2450 					&transition_ms, &last_lt_index))
2451 				continue;
2452 		} else {
2453 			transition_ms = total_latency_us + 19;
2454 			do_div(transition_ms, 20);
2455 			if (transition_ms > (1 << 24) - 1)
2456 				transition_ms = (1 << 24) - 1;
2457 		}
2458 
2459 		target = cpu_to_le64((state << 3) | (transition_ms << 8));
2460 		if (max_ps == -1)
2461 			max_ps = state;
2462 		if (total_latency_us > max_lat_us)
2463 			max_lat_us = total_latency_us;
2464 	}
2465 
2466 	if (max_ps == -1)
2467 		dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2468 	else
2469 		dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2470 			max_ps, max_lat_us, (int)sizeof(*table), table);
2471 	apste = 1;
2472 
2473 done:
2474 	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2475 				table, sizeof(*table), NULL);
2476 	if (ret)
2477 		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2478 	kfree(table);
2479 	return ret;
2480 }
2481 
2482 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2483 {
2484 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2485 	u64 latency;
2486 
2487 	switch (val) {
2488 	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2489 	case PM_QOS_LATENCY_ANY:
2490 		latency = U64_MAX;
2491 		break;
2492 
2493 	default:
2494 		latency = val;
2495 	}
2496 
2497 	if (ctrl->ps_max_latency_us != latency) {
2498 		ctrl->ps_max_latency_us = latency;
2499 		if (ctrl->state == NVME_CTRL_LIVE)
2500 			nvme_configure_apst(ctrl);
2501 	}
2502 }
2503 
2504 struct nvme_core_quirk_entry {
2505 	/*
2506 	 * NVMe model and firmware strings are padded with spaces.  For
2507 	 * simplicity, strings in the quirk table are padded with NULLs
2508 	 * instead.
2509 	 */
2510 	u16 vid;
2511 	const char *mn;
2512 	const char *fr;
2513 	unsigned long quirks;
2514 };
2515 
2516 static const struct nvme_core_quirk_entry core_quirks[] = {
2517 	{
2518 		/*
2519 		 * This Toshiba device seems to die using any APST states.  See:
2520 		 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2521 		 */
2522 		.vid = 0x1179,
2523 		.mn = "THNSF5256GPUK TOSHIBA",
2524 		.quirks = NVME_QUIRK_NO_APST,
2525 	},
2526 	{
2527 		/*
2528 		 * This LiteON CL1-3D*-Q11 firmware version has a race
2529 		 * condition associated with actions related to suspend to idle
2530 		 * LiteON has resolved the problem in future firmware
2531 		 */
2532 		.vid = 0x14a4,
2533 		.fr = "22301111",
2534 		.quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2535 	},
2536 	{
2537 		/*
2538 		 * This Kioxia CD6-V Series / HPE PE8030 device times out and
2539 		 * aborts I/O during any load, but more easily reproducible
2540 		 * with discards (fstrim).
2541 		 *
2542 		 * The device is left in a state where it is also not possible
2543 		 * to use "nvme set-feature" to disable APST, but booting with
2544 		 * nvme_core.default_ps_max_latency=0 works.
2545 		 */
2546 		.vid = 0x1e0f,
2547 		.mn = "KCD6XVUL6T40",
2548 		.quirks = NVME_QUIRK_NO_APST,
2549 	}
2550 };
2551 
2552 /* match is null-terminated but idstr is space-padded. */
2553 static bool string_matches(const char *idstr, const char *match, size_t len)
2554 {
2555 	size_t matchlen;
2556 
2557 	if (!match)
2558 		return true;
2559 
2560 	matchlen = strlen(match);
2561 	WARN_ON_ONCE(matchlen > len);
2562 
2563 	if (memcmp(idstr, match, matchlen))
2564 		return false;
2565 
2566 	for (; matchlen < len; matchlen++)
2567 		if (idstr[matchlen] != ' ')
2568 			return false;
2569 
2570 	return true;
2571 }
2572 
2573 static bool quirk_matches(const struct nvme_id_ctrl *id,
2574 			  const struct nvme_core_quirk_entry *q)
2575 {
2576 	return q->vid == le16_to_cpu(id->vid) &&
2577 		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2578 		string_matches(id->fr, q->fr, sizeof(id->fr));
2579 }
2580 
2581 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2582 		struct nvme_id_ctrl *id)
2583 {
2584 	size_t nqnlen;
2585 	int off;
2586 
2587 	if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2588 		nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2589 		if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2590 			strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2591 			return;
2592 		}
2593 
2594 		if (ctrl->vs >= NVME_VS(1, 2, 1))
2595 			dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2596 	}
2597 
2598 	/* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2599 	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2600 			"nqn.2014.08.org.nvmexpress:%04x%04x",
2601 			le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2602 	memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2603 	off += sizeof(id->sn);
2604 	memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2605 	off += sizeof(id->mn);
2606 	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2607 }
2608 
2609 static void nvme_release_subsystem(struct device *dev)
2610 {
2611 	struct nvme_subsystem *subsys =
2612 		container_of(dev, struct nvme_subsystem, dev);
2613 
2614 	if (subsys->instance >= 0)
2615 		ida_free(&nvme_instance_ida, subsys->instance);
2616 	kfree(subsys);
2617 }
2618 
2619 static void nvme_destroy_subsystem(struct kref *ref)
2620 {
2621 	struct nvme_subsystem *subsys =
2622 			container_of(ref, struct nvme_subsystem, ref);
2623 
2624 	mutex_lock(&nvme_subsystems_lock);
2625 	list_del(&subsys->entry);
2626 	mutex_unlock(&nvme_subsystems_lock);
2627 
2628 	ida_destroy(&subsys->ns_ida);
2629 	device_del(&subsys->dev);
2630 	put_device(&subsys->dev);
2631 }
2632 
2633 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2634 {
2635 	kref_put(&subsys->ref, nvme_destroy_subsystem);
2636 }
2637 
2638 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2639 {
2640 	struct nvme_subsystem *subsys;
2641 
2642 	lockdep_assert_held(&nvme_subsystems_lock);
2643 
2644 	/*
2645 	 * Fail matches for discovery subsystems. This results
2646 	 * in each discovery controller bound to a unique subsystem.
2647 	 * This avoids issues with validating controller values
2648 	 * that can only be true when there is a single unique subsystem.
2649 	 * There may be multiple and completely independent entities
2650 	 * that provide discovery controllers.
2651 	 */
2652 	if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2653 		return NULL;
2654 
2655 	list_for_each_entry(subsys, &nvme_subsystems, entry) {
2656 		if (strcmp(subsys->subnqn, subsysnqn))
2657 			continue;
2658 		if (!kref_get_unless_zero(&subsys->ref))
2659 			continue;
2660 		return subsys;
2661 	}
2662 
2663 	return NULL;
2664 }
2665 
2666 #define SUBSYS_ATTR_RO(_name, _mode, _show)			\
2667 	struct device_attribute subsys_attr_##_name = \
2668 		__ATTR(_name, _mode, _show, NULL)
2669 
2670 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2671 				    struct device_attribute *attr,
2672 				    char *buf)
2673 {
2674 	struct nvme_subsystem *subsys =
2675 		container_of(dev, struct nvme_subsystem, dev);
2676 
2677 	return sysfs_emit(buf, "%s\n", subsys->subnqn);
2678 }
2679 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2680 
2681 static ssize_t nvme_subsys_show_type(struct device *dev,
2682 				    struct device_attribute *attr,
2683 				    char *buf)
2684 {
2685 	struct nvme_subsystem *subsys =
2686 		container_of(dev, struct nvme_subsystem, dev);
2687 
2688 	switch (subsys->subtype) {
2689 	case NVME_NQN_DISC:
2690 		return sysfs_emit(buf, "discovery\n");
2691 	case NVME_NQN_NVME:
2692 		return sysfs_emit(buf, "nvm\n");
2693 	default:
2694 		return sysfs_emit(buf, "reserved\n");
2695 	}
2696 }
2697 static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2698 
2699 #define nvme_subsys_show_str_function(field)				\
2700 static ssize_t subsys_##field##_show(struct device *dev,		\
2701 			    struct device_attribute *attr, char *buf)	\
2702 {									\
2703 	struct nvme_subsystem *subsys =					\
2704 		container_of(dev, struct nvme_subsystem, dev);		\
2705 	return sysfs_emit(buf, "%.*s\n",				\
2706 			   (int)sizeof(subsys->field), subsys->field);	\
2707 }									\
2708 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2709 
2710 nvme_subsys_show_str_function(model);
2711 nvme_subsys_show_str_function(serial);
2712 nvme_subsys_show_str_function(firmware_rev);
2713 
2714 static struct attribute *nvme_subsys_attrs[] = {
2715 	&subsys_attr_model.attr,
2716 	&subsys_attr_serial.attr,
2717 	&subsys_attr_firmware_rev.attr,
2718 	&subsys_attr_subsysnqn.attr,
2719 	&subsys_attr_subsystype.attr,
2720 #ifdef CONFIG_NVME_MULTIPATH
2721 	&subsys_attr_iopolicy.attr,
2722 #endif
2723 	NULL,
2724 };
2725 
2726 static const struct attribute_group nvme_subsys_attrs_group = {
2727 	.attrs = nvme_subsys_attrs,
2728 };
2729 
2730 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2731 	&nvme_subsys_attrs_group,
2732 	NULL,
2733 };
2734 
2735 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2736 {
2737 	return ctrl->opts && ctrl->opts->discovery_nqn;
2738 }
2739 
2740 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2741 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2742 {
2743 	struct nvme_ctrl *tmp;
2744 
2745 	lockdep_assert_held(&nvme_subsystems_lock);
2746 
2747 	list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2748 		if (nvme_state_terminal(tmp))
2749 			continue;
2750 
2751 		if (tmp->cntlid == ctrl->cntlid) {
2752 			dev_err(ctrl->device,
2753 				"Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2754 				ctrl->cntlid, dev_name(tmp->device),
2755 				subsys->subnqn);
2756 			return false;
2757 		}
2758 
2759 		if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2760 		    nvme_discovery_ctrl(ctrl))
2761 			continue;
2762 
2763 		dev_err(ctrl->device,
2764 			"Subsystem does not support multiple controllers\n");
2765 		return false;
2766 	}
2767 
2768 	return true;
2769 }
2770 
2771 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2772 {
2773 	struct nvme_subsystem *subsys, *found;
2774 	int ret;
2775 
2776 	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2777 	if (!subsys)
2778 		return -ENOMEM;
2779 
2780 	subsys->instance = -1;
2781 	mutex_init(&subsys->lock);
2782 	kref_init(&subsys->ref);
2783 	INIT_LIST_HEAD(&subsys->ctrls);
2784 	INIT_LIST_HEAD(&subsys->nsheads);
2785 	nvme_init_subnqn(subsys, ctrl, id);
2786 	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2787 	memcpy(subsys->model, id->mn, sizeof(subsys->model));
2788 	memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2789 	subsys->vendor_id = le16_to_cpu(id->vid);
2790 	subsys->cmic = id->cmic;
2791 
2792 	/* Versions prior to 1.4 don't necessarily report a valid type */
2793 	if (id->cntrltype == NVME_CTRL_DISC ||
2794 	    !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2795 		subsys->subtype = NVME_NQN_DISC;
2796 	else
2797 		subsys->subtype = NVME_NQN_NVME;
2798 
2799 	if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2800 		dev_err(ctrl->device,
2801 			"Subsystem %s is not a discovery controller",
2802 			subsys->subnqn);
2803 		kfree(subsys);
2804 		return -EINVAL;
2805 	}
2806 	subsys->awupf = le16_to_cpu(id->awupf);
2807 	nvme_mpath_default_iopolicy(subsys);
2808 
2809 	subsys->dev.class = nvme_subsys_class;
2810 	subsys->dev.release = nvme_release_subsystem;
2811 	subsys->dev.groups = nvme_subsys_attrs_groups;
2812 	dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2813 	device_initialize(&subsys->dev);
2814 
2815 	mutex_lock(&nvme_subsystems_lock);
2816 	found = __nvme_find_get_subsystem(subsys->subnqn);
2817 	if (found) {
2818 		put_device(&subsys->dev);
2819 		subsys = found;
2820 
2821 		if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2822 			ret = -EINVAL;
2823 			goto out_put_subsystem;
2824 		}
2825 	} else {
2826 		ret = device_add(&subsys->dev);
2827 		if (ret) {
2828 			dev_err(ctrl->device,
2829 				"failed to register subsystem device.\n");
2830 			put_device(&subsys->dev);
2831 			goto out_unlock;
2832 		}
2833 		ida_init(&subsys->ns_ida);
2834 		list_add_tail(&subsys->entry, &nvme_subsystems);
2835 	}
2836 
2837 	ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2838 				dev_name(ctrl->device));
2839 	if (ret) {
2840 		dev_err(ctrl->device,
2841 			"failed to create sysfs link from subsystem.\n");
2842 		goto out_put_subsystem;
2843 	}
2844 
2845 	if (!found)
2846 		subsys->instance = ctrl->instance;
2847 	ctrl->subsys = subsys;
2848 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2849 	mutex_unlock(&nvme_subsystems_lock);
2850 	return 0;
2851 
2852 out_put_subsystem:
2853 	nvme_put_subsystem(subsys);
2854 out_unlock:
2855 	mutex_unlock(&nvme_subsystems_lock);
2856 	return ret;
2857 }
2858 
2859 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2860 		void *log, size_t size, u64 offset)
2861 {
2862 	struct nvme_command c = { };
2863 	u32 dwlen = nvme_bytes_to_numd(size);
2864 
2865 	c.get_log_page.opcode = nvme_admin_get_log_page;
2866 	c.get_log_page.nsid = cpu_to_le32(nsid);
2867 	c.get_log_page.lid = log_page;
2868 	c.get_log_page.lsp = lsp;
2869 	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2870 	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2871 	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2872 	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2873 	c.get_log_page.csi = csi;
2874 
2875 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2876 }
2877 
2878 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2879 				struct nvme_effects_log **log)
2880 {
2881 	struct nvme_effects_log	*cel = xa_load(&ctrl->cels, csi);
2882 	int ret;
2883 
2884 	if (cel)
2885 		goto out;
2886 
2887 	cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2888 	if (!cel)
2889 		return -ENOMEM;
2890 
2891 	ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
2892 			cel, sizeof(*cel), 0);
2893 	if (ret) {
2894 		kfree(cel);
2895 		return ret;
2896 	}
2897 
2898 	xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
2899 out:
2900 	*log = cel;
2901 	return 0;
2902 }
2903 
2904 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
2905 {
2906 	u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
2907 
2908 	if (check_shl_overflow(1U, units + page_shift - 9, &val))
2909 		return UINT_MAX;
2910 	return val;
2911 }
2912 
2913 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
2914 {
2915 	struct nvme_command c = { };
2916 	struct nvme_id_ctrl_nvm *id;
2917 	int ret;
2918 
2919 	if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
2920 		ctrl->max_discard_sectors = UINT_MAX;
2921 		ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
2922 	} else {
2923 		ctrl->max_discard_sectors = 0;
2924 		ctrl->max_discard_segments = 0;
2925 	}
2926 
2927 	/*
2928 	 * Even though NVMe spec explicitly states that MDTS is not applicable
2929 	 * to the write-zeroes, we are cautious and limit the size to the
2930 	 * controllers max_hw_sectors value, which is based on the MDTS field
2931 	 * and possibly other limiting factors.
2932 	 */
2933 	if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
2934 	    !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
2935 		ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
2936 	else
2937 		ctrl->max_zeroes_sectors = 0;
2938 
2939 	if (nvme_ctrl_limited_cns(ctrl))
2940 		return 0;
2941 
2942 	id = kzalloc(sizeof(*id), GFP_KERNEL);
2943 	if (!id)
2944 		return 0;
2945 
2946 	c.identify.opcode = nvme_admin_identify;
2947 	c.identify.cns = NVME_ID_CNS_CS_CTRL;
2948 	c.identify.csi = NVME_CSI_NVM;
2949 
2950 	ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
2951 	if (ret)
2952 		goto free_data;
2953 
2954 	if (id->dmrl)
2955 		ctrl->max_discard_segments = id->dmrl;
2956 	ctrl->dmrsl = le32_to_cpu(id->dmrsl);
2957 	if (id->wzsl)
2958 		ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
2959 
2960 free_data:
2961 	kfree(id);
2962 	return ret;
2963 }
2964 
2965 static int nvme_init_identify(struct nvme_ctrl *ctrl)
2966 {
2967 	struct nvme_id_ctrl *id;
2968 	u32 max_hw_sectors;
2969 	bool prev_apst_enabled;
2970 	int ret;
2971 
2972 	ret = nvme_identify_ctrl(ctrl, &id);
2973 	if (ret) {
2974 		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2975 		return -EIO;
2976 	}
2977 
2978 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2979 		ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
2980 		if (ret < 0)
2981 			goto out_free;
2982 	}
2983 
2984 	if (!(ctrl->ops->flags & NVME_F_FABRICS))
2985 		ctrl->cntlid = le16_to_cpu(id->cntlid);
2986 
2987 	if (!ctrl->identified) {
2988 		unsigned int i;
2989 
2990 		ret = nvme_init_subsystem(ctrl, id);
2991 		if (ret)
2992 			goto out_free;
2993 
2994 		/*
2995 		 * Check for quirks.  Quirk can depend on firmware version,
2996 		 * so, in principle, the set of quirks present can change
2997 		 * across a reset.  As a possible future enhancement, we
2998 		 * could re-scan for quirks every time we reinitialize
2999 		 * the device, but we'd have to make sure that the driver
3000 		 * behaves intelligently if the quirks change.
3001 		 */
3002 		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3003 			if (quirk_matches(id, &core_quirks[i]))
3004 				ctrl->quirks |= core_quirks[i].quirks;
3005 		}
3006 	}
3007 
3008 	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3009 		dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3010 		ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3011 	}
3012 
3013 	ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3014 	ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3015 	ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3016 
3017 	ctrl->oacs = le16_to_cpu(id->oacs);
3018 	ctrl->oncs = le16_to_cpu(id->oncs);
3019 	ctrl->mtfa = le16_to_cpu(id->mtfa);
3020 	ctrl->oaes = le32_to_cpu(id->oaes);
3021 	ctrl->wctemp = le16_to_cpu(id->wctemp);
3022 	ctrl->cctemp = le16_to_cpu(id->cctemp);
3023 
3024 	atomic_set(&ctrl->abort_limit, id->acl + 1);
3025 	ctrl->vwc = id->vwc;
3026 	if (id->mdts)
3027 		max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3028 	else
3029 		max_hw_sectors = UINT_MAX;
3030 	ctrl->max_hw_sectors =
3031 		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3032 
3033 	nvme_set_queue_limits(ctrl, ctrl->admin_q);
3034 	ctrl->sgls = le32_to_cpu(id->sgls);
3035 	ctrl->kas = le16_to_cpu(id->kas);
3036 	ctrl->max_namespaces = le32_to_cpu(id->mnan);
3037 	ctrl->ctratt = le32_to_cpu(id->ctratt);
3038 
3039 	ctrl->cntrltype = id->cntrltype;
3040 	ctrl->dctype = id->dctype;
3041 
3042 	if (id->rtd3e) {
3043 		/* us -> s */
3044 		u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3045 
3046 		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3047 						 shutdown_timeout, 60);
3048 
3049 		if (ctrl->shutdown_timeout != shutdown_timeout)
3050 			dev_info(ctrl->device,
3051 				 "Shutdown timeout set to %u seconds\n",
3052 				 ctrl->shutdown_timeout);
3053 	} else
3054 		ctrl->shutdown_timeout = shutdown_timeout;
3055 
3056 	ctrl->npss = id->npss;
3057 	ctrl->apsta = id->apsta;
3058 	prev_apst_enabled = ctrl->apst_enabled;
3059 	if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3060 		if (force_apst && id->apsta) {
3061 			dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3062 			ctrl->apst_enabled = true;
3063 		} else {
3064 			ctrl->apst_enabled = false;
3065 		}
3066 	} else {
3067 		ctrl->apst_enabled = id->apsta;
3068 	}
3069 	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3070 
3071 	if (ctrl->ops->flags & NVME_F_FABRICS) {
3072 		ctrl->icdoff = le16_to_cpu(id->icdoff);
3073 		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3074 		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3075 		ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3076 
3077 		/*
3078 		 * In fabrics we need to verify the cntlid matches the
3079 		 * admin connect
3080 		 */
3081 		if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3082 			dev_err(ctrl->device,
3083 				"Mismatching cntlid: Connect %u vs Identify "
3084 				"%u, rejecting\n",
3085 				ctrl->cntlid, le16_to_cpu(id->cntlid));
3086 			ret = -EINVAL;
3087 			goto out_free;
3088 		}
3089 
3090 		if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3091 			dev_err(ctrl->device,
3092 				"keep-alive support is mandatory for fabrics\n");
3093 			ret = -EINVAL;
3094 			goto out_free;
3095 		}
3096 	} else {
3097 		ctrl->hmpre = le32_to_cpu(id->hmpre);
3098 		ctrl->hmmin = le32_to_cpu(id->hmmin);
3099 		ctrl->hmminds = le32_to_cpu(id->hmminds);
3100 		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3101 	}
3102 
3103 	ret = nvme_mpath_init_identify(ctrl, id);
3104 	if (ret < 0)
3105 		goto out_free;
3106 
3107 	if (ctrl->apst_enabled && !prev_apst_enabled)
3108 		dev_pm_qos_expose_latency_tolerance(ctrl->device);
3109 	else if (!ctrl->apst_enabled && prev_apst_enabled)
3110 		dev_pm_qos_hide_latency_tolerance(ctrl->device);
3111 
3112 out_free:
3113 	kfree(id);
3114 	return ret;
3115 }
3116 
3117 /*
3118  * Initialize the cached copies of the Identify data and various controller
3119  * register in our nvme_ctrl structure.  This should be called as soon as
3120  * the admin queue is fully up and running.
3121  */
3122 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)
3123 {
3124 	int ret;
3125 
3126 	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3127 	if (ret) {
3128 		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3129 		return ret;
3130 	}
3131 
3132 	ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3133 
3134 	if (ctrl->vs >= NVME_VS(1, 1, 0))
3135 		ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3136 
3137 	ret = nvme_init_identify(ctrl);
3138 	if (ret)
3139 		return ret;
3140 
3141 	ret = nvme_configure_apst(ctrl);
3142 	if (ret < 0)
3143 		return ret;
3144 
3145 	ret = nvme_configure_timestamp(ctrl);
3146 	if (ret < 0)
3147 		return ret;
3148 
3149 	ret = nvme_configure_host_options(ctrl);
3150 	if (ret < 0)
3151 		return ret;
3152 
3153 	if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3154 		ret = nvme_hwmon_init(ctrl);
3155 		if (ret < 0)
3156 			return ret;
3157 	}
3158 
3159 	ctrl->identified = true;
3160 
3161 	return 0;
3162 }
3163 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3164 
3165 static int nvme_dev_open(struct inode *inode, struct file *file)
3166 {
3167 	struct nvme_ctrl *ctrl =
3168 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3169 
3170 	switch (ctrl->state) {
3171 	case NVME_CTRL_LIVE:
3172 		break;
3173 	default:
3174 		return -EWOULDBLOCK;
3175 	}
3176 
3177 	nvme_get_ctrl(ctrl);
3178 	if (!try_module_get(ctrl->ops->module)) {
3179 		nvme_put_ctrl(ctrl);
3180 		return -EINVAL;
3181 	}
3182 
3183 	file->private_data = ctrl;
3184 	return 0;
3185 }
3186 
3187 static int nvme_dev_release(struct inode *inode, struct file *file)
3188 {
3189 	struct nvme_ctrl *ctrl =
3190 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3191 
3192 	module_put(ctrl->ops->module);
3193 	nvme_put_ctrl(ctrl);
3194 	return 0;
3195 }
3196 
3197 static const struct file_operations nvme_dev_fops = {
3198 	.owner		= THIS_MODULE,
3199 	.open		= nvme_dev_open,
3200 	.release	= nvme_dev_release,
3201 	.unlocked_ioctl	= nvme_dev_ioctl,
3202 	.compat_ioctl	= compat_ptr_ioctl,
3203 	.uring_cmd	= nvme_dev_uring_cmd,
3204 };
3205 
3206 static ssize_t nvme_sysfs_reset(struct device *dev,
3207 				struct device_attribute *attr, const char *buf,
3208 				size_t count)
3209 {
3210 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3211 	int ret;
3212 
3213 	ret = nvme_reset_ctrl_sync(ctrl);
3214 	if (ret < 0)
3215 		return ret;
3216 	return count;
3217 }
3218 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3219 
3220 static ssize_t nvme_sysfs_rescan(struct device *dev,
3221 				struct device_attribute *attr, const char *buf,
3222 				size_t count)
3223 {
3224 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3225 
3226 	nvme_queue_scan(ctrl);
3227 	return count;
3228 }
3229 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3230 
3231 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3232 {
3233 	struct gendisk *disk = dev_to_disk(dev);
3234 
3235 	if (disk->fops == &nvme_bdev_ops)
3236 		return nvme_get_ns_from_dev(dev)->head;
3237 	else
3238 		return disk->private_data;
3239 }
3240 
3241 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3242 		char *buf)
3243 {
3244 	struct nvme_ns_head *head = dev_to_ns_head(dev);
3245 	struct nvme_ns_ids *ids = &head->ids;
3246 	struct nvme_subsystem *subsys = head->subsys;
3247 	int serial_len = sizeof(subsys->serial);
3248 	int model_len = sizeof(subsys->model);
3249 
3250 	if (!uuid_is_null(&ids->uuid))
3251 		return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3252 
3253 	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3254 		return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3255 
3256 	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3257 		return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3258 
3259 	while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3260 				  subsys->serial[serial_len - 1] == '\0'))
3261 		serial_len--;
3262 	while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3263 				 subsys->model[model_len - 1] == '\0'))
3264 		model_len--;
3265 
3266 	return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3267 		serial_len, subsys->serial, model_len, subsys->model,
3268 		head->ns_id);
3269 }
3270 static DEVICE_ATTR_RO(wwid);
3271 
3272 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3273 		char *buf)
3274 {
3275 	return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3276 }
3277 static DEVICE_ATTR_RO(nguid);
3278 
3279 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3280 		char *buf)
3281 {
3282 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3283 
3284 	/* For backward compatibility expose the NGUID to userspace if
3285 	 * we have no UUID set
3286 	 */
3287 	if (uuid_is_null(&ids->uuid)) {
3288 		dev_warn_ratelimited(dev,
3289 			"No UUID available providing old NGUID\n");
3290 		return sysfs_emit(buf, "%pU\n", ids->nguid);
3291 	}
3292 	return sysfs_emit(buf, "%pU\n", &ids->uuid);
3293 }
3294 static DEVICE_ATTR_RO(uuid);
3295 
3296 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3297 		char *buf)
3298 {
3299 	return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3300 }
3301 static DEVICE_ATTR_RO(eui);
3302 
3303 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3304 		char *buf)
3305 {
3306 	return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3307 }
3308 static DEVICE_ATTR_RO(nsid);
3309 
3310 static struct attribute *nvme_ns_id_attrs[] = {
3311 	&dev_attr_wwid.attr,
3312 	&dev_attr_uuid.attr,
3313 	&dev_attr_nguid.attr,
3314 	&dev_attr_eui.attr,
3315 	&dev_attr_nsid.attr,
3316 #ifdef CONFIG_NVME_MULTIPATH
3317 	&dev_attr_ana_grpid.attr,
3318 	&dev_attr_ana_state.attr,
3319 #endif
3320 	NULL,
3321 };
3322 
3323 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3324 		struct attribute *a, int n)
3325 {
3326 	struct device *dev = container_of(kobj, struct device, kobj);
3327 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3328 
3329 	if (a == &dev_attr_uuid.attr) {
3330 		if (uuid_is_null(&ids->uuid) &&
3331 		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3332 			return 0;
3333 	}
3334 	if (a == &dev_attr_nguid.attr) {
3335 		if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3336 			return 0;
3337 	}
3338 	if (a == &dev_attr_eui.attr) {
3339 		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3340 			return 0;
3341 	}
3342 #ifdef CONFIG_NVME_MULTIPATH
3343 	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3344 		if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
3345 			return 0;
3346 		if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3347 			return 0;
3348 	}
3349 #endif
3350 	return a->mode;
3351 }
3352 
3353 static const struct attribute_group nvme_ns_id_attr_group = {
3354 	.attrs		= nvme_ns_id_attrs,
3355 	.is_visible	= nvme_ns_id_attrs_are_visible,
3356 };
3357 
3358 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3359 	&nvme_ns_id_attr_group,
3360 	NULL,
3361 };
3362 
3363 #define nvme_show_str_function(field)						\
3364 static ssize_t  field##_show(struct device *dev,				\
3365 			    struct device_attribute *attr, char *buf)		\
3366 {										\
3367         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3368         return sysfs_emit(buf, "%.*s\n",					\
3369 		(int)sizeof(ctrl->subsys->field), ctrl->subsys->field);		\
3370 }										\
3371 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3372 
3373 nvme_show_str_function(model);
3374 nvme_show_str_function(serial);
3375 nvme_show_str_function(firmware_rev);
3376 
3377 #define nvme_show_int_function(field)						\
3378 static ssize_t  field##_show(struct device *dev,				\
3379 			    struct device_attribute *attr, char *buf)		\
3380 {										\
3381         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3382         return sysfs_emit(buf, "%d\n", ctrl->field);				\
3383 }										\
3384 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3385 
3386 nvme_show_int_function(cntlid);
3387 nvme_show_int_function(numa_node);
3388 nvme_show_int_function(queue_count);
3389 nvme_show_int_function(sqsize);
3390 nvme_show_int_function(kato);
3391 
3392 static ssize_t nvme_sysfs_delete(struct device *dev,
3393 				struct device_attribute *attr, const char *buf,
3394 				size_t count)
3395 {
3396 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3397 
3398 	if (device_remove_file_self(dev, attr))
3399 		nvme_delete_ctrl_sync(ctrl);
3400 	return count;
3401 }
3402 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3403 
3404 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3405 					 struct device_attribute *attr,
3406 					 char *buf)
3407 {
3408 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3409 
3410 	return sysfs_emit(buf, "%s\n", ctrl->ops->name);
3411 }
3412 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3413 
3414 static ssize_t nvme_sysfs_show_state(struct device *dev,
3415 				     struct device_attribute *attr,
3416 				     char *buf)
3417 {
3418 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3419 	static const char *const state_name[] = {
3420 		[NVME_CTRL_NEW]		= "new",
3421 		[NVME_CTRL_LIVE]	= "live",
3422 		[NVME_CTRL_RESETTING]	= "resetting",
3423 		[NVME_CTRL_CONNECTING]	= "connecting",
3424 		[NVME_CTRL_DELETING]	= "deleting",
3425 		[NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3426 		[NVME_CTRL_DEAD]	= "dead",
3427 	};
3428 
3429 	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3430 	    state_name[ctrl->state])
3431 		return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3432 
3433 	return sysfs_emit(buf, "unknown state\n");
3434 }
3435 
3436 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3437 
3438 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3439 					 struct device_attribute *attr,
3440 					 char *buf)
3441 {
3442 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3443 
3444 	return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
3445 }
3446 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3447 
3448 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3449 					struct device_attribute *attr,
3450 					char *buf)
3451 {
3452 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3453 
3454 	return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
3455 }
3456 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3457 
3458 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3459 					struct device_attribute *attr,
3460 					char *buf)
3461 {
3462 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3463 
3464 	return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
3465 }
3466 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3467 
3468 static ssize_t nvme_sysfs_show_address(struct device *dev,
3469 					 struct device_attribute *attr,
3470 					 char *buf)
3471 {
3472 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3473 
3474 	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3475 }
3476 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3477 
3478 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3479 		struct device_attribute *attr, char *buf)
3480 {
3481 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3482 	struct nvmf_ctrl_options *opts = ctrl->opts;
3483 
3484 	if (ctrl->opts->max_reconnects == -1)
3485 		return sysfs_emit(buf, "off\n");
3486 	return sysfs_emit(buf, "%d\n",
3487 			  opts->max_reconnects * opts->reconnect_delay);
3488 }
3489 
3490 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3491 		struct device_attribute *attr, const char *buf, size_t count)
3492 {
3493 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3494 	struct nvmf_ctrl_options *opts = ctrl->opts;
3495 	int ctrl_loss_tmo, err;
3496 
3497 	err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3498 	if (err)
3499 		return -EINVAL;
3500 
3501 	if (ctrl_loss_tmo < 0)
3502 		opts->max_reconnects = -1;
3503 	else
3504 		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3505 						opts->reconnect_delay);
3506 	return count;
3507 }
3508 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3509 	nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3510 
3511 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3512 		struct device_attribute *attr, char *buf)
3513 {
3514 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3515 
3516 	if (ctrl->opts->reconnect_delay == -1)
3517 		return sysfs_emit(buf, "off\n");
3518 	return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3519 }
3520 
3521 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3522 		struct device_attribute *attr, const char *buf, size_t count)
3523 {
3524 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3525 	unsigned int v;
3526 	int err;
3527 
3528 	err = kstrtou32(buf, 10, &v);
3529 	if (err)
3530 		return err;
3531 
3532 	ctrl->opts->reconnect_delay = v;
3533 	return count;
3534 }
3535 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3536 	nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3537 
3538 static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3539 		struct device_attribute *attr, char *buf)
3540 {
3541 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3542 
3543 	if (ctrl->opts->fast_io_fail_tmo == -1)
3544 		return sysfs_emit(buf, "off\n");
3545 	return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3546 }
3547 
3548 static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3549 		struct device_attribute *attr, const char *buf, size_t count)
3550 {
3551 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3552 	struct nvmf_ctrl_options *opts = ctrl->opts;
3553 	int fast_io_fail_tmo, err;
3554 
3555 	err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3556 	if (err)
3557 		return -EINVAL;
3558 
3559 	if (fast_io_fail_tmo < 0)
3560 		opts->fast_io_fail_tmo = -1;
3561 	else
3562 		opts->fast_io_fail_tmo = fast_io_fail_tmo;
3563 	return count;
3564 }
3565 static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3566 	nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3567 
3568 static ssize_t cntrltype_show(struct device *dev,
3569 			      struct device_attribute *attr, char *buf)
3570 {
3571 	static const char * const type[] = {
3572 		[NVME_CTRL_IO] = "io\n",
3573 		[NVME_CTRL_DISC] = "discovery\n",
3574 		[NVME_CTRL_ADMIN] = "admin\n",
3575 	};
3576 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3577 
3578 	if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3579 		return sysfs_emit(buf, "reserved\n");
3580 
3581 	return sysfs_emit(buf, type[ctrl->cntrltype]);
3582 }
3583 static DEVICE_ATTR_RO(cntrltype);
3584 
3585 static ssize_t dctype_show(struct device *dev,
3586 			   struct device_attribute *attr, char *buf)
3587 {
3588 	static const char * const type[] = {
3589 		[NVME_DCTYPE_NOT_REPORTED] = "none\n",
3590 		[NVME_DCTYPE_DDC] = "ddc\n",
3591 		[NVME_DCTYPE_CDC] = "cdc\n",
3592 	};
3593 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3594 
3595 	if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3596 		return sysfs_emit(buf, "reserved\n");
3597 
3598 	return sysfs_emit(buf, type[ctrl->dctype]);
3599 }
3600 static DEVICE_ATTR_RO(dctype);
3601 
3602 static struct attribute *nvme_dev_attrs[] = {
3603 	&dev_attr_reset_controller.attr,
3604 	&dev_attr_rescan_controller.attr,
3605 	&dev_attr_model.attr,
3606 	&dev_attr_serial.attr,
3607 	&dev_attr_firmware_rev.attr,
3608 	&dev_attr_cntlid.attr,
3609 	&dev_attr_delete_controller.attr,
3610 	&dev_attr_transport.attr,
3611 	&dev_attr_subsysnqn.attr,
3612 	&dev_attr_address.attr,
3613 	&dev_attr_state.attr,
3614 	&dev_attr_numa_node.attr,
3615 	&dev_attr_queue_count.attr,
3616 	&dev_attr_sqsize.attr,
3617 	&dev_attr_hostnqn.attr,
3618 	&dev_attr_hostid.attr,
3619 	&dev_attr_ctrl_loss_tmo.attr,
3620 	&dev_attr_reconnect_delay.attr,
3621 	&dev_attr_fast_io_fail_tmo.attr,
3622 	&dev_attr_kato.attr,
3623 	&dev_attr_cntrltype.attr,
3624 	&dev_attr_dctype.attr,
3625 	NULL
3626 };
3627 
3628 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3629 		struct attribute *a, int n)
3630 {
3631 	struct device *dev = container_of(kobj, struct device, kobj);
3632 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3633 
3634 	if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3635 		return 0;
3636 	if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3637 		return 0;
3638 	if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3639 		return 0;
3640 	if (a == &dev_attr_hostid.attr && !ctrl->opts)
3641 		return 0;
3642 	if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3643 		return 0;
3644 	if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3645 		return 0;
3646 	if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3647 		return 0;
3648 
3649 	return a->mode;
3650 }
3651 
3652 static const struct attribute_group nvme_dev_attrs_group = {
3653 	.attrs		= nvme_dev_attrs,
3654 	.is_visible	= nvme_dev_attrs_are_visible,
3655 };
3656 
3657 static const struct attribute_group *nvme_dev_attr_groups[] = {
3658 	&nvme_dev_attrs_group,
3659 	NULL,
3660 };
3661 
3662 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3663 		unsigned nsid)
3664 {
3665 	struct nvme_ns_head *h;
3666 
3667 	lockdep_assert_held(&ctrl->subsys->lock);
3668 
3669 	list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3670 		/*
3671 		 * Private namespaces can share NSIDs under some conditions.
3672 		 * In that case we can't use the same ns_head for namespaces
3673 		 * with the same NSID.
3674 		 */
3675 		if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3676 			continue;
3677 		if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3678 			return h;
3679 	}
3680 
3681 	return NULL;
3682 }
3683 
3684 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3685 		struct nvme_ns_ids *ids)
3686 {
3687 	bool has_uuid = !uuid_is_null(&ids->uuid);
3688 	bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3689 	bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3690 	struct nvme_ns_head *h;
3691 
3692 	lockdep_assert_held(&subsys->lock);
3693 
3694 	list_for_each_entry(h, &subsys->nsheads, entry) {
3695 		if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3696 			return -EINVAL;
3697 		if (has_nguid &&
3698 		    memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3699 			return -EINVAL;
3700 		if (has_eui64 &&
3701 		    memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
3702 			return -EINVAL;
3703 	}
3704 
3705 	return 0;
3706 }
3707 
3708 static void nvme_cdev_rel(struct device *dev)
3709 {
3710 	ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
3711 }
3712 
3713 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3714 {
3715 	cdev_device_del(cdev, cdev_device);
3716 	put_device(cdev_device);
3717 }
3718 
3719 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3720 		const struct file_operations *fops, struct module *owner)
3721 {
3722 	int minor, ret;
3723 
3724 	minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
3725 	if (minor < 0)
3726 		return minor;
3727 	cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3728 	cdev_device->class = nvme_ns_chr_class;
3729 	cdev_device->release = nvme_cdev_rel;
3730 	device_initialize(cdev_device);
3731 	cdev_init(cdev, fops);
3732 	cdev->owner = owner;
3733 	ret = cdev_device_add(cdev, cdev_device);
3734 	if (ret)
3735 		put_device(cdev_device);
3736 
3737 	return ret;
3738 }
3739 
3740 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3741 {
3742 	return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3743 }
3744 
3745 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
3746 {
3747 	nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
3748 	return 0;
3749 }
3750 
3751 static const struct file_operations nvme_ns_chr_fops = {
3752 	.owner		= THIS_MODULE,
3753 	.open		= nvme_ns_chr_open,
3754 	.release	= nvme_ns_chr_release,
3755 	.unlocked_ioctl	= nvme_ns_chr_ioctl,
3756 	.compat_ioctl	= compat_ptr_ioctl,
3757 	.uring_cmd	= nvme_ns_chr_uring_cmd,
3758 };
3759 
3760 static int nvme_add_ns_cdev(struct nvme_ns *ns)
3761 {
3762 	int ret;
3763 
3764 	ns->cdev_device.parent = ns->ctrl->device;
3765 	ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
3766 			   ns->ctrl->instance, ns->head->instance);
3767 	if (ret)
3768 		return ret;
3769 
3770 	return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
3771 			     ns->ctrl->ops->module);
3772 }
3773 
3774 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3775 		unsigned nsid, struct nvme_ns_ids *ids)
3776 {
3777 	struct nvme_ns_head *head;
3778 	size_t size = sizeof(*head);
3779 	int ret = -ENOMEM;
3780 
3781 #ifdef CONFIG_NVME_MULTIPATH
3782 	size += num_possible_nodes() * sizeof(struct nvme_ns *);
3783 #endif
3784 
3785 	head = kzalloc(size, GFP_KERNEL);
3786 	if (!head)
3787 		goto out;
3788 	ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
3789 	if (ret < 0)
3790 		goto out_free_head;
3791 	head->instance = ret;
3792 	INIT_LIST_HEAD(&head->list);
3793 	ret = init_srcu_struct(&head->srcu);
3794 	if (ret)
3795 		goto out_ida_remove;
3796 	head->subsys = ctrl->subsys;
3797 	head->ns_id = nsid;
3798 	head->ids = *ids;
3799 	kref_init(&head->ref);
3800 
3801 	if (head->ids.csi) {
3802 		ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3803 		if (ret)
3804 			goto out_cleanup_srcu;
3805 	} else
3806 		head->effects = ctrl->effects;
3807 
3808 	ret = nvme_mpath_alloc_disk(ctrl, head);
3809 	if (ret)
3810 		goto out_cleanup_srcu;
3811 
3812 	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3813 
3814 	kref_get(&ctrl->subsys->ref);
3815 
3816 	return head;
3817 out_cleanup_srcu:
3818 	cleanup_srcu_struct(&head->srcu);
3819 out_ida_remove:
3820 	ida_free(&ctrl->subsys->ns_ida, head->instance);
3821 out_free_head:
3822 	kfree(head);
3823 out:
3824 	if (ret > 0)
3825 		ret = blk_status_to_errno(nvme_error_status(ret));
3826 	return ERR_PTR(ret);
3827 }
3828 
3829 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
3830 		struct nvme_ns_ids *ids)
3831 {
3832 	struct nvme_subsystem *s;
3833 	int ret = 0;
3834 
3835 	/*
3836 	 * Note that this check is racy as we try to avoid holding the global
3837 	 * lock over the whole ns_head creation.  But it is only intended as
3838 	 * a sanity check anyway.
3839 	 */
3840 	mutex_lock(&nvme_subsystems_lock);
3841 	list_for_each_entry(s, &nvme_subsystems, entry) {
3842 		if (s == this)
3843 			continue;
3844 		mutex_lock(&s->lock);
3845 		ret = nvme_subsys_check_duplicate_ids(s, ids);
3846 		mutex_unlock(&s->lock);
3847 		if (ret)
3848 			break;
3849 	}
3850 	mutex_unlock(&nvme_subsystems_lock);
3851 
3852 	return ret;
3853 }
3854 
3855 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3856 		struct nvme_ns_ids *ids, bool is_shared)
3857 {
3858 	struct nvme_ctrl *ctrl = ns->ctrl;
3859 	struct nvme_ns_head *head = NULL;
3860 	int ret;
3861 
3862 	ret = nvme_global_check_duplicate_ids(ctrl->subsys, ids);
3863 	if (ret) {
3864 		dev_err(ctrl->device,
3865 			"globally duplicate IDs for nsid %d\n", nsid);
3866 		nvme_print_device_info(ctrl);
3867 		return ret;
3868 	}
3869 
3870 	mutex_lock(&ctrl->subsys->lock);
3871 	head = nvme_find_ns_head(ctrl, nsid);
3872 	if (!head) {
3873 		ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, ids);
3874 		if (ret) {
3875 			dev_err(ctrl->device,
3876 				"duplicate IDs in subsystem for nsid %d\n",
3877 				nsid);
3878 			goto out_unlock;
3879 		}
3880 		head = nvme_alloc_ns_head(ctrl, nsid, ids);
3881 		if (IS_ERR(head)) {
3882 			ret = PTR_ERR(head);
3883 			goto out_unlock;
3884 		}
3885 		head->shared = is_shared;
3886 	} else {
3887 		ret = -EINVAL;
3888 		if (!is_shared || !head->shared) {
3889 			dev_err(ctrl->device,
3890 				"Duplicate unshared namespace %d\n", nsid);
3891 			goto out_put_ns_head;
3892 		}
3893 		if (!nvme_ns_ids_equal(&head->ids, ids)) {
3894 			dev_err(ctrl->device,
3895 				"IDs don't match for shared namespace %d\n",
3896 					nsid);
3897 			goto out_put_ns_head;
3898 		}
3899 
3900 		if (!multipath && !list_empty(&head->list)) {
3901 			dev_warn(ctrl->device,
3902 				"Found shared namespace %d, but multipathing not supported.\n",
3903 				nsid);
3904 			dev_warn_once(ctrl->device,
3905 				"Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
3906 		}
3907 	}
3908 
3909 	list_add_tail_rcu(&ns->siblings, &head->list);
3910 	ns->head = head;
3911 	mutex_unlock(&ctrl->subsys->lock);
3912 	return 0;
3913 
3914 out_put_ns_head:
3915 	nvme_put_ns_head(head);
3916 out_unlock:
3917 	mutex_unlock(&ctrl->subsys->lock);
3918 	return ret;
3919 }
3920 
3921 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3922 {
3923 	struct nvme_ns *ns, *ret = NULL;
3924 
3925 	down_read(&ctrl->namespaces_rwsem);
3926 	list_for_each_entry(ns, &ctrl->namespaces, list) {
3927 		if (ns->head->ns_id == nsid) {
3928 			if (!nvme_get_ns(ns))
3929 				continue;
3930 			ret = ns;
3931 			break;
3932 		}
3933 		if (ns->head->ns_id > nsid)
3934 			break;
3935 	}
3936 	up_read(&ctrl->namespaces_rwsem);
3937 	return ret;
3938 }
3939 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3940 
3941 /*
3942  * Add the namespace to the controller list while keeping the list ordered.
3943  */
3944 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3945 {
3946 	struct nvme_ns *tmp;
3947 
3948 	list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
3949 		if (tmp->head->ns_id < ns->head->ns_id) {
3950 			list_add(&ns->list, &tmp->list);
3951 			return;
3952 		}
3953 	}
3954 	list_add(&ns->list, &ns->ctrl->namespaces);
3955 }
3956 
3957 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
3958 		struct nvme_ns_ids *ids)
3959 {
3960 	struct nvme_ns *ns;
3961 	struct gendisk *disk;
3962 	struct nvme_id_ns *id;
3963 	int node = ctrl->numa_node;
3964 
3965 	if (nvme_identify_ns(ctrl, nsid, ids, &id))
3966 		return;
3967 
3968 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3969 	if (!ns)
3970 		goto out_free_id;
3971 
3972 	disk = blk_mq_alloc_disk(ctrl->tagset, ns);
3973 	if (IS_ERR(disk))
3974 		goto out_free_ns;
3975 	disk->fops = &nvme_bdev_ops;
3976 	disk->private_data = ns;
3977 
3978 	ns->disk = disk;
3979 	ns->queue = disk->queue;
3980 
3981 	if (ctrl->opts && ctrl->opts->data_digest)
3982 		blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3983 
3984 	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3985 	if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3986 		blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3987 
3988 	ns->ctrl = ctrl;
3989 	kref_init(&ns->kref);
3990 
3991 	if (nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED))
3992 		goto out_cleanup_disk;
3993 
3994 	/*
3995 	 * If multipathing is enabled, the device name for all disks and not
3996 	 * just those that represent shared namespaces needs to be based on the
3997 	 * subsystem instance.  Using the controller instance for private
3998 	 * namespaces could lead to naming collisions between shared and private
3999 	 * namespaces if they don't use a common numbering scheme.
4000 	 *
4001 	 * If multipathing is not enabled, disk names must use the controller
4002 	 * instance as shared namespaces will show up as multiple block
4003 	 * devices.
4004 	 */
4005 	if (ns->head->disk) {
4006 		sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
4007 			ctrl->instance, ns->head->instance);
4008 		disk->flags |= GENHD_FL_HIDDEN;
4009 	} else if (multipath) {
4010 		sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
4011 			ns->head->instance);
4012 	} else {
4013 		sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
4014 			ns->head->instance);
4015 	}
4016 
4017 	if (nvme_update_ns_info(ns, id))
4018 		goto out_unlink_ns;
4019 
4020 	down_write(&ctrl->namespaces_rwsem);
4021 	nvme_ns_add_to_ctrl_list(ns);
4022 	up_write(&ctrl->namespaces_rwsem);
4023 	nvme_get_ctrl(ctrl);
4024 
4025 	if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
4026 		goto out_cleanup_ns_from_list;
4027 
4028 	if (!nvme_ns_head_multipath(ns->head))
4029 		nvme_add_ns_cdev(ns);
4030 
4031 	nvme_mpath_add_disk(ns, id);
4032 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
4033 	kfree(id);
4034 
4035 	return;
4036 
4037  out_cleanup_ns_from_list:
4038 	nvme_put_ctrl(ctrl);
4039 	down_write(&ctrl->namespaces_rwsem);
4040 	list_del_init(&ns->list);
4041 	up_write(&ctrl->namespaces_rwsem);
4042  out_unlink_ns:
4043 	mutex_lock(&ctrl->subsys->lock);
4044 	list_del_rcu(&ns->siblings);
4045 	if (list_empty(&ns->head->list))
4046 		list_del_init(&ns->head->entry);
4047 	mutex_unlock(&ctrl->subsys->lock);
4048 	nvme_put_ns_head(ns->head);
4049  out_cleanup_disk:
4050 	blk_cleanup_disk(disk);
4051  out_free_ns:
4052 	kfree(ns);
4053  out_free_id:
4054 	kfree(id);
4055 }
4056 
4057 static void nvme_ns_remove(struct nvme_ns *ns)
4058 {
4059 	bool last_path = false;
4060 
4061 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4062 		return;
4063 
4064 	clear_bit(NVME_NS_READY, &ns->flags);
4065 	set_capacity(ns->disk, 0);
4066 	nvme_fault_inject_fini(&ns->fault_inject);
4067 
4068 	/*
4069 	 * Ensure that !NVME_NS_READY is seen by other threads to prevent
4070 	 * this ns going back into current_path.
4071 	 */
4072 	synchronize_srcu(&ns->head->srcu);
4073 
4074 	/* wait for concurrent submissions */
4075 	if (nvme_mpath_clear_current_path(ns))
4076 		synchronize_srcu(&ns->head->srcu);
4077 
4078 	mutex_lock(&ns->ctrl->subsys->lock);
4079 	list_del_rcu(&ns->siblings);
4080 	if (list_empty(&ns->head->list)) {
4081 		list_del_init(&ns->head->entry);
4082 		last_path = true;
4083 	}
4084 	mutex_unlock(&ns->ctrl->subsys->lock);
4085 
4086 	/* guarantee not available in head->list */
4087 	synchronize_rcu();
4088 
4089 	if (!nvme_ns_head_multipath(ns->head))
4090 		nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4091 	del_gendisk(ns->disk);
4092 	blk_cleanup_queue(ns->queue);
4093 
4094 	down_write(&ns->ctrl->namespaces_rwsem);
4095 	list_del_init(&ns->list);
4096 	up_write(&ns->ctrl->namespaces_rwsem);
4097 
4098 	if (last_path)
4099 		nvme_mpath_shutdown_disk(ns->head);
4100 	nvme_put_ns(ns);
4101 }
4102 
4103 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4104 {
4105 	struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4106 
4107 	if (ns) {
4108 		nvme_ns_remove(ns);
4109 		nvme_put_ns(ns);
4110 	}
4111 }
4112 
4113 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids)
4114 {
4115 	struct nvme_id_ns *id;
4116 	int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4117 
4118 	if (test_bit(NVME_NS_DEAD, &ns->flags))
4119 		goto out;
4120 
4121 	ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
4122 	if (ret)
4123 		goto out;
4124 
4125 	ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4126 	if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
4127 		dev_err(ns->ctrl->device,
4128 			"identifiers changed for nsid %d\n", ns->head->ns_id);
4129 		goto out_free_id;
4130 	}
4131 
4132 	ret = nvme_update_ns_info(ns, id);
4133 
4134 out_free_id:
4135 	kfree(id);
4136 out:
4137 	/*
4138 	 * Only remove the namespace if we got a fatal error back from the
4139 	 * device, otherwise ignore the error and just move on.
4140 	 *
4141 	 * TODO: we should probably schedule a delayed retry here.
4142 	 */
4143 	if (ret > 0 && (ret & NVME_SC_DNR))
4144 		nvme_ns_remove(ns);
4145 }
4146 
4147 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4148 {
4149 	struct nvme_ns_ids ids = { };
4150 	struct nvme_id_ns_cs_indep *id;
4151 	struct nvme_ns *ns;
4152 	bool ready = true;
4153 
4154 	if (nvme_identify_ns_descs(ctrl, nsid, &ids))
4155 		return;
4156 
4157 	/*
4158 	 * Check if the namespace is ready.  If not ignore it, we will get an
4159 	 * AEN once it becomes ready and restart the scan.
4160 	 */
4161 	if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) &&
4162 	    !nvme_identify_ns_cs_indep(ctrl, nsid, &id)) {
4163 		ready = id->nstat & NVME_NSTAT_NRDY;
4164 		kfree(id);
4165 	}
4166 
4167 	if (!ready)
4168 		return;
4169 
4170 	ns = nvme_find_get_ns(ctrl, nsid);
4171 	if (ns) {
4172 		nvme_validate_ns(ns, &ids);
4173 		nvme_put_ns(ns);
4174 		return;
4175 	}
4176 
4177 	switch (ids.csi) {
4178 	case NVME_CSI_NVM:
4179 		nvme_alloc_ns(ctrl, nsid, &ids);
4180 		break;
4181 	case NVME_CSI_ZNS:
4182 		if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
4183 			dev_warn(ctrl->device,
4184 				"nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
4185 				nsid);
4186 			break;
4187 		}
4188 		if (!nvme_multi_css(ctrl)) {
4189 			dev_warn(ctrl->device,
4190 				"command set not reported for nsid: %d\n",
4191 				nsid);
4192 			break;
4193 		}
4194 		nvme_alloc_ns(ctrl, nsid, &ids);
4195 		break;
4196 	default:
4197 		dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
4198 			ids.csi, nsid);
4199 		break;
4200 	}
4201 }
4202 
4203 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4204 					unsigned nsid)
4205 {
4206 	struct nvme_ns *ns, *next;
4207 	LIST_HEAD(rm_list);
4208 
4209 	down_write(&ctrl->namespaces_rwsem);
4210 	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4211 		if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4212 			list_move_tail(&ns->list, &rm_list);
4213 	}
4214 	up_write(&ctrl->namespaces_rwsem);
4215 
4216 	list_for_each_entry_safe(ns, next, &rm_list, list)
4217 		nvme_ns_remove(ns);
4218 
4219 }
4220 
4221 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4222 {
4223 	const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4224 	__le32 *ns_list;
4225 	u32 prev = 0;
4226 	int ret = 0, i;
4227 
4228 	if (nvme_ctrl_limited_cns(ctrl))
4229 		return -EOPNOTSUPP;
4230 
4231 	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4232 	if (!ns_list)
4233 		return -ENOMEM;
4234 
4235 	for (;;) {
4236 		struct nvme_command cmd = {
4237 			.identify.opcode	= nvme_admin_identify,
4238 			.identify.cns		= NVME_ID_CNS_NS_ACTIVE_LIST,
4239 			.identify.nsid		= cpu_to_le32(prev),
4240 		};
4241 
4242 		ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4243 					    NVME_IDENTIFY_DATA_SIZE);
4244 		if (ret) {
4245 			dev_warn(ctrl->device,
4246 				"Identify NS List failed (status=0x%x)\n", ret);
4247 			goto free;
4248 		}
4249 
4250 		for (i = 0; i < nr_entries; i++) {
4251 			u32 nsid = le32_to_cpu(ns_list[i]);
4252 
4253 			if (!nsid)	/* end of the list? */
4254 				goto out;
4255 			nvme_validate_or_alloc_ns(ctrl, nsid);
4256 			while (++prev < nsid)
4257 				nvme_ns_remove_by_nsid(ctrl, prev);
4258 		}
4259 	}
4260  out:
4261 	nvme_remove_invalid_namespaces(ctrl, prev);
4262  free:
4263 	kfree(ns_list);
4264 	return ret;
4265 }
4266 
4267 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4268 {
4269 	struct nvme_id_ctrl *id;
4270 	u32 nn, i;
4271 
4272 	if (nvme_identify_ctrl(ctrl, &id))
4273 		return;
4274 	nn = le32_to_cpu(id->nn);
4275 	kfree(id);
4276 
4277 	for (i = 1; i <= nn; i++)
4278 		nvme_validate_or_alloc_ns(ctrl, i);
4279 
4280 	nvme_remove_invalid_namespaces(ctrl, nn);
4281 }
4282 
4283 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4284 {
4285 	size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4286 	__le32 *log;
4287 	int error;
4288 
4289 	log = kzalloc(log_size, GFP_KERNEL);
4290 	if (!log)
4291 		return;
4292 
4293 	/*
4294 	 * We need to read the log to clear the AEN, but we don't want to rely
4295 	 * on it for the changed namespace information as userspace could have
4296 	 * raced with us in reading the log page, which could cause us to miss
4297 	 * updates.
4298 	 */
4299 	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4300 			NVME_CSI_NVM, log, log_size, 0);
4301 	if (error)
4302 		dev_warn(ctrl->device,
4303 			"reading changed ns log failed: %d\n", error);
4304 
4305 	kfree(log);
4306 }
4307 
4308 static void nvme_scan_work(struct work_struct *work)
4309 {
4310 	struct nvme_ctrl *ctrl =
4311 		container_of(work, struct nvme_ctrl, scan_work);
4312 	int ret;
4313 
4314 	/* No tagset on a live ctrl means IO queues could not created */
4315 	if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4316 		return;
4317 
4318 	/*
4319 	 * Identify controller limits can change at controller reset due to
4320 	 * new firmware download, even though it is not common we cannot ignore
4321 	 * such scenario. Controller's non-mdts limits are reported in the unit
4322 	 * of logical blocks that is dependent on the format of attached
4323 	 * namespace. Hence re-read the limits at the time of ns allocation.
4324 	 */
4325 	ret = nvme_init_non_mdts_limits(ctrl);
4326 	if (ret < 0) {
4327 		dev_warn(ctrl->device,
4328 			"reading non-mdts-limits failed: %d\n", ret);
4329 		return;
4330 	}
4331 
4332 	if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4333 		dev_info(ctrl->device, "rescanning namespaces.\n");
4334 		nvme_clear_changed_ns_log(ctrl);
4335 	}
4336 
4337 	mutex_lock(&ctrl->scan_lock);
4338 	if (nvme_scan_ns_list(ctrl) != 0)
4339 		nvme_scan_ns_sequential(ctrl);
4340 	mutex_unlock(&ctrl->scan_lock);
4341 }
4342 
4343 /*
4344  * This function iterates the namespace list unlocked to allow recovery from
4345  * controller failure. It is up to the caller to ensure the namespace list is
4346  * not modified by scan work while this function is executing.
4347  */
4348 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4349 {
4350 	struct nvme_ns *ns, *next;
4351 	LIST_HEAD(ns_list);
4352 
4353 	/*
4354 	 * make sure to requeue I/O to all namespaces as these
4355 	 * might result from the scan itself and must complete
4356 	 * for the scan_work to make progress
4357 	 */
4358 	nvme_mpath_clear_ctrl_paths(ctrl);
4359 
4360 	/* prevent racing with ns scanning */
4361 	flush_work(&ctrl->scan_work);
4362 
4363 	/*
4364 	 * The dead states indicates the controller was not gracefully
4365 	 * disconnected. In that case, we won't be able to flush any data while
4366 	 * removing the namespaces' disks; fail all the queues now to avoid
4367 	 * potentially having to clean up the failed sync later.
4368 	 */
4369 	if (ctrl->state == NVME_CTRL_DEAD)
4370 		nvme_kill_queues(ctrl);
4371 
4372 	/* this is a no-op when called from the controller reset handler */
4373 	nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4374 
4375 	down_write(&ctrl->namespaces_rwsem);
4376 	list_splice_init(&ctrl->namespaces, &ns_list);
4377 	up_write(&ctrl->namespaces_rwsem);
4378 
4379 	list_for_each_entry_safe(ns, next, &ns_list, list)
4380 		nvme_ns_remove(ns);
4381 }
4382 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4383 
4384 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4385 {
4386 	struct nvme_ctrl *ctrl =
4387 		container_of(dev, struct nvme_ctrl, ctrl_device);
4388 	struct nvmf_ctrl_options *opts = ctrl->opts;
4389 	int ret;
4390 
4391 	ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4392 	if (ret)
4393 		return ret;
4394 
4395 	if (opts) {
4396 		ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4397 		if (ret)
4398 			return ret;
4399 
4400 		ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4401 				opts->trsvcid ?: "none");
4402 		if (ret)
4403 			return ret;
4404 
4405 		ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4406 				opts->host_traddr ?: "none");
4407 		if (ret)
4408 			return ret;
4409 
4410 		ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4411 				opts->host_iface ?: "none");
4412 	}
4413 	return ret;
4414 }
4415 
4416 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4417 {
4418 	char *envp[2] = { envdata, NULL };
4419 
4420 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4421 }
4422 
4423 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4424 {
4425 	char *envp[2] = { NULL, NULL };
4426 	u32 aen_result = ctrl->aen_result;
4427 
4428 	ctrl->aen_result = 0;
4429 	if (!aen_result)
4430 		return;
4431 
4432 	envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4433 	if (!envp[0])
4434 		return;
4435 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4436 	kfree(envp[0]);
4437 }
4438 
4439 static void nvme_async_event_work(struct work_struct *work)
4440 {
4441 	struct nvme_ctrl *ctrl =
4442 		container_of(work, struct nvme_ctrl, async_event_work);
4443 
4444 	nvme_aen_uevent(ctrl);
4445 
4446 	/*
4447 	 * The transport drivers must guarantee AER submission here is safe by
4448 	 * flushing ctrl async_event_work after changing the controller state
4449 	 * from LIVE and before freeing the admin queue.
4450 	*/
4451 	if (ctrl->state == NVME_CTRL_LIVE)
4452 		ctrl->ops->submit_async_event(ctrl);
4453 }
4454 
4455 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4456 {
4457 
4458 	u32 csts;
4459 
4460 	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4461 		return false;
4462 
4463 	if (csts == ~0)
4464 		return false;
4465 
4466 	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4467 }
4468 
4469 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4470 {
4471 	struct nvme_fw_slot_info_log *log;
4472 
4473 	log = kmalloc(sizeof(*log), GFP_KERNEL);
4474 	if (!log)
4475 		return;
4476 
4477 	if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4478 			log, sizeof(*log), 0))
4479 		dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4480 	kfree(log);
4481 }
4482 
4483 static void nvme_fw_act_work(struct work_struct *work)
4484 {
4485 	struct nvme_ctrl *ctrl = container_of(work,
4486 				struct nvme_ctrl, fw_act_work);
4487 	unsigned long fw_act_timeout;
4488 
4489 	if (ctrl->mtfa)
4490 		fw_act_timeout = jiffies +
4491 				msecs_to_jiffies(ctrl->mtfa * 100);
4492 	else
4493 		fw_act_timeout = jiffies +
4494 				msecs_to_jiffies(admin_timeout * 1000);
4495 
4496 	nvme_stop_queues(ctrl);
4497 	while (nvme_ctrl_pp_status(ctrl)) {
4498 		if (time_after(jiffies, fw_act_timeout)) {
4499 			dev_warn(ctrl->device,
4500 				"Fw activation timeout, reset controller\n");
4501 			nvme_try_sched_reset(ctrl);
4502 			return;
4503 		}
4504 		msleep(100);
4505 	}
4506 
4507 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4508 		return;
4509 
4510 	nvme_start_queues(ctrl);
4511 	/* read FW slot information to clear the AER */
4512 	nvme_get_fw_slot_info(ctrl);
4513 }
4514 
4515 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4516 {
4517 	u32 aer_notice_type = (result & 0xff00) >> 8;
4518 
4519 	trace_nvme_async_event(ctrl, aer_notice_type);
4520 
4521 	switch (aer_notice_type) {
4522 	case NVME_AER_NOTICE_NS_CHANGED:
4523 		set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4524 		nvme_queue_scan(ctrl);
4525 		break;
4526 	case NVME_AER_NOTICE_FW_ACT_STARTING:
4527 		/*
4528 		 * We are (ab)using the RESETTING state to prevent subsequent
4529 		 * recovery actions from interfering with the controller's
4530 		 * firmware activation.
4531 		 */
4532 		if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4533 			queue_work(nvme_wq, &ctrl->fw_act_work);
4534 		break;
4535 #ifdef CONFIG_NVME_MULTIPATH
4536 	case NVME_AER_NOTICE_ANA:
4537 		if (!ctrl->ana_log_buf)
4538 			break;
4539 		queue_work(nvme_wq, &ctrl->ana_work);
4540 		break;
4541 #endif
4542 	case NVME_AER_NOTICE_DISC_CHANGED:
4543 		ctrl->aen_result = result;
4544 		break;
4545 	default:
4546 		dev_warn(ctrl->device, "async event result %08x\n", result);
4547 	}
4548 }
4549 
4550 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4551 		volatile union nvme_result *res)
4552 {
4553 	u32 result = le32_to_cpu(res->u32);
4554 	u32 aer_type = result & 0x07;
4555 
4556 	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4557 		return;
4558 
4559 	switch (aer_type) {
4560 	case NVME_AER_NOTICE:
4561 		nvme_handle_aen_notice(ctrl, result);
4562 		break;
4563 	case NVME_AER_ERROR:
4564 	case NVME_AER_SMART:
4565 	case NVME_AER_CSS:
4566 	case NVME_AER_VS:
4567 		trace_nvme_async_event(ctrl, aer_type);
4568 		ctrl->aen_result = result;
4569 		break;
4570 	default:
4571 		break;
4572 	}
4573 	queue_work(nvme_wq, &ctrl->async_event_work);
4574 }
4575 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4576 
4577 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4578 {
4579 	nvme_mpath_stop(ctrl);
4580 	nvme_stop_keep_alive(ctrl);
4581 	nvme_stop_failfast_work(ctrl);
4582 	flush_work(&ctrl->async_event_work);
4583 	cancel_work_sync(&ctrl->fw_act_work);
4584 }
4585 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4586 
4587 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4588 {
4589 	nvme_start_keep_alive(ctrl);
4590 
4591 	nvme_enable_aen(ctrl);
4592 
4593 	if (ctrl->queue_count > 1) {
4594 		nvme_queue_scan(ctrl);
4595 		nvme_start_queues(ctrl);
4596 		nvme_mpath_update(ctrl);
4597 	}
4598 
4599 	nvme_change_uevent(ctrl, "NVME_EVENT=connected");
4600 }
4601 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4602 
4603 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4604 {
4605 	nvme_hwmon_exit(ctrl);
4606 	nvme_fault_inject_fini(&ctrl->fault_inject);
4607 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
4608 	cdev_device_del(&ctrl->cdev, ctrl->device);
4609 	nvme_put_ctrl(ctrl);
4610 }
4611 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4612 
4613 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4614 {
4615 	struct nvme_effects_log	*cel;
4616 	unsigned long i;
4617 
4618 	xa_for_each(&ctrl->cels, i, cel) {
4619 		xa_erase(&ctrl->cels, i);
4620 		kfree(cel);
4621 	}
4622 
4623 	xa_destroy(&ctrl->cels);
4624 }
4625 
4626 static void nvme_free_ctrl(struct device *dev)
4627 {
4628 	struct nvme_ctrl *ctrl =
4629 		container_of(dev, struct nvme_ctrl, ctrl_device);
4630 	struct nvme_subsystem *subsys = ctrl->subsys;
4631 
4632 	if (!subsys || ctrl->instance != subsys->instance)
4633 		ida_free(&nvme_instance_ida, ctrl->instance);
4634 
4635 	nvme_free_cels(ctrl);
4636 	nvme_mpath_uninit(ctrl);
4637 	__free_page(ctrl->discard_page);
4638 
4639 	if (subsys) {
4640 		mutex_lock(&nvme_subsystems_lock);
4641 		list_del(&ctrl->subsys_entry);
4642 		sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4643 		mutex_unlock(&nvme_subsystems_lock);
4644 	}
4645 
4646 	ctrl->ops->free_ctrl(ctrl);
4647 
4648 	if (subsys)
4649 		nvme_put_subsystem(subsys);
4650 }
4651 
4652 /*
4653  * Initialize a NVMe controller structures.  This needs to be called during
4654  * earliest initialization so that we have the initialized structured around
4655  * during probing.
4656  */
4657 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4658 		const struct nvme_ctrl_ops *ops, unsigned long quirks)
4659 {
4660 	int ret;
4661 
4662 	ctrl->state = NVME_CTRL_NEW;
4663 	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
4664 	spin_lock_init(&ctrl->lock);
4665 	mutex_init(&ctrl->scan_lock);
4666 	INIT_LIST_HEAD(&ctrl->namespaces);
4667 	xa_init(&ctrl->cels);
4668 	init_rwsem(&ctrl->namespaces_rwsem);
4669 	ctrl->dev = dev;
4670 	ctrl->ops = ops;
4671 	ctrl->quirks = quirks;
4672 	ctrl->numa_node = NUMA_NO_NODE;
4673 	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4674 	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4675 	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4676 	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4677 	init_waitqueue_head(&ctrl->state_wq);
4678 
4679 	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4680 	INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
4681 	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4682 	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4683 
4684 	BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4685 			PAGE_SIZE);
4686 	ctrl->discard_page = alloc_page(GFP_KERNEL);
4687 	if (!ctrl->discard_page) {
4688 		ret = -ENOMEM;
4689 		goto out;
4690 	}
4691 
4692 	ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
4693 	if (ret < 0)
4694 		goto out;
4695 	ctrl->instance = ret;
4696 
4697 	device_initialize(&ctrl->ctrl_device);
4698 	ctrl->device = &ctrl->ctrl_device;
4699 	ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
4700 			ctrl->instance);
4701 	ctrl->device->class = nvme_class;
4702 	ctrl->device->parent = ctrl->dev;
4703 	ctrl->device->groups = nvme_dev_attr_groups;
4704 	ctrl->device->release = nvme_free_ctrl;
4705 	dev_set_drvdata(ctrl->device, ctrl);
4706 	ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4707 	if (ret)
4708 		goto out_release_instance;
4709 
4710 	nvme_get_ctrl(ctrl);
4711 	cdev_init(&ctrl->cdev, &nvme_dev_fops);
4712 	ctrl->cdev.owner = ops->module;
4713 	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4714 	if (ret)
4715 		goto out_free_name;
4716 
4717 	/*
4718 	 * Initialize latency tolerance controls.  The sysfs files won't
4719 	 * be visible to userspace unless the device actually supports APST.
4720 	 */
4721 	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4722 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4723 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4724 
4725 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4726 	nvme_mpath_init_ctrl(ctrl);
4727 
4728 	return 0;
4729 out_free_name:
4730 	nvme_put_ctrl(ctrl);
4731 	kfree_const(ctrl->device->kobj.name);
4732 out_release_instance:
4733 	ida_free(&nvme_instance_ida, ctrl->instance);
4734 out:
4735 	if (ctrl->discard_page)
4736 		__free_page(ctrl->discard_page);
4737 	return ret;
4738 }
4739 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4740 
4741 static void nvme_start_ns_queue(struct nvme_ns *ns)
4742 {
4743 	if (test_and_clear_bit(NVME_NS_STOPPED, &ns->flags))
4744 		blk_mq_unquiesce_queue(ns->queue);
4745 }
4746 
4747 static void nvme_stop_ns_queue(struct nvme_ns *ns)
4748 {
4749 	if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
4750 		blk_mq_quiesce_queue(ns->queue);
4751 	else
4752 		blk_mq_wait_quiesce_done(ns->queue);
4753 }
4754 
4755 /*
4756  * Prepare a queue for teardown.
4757  *
4758  * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
4759  * the capacity to 0 after that to avoid blocking dispatchers that may be
4760  * holding bd_butex.  This will end buffered writers dirtying pages that can't
4761  * be synced.
4762  */
4763 static void nvme_set_queue_dying(struct nvme_ns *ns)
4764 {
4765 	if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
4766 		return;
4767 
4768 	blk_mark_disk_dead(ns->disk);
4769 	nvme_start_ns_queue(ns);
4770 
4771 	set_capacity_and_notify(ns->disk, 0);
4772 }
4773 
4774 /**
4775  * nvme_kill_queues(): Ends all namespace queues
4776  * @ctrl: the dead controller that needs to end
4777  *
4778  * Call this function when the driver determines it is unable to get the
4779  * controller in a state capable of servicing IO.
4780  */
4781 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4782 {
4783 	struct nvme_ns *ns;
4784 
4785 	down_read(&ctrl->namespaces_rwsem);
4786 
4787 	/* Forcibly unquiesce queues to avoid blocking dispatch */
4788 	if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4789 		nvme_start_admin_queue(ctrl);
4790 
4791 	list_for_each_entry(ns, &ctrl->namespaces, list)
4792 		nvme_set_queue_dying(ns);
4793 
4794 	up_read(&ctrl->namespaces_rwsem);
4795 }
4796 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4797 
4798 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4799 {
4800 	struct nvme_ns *ns;
4801 
4802 	down_read(&ctrl->namespaces_rwsem);
4803 	list_for_each_entry(ns, &ctrl->namespaces, list)
4804 		blk_mq_unfreeze_queue(ns->queue);
4805 	up_read(&ctrl->namespaces_rwsem);
4806 }
4807 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4808 
4809 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4810 {
4811 	struct nvme_ns *ns;
4812 
4813 	down_read(&ctrl->namespaces_rwsem);
4814 	list_for_each_entry(ns, &ctrl->namespaces, list) {
4815 		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4816 		if (timeout <= 0)
4817 			break;
4818 	}
4819 	up_read(&ctrl->namespaces_rwsem);
4820 	return timeout;
4821 }
4822 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4823 
4824 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4825 {
4826 	struct nvme_ns *ns;
4827 
4828 	down_read(&ctrl->namespaces_rwsem);
4829 	list_for_each_entry(ns, &ctrl->namespaces, list)
4830 		blk_mq_freeze_queue_wait(ns->queue);
4831 	up_read(&ctrl->namespaces_rwsem);
4832 }
4833 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4834 
4835 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4836 {
4837 	struct nvme_ns *ns;
4838 
4839 	down_read(&ctrl->namespaces_rwsem);
4840 	list_for_each_entry(ns, &ctrl->namespaces, list)
4841 		blk_freeze_queue_start(ns->queue);
4842 	up_read(&ctrl->namespaces_rwsem);
4843 }
4844 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4845 
4846 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4847 {
4848 	struct nvme_ns *ns;
4849 
4850 	down_read(&ctrl->namespaces_rwsem);
4851 	list_for_each_entry(ns, &ctrl->namespaces, list)
4852 		nvme_stop_ns_queue(ns);
4853 	up_read(&ctrl->namespaces_rwsem);
4854 }
4855 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4856 
4857 void nvme_start_queues(struct nvme_ctrl *ctrl)
4858 {
4859 	struct nvme_ns *ns;
4860 
4861 	down_read(&ctrl->namespaces_rwsem);
4862 	list_for_each_entry(ns, &ctrl->namespaces, list)
4863 		nvme_start_ns_queue(ns);
4864 	up_read(&ctrl->namespaces_rwsem);
4865 }
4866 EXPORT_SYMBOL_GPL(nvme_start_queues);
4867 
4868 void nvme_stop_admin_queue(struct nvme_ctrl *ctrl)
4869 {
4870 	if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4871 		blk_mq_quiesce_queue(ctrl->admin_q);
4872 	else
4873 		blk_mq_wait_quiesce_done(ctrl->admin_q);
4874 }
4875 EXPORT_SYMBOL_GPL(nvme_stop_admin_queue);
4876 
4877 void nvme_start_admin_queue(struct nvme_ctrl *ctrl)
4878 {
4879 	if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4880 		blk_mq_unquiesce_queue(ctrl->admin_q);
4881 }
4882 EXPORT_SYMBOL_GPL(nvme_start_admin_queue);
4883 
4884 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
4885 {
4886 	struct nvme_ns *ns;
4887 
4888 	down_read(&ctrl->namespaces_rwsem);
4889 	list_for_each_entry(ns, &ctrl->namespaces, list)
4890 		blk_sync_queue(ns->queue);
4891 	up_read(&ctrl->namespaces_rwsem);
4892 }
4893 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
4894 
4895 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4896 {
4897 	nvme_sync_io_queues(ctrl);
4898 	if (ctrl->admin_q)
4899 		blk_sync_queue(ctrl->admin_q);
4900 }
4901 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4902 
4903 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4904 {
4905 	if (file->f_op != &nvme_dev_fops)
4906 		return NULL;
4907 	return file->private_data;
4908 }
4909 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4910 
4911 /*
4912  * Check we didn't inadvertently grow the command structure sizes:
4913  */
4914 static inline void _nvme_check_size(void)
4915 {
4916 	BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4917 	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4918 	BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4919 	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4920 	BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4921 	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4922 	BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4923 	BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4924 	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4925 	BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4926 	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4927 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4928 	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4929 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
4930 			NVME_IDENTIFY_DATA_SIZE);
4931 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4932 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
4933 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4934 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
4935 	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4936 	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4937 	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4938 	BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4939 	BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
4940 }
4941 
4942 
4943 static int __init nvme_core_init(void)
4944 {
4945 	int result = -ENOMEM;
4946 
4947 	_nvme_check_size();
4948 
4949 	nvme_wq = alloc_workqueue("nvme-wq",
4950 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4951 	if (!nvme_wq)
4952 		goto out;
4953 
4954 	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4955 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4956 	if (!nvme_reset_wq)
4957 		goto destroy_wq;
4958 
4959 	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4960 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4961 	if (!nvme_delete_wq)
4962 		goto destroy_reset_wq;
4963 
4964 	result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
4965 			NVME_MINORS, "nvme");
4966 	if (result < 0)
4967 		goto destroy_delete_wq;
4968 
4969 	nvme_class = class_create(THIS_MODULE, "nvme");
4970 	if (IS_ERR(nvme_class)) {
4971 		result = PTR_ERR(nvme_class);
4972 		goto unregister_chrdev;
4973 	}
4974 	nvme_class->dev_uevent = nvme_class_uevent;
4975 
4976 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4977 	if (IS_ERR(nvme_subsys_class)) {
4978 		result = PTR_ERR(nvme_subsys_class);
4979 		goto destroy_class;
4980 	}
4981 
4982 	result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
4983 				     "nvme-generic");
4984 	if (result < 0)
4985 		goto destroy_subsys_class;
4986 
4987 	nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic");
4988 	if (IS_ERR(nvme_ns_chr_class)) {
4989 		result = PTR_ERR(nvme_ns_chr_class);
4990 		goto unregister_generic_ns;
4991 	}
4992 
4993 	return 0;
4994 
4995 unregister_generic_ns:
4996 	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
4997 destroy_subsys_class:
4998 	class_destroy(nvme_subsys_class);
4999 destroy_class:
5000 	class_destroy(nvme_class);
5001 unregister_chrdev:
5002 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5003 destroy_delete_wq:
5004 	destroy_workqueue(nvme_delete_wq);
5005 destroy_reset_wq:
5006 	destroy_workqueue(nvme_reset_wq);
5007 destroy_wq:
5008 	destroy_workqueue(nvme_wq);
5009 out:
5010 	return result;
5011 }
5012 
5013 static void __exit nvme_core_exit(void)
5014 {
5015 	class_destroy(nvme_ns_chr_class);
5016 	class_destroy(nvme_subsys_class);
5017 	class_destroy(nvme_class);
5018 	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5019 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5020 	destroy_workqueue(nvme_delete_wq);
5021 	destroy_workqueue(nvme_reset_wq);
5022 	destroy_workqueue(nvme_wq);
5023 	ida_destroy(&nvme_ns_chr_minor_ida);
5024 	ida_destroy(&nvme_instance_ida);
5025 }
5026 
5027 MODULE_LICENSE("GPL");
5028 MODULE_VERSION("1.0");
5029 module_init(nvme_core_init);
5030 module_exit(nvme_core_exit);
5031