xref: /openbmc/linux/drivers/acpi/nfit/core.c (revision 2010776f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4  */
5 #include <linux/list_sort.h>
6 #include <linux/libnvdimm.h>
7 #include <linux/module.h>
8 #include <linux/nospec.h>
9 #include <linux/mutex.h>
10 #include <linux/ndctl.h>
11 #include <linux/sysfs.h>
12 #include <linux/delay.h>
13 #include <linux/list.h>
14 #include <linux/acpi.h>
15 #include <linux/sort.h>
16 #include <linux/io.h>
17 #include <linux/nd.h>
18 #include <asm/cacheflush.h>
19 #include <acpi/nfit.h>
20 #include "intel.h"
21 #include "nfit.h"
22 
23 /*
24  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
25  * irrelevant.
26  */
27 #include <linux/io-64-nonatomic-hi-lo.h>
28 
29 static bool force_enable_dimms;
30 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
31 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
32 
33 static bool disable_vendor_specific;
34 module_param(disable_vendor_specific, bool, S_IRUGO);
35 MODULE_PARM_DESC(disable_vendor_specific,
36 		"Limit commands to the publicly specified set");
37 
38 static unsigned long override_dsm_mask;
39 module_param(override_dsm_mask, ulong, S_IRUGO);
40 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
41 
42 static int default_dsm_family = -1;
43 module_param(default_dsm_family, int, S_IRUGO);
44 MODULE_PARM_DESC(default_dsm_family,
45 		"Try this DSM type first when identifying NVDIMM family");
46 
47 static bool no_init_ars;
48 module_param(no_init_ars, bool, 0644);
49 MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time");
50 
51 static bool force_labels;
52 module_param(force_labels, bool, 0444);
53 MODULE_PARM_DESC(force_labels, "Opt-in to labels despite missing methods");
54 
55 LIST_HEAD(acpi_descs);
56 DEFINE_MUTEX(acpi_desc_lock);
57 
58 static struct workqueue_struct *nfit_wq;
59 
60 struct nfit_table_prev {
61 	struct list_head spas;
62 	struct list_head memdevs;
63 	struct list_head dcrs;
64 	struct list_head bdws;
65 	struct list_head idts;
66 	struct list_head flushes;
67 };
68 
69 static guid_t nfit_uuid[NFIT_UUID_MAX];
70 
71 const guid_t *to_nfit_uuid(enum nfit_uuids id)
72 {
73 	return &nfit_uuid[id];
74 }
75 EXPORT_SYMBOL(to_nfit_uuid);
76 
77 static const guid_t *to_nfit_bus_uuid(int family)
78 {
79 	if (WARN_ONCE(family == NVDIMM_BUS_FAMILY_NFIT,
80 			"only secondary bus families can be translated\n"))
81 		return NULL;
82 	/*
83 	 * The index of bus UUIDs starts immediately following the last
84 	 * NVDIMM/leaf family.
85 	 */
86 	return to_nfit_uuid(family + NVDIMM_FAMILY_MAX);
87 }
88 
89 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
90 {
91 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
92 
93 	/*
94 	 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
95 	 * acpi_device.
96 	 */
97 	if (!nd_desc->provider_name
98 			|| strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
99 		return NULL;
100 
101 	return to_acpi_device(acpi_desc->dev);
102 }
103 
104 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
105 {
106 	struct nd_cmd_clear_error *clear_err;
107 	struct nd_cmd_ars_status *ars_status;
108 	u16 flags;
109 
110 	switch (cmd) {
111 	case ND_CMD_ARS_CAP:
112 		if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
113 			return -ENOTTY;
114 
115 		/* Command failed */
116 		if (status & 0xffff)
117 			return -EIO;
118 
119 		/* No supported scan types for this range */
120 		flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
121 		if ((status >> 16 & flags) == 0)
122 			return -ENOTTY;
123 		return 0;
124 	case ND_CMD_ARS_START:
125 		/* ARS is in progress */
126 		if ((status & 0xffff) == NFIT_ARS_START_BUSY)
127 			return -EBUSY;
128 
129 		/* Command failed */
130 		if (status & 0xffff)
131 			return -EIO;
132 		return 0;
133 	case ND_CMD_ARS_STATUS:
134 		ars_status = buf;
135 		/* Command failed */
136 		if (status & 0xffff)
137 			return -EIO;
138 		/* Check extended status (Upper two bytes) */
139 		if (status == NFIT_ARS_STATUS_DONE)
140 			return 0;
141 
142 		/* ARS is in progress */
143 		if (status == NFIT_ARS_STATUS_BUSY)
144 			return -EBUSY;
145 
146 		/* No ARS performed for the current boot */
147 		if (status == NFIT_ARS_STATUS_NONE)
148 			return -EAGAIN;
149 
150 		/*
151 		 * ARS interrupted, either we overflowed or some other
152 		 * agent wants the scan to stop.  If we didn't overflow
153 		 * then just continue with the returned results.
154 		 */
155 		if (status == NFIT_ARS_STATUS_INTR) {
156 			if (ars_status->out_length >= 40 && (ars_status->flags
157 						& NFIT_ARS_F_OVERFLOW))
158 				return -ENOSPC;
159 			return 0;
160 		}
161 
162 		/* Unknown status */
163 		if (status >> 16)
164 			return -EIO;
165 		return 0;
166 	case ND_CMD_CLEAR_ERROR:
167 		clear_err = buf;
168 		if (status & 0xffff)
169 			return -EIO;
170 		if (!clear_err->cleared)
171 			return -EIO;
172 		if (clear_err->length > clear_err->cleared)
173 			return clear_err->cleared;
174 		return 0;
175 	default:
176 		break;
177 	}
178 
179 	/* all other non-zero status results in an error */
180 	if (status)
181 		return -EIO;
182 	return 0;
183 }
184 
185 #define ACPI_LABELS_LOCKED 3
186 
187 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
188 		u32 status)
189 {
190 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
191 
192 	switch (cmd) {
193 	case ND_CMD_GET_CONFIG_SIZE:
194 		/*
195 		 * In the _LSI, _LSR, _LSW case the locked status is
196 		 * communicated via the read/write commands
197 		 */
198 		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
199 			break;
200 
201 		if (status >> 16 & ND_CONFIG_LOCKED)
202 			return -EACCES;
203 		break;
204 	case ND_CMD_GET_CONFIG_DATA:
205 		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
206 				&& status == ACPI_LABELS_LOCKED)
207 			return -EACCES;
208 		break;
209 	case ND_CMD_SET_CONFIG_DATA:
210 		if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
211 				&& status == ACPI_LABELS_LOCKED)
212 			return -EACCES;
213 		break;
214 	default:
215 		break;
216 	}
217 
218 	/* all other non-zero status results in an error */
219 	if (status)
220 		return -EIO;
221 	return 0;
222 }
223 
224 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
225 		u32 status)
226 {
227 	if (!nvdimm)
228 		return xlat_bus_status(buf, cmd, status);
229 	return xlat_nvdimm_status(nvdimm, buf, cmd, status);
230 }
231 
232 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */
233 static union acpi_object *pkg_to_buf(union acpi_object *pkg)
234 {
235 	int i;
236 	void *dst;
237 	size_t size = 0;
238 	union acpi_object *buf = NULL;
239 
240 	if (pkg->type != ACPI_TYPE_PACKAGE) {
241 		WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
242 				pkg->type);
243 		goto err;
244 	}
245 
246 	for (i = 0; i < pkg->package.count; i++) {
247 		union acpi_object *obj = &pkg->package.elements[i];
248 
249 		if (obj->type == ACPI_TYPE_INTEGER)
250 			size += 4;
251 		else if (obj->type == ACPI_TYPE_BUFFER)
252 			size += obj->buffer.length;
253 		else {
254 			WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
255 					obj->type);
256 			goto err;
257 		}
258 	}
259 
260 	buf = ACPI_ALLOCATE(sizeof(*buf) + size);
261 	if (!buf)
262 		goto err;
263 
264 	dst = buf + 1;
265 	buf->type = ACPI_TYPE_BUFFER;
266 	buf->buffer.length = size;
267 	buf->buffer.pointer = dst;
268 	for (i = 0; i < pkg->package.count; i++) {
269 		union acpi_object *obj = &pkg->package.elements[i];
270 
271 		if (obj->type == ACPI_TYPE_INTEGER) {
272 			memcpy(dst, &obj->integer.value, 4);
273 			dst += 4;
274 		} else if (obj->type == ACPI_TYPE_BUFFER) {
275 			memcpy(dst, obj->buffer.pointer, obj->buffer.length);
276 			dst += obj->buffer.length;
277 		}
278 	}
279 err:
280 	ACPI_FREE(pkg);
281 	return buf;
282 }
283 
284 static union acpi_object *int_to_buf(union acpi_object *integer)
285 {
286 	union acpi_object *buf = NULL;
287 	void *dst = NULL;
288 
289 	if (integer->type != ACPI_TYPE_INTEGER) {
290 		WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
291 				integer->type);
292 		goto err;
293 	}
294 
295 	buf = ACPI_ALLOCATE(sizeof(*buf) + 4);
296 	if (!buf)
297 		goto err;
298 
299 	dst = buf + 1;
300 	buf->type = ACPI_TYPE_BUFFER;
301 	buf->buffer.length = 4;
302 	buf->buffer.pointer = dst;
303 	memcpy(dst, &integer->integer.value, 4);
304 err:
305 	ACPI_FREE(integer);
306 	return buf;
307 }
308 
309 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset,
310 		u32 len, void *data)
311 {
312 	acpi_status rc;
313 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
314 	struct acpi_object_list input = {
315 		.count = 3,
316 		.pointer = (union acpi_object []) {
317 			[0] = {
318 				.integer.type = ACPI_TYPE_INTEGER,
319 				.integer.value = offset,
320 			},
321 			[1] = {
322 				.integer.type = ACPI_TYPE_INTEGER,
323 				.integer.value = len,
324 			},
325 			[2] = {
326 				.buffer.type = ACPI_TYPE_BUFFER,
327 				.buffer.pointer = data,
328 				.buffer.length = len,
329 			},
330 		},
331 	};
332 
333 	rc = acpi_evaluate_object(handle, "_LSW", &input, &buf);
334 	if (ACPI_FAILURE(rc))
335 		return NULL;
336 	return int_to_buf(buf.pointer);
337 }
338 
339 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset,
340 		u32 len)
341 {
342 	acpi_status rc;
343 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
344 	struct acpi_object_list input = {
345 		.count = 2,
346 		.pointer = (union acpi_object []) {
347 			[0] = {
348 				.integer.type = ACPI_TYPE_INTEGER,
349 				.integer.value = offset,
350 			},
351 			[1] = {
352 				.integer.type = ACPI_TYPE_INTEGER,
353 				.integer.value = len,
354 			},
355 		},
356 	};
357 
358 	rc = acpi_evaluate_object(handle, "_LSR", &input, &buf);
359 	if (ACPI_FAILURE(rc))
360 		return NULL;
361 	return pkg_to_buf(buf.pointer);
362 }
363 
364 static union acpi_object *acpi_label_info(acpi_handle handle)
365 {
366 	acpi_status rc;
367 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
368 
369 	rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf);
370 	if (ACPI_FAILURE(rc))
371 		return NULL;
372 	return pkg_to_buf(buf.pointer);
373 }
374 
375 static u8 nfit_dsm_revid(unsigned family, unsigned func)
376 {
377 	static const u8 revid_table[NVDIMM_FAMILY_MAX+1][NVDIMM_CMD_MAX+1] = {
378 		[NVDIMM_FAMILY_INTEL] = {
379 			[NVDIMM_INTEL_GET_MODES ...
380 				NVDIMM_INTEL_FW_ACTIVATE_ARM] = 2,
381 		},
382 	};
383 	u8 id;
384 
385 	if (family > NVDIMM_FAMILY_MAX)
386 		return 0;
387 	if (func > NVDIMM_CMD_MAX)
388 		return 0;
389 	id = revid_table[family][func];
390 	if (id == 0)
391 		return 1; /* default */
392 	return id;
393 }
394 
395 static bool payload_dumpable(struct nvdimm *nvdimm, unsigned int func)
396 {
397 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
398 
399 	if (nfit_mem && nfit_mem->family == NVDIMM_FAMILY_INTEL
400 			&& func >= NVDIMM_INTEL_GET_SECURITY_STATE
401 			&& func <= NVDIMM_INTEL_MASTER_SECURE_ERASE)
402 		return IS_ENABLED(CONFIG_NFIT_SECURITY_DEBUG);
403 	return true;
404 }
405 
406 static int cmd_to_func(struct nfit_mem *nfit_mem, unsigned int cmd,
407 		struct nd_cmd_pkg *call_pkg, int *family)
408 {
409 	if (call_pkg) {
410 		int i;
411 
412 		if (nfit_mem && nfit_mem->family != call_pkg->nd_family)
413 			return -ENOTTY;
414 
415 		for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
416 			if (call_pkg->nd_reserved2[i])
417 				return -EINVAL;
418 		*family = call_pkg->nd_family;
419 		return call_pkg->nd_command;
420 	}
421 
422 	/* In the !call_pkg case, bus commands == bus functions */
423 	if (!nfit_mem)
424 		return cmd;
425 
426 	/* Linux ND commands == NVDIMM_FAMILY_INTEL function numbers */
427 	if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
428 		return cmd;
429 
430 	/*
431 	 * Force function number validation to fail since 0 is never
432 	 * published as a valid function in dsm_mask.
433 	 */
434 	return 0;
435 }
436 
437 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
438 		unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
439 {
440 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
441 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
442 	union acpi_object in_obj, in_buf, *out_obj;
443 	const struct nd_cmd_desc *desc = NULL;
444 	struct device *dev = acpi_desc->dev;
445 	struct nd_cmd_pkg *call_pkg = NULL;
446 	const char *cmd_name, *dimm_name;
447 	unsigned long cmd_mask, dsm_mask;
448 	u32 offset, fw_status = 0;
449 	acpi_handle handle;
450 	const guid_t *guid;
451 	int func, rc, i;
452 	int family = 0;
453 
454 	if (cmd_rc)
455 		*cmd_rc = -EINVAL;
456 
457 	if (cmd == ND_CMD_CALL)
458 		call_pkg = buf;
459 	func = cmd_to_func(nfit_mem, cmd, call_pkg, &family);
460 	if (func < 0)
461 		return func;
462 
463 	if (nvdimm) {
464 		struct acpi_device *adev = nfit_mem->adev;
465 
466 		if (!adev)
467 			return -ENOTTY;
468 
469 		dimm_name = nvdimm_name(nvdimm);
470 		cmd_name = nvdimm_cmd_name(cmd);
471 		cmd_mask = nvdimm_cmd_mask(nvdimm);
472 		dsm_mask = nfit_mem->dsm_mask;
473 		desc = nd_cmd_dimm_desc(cmd);
474 		guid = to_nfit_uuid(nfit_mem->family);
475 		handle = adev->handle;
476 	} else {
477 		struct acpi_device *adev = to_acpi_dev(acpi_desc);
478 
479 		cmd_name = nvdimm_bus_cmd_name(cmd);
480 		cmd_mask = nd_desc->cmd_mask;
481 		if (cmd == ND_CMD_CALL && call_pkg->nd_family) {
482 			family = call_pkg->nd_family;
483 			if (family > NVDIMM_BUS_FAMILY_MAX ||
484 			    !test_bit(family, &nd_desc->bus_family_mask))
485 				return -EINVAL;
486 			family = array_index_nospec(family,
487 						    NVDIMM_BUS_FAMILY_MAX + 1);
488 			dsm_mask = acpi_desc->family_dsm_mask[family];
489 			guid = to_nfit_bus_uuid(family);
490 		} else {
491 			dsm_mask = acpi_desc->bus_dsm_mask;
492 			guid = to_nfit_uuid(NFIT_DEV_BUS);
493 		}
494 		desc = nd_cmd_bus_desc(cmd);
495 		handle = adev->handle;
496 		dimm_name = "bus";
497 	}
498 
499 	if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
500 		return -ENOTTY;
501 
502 	/*
503 	 * Check for a valid command.  For ND_CMD_CALL, we also have to
504 	 * make sure that the DSM function is supported.
505 	 */
506 	if (cmd == ND_CMD_CALL &&
507 	    (func > NVDIMM_CMD_MAX || !test_bit(func, &dsm_mask)))
508 		return -ENOTTY;
509 	else if (!test_bit(cmd, &cmd_mask))
510 		return -ENOTTY;
511 
512 	in_obj.type = ACPI_TYPE_PACKAGE;
513 	in_obj.package.count = 1;
514 	in_obj.package.elements = &in_buf;
515 	in_buf.type = ACPI_TYPE_BUFFER;
516 	in_buf.buffer.pointer = buf;
517 	in_buf.buffer.length = 0;
518 
519 	/* libnvdimm has already validated the input envelope */
520 	for (i = 0; i < desc->in_num; i++)
521 		in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
522 				i, buf);
523 
524 	if (call_pkg) {
525 		/* skip over package wrapper */
526 		in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
527 		in_buf.buffer.length = call_pkg->nd_size_in;
528 	}
529 
530 	dev_dbg(dev, "%s cmd: %d: family: %d func: %d input length: %d\n",
531 		dimm_name, cmd, family, func, in_buf.buffer.length);
532 	if (payload_dumpable(nvdimm, func))
533 		print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
534 				in_buf.buffer.pointer,
535 				min_t(u32, 256, in_buf.buffer.length), true);
536 
537 	/* call the BIOS, prefer the named methods over _DSM if available */
538 	if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE
539 			&& test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
540 		out_obj = acpi_label_info(handle);
541 	else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA
542 			&& test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
543 		struct nd_cmd_get_config_data_hdr *p = buf;
544 
545 		out_obj = acpi_label_read(handle, p->in_offset, p->in_length);
546 	} else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA
547 			&& test_bit(NFIT_MEM_LSW, &nfit_mem->flags)) {
548 		struct nd_cmd_set_config_hdr *p = buf;
549 
550 		out_obj = acpi_label_write(handle, p->in_offset, p->in_length,
551 				p->in_buf);
552 	} else {
553 		u8 revid;
554 
555 		if (nvdimm)
556 			revid = nfit_dsm_revid(nfit_mem->family, func);
557 		else
558 			revid = 1;
559 		out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
560 	}
561 
562 	if (!out_obj) {
563 		dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name);
564 		return -EINVAL;
565 	}
566 
567 	if (out_obj->type != ACPI_TYPE_BUFFER) {
568 		dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n",
569 				dimm_name, cmd_name, out_obj->type);
570 		rc = -EINVAL;
571 		goto out;
572 	}
573 
574 	dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name,
575 			cmd_name, out_obj->buffer.length);
576 	print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
577 			out_obj->buffer.pointer,
578 			min_t(u32, 128, out_obj->buffer.length), true);
579 
580 	if (call_pkg) {
581 		call_pkg->nd_fw_size = out_obj->buffer.length;
582 		memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
583 			out_obj->buffer.pointer,
584 			min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
585 
586 		ACPI_FREE(out_obj);
587 		/*
588 		 * Need to support FW function w/o known size in advance.
589 		 * Caller can determine required size based upon nd_fw_size.
590 		 * If we return an error (like elsewhere) then caller wouldn't
591 		 * be able to rely upon data returned to make calculation.
592 		 */
593 		if (cmd_rc)
594 			*cmd_rc = 0;
595 		return 0;
596 	}
597 
598 	for (i = 0, offset = 0; i < desc->out_num; i++) {
599 		u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
600 				(u32 *) out_obj->buffer.pointer,
601 				out_obj->buffer.length - offset);
602 
603 		if (offset + out_size > out_obj->buffer.length) {
604 			dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n",
605 					dimm_name, cmd_name, i);
606 			break;
607 		}
608 
609 		if (in_buf.buffer.length + offset + out_size > buf_len) {
610 			dev_dbg(dev, "%s output overrun cmd: %s field: %d\n",
611 					dimm_name, cmd_name, i);
612 			rc = -ENXIO;
613 			goto out;
614 		}
615 		memcpy(buf + in_buf.buffer.length + offset,
616 				out_obj->buffer.pointer + offset, out_size);
617 		offset += out_size;
618 	}
619 
620 	/*
621 	 * Set fw_status for all the commands with a known format to be
622 	 * later interpreted by xlat_status().
623 	 */
624 	if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP
625 					&& cmd <= ND_CMD_CLEAR_ERROR)
626 				|| (nvdimm && cmd >= ND_CMD_SMART
627 					&& cmd <= ND_CMD_VENDOR)))
628 		fw_status = *(u32 *) out_obj->buffer.pointer;
629 
630 	if (offset + in_buf.buffer.length < buf_len) {
631 		if (i >= 1) {
632 			/*
633 			 * status valid, return the number of bytes left
634 			 * unfilled in the output buffer
635 			 */
636 			rc = buf_len - offset - in_buf.buffer.length;
637 			if (cmd_rc)
638 				*cmd_rc = xlat_status(nvdimm, buf, cmd,
639 						fw_status);
640 		} else {
641 			dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
642 					__func__, dimm_name, cmd_name, buf_len,
643 					offset);
644 			rc = -ENXIO;
645 		}
646 	} else {
647 		rc = 0;
648 		if (cmd_rc)
649 			*cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status);
650 	}
651 
652  out:
653 	ACPI_FREE(out_obj);
654 
655 	return rc;
656 }
657 EXPORT_SYMBOL_GPL(acpi_nfit_ctl);
658 
659 static const char *spa_type_name(u16 type)
660 {
661 	static const char *to_name[] = {
662 		[NFIT_SPA_VOLATILE] = "volatile",
663 		[NFIT_SPA_PM] = "pmem",
664 		[NFIT_SPA_DCR] = "dimm-control-region",
665 		[NFIT_SPA_BDW] = "block-data-window",
666 		[NFIT_SPA_VDISK] = "volatile-disk",
667 		[NFIT_SPA_VCD] = "volatile-cd",
668 		[NFIT_SPA_PDISK] = "persistent-disk",
669 		[NFIT_SPA_PCD] = "persistent-cd",
670 
671 	};
672 
673 	if (type > NFIT_SPA_PCD)
674 		return "unknown";
675 
676 	return to_name[type];
677 }
678 
679 int nfit_spa_type(struct acpi_nfit_system_address *spa)
680 {
681 	int i;
682 
683 	for (i = 0; i < NFIT_UUID_MAX; i++)
684 		if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid))
685 			return i;
686 	return -1;
687 }
688 
689 static size_t sizeof_spa(struct acpi_nfit_system_address *spa)
690 {
691 	if (spa->flags & ACPI_NFIT_LOCATION_COOKIE_VALID)
692 		return sizeof(*spa);
693 	return sizeof(*spa) - 8;
694 }
695 
696 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
697 		struct nfit_table_prev *prev,
698 		struct acpi_nfit_system_address *spa)
699 {
700 	struct device *dev = acpi_desc->dev;
701 	struct nfit_spa *nfit_spa;
702 
703 	if (spa->header.length != sizeof_spa(spa))
704 		return false;
705 
706 	list_for_each_entry(nfit_spa, &prev->spas, list) {
707 		if (memcmp(nfit_spa->spa, spa, sizeof_spa(spa)) == 0) {
708 			list_move_tail(&nfit_spa->list, &acpi_desc->spas);
709 			return true;
710 		}
711 	}
712 
713 	nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof_spa(spa),
714 			GFP_KERNEL);
715 	if (!nfit_spa)
716 		return false;
717 	INIT_LIST_HEAD(&nfit_spa->list);
718 	memcpy(nfit_spa->spa, spa, sizeof_spa(spa));
719 	list_add_tail(&nfit_spa->list, &acpi_desc->spas);
720 	dev_dbg(dev, "spa index: %d type: %s\n",
721 			spa->range_index,
722 			spa_type_name(nfit_spa_type(spa)));
723 	return true;
724 }
725 
726 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
727 		struct nfit_table_prev *prev,
728 		struct acpi_nfit_memory_map *memdev)
729 {
730 	struct device *dev = acpi_desc->dev;
731 	struct nfit_memdev *nfit_memdev;
732 
733 	if (memdev->header.length != sizeof(*memdev))
734 		return false;
735 
736 	list_for_each_entry(nfit_memdev, &prev->memdevs, list)
737 		if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
738 			list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
739 			return true;
740 		}
741 
742 	nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
743 			GFP_KERNEL);
744 	if (!nfit_memdev)
745 		return false;
746 	INIT_LIST_HEAD(&nfit_memdev->list);
747 	memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
748 	list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
749 	dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
750 			memdev->device_handle, memdev->range_index,
751 			memdev->region_index, memdev->flags);
752 	return true;
753 }
754 
755 int nfit_get_smbios_id(u32 device_handle, u16 *flags)
756 {
757 	struct acpi_nfit_memory_map *memdev;
758 	struct acpi_nfit_desc *acpi_desc;
759 	struct nfit_mem *nfit_mem;
760 	u16 physical_id;
761 
762 	mutex_lock(&acpi_desc_lock);
763 	list_for_each_entry(acpi_desc, &acpi_descs, list) {
764 		mutex_lock(&acpi_desc->init_mutex);
765 		list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
766 			memdev = __to_nfit_memdev(nfit_mem);
767 			if (memdev->device_handle == device_handle) {
768 				*flags = memdev->flags;
769 				physical_id = memdev->physical_id;
770 				mutex_unlock(&acpi_desc->init_mutex);
771 				mutex_unlock(&acpi_desc_lock);
772 				return physical_id;
773 			}
774 		}
775 		mutex_unlock(&acpi_desc->init_mutex);
776 	}
777 	mutex_unlock(&acpi_desc_lock);
778 
779 	return -ENODEV;
780 }
781 EXPORT_SYMBOL_GPL(nfit_get_smbios_id);
782 
783 /*
784  * An implementation may provide a truncated control region if no block windows
785  * are defined.
786  */
787 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
788 {
789 	if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
790 				window_size))
791 		return 0;
792 	if (dcr->windows)
793 		return sizeof(*dcr);
794 	return offsetof(struct acpi_nfit_control_region, window_size);
795 }
796 
797 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
798 		struct nfit_table_prev *prev,
799 		struct acpi_nfit_control_region *dcr)
800 {
801 	struct device *dev = acpi_desc->dev;
802 	struct nfit_dcr *nfit_dcr;
803 
804 	if (!sizeof_dcr(dcr))
805 		return false;
806 
807 	list_for_each_entry(nfit_dcr, &prev->dcrs, list)
808 		if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
809 			list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
810 			return true;
811 		}
812 
813 	nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
814 			GFP_KERNEL);
815 	if (!nfit_dcr)
816 		return false;
817 	INIT_LIST_HEAD(&nfit_dcr->list);
818 	memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
819 	list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
820 	dev_dbg(dev, "dcr index: %d windows: %d\n",
821 			dcr->region_index, dcr->windows);
822 	return true;
823 }
824 
825 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
826 		struct nfit_table_prev *prev,
827 		struct acpi_nfit_data_region *bdw)
828 {
829 	struct device *dev = acpi_desc->dev;
830 	struct nfit_bdw *nfit_bdw;
831 
832 	if (bdw->header.length != sizeof(*bdw))
833 		return false;
834 	list_for_each_entry(nfit_bdw, &prev->bdws, list)
835 		if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
836 			list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
837 			return true;
838 		}
839 
840 	nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
841 			GFP_KERNEL);
842 	if (!nfit_bdw)
843 		return false;
844 	INIT_LIST_HEAD(&nfit_bdw->list);
845 	memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
846 	list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
847 	dev_dbg(dev, "bdw dcr: %d windows: %d\n",
848 			bdw->region_index, bdw->windows);
849 	return true;
850 }
851 
852 static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
853 {
854 	if (idt->header.length < sizeof(*idt))
855 		return 0;
856 	return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
857 }
858 
859 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
860 		struct nfit_table_prev *prev,
861 		struct acpi_nfit_interleave *idt)
862 {
863 	struct device *dev = acpi_desc->dev;
864 	struct nfit_idt *nfit_idt;
865 
866 	if (!sizeof_idt(idt))
867 		return false;
868 
869 	list_for_each_entry(nfit_idt, &prev->idts, list) {
870 		if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
871 			continue;
872 
873 		if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
874 			list_move_tail(&nfit_idt->list, &acpi_desc->idts);
875 			return true;
876 		}
877 	}
878 
879 	nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
880 			GFP_KERNEL);
881 	if (!nfit_idt)
882 		return false;
883 	INIT_LIST_HEAD(&nfit_idt->list);
884 	memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
885 	list_add_tail(&nfit_idt->list, &acpi_desc->idts);
886 	dev_dbg(dev, "idt index: %d num_lines: %d\n",
887 			idt->interleave_index, idt->line_count);
888 	return true;
889 }
890 
891 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
892 {
893 	if (flush->header.length < sizeof(*flush))
894 		return 0;
895 	return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
896 }
897 
898 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
899 		struct nfit_table_prev *prev,
900 		struct acpi_nfit_flush_address *flush)
901 {
902 	struct device *dev = acpi_desc->dev;
903 	struct nfit_flush *nfit_flush;
904 
905 	if (!sizeof_flush(flush))
906 		return false;
907 
908 	list_for_each_entry(nfit_flush, &prev->flushes, list) {
909 		if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
910 			continue;
911 
912 		if (memcmp(nfit_flush->flush, flush,
913 					sizeof_flush(flush)) == 0) {
914 			list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
915 			return true;
916 		}
917 	}
918 
919 	nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
920 			+ sizeof_flush(flush), GFP_KERNEL);
921 	if (!nfit_flush)
922 		return false;
923 	INIT_LIST_HEAD(&nfit_flush->list);
924 	memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
925 	list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
926 	dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n",
927 			flush->device_handle, flush->hint_count);
928 	return true;
929 }
930 
931 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
932 		struct acpi_nfit_capabilities *pcap)
933 {
934 	struct device *dev = acpi_desc->dev;
935 	u32 mask;
936 
937 	mask = (1 << (pcap->highest_capability + 1)) - 1;
938 	acpi_desc->platform_cap = pcap->capabilities & mask;
939 	dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap);
940 	return true;
941 }
942 
943 static void *add_table(struct acpi_nfit_desc *acpi_desc,
944 		struct nfit_table_prev *prev, void *table, const void *end)
945 {
946 	struct device *dev = acpi_desc->dev;
947 	struct acpi_nfit_header *hdr;
948 	void *err = ERR_PTR(-ENOMEM);
949 
950 	if (table >= end)
951 		return NULL;
952 
953 	hdr = table;
954 	if (!hdr->length) {
955 		dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
956 			hdr->type);
957 		return NULL;
958 	}
959 
960 	switch (hdr->type) {
961 	case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
962 		if (!add_spa(acpi_desc, prev, table))
963 			return err;
964 		break;
965 	case ACPI_NFIT_TYPE_MEMORY_MAP:
966 		if (!add_memdev(acpi_desc, prev, table))
967 			return err;
968 		break;
969 	case ACPI_NFIT_TYPE_CONTROL_REGION:
970 		if (!add_dcr(acpi_desc, prev, table))
971 			return err;
972 		break;
973 	case ACPI_NFIT_TYPE_DATA_REGION:
974 		if (!add_bdw(acpi_desc, prev, table))
975 			return err;
976 		break;
977 	case ACPI_NFIT_TYPE_INTERLEAVE:
978 		if (!add_idt(acpi_desc, prev, table))
979 			return err;
980 		break;
981 	case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
982 		if (!add_flush(acpi_desc, prev, table))
983 			return err;
984 		break;
985 	case ACPI_NFIT_TYPE_SMBIOS:
986 		dev_dbg(dev, "smbios\n");
987 		break;
988 	case ACPI_NFIT_TYPE_CAPABILITIES:
989 		if (!add_platform_cap(acpi_desc, table))
990 			return err;
991 		break;
992 	default:
993 		dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
994 		break;
995 	}
996 
997 	return table + hdr->length;
998 }
999 
1000 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
1001 		struct nfit_mem *nfit_mem)
1002 {
1003 	u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1004 	u16 dcr = nfit_mem->dcr->region_index;
1005 	struct nfit_spa *nfit_spa;
1006 
1007 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1008 		u16 range_index = nfit_spa->spa->range_index;
1009 		int type = nfit_spa_type(nfit_spa->spa);
1010 		struct nfit_memdev *nfit_memdev;
1011 
1012 		if (type != NFIT_SPA_BDW)
1013 			continue;
1014 
1015 		list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1016 			if (nfit_memdev->memdev->range_index != range_index)
1017 				continue;
1018 			if (nfit_memdev->memdev->device_handle != device_handle)
1019 				continue;
1020 			if (nfit_memdev->memdev->region_index != dcr)
1021 				continue;
1022 
1023 			nfit_mem->spa_bdw = nfit_spa->spa;
1024 			return;
1025 		}
1026 	}
1027 
1028 	dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
1029 			nfit_mem->spa_dcr->range_index);
1030 	nfit_mem->bdw = NULL;
1031 }
1032 
1033 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
1034 		struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
1035 {
1036 	u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
1037 	struct nfit_memdev *nfit_memdev;
1038 	struct nfit_bdw *nfit_bdw;
1039 	struct nfit_idt *nfit_idt;
1040 	u16 idt_idx, range_index;
1041 
1042 	list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
1043 		if (nfit_bdw->bdw->region_index != dcr)
1044 			continue;
1045 		nfit_mem->bdw = nfit_bdw->bdw;
1046 		break;
1047 	}
1048 
1049 	if (!nfit_mem->bdw)
1050 		return;
1051 
1052 	nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
1053 
1054 	if (!nfit_mem->spa_bdw)
1055 		return;
1056 
1057 	range_index = nfit_mem->spa_bdw->range_index;
1058 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1059 		if (nfit_memdev->memdev->range_index != range_index ||
1060 				nfit_memdev->memdev->region_index != dcr)
1061 			continue;
1062 		nfit_mem->memdev_bdw = nfit_memdev->memdev;
1063 		idt_idx = nfit_memdev->memdev->interleave_index;
1064 		list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1065 			if (nfit_idt->idt->interleave_index != idt_idx)
1066 				continue;
1067 			nfit_mem->idt_bdw = nfit_idt->idt;
1068 			break;
1069 		}
1070 		break;
1071 	}
1072 }
1073 
1074 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
1075 		struct acpi_nfit_system_address *spa)
1076 {
1077 	struct nfit_mem *nfit_mem, *found;
1078 	struct nfit_memdev *nfit_memdev;
1079 	int type = spa ? nfit_spa_type(spa) : 0;
1080 
1081 	switch (type) {
1082 	case NFIT_SPA_DCR:
1083 	case NFIT_SPA_PM:
1084 		break;
1085 	default:
1086 		if (spa)
1087 			return 0;
1088 	}
1089 
1090 	/*
1091 	 * This loop runs in two modes, when a dimm is mapped the loop
1092 	 * adds memdev associations to an existing dimm, or creates a
1093 	 * dimm. In the unmapped dimm case this loop sweeps for memdev
1094 	 * instances with an invalid / zero range_index and adds those
1095 	 * dimms without spa associations.
1096 	 */
1097 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1098 		struct nfit_flush *nfit_flush;
1099 		struct nfit_dcr *nfit_dcr;
1100 		u32 device_handle;
1101 		u16 dcr;
1102 
1103 		if (spa && nfit_memdev->memdev->range_index != spa->range_index)
1104 			continue;
1105 		if (!spa && nfit_memdev->memdev->range_index)
1106 			continue;
1107 		found = NULL;
1108 		dcr = nfit_memdev->memdev->region_index;
1109 		device_handle = nfit_memdev->memdev->device_handle;
1110 		list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1111 			if (__to_nfit_memdev(nfit_mem)->device_handle
1112 					== device_handle) {
1113 				found = nfit_mem;
1114 				break;
1115 			}
1116 
1117 		if (found)
1118 			nfit_mem = found;
1119 		else {
1120 			nfit_mem = devm_kzalloc(acpi_desc->dev,
1121 					sizeof(*nfit_mem), GFP_KERNEL);
1122 			if (!nfit_mem)
1123 				return -ENOMEM;
1124 			INIT_LIST_HEAD(&nfit_mem->list);
1125 			nfit_mem->acpi_desc = acpi_desc;
1126 			list_add(&nfit_mem->list, &acpi_desc->dimms);
1127 		}
1128 
1129 		list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1130 			if (nfit_dcr->dcr->region_index != dcr)
1131 				continue;
1132 			/*
1133 			 * Record the control region for the dimm.  For
1134 			 * the ACPI 6.1 case, where there are separate
1135 			 * control regions for the pmem vs blk
1136 			 * interfaces, be sure to record the extended
1137 			 * blk details.
1138 			 */
1139 			if (!nfit_mem->dcr)
1140 				nfit_mem->dcr = nfit_dcr->dcr;
1141 			else if (nfit_mem->dcr->windows == 0
1142 					&& nfit_dcr->dcr->windows)
1143 				nfit_mem->dcr = nfit_dcr->dcr;
1144 			break;
1145 		}
1146 
1147 		list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
1148 			struct acpi_nfit_flush_address *flush;
1149 			u16 i;
1150 
1151 			if (nfit_flush->flush->device_handle != device_handle)
1152 				continue;
1153 			nfit_mem->nfit_flush = nfit_flush;
1154 			flush = nfit_flush->flush;
1155 			nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev,
1156 					flush->hint_count,
1157 					sizeof(struct resource),
1158 					GFP_KERNEL);
1159 			if (!nfit_mem->flush_wpq)
1160 				return -ENOMEM;
1161 			for (i = 0; i < flush->hint_count; i++) {
1162 				struct resource *res = &nfit_mem->flush_wpq[i];
1163 
1164 				res->start = flush->hint_address[i];
1165 				res->end = res->start + 8 - 1;
1166 			}
1167 			break;
1168 		}
1169 
1170 		if (dcr && !nfit_mem->dcr) {
1171 			dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
1172 					spa->range_index, dcr);
1173 			return -ENODEV;
1174 		}
1175 
1176 		if (type == NFIT_SPA_DCR) {
1177 			struct nfit_idt *nfit_idt;
1178 			u16 idt_idx;
1179 
1180 			/* multiple dimms may share a SPA when interleaved */
1181 			nfit_mem->spa_dcr = spa;
1182 			nfit_mem->memdev_dcr = nfit_memdev->memdev;
1183 			idt_idx = nfit_memdev->memdev->interleave_index;
1184 			list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1185 				if (nfit_idt->idt->interleave_index != idt_idx)
1186 					continue;
1187 				nfit_mem->idt_dcr = nfit_idt->idt;
1188 				break;
1189 			}
1190 			nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
1191 		} else if (type == NFIT_SPA_PM) {
1192 			/*
1193 			 * A single dimm may belong to multiple SPA-PM
1194 			 * ranges, record at least one in addition to
1195 			 * any SPA-DCR range.
1196 			 */
1197 			nfit_mem->memdev_pmem = nfit_memdev->memdev;
1198 		} else
1199 			nfit_mem->memdev_dcr = nfit_memdev->memdev;
1200 	}
1201 
1202 	return 0;
1203 }
1204 
1205 static int nfit_mem_cmp(void *priv, const struct list_head *_a,
1206 		const struct list_head *_b)
1207 {
1208 	struct nfit_mem *a = container_of(_a, typeof(*a), list);
1209 	struct nfit_mem *b = container_of(_b, typeof(*b), list);
1210 	u32 handleA, handleB;
1211 
1212 	handleA = __to_nfit_memdev(a)->device_handle;
1213 	handleB = __to_nfit_memdev(b)->device_handle;
1214 	if (handleA < handleB)
1215 		return -1;
1216 	else if (handleA > handleB)
1217 		return 1;
1218 	return 0;
1219 }
1220 
1221 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
1222 {
1223 	struct nfit_spa *nfit_spa;
1224 	int rc;
1225 
1226 
1227 	/*
1228 	 * For each SPA-DCR or SPA-PMEM address range find its
1229 	 * corresponding MEMDEV(s).  From each MEMDEV find the
1230 	 * corresponding DCR.  Then, if we're operating on a SPA-DCR,
1231 	 * try to find a SPA-BDW and a corresponding BDW that references
1232 	 * the DCR.  Throw it all into an nfit_mem object.  Note, that
1233 	 * BDWs are optional.
1234 	 */
1235 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1236 		rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
1237 		if (rc)
1238 			return rc;
1239 	}
1240 
1241 	/*
1242 	 * If a DIMM has failed to be mapped into SPA there will be no
1243 	 * SPA entries above. Find and register all the unmapped DIMMs
1244 	 * for reporting and recovery purposes.
1245 	 */
1246 	rc = __nfit_mem_init(acpi_desc, NULL);
1247 	if (rc)
1248 		return rc;
1249 
1250 	list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
1251 
1252 	return 0;
1253 }
1254 
1255 static ssize_t bus_dsm_mask_show(struct device *dev,
1256 		struct device_attribute *attr, char *buf)
1257 {
1258 	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1259 	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1260 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1261 
1262 	return sprintf(buf, "%#lx\n", acpi_desc->bus_dsm_mask);
1263 }
1264 static struct device_attribute dev_attr_bus_dsm_mask =
1265 		__ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL);
1266 
1267 static ssize_t revision_show(struct device *dev,
1268 		struct device_attribute *attr, char *buf)
1269 {
1270 	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1271 	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1272 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1273 
1274 	return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
1275 }
1276 static DEVICE_ATTR_RO(revision);
1277 
1278 static ssize_t hw_error_scrub_show(struct device *dev,
1279 		struct device_attribute *attr, char *buf)
1280 {
1281 	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1282 	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1283 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1284 
1285 	return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
1286 }
1287 
1288 /*
1289  * The 'hw_error_scrub' attribute can have the following values written to it:
1290  * '0': Switch to the default mode where an exception will only insert
1291  *      the address of the memory error into the poison and badblocks lists.
1292  * '1': Enable a full scrub to happen if an exception for a memory error is
1293  *      received.
1294  */
1295 static ssize_t hw_error_scrub_store(struct device *dev,
1296 		struct device_attribute *attr, const char *buf, size_t size)
1297 {
1298 	struct nvdimm_bus_descriptor *nd_desc;
1299 	ssize_t rc;
1300 	long val;
1301 
1302 	rc = kstrtol(buf, 0, &val);
1303 	if (rc)
1304 		return rc;
1305 
1306 	nfit_device_lock(dev);
1307 	nd_desc = dev_get_drvdata(dev);
1308 	if (nd_desc) {
1309 		struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1310 
1311 		switch (val) {
1312 		case HW_ERROR_SCRUB_ON:
1313 			acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
1314 			break;
1315 		case HW_ERROR_SCRUB_OFF:
1316 			acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
1317 			break;
1318 		default:
1319 			rc = -EINVAL;
1320 			break;
1321 		}
1322 	}
1323 	nfit_device_unlock(dev);
1324 	if (rc)
1325 		return rc;
1326 	return size;
1327 }
1328 static DEVICE_ATTR_RW(hw_error_scrub);
1329 
1330 /*
1331  * This shows the number of full Address Range Scrubs that have been
1332  * completed since driver load time. Userspace can wait on this using
1333  * select/poll etc. A '+' at the end indicates an ARS is in progress
1334  */
1335 static ssize_t scrub_show(struct device *dev,
1336 		struct device_attribute *attr, char *buf)
1337 {
1338 	struct nvdimm_bus_descriptor *nd_desc;
1339 	struct acpi_nfit_desc *acpi_desc;
1340 	ssize_t rc = -ENXIO;
1341 	bool busy;
1342 
1343 	nfit_device_lock(dev);
1344 	nd_desc = dev_get_drvdata(dev);
1345 	if (!nd_desc) {
1346 		nfit_device_unlock(dev);
1347 		return rc;
1348 	}
1349 	acpi_desc = to_acpi_desc(nd_desc);
1350 
1351 	mutex_lock(&acpi_desc->init_mutex);
1352 	busy = test_bit(ARS_BUSY, &acpi_desc->scrub_flags)
1353 		&& !test_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
1354 	rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, busy ? "+\n" : "\n");
1355 	/* Allow an admin to poll the busy state at a higher rate */
1356 	if (busy && capable(CAP_SYS_RAWIO) && !test_and_set_bit(ARS_POLL,
1357 				&acpi_desc->scrub_flags)) {
1358 		acpi_desc->scrub_tmo = 1;
1359 		mod_delayed_work(nfit_wq, &acpi_desc->dwork, HZ);
1360 	}
1361 
1362 	mutex_unlock(&acpi_desc->init_mutex);
1363 	nfit_device_unlock(dev);
1364 	return rc;
1365 }
1366 
1367 static ssize_t scrub_store(struct device *dev,
1368 		struct device_attribute *attr, const char *buf, size_t size)
1369 {
1370 	struct nvdimm_bus_descriptor *nd_desc;
1371 	ssize_t rc;
1372 	long val;
1373 
1374 	rc = kstrtol(buf, 0, &val);
1375 	if (rc)
1376 		return rc;
1377 	if (val != 1)
1378 		return -EINVAL;
1379 
1380 	nfit_device_lock(dev);
1381 	nd_desc = dev_get_drvdata(dev);
1382 	if (nd_desc) {
1383 		struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1384 
1385 		rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
1386 	}
1387 	nfit_device_unlock(dev);
1388 	if (rc)
1389 		return rc;
1390 	return size;
1391 }
1392 static DEVICE_ATTR_RW(scrub);
1393 
1394 static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
1395 {
1396 	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1397 	const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
1398 		| 1 << ND_CMD_ARS_STATUS;
1399 
1400 	return (nd_desc->cmd_mask & mask) == mask;
1401 }
1402 
1403 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1404 {
1405 	struct device *dev = kobj_to_dev(kobj);
1406 	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1407 
1408 	if (a == &dev_attr_scrub.attr)
1409 		return ars_supported(nvdimm_bus) ? a->mode : 0;
1410 
1411 	if (a == &dev_attr_firmware_activate_noidle.attr)
1412 		return intel_fwa_supported(nvdimm_bus) ? a->mode : 0;
1413 
1414 	return a->mode;
1415 }
1416 
1417 static struct attribute *acpi_nfit_attributes[] = {
1418 	&dev_attr_revision.attr,
1419 	&dev_attr_scrub.attr,
1420 	&dev_attr_hw_error_scrub.attr,
1421 	&dev_attr_bus_dsm_mask.attr,
1422 	&dev_attr_firmware_activate_noidle.attr,
1423 	NULL,
1424 };
1425 
1426 static const struct attribute_group acpi_nfit_attribute_group = {
1427 	.name = "nfit",
1428 	.attrs = acpi_nfit_attributes,
1429 	.is_visible = nfit_visible,
1430 };
1431 
1432 static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1433 	&acpi_nfit_attribute_group,
1434 	NULL,
1435 };
1436 
1437 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1438 {
1439 	struct nvdimm *nvdimm = to_nvdimm(dev);
1440 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1441 
1442 	return __to_nfit_memdev(nfit_mem);
1443 }
1444 
1445 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1446 {
1447 	struct nvdimm *nvdimm = to_nvdimm(dev);
1448 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1449 
1450 	return nfit_mem->dcr;
1451 }
1452 
1453 static ssize_t handle_show(struct device *dev,
1454 		struct device_attribute *attr, char *buf)
1455 {
1456 	struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1457 
1458 	return sprintf(buf, "%#x\n", memdev->device_handle);
1459 }
1460 static DEVICE_ATTR_RO(handle);
1461 
1462 static ssize_t phys_id_show(struct device *dev,
1463 		struct device_attribute *attr, char *buf)
1464 {
1465 	struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1466 
1467 	return sprintf(buf, "%#x\n", memdev->physical_id);
1468 }
1469 static DEVICE_ATTR_RO(phys_id);
1470 
1471 static ssize_t vendor_show(struct device *dev,
1472 		struct device_attribute *attr, char *buf)
1473 {
1474 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1475 
1476 	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1477 }
1478 static DEVICE_ATTR_RO(vendor);
1479 
1480 static ssize_t rev_id_show(struct device *dev,
1481 		struct device_attribute *attr, char *buf)
1482 {
1483 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1484 
1485 	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1486 }
1487 static DEVICE_ATTR_RO(rev_id);
1488 
1489 static ssize_t device_show(struct device *dev,
1490 		struct device_attribute *attr, char *buf)
1491 {
1492 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1493 
1494 	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1495 }
1496 static DEVICE_ATTR_RO(device);
1497 
1498 static ssize_t subsystem_vendor_show(struct device *dev,
1499 		struct device_attribute *attr, char *buf)
1500 {
1501 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1502 
1503 	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1504 }
1505 static DEVICE_ATTR_RO(subsystem_vendor);
1506 
1507 static ssize_t subsystem_rev_id_show(struct device *dev,
1508 		struct device_attribute *attr, char *buf)
1509 {
1510 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1511 
1512 	return sprintf(buf, "0x%04x\n",
1513 			be16_to_cpu(dcr->subsystem_revision_id));
1514 }
1515 static DEVICE_ATTR_RO(subsystem_rev_id);
1516 
1517 static ssize_t subsystem_device_show(struct device *dev,
1518 		struct device_attribute *attr, char *buf)
1519 {
1520 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1521 
1522 	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1523 }
1524 static DEVICE_ATTR_RO(subsystem_device);
1525 
1526 static int num_nvdimm_formats(struct nvdimm *nvdimm)
1527 {
1528 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1529 	int formats = 0;
1530 
1531 	if (nfit_mem->memdev_pmem)
1532 		formats++;
1533 	if (nfit_mem->memdev_bdw)
1534 		formats++;
1535 	return formats;
1536 }
1537 
1538 static ssize_t format_show(struct device *dev,
1539 		struct device_attribute *attr, char *buf)
1540 {
1541 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1542 
1543 	return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1544 }
1545 static DEVICE_ATTR_RO(format);
1546 
1547 static ssize_t format1_show(struct device *dev,
1548 		struct device_attribute *attr, char *buf)
1549 {
1550 	u32 handle;
1551 	ssize_t rc = -ENXIO;
1552 	struct nfit_mem *nfit_mem;
1553 	struct nfit_memdev *nfit_memdev;
1554 	struct acpi_nfit_desc *acpi_desc;
1555 	struct nvdimm *nvdimm = to_nvdimm(dev);
1556 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1557 
1558 	nfit_mem = nvdimm_provider_data(nvdimm);
1559 	acpi_desc = nfit_mem->acpi_desc;
1560 	handle = to_nfit_memdev(dev)->device_handle;
1561 
1562 	/* assumes DIMMs have at most 2 published interface codes */
1563 	mutex_lock(&acpi_desc->init_mutex);
1564 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1565 		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1566 		struct nfit_dcr *nfit_dcr;
1567 
1568 		if (memdev->device_handle != handle)
1569 			continue;
1570 
1571 		list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1572 			if (nfit_dcr->dcr->region_index != memdev->region_index)
1573 				continue;
1574 			if (nfit_dcr->dcr->code == dcr->code)
1575 				continue;
1576 			rc = sprintf(buf, "0x%04x\n",
1577 					le16_to_cpu(nfit_dcr->dcr->code));
1578 			break;
1579 		}
1580 		if (rc != -ENXIO)
1581 			break;
1582 	}
1583 	mutex_unlock(&acpi_desc->init_mutex);
1584 	return rc;
1585 }
1586 static DEVICE_ATTR_RO(format1);
1587 
1588 static ssize_t formats_show(struct device *dev,
1589 		struct device_attribute *attr, char *buf)
1590 {
1591 	struct nvdimm *nvdimm = to_nvdimm(dev);
1592 
1593 	return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1594 }
1595 static DEVICE_ATTR_RO(formats);
1596 
1597 static ssize_t serial_show(struct device *dev,
1598 		struct device_attribute *attr, char *buf)
1599 {
1600 	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1601 
1602 	return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1603 }
1604 static DEVICE_ATTR_RO(serial);
1605 
1606 static ssize_t family_show(struct device *dev,
1607 		struct device_attribute *attr, char *buf)
1608 {
1609 	struct nvdimm *nvdimm = to_nvdimm(dev);
1610 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1611 
1612 	if (nfit_mem->family < 0)
1613 		return -ENXIO;
1614 	return sprintf(buf, "%d\n", nfit_mem->family);
1615 }
1616 static DEVICE_ATTR_RO(family);
1617 
1618 static ssize_t dsm_mask_show(struct device *dev,
1619 		struct device_attribute *attr, char *buf)
1620 {
1621 	struct nvdimm *nvdimm = to_nvdimm(dev);
1622 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1623 
1624 	if (nfit_mem->family < 0)
1625 		return -ENXIO;
1626 	return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1627 }
1628 static DEVICE_ATTR_RO(dsm_mask);
1629 
1630 static ssize_t flags_show(struct device *dev,
1631 		struct device_attribute *attr, char *buf)
1632 {
1633 	struct nvdimm *nvdimm = to_nvdimm(dev);
1634 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1635 	u16 flags = __to_nfit_memdev(nfit_mem)->flags;
1636 
1637 	if (test_bit(NFIT_MEM_DIRTY, &nfit_mem->flags))
1638 		flags |= ACPI_NFIT_MEM_FLUSH_FAILED;
1639 
1640 	return sprintf(buf, "%s%s%s%s%s%s%s\n",
1641 		flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1642 		flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1643 		flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1644 		flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1645 		flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
1646 		flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
1647 		flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
1648 }
1649 static DEVICE_ATTR_RO(flags);
1650 
1651 static ssize_t id_show(struct device *dev,
1652 		struct device_attribute *attr, char *buf)
1653 {
1654 	struct nvdimm *nvdimm = to_nvdimm(dev);
1655 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1656 
1657 	return sprintf(buf, "%s\n", nfit_mem->id);
1658 }
1659 static DEVICE_ATTR_RO(id);
1660 
1661 static ssize_t dirty_shutdown_show(struct device *dev,
1662 		struct device_attribute *attr, char *buf)
1663 {
1664 	struct nvdimm *nvdimm = to_nvdimm(dev);
1665 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1666 
1667 	return sprintf(buf, "%d\n", nfit_mem->dirty_shutdown);
1668 }
1669 static DEVICE_ATTR_RO(dirty_shutdown);
1670 
1671 static struct attribute *acpi_nfit_dimm_attributes[] = {
1672 	&dev_attr_handle.attr,
1673 	&dev_attr_phys_id.attr,
1674 	&dev_attr_vendor.attr,
1675 	&dev_attr_device.attr,
1676 	&dev_attr_rev_id.attr,
1677 	&dev_attr_subsystem_vendor.attr,
1678 	&dev_attr_subsystem_device.attr,
1679 	&dev_attr_subsystem_rev_id.attr,
1680 	&dev_attr_format.attr,
1681 	&dev_attr_formats.attr,
1682 	&dev_attr_format1.attr,
1683 	&dev_attr_serial.attr,
1684 	&dev_attr_flags.attr,
1685 	&dev_attr_id.attr,
1686 	&dev_attr_family.attr,
1687 	&dev_attr_dsm_mask.attr,
1688 	&dev_attr_dirty_shutdown.attr,
1689 	NULL,
1690 };
1691 
1692 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1693 		struct attribute *a, int n)
1694 {
1695 	struct device *dev = kobj_to_dev(kobj);
1696 	struct nvdimm *nvdimm = to_nvdimm(dev);
1697 	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1698 
1699 	if (!to_nfit_dcr(dev)) {
1700 		/* Without a dcr only the memdev attributes can be surfaced */
1701 		if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
1702 				|| a == &dev_attr_flags.attr
1703 				|| a == &dev_attr_family.attr
1704 				|| a == &dev_attr_dsm_mask.attr)
1705 			return a->mode;
1706 		return 0;
1707 	}
1708 
1709 	if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1710 		return 0;
1711 
1712 	if (!test_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags)
1713 			&& a == &dev_attr_dirty_shutdown.attr)
1714 		return 0;
1715 
1716 	return a->mode;
1717 }
1718 
1719 static const struct attribute_group acpi_nfit_dimm_attribute_group = {
1720 	.name = "nfit",
1721 	.attrs = acpi_nfit_dimm_attributes,
1722 	.is_visible = acpi_nfit_dimm_attr_visible,
1723 };
1724 
1725 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1726 	&acpi_nfit_dimm_attribute_group,
1727 	NULL,
1728 };
1729 
1730 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1731 		u32 device_handle)
1732 {
1733 	struct nfit_mem *nfit_mem;
1734 
1735 	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1736 		if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1737 			return nfit_mem->nvdimm;
1738 
1739 	return NULL;
1740 }
1741 
1742 void __acpi_nvdimm_notify(struct device *dev, u32 event)
1743 {
1744 	struct nfit_mem *nfit_mem;
1745 	struct acpi_nfit_desc *acpi_desc;
1746 
1747 	dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev),
1748 			event);
1749 
1750 	if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1751 		dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1752 				event);
1753 		return;
1754 	}
1755 
1756 	acpi_desc = dev_get_drvdata(dev->parent);
1757 	if (!acpi_desc)
1758 		return;
1759 
1760 	/*
1761 	 * If we successfully retrieved acpi_desc, then we know nfit_mem data
1762 	 * is still valid.
1763 	 */
1764 	nfit_mem = dev_get_drvdata(dev);
1765 	if (nfit_mem && nfit_mem->flags_attr)
1766 		sysfs_notify_dirent(nfit_mem->flags_attr);
1767 }
1768 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1769 
1770 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1771 {
1772 	struct acpi_device *adev = data;
1773 	struct device *dev = &adev->dev;
1774 
1775 	nfit_device_lock(dev->parent);
1776 	__acpi_nvdimm_notify(dev, event);
1777 	nfit_device_unlock(dev->parent);
1778 }
1779 
1780 static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
1781 {
1782 	acpi_handle handle;
1783 	acpi_status status;
1784 
1785 	status = acpi_get_handle(adev->handle, method, &handle);
1786 
1787 	if (ACPI_SUCCESS(status))
1788 		return true;
1789 	return false;
1790 }
1791 
1792 __weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem)
1793 {
1794 	struct device *dev = &nfit_mem->adev->dev;
1795 	struct nd_intel_smart smart = { 0 };
1796 	union acpi_object in_buf = {
1797 		.buffer.type = ACPI_TYPE_BUFFER,
1798 		.buffer.length = 0,
1799 	};
1800 	union acpi_object in_obj = {
1801 		.package.type = ACPI_TYPE_PACKAGE,
1802 		.package.count = 1,
1803 		.package.elements = &in_buf,
1804 	};
1805 	const u8 func = ND_INTEL_SMART;
1806 	const guid_t *guid = to_nfit_uuid(nfit_mem->family);
1807 	u8 revid = nfit_dsm_revid(nfit_mem->family, func);
1808 	struct acpi_device *adev = nfit_mem->adev;
1809 	acpi_handle handle = adev->handle;
1810 	union acpi_object *out_obj;
1811 
1812 	if ((nfit_mem->dsm_mask & (1 << func)) == 0)
1813 		return;
1814 
1815 	out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
1816 	if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER
1817 			|| out_obj->buffer.length < sizeof(smart)) {
1818 		dev_dbg(dev->parent, "%s: failed to retrieve initial health\n",
1819 				dev_name(dev));
1820 		ACPI_FREE(out_obj);
1821 		return;
1822 	}
1823 	memcpy(&smart, out_obj->buffer.pointer, sizeof(smart));
1824 	ACPI_FREE(out_obj);
1825 
1826 	if (smart.flags & ND_INTEL_SMART_SHUTDOWN_VALID) {
1827 		if (smart.shutdown_state)
1828 			set_bit(NFIT_MEM_DIRTY, &nfit_mem->flags);
1829 	}
1830 
1831 	if (smart.flags & ND_INTEL_SMART_SHUTDOWN_COUNT_VALID) {
1832 		set_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags);
1833 		nfit_mem->dirty_shutdown = smart.shutdown_count;
1834 	}
1835 }
1836 
1837 static void populate_shutdown_status(struct nfit_mem *nfit_mem)
1838 {
1839 	/*
1840 	 * For DIMMs that provide a dynamic facility to retrieve a
1841 	 * dirty-shutdown status and/or a dirty-shutdown count, cache
1842 	 * these values in nfit_mem.
1843 	 */
1844 	if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1845 		nfit_intel_shutdown_status(nfit_mem);
1846 }
1847 
1848 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1849 		struct nfit_mem *nfit_mem, u32 device_handle)
1850 {
1851 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1852 	struct acpi_device *adev, *adev_dimm;
1853 	struct device *dev = acpi_desc->dev;
1854 	unsigned long dsm_mask, label_mask;
1855 	const guid_t *guid;
1856 	int i;
1857 	int family = -1;
1858 	struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
1859 
1860 	/* nfit test assumes 1:1 relationship between commands and dsms */
1861 	nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1862 	nfit_mem->family = NVDIMM_FAMILY_INTEL;
1863 	set_bit(NVDIMM_FAMILY_INTEL, &nd_desc->dimm_family_mask);
1864 
1865 	if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1866 		sprintf(nfit_mem->id, "%04x-%02x-%04x-%08x",
1867 				be16_to_cpu(dcr->vendor_id),
1868 				dcr->manufacturing_location,
1869 				be16_to_cpu(dcr->manufacturing_date),
1870 				be32_to_cpu(dcr->serial_number));
1871 	else
1872 		sprintf(nfit_mem->id, "%04x-%08x",
1873 				be16_to_cpu(dcr->vendor_id),
1874 				be32_to_cpu(dcr->serial_number));
1875 
1876 	adev = to_acpi_dev(acpi_desc);
1877 	if (!adev) {
1878 		/* unit test case */
1879 		populate_shutdown_status(nfit_mem);
1880 		return 0;
1881 	}
1882 
1883 	adev_dimm = acpi_find_child_device(adev, device_handle, false);
1884 	nfit_mem->adev = adev_dimm;
1885 	if (!adev_dimm) {
1886 		dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1887 				device_handle);
1888 		return force_enable_dimms ? 0 : -ENODEV;
1889 	}
1890 
1891 	if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1892 		ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1893 		dev_err(dev, "%s: notification registration failed\n",
1894 				dev_name(&adev_dimm->dev));
1895 		return -ENXIO;
1896 	}
1897 	/*
1898 	 * Record nfit_mem for the notification path to track back to
1899 	 * the nfit sysfs attributes for this dimm device object.
1900 	 */
1901 	dev_set_drvdata(&adev_dimm->dev, nfit_mem);
1902 
1903 	/*
1904 	 * There are 4 "legacy" NVDIMM command sets
1905 	 * (NVDIMM_FAMILY_{INTEL,MSFT,HPE1,HPE2}) that were created before
1906 	 * an EFI working group was established to constrain this
1907 	 * proliferation. The nfit driver probes for the supported command
1908 	 * set by GUID. Note, if you're a platform developer looking to add
1909 	 * a new command set to this probe, consider using an existing set,
1910 	 * or otherwise seek approval to publish the command set at
1911 	 * http://www.uefi.org/RFIC_LIST.
1912 	 *
1913 	 * Note, that checking for function0 (bit0) tells us if any commands
1914 	 * are reachable through this GUID.
1915 	 */
1916 	clear_bit(NVDIMM_FAMILY_INTEL, &nd_desc->dimm_family_mask);
1917 	for (i = 0; i <= NVDIMM_FAMILY_MAX; i++)
1918 		if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) {
1919 			set_bit(i, &nd_desc->dimm_family_mask);
1920 			if (family < 0 || i == default_dsm_family)
1921 				family = i;
1922 		}
1923 
1924 	/* limit the supported commands to those that are publicly documented */
1925 	nfit_mem->family = family;
1926 	if (override_dsm_mask && !disable_vendor_specific)
1927 		dsm_mask = override_dsm_mask;
1928 	else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1929 		dsm_mask = NVDIMM_INTEL_CMDMASK;
1930 		if (disable_vendor_specific)
1931 			dsm_mask &= ~(1 << ND_CMD_VENDOR);
1932 	} else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1933 		dsm_mask = 0x1c3c76;
1934 	} else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1935 		dsm_mask = 0x1fe;
1936 		if (disable_vendor_specific)
1937 			dsm_mask &= ~(1 << 8);
1938 	} else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1939 		dsm_mask = 0xffffffff;
1940 	} else if (nfit_mem->family == NVDIMM_FAMILY_HYPERV) {
1941 		dsm_mask = 0x1f;
1942 	} else {
1943 		dev_dbg(dev, "unknown dimm command family\n");
1944 		nfit_mem->family = -1;
1945 		/* DSMs are optional, continue loading the driver... */
1946 		return 0;
1947 	}
1948 
1949 	/*
1950 	 * Function 0 is the command interrogation function, don't
1951 	 * export it to potential userspace use, and enable it to be
1952 	 * used as an error value in acpi_nfit_ctl().
1953 	 */
1954 	dsm_mask &= ~1UL;
1955 
1956 	guid = to_nfit_uuid(nfit_mem->family);
1957 	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1958 		if (acpi_check_dsm(adev_dimm->handle, guid,
1959 					nfit_dsm_revid(nfit_mem->family, i),
1960 					1ULL << i))
1961 			set_bit(i, &nfit_mem->dsm_mask);
1962 
1963 	/*
1964 	 * Prefer the NVDIMM_FAMILY_INTEL label read commands if present
1965 	 * due to their better semantics handling locked capacity.
1966 	 */
1967 	label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA
1968 		| 1 << ND_CMD_SET_CONFIG_DATA;
1969 	if (family == NVDIMM_FAMILY_INTEL
1970 			&& (dsm_mask & label_mask) == label_mask)
1971 		/* skip _LS{I,R,W} enabling */;
1972 	else {
1973 		if (acpi_nvdimm_has_method(adev_dimm, "_LSI")
1974 				&& acpi_nvdimm_has_method(adev_dimm, "_LSR")) {
1975 			dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev));
1976 			set_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1977 		}
1978 
1979 		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
1980 				&& acpi_nvdimm_has_method(adev_dimm, "_LSW")) {
1981 			dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev));
1982 			set_bit(NFIT_MEM_LSW, &nfit_mem->flags);
1983 		}
1984 
1985 		/*
1986 		 * Quirk read-only label configurations to preserve
1987 		 * access to label-less namespaces by default.
1988 		 */
1989 		if (!test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
1990 				&& !force_labels) {
1991 			dev_dbg(dev, "%s: No _LSW, disable labels\n",
1992 					dev_name(&adev_dimm->dev));
1993 			clear_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1994 		} else
1995 			dev_dbg(dev, "%s: Force enable labels\n",
1996 					dev_name(&adev_dimm->dev));
1997 	}
1998 
1999 	populate_shutdown_status(nfit_mem);
2000 
2001 	return 0;
2002 }
2003 
2004 static void shutdown_dimm_notify(void *data)
2005 {
2006 	struct acpi_nfit_desc *acpi_desc = data;
2007 	struct nfit_mem *nfit_mem;
2008 
2009 	mutex_lock(&acpi_desc->init_mutex);
2010 	/*
2011 	 * Clear out the nfit_mem->flags_attr and shut down dimm event
2012 	 * notifications.
2013 	 */
2014 	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2015 		struct acpi_device *adev_dimm = nfit_mem->adev;
2016 
2017 		if (nfit_mem->flags_attr) {
2018 			sysfs_put(nfit_mem->flags_attr);
2019 			nfit_mem->flags_attr = NULL;
2020 		}
2021 		if (adev_dimm) {
2022 			acpi_remove_notify_handler(adev_dimm->handle,
2023 					ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
2024 			dev_set_drvdata(&adev_dimm->dev, NULL);
2025 		}
2026 	}
2027 	mutex_unlock(&acpi_desc->init_mutex);
2028 }
2029 
2030 static const struct nvdimm_security_ops *acpi_nfit_get_security_ops(int family)
2031 {
2032 	switch (family) {
2033 	case NVDIMM_FAMILY_INTEL:
2034 		return intel_security_ops;
2035 	default:
2036 		return NULL;
2037 	}
2038 }
2039 
2040 static const struct nvdimm_fw_ops *acpi_nfit_get_fw_ops(
2041 		struct nfit_mem *nfit_mem)
2042 {
2043 	unsigned long mask;
2044 	struct acpi_nfit_desc *acpi_desc = nfit_mem->acpi_desc;
2045 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2046 
2047 	if (!nd_desc->fw_ops)
2048 		return NULL;
2049 
2050 	if (nfit_mem->family != NVDIMM_FAMILY_INTEL)
2051 		return NULL;
2052 
2053 	mask = nfit_mem->dsm_mask & NVDIMM_INTEL_FW_ACTIVATE_CMDMASK;
2054 	if (mask != NVDIMM_INTEL_FW_ACTIVATE_CMDMASK)
2055 		return NULL;
2056 
2057 	return intel_fw_ops;
2058 }
2059 
2060 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
2061 {
2062 	struct nfit_mem *nfit_mem;
2063 	int dimm_count = 0, rc;
2064 	struct nvdimm *nvdimm;
2065 
2066 	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2067 		struct acpi_nfit_flush_address *flush;
2068 		unsigned long flags = 0, cmd_mask;
2069 		struct nfit_memdev *nfit_memdev;
2070 		u32 device_handle;
2071 		u16 mem_flags;
2072 
2073 		device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
2074 		nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
2075 		if (nvdimm) {
2076 			dimm_count++;
2077 			continue;
2078 		}
2079 
2080 		if (nfit_mem->bdw && nfit_mem->memdev_pmem) {
2081 			set_bit(NDD_ALIASING, &flags);
2082 			set_bit(NDD_LABELING, &flags);
2083 		}
2084 
2085 		/* collate flags across all memdevs for this dimm */
2086 		list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2087 			struct acpi_nfit_memory_map *dimm_memdev;
2088 
2089 			dimm_memdev = __to_nfit_memdev(nfit_mem);
2090 			if (dimm_memdev->device_handle
2091 					!= nfit_memdev->memdev->device_handle)
2092 				continue;
2093 			dimm_memdev->flags |= nfit_memdev->memdev->flags;
2094 		}
2095 
2096 		mem_flags = __to_nfit_memdev(nfit_mem)->flags;
2097 		if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
2098 			set_bit(NDD_UNARMED, &flags);
2099 
2100 		rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
2101 		if (rc)
2102 			continue;
2103 
2104 		/*
2105 		 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
2106 		 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
2107 		 * userspace interface.
2108 		 */
2109 		cmd_mask = 1UL << ND_CMD_CALL;
2110 		if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
2111 			/*
2112 			 * These commands have a 1:1 correspondence
2113 			 * between DSM payload and libnvdimm ioctl
2114 			 * payload format.
2115 			 */
2116 			cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK;
2117 		}
2118 
2119 		/* Quirk to ignore LOCAL for labels on HYPERV DIMMs */
2120 		if (nfit_mem->family == NVDIMM_FAMILY_HYPERV)
2121 			set_bit(NDD_NOBLK, &flags);
2122 
2123 		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
2124 			set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
2125 			set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
2126 		}
2127 		if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags))
2128 			set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
2129 
2130 		flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
2131 			: NULL;
2132 		nvdimm = __nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
2133 				acpi_nfit_dimm_attribute_groups,
2134 				flags, cmd_mask, flush ? flush->hint_count : 0,
2135 				nfit_mem->flush_wpq, &nfit_mem->id[0],
2136 				acpi_nfit_get_security_ops(nfit_mem->family),
2137 				acpi_nfit_get_fw_ops(nfit_mem));
2138 		if (!nvdimm)
2139 			return -ENOMEM;
2140 
2141 		nfit_mem->nvdimm = nvdimm;
2142 		dimm_count++;
2143 
2144 		if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
2145 			continue;
2146 
2147 		dev_err(acpi_desc->dev, "Error found in NVDIMM %s flags:%s%s%s%s%s\n",
2148 				nvdimm_name(nvdimm),
2149 		  mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
2150 		  mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
2151 		  mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
2152 		  mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
2153 		  mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
2154 
2155 	}
2156 
2157 	rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
2158 	if (rc)
2159 		return rc;
2160 
2161 	/*
2162 	 * Now that dimms are successfully registered, and async registration
2163 	 * is flushed, attempt to enable event notification.
2164 	 */
2165 	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2166 		struct kernfs_node *nfit_kernfs;
2167 
2168 		nvdimm = nfit_mem->nvdimm;
2169 		if (!nvdimm)
2170 			continue;
2171 
2172 		nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
2173 		if (nfit_kernfs)
2174 			nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
2175 					"flags");
2176 		sysfs_put(nfit_kernfs);
2177 		if (!nfit_mem->flags_attr)
2178 			dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
2179 					nvdimm_name(nvdimm));
2180 	}
2181 
2182 	return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
2183 			acpi_desc);
2184 }
2185 
2186 /*
2187  * These constants are private because there are no kernel consumers of
2188  * these commands.
2189  */
2190 enum nfit_aux_cmds {
2191 	NFIT_CMD_TRANSLATE_SPA = 5,
2192 	NFIT_CMD_ARS_INJECT_SET = 7,
2193 	NFIT_CMD_ARS_INJECT_CLEAR = 8,
2194 	NFIT_CMD_ARS_INJECT_GET = 9,
2195 };
2196 
2197 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
2198 {
2199 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2200 	const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS);
2201 	unsigned long dsm_mask, *mask;
2202 	struct acpi_device *adev;
2203 	int i;
2204 
2205 	set_bit(ND_CMD_CALL, &nd_desc->cmd_mask);
2206 	set_bit(NVDIMM_BUS_FAMILY_NFIT, &nd_desc->bus_family_mask);
2207 
2208 	/* enable nfit_test to inject bus command emulation */
2209 	if (acpi_desc->bus_cmd_force_en) {
2210 		nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
2211 		mask = &nd_desc->bus_family_mask;
2212 		if (acpi_desc->family_dsm_mask[NVDIMM_BUS_FAMILY_INTEL]) {
2213 			set_bit(NVDIMM_BUS_FAMILY_INTEL, mask);
2214 			nd_desc->fw_ops = intel_bus_fw_ops;
2215 		}
2216 	}
2217 
2218 	adev = to_acpi_dev(acpi_desc);
2219 	if (!adev)
2220 		return;
2221 
2222 	for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
2223 		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2224 			set_bit(i, &nd_desc->cmd_mask);
2225 
2226 	dsm_mask =
2227 		(1 << ND_CMD_ARS_CAP) |
2228 		(1 << ND_CMD_ARS_START) |
2229 		(1 << ND_CMD_ARS_STATUS) |
2230 		(1 << ND_CMD_CLEAR_ERROR) |
2231 		(1 << NFIT_CMD_TRANSLATE_SPA) |
2232 		(1 << NFIT_CMD_ARS_INJECT_SET) |
2233 		(1 << NFIT_CMD_ARS_INJECT_CLEAR) |
2234 		(1 << NFIT_CMD_ARS_INJECT_GET);
2235 	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2236 		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2237 			set_bit(i, &acpi_desc->bus_dsm_mask);
2238 
2239 	/* Enumerate allowed NVDIMM_BUS_FAMILY_INTEL commands */
2240 	dsm_mask = NVDIMM_BUS_INTEL_FW_ACTIVATE_CMDMASK;
2241 	guid = to_nfit_bus_uuid(NVDIMM_BUS_FAMILY_INTEL);
2242 	mask = &acpi_desc->family_dsm_mask[NVDIMM_BUS_FAMILY_INTEL];
2243 	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2244 		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2245 			set_bit(i, mask);
2246 
2247 	if (*mask == dsm_mask) {
2248 		set_bit(NVDIMM_BUS_FAMILY_INTEL, &nd_desc->bus_family_mask);
2249 		nd_desc->fw_ops = intel_bus_fw_ops;
2250 	}
2251 }
2252 
2253 static ssize_t range_index_show(struct device *dev,
2254 		struct device_attribute *attr, char *buf)
2255 {
2256 	struct nd_region *nd_region = to_nd_region(dev);
2257 	struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
2258 
2259 	return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
2260 }
2261 static DEVICE_ATTR_RO(range_index);
2262 
2263 static struct attribute *acpi_nfit_region_attributes[] = {
2264 	&dev_attr_range_index.attr,
2265 	NULL,
2266 };
2267 
2268 static const struct attribute_group acpi_nfit_region_attribute_group = {
2269 	.name = "nfit",
2270 	.attrs = acpi_nfit_region_attributes,
2271 };
2272 
2273 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
2274 	&acpi_nfit_region_attribute_group,
2275 	NULL,
2276 };
2277 
2278 /* enough info to uniquely specify an interleave set */
2279 struct nfit_set_info {
2280 	u64 region_offset;
2281 	u32 serial_number;
2282 	u32 pad;
2283 };
2284 
2285 struct nfit_set_info2 {
2286 	u64 region_offset;
2287 	u32 serial_number;
2288 	u16 vendor_id;
2289 	u16 manufacturing_date;
2290 	u8 manufacturing_location;
2291 	u8 reserved[31];
2292 };
2293 
2294 static int cmp_map_compat(const void *m0, const void *m1)
2295 {
2296 	const struct nfit_set_info *map0 = m0;
2297 	const struct nfit_set_info *map1 = m1;
2298 
2299 	return memcmp(&map0->region_offset, &map1->region_offset,
2300 			sizeof(u64));
2301 }
2302 
2303 static int cmp_map(const void *m0, const void *m1)
2304 {
2305 	const struct nfit_set_info *map0 = m0;
2306 	const struct nfit_set_info *map1 = m1;
2307 
2308 	if (map0->region_offset < map1->region_offset)
2309 		return -1;
2310 	else if (map0->region_offset > map1->region_offset)
2311 		return 1;
2312 	return 0;
2313 }
2314 
2315 static int cmp_map2(const void *m0, const void *m1)
2316 {
2317 	const struct nfit_set_info2 *map0 = m0;
2318 	const struct nfit_set_info2 *map1 = m1;
2319 
2320 	if (map0->region_offset < map1->region_offset)
2321 		return -1;
2322 	else if (map0->region_offset > map1->region_offset)
2323 		return 1;
2324 	return 0;
2325 }
2326 
2327 /* Retrieve the nth entry referencing this spa */
2328 static struct acpi_nfit_memory_map *memdev_from_spa(
2329 		struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
2330 {
2331 	struct nfit_memdev *nfit_memdev;
2332 
2333 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
2334 		if (nfit_memdev->memdev->range_index == range_index)
2335 			if (n-- == 0)
2336 				return nfit_memdev->memdev;
2337 	return NULL;
2338 }
2339 
2340 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
2341 		struct nd_region_desc *ndr_desc,
2342 		struct acpi_nfit_system_address *spa)
2343 {
2344 	struct device *dev = acpi_desc->dev;
2345 	struct nd_interleave_set *nd_set;
2346 	u16 nr = ndr_desc->num_mappings;
2347 	struct nfit_set_info2 *info2;
2348 	struct nfit_set_info *info;
2349 	int i;
2350 
2351 	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
2352 	if (!nd_set)
2353 		return -ENOMEM;
2354 	import_guid(&nd_set->type_guid, spa->range_guid);
2355 
2356 	info = devm_kcalloc(dev, nr, sizeof(*info), GFP_KERNEL);
2357 	if (!info)
2358 		return -ENOMEM;
2359 
2360 	info2 = devm_kcalloc(dev, nr, sizeof(*info2), GFP_KERNEL);
2361 	if (!info2)
2362 		return -ENOMEM;
2363 
2364 	for (i = 0; i < nr; i++) {
2365 		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
2366 		struct nvdimm *nvdimm = mapping->nvdimm;
2367 		struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2368 		struct nfit_set_info *map = &info[i];
2369 		struct nfit_set_info2 *map2 = &info2[i];
2370 		struct acpi_nfit_memory_map *memdev =
2371 			memdev_from_spa(acpi_desc, spa->range_index, i);
2372 		struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2373 
2374 		if (!memdev || !nfit_mem->dcr) {
2375 			dev_err(dev, "%s: failed to find DCR\n", __func__);
2376 			return -ENODEV;
2377 		}
2378 
2379 		map->region_offset = memdev->region_offset;
2380 		map->serial_number = dcr->serial_number;
2381 
2382 		map2->region_offset = memdev->region_offset;
2383 		map2->serial_number = dcr->serial_number;
2384 		map2->vendor_id = dcr->vendor_id;
2385 		map2->manufacturing_date = dcr->manufacturing_date;
2386 		map2->manufacturing_location = dcr->manufacturing_location;
2387 	}
2388 
2389 	/* v1.1 namespaces */
2390 	sort(info, nr, sizeof(*info), cmp_map, NULL);
2391 	nd_set->cookie1 = nd_fletcher64(info, sizeof(*info) * nr, 0);
2392 
2393 	/* v1.2 namespaces */
2394 	sort(info2, nr, sizeof(*info2), cmp_map2, NULL);
2395 	nd_set->cookie2 = nd_fletcher64(info2, sizeof(*info2) * nr, 0);
2396 
2397 	/* support v1.1 namespaces created with the wrong sort order */
2398 	sort(info, nr, sizeof(*info), cmp_map_compat, NULL);
2399 	nd_set->altcookie = nd_fletcher64(info, sizeof(*info) * nr, 0);
2400 
2401 	/* record the result of the sort for the mapping position */
2402 	for (i = 0; i < nr; i++) {
2403 		struct nfit_set_info2 *map2 = &info2[i];
2404 		int j;
2405 
2406 		for (j = 0; j < nr; j++) {
2407 			struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
2408 			struct nvdimm *nvdimm = mapping->nvdimm;
2409 			struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2410 			struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2411 
2412 			if (map2->serial_number == dcr->serial_number &&
2413 			    map2->vendor_id == dcr->vendor_id &&
2414 			    map2->manufacturing_date == dcr->manufacturing_date &&
2415 			    map2->manufacturing_location
2416 				    == dcr->manufacturing_location) {
2417 				mapping->position = i;
2418 				break;
2419 			}
2420 		}
2421 	}
2422 
2423 	ndr_desc->nd_set = nd_set;
2424 	devm_kfree(dev, info);
2425 	devm_kfree(dev, info2);
2426 
2427 	return 0;
2428 }
2429 
2430 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
2431 {
2432 	struct acpi_nfit_interleave *idt = mmio->idt;
2433 	u32 sub_line_offset, line_index, line_offset;
2434 	u64 line_no, table_skip_count, table_offset;
2435 
2436 	line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
2437 	table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
2438 	line_offset = idt->line_offset[line_index]
2439 		* mmio->line_size;
2440 	table_offset = table_skip_count * mmio->table_size;
2441 
2442 	return mmio->base_offset + line_offset + table_offset + sub_line_offset;
2443 }
2444 
2445 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
2446 {
2447 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2448 	u64 offset = nfit_blk->stat_offset + mmio->size * bw;
2449 	const u32 STATUS_MASK = 0x80000037;
2450 
2451 	if (mmio->num_lines)
2452 		offset = to_interleave_offset(offset, mmio);
2453 
2454 	return readl(mmio->addr.base + offset) & STATUS_MASK;
2455 }
2456 
2457 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
2458 		resource_size_t dpa, unsigned int len, unsigned int write)
2459 {
2460 	u64 cmd, offset;
2461 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2462 
2463 	enum {
2464 		BCW_OFFSET_MASK = (1ULL << 48)-1,
2465 		BCW_LEN_SHIFT = 48,
2466 		BCW_LEN_MASK = (1ULL << 8) - 1,
2467 		BCW_CMD_SHIFT = 56,
2468 	};
2469 
2470 	cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
2471 	len = len >> L1_CACHE_SHIFT;
2472 	cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
2473 	cmd |= ((u64) write) << BCW_CMD_SHIFT;
2474 
2475 	offset = nfit_blk->cmd_offset + mmio->size * bw;
2476 	if (mmio->num_lines)
2477 		offset = to_interleave_offset(offset, mmio);
2478 
2479 	writeq(cmd, mmio->addr.base + offset);
2480 	nvdimm_flush(nfit_blk->nd_region, NULL);
2481 
2482 	if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
2483 		readq(mmio->addr.base + offset);
2484 }
2485 
2486 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
2487 		resource_size_t dpa, void *iobuf, size_t len, int rw,
2488 		unsigned int lane)
2489 {
2490 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2491 	unsigned int copied = 0;
2492 	u64 base_offset;
2493 	int rc;
2494 
2495 	base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
2496 		+ lane * mmio->size;
2497 	write_blk_ctl(nfit_blk, lane, dpa, len, rw);
2498 	while (len) {
2499 		unsigned int c;
2500 		u64 offset;
2501 
2502 		if (mmio->num_lines) {
2503 			u32 line_offset;
2504 
2505 			offset = to_interleave_offset(base_offset + copied,
2506 					mmio);
2507 			div_u64_rem(offset, mmio->line_size, &line_offset);
2508 			c = min_t(size_t, len, mmio->line_size - line_offset);
2509 		} else {
2510 			offset = base_offset + nfit_blk->bdw_offset;
2511 			c = len;
2512 		}
2513 
2514 		if (rw)
2515 			memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
2516 		else {
2517 			if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
2518 				arch_invalidate_pmem((void __force *)
2519 					mmio->addr.aperture + offset, c);
2520 
2521 			memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
2522 		}
2523 
2524 		copied += c;
2525 		len -= c;
2526 	}
2527 
2528 	if (rw)
2529 		nvdimm_flush(nfit_blk->nd_region, NULL);
2530 
2531 	rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
2532 	return rc;
2533 }
2534 
2535 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
2536 		resource_size_t dpa, void *iobuf, u64 len, int rw)
2537 {
2538 	struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
2539 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2540 	struct nd_region *nd_region = nfit_blk->nd_region;
2541 	unsigned int lane, copied = 0;
2542 	int rc = 0;
2543 
2544 	lane = nd_region_acquire_lane(nd_region);
2545 	while (len) {
2546 		u64 c = min(len, mmio->size);
2547 
2548 		rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
2549 				iobuf + copied, c, rw, lane);
2550 		if (rc)
2551 			break;
2552 
2553 		copied += c;
2554 		len -= c;
2555 	}
2556 	nd_region_release_lane(nd_region, lane);
2557 
2558 	return rc;
2559 }
2560 
2561 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
2562 		struct acpi_nfit_interleave *idt, u16 interleave_ways)
2563 {
2564 	if (idt) {
2565 		mmio->num_lines = idt->line_count;
2566 		mmio->line_size = idt->line_size;
2567 		if (interleave_ways == 0)
2568 			return -ENXIO;
2569 		mmio->table_size = mmio->num_lines * interleave_ways
2570 			* mmio->line_size;
2571 	}
2572 
2573 	return 0;
2574 }
2575 
2576 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
2577 		struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
2578 {
2579 	struct nd_cmd_dimm_flags flags;
2580 	int rc;
2581 
2582 	memset(&flags, 0, sizeof(flags));
2583 	rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
2584 			sizeof(flags), NULL);
2585 
2586 	if (rc >= 0 && flags.status == 0)
2587 		nfit_blk->dimm_flags = flags.flags;
2588 	else if (rc == -ENOTTY) {
2589 		/* fall back to a conservative default */
2590 		nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
2591 		rc = 0;
2592 	} else
2593 		rc = -ENXIO;
2594 
2595 	return rc;
2596 }
2597 
2598 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
2599 		struct device *dev)
2600 {
2601 	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
2602 	struct nd_blk_region *ndbr = to_nd_blk_region(dev);
2603 	struct nfit_blk_mmio *mmio;
2604 	struct nfit_blk *nfit_blk;
2605 	struct nfit_mem *nfit_mem;
2606 	struct nvdimm *nvdimm;
2607 	int rc;
2608 
2609 	nvdimm = nd_blk_region_to_dimm(ndbr);
2610 	nfit_mem = nvdimm_provider_data(nvdimm);
2611 	if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
2612 		dev_dbg(dev, "missing%s%s%s\n",
2613 				nfit_mem ? "" : " nfit_mem",
2614 				(nfit_mem && nfit_mem->dcr) ? "" : " dcr",
2615 				(nfit_mem && nfit_mem->bdw) ? "" : " bdw");
2616 		return -ENXIO;
2617 	}
2618 
2619 	nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
2620 	if (!nfit_blk)
2621 		return -ENOMEM;
2622 	nd_blk_region_set_provider_data(ndbr, nfit_blk);
2623 	nfit_blk->nd_region = to_nd_region(dev);
2624 
2625 	/* map block aperture memory */
2626 	nfit_blk->bdw_offset = nfit_mem->bdw->offset;
2627 	mmio = &nfit_blk->mmio[BDW];
2628 	mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
2629 			nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr));
2630 	if (!mmio->addr.base) {
2631 		dev_dbg(dev, "%s failed to map bdw\n",
2632 				nvdimm_name(nvdimm));
2633 		return -ENOMEM;
2634 	}
2635 	mmio->size = nfit_mem->bdw->size;
2636 	mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
2637 	mmio->idt = nfit_mem->idt_bdw;
2638 	mmio->spa = nfit_mem->spa_bdw;
2639 	rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
2640 			nfit_mem->memdev_bdw->interleave_ways);
2641 	if (rc) {
2642 		dev_dbg(dev, "%s failed to init bdw interleave\n",
2643 				nvdimm_name(nvdimm));
2644 		return rc;
2645 	}
2646 
2647 	/* map block control memory */
2648 	nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
2649 	nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
2650 	mmio = &nfit_blk->mmio[DCR];
2651 	mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
2652 			nfit_mem->spa_dcr->length);
2653 	if (!mmio->addr.base) {
2654 		dev_dbg(dev, "%s failed to map dcr\n",
2655 				nvdimm_name(nvdimm));
2656 		return -ENOMEM;
2657 	}
2658 	mmio->size = nfit_mem->dcr->window_size;
2659 	mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
2660 	mmio->idt = nfit_mem->idt_dcr;
2661 	mmio->spa = nfit_mem->spa_dcr;
2662 	rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
2663 			nfit_mem->memdev_dcr->interleave_ways);
2664 	if (rc) {
2665 		dev_dbg(dev, "%s failed to init dcr interleave\n",
2666 				nvdimm_name(nvdimm));
2667 		return rc;
2668 	}
2669 
2670 	rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
2671 	if (rc < 0) {
2672 		dev_dbg(dev, "%s failed get DIMM flags\n",
2673 				nvdimm_name(nvdimm));
2674 		return rc;
2675 	}
2676 
2677 	if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
2678 		dev_warn(dev, "unable to guarantee persistence of writes\n");
2679 
2680 	if (mmio->line_size == 0)
2681 		return 0;
2682 
2683 	if ((u32) nfit_blk->cmd_offset % mmio->line_size
2684 			+ 8 > mmio->line_size) {
2685 		dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
2686 		return -ENXIO;
2687 	} else if ((u32) nfit_blk->stat_offset % mmio->line_size
2688 			+ 8 > mmio->line_size) {
2689 		dev_dbg(dev, "stat_offset crosses interleave boundary\n");
2690 		return -ENXIO;
2691 	}
2692 
2693 	return 0;
2694 }
2695 
2696 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
2697 		struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
2698 {
2699 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2700 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2701 	int cmd_rc, rc;
2702 
2703 	cmd->address = spa->address;
2704 	cmd->length = spa->length;
2705 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
2706 			sizeof(*cmd), &cmd_rc);
2707 	if (rc < 0)
2708 		return rc;
2709 	return cmd_rc;
2710 }
2711 
2712 static int ars_start(struct acpi_nfit_desc *acpi_desc,
2713 		struct nfit_spa *nfit_spa, enum nfit_ars_state req_type)
2714 {
2715 	int rc;
2716 	int cmd_rc;
2717 	struct nd_cmd_ars_start ars_start;
2718 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2719 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2720 
2721 	memset(&ars_start, 0, sizeof(ars_start));
2722 	ars_start.address = spa->address;
2723 	ars_start.length = spa->length;
2724 	if (req_type == ARS_REQ_SHORT)
2725 		ars_start.flags = ND_ARS_RETURN_PREV_DATA;
2726 	if (nfit_spa_type(spa) == NFIT_SPA_PM)
2727 		ars_start.type = ND_ARS_PERSISTENT;
2728 	else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
2729 		ars_start.type = ND_ARS_VOLATILE;
2730 	else
2731 		return -ENOTTY;
2732 
2733 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2734 			sizeof(ars_start), &cmd_rc);
2735 
2736 	if (rc < 0)
2737 		return rc;
2738 	if (cmd_rc < 0)
2739 		return cmd_rc;
2740 	set_bit(ARS_VALID, &acpi_desc->scrub_flags);
2741 	return 0;
2742 }
2743 
2744 static int ars_continue(struct acpi_nfit_desc *acpi_desc)
2745 {
2746 	int rc, cmd_rc;
2747 	struct nd_cmd_ars_start ars_start;
2748 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2749 	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2750 
2751 	ars_start = (struct nd_cmd_ars_start) {
2752 		.address = ars_status->restart_address,
2753 		.length = ars_status->restart_length,
2754 		.type = ars_status->type,
2755 	};
2756 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2757 			sizeof(ars_start), &cmd_rc);
2758 	if (rc < 0)
2759 		return rc;
2760 	return cmd_rc;
2761 }
2762 
2763 static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
2764 {
2765 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2766 	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2767 	int rc, cmd_rc;
2768 
2769 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
2770 			acpi_desc->max_ars, &cmd_rc);
2771 	if (rc < 0)
2772 		return rc;
2773 	return cmd_rc;
2774 }
2775 
2776 static void ars_complete(struct acpi_nfit_desc *acpi_desc,
2777 		struct nfit_spa *nfit_spa)
2778 {
2779 	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2780 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2781 	struct nd_region *nd_region = nfit_spa->nd_region;
2782 	struct device *dev;
2783 
2784 	lockdep_assert_held(&acpi_desc->init_mutex);
2785 	/*
2786 	 * Only advance the ARS state for ARS runs initiated by the
2787 	 * kernel, ignore ARS results from BIOS initiated runs for scrub
2788 	 * completion tracking.
2789 	 */
2790 	if (acpi_desc->scrub_spa != nfit_spa)
2791 		return;
2792 
2793 	if ((ars_status->address >= spa->address && ars_status->address
2794 				< spa->address + spa->length)
2795 			|| (ars_status->address < spa->address)) {
2796 		/*
2797 		 * Assume that if a scrub starts at an offset from the
2798 		 * start of nfit_spa that we are in the continuation
2799 		 * case.
2800 		 *
2801 		 * Otherwise, if the scrub covers the spa range, mark
2802 		 * any pending request complete.
2803 		 */
2804 		if (ars_status->address + ars_status->length
2805 				>= spa->address + spa->length)
2806 				/* complete */;
2807 		else
2808 			return;
2809 	} else
2810 		return;
2811 
2812 	acpi_desc->scrub_spa = NULL;
2813 	if (nd_region) {
2814 		dev = nd_region_dev(nd_region);
2815 		nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON);
2816 	} else
2817 		dev = acpi_desc->dev;
2818 	dev_dbg(dev, "ARS: range %d complete\n", spa->range_index);
2819 }
2820 
2821 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc)
2822 {
2823 	struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus;
2824 	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2825 	int rc;
2826 	u32 i;
2827 
2828 	/*
2829 	 * First record starts at 44 byte offset from the start of the
2830 	 * payload.
2831 	 */
2832 	if (ars_status->out_length < 44)
2833 		return 0;
2834 
2835 	/*
2836 	 * Ignore potentially stale results that are only refreshed
2837 	 * after a start-ARS event.
2838 	 */
2839 	if (!test_and_clear_bit(ARS_VALID, &acpi_desc->scrub_flags)) {
2840 		dev_dbg(acpi_desc->dev, "skip %d stale records\n",
2841 				ars_status->num_records);
2842 		return 0;
2843 	}
2844 
2845 	for (i = 0; i < ars_status->num_records; i++) {
2846 		/* only process full records */
2847 		if (ars_status->out_length
2848 				< 44 + sizeof(struct nd_ars_record) * (i + 1))
2849 			break;
2850 		rc = nvdimm_bus_add_badrange(nvdimm_bus,
2851 				ars_status->records[i].err_address,
2852 				ars_status->records[i].length);
2853 		if (rc)
2854 			return rc;
2855 	}
2856 	if (i < ars_status->num_records)
2857 		dev_warn(acpi_desc->dev, "detected truncated ars results\n");
2858 
2859 	return 0;
2860 }
2861 
2862 static void acpi_nfit_remove_resource(void *data)
2863 {
2864 	struct resource *res = data;
2865 
2866 	remove_resource(res);
2867 }
2868 
2869 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2870 		struct nd_region_desc *ndr_desc)
2871 {
2872 	struct resource *res, *nd_res = ndr_desc->res;
2873 	int is_pmem, ret;
2874 
2875 	/* No operation if the region is already registered as PMEM */
2876 	is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2877 				IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2878 	if (is_pmem == REGION_INTERSECTS)
2879 		return 0;
2880 
2881 	res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2882 	if (!res)
2883 		return -ENOMEM;
2884 
2885 	res->name = "Persistent Memory";
2886 	res->start = nd_res->start;
2887 	res->end = nd_res->end;
2888 	res->flags = IORESOURCE_MEM;
2889 	res->desc = IORES_DESC_PERSISTENT_MEMORY;
2890 
2891 	ret = insert_resource(&iomem_resource, res);
2892 	if (ret)
2893 		return ret;
2894 
2895 	ret = devm_add_action_or_reset(acpi_desc->dev,
2896 					acpi_nfit_remove_resource,
2897 					res);
2898 	if (ret)
2899 		return ret;
2900 
2901 	return 0;
2902 }
2903 
2904 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2905 		struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2906 		struct acpi_nfit_memory_map *memdev,
2907 		struct nfit_spa *nfit_spa)
2908 {
2909 	struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2910 			memdev->device_handle);
2911 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2912 	struct nd_blk_region_desc *ndbr_desc;
2913 	struct nfit_mem *nfit_mem;
2914 	int rc;
2915 
2916 	if (!nvdimm) {
2917 		dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2918 				spa->range_index, memdev->device_handle);
2919 		return -ENODEV;
2920 	}
2921 
2922 	mapping->nvdimm = nvdimm;
2923 	switch (nfit_spa_type(spa)) {
2924 	case NFIT_SPA_PM:
2925 	case NFIT_SPA_VOLATILE:
2926 		mapping->start = memdev->address;
2927 		mapping->size = memdev->region_size;
2928 		break;
2929 	case NFIT_SPA_DCR:
2930 		nfit_mem = nvdimm_provider_data(nvdimm);
2931 		if (!nfit_mem || !nfit_mem->bdw) {
2932 			dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2933 					spa->range_index, nvdimm_name(nvdimm));
2934 			break;
2935 		}
2936 
2937 		mapping->size = nfit_mem->bdw->capacity;
2938 		mapping->start = nfit_mem->bdw->start_address;
2939 		ndr_desc->num_lanes = nfit_mem->bdw->windows;
2940 		ndr_desc->mapping = mapping;
2941 		ndr_desc->num_mappings = 1;
2942 		ndbr_desc = to_blk_region_desc(ndr_desc);
2943 		ndbr_desc->enable = acpi_nfit_blk_region_enable;
2944 		ndbr_desc->do_io = acpi_desc->blk_do_io;
2945 		rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2946 		if (rc)
2947 			return rc;
2948 		nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2949 				ndr_desc);
2950 		if (!nfit_spa->nd_region)
2951 			return -ENOMEM;
2952 		break;
2953 	}
2954 
2955 	return 0;
2956 }
2957 
2958 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2959 {
2960 	return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2961 		nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2962 		nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2963 		nfit_spa_type(spa) == NFIT_SPA_PCD);
2964 }
2965 
2966 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa)
2967 {
2968 	return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2969 		nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2970 		nfit_spa_type(spa) == NFIT_SPA_VOLATILE);
2971 }
2972 
2973 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2974 		struct nfit_spa *nfit_spa)
2975 {
2976 	static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2977 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2978 	struct nd_blk_region_desc ndbr_desc;
2979 	struct nd_region_desc *ndr_desc;
2980 	struct nfit_memdev *nfit_memdev;
2981 	struct nvdimm_bus *nvdimm_bus;
2982 	struct resource res;
2983 	int count = 0, rc;
2984 
2985 	if (nfit_spa->nd_region)
2986 		return 0;
2987 
2988 	if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2989 		dev_dbg(acpi_desc->dev, "detected invalid spa index\n");
2990 		return 0;
2991 	}
2992 
2993 	memset(&res, 0, sizeof(res));
2994 	memset(&mappings, 0, sizeof(mappings));
2995 	memset(&ndbr_desc, 0, sizeof(ndbr_desc));
2996 	res.start = spa->address;
2997 	res.end = res.start + spa->length - 1;
2998 	ndr_desc = &ndbr_desc.ndr_desc;
2999 	ndr_desc->res = &res;
3000 	ndr_desc->provider_data = nfit_spa;
3001 	ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
3002 	if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) {
3003 		ndr_desc->numa_node = pxm_to_online_node(spa->proximity_domain);
3004 		ndr_desc->target_node = pxm_to_node(spa->proximity_domain);
3005 	} else {
3006 		ndr_desc->numa_node = NUMA_NO_NODE;
3007 		ndr_desc->target_node = NUMA_NO_NODE;
3008 	}
3009 
3010 	/*
3011 	 * Persistence domain bits are hierarchical, if
3012 	 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
3013 	 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
3014 	 */
3015 	if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
3016 		set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
3017 	else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
3018 		set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
3019 
3020 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
3021 		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
3022 		struct nd_mapping_desc *mapping;
3023 
3024 		/* range index 0 == unmapped in SPA or invalid-SPA */
3025 		if (memdev->range_index == 0 || spa->range_index == 0)
3026 			continue;
3027 		if (memdev->range_index != spa->range_index)
3028 			continue;
3029 		if (count >= ND_MAX_MAPPINGS) {
3030 			dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
3031 					spa->range_index, ND_MAX_MAPPINGS);
3032 			return -ENXIO;
3033 		}
3034 		mapping = &mappings[count++];
3035 		rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
3036 				memdev, nfit_spa);
3037 		if (rc)
3038 			goto out;
3039 	}
3040 
3041 	ndr_desc->mapping = mappings;
3042 	ndr_desc->num_mappings = count;
3043 	rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
3044 	if (rc)
3045 		goto out;
3046 
3047 	nvdimm_bus = acpi_desc->nvdimm_bus;
3048 	if (nfit_spa_type(spa) == NFIT_SPA_PM) {
3049 		rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
3050 		if (rc) {
3051 			dev_warn(acpi_desc->dev,
3052 				"failed to insert pmem resource to iomem: %d\n",
3053 				rc);
3054 			goto out;
3055 		}
3056 
3057 		nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
3058 				ndr_desc);
3059 		if (!nfit_spa->nd_region)
3060 			rc = -ENOMEM;
3061 	} else if (nfit_spa_is_volatile(spa)) {
3062 		nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
3063 				ndr_desc);
3064 		if (!nfit_spa->nd_region)
3065 			rc = -ENOMEM;
3066 	} else if (nfit_spa_is_virtual(spa)) {
3067 		nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
3068 				ndr_desc);
3069 		if (!nfit_spa->nd_region)
3070 			rc = -ENOMEM;
3071 	}
3072 
3073  out:
3074 	if (rc)
3075 		dev_err(acpi_desc->dev, "failed to register spa range %d\n",
3076 				nfit_spa->spa->range_index);
3077 	return rc;
3078 }
3079 
3080 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc)
3081 {
3082 	struct device *dev = acpi_desc->dev;
3083 	struct nd_cmd_ars_status *ars_status;
3084 
3085 	if (acpi_desc->ars_status) {
3086 		memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3087 		return 0;
3088 	}
3089 
3090 	ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL);
3091 	if (!ars_status)
3092 		return -ENOMEM;
3093 	acpi_desc->ars_status = ars_status;
3094 	return 0;
3095 }
3096 
3097 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc)
3098 {
3099 	int rc;
3100 
3101 	if (ars_status_alloc(acpi_desc))
3102 		return -ENOMEM;
3103 
3104 	rc = ars_get_status(acpi_desc);
3105 
3106 	if (rc < 0 && rc != -ENOSPC)
3107 		return rc;
3108 
3109 	if (ars_status_process_records(acpi_desc))
3110 		dev_err(acpi_desc->dev, "Failed to process ARS records\n");
3111 
3112 	return rc;
3113 }
3114 
3115 static int ars_register(struct acpi_nfit_desc *acpi_desc,
3116 		struct nfit_spa *nfit_spa)
3117 {
3118 	int rc;
3119 
3120 	if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3121 		return acpi_nfit_register_region(acpi_desc, nfit_spa);
3122 
3123 	set_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3124 	if (!no_init_ars)
3125 		set_bit(ARS_REQ_LONG, &nfit_spa->ars_state);
3126 
3127 	switch (acpi_nfit_query_poison(acpi_desc)) {
3128 	case 0:
3129 	case -ENOSPC:
3130 	case -EAGAIN:
3131 		rc = ars_start(acpi_desc, nfit_spa, ARS_REQ_SHORT);
3132 		/* shouldn't happen, try again later */
3133 		if (rc == -EBUSY)
3134 			break;
3135 		if (rc) {
3136 			set_bit(ARS_FAILED, &nfit_spa->ars_state);
3137 			break;
3138 		}
3139 		clear_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3140 		rc = acpi_nfit_query_poison(acpi_desc);
3141 		if (rc)
3142 			break;
3143 		acpi_desc->scrub_spa = nfit_spa;
3144 		ars_complete(acpi_desc, nfit_spa);
3145 		/*
3146 		 * If ars_complete() says we didn't complete the
3147 		 * short scrub, we'll try again with a long
3148 		 * request.
3149 		 */
3150 		acpi_desc->scrub_spa = NULL;
3151 		break;
3152 	case -EBUSY:
3153 	case -ENOMEM:
3154 		/*
3155 		 * BIOS was using ARS, wait for it to complete (or
3156 		 * resources to become available) and then perform our
3157 		 * own scrubs.
3158 		 */
3159 		break;
3160 	default:
3161 		set_bit(ARS_FAILED, &nfit_spa->ars_state);
3162 		break;
3163 	}
3164 
3165 	return acpi_nfit_register_region(acpi_desc, nfit_spa);
3166 }
3167 
3168 static void ars_complete_all(struct acpi_nfit_desc *acpi_desc)
3169 {
3170 	struct nfit_spa *nfit_spa;
3171 
3172 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3173 		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3174 			continue;
3175 		ars_complete(acpi_desc, nfit_spa);
3176 	}
3177 }
3178 
3179 static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc,
3180 		int query_rc)
3181 {
3182 	unsigned int tmo = acpi_desc->scrub_tmo;
3183 	struct device *dev = acpi_desc->dev;
3184 	struct nfit_spa *nfit_spa;
3185 
3186 	lockdep_assert_held(&acpi_desc->init_mutex);
3187 
3188 	if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags))
3189 		return 0;
3190 
3191 	if (query_rc == -EBUSY) {
3192 		dev_dbg(dev, "ARS: ARS busy\n");
3193 		return min(30U * 60U, tmo * 2);
3194 	}
3195 	if (query_rc == -ENOSPC) {
3196 		dev_dbg(dev, "ARS: ARS continue\n");
3197 		ars_continue(acpi_desc);
3198 		return 1;
3199 	}
3200 	if (query_rc && query_rc != -EAGAIN) {
3201 		unsigned long long addr, end;
3202 
3203 		addr = acpi_desc->ars_status->address;
3204 		end = addr + acpi_desc->ars_status->length;
3205 		dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end,
3206 				query_rc);
3207 	}
3208 
3209 	ars_complete_all(acpi_desc);
3210 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3211 		enum nfit_ars_state req_type;
3212 		int rc;
3213 
3214 		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3215 			continue;
3216 
3217 		/* prefer short ARS requests first */
3218 		if (test_bit(ARS_REQ_SHORT, &nfit_spa->ars_state))
3219 			req_type = ARS_REQ_SHORT;
3220 		else if (test_bit(ARS_REQ_LONG, &nfit_spa->ars_state))
3221 			req_type = ARS_REQ_LONG;
3222 		else
3223 			continue;
3224 		rc = ars_start(acpi_desc, nfit_spa, req_type);
3225 
3226 		dev = nd_region_dev(nfit_spa->nd_region);
3227 		dev_dbg(dev, "ARS: range %d ARS start %s (%d)\n",
3228 				nfit_spa->spa->range_index,
3229 				req_type == ARS_REQ_SHORT ? "short" : "long",
3230 				rc);
3231 		/*
3232 		 * Hmm, we raced someone else starting ARS? Try again in
3233 		 * a bit.
3234 		 */
3235 		if (rc == -EBUSY)
3236 			return 1;
3237 		if (rc == 0) {
3238 			dev_WARN_ONCE(dev, acpi_desc->scrub_spa,
3239 					"scrub start while range %d active\n",
3240 					acpi_desc->scrub_spa->spa->range_index);
3241 			clear_bit(req_type, &nfit_spa->ars_state);
3242 			acpi_desc->scrub_spa = nfit_spa;
3243 			/*
3244 			 * Consider this spa last for future scrub
3245 			 * requests
3246 			 */
3247 			list_move_tail(&nfit_spa->list, &acpi_desc->spas);
3248 			return 1;
3249 		}
3250 
3251 		dev_err(dev, "ARS: range %d ARS failed (%d)\n",
3252 				nfit_spa->spa->range_index, rc);
3253 		set_bit(ARS_FAILED, &nfit_spa->ars_state);
3254 	}
3255 	return 0;
3256 }
3257 
3258 static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo)
3259 {
3260 	lockdep_assert_held(&acpi_desc->init_mutex);
3261 
3262 	set_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3263 	/* note this should only be set from within the workqueue */
3264 	if (tmo)
3265 		acpi_desc->scrub_tmo = tmo;
3266 	queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ);
3267 }
3268 
3269 static void sched_ars(struct acpi_nfit_desc *acpi_desc)
3270 {
3271 	__sched_ars(acpi_desc, 0);
3272 }
3273 
3274 static void notify_ars_done(struct acpi_nfit_desc *acpi_desc)
3275 {
3276 	lockdep_assert_held(&acpi_desc->init_mutex);
3277 
3278 	clear_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3279 	acpi_desc->scrub_count++;
3280 	if (acpi_desc->scrub_count_state)
3281 		sysfs_notify_dirent(acpi_desc->scrub_count_state);
3282 }
3283 
3284 static void acpi_nfit_scrub(struct work_struct *work)
3285 {
3286 	struct acpi_nfit_desc *acpi_desc;
3287 	unsigned int tmo;
3288 	int query_rc;
3289 
3290 	acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work);
3291 	mutex_lock(&acpi_desc->init_mutex);
3292 	query_rc = acpi_nfit_query_poison(acpi_desc);
3293 	tmo = __acpi_nfit_scrub(acpi_desc, query_rc);
3294 	if (tmo)
3295 		__sched_ars(acpi_desc, tmo);
3296 	else
3297 		notify_ars_done(acpi_desc);
3298 	memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3299 	clear_bit(ARS_POLL, &acpi_desc->scrub_flags);
3300 	mutex_unlock(&acpi_desc->init_mutex);
3301 }
3302 
3303 static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc,
3304 		struct nfit_spa *nfit_spa)
3305 {
3306 	int type = nfit_spa_type(nfit_spa->spa);
3307 	struct nd_cmd_ars_cap ars_cap;
3308 	int rc;
3309 
3310 	set_bit(ARS_FAILED, &nfit_spa->ars_state);
3311 	memset(&ars_cap, 0, sizeof(ars_cap));
3312 	rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
3313 	if (rc < 0)
3314 		return;
3315 	/* check that the supported scrub types match the spa type */
3316 	if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16)
3317 				& ND_ARS_VOLATILE) == 0)
3318 		return;
3319 	if (type == NFIT_SPA_PM && ((ars_cap.status >> 16)
3320 				& ND_ARS_PERSISTENT) == 0)
3321 		return;
3322 
3323 	nfit_spa->max_ars = ars_cap.max_ars_out;
3324 	nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
3325 	acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars);
3326 	clear_bit(ARS_FAILED, &nfit_spa->ars_state);
3327 }
3328 
3329 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
3330 {
3331 	struct nfit_spa *nfit_spa;
3332 	int rc, do_sched_ars = 0;
3333 
3334 	set_bit(ARS_VALID, &acpi_desc->scrub_flags);
3335 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3336 		switch (nfit_spa_type(nfit_spa->spa)) {
3337 		case NFIT_SPA_VOLATILE:
3338 		case NFIT_SPA_PM:
3339 			acpi_nfit_init_ars(acpi_desc, nfit_spa);
3340 			break;
3341 		}
3342 	}
3343 
3344 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3345 		switch (nfit_spa_type(nfit_spa->spa)) {
3346 		case NFIT_SPA_VOLATILE:
3347 		case NFIT_SPA_PM:
3348 			/* register regions and kick off initial ARS run */
3349 			rc = ars_register(acpi_desc, nfit_spa);
3350 			if (rc)
3351 				return rc;
3352 
3353 			/*
3354 			 * Kick off background ARS if at least one
3355 			 * region successfully registered ARS
3356 			 */
3357 			if (!test_bit(ARS_FAILED, &nfit_spa->ars_state))
3358 				do_sched_ars++;
3359 			break;
3360 		case NFIT_SPA_BDW:
3361 			/* nothing to register */
3362 			break;
3363 		case NFIT_SPA_DCR:
3364 		case NFIT_SPA_VDISK:
3365 		case NFIT_SPA_VCD:
3366 		case NFIT_SPA_PDISK:
3367 		case NFIT_SPA_PCD:
3368 			/* register known regions that don't support ARS */
3369 			rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
3370 			if (rc)
3371 				return rc;
3372 			break;
3373 		default:
3374 			/* don't register unknown regions */
3375 			break;
3376 		}
3377 	}
3378 
3379 	if (do_sched_ars)
3380 		sched_ars(acpi_desc);
3381 	return 0;
3382 }
3383 
3384 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
3385 		struct nfit_table_prev *prev)
3386 {
3387 	struct device *dev = acpi_desc->dev;
3388 
3389 	if (!list_empty(&prev->spas) ||
3390 			!list_empty(&prev->memdevs) ||
3391 			!list_empty(&prev->dcrs) ||
3392 			!list_empty(&prev->bdws) ||
3393 			!list_empty(&prev->idts) ||
3394 			!list_empty(&prev->flushes)) {
3395 		dev_err(dev, "new nfit deletes entries (unsupported)\n");
3396 		return -ENXIO;
3397 	}
3398 	return 0;
3399 }
3400 
3401 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
3402 {
3403 	struct device *dev = acpi_desc->dev;
3404 	struct kernfs_node *nfit;
3405 	struct device *bus_dev;
3406 
3407 	if (!ars_supported(acpi_desc->nvdimm_bus))
3408 		return 0;
3409 
3410 	bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3411 	nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
3412 	if (!nfit) {
3413 		dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
3414 		return -ENODEV;
3415 	}
3416 	acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
3417 	sysfs_put(nfit);
3418 	if (!acpi_desc->scrub_count_state) {
3419 		dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
3420 		return -ENODEV;
3421 	}
3422 
3423 	return 0;
3424 }
3425 
3426 static void acpi_nfit_unregister(void *data)
3427 {
3428 	struct acpi_nfit_desc *acpi_desc = data;
3429 
3430 	nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
3431 }
3432 
3433 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
3434 {
3435 	struct device *dev = acpi_desc->dev;
3436 	struct nfit_table_prev prev;
3437 	const void *end;
3438 	int rc;
3439 
3440 	if (!acpi_desc->nvdimm_bus) {
3441 		acpi_nfit_init_dsms(acpi_desc);
3442 
3443 		acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
3444 				&acpi_desc->nd_desc);
3445 		if (!acpi_desc->nvdimm_bus)
3446 			return -ENOMEM;
3447 
3448 		rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
3449 				acpi_desc);
3450 		if (rc)
3451 			return rc;
3452 
3453 		rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
3454 		if (rc)
3455 			return rc;
3456 
3457 		/* register this acpi_desc for mce notifications */
3458 		mutex_lock(&acpi_desc_lock);
3459 		list_add_tail(&acpi_desc->list, &acpi_descs);
3460 		mutex_unlock(&acpi_desc_lock);
3461 	}
3462 
3463 	mutex_lock(&acpi_desc->init_mutex);
3464 
3465 	INIT_LIST_HEAD(&prev.spas);
3466 	INIT_LIST_HEAD(&prev.memdevs);
3467 	INIT_LIST_HEAD(&prev.dcrs);
3468 	INIT_LIST_HEAD(&prev.bdws);
3469 	INIT_LIST_HEAD(&prev.idts);
3470 	INIT_LIST_HEAD(&prev.flushes);
3471 
3472 	list_cut_position(&prev.spas, &acpi_desc->spas,
3473 				acpi_desc->spas.prev);
3474 	list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
3475 				acpi_desc->memdevs.prev);
3476 	list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
3477 				acpi_desc->dcrs.prev);
3478 	list_cut_position(&prev.bdws, &acpi_desc->bdws,
3479 				acpi_desc->bdws.prev);
3480 	list_cut_position(&prev.idts, &acpi_desc->idts,
3481 				acpi_desc->idts.prev);
3482 	list_cut_position(&prev.flushes, &acpi_desc->flushes,
3483 				acpi_desc->flushes.prev);
3484 
3485 	end = data + sz;
3486 	while (!IS_ERR_OR_NULL(data))
3487 		data = add_table(acpi_desc, &prev, data, end);
3488 
3489 	if (IS_ERR(data)) {
3490 		dev_dbg(dev, "nfit table parsing error: %ld\n",	PTR_ERR(data));
3491 		rc = PTR_ERR(data);
3492 		goto out_unlock;
3493 	}
3494 
3495 	rc = acpi_nfit_check_deletions(acpi_desc, &prev);
3496 	if (rc)
3497 		goto out_unlock;
3498 
3499 	rc = nfit_mem_init(acpi_desc);
3500 	if (rc)
3501 		goto out_unlock;
3502 
3503 	rc = acpi_nfit_register_dimms(acpi_desc);
3504 	if (rc)
3505 		goto out_unlock;
3506 
3507 	rc = acpi_nfit_register_regions(acpi_desc);
3508 
3509  out_unlock:
3510 	mutex_unlock(&acpi_desc->init_mutex);
3511 	return rc;
3512 }
3513 EXPORT_SYMBOL_GPL(acpi_nfit_init);
3514 
3515 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
3516 {
3517 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3518 	struct device *dev = acpi_desc->dev;
3519 
3520 	/* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
3521 	nfit_device_lock(dev);
3522 	nfit_device_unlock(dev);
3523 
3524 	/* Bounce the init_mutex to complete initial registration */
3525 	mutex_lock(&acpi_desc->init_mutex);
3526 	mutex_unlock(&acpi_desc->init_mutex);
3527 
3528 	return 0;
3529 }
3530 
3531 static int __acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3532 		struct nvdimm *nvdimm, unsigned int cmd)
3533 {
3534 	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3535 
3536 	if (nvdimm)
3537 		return 0;
3538 	if (cmd != ND_CMD_ARS_START)
3539 		return 0;
3540 
3541 	/*
3542 	 * The kernel and userspace may race to initiate a scrub, but
3543 	 * the scrub thread is prepared to lose that initial race.  It
3544 	 * just needs guarantees that any ARS it initiates are not
3545 	 * interrupted by any intervening start requests from userspace.
3546 	 */
3547 	if (work_busy(&acpi_desc->dwork.work))
3548 		return -EBUSY;
3549 
3550 	return 0;
3551 }
3552 
3553 /*
3554  * Prevent security and firmware activate commands from being issued via
3555  * ioctl.
3556  */
3557 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3558 		struct nvdimm *nvdimm, unsigned int cmd, void *buf)
3559 {
3560 	struct nd_cmd_pkg *call_pkg = buf;
3561 	unsigned int func;
3562 
3563 	if (nvdimm && cmd == ND_CMD_CALL &&
3564 			call_pkg->nd_family == NVDIMM_FAMILY_INTEL) {
3565 		func = call_pkg->nd_command;
3566 		if (func > NVDIMM_CMD_MAX ||
3567 		    (1 << func) & NVDIMM_INTEL_DENY_CMDMASK)
3568 			return -EOPNOTSUPP;
3569 	}
3570 
3571 	/* block all non-nfit bus commands */
3572 	if (!nvdimm && cmd == ND_CMD_CALL &&
3573 			call_pkg->nd_family != NVDIMM_BUS_FAMILY_NFIT)
3574 		return -EOPNOTSUPP;
3575 
3576 	return __acpi_nfit_clear_to_send(nd_desc, nvdimm, cmd);
3577 }
3578 
3579 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc,
3580 		enum nfit_ars_state req_type)
3581 {
3582 	struct device *dev = acpi_desc->dev;
3583 	int scheduled = 0, busy = 0;
3584 	struct nfit_spa *nfit_spa;
3585 
3586 	mutex_lock(&acpi_desc->init_mutex);
3587 	if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags)) {
3588 		mutex_unlock(&acpi_desc->init_mutex);
3589 		return 0;
3590 	}
3591 
3592 	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3593 		int type = nfit_spa_type(nfit_spa->spa);
3594 
3595 		if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE)
3596 			continue;
3597 		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3598 			continue;
3599 
3600 		if (test_and_set_bit(req_type, &nfit_spa->ars_state))
3601 			busy++;
3602 		else
3603 			scheduled++;
3604 	}
3605 	if (scheduled) {
3606 		sched_ars(acpi_desc);
3607 		dev_dbg(dev, "ars_scan triggered\n");
3608 	}
3609 	mutex_unlock(&acpi_desc->init_mutex);
3610 
3611 	if (scheduled)
3612 		return 0;
3613 	if (busy)
3614 		return -EBUSY;
3615 	return -ENOTTY;
3616 }
3617 
3618 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
3619 {
3620 	struct nvdimm_bus_descriptor *nd_desc;
3621 
3622 	dev_set_drvdata(dev, acpi_desc);
3623 	acpi_desc->dev = dev;
3624 	acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
3625 	nd_desc = &acpi_desc->nd_desc;
3626 	nd_desc->provider_name = "ACPI.NFIT";
3627 	nd_desc->module = THIS_MODULE;
3628 	nd_desc->ndctl = acpi_nfit_ctl;
3629 	nd_desc->flush_probe = acpi_nfit_flush_probe;
3630 	nd_desc->clear_to_send = acpi_nfit_clear_to_send;
3631 	nd_desc->attr_groups = acpi_nfit_attribute_groups;
3632 
3633 	INIT_LIST_HEAD(&acpi_desc->spas);
3634 	INIT_LIST_HEAD(&acpi_desc->dcrs);
3635 	INIT_LIST_HEAD(&acpi_desc->bdws);
3636 	INIT_LIST_HEAD(&acpi_desc->idts);
3637 	INIT_LIST_HEAD(&acpi_desc->flushes);
3638 	INIT_LIST_HEAD(&acpi_desc->memdevs);
3639 	INIT_LIST_HEAD(&acpi_desc->dimms);
3640 	INIT_LIST_HEAD(&acpi_desc->list);
3641 	mutex_init(&acpi_desc->init_mutex);
3642 	acpi_desc->scrub_tmo = 1;
3643 	INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub);
3644 }
3645 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
3646 
3647 static void acpi_nfit_put_table(void *table)
3648 {
3649 	acpi_put_table(table);
3650 }
3651 
3652 void acpi_nfit_shutdown(void *data)
3653 {
3654 	struct acpi_nfit_desc *acpi_desc = data;
3655 	struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3656 
3657 	/*
3658 	 * Destruct under acpi_desc_lock so that nfit_handle_mce does not
3659 	 * race teardown
3660 	 */
3661 	mutex_lock(&acpi_desc_lock);
3662 	list_del(&acpi_desc->list);
3663 	mutex_unlock(&acpi_desc_lock);
3664 
3665 	mutex_lock(&acpi_desc->init_mutex);
3666 	set_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
3667 	cancel_delayed_work_sync(&acpi_desc->dwork);
3668 	mutex_unlock(&acpi_desc->init_mutex);
3669 
3670 	/*
3671 	 * Bounce the nvdimm bus lock to make sure any in-flight
3672 	 * acpi_nfit_ars_rescan() submissions have had a chance to
3673 	 * either submit or see ->cancel set.
3674 	 */
3675 	nfit_device_lock(bus_dev);
3676 	nfit_device_unlock(bus_dev);
3677 
3678 	flush_workqueue(nfit_wq);
3679 }
3680 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
3681 
3682 static int acpi_nfit_add(struct acpi_device *adev)
3683 {
3684 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3685 	struct acpi_nfit_desc *acpi_desc;
3686 	struct device *dev = &adev->dev;
3687 	struct acpi_table_header *tbl;
3688 	acpi_status status = AE_OK;
3689 	acpi_size sz;
3690 	int rc = 0;
3691 
3692 	status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl);
3693 	if (ACPI_FAILURE(status)) {
3694 		/* The NVDIMM root device allows OS to trigger enumeration of
3695 		 * NVDIMMs through NFIT at boot time and re-enumeration at
3696 		 * root level via the _FIT method during runtime.
3697 		 * This is ok to return 0 here, we could have an nvdimm
3698 		 * hotplugged later and evaluate _FIT method which returns
3699 		 * data in the format of a series of NFIT Structures.
3700 		 */
3701 		dev_dbg(dev, "failed to find NFIT at startup\n");
3702 		return 0;
3703 	}
3704 
3705 	rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
3706 	if (rc)
3707 		return rc;
3708 	sz = tbl->length;
3709 
3710 	acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3711 	if (!acpi_desc)
3712 		return -ENOMEM;
3713 	acpi_nfit_desc_init(acpi_desc, &adev->dev);
3714 
3715 	/* Save the acpi header for exporting the revision via sysfs */
3716 	acpi_desc->acpi_header = *tbl;
3717 
3718 	/* Evaluate _FIT and override with that if present */
3719 	status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
3720 	if (ACPI_SUCCESS(status) && buf.length > 0) {
3721 		union acpi_object *obj = buf.pointer;
3722 
3723 		if (obj->type == ACPI_TYPE_BUFFER)
3724 			rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3725 					obj->buffer.length);
3726 		else
3727 			dev_dbg(dev, "invalid type %d, ignoring _FIT\n",
3728 				(int) obj->type);
3729 		kfree(buf.pointer);
3730 	} else
3731 		/* skip over the lead-in header table */
3732 		rc = acpi_nfit_init(acpi_desc, (void *) tbl
3733 				+ sizeof(struct acpi_table_nfit),
3734 				sz - sizeof(struct acpi_table_nfit));
3735 
3736 	if (rc)
3737 		return rc;
3738 	return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
3739 }
3740 
3741 static int acpi_nfit_remove(struct acpi_device *adev)
3742 {
3743 	/* see acpi_nfit_unregister */
3744 	return 0;
3745 }
3746 
3747 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle)
3748 {
3749 	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3750 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3751 	union acpi_object *obj;
3752 	acpi_status status;
3753 	int ret;
3754 
3755 	if (!dev->driver) {
3756 		/* dev->driver may be null if we're being removed */
3757 		dev_dbg(dev, "no driver found for dev\n");
3758 		return;
3759 	}
3760 
3761 	if (!acpi_desc) {
3762 		acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3763 		if (!acpi_desc)
3764 			return;
3765 		acpi_nfit_desc_init(acpi_desc, dev);
3766 	} else {
3767 		/*
3768 		 * Finish previous registration before considering new
3769 		 * regions.
3770 		 */
3771 		flush_workqueue(nfit_wq);
3772 	}
3773 
3774 	/* Evaluate _FIT */
3775 	status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
3776 	if (ACPI_FAILURE(status)) {
3777 		dev_err(dev, "failed to evaluate _FIT\n");
3778 		return;
3779 	}
3780 
3781 	obj = buf.pointer;
3782 	if (obj->type == ACPI_TYPE_BUFFER) {
3783 		ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3784 				obj->buffer.length);
3785 		if (ret)
3786 			dev_err(dev, "failed to merge updated NFIT\n");
3787 	} else
3788 		dev_err(dev, "Invalid _FIT\n");
3789 	kfree(buf.pointer);
3790 }
3791 
3792 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle)
3793 {
3794 	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3795 
3796 	if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON)
3797 		acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
3798 	else
3799 		acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_SHORT);
3800 }
3801 
3802 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
3803 {
3804 	dev_dbg(dev, "event: 0x%x\n", event);
3805 
3806 	switch (event) {
3807 	case NFIT_NOTIFY_UPDATE:
3808 		return acpi_nfit_update_notify(dev, handle);
3809 	case NFIT_NOTIFY_UC_MEMORY_ERROR:
3810 		return acpi_nfit_uc_error_notify(dev, handle);
3811 	default:
3812 		return;
3813 	}
3814 }
3815 EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
3816 
3817 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
3818 {
3819 	nfit_device_lock(&adev->dev);
3820 	__acpi_nfit_notify(&adev->dev, adev->handle, event);
3821 	nfit_device_unlock(&adev->dev);
3822 }
3823 
3824 static const struct acpi_device_id acpi_nfit_ids[] = {
3825 	{ "ACPI0012", 0 },
3826 	{ "", 0 },
3827 };
3828 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
3829 
3830 static struct acpi_driver acpi_nfit_driver = {
3831 	.name = KBUILD_MODNAME,
3832 	.ids = acpi_nfit_ids,
3833 	.ops = {
3834 		.add = acpi_nfit_add,
3835 		.remove = acpi_nfit_remove,
3836 		.notify = acpi_nfit_notify,
3837 	},
3838 };
3839 
3840 static __init int nfit_init(void)
3841 {
3842 	int ret;
3843 
3844 	BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
3845 	BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 64);
3846 	BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
3847 	BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
3848 	BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
3849 	BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
3850 	BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
3851 	BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
3852 
3853 	guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
3854 	guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
3855 	guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]);
3856 	guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]);
3857 	guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]);
3858 	guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]);
3859 	guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]);
3860 	guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]);
3861 	guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]);
3862 	guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]);
3863 	guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
3864 	guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
3865 	guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
3866 	guid_parse(UUID_NFIT_DIMM_N_HYPERV, &nfit_uuid[NFIT_DEV_DIMM_N_HYPERV]);
3867 	guid_parse(UUID_INTEL_BUS, &nfit_uuid[NFIT_BUS_INTEL]);
3868 
3869 	nfit_wq = create_singlethread_workqueue("nfit");
3870 	if (!nfit_wq)
3871 		return -ENOMEM;
3872 
3873 	nfit_mce_register();
3874 	ret = acpi_bus_register_driver(&acpi_nfit_driver);
3875 	if (ret) {
3876 		nfit_mce_unregister();
3877 		destroy_workqueue(nfit_wq);
3878 	}
3879 
3880 	return ret;
3881 
3882 }
3883 
3884 static __exit void nfit_exit(void)
3885 {
3886 	nfit_mce_unregister();
3887 	acpi_bus_unregister_driver(&acpi_nfit_driver);
3888 	destroy_workqueue(nfit_wq);
3889 	WARN_ON(!list_empty(&acpi_descs));
3890 }
3891 
3892 module_init(nfit_init);
3893 module_exit(nfit_exit);
3894 MODULE_LICENSE("GPL v2");
3895 MODULE_AUTHOR("Intel Corporation");
3896