1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/debugfs.h>
25 #include <linux/list.h>
26 #include <linux/module.h>
27 #include <linux/uaccess.h>
28 #include <linux/reboot.h>
29 #include <linux/syscalls.h>
30 #include <linux/pm_runtime.h>
31 
32 #include "amdgpu.h"
33 #include "amdgpu_ras.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_xgmi.h"
36 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
37 #include "atom.h"
38 #include "amdgpu_reset.h"
39 
40 #ifdef CONFIG_X86_MCE_AMD
41 #include <asm/mce.h>
42 
43 static bool notifier_registered;
44 #endif
45 static const char *RAS_FS_NAME = "ras";
46 
47 const char *ras_error_string[] = {
48 	"none",
49 	"parity",
50 	"single_correctable",
51 	"multi_uncorrectable",
52 	"poison",
53 };
54 
55 const char *ras_block_string[] = {
56 	"umc",
57 	"sdma",
58 	"gfx",
59 	"mmhub",
60 	"athub",
61 	"pcie_bif",
62 	"hdp",
63 	"xgmi_wafl",
64 	"df",
65 	"smn",
66 	"sem",
67 	"mp0",
68 	"mp1",
69 	"fuse",
70 	"mca",
71 	"vcn",
72 	"jpeg",
73 };
74 
75 const char *ras_mca_block_string[] = {
76 	"mca_mp0",
77 	"mca_mp1",
78 	"mca_mpio",
79 	"mca_iohc",
80 };
81 
82 struct amdgpu_ras_block_list {
83 	/* ras block link */
84 	struct list_head node;
85 
86 	struct amdgpu_ras_block_object *ras_obj;
87 };
88 
89 const char *get_ras_block_str(struct ras_common_if *ras_block)
90 {
91 	if (!ras_block)
92 		return "NULL";
93 
94 	if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
95 		return "OUT OF RANGE";
96 
97 	if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
98 		return ras_mca_block_string[ras_block->sub_block_index];
99 
100 	return ras_block_string[ras_block->block];
101 }
102 
103 #define ras_block_str(_BLOCK_) \
104 	(((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
105 
106 #define ras_err_str(i) (ras_error_string[ffs(i)])
107 
108 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
109 
110 /* inject address is 52 bits */
111 #define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
112 
113 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
114 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
115 
116 enum amdgpu_ras_retire_page_reservation {
117 	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
118 	AMDGPU_RAS_RETIRE_PAGE_PENDING,
119 	AMDGPU_RAS_RETIRE_PAGE_FAULT,
120 };
121 
122 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
123 
124 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
125 				uint64_t addr);
126 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
127 				uint64_t addr);
128 #ifdef CONFIG_X86_MCE_AMD
129 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
130 struct mce_notifier_adev_list {
131 	struct amdgpu_device *devs[MAX_GPU_INSTANCE];
132 	int num_gpu;
133 };
134 static struct mce_notifier_adev_list mce_adev_list;
135 #endif
136 
137 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
138 {
139 	if (adev && amdgpu_ras_get_context(adev))
140 		amdgpu_ras_get_context(adev)->error_query_ready = ready;
141 }
142 
143 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
144 {
145 	if (adev && amdgpu_ras_get_context(adev))
146 		return amdgpu_ras_get_context(adev)->error_query_ready;
147 
148 	return false;
149 }
150 
151 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
152 {
153 	struct ras_err_data err_data = {0, 0, 0, NULL};
154 	struct eeprom_table_record err_rec;
155 
156 	if ((address >= adev->gmc.mc_vram_size) ||
157 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
158 		dev_warn(adev->dev,
159 		         "RAS WARN: input address 0x%llx is invalid.\n",
160 		         address);
161 		return -EINVAL;
162 	}
163 
164 	if (amdgpu_ras_check_bad_page(adev, address)) {
165 		dev_warn(adev->dev,
166 			 "RAS WARN: 0x%llx has already been marked as bad page!\n",
167 			 address);
168 		return 0;
169 	}
170 
171 	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
172 	err_data.err_addr = &err_rec;
173 	amdgpu_umc_fill_error_record(&err_data, address,
174 			(address >> AMDGPU_GPU_PAGE_SHIFT), 0, 0);
175 
176 	if (amdgpu_bad_page_threshold != 0) {
177 		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
178 					 err_data.err_addr_cnt);
179 		amdgpu_ras_save_bad_pages(adev);
180 	}
181 
182 	dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
183 	dev_warn(adev->dev, "Clear EEPROM:\n");
184 	dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
185 
186 	return 0;
187 }
188 
189 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
190 					size_t size, loff_t *pos)
191 {
192 	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
193 	struct ras_query_if info = {
194 		.head = obj->head,
195 	};
196 	ssize_t s;
197 	char val[128];
198 
199 	if (amdgpu_ras_query_error_status(obj->adev, &info))
200 		return -EINVAL;
201 
202 	/* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
203 	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
204 	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
205 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
206 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
207 	}
208 
209 	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
210 			"ue", info.ue_count,
211 			"ce", info.ce_count);
212 	if (*pos >= s)
213 		return 0;
214 
215 	s -= *pos;
216 	s = min_t(u64, s, size);
217 
218 
219 	if (copy_to_user(buf, &val[*pos], s))
220 		return -EINVAL;
221 
222 	*pos += s;
223 
224 	return s;
225 }
226 
227 static const struct file_operations amdgpu_ras_debugfs_ops = {
228 	.owner = THIS_MODULE,
229 	.read = amdgpu_ras_debugfs_read,
230 	.write = NULL,
231 	.llseek = default_llseek
232 };
233 
234 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
235 {
236 	int i;
237 
238 	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
239 		*block_id = i;
240 		if (strcmp(name, ras_block_string[i]) == 0)
241 			return 0;
242 	}
243 	return -EINVAL;
244 }
245 
246 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
247 		const char __user *buf, size_t size,
248 		loff_t *pos, struct ras_debug_if *data)
249 {
250 	ssize_t s = min_t(u64, 64, size);
251 	char str[65];
252 	char block_name[33];
253 	char err[9] = "ue";
254 	int op = -1;
255 	int block_id;
256 	uint32_t sub_block;
257 	u64 address, value;
258 
259 	if (*pos)
260 		return -EINVAL;
261 	*pos = size;
262 
263 	memset(str, 0, sizeof(str));
264 	memset(data, 0, sizeof(*data));
265 
266 	if (copy_from_user(str, buf, s))
267 		return -EINVAL;
268 
269 	if (sscanf(str, "disable %32s", block_name) == 1)
270 		op = 0;
271 	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
272 		op = 1;
273 	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
274 		op = 2;
275 	else if (strstr(str, "retire_page") != NULL)
276 		op = 3;
277 	else if (str[0] && str[1] && str[2] && str[3])
278 		/* ascii string, but commands are not matched. */
279 		return -EINVAL;
280 
281 	if (op != -1) {
282 		if (op == 3) {
283 			if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
284 			    sscanf(str, "%*s %llu", &address) != 1)
285 				return -EINVAL;
286 
287 			data->op = op;
288 			data->inject.address = address;
289 
290 			return 0;
291 		}
292 
293 		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
294 			return -EINVAL;
295 
296 		data->head.block = block_id;
297 		/* only ue and ce errors are supported */
298 		if (!memcmp("ue", err, 2))
299 			data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
300 		else if (!memcmp("ce", err, 2))
301 			data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
302 		else
303 			return -EINVAL;
304 
305 		data->op = op;
306 
307 		if (op == 2) {
308 			if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
309 				   &sub_block, &address, &value) != 3 &&
310 			    sscanf(str, "%*s %*s %*s %u %llu %llu",
311 				   &sub_block, &address, &value) != 3)
312 				return -EINVAL;
313 			data->head.sub_block_index = sub_block;
314 			data->inject.address = address;
315 			data->inject.value = value;
316 		}
317 	} else {
318 		if (size < sizeof(*data))
319 			return -EINVAL;
320 
321 		if (copy_from_user(data, buf, sizeof(*data)))
322 			return -EINVAL;
323 	}
324 
325 	return 0;
326 }
327 
328 /**
329  * DOC: AMDGPU RAS debugfs control interface
330  *
331  * The control interface accepts struct ras_debug_if which has two members.
332  *
333  * First member: ras_debug_if::head or ras_debug_if::inject.
334  *
335  * head is used to indicate which IP block will be under control.
336  *
337  * head has four members, they are block, type, sub_block_index, name.
338  * block: which IP will be under control.
339  * type: what kind of error will be enabled/disabled/injected.
340  * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
341  * name: the name of IP.
342  *
343  * inject has two more members than head, they are address, value.
344  * As their names indicate, inject operation will write the
345  * value to the address.
346  *
347  * The second member: struct ras_debug_if::op.
348  * It has three kinds of operations.
349  *
350  * - 0: disable RAS on the block. Take ::head as its data.
351  * - 1: enable RAS on the block. Take ::head as its data.
352  * - 2: inject errors on the block. Take ::inject as its data.
353  *
354  * How to use the interface?
355  *
356  * In a program
357  *
358  * Copy the struct ras_debug_if in your code and initialize it.
359  * Write the struct to the control interface.
360  *
361  * From shell
362  *
363  * .. code-block:: bash
364  *
365  *	echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
366  *	echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
367  *	echo "inject  <block> <error> <sub-block> <address> <value> > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
368  *
369  * Where N, is the card which you want to affect.
370  *
371  * "disable" requires only the block.
372  * "enable" requires the block and error type.
373  * "inject" requires the block, error type, address, and value.
374  *
375  * The block is one of: umc, sdma, gfx, etc.
376  *	see ras_block_string[] for details
377  *
378  * The error type is one of: ue, ce, where,
379  *	ue is multi-uncorrectable
380  *	ce is single-correctable
381  *
382  * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
383  * The address and value are hexadecimal numbers, leading 0x is optional.
384  *
385  * For instance,
386  *
387  * .. code-block:: bash
388  *
389  *	echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
390  *	echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
391  *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
392  *
393  * How to check the result of the operation?
394  *
395  * To check disable/enable, see "ras" features at,
396  * /sys/class/drm/card[0/1/2...]/device/ras/features
397  *
398  * To check inject, see the corresponding error count at,
399  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
400  *
401  * .. note::
402  *	Operations are only allowed on blocks which are supported.
403  *	Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
404  *	to see which blocks support RAS on a particular asic.
405  *
406  */
407 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
408 					     const char __user *buf,
409 					     size_t size, loff_t *pos)
410 {
411 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
412 	struct ras_debug_if data;
413 	int ret = 0;
414 
415 	if (!amdgpu_ras_get_error_query_ready(adev)) {
416 		dev_warn(adev->dev, "RAS WARN: error injection "
417 				"currently inaccessible\n");
418 		return size;
419 	}
420 
421 	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
422 	if (ret)
423 		return ret;
424 
425 	if (data.op == 3) {
426 		ret = amdgpu_reserve_page_direct(adev, data.inject.address);
427 		if (!ret)
428 			return size;
429 		else
430 			return ret;
431 	}
432 
433 	if (!amdgpu_ras_is_supported(adev, data.head.block))
434 		return -EINVAL;
435 
436 	switch (data.op) {
437 	case 0:
438 		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
439 		break;
440 	case 1:
441 		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
442 		break;
443 	case 2:
444 		if ((data.inject.address >= adev->gmc.mc_vram_size) ||
445 		    (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
446 			dev_warn(adev->dev, "RAS WARN: input address "
447 					"0x%llx is invalid.",
448 					data.inject.address);
449 			ret = -EINVAL;
450 			break;
451 		}
452 
453 		/* umc ce/ue error injection for a bad page is not allowed */
454 		if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
455 		    amdgpu_ras_check_bad_page(adev, data.inject.address)) {
456 			dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
457 				 "already been marked as bad!\n",
458 				 data.inject.address);
459 			break;
460 		}
461 
462 		/* data.inject.address is offset instead of absolute gpu address */
463 		ret = amdgpu_ras_error_inject(adev, &data.inject);
464 		break;
465 	default:
466 		ret = -EINVAL;
467 		break;
468 	}
469 
470 	if (ret)
471 		return ret;
472 
473 	return size;
474 }
475 
476 /**
477  * DOC: AMDGPU RAS debugfs EEPROM table reset interface
478  *
479  * Some boards contain an EEPROM which is used to persistently store a list of
480  * bad pages which experiences ECC errors in vram.  This interface provides
481  * a way to reset the EEPROM, e.g., after testing error injection.
482  *
483  * Usage:
484  *
485  * .. code-block:: bash
486  *
487  *	echo 1 > ../ras/ras_eeprom_reset
488  *
489  * will reset EEPROM table to 0 entries.
490  *
491  */
492 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
493 					       const char __user *buf,
494 					       size_t size, loff_t *pos)
495 {
496 	struct amdgpu_device *adev =
497 		(struct amdgpu_device *)file_inode(f)->i_private;
498 	int ret;
499 
500 	ret = amdgpu_ras_eeprom_reset_table(
501 		&(amdgpu_ras_get_context(adev)->eeprom_control));
502 
503 	if (!ret) {
504 		/* Something was written to EEPROM.
505 		 */
506 		amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
507 		return size;
508 	} else {
509 		return ret;
510 	}
511 }
512 
513 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
514 	.owner = THIS_MODULE,
515 	.read = NULL,
516 	.write = amdgpu_ras_debugfs_ctrl_write,
517 	.llseek = default_llseek
518 };
519 
520 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
521 	.owner = THIS_MODULE,
522 	.read = NULL,
523 	.write = amdgpu_ras_debugfs_eeprom_write,
524 	.llseek = default_llseek
525 };
526 
527 /**
528  * DOC: AMDGPU RAS sysfs Error Count Interface
529  *
530  * It allows the user to read the error count for each IP block on the gpu through
531  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
532  *
533  * It outputs the multiple lines which report the uncorrected (ue) and corrected
534  * (ce) error counts.
535  *
536  * The format of one line is below,
537  *
538  * [ce|ue]: count
539  *
540  * Example:
541  *
542  * .. code-block:: bash
543  *
544  *	ue: 0
545  *	ce: 1
546  *
547  */
548 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
549 		struct device_attribute *attr, char *buf)
550 {
551 	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
552 	struct ras_query_if info = {
553 		.head = obj->head,
554 	};
555 
556 	if (!amdgpu_ras_get_error_query_ready(obj->adev))
557 		return sysfs_emit(buf, "Query currently inaccessible\n");
558 
559 	if (amdgpu_ras_query_error_status(obj->adev, &info))
560 		return -EINVAL;
561 
562 	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
563 	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
564 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
565 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
566 	}
567 
568 	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
569 			  "ce", info.ce_count);
570 }
571 
572 /* obj begin */
573 
574 #define get_obj(obj) do { (obj)->use++; } while (0)
575 #define alive_obj(obj) ((obj)->use)
576 
577 static inline void put_obj(struct ras_manager *obj)
578 {
579 	if (obj && (--obj->use == 0))
580 		list_del(&obj->node);
581 	if (obj && (obj->use < 0))
582 		DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
583 }
584 
585 /* make one obj and return it. */
586 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
587 		struct ras_common_if *head)
588 {
589 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
590 	struct ras_manager *obj;
591 
592 	if (!adev->ras_enabled || !con)
593 		return NULL;
594 
595 	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
596 		return NULL;
597 
598 	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
599 		if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
600 			return NULL;
601 
602 		obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
603 	} else
604 		obj = &con->objs[head->block];
605 
606 	/* already exist. return obj? */
607 	if (alive_obj(obj))
608 		return NULL;
609 
610 	obj->head = *head;
611 	obj->adev = adev;
612 	list_add(&obj->node, &con->head);
613 	get_obj(obj);
614 
615 	return obj;
616 }
617 
618 /* return an obj equal to head, or the first when head is NULL */
619 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
620 		struct ras_common_if *head)
621 {
622 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
623 	struct ras_manager *obj;
624 	int i;
625 
626 	if (!adev->ras_enabled || !con)
627 		return NULL;
628 
629 	if (head) {
630 		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
631 			return NULL;
632 
633 		if (head->block == AMDGPU_RAS_BLOCK__MCA) {
634 			if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
635 				return NULL;
636 
637 			obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
638 		} else
639 			obj = &con->objs[head->block];
640 
641 		if (alive_obj(obj))
642 			return obj;
643 	} else {
644 		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
645 			obj = &con->objs[i];
646 			if (alive_obj(obj))
647 				return obj;
648 		}
649 	}
650 
651 	return NULL;
652 }
653 /* obj end */
654 
655 /* feature ctl begin */
656 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
657 					 struct ras_common_if *head)
658 {
659 	return adev->ras_hw_enabled & BIT(head->block);
660 }
661 
662 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
663 		struct ras_common_if *head)
664 {
665 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
666 
667 	return con->features & BIT(head->block);
668 }
669 
670 /*
671  * if obj is not created, then create one.
672  * set feature enable flag.
673  */
674 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
675 		struct ras_common_if *head, int enable)
676 {
677 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
678 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
679 
680 	/* If hardware does not support ras, then do not create obj.
681 	 * But if hardware support ras, we can create the obj.
682 	 * Ras framework checks con->hw_supported to see if it need do
683 	 * corresponding initialization.
684 	 * IP checks con->support to see if it need disable ras.
685 	 */
686 	if (!amdgpu_ras_is_feature_allowed(adev, head))
687 		return 0;
688 
689 	if (enable) {
690 		if (!obj) {
691 			obj = amdgpu_ras_create_obj(adev, head);
692 			if (!obj)
693 				return -EINVAL;
694 		} else {
695 			/* In case we create obj somewhere else */
696 			get_obj(obj);
697 		}
698 		con->features |= BIT(head->block);
699 	} else {
700 		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
701 			con->features &= ~BIT(head->block);
702 			put_obj(obj);
703 		}
704 	}
705 
706 	return 0;
707 }
708 
709 /* wrapper of psp_ras_enable_features */
710 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
711 		struct ras_common_if *head, bool enable)
712 {
713 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
714 	union ta_ras_cmd_input *info;
715 	int ret;
716 
717 	if (!con)
718 		return -EINVAL;
719 
720 	if (head->block == AMDGPU_RAS_BLOCK__GFX) {
721 		info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
722 		if (!info)
723 			return -ENOMEM;
724 
725 		if (!enable) {
726 			info->disable_features = (struct ta_ras_disable_features_input) {
727 				.block_id =  amdgpu_ras_block_to_ta(head->block),
728 				.error_type = amdgpu_ras_error_to_ta(head->type),
729 			};
730 		} else {
731 			info->enable_features = (struct ta_ras_enable_features_input) {
732 				.block_id =  amdgpu_ras_block_to_ta(head->block),
733 				.error_type = amdgpu_ras_error_to_ta(head->type),
734 			};
735 		}
736 	}
737 
738 	/* Do not enable if it is not allowed. */
739 	WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
740 
741 	/* Only enable ras feature operation handle on host side */
742 	if (head->block == AMDGPU_RAS_BLOCK__GFX &&
743 		!amdgpu_sriov_vf(adev) &&
744 		!amdgpu_ras_intr_triggered()) {
745 		ret = psp_ras_enable_features(&adev->psp, info, enable);
746 		if (ret) {
747 			dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
748 				enable ? "enable":"disable",
749 				get_ras_block_str(head),
750 				amdgpu_ras_is_poison_mode_supported(adev), ret);
751 			goto out;
752 		}
753 	}
754 
755 	/* setup the obj */
756 	__amdgpu_ras_feature_enable(adev, head, enable);
757 	ret = 0;
758 out:
759 	if (head->block == AMDGPU_RAS_BLOCK__GFX)
760 		kfree(info);
761 	return ret;
762 }
763 
764 /* Only used in device probe stage and called only once. */
765 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
766 		struct ras_common_if *head, bool enable)
767 {
768 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
769 	int ret;
770 
771 	if (!con)
772 		return -EINVAL;
773 
774 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
775 		if (enable) {
776 			/* There is no harm to issue a ras TA cmd regardless of
777 			 * the currecnt ras state.
778 			 * If current state == target state, it will do nothing
779 			 * But sometimes it requests driver to reset and repost
780 			 * with error code -EAGAIN.
781 			 */
782 			ret = amdgpu_ras_feature_enable(adev, head, 1);
783 			/* With old ras TA, we might fail to enable ras.
784 			 * Log it and just setup the object.
785 			 * TODO need remove this WA in the future.
786 			 */
787 			if (ret == -EINVAL) {
788 				ret = __amdgpu_ras_feature_enable(adev, head, 1);
789 				if (!ret)
790 					dev_info(adev->dev,
791 						"RAS INFO: %s setup object\n",
792 						get_ras_block_str(head));
793 			}
794 		} else {
795 			/* setup the object then issue a ras TA disable cmd.*/
796 			ret = __amdgpu_ras_feature_enable(adev, head, 1);
797 			if (ret)
798 				return ret;
799 
800 			/* gfx block ras dsiable cmd must send to ras-ta */
801 			if (head->block == AMDGPU_RAS_BLOCK__GFX)
802 				con->features |= BIT(head->block);
803 
804 			ret = amdgpu_ras_feature_enable(adev, head, 0);
805 
806 			/* clean gfx block ras features flag */
807 			if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
808 				con->features &= ~BIT(head->block);
809 		}
810 	} else
811 		ret = amdgpu_ras_feature_enable(adev, head, enable);
812 
813 	return ret;
814 }
815 
816 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
817 		bool bypass)
818 {
819 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
820 	struct ras_manager *obj, *tmp;
821 
822 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
823 		/* bypass psp.
824 		 * aka just release the obj and corresponding flags
825 		 */
826 		if (bypass) {
827 			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
828 				break;
829 		} else {
830 			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
831 				break;
832 		}
833 	}
834 
835 	return con->features;
836 }
837 
838 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
839 		bool bypass)
840 {
841 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
842 	int i;
843 	const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
844 
845 	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
846 		struct ras_common_if head = {
847 			.block = i,
848 			.type = default_ras_type,
849 			.sub_block_index = 0,
850 		};
851 
852 		if (i == AMDGPU_RAS_BLOCK__MCA)
853 			continue;
854 
855 		if (bypass) {
856 			/*
857 			 * bypass psp. vbios enable ras for us.
858 			 * so just create the obj
859 			 */
860 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
861 				break;
862 		} else {
863 			if (amdgpu_ras_feature_enable(adev, &head, 1))
864 				break;
865 		}
866 	}
867 
868 	for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
869 		struct ras_common_if head = {
870 			.block = AMDGPU_RAS_BLOCK__MCA,
871 			.type = default_ras_type,
872 			.sub_block_index = i,
873 		};
874 
875 		if (bypass) {
876 			/*
877 			 * bypass psp. vbios enable ras for us.
878 			 * so just create the obj
879 			 */
880 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
881 				break;
882 		} else {
883 			if (amdgpu_ras_feature_enable(adev, &head, 1))
884 				break;
885 		}
886 	}
887 
888 	return con->features;
889 }
890 /* feature ctl end */
891 
892 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
893 		enum amdgpu_ras_block block)
894 {
895 	if (!block_obj)
896 		return -EINVAL;
897 
898 	if (block_obj->ras_comm.block == block)
899 		return 0;
900 
901 	return -EINVAL;
902 }
903 
904 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
905 					enum amdgpu_ras_block block, uint32_t sub_block_index)
906 {
907 	struct amdgpu_ras_block_list *node, *tmp;
908 	struct amdgpu_ras_block_object *obj;
909 
910 	if (block >= AMDGPU_RAS_BLOCK__LAST)
911 		return NULL;
912 
913 	if (!amdgpu_ras_is_supported(adev, block))
914 		return NULL;
915 
916 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
917 		if (!node->ras_obj) {
918 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
919 			continue;
920 		}
921 
922 		obj = node->ras_obj;
923 		if (obj->ras_block_match) {
924 			if (obj->ras_block_match(obj, block, sub_block_index) == 0)
925 				return obj;
926 		} else {
927 			if (amdgpu_ras_block_match_default(obj, block) == 0)
928 				return obj;
929 		}
930 	}
931 
932 	return NULL;
933 }
934 
935 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
936 {
937 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
938 	int ret = 0;
939 
940 	/*
941 	 * choosing right query method according to
942 	 * whether smu support query error information
943 	 */
944 	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
945 	if (ret == -EOPNOTSUPP) {
946 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
947 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
948 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
949 
950 		/* umc query_ras_error_address is also responsible for clearing
951 		 * error status
952 		 */
953 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
954 		    adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
955 			adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
956 	} else if (!ret) {
957 		if (adev->umc.ras &&
958 			adev->umc.ras->ecc_info_query_ras_error_count)
959 			adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
960 
961 		if (adev->umc.ras &&
962 			adev->umc.ras->ecc_info_query_ras_error_address)
963 			adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
964 	}
965 }
966 
967 /* query/inject/cure begin */
968 int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
969 				  struct ras_query_if *info)
970 {
971 	struct amdgpu_ras_block_object *block_obj = NULL;
972 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
973 	struct ras_err_data err_data = {0, 0, 0, NULL};
974 
975 	if (!obj)
976 		return -EINVAL;
977 
978 	if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
979 		amdgpu_ras_get_ecc_info(adev, &err_data);
980 	} else {
981 		block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
982 		if (!block_obj || !block_obj->hw_ops)   {
983 			dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
984 				     get_ras_block_str(&info->head));
985 			return -EINVAL;
986 		}
987 
988 		if (block_obj->hw_ops->query_ras_error_count)
989 			block_obj->hw_ops->query_ras_error_count(adev, &err_data);
990 
991 		if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
992 		    (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
993 		    (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
994 				if (block_obj->hw_ops->query_ras_error_status)
995 					block_obj->hw_ops->query_ras_error_status(adev);
996 			}
997 	}
998 
999 	obj->err_data.ue_count += err_data.ue_count;
1000 	obj->err_data.ce_count += err_data.ce_count;
1001 
1002 	info->ue_count = obj->err_data.ue_count;
1003 	info->ce_count = obj->err_data.ce_count;
1004 
1005 	if (err_data.ce_count) {
1006 		if (adev->smuio.funcs &&
1007 		    adev->smuio.funcs->get_socket_id &&
1008 		    adev->smuio.funcs->get_die_id) {
1009 			dev_info(adev->dev, "socket: %d, die: %d "
1010 					"%ld correctable hardware errors "
1011 					"detected in %s block, no user "
1012 					"action is needed.\n",
1013 					adev->smuio.funcs->get_socket_id(adev),
1014 					adev->smuio.funcs->get_die_id(adev),
1015 					obj->err_data.ce_count,
1016 					get_ras_block_str(&info->head));
1017 		} else {
1018 			dev_info(adev->dev, "%ld correctable hardware errors "
1019 					"detected in %s block, no user "
1020 					"action is needed.\n",
1021 					obj->err_data.ce_count,
1022 					get_ras_block_str(&info->head));
1023 		}
1024 	}
1025 	if (err_data.ue_count) {
1026 		if (adev->smuio.funcs &&
1027 		    adev->smuio.funcs->get_socket_id &&
1028 		    adev->smuio.funcs->get_die_id) {
1029 			dev_info(adev->dev, "socket: %d, die: %d "
1030 					"%ld uncorrectable hardware errors "
1031 					"detected in %s block\n",
1032 					adev->smuio.funcs->get_socket_id(adev),
1033 					adev->smuio.funcs->get_die_id(adev),
1034 					obj->err_data.ue_count,
1035 					get_ras_block_str(&info->head));
1036 		} else {
1037 			dev_info(adev->dev, "%ld uncorrectable hardware errors "
1038 					"detected in %s block\n",
1039 					obj->err_data.ue_count,
1040 					get_ras_block_str(&info->head));
1041 		}
1042 	}
1043 
1044 	return 0;
1045 }
1046 
1047 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1048 		enum amdgpu_ras_block block)
1049 {
1050 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1051 
1052 	if (!amdgpu_ras_is_supported(adev, block))
1053 		return -EINVAL;
1054 
1055 	if (!block_obj || !block_obj->hw_ops)   {
1056 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1057 			     ras_block_str(block));
1058 		return -EINVAL;
1059 	}
1060 
1061 	if (block_obj->hw_ops->reset_ras_error_count)
1062 		block_obj->hw_ops->reset_ras_error_count(adev);
1063 
1064 	if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1065 	    (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1066 		if (block_obj->hw_ops->reset_ras_error_status)
1067 			block_obj->hw_ops->reset_ras_error_status(adev);
1068 	}
1069 
1070 	return 0;
1071 }
1072 
1073 /* wrapper of psp_ras_trigger_error */
1074 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1075 		struct ras_inject_if *info)
1076 {
1077 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1078 	struct ta_ras_trigger_error_input block_info = {
1079 		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
1080 		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1081 		.sub_block_index = info->head.sub_block_index,
1082 		.address = info->address,
1083 		.value = info->value,
1084 	};
1085 	int ret = -EINVAL;
1086 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1087 							info->head.block,
1088 							info->head.sub_block_index);
1089 
1090 	if (!obj)
1091 		return -EINVAL;
1092 
1093 	if (!block_obj || !block_obj->hw_ops)	{
1094 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1095 			     get_ras_block_str(&info->head));
1096 		return -EINVAL;
1097 	}
1098 
1099 	/* Calculate XGMI relative offset */
1100 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
1101 		block_info.address =
1102 			amdgpu_xgmi_get_relative_phy_addr(adev,
1103 							  block_info.address);
1104 	}
1105 
1106 	if (info->head.block == AMDGPU_RAS_BLOCK__GFX) {
1107 		if (block_obj->hw_ops->ras_error_inject)
1108 			ret = block_obj->hw_ops->ras_error_inject(adev, info);
1109 	} else {
1110 		/* If defined special ras_error_inject(e.g: xgmi), implement special ras_error_inject */
1111 		if (block_obj->hw_ops->ras_error_inject)
1112 			ret = block_obj->hw_ops->ras_error_inject(adev, &block_info);
1113 		else  /*If not defined .ras_error_inject, use default ras_error_inject*/
1114 			ret = psp_ras_trigger_error(&adev->psp, &block_info);
1115 	}
1116 
1117 	if (ret)
1118 		dev_err(adev->dev, "ras inject %s failed %d\n",
1119 			get_ras_block_str(&info->head), ret);
1120 
1121 	return ret;
1122 }
1123 
1124 /**
1125  * amdgpu_ras_query_error_count -- Get error counts of all IPs
1126  * @adev: pointer to AMD GPU device
1127  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1128  * @ue_count: pointer to an integer to be set to the count of uncorrectible
1129  * errors.
1130  *
1131  * If set, @ce_count or @ue_count, count and return the corresponding
1132  * error counts in those integer pointers. Return 0 if the device
1133  * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1134  */
1135 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1136 				 unsigned long *ce_count,
1137 				 unsigned long *ue_count)
1138 {
1139 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1140 	struct ras_manager *obj;
1141 	unsigned long ce, ue;
1142 
1143 	if (!adev->ras_enabled || !con)
1144 		return -EOPNOTSUPP;
1145 
1146 	/* Don't count since no reporting.
1147 	 */
1148 	if (!ce_count && !ue_count)
1149 		return 0;
1150 
1151 	ce = 0;
1152 	ue = 0;
1153 	list_for_each_entry(obj, &con->head, node) {
1154 		struct ras_query_if info = {
1155 			.head = obj->head,
1156 		};
1157 		int res;
1158 
1159 		res = amdgpu_ras_query_error_status(adev, &info);
1160 		if (res)
1161 			return res;
1162 
1163 		if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1164 		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1165 			if (amdgpu_ras_reset_error_status(adev, info.head.block))
1166 				dev_warn(adev->dev, "Failed to reset error counter and error status");
1167 		}
1168 
1169 		ce += info.ce_count;
1170 		ue += info.ue_count;
1171 	}
1172 
1173 	if (ce_count)
1174 		*ce_count = ce;
1175 
1176 	if (ue_count)
1177 		*ue_count = ue;
1178 
1179 	return 0;
1180 }
1181 /* query/inject/cure end */
1182 
1183 
1184 /* sysfs begin */
1185 
1186 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1187 		struct ras_badpage **bps, unsigned int *count);
1188 
1189 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1190 {
1191 	switch (flags) {
1192 	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1193 		return "R";
1194 	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1195 		return "P";
1196 	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1197 	default:
1198 		return "F";
1199 	}
1200 }
1201 
1202 /**
1203  * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1204  *
1205  * It allows user to read the bad pages of vram on the gpu through
1206  * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1207  *
1208  * It outputs multiple lines, and each line stands for one gpu page.
1209  *
1210  * The format of one line is below,
1211  * gpu pfn : gpu page size : flags
1212  *
1213  * gpu pfn and gpu page size are printed in hex format.
1214  * flags can be one of below character,
1215  *
1216  * R: reserved, this gpu page is reserved and not able to use.
1217  *
1218  * P: pending for reserve, this gpu page is marked as bad, will be reserved
1219  * in next window of page_reserve.
1220  *
1221  * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1222  *
1223  * Examples:
1224  *
1225  * .. code-block:: bash
1226  *
1227  *	0x00000001 : 0x00001000 : R
1228  *	0x00000002 : 0x00001000 : P
1229  *
1230  */
1231 
1232 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1233 		struct kobject *kobj, struct bin_attribute *attr,
1234 		char *buf, loff_t ppos, size_t count)
1235 {
1236 	struct amdgpu_ras *con =
1237 		container_of(attr, struct amdgpu_ras, badpages_attr);
1238 	struct amdgpu_device *adev = con->adev;
1239 	const unsigned int element_size =
1240 		sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1241 	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1242 	unsigned int end = div64_ul(ppos + count - 1, element_size);
1243 	ssize_t s = 0;
1244 	struct ras_badpage *bps = NULL;
1245 	unsigned int bps_count = 0;
1246 
1247 	memset(buf, 0, count);
1248 
1249 	if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1250 		return 0;
1251 
1252 	for (; start < end && start < bps_count; start++)
1253 		s += scnprintf(&buf[s], element_size + 1,
1254 				"0x%08x : 0x%08x : %1s\n",
1255 				bps[start].bp,
1256 				bps[start].size,
1257 				amdgpu_ras_badpage_flags_str(bps[start].flags));
1258 
1259 	kfree(bps);
1260 
1261 	return s;
1262 }
1263 
1264 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1265 		struct device_attribute *attr, char *buf)
1266 {
1267 	struct amdgpu_ras *con =
1268 		container_of(attr, struct amdgpu_ras, features_attr);
1269 
1270 	return scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features);
1271 }
1272 
1273 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1274 {
1275 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1276 
1277 	sysfs_remove_file_from_group(&adev->dev->kobj,
1278 				&con->badpages_attr.attr,
1279 				RAS_FS_NAME);
1280 }
1281 
1282 static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1283 {
1284 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1285 	struct attribute *attrs[] = {
1286 		&con->features_attr.attr,
1287 		NULL
1288 	};
1289 	struct attribute_group group = {
1290 		.name = RAS_FS_NAME,
1291 		.attrs = attrs,
1292 	};
1293 
1294 	sysfs_remove_group(&adev->dev->kobj, &group);
1295 
1296 	return 0;
1297 }
1298 
1299 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1300 		struct ras_common_if *head)
1301 {
1302 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1303 
1304 	if (!obj || obj->attr_inuse)
1305 		return -EINVAL;
1306 
1307 	get_obj(obj);
1308 
1309 	snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1310 		"%s_err_count", head->name);
1311 
1312 	obj->sysfs_attr = (struct device_attribute){
1313 		.attr = {
1314 			.name = obj->fs_data.sysfs_name,
1315 			.mode = S_IRUGO,
1316 		},
1317 			.show = amdgpu_ras_sysfs_read,
1318 	};
1319 	sysfs_attr_init(&obj->sysfs_attr.attr);
1320 
1321 	if (sysfs_add_file_to_group(&adev->dev->kobj,
1322 				&obj->sysfs_attr.attr,
1323 				RAS_FS_NAME)) {
1324 		put_obj(obj);
1325 		return -EINVAL;
1326 	}
1327 
1328 	obj->attr_inuse = 1;
1329 
1330 	return 0;
1331 }
1332 
1333 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1334 		struct ras_common_if *head)
1335 {
1336 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1337 
1338 	if (!obj || !obj->attr_inuse)
1339 		return -EINVAL;
1340 
1341 	sysfs_remove_file_from_group(&adev->dev->kobj,
1342 				&obj->sysfs_attr.attr,
1343 				RAS_FS_NAME);
1344 	obj->attr_inuse = 0;
1345 	put_obj(obj);
1346 
1347 	return 0;
1348 }
1349 
1350 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1351 {
1352 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1353 	struct ras_manager *obj, *tmp;
1354 
1355 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1356 		amdgpu_ras_sysfs_remove(adev, &obj->head);
1357 	}
1358 
1359 	if (amdgpu_bad_page_threshold != 0)
1360 		amdgpu_ras_sysfs_remove_bad_page_node(adev);
1361 
1362 	amdgpu_ras_sysfs_remove_feature_node(adev);
1363 
1364 	return 0;
1365 }
1366 /* sysfs end */
1367 
1368 /**
1369  * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1370  *
1371  * Normally when there is an uncorrectable error, the driver will reset
1372  * the GPU to recover.  However, in the event of an unrecoverable error,
1373  * the driver provides an interface to reboot the system automatically
1374  * in that event.
1375  *
1376  * The following file in debugfs provides that interface:
1377  * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1378  *
1379  * Usage:
1380  *
1381  * .. code-block:: bash
1382  *
1383  *	echo true > .../ras/auto_reboot
1384  *
1385  */
1386 /* debugfs begin */
1387 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1388 {
1389 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1390 	struct drm_minor  *minor = adev_to_drm(adev)->primary;
1391 	struct dentry     *dir;
1392 
1393 	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1394 	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1395 			    &amdgpu_ras_debugfs_ctrl_ops);
1396 	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1397 			    &amdgpu_ras_debugfs_eeprom_ops);
1398 	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1399 			   &con->bad_page_cnt_threshold);
1400 	debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1401 	debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1402 	debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1403 			    &amdgpu_ras_debugfs_eeprom_size_ops);
1404 	con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1405 						       S_IRUGO, dir, adev,
1406 						       &amdgpu_ras_debugfs_eeprom_table_ops);
1407 	amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1408 
1409 	/*
1410 	 * After one uncorrectable error happens, usually GPU recovery will
1411 	 * be scheduled. But due to the known problem in GPU recovery failing
1412 	 * to bring GPU back, below interface provides one direct way to
1413 	 * user to reboot system automatically in such case within
1414 	 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1415 	 * will never be called.
1416 	 */
1417 	debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1418 
1419 	/*
1420 	 * User could set this not to clean up hardware's error count register
1421 	 * of RAS IPs during ras recovery.
1422 	 */
1423 	debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1424 			    &con->disable_ras_err_cnt_harvest);
1425 	return dir;
1426 }
1427 
1428 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1429 				      struct ras_fs_if *head,
1430 				      struct dentry *dir)
1431 {
1432 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1433 
1434 	if (!obj || !dir)
1435 		return;
1436 
1437 	get_obj(obj);
1438 
1439 	memcpy(obj->fs_data.debugfs_name,
1440 			head->debugfs_name,
1441 			sizeof(obj->fs_data.debugfs_name));
1442 
1443 	debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1444 			    obj, &amdgpu_ras_debugfs_ops);
1445 }
1446 
1447 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1448 {
1449 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1450 	struct dentry *dir;
1451 	struct ras_manager *obj;
1452 	struct ras_fs_if fs_info;
1453 
1454 	/*
1455 	 * it won't be called in resume path, no need to check
1456 	 * suspend and gpu reset status
1457 	 */
1458 	if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
1459 		return;
1460 
1461 	dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
1462 
1463 	list_for_each_entry(obj, &con->head, node) {
1464 		if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1465 			(obj->attr_inuse == 1)) {
1466 			sprintf(fs_info.debugfs_name, "%s_err_inject",
1467 					get_ras_block_str(&obj->head));
1468 			fs_info.head = obj->head;
1469 			amdgpu_ras_debugfs_create(adev, &fs_info, dir);
1470 		}
1471 	}
1472 }
1473 
1474 /* debugfs end */
1475 
1476 /* ras fs */
1477 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1478 		amdgpu_ras_sysfs_badpages_read, NULL, 0);
1479 static DEVICE_ATTR(features, S_IRUGO,
1480 		amdgpu_ras_sysfs_features_read, NULL);
1481 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1482 {
1483 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1484 	struct attribute_group group = {
1485 		.name = RAS_FS_NAME,
1486 	};
1487 	struct attribute *attrs[] = {
1488 		&con->features_attr.attr,
1489 		NULL
1490 	};
1491 	struct bin_attribute *bin_attrs[] = {
1492 		NULL,
1493 		NULL,
1494 	};
1495 	int r;
1496 
1497 	/* add features entry */
1498 	con->features_attr = dev_attr_features;
1499 	group.attrs = attrs;
1500 	sysfs_attr_init(attrs[0]);
1501 
1502 	if (amdgpu_bad_page_threshold != 0) {
1503 		/* add bad_page_features entry */
1504 		bin_attr_gpu_vram_bad_pages.private = NULL;
1505 		con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1506 		bin_attrs[0] = &con->badpages_attr;
1507 		group.bin_attrs = bin_attrs;
1508 		sysfs_bin_attr_init(bin_attrs[0]);
1509 	}
1510 
1511 	r = sysfs_create_group(&adev->dev->kobj, &group);
1512 	if (r)
1513 		dev_err(adev->dev, "Failed to create RAS sysfs group!");
1514 
1515 	return 0;
1516 }
1517 
1518 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1519 {
1520 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1521 	struct ras_manager *con_obj, *ip_obj, *tmp;
1522 
1523 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1524 		list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1525 			ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1526 			if (ip_obj)
1527 				put_obj(ip_obj);
1528 		}
1529 	}
1530 
1531 	amdgpu_ras_sysfs_remove_all(adev);
1532 	return 0;
1533 }
1534 /* ras fs end */
1535 
1536 /* ih begin */
1537 
1538 /* For the hardware that cannot enable bif ring for both ras_controller_irq
1539  * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1540  * register to check whether the interrupt is triggered or not, and properly
1541  * ack the interrupt if it is there
1542  */
1543 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1544 {
1545 	/* Fatal error events are handled on host side */
1546 	if (amdgpu_sriov_vf(adev) ||
1547 		!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF))
1548 		return;
1549 
1550 	if (adev->nbio.ras &&
1551 	    adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1552 		adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1553 
1554 	if (adev->nbio.ras &&
1555 	    adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1556 		adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1557 }
1558 
1559 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1560 				struct amdgpu_iv_entry *entry)
1561 {
1562 	bool poison_stat = false;
1563 	struct amdgpu_device *adev = obj->adev;
1564 	struct ras_err_data err_data = {0, 0, 0, NULL};
1565 	struct amdgpu_ras_block_object *block_obj =
1566 		amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1567 
1568 	if (!block_obj || !block_obj->hw_ops)
1569 		return;
1570 
1571 	/* both query_poison_status and handle_poison_consumption are optional,
1572 	 * but at least one of them should be implemented if we need poison
1573 	 * consumption handler
1574 	 */
1575 	if (block_obj->hw_ops->query_poison_status) {
1576 		poison_stat = block_obj->hw_ops->query_poison_status(adev);
1577 		if (!poison_stat) {
1578 			/* Not poison consumption interrupt, no need to handle it */
1579 			dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1580 					block_obj->ras_comm.name);
1581 
1582 			return;
1583 		}
1584 	}
1585 
1586 	if (!adev->gmc.xgmi.connected_to_cpu)
1587 		amdgpu_umc_poison_handler(adev, &err_data, false);
1588 
1589 	if (block_obj->hw_ops->handle_poison_consumption)
1590 		poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1591 
1592 	/* gpu reset is fallback for failed and default cases */
1593 	if (poison_stat) {
1594 		dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1595 				block_obj->ras_comm.name);
1596 		amdgpu_ras_reset_gpu(adev);
1597 	}
1598 }
1599 
1600 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1601 				struct amdgpu_iv_entry *entry)
1602 {
1603 	dev_info(obj->adev->dev,
1604 		"Poison is created, no user action is needed.\n");
1605 }
1606 
1607 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1608 				struct amdgpu_iv_entry *entry)
1609 {
1610 	struct ras_ih_data *data = &obj->ih_data;
1611 	struct ras_err_data err_data = {0, 0, 0, NULL};
1612 	int ret;
1613 
1614 	if (!data->cb)
1615 		return;
1616 
1617 	/* Let IP handle its data, maybe we need get the output
1618 	 * from the callback to update the error type/count, etc
1619 	 */
1620 	ret = data->cb(obj->adev, &err_data, entry);
1621 	/* ue will trigger an interrupt, and in that case
1622 	 * we need do a reset to recovery the whole system.
1623 	 * But leave IP do that recovery, here we just dispatch
1624 	 * the error.
1625 	 */
1626 	if (ret == AMDGPU_RAS_SUCCESS) {
1627 		/* these counts could be left as 0 if
1628 		 * some blocks do not count error number
1629 		 */
1630 		obj->err_data.ue_count += err_data.ue_count;
1631 		obj->err_data.ce_count += err_data.ce_count;
1632 	}
1633 }
1634 
1635 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1636 {
1637 	struct ras_ih_data *data = &obj->ih_data;
1638 	struct amdgpu_iv_entry entry;
1639 
1640 	while (data->rptr != data->wptr) {
1641 		rmb();
1642 		memcpy(&entry, &data->ring[data->rptr],
1643 				data->element_size);
1644 
1645 		wmb();
1646 		data->rptr = (data->aligned_element_size +
1647 				data->rptr) % data->ring_size;
1648 
1649 		if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1650 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1651 				amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
1652 			else
1653 				amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
1654 		} else {
1655 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1656 				amdgpu_ras_interrupt_umc_handler(obj, &entry);
1657 			else
1658 				dev_warn(obj->adev->dev,
1659 					"No RAS interrupt handler for non-UMC block with poison disabled.\n");
1660 		}
1661 	}
1662 }
1663 
1664 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1665 {
1666 	struct ras_ih_data *data =
1667 		container_of(work, struct ras_ih_data, ih_work);
1668 	struct ras_manager *obj =
1669 		container_of(data, struct ras_manager, ih_data);
1670 
1671 	amdgpu_ras_interrupt_handler(obj);
1672 }
1673 
1674 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1675 		struct ras_dispatch_if *info)
1676 {
1677 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1678 	struct ras_ih_data *data = &obj->ih_data;
1679 
1680 	if (!obj)
1681 		return -EINVAL;
1682 
1683 	if (data->inuse == 0)
1684 		return 0;
1685 
1686 	/* Might be overflow... */
1687 	memcpy(&data->ring[data->wptr], info->entry,
1688 			data->element_size);
1689 
1690 	wmb();
1691 	data->wptr = (data->aligned_element_size +
1692 			data->wptr) % data->ring_size;
1693 
1694 	schedule_work(&data->ih_work);
1695 
1696 	return 0;
1697 }
1698 
1699 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
1700 		struct ras_common_if *head)
1701 {
1702 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1703 	struct ras_ih_data *data;
1704 
1705 	if (!obj)
1706 		return -EINVAL;
1707 
1708 	data = &obj->ih_data;
1709 	if (data->inuse == 0)
1710 		return 0;
1711 
1712 	cancel_work_sync(&data->ih_work);
1713 
1714 	kfree(data->ring);
1715 	memset(data, 0, sizeof(*data));
1716 	put_obj(obj);
1717 
1718 	return 0;
1719 }
1720 
1721 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
1722 		struct ras_common_if *head)
1723 {
1724 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1725 	struct ras_ih_data *data;
1726 	struct amdgpu_ras_block_object *ras_obj;
1727 
1728 	if (!obj) {
1729 		/* in case we registe the IH before enable ras feature */
1730 		obj = amdgpu_ras_create_obj(adev, head);
1731 		if (!obj)
1732 			return -EINVAL;
1733 	} else
1734 		get_obj(obj);
1735 
1736 	ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1737 
1738 	data = &obj->ih_data;
1739 	/* add the callback.etc */
1740 	*data = (struct ras_ih_data) {
1741 		.inuse = 0,
1742 		.cb = ras_obj->ras_cb,
1743 		.element_size = sizeof(struct amdgpu_iv_entry),
1744 		.rptr = 0,
1745 		.wptr = 0,
1746 	};
1747 
1748 	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1749 
1750 	data->aligned_element_size = ALIGN(data->element_size, 8);
1751 	/* the ring can store 64 iv entries. */
1752 	data->ring_size = 64 * data->aligned_element_size;
1753 	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1754 	if (!data->ring) {
1755 		put_obj(obj);
1756 		return -ENOMEM;
1757 	}
1758 
1759 	/* IH is ready */
1760 	data->inuse = 1;
1761 
1762 	return 0;
1763 }
1764 
1765 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1766 {
1767 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1768 	struct ras_manager *obj, *tmp;
1769 
1770 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1771 		amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
1772 	}
1773 
1774 	return 0;
1775 }
1776 /* ih end */
1777 
1778 /* traversal all IPs except NBIO to query error counter */
1779 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1780 {
1781 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1782 	struct ras_manager *obj;
1783 
1784 	if (!adev->ras_enabled || !con)
1785 		return;
1786 
1787 	list_for_each_entry(obj, &con->head, node) {
1788 		struct ras_query_if info = {
1789 			.head = obj->head,
1790 		};
1791 
1792 		/*
1793 		 * PCIE_BIF IP has one different isr by ras controller
1794 		 * interrupt, the specific ras counter query will be
1795 		 * done in that isr. So skip such block from common
1796 		 * sync flood interrupt isr calling.
1797 		 */
1798 		if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1799 			continue;
1800 
1801 		/*
1802 		 * this is a workaround for aldebaran, skip send msg to
1803 		 * smu to get ecc_info table due to smu handle get ecc
1804 		 * info table failed temporarily.
1805 		 * should be removed until smu fix handle ecc_info table.
1806 		 */
1807 		if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1808 			(adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1809 			continue;
1810 
1811 		amdgpu_ras_query_error_status(adev, &info);
1812 
1813 		if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1814 		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1815 		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
1816 			if (amdgpu_ras_reset_error_status(adev, info.head.block))
1817 				dev_warn(adev->dev, "Failed to reset error counter and error status");
1818 		}
1819 	}
1820 }
1821 
1822 /* Parse RdRspStatus and WrRspStatus */
1823 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1824 					  struct ras_query_if *info)
1825 {
1826 	struct amdgpu_ras_block_object *block_obj;
1827 	/*
1828 	 * Only two block need to query read/write
1829 	 * RspStatus at current state
1830 	 */
1831 	if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1832 		(info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
1833 		return;
1834 
1835 	block_obj = amdgpu_ras_get_ras_block(adev,
1836 					info->head.block,
1837 					info->head.sub_block_index);
1838 
1839 	if (!block_obj || !block_obj->hw_ops) {
1840 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1841 			     get_ras_block_str(&info->head));
1842 		return;
1843 	}
1844 
1845 	if (block_obj->hw_ops->query_ras_error_status)
1846 		block_obj->hw_ops->query_ras_error_status(adev);
1847 
1848 }
1849 
1850 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1851 {
1852 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1853 	struct ras_manager *obj;
1854 
1855 	if (!adev->ras_enabled || !con)
1856 		return;
1857 
1858 	list_for_each_entry(obj, &con->head, node) {
1859 		struct ras_query_if info = {
1860 			.head = obj->head,
1861 		};
1862 
1863 		amdgpu_ras_error_status_query(adev, &info);
1864 	}
1865 }
1866 
1867 /* recovery begin */
1868 
1869 /* return 0 on success.
1870  * caller need free bps.
1871  */
1872 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1873 		struct ras_badpage **bps, unsigned int *count)
1874 {
1875 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1876 	struct ras_err_handler_data *data;
1877 	int i = 0;
1878 	int ret = 0, status;
1879 
1880 	if (!con || !con->eh_data || !bps || !count)
1881 		return -EINVAL;
1882 
1883 	mutex_lock(&con->recovery_lock);
1884 	data = con->eh_data;
1885 	if (!data || data->count == 0) {
1886 		*bps = NULL;
1887 		ret = -EINVAL;
1888 		goto out;
1889 	}
1890 
1891 	*bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1892 	if (!*bps) {
1893 		ret = -ENOMEM;
1894 		goto out;
1895 	}
1896 
1897 	for (; i < data->count; i++) {
1898 		(*bps)[i] = (struct ras_badpage){
1899 			.bp = data->bps[i].retired_page,
1900 			.size = AMDGPU_GPU_PAGE_SIZE,
1901 			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
1902 		};
1903 		status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
1904 				data->bps[i].retired_page);
1905 		if (status == -EBUSY)
1906 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
1907 		else if (status == -ENOENT)
1908 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
1909 	}
1910 
1911 	*count = data->count;
1912 out:
1913 	mutex_unlock(&con->recovery_lock);
1914 	return ret;
1915 }
1916 
1917 static void amdgpu_ras_do_recovery(struct work_struct *work)
1918 {
1919 	struct amdgpu_ras *ras =
1920 		container_of(work, struct amdgpu_ras, recovery_work);
1921 	struct amdgpu_device *remote_adev = NULL;
1922 	struct amdgpu_device *adev = ras->adev;
1923 	struct list_head device_list, *device_list_handle =  NULL;
1924 
1925 	if (!ras->disable_ras_err_cnt_harvest) {
1926 		struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
1927 
1928 		/* Build list of devices to query RAS related errors */
1929 		if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
1930 			device_list_handle = &hive->device_list;
1931 		} else {
1932 			INIT_LIST_HEAD(&device_list);
1933 			list_add_tail(&adev->gmc.xgmi.head, &device_list);
1934 			device_list_handle = &device_list;
1935 		}
1936 
1937 		list_for_each_entry(remote_adev,
1938 				device_list_handle, gmc.xgmi.head) {
1939 			amdgpu_ras_query_err_status(remote_adev);
1940 			amdgpu_ras_log_on_err_counter(remote_adev);
1941 		}
1942 
1943 		amdgpu_put_xgmi_hive(hive);
1944 	}
1945 
1946 	if (amdgpu_device_should_recover_gpu(ras->adev)) {
1947 		struct amdgpu_reset_context reset_context;
1948 		memset(&reset_context, 0, sizeof(reset_context));
1949 
1950 		reset_context.method = AMD_RESET_METHOD_NONE;
1951 		reset_context.reset_req_dev = adev;
1952 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1953 
1954 		amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
1955 	}
1956 	atomic_set(&ras->in_recovery, 0);
1957 }
1958 
1959 /* alloc/realloc bps array */
1960 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
1961 		struct ras_err_handler_data *data, int pages)
1962 {
1963 	unsigned int old_space = data->count + data->space_left;
1964 	unsigned int new_space = old_space + pages;
1965 	unsigned int align_space = ALIGN(new_space, 512);
1966 	void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
1967 
1968 	if (!bps) {
1969 		return -ENOMEM;
1970 	}
1971 
1972 	if (data->bps) {
1973 		memcpy(bps, data->bps,
1974 				data->count * sizeof(*data->bps));
1975 		kfree(data->bps);
1976 	}
1977 
1978 	data->bps = bps;
1979 	data->space_left += align_space - old_space;
1980 	return 0;
1981 }
1982 
1983 /* it deal with vram only. */
1984 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
1985 		struct eeprom_table_record *bps, int pages)
1986 {
1987 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1988 	struct ras_err_handler_data *data;
1989 	int ret = 0;
1990 	uint32_t i;
1991 
1992 	if (!con || !con->eh_data || !bps || pages <= 0)
1993 		return 0;
1994 
1995 	mutex_lock(&con->recovery_lock);
1996 	data = con->eh_data;
1997 	if (!data)
1998 		goto out;
1999 
2000 	for (i = 0; i < pages; i++) {
2001 		if (amdgpu_ras_check_bad_page_unlock(con,
2002 			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2003 			continue;
2004 
2005 		if (!data->space_left &&
2006 			amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2007 			ret = -ENOMEM;
2008 			goto out;
2009 		}
2010 
2011 		amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
2012 			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2013 			AMDGPU_GPU_PAGE_SIZE);
2014 
2015 		memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2016 		data->count++;
2017 		data->space_left--;
2018 	}
2019 out:
2020 	mutex_unlock(&con->recovery_lock);
2021 
2022 	return ret;
2023 }
2024 
2025 /*
2026  * write error record array to eeprom, the function should be
2027  * protected by recovery_lock
2028  */
2029 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
2030 {
2031 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2032 	struct ras_err_handler_data *data;
2033 	struct amdgpu_ras_eeprom_control *control;
2034 	int save_count;
2035 
2036 	if (!con || !con->eh_data)
2037 		return 0;
2038 
2039 	mutex_lock(&con->recovery_lock);
2040 	control = &con->eeprom_control;
2041 	data = con->eh_data;
2042 	save_count = data->count - control->ras_num_recs;
2043 	mutex_unlock(&con->recovery_lock);
2044 	/* only new entries are saved */
2045 	if (save_count > 0) {
2046 		if (amdgpu_ras_eeprom_append(control,
2047 					     &data->bps[control->ras_num_recs],
2048 					     save_count)) {
2049 			dev_err(adev->dev, "Failed to save EEPROM table data!");
2050 			return -EIO;
2051 		}
2052 
2053 		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2054 	}
2055 
2056 	return 0;
2057 }
2058 
2059 /*
2060  * read error record array in eeprom and reserve enough space for
2061  * storing new bad pages
2062  */
2063 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2064 {
2065 	struct amdgpu_ras_eeprom_control *control =
2066 		&adev->psp.ras_context.ras->eeprom_control;
2067 	struct eeprom_table_record *bps;
2068 	int ret;
2069 
2070 	/* no bad page record, skip eeprom access */
2071 	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
2072 		return 0;
2073 
2074 	bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
2075 	if (!bps)
2076 		return -ENOMEM;
2077 
2078 	ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
2079 	if (ret)
2080 		dev_err(adev->dev, "Failed to load EEPROM table records!");
2081 	else
2082 		ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
2083 
2084 	kfree(bps);
2085 	return ret;
2086 }
2087 
2088 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2089 				uint64_t addr)
2090 {
2091 	struct ras_err_handler_data *data = con->eh_data;
2092 	int i;
2093 
2094 	addr >>= AMDGPU_GPU_PAGE_SHIFT;
2095 	for (i = 0; i < data->count; i++)
2096 		if (addr == data->bps[i].retired_page)
2097 			return true;
2098 
2099 	return false;
2100 }
2101 
2102 /*
2103  * check if an address belongs to bad page
2104  *
2105  * Note: this check is only for umc block
2106  */
2107 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2108 				uint64_t addr)
2109 {
2110 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2111 	bool ret = false;
2112 
2113 	if (!con || !con->eh_data)
2114 		return ret;
2115 
2116 	mutex_lock(&con->recovery_lock);
2117 	ret = amdgpu_ras_check_bad_page_unlock(con, addr);
2118 	mutex_unlock(&con->recovery_lock);
2119 	return ret;
2120 }
2121 
2122 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
2123 					  uint32_t max_count)
2124 {
2125 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2126 
2127 	/*
2128 	 * Justification of value bad_page_cnt_threshold in ras structure
2129 	 *
2130 	 * Generally, -1 <= amdgpu_bad_page_threshold <= max record length
2131 	 * in eeprom, and introduce two scenarios accordingly.
2132 	 *
2133 	 * Bad page retirement enablement:
2134 	 *    - If amdgpu_bad_page_threshold = -1,
2135 	 *      bad_page_cnt_threshold = typical value by formula.
2136 	 *
2137 	 *    - When the value from user is 0 < amdgpu_bad_page_threshold <
2138 	 *      max record length in eeprom, use it directly.
2139 	 *
2140 	 * Bad page retirement disablement:
2141 	 *    - If amdgpu_bad_page_threshold = 0, bad page retirement
2142 	 *      functionality is disabled, and bad_page_cnt_threshold will
2143 	 *      take no effect.
2144 	 */
2145 
2146 	if (amdgpu_bad_page_threshold < 0) {
2147 		u64 val = adev->gmc.mc_vram_size;
2148 
2149 		do_div(val, RAS_BAD_PAGE_COVER);
2150 		con->bad_page_cnt_threshold = min(lower_32_bits(val),
2151 						  max_count);
2152 	} else {
2153 		con->bad_page_cnt_threshold = min_t(int, max_count,
2154 						    amdgpu_bad_page_threshold);
2155 	}
2156 }
2157 
2158 int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
2159 {
2160 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2161 	struct ras_err_handler_data **data;
2162 	u32  max_eeprom_records_count = 0;
2163 	bool exc_err_limit = false;
2164 	int ret;
2165 
2166 	if (!con || amdgpu_sriov_vf(adev))
2167 		return 0;
2168 
2169 	/* Allow access to RAS EEPROM via debugfs, when the ASIC
2170 	 * supports RAS and debugfs is enabled, but when
2171 	 * adev->ras_enabled is unset, i.e. when "ras_enable"
2172 	 * module parameter is set to 0.
2173 	 */
2174 	con->adev = adev;
2175 
2176 	if (!adev->ras_enabled)
2177 		return 0;
2178 
2179 	data = &con->eh_data;
2180 	*data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2181 	if (!*data) {
2182 		ret = -ENOMEM;
2183 		goto out;
2184 	}
2185 
2186 	mutex_init(&con->recovery_lock);
2187 	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2188 	atomic_set(&con->in_recovery, 0);
2189 	con->eeprom_control.bad_channel_bitmap = 0;
2190 
2191 	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count();
2192 	amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
2193 
2194 	/* Todo: During test the SMU might fail to read the eeprom through I2C
2195 	 * when the GPU is pending on XGMI reset during probe time
2196 	 * (Mostly after second bus reset), skip it now
2197 	 */
2198 	if (adev->gmc.xgmi.pending_reset)
2199 		return 0;
2200 	ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2201 	/*
2202 	 * This calling fails when exc_err_limit is true or
2203 	 * ret != 0.
2204 	 */
2205 	if (exc_err_limit || ret)
2206 		goto free;
2207 
2208 	if (con->eeprom_control.ras_num_recs) {
2209 		ret = amdgpu_ras_load_bad_pages(adev);
2210 		if (ret)
2211 			goto free;
2212 
2213 		amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
2214 
2215 		if (con->update_channel_flag == true) {
2216 			amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2217 			con->update_channel_flag = false;
2218 		}
2219 	}
2220 
2221 #ifdef CONFIG_X86_MCE_AMD
2222 	if ((adev->asic_type == CHIP_ALDEBARAN) &&
2223 	    (adev->gmc.xgmi.connected_to_cpu))
2224 		amdgpu_register_bad_pages_mca_notifier(adev);
2225 #endif
2226 	return 0;
2227 
2228 free:
2229 	kfree((*data)->bps);
2230 	kfree(*data);
2231 	con->eh_data = NULL;
2232 out:
2233 	dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
2234 
2235 	/*
2236 	 * Except error threshold exceeding case, other failure cases in this
2237 	 * function would not fail amdgpu driver init.
2238 	 */
2239 	if (!exc_err_limit)
2240 		ret = 0;
2241 	else
2242 		ret = -EINVAL;
2243 
2244 	return ret;
2245 }
2246 
2247 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2248 {
2249 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2250 	struct ras_err_handler_data *data = con->eh_data;
2251 
2252 	/* recovery_init failed to init it, fini is useless */
2253 	if (!data)
2254 		return 0;
2255 
2256 	cancel_work_sync(&con->recovery_work);
2257 
2258 	mutex_lock(&con->recovery_lock);
2259 	con->eh_data = NULL;
2260 	kfree(data->bps);
2261 	kfree(data);
2262 	mutex_unlock(&con->recovery_lock);
2263 
2264 	return 0;
2265 }
2266 /* recovery end */
2267 
2268 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
2269 {
2270 	if (amdgpu_sriov_vf(adev)) {
2271 		switch (adev->ip_versions[MP0_HWIP][0]) {
2272 		case IP_VERSION(13, 0, 2):
2273 			return true;
2274 		default:
2275 			return false;
2276 		}
2277 	}
2278 
2279 	if (adev->asic_type == CHIP_IP_DISCOVERY) {
2280 		switch (adev->ip_versions[MP0_HWIP][0]) {
2281 		case IP_VERSION(13, 0, 0):
2282 		case IP_VERSION(13, 0, 10):
2283 			return true;
2284 		default:
2285 			return false;
2286 		}
2287 	}
2288 
2289 	return adev->asic_type == CHIP_VEGA10 ||
2290 		adev->asic_type == CHIP_VEGA20 ||
2291 		adev->asic_type == CHIP_ARCTURUS ||
2292 		adev->asic_type == CHIP_ALDEBARAN ||
2293 		adev->asic_type == CHIP_SIENNA_CICHLID;
2294 }
2295 
2296 /*
2297  * this is workaround for vega20 workstation sku,
2298  * force enable gfx ras, ignore vbios gfx ras flag
2299  * due to GC EDC can not write
2300  */
2301 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
2302 {
2303 	struct atom_context *ctx = adev->mode_info.atom_context;
2304 
2305 	if (!ctx)
2306 		return;
2307 
2308 	if (strnstr(ctx->vbios_version, "D16406",
2309 		    sizeof(ctx->vbios_version)) ||
2310 		strnstr(ctx->vbios_version, "D36002",
2311 			sizeof(ctx->vbios_version)))
2312 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
2313 }
2314 
2315 /*
2316  * check hardware's ras ability which will be saved in hw_supported.
2317  * if hardware does not support ras, we can skip some ras initializtion and
2318  * forbid some ras operations from IP.
2319  * if software itself, say boot parameter, limit the ras ability. We still
2320  * need allow IP do some limited operations, like disable. In such case,
2321  * we have to initialize ras as normal. but need check if operation is
2322  * allowed or not in each function.
2323  */
2324 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2325 {
2326 	adev->ras_hw_enabled = adev->ras_enabled = 0;
2327 
2328 	if (!adev->is_atom_fw ||
2329 	    !amdgpu_ras_asic_supported(adev))
2330 		return;
2331 
2332 	if (!adev->gmc.xgmi.connected_to_cpu) {
2333 		if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2334 			dev_info(adev->dev, "MEM ECC is active.\n");
2335 			adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
2336 						   1 << AMDGPU_RAS_BLOCK__DF);
2337 		} else {
2338 			dev_info(adev->dev, "MEM ECC is not presented.\n");
2339 		}
2340 
2341 		if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2342 			dev_info(adev->dev, "SRAM ECC is active.\n");
2343 			if (!amdgpu_sriov_vf(adev)) {
2344 				adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2345 							    1 << AMDGPU_RAS_BLOCK__DF);
2346 
2347 				if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0))
2348 					adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2349 							1 << AMDGPU_RAS_BLOCK__JPEG);
2350 				else
2351 					adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2352 							1 << AMDGPU_RAS_BLOCK__JPEG);
2353 			} else {
2354 				adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2355 								1 << AMDGPU_RAS_BLOCK__SDMA |
2356 								1 << AMDGPU_RAS_BLOCK__GFX);
2357 			}
2358 		} else {
2359 			dev_info(adev->dev, "SRAM ECC is not presented.\n");
2360 		}
2361 	} else {
2362 		/* driver only manages a few IP blocks RAS feature
2363 		 * when GPU is connected cpu through XGMI */
2364 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
2365 					   1 << AMDGPU_RAS_BLOCK__SDMA |
2366 					   1 << AMDGPU_RAS_BLOCK__MMHUB);
2367 	}
2368 
2369 	amdgpu_ras_get_quirks(adev);
2370 
2371 	/* hw_supported needs to be aligned with RAS block mask. */
2372 	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2373 
2374 	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2375 		adev->ras_hw_enabled & amdgpu_ras_mask;
2376 }
2377 
2378 static void amdgpu_ras_counte_dw(struct work_struct *work)
2379 {
2380 	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2381 					      ras_counte_delay_work.work);
2382 	struct amdgpu_device *adev = con->adev;
2383 	struct drm_device *dev = adev_to_drm(adev);
2384 	unsigned long ce_count, ue_count;
2385 	int res;
2386 
2387 	res = pm_runtime_get_sync(dev->dev);
2388 	if (res < 0)
2389 		goto Out;
2390 
2391 	/* Cache new values.
2392 	 */
2393 	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2394 		atomic_set(&con->ras_ce_count, ce_count);
2395 		atomic_set(&con->ras_ue_count, ue_count);
2396 	}
2397 
2398 	pm_runtime_mark_last_busy(dev->dev);
2399 Out:
2400 	pm_runtime_put_autosuspend(dev->dev);
2401 }
2402 
2403 int amdgpu_ras_init(struct amdgpu_device *adev)
2404 {
2405 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2406 	int r;
2407 	bool df_poison, umc_poison;
2408 
2409 	if (con)
2410 		return 0;
2411 
2412 	con = kmalloc(sizeof(struct amdgpu_ras) +
2413 			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2414 			sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
2415 			GFP_KERNEL|__GFP_ZERO);
2416 	if (!con)
2417 		return -ENOMEM;
2418 
2419 	con->adev = adev;
2420 	INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2421 	atomic_set(&con->ras_ce_count, 0);
2422 	atomic_set(&con->ras_ue_count, 0);
2423 
2424 	con->objs = (struct ras_manager *)(con + 1);
2425 
2426 	amdgpu_ras_set_context(adev, con);
2427 
2428 	amdgpu_ras_check_supported(adev);
2429 
2430 	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
2431 		/* set gfx block ras context feature for VEGA20 Gaming
2432 		 * send ras disable cmd to ras ta during ras late init.
2433 		 */
2434 		if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2435 			con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2436 
2437 			return 0;
2438 		}
2439 
2440 		r = 0;
2441 		goto release_con;
2442 	}
2443 
2444 	con->update_channel_flag = false;
2445 	con->features = 0;
2446 	INIT_LIST_HEAD(&con->head);
2447 	/* Might need get this flag from vbios. */
2448 	con->flags = RAS_DEFAULT_FLAGS;
2449 
2450 	/* initialize nbio ras function ahead of any other
2451 	 * ras functions so hardware fatal error interrupt
2452 	 * can be enabled as early as possible */
2453 	switch (adev->asic_type) {
2454 	case CHIP_VEGA20:
2455 	case CHIP_ARCTURUS:
2456 	case CHIP_ALDEBARAN:
2457 		if (!adev->gmc.xgmi.connected_to_cpu) {
2458 			adev->nbio.ras = &nbio_v7_4_ras;
2459 			amdgpu_ras_register_ras_block(adev, &adev->nbio.ras->ras_block);
2460 			adev->nbio.ras_if = &adev->nbio.ras->ras_block.ras_comm;
2461 		}
2462 		break;
2463 	default:
2464 		/* nbio ras is not available */
2465 		break;
2466 	}
2467 
2468 	if (adev->nbio.ras &&
2469 	    adev->nbio.ras->init_ras_controller_interrupt) {
2470 		r = adev->nbio.ras->init_ras_controller_interrupt(adev);
2471 		if (r)
2472 			goto release_con;
2473 	}
2474 
2475 	if (adev->nbio.ras &&
2476 	    adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2477 		r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
2478 		if (r)
2479 			goto release_con;
2480 	}
2481 
2482 	/* Init poison supported flag, the default value is false */
2483 	if (adev->gmc.xgmi.connected_to_cpu) {
2484 		/* enabled by default when GPU is connected to CPU */
2485 		con->poison_supported = true;
2486 	}
2487 	else if (adev->df.funcs &&
2488 	    adev->df.funcs->query_ras_poison_mode &&
2489 	    adev->umc.ras &&
2490 	    adev->umc.ras->query_ras_poison_mode) {
2491 		df_poison =
2492 			adev->df.funcs->query_ras_poison_mode(adev);
2493 		umc_poison =
2494 			adev->umc.ras->query_ras_poison_mode(adev);
2495 		/* Only poison is set in both DF and UMC, we can support it */
2496 		if (df_poison && umc_poison)
2497 			con->poison_supported = true;
2498 		else if (df_poison != umc_poison)
2499 			dev_warn(adev->dev, "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2500 					df_poison, umc_poison);
2501 	}
2502 
2503 	if (amdgpu_ras_fs_init(adev)) {
2504 		r = -EINVAL;
2505 		goto release_con;
2506 	}
2507 
2508 	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2509 		 "hardware ability[%x] ras_mask[%x]\n",
2510 		 adev->ras_hw_enabled, adev->ras_enabled);
2511 
2512 	return 0;
2513 release_con:
2514 	amdgpu_ras_set_context(adev, NULL);
2515 	kfree(con);
2516 
2517 	return r;
2518 }
2519 
2520 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
2521 {
2522 	if (adev->gmc.xgmi.connected_to_cpu)
2523 		return 1;
2524 	return 0;
2525 }
2526 
2527 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2528 					struct ras_common_if *ras_block)
2529 {
2530 	struct ras_query_if info = {
2531 		.head = *ras_block,
2532 	};
2533 
2534 	if (!amdgpu_persistent_edc_harvesting_supported(adev))
2535 		return 0;
2536 
2537 	if (amdgpu_ras_query_error_status(adev, &info) != 0)
2538 		DRM_WARN("RAS init harvest failure");
2539 
2540 	if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2541 		DRM_WARN("RAS init harvest reset failure");
2542 
2543 	return 0;
2544 }
2545 
2546 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2547 {
2548        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2549 
2550        if (!con)
2551                return false;
2552 
2553        return con->poison_supported;
2554 }
2555 
2556 /* helper function to handle common stuff in ip late init phase */
2557 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2558 			 struct ras_common_if *ras_block)
2559 {
2560 	struct amdgpu_ras_block_object *ras_obj = NULL;
2561 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2562 	unsigned long ue_count, ce_count;
2563 	int r;
2564 
2565 	/* disable RAS feature per IP block if it is not supported */
2566 	if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2567 		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2568 		return 0;
2569 	}
2570 
2571 	r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2572 	if (r) {
2573 		if (adev->in_suspend || amdgpu_in_reset(adev)) {
2574 			/* in resume phase, if fail to enable ras,
2575 			 * clean up all ras fs nodes, and disable ras */
2576 			goto cleanup;
2577 		} else
2578 			return r;
2579 	}
2580 
2581 	/* check for errors on warm reset edc persisant supported ASIC */
2582 	amdgpu_persistent_edc_harvesting(adev, ras_block);
2583 
2584 	/* in resume phase, no need to create ras fs node */
2585 	if (adev->in_suspend || amdgpu_in_reset(adev))
2586 		return 0;
2587 
2588 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2589 	if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2590 	    (ras_obj->hw_ops->query_poison_status ||
2591 	    ras_obj->hw_ops->handle_poison_consumption))) {
2592 		r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
2593 		if (r)
2594 			goto cleanup;
2595 	}
2596 
2597 	r = amdgpu_ras_sysfs_create(adev, ras_block);
2598 	if (r)
2599 		goto interrupt;
2600 
2601 	/* Those are the cached values at init.
2602 	 */
2603 	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2604 		atomic_set(&con->ras_ce_count, ce_count);
2605 		atomic_set(&con->ras_ue_count, ue_count);
2606 	}
2607 
2608 	return 0;
2609 
2610 interrupt:
2611 	if (ras_obj->ras_cb)
2612 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2613 cleanup:
2614 	amdgpu_ras_feature_enable(adev, ras_block, 0);
2615 	return r;
2616 }
2617 
2618 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
2619 			 struct ras_common_if *ras_block)
2620 {
2621 	return amdgpu_ras_block_late_init(adev, ras_block);
2622 }
2623 
2624 /* helper function to remove ras fs node and interrupt handler */
2625 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2626 			  struct ras_common_if *ras_block)
2627 {
2628 	struct amdgpu_ras_block_object *ras_obj;
2629 	if (!ras_block)
2630 		return;
2631 
2632 	amdgpu_ras_sysfs_remove(adev, ras_block);
2633 
2634 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2635 	if (ras_obj->ras_cb)
2636 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2637 }
2638 
2639 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2640 			  struct ras_common_if *ras_block)
2641 {
2642 	return amdgpu_ras_block_late_fini(adev, ras_block);
2643 }
2644 
2645 /* do some init work after IP late init as dependence.
2646  * and it runs in resume/gpu reset/booting up cases.
2647  */
2648 void amdgpu_ras_resume(struct amdgpu_device *adev)
2649 {
2650 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2651 	struct ras_manager *obj, *tmp;
2652 
2653 	if (!adev->ras_enabled || !con) {
2654 		/* clean ras context for VEGA20 Gaming after send ras disable cmd */
2655 		amdgpu_release_ras_context(adev);
2656 
2657 		return;
2658 	}
2659 
2660 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
2661 		/* Set up all other IPs which are not implemented. There is a
2662 		 * tricky thing that IP's actual ras error type should be
2663 		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2664 		 * ERROR_NONE make sense anyway.
2665 		 */
2666 		amdgpu_ras_enable_all_features(adev, 1);
2667 
2668 		/* We enable ras on all hw_supported block, but as boot
2669 		 * parameter might disable some of them and one or more IP has
2670 		 * not implemented yet. So we disable them on behalf.
2671 		 */
2672 		list_for_each_entry_safe(obj, tmp, &con->head, node) {
2673 			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2674 				amdgpu_ras_feature_enable(adev, &obj->head, 0);
2675 				/* there should be no any reference. */
2676 				WARN_ON(alive_obj(obj));
2677 			}
2678 		}
2679 	}
2680 }
2681 
2682 void amdgpu_ras_suspend(struct amdgpu_device *adev)
2683 {
2684 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2685 
2686 	if (!adev->ras_enabled || !con)
2687 		return;
2688 
2689 	amdgpu_ras_disable_all_features(adev, 0);
2690 	/* Make sure all ras objects are disabled. */
2691 	if (con->features)
2692 		amdgpu_ras_disable_all_features(adev, 1);
2693 }
2694 
2695 int amdgpu_ras_late_init(struct amdgpu_device *adev)
2696 {
2697 	struct amdgpu_ras_block_list *node, *tmp;
2698 	struct amdgpu_ras_block_object *obj;
2699 	int r;
2700 
2701 	/* Guest side doesn't need init ras feature */
2702 	if (amdgpu_sriov_vf(adev))
2703 		return 0;
2704 
2705 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2706 		if (!node->ras_obj) {
2707 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2708 			continue;
2709 		}
2710 
2711 		obj = node->ras_obj;
2712 		if (obj->ras_late_init) {
2713 			r = obj->ras_late_init(adev, &obj->ras_comm);
2714 			if (r) {
2715 				dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2716 					obj->ras_comm.name, r);
2717 				return r;
2718 			}
2719 		} else
2720 			amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
2721 	}
2722 
2723 	return 0;
2724 }
2725 
2726 /* do some fini work before IP fini as dependence */
2727 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2728 {
2729 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2730 
2731 	if (!adev->ras_enabled || !con)
2732 		return 0;
2733 
2734 
2735 	/* Need disable ras on all IPs here before ip [hw/sw]fini */
2736 	if (con->features)
2737 		amdgpu_ras_disable_all_features(adev, 0);
2738 	amdgpu_ras_recovery_fini(adev);
2739 	return 0;
2740 }
2741 
2742 int amdgpu_ras_fini(struct amdgpu_device *adev)
2743 {
2744 	struct amdgpu_ras_block_list *ras_node, *tmp;
2745 	struct amdgpu_ras_block_object *obj = NULL;
2746 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2747 
2748 	if (!adev->ras_enabled || !con)
2749 		return 0;
2750 
2751 	list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2752 		if (ras_node->ras_obj) {
2753 			obj = ras_node->ras_obj;
2754 			if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2755 			    obj->ras_fini)
2756 				obj->ras_fini(adev, &obj->ras_comm);
2757 			else
2758 				amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
2759 		}
2760 
2761 		/* Clear ras blocks from ras_list and free ras block list node */
2762 		list_del(&ras_node->node);
2763 		kfree(ras_node);
2764 	}
2765 
2766 	amdgpu_ras_fs_fini(adev);
2767 	amdgpu_ras_interrupt_remove_all(adev);
2768 
2769 	WARN(con->features, "Feature mask is not cleared");
2770 
2771 	if (con->features)
2772 		amdgpu_ras_disable_all_features(adev, 1);
2773 
2774 	cancel_delayed_work_sync(&con->ras_counte_delay_work);
2775 
2776 	amdgpu_ras_set_context(adev, NULL);
2777 	kfree(con);
2778 
2779 	return 0;
2780 }
2781 
2782 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2783 {
2784 	amdgpu_ras_check_supported(adev);
2785 	if (!adev->ras_hw_enabled)
2786 		return;
2787 
2788 	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2789 		dev_info(adev->dev, "uncorrectable hardware error"
2790 			"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
2791 
2792 		amdgpu_ras_reset_gpu(adev);
2793 	}
2794 }
2795 
2796 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2797 {
2798 	if (adev->asic_type == CHIP_VEGA20 &&
2799 	    adev->pm.fw_version <= 0x283400) {
2800 		return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2801 				amdgpu_ras_intr_triggered();
2802 	}
2803 
2804 	return false;
2805 }
2806 
2807 void amdgpu_release_ras_context(struct amdgpu_device *adev)
2808 {
2809 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2810 
2811 	if (!con)
2812 		return;
2813 
2814 	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
2815 		con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2816 		amdgpu_ras_set_context(adev, NULL);
2817 		kfree(con);
2818 	}
2819 }
2820 
2821 #ifdef CONFIG_X86_MCE_AMD
2822 static struct amdgpu_device *find_adev(uint32_t node_id)
2823 {
2824 	int i;
2825 	struct amdgpu_device *adev = NULL;
2826 
2827 	for (i = 0; i < mce_adev_list.num_gpu; i++) {
2828 		adev = mce_adev_list.devs[i];
2829 
2830 		if (adev && adev->gmc.xgmi.connected_to_cpu &&
2831 		    adev->gmc.xgmi.physical_node_id == node_id)
2832 			break;
2833 		adev = NULL;
2834 	}
2835 
2836 	return adev;
2837 }
2838 
2839 #define GET_MCA_IPID_GPUID(m)	(((m) >> 44) & 0xF)
2840 #define GET_UMC_INST(m)		(((m) >> 21) & 0x7)
2841 #define GET_CHAN_INDEX(m)	((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
2842 #define GPU_ID_OFFSET		8
2843 
2844 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
2845 				    unsigned long val, void *data)
2846 {
2847 	struct mce *m = (struct mce *)data;
2848 	struct amdgpu_device *adev = NULL;
2849 	uint32_t gpu_id = 0;
2850 	uint32_t umc_inst = 0, ch_inst = 0;
2851 	struct ras_err_data err_data = {0, 0, 0, NULL};
2852 
2853 	/*
2854 	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
2855 	 * and error occurred in DramECC (Extended error code = 0) then only
2856 	 * process the error, else bail out.
2857 	 */
2858 	if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
2859 		    (XEC(m->status, 0x3f) == 0x0)))
2860 		return NOTIFY_DONE;
2861 
2862 	/*
2863 	 * If it is correctable error, return.
2864 	 */
2865 	if (mce_is_correctable(m))
2866 		return NOTIFY_OK;
2867 
2868 	/*
2869 	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
2870 	 */
2871 	gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
2872 
2873 	adev = find_adev(gpu_id);
2874 	if (!adev) {
2875 		DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
2876 								gpu_id);
2877 		return NOTIFY_DONE;
2878 	}
2879 
2880 	/*
2881 	 * If it is uncorrectable error, then find out UMC instance and
2882 	 * channel index.
2883 	 */
2884 	umc_inst = GET_UMC_INST(m->ipid);
2885 	ch_inst = GET_CHAN_INDEX(m->ipid);
2886 
2887 	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
2888 			     umc_inst, ch_inst);
2889 
2890 	err_data.err_addr =
2891 		kcalloc(adev->umc.max_ras_err_cnt_per_query,
2892 			sizeof(struct eeprom_table_record), GFP_KERNEL);
2893 	if (!err_data.err_addr) {
2894 		dev_warn(adev->dev,
2895 			"Failed to alloc memory for umc error record in mca notifier!\n");
2896 		return NOTIFY_DONE;
2897 	}
2898 
2899 	/*
2900 	 * Translate UMC channel address to Physical address
2901 	 */
2902 	if (adev->umc.ras &&
2903 	    adev->umc.ras->convert_ras_error_address)
2904 		adev->umc.ras->convert_ras_error_address(adev,
2905 			&err_data, m->addr, ch_inst, umc_inst);
2906 
2907 	if (amdgpu_bad_page_threshold != 0) {
2908 		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
2909 						err_data.err_addr_cnt);
2910 		amdgpu_ras_save_bad_pages(adev);
2911 	}
2912 
2913 	kfree(err_data.err_addr);
2914 	return NOTIFY_OK;
2915 }
2916 
2917 static struct notifier_block amdgpu_bad_page_nb = {
2918 	.notifier_call  = amdgpu_bad_page_notifier,
2919 	.priority       = MCE_PRIO_UC,
2920 };
2921 
2922 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
2923 {
2924 	/*
2925 	 * Add the adev to the mce_adev_list.
2926 	 * During mode2 reset, amdgpu device is temporarily
2927 	 * removed from the mgpu_info list which can cause
2928 	 * page retirement to fail.
2929 	 * Use this list instead of mgpu_info to find the amdgpu
2930 	 * device on which the UMC error was reported.
2931 	 */
2932 	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
2933 
2934 	/*
2935 	 * Register the x86 notifier only once
2936 	 * with MCE subsystem.
2937 	 */
2938 	if (notifier_registered == false) {
2939 		mce_register_decode_chain(&amdgpu_bad_page_nb);
2940 		notifier_registered = true;
2941 	}
2942 }
2943 #endif
2944 
2945 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
2946 {
2947 	if (!adev)
2948 		return NULL;
2949 
2950 	return adev->psp.ras_context.ras;
2951 }
2952 
2953 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
2954 {
2955 	if (!adev)
2956 		return -EINVAL;
2957 
2958 	adev->psp.ras_context.ras = ras_con;
2959 	return 0;
2960 }
2961 
2962 /* check if ras is supported on block, say, sdma, gfx */
2963 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
2964 		unsigned int block)
2965 {
2966 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2967 
2968 	if (block >= AMDGPU_RAS_BLOCK_COUNT)
2969 		return 0;
2970 	return ras && (adev->ras_enabled & (1 << block));
2971 }
2972 
2973 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
2974 {
2975 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2976 
2977 	if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
2978 		amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
2979 	return 0;
2980 }
2981 
2982 
2983 /* Register each ip ras block into amdgpu ras */
2984 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
2985 		struct amdgpu_ras_block_object *ras_block_obj)
2986 {
2987 	struct amdgpu_ras_block_list *ras_node;
2988 	if (!adev || !ras_block_obj)
2989 		return -EINVAL;
2990 
2991 	if (!amdgpu_ras_asic_supported(adev))
2992 		return 0;
2993 
2994 	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
2995 	if (!ras_node)
2996 		return -ENOMEM;
2997 
2998 	INIT_LIST_HEAD(&ras_node->node);
2999 	ras_node->ras_obj = ras_block_obj;
3000 	list_add_tail(&ras_node->node, &adev->ras_list);
3001 
3002 	return 0;
3003 }
3004