1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/debugfs.h>
25 #include <linux/list.h>
26 #include <linux/module.h>
27 #include <linux/uaccess.h>
28 #include <linux/reboot.h>
29 #include <linux/syscalls.h>
30 #include <linux/pm_runtime.h>
31 
32 #include "amdgpu.h"
33 #include "amdgpu_ras.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_xgmi.h"
36 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
37 #include "nbio_v4_3.h"
38 #include "nbio_v7_9.h"
39 #include "atom.h"
40 #include "amdgpu_reset.h"
41 
42 #ifdef CONFIG_X86_MCE_AMD
43 #include <asm/mce.h>
44 
45 static bool notifier_registered;
46 #endif
47 static const char *RAS_FS_NAME = "ras";
48 
49 const char *ras_error_string[] = {
50 	"none",
51 	"parity",
52 	"single_correctable",
53 	"multi_uncorrectable",
54 	"poison",
55 };
56 
57 const char *ras_block_string[] = {
58 	"umc",
59 	"sdma",
60 	"gfx",
61 	"mmhub",
62 	"athub",
63 	"pcie_bif",
64 	"hdp",
65 	"xgmi_wafl",
66 	"df",
67 	"smn",
68 	"sem",
69 	"mp0",
70 	"mp1",
71 	"fuse",
72 	"mca",
73 	"vcn",
74 	"jpeg",
75 };
76 
77 const char *ras_mca_block_string[] = {
78 	"mca_mp0",
79 	"mca_mp1",
80 	"mca_mpio",
81 	"mca_iohc",
82 };
83 
84 struct amdgpu_ras_block_list {
85 	/* ras block link */
86 	struct list_head node;
87 
88 	struct amdgpu_ras_block_object *ras_obj;
89 };
90 
91 const char *get_ras_block_str(struct ras_common_if *ras_block)
92 {
93 	if (!ras_block)
94 		return "NULL";
95 
96 	if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
97 		return "OUT OF RANGE";
98 
99 	if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
100 		return ras_mca_block_string[ras_block->sub_block_index];
101 
102 	return ras_block_string[ras_block->block];
103 }
104 
105 #define ras_block_str(_BLOCK_) \
106 	(((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
107 
108 #define ras_err_str(i) (ras_error_string[ffs(i)])
109 
110 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
111 
112 /* inject address is 52 bits */
113 #define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
114 
115 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
116 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
117 
118 enum amdgpu_ras_retire_page_reservation {
119 	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
120 	AMDGPU_RAS_RETIRE_PAGE_PENDING,
121 	AMDGPU_RAS_RETIRE_PAGE_FAULT,
122 };
123 
124 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
125 
126 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
127 				uint64_t addr);
128 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
129 				uint64_t addr);
130 #ifdef CONFIG_X86_MCE_AMD
131 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
132 struct mce_notifier_adev_list {
133 	struct amdgpu_device *devs[MAX_GPU_INSTANCE];
134 	int num_gpu;
135 };
136 static struct mce_notifier_adev_list mce_adev_list;
137 #endif
138 
139 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
140 {
141 	if (adev && amdgpu_ras_get_context(adev))
142 		amdgpu_ras_get_context(adev)->error_query_ready = ready;
143 }
144 
145 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
146 {
147 	if (adev && amdgpu_ras_get_context(adev))
148 		return amdgpu_ras_get_context(adev)->error_query_ready;
149 
150 	return false;
151 }
152 
153 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
154 {
155 	struct ras_err_data err_data = {0, 0, 0, NULL};
156 	struct eeprom_table_record err_rec;
157 
158 	if ((address >= adev->gmc.mc_vram_size) ||
159 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
160 		dev_warn(adev->dev,
161 		         "RAS WARN: input address 0x%llx is invalid.\n",
162 		         address);
163 		return -EINVAL;
164 	}
165 
166 	if (amdgpu_ras_check_bad_page(adev, address)) {
167 		dev_warn(adev->dev,
168 			 "RAS WARN: 0x%llx has already been marked as bad page!\n",
169 			 address);
170 		return 0;
171 	}
172 
173 	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
174 	err_data.err_addr = &err_rec;
175 	amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
176 
177 	if (amdgpu_bad_page_threshold != 0) {
178 		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
179 					 err_data.err_addr_cnt);
180 		amdgpu_ras_save_bad_pages(adev, NULL);
181 	}
182 
183 	dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
184 	dev_warn(adev->dev, "Clear EEPROM:\n");
185 	dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
186 
187 	return 0;
188 }
189 
190 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
191 					size_t size, loff_t *pos)
192 {
193 	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
194 	struct ras_query_if info = {
195 		.head = obj->head,
196 	};
197 	ssize_t s;
198 	char val[128];
199 
200 	if (amdgpu_ras_query_error_status(obj->adev, &info))
201 		return -EINVAL;
202 
203 	/* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
204 	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
205 	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
206 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
207 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
208 	}
209 
210 	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
211 			"ue", info.ue_count,
212 			"ce", info.ce_count);
213 	if (*pos >= s)
214 		return 0;
215 
216 	s -= *pos;
217 	s = min_t(u64, s, size);
218 
219 
220 	if (copy_to_user(buf, &val[*pos], s))
221 		return -EINVAL;
222 
223 	*pos += s;
224 
225 	return s;
226 }
227 
228 static const struct file_operations amdgpu_ras_debugfs_ops = {
229 	.owner = THIS_MODULE,
230 	.read = amdgpu_ras_debugfs_read,
231 	.write = NULL,
232 	.llseek = default_llseek
233 };
234 
235 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
236 {
237 	int i;
238 
239 	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
240 		*block_id = i;
241 		if (strcmp(name, ras_block_string[i]) == 0)
242 			return 0;
243 	}
244 	return -EINVAL;
245 }
246 
247 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
248 		const char __user *buf, size_t size,
249 		loff_t *pos, struct ras_debug_if *data)
250 {
251 	ssize_t s = min_t(u64, 64, size);
252 	char str[65];
253 	char block_name[33];
254 	char err[9] = "ue";
255 	int op = -1;
256 	int block_id;
257 	uint32_t sub_block;
258 	u64 address, value;
259 	/* default value is 0 if the mask is not set by user */
260 	u32 instance_mask = 0;
261 
262 	if (*pos)
263 		return -EINVAL;
264 	*pos = size;
265 
266 	memset(str, 0, sizeof(str));
267 	memset(data, 0, sizeof(*data));
268 
269 	if (copy_from_user(str, buf, s))
270 		return -EINVAL;
271 
272 	if (sscanf(str, "disable %32s", block_name) == 1)
273 		op = 0;
274 	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
275 		op = 1;
276 	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
277 		op = 2;
278 	else if (strstr(str, "retire_page") != NULL)
279 		op = 3;
280 	else if (str[0] && str[1] && str[2] && str[3])
281 		/* ascii string, but commands are not matched. */
282 		return -EINVAL;
283 
284 	if (op != -1) {
285 		if (op == 3) {
286 			if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
287 			    sscanf(str, "%*s %llu", &address) != 1)
288 				return -EINVAL;
289 
290 			data->op = op;
291 			data->inject.address = address;
292 
293 			return 0;
294 		}
295 
296 		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
297 			return -EINVAL;
298 
299 		data->head.block = block_id;
300 		/* only ue and ce errors are supported */
301 		if (!memcmp("ue", err, 2))
302 			data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
303 		else if (!memcmp("ce", err, 2))
304 			data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
305 		else
306 			return -EINVAL;
307 
308 		data->op = op;
309 
310 		if (op == 2) {
311 			if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
312 				   &sub_block, &address, &value, &instance_mask) != 4 &&
313 			    sscanf(str, "%*s %*s %*s %u %llu %llu %u",
314 				   &sub_block, &address, &value, &instance_mask) != 4 &&
315 				sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
316 				   &sub_block, &address, &value) != 3 &&
317 			    sscanf(str, "%*s %*s %*s %u %llu %llu",
318 				   &sub_block, &address, &value) != 3)
319 				return -EINVAL;
320 			data->head.sub_block_index = sub_block;
321 			data->inject.address = address;
322 			data->inject.value = value;
323 			data->inject.instance_mask = instance_mask;
324 		}
325 	} else {
326 		if (size < sizeof(*data))
327 			return -EINVAL;
328 
329 		if (copy_from_user(data, buf, sizeof(*data)))
330 			return -EINVAL;
331 	}
332 
333 	return 0;
334 }
335 
336 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
337 				struct ras_debug_if *data)
338 {
339 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
340 	uint32_t mask, inst_mask = data->inject.instance_mask;
341 
342 	/* no need to set instance mask if there is only one instance */
343 	if (num_xcc <= 1 && inst_mask) {
344 		data->inject.instance_mask = 0;
345 		dev_dbg(adev->dev,
346 			"RAS inject mask(0x%x) isn't supported and force it to 0.\n",
347 			inst_mask);
348 
349 		return;
350 	}
351 
352 	switch (data->head.block) {
353 	case AMDGPU_RAS_BLOCK__GFX:
354 		mask = GENMASK(num_xcc - 1, 0);
355 		break;
356 	case AMDGPU_RAS_BLOCK__SDMA:
357 		mask = GENMASK(adev->sdma.num_instances - 1, 0);
358 		break;
359 	case AMDGPU_RAS_BLOCK__VCN:
360 	case AMDGPU_RAS_BLOCK__JPEG:
361 		mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
362 		break;
363 	default:
364 		mask = inst_mask;
365 		break;
366 	}
367 
368 	/* remove invalid bits in instance mask */
369 	data->inject.instance_mask &= mask;
370 	if (inst_mask != data->inject.instance_mask)
371 		dev_dbg(adev->dev,
372 			"Adjust RAS inject mask 0x%x to 0x%x\n",
373 			inst_mask, data->inject.instance_mask);
374 }
375 
376 /**
377  * DOC: AMDGPU RAS debugfs control interface
378  *
379  * The control interface accepts struct ras_debug_if which has two members.
380  *
381  * First member: ras_debug_if::head or ras_debug_if::inject.
382  *
383  * head is used to indicate which IP block will be under control.
384  *
385  * head has four members, they are block, type, sub_block_index, name.
386  * block: which IP will be under control.
387  * type: what kind of error will be enabled/disabled/injected.
388  * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
389  * name: the name of IP.
390  *
391  * inject has three more members than head, they are address, value and mask.
392  * As their names indicate, inject operation will write the
393  * value to the address.
394  *
395  * The second member: struct ras_debug_if::op.
396  * It has three kinds of operations.
397  *
398  * - 0: disable RAS on the block. Take ::head as its data.
399  * - 1: enable RAS on the block. Take ::head as its data.
400  * - 2: inject errors on the block. Take ::inject as its data.
401  *
402  * How to use the interface?
403  *
404  * In a program
405  *
406  * Copy the struct ras_debug_if in your code and initialize it.
407  * Write the struct to the control interface.
408  *
409  * From shell
410  *
411  * .. code-block:: bash
412  *
413  *	echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
414  *	echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
415  *	echo "inject  <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
416  *
417  * Where N, is the card which you want to affect.
418  *
419  * "disable" requires only the block.
420  * "enable" requires the block and error type.
421  * "inject" requires the block, error type, address, and value.
422  *
423  * The block is one of: umc, sdma, gfx, etc.
424  *	see ras_block_string[] for details
425  *
426  * The error type is one of: ue, ce, where,
427  *	ue is multi-uncorrectable
428  *	ce is single-correctable
429  *
430  * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
431  * The address and value are hexadecimal numbers, leading 0x is optional.
432  * The mask means instance mask, is optional, default value is 0x1.
433  *
434  * For instance,
435  *
436  * .. code-block:: bash
437  *
438  *	echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
439  *	echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
440  *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
441  *
442  * How to check the result of the operation?
443  *
444  * To check disable/enable, see "ras" features at,
445  * /sys/class/drm/card[0/1/2...]/device/ras/features
446  *
447  * To check inject, see the corresponding error count at,
448  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
449  *
450  * .. note::
451  *	Operations are only allowed on blocks which are supported.
452  *	Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
453  *	to see which blocks support RAS on a particular asic.
454  *
455  */
456 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
457 					     const char __user *buf,
458 					     size_t size, loff_t *pos)
459 {
460 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
461 	struct ras_debug_if data;
462 	int ret = 0;
463 
464 	if (!amdgpu_ras_get_error_query_ready(adev)) {
465 		dev_warn(adev->dev, "RAS WARN: error injection "
466 				"currently inaccessible\n");
467 		return size;
468 	}
469 
470 	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
471 	if (ret)
472 		return ret;
473 
474 	if (data.op == 3) {
475 		ret = amdgpu_reserve_page_direct(adev, data.inject.address);
476 		if (!ret)
477 			return size;
478 		else
479 			return ret;
480 	}
481 
482 	if (!amdgpu_ras_is_supported(adev, data.head.block))
483 		return -EINVAL;
484 
485 	switch (data.op) {
486 	case 0:
487 		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
488 		break;
489 	case 1:
490 		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
491 		break;
492 	case 2:
493 		if ((data.inject.address >= adev->gmc.mc_vram_size &&
494 		    adev->gmc.mc_vram_size) ||
495 		    (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
496 			dev_warn(adev->dev, "RAS WARN: input address "
497 					"0x%llx is invalid.",
498 					data.inject.address);
499 			ret = -EINVAL;
500 			break;
501 		}
502 
503 		/* umc ce/ue error injection for a bad page is not allowed */
504 		if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
505 		    amdgpu_ras_check_bad_page(adev, data.inject.address)) {
506 			dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
507 				 "already been marked as bad!\n",
508 				 data.inject.address);
509 			break;
510 		}
511 
512 		amdgpu_ras_instance_mask_check(adev, &data);
513 
514 		/* data.inject.address is offset instead of absolute gpu address */
515 		ret = amdgpu_ras_error_inject(adev, &data.inject);
516 		break;
517 	default:
518 		ret = -EINVAL;
519 		break;
520 	}
521 
522 	if (ret)
523 		return ret;
524 
525 	return size;
526 }
527 
528 /**
529  * DOC: AMDGPU RAS debugfs EEPROM table reset interface
530  *
531  * Some boards contain an EEPROM which is used to persistently store a list of
532  * bad pages which experiences ECC errors in vram.  This interface provides
533  * a way to reset the EEPROM, e.g., after testing error injection.
534  *
535  * Usage:
536  *
537  * .. code-block:: bash
538  *
539  *	echo 1 > ../ras/ras_eeprom_reset
540  *
541  * will reset EEPROM table to 0 entries.
542  *
543  */
544 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
545 					       const char __user *buf,
546 					       size_t size, loff_t *pos)
547 {
548 	struct amdgpu_device *adev =
549 		(struct amdgpu_device *)file_inode(f)->i_private;
550 	int ret;
551 
552 	ret = amdgpu_ras_eeprom_reset_table(
553 		&(amdgpu_ras_get_context(adev)->eeprom_control));
554 
555 	if (!ret) {
556 		/* Something was written to EEPROM.
557 		 */
558 		amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
559 		return size;
560 	} else {
561 		return ret;
562 	}
563 }
564 
565 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
566 	.owner = THIS_MODULE,
567 	.read = NULL,
568 	.write = amdgpu_ras_debugfs_ctrl_write,
569 	.llseek = default_llseek
570 };
571 
572 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
573 	.owner = THIS_MODULE,
574 	.read = NULL,
575 	.write = amdgpu_ras_debugfs_eeprom_write,
576 	.llseek = default_llseek
577 };
578 
579 /**
580  * DOC: AMDGPU RAS sysfs Error Count Interface
581  *
582  * It allows the user to read the error count for each IP block on the gpu through
583  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
584  *
585  * It outputs the multiple lines which report the uncorrected (ue) and corrected
586  * (ce) error counts.
587  *
588  * The format of one line is below,
589  *
590  * [ce|ue]: count
591  *
592  * Example:
593  *
594  * .. code-block:: bash
595  *
596  *	ue: 0
597  *	ce: 1
598  *
599  */
600 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
601 		struct device_attribute *attr, char *buf)
602 {
603 	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
604 	struct ras_query_if info = {
605 		.head = obj->head,
606 	};
607 
608 	if (!amdgpu_ras_get_error_query_ready(obj->adev))
609 		return sysfs_emit(buf, "Query currently inaccessible\n");
610 
611 	if (amdgpu_ras_query_error_status(obj->adev, &info))
612 		return -EINVAL;
613 
614 	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
615 	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
616 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
617 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
618 	}
619 
620 	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
621 			  "ce", info.ce_count);
622 }
623 
624 /* obj begin */
625 
626 #define get_obj(obj) do { (obj)->use++; } while (0)
627 #define alive_obj(obj) ((obj)->use)
628 
629 static inline void put_obj(struct ras_manager *obj)
630 {
631 	if (obj && (--obj->use == 0))
632 		list_del(&obj->node);
633 	if (obj && (obj->use < 0))
634 		DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
635 }
636 
637 /* make one obj and return it. */
638 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
639 		struct ras_common_if *head)
640 {
641 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
642 	struct ras_manager *obj;
643 
644 	if (!adev->ras_enabled || !con)
645 		return NULL;
646 
647 	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
648 		return NULL;
649 
650 	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
651 		if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
652 			return NULL;
653 
654 		obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
655 	} else
656 		obj = &con->objs[head->block];
657 
658 	/* already exist. return obj? */
659 	if (alive_obj(obj))
660 		return NULL;
661 
662 	obj->head = *head;
663 	obj->adev = adev;
664 	list_add(&obj->node, &con->head);
665 	get_obj(obj);
666 
667 	return obj;
668 }
669 
670 /* return an obj equal to head, or the first when head is NULL */
671 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
672 		struct ras_common_if *head)
673 {
674 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
675 	struct ras_manager *obj;
676 	int i;
677 
678 	if (!adev->ras_enabled || !con)
679 		return NULL;
680 
681 	if (head) {
682 		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
683 			return NULL;
684 
685 		if (head->block == AMDGPU_RAS_BLOCK__MCA) {
686 			if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
687 				return NULL;
688 
689 			obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
690 		} else
691 			obj = &con->objs[head->block];
692 
693 		if (alive_obj(obj))
694 			return obj;
695 	} else {
696 		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
697 			obj = &con->objs[i];
698 			if (alive_obj(obj))
699 				return obj;
700 		}
701 	}
702 
703 	return NULL;
704 }
705 /* obj end */
706 
707 /* feature ctl begin */
708 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
709 					 struct ras_common_if *head)
710 {
711 	return adev->ras_hw_enabled & BIT(head->block);
712 }
713 
714 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
715 		struct ras_common_if *head)
716 {
717 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
718 
719 	return con->features & BIT(head->block);
720 }
721 
722 /*
723  * if obj is not created, then create one.
724  * set feature enable flag.
725  */
726 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
727 		struct ras_common_if *head, int enable)
728 {
729 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
730 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
731 
732 	/* If hardware does not support ras, then do not create obj.
733 	 * But if hardware support ras, we can create the obj.
734 	 * Ras framework checks con->hw_supported to see if it need do
735 	 * corresponding initialization.
736 	 * IP checks con->support to see if it need disable ras.
737 	 */
738 	if (!amdgpu_ras_is_feature_allowed(adev, head))
739 		return 0;
740 
741 	if (enable) {
742 		if (!obj) {
743 			obj = amdgpu_ras_create_obj(adev, head);
744 			if (!obj)
745 				return -EINVAL;
746 		} else {
747 			/* In case we create obj somewhere else */
748 			get_obj(obj);
749 		}
750 		con->features |= BIT(head->block);
751 	} else {
752 		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
753 			con->features &= ~BIT(head->block);
754 			put_obj(obj);
755 		}
756 	}
757 
758 	return 0;
759 }
760 
761 /* wrapper of psp_ras_enable_features */
762 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
763 		struct ras_common_if *head, bool enable)
764 {
765 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
766 	union ta_ras_cmd_input *info;
767 	int ret;
768 
769 	if (!con)
770 		return -EINVAL;
771 
772 	/* Do not enable ras feature if it is not allowed */
773 	if (enable &&
774 	    head->block != AMDGPU_RAS_BLOCK__GFX &&
775 	    !amdgpu_ras_is_feature_allowed(adev, head))
776 		return 0;
777 
778 	/* Only enable gfx ras feature from host side */
779 	if (head->block == AMDGPU_RAS_BLOCK__GFX &&
780 	    !amdgpu_sriov_vf(adev) &&
781 	    !amdgpu_ras_intr_triggered()) {
782 		info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
783 		if (!info)
784 			return -ENOMEM;
785 
786 		if (!enable) {
787 			info->disable_features = (struct ta_ras_disable_features_input) {
788 				.block_id =  amdgpu_ras_block_to_ta(head->block),
789 				.error_type = amdgpu_ras_error_to_ta(head->type),
790 			};
791 		} else {
792 			info->enable_features = (struct ta_ras_enable_features_input) {
793 				.block_id =  amdgpu_ras_block_to_ta(head->block),
794 				.error_type = amdgpu_ras_error_to_ta(head->type),
795 			};
796 		}
797 
798 		ret = psp_ras_enable_features(&adev->psp, info, enable);
799 		if (ret) {
800 			dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
801 				enable ? "enable":"disable",
802 				get_ras_block_str(head),
803 				amdgpu_ras_is_poison_mode_supported(adev), ret);
804 			kfree(info);
805 			return ret;
806 		}
807 
808 		kfree(info);
809 	}
810 
811 	/* setup the obj */
812 	__amdgpu_ras_feature_enable(adev, head, enable);
813 
814 	return 0;
815 }
816 
817 /* Only used in device probe stage and called only once. */
818 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
819 		struct ras_common_if *head, bool enable)
820 {
821 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
822 	int ret;
823 
824 	if (!con)
825 		return -EINVAL;
826 
827 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
828 		if (enable) {
829 			/* There is no harm to issue a ras TA cmd regardless of
830 			 * the currecnt ras state.
831 			 * If current state == target state, it will do nothing
832 			 * But sometimes it requests driver to reset and repost
833 			 * with error code -EAGAIN.
834 			 */
835 			ret = amdgpu_ras_feature_enable(adev, head, 1);
836 			/* With old ras TA, we might fail to enable ras.
837 			 * Log it and just setup the object.
838 			 * TODO need remove this WA in the future.
839 			 */
840 			if (ret == -EINVAL) {
841 				ret = __amdgpu_ras_feature_enable(adev, head, 1);
842 				if (!ret)
843 					dev_info(adev->dev,
844 						"RAS INFO: %s setup object\n",
845 						get_ras_block_str(head));
846 			}
847 		} else {
848 			/* setup the object then issue a ras TA disable cmd.*/
849 			ret = __amdgpu_ras_feature_enable(adev, head, 1);
850 			if (ret)
851 				return ret;
852 
853 			/* gfx block ras dsiable cmd must send to ras-ta */
854 			if (head->block == AMDGPU_RAS_BLOCK__GFX)
855 				con->features |= BIT(head->block);
856 
857 			ret = amdgpu_ras_feature_enable(adev, head, 0);
858 
859 			/* clean gfx block ras features flag */
860 			if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
861 				con->features &= ~BIT(head->block);
862 		}
863 	} else
864 		ret = amdgpu_ras_feature_enable(adev, head, enable);
865 
866 	return ret;
867 }
868 
869 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
870 		bool bypass)
871 {
872 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
873 	struct ras_manager *obj, *tmp;
874 
875 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
876 		/* bypass psp.
877 		 * aka just release the obj and corresponding flags
878 		 */
879 		if (bypass) {
880 			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
881 				break;
882 		} else {
883 			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
884 				break;
885 		}
886 	}
887 
888 	return con->features;
889 }
890 
891 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
892 		bool bypass)
893 {
894 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
895 	int i;
896 	const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
897 
898 	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
899 		struct ras_common_if head = {
900 			.block = i,
901 			.type = default_ras_type,
902 			.sub_block_index = 0,
903 		};
904 
905 		if (i == AMDGPU_RAS_BLOCK__MCA)
906 			continue;
907 
908 		if (bypass) {
909 			/*
910 			 * bypass psp. vbios enable ras for us.
911 			 * so just create the obj
912 			 */
913 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
914 				break;
915 		} else {
916 			if (amdgpu_ras_feature_enable(adev, &head, 1))
917 				break;
918 		}
919 	}
920 
921 	for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
922 		struct ras_common_if head = {
923 			.block = AMDGPU_RAS_BLOCK__MCA,
924 			.type = default_ras_type,
925 			.sub_block_index = i,
926 		};
927 
928 		if (bypass) {
929 			/*
930 			 * bypass psp. vbios enable ras for us.
931 			 * so just create the obj
932 			 */
933 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
934 				break;
935 		} else {
936 			if (amdgpu_ras_feature_enable(adev, &head, 1))
937 				break;
938 		}
939 	}
940 
941 	return con->features;
942 }
943 /* feature ctl end */
944 
945 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
946 		enum amdgpu_ras_block block)
947 {
948 	if (!block_obj)
949 		return -EINVAL;
950 
951 	if (block_obj->ras_comm.block == block)
952 		return 0;
953 
954 	return -EINVAL;
955 }
956 
957 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
958 					enum amdgpu_ras_block block, uint32_t sub_block_index)
959 {
960 	struct amdgpu_ras_block_list *node, *tmp;
961 	struct amdgpu_ras_block_object *obj;
962 
963 	if (block >= AMDGPU_RAS_BLOCK__LAST)
964 		return NULL;
965 
966 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
967 		if (!node->ras_obj) {
968 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
969 			continue;
970 		}
971 
972 		obj = node->ras_obj;
973 		if (obj->ras_block_match) {
974 			if (obj->ras_block_match(obj, block, sub_block_index) == 0)
975 				return obj;
976 		} else {
977 			if (amdgpu_ras_block_match_default(obj, block) == 0)
978 				return obj;
979 		}
980 	}
981 
982 	return NULL;
983 }
984 
985 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
986 {
987 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
988 	int ret = 0;
989 
990 	/*
991 	 * choosing right query method according to
992 	 * whether smu support query error information
993 	 */
994 	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
995 	if (ret == -EOPNOTSUPP) {
996 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
997 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
998 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
999 
1000 		/* umc query_ras_error_address is also responsible for clearing
1001 		 * error status
1002 		 */
1003 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1004 		    adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1005 			adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1006 	} else if (!ret) {
1007 		if (adev->umc.ras &&
1008 			adev->umc.ras->ecc_info_query_ras_error_count)
1009 			adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1010 
1011 		if (adev->umc.ras &&
1012 			adev->umc.ras->ecc_info_query_ras_error_address)
1013 			adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1014 	}
1015 }
1016 
1017 /* query/inject/cure begin */
1018 int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
1019 				  struct ras_query_if *info)
1020 {
1021 	struct amdgpu_ras_block_object *block_obj = NULL;
1022 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1023 	struct ras_err_data err_data = {0, 0, 0, NULL};
1024 
1025 	if (!obj)
1026 		return -EINVAL;
1027 
1028 	if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1029 		amdgpu_ras_get_ecc_info(adev, &err_data);
1030 	} else {
1031 		block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1032 		if (!block_obj || !block_obj->hw_ops)   {
1033 			dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1034 				     get_ras_block_str(&info->head));
1035 			return -EINVAL;
1036 		}
1037 
1038 		if (block_obj->hw_ops->query_ras_error_count)
1039 			block_obj->hw_ops->query_ras_error_count(adev, &err_data);
1040 
1041 		if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1042 		    (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1043 		    (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1044 				if (block_obj->hw_ops->query_ras_error_status)
1045 					block_obj->hw_ops->query_ras_error_status(adev);
1046 			}
1047 	}
1048 
1049 	obj->err_data.ue_count += err_data.ue_count;
1050 	obj->err_data.ce_count += err_data.ce_count;
1051 
1052 	info->ue_count = obj->err_data.ue_count;
1053 	info->ce_count = obj->err_data.ce_count;
1054 
1055 	if (err_data.ce_count) {
1056 		if (!adev->aid_mask &&
1057 		    adev->smuio.funcs &&
1058 		    adev->smuio.funcs->get_socket_id &&
1059 		    adev->smuio.funcs->get_die_id) {
1060 			dev_info(adev->dev, "socket: %d, die: %d "
1061 					"%ld correctable hardware errors "
1062 					"detected in %s block, no user "
1063 					"action is needed.\n",
1064 					adev->smuio.funcs->get_socket_id(adev),
1065 					adev->smuio.funcs->get_die_id(adev),
1066 					obj->err_data.ce_count,
1067 					get_ras_block_str(&info->head));
1068 		} else {
1069 			dev_info(adev->dev, "%ld correctable hardware errors "
1070 					"detected in %s block, no user "
1071 					"action is needed.\n",
1072 					obj->err_data.ce_count,
1073 					get_ras_block_str(&info->head));
1074 		}
1075 	}
1076 	if (err_data.ue_count) {
1077 		if (!adev->aid_mask &&
1078 		    adev->smuio.funcs &&
1079 		    adev->smuio.funcs->get_socket_id &&
1080 		    adev->smuio.funcs->get_die_id) {
1081 			dev_info(adev->dev, "socket: %d, die: %d "
1082 					"%ld uncorrectable hardware errors "
1083 					"detected in %s block\n",
1084 					adev->smuio.funcs->get_socket_id(adev),
1085 					adev->smuio.funcs->get_die_id(adev),
1086 					obj->err_data.ue_count,
1087 					get_ras_block_str(&info->head));
1088 		} else {
1089 			dev_info(adev->dev, "%ld uncorrectable hardware errors "
1090 					"detected in %s block\n",
1091 					obj->err_data.ue_count,
1092 					get_ras_block_str(&info->head));
1093 		}
1094 	}
1095 
1096 	return 0;
1097 }
1098 
1099 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1100 		enum amdgpu_ras_block block)
1101 {
1102 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1103 
1104 	if (!amdgpu_ras_is_supported(adev, block))
1105 		return -EINVAL;
1106 
1107 	if (!block_obj || !block_obj->hw_ops)   {
1108 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1109 			     ras_block_str(block));
1110 		return -EINVAL;
1111 	}
1112 
1113 	if (block_obj->hw_ops->reset_ras_error_count)
1114 		block_obj->hw_ops->reset_ras_error_count(adev);
1115 
1116 	if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1117 	    (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1118 		if (block_obj->hw_ops->reset_ras_error_status)
1119 			block_obj->hw_ops->reset_ras_error_status(adev);
1120 	}
1121 
1122 	return 0;
1123 }
1124 
1125 /* wrapper of psp_ras_trigger_error */
1126 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1127 		struct ras_inject_if *info)
1128 {
1129 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1130 	struct ta_ras_trigger_error_input block_info = {
1131 		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
1132 		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1133 		.sub_block_index = info->head.sub_block_index,
1134 		.address = info->address,
1135 		.value = info->value,
1136 	};
1137 	int ret = -EINVAL;
1138 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1139 							info->head.block,
1140 							info->head.sub_block_index);
1141 
1142 	/* inject on guest isn't allowed, return success directly */
1143 	if (amdgpu_sriov_vf(adev))
1144 		return 0;
1145 
1146 	if (!obj)
1147 		return -EINVAL;
1148 
1149 	if (!block_obj || !block_obj->hw_ops)	{
1150 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1151 			     get_ras_block_str(&info->head));
1152 		return -EINVAL;
1153 	}
1154 
1155 	/* Calculate XGMI relative offset */
1156 	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1157 	    info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1158 		block_info.address =
1159 			amdgpu_xgmi_get_relative_phy_addr(adev,
1160 							  block_info.address);
1161 	}
1162 
1163 	if (block_obj->hw_ops->ras_error_inject) {
1164 		if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1165 			ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1166 		else /* Special ras_error_inject is defined (e.g: xgmi) */
1167 			ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1168 						info->instance_mask);
1169 	} else {
1170 		/* default path */
1171 		ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1172 	}
1173 
1174 	if (ret)
1175 		dev_err(adev->dev, "ras inject %s failed %d\n",
1176 			get_ras_block_str(&info->head), ret);
1177 
1178 	return ret;
1179 }
1180 
1181 /**
1182  * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1183  * @adev: pointer to AMD GPU device
1184  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1185  * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1186  * @query_info: pointer to ras_query_if
1187  *
1188  * Return 0 for query success or do nothing, otherwise return an error
1189  * on failures
1190  */
1191 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1192 					       unsigned long *ce_count,
1193 					       unsigned long *ue_count,
1194 					       struct ras_query_if *query_info)
1195 {
1196 	int ret;
1197 
1198 	if (!query_info)
1199 		/* do nothing if query_info is not specified */
1200 		return 0;
1201 
1202 	ret = amdgpu_ras_query_error_status(adev, query_info);
1203 	if (ret)
1204 		return ret;
1205 
1206 	*ce_count += query_info->ce_count;
1207 	*ue_count += query_info->ue_count;
1208 
1209 	/* some hardware/IP supports read to clear
1210 	 * no need to explictly reset the err status after the query call */
1211 	if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1212 	    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1213 		if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1214 			dev_warn(adev->dev,
1215 				 "Failed to reset error counter and error status\n");
1216 	}
1217 
1218 	return 0;
1219 }
1220 
1221 /**
1222  * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1223  * @adev: pointer to AMD GPU device
1224  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1225  * @ue_count: pointer to an integer to be set to the count of uncorrectible
1226  * errors.
1227  * @query_info: pointer to ras_query_if if the query request is only for
1228  * specific ip block; if info is NULL, then the qurey request is for
1229  * all the ip blocks that support query ras error counters/status
1230  *
1231  * If set, @ce_count or @ue_count, count and return the corresponding
1232  * error counts in those integer pointers. Return 0 if the device
1233  * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1234  */
1235 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1236 				 unsigned long *ce_count,
1237 				 unsigned long *ue_count,
1238 				 struct ras_query_if *query_info)
1239 {
1240 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1241 	struct ras_manager *obj;
1242 	unsigned long ce, ue;
1243 	int ret;
1244 
1245 	if (!adev->ras_enabled || !con)
1246 		return -EOPNOTSUPP;
1247 
1248 	/* Don't count since no reporting.
1249 	 */
1250 	if (!ce_count && !ue_count)
1251 		return 0;
1252 
1253 	ce = 0;
1254 	ue = 0;
1255 	if (!query_info) {
1256 		/* query all the ip blocks that support ras query interface */
1257 		list_for_each_entry(obj, &con->head, node) {
1258 			struct ras_query_if info = {
1259 				.head = obj->head,
1260 			};
1261 
1262 			ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1263 		}
1264 	} else {
1265 		/* query specific ip block */
1266 		ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1267 	}
1268 
1269 	if (ret)
1270 		return ret;
1271 
1272 	if (ce_count)
1273 		*ce_count = ce;
1274 
1275 	if (ue_count)
1276 		*ue_count = ue;
1277 
1278 	return 0;
1279 }
1280 /* query/inject/cure end */
1281 
1282 
1283 /* sysfs begin */
1284 
1285 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1286 		struct ras_badpage **bps, unsigned int *count);
1287 
1288 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1289 {
1290 	switch (flags) {
1291 	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1292 		return "R";
1293 	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1294 		return "P";
1295 	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1296 	default:
1297 		return "F";
1298 	}
1299 }
1300 
1301 /**
1302  * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1303  *
1304  * It allows user to read the bad pages of vram on the gpu through
1305  * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1306  *
1307  * It outputs multiple lines, and each line stands for one gpu page.
1308  *
1309  * The format of one line is below,
1310  * gpu pfn : gpu page size : flags
1311  *
1312  * gpu pfn and gpu page size are printed in hex format.
1313  * flags can be one of below character,
1314  *
1315  * R: reserved, this gpu page is reserved and not able to use.
1316  *
1317  * P: pending for reserve, this gpu page is marked as bad, will be reserved
1318  * in next window of page_reserve.
1319  *
1320  * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1321  *
1322  * Examples:
1323  *
1324  * .. code-block:: bash
1325  *
1326  *	0x00000001 : 0x00001000 : R
1327  *	0x00000002 : 0x00001000 : P
1328  *
1329  */
1330 
1331 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1332 		struct kobject *kobj, struct bin_attribute *attr,
1333 		char *buf, loff_t ppos, size_t count)
1334 {
1335 	struct amdgpu_ras *con =
1336 		container_of(attr, struct amdgpu_ras, badpages_attr);
1337 	struct amdgpu_device *adev = con->adev;
1338 	const unsigned int element_size =
1339 		sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1340 	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1341 	unsigned int end = div64_ul(ppos + count - 1, element_size);
1342 	ssize_t s = 0;
1343 	struct ras_badpage *bps = NULL;
1344 	unsigned int bps_count = 0;
1345 
1346 	memset(buf, 0, count);
1347 
1348 	if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1349 		return 0;
1350 
1351 	for (; start < end && start < bps_count; start++)
1352 		s += scnprintf(&buf[s], element_size + 1,
1353 				"0x%08x : 0x%08x : %1s\n",
1354 				bps[start].bp,
1355 				bps[start].size,
1356 				amdgpu_ras_badpage_flags_str(bps[start].flags));
1357 
1358 	kfree(bps);
1359 
1360 	return s;
1361 }
1362 
1363 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1364 		struct device_attribute *attr, char *buf)
1365 {
1366 	struct amdgpu_ras *con =
1367 		container_of(attr, struct amdgpu_ras, features_attr);
1368 
1369 	return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1370 }
1371 
1372 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1373 {
1374 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1375 
1376 	sysfs_remove_file_from_group(&adev->dev->kobj,
1377 				&con->badpages_attr.attr,
1378 				RAS_FS_NAME);
1379 }
1380 
1381 static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1382 {
1383 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1384 	struct attribute *attrs[] = {
1385 		&con->features_attr.attr,
1386 		NULL
1387 	};
1388 	struct attribute_group group = {
1389 		.name = RAS_FS_NAME,
1390 		.attrs = attrs,
1391 	};
1392 
1393 	sysfs_remove_group(&adev->dev->kobj, &group);
1394 
1395 	return 0;
1396 }
1397 
1398 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1399 		struct ras_common_if *head)
1400 {
1401 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1402 
1403 	if (!obj || obj->attr_inuse)
1404 		return -EINVAL;
1405 
1406 	get_obj(obj);
1407 
1408 	snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1409 		"%s_err_count", head->name);
1410 
1411 	obj->sysfs_attr = (struct device_attribute){
1412 		.attr = {
1413 			.name = obj->fs_data.sysfs_name,
1414 			.mode = S_IRUGO,
1415 		},
1416 			.show = amdgpu_ras_sysfs_read,
1417 	};
1418 	sysfs_attr_init(&obj->sysfs_attr.attr);
1419 
1420 	if (sysfs_add_file_to_group(&adev->dev->kobj,
1421 				&obj->sysfs_attr.attr,
1422 				RAS_FS_NAME)) {
1423 		put_obj(obj);
1424 		return -EINVAL;
1425 	}
1426 
1427 	obj->attr_inuse = 1;
1428 
1429 	return 0;
1430 }
1431 
1432 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1433 		struct ras_common_if *head)
1434 {
1435 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1436 
1437 	if (!obj || !obj->attr_inuse)
1438 		return -EINVAL;
1439 
1440 	sysfs_remove_file_from_group(&adev->dev->kobj,
1441 				&obj->sysfs_attr.attr,
1442 				RAS_FS_NAME);
1443 	obj->attr_inuse = 0;
1444 	put_obj(obj);
1445 
1446 	return 0;
1447 }
1448 
1449 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1450 {
1451 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1452 	struct ras_manager *obj, *tmp;
1453 
1454 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1455 		amdgpu_ras_sysfs_remove(adev, &obj->head);
1456 	}
1457 
1458 	if (amdgpu_bad_page_threshold != 0)
1459 		amdgpu_ras_sysfs_remove_bad_page_node(adev);
1460 
1461 	amdgpu_ras_sysfs_remove_feature_node(adev);
1462 
1463 	return 0;
1464 }
1465 /* sysfs end */
1466 
1467 /**
1468  * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1469  *
1470  * Normally when there is an uncorrectable error, the driver will reset
1471  * the GPU to recover.  However, in the event of an unrecoverable error,
1472  * the driver provides an interface to reboot the system automatically
1473  * in that event.
1474  *
1475  * The following file in debugfs provides that interface:
1476  * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1477  *
1478  * Usage:
1479  *
1480  * .. code-block:: bash
1481  *
1482  *	echo true > .../ras/auto_reboot
1483  *
1484  */
1485 /* debugfs begin */
1486 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1487 {
1488 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1489 	struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1490 	struct drm_minor  *minor = adev_to_drm(adev)->primary;
1491 	struct dentry     *dir;
1492 
1493 	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1494 	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1495 			    &amdgpu_ras_debugfs_ctrl_ops);
1496 	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1497 			    &amdgpu_ras_debugfs_eeprom_ops);
1498 	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1499 			   &con->bad_page_cnt_threshold);
1500 	debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1501 	debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1502 	debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1503 	debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1504 			    &amdgpu_ras_debugfs_eeprom_size_ops);
1505 	con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1506 						       S_IRUGO, dir, adev,
1507 						       &amdgpu_ras_debugfs_eeprom_table_ops);
1508 	amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1509 
1510 	/*
1511 	 * After one uncorrectable error happens, usually GPU recovery will
1512 	 * be scheduled. But due to the known problem in GPU recovery failing
1513 	 * to bring GPU back, below interface provides one direct way to
1514 	 * user to reboot system automatically in such case within
1515 	 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1516 	 * will never be called.
1517 	 */
1518 	debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1519 
1520 	/*
1521 	 * User could set this not to clean up hardware's error count register
1522 	 * of RAS IPs during ras recovery.
1523 	 */
1524 	debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1525 			    &con->disable_ras_err_cnt_harvest);
1526 	return dir;
1527 }
1528 
1529 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1530 				      struct ras_fs_if *head,
1531 				      struct dentry *dir)
1532 {
1533 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1534 
1535 	if (!obj || !dir)
1536 		return;
1537 
1538 	get_obj(obj);
1539 
1540 	memcpy(obj->fs_data.debugfs_name,
1541 			head->debugfs_name,
1542 			sizeof(obj->fs_data.debugfs_name));
1543 
1544 	debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1545 			    obj, &amdgpu_ras_debugfs_ops);
1546 }
1547 
1548 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1549 {
1550 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1551 	struct dentry *dir;
1552 	struct ras_manager *obj;
1553 	struct ras_fs_if fs_info;
1554 
1555 	/*
1556 	 * it won't be called in resume path, no need to check
1557 	 * suspend and gpu reset status
1558 	 */
1559 	if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
1560 		return;
1561 
1562 	dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
1563 
1564 	list_for_each_entry(obj, &con->head, node) {
1565 		if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1566 			(obj->attr_inuse == 1)) {
1567 			sprintf(fs_info.debugfs_name, "%s_err_inject",
1568 					get_ras_block_str(&obj->head));
1569 			fs_info.head = obj->head;
1570 			amdgpu_ras_debugfs_create(adev, &fs_info, dir);
1571 		}
1572 	}
1573 }
1574 
1575 /* debugfs end */
1576 
1577 /* ras fs */
1578 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1579 		amdgpu_ras_sysfs_badpages_read, NULL, 0);
1580 static DEVICE_ATTR(features, S_IRUGO,
1581 		amdgpu_ras_sysfs_features_read, NULL);
1582 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1583 {
1584 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1585 	struct attribute_group group = {
1586 		.name = RAS_FS_NAME,
1587 	};
1588 	struct attribute *attrs[] = {
1589 		&con->features_attr.attr,
1590 		NULL
1591 	};
1592 	struct bin_attribute *bin_attrs[] = {
1593 		NULL,
1594 		NULL,
1595 	};
1596 	int r;
1597 
1598 	/* add features entry */
1599 	con->features_attr = dev_attr_features;
1600 	group.attrs = attrs;
1601 	sysfs_attr_init(attrs[0]);
1602 
1603 	if (amdgpu_bad_page_threshold != 0) {
1604 		/* add bad_page_features entry */
1605 		bin_attr_gpu_vram_bad_pages.private = NULL;
1606 		con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1607 		bin_attrs[0] = &con->badpages_attr;
1608 		group.bin_attrs = bin_attrs;
1609 		sysfs_bin_attr_init(bin_attrs[0]);
1610 	}
1611 
1612 	r = sysfs_create_group(&adev->dev->kobj, &group);
1613 	if (r)
1614 		dev_err(adev->dev, "Failed to create RAS sysfs group!");
1615 
1616 	return 0;
1617 }
1618 
1619 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1620 {
1621 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1622 	struct ras_manager *con_obj, *ip_obj, *tmp;
1623 
1624 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1625 		list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1626 			ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1627 			if (ip_obj)
1628 				put_obj(ip_obj);
1629 		}
1630 	}
1631 
1632 	amdgpu_ras_sysfs_remove_all(adev);
1633 	return 0;
1634 }
1635 /* ras fs end */
1636 
1637 /* ih begin */
1638 
1639 /* For the hardware that cannot enable bif ring for both ras_controller_irq
1640  * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1641  * register to check whether the interrupt is triggered or not, and properly
1642  * ack the interrupt if it is there
1643  */
1644 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1645 {
1646 	/* Fatal error events are handled on host side */
1647 	if (amdgpu_sriov_vf(adev))
1648 		return;
1649 
1650 	if (adev->nbio.ras &&
1651 	    adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1652 		adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1653 
1654 	if (adev->nbio.ras &&
1655 	    adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1656 		adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1657 }
1658 
1659 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1660 				struct amdgpu_iv_entry *entry)
1661 {
1662 	bool poison_stat = false;
1663 	struct amdgpu_device *adev = obj->adev;
1664 	struct amdgpu_ras_block_object *block_obj =
1665 		amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1666 
1667 	if (!block_obj)
1668 		return;
1669 
1670 	/* both query_poison_status and handle_poison_consumption are optional,
1671 	 * but at least one of them should be implemented if we need poison
1672 	 * consumption handler
1673 	 */
1674 	if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
1675 		poison_stat = block_obj->hw_ops->query_poison_status(adev);
1676 		if (!poison_stat) {
1677 			/* Not poison consumption interrupt, no need to handle it */
1678 			dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1679 					block_obj->ras_comm.name);
1680 
1681 			return;
1682 		}
1683 	}
1684 
1685 	amdgpu_umc_poison_handler(adev, false);
1686 
1687 	if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
1688 		poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1689 
1690 	/* gpu reset is fallback for failed and default cases */
1691 	if (poison_stat) {
1692 		dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1693 				block_obj->ras_comm.name);
1694 		amdgpu_ras_reset_gpu(adev);
1695 	} else {
1696 		amdgpu_gfx_poison_consumption_handler(adev, entry);
1697 	}
1698 }
1699 
1700 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1701 				struct amdgpu_iv_entry *entry)
1702 {
1703 	dev_info(obj->adev->dev,
1704 		"Poison is created, no user action is needed.\n");
1705 }
1706 
1707 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1708 				struct amdgpu_iv_entry *entry)
1709 {
1710 	struct ras_ih_data *data = &obj->ih_data;
1711 	struct ras_err_data err_data = {0, 0, 0, NULL};
1712 	int ret;
1713 
1714 	if (!data->cb)
1715 		return;
1716 
1717 	/* Let IP handle its data, maybe we need get the output
1718 	 * from the callback to update the error type/count, etc
1719 	 */
1720 	ret = data->cb(obj->adev, &err_data, entry);
1721 	/* ue will trigger an interrupt, and in that case
1722 	 * we need do a reset to recovery the whole system.
1723 	 * But leave IP do that recovery, here we just dispatch
1724 	 * the error.
1725 	 */
1726 	if (ret == AMDGPU_RAS_SUCCESS) {
1727 		/* these counts could be left as 0 if
1728 		 * some blocks do not count error number
1729 		 */
1730 		obj->err_data.ue_count += err_data.ue_count;
1731 		obj->err_data.ce_count += err_data.ce_count;
1732 	}
1733 }
1734 
1735 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1736 {
1737 	struct ras_ih_data *data = &obj->ih_data;
1738 	struct amdgpu_iv_entry entry;
1739 
1740 	while (data->rptr != data->wptr) {
1741 		rmb();
1742 		memcpy(&entry, &data->ring[data->rptr],
1743 				data->element_size);
1744 
1745 		wmb();
1746 		data->rptr = (data->aligned_element_size +
1747 				data->rptr) % data->ring_size;
1748 
1749 		if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1750 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1751 				amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
1752 			else
1753 				amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
1754 		} else {
1755 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1756 				amdgpu_ras_interrupt_umc_handler(obj, &entry);
1757 			else
1758 				dev_warn(obj->adev->dev,
1759 					"No RAS interrupt handler for non-UMC block with poison disabled.\n");
1760 		}
1761 	}
1762 }
1763 
1764 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1765 {
1766 	struct ras_ih_data *data =
1767 		container_of(work, struct ras_ih_data, ih_work);
1768 	struct ras_manager *obj =
1769 		container_of(data, struct ras_manager, ih_data);
1770 
1771 	amdgpu_ras_interrupt_handler(obj);
1772 }
1773 
1774 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1775 		struct ras_dispatch_if *info)
1776 {
1777 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1778 	struct ras_ih_data *data = &obj->ih_data;
1779 
1780 	if (!obj)
1781 		return -EINVAL;
1782 
1783 	if (data->inuse == 0)
1784 		return 0;
1785 
1786 	/* Might be overflow... */
1787 	memcpy(&data->ring[data->wptr], info->entry,
1788 			data->element_size);
1789 
1790 	wmb();
1791 	data->wptr = (data->aligned_element_size +
1792 			data->wptr) % data->ring_size;
1793 
1794 	schedule_work(&data->ih_work);
1795 
1796 	return 0;
1797 }
1798 
1799 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
1800 		struct ras_common_if *head)
1801 {
1802 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1803 	struct ras_ih_data *data;
1804 
1805 	if (!obj)
1806 		return -EINVAL;
1807 
1808 	data = &obj->ih_data;
1809 	if (data->inuse == 0)
1810 		return 0;
1811 
1812 	cancel_work_sync(&data->ih_work);
1813 
1814 	kfree(data->ring);
1815 	memset(data, 0, sizeof(*data));
1816 	put_obj(obj);
1817 
1818 	return 0;
1819 }
1820 
1821 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
1822 		struct ras_common_if *head)
1823 {
1824 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1825 	struct ras_ih_data *data;
1826 	struct amdgpu_ras_block_object *ras_obj;
1827 
1828 	if (!obj) {
1829 		/* in case we registe the IH before enable ras feature */
1830 		obj = amdgpu_ras_create_obj(adev, head);
1831 		if (!obj)
1832 			return -EINVAL;
1833 	} else
1834 		get_obj(obj);
1835 
1836 	ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1837 
1838 	data = &obj->ih_data;
1839 	/* add the callback.etc */
1840 	*data = (struct ras_ih_data) {
1841 		.inuse = 0,
1842 		.cb = ras_obj->ras_cb,
1843 		.element_size = sizeof(struct amdgpu_iv_entry),
1844 		.rptr = 0,
1845 		.wptr = 0,
1846 	};
1847 
1848 	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1849 
1850 	data->aligned_element_size = ALIGN(data->element_size, 8);
1851 	/* the ring can store 64 iv entries. */
1852 	data->ring_size = 64 * data->aligned_element_size;
1853 	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1854 	if (!data->ring) {
1855 		put_obj(obj);
1856 		return -ENOMEM;
1857 	}
1858 
1859 	/* IH is ready */
1860 	data->inuse = 1;
1861 
1862 	return 0;
1863 }
1864 
1865 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1866 {
1867 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1868 	struct ras_manager *obj, *tmp;
1869 
1870 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1871 		amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
1872 	}
1873 
1874 	return 0;
1875 }
1876 /* ih end */
1877 
1878 /* traversal all IPs except NBIO to query error counter */
1879 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1880 {
1881 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1882 	struct ras_manager *obj;
1883 
1884 	if (!adev->ras_enabled || !con)
1885 		return;
1886 
1887 	list_for_each_entry(obj, &con->head, node) {
1888 		struct ras_query_if info = {
1889 			.head = obj->head,
1890 		};
1891 
1892 		/*
1893 		 * PCIE_BIF IP has one different isr by ras controller
1894 		 * interrupt, the specific ras counter query will be
1895 		 * done in that isr. So skip such block from common
1896 		 * sync flood interrupt isr calling.
1897 		 */
1898 		if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1899 			continue;
1900 
1901 		/*
1902 		 * this is a workaround for aldebaran, skip send msg to
1903 		 * smu to get ecc_info table due to smu handle get ecc
1904 		 * info table failed temporarily.
1905 		 * should be removed until smu fix handle ecc_info table.
1906 		 */
1907 		if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1908 			(adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1909 			continue;
1910 
1911 		amdgpu_ras_query_error_status(adev, &info);
1912 
1913 		if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1914 		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1915 		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
1916 			if (amdgpu_ras_reset_error_status(adev, info.head.block))
1917 				dev_warn(adev->dev, "Failed to reset error counter and error status");
1918 		}
1919 	}
1920 }
1921 
1922 /* Parse RdRspStatus and WrRspStatus */
1923 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1924 					  struct ras_query_if *info)
1925 {
1926 	struct amdgpu_ras_block_object *block_obj;
1927 	/*
1928 	 * Only two block need to query read/write
1929 	 * RspStatus at current state
1930 	 */
1931 	if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1932 		(info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
1933 		return;
1934 
1935 	block_obj = amdgpu_ras_get_ras_block(adev,
1936 					info->head.block,
1937 					info->head.sub_block_index);
1938 
1939 	if (!block_obj || !block_obj->hw_ops) {
1940 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1941 			     get_ras_block_str(&info->head));
1942 		return;
1943 	}
1944 
1945 	if (block_obj->hw_ops->query_ras_error_status)
1946 		block_obj->hw_ops->query_ras_error_status(adev);
1947 
1948 }
1949 
1950 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1951 {
1952 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1953 	struct ras_manager *obj;
1954 
1955 	if (!adev->ras_enabled || !con)
1956 		return;
1957 
1958 	list_for_each_entry(obj, &con->head, node) {
1959 		struct ras_query_if info = {
1960 			.head = obj->head,
1961 		};
1962 
1963 		amdgpu_ras_error_status_query(adev, &info);
1964 	}
1965 }
1966 
1967 /* recovery begin */
1968 
1969 /* return 0 on success.
1970  * caller need free bps.
1971  */
1972 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1973 		struct ras_badpage **bps, unsigned int *count)
1974 {
1975 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1976 	struct ras_err_handler_data *data;
1977 	int i = 0;
1978 	int ret = 0, status;
1979 
1980 	if (!con || !con->eh_data || !bps || !count)
1981 		return -EINVAL;
1982 
1983 	mutex_lock(&con->recovery_lock);
1984 	data = con->eh_data;
1985 	if (!data || data->count == 0) {
1986 		*bps = NULL;
1987 		ret = -EINVAL;
1988 		goto out;
1989 	}
1990 
1991 	*bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1992 	if (!*bps) {
1993 		ret = -ENOMEM;
1994 		goto out;
1995 	}
1996 
1997 	for (; i < data->count; i++) {
1998 		(*bps)[i] = (struct ras_badpage){
1999 			.bp = data->bps[i].retired_page,
2000 			.size = AMDGPU_GPU_PAGE_SIZE,
2001 			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2002 		};
2003 		status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2004 				data->bps[i].retired_page);
2005 		if (status == -EBUSY)
2006 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2007 		else if (status == -ENOENT)
2008 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2009 	}
2010 
2011 	*count = data->count;
2012 out:
2013 	mutex_unlock(&con->recovery_lock);
2014 	return ret;
2015 }
2016 
2017 static void amdgpu_ras_do_recovery(struct work_struct *work)
2018 {
2019 	struct amdgpu_ras *ras =
2020 		container_of(work, struct amdgpu_ras, recovery_work);
2021 	struct amdgpu_device *remote_adev = NULL;
2022 	struct amdgpu_device *adev = ras->adev;
2023 	struct list_head device_list, *device_list_handle =  NULL;
2024 
2025 	if (!ras->disable_ras_err_cnt_harvest) {
2026 		struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2027 
2028 		/* Build list of devices to query RAS related errors */
2029 		if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2030 			device_list_handle = &hive->device_list;
2031 		} else {
2032 			INIT_LIST_HEAD(&device_list);
2033 			list_add_tail(&adev->gmc.xgmi.head, &device_list);
2034 			device_list_handle = &device_list;
2035 		}
2036 
2037 		list_for_each_entry(remote_adev,
2038 				device_list_handle, gmc.xgmi.head) {
2039 			amdgpu_ras_query_err_status(remote_adev);
2040 			amdgpu_ras_log_on_err_counter(remote_adev);
2041 		}
2042 
2043 		amdgpu_put_xgmi_hive(hive);
2044 	}
2045 
2046 	if (amdgpu_device_should_recover_gpu(ras->adev)) {
2047 		struct amdgpu_reset_context reset_context;
2048 		memset(&reset_context, 0, sizeof(reset_context));
2049 
2050 		reset_context.method = AMD_RESET_METHOD_NONE;
2051 		reset_context.reset_req_dev = adev;
2052 
2053 		/* Perform full reset in fatal error mode */
2054 		if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2055 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2056 		else {
2057 			clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2058 
2059 			if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2060 				ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2061 				reset_context.method = AMD_RESET_METHOD_MODE2;
2062 			}
2063 
2064 			/* Fatal error occurs in poison mode, mode1 reset is used to
2065 			 * recover gpu.
2066 			 */
2067 			if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2068 				ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2069 				set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2070 
2071 				psp_fatal_error_recovery_quirk(&adev->psp);
2072 			}
2073 		}
2074 
2075 		amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2076 	}
2077 	atomic_set(&ras->in_recovery, 0);
2078 }
2079 
2080 /* alloc/realloc bps array */
2081 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2082 		struct ras_err_handler_data *data, int pages)
2083 {
2084 	unsigned int old_space = data->count + data->space_left;
2085 	unsigned int new_space = old_space + pages;
2086 	unsigned int align_space = ALIGN(new_space, 512);
2087 	void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2088 
2089 	if (!bps) {
2090 		return -ENOMEM;
2091 	}
2092 
2093 	if (data->bps) {
2094 		memcpy(bps, data->bps,
2095 				data->count * sizeof(*data->bps));
2096 		kfree(data->bps);
2097 	}
2098 
2099 	data->bps = bps;
2100 	data->space_left += align_space - old_space;
2101 	return 0;
2102 }
2103 
2104 /* it deal with vram only. */
2105 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2106 		struct eeprom_table_record *bps, int pages)
2107 {
2108 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2109 	struct ras_err_handler_data *data;
2110 	int ret = 0;
2111 	uint32_t i;
2112 
2113 	if (!con || !con->eh_data || !bps || pages <= 0)
2114 		return 0;
2115 
2116 	mutex_lock(&con->recovery_lock);
2117 	data = con->eh_data;
2118 	if (!data)
2119 		goto out;
2120 
2121 	for (i = 0; i < pages; i++) {
2122 		if (amdgpu_ras_check_bad_page_unlock(con,
2123 			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2124 			continue;
2125 
2126 		if (!data->space_left &&
2127 			amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2128 			ret = -ENOMEM;
2129 			goto out;
2130 		}
2131 
2132 		amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
2133 			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2134 			AMDGPU_GPU_PAGE_SIZE);
2135 
2136 		memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2137 		data->count++;
2138 		data->space_left--;
2139 	}
2140 out:
2141 	mutex_unlock(&con->recovery_lock);
2142 
2143 	return ret;
2144 }
2145 
2146 /*
2147  * write error record array to eeprom, the function should be
2148  * protected by recovery_lock
2149  * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
2150  */
2151 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2152 		unsigned long *new_cnt)
2153 {
2154 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2155 	struct ras_err_handler_data *data;
2156 	struct amdgpu_ras_eeprom_control *control;
2157 	int save_count;
2158 
2159 	if (!con || !con->eh_data) {
2160 		if (new_cnt)
2161 			*new_cnt = 0;
2162 
2163 		return 0;
2164 	}
2165 
2166 	mutex_lock(&con->recovery_lock);
2167 	control = &con->eeprom_control;
2168 	data = con->eh_data;
2169 	save_count = data->count - control->ras_num_recs;
2170 	mutex_unlock(&con->recovery_lock);
2171 
2172 	if (new_cnt)
2173 		*new_cnt = save_count / adev->umc.retire_unit;
2174 
2175 	/* only new entries are saved */
2176 	if (save_count > 0) {
2177 		if (amdgpu_ras_eeprom_append(control,
2178 					     &data->bps[control->ras_num_recs],
2179 					     save_count)) {
2180 			dev_err(adev->dev, "Failed to save EEPROM table data!");
2181 			return -EIO;
2182 		}
2183 
2184 		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2185 	}
2186 
2187 	return 0;
2188 }
2189 
2190 /*
2191  * read error record array in eeprom and reserve enough space for
2192  * storing new bad pages
2193  */
2194 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2195 {
2196 	struct amdgpu_ras_eeprom_control *control =
2197 		&adev->psp.ras_context.ras->eeprom_control;
2198 	struct eeprom_table_record *bps;
2199 	int ret;
2200 
2201 	/* no bad page record, skip eeprom access */
2202 	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
2203 		return 0;
2204 
2205 	bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
2206 	if (!bps)
2207 		return -ENOMEM;
2208 
2209 	ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
2210 	if (ret)
2211 		dev_err(adev->dev, "Failed to load EEPROM table records!");
2212 	else
2213 		ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
2214 
2215 	kfree(bps);
2216 	return ret;
2217 }
2218 
2219 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2220 				uint64_t addr)
2221 {
2222 	struct ras_err_handler_data *data = con->eh_data;
2223 	int i;
2224 
2225 	addr >>= AMDGPU_GPU_PAGE_SHIFT;
2226 	for (i = 0; i < data->count; i++)
2227 		if (addr == data->bps[i].retired_page)
2228 			return true;
2229 
2230 	return false;
2231 }
2232 
2233 /*
2234  * check if an address belongs to bad page
2235  *
2236  * Note: this check is only for umc block
2237  */
2238 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2239 				uint64_t addr)
2240 {
2241 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2242 	bool ret = false;
2243 
2244 	if (!con || !con->eh_data)
2245 		return ret;
2246 
2247 	mutex_lock(&con->recovery_lock);
2248 	ret = amdgpu_ras_check_bad_page_unlock(con, addr);
2249 	mutex_unlock(&con->recovery_lock);
2250 	return ret;
2251 }
2252 
2253 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
2254 					  uint32_t max_count)
2255 {
2256 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2257 
2258 	/*
2259 	 * Justification of value bad_page_cnt_threshold in ras structure
2260 	 *
2261 	 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
2262 	 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
2263 	 * scenarios accordingly.
2264 	 *
2265 	 * Bad page retirement enablement:
2266 	 *    - If amdgpu_bad_page_threshold = -2,
2267 	 *      bad_page_cnt_threshold = typical value by formula.
2268 	 *
2269 	 *    - When the value from user is 0 < amdgpu_bad_page_threshold <
2270 	 *      max record length in eeprom, use it directly.
2271 	 *
2272 	 * Bad page retirement disablement:
2273 	 *    - If amdgpu_bad_page_threshold = 0, bad page retirement
2274 	 *      functionality is disabled, and bad_page_cnt_threshold will
2275 	 *      take no effect.
2276 	 */
2277 
2278 	if (amdgpu_bad_page_threshold < 0) {
2279 		u64 val = adev->gmc.mc_vram_size;
2280 
2281 		do_div(val, RAS_BAD_PAGE_COVER);
2282 		con->bad_page_cnt_threshold = min(lower_32_bits(val),
2283 						  max_count);
2284 	} else {
2285 		con->bad_page_cnt_threshold = min_t(int, max_count,
2286 						    amdgpu_bad_page_threshold);
2287 	}
2288 }
2289 
2290 int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
2291 {
2292 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2293 	struct ras_err_handler_data **data;
2294 	u32  max_eeprom_records_count = 0;
2295 	bool exc_err_limit = false;
2296 	int ret;
2297 
2298 	if (!con || amdgpu_sriov_vf(adev))
2299 		return 0;
2300 
2301 	/* Allow access to RAS EEPROM via debugfs, when the ASIC
2302 	 * supports RAS and debugfs is enabled, but when
2303 	 * adev->ras_enabled is unset, i.e. when "ras_enable"
2304 	 * module parameter is set to 0.
2305 	 */
2306 	con->adev = adev;
2307 
2308 	if (!adev->ras_enabled)
2309 		return 0;
2310 
2311 	data = &con->eh_data;
2312 	*data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2313 	if (!*data) {
2314 		ret = -ENOMEM;
2315 		goto out;
2316 	}
2317 
2318 	mutex_init(&con->recovery_lock);
2319 	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2320 	atomic_set(&con->in_recovery, 0);
2321 	con->eeprom_control.bad_channel_bitmap = 0;
2322 
2323 	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
2324 	amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
2325 
2326 	/* Todo: During test the SMU might fail to read the eeprom through I2C
2327 	 * when the GPU is pending on XGMI reset during probe time
2328 	 * (Mostly after second bus reset), skip it now
2329 	 */
2330 	if (adev->gmc.xgmi.pending_reset)
2331 		return 0;
2332 	ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2333 	/*
2334 	 * This calling fails when exc_err_limit is true or
2335 	 * ret != 0.
2336 	 */
2337 	if (exc_err_limit || ret)
2338 		goto free;
2339 
2340 	if (con->eeprom_control.ras_num_recs) {
2341 		ret = amdgpu_ras_load_bad_pages(adev);
2342 		if (ret)
2343 			goto free;
2344 
2345 		amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
2346 
2347 		if (con->update_channel_flag == true) {
2348 			amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2349 			con->update_channel_flag = false;
2350 		}
2351 	}
2352 
2353 #ifdef CONFIG_X86_MCE_AMD
2354 	if ((adev->asic_type == CHIP_ALDEBARAN) &&
2355 	    (adev->gmc.xgmi.connected_to_cpu))
2356 		amdgpu_register_bad_pages_mca_notifier(adev);
2357 #endif
2358 	return 0;
2359 
2360 free:
2361 	kfree((*data)->bps);
2362 	kfree(*data);
2363 	con->eh_data = NULL;
2364 out:
2365 	dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
2366 
2367 	/*
2368 	 * Except error threshold exceeding case, other failure cases in this
2369 	 * function would not fail amdgpu driver init.
2370 	 */
2371 	if (!exc_err_limit)
2372 		ret = 0;
2373 	else
2374 		ret = -EINVAL;
2375 
2376 	return ret;
2377 }
2378 
2379 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2380 {
2381 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2382 	struct ras_err_handler_data *data = con->eh_data;
2383 
2384 	/* recovery_init failed to init it, fini is useless */
2385 	if (!data)
2386 		return 0;
2387 
2388 	cancel_work_sync(&con->recovery_work);
2389 
2390 	mutex_lock(&con->recovery_lock);
2391 	con->eh_data = NULL;
2392 	kfree(data->bps);
2393 	kfree(data);
2394 	mutex_unlock(&con->recovery_lock);
2395 
2396 	return 0;
2397 }
2398 /* recovery end */
2399 
2400 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
2401 {
2402 	if (amdgpu_sriov_vf(adev)) {
2403 		switch (adev->ip_versions[MP0_HWIP][0]) {
2404 		case IP_VERSION(13, 0, 2):
2405 		case IP_VERSION(13, 0, 6):
2406 			return true;
2407 		default:
2408 			return false;
2409 		}
2410 	}
2411 
2412 	if (adev->asic_type == CHIP_IP_DISCOVERY) {
2413 		switch (adev->ip_versions[MP0_HWIP][0]) {
2414 		case IP_VERSION(13, 0, 0):
2415 		case IP_VERSION(13, 0, 6):
2416 		case IP_VERSION(13, 0, 10):
2417 			return true;
2418 		default:
2419 			return false;
2420 		}
2421 	}
2422 
2423 	return adev->asic_type == CHIP_VEGA10 ||
2424 		adev->asic_type == CHIP_VEGA20 ||
2425 		adev->asic_type == CHIP_ARCTURUS ||
2426 		adev->asic_type == CHIP_ALDEBARAN ||
2427 		adev->asic_type == CHIP_SIENNA_CICHLID;
2428 }
2429 
2430 /*
2431  * this is workaround for vega20 workstation sku,
2432  * force enable gfx ras, ignore vbios gfx ras flag
2433  * due to GC EDC can not write
2434  */
2435 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
2436 {
2437 	struct atom_context *ctx = adev->mode_info.atom_context;
2438 
2439 	if (!ctx)
2440 		return;
2441 
2442 	if (strnstr(ctx->vbios_pn, "D16406",
2443 		    sizeof(ctx->vbios_pn)) ||
2444 		strnstr(ctx->vbios_pn, "D36002",
2445 			sizeof(ctx->vbios_pn)))
2446 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
2447 }
2448 
2449 /*
2450  * check hardware's ras ability which will be saved in hw_supported.
2451  * if hardware does not support ras, we can skip some ras initializtion and
2452  * forbid some ras operations from IP.
2453  * if software itself, say boot parameter, limit the ras ability. We still
2454  * need allow IP do some limited operations, like disable. In such case,
2455  * we have to initialize ras as normal. but need check if operation is
2456  * allowed or not in each function.
2457  */
2458 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2459 {
2460 	adev->ras_hw_enabled = adev->ras_enabled = 0;
2461 
2462 	if (!amdgpu_ras_asic_supported(adev))
2463 		return;
2464 
2465 	if (!adev->gmc.xgmi.connected_to_cpu &&	!adev->gmc.is_app_apu) {
2466 		if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2467 			dev_info(adev->dev, "MEM ECC is active.\n");
2468 			adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
2469 						   1 << AMDGPU_RAS_BLOCK__DF);
2470 		} else {
2471 			dev_info(adev->dev, "MEM ECC is not presented.\n");
2472 		}
2473 
2474 		if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2475 			dev_info(adev->dev, "SRAM ECC is active.\n");
2476 			if (!amdgpu_sriov_vf(adev))
2477 				adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2478 							    1 << AMDGPU_RAS_BLOCK__DF);
2479 			else
2480 				adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2481 								1 << AMDGPU_RAS_BLOCK__SDMA |
2482 								1 << AMDGPU_RAS_BLOCK__GFX);
2483 
2484 			/* VCN/JPEG RAS can be supported on both bare metal and
2485 			 * SRIOV environment
2486 			 */
2487 			if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0) ||
2488 			    adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 0))
2489 				adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2490 							1 << AMDGPU_RAS_BLOCK__JPEG);
2491 			else
2492 				adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2493 							1 << AMDGPU_RAS_BLOCK__JPEG);
2494 
2495 			/*
2496 			 * XGMI RAS is not supported if xgmi num physical nodes
2497 			 * is zero
2498 			 */
2499 			if (!adev->gmc.xgmi.num_physical_nodes)
2500 				adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
2501 		} else {
2502 			dev_info(adev->dev, "SRAM ECC is not presented.\n");
2503 		}
2504 	} else {
2505 		/* driver only manages a few IP blocks RAS feature
2506 		 * when GPU is connected cpu through XGMI */
2507 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
2508 					   1 << AMDGPU_RAS_BLOCK__SDMA |
2509 					   1 << AMDGPU_RAS_BLOCK__MMHUB);
2510 	}
2511 
2512 	amdgpu_ras_get_quirks(adev);
2513 
2514 	/* hw_supported needs to be aligned with RAS block mask. */
2515 	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2516 
2517 
2518 	/*
2519 	 * Disable ras feature for aqua vanjaram
2520 	 * by default on apu platform.
2521 	 */
2522 	if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 6) &&
2523 	    adev->gmc.is_app_apu)
2524 		adev->ras_enabled = amdgpu_ras_enable != 1 ? 0 :
2525 			adev->ras_hw_enabled & amdgpu_ras_mask;
2526 	else
2527 		adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2528 			adev->ras_hw_enabled & amdgpu_ras_mask;
2529 }
2530 
2531 static void amdgpu_ras_counte_dw(struct work_struct *work)
2532 {
2533 	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2534 					      ras_counte_delay_work.work);
2535 	struct amdgpu_device *adev = con->adev;
2536 	struct drm_device *dev = adev_to_drm(adev);
2537 	unsigned long ce_count, ue_count;
2538 	int res;
2539 
2540 	res = pm_runtime_get_sync(dev->dev);
2541 	if (res < 0)
2542 		goto Out;
2543 
2544 	/* Cache new values.
2545 	 */
2546 	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
2547 		atomic_set(&con->ras_ce_count, ce_count);
2548 		atomic_set(&con->ras_ue_count, ue_count);
2549 	}
2550 
2551 	pm_runtime_mark_last_busy(dev->dev);
2552 Out:
2553 	pm_runtime_put_autosuspend(dev->dev);
2554 }
2555 
2556 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
2557 {
2558 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2559 	bool df_poison, umc_poison;
2560 
2561 	/* poison setting is useless on SRIOV guest */
2562 	if (amdgpu_sriov_vf(adev) || !con)
2563 		return;
2564 
2565 	/* Init poison supported flag, the default value is false */
2566 	if (adev->gmc.xgmi.connected_to_cpu) {
2567 		/* enabled by default when GPU is connected to CPU */
2568 		con->poison_supported = true;
2569 	} else if (adev->df.funcs &&
2570 	    adev->df.funcs->query_ras_poison_mode &&
2571 	    adev->umc.ras &&
2572 	    adev->umc.ras->query_ras_poison_mode) {
2573 		df_poison =
2574 			adev->df.funcs->query_ras_poison_mode(adev);
2575 		umc_poison =
2576 			adev->umc.ras->query_ras_poison_mode(adev);
2577 
2578 		/* Only poison is set in both DF and UMC, we can support it */
2579 		if (df_poison && umc_poison)
2580 			con->poison_supported = true;
2581 		else if (df_poison != umc_poison)
2582 			dev_warn(adev->dev,
2583 				"Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2584 				df_poison, umc_poison);
2585 	}
2586 }
2587 
2588 int amdgpu_ras_init(struct amdgpu_device *adev)
2589 {
2590 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2591 	int r;
2592 
2593 	if (con)
2594 		return 0;
2595 
2596 	con = kmalloc(sizeof(struct amdgpu_ras) +
2597 			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2598 			sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
2599 			GFP_KERNEL|__GFP_ZERO);
2600 	if (!con)
2601 		return -ENOMEM;
2602 
2603 	con->adev = adev;
2604 	INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2605 	atomic_set(&con->ras_ce_count, 0);
2606 	atomic_set(&con->ras_ue_count, 0);
2607 
2608 	con->objs = (struct ras_manager *)(con + 1);
2609 
2610 	amdgpu_ras_set_context(adev, con);
2611 
2612 	amdgpu_ras_check_supported(adev);
2613 
2614 	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
2615 		/* set gfx block ras context feature for VEGA20 Gaming
2616 		 * send ras disable cmd to ras ta during ras late init.
2617 		 */
2618 		if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2619 			con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2620 
2621 			return 0;
2622 		}
2623 
2624 		r = 0;
2625 		goto release_con;
2626 	}
2627 
2628 	con->update_channel_flag = false;
2629 	con->features = 0;
2630 	INIT_LIST_HEAD(&con->head);
2631 	/* Might need get this flag from vbios. */
2632 	con->flags = RAS_DEFAULT_FLAGS;
2633 
2634 	/* initialize nbio ras function ahead of any other
2635 	 * ras functions so hardware fatal error interrupt
2636 	 * can be enabled as early as possible */
2637 	switch (adev->ip_versions[NBIO_HWIP][0]) {
2638 	case IP_VERSION(7, 4, 0):
2639 	case IP_VERSION(7, 4, 1):
2640 	case IP_VERSION(7, 4, 4):
2641 		if (!adev->gmc.xgmi.connected_to_cpu)
2642 			adev->nbio.ras = &nbio_v7_4_ras;
2643 		break;
2644 	case IP_VERSION(4, 3, 0):
2645 		if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
2646 			/* unlike other generation of nbio ras,
2647 			 * nbio v4_3 only support fatal error interrupt
2648 			 * to inform software that DF is freezed due to
2649 			 * system fatal error event. driver should not
2650 			 * enable nbio ras in such case. Instead,
2651 			 * check DF RAS */
2652 			adev->nbio.ras = &nbio_v4_3_ras;
2653 		break;
2654 	case IP_VERSION(7, 9, 0):
2655 		if (!adev->gmc.is_app_apu)
2656 			adev->nbio.ras = &nbio_v7_9_ras;
2657 		break;
2658 	default:
2659 		/* nbio ras is not available */
2660 		break;
2661 	}
2662 
2663 	/* nbio ras block needs to be enabled ahead of other ras blocks
2664 	 * to handle fatal error */
2665 	r = amdgpu_nbio_ras_sw_init(adev);
2666 	if (r)
2667 		return r;
2668 
2669 	if (adev->nbio.ras &&
2670 	    adev->nbio.ras->init_ras_controller_interrupt) {
2671 		r = adev->nbio.ras->init_ras_controller_interrupt(adev);
2672 		if (r)
2673 			goto release_con;
2674 	}
2675 
2676 	if (adev->nbio.ras &&
2677 	    adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2678 		r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
2679 		if (r)
2680 			goto release_con;
2681 	}
2682 
2683 	amdgpu_ras_query_poison_mode(adev);
2684 
2685 	if (amdgpu_ras_fs_init(adev)) {
2686 		r = -EINVAL;
2687 		goto release_con;
2688 	}
2689 
2690 	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2691 		 "hardware ability[%x] ras_mask[%x]\n",
2692 		 adev->ras_hw_enabled, adev->ras_enabled);
2693 
2694 	return 0;
2695 release_con:
2696 	amdgpu_ras_set_context(adev, NULL);
2697 	kfree(con);
2698 
2699 	return r;
2700 }
2701 
2702 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
2703 {
2704 	if (adev->gmc.xgmi.connected_to_cpu ||
2705 	    adev->gmc.is_app_apu)
2706 		return 1;
2707 	return 0;
2708 }
2709 
2710 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2711 					struct ras_common_if *ras_block)
2712 {
2713 	struct ras_query_if info = {
2714 		.head = *ras_block,
2715 	};
2716 
2717 	if (!amdgpu_persistent_edc_harvesting_supported(adev))
2718 		return 0;
2719 
2720 	if (amdgpu_ras_query_error_status(adev, &info) != 0)
2721 		DRM_WARN("RAS init harvest failure");
2722 
2723 	if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2724 		DRM_WARN("RAS init harvest reset failure");
2725 
2726 	return 0;
2727 }
2728 
2729 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2730 {
2731        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2732 
2733        if (!con)
2734                return false;
2735 
2736        return con->poison_supported;
2737 }
2738 
2739 /* helper function to handle common stuff in ip late init phase */
2740 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2741 			 struct ras_common_if *ras_block)
2742 {
2743 	struct amdgpu_ras_block_object *ras_obj = NULL;
2744 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2745 	struct ras_query_if *query_info;
2746 	unsigned long ue_count, ce_count;
2747 	int r;
2748 
2749 	/* disable RAS feature per IP block if it is not supported */
2750 	if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2751 		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2752 		return 0;
2753 	}
2754 
2755 	r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2756 	if (r) {
2757 		if (adev->in_suspend || amdgpu_in_reset(adev)) {
2758 			/* in resume phase, if fail to enable ras,
2759 			 * clean up all ras fs nodes, and disable ras */
2760 			goto cleanup;
2761 		} else
2762 			return r;
2763 	}
2764 
2765 	/* check for errors on warm reset edc persisant supported ASIC */
2766 	amdgpu_persistent_edc_harvesting(adev, ras_block);
2767 
2768 	/* in resume phase, no need to create ras fs node */
2769 	if (adev->in_suspend || amdgpu_in_reset(adev))
2770 		return 0;
2771 
2772 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2773 	if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2774 	    (ras_obj->hw_ops->query_poison_status ||
2775 	    ras_obj->hw_ops->handle_poison_consumption))) {
2776 		r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
2777 		if (r)
2778 			goto cleanup;
2779 	}
2780 
2781 	if (ras_obj->hw_ops &&
2782 	    (ras_obj->hw_ops->query_ras_error_count ||
2783 	     ras_obj->hw_ops->query_ras_error_status)) {
2784 		r = amdgpu_ras_sysfs_create(adev, ras_block);
2785 		if (r)
2786 			goto interrupt;
2787 
2788 		/* Those are the cached values at init.
2789 		 */
2790 		query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
2791 		if (!query_info)
2792 			return -ENOMEM;
2793 		memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
2794 
2795 		if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
2796 			atomic_set(&con->ras_ce_count, ce_count);
2797 			atomic_set(&con->ras_ue_count, ue_count);
2798 		}
2799 
2800 		kfree(query_info);
2801 	}
2802 
2803 	return 0;
2804 
2805 interrupt:
2806 	if (ras_obj->ras_cb)
2807 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2808 cleanup:
2809 	amdgpu_ras_feature_enable(adev, ras_block, 0);
2810 	return r;
2811 }
2812 
2813 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
2814 			 struct ras_common_if *ras_block)
2815 {
2816 	return amdgpu_ras_block_late_init(adev, ras_block);
2817 }
2818 
2819 /* helper function to remove ras fs node and interrupt handler */
2820 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2821 			  struct ras_common_if *ras_block)
2822 {
2823 	struct amdgpu_ras_block_object *ras_obj;
2824 	if (!ras_block)
2825 		return;
2826 
2827 	amdgpu_ras_sysfs_remove(adev, ras_block);
2828 
2829 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2830 	if (ras_obj->ras_cb)
2831 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2832 }
2833 
2834 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2835 			  struct ras_common_if *ras_block)
2836 {
2837 	return amdgpu_ras_block_late_fini(adev, ras_block);
2838 }
2839 
2840 /* do some init work after IP late init as dependence.
2841  * and it runs in resume/gpu reset/booting up cases.
2842  */
2843 void amdgpu_ras_resume(struct amdgpu_device *adev)
2844 {
2845 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2846 	struct ras_manager *obj, *tmp;
2847 
2848 	if (!adev->ras_enabled || !con) {
2849 		/* clean ras context for VEGA20 Gaming after send ras disable cmd */
2850 		amdgpu_release_ras_context(adev);
2851 
2852 		return;
2853 	}
2854 
2855 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
2856 		/* Set up all other IPs which are not implemented. There is a
2857 		 * tricky thing that IP's actual ras error type should be
2858 		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2859 		 * ERROR_NONE make sense anyway.
2860 		 */
2861 		amdgpu_ras_enable_all_features(adev, 1);
2862 
2863 		/* We enable ras on all hw_supported block, but as boot
2864 		 * parameter might disable some of them and one or more IP has
2865 		 * not implemented yet. So we disable them on behalf.
2866 		 */
2867 		list_for_each_entry_safe(obj, tmp, &con->head, node) {
2868 			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2869 				amdgpu_ras_feature_enable(adev, &obj->head, 0);
2870 				/* there should be no any reference. */
2871 				WARN_ON(alive_obj(obj));
2872 			}
2873 		}
2874 	}
2875 }
2876 
2877 void amdgpu_ras_suspend(struct amdgpu_device *adev)
2878 {
2879 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2880 
2881 	if (!adev->ras_enabled || !con)
2882 		return;
2883 
2884 	amdgpu_ras_disable_all_features(adev, 0);
2885 	/* Make sure all ras objects are disabled. */
2886 	if (con->features)
2887 		amdgpu_ras_disable_all_features(adev, 1);
2888 }
2889 
2890 int amdgpu_ras_late_init(struct amdgpu_device *adev)
2891 {
2892 	struct amdgpu_ras_block_list *node, *tmp;
2893 	struct amdgpu_ras_block_object *obj;
2894 	int r;
2895 
2896 	/* Guest side doesn't need init ras feature */
2897 	if (amdgpu_sriov_vf(adev))
2898 		return 0;
2899 
2900 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2901 		if (!node->ras_obj) {
2902 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2903 			continue;
2904 		}
2905 
2906 		obj = node->ras_obj;
2907 		if (obj->ras_late_init) {
2908 			r = obj->ras_late_init(adev, &obj->ras_comm);
2909 			if (r) {
2910 				dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2911 					obj->ras_comm.name, r);
2912 				return r;
2913 			}
2914 		} else
2915 			amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
2916 	}
2917 
2918 	return 0;
2919 }
2920 
2921 /* do some fini work before IP fini as dependence */
2922 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2923 {
2924 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2925 
2926 	if (!adev->ras_enabled || !con)
2927 		return 0;
2928 
2929 
2930 	/* Need disable ras on all IPs here before ip [hw/sw]fini */
2931 	if (con->features)
2932 		amdgpu_ras_disable_all_features(adev, 0);
2933 	amdgpu_ras_recovery_fini(adev);
2934 	return 0;
2935 }
2936 
2937 int amdgpu_ras_fini(struct amdgpu_device *adev)
2938 {
2939 	struct amdgpu_ras_block_list *ras_node, *tmp;
2940 	struct amdgpu_ras_block_object *obj = NULL;
2941 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2942 
2943 	if (!adev->ras_enabled || !con)
2944 		return 0;
2945 
2946 	list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2947 		if (ras_node->ras_obj) {
2948 			obj = ras_node->ras_obj;
2949 			if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2950 			    obj->ras_fini)
2951 				obj->ras_fini(adev, &obj->ras_comm);
2952 			else
2953 				amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
2954 		}
2955 
2956 		/* Clear ras blocks from ras_list and free ras block list node */
2957 		list_del(&ras_node->node);
2958 		kfree(ras_node);
2959 	}
2960 
2961 	amdgpu_ras_fs_fini(adev);
2962 	amdgpu_ras_interrupt_remove_all(adev);
2963 
2964 	WARN(con->features, "Feature mask is not cleared");
2965 
2966 	if (con->features)
2967 		amdgpu_ras_disable_all_features(adev, 1);
2968 
2969 	cancel_delayed_work_sync(&con->ras_counte_delay_work);
2970 
2971 	amdgpu_ras_set_context(adev, NULL);
2972 	kfree(con);
2973 
2974 	return 0;
2975 }
2976 
2977 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2978 {
2979 	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2980 		struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2981 
2982 		dev_info(adev->dev, "uncorrectable hardware error"
2983 			"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
2984 
2985 		ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2986 		amdgpu_ras_reset_gpu(adev);
2987 	}
2988 }
2989 
2990 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2991 {
2992 	if (adev->asic_type == CHIP_VEGA20 &&
2993 	    adev->pm.fw_version <= 0x283400) {
2994 		return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2995 				amdgpu_ras_intr_triggered();
2996 	}
2997 
2998 	return false;
2999 }
3000 
3001 void amdgpu_release_ras_context(struct amdgpu_device *adev)
3002 {
3003 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3004 
3005 	if (!con)
3006 		return;
3007 
3008 	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
3009 		con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
3010 		amdgpu_ras_set_context(adev, NULL);
3011 		kfree(con);
3012 	}
3013 }
3014 
3015 #ifdef CONFIG_X86_MCE_AMD
3016 static struct amdgpu_device *find_adev(uint32_t node_id)
3017 {
3018 	int i;
3019 	struct amdgpu_device *adev = NULL;
3020 
3021 	for (i = 0; i < mce_adev_list.num_gpu; i++) {
3022 		adev = mce_adev_list.devs[i];
3023 
3024 		if (adev && adev->gmc.xgmi.connected_to_cpu &&
3025 		    adev->gmc.xgmi.physical_node_id == node_id)
3026 			break;
3027 		adev = NULL;
3028 	}
3029 
3030 	return adev;
3031 }
3032 
3033 #define GET_MCA_IPID_GPUID(m)	(((m) >> 44) & 0xF)
3034 #define GET_UMC_INST(m)		(((m) >> 21) & 0x7)
3035 #define GET_CHAN_INDEX(m)	((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
3036 #define GPU_ID_OFFSET		8
3037 
3038 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
3039 				    unsigned long val, void *data)
3040 {
3041 	struct mce *m = (struct mce *)data;
3042 	struct amdgpu_device *adev = NULL;
3043 	uint32_t gpu_id = 0;
3044 	uint32_t umc_inst = 0, ch_inst = 0;
3045 
3046 	/*
3047 	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
3048 	 * and error occurred in DramECC (Extended error code = 0) then only
3049 	 * process the error, else bail out.
3050 	 */
3051 	if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
3052 		    (XEC(m->status, 0x3f) == 0x0)))
3053 		return NOTIFY_DONE;
3054 
3055 	/*
3056 	 * If it is correctable error, return.
3057 	 */
3058 	if (mce_is_correctable(m))
3059 		return NOTIFY_OK;
3060 
3061 	/*
3062 	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
3063 	 */
3064 	gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
3065 
3066 	adev = find_adev(gpu_id);
3067 	if (!adev) {
3068 		DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
3069 								gpu_id);
3070 		return NOTIFY_DONE;
3071 	}
3072 
3073 	/*
3074 	 * If it is uncorrectable error, then find out UMC instance and
3075 	 * channel index.
3076 	 */
3077 	umc_inst = GET_UMC_INST(m->ipid);
3078 	ch_inst = GET_CHAN_INDEX(m->ipid);
3079 
3080 	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
3081 			     umc_inst, ch_inst);
3082 
3083 	if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
3084 		return NOTIFY_OK;
3085 	else
3086 		return NOTIFY_DONE;
3087 }
3088 
3089 static struct notifier_block amdgpu_bad_page_nb = {
3090 	.notifier_call  = amdgpu_bad_page_notifier,
3091 	.priority       = MCE_PRIO_UC,
3092 };
3093 
3094 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
3095 {
3096 	/*
3097 	 * Add the adev to the mce_adev_list.
3098 	 * During mode2 reset, amdgpu device is temporarily
3099 	 * removed from the mgpu_info list which can cause
3100 	 * page retirement to fail.
3101 	 * Use this list instead of mgpu_info to find the amdgpu
3102 	 * device on which the UMC error was reported.
3103 	 */
3104 	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
3105 
3106 	/*
3107 	 * Register the x86 notifier only once
3108 	 * with MCE subsystem.
3109 	 */
3110 	if (notifier_registered == false) {
3111 		mce_register_decode_chain(&amdgpu_bad_page_nb);
3112 		notifier_registered = true;
3113 	}
3114 }
3115 #endif
3116 
3117 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
3118 {
3119 	if (!adev)
3120 		return NULL;
3121 
3122 	return adev->psp.ras_context.ras;
3123 }
3124 
3125 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
3126 {
3127 	if (!adev)
3128 		return -EINVAL;
3129 
3130 	adev->psp.ras_context.ras = ras_con;
3131 	return 0;
3132 }
3133 
3134 /* check if ras is supported on block, say, sdma, gfx */
3135 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
3136 		unsigned int block)
3137 {
3138 	int ret = 0;
3139 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3140 
3141 	if (block >= AMDGPU_RAS_BLOCK_COUNT)
3142 		return 0;
3143 
3144 	ret = ras && (adev->ras_enabled & (1 << block));
3145 
3146 	/* For the special asic with mem ecc enabled but sram ecc
3147 	 * not enabled, even if the ras block is not supported on
3148 	 * .ras_enabled, if the asic supports poison mode and the
3149 	 * ras block has ras configuration, it can be considered
3150 	 * that the ras block supports ras function.
3151 	 */
3152 	if (!ret &&
3153 	    (block == AMDGPU_RAS_BLOCK__GFX ||
3154 	     block == AMDGPU_RAS_BLOCK__SDMA ||
3155 	     block == AMDGPU_RAS_BLOCK__VCN ||
3156 	     block == AMDGPU_RAS_BLOCK__JPEG) &&
3157 	    amdgpu_ras_is_poison_mode_supported(adev) &&
3158 	    amdgpu_ras_get_ras_block(adev, block, 0))
3159 		ret = 1;
3160 
3161 	return ret;
3162 }
3163 
3164 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
3165 {
3166 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3167 
3168 	if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
3169 		amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
3170 	return 0;
3171 }
3172 
3173 
3174 /* Register each ip ras block into amdgpu ras */
3175 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
3176 		struct amdgpu_ras_block_object *ras_block_obj)
3177 {
3178 	struct amdgpu_ras_block_list *ras_node;
3179 	if (!adev || !ras_block_obj)
3180 		return -EINVAL;
3181 
3182 	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
3183 	if (!ras_node)
3184 		return -ENOMEM;
3185 
3186 	INIT_LIST_HEAD(&ras_node->node);
3187 	ras_node->ras_obj = ras_block_obj;
3188 	list_add_tail(&ras_node->node, &adev->ras_list);
3189 
3190 	return 0;
3191 }
3192 
3193 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
3194 {
3195 	if (!err_type_name)
3196 		return;
3197 
3198 	switch (err_type) {
3199 	case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
3200 		sprintf(err_type_name, "correctable");
3201 		break;
3202 	case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
3203 		sprintf(err_type_name, "uncorrectable");
3204 		break;
3205 	default:
3206 		sprintf(err_type_name, "unknown");
3207 		break;
3208 	}
3209 }
3210 
3211 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
3212 					 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
3213 					 uint32_t instance,
3214 					 uint32_t *memory_id)
3215 {
3216 	uint32_t err_status_lo_data, err_status_lo_offset;
3217 
3218 	if (!reg_entry)
3219 		return false;
3220 
3221 	err_status_lo_offset =
3222 		AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
3223 					    reg_entry->seg_lo, reg_entry->reg_lo);
3224 	err_status_lo_data = RREG32(err_status_lo_offset);
3225 
3226 	if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
3227 	    !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
3228 		return false;
3229 
3230 	*memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
3231 
3232 	return true;
3233 }
3234 
3235 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
3236 				       const struct amdgpu_ras_err_status_reg_entry *reg_entry,
3237 				       uint32_t instance,
3238 				       unsigned long *err_cnt)
3239 {
3240 	uint32_t err_status_hi_data, err_status_hi_offset;
3241 
3242 	if (!reg_entry)
3243 		return false;
3244 
3245 	err_status_hi_offset =
3246 		AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
3247 					    reg_entry->seg_hi, reg_entry->reg_hi);
3248 	err_status_hi_data = RREG32(err_status_hi_offset);
3249 
3250 	if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
3251 	    !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
3252 		/* keep the check here in case we need to refer to the result later */
3253 		dev_dbg(adev->dev, "Invalid err_info field\n");
3254 
3255 	/* read err count */
3256 	*err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
3257 
3258 	return true;
3259 }
3260 
3261 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
3262 					   const struct amdgpu_ras_err_status_reg_entry *reg_list,
3263 					   uint32_t reg_list_size,
3264 					   const struct amdgpu_ras_memory_id_entry *mem_list,
3265 					   uint32_t mem_list_size,
3266 					   uint32_t instance,
3267 					   uint32_t err_type,
3268 					   unsigned long *err_count)
3269 {
3270 	uint32_t memory_id;
3271 	unsigned long err_cnt;
3272 	char err_type_name[16];
3273 	uint32_t i, j;
3274 
3275 	for (i = 0; i < reg_list_size; i++) {
3276 		/* query memory_id from err_status_lo */
3277 		if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
3278 							 instance, &memory_id))
3279 			continue;
3280 
3281 		/* query err_cnt from err_status_hi */
3282 		if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
3283 						       instance, &err_cnt) ||
3284 		    !err_cnt)
3285 			continue;
3286 
3287 		*err_count += err_cnt;
3288 
3289 		/* log the errors */
3290 		amdgpu_ras_get_error_type_name(err_type, err_type_name);
3291 		if (!mem_list) {
3292 			/* memory_list is not supported */
3293 			dev_info(adev->dev,
3294 				 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
3295 				 err_cnt, err_type_name,
3296 				 reg_list[i].block_name,
3297 				 instance, memory_id);
3298 		} else {
3299 			for (j = 0; j < mem_list_size; j++) {
3300 				if (memory_id == mem_list[j].memory_id) {
3301 					dev_info(adev->dev,
3302 						 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
3303 						 err_cnt, err_type_name,
3304 						 reg_list[i].block_name,
3305 						 instance, mem_list[j].name);
3306 					break;
3307 				}
3308 			}
3309 		}
3310 	}
3311 }
3312 
3313 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
3314 					   const struct amdgpu_ras_err_status_reg_entry *reg_list,
3315 					   uint32_t reg_list_size,
3316 					   uint32_t instance)
3317 {
3318 	uint32_t err_status_lo_offset, err_status_hi_offset;
3319 	uint32_t i;
3320 
3321 	for (i = 0; i < reg_list_size; i++) {
3322 		err_status_lo_offset =
3323 			AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
3324 						    reg_list[i].seg_lo, reg_list[i].reg_lo);
3325 		err_status_hi_offset =
3326 			AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
3327 						    reg_list[i].seg_hi, reg_list[i].reg_hi);
3328 		WREG32(err_status_lo_offset, 0);
3329 		WREG32(err_status_hi_offset, 0);
3330 	}
3331 }
3332