1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 32 #include "amdgpu.h" 33 #include "amdgpu_ras.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_xgmi.h" 36 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 37 #include "nbio_v4_3.h" 38 #include "atom.h" 39 #include "amdgpu_reset.h" 40 41 #ifdef CONFIG_X86_MCE_AMD 42 #include <asm/mce.h> 43 44 static bool notifier_registered; 45 #endif 46 static const char *RAS_FS_NAME = "ras"; 47 48 const char *ras_error_string[] = { 49 "none", 50 "parity", 51 "single_correctable", 52 "multi_uncorrectable", 53 "poison", 54 }; 55 56 const char *ras_block_string[] = { 57 "umc", 58 "sdma", 59 "gfx", 60 "mmhub", 61 "athub", 62 "pcie_bif", 63 "hdp", 64 "xgmi_wafl", 65 "df", 66 "smn", 67 "sem", 68 "mp0", 69 "mp1", 70 "fuse", 71 "mca", 72 "vcn", 73 "jpeg", 74 }; 75 76 const char *ras_mca_block_string[] = { 77 "mca_mp0", 78 "mca_mp1", 79 "mca_mpio", 80 "mca_iohc", 81 }; 82 83 struct amdgpu_ras_block_list { 84 /* ras block link */ 85 struct list_head node; 86 87 struct amdgpu_ras_block_object *ras_obj; 88 }; 89 90 const char *get_ras_block_str(struct ras_common_if *ras_block) 91 { 92 if (!ras_block) 93 return "NULL"; 94 95 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT) 96 return "OUT OF RANGE"; 97 98 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 99 return ras_mca_block_string[ras_block->sub_block_index]; 100 101 return ras_block_string[ras_block->block]; 102 } 103 104 #define ras_block_str(_BLOCK_) \ 105 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 106 107 #define ras_err_str(i) (ras_error_string[ffs(i)]) 108 109 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 110 111 /* inject address is 52 bits */ 112 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 113 114 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 115 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 116 117 enum amdgpu_ras_retire_page_reservation { 118 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 119 AMDGPU_RAS_RETIRE_PAGE_PENDING, 120 AMDGPU_RAS_RETIRE_PAGE_FAULT, 121 }; 122 123 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 124 125 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 126 uint64_t addr); 127 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 128 uint64_t addr); 129 #ifdef CONFIG_X86_MCE_AMD 130 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 131 struct mce_notifier_adev_list { 132 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 133 int num_gpu; 134 }; 135 static struct mce_notifier_adev_list mce_adev_list; 136 #endif 137 138 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 139 { 140 if (adev && amdgpu_ras_get_context(adev)) 141 amdgpu_ras_get_context(adev)->error_query_ready = ready; 142 } 143 144 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 145 { 146 if (adev && amdgpu_ras_get_context(adev)) 147 return amdgpu_ras_get_context(adev)->error_query_ready; 148 149 return false; 150 } 151 152 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 153 { 154 struct ras_err_data err_data = {0, 0, 0, NULL}; 155 struct eeprom_table_record err_rec; 156 157 if ((address >= adev->gmc.mc_vram_size) || 158 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 159 dev_warn(adev->dev, 160 "RAS WARN: input address 0x%llx is invalid.\n", 161 address); 162 return -EINVAL; 163 } 164 165 if (amdgpu_ras_check_bad_page(adev, address)) { 166 dev_warn(adev->dev, 167 "RAS WARN: 0x%llx has already been marked as bad page!\n", 168 address); 169 return 0; 170 } 171 172 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 173 err_data.err_addr = &err_rec; 174 amdgpu_umc_fill_error_record(&err_data, address, 175 (address >> AMDGPU_GPU_PAGE_SHIFT), 0, 0); 176 177 if (amdgpu_bad_page_threshold != 0) { 178 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 179 err_data.err_addr_cnt); 180 amdgpu_ras_save_bad_pages(adev, NULL); 181 } 182 183 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 184 dev_warn(adev->dev, "Clear EEPROM:\n"); 185 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 186 187 return 0; 188 } 189 190 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 191 size_t size, loff_t *pos) 192 { 193 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 194 struct ras_query_if info = { 195 .head = obj->head, 196 }; 197 ssize_t s; 198 char val[128]; 199 200 if (amdgpu_ras_query_error_status(obj->adev, &info)) 201 return -EINVAL; 202 203 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 204 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) && 205 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) { 206 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 207 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 208 } 209 210 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 211 "ue", info.ue_count, 212 "ce", info.ce_count); 213 if (*pos >= s) 214 return 0; 215 216 s -= *pos; 217 s = min_t(u64, s, size); 218 219 220 if (copy_to_user(buf, &val[*pos], s)) 221 return -EINVAL; 222 223 *pos += s; 224 225 return s; 226 } 227 228 static const struct file_operations amdgpu_ras_debugfs_ops = { 229 .owner = THIS_MODULE, 230 .read = amdgpu_ras_debugfs_read, 231 .write = NULL, 232 .llseek = default_llseek 233 }; 234 235 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 236 { 237 int i; 238 239 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 240 *block_id = i; 241 if (strcmp(name, ras_block_string[i]) == 0) 242 return 0; 243 } 244 return -EINVAL; 245 } 246 247 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 248 const char __user *buf, size_t size, 249 loff_t *pos, struct ras_debug_if *data) 250 { 251 ssize_t s = min_t(u64, 64, size); 252 char str[65]; 253 char block_name[33]; 254 char err[9] = "ue"; 255 int op = -1; 256 int block_id; 257 uint32_t sub_block; 258 u64 address, value; 259 260 if (*pos) 261 return -EINVAL; 262 *pos = size; 263 264 memset(str, 0, sizeof(str)); 265 memset(data, 0, sizeof(*data)); 266 267 if (copy_from_user(str, buf, s)) 268 return -EINVAL; 269 270 if (sscanf(str, "disable %32s", block_name) == 1) 271 op = 0; 272 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 273 op = 1; 274 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 275 op = 2; 276 else if (strstr(str, "retire_page") != NULL) 277 op = 3; 278 else if (str[0] && str[1] && str[2] && str[3]) 279 /* ascii string, but commands are not matched. */ 280 return -EINVAL; 281 282 if (op != -1) { 283 if (op == 3) { 284 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 285 sscanf(str, "%*s %llu", &address) != 1) 286 return -EINVAL; 287 288 data->op = op; 289 data->inject.address = address; 290 291 return 0; 292 } 293 294 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 295 return -EINVAL; 296 297 data->head.block = block_id; 298 /* only ue and ce errors are supported */ 299 if (!memcmp("ue", err, 2)) 300 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 301 else if (!memcmp("ce", err, 2)) 302 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 303 else 304 return -EINVAL; 305 306 data->op = op; 307 308 if (op == 2) { 309 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 310 &sub_block, &address, &value) != 3 && 311 sscanf(str, "%*s %*s %*s %u %llu %llu", 312 &sub_block, &address, &value) != 3) 313 return -EINVAL; 314 data->head.sub_block_index = sub_block; 315 data->inject.address = address; 316 data->inject.value = value; 317 } 318 } else { 319 if (size < sizeof(*data)) 320 return -EINVAL; 321 322 if (copy_from_user(data, buf, sizeof(*data))) 323 return -EINVAL; 324 } 325 326 return 0; 327 } 328 329 /** 330 * DOC: AMDGPU RAS debugfs control interface 331 * 332 * The control interface accepts struct ras_debug_if which has two members. 333 * 334 * First member: ras_debug_if::head or ras_debug_if::inject. 335 * 336 * head is used to indicate which IP block will be under control. 337 * 338 * head has four members, they are block, type, sub_block_index, name. 339 * block: which IP will be under control. 340 * type: what kind of error will be enabled/disabled/injected. 341 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 342 * name: the name of IP. 343 * 344 * inject has two more members than head, they are address, value. 345 * As their names indicate, inject operation will write the 346 * value to the address. 347 * 348 * The second member: struct ras_debug_if::op. 349 * It has three kinds of operations. 350 * 351 * - 0: disable RAS on the block. Take ::head as its data. 352 * - 1: enable RAS on the block. Take ::head as its data. 353 * - 2: inject errors on the block. Take ::inject as its data. 354 * 355 * How to use the interface? 356 * 357 * In a program 358 * 359 * Copy the struct ras_debug_if in your code and initialize it. 360 * Write the struct to the control interface. 361 * 362 * From shell 363 * 364 * .. code-block:: bash 365 * 366 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 367 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 368 * echo "inject <block> <error> <sub-block> <address> <value> > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 369 * 370 * Where N, is the card which you want to affect. 371 * 372 * "disable" requires only the block. 373 * "enable" requires the block and error type. 374 * "inject" requires the block, error type, address, and value. 375 * 376 * The block is one of: umc, sdma, gfx, etc. 377 * see ras_block_string[] for details 378 * 379 * The error type is one of: ue, ce, where, 380 * ue is multi-uncorrectable 381 * ce is single-correctable 382 * 383 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 384 * The address and value are hexadecimal numbers, leading 0x is optional. 385 * 386 * For instance, 387 * 388 * .. code-block:: bash 389 * 390 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 391 * echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 392 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 393 * 394 * How to check the result of the operation? 395 * 396 * To check disable/enable, see "ras" features at, 397 * /sys/class/drm/card[0/1/2...]/device/ras/features 398 * 399 * To check inject, see the corresponding error count at, 400 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 401 * 402 * .. note:: 403 * Operations are only allowed on blocks which are supported. 404 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 405 * to see which blocks support RAS on a particular asic. 406 * 407 */ 408 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 409 const char __user *buf, 410 size_t size, loff_t *pos) 411 { 412 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 413 struct ras_debug_if data; 414 int ret = 0; 415 416 if (!amdgpu_ras_get_error_query_ready(adev)) { 417 dev_warn(adev->dev, "RAS WARN: error injection " 418 "currently inaccessible\n"); 419 return size; 420 } 421 422 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 423 if (ret) 424 return ret; 425 426 if (data.op == 3) { 427 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 428 if (!ret) 429 return size; 430 else 431 return ret; 432 } 433 434 if (!amdgpu_ras_is_supported(adev, data.head.block)) 435 return -EINVAL; 436 437 switch (data.op) { 438 case 0: 439 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 440 break; 441 case 1: 442 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 443 break; 444 case 2: 445 if ((data.inject.address >= adev->gmc.mc_vram_size) || 446 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 447 dev_warn(adev->dev, "RAS WARN: input address " 448 "0x%llx is invalid.", 449 data.inject.address); 450 ret = -EINVAL; 451 break; 452 } 453 454 /* umc ce/ue error injection for a bad page is not allowed */ 455 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) && 456 amdgpu_ras_check_bad_page(adev, data.inject.address)) { 457 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has " 458 "already been marked as bad!\n", 459 data.inject.address); 460 break; 461 } 462 463 /* data.inject.address is offset instead of absolute gpu address */ 464 ret = amdgpu_ras_error_inject(adev, &data.inject); 465 break; 466 default: 467 ret = -EINVAL; 468 break; 469 } 470 471 if (ret) 472 return ret; 473 474 return size; 475 } 476 477 /** 478 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 479 * 480 * Some boards contain an EEPROM which is used to persistently store a list of 481 * bad pages which experiences ECC errors in vram. This interface provides 482 * a way to reset the EEPROM, e.g., after testing error injection. 483 * 484 * Usage: 485 * 486 * .. code-block:: bash 487 * 488 * echo 1 > ../ras/ras_eeprom_reset 489 * 490 * will reset EEPROM table to 0 entries. 491 * 492 */ 493 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 494 const char __user *buf, 495 size_t size, loff_t *pos) 496 { 497 struct amdgpu_device *adev = 498 (struct amdgpu_device *)file_inode(f)->i_private; 499 int ret; 500 501 ret = amdgpu_ras_eeprom_reset_table( 502 &(amdgpu_ras_get_context(adev)->eeprom_control)); 503 504 if (!ret) { 505 /* Something was written to EEPROM. 506 */ 507 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 508 return size; 509 } else { 510 return ret; 511 } 512 } 513 514 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 515 .owner = THIS_MODULE, 516 .read = NULL, 517 .write = amdgpu_ras_debugfs_ctrl_write, 518 .llseek = default_llseek 519 }; 520 521 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 522 .owner = THIS_MODULE, 523 .read = NULL, 524 .write = amdgpu_ras_debugfs_eeprom_write, 525 .llseek = default_llseek 526 }; 527 528 /** 529 * DOC: AMDGPU RAS sysfs Error Count Interface 530 * 531 * It allows the user to read the error count for each IP block on the gpu through 532 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 533 * 534 * It outputs the multiple lines which report the uncorrected (ue) and corrected 535 * (ce) error counts. 536 * 537 * The format of one line is below, 538 * 539 * [ce|ue]: count 540 * 541 * Example: 542 * 543 * .. code-block:: bash 544 * 545 * ue: 0 546 * ce: 1 547 * 548 */ 549 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 550 struct device_attribute *attr, char *buf) 551 { 552 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 553 struct ras_query_if info = { 554 .head = obj->head, 555 }; 556 557 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 558 return sysfs_emit(buf, "Query currently inaccessible\n"); 559 560 if (amdgpu_ras_query_error_status(obj->adev, &info)) 561 return -EINVAL; 562 563 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) && 564 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) { 565 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 566 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 567 } 568 569 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 570 "ce", info.ce_count); 571 } 572 573 /* obj begin */ 574 575 #define get_obj(obj) do { (obj)->use++; } while (0) 576 #define alive_obj(obj) ((obj)->use) 577 578 static inline void put_obj(struct ras_manager *obj) 579 { 580 if (obj && (--obj->use == 0)) 581 list_del(&obj->node); 582 if (obj && (obj->use < 0)) 583 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 584 } 585 586 /* make one obj and return it. */ 587 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 588 struct ras_common_if *head) 589 { 590 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 591 struct ras_manager *obj; 592 593 if (!adev->ras_enabled || !con) 594 return NULL; 595 596 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 597 return NULL; 598 599 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 600 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 601 return NULL; 602 603 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 604 } else 605 obj = &con->objs[head->block]; 606 607 /* already exist. return obj? */ 608 if (alive_obj(obj)) 609 return NULL; 610 611 obj->head = *head; 612 obj->adev = adev; 613 list_add(&obj->node, &con->head); 614 get_obj(obj); 615 616 return obj; 617 } 618 619 /* return an obj equal to head, or the first when head is NULL */ 620 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 621 struct ras_common_if *head) 622 { 623 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 624 struct ras_manager *obj; 625 int i; 626 627 if (!adev->ras_enabled || !con) 628 return NULL; 629 630 if (head) { 631 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 632 return NULL; 633 634 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 635 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 636 return NULL; 637 638 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 639 } else 640 obj = &con->objs[head->block]; 641 642 if (alive_obj(obj)) 643 return obj; 644 } else { 645 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 646 obj = &con->objs[i]; 647 if (alive_obj(obj)) 648 return obj; 649 } 650 } 651 652 return NULL; 653 } 654 /* obj end */ 655 656 /* feature ctl begin */ 657 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 658 struct ras_common_if *head) 659 { 660 return adev->ras_hw_enabled & BIT(head->block); 661 } 662 663 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 664 struct ras_common_if *head) 665 { 666 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 667 668 return con->features & BIT(head->block); 669 } 670 671 /* 672 * if obj is not created, then create one. 673 * set feature enable flag. 674 */ 675 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 676 struct ras_common_if *head, int enable) 677 { 678 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 679 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 680 681 /* If hardware does not support ras, then do not create obj. 682 * But if hardware support ras, we can create the obj. 683 * Ras framework checks con->hw_supported to see if it need do 684 * corresponding initialization. 685 * IP checks con->support to see if it need disable ras. 686 */ 687 if (!amdgpu_ras_is_feature_allowed(adev, head)) 688 return 0; 689 690 if (enable) { 691 if (!obj) { 692 obj = amdgpu_ras_create_obj(adev, head); 693 if (!obj) 694 return -EINVAL; 695 } else { 696 /* In case we create obj somewhere else */ 697 get_obj(obj); 698 } 699 con->features |= BIT(head->block); 700 } else { 701 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 702 con->features &= ~BIT(head->block); 703 put_obj(obj); 704 } 705 } 706 707 return 0; 708 } 709 710 static int amdgpu_ras_check_feature_allowed(struct amdgpu_device *adev, 711 struct ras_common_if *head) 712 { 713 if (amdgpu_ras_is_feature_allowed(adev, head) || 714 amdgpu_ras_is_poison_mode_supported(adev)) 715 return 1; 716 else 717 return 0; 718 } 719 720 /* wrapper of psp_ras_enable_features */ 721 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 722 struct ras_common_if *head, bool enable) 723 { 724 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 725 union ta_ras_cmd_input *info; 726 int ret = 0; 727 728 if (!con) 729 return -EINVAL; 730 731 if (head->block == AMDGPU_RAS_BLOCK__GFX) { 732 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 733 if (!info) 734 return -ENOMEM; 735 736 if (!enable) { 737 info->disable_features = (struct ta_ras_disable_features_input) { 738 .block_id = amdgpu_ras_block_to_ta(head->block), 739 .error_type = amdgpu_ras_error_to_ta(head->type), 740 }; 741 } else { 742 info->enable_features = (struct ta_ras_enable_features_input) { 743 .block_id = amdgpu_ras_block_to_ta(head->block), 744 .error_type = amdgpu_ras_error_to_ta(head->type), 745 }; 746 } 747 } 748 749 /* Do not enable if it is not allowed. */ 750 if (enable && !amdgpu_ras_check_feature_allowed(adev, head)) 751 goto out; 752 753 /* Only enable ras feature operation handle on host side */ 754 if (head->block == AMDGPU_RAS_BLOCK__GFX && 755 !amdgpu_sriov_vf(adev) && 756 !amdgpu_ras_intr_triggered()) { 757 ret = psp_ras_enable_features(&adev->psp, info, enable); 758 if (ret) { 759 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 760 enable ? "enable":"disable", 761 get_ras_block_str(head), 762 amdgpu_ras_is_poison_mode_supported(adev), ret); 763 goto out; 764 } 765 } 766 767 /* setup the obj */ 768 __amdgpu_ras_feature_enable(adev, head, enable); 769 out: 770 if (head->block == AMDGPU_RAS_BLOCK__GFX) 771 kfree(info); 772 return ret; 773 } 774 775 /* Only used in device probe stage and called only once. */ 776 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 777 struct ras_common_if *head, bool enable) 778 { 779 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 780 int ret; 781 782 if (!con) 783 return -EINVAL; 784 785 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 786 if (enable) { 787 /* There is no harm to issue a ras TA cmd regardless of 788 * the currecnt ras state. 789 * If current state == target state, it will do nothing 790 * But sometimes it requests driver to reset and repost 791 * with error code -EAGAIN. 792 */ 793 ret = amdgpu_ras_feature_enable(adev, head, 1); 794 /* With old ras TA, we might fail to enable ras. 795 * Log it and just setup the object. 796 * TODO need remove this WA in the future. 797 */ 798 if (ret == -EINVAL) { 799 ret = __amdgpu_ras_feature_enable(adev, head, 1); 800 if (!ret) 801 dev_info(adev->dev, 802 "RAS INFO: %s setup object\n", 803 get_ras_block_str(head)); 804 } 805 } else { 806 /* setup the object then issue a ras TA disable cmd.*/ 807 ret = __amdgpu_ras_feature_enable(adev, head, 1); 808 if (ret) 809 return ret; 810 811 /* gfx block ras dsiable cmd must send to ras-ta */ 812 if (head->block == AMDGPU_RAS_BLOCK__GFX) 813 con->features |= BIT(head->block); 814 815 ret = amdgpu_ras_feature_enable(adev, head, 0); 816 817 /* clean gfx block ras features flag */ 818 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 819 con->features &= ~BIT(head->block); 820 } 821 } else 822 ret = amdgpu_ras_feature_enable(adev, head, enable); 823 824 return ret; 825 } 826 827 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 828 bool bypass) 829 { 830 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 831 struct ras_manager *obj, *tmp; 832 833 list_for_each_entry_safe(obj, tmp, &con->head, node) { 834 /* bypass psp. 835 * aka just release the obj and corresponding flags 836 */ 837 if (bypass) { 838 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 839 break; 840 } else { 841 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 842 break; 843 } 844 } 845 846 return con->features; 847 } 848 849 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 850 bool bypass) 851 { 852 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 853 int i; 854 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 855 856 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 857 struct ras_common_if head = { 858 .block = i, 859 .type = default_ras_type, 860 .sub_block_index = 0, 861 }; 862 863 if (i == AMDGPU_RAS_BLOCK__MCA) 864 continue; 865 866 if (bypass) { 867 /* 868 * bypass psp. vbios enable ras for us. 869 * so just create the obj 870 */ 871 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 872 break; 873 } else { 874 if (amdgpu_ras_feature_enable(adev, &head, 1)) 875 break; 876 } 877 } 878 879 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 880 struct ras_common_if head = { 881 .block = AMDGPU_RAS_BLOCK__MCA, 882 .type = default_ras_type, 883 .sub_block_index = i, 884 }; 885 886 if (bypass) { 887 /* 888 * bypass psp. vbios enable ras for us. 889 * so just create the obj 890 */ 891 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 892 break; 893 } else { 894 if (amdgpu_ras_feature_enable(adev, &head, 1)) 895 break; 896 } 897 } 898 899 return con->features; 900 } 901 /* feature ctl end */ 902 903 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 904 enum amdgpu_ras_block block) 905 { 906 if (!block_obj) 907 return -EINVAL; 908 909 if (block_obj->ras_comm.block == block) 910 return 0; 911 912 return -EINVAL; 913 } 914 915 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 916 enum amdgpu_ras_block block, uint32_t sub_block_index) 917 { 918 struct amdgpu_ras_block_list *node, *tmp; 919 struct amdgpu_ras_block_object *obj; 920 921 if (block >= AMDGPU_RAS_BLOCK__LAST) 922 return NULL; 923 924 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 925 if (!node->ras_obj) { 926 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 927 continue; 928 } 929 930 obj = node->ras_obj; 931 if (obj->ras_block_match) { 932 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 933 return obj; 934 } else { 935 if (amdgpu_ras_block_match_default(obj, block) == 0) 936 return obj; 937 } 938 } 939 940 return NULL; 941 } 942 943 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 944 { 945 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 946 int ret = 0; 947 948 /* 949 * choosing right query method according to 950 * whether smu support query error information 951 */ 952 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 953 if (ret == -EOPNOTSUPP) { 954 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 955 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 956 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 957 958 /* umc query_ras_error_address is also responsible for clearing 959 * error status 960 */ 961 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 962 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 963 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 964 } else if (!ret) { 965 if (adev->umc.ras && 966 adev->umc.ras->ecc_info_query_ras_error_count) 967 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 968 969 if (adev->umc.ras && 970 adev->umc.ras->ecc_info_query_ras_error_address) 971 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 972 } 973 } 974 975 /* query/inject/cure begin */ 976 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, 977 struct ras_query_if *info) 978 { 979 struct amdgpu_ras_block_object *block_obj = NULL; 980 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 981 struct ras_err_data err_data = {0, 0, 0, NULL}; 982 983 if (!obj) 984 return -EINVAL; 985 986 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 987 amdgpu_ras_get_ecc_info(adev, &err_data); 988 } else { 989 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 990 if (!block_obj || !block_obj->hw_ops) { 991 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 992 get_ras_block_str(&info->head)); 993 return -EINVAL; 994 } 995 996 if (block_obj->hw_ops->query_ras_error_count) 997 block_obj->hw_ops->query_ras_error_count(adev, &err_data); 998 999 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1000 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1001 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1002 if (block_obj->hw_ops->query_ras_error_status) 1003 block_obj->hw_ops->query_ras_error_status(adev); 1004 } 1005 } 1006 1007 obj->err_data.ue_count += err_data.ue_count; 1008 obj->err_data.ce_count += err_data.ce_count; 1009 1010 info->ue_count = obj->err_data.ue_count; 1011 info->ce_count = obj->err_data.ce_count; 1012 1013 if (err_data.ce_count) { 1014 if (adev->smuio.funcs && 1015 adev->smuio.funcs->get_socket_id && 1016 adev->smuio.funcs->get_die_id) { 1017 dev_info(adev->dev, "socket: %d, die: %d " 1018 "%ld correctable hardware errors " 1019 "detected in %s block, no user " 1020 "action is needed.\n", 1021 adev->smuio.funcs->get_socket_id(adev), 1022 adev->smuio.funcs->get_die_id(adev), 1023 obj->err_data.ce_count, 1024 get_ras_block_str(&info->head)); 1025 } else { 1026 dev_info(adev->dev, "%ld correctable hardware errors " 1027 "detected in %s block, no user " 1028 "action is needed.\n", 1029 obj->err_data.ce_count, 1030 get_ras_block_str(&info->head)); 1031 } 1032 } 1033 if (err_data.ue_count) { 1034 if (adev->smuio.funcs && 1035 adev->smuio.funcs->get_socket_id && 1036 adev->smuio.funcs->get_die_id) { 1037 dev_info(adev->dev, "socket: %d, die: %d " 1038 "%ld uncorrectable hardware errors " 1039 "detected in %s block\n", 1040 adev->smuio.funcs->get_socket_id(adev), 1041 adev->smuio.funcs->get_die_id(adev), 1042 obj->err_data.ue_count, 1043 get_ras_block_str(&info->head)); 1044 } else { 1045 dev_info(adev->dev, "%ld uncorrectable hardware errors " 1046 "detected in %s block\n", 1047 obj->err_data.ue_count, 1048 get_ras_block_str(&info->head)); 1049 } 1050 } 1051 1052 return 0; 1053 } 1054 1055 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1056 enum amdgpu_ras_block block) 1057 { 1058 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1059 1060 if (!amdgpu_ras_is_supported(adev, block)) 1061 return -EINVAL; 1062 1063 if (!block_obj || !block_obj->hw_ops) { 1064 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1065 ras_block_str(block)); 1066 return -EINVAL; 1067 } 1068 1069 if (block_obj->hw_ops->reset_ras_error_count) 1070 block_obj->hw_ops->reset_ras_error_count(adev); 1071 1072 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1073 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1074 if (block_obj->hw_ops->reset_ras_error_status) 1075 block_obj->hw_ops->reset_ras_error_status(adev); 1076 } 1077 1078 return 0; 1079 } 1080 1081 /* wrapper of psp_ras_trigger_error */ 1082 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1083 struct ras_inject_if *info) 1084 { 1085 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1086 struct ta_ras_trigger_error_input block_info = { 1087 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1088 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1089 .sub_block_index = info->head.sub_block_index, 1090 .address = info->address, 1091 .value = info->value, 1092 }; 1093 int ret = -EINVAL; 1094 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1095 info->head.block, 1096 info->head.sub_block_index); 1097 1098 /* inject on guest isn't allowed, return success directly */ 1099 if (amdgpu_sriov_vf(adev)) 1100 return 0; 1101 1102 if (!obj) 1103 return -EINVAL; 1104 1105 if (!block_obj || !block_obj->hw_ops) { 1106 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1107 get_ras_block_str(&info->head)); 1108 return -EINVAL; 1109 } 1110 1111 /* Calculate XGMI relative offset */ 1112 if (adev->gmc.xgmi.num_physical_nodes > 1) { 1113 block_info.address = 1114 amdgpu_xgmi_get_relative_phy_addr(adev, 1115 block_info.address); 1116 } 1117 1118 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) { 1119 if (block_obj->hw_ops->ras_error_inject) 1120 ret = block_obj->hw_ops->ras_error_inject(adev, info); 1121 } else { 1122 /* If defined special ras_error_inject(e.g: xgmi), implement special ras_error_inject */ 1123 if (block_obj->hw_ops->ras_error_inject) 1124 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info); 1125 else /*If not defined .ras_error_inject, use default ras_error_inject*/ 1126 ret = psp_ras_trigger_error(&adev->psp, &block_info); 1127 } 1128 1129 if (ret) 1130 dev_err(adev->dev, "ras inject %s failed %d\n", 1131 get_ras_block_str(&info->head), ret); 1132 1133 return ret; 1134 } 1135 1136 /** 1137 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1138 * @adev: pointer to AMD GPU device 1139 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1140 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1141 * @query_info: pointer to ras_query_if 1142 * 1143 * Return 0 for query success or do nothing, otherwise return an error 1144 * on failures 1145 */ 1146 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1147 unsigned long *ce_count, 1148 unsigned long *ue_count, 1149 struct ras_query_if *query_info) 1150 { 1151 int ret; 1152 1153 if (!query_info) 1154 /* do nothing if query_info is not specified */ 1155 return 0; 1156 1157 ret = amdgpu_ras_query_error_status(adev, query_info); 1158 if (ret) 1159 return ret; 1160 1161 *ce_count += query_info->ce_count; 1162 *ue_count += query_info->ue_count; 1163 1164 /* some hardware/IP supports read to clear 1165 * no need to explictly reset the err status after the query call */ 1166 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) && 1167 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) { 1168 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1169 dev_warn(adev->dev, 1170 "Failed to reset error counter and error status\n"); 1171 } 1172 1173 return 0; 1174 } 1175 1176 /** 1177 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1178 * @adev: pointer to AMD GPU device 1179 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1180 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1181 * errors. 1182 * @query_info: pointer to ras_query_if if the query request is only for 1183 * specific ip block; if info is NULL, then the qurey request is for 1184 * all the ip blocks that support query ras error counters/status 1185 * 1186 * If set, @ce_count or @ue_count, count and return the corresponding 1187 * error counts in those integer pointers. Return 0 if the device 1188 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1189 */ 1190 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1191 unsigned long *ce_count, 1192 unsigned long *ue_count, 1193 struct ras_query_if *query_info) 1194 { 1195 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1196 struct ras_manager *obj; 1197 unsigned long ce, ue; 1198 int ret; 1199 1200 if (!adev->ras_enabled || !con) 1201 return -EOPNOTSUPP; 1202 1203 /* Don't count since no reporting. 1204 */ 1205 if (!ce_count && !ue_count) 1206 return 0; 1207 1208 ce = 0; 1209 ue = 0; 1210 if (!query_info) { 1211 /* query all the ip blocks that support ras query interface */ 1212 list_for_each_entry(obj, &con->head, node) { 1213 struct ras_query_if info = { 1214 .head = obj->head, 1215 }; 1216 1217 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1218 } 1219 } else { 1220 /* query specific ip block */ 1221 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1222 } 1223 1224 if (ret) 1225 return ret; 1226 1227 if (ce_count) 1228 *ce_count = ce; 1229 1230 if (ue_count) 1231 *ue_count = ue; 1232 1233 return 0; 1234 } 1235 /* query/inject/cure end */ 1236 1237 1238 /* sysfs begin */ 1239 1240 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1241 struct ras_badpage **bps, unsigned int *count); 1242 1243 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1244 { 1245 switch (flags) { 1246 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1247 return "R"; 1248 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1249 return "P"; 1250 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1251 default: 1252 return "F"; 1253 } 1254 } 1255 1256 /** 1257 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1258 * 1259 * It allows user to read the bad pages of vram on the gpu through 1260 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1261 * 1262 * It outputs multiple lines, and each line stands for one gpu page. 1263 * 1264 * The format of one line is below, 1265 * gpu pfn : gpu page size : flags 1266 * 1267 * gpu pfn and gpu page size are printed in hex format. 1268 * flags can be one of below character, 1269 * 1270 * R: reserved, this gpu page is reserved and not able to use. 1271 * 1272 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1273 * in next window of page_reserve. 1274 * 1275 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1276 * 1277 * Examples: 1278 * 1279 * .. code-block:: bash 1280 * 1281 * 0x00000001 : 0x00001000 : R 1282 * 0x00000002 : 0x00001000 : P 1283 * 1284 */ 1285 1286 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1287 struct kobject *kobj, struct bin_attribute *attr, 1288 char *buf, loff_t ppos, size_t count) 1289 { 1290 struct amdgpu_ras *con = 1291 container_of(attr, struct amdgpu_ras, badpages_attr); 1292 struct amdgpu_device *adev = con->adev; 1293 const unsigned int element_size = 1294 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1295 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1296 unsigned int end = div64_ul(ppos + count - 1, element_size); 1297 ssize_t s = 0; 1298 struct ras_badpage *bps = NULL; 1299 unsigned int bps_count = 0; 1300 1301 memset(buf, 0, count); 1302 1303 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1304 return 0; 1305 1306 for (; start < end && start < bps_count; start++) 1307 s += scnprintf(&buf[s], element_size + 1, 1308 "0x%08x : 0x%08x : %1s\n", 1309 bps[start].bp, 1310 bps[start].size, 1311 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1312 1313 kfree(bps); 1314 1315 return s; 1316 } 1317 1318 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1319 struct device_attribute *attr, char *buf) 1320 { 1321 struct amdgpu_ras *con = 1322 container_of(attr, struct amdgpu_ras, features_attr); 1323 1324 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1325 } 1326 1327 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1328 { 1329 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1330 1331 sysfs_remove_file_from_group(&adev->dev->kobj, 1332 &con->badpages_attr.attr, 1333 RAS_FS_NAME); 1334 } 1335 1336 static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev) 1337 { 1338 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1339 struct attribute *attrs[] = { 1340 &con->features_attr.attr, 1341 NULL 1342 }; 1343 struct attribute_group group = { 1344 .name = RAS_FS_NAME, 1345 .attrs = attrs, 1346 }; 1347 1348 sysfs_remove_group(&adev->dev->kobj, &group); 1349 1350 return 0; 1351 } 1352 1353 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1354 struct ras_common_if *head) 1355 { 1356 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1357 1358 if (!obj || obj->attr_inuse) 1359 return -EINVAL; 1360 1361 get_obj(obj); 1362 1363 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1364 "%s_err_count", head->name); 1365 1366 obj->sysfs_attr = (struct device_attribute){ 1367 .attr = { 1368 .name = obj->fs_data.sysfs_name, 1369 .mode = S_IRUGO, 1370 }, 1371 .show = amdgpu_ras_sysfs_read, 1372 }; 1373 sysfs_attr_init(&obj->sysfs_attr.attr); 1374 1375 if (sysfs_add_file_to_group(&adev->dev->kobj, 1376 &obj->sysfs_attr.attr, 1377 RAS_FS_NAME)) { 1378 put_obj(obj); 1379 return -EINVAL; 1380 } 1381 1382 obj->attr_inuse = 1; 1383 1384 return 0; 1385 } 1386 1387 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1388 struct ras_common_if *head) 1389 { 1390 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1391 1392 if (!obj || !obj->attr_inuse) 1393 return -EINVAL; 1394 1395 sysfs_remove_file_from_group(&adev->dev->kobj, 1396 &obj->sysfs_attr.attr, 1397 RAS_FS_NAME); 1398 obj->attr_inuse = 0; 1399 put_obj(obj); 1400 1401 return 0; 1402 } 1403 1404 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1405 { 1406 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1407 struct ras_manager *obj, *tmp; 1408 1409 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1410 amdgpu_ras_sysfs_remove(adev, &obj->head); 1411 } 1412 1413 if (amdgpu_bad_page_threshold != 0) 1414 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1415 1416 amdgpu_ras_sysfs_remove_feature_node(adev); 1417 1418 return 0; 1419 } 1420 /* sysfs end */ 1421 1422 /** 1423 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 1424 * 1425 * Normally when there is an uncorrectable error, the driver will reset 1426 * the GPU to recover. However, in the event of an unrecoverable error, 1427 * the driver provides an interface to reboot the system automatically 1428 * in that event. 1429 * 1430 * The following file in debugfs provides that interface: 1431 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 1432 * 1433 * Usage: 1434 * 1435 * .. code-block:: bash 1436 * 1437 * echo true > .../ras/auto_reboot 1438 * 1439 */ 1440 /* debugfs begin */ 1441 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 1442 { 1443 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1444 struct drm_minor *minor = adev_to_drm(adev)->primary; 1445 struct dentry *dir; 1446 1447 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 1448 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 1449 &amdgpu_ras_debugfs_ctrl_ops); 1450 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 1451 &amdgpu_ras_debugfs_eeprom_ops); 1452 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 1453 &con->bad_page_cnt_threshold); 1454 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 1455 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 1456 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 1457 &amdgpu_ras_debugfs_eeprom_size_ops); 1458 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 1459 S_IRUGO, dir, adev, 1460 &amdgpu_ras_debugfs_eeprom_table_ops); 1461 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 1462 1463 /* 1464 * After one uncorrectable error happens, usually GPU recovery will 1465 * be scheduled. But due to the known problem in GPU recovery failing 1466 * to bring GPU back, below interface provides one direct way to 1467 * user to reboot system automatically in such case within 1468 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 1469 * will never be called. 1470 */ 1471 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 1472 1473 /* 1474 * User could set this not to clean up hardware's error count register 1475 * of RAS IPs during ras recovery. 1476 */ 1477 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 1478 &con->disable_ras_err_cnt_harvest); 1479 return dir; 1480 } 1481 1482 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 1483 struct ras_fs_if *head, 1484 struct dentry *dir) 1485 { 1486 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 1487 1488 if (!obj || !dir) 1489 return; 1490 1491 get_obj(obj); 1492 1493 memcpy(obj->fs_data.debugfs_name, 1494 head->debugfs_name, 1495 sizeof(obj->fs_data.debugfs_name)); 1496 1497 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 1498 obj, &amdgpu_ras_debugfs_ops); 1499 } 1500 1501 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 1502 { 1503 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1504 struct dentry *dir; 1505 struct ras_manager *obj; 1506 struct ras_fs_if fs_info; 1507 1508 /* 1509 * it won't be called in resume path, no need to check 1510 * suspend and gpu reset status 1511 */ 1512 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 1513 return; 1514 1515 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 1516 1517 list_for_each_entry(obj, &con->head, node) { 1518 if (amdgpu_ras_is_supported(adev, obj->head.block) && 1519 (obj->attr_inuse == 1)) { 1520 sprintf(fs_info.debugfs_name, "%s_err_inject", 1521 get_ras_block_str(&obj->head)); 1522 fs_info.head = obj->head; 1523 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 1524 } 1525 } 1526 } 1527 1528 /* debugfs end */ 1529 1530 /* ras fs */ 1531 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 1532 amdgpu_ras_sysfs_badpages_read, NULL, 0); 1533 static DEVICE_ATTR(features, S_IRUGO, 1534 amdgpu_ras_sysfs_features_read, NULL); 1535 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 1536 { 1537 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1538 struct attribute_group group = { 1539 .name = RAS_FS_NAME, 1540 }; 1541 struct attribute *attrs[] = { 1542 &con->features_attr.attr, 1543 NULL 1544 }; 1545 struct bin_attribute *bin_attrs[] = { 1546 NULL, 1547 NULL, 1548 }; 1549 int r; 1550 1551 /* add features entry */ 1552 con->features_attr = dev_attr_features; 1553 group.attrs = attrs; 1554 sysfs_attr_init(attrs[0]); 1555 1556 if (amdgpu_bad_page_threshold != 0) { 1557 /* add bad_page_features entry */ 1558 bin_attr_gpu_vram_bad_pages.private = NULL; 1559 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 1560 bin_attrs[0] = &con->badpages_attr; 1561 group.bin_attrs = bin_attrs; 1562 sysfs_bin_attr_init(bin_attrs[0]); 1563 } 1564 1565 r = sysfs_create_group(&adev->dev->kobj, &group); 1566 if (r) 1567 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 1568 1569 return 0; 1570 } 1571 1572 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 1573 { 1574 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1575 struct ras_manager *con_obj, *ip_obj, *tmp; 1576 1577 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 1578 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 1579 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 1580 if (ip_obj) 1581 put_obj(ip_obj); 1582 } 1583 } 1584 1585 amdgpu_ras_sysfs_remove_all(adev); 1586 return 0; 1587 } 1588 /* ras fs end */ 1589 1590 /* ih begin */ 1591 1592 /* For the hardware that cannot enable bif ring for both ras_controller_irq 1593 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 1594 * register to check whether the interrupt is triggered or not, and properly 1595 * ack the interrupt if it is there 1596 */ 1597 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 1598 { 1599 /* Fatal error events are handled on host side */ 1600 if (amdgpu_sriov_vf(adev) || 1601 !amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF)) 1602 return; 1603 1604 if (adev->nbio.ras && 1605 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 1606 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 1607 1608 if (adev->nbio.ras && 1609 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 1610 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 1611 } 1612 1613 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 1614 struct amdgpu_iv_entry *entry) 1615 { 1616 bool poison_stat = false; 1617 struct amdgpu_device *adev = obj->adev; 1618 struct amdgpu_ras_block_object *block_obj = 1619 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 1620 1621 if (!block_obj) 1622 return; 1623 1624 /* both query_poison_status and handle_poison_consumption are optional, 1625 * but at least one of them should be implemented if we need poison 1626 * consumption handler 1627 */ 1628 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 1629 poison_stat = block_obj->hw_ops->query_poison_status(adev); 1630 if (!poison_stat) { 1631 /* Not poison consumption interrupt, no need to handle it */ 1632 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 1633 block_obj->ras_comm.name); 1634 1635 return; 1636 } 1637 } 1638 1639 if (!adev->gmc.xgmi.connected_to_cpu) 1640 amdgpu_umc_poison_handler(adev, false); 1641 1642 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 1643 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 1644 1645 /* gpu reset is fallback for failed and default cases */ 1646 if (poison_stat) { 1647 dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n", 1648 block_obj->ras_comm.name); 1649 amdgpu_ras_reset_gpu(adev); 1650 } else { 1651 amdgpu_gfx_poison_consumption_handler(adev, entry); 1652 } 1653 } 1654 1655 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 1656 struct amdgpu_iv_entry *entry) 1657 { 1658 dev_info(obj->adev->dev, 1659 "Poison is created, no user action is needed.\n"); 1660 } 1661 1662 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 1663 struct amdgpu_iv_entry *entry) 1664 { 1665 struct ras_ih_data *data = &obj->ih_data; 1666 struct ras_err_data err_data = {0, 0, 0, NULL}; 1667 int ret; 1668 1669 if (!data->cb) 1670 return; 1671 1672 /* Let IP handle its data, maybe we need get the output 1673 * from the callback to update the error type/count, etc 1674 */ 1675 ret = data->cb(obj->adev, &err_data, entry); 1676 /* ue will trigger an interrupt, and in that case 1677 * we need do a reset to recovery the whole system. 1678 * But leave IP do that recovery, here we just dispatch 1679 * the error. 1680 */ 1681 if (ret == AMDGPU_RAS_SUCCESS) { 1682 /* these counts could be left as 0 if 1683 * some blocks do not count error number 1684 */ 1685 obj->err_data.ue_count += err_data.ue_count; 1686 obj->err_data.ce_count += err_data.ce_count; 1687 } 1688 } 1689 1690 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 1691 { 1692 struct ras_ih_data *data = &obj->ih_data; 1693 struct amdgpu_iv_entry entry; 1694 1695 while (data->rptr != data->wptr) { 1696 rmb(); 1697 memcpy(&entry, &data->ring[data->rptr], 1698 data->element_size); 1699 1700 wmb(); 1701 data->rptr = (data->aligned_element_size + 1702 data->rptr) % data->ring_size; 1703 1704 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 1705 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 1706 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 1707 else 1708 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 1709 } else { 1710 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 1711 amdgpu_ras_interrupt_umc_handler(obj, &entry); 1712 else 1713 dev_warn(obj->adev->dev, 1714 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 1715 } 1716 } 1717 } 1718 1719 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 1720 { 1721 struct ras_ih_data *data = 1722 container_of(work, struct ras_ih_data, ih_work); 1723 struct ras_manager *obj = 1724 container_of(data, struct ras_manager, ih_data); 1725 1726 amdgpu_ras_interrupt_handler(obj); 1727 } 1728 1729 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 1730 struct ras_dispatch_if *info) 1731 { 1732 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1733 struct ras_ih_data *data = &obj->ih_data; 1734 1735 if (!obj) 1736 return -EINVAL; 1737 1738 if (data->inuse == 0) 1739 return 0; 1740 1741 /* Might be overflow... */ 1742 memcpy(&data->ring[data->wptr], info->entry, 1743 data->element_size); 1744 1745 wmb(); 1746 data->wptr = (data->aligned_element_size + 1747 data->wptr) % data->ring_size; 1748 1749 schedule_work(&data->ih_work); 1750 1751 return 0; 1752 } 1753 1754 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 1755 struct ras_common_if *head) 1756 { 1757 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1758 struct ras_ih_data *data; 1759 1760 if (!obj) 1761 return -EINVAL; 1762 1763 data = &obj->ih_data; 1764 if (data->inuse == 0) 1765 return 0; 1766 1767 cancel_work_sync(&data->ih_work); 1768 1769 kfree(data->ring); 1770 memset(data, 0, sizeof(*data)); 1771 put_obj(obj); 1772 1773 return 0; 1774 } 1775 1776 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 1777 struct ras_common_if *head) 1778 { 1779 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1780 struct ras_ih_data *data; 1781 struct amdgpu_ras_block_object *ras_obj; 1782 1783 if (!obj) { 1784 /* in case we registe the IH before enable ras feature */ 1785 obj = amdgpu_ras_create_obj(adev, head); 1786 if (!obj) 1787 return -EINVAL; 1788 } else 1789 get_obj(obj); 1790 1791 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 1792 1793 data = &obj->ih_data; 1794 /* add the callback.etc */ 1795 *data = (struct ras_ih_data) { 1796 .inuse = 0, 1797 .cb = ras_obj->ras_cb, 1798 .element_size = sizeof(struct amdgpu_iv_entry), 1799 .rptr = 0, 1800 .wptr = 0, 1801 }; 1802 1803 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 1804 1805 data->aligned_element_size = ALIGN(data->element_size, 8); 1806 /* the ring can store 64 iv entries. */ 1807 data->ring_size = 64 * data->aligned_element_size; 1808 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 1809 if (!data->ring) { 1810 put_obj(obj); 1811 return -ENOMEM; 1812 } 1813 1814 /* IH is ready */ 1815 data->inuse = 1; 1816 1817 return 0; 1818 } 1819 1820 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 1821 { 1822 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1823 struct ras_manager *obj, *tmp; 1824 1825 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1826 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 1827 } 1828 1829 return 0; 1830 } 1831 /* ih end */ 1832 1833 /* traversal all IPs except NBIO to query error counter */ 1834 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev) 1835 { 1836 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1837 struct ras_manager *obj; 1838 1839 if (!adev->ras_enabled || !con) 1840 return; 1841 1842 list_for_each_entry(obj, &con->head, node) { 1843 struct ras_query_if info = { 1844 .head = obj->head, 1845 }; 1846 1847 /* 1848 * PCIE_BIF IP has one different isr by ras controller 1849 * interrupt, the specific ras counter query will be 1850 * done in that isr. So skip such block from common 1851 * sync flood interrupt isr calling. 1852 */ 1853 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 1854 continue; 1855 1856 /* 1857 * this is a workaround for aldebaran, skip send msg to 1858 * smu to get ecc_info table due to smu handle get ecc 1859 * info table failed temporarily. 1860 * should be removed until smu fix handle ecc_info table. 1861 */ 1862 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 1863 (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2))) 1864 continue; 1865 1866 amdgpu_ras_query_error_status(adev, &info); 1867 1868 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) && 1869 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) && 1870 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) { 1871 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 1872 dev_warn(adev->dev, "Failed to reset error counter and error status"); 1873 } 1874 } 1875 } 1876 1877 /* Parse RdRspStatus and WrRspStatus */ 1878 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 1879 struct ras_query_if *info) 1880 { 1881 struct amdgpu_ras_block_object *block_obj; 1882 /* 1883 * Only two block need to query read/write 1884 * RspStatus at current state 1885 */ 1886 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 1887 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 1888 return; 1889 1890 block_obj = amdgpu_ras_get_ras_block(adev, 1891 info->head.block, 1892 info->head.sub_block_index); 1893 1894 if (!block_obj || !block_obj->hw_ops) { 1895 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1896 get_ras_block_str(&info->head)); 1897 return; 1898 } 1899 1900 if (block_obj->hw_ops->query_ras_error_status) 1901 block_obj->hw_ops->query_ras_error_status(adev); 1902 1903 } 1904 1905 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 1906 { 1907 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1908 struct ras_manager *obj; 1909 1910 if (!adev->ras_enabled || !con) 1911 return; 1912 1913 list_for_each_entry(obj, &con->head, node) { 1914 struct ras_query_if info = { 1915 .head = obj->head, 1916 }; 1917 1918 amdgpu_ras_error_status_query(adev, &info); 1919 } 1920 } 1921 1922 /* recovery begin */ 1923 1924 /* return 0 on success. 1925 * caller need free bps. 1926 */ 1927 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1928 struct ras_badpage **bps, unsigned int *count) 1929 { 1930 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1931 struct ras_err_handler_data *data; 1932 int i = 0; 1933 int ret = 0, status; 1934 1935 if (!con || !con->eh_data || !bps || !count) 1936 return -EINVAL; 1937 1938 mutex_lock(&con->recovery_lock); 1939 data = con->eh_data; 1940 if (!data || data->count == 0) { 1941 *bps = NULL; 1942 ret = -EINVAL; 1943 goto out; 1944 } 1945 1946 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL); 1947 if (!*bps) { 1948 ret = -ENOMEM; 1949 goto out; 1950 } 1951 1952 for (; i < data->count; i++) { 1953 (*bps)[i] = (struct ras_badpage){ 1954 .bp = data->bps[i].retired_page, 1955 .size = AMDGPU_GPU_PAGE_SIZE, 1956 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 1957 }; 1958 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1959 data->bps[i].retired_page); 1960 if (status == -EBUSY) 1961 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1962 else if (status == -ENOENT) 1963 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1964 } 1965 1966 *count = data->count; 1967 out: 1968 mutex_unlock(&con->recovery_lock); 1969 return ret; 1970 } 1971 1972 static void amdgpu_ras_do_recovery(struct work_struct *work) 1973 { 1974 struct amdgpu_ras *ras = 1975 container_of(work, struct amdgpu_ras, recovery_work); 1976 struct amdgpu_device *remote_adev = NULL; 1977 struct amdgpu_device *adev = ras->adev; 1978 struct list_head device_list, *device_list_handle = NULL; 1979 1980 if (!ras->disable_ras_err_cnt_harvest) { 1981 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 1982 1983 /* Build list of devices to query RAS related errors */ 1984 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 1985 device_list_handle = &hive->device_list; 1986 } else { 1987 INIT_LIST_HEAD(&device_list); 1988 list_add_tail(&adev->gmc.xgmi.head, &device_list); 1989 device_list_handle = &device_list; 1990 } 1991 1992 list_for_each_entry(remote_adev, 1993 device_list_handle, gmc.xgmi.head) { 1994 amdgpu_ras_query_err_status(remote_adev); 1995 amdgpu_ras_log_on_err_counter(remote_adev); 1996 } 1997 1998 amdgpu_put_xgmi_hive(hive); 1999 } 2000 2001 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2002 struct amdgpu_reset_context reset_context; 2003 memset(&reset_context, 0, sizeof(reset_context)); 2004 2005 reset_context.method = AMD_RESET_METHOD_NONE; 2006 reset_context.reset_req_dev = adev; 2007 2008 /* Perform full reset in fatal error mode */ 2009 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2010 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2011 else 2012 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2013 2014 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2015 } 2016 atomic_set(&ras->in_recovery, 0); 2017 } 2018 2019 /* alloc/realloc bps array */ 2020 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2021 struct ras_err_handler_data *data, int pages) 2022 { 2023 unsigned int old_space = data->count + data->space_left; 2024 unsigned int new_space = old_space + pages; 2025 unsigned int align_space = ALIGN(new_space, 512); 2026 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL); 2027 2028 if (!bps) { 2029 return -ENOMEM; 2030 } 2031 2032 if (data->bps) { 2033 memcpy(bps, data->bps, 2034 data->count * sizeof(*data->bps)); 2035 kfree(data->bps); 2036 } 2037 2038 data->bps = bps; 2039 data->space_left += align_space - old_space; 2040 return 0; 2041 } 2042 2043 /* it deal with vram only. */ 2044 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 2045 struct eeprom_table_record *bps, int pages) 2046 { 2047 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2048 struct ras_err_handler_data *data; 2049 int ret = 0; 2050 uint32_t i; 2051 2052 if (!con || !con->eh_data || !bps || pages <= 0) 2053 return 0; 2054 2055 mutex_lock(&con->recovery_lock); 2056 data = con->eh_data; 2057 if (!data) 2058 goto out; 2059 2060 for (i = 0; i < pages; i++) { 2061 if (amdgpu_ras_check_bad_page_unlock(con, 2062 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2063 continue; 2064 2065 if (!data->space_left && 2066 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2067 ret = -ENOMEM; 2068 goto out; 2069 } 2070 2071 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr, 2072 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT, 2073 AMDGPU_GPU_PAGE_SIZE); 2074 2075 memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps)); 2076 data->count++; 2077 data->space_left--; 2078 } 2079 out: 2080 mutex_unlock(&con->recovery_lock); 2081 2082 return ret; 2083 } 2084 2085 /* 2086 * write error record array to eeprom, the function should be 2087 * protected by recovery_lock 2088 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 2089 */ 2090 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 2091 unsigned long *new_cnt) 2092 { 2093 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2094 struct ras_err_handler_data *data; 2095 struct amdgpu_ras_eeprom_control *control; 2096 int save_count; 2097 2098 if (!con || !con->eh_data) { 2099 if (new_cnt) 2100 *new_cnt = 0; 2101 2102 return 0; 2103 } 2104 2105 mutex_lock(&con->recovery_lock); 2106 control = &con->eeprom_control; 2107 data = con->eh_data; 2108 save_count = data->count - control->ras_num_recs; 2109 mutex_unlock(&con->recovery_lock); 2110 2111 if (new_cnt) 2112 *new_cnt = save_count / adev->umc.retire_unit; 2113 2114 /* only new entries are saved */ 2115 if (save_count > 0) { 2116 if (amdgpu_ras_eeprom_append(control, 2117 &data->bps[control->ras_num_recs], 2118 save_count)) { 2119 dev_err(adev->dev, "Failed to save EEPROM table data!"); 2120 return -EIO; 2121 } 2122 2123 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 2124 } 2125 2126 return 0; 2127 } 2128 2129 /* 2130 * read error record array in eeprom and reserve enough space for 2131 * storing new bad pages 2132 */ 2133 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 2134 { 2135 struct amdgpu_ras_eeprom_control *control = 2136 &adev->psp.ras_context.ras->eeprom_control; 2137 struct eeprom_table_record *bps; 2138 int ret; 2139 2140 /* no bad page record, skip eeprom access */ 2141 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 2142 return 0; 2143 2144 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 2145 if (!bps) 2146 return -ENOMEM; 2147 2148 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 2149 if (ret) 2150 dev_err(adev->dev, "Failed to load EEPROM table records!"); 2151 else 2152 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs); 2153 2154 kfree(bps); 2155 return ret; 2156 } 2157 2158 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 2159 uint64_t addr) 2160 { 2161 struct ras_err_handler_data *data = con->eh_data; 2162 int i; 2163 2164 addr >>= AMDGPU_GPU_PAGE_SHIFT; 2165 for (i = 0; i < data->count; i++) 2166 if (addr == data->bps[i].retired_page) 2167 return true; 2168 2169 return false; 2170 } 2171 2172 /* 2173 * check if an address belongs to bad page 2174 * 2175 * Note: this check is only for umc block 2176 */ 2177 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 2178 uint64_t addr) 2179 { 2180 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2181 bool ret = false; 2182 2183 if (!con || !con->eh_data) 2184 return ret; 2185 2186 mutex_lock(&con->recovery_lock); 2187 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 2188 mutex_unlock(&con->recovery_lock); 2189 return ret; 2190 } 2191 2192 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 2193 uint32_t max_count) 2194 { 2195 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2196 2197 /* 2198 * Justification of value bad_page_cnt_threshold in ras structure 2199 * 2200 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length 2201 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two 2202 * scenarios accordingly. 2203 * 2204 * Bad page retirement enablement: 2205 * - If amdgpu_bad_page_threshold = -2, 2206 * bad_page_cnt_threshold = typical value by formula. 2207 * 2208 * - When the value from user is 0 < amdgpu_bad_page_threshold < 2209 * max record length in eeprom, use it directly. 2210 * 2211 * Bad page retirement disablement: 2212 * - If amdgpu_bad_page_threshold = 0, bad page retirement 2213 * functionality is disabled, and bad_page_cnt_threshold will 2214 * take no effect. 2215 */ 2216 2217 if (amdgpu_bad_page_threshold < 0) { 2218 u64 val = adev->gmc.mc_vram_size; 2219 2220 do_div(val, RAS_BAD_PAGE_COVER); 2221 con->bad_page_cnt_threshold = min(lower_32_bits(val), 2222 max_count); 2223 } else { 2224 con->bad_page_cnt_threshold = min_t(int, max_count, 2225 amdgpu_bad_page_threshold); 2226 } 2227 } 2228 2229 int amdgpu_ras_recovery_init(struct amdgpu_device *adev) 2230 { 2231 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2232 struct ras_err_handler_data **data; 2233 u32 max_eeprom_records_count = 0; 2234 bool exc_err_limit = false; 2235 int ret; 2236 2237 if (!con || amdgpu_sriov_vf(adev)) 2238 return 0; 2239 2240 /* Allow access to RAS EEPROM via debugfs, when the ASIC 2241 * supports RAS and debugfs is enabled, but when 2242 * adev->ras_enabled is unset, i.e. when "ras_enable" 2243 * module parameter is set to 0. 2244 */ 2245 con->adev = adev; 2246 2247 if (!adev->ras_enabled) 2248 return 0; 2249 2250 data = &con->eh_data; 2251 *data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO); 2252 if (!*data) { 2253 ret = -ENOMEM; 2254 goto out; 2255 } 2256 2257 mutex_init(&con->recovery_lock); 2258 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 2259 atomic_set(&con->in_recovery, 0); 2260 con->eeprom_control.bad_channel_bitmap = 0; 2261 2262 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(); 2263 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 2264 2265 /* Todo: During test the SMU might fail to read the eeprom through I2C 2266 * when the GPU is pending on XGMI reset during probe time 2267 * (Mostly after second bus reset), skip it now 2268 */ 2269 if (adev->gmc.xgmi.pending_reset) 2270 return 0; 2271 ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit); 2272 /* 2273 * This calling fails when exc_err_limit is true or 2274 * ret != 0. 2275 */ 2276 if (exc_err_limit || ret) 2277 goto free; 2278 2279 if (con->eeprom_control.ras_num_recs) { 2280 ret = amdgpu_ras_load_bad_pages(adev); 2281 if (ret) 2282 goto free; 2283 2284 amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs); 2285 2286 if (con->update_channel_flag == true) { 2287 amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap); 2288 con->update_channel_flag = false; 2289 } 2290 } 2291 2292 #ifdef CONFIG_X86_MCE_AMD 2293 if ((adev->asic_type == CHIP_ALDEBARAN) && 2294 (adev->gmc.xgmi.connected_to_cpu)) 2295 amdgpu_register_bad_pages_mca_notifier(adev); 2296 #endif 2297 return 0; 2298 2299 free: 2300 kfree((*data)->bps); 2301 kfree(*data); 2302 con->eh_data = NULL; 2303 out: 2304 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 2305 2306 /* 2307 * Except error threshold exceeding case, other failure cases in this 2308 * function would not fail amdgpu driver init. 2309 */ 2310 if (!exc_err_limit) 2311 ret = 0; 2312 else 2313 ret = -EINVAL; 2314 2315 return ret; 2316 } 2317 2318 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 2319 { 2320 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2321 struct ras_err_handler_data *data = con->eh_data; 2322 2323 /* recovery_init failed to init it, fini is useless */ 2324 if (!data) 2325 return 0; 2326 2327 cancel_work_sync(&con->recovery_work); 2328 2329 mutex_lock(&con->recovery_lock); 2330 con->eh_data = NULL; 2331 kfree(data->bps); 2332 kfree(data); 2333 mutex_unlock(&con->recovery_lock); 2334 2335 return 0; 2336 } 2337 /* recovery end */ 2338 2339 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 2340 { 2341 if (amdgpu_sriov_vf(adev)) { 2342 switch (adev->ip_versions[MP0_HWIP][0]) { 2343 case IP_VERSION(13, 0, 2): 2344 return true; 2345 default: 2346 return false; 2347 } 2348 } 2349 2350 if (adev->asic_type == CHIP_IP_DISCOVERY) { 2351 switch (adev->ip_versions[MP0_HWIP][0]) { 2352 case IP_VERSION(13, 0, 0): 2353 case IP_VERSION(13, 0, 10): 2354 return true; 2355 default: 2356 return false; 2357 } 2358 } 2359 2360 return adev->asic_type == CHIP_VEGA10 || 2361 adev->asic_type == CHIP_VEGA20 || 2362 adev->asic_type == CHIP_ARCTURUS || 2363 adev->asic_type == CHIP_ALDEBARAN || 2364 adev->asic_type == CHIP_SIENNA_CICHLID; 2365 } 2366 2367 /* 2368 * this is workaround for vega20 workstation sku, 2369 * force enable gfx ras, ignore vbios gfx ras flag 2370 * due to GC EDC can not write 2371 */ 2372 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 2373 { 2374 struct atom_context *ctx = adev->mode_info.atom_context; 2375 2376 if (!ctx) 2377 return; 2378 2379 if (strnstr(ctx->vbios_version, "D16406", 2380 sizeof(ctx->vbios_version)) || 2381 strnstr(ctx->vbios_version, "D36002", 2382 sizeof(ctx->vbios_version))) 2383 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 2384 } 2385 2386 /* 2387 * check hardware's ras ability which will be saved in hw_supported. 2388 * if hardware does not support ras, we can skip some ras initializtion and 2389 * forbid some ras operations from IP. 2390 * if software itself, say boot parameter, limit the ras ability. We still 2391 * need allow IP do some limited operations, like disable. In such case, 2392 * we have to initialize ras as normal. but need check if operation is 2393 * allowed or not in each function. 2394 */ 2395 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 2396 { 2397 adev->ras_hw_enabled = adev->ras_enabled = 0; 2398 2399 if (!adev->is_atom_fw || 2400 !amdgpu_ras_asic_supported(adev)) 2401 return; 2402 2403 if (!adev->gmc.xgmi.connected_to_cpu) { 2404 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 2405 dev_info(adev->dev, "MEM ECC is active.\n"); 2406 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 2407 1 << AMDGPU_RAS_BLOCK__DF); 2408 } else { 2409 dev_info(adev->dev, "MEM ECC is not presented.\n"); 2410 } 2411 2412 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 2413 dev_info(adev->dev, "SRAM ECC is active.\n"); 2414 if (!amdgpu_sriov_vf(adev)) 2415 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 2416 1 << AMDGPU_RAS_BLOCK__DF); 2417 else 2418 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 2419 1 << AMDGPU_RAS_BLOCK__SDMA | 2420 1 << AMDGPU_RAS_BLOCK__GFX); 2421 2422 /* VCN/JPEG RAS can be supported on both bare metal and 2423 * SRIOV environment 2424 */ 2425 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0) || 2426 adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 0)) 2427 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 2428 1 << AMDGPU_RAS_BLOCK__JPEG); 2429 else 2430 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 2431 1 << AMDGPU_RAS_BLOCK__JPEG); 2432 2433 /* 2434 * XGMI RAS is not supported if xgmi num physical nodes 2435 * is zero 2436 */ 2437 if (!adev->gmc.xgmi.num_physical_nodes) 2438 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 2439 } else { 2440 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 2441 } 2442 } else { 2443 /* driver only manages a few IP blocks RAS feature 2444 * when GPU is connected cpu through XGMI */ 2445 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 2446 1 << AMDGPU_RAS_BLOCK__SDMA | 2447 1 << AMDGPU_RAS_BLOCK__MMHUB); 2448 } 2449 2450 amdgpu_ras_get_quirks(adev); 2451 2452 /* hw_supported needs to be aligned with RAS block mask. */ 2453 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 2454 2455 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 2456 adev->ras_hw_enabled & amdgpu_ras_mask; 2457 } 2458 2459 static void amdgpu_ras_counte_dw(struct work_struct *work) 2460 { 2461 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 2462 ras_counte_delay_work.work); 2463 struct amdgpu_device *adev = con->adev; 2464 struct drm_device *dev = adev_to_drm(adev); 2465 unsigned long ce_count, ue_count; 2466 int res; 2467 2468 res = pm_runtime_get_sync(dev->dev); 2469 if (res < 0) 2470 goto Out; 2471 2472 /* Cache new values. 2473 */ 2474 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 2475 atomic_set(&con->ras_ce_count, ce_count); 2476 atomic_set(&con->ras_ue_count, ue_count); 2477 } 2478 2479 pm_runtime_mark_last_busy(dev->dev); 2480 Out: 2481 pm_runtime_put_autosuspend(dev->dev); 2482 } 2483 2484 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 2485 { 2486 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2487 bool df_poison, umc_poison; 2488 2489 /* poison setting is useless on SRIOV guest */ 2490 if (amdgpu_sriov_vf(adev) || !con) 2491 return; 2492 2493 /* Init poison supported flag, the default value is false */ 2494 if (adev->gmc.xgmi.connected_to_cpu) { 2495 /* enabled by default when GPU is connected to CPU */ 2496 con->poison_supported = true; 2497 } else if (adev->df.funcs && 2498 adev->df.funcs->query_ras_poison_mode && 2499 adev->umc.ras && 2500 adev->umc.ras->query_ras_poison_mode) { 2501 df_poison = 2502 adev->df.funcs->query_ras_poison_mode(adev); 2503 umc_poison = 2504 adev->umc.ras->query_ras_poison_mode(adev); 2505 2506 /* Only poison is set in both DF and UMC, we can support it */ 2507 if (df_poison && umc_poison) 2508 con->poison_supported = true; 2509 else if (df_poison != umc_poison) 2510 dev_warn(adev->dev, 2511 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 2512 df_poison, umc_poison); 2513 } 2514 } 2515 2516 int amdgpu_ras_init(struct amdgpu_device *adev) 2517 { 2518 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2519 int r; 2520 2521 if (con) 2522 return 0; 2523 2524 con = kmalloc(sizeof(struct amdgpu_ras) + 2525 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 2526 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 2527 GFP_KERNEL|__GFP_ZERO); 2528 if (!con) 2529 return -ENOMEM; 2530 2531 con->adev = adev; 2532 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 2533 atomic_set(&con->ras_ce_count, 0); 2534 atomic_set(&con->ras_ue_count, 0); 2535 2536 con->objs = (struct ras_manager *)(con + 1); 2537 2538 amdgpu_ras_set_context(adev, con); 2539 2540 amdgpu_ras_check_supported(adev); 2541 2542 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 2543 /* set gfx block ras context feature for VEGA20 Gaming 2544 * send ras disable cmd to ras ta during ras late init. 2545 */ 2546 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 2547 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 2548 2549 return 0; 2550 } 2551 2552 r = 0; 2553 goto release_con; 2554 } 2555 2556 con->update_channel_flag = false; 2557 con->features = 0; 2558 INIT_LIST_HEAD(&con->head); 2559 /* Might need get this flag from vbios. */ 2560 con->flags = RAS_DEFAULT_FLAGS; 2561 2562 /* initialize nbio ras function ahead of any other 2563 * ras functions so hardware fatal error interrupt 2564 * can be enabled as early as possible */ 2565 switch (adev->ip_versions[NBIO_HWIP][0]) { 2566 case IP_VERSION(7, 4, 0): 2567 case IP_VERSION(7, 4, 1): 2568 case IP_VERSION(7, 4, 4): 2569 if (!adev->gmc.xgmi.connected_to_cpu) 2570 adev->nbio.ras = &nbio_v7_4_ras; 2571 break; 2572 case IP_VERSION(4, 3, 0): 2573 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 2574 /* unlike other generation of nbio ras, 2575 * nbio v4_3 only support fatal error interrupt 2576 * to inform software that DF is freezed due to 2577 * system fatal error event. driver should not 2578 * enable nbio ras in such case. Instead, 2579 * check DF RAS */ 2580 adev->nbio.ras = &nbio_v4_3_ras; 2581 break; 2582 default: 2583 /* nbio ras is not available */ 2584 break; 2585 } 2586 2587 /* nbio ras block needs to be enabled ahead of other ras blocks 2588 * to handle fatal error */ 2589 r = amdgpu_nbio_ras_sw_init(adev); 2590 if (r) 2591 return r; 2592 2593 if (adev->nbio.ras && 2594 adev->nbio.ras->init_ras_controller_interrupt) { 2595 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 2596 if (r) 2597 goto release_con; 2598 } 2599 2600 if (adev->nbio.ras && 2601 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 2602 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 2603 if (r) 2604 goto release_con; 2605 } 2606 2607 amdgpu_ras_query_poison_mode(adev); 2608 2609 if (amdgpu_ras_fs_init(adev)) { 2610 r = -EINVAL; 2611 goto release_con; 2612 } 2613 2614 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 2615 "hardware ability[%x] ras_mask[%x]\n", 2616 adev->ras_hw_enabled, adev->ras_enabled); 2617 2618 return 0; 2619 release_con: 2620 amdgpu_ras_set_context(adev, NULL); 2621 kfree(con); 2622 2623 return r; 2624 } 2625 2626 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 2627 { 2628 if (adev->gmc.xgmi.connected_to_cpu) 2629 return 1; 2630 return 0; 2631 } 2632 2633 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 2634 struct ras_common_if *ras_block) 2635 { 2636 struct ras_query_if info = { 2637 .head = *ras_block, 2638 }; 2639 2640 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 2641 return 0; 2642 2643 if (amdgpu_ras_query_error_status(adev, &info) != 0) 2644 DRM_WARN("RAS init harvest failure"); 2645 2646 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 2647 DRM_WARN("RAS init harvest reset failure"); 2648 2649 return 0; 2650 } 2651 2652 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 2653 { 2654 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2655 2656 if (!con) 2657 return false; 2658 2659 return con->poison_supported; 2660 } 2661 2662 /* helper function to handle common stuff in ip late init phase */ 2663 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 2664 struct ras_common_if *ras_block) 2665 { 2666 struct amdgpu_ras_block_object *ras_obj = NULL; 2667 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2668 struct ras_query_if *query_info; 2669 unsigned long ue_count, ce_count; 2670 int r; 2671 2672 /* disable RAS feature per IP block if it is not supported */ 2673 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 2674 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 2675 return 0; 2676 } 2677 2678 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 2679 if (r) { 2680 if (adev->in_suspend || amdgpu_in_reset(adev)) { 2681 /* in resume phase, if fail to enable ras, 2682 * clean up all ras fs nodes, and disable ras */ 2683 goto cleanup; 2684 } else 2685 return r; 2686 } 2687 2688 /* check for errors on warm reset edc persisant supported ASIC */ 2689 amdgpu_persistent_edc_harvesting(adev, ras_block); 2690 2691 /* in resume phase, no need to create ras fs node */ 2692 if (adev->in_suspend || amdgpu_in_reset(adev)) 2693 return 0; 2694 2695 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 2696 if (ras_obj->ras_cb || (ras_obj->hw_ops && 2697 (ras_obj->hw_ops->query_poison_status || 2698 ras_obj->hw_ops->handle_poison_consumption))) { 2699 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 2700 if (r) 2701 goto cleanup; 2702 } 2703 2704 r = amdgpu_ras_sysfs_create(adev, ras_block); 2705 if (r) 2706 goto interrupt; 2707 2708 /* Those are the cached values at init. 2709 */ 2710 query_info = kzalloc(sizeof(struct ras_query_if), GFP_KERNEL); 2711 if (!query_info) 2712 return -ENOMEM; 2713 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 2714 2715 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 2716 atomic_set(&con->ras_ce_count, ce_count); 2717 atomic_set(&con->ras_ue_count, ue_count); 2718 } 2719 2720 kfree(query_info); 2721 return 0; 2722 2723 interrupt: 2724 if (ras_obj->ras_cb) 2725 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 2726 cleanup: 2727 amdgpu_ras_feature_enable(adev, ras_block, 0); 2728 return r; 2729 } 2730 2731 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 2732 struct ras_common_if *ras_block) 2733 { 2734 return amdgpu_ras_block_late_init(adev, ras_block); 2735 } 2736 2737 /* helper function to remove ras fs node and interrupt handler */ 2738 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 2739 struct ras_common_if *ras_block) 2740 { 2741 struct amdgpu_ras_block_object *ras_obj; 2742 if (!ras_block) 2743 return; 2744 2745 amdgpu_ras_sysfs_remove(adev, ras_block); 2746 2747 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 2748 if (ras_obj->ras_cb) 2749 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 2750 } 2751 2752 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 2753 struct ras_common_if *ras_block) 2754 { 2755 return amdgpu_ras_block_late_fini(adev, ras_block); 2756 } 2757 2758 /* do some init work after IP late init as dependence. 2759 * and it runs in resume/gpu reset/booting up cases. 2760 */ 2761 void amdgpu_ras_resume(struct amdgpu_device *adev) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_manager *obj, *tmp; 2765 2766 if (!adev->ras_enabled || !con) { 2767 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 2768 amdgpu_release_ras_context(adev); 2769 2770 return; 2771 } 2772 2773 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 2774 /* Set up all other IPs which are not implemented. There is a 2775 * tricky thing that IP's actual ras error type should be 2776 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 2777 * ERROR_NONE make sense anyway. 2778 */ 2779 amdgpu_ras_enable_all_features(adev, 1); 2780 2781 /* We enable ras on all hw_supported block, but as boot 2782 * parameter might disable some of them and one or more IP has 2783 * not implemented yet. So we disable them on behalf. 2784 */ 2785 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2786 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 2787 amdgpu_ras_feature_enable(adev, &obj->head, 0); 2788 /* there should be no any reference. */ 2789 WARN_ON(alive_obj(obj)); 2790 } 2791 } 2792 } 2793 } 2794 2795 void amdgpu_ras_suspend(struct amdgpu_device *adev) 2796 { 2797 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2798 2799 if (!adev->ras_enabled || !con) 2800 return; 2801 2802 amdgpu_ras_disable_all_features(adev, 0); 2803 /* Make sure all ras objects are disabled. */ 2804 if (con->features) 2805 amdgpu_ras_disable_all_features(adev, 1); 2806 } 2807 2808 int amdgpu_ras_late_init(struct amdgpu_device *adev) 2809 { 2810 struct amdgpu_ras_block_list *node, *tmp; 2811 struct amdgpu_ras_block_object *obj; 2812 int r; 2813 2814 /* Guest side doesn't need init ras feature */ 2815 if (amdgpu_sriov_vf(adev)) 2816 return 0; 2817 2818 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 2819 if (!node->ras_obj) { 2820 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 2821 continue; 2822 } 2823 2824 obj = node->ras_obj; 2825 if (obj->ras_late_init) { 2826 r = obj->ras_late_init(adev, &obj->ras_comm); 2827 if (r) { 2828 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 2829 obj->ras_comm.name, r); 2830 return r; 2831 } 2832 } else 2833 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 2834 } 2835 2836 return 0; 2837 } 2838 2839 /* do some fini work before IP fini as dependence */ 2840 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 2841 { 2842 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2843 2844 if (!adev->ras_enabled || !con) 2845 return 0; 2846 2847 2848 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 2849 if (con->features) 2850 amdgpu_ras_disable_all_features(adev, 0); 2851 amdgpu_ras_recovery_fini(adev); 2852 return 0; 2853 } 2854 2855 int amdgpu_ras_fini(struct amdgpu_device *adev) 2856 { 2857 struct amdgpu_ras_block_list *ras_node, *tmp; 2858 struct amdgpu_ras_block_object *obj = NULL; 2859 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2860 2861 if (!adev->ras_enabled || !con) 2862 return 0; 2863 2864 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 2865 if (ras_node->ras_obj) { 2866 obj = ras_node->ras_obj; 2867 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 2868 obj->ras_fini) 2869 obj->ras_fini(adev, &obj->ras_comm); 2870 else 2871 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 2872 } 2873 2874 /* Clear ras blocks from ras_list and free ras block list node */ 2875 list_del(&ras_node->node); 2876 kfree(ras_node); 2877 } 2878 2879 amdgpu_ras_fs_fini(adev); 2880 amdgpu_ras_interrupt_remove_all(adev); 2881 2882 WARN(con->features, "Feature mask is not cleared"); 2883 2884 if (con->features) 2885 amdgpu_ras_disable_all_features(adev, 1); 2886 2887 cancel_delayed_work_sync(&con->ras_counte_delay_work); 2888 2889 amdgpu_ras_set_context(adev, NULL); 2890 kfree(con); 2891 2892 return 0; 2893 } 2894 2895 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 2896 { 2897 amdgpu_ras_check_supported(adev); 2898 if (!adev->ras_hw_enabled) 2899 return; 2900 2901 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 2902 dev_info(adev->dev, "uncorrectable hardware error" 2903 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 2904 2905 amdgpu_ras_reset_gpu(adev); 2906 } 2907 } 2908 2909 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 2910 { 2911 if (adev->asic_type == CHIP_VEGA20 && 2912 adev->pm.fw_version <= 0x283400) { 2913 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 2914 amdgpu_ras_intr_triggered(); 2915 } 2916 2917 return false; 2918 } 2919 2920 void amdgpu_release_ras_context(struct amdgpu_device *adev) 2921 { 2922 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2923 2924 if (!con) 2925 return; 2926 2927 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 2928 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 2929 amdgpu_ras_set_context(adev, NULL); 2930 kfree(con); 2931 } 2932 } 2933 2934 #ifdef CONFIG_X86_MCE_AMD 2935 static struct amdgpu_device *find_adev(uint32_t node_id) 2936 { 2937 int i; 2938 struct amdgpu_device *adev = NULL; 2939 2940 for (i = 0; i < mce_adev_list.num_gpu; i++) { 2941 adev = mce_adev_list.devs[i]; 2942 2943 if (adev && adev->gmc.xgmi.connected_to_cpu && 2944 adev->gmc.xgmi.physical_node_id == node_id) 2945 break; 2946 adev = NULL; 2947 } 2948 2949 return adev; 2950 } 2951 2952 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 2953 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 2954 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 2955 #define GPU_ID_OFFSET 8 2956 2957 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 2958 unsigned long val, void *data) 2959 { 2960 struct mce *m = (struct mce *)data; 2961 struct amdgpu_device *adev = NULL; 2962 uint32_t gpu_id = 0; 2963 uint32_t umc_inst = 0, ch_inst = 0; 2964 2965 /* 2966 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 2967 * and error occurred in DramECC (Extended error code = 0) then only 2968 * process the error, else bail out. 2969 */ 2970 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 2971 (XEC(m->status, 0x3f) == 0x0))) 2972 return NOTIFY_DONE; 2973 2974 /* 2975 * If it is correctable error, return. 2976 */ 2977 if (mce_is_correctable(m)) 2978 return NOTIFY_OK; 2979 2980 /* 2981 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 2982 */ 2983 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 2984 2985 adev = find_adev(gpu_id); 2986 if (!adev) { 2987 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 2988 gpu_id); 2989 return NOTIFY_DONE; 2990 } 2991 2992 /* 2993 * If it is uncorrectable error, then find out UMC instance and 2994 * channel index. 2995 */ 2996 umc_inst = GET_UMC_INST(m->ipid); 2997 ch_inst = GET_CHAN_INDEX(m->ipid); 2998 2999 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 3000 umc_inst, ch_inst); 3001 3002 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 3003 return NOTIFY_OK; 3004 else 3005 return NOTIFY_DONE; 3006 } 3007 3008 static struct notifier_block amdgpu_bad_page_nb = { 3009 .notifier_call = amdgpu_bad_page_notifier, 3010 .priority = MCE_PRIO_UC, 3011 }; 3012 3013 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 3014 { 3015 /* 3016 * Add the adev to the mce_adev_list. 3017 * During mode2 reset, amdgpu device is temporarily 3018 * removed from the mgpu_info list which can cause 3019 * page retirement to fail. 3020 * Use this list instead of mgpu_info to find the amdgpu 3021 * device on which the UMC error was reported. 3022 */ 3023 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 3024 3025 /* 3026 * Register the x86 notifier only once 3027 * with MCE subsystem. 3028 */ 3029 if (notifier_registered == false) { 3030 mce_register_decode_chain(&amdgpu_bad_page_nb); 3031 notifier_registered = true; 3032 } 3033 } 3034 #endif 3035 3036 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 3037 { 3038 if (!adev) 3039 return NULL; 3040 3041 return adev->psp.ras_context.ras; 3042 } 3043 3044 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 3045 { 3046 if (!adev) 3047 return -EINVAL; 3048 3049 adev->psp.ras_context.ras = ras_con; 3050 return 0; 3051 } 3052 3053 /* check if ras is supported on block, say, sdma, gfx */ 3054 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 3055 unsigned int block) 3056 { 3057 int ret = 0; 3058 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3059 3060 if (block >= AMDGPU_RAS_BLOCK_COUNT) 3061 return 0; 3062 3063 ret = ras && (adev->ras_enabled & (1 << block)); 3064 3065 /* For the special asic with mem ecc enabled but sram ecc 3066 * not enabled, even if the ras block is not supported on 3067 * .ras_enabled, if the asic supports poison mode and the 3068 * ras block has ras configuration, it can be considered 3069 * that the ras block supports ras function. 3070 */ 3071 if (!ret && 3072 amdgpu_ras_is_poison_mode_supported(adev) && 3073 amdgpu_ras_get_ras_block(adev, block, 0)) 3074 ret = 1; 3075 3076 return ret; 3077 } 3078 3079 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 3080 { 3081 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3082 3083 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) 3084 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 3085 return 0; 3086 } 3087 3088 3089 /* Register each ip ras block into amdgpu ras */ 3090 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 3091 struct amdgpu_ras_block_object *ras_block_obj) 3092 { 3093 struct amdgpu_ras_block_list *ras_node; 3094 if (!adev || !ras_block_obj) 3095 return -EINVAL; 3096 3097 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 3098 if (!ras_node) 3099 return -ENOMEM; 3100 3101 INIT_LIST_HEAD(&ras_node->node); 3102 ras_node->ras_obj = ras_block_obj; 3103 list_add_tail(&ras_node->node, &adev->ras_list); 3104 3105 return 0; 3106 } 3107