1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/list.h> 25 #include "amdgpu.h" 26 #include "amdgpu_xgmi.h" 27 #include "amdgpu_smu.h" 28 #include "amdgpu_ras.h" 29 #include "df/df_3_6_offset.h" 30 31 static DEFINE_MUTEX(xgmi_mutex); 32 33 #define AMDGPU_MAX_XGMI_HIVE 8 34 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4 35 36 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE]; 37 static unsigned hive_count = 0; 38 39 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive) 40 { 41 return &hive->device_list; 42 } 43 44 /** 45 * DOC: AMDGPU XGMI Support 46 * 47 * XGMI is a high speed interconnect that joins multiple GPU cards 48 * into a homogeneous memory space that is organized by a collective 49 * hive ID and individual node IDs, both of which are 64-bit numbers. 50 * 51 * The file xgmi_device_id contains the unique per GPU device ID and 52 * is stored in the /sys/class/drm/card${cardno}/device/ directory. 53 * 54 * Inside the device directory a sub-directory 'xgmi_hive_info' is 55 * created which contains the hive ID and the list of nodes. 56 * 57 * The hive ID is stored in: 58 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id 59 * 60 * The node information is stored in numbered directories: 61 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id 62 * 63 * Each device has their own xgmi_hive_info direction with a mirror 64 * set of node sub-directories. 65 * 66 * The XGMI memory space is built by contiguously adding the power of 67 * two padded VRAM space from each node to each other. 68 * 69 */ 70 71 72 static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev, 73 struct device_attribute *attr, char *buf) 74 { 75 struct amdgpu_hive_info *hive = 76 container_of(attr, struct amdgpu_hive_info, dev_attr); 77 78 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id); 79 } 80 81 static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev, 82 struct amdgpu_hive_info *hive) 83 { 84 int ret = 0; 85 86 if (WARN_ON(hive->kobj)) 87 return -EINVAL; 88 89 hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj); 90 if (!hive->kobj) { 91 dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n"); 92 return -EINVAL; 93 } 94 95 hive->dev_attr = (struct device_attribute) { 96 .attr = { 97 .name = "xgmi_hive_id", 98 .mode = S_IRUGO, 99 100 }, 101 .show = amdgpu_xgmi_show_hive_id, 102 }; 103 104 ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr); 105 if (ret) { 106 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n"); 107 kobject_del(hive->kobj); 108 kobject_put(hive->kobj); 109 hive->kobj = NULL; 110 } 111 112 return ret; 113 } 114 115 static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev, 116 struct amdgpu_hive_info *hive) 117 { 118 sysfs_remove_file(hive->kobj, &hive->dev_attr.attr); 119 kobject_del(hive->kobj); 120 kobject_put(hive->kobj); 121 hive->kobj = NULL; 122 } 123 124 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev, 125 struct device_attribute *attr, 126 char *buf) 127 { 128 struct drm_device *ddev = dev_get_drvdata(dev); 129 struct amdgpu_device *adev = ddev->dev_private; 130 131 return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id); 132 133 } 134 135 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801) 136 static ssize_t amdgpu_xgmi_show_error(struct device *dev, 137 struct device_attribute *attr, 138 char *buf) 139 { 140 struct drm_device *ddev = dev_get_drvdata(dev); 141 struct amdgpu_device *adev = ddev->dev_private; 142 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in; 143 uint64_t fica_out; 144 unsigned int error_count = 0; 145 146 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200); 147 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208); 148 149 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_ctl_in); 150 if (fica_out != 0x1f) 151 pr_err("xGMI error counters not enabled!\n"); 152 153 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_status_in); 154 155 if ((fica_out & 0xffff) == 2) 156 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63); 157 158 adev->df_funcs->set_fica(adev, ficaa_pie_status_in, 0, 0); 159 160 return snprintf(buf, PAGE_SIZE, "%d\n", error_count); 161 } 162 163 164 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL); 165 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL); 166 167 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, 168 struct amdgpu_hive_info *hive) 169 { 170 int ret = 0; 171 char node[10] = { 0 }; 172 173 /* Create xgmi device id file */ 174 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id); 175 if (ret) { 176 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n"); 177 return ret; 178 } 179 180 /* Create xgmi error file */ 181 ret = device_create_file(adev->dev, &dev_attr_xgmi_error); 182 if (ret) 183 pr_err("failed to create xgmi_error\n"); 184 185 186 /* Create sysfs link to hive info folder on the first device */ 187 if (adev != hive->adev) { 188 ret = sysfs_create_link(&adev->dev->kobj, hive->kobj, 189 "xgmi_hive_info"); 190 if (ret) { 191 dev_err(adev->dev, "XGMI: Failed to create link to hive info"); 192 goto remove_file; 193 } 194 } 195 196 sprintf(node, "node%d", hive->number_devices); 197 /* Create sysfs link form the hive folder to yourself */ 198 ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node); 199 if (ret) { 200 dev_err(adev->dev, "XGMI: Failed to create link from hive info"); 201 goto remove_link; 202 } 203 204 goto success; 205 206 207 remove_link: 208 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); 209 210 remove_file: 211 device_remove_file(adev->dev, &dev_attr_xgmi_device_id); 212 213 success: 214 return ret; 215 } 216 217 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, 218 struct amdgpu_hive_info *hive) 219 { 220 device_remove_file(adev->dev, &dev_attr_xgmi_device_id); 221 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); 222 sysfs_remove_link(hive->kobj, adev->ddev->unique); 223 } 224 225 226 227 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock) 228 { 229 int i; 230 struct amdgpu_hive_info *tmp; 231 232 if (!adev->gmc.xgmi.hive_id) 233 return NULL; 234 235 mutex_lock(&xgmi_mutex); 236 237 for (i = 0 ; i < hive_count; ++i) { 238 tmp = &xgmi_hives[i]; 239 if (tmp->hive_id == adev->gmc.xgmi.hive_id) { 240 if (lock) 241 mutex_lock(&tmp->hive_lock); 242 mutex_unlock(&xgmi_mutex); 243 return tmp; 244 } 245 } 246 if (i >= AMDGPU_MAX_XGMI_HIVE) { 247 mutex_unlock(&xgmi_mutex); 248 return NULL; 249 } 250 251 /* initialize new hive if not exist */ 252 tmp = &xgmi_hives[hive_count++]; 253 254 if (amdgpu_xgmi_sysfs_create(adev, tmp)) { 255 mutex_unlock(&xgmi_mutex); 256 return NULL; 257 } 258 259 tmp->adev = adev; 260 tmp->hive_id = adev->gmc.xgmi.hive_id; 261 INIT_LIST_HEAD(&tmp->device_list); 262 mutex_init(&tmp->hive_lock); 263 mutex_init(&tmp->reset_lock); 264 265 if (lock) 266 mutex_lock(&tmp->hive_lock); 267 tmp->pstate = -1; 268 mutex_unlock(&xgmi_mutex); 269 270 return tmp; 271 } 272 273 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) 274 { 275 int ret = 0; 276 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); 277 struct amdgpu_device *tmp_adev; 278 bool update_hive_pstate = true; 279 bool is_high_pstate = pstate && adev->asic_type == CHIP_VEGA20; 280 281 if (!hive) 282 return 0; 283 284 mutex_lock(&hive->hive_lock); 285 286 if (hive->pstate == pstate) { 287 adev->pstate = is_high_pstate ? pstate : adev->pstate; 288 goto out; 289 } 290 291 dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate); 292 293 if (is_support_sw_smu_xgmi(adev)) 294 ret = smu_set_xgmi_pstate(&adev->smu, pstate); 295 else if (adev->powerplay.pp_funcs && 296 adev->powerplay.pp_funcs->set_xgmi_pstate) 297 ret = adev->powerplay.pp_funcs->set_xgmi_pstate(adev->powerplay.pp_handle, 298 pstate); 299 300 if (ret) { 301 dev_err(adev->dev, 302 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d", 303 adev->gmc.xgmi.node_id, 304 adev->gmc.xgmi.hive_id, ret); 305 goto out; 306 } 307 308 /* Update device pstate */ 309 adev->pstate = pstate; 310 311 /* 312 * Update the hive pstate only all devices of the hive 313 * are in the same pstate 314 */ 315 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 316 if (tmp_adev->pstate != adev->pstate) { 317 update_hive_pstate = false; 318 break; 319 } 320 } 321 if (update_hive_pstate || is_high_pstate) 322 hive->pstate = pstate; 323 324 out: 325 mutex_unlock(&hive->hive_lock); 326 327 return ret; 328 } 329 330 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev) 331 { 332 int ret = -EINVAL; 333 334 /* Each psp need to set the latest topology */ 335 ret = psp_xgmi_set_topology_info(&adev->psp, 336 hive->number_devices, 337 &adev->psp.xgmi_context.top_info); 338 if (ret) 339 dev_err(adev->dev, 340 "XGMI: Set topology failure on device %llx, hive %llx, ret %d", 341 adev->gmc.xgmi.node_id, 342 adev->gmc.xgmi.hive_id, ret); 343 344 return ret; 345 } 346 347 348 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev, 349 struct amdgpu_device *peer_adev) 350 { 351 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info; 352 int i; 353 354 for (i = 0 ; i < top->num_nodes; ++i) 355 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id) 356 return top->nodes[i].num_hops; 357 return -EINVAL; 358 } 359 360 int amdgpu_xgmi_add_device(struct amdgpu_device *adev) 361 { 362 struct psp_xgmi_topology_info *top_info; 363 struct amdgpu_hive_info *hive; 364 struct amdgpu_xgmi *entry; 365 struct amdgpu_device *tmp_adev = NULL; 366 367 int count = 0, ret = 0; 368 369 if (!adev->gmc.xgmi.supported) 370 return 0; 371 372 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) { 373 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id); 374 if (ret) { 375 dev_err(adev->dev, 376 "XGMI: Failed to get hive id\n"); 377 return ret; 378 } 379 380 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id); 381 if (ret) { 382 dev_err(adev->dev, 383 "XGMI: Failed to get node id\n"); 384 return ret; 385 } 386 } else { 387 adev->gmc.xgmi.hive_id = 16; 388 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16; 389 } 390 391 hive = amdgpu_get_xgmi_hive(adev, 1); 392 if (!hive) { 393 ret = -EINVAL; 394 dev_err(adev->dev, 395 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n", 396 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id); 397 goto exit; 398 } 399 400 /* Set default device pstate */ 401 adev->pstate = -1; 402 403 top_info = &adev->psp.xgmi_context.top_info; 404 405 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list); 406 list_for_each_entry(entry, &hive->device_list, head) 407 top_info->nodes[count++].node_id = entry->node_id; 408 top_info->num_nodes = count; 409 hive->number_devices = count; 410 411 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) { 412 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 413 /* update node list for other device in the hive */ 414 if (tmp_adev != adev) { 415 top_info = &tmp_adev->psp.xgmi_context.top_info; 416 top_info->nodes[count - 1].node_id = 417 adev->gmc.xgmi.node_id; 418 top_info->num_nodes = count; 419 } 420 ret = amdgpu_xgmi_update_topology(hive, tmp_adev); 421 if (ret) 422 goto exit; 423 } 424 425 /* get latest topology info for each device from psp */ 426 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 427 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, 428 &tmp_adev->psp.xgmi_context.top_info); 429 if (ret) { 430 dev_err(tmp_adev->dev, 431 "XGMI: Get topology failure on device %llx, hive %llx, ret %d", 432 tmp_adev->gmc.xgmi.node_id, 433 tmp_adev->gmc.xgmi.hive_id, ret); 434 /* To do : continue with some node failed or disable the whole hive */ 435 goto exit; 436 } 437 } 438 } 439 440 if (!ret) 441 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive); 442 443 444 mutex_unlock(&hive->hive_lock); 445 exit: 446 if (!ret) 447 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n", 448 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id); 449 else 450 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n", 451 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id, 452 ret); 453 454 return ret; 455 } 456 457 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev) 458 { 459 struct amdgpu_hive_info *hive; 460 461 if (!adev->gmc.xgmi.supported) 462 return; 463 464 hive = amdgpu_get_xgmi_hive(adev, 1); 465 if (!hive) 466 return; 467 468 if (!(hive->number_devices--)) { 469 amdgpu_xgmi_sysfs_destroy(adev, hive); 470 mutex_destroy(&hive->hive_lock); 471 mutex_destroy(&hive->reset_lock); 472 } else { 473 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive); 474 mutex_unlock(&hive->hive_lock); 475 } 476 } 477 478 int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev) 479 { 480 int r; 481 struct ras_ih_if ih_info = { 482 .cb = NULL, 483 }; 484 struct ras_fs_if fs_info = { 485 .sysfs_name = "xgmi_wafl_err_count", 486 .debugfs_name = "xgmi_wafl_err_inject", 487 }; 488 489 if (!adev->gmc.xgmi.supported || 490 adev->gmc.xgmi.num_physical_nodes == 0) 491 return 0; 492 493 if (!adev->gmc.xgmi.ras_if) { 494 adev->gmc.xgmi.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL); 495 if (!adev->gmc.xgmi.ras_if) 496 return -ENOMEM; 497 adev->gmc.xgmi.ras_if->block = AMDGPU_RAS_BLOCK__XGMI_WAFL; 498 adev->gmc.xgmi.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 499 adev->gmc.xgmi.ras_if->sub_block_index = 0; 500 strcpy(adev->gmc.xgmi.ras_if->name, "xgmi_wafl"); 501 } 502 ih_info.head = fs_info.head = *adev->gmc.xgmi.ras_if; 503 r = amdgpu_ras_late_init(adev, adev->gmc.xgmi.ras_if, 504 &fs_info, &ih_info); 505 if (r || !amdgpu_ras_is_supported(adev, adev->gmc.xgmi.ras_if->block)) { 506 kfree(adev->gmc.xgmi.ras_if); 507 adev->gmc.xgmi.ras_if = NULL; 508 } 509 510 return r; 511 } 512 513 void amdgpu_xgmi_ras_fini(struct amdgpu_device *adev) 514 { 515 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL) && 516 adev->gmc.xgmi.ras_if) { 517 struct ras_common_if *ras_if = adev->gmc.xgmi.ras_if; 518 struct ras_ih_if ih_info = { 519 .cb = NULL, 520 }; 521 522 amdgpu_ras_late_fini(adev, ras_if, &ih_info); 523 kfree(ras_if); 524 } 525 } 526