1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include "amdgpu.h" 27 #include "amdgpu_gfx.h" 28 #include "amdgpu_rlc.h" 29 #include "amdgpu_ras.h" 30 31 /* delay 0.1 second to enable gfx off feature */ 32 #define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100) 33 34 /* 35 * GPU GFX IP block helpers function. 36 */ 37 38 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, 39 int pipe, int queue) 40 { 41 int bit = 0; 42 43 bit += mec * adev->gfx.mec.num_pipe_per_mec 44 * adev->gfx.mec.num_queue_per_pipe; 45 bit += pipe * adev->gfx.mec.num_queue_per_pipe; 46 bit += queue; 47 48 return bit; 49 } 50 51 void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit, 52 int *mec, int *pipe, int *queue) 53 { 54 *queue = bit % adev->gfx.mec.num_queue_per_pipe; 55 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) 56 % adev->gfx.mec.num_pipe_per_mec; 57 *mec = (bit / adev->gfx.mec.num_queue_per_pipe) 58 / adev->gfx.mec.num_pipe_per_mec; 59 60 } 61 62 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, 63 int mec, int pipe, int queue) 64 { 65 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), 66 adev->gfx.mec.queue_bitmap); 67 } 68 69 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, 70 int me, int pipe, int queue) 71 { 72 int bit = 0; 73 74 bit += me * adev->gfx.me.num_pipe_per_me 75 * adev->gfx.me.num_queue_per_pipe; 76 bit += pipe * adev->gfx.me.num_queue_per_pipe; 77 bit += queue; 78 79 return bit; 80 } 81 82 void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, 83 int *me, int *pipe, int *queue) 84 { 85 *queue = bit % adev->gfx.me.num_queue_per_pipe; 86 *pipe = (bit / adev->gfx.me.num_queue_per_pipe) 87 % adev->gfx.me.num_pipe_per_me; 88 *me = (bit / adev->gfx.me.num_queue_per_pipe) 89 / adev->gfx.me.num_pipe_per_me; 90 } 91 92 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, 93 int me, int pipe, int queue) 94 { 95 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), 96 adev->gfx.me.queue_bitmap); 97 } 98 99 /** 100 * amdgpu_gfx_scratch_get - Allocate a scratch register 101 * 102 * @adev: amdgpu_device pointer 103 * @reg: scratch register mmio offset 104 * 105 * Allocate a CP scratch register for use by the driver (all asics). 106 * Returns 0 on success or -EINVAL on failure. 107 */ 108 int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg) 109 { 110 int i; 111 112 i = ffs(adev->gfx.scratch.free_mask); 113 if (i != 0 && i <= adev->gfx.scratch.num_reg) { 114 i--; 115 adev->gfx.scratch.free_mask &= ~(1u << i); 116 *reg = adev->gfx.scratch.reg_base + i; 117 return 0; 118 } 119 return -EINVAL; 120 } 121 122 /** 123 * amdgpu_gfx_scratch_free - Free a scratch register 124 * 125 * @adev: amdgpu_device pointer 126 * @reg: scratch register mmio offset 127 * 128 * Free a CP scratch register allocated for use by the driver (all asics) 129 */ 130 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg) 131 { 132 adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base); 133 } 134 135 /** 136 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter 137 * 138 * @mask: array in which the per-shader array disable masks will be stored 139 * @max_se: number of SEs 140 * @max_sh: number of SHs 141 * 142 * The bitmask of CUs to be disabled in the shader array determined by se and 143 * sh is stored in mask[se * max_sh + sh]. 144 */ 145 void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh) 146 { 147 unsigned se, sh, cu; 148 const char *p; 149 150 memset(mask, 0, sizeof(*mask) * max_se * max_sh); 151 152 if (!amdgpu_disable_cu || !*amdgpu_disable_cu) 153 return; 154 155 p = amdgpu_disable_cu; 156 for (;;) { 157 char *next; 158 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); 159 if (ret < 3) { 160 DRM_ERROR("amdgpu: could not parse disable_cu\n"); 161 return; 162 } 163 164 if (se < max_se && sh < max_sh && cu < 16) { 165 DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); 166 mask[se * max_sh + sh] |= 1u << cu; 167 } else { 168 DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", 169 se, sh, cu); 170 } 171 172 next = strchr(p, ','); 173 if (!next) 174 break; 175 p = next + 1; 176 } 177 } 178 179 static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev) 180 { 181 if (amdgpu_compute_multipipe != -1) { 182 DRM_INFO("amdgpu: forcing compute pipe policy %d\n", 183 amdgpu_compute_multipipe); 184 return amdgpu_compute_multipipe == 1; 185 } 186 187 /* FIXME: spreading the queues across pipes causes perf regressions 188 * on POLARIS11 compute workloads */ 189 if (adev->asic_type == CHIP_POLARIS11) 190 return false; 191 192 return adev->gfx.mec.num_mec > 1; 193 } 194 195 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, 196 int queue) 197 { 198 /* Policy: make queue 0 of each pipe as high priority compute queue */ 199 return (queue == 0); 200 201 } 202 203 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) 204 { 205 int i, queue, pipe, mec; 206 bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); 207 208 /* policy for amdgpu compute queue ownership */ 209 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 210 queue = i % adev->gfx.mec.num_queue_per_pipe; 211 pipe = (i / adev->gfx.mec.num_queue_per_pipe) 212 % adev->gfx.mec.num_pipe_per_mec; 213 mec = (i / adev->gfx.mec.num_queue_per_pipe) 214 / adev->gfx.mec.num_pipe_per_mec; 215 216 /* we've run out of HW */ 217 if (mec >= adev->gfx.mec.num_mec) 218 break; 219 220 if (multipipe_policy) { 221 /* policy: amdgpu owns the first two queues of the first MEC */ 222 if (mec == 0 && queue < 2) 223 set_bit(i, adev->gfx.mec.queue_bitmap); 224 } else { 225 /* policy: amdgpu owns all queues in the first pipe */ 226 if (mec == 0 && pipe == 0) 227 set_bit(i, adev->gfx.mec.queue_bitmap); 228 } 229 } 230 231 /* update the number of active compute rings */ 232 adev->gfx.num_compute_rings = 233 bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 234 235 /* If you hit this case and edited the policy, you probably just 236 * need to increase AMDGPU_MAX_COMPUTE_RINGS */ 237 if (WARN_ON(adev->gfx.num_compute_rings > AMDGPU_MAX_COMPUTE_RINGS)) 238 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; 239 } 240 241 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) 242 { 243 int i, queue, me; 244 245 for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) { 246 queue = i % adev->gfx.me.num_queue_per_pipe; 247 me = (i / adev->gfx.me.num_queue_per_pipe) 248 / adev->gfx.me.num_pipe_per_me; 249 250 if (me >= adev->gfx.me.num_me) 251 break; 252 /* policy: amdgpu owns the first queue per pipe at this stage 253 * will extend to mulitple queues per pipe later */ 254 if (me == 0 && queue < 1) 255 set_bit(i, adev->gfx.me.queue_bitmap); 256 } 257 258 /* update the number of active graphics rings */ 259 adev->gfx.num_gfx_rings = 260 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 261 } 262 263 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, 264 struct amdgpu_ring *ring) 265 { 266 int queue_bit; 267 int mec, pipe, queue; 268 269 queue_bit = adev->gfx.mec.num_mec 270 * adev->gfx.mec.num_pipe_per_mec 271 * adev->gfx.mec.num_queue_per_pipe; 272 273 while (queue_bit-- >= 0) { 274 if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap)) 275 continue; 276 277 amdgpu_gfx_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 278 279 /* 280 * 1. Using pipes 2/3 from MEC 2 seems cause problems. 281 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN 282 * only can be issued on queue 0. 283 */ 284 if ((mec == 1 && pipe > 1) || queue != 0) 285 continue; 286 287 ring->me = mec + 1; 288 ring->pipe = pipe; 289 ring->queue = queue; 290 291 return 0; 292 } 293 294 dev_err(adev->dev, "Failed to find a queue for KIQ\n"); 295 return -EINVAL; 296 } 297 298 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, 299 struct amdgpu_ring *ring, 300 struct amdgpu_irq_src *irq) 301 { 302 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 303 int r = 0; 304 305 spin_lock_init(&kiq->ring_lock); 306 307 r = amdgpu_device_wb_get(adev, &kiq->reg_val_offs); 308 if (r) 309 return r; 310 311 ring->adev = NULL; 312 ring->ring_obj = NULL; 313 ring->use_doorbell = true; 314 ring->doorbell_index = adev->doorbell_index.kiq; 315 316 r = amdgpu_gfx_kiq_acquire(adev, ring); 317 if (r) 318 return r; 319 320 ring->eop_gpu_addr = kiq->eop_gpu_addr; 321 sprintf(ring->name, "kiq_%d.%d.%d", ring->me, ring->pipe, ring->queue); 322 r = amdgpu_ring_init(adev, ring, 1024, 323 irq, AMDGPU_CP_KIQ_IRQ_DRIVER0); 324 if (r) 325 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); 326 327 return r; 328 } 329 330 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring) 331 { 332 amdgpu_device_wb_free(ring->adev, ring->adev->gfx.kiq.reg_val_offs); 333 amdgpu_ring_fini(ring); 334 } 335 336 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev) 337 { 338 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 339 340 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL); 341 } 342 343 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, 344 unsigned hpd_size) 345 { 346 int r; 347 u32 *hpd; 348 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 349 350 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, 351 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj, 352 &kiq->eop_gpu_addr, (void **)&hpd); 353 if (r) { 354 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); 355 return r; 356 } 357 358 memset(hpd, 0, hpd_size); 359 360 r = amdgpu_bo_reserve(kiq->eop_obj, true); 361 if (unlikely(r != 0)) 362 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 363 amdgpu_bo_kunmap(kiq->eop_obj); 364 amdgpu_bo_unreserve(kiq->eop_obj); 365 366 return 0; 367 } 368 369 /* create MQD for each compute/gfx queue */ 370 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, 371 unsigned mqd_size) 372 { 373 struct amdgpu_ring *ring = NULL; 374 int r, i; 375 376 /* create MQD for KIQ */ 377 ring = &adev->gfx.kiq.ring; 378 if (!ring->mqd_obj) { 379 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must 380 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD 381 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for 382 * KIQ MQD no matter SRIOV or Bare-metal 383 */ 384 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 385 AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj, 386 &ring->mqd_gpu_addr, &ring->mqd_ptr); 387 if (r) { 388 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); 389 return r; 390 } 391 392 /* prepare MQD backup */ 393 adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL); 394 if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]) 395 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 396 } 397 398 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 399 /* create MQD for each KGQ */ 400 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 401 ring = &adev->gfx.gfx_ring[i]; 402 if (!ring->mqd_obj) { 403 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 404 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, 405 &ring->mqd_gpu_addr, &ring->mqd_ptr); 406 if (r) { 407 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 408 return r; 409 } 410 411 /* prepare MQD backup */ 412 adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); 413 if (!adev->gfx.me.mqd_backup[i]) 414 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 415 } 416 } 417 } 418 419 /* create MQD for each KCQ */ 420 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 421 ring = &adev->gfx.compute_ring[i]; 422 if (!ring->mqd_obj) { 423 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 424 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, 425 &ring->mqd_gpu_addr, &ring->mqd_ptr); 426 if (r) { 427 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 428 return r; 429 } 430 431 /* prepare MQD backup */ 432 adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); 433 if (!adev->gfx.mec.mqd_backup[i]) 434 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 435 } 436 } 437 438 return 0; 439 } 440 441 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev) 442 { 443 struct amdgpu_ring *ring = NULL; 444 int i; 445 446 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 447 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 448 ring = &adev->gfx.gfx_ring[i]; 449 kfree(adev->gfx.me.mqd_backup[i]); 450 amdgpu_bo_free_kernel(&ring->mqd_obj, 451 &ring->mqd_gpu_addr, 452 &ring->mqd_ptr); 453 } 454 } 455 456 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 457 ring = &adev->gfx.compute_ring[i]; 458 kfree(adev->gfx.mec.mqd_backup[i]); 459 amdgpu_bo_free_kernel(&ring->mqd_obj, 460 &ring->mqd_gpu_addr, 461 &ring->mqd_ptr); 462 } 463 464 ring = &adev->gfx.kiq.ring; 465 kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]); 466 amdgpu_bo_free_kernel(&ring->mqd_obj, 467 &ring->mqd_gpu_addr, 468 &ring->mqd_ptr); 469 } 470 471 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) 472 { 473 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 474 struct amdgpu_ring *kiq_ring = &kiq->ring; 475 int i; 476 477 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 478 return -EINVAL; 479 480 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 481 adev->gfx.num_compute_rings)) 482 return -ENOMEM; 483 484 for (i = 0; i < adev->gfx.num_compute_rings; i++) 485 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i], 486 RESET_QUEUES, 0, 0); 487 488 return amdgpu_ring_test_helper(kiq_ring); 489 } 490 491 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev) 492 { 493 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 494 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 495 uint64_t queue_mask = 0; 496 int r, i; 497 498 if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources) 499 return -EINVAL; 500 501 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 502 if (!test_bit(i, adev->gfx.mec.queue_bitmap)) 503 continue; 504 505 /* This situation may be hit in the future if a new HW 506 * generation exposes more than 64 queues. If so, the 507 * definition of queue_mask needs updating */ 508 if (WARN_ON(i > (sizeof(queue_mask)*8))) { 509 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 510 break; 511 } 512 513 queue_mask |= (1ull << i); 514 } 515 516 DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe, 517 kiq_ring->queue); 518 519 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 520 adev->gfx.num_compute_rings + 521 kiq->pmf->set_resources_size); 522 if (r) { 523 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 524 return r; 525 } 526 527 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 528 for (i = 0; i < adev->gfx.num_compute_rings; i++) 529 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]); 530 531 r = amdgpu_ring_test_helper(kiq_ring); 532 if (r) 533 DRM_ERROR("KCQ enable failed\n"); 534 535 return r; 536 } 537 538 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable 539 * 540 * @adev: amdgpu_device pointer 541 * @bool enable true: enable gfx off feature, false: disable gfx off feature 542 * 543 * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled. 544 * 2. other client can send request to disable gfx off feature, the request should be honored. 545 * 3. other client can cancel their request of disable gfx off feature 546 * 4. other client should not send request to enable gfx off feature before disable gfx off feature. 547 */ 548 549 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) 550 { 551 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) 552 return; 553 554 mutex_lock(&adev->gfx.gfx_off_mutex); 555 556 if (!enable) 557 adev->gfx.gfx_off_req_count++; 558 else if (adev->gfx.gfx_off_req_count > 0) 559 adev->gfx.gfx_off_req_count--; 560 561 if (enable && !adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { 562 schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); 563 } else if (!enable && adev->gfx.gfx_off_state) { 564 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) 565 adev->gfx.gfx_off_state = false; 566 } 567 568 mutex_unlock(&adev->gfx.gfx_off_mutex); 569 } 570 571 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev) 572 { 573 int r; 574 struct ras_fs_if fs_info = { 575 .sysfs_name = "gfx_err_count", 576 }; 577 struct ras_ih_if ih_info = { 578 .cb = amdgpu_gfx_process_ras_data_cb, 579 }; 580 581 if (!adev->gfx.ras_if) { 582 adev->gfx.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL); 583 if (!adev->gfx.ras_if) 584 return -ENOMEM; 585 adev->gfx.ras_if->block = AMDGPU_RAS_BLOCK__GFX; 586 adev->gfx.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 587 adev->gfx.ras_if->sub_block_index = 0; 588 strcpy(adev->gfx.ras_if->name, "gfx"); 589 } 590 fs_info.head = ih_info.head = *adev->gfx.ras_if; 591 592 r = amdgpu_ras_late_init(adev, adev->gfx.ras_if, 593 &fs_info, &ih_info); 594 if (r) 595 goto free; 596 597 if (amdgpu_ras_is_supported(adev, adev->gfx.ras_if->block)) { 598 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 599 if (r) 600 goto late_fini; 601 } else { 602 /* free gfx ras_if if ras is not supported */ 603 r = 0; 604 goto free; 605 } 606 607 return 0; 608 late_fini: 609 amdgpu_ras_late_fini(adev, adev->gfx.ras_if, &ih_info); 610 free: 611 kfree(adev->gfx.ras_if); 612 adev->gfx.ras_if = NULL; 613 return r; 614 } 615 616 void amdgpu_gfx_ras_fini(struct amdgpu_device *adev) 617 { 618 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX) && 619 adev->gfx.ras_if) { 620 struct ras_common_if *ras_if = adev->gfx.ras_if; 621 struct ras_ih_if ih_info = { 622 .head = *ras_if, 623 .cb = amdgpu_gfx_process_ras_data_cb, 624 }; 625 626 amdgpu_ras_late_fini(adev, ras_if, &ih_info); 627 kfree(ras_if); 628 } 629 } 630 631 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, 632 void *err_data, 633 struct amdgpu_iv_entry *entry) 634 { 635 /* TODO ue will trigger an interrupt. 636 * 637 * When “Full RAS” is enabled, the per-IP interrupt sources should 638 * be disabled and the driver should only look for the aggregated 639 * interrupt via sync flood 640 */ 641 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) { 642 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 643 if (adev->gfx.funcs->query_ras_error_count) 644 adev->gfx.funcs->query_ras_error_count(adev, err_data); 645 amdgpu_ras_reset_gpu(adev); 646 } 647 return AMDGPU_RAS_SUCCESS; 648 } 649 650 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, 651 struct amdgpu_irq_src *source, 652 struct amdgpu_iv_entry *entry) 653 { 654 struct ras_common_if *ras_if = adev->gfx.ras_if; 655 struct ras_dispatch_if ih_data = { 656 .entry = entry, 657 }; 658 659 if (!ras_if) 660 return 0; 661 662 ih_data.head = *ras_if; 663 664 DRM_ERROR("CP ECC ERROR IRQ\n"); 665 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 666 return 0; 667 } 668 669 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) 670 { 671 signed long r, cnt = 0; 672 unsigned long flags; 673 uint32_t seq; 674 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 675 struct amdgpu_ring *ring = &kiq->ring; 676 677 BUG_ON(!ring->funcs->emit_rreg); 678 679 spin_lock_irqsave(&kiq->ring_lock, flags); 680 amdgpu_ring_alloc(ring, 32); 681 amdgpu_ring_emit_rreg(ring, reg); 682 amdgpu_fence_emit_polling(ring, &seq); 683 amdgpu_ring_commit(ring); 684 spin_unlock_irqrestore(&kiq->ring_lock, flags); 685 686 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 687 688 /* don't wait anymore for gpu reset case because this way may 689 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 690 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 691 * never return if we keep waiting in virt_kiq_rreg, which cause 692 * gpu_recover() hang there. 693 * 694 * also don't wait anymore for IRQ context 695 * */ 696 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 697 goto failed_kiq_read; 698 699 might_sleep(); 700 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 701 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 702 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 703 } 704 705 if (cnt > MAX_KIQ_REG_TRY) 706 goto failed_kiq_read; 707 708 return adev->wb.wb[kiq->reg_val_offs]; 709 710 failed_kiq_read: 711 pr_err("failed to read reg:%x\n", reg); 712 return ~0; 713 } 714 715 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 716 { 717 signed long r, cnt = 0; 718 unsigned long flags; 719 uint32_t seq; 720 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 721 struct amdgpu_ring *ring = &kiq->ring; 722 723 BUG_ON(!ring->funcs->emit_wreg); 724 725 spin_lock_irqsave(&kiq->ring_lock, flags); 726 amdgpu_ring_alloc(ring, 32); 727 amdgpu_ring_emit_wreg(ring, reg, v); 728 amdgpu_fence_emit_polling(ring, &seq); 729 amdgpu_ring_commit(ring); 730 spin_unlock_irqrestore(&kiq->ring_lock, flags); 731 732 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 733 734 /* don't wait anymore for gpu reset case because this way may 735 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 736 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 737 * never return if we keep waiting in virt_kiq_rreg, which cause 738 * gpu_recover() hang there. 739 * 740 * also don't wait anymore for IRQ context 741 * */ 742 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 743 goto failed_kiq_write; 744 745 might_sleep(); 746 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 747 748 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 749 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 750 } 751 752 if (cnt > MAX_KIQ_REG_TRY) 753 goto failed_kiq_write; 754 755 return; 756 757 failed_kiq_write: 758 pr_err("failed to write reg:%x\n", reg); 759 } 760