1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/module.h> 25 26 #include <drm/drm_drv.h> 27 28 #include "amdgpu.h" 29 30 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) 31 { 32 /* By now all MMIO pages except mailbox are blocked */ 33 /* if blocking is enabled in hypervisor. Choose the */ 34 /* SCRATCH_REG0 to test. */ 35 return RREG32_NO_KIQ(0xc040) == 0xffffffff; 36 } 37 38 void amdgpu_virt_init_setting(struct amdgpu_device *adev) 39 { 40 /* enable virtual display */ 41 adev->mode_info.num_crtc = 1; 42 adev->enable_virtual_display = true; 43 adev->ddev->driver->driver_features &= ~DRIVER_ATOMIC; 44 adev->cg_flags = 0; 45 adev->pg_flags = 0; 46 } 47 48 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) 49 { 50 signed long r, cnt = 0; 51 unsigned long flags; 52 uint32_t seq; 53 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 54 struct amdgpu_ring *ring = &kiq->ring; 55 56 BUG_ON(!ring->funcs->emit_rreg); 57 58 spin_lock_irqsave(&kiq->ring_lock, flags); 59 amdgpu_ring_alloc(ring, 32); 60 amdgpu_ring_emit_rreg(ring, reg); 61 amdgpu_fence_emit_polling(ring, &seq); 62 amdgpu_ring_commit(ring); 63 spin_unlock_irqrestore(&kiq->ring_lock, flags); 64 65 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 66 67 /* don't wait anymore for gpu reset case because this way may 68 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 69 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 70 * never return if we keep waiting in virt_kiq_rreg, which cause 71 * gpu_recover() hang there. 72 * 73 * also don't wait anymore for IRQ context 74 * */ 75 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 76 goto failed_kiq_read; 77 78 might_sleep(); 79 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 80 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 81 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 82 } 83 84 if (cnt > MAX_KIQ_REG_TRY) 85 goto failed_kiq_read; 86 87 return adev->wb.wb[adev->virt.reg_val_offs]; 88 89 failed_kiq_read: 90 pr_err("failed to read reg:%x\n", reg); 91 return ~0; 92 } 93 94 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 95 { 96 signed long r, cnt = 0; 97 unsigned long flags; 98 uint32_t seq; 99 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 100 struct amdgpu_ring *ring = &kiq->ring; 101 102 BUG_ON(!ring->funcs->emit_wreg); 103 104 spin_lock_irqsave(&kiq->ring_lock, flags); 105 amdgpu_ring_alloc(ring, 32); 106 amdgpu_ring_emit_wreg(ring, reg, v); 107 amdgpu_fence_emit_polling(ring, &seq); 108 amdgpu_ring_commit(ring); 109 spin_unlock_irqrestore(&kiq->ring_lock, flags); 110 111 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 112 113 /* don't wait anymore for gpu reset case because this way may 114 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 115 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 116 * never return if we keep waiting in virt_kiq_rreg, which cause 117 * gpu_recover() hang there. 118 * 119 * also don't wait anymore for IRQ context 120 * */ 121 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 122 goto failed_kiq_write; 123 124 might_sleep(); 125 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 126 127 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 128 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 129 } 130 131 if (cnt > MAX_KIQ_REG_TRY) 132 goto failed_kiq_write; 133 134 return; 135 136 failed_kiq_write: 137 pr_err("failed to write reg:%x\n", reg); 138 } 139 140 void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, 141 uint32_t reg0, uint32_t reg1, 142 uint32_t ref, uint32_t mask) 143 { 144 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 145 struct amdgpu_ring *ring = &kiq->ring; 146 signed long r, cnt = 0; 147 unsigned long flags; 148 uint32_t seq; 149 150 spin_lock_irqsave(&kiq->ring_lock, flags); 151 amdgpu_ring_alloc(ring, 32); 152 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 153 ref, mask); 154 amdgpu_fence_emit_polling(ring, &seq); 155 amdgpu_ring_commit(ring); 156 spin_unlock_irqrestore(&kiq->ring_lock, flags); 157 158 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 159 160 /* don't wait anymore for IRQ context */ 161 if (r < 1 && in_interrupt()) 162 goto failed_kiq; 163 164 might_sleep(); 165 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 166 167 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 168 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 169 } 170 171 if (cnt > MAX_KIQ_REG_TRY) 172 goto failed_kiq; 173 174 return; 175 176 failed_kiq: 177 pr_err("failed to write reg %x wait reg %x\n", reg0, reg1); 178 } 179 180 /** 181 * amdgpu_virt_request_full_gpu() - request full gpu access 182 * @amdgpu: amdgpu device. 183 * @init: is driver init time. 184 * When start to init/fini driver, first need to request full gpu access. 185 * Return: Zero if request success, otherwise will return error. 186 */ 187 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 188 { 189 struct amdgpu_virt *virt = &adev->virt; 190 int r; 191 192 if (virt->ops && virt->ops->req_full_gpu) { 193 r = virt->ops->req_full_gpu(adev, init); 194 if (r) 195 return r; 196 197 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 198 } 199 200 return 0; 201 } 202 203 /** 204 * amdgpu_virt_release_full_gpu() - release full gpu access 205 * @amdgpu: amdgpu device. 206 * @init: is driver init time. 207 * When finishing driver init/fini, need to release full gpu access. 208 * Return: Zero if release success, otherwise will returen error. 209 */ 210 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 211 { 212 struct amdgpu_virt *virt = &adev->virt; 213 int r; 214 215 if (virt->ops && virt->ops->rel_full_gpu) { 216 r = virt->ops->rel_full_gpu(adev, init); 217 if (r) 218 return r; 219 220 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 221 } 222 return 0; 223 } 224 225 /** 226 * amdgpu_virt_reset_gpu() - reset gpu 227 * @amdgpu: amdgpu device. 228 * Send reset command to GPU hypervisor to reset GPU that VM is using 229 * Return: Zero if reset success, otherwise will return error. 230 */ 231 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) 232 { 233 struct amdgpu_virt *virt = &adev->virt; 234 int r; 235 236 if (virt->ops && virt->ops->reset_gpu) { 237 r = virt->ops->reset_gpu(adev); 238 if (r) 239 return r; 240 241 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 242 } 243 244 return 0; 245 } 246 247 /** 248 * amdgpu_virt_wait_reset() - wait for reset gpu completed 249 * @amdgpu: amdgpu device. 250 * Wait for GPU reset completed. 251 * Return: Zero if reset success, otherwise will return error. 252 */ 253 int amdgpu_virt_wait_reset(struct amdgpu_device *adev) 254 { 255 struct amdgpu_virt *virt = &adev->virt; 256 257 if (!virt->ops || !virt->ops->wait_reset) 258 return -EINVAL; 259 260 return virt->ops->wait_reset(adev); 261 } 262 263 /** 264 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table 265 * @amdgpu: amdgpu device. 266 * MM table is used by UVD and VCE for its initialization 267 * Return: Zero if allocate success. 268 */ 269 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) 270 { 271 int r; 272 273 if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) 274 return 0; 275 276 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 277 AMDGPU_GEM_DOMAIN_VRAM, 278 &adev->virt.mm_table.bo, 279 &adev->virt.mm_table.gpu_addr, 280 (void *)&adev->virt.mm_table.cpu_addr); 281 if (r) { 282 DRM_ERROR("failed to alloc mm table and error = %d.\n", r); 283 return r; 284 } 285 286 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); 287 DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n", 288 adev->virt.mm_table.gpu_addr, 289 adev->virt.mm_table.cpu_addr); 290 return 0; 291 } 292 293 /** 294 * amdgpu_virt_free_mm_table() - free mm table memory 295 * @amdgpu: amdgpu device. 296 * Free MM table memory 297 */ 298 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) 299 { 300 if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) 301 return; 302 303 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo, 304 &adev->virt.mm_table.gpu_addr, 305 (void *)&adev->virt.mm_table.cpu_addr); 306 adev->virt.mm_table.gpu_addr = 0; 307 } 308 309 310 int amdgpu_virt_fw_reserve_get_checksum(void *obj, 311 unsigned long obj_size, 312 unsigned int key, 313 unsigned int chksum) 314 { 315 unsigned int ret = key; 316 unsigned long i = 0; 317 unsigned char *pos; 318 319 pos = (char *)obj; 320 /* calculate checksum */ 321 for (i = 0; i < obj_size; ++i) 322 ret += *(pos + i); 323 /* minus the chksum itself */ 324 pos = (char *)&chksum; 325 for (i = 0; i < sizeof(chksum); ++i) 326 ret -= *(pos + i); 327 return ret; 328 } 329 330 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) 331 { 332 uint32_t pf2vf_size = 0; 333 uint32_t checksum = 0; 334 uint32_t checkval; 335 char *str; 336 337 adev->virt.fw_reserve.p_pf2vf = NULL; 338 adev->virt.fw_reserve.p_vf2pf = NULL; 339 340 if (adev->fw_vram_usage.va != NULL) { 341 adev->virt.fw_reserve.p_pf2vf = 342 (struct amd_sriov_msg_pf2vf_info_header *)( 343 adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET); 344 AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size); 345 AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum); 346 AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature); 347 348 /* pf2vf message must be in 4K */ 349 if (pf2vf_size > 0 && pf2vf_size < 4096) { 350 checkval = amdgpu_virt_fw_reserve_get_checksum( 351 adev->virt.fw_reserve.p_pf2vf, pf2vf_size, 352 adev->virt.fw_reserve.checksum_key, checksum); 353 if (checkval == checksum) { 354 adev->virt.fw_reserve.p_vf2pf = 355 ((void *)adev->virt.fw_reserve.p_pf2vf + 356 pf2vf_size); 357 memset((void *)adev->virt.fw_reserve.p_vf2pf, 0, 358 sizeof(amdgim_vf2pf_info)); 359 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.version, 360 AMDGPU_FW_VRAM_VF2PF_VER); 361 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.size, 362 sizeof(amdgim_vf2pf_info)); 363 AMDGPU_FW_VRAM_VF2PF_READ(adev, driver_version, 364 &str); 365 #ifdef MODULE 366 if (THIS_MODULE->version != NULL) 367 strcpy(str, THIS_MODULE->version); 368 else 369 #endif 370 strcpy(str, "N/A"); 371 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, driver_cert, 372 0); 373 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, checksum, 374 amdgpu_virt_fw_reserve_get_checksum( 375 adev->virt.fw_reserve.p_vf2pf, 376 pf2vf_size, 377 adev->virt.fw_reserve.checksum_key, 0)); 378 } 379 } 380 } 381 } 382 383 static uint32_t parse_clk(char *buf, bool min) 384 { 385 char *ptr = buf; 386 uint32_t clk = 0; 387 388 do { 389 ptr = strchr(ptr, ':'); 390 if (!ptr) 391 break; 392 ptr+=2; 393 if (kstrtou32(ptr, 10, &clk)) 394 return 0; 395 } while (!min); 396 397 return clk * 100; 398 } 399 400 uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest) 401 { 402 char *buf = NULL; 403 uint32_t clk = 0; 404 405 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 406 if (!buf) 407 return -ENOMEM; 408 409 adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf); 410 clk = parse_clk(buf, lowest); 411 412 kfree(buf); 413 414 return clk; 415 } 416 417 uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest) 418 { 419 char *buf = NULL; 420 uint32_t clk = 0; 421 422 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 423 if (!buf) 424 return -ENOMEM; 425 426 adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf); 427 clk = parse_clk(buf, lowest); 428 429 kfree(buf); 430 431 return clk; 432 } 433