1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include "amdgpu.h" 25 26 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) 27 { 28 /* By now all MMIO pages except mailbox are blocked */ 29 /* if blocking is enabled in hypervisor. Choose the */ 30 /* SCRATCH_REG0 to test. */ 31 return RREG32_NO_KIQ(0xc040) == 0xffffffff; 32 } 33 34 void amdgpu_virt_init_setting(struct amdgpu_device *adev) 35 { 36 /* enable virtual display */ 37 adev->mode_info.num_crtc = 1; 38 adev->enable_virtual_display = true; 39 adev->ddev->driver->driver_features &= ~DRIVER_ATOMIC; 40 adev->cg_flags = 0; 41 adev->pg_flags = 0; 42 } 43 44 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) 45 { 46 signed long r, cnt = 0; 47 unsigned long flags; 48 uint32_t seq; 49 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 50 struct amdgpu_ring *ring = &kiq->ring; 51 52 BUG_ON(!ring->funcs->emit_rreg); 53 54 spin_lock_irqsave(&kiq->ring_lock, flags); 55 amdgpu_ring_alloc(ring, 32); 56 amdgpu_ring_emit_rreg(ring, reg); 57 amdgpu_fence_emit_polling(ring, &seq); 58 amdgpu_ring_commit(ring); 59 spin_unlock_irqrestore(&kiq->ring_lock, flags); 60 61 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 62 63 /* don't wait anymore for gpu reset case because this way may 64 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 65 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 66 * never return if we keep waiting in virt_kiq_rreg, which cause 67 * gpu_recover() hang there. 68 * 69 * also don't wait anymore for IRQ context 70 * */ 71 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 72 goto failed_kiq_read; 73 74 might_sleep(); 75 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 76 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 77 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 78 } 79 80 if (cnt > MAX_KIQ_REG_TRY) 81 goto failed_kiq_read; 82 83 return adev->wb.wb[adev->virt.reg_val_offs]; 84 85 failed_kiq_read: 86 pr_err("failed to read reg:%x\n", reg); 87 return ~0; 88 } 89 90 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 91 { 92 signed long r, cnt = 0; 93 unsigned long flags; 94 uint32_t seq; 95 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 96 struct amdgpu_ring *ring = &kiq->ring; 97 98 BUG_ON(!ring->funcs->emit_wreg); 99 100 spin_lock_irqsave(&kiq->ring_lock, flags); 101 amdgpu_ring_alloc(ring, 32); 102 amdgpu_ring_emit_wreg(ring, reg, v); 103 amdgpu_fence_emit_polling(ring, &seq); 104 amdgpu_ring_commit(ring); 105 spin_unlock_irqrestore(&kiq->ring_lock, flags); 106 107 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 108 109 /* don't wait anymore for gpu reset case because this way may 110 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 111 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 112 * never return if we keep waiting in virt_kiq_rreg, which cause 113 * gpu_recover() hang there. 114 * 115 * also don't wait anymore for IRQ context 116 * */ 117 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 118 goto failed_kiq_write; 119 120 might_sleep(); 121 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 122 123 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 124 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 125 } 126 127 if (cnt > MAX_KIQ_REG_TRY) 128 goto failed_kiq_write; 129 130 return; 131 132 failed_kiq_write: 133 pr_err("failed to write reg:%x\n", reg); 134 } 135 136 void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, 137 uint32_t reg0, uint32_t reg1, 138 uint32_t ref, uint32_t mask) 139 { 140 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 141 struct amdgpu_ring *ring = &kiq->ring; 142 signed long r, cnt = 0; 143 unsigned long flags; 144 uint32_t seq; 145 146 spin_lock_irqsave(&kiq->ring_lock, flags); 147 amdgpu_ring_alloc(ring, 32); 148 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 149 ref, mask); 150 amdgpu_fence_emit_polling(ring, &seq); 151 amdgpu_ring_commit(ring); 152 spin_unlock_irqrestore(&kiq->ring_lock, flags); 153 154 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 155 156 /* don't wait anymore for IRQ context */ 157 if (r < 1 && in_interrupt()) 158 goto failed_kiq; 159 160 might_sleep(); 161 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 162 163 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 164 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 165 } 166 167 if (cnt > MAX_KIQ_REG_TRY) 168 goto failed_kiq; 169 170 return; 171 172 failed_kiq: 173 pr_err("failed to write reg %x wait reg %x\n", reg0, reg1); 174 } 175 176 /** 177 * amdgpu_virt_request_full_gpu() - request full gpu access 178 * @amdgpu: amdgpu device. 179 * @init: is driver init time. 180 * When start to init/fini driver, first need to request full gpu access. 181 * Return: Zero if request success, otherwise will return error. 182 */ 183 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 184 { 185 struct amdgpu_virt *virt = &adev->virt; 186 int r; 187 188 if (virt->ops && virt->ops->req_full_gpu) { 189 r = virt->ops->req_full_gpu(adev, init); 190 if (r) 191 return r; 192 193 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 194 } 195 196 return 0; 197 } 198 199 /** 200 * amdgpu_virt_release_full_gpu() - release full gpu access 201 * @amdgpu: amdgpu device. 202 * @init: is driver init time. 203 * When finishing driver init/fini, need to release full gpu access. 204 * Return: Zero if release success, otherwise will returen error. 205 */ 206 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 207 { 208 struct amdgpu_virt *virt = &adev->virt; 209 int r; 210 211 if (virt->ops && virt->ops->rel_full_gpu) { 212 r = virt->ops->rel_full_gpu(adev, init); 213 if (r) 214 return r; 215 216 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 217 } 218 return 0; 219 } 220 221 /** 222 * amdgpu_virt_reset_gpu() - reset gpu 223 * @amdgpu: amdgpu device. 224 * Send reset command to GPU hypervisor to reset GPU that VM is using 225 * Return: Zero if reset success, otherwise will return error. 226 */ 227 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) 228 { 229 struct amdgpu_virt *virt = &adev->virt; 230 int r; 231 232 if (virt->ops && virt->ops->reset_gpu) { 233 r = virt->ops->reset_gpu(adev); 234 if (r) 235 return r; 236 237 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 238 } 239 240 return 0; 241 } 242 243 /** 244 * amdgpu_virt_wait_reset() - wait for reset gpu completed 245 * @amdgpu: amdgpu device. 246 * Wait for GPU reset completed. 247 * Return: Zero if reset success, otherwise will return error. 248 */ 249 int amdgpu_virt_wait_reset(struct amdgpu_device *adev) 250 { 251 struct amdgpu_virt *virt = &adev->virt; 252 253 if (!virt->ops || !virt->ops->wait_reset) 254 return -EINVAL; 255 256 return virt->ops->wait_reset(adev); 257 } 258 259 /** 260 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table 261 * @amdgpu: amdgpu device. 262 * MM table is used by UVD and VCE for its initialization 263 * Return: Zero if allocate success. 264 */ 265 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) 266 { 267 int r; 268 269 if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) 270 return 0; 271 272 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 273 AMDGPU_GEM_DOMAIN_VRAM, 274 &adev->virt.mm_table.bo, 275 &adev->virt.mm_table.gpu_addr, 276 (void *)&adev->virt.mm_table.cpu_addr); 277 if (r) { 278 DRM_ERROR("failed to alloc mm table and error = %d.\n", r); 279 return r; 280 } 281 282 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); 283 DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n", 284 adev->virt.mm_table.gpu_addr, 285 adev->virt.mm_table.cpu_addr); 286 return 0; 287 } 288 289 /** 290 * amdgpu_virt_free_mm_table() - free mm table memory 291 * @amdgpu: amdgpu device. 292 * Free MM table memory 293 */ 294 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) 295 { 296 if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) 297 return; 298 299 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo, 300 &adev->virt.mm_table.gpu_addr, 301 (void *)&adev->virt.mm_table.cpu_addr); 302 adev->virt.mm_table.gpu_addr = 0; 303 } 304 305 306 int amdgpu_virt_fw_reserve_get_checksum(void *obj, 307 unsigned long obj_size, 308 unsigned int key, 309 unsigned int chksum) 310 { 311 unsigned int ret = key; 312 unsigned long i = 0; 313 unsigned char *pos; 314 315 pos = (char *)obj; 316 /* calculate checksum */ 317 for (i = 0; i < obj_size; ++i) 318 ret += *(pos + i); 319 /* minus the chksum itself */ 320 pos = (char *)&chksum; 321 for (i = 0; i < sizeof(chksum); ++i) 322 ret -= *(pos + i); 323 return ret; 324 } 325 326 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) 327 { 328 uint32_t pf2vf_size = 0; 329 uint32_t checksum = 0; 330 uint32_t checkval; 331 char *str; 332 333 adev->virt.fw_reserve.p_pf2vf = NULL; 334 adev->virt.fw_reserve.p_vf2pf = NULL; 335 336 if (adev->fw_vram_usage.va != NULL) { 337 adev->virt.fw_reserve.p_pf2vf = 338 (struct amd_sriov_msg_pf2vf_info_header *)( 339 adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET); 340 AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size); 341 AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum); 342 AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature); 343 344 /* pf2vf message must be in 4K */ 345 if (pf2vf_size > 0 && pf2vf_size < 4096) { 346 checkval = amdgpu_virt_fw_reserve_get_checksum( 347 adev->virt.fw_reserve.p_pf2vf, pf2vf_size, 348 adev->virt.fw_reserve.checksum_key, checksum); 349 if (checkval == checksum) { 350 adev->virt.fw_reserve.p_vf2pf = 351 ((void *)adev->virt.fw_reserve.p_pf2vf + 352 pf2vf_size); 353 memset((void *)adev->virt.fw_reserve.p_vf2pf, 0, 354 sizeof(amdgim_vf2pf_info)); 355 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.version, 356 AMDGPU_FW_VRAM_VF2PF_VER); 357 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.size, 358 sizeof(amdgim_vf2pf_info)); 359 AMDGPU_FW_VRAM_VF2PF_READ(adev, driver_version, 360 &str); 361 #ifdef MODULE 362 if (THIS_MODULE->version != NULL) 363 strcpy(str, THIS_MODULE->version); 364 else 365 #endif 366 strcpy(str, "N/A"); 367 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, driver_cert, 368 0); 369 AMDGPU_FW_VRAM_VF2PF_WRITE(adev, checksum, 370 amdgpu_virt_fw_reserve_get_checksum( 371 adev->virt.fw_reserve.p_vf2pf, 372 pf2vf_size, 373 adev->virt.fw_reserve.checksum_key, 0)); 374 } 375 } 376 } 377 } 378 379 static uint32_t parse_clk(char *buf, bool min) 380 { 381 char *ptr = buf; 382 uint32_t clk = 0; 383 384 do { 385 ptr = strchr(ptr, ':'); 386 if (!ptr) 387 break; 388 ptr+=2; 389 clk = simple_strtoul(ptr, NULL, 10); 390 } while (!min); 391 392 return clk * 100; 393 } 394 395 uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest) 396 { 397 char *buf = NULL; 398 uint32_t clk = 0; 399 400 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 401 if (!buf) 402 return -ENOMEM; 403 404 adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf); 405 clk = parse_clk(buf, lowest); 406 407 kfree(buf); 408 409 return clk; 410 } 411 412 uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest) 413 { 414 char *buf = NULL; 415 uint32_t clk = 0; 416 417 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 418 if (!buf) 419 return -ENOMEM; 420 421 adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf); 422 clk = parse_clk(buf, lowest); 423 424 kfree(buf); 425 426 return clk; 427 } 428 429