1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016-2019 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/ktime.h> 8 #include <linux/time64.h> 9 #include <linux/string_helpers.h> 10 #include <linux/timekeeping.h> 11 12 #include "i915_drv.h" 13 #include "intel_guc_ct.h" 14 #include "gt/intel_gt.h" 15 16 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) 17 { 18 return container_of(ct, struct intel_guc, ct); 19 } 20 21 static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct) 22 { 23 return guc_to_gt(ct_to_guc(ct)); 24 } 25 26 static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct) 27 { 28 return ct_to_gt(ct)->i915; 29 } 30 31 static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct) 32 { 33 return &ct_to_i915(ct)->drm; 34 } 35 36 #define CT_ERROR(_ct, _fmt, ...) \ 37 drm_err(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) 38 #ifdef CONFIG_DRM_I915_DEBUG_GUC 39 #define CT_DEBUG(_ct, _fmt, ...) \ 40 drm_dbg(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) 41 #else 42 #define CT_DEBUG(...) do { } while (0) 43 #endif 44 #define CT_PROBE_ERROR(_ct, _fmt, ...) \ 45 i915_probe_error(ct_to_i915(ct), "CT: " _fmt, ##__VA_ARGS__) 46 47 /** 48 * DOC: CTB Blob 49 * 50 * We allocate single blob to hold both CTB descriptors and buffers: 51 * 52 * +--------+-----------------------------------------------+------+ 53 * | offset | contents | size | 54 * +========+===============================================+======+ 55 * | 0x0000 | H2G `CTB Descriptor`_ (send) | | 56 * +--------+-----------------------------------------------+ 4K | 57 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | | 58 * +--------+-----------------------------------------------+------+ 59 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K | 60 * | | | | 61 * +--------+-----------------------------------------------+------+ 62 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K | 63 * | + n*4K | | | 64 * +--------+-----------------------------------------------+------+ 65 * 66 * Size of each `CT Buffer`_ must be multiple of 4K. 67 * We don't expect too many messages in flight at any time, unless we are 68 * using the GuC submission. In that case each request requires a minimum 69 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 70 * enough space to avoid backpressure on the driver. We increase the size 71 * of the receive buffer (relative to the send) to ensure a G2H response 72 * CTB has a landing spot. 73 */ 74 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 75 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 76 #define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) 77 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) 78 79 struct ct_request { 80 struct list_head link; 81 u32 fence; 82 u32 status; 83 u32 response_len; 84 u32 *response_buf; 85 }; 86 87 struct ct_incoming_msg { 88 struct list_head link; 89 u32 size; 90 u32 msg[]; 91 }; 92 93 enum { CTB_SEND = 0, CTB_RECV = 1 }; 94 95 enum { CTB_OWNER_HOST = 0 }; 96 97 static void ct_receive_tasklet_func(struct tasklet_struct *t); 98 static void ct_incoming_request_worker_func(struct work_struct *w); 99 100 /** 101 * intel_guc_ct_init_early - Initialize CT state without requiring device access 102 * @ct: pointer to CT struct 103 */ 104 void intel_guc_ct_init_early(struct intel_guc_ct *ct) 105 { 106 spin_lock_init(&ct->ctbs.send.lock); 107 spin_lock_init(&ct->ctbs.recv.lock); 108 spin_lock_init(&ct->requests.lock); 109 INIT_LIST_HEAD(&ct->requests.pending); 110 INIT_LIST_HEAD(&ct->requests.incoming); 111 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); 112 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); 113 init_waitqueue_head(&ct->wq); 114 } 115 116 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) 117 { 118 memset(desc, 0, sizeof(*desc)); 119 } 120 121 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) 122 { 123 u32 space; 124 125 ctb->broken = false; 126 ctb->tail = 0; 127 ctb->head = 0; 128 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; 129 atomic_set(&ctb->space, space); 130 131 guc_ct_buffer_desc_init(ctb->desc); 132 } 133 134 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, 135 struct guc_ct_buffer_desc *desc, 136 u32 *cmds, u32 size_in_bytes, u32 resv_space) 137 { 138 GEM_BUG_ON(size_in_bytes % 4); 139 140 ctb->desc = desc; 141 ctb->cmds = cmds; 142 ctb->size = size_in_bytes / 4; 143 ctb->resv_space = resv_space / 4; 144 145 guc_ct_buffer_reset(ctb); 146 } 147 148 static int guc_action_control_ctb(struct intel_guc *guc, u32 control) 149 { 150 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 151 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 152 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 153 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB), 154 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control), 155 }; 156 int ret; 157 158 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE); 159 160 /* CT control must go over MMIO */ 161 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); 162 163 return ret > 0 ? -EPROTO : ret; 164 } 165 166 static int ct_control_enable(struct intel_guc_ct *ct, bool enable) 167 { 168 int err; 169 170 err = guc_action_control_ctb(ct_to_guc(ct), enable ? 171 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE); 172 if (unlikely(err)) 173 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n", 174 str_enable_disable(enable), ERR_PTR(err)); 175 176 return err; 177 } 178 179 static int ct_register_buffer(struct intel_guc_ct *ct, bool send, 180 u32 desc_addr, u32 buff_addr, u32 size) 181 { 182 int err; 183 184 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 185 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY : 186 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 187 desc_addr); 188 if (unlikely(err)) 189 goto failed; 190 191 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 192 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY : 193 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 194 buff_addr); 195 if (unlikely(err)) 196 goto failed; 197 198 err = intel_guc_self_cfg32(ct_to_guc(ct), send ? 199 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY : 200 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 201 size); 202 if (unlikely(err)) 203 failed: 204 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n", 205 send ? "SEND" : "RECV", ERR_PTR(err)); 206 207 return err; 208 } 209 210 /** 211 * intel_guc_ct_init - Init buffer-based communication 212 * @ct: pointer to CT struct 213 * 214 * Allocate memory required for buffer-based communication. 215 * 216 * Return: 0 on success, a negative errno code on failure. 217 */ 218 int intel_guc_ct_init(struct intel_guc_ct *ct) 219 { 220 struct intel_guc *guc = ct_to_guc(ct); 221 struct guc_ct_buffer_desc *desc; 222 u32 blob_size; 223 u32 cmds_size; 224 u32 resv_space; 225 void *blob; 226 u32 *cmds; 227 int err; 228 229 err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO); 230 if (err) 231 return err; 232 233 GEM_BUG_ON(ct->vma); 234 235 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; 236 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob); 237 if (unlikely(err)) { 238 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n", 239 blob_size, ERR_PTR(err)); 240 return err; 241 } 242 243 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size); 244 245 /* store pointers to desc and cmds for send ctb */ 246 desc = blob; 247 cmds = blob + 2 * CTB_DESC_SIZE; 248 cmds_size = CTB_H2G_BUFFER_SIZE; 249 resv_space = 0; 250 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", 251 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 252 resv_space); 253 254 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); 255 256 /* store pointers to desc and cmds for recv ctb */ 257 desc = blob + CTB_DESC_SIZE; 258 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; 259 cmds_size = CTB_G2H_BUFFER_SIZE; 260 resv_space = G2H_ROOM_BUFFER_SIZE; 261 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", 262 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 263 resv_space); 264 265 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); 266 267 return 0; 268 } 269 270 /** 271 * intel_guc_ct_fini - Fini buffer-based communication 272 * @ct: pointer to CT struct 273 * 274 * Deallocate memory required for buffer-based communication. 275 */ 276 void intel_guc_ct_fini(struct intel_guc_ct *ct) 277 { 278 GEM_BUG_ON(ct->enabled); 279 280 tasklet_kill(&ct->receive_tasklet); 281 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); 282 memset(ct, 0, sizeof(*ct)); 283 } 284 285 /** 286 * intel_guc_ct_enable - Enable buffer based command transport. 287 * @ct: pointer to CT struct 288 * 289 * Return: 0 on success, a negative errno code on failure. 290 */ 291 int intel_guc_ct_enable(struct intel_guc_ct *ct) 292 { 293 struct intel_guc *guc = ct_to_guc(ct); 294 u32 base, desc, cmds, size; 295 void *blob; 296 int err; 297 298 GEM_BUG_ON(ct->enabled); 299 300 /* vma should be already allocated and map'ed */ 301 GEM_BUG_ON(!ct->vma); 302 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj)); 303 base = intel_guc_ggtt_offset(guc, ct->vma); 304 305 /* blob should start with send descriptor */ 306 blob = __px_vaddr(ct->vma->obj); 307 GEM_BUG_ON(blob != ct->ctbs.send.desc); 308 309 /* (re)initialize descriptors */ 310 guc_ct_buffer_reset(&ct->ctbs.send); 311 guc_ct_buffer_reset(&ct->ctbs.recv); 312 313 /* 314 * Register both CT buffers starting with RECV buffer. 315 * Descriptors are in first half of the blob. 316 */ 317 desc = base + ptrdiff(ct->ctbs.recv.desc, blob); 318 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); 319 size = ct->ctbs.recv.size * 4; 320 err = ct_register_buffer(ct, false, desc, cmds, size); 321 if (unlikely(err)) 322 goto err_out; 323 324 desc = base + ptrdiff(ct->ctbs.send.desc, blob); 325 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); 326 size = ct->ctbs.send.size * 4; 327 err = ct_register_buffer(ct, true, desc, cmds, size); 328 if (unlikely(err)) 329 goto err_out; 330 331 err = ct_control_enable(ct, true); 332 if (unlikely(err)) 333 goto err_out; 334 335 ct->enabled = true; 336 ct->stall_time = KTIME_MAX; 337 338 return 0; 339 340 err_out: 341 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); 342 return err; 343 } 344 345 /** 346 * intel_guc_ct_disable - Disable buffer based command transport. 347 * @ct: pointer to CT struct 348 */ 349 void intel_guc_ct_disable(struct intel_guc_ct *ct) 350 { 351 struct intel_guc *guc = ct_to_guc(ct); 352 353 GEM_BUG_ON(!ct->enabled); 354 355 ct->enabled = false; 356 357 if (intel_guc_is_fw_running(guc)) { 358 ct_control_enable(ct, false); 359 } 360 } 361 362 static u32 ct_get_next_fence(struct intel_guc_ct *ct) 363 { 364 /* For now it's trivial */ 365 return ++ct->requests.last_fence; 366 } 367 368 static int ct_write(struct intel_guc_ct *ct, 369 const u32 *action, 370 u32 len /* in dwords */, 371 u32 fence, u32 flags) 372 { 373 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 374 struct guc_ct_buffer_desc *desc = ctb->desc; 375 u32 tail = ctb->tail; 376 u32 size = ctb->size; 377 u32 header; 378 u32 hxg; 379 u32 type; 380 u32 *cmds = ctb->cmds; 381 unsigned int i; 382 383 if (unlikely(desc->status)) 384 goto corrupted; 385 386 GEM_BUG_ON(tail > size); 387 388 #ifdef CONFIG_DRM_I915_DEBUG_GUC 389 if (unlikely(tail != READ_ONCE(desc->tail))) { 390 CT_ERROR(ct, "Tail was modified %u != %u\n", 391 desc->tail, tail); 392 desc->status |= GUC_CTB_STATUS_MISMATCH; 393 goto corrupted; 394 } 395 if (unlikely(READ_ONCE(desc->head) >= size)) { 396 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 397 desc->head, size); 398 desc->status |= GUC_CTB_STATUS_OVERFLOW; 399 goto corrupted; 400 } 401 #endif 402 403 /* 404 * dw0: CT header (including fence) 405 * dw1: HXG header (including action code) 406 * dw2+: action data 407 */ 408 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 409 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 410 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); 411 412 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT : 413 GUC_HXG_TYPE_REQUEST; 414 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | 415 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 416 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 417 418 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", 419 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); 420 421 cmds[tail] = header; 422 tail = (tail + 1) % size; 423 424 cmds[tail] = hxg; 425 tail = (tail + 1) % size; 426 427 for (i = 1; i < len; i++) { 428 cmds[tail] = action[i]; 429 tail = (tail + 1) % size; 430 } 431 GEM_BUG_ON(tail > size); 432 433 /* 434 * make sure H2G buffer update and LRC tail update (if this triggering a 435 * submission) are visible before updating the descriptor tail 436 */ 437 intel_guc_write_barrier(ct_to_guc(ct)); 438 439 /* update local copies */ 440 ctb->tail = tail; 441 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); 442 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); 443 444 /* now update descriptor */ 445 WRITE_ONCE(desc->tail, tail); 446 447 return 0; 448 449 corrupted: 450 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 451 desc->head, desc->tail, desc->status); 452 ctb->broken = true; 453 return -EPIPE; 454 } 455 456 /** 457 * wait_for_ct_request_update - Wait for CT request state update. 458 * @ct: pointer to CT 459 * @req: pointer to pending request 460 * @status: placeholder for status 461 * 462 * For each sent request, GuC shall send back CT response message. 463 * Our message handler will update status of tracked request once 464 * response message with given fence is received. Wait here and 465 * check for valid response status value. 466 * 467 * Return: 468 * * 0 response received (status is valid) 469 * * -ETIMEDOUT no response within hardcoded timeout 470 */ 471 static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status) 472 { 473 int err; 474 bool ct_enabled; 475 476 /* 477 * Fast commands should complete in less than 10us, so sample quickly 478 * up to that length of time, then switch to a slower sleep-wait loop. 479 * No GuC command should ever take longer than 10ms but many GuC 480 * commands can be inflight at time, so use a 1s timeout on the slower 481 * sleep-wait loop. 482 */ 483 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 484 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 485 #define done \ 486 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \ 487 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ 488 GUC_HXG_ORIGIN_GUC) 489 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); 490 if (err) 491 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); 492 #undef done 493 if (!ct_enabled) 494 err = -ENODEV; 495 496 *status = req->status; 497 return err; 498 } 499 500 #define GUC_CTB_TIMEOUT_MS 1500 501 static inline bool ct_deadlocked(struct intel_guc_ct *ct) 502 { 503 long timeout = GUC_CTB_TIMEOUT_MS; 504 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; 505 506 if (unlikely(ret)) { 507 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; 508 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; 509 510 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", 511 ktime_ms_delta(ktime_get(), ct->stall_time), 512 send->status, recv->status); 513 CT_ERROR(ct, "H2G Space: %u (Bytes)\n", 514 atomic_read(&ct->ctbs.send.space) * 4); 515 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head); 516 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail); 517 CT_ERROR(ct, "G2H Space: %u (Bytes)\n", 518 atomic_read(&ct->ctbs.recv.space) * 4); 519 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); 520 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); 521 522 ct->ctbs.send.broken = true; 523 } 524 525 return ret; 526 } 527 528 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) 529 { 530 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 531 532 /* 533 * We leave a certain amount of space in the G2H CTB buffer for 534 * unexpected G2H CTBs (e.g. logging, engine hang, etc...) 535 */ 536 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; 537 } 538 539 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 540 { 541 lockdep_assert_held(&ct->ctbs.send.lock); 542 543 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); 544 545 if (g2h_len_dw) 546 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); 547 } 548 549 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 550 { 551 atomic_add(g2h_len_dw, &ct->ctbs.recv.space); 552 } 553 554 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) 555 { 556 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 557 struct guc_ct_buffer_desc *desc = ctb->desc; 558 u32 head; 559 u32 space; 560 561 if (atomic_read(&ctb->space) >= len_dw) 562 return true; 563 564 head = READ_ONCE(desc->head); 565 if (unlikely(head > ctb->size)) { 566 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 567 head, ctb->size); 568 desc->status |= GUC_CTB_STATUS_OVERFLOW; 569 ctb->broken = true; 570 return false; 571 } 572 573 space = CIRC_SPACE(ctb->tail, head, ctb->size); 574 atomic_set(&ctb->space, space); 575 576 return space >= len_dw; 577 } 578 579 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) 580 { 581 bool h2g = h2g_has_room(ct, h2g_dw); 582 bool g2h = g2h_has_room(ct, g2h_dw); 583 584 lockdep_assert_held(&ct->ctbs.send.lock); 585 586 if (unlikely(!h2g || !g2h)) { 587 if (ct->stall_time == KTIME_MAX) 588 ct->stall_time = ktime_get(); 589 590 /* Be paranoid and kick G2H tasklet to free credits */ 591 if (!g2h) 592 tasklet_hi_schedule(&ct->receive_tasklet); 593 594 if (unlikely(ct_deadlocked(ct))) 595 return -EPIPE; 596 else 597 return -EBUSY; 598 } 599 600 ct->stall_time = KTIME_MAX; 601 return 0; 602 } 603 604 #define G2H_LEN_DW(f) ({ \ 605 typeof(f) f_ = (f); \ 606 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ 607 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ 608 GUC_CTB_HXG_MSG_MIN_LEN : 0; \ 609 }) 610 static int ct_send_nb(struct intel_guc_ct *ct, 611 const u32 *action, 612 u32 len, 613 u32 flags) 614 { 615 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 616 unsigned long spin_flags; 617 u32 g2h_len_dw = G2H_LEN_DW(flags); 618 u32 fence; 619 int ret; 620 621 spin_lock_irqsave(&ctb->lock, spin_flags); 622 623 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); 624 if (unlikely(ret)) 625 goto out; 626 627 fence = ct_get_next_fence(ct); 628 ret = ct_write(ct, action, len, fence, flags); 629 if (unlikely(ret)) 630 goto out; 631 632 g2h_reserve_space(ct, g2h_len_dw); 633 intel_guc_notify(ct_to_guc(ct)); 634 635 out: 636 spin_unlock_irqrestore(&ctb->lock, spin_flags); 637 638 return ret; 639 } 640 641 static int ct_send(struct intel_guc_ct *ct, 642 const u32 *action, 643 u32 len, 644 u32 *response_buf, 645 u32 response_buf_size, 646 u32 *status) 647 { 648 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 649 struct ct_request request; 650 unsigned long flags; 651 unsigned int sleep_period_ms = 1; 652 bool send_again; 653 u32 fence; 654 int err; 655 656 GEM_BUG_ON(!ct->enabled); 657 GEM_BUG_ON(!len); 658 GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK); 659 GEM_BUG_ON(!response_buf && response_buf_size); 660 might_sleep(); 661 662 resend: 663 send_again = false; 664 665 /* 666 * We use a lazy spin wait loop here as we believe that if the CT 667 * buffers are sized correctly the flow control condition should be 668 * rare. Reserving the maximum size in the G2H credits as we don't know 669 * how big the response is going to be. 670 */ 671 retry: 672 spin_lock_irqsave(&ctb->lock, flags); 673 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || 674 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { 675 if (ct->stall_time == KTIME_MAX) 676 ct->stall_time = ktime_get(); 677 spin_unlock_irqrestore(&ctb->lock, flags); 678 679 if (unlikely(ct_deadlocked(ct))) 680 return -EPIPE; 681 682 if (msleep_interruptible(sleep_period_ms)) 683 return -EINTR; 684 sleep_period_ms = sleep_period_ms << 1; 685 686 goto retry; 687 } 688 689 ct->stall_time = KTIME_MAX; 690 691 fence = ct_get_next_fence(ct); 692 request.fence = fence; 693 request.status = 0; 694 request.response_len = response_buf_size; 695 request.response_buf = response_buf; 696 697 spin_lock(&ct->requests.lock); 698 list_add_tail(&request.link, &ct->requests.pending); 699 spin_unlock(&ct->requests.lock); 700 701 err = ct_write(ct, action, len, fence, 0); 702 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 703 704 spin_unlock_irqrestore(&ctb->lock, flags); 705 706 if (unlikely(err)) 707 goto unlink; 708 709 intel_guc_notify(ct_to_guc(ct)); 710 711 err = wait_for_ct_request_update(ct, &request, status); 712 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 713 if (unlikely(err)) { 714 if (err == -ENODEV) 715 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress. 716 * In this case, output is debug rather than error info 717 */ 718 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n", 719 action[0], request.fence); 720 else 721 CT_ERROR(ct, "No response for request %#x (fence %u)\n", 722 action[0], request.fence); 723 goto unlink; 724 } 725 726 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 727 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action, 728 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status)); 729 send_again = true; 730 goto unlink; 731 } 732 733 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { 734 err = -EIO; 735 goto unlink; 736 } 737 738 if (response_buf) { 739 /* There shall be no data in the status */ 740 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); 741 /* Return actual response len */ 742 err = request.response_len; 743 } else { 744 /* There shall be no response payload */ 745 WARN_ON(request.response_len); 746 /* Return data decoded from the status dword */ 747 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); 748 } 749 750 unlink: 751 spin_lock_irqsave(&ct->requests.lock, flags); 752 list_del(&request.link); 753 spin_unlock_irqrestore(&ct->requests.lock, flags); 754 755 if (unlikely(send_again)) 756 goto resend; 757 758 return err; 759 } 760 761 /* 762 * Command Transport (CT) buffer based GuC send function. 763 */ 764 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, 765 u32 *response_buf, u32 response_buf_size, u32 flags) 766 { 767 u32 status = ~0; /* undefined */ 768 int ret; 769 770 if (unlikely(!ct->enabled)) { 771 struct intel_guc *guc = ct_to_guc(ct); 772 struct intel_uc *uc = container_of(guc, struct intel_uc, guc); 773 774 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); 775 return -ENODEV; 776 } 777 778 if (unlikely(ct->ctbs.send.broken)) 779 return -EPIPE; 780 781 if (flags & INTEL_GUC_CT_SEND_NB) 782 return ct_send_nb(ct, action, len, flags); 783 784 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); 785 if (unlikely(ret < 0)) { 786 if (ret != -ENODEV) 787 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", 788 action[0], ERR_PTR(ret), status); 789 } else if (unlikely(ret)) { 790 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", 791 action[0], ret, ret); 792 } 793 794 return ret; 795 } 796 797 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) 798 { 799 struct ct_incoming_msg *msg; 800 801 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC); 802 if (msg) 803 msg->size = num_dwords; 804 return msg; 805 } 806 807 static void ct_free_msg(struct ct_incoming_msg *msg) 808 { 809 kfree(msg); 810 } 811 812 /* 813 * Return: number available remaining dwords to read (0 if empty) 814 * or a negative error code on failure 815 */ 816 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) 817 { 818 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 819 struct guc_ct_buffer_desc *desc = ctb->desc; 820 u32 head = ctb->head; 821 u32 tail = READ_ONCE(desc->tail); 822 u32 size = ctb->size; 823 u32 *cmds = ctb->cmds; 824 s32 available; 825 unsigned int len; 826 unsigned int i; 827 u32 header; 828 829 if (unlikely(ctb->broken)) 830 return -EPIPE; 831 832 if (unlikely(desc->status)) { 833 u32 status = desc->status; 834 835 if (status & GUC_CTB_STATUS_UNUSED) { 836 /* 837 * Potentially valid if a CLIENT_RESET request resulted in 838 * contexts/engines being reset. But should never happen as 839 * no contexts should be active when CLIENT_RESET is sent. 840 */ 841 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n"); 842 status &= ~GUC_CTB_STATUS_UNUSED; 843 } 844 845 if (status) 846 goto corrupted; 847 } 848 849 GEM_BUG_ON(head > size); 850 851 #ifdef CONFIG_DRM_I915_DEBUG_GUC 852 if (unlikely(head != READ_ONCE(desc->head))) { 853 CT_ERROR(ct, "Head was modified %u != %u\n", 854 desc->head, head); 855 desc->status |= GUC_CTB_STATUS_MISMATCH; 856 goto corrupted; 857 } 858 #endif 859 if (unlikely(tail >= size)) { 860 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", 861 tail, size); 862 desc->status |= GUC_CTB_STATUS_OVERFLOW; 863 goto corrupted; 864 } 865 866 /* tail == head condition indicates empty */ 867 available = tail - head; 868 if (unlikely(available == 0)) { 869 *msg = NULL; 870 return 0; 871 } 872 873 /* beware of buffer wrap case */ 874 if (unlikely(available < 0)) 875 available += size; 876 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); 877 GEM_BUG_ON(available < 0); 878 879 header = cmds[head]; 880 head = (head + 1) % size; 881 882 /* message len with header */ 883 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; 884 if (unlikely(len > (u32)available)) { 885 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 886 4, &header, 887 4 * (head + available - 1 > size ? 888 size - head : available - 1), &cmds[head], 889 4 * (head + available - 1 > size ? 890 available - 1 - size + head : 0), &cmds[0]); 891 desc->status |= GUC_CTB_STATUS_UNDERFLOW; 892 goto corrupted; 893 } 894 895 *msg = ct_alloc_msg(len); 896 if (!*msg) { 897 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n", 898 4, &header, 899 4 * (head + available - 1 > size ? 900 size - head : available - 1), &cmds[head], 901 4 * (head + available - 1 > size ? 902 available - 1 - size + head : 0), &cmds[0]); 903 return available; 904 } 905 906 (*msg)->msg[0] = header; 907 908 for (i = 1; i < len; i++) { 909 (*msg)->msg[i] = cmds[head]; 910 head = (head + 1) % size; 911 } 912 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); 913 914 /* update local copies */ 915 ctb->head = head; 916 917 /* now update descriptor */ 918 WRITE_ONCE(desc->head, head); 919 920 return available - len; 921 922 corrupted: 923 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 924 desc->head, desc->tail, desc->status); 925 ctb->broken = true; 926 return -EPIPE; 927 } 928 929 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) 930 { 931 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); 932 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); 933 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; 934 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; 935 u32 datalen = len - GUC_HXG_MSG_MIN_LEN; 936 struct ct_request *req; 937 unsigned long flags; 938 bool found = false; 939 int err = 0; 940 941 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); 942 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); 943 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && 944 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY && 945 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); 946 947 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); 948 949 spin_lock_irqsave(&ct->requests.lock, flags); 950 list_for_each_entry(req, &ct->requests.pending, link) { 951 if (unlikely(fence != req->fence)) { 952 CT_DEBUG(ct, "request %u awaits response\n", 953 req->fence); 954 continue; 955 } 956 if (unlikely(datalen > req->response_len)) { 957 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n", 958 req->fence, datalen, req->response_len); 959 datalen = min(datalen, req->response_len); 960 err = -EMSGSIZE; 961 } 962 if (datalen) 963 memcpy(req->response_buf, data, 4 * datalen); 964 req->response_len = datalen; 965 WRITE_ONCE(req->status, hxg[0]); 966 found = true; 967 break; 968 } 969 if (!found) { 970 CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence); 971 CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence, 972 ct->requests.last_fence); 973 list_for_each_entry(req, &ct->requests.pending, link) 974 CT_ERROR(ct, "request %u awaits response\n", 975 req->fence); 976 err = -ENOKEY; 977 } 978 spin_unlock_irqrestore(&ct->requests.lock, flags); 979 980 if (unlikely(err)) 981 return err; 982 983 ct_free_msg(response); 984 return 0; 985 } 986 987 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 988 { 989 struct intel_guc *guc = ct_to_guc(ct); 990 const u32 *hxg; 991 const u32 *payload; 992 u32 hxg_len, action, len; 993 int ret; 994 995 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 996 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; 997 payload = &hxg[GUC_HXG_MSG_MIN_LEN]; 998 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 999 len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1000 1001 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); 1002 1003 switch (action) { 1004 case INTEL_GUC_ACTION_DEFAULT: 1005 ret = intel_guc_to_host_process_recv_msg(guc, payload, len); 1006 break; 1007 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1008 ret = intel_guc_deregister_done_process_msg(guc, payload, 1009 len); 1010 break; 1011 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1012 ret = intel_guc_sched_done_process_msg(guc, payload, len); 1013 break; 1014 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1015 ret = intel_guc_context_reset_process_msg(guc, payload, len); 1016 break; 1017 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1018 ret = intel_guc_error_capture_process_msg(guc, payload, len); 1019 if (unlikely(ret)) 1020 CT_ERROR(ct, "error capture notification failed %x %*ph\n", 1021 action, 4 * len, payload); 1022 break; 1023 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1024 ret = intel_guc_engine_failure_process_msg(guc, payload, len); 1025 break; 1026 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1027 intel_guc_log_handle_flush_event(&guc->log); 1028 ret = 0; 1029 break; 1030 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1031 CT_ERROR(ct, "Received GuC crash dump notification!\n"); 1032 ret = 0; 1033 break; 1034 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION: 1035 CT_ERROR(ct, "Received GuC exception notification!\n"); 1036 ret = 0; 1037 break; 1038 default: 1039 ret = -EOPNOTSUPP; 1040 break; 1041 } 1042 1043 if (unlikely(ret)) { 1044 CT_ERROR(ct, "Failed to process request %04x (%pe)\n", 1045 action, ERR_PTR(ret)); 1046 return ret; 1047 } 1048 1049 ct_free_msg(request); 1050 return 0; 1051 } 1052 1053 static bool ct_process_incoming_requests(struct intel_guc_ct *ct) 1054 { 1055 unsigned long flags; 1056 struct ct_incoming_msg *request; 1057 bool done; 1058 int err; 1059 1060 spin_lock_irqsave(&ct->requests.lock, flags); 1061 request = list_first_entry_or_null(&ct->requests.incoming, 1062 struct ct_incoming_msg, link); 1063 if (request) 1064 list_del(&request->link); 1065 done = !!list_empty(&ct->requests.incoming); 1066 spin_unlock_irqrestore(&ct->requests.lock, flags); 1067 1068 if (!request) 1069 return true; 1070 1071 err = ct_process_request(ct, request); 1072 if (unlikely(err)) { 1073 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1074 ERR_PTR(err), 4 * request->size, request->msg); 1075 ct_free_msg(request); 1076 } 1077 1078 return done; 1079 } 1080 1081 static void ct_incoming_request_worker_func(struct work_struct *w) 1082 { 1083 struct intel_guc_ct *ct = 1084 container_of(w, struct intel_guc_ct, requests.worker); 1085 bool done; 1086 1087 do { 1088 done = ct_process_incoming_requests(ct); 1089 } while (!done); 1090 } 1091 1092 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1093 { 1094 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1095 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1096 unsigned long flags; 1097 1098 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); 1099 1100 /* 1101 * Adjusting the space must be done in IRQ or deadlock can occur as the 1102 * CTB processing in the below workqueue can send CTBs which creates a 1103 * circular dependency if the space was returned there. 1104 */ 1105 switch (action) { 1106 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1107 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1108 g2h_release_space(ct, request->size); 1109 } 1110 1111 spin_lock_irqsave(&ct->requests.lock, flags); 1112 list_add_tail(&request->link, &ct->requests.incoming); 1113 spin_unlock_irqrestore(&ct->requests.lock, flags); 1114 1115 queue_work(system_unbound_wq, &ct->requests.worker); 1116 return 0; 1117 } 1118 1119 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1120 { 1121 u32 origin, type; 1122 u32 *hxg; 1123 int err; 1124 1125 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) 1126 return -EBADMSG; 1127 1128 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; 1129 1130 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1131 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1132 err = -EPROTO; 1133 goto failed; 1134 } 1135 1136 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1137 switch (type) { 1138 case GUC_HXG_TYPE_EVENT: 1139 err = ct_handle_event(ct, msg); 1140 break; 1141 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1142 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1143 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1144 err = ct_handle_response(ct, msg); 1145 break; 1146 default: 1147 err = -EOPNOTSUPP; 1148 } 1149 1150 if (unlikely(err)) { 1151 failed: 1152 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", 1153 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); 1154 } 1155 return err; 1156 } 1157 1158 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1159 { 1160 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); 1161 int err; 1162 1163 if (format == GUC_CTB_FORMAT_HXG) 1164 err = ct_handle_hxg(ct, msg); 1165 else 1166 err = -EOPNOTSUPP; 1167 1168 if (unlikely(err)) { 1169 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1170 ERR_PTR(err), 4 * msg->size, msg->msg); 1171 ct_free_msg(msg); 1172 } 1173 } 1174 1175 /* 1176 * Return: number available remaining dwords to read (0 if empty) 1177 * or a negative error code on failure 1178 */ 1179 static int ct_receive(struct intel_guc_ct *ct) 1180 { 1181 struct ct_incoming_msg *msg = NULL; 1182 unsigned long flags; 1183 int ret; 1184 1185 spin_lock_irqsave(&ct->ctbs.recv.lock, flags); 1186 ret = ct_read(ct, &msg); 1187 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags); 1188 if (ret < 0) 1189 return ret; 1190 1191 if (msg) 1192 ct_handle_msg(ct, msg); 1193 1194 return ret; 1195 } 1196 1197 static void ct_try_receive_message(struct intel_guc_ct *ct) 1198 { 1199 int ret; 1200 1201 if (GEM_WARN_ON(!ct->enabled)) 1202 return; 1203 1204 ret = ct_receive(ct); 1205 if (ret > 0) 1206 tasklet_hi_schedule(&ct->receive_tasklet); 1207 } 1208 1209 static void ct_receive_tasklet_func(struct tasklet_struct *t) 1210 { 1211 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet); 1212 1213 ct_try_receive_message(ct); 1214 } 1215 1216 /* 1217 * When we're communicating with the GuC over CT, GuC uses events 1218 * to notify us about new messages being posted on the RECV buffer. 1219 */ 1220 void intel_guc_ct_event_handler(struct intel_guc_ct *ct) 1221 { 1222 if (unlikely(!ct->enabled)) { 1223 WARN(1, "Unexpected GuC event received while CT disabled!\n"); 1224 return; 1225 } 1226 1227 ct_try_receive_message(ct); 1228 } 1229 1230 void intel_guc_ct_print_info(struct intel_guc_ct *ct, 1231 struct drm_printer *p) 1232 { 1233 drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled)); 1234 1235 if (!ct->enabled) 1236 return; 1237 1238 drm_printf(p, "H2G Space: %u\n", 1239 atomic_read(&ct->ctbs.send.space) * 4); 1240 drm_printf(p, "Head: %u\n", 1241 ct->ctbs.send.desc->head); 1242 drm_printf(p, "Tail: %u\n", 1243 ct->ctbs.send.desc->tail); 1244 drm_printf(p, "G2H Space: %u\n", 1245 atomic_read(&ct->ctbs.recv.space) * 4); 1246 drm_printf(p, "Head: %u\n", 1247 ct->ctbs.recv.desc->head); 1248 drm_printf(p, "Tail: %u\n", 1249 ct->ctbs.recv.desc->tail); 1250 } 1251