1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016-2019 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/ktime.h> 8 #include <linux/time64.h> 9 #include <linux/string_helpers.h> 10 #include <linux/timekeeping.h> 11 12 #include "i915_drv.h" 13 #include "intel_guc_ct.h" 14 #include "intel_guc_print.h" 15 16 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 17 enum { 18 CT_DEAD_ALIVE = 0, 19 CT_DEAD_SETUP, 20 CT_DEAD_WRITE, 21 CT_DEAD_DEADLOCK, 22 CT_DEAD_H2G_HAS_ROOM, 23 CT_DEAD_READ, 24 CT_DEAD_PROCESS_FAILED, 25 }; 26 27 static void ct_dead_ct_worker_func(struct work_struct *w); 28 29 #define CT_DEAD(ct, reason) \ 30 do { \ 31 if (!(ct)->dead_ct_reported) { \ 32 (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \ 33 queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \ 34 } \ 35 } while (0) 36 #else 37 #define CT_DEAD(ct, reason) do { } while (0) 38 #endif 39 40 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) 41 { 42 return container_of(ct, struct intel_guc, ct); 43 } 44 45 #define CT_ERROR(_ct, _fmt, ...) \ 46 guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__) 47 #ifdef CONFIG_DRM_I915_DEBUG_GUC 48 #define CT_DEBUG(_ct, _fmt, ...) \ 49 guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__) 50 #else 51 #define CT_DEBUG(...) do { } while (0) 52 #endif 53 #define CT_PROBE_ERROR(_ct, _fmt, ...) \ 54 guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__) 55 56 /** 57 * DOC: CTB Blob 58 * 59 * We allocate single blob to hold both CTB descriptors and buffers: 60 * 61 * +--------+-----------------------------------------------+------+ 62 * | offset | contents | size | 63 * +========+===============================================+======+ 64 * | 0x0000 | H2G `CTB Descriptor`_ (send) | | 65 * +--------+-----------------------------------------------+ 4K | 66 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | | 67 * +--------+-----------------------------------------------+------+ 68 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K | 69 * | | | | 70 * +--------+-----------------------------------------------+------+ 71 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K | 72 * | + n*4K | | | 73 * +--------+-----------------------------------------------+------+ 74 * 75 * Size of each `CT Buffer`_ must be multiple of 4K. 76 * We don't expect too many messages in flight at any time, unless we are 77 * using the GuC submission. In that case each request requires a minimum 78 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 79 * enough space to avoid backpressure on the driver. We increase the size 80 * of the receive buffer (relative to the send) to ensure a G2H response 81 * CTB has a landing spot. 82 */ 83 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 84 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 85 #define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) 86 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) 87 88 struct ct_request { 89 struct list_head link; 90 u32 fence; 91 u32 status; 92 u32 response_len; 93 u32 *response_buf; 94 }; 95 96 struct ct_incoming_msg { 97 struct list_head link; 98 u32 size; 99 u32 msg[]; 100 }; 101 102 enum { CTB_SEND = 0, CTB_RECV = 1 }; 103 104 enum { CTB_OWNER_HOST = 0 }; 105 106 static void ct_receive_tasklet_func(struct tasklet_struct *t); 107 static void ct_incoming_request_worker_func(struct work_struct *w); 108 109 /** 110 * intel_guc_ct_init_early - Initialize CT state without requiring device access 111 * @ct: pointer to CT struct 112 */ 113 void intel_guc_ct_init_early(struct intel_guc_ct *ct) 114 { 115 spin_lock_init(&ct->ctbs.send.lock); 116 spin_lock_init(&ct->ctbs.recv.lock); 117 spin_lock_init(&ct->requests.lock); 118 INIT_LIST_HEAD(&ct->requests.pending); 119 INIT_LIST_HEAD(&ct->requests.incoming); 120 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 121 INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func); 122 #endif 123 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); 124 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); 125 init_waitqueue_head(&ct->wq); 126 } 127 128 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) 129 { 130 memset(desc, 0, sizeof(*desc)); 131 } 132 133 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) 134 { 135 u32 space; 136 137 ctb->broken = false; 138 ctb->tail = 0; 139 ctb->head = 0; 140 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; 141 atomic_set(&ctb->space, space); 142 143 guc_ct_buffer_desc_init(ctb->desc); 144 } 145 146 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, 147 struct guc_ct_buffer_desc *desc, 148 u32 *cmds, u32 size_in_bytes, u32 resv_space) 149 { 150 GEM_BUG_ON(size_in_bytes % 4); 151 152 ctb->desc = desc; 153 ctb->cmds = cmds; 154 ctb->size = size_in_bytes / 4; 155 ctb->resv_space = resv_space / 4; 156 157 guc_ct_buffer_reset(ctb); 158 } 159 160 static int guc_action_control_ctb(struct intel_guc *guc, u32 control) 161 { 162 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 163 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 164 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 165 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB), 166 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control), 167 }; 168 int ret; 169 170 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE); 171 172 /* CT control must go over MMIO */ 173 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); 174 175 return ret > 0 ? -EPROTO : ret; 176 } 177 178 static int ct_control_enable(struct intel_guc_ct *ct, bool enable) 179 { 180 int err; 181 182 err = guc_action_control_ctb(ct_to_guc(ct), enable ? 183 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE); 184 if (unlikely(err)) 185 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n", 186 str_enable_disable(enable), ERR_PTR(err)); 187 188 return err; 189 } 190 191 static int ct_register_buffer(struct intel_guc_ct *ct, bool send, 192 u32 desc_addr, u32 buff_addr, u32 size) 193 { 194 int err; 195 196 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 197 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY : 198 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 199 desc_addr); 200 if (unlikely(err)) 201 goto failed; 202 203 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 204 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY : 205 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 206 buff_addr); 207 if (unlikely(err)) 208 goto failed; 209 210 err = intel_guc_self_cfg32(ct_to_guc(ct), send ? 211 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY : 212 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 213 size); 214 if (unlikely(err)) 215 failed: 216 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n", 217 send ? "SEND" : "RECV", ERR_PTR(err)); 218 219 return err; 220 } 221 222 /** 223 * intel_guc_ct_init - Init buffer-based communication 224 * @ct: pointer to CT struct 225 * 226 * Allocate memory required for buffer-based communication. 227 * 228 * Return: 0 on success, a negative errno code on failure. 229 */ 230 int intel_guc_ct_init(struct intel_guc_ct *ct) 231 { 232 struct intel_guc *guc = ct_to_guc(ct); 233 struct guc_ct_buffer_desc *desc; 234 u32 blob_size; 235 u32 cmds_size; 236 u32 resv_space; 237 void *blob; 238 u32 *cmds; 239 int err; 240 241 err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO); 242 if (err) 243 return err; 244 245 GEM_BUG_ON(ct->vma); 246 247 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; 248 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob); 249 if (unlikely(err)) { 250 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n", 251 blob_size, ERR_PTR(err)); 252 return err; 253 } 254 255 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size); 256 257 /* store pointers to desc and cmds for send ctb */ 258 desc = blob; 259 cmds = blob + 2 * CTB_DESC_SIZE; 260 cmds_size = CTB_H2G_BUFFER_SIZE; 261 resv_space = 0; 262 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", 263 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 264 resv_space); 265 266 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); 267 268 /* store pointers to desc and cmds for recv ctb */ 269 desc = blob + CTB_DESC_SIZE; 270 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; 271 cmds_size = CTB_G2H_BUFFER_SIZE; 272 resv_space = G2H_ROOM_BUFFER_SIZE; 273 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", 274 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 275 resv_space); 276 277 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); 278 279 return 0; 280 } 281 282 /** 283 * intel_guc_ct_fini - Fini buffer-based communication 284 * @ct: pointer to CT struct 285 * 286 * Deallocate memory required for buffer-based communication. 287 */ 288 void intel_guc_ct_fini(struct intel_guc_ct *ct) 289 { 290 GEM_BUG_ON(ct->enabled); 291 292 tasklet_kill(&ct->receive_tasklet); 293 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); 294 memset(ct, 0, sizeof(*ct)); 295 } 296 297 /** 298 * intel_guc_ct_enable - Enable buffer based command transport. 299 * @ct: pointer to CT struct 300 * 301 * Return: 0 on success, a negative errno code on failure. 302 */ 303 int intel_guc_ct_enable(struct intel_guc_ct *ct) 304 { 305 struct intel_guc *guc = ct_to_guc(ct); 306 u32 base, desc, cmds, size; 307 void *blob; 308 int err; 309 310 GEM_BUG_ON(ct->enabled); 311 312 /* vma should be already allocated and map'ed */ 313 GEM_BUG_ON(!ct->vma); 314 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj)); 315 base = intel_guc_ggtt_offset(guc, ct->vma); 316 317 /* blob should start with send descriptor */ 318 blob = __px_vaddr(ct->vma->obj); 319 GEM_BUG_ON(blob != ct->ctbs.send.desc); 320 321 /* (re)initialize descriptors */ 322 guc_ct_buffer_reset(&ct->ctbs.send); 323 guc_ct_buffer_reset(&ct->ctbs.recv); 324 325 /* 326 * Register both CT buffers starting with RECV buffer. 327 * Descriptors are in first half of the blob. 328 */ 329 desc = base + ptrdiff(ct->ctbs.recv.desc, blob); 330 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); 331 size = ct->ctbs.recv.size * 4; 332 err = ct_register_buffer(ct, false, desc, cmds, size); 333 if (unlikely(err)) 334 goto err_out; 335 336 desc = base + ptrdiff(ct->ctbs.send.desc, blob); 337 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); 338 size = ct->ctbs.send.size * 4; 339 err = ct_register_buffer(ct, true, desc, cmds, size); 340 if (unlikely(err)) 341 goto err_out; 342 343 err = ct_control_enable(ct, true); 344 if (unlikely(err)) 345 goto err_out; 346 347 ct->enabled = true; 348 ct->stall_time = KTIME_MAX; 349 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 350 ct->dead_ct_reported = false; 351 ct->dead_ct_reason = CT_DEAD_ALIVE; 352 #endif 353 354 return 0; 355 356 err_out: 357 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); 358 CT_DEAD(ct, SETUP); 359 return err; 360 } 361 362 /** 363 * intel_guc_ct_disable - Disable buffer based command transport. 364 * @ct: pointer to CT struct 365 */ 366 void intel_guc_ct_disable(struct intel_guc_ct *ct) 367 { 368 struct intel_guc *guc = ct_to_guc(ct); 369 370 GEM_BUG_ON(!ct->enabled); 371 372 ct->enabled = false; 373 374 if (intel_guc_is_fw_running(guc)) { 375 ct_control_enable(ct, false); 376 } 377 } 378 379 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 380 static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action) 381 { 382 unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found); 383 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 384 unsigned long entries[SZ_32]; 385 unsigned int n; 386 387 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 388 389 /* May be called under spinlock, so avoid sleeping */ 390 ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT); 391 #endif 392 ct->requests.lost_and_found[lost].fence = fence; 393 ct->requests.lost_and_found[lost].action = action; 394 } 395 #endif 396 397 static u32 ct_get_next_fence(struct intel_guc_ct *ct) 398 { 399 /* For now it's trivial */ 400 return ++ct->requests.last_fence; 401 } 402 403 static int ct_write(struct intel_guc_ct *ct, 404 const u32 *action, 405 u32 len /* in dwords */, 406 u32 fence, u32 flags) 407 { 408 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 409 struct guc_ct_buffer_desc *desc = ctb->desc; 410 u32 tail = ctb->tail; 411 u32 size = ctb->size; 412 u32 header; 413 u32 hxg; 414 u32 type; 415 u32 *cmds = ctb->cmds; 416 unsigned int i; 417 418 if (unlikely(desc->status)) 419 goto corrupted; 420 421 GEM_BUG_ON(tail > size); 422 423 #ifdef CONFIG_DRM_I915_DEBUG_GUC 424 if (unlikely(tail != READ_ONCE(desc->tail))) { 425 CT_ERROR(ct, "Tail was modified %u != %u\n", 426 desc->tail, tail); 427 desc->status |= GUC_CTB_STATUS_MISMATCH; 428 goto corrupted; 429 } 430 if (unlikely(READ_ONCE(desc->head) >= size)) { 431 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 432 desc->head, size); 433 desc->status |= GUC_CTB_STATUS_OVERFLOW; 434 goto corrupted; 435 } 436 #endif 437 438 /* 439 * dw0: CT header (including fence) 440 * dw1: HXG header (including action code) 441 * dw2+: action data 442 */ 443 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 444 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 445 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); 446 447 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST : 448 GUC_HXG_TYPE_REQUEST; 449 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | 450 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION | 451 GUC_HXG_REQUEST_MSG_0_DATA0, action[0]); 452 453 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", 454 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); 455 456 cmds[tail] = header; 457 tail = (tail + 1) % size; 458 459 cmds[tail] = hxg; 460 tail = (tail + 1) % size; 461 462 for (i = 1; i < len; i++) { 463 cmds[tail] = action[i]; 464 tail = (tail + 1) % size; 465 } 466 GEM_BUG_ON(tail > size); 467 468 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 469 ct_track_lost_and_found(ct, fence, 470 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 471 #endif 472 473 /* 474 * make sure H2G buffer update and LRC tail update (if this triggering a 475 * submission) are visible before updating the descriptor tail 476 */ 477 intel_guc_write_barrier(ct_to_guc(ct)); 478 479 /* update local copies */ 480 ctb->tail = tail; 481 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); 482 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); 483 484 /* now update descriptor */ 485 WRITE_ONCE(desc->tail, tail); 486 487 return 0; 488 489 corrupted: 490 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 491 desc->head, desc->tail, desc->status); 492 CT_DEAD(ct, WRITE); 493 ctb->broken = true; 494 return -EPIPE; 495 } 496 497 /** 498 * wait_for_ct_request_update - Wait for CT request state update. 499 * @ct: pointer to CT 500 * @req: pointer to pending request 501 * @status: placeholder for status 502 * 503 * For each sent request, GuC shall send back CT response message. 504 * Our message handler will update status of tracked request once 505 * response message with given fence is received. Wait here and 506 * check for valid response status value. 507 * 508 * Return: 509 * * 0 response received (status is valid) 510 * * -ETIMEDOUT no response within hardcoded timeout 511 */ 512 static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status) 513 { 514 int err; 515 bool ct_enabled; 516 517 /* 518 * Fast commands should complete in less than 10us, so sample quickly 519 * up to that length of time, then switch to a slower sleep-wait loop. 520 * No GuC command should ever take longer than 10ms but many GuC 521 * commands can be inflight at time, so use a 1s timeout on the slower 522 * sleep-wait loop. 523 */ 524 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 525 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 526 #define done \ 527 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \ 528 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ 529 GUC_HXG_ORIGIN_GUC) 530 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); 531 if (err) 532 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); 533 #undef done 534 if (!ct_enabled) 535 err = -ENODEV; 536 537 *status = req->status; 538 return err; 539 } 540 541 #define GUC_CTB_TIMEOUT_MS 1500 542 static inline bool ct_deadlocked(struct intel_guc_ct *ct) 543 { 544 long timeout = GUC_CTB_TIMEOUT_MS; 545 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; 546 547 if (unlikely(ret)) { 548 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; 549 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; 550 551 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", 552 ktime_ms_delta(ktime_get(), ct->stall_time), 553 send->status, recv->status); 554 CT_ERROR(ct, "H2G Space: %u (Bytes)\n", 555 atomic_read(&ct->ctbs.send.space) * 4); 556 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head); 557 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail); 558 CT_ERROR(ct, "G2H Space: %u (Bytes)\n", 559 atomic_read(&ct->ctbs.recv.space) * 4); 560 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); 561 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); 562 563 CT_DEAD(ct, DEADLOCK); 564 ct->ctbs.send.broken = true; 565 } 566 567 return ret; 568 } 569 570 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) 571 { 572 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 573 574 /* 575 * We leave a certain amount of space in the G2H CTB buffer for 576 * unexpected G2H CTBs (e.g. logging, engine hang, etc...) 577 */ 578 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; 579 } 580 581 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 582 { 583 lockdep_assert_held(&ct->ctbs.send.lock); 584 585 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); 586 587 if (g2h_len_dw) 588 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); 589 } 590 591 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 592 { 593 atomic_add(g2h_len_dw, &ct->ctbs.recv.space); 594 } 595 596 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) 597 { 598 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 599 struct guc_ct_buffer_desc *desc = ctb->desc; 600 u32 head; 601 u32 space; 602 603 if (atomic_read(&ctb->space) >= len_dw) 604 return true; 605 606 head = READ_ONCE(desc->head); 607 if (unlikely(head > ctb->size)) { 608 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 609 head, ctb->size); 610 desc->status |= GUC_CTB_STATUS_OVERFLOW; 611 ctb->broken = true; 612 CT_DEAD(ct, H2G_HAS_ROOM); 613 return false; 614 } 615 616 space = CIRC_SPACE(ctb->tail, head, ctb->size); 617 atomic_set(&ctb->space, space); 618 619 return space >= len_dw; 620 } 621 622 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) 623 { 624 bool h2g = h2g_has_room(ct, h2g_dw); 625 bool g2h = g2h_has_room(ct, g2h_dw); 626 627 lockdep_assert_held(&ct->ctbs.send.lock); 628 629 if (unlikely(!h2g || !g2h)) { 630 if (ct->stall_time == KTIME_MAX) 631 ct->stall_time = ktime_get(); 632 633 /* Be paranoid and kick G2H tasklet to free credits */ 634 if (!g2h) 635 tasklet_hi_schedule(&ct->receive_tasklet); 636 637 if (unlikely(ct_deadlocked(ct))) 638 return -EPIPE; 639 else 640 return -EBUSY; 641 } 642 643 ct->stall_time = KTIME_MAX; 644 return 0; 645 } 646 647 #define G2H_LEN_DW(f) ({ \ 648 typeof(f) f_ = (f); \ 649 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ 650 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ 651 GUC_CTB_HXG_MSG_MIN_LEN : 0; \ 652 }) 653 static int ct_send_nb(struct intel_guc_ct *ct, 654 const u32 *action, 655 u32 len, 656 u32 flags) 657 { 658 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 659 unsigned long spin_flags; 660 u32 g2h_len_dw = G2H_LEN_DW(flags); 661 u32 fence; 662 int ret; 663 664 spin_lock_irqsave(&ctb->lock, spin_flags); 665 666 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); 667 if (unlikely(ret)) 668 goto out; 669 670 fence = ct_get_next_fence(ct); 671 ret = ct_write(ct, action, len, fence, flags); 672 if (unlikely(ret)) 673 goto out; 674 675 g2h_reserve_space(ct, g2h_len_dw); 676 intel_guc_notify(ct_to_guc(ct)); 677 678 out: 679 spin_unlock_irqrestore(&ctb->lock, spin_flags); 680 681 return ret; 682 } 683 684 static int ct_send(struct intel_guc_ct *ct, 685 const u32 *action, 686 u32 len, 687 u32 *response_buf, 688 u32 response_buf_size, 689 u32 *status) 690 { 691 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 692 struct ct_request request; 693 unsigned long flags; 694 unsigned int sleep_period_ms = 1; 695 bool send_again; 696 u32 fence; 697 int err; 698 699 GEM_BUG_ON(!ct->enabled); 700 GEM_BUG_ON(!len); 701 GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN); 702 GEM_BUG_ON(!response_buf && response_buf_size); 703 might_sleep(); 704 705 resend: 706 send_again = false; 707 708 /* 709 * We use a lazy spin wait loop here as we believe that if the CT 710 * buffers are sized correctly the flow control condition should be 711 * rare. Reserving the maximum size in the G2H credits as we don't know 712 * how big the response is going to be. 713 */ 714 retry: 715 spin_lock_irqsave(&ctb->lock, flags); 716 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || 717 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { 718 if (ct->stall_time == KTIME_MAX) 719 ct->stall_time = ktime_get(); 720 spin_unlock_irqrestore(&ctb->lock, flags); 721 722 if (unlikely(ct_deadlocked(ct))) 723 return -EPIPE; 724 725 if (msleep_interruptible(sleep_period_ms)) 726 return -EINTR; 727 sleep_period_ms = sleep_period_ms << 1; 728 729 goto retry; 730 } 731 732 ct->stall_time = KTIME_MAX; 733 734 fence = ct_get_next_fence(ct); 735 request.fence = fence; 736 request.status = 0; 737 request.response_len = response_buf_size; 738 request.response_buf = response_buf; 739 740 spin_lock(&ct->requests.lock); 741 list_add_tail(&request.link, &ct->requests.pending); 742 spin_unlock(&ct->requests.lock); 743 744 err = ct_write(ct, action, len, fence, 0); 745 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 746 747 spin_unlock_irqrestore(&ctb->lock, flags); 748 749 if (unlikely(err)) 750 goto unlink; 751 752 intel_guc_notify(ct_to_guc(ct)); 753 754 err = wait_for_ct_request_update(ct, &request, status); 755 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 756 if (unlikely(err)) { 757 if (err == -ENODEV) 758 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress. 759 * In this case, output is debug rather than error info 760 */ 761 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n", 762 action[0], request.fence); 763 else 764 CT_ERROR(ct, "No response for request %#x (fence %u)\n", 765 action[0], request.fence); 766 goto unlink; 767 } 768 769 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 770 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action, 771 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status)); 772 send_again = true; 773 goto unlink; 774 } 775 776 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { 777 err = -EIO; 778 goto unlink; 779 } 780 781 if (response_buf) { 782 /* There shall be no data in the status */ 783 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); 784 /* Return actual response len */ 785 err = request.response_len; 786 } else { 787 /* There shall be no response payload */ 788 WARN_ON(request.response_len); 789 /* Return data decoded from the status dword */ 790 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); 791 } 792 793 unlink: 794 spin_lock_irqsave(&ct->requests.lock, flags); 795 list_del(&request.link); 796 spin_unlock_irqrestore(&ct->requests.lock, flags); 797 798 if (unlikely(send_again)) 799 goto resend; 800 801 return err; 802 } 803 804 /* 805 * Command Transport (CT) buffer based GuC send function. 806 */ 807 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, 808 u32 *response_buf, u32 response_buf_size, u32 flags) 809 { 810 u32 status = ~0; /* undefined */ 811 int ret; 812 813 if (unlikely(!ct->enabled)) { 814 struct intel_guc *guc = ct_to_guc(ct); 815 struct intel_uc *uc = container_of(guc, struct intel_uc, guc); 816 817 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); 818 return -ENODEV; 819 } 820 821 if (unlikely(ct->ctbs.send.broken)) 822 return -EPIPE; 823 824 if (flags & INTEL_GUC_CT_SEND_NB) 825 return ct_send_nb(ct, action, len, flags); 826 827 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); 828 if (unlikely(ret < 0)) { 829 if (ret != -ENODEV) 830 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", 831 action[0], ERR_PTR(ret), status); 832 } else if (unlikely(ret)) { 833 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", 834 action[0], ret, ret); 835 } 836 837 return ret; 838 } 839 840 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) 841 { 842 struct ct_incoming_msg *msg; 843 844 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC); 845 if (msg) 846 msg->size = num_dwords; 847 return msg; 848 } 849 850 static void ct_free_msg(struct ct_incoming_msg *msg) 851 { 852 kfree(msg); 853 } 854 855 /* 856 * Return: number available remaining dwords to read (0 if empty) 857 * or a negative error code on failure 858 */ 859 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) 860 { 861 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 862 struct guc_ct_buffer_desc *desc = ctb->desc; 863 u32 head = ctb->head; 864 u32 tail = READ_ONCE(desc->tail); 865 u32 size = ctb->size; 866 u32 *cmds = ctb->cmds; 867 s32 available; 868 unsigned int len; 869 unsigned int i; 870 u32 header; 871 872 if (unlikely(ctb->broken)) 873 return -EPIPE; 874 875 if (unlikely(desc->status)) { 876 u32 status = desc->status; 877 878 if (status & GUC_CTB_STATUS_UNUSED) { 879 /* 880 * Potentially valid if a CLIENT_RESET request resulted in 881 * contexts/engines being reset. But should never happen as 882 * no contexts should be active when CLIENT_RESET is sent. 883 */ 884 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n"); 885 status &= ~GUC_CTB_STATUS_UNUSED; 886 } 887 888 if (status) 889 goto corrupted; 890 } 891 892 GEM_BUG_ON(head > size); 893 894 #ifdef CONFIG_DRM_I915_DEBUG_GUC 895 if (unlikely(head != READ_ONCE(desc->head))) { 896 CT_ERROR(ct, "Head was modified %u != %u\n", 897 desc->head, head); 898 desc->status |= GUC_CTB_STATUS_MISMATCH; 899 goto corrupted; 900 } 901 #endif 902 if (unlikely(tail >= size)) { 903 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", 904 tail, size); 905 desc->status |= GUC_CTB_STATUS_OVERFLOW; 906 goto corrupted; 907 } 908 909 /* tail == head condition indicates empty */ 910 available = tail - head; 911 if (unlikely(available == 0)) { 912 *msg = NULL; 913 return 0; 914 } 915 916 /* beware of buffer wrap case */ 917 if (unlikely(available < 0)) 918 available += size; 919 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); 920 GEM_BUG_ON(available < 0); 921 922 header = cmds[head]; 923 head = (head + 1) % size; 924 925 /* message len with header */ 926 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; 927 if (unlikely(len > (u32)available)) { 928 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 929 4, &header, 930 4 * (head + available - 1 > size ? 931 size - head : available - 1), &cmds[head], 932 4 * (head + available - 1 > size ? 933 available - 1 - size + head : 0), &cmds[0]); 934 desc->status |= GUC_CTB_STATUS_UNDERFLOW; 935 goto corrupted; 936 } 937 938 *msg = ct_alloc_msg(len); 939 if (!*msg) { 940 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n", 941 4, &header, 942 4 * (head + available - 1 > size ? 943 size - head : available - 1), &cmds[head], 944 4 * (head + available - 1 > size ? 945 available - 1 - size + head : 0), &cmds[0]); 946 return available; 947 } 948 949 (*msg)->msg[0] = header; 950 951 for (i = 1; i < len; i++) { 952 (*msg)->msg[i] = cmds[head]; 953 head = (head + 1) % size; 954 } 955 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); 956 957 /* update local copies */ 958 ctb->head = head; 959 960 /* now update descriptor */ 961 WRITE_ONCE(desc->head, head); 962 963 /* 964 * Wa_22016122933: Making sure the head update is 965 * visible to GuC right away 966 */ 967 intel_guc_write_barrier(ct_to_guc(ct)); 968 969 return available - len; 970 971 corrupted: 972 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 973 desc->head, desc->tail, desc->status); 974 ctb->broken = true; 975 CT_DEAD(ct, READ); 976 return -EPIPE; 977 } 978 979 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 980 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) 981 { 982 unsigned int n; 983 char *buf = NULL; 984 bool found = false; 985 986 lockdep_assert_held(&ct->requests.lock); 987 988 for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) { 989 if (ct->requests.lost_and_found[n].fence != fence) 990 continue; 991 found = true; 992 993 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 994 buf = kmalloc(SZ_4K, GFP_NOWAIT); 995 if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack, 996 buf, SZ_4K, 0)) { 997 CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s", 998 fence, ct->requests.lost_and_found[n].action, buf); 999 break; 1000 } 1001 #endif 1002 CT_ERROR(ct, "Fence %u was used by action %#04x\n", 1003 fence, ct->requests.lost_and_found[n].action); 1004 break; 1005 } 1006 kfree(buf); 1007 return found; 1008 } 1009 #else 1010 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) 1011 { 1012 return false; 1013 } 1014 #endif 1015 1016 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) 1017 { 1018 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); 1019 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); 1020 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; 1021 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; 1022 u32 datalen = len - GUC_HXG_MSG_MIN_LEN; 1023 struct ct_request *req; 1024 unsigned long flags; 1025 bool found = false; 1026 int err = 0; 1027 1028 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); 1029 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); 1030 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && 1031 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY && 1032 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); 1033 1034 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); 1035 1036 spin_lock_irqsave(&ct->requests.lock, flags); 1037 list_for_each_entry(req, &ct->requests.pending, link) { 1038 if (unlikely(fence != req->fence)) { 1039 CT_DEBUG(ct, "request %u awaits response\n", 1040 req->fence); 1041 continue; 1042 } 1043 if (unlikely(datalen > req->response_len)) { 1044 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n", 1045 req->fence, datalen, req->response_len); 1046 datalen = min(datalen, req->response_len); 1047 err = -EMSGSIZE; 1048 } 1049 if (datalen) 1050 memcpy(req->response_buf, data, 4 * datalen); 1051 req->response_len = datalen; 1052 WRITE_ONCE(req->status, hxg[0]); 1053 found = true; 1054 break; 1055 } 1056 if (!found) { 1057 CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n", 1058 len, hxg[0], fence, ct->requests.last_fence); 1059 if (!ct_check_lost_and_found(ct, fence)) { 1060 list_for_each_entry(req, &ct->requests.pending, link) 1061 CT_ERROR(ct, "request %u awaits response\n", 1062 req->fence); 1063 } 1064 err = -ENOKEY; 1065 } 1066 spin_unlock_irqrestore(&ct->requests.lock, flags); 1067 1068 if (unlikely(err)) 1069 return err; 1070 1071 ct_free_msg(response); 1072 return 0; 1073 } 1074 1075 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1076 { 1077 struct intel_guc *guc = ct_to_guc(ct); 1078 const u32 *hxg; 1079 const u32 *payload; 1080 u32 hxg_len, action, len; 1081 int ret; 1082 1083 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1084 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; 1085 payload = &hxg[GUC_HXG_MSG_MIN_LEN]; 1086 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1087 len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1088 1089 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); 1090 1091 switch (action) { 1092 case INTEL_GUC_ACTION_DEFAULT: 1093 ret = intel_guc_to_host_process_recv_msg(guc, payload, len); 1094 break; 1095 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1096 ret = intel_guc_deregister_done_process_msg(guc, payload, 1097 len); 1098 break; 1099 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1100 ret = intel_guc_sched_done_process_msg(guc, payload, len); 1101 break; 1102 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1103 ret = intel_guc_context_reset_process_msg(guc, payload, len); 1104 break; 1105 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1106 ret = intel_guc_error_capture_process_msg(guc, payload, len); 1107 if (unlikely(ret)) 1108 CT_ERROR(ct, "error capture notification failed %x %*ph\n", 1109 action, 4 * len, payload); 1110 break; 1111 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1112 ret = intel_guc_engine_failure_process_msg(guc, payload, len); 1113 break; 1114 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1115 intel_guc_log_handle_flush_event(&guc->log); 1116 ret = 0; 1117 break; 1118 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1119 CT_ERROR(ct, "Received GuC crash dump notification!\n"); 1120 ret = 0; 1121 break; 1122 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION: 1123 CT_ERROR(ct, "Received GuC exception notification!\n"); 1124 ret = 0; 1125 break; 1126 default: 1127 ret = -EOPNOTSUPP; 1128 break; 1129 } 1130 1131 if (unlikely(ret)) { 1132 CT_ERROR(ct, "Failed to process request %04x (%pe)\n", 1133 action, ERR_PTR(ret)); 1134 return ret; 1135 } 1136 1137 ct_free_msg(request); 1138 return 0; 1139 } 1140 1141 static bool ct_process_incoming_requests(struct intel_guc_ct *ct) 1142 { 1143 unsigned long flags; 1144 struct ct_incoming_msg *request; 1145 bool done; 1146 int err; 1147 1148 spin_lock_irqsave(&ct->requests.lock, flags); 1149 request = list_first_entry_or_null(&ct->requests.incoming, 1150 struct ct_incoming_msg, link); 1151 if (request) 1152 list_del(&request->link); 1153 done = !!list_empty(&ct->requests.incoming); 1154 spin_unlock_irqrestore(&ct->requests.lock, flags); 1155 1156 if (!request) 1157 return true; 1158 1159 err = ct_process_request(ct, request); 1160 if (unlikely(err)) { 1161 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1162 ERR_PTR(err), 4 * request->size, request->msg); 1163 CT_DEAD(ct, PROCESS_FAILED); 1164 ct_free_msg(request); 1165 } 1166 1167 return done; 1168 } 1169 1170 static void ct_incoming_request_worker_func(struct work_struct *w) 1171 { 1172 struct intel_guc_ct *ct = 1173 container_of(w, struct intel_guc_ct, requests.worker); 1174 bool done; 1175 1176 do { 1177 done = ct_process_incoming_requests(ct); 1178 } while (!done); 1179 } 1180 1181 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1182 { 1183 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1184 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1185 unsigned long flags; 1186 1187 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); 1188 1189 /* 1190 * Adjusting the space must be done in IRQ or deadlock can occur as the 1191 * CTB processing in the below workqueue can send CTBs which creates a 1192 * circular dependency if the space was returned there. 1193 */ 1194 switch (action) { 1195 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1196 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1197 g2h_release_space(ct, request->size); 1198 } 1199 1200 spin_lock_irqsave(&ct->requests.lock, flags); 1201 list_add_tail(&request->link, &ct->requests.incoming); 1202 spin_unlock_irqrestore(&ct->requests.lock, flags); 1203 1204 queue_work(system_unbound_wq, &ct->requests.worker); 1205 return 0; 1206 } 1207 1208 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1209 { 1210 u32 origin, type; 1211 u32 *hxg; 1212 int err; 1213 1214 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) 1215 return -EBADMSG; 1216 1217 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; 1218 1219 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1220 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1221 err = -EPROTO; 1222 goto failed; 1223 } 1224 1225 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1226 switch (type) { 1227 case GUC_HXG_TYPE_EVENT: 1228 err = ct_handle_event(ct, msg); 1229 break; 1230 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1231 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1232 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1233 err = ct_handle_response(ct, msg); 1234 break; 1235 default: 1236 err = -EOPNOTSUPP; 1237 } 1238 1239 if (unlikely(err)) { 1240 failed: 1241 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", 1242 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); 1243 } 1244 return err; 1245 } 1246 1247 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1248 { 1249 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); 1250 int err; 1251 1252 if (format == GUC_CTB_FORMAT_HXG) 1253 err = ct_handle_hxg(ct, msg); 1254 else 1255 err = -EOPNOTSUPP; 1256 1257 if (unlikely(err)) { 1258 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1259 ERR_PTR(err), 4 * msg->size, msg->msg); 1260 ct_free_msg(msg); 1261 } 1262 } 1263 1264 /* 1265 * Return: number available remaining dwords to read (0 if empty) 1266 * or a negative error code on failure 1267 */ 1268 static int ct_receive(struct intel_guc_ct *ct) 1269 { 1270 struct ct_incoming_msg *msg = NULL; 1271 unsigned long flags; 1272 int ret; 1273 1274 spin_lock_irqsave(&ct->ctbs.recv.lock, flags); 1275 ret = ct_read(ct, &msg); 1276 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags); 1277 if (ret < 0) 1278 return ret; 1279 1280 if (msg) 1281 ct_handle_msg(ct, msg); 1282 1283 return ret; 1284 } 1285 1286 static void ct_try_receive_message(struct intel_guc_ct *ct) 1287 { 1288 int ret; 1289 1290 if (GEM_WARN_ON(!ct->enabled)) 1291 return; 1292 1293 ret = ct_receive(ct); 1294 if (ret > 0) 1295 tasklet_hi_schedule(&ct->receive_tasklet); 1296 } 1297 1298 static void ct_receive_tasklet_func(struct tasklet_struct *t) 1299 { 1300 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet); 1301 1302 ct_try_receive_message(ct); 1303 } 1304 1305 /* 1306 * When we're communicating with the GuC over CT, GuC uses events 1307 * to notify us about new messages being posted on the RECV buffer. 1308 */ 1309 void intel_guc_ct_event_handler(struct intel_guc_ct *ct) 1310 { 1311 if (unlikely(!ct->enabled)) { 1312 WARN(1, "Unexpected GuC event received while CT disabled!\n"); 1313 return; 1314 } 1315 1316 ct_try_receive_message(ct); 1317 } 1318 1319 void intel_guc_ct_print_info(struct intel_guc_ct *ct, 1320 struct drm_printer *p) 1321 { 1322 drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled)); 1323 1324 if (!ct->enabled) 1325 return; 1326 1327 drm_printf(p, "H2G Space: %u\n", 1328 atomic_read(&ct->ctbs.send.space) * 4); 1329 drm_printf(p, "Head: %u\n", 1330 ct->ctbs.send.desc->head); 1331 drm_printf(p, "Tail: %u\n", 1332 ct->ctbs.send.desc->tail); 1333 drm_printf(p, "G2H Space: %u\n", 1334 atomic_read(&ct->ctbs.recv.space) * 4); 1335 drm_printf(p, "Head: %u\n", 1336 ct->ctbs.recv.desc->head); 1337 drm_printf(p, "Tail: %u\n", 1338 ct->ctbs.recv.desc->tail); 1339 } 1340 1341 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 1342 static void ct_dead_ct_worker_func(struct work_struct *w) 1343 { 1344 struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker); 1345 struct intel_guc *guc = ct_to_guc(ct); 1346 1347 if (ct->dead_ct_reported) 1348 return; 1349 1350 ct->dead_ct_reported = true; 1351 1352 guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason); 1353 intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U); 1354 } 1355 #endif 1356