1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016-2019 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/ktime.h> 8 #include <linux/time64.h> 9 #include <linux/timekeeping.h> 10 11 #include "i915_drv.h" 12 #include "intel_guc_ct.h" 13 #include "gt/intel_gt.h" 14 15 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) 16 { 17 return container_of(ct, struct intel_guc, ct); 18 } 19 20 static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct) 21 { 22 return guc_to_gt(ct_to_guc(ct)); 23 } 24 25 static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct) 26 { 27 return ct_to_gt(ct)->i915; 28 } 29 30 static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct) 31 { 32 return &ct_to_i915(ct)->drm; 33 } 34 35 #define CT_ERROR(_ct, _fmt, ...) \ 36 drm_err(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) 37 #ifdef CONFIG_DRM_I915_DEBUG_GUC 38 #define CT_DEBUG(_ct, _fmt, ...) \ 39 drm_dbg(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) 40 #else 41 #define CT_DEBUG(...) do { } while (0) 42 #endif 43 #define CT_PROBE_ERROR(_ct, _fmt, ...) \ 44 i915_probe_error(ct_to_i915(ct), "CT: " _fmt, ##__VA_ARGS__) 45 46 /** 47 * DOC: CTB Blob 48 * 49 * We allocate single blob to hold both CTB descriptors and buffers: 50 * 51 * +--------+-----------------------------------------------+------+ 52 * | offset | contents | size | 53 * +========+===============================================+======+ 54 * | 0x0000 | H2G `CTB Descriptor`_ (send) | | 55 * +--------+-----------------------------------------------+ 4K | 56 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | | 57 * +--------+-----------------------------------------------+------+ 58 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K | 59 * | | | | 60 * +--------+-----------------------------------------------+------+ 61 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K | 62 * | + n*4K | | | 63 * +--------+-----------------------------------------------+------+ 64 * 65 * Size of each `CT Buffer`_ must be multiple of 4K. 66 * We don't expect too many messages in flight at any time, unless we are 67 * using the GuC submission. In that case each request requires a minimum 68 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 69 * enough space to avoid backpressure on the driver. We increase the size 70 * of the receive buffer (relative to the send) to ensure a G2H response 71 * CTB has a landing spot. 72 */ 73 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 74 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 75 #define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) 76 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) 77 78 struct ct_request { 79 struct list_head link; 80 u32 fence; 81 u32 status; 82 u32 response_len; 83 u32 *response_buf; 84 }; 85 86 struct ct_incoming_msg { 87 struct list_head link; 88 u32 size; 89 u32 msg[]; 90 }; 91 92 enum { CTB_SEND = 0, CTB_RECV = 1 }; 93 94 enum { CTB_OWNER_HOST = 0 }; 95 96 static void ct_receive_tasklet_func(struct tasklet_struct *t); 97 static void ct_incoming_request_worker_func(struct work_struct *w); 98 99 /** 100 * intel_guc_ct_init_early - Initialize CT state without requiring device access 101 * @ct: pointer to CT struct 102 */ 103 void intel_guc_ct_init_early(struct intel_guc_ct *ct) 104 { 105 spin_lock_init(&ct->ctbs.send.lock); 106 spin_lock_init(&ct->ctbs.recv.lock); 107 spin_lock_init(&ct->requests.lock); 108 INIT_LIST_HEAD(&ct->requests.pending); 109 INIT_LIST_HEAD(&ct->requests.incoming); 110 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); 111 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); 112 init_waitqueue_head(&ct->wq); 113 } 114 115 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) 116 { 117 memset(desc, 0, sizeof(*desc)); 118 } 119 120 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) 121 { 122 u32 space; 123 124 ctb->broken = false; 125 ctb->tail = 0; 126 ctb->head = 0; 127 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; 128 atomic_set(&ctb->space, space); 129 130 guc_ct_buffer_desc_init(ctb->desc); 131 } 132 133 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, 134 struct guc_ct_buffer_desc *desc, 135 u32 *cmds, u32 size_in_bytes, u32 resv_space) 136 { 137 GEM_BUG_ON(size_in_bytes % 4); 138 139 ctb->desc = desc; 140 ctb->cmds = cmds; 141 ctb->size = size_in_bytes / 4; 142 ctb->resv_space = resv_space / 4; 143 144 guc_ct_buffer_reset(ctb); 145 } 146 147 static int guc_action_control_ctb(struct intel_guc *guc, u32 control) 148 { 149 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 150 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 151 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 152 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB), 153 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control), 154 }; 155 int ret; 156 157 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE); 158 159 /* CT control must go over MMIO */ 160 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); 161 162 return ret > 0 ? -EPROTO : ret; 163 } 164 165 static int ct_control_enable(struct intel_guc_ct *ct, bool enable) 166 { 167 int err; 168 169 err = guc_action_control_ctb(ct_to_guc(ct), enable ? 170 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE); 171 if (unlikely(err)) 172 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n", 173 enabledisable(enable), ERR_PTR(err)); 174 175 return err; 176 } 177 178 static int ct_register_buffer(struct intel_guc_ct *ct, bool send, 179 u32 desc_addr, u32 buff_addr, u32 size) 180 { 181 int err; 182 183 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 184 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY : 185 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 186 desc_addr); 187 if (unlikely(err)) 188 goto failed; 189 190 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 191 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY : 192 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 193 buff_addr); 194 if (unlikely(err)) 195 goto failed; 196 197 err = intel_guc_self_cfg32(ct_to_guc(ct), send ? 198 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY : 199 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 200 size); 201 if (unlikely(err)) 202 failed: 203 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n", 204 send ? "SEND" : "RECV", ERR_PTR(err)); 205 206 return err; 207 } 208 209 /** 210 * intel_guc_ct_init - Init buffer-based communication 211 * @ct: pointer to CT struct 212 * 213 * Allocate memory required for buffer-based communication. 214 * 215 * Return: 0 on success, a negative errno code on failure. 216 */ 217 int intel_guc_ct_init(struct intel_guc_ct *ct) 218 { 219 struct intel_guc *guc = ct_to_guc(ct); 220 struct guc_ct_buffer_desc *desc; 221 u32 blob_size; 222 u32 cmds_size; 223 u32 resv_space; 224 void *blob; 225 u32 *cmds; 226 int err; 227 228 err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO); 229 if (err) 230 return err; 231 232 GEM_BUG_ON(ct->vma); 233 234 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; 235 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob); 236 if (unlikely(err)) { 237 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n", 238 blob_size, ERR_PTR(err)); 239 return err; 240 } 241 242 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size); 243 244 /* store pointers to desc and cmds for send ctb */ 245 desc = blob; 246 cmds = blob + 2 * CTB_DESC_SIZE; 247 cmds_size = CTB_H2G_BUFFER_SIZE; 248 resv_space = 0; 249 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", 250 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 251 resv_space); 252 253 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); 254 255 /* store pointers to desc and cmds for recv ctb */ 256 desc = blob + CTB_DESC_SIZE; 257 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; 258 cmds_size = CTB_G2H_BUFFER_SIZE; 259 resv_space = G2H_ROOM_BUFFER_SIZE; 260 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", 261 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 262 resv_space); 263 264 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); 265 266 return 0; 267 } 268 269 /** 270 * intel_guc_ct_fini - Fini buffer-based communication 271 * @ct: pointer to CT struct 272 * 273 * Deallocate memory required for buffer-based communication. 274 */ 275 void intel_guc_ct_fini(struct intel_guc_ct *ct) 276 { 277 GEM_BUG_ON(ct->enabled); 278 279 tasklet_kill(&ct->receive_tasklet); 280 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); 281 memset(ct, 0, sizeof(*ct)); 282 } 283 284 /** 285 * intel_guc_ct_enable - Enable buffer based command transport. 286 * @ct: pointer to CT struct 287 * 288 * Return: 0 on success, a negative errno code on failure. 289 */ 290 int intel_guc_ct_enable(struct intel_guc_ct *ct) 291 { 292 struct intel_guc *guc = ct_to_guc(ct); 293 u32 base, desc, cmds, size; 294 void *blob; 295 int err; 296 297 GEM_BUG_ON(ct->enabled); 298 299 /* vma should be already allocated and map'ed */ 300 GEM_BUG_ON(!ct->vma); 301 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj)); 302 base = intel_guc_ggtt_offset(guc, ct->vma); 303 304 /* blob should start with send descriptor */ 305 blob = __px_vaddr(ct->vma->obj); 306 GEM_BUG_ON(blob != ct->ctbs.send.desc); 307 308 /* (re)initialize descriptors */ 309 guc_ct_buffer_reset(&ct->ctbs.send); 310 guc_ct_buffer_reset(&ct->ctbs.recv); 311 312 /* 313 * Register both CT buffers starting with RECV buffer. 314 * Descriptors are in first half of the blob. 315 */ 316 desc = base + ptrdiff(ct->ctbs.recv.desc, blob); 317 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); 318 size = ct->ctbs.recv.size * 4; 319 err = ct_register_buffer(ct, false, desc, cmds, size); 320 if (unlikely(err)) 321 goto err_out; 322 323 desc = base + ptrdiff(ct->ctbs.send.desc, blob); 324 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); 325 size = ct->ctbs.send.size * 4; 326 err = ct_register_buffer(ct, true, desc, cmds, size); 327 if (unlikely(err)) 328 goto err_out; 329 330 err = ct_control_enable(ct, true); 331 if (unlikely(err)) 332 goto err_out; 333 334 ct->enabled = true; 335 ct->stall_time = KTIME_MAX; 336 337 return 0; 338 339 err_out: 340 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); 341 return err; 342 } 343 344 /** 345 * intel_guc_ct_disable - Disable buffer based command transport. 346 * @ct: pointer to CT struct 347 */ 348 void intel_guc_ct_disable(struct intel_guc_ct *ct) 349 { 350 struct intel_guc *guc = ct_to_guc(ct); 351 352 GEM_BUG_ON(!ct->enabled); 353 354 ct->enabled = false; 355 356 if (intel_guc_is_fw_running(guc)) { 357 ct_control_enable(ct, false); 358 } 359 } 360 361 static u32 ct_get_next_fence(struct intel_guc_ct *ct) 362 { 363 /* For now it's trivial */ 364 return ++ct->requests.last_fence; 365 } 366 367 static int ct_write(struct intel_guc_ct *ct, 368 const u32 *action, 369 u32 len /* in dwords */, 370 u32 fence, u32 flags) 371 { 372 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 373 struct guc_ct_buffer_desc *desc = ctb->desc; 374 u32 tail = ctb->tail; 375 u32 size = ctb->size; 376 u32 header; 377 u32 hxg; 378 u32 type; 379 u32 *cmds = ctb->cmds; 380 unsigned int i; 381 382 if (unlikely(desc->status)) 383 goto corrupted; 384 385 GEM_BUG_ON(tail > size); 386 387 #ifdef CONFIG_DRM_I915_DEBUG_GUC 388 if (unlikely(tail != READ_ONCE(desc->tail))) { 389 CT_ERROR(ct, "Tail was modified %u != %u\n", 390 desc->tail, tail); 391 desc->status |= GUC_CTB_STATUS_MISMATCH; 392 goto corrupted; 393 } 394 if (unlikely(READ_ONCE(desc->head) >= size)) { 395 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 396 desc->head, size); 397 desc->status |= GUC_CTB_STATUS_OVERFLOW; 398 goto corrupted; 399 } 400 #endif 401 402 /* 403 * dw0: CT header (including fence) 404 * dw1: HXG header (including action code) 405 * dw2+: action data 406 */ 407 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 408 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 409 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); 410 411 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT : 412 GUC_HXG_TYPE_REQUEST; 413 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | 414 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 415 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 416 417 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", 418 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); 419 420 cmds[tail] = header; 421 tail = (tail + 1) % size; 422 423 cmds[tail] = hxg; 424 tail = (tail + 1) % size; 425 426 for (i = 1; i < len; i++) { 427 cmds[tail] = action[i]; 428 tail = (tail + 1) % size; 429 } 430 GEM_BUG_ON(tail > size); 431 432 /* 433 * make sure H2G buffer update and LRC tail update (if this triggering a 434 * submission) are visible before updating the descriptor tail 435 */ 436 intel_guc_write_barrier(ct_to_guc(ct)); 437 438 /* update local copies */ 439 ctb->tail = tail; 440 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); 441 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); 442 443 /* now update descriptor */ 444 WRITE_ONCE(desc->tail, tail); 445 446 return 0; 447 448 corrupted: 449 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 450 desc->head, desc->tail, desc->status); 451 ctb->broken = true; 452 return -EPIPE; 453 } 454 455 /** 456 * wait_for_ct_request_update - Wait for CT request state update. 457 * @req: pointer to pending request 458 * @status: placeholder for status 459 * 460 * For each sent request, GuC shall send back CT response message. 461 * Our message handler will update status of tracked request once 462 * response message with given fence is received. Wait here and 463 * check for valid response status value. 464 * 465 * Return: 466 * * 0 response received (status is valid) 467 * * -ETIMEDOUT no response within hardcoded timeout 468 */ 469 static int wait_for_ct_request_update(struct ct_request *req, u32 *status) 470 { 471 int err; 472 473 /* 474 * Fast commands should complete in less than 10us, so sample quickly 475 * up to that length of time, then switch to a slower sleep-wait loop. 476 * No GuC command should ever take longer than 10ms but many GuC 477 * commands can be inflight at time, so use a 1s timeout on the slower 478 * sleep-wait loop. 479 */ 480 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 481 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 482 #define done \ 483 (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ 484 GUC_HXG_ORIGIN_GUC) 485 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); 486 if (err) 487 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); 488 #undef done 489 490 *status = req->status; 491 return err; 492 } 493 494 #define GUC_CTB_TIMEOUT_MS 1500 495 static inline bool ct_deadlocked(struct intel_guc_ct *ct) 496 { 497 long timeout = GUC_CTB_TIMEOUT_MS; 498 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; 499 500 if (unlikely(ret)) { 501 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; 502 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; 503 504 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", 505 ktime_ms_delta(ktime_get(), ct->stall_time), 506 send->status, recv->status); 507 CT_ERROR(ct, "H2G Space: %u (Bytes)\n", 508 atomic_read(&ct->ctbs.send.space) * 4); 509 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head); 510 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail); 511 CT_ERROR(ct, "G2H Space: %u (Bytes)\n", 512 atomic_read(&ct->ctbs.recv.space) * 4); 513 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); 514 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); 515 516 ct->ctbs.send.broken = true; 517 } 518 519 return ret; 520 } 521 522 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) 523 { 524 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 525 526 /* 527 * We leave a certain amount of space in the G2H CTB buffer for 528 * unexpected G2H CTBs (e.g. logging, engine hang, etc...) 529 */ 530 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; 531 } 532 533 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 534 { 535 lockdep_assert_held(&ct->ctbs.send.lock); 536 537 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); 538 539 if (g2h_len_dw) 540 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); 541 } 542 543 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 544 { 545 atomic_add(g2h_len_dw, &ct->ctbs.recv.space); 546 } 547 548 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) 549 { 550 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 551 struct guc_ct_buffer_desc *desc = ctb->desc; 552 u32 head; 553 u32 space; 554 555 if (atomic_read(&ctb->space) >= len_dw) 556 return true; 557 558 head = READ_ONCE(desc->head); 559 if (unlikely(head > ctb->size)) { 560 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 561 head, ctb->size); 562 desc->status |= GUC_CTB_STATUS_OVERFLOW; 563 ctb->broken = true; 564 return false; 565 } 566 567 space = CIRC_SPACE(ctb->tail, head, ctb->size); 568 atomic_set(&ctb->space, space); 569 570 return space >= len_dw; 571 } 572 573 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) 574 { 575 bool h2g = h2g_has_room(ct, h2g_dw); 576 bool g2h = g2h_has_room(ct, g2h_dw); 577 578 lockdep_assert_held(&ct->ctbs.send.lock); 579 580 if (unlikely(!h2g || !g2h)) { 581 if (ct->stall_time == KTIME_MAX) 582 ct->stall_time = ktime_get(); 583 584 /* Be paranoid and kick G2H tasklet to free credits */ 585 if (!g2h) 586 tasklet_hi_schedule(&ct->receive_tasklet); 587 588 if (unlikely(ct_deadlocked(ct))) 589 return -EPIPE; 590 else 591 return -EBUSY; 592 } 593 594 ct->stall_time = KTIME_MAX; 595 return 0; 596 } 597 598 #define G2H_LEN_DW(f) ({ \ 599 typeof(f) f_ = (f); \ 600 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ 601 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ 602 GUC_CTB_HXG_MSG_MIN_LEN : 0; \ 603 }) 604 static int ct_send_nb(struct intel_guc_ct *ct, 605 const u32 *action, 606 u32 len, 607 u32 flags) 608 { 609 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 610 unsigned long spin_flags; 611 u32 g2h_len_dw = G2H_LEN_DW(flags); 612 u32 fence; 613 int ret; 614 615 spin_lock_irqsave(&ctb->lock, spin_flags); 616 617 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); 618 if (unlikely(ret)) 619 goto out; 620 621 fence = ct_get_next_fence(ct); 622 ret = ct_write(ct, action, len, fence, flags); 623 if (unlikely(ret)) 624 goto out; 625 626 g2h_reserve_space(ct, g2h_len_dw); 627 intel_guc_notify(ct_to_guc(ct)); 628 629 out: 630 spin_unlock_irqrestore(&ctb->lock, spin_flags); 631 632 return ret; 633 } 634 635 static int ct_send(struct intel_guc_ct *ct, 636 const u32 *action, 637 u32 len, 638 u32 *response_buf, 639 u32 response_buf_size, 640 u32 *status) 641 { 642 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 643 struct ct_request request; 644 unsigned long flags; 645 unsigned int sleep_period_ms = 1; 646 bool send_again; 647 u32 fence; 648 int err; 649 650 GEM_BUG_ON(!ct->enabled); 651 GEM_BUG_ON(!len); 652 GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK); 653 GEM_BUG_ON(!response_buf && response_buf_size); 654 might_sleep(); 655 656 resend: 657 send_again = false; 658 659 /* 660 * We use a lazy spin wait loop here as we believe that if the CT 661 * buffers are sized correctly the flow control condition should be 662 * rare. Reserving the maximum size in the G2H credits as we don't know 663 * how big the response is going to be. 664 */ 665 retry: 666 spin_lock_irqsave(&ctb->lock, flags); 667 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || 668 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { 669 if (ct->stall_time == KTIME_MAX) 670 ct->stall_time = ktime_get(); 671 spin_unlock_irqrestore(&ctb->lock, flags); 672 673 if (unlikely(ct_deadlocked(ct))) 674 return -EPIPE; 675 676 if (msleep_interruptible(sleep_period_ms)) 677 return -EINTR; 678 sleep_period_ms = sleep_period_ms << 1; 679 680 goto retry; 681 } 682 683 ct->stall_time = KTIME_MAX; 684 685 fence = ct_get_next_fence(ct); 686 request.fence = fence; 687 request.status = 0; 688 request.response_len = response_buf_size; 689 request.response_buf = response_buf; 690 691 spin_lock(&ct->requests.lock); 692 list_add_tail(&request.link, &ct->requests.pending); 693 spin_unlock(&ct->requests.lock); 694 695 err = ct_write(ct, action, len, fence, 0); 696 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 697 698 spin_unlock_irqrestore(&ctb->lock, flags); 699 700 if (unlikely(err)) 701 goto unlink; 702 703 intel_guc_notify(ct_to_guc(ct)); 704 705 err = wait_for_ct_request_update(&request, status); 706 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 707 if (unlikely(err)) { 708 CT_ERROR(ct, "No response for request %#x (fence %u)\n", 709 action[0], request.fence); 710 goto unlink; 711 } 712 713 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 714 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action, 715 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status)); 716 send_again = true; 717 goto unlink; 718 } 719 720 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { 721 err = -EIO; 722 goto unlink; 723 } 724 725 if (response_buf) { 726 /* There shall be no data in the status */ 727 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); 728 /* Return actual response len */ 729 err = request.response_len; 730 } else { 731 /* There shall be no response payload */ 732 WARN_ON(request.response_len); 733 /* Return data decoded from the status dword */ 734 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); 735 } 736 737 unlink: 738 spin_lock_irqsave(&ct->requests.lock, flags); 739 list_del(&request.link); 740 spin_unlock_irqrestore(&ct->requests.lock, flags); 741 742 if (unlikely(send_again)) 743 goto resend; 744 745 return err; 746 } 747 748 /* 749 * Command Transport (CT) buffer based GuC send function. 750 */ 751 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, 752 u32 *response_buf, u32 response_buf_size, u32 flags) 753 { 754 u32 status = ~0; /* undefined */ 755 int ret; 756 757 if (unlikely(!ct->enabled)) { 758 struct intel_guc *guc = ct_to_guc(ct); 759 struct intel_uc *uc = container_of(guc, struct intel_uc, guc); 760 761 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); 762 return -ENODEV; 763 } 764 765 if (unlikely(ct->ctbs.send.broken)) 766 return -EPIPE; 767 768 if (flags & INTEL_GUC_CT_SEND_NB) 769 return ct_send_nb(ct, action, len, flags); 770 771 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); 772 if (unlikely(ret < 0)) { 773 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", 774 action[0], ERR_PTR(ret), status); 775 } else if (unlikely(ret)) { 776 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", 777 action[0], ret, ret); 778 } 779 780 return ret; 781 } 782 783 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) 784 { 785 struct ct_incoming_msg *msg; 786 787 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC); 788 if (msg) 789 msg->size = num_dwords; 790 return msg; 791 } 792 793 static void ct_free_msg(struct ct_incoming_msg *msg) 794 { 795 kfree(msg); 796 } 797 798 /* 799 * Return: number available remaining dwords to read (0 if empty) 800 * or a negative error code on failure 801 */ 802 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) 803 { 804 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 805 struct guc_ct_buffer_desc *desc = ctb->desc; 806 u32 head = ctb->head; 807 u32 tail = READ_ONCE(desc->tail); 808 u32 size = ctb->size; 809 u32 *cmds = ctb->cmds; 810 s32 available; 811 unsigned int len; 812 unsigned int i; 813 u32 header; 814 815 if (unlikely(ctb->broken)) 816 return -EPIPE; 817 818 if (unlikely(desc->status)) 819 goto corrupted; 820 821 GEM_BUG_ON(head > size); 822 823 #ifdef CONFIG_DRM_I915_DEBUG_GUC 824 if (unlikely(head != READ_ONCE(desc->head))) { 825 CT_ERROR(ct, "Head was modified %u != %u\n", 826 desc->head, head); 827 desc->status |= GUC_CTB_STATUS_MISMATCH; 828 goto corrupted; 829 } 830 #endif 831 if (unlikely(tail >= size)) { 832 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", 833 tail, size); 834 desc->status |= GUC_CTB_STATUS_OVERFLOW; 835 goto corrupted; 836 } 837 838 /* tail == head condition indicates empty */ 839 available = tail - head; 840 if (unlikely(available == 0)) { 841 *msg = NULL; 842 return 0; 843 } 844 845 /* beware of buffer wrap case */ 846 if (unlikely(available < 0)) 847 available += size; 848 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); 849 GEM_BUG_ON(available < 0); 850 851 header = cmds[head]; 852 head = (head + 1) % size; 853 854 /* message len with header */ 855 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; 856 if (unlikely(len > (u32)available)) { 857 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 858 4, &header, 859 4 * (head + available - 1 > size ? 860 size - head : available - 1), &cmds[head], 861 4 * (head + available - 1 > size ? 862 available - 1 - size + head : 0), &cmds[0]); 863 desc->status |= GUC_CTB_STATUS_UNDERFLOW; 864 goto corrupted; 865 } 866 867 *msg = ct_alloc_msg(len); 868 if (!*msg) { 869 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n", 870 4, &header, 871 4 * (head + available - 1 > size ? 872 size - head : available - 1), &cmds[head], 873 4 * (head + available - 1 > size ? 874 available - 1 - size + head : 0), &cmds[0]); 875 return available; 876 } 877 878 (*msg)->msg[0] = header; 879 880 for (i = 1; i < len; i++) { 881 (*msg)->msg[i] = cmds[head]; 882 head = (head + 1) % size; 883 } 884 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); 885 886 /* update local copies */ 887 ctb->head = head; 888 889 /* now update descriptor */ 890 WRITE_ONCE(desc->head, head); 891 892 return available - len; 893 894 corrupted: 895 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 896 desc->head, desc->tail, desc->status); 897 ctb->broken = true; 898 return -EPIPE; 899 } 900 901 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) 902 { 903 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); 904 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); 905 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; 906 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; 907 u32 datalen = len - GUC_HXG_MSG_MIN_LEN; 908 struct ct_request *req; 909 unsigned long flags; 910 bool found = false; 911 int err = 0; 912 913 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); 914 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); 915 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && 916 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY && 917 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); 918 919 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); 920 921 spin_lock_irqsave(&ct->requests.lock, flags); 922 list_for_each_entry(req, &ct->requests.pending, link) { 923 if (unlikely(fence != req->fence)) { 924 CT_DEBUG(ct, "request %u awaits response\n", 925 req->fence); 926 continue; 927 } 928 if (unlikely(datalen > req->response_len)) { 929 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n", 930 req->fence, datalen, req->response_len); 931 datalen = min(datalen, req->response_len); 932 err = -EMSGSIZE; 933 } 934 if (datalen) 935 memcpy(req->response_buf, data, 4 * datalen); 936 req->response_len = datalen; 937 WRITE_ONCE(req->status, hxg[0]); 938 found = true; 939 break; 940 } 941 if (!found) { 942 CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence); 943 CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence, 944 ct->requests.last_fence); 945 list_for_each_entry(req, &ct->requests.pending, link) 946 CT_ERROR(ct, "request %u awaits response\n", 947 req->fence); 948 err = -ENOKEY; 949 } 950 spin_unlock_irqrestore(&ct->requests.lock, flags); 951 952 if (unlikely(err)) 953 return err; 954 955 ct_free_msg(response); 956 return 0; 957 } 958 959 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 960 { 961 struct intel_guc *guc = ct_to_guc(ct); 962 const u32 *hxg; 963 const u32 *payload; 964 u32 hxg_len, action, len; 965 int ret; 966 967 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 968 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; 969 payload = &hxg[GUC_HXG_MSG_MIN_LEN]; 970 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 971 len = hxg_len - GUC_HXG_MSG_MIN_LEN; 972 973 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); 974 975 switch (action) { 976 case INTEL_GUC_ACTION_DEFAULT: 977 ret = intel_guc_to_host_process_recv_msg(guc, payload, len); 978 break; 979 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 980 ret = intel_guc_deregister_done_process_msg(guc, payload, 981 len); 982 break; 983 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 984 ret = intel_guc_sched_done_process_msg(guc, payload, len); 985 break; 986 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 987 ret = intel_guc_context_reset_process_msg(guc, payload, len); 988 break; 989 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 990 ret = intel_guc_error_capture_process_msg(guc, payload, len); 991 if (unlikely(ret)) 992 CT_ERROR(ct, "error capture notification failed %x %*ph\n", 993 action, 4 * len, payload); 994 break; 995 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 996 ret = intel_guc_engine_failure_process_msg(guc, payload, len); 997 break; 998 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 999 intel_guc_log_handle_flush_event(&guc->log); 1000 ret = 0; 1001 break; 1002 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1003 CT_ERROR(ct, "Received GuC crash dump notification!\n"); 1004 ret = 0; 1005 break; 1006 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION: 1007 CT_ERROR(ct, "Received GuC exception notification!\n"); 1008 ret = 0; 1009 break; 1010 default: 1011 ret = -EOPNOTSUPP; 1012 break; 1013 } 1014 1015 if (unlikely(ret)) { 1016 CT_ERROR(ct, "Failed to process request %04x (%pe)\n", 1017 action, ERR_PTR(ret)); 1018 return ret; 1019 } 1020 1021 ct_free_msg(request); 1022 return 0; 1023 } 1024 1025 static bool ct_process_incoming_requests(struct intel_guc_ct *ct) 1026 { 1027 unsigned long flags; 1028 struct ct_incoming_msg *request; 1029 bool done; 1030 int err; 1031 1032 spin_lock_irqsave(&ct->requests.lock, flags); 1033 request = list_first_entry_or_null(&ct->requests.incoming, 1034 struct ct_incoming_msg, link); 1035 if (request) 1036 list_del(&request->link); 1037 done = !!list_empty(&ct->requests.incoming); 1038 spin_unlock_irqrestore(&ct->requests.lock, flags); 1039 1040 if (!request) 1041 return true; 1042 1043 err = ct_process_request(ct, request); 1044 if (unlikely(err)) { 1045 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1046 ERR_PTR(err), 4 * request->size, request->msg); 1047 ct_free_msg(request); 1048 } 1049 1050 return done; 1051 } 1052 1053 static void ct_incoming_request_worker_func(struct work_struct *w) 1054 { 1055 struct intel_guc_ct *ct = 1056 container_of(w, struct intel_guc_ct, requests.worker); 1057 bool done; 1058 1059 do { 1060 done = ct_process_incoming_requests(ct); 1061 } while (!done); 1062 } 1063 1064 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1065 { 1066 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1067 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1068 unsigned long flags; 1069 1070 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); 1071 1072 /* 1073 * Adjusting the space must be done in IRQ or deadlock can occur as the 1074 * CTB processing in the below workqueue can send CTBs which creates a 1075 * circular dependency if the space was returned there. 1076 */ 1077 switch (action) { 1078 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1079 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1080 g2h_release_space(ct, request->size); 1081 } 1082 1083 spin_lock_irqsave(&ct->requests.lock, flags); 1084 list_add_tail(&request->link, &ct->requests.incoming); 1085 spin_unlock_irqrestore(&ct->requests.lock, flags); 1086 1087 queue_work(system_unbound_wq, &ct->requests.worker); 1088 return 0; 1089 } 1090 1091 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1092 { 1093 u32 origin, type; 1094 u32 *hxg; 1095 int err; 1096 1097 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) 1098 return -EBADMSG; 1099 1100 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; 1101 1102 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1103 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1104 err = -EPROTO; 1105 goto failed; 1106 } 1107 1108 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1109 switch (type) { 1110 case GUC_HXG_TYPE_EVENT: 1111 err = ct_handle_event(ct, msg); 1112 break; 1113 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1114 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1115 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1116 err = ct_handle_response(ct, msg); 1117 break; 1118 default: 1119 err = -EOPNOTSUPP; 1120 } 1121 1122 if (unlikely(err)) { 1123 failed: 1124 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", 1125 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); 1126 } 1127 return err; 1128 } 1129 1130 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1131 { 1132 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); 1133 int err; 1134 1135 if (format == GUC_CTB_FORMAT_HXG) 1136 err = ct_handle_hxg(ct, msg); 1137 else 1138 err = -EOPNOTSUPP; 1139 1140 if (unlikely(err)) { 1141 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1142 ERR_PTR(err), 4 * msg->size, msg->msg); 1143 ct_free_msg(msg); 1144 } 1145 } 1146 1147 /* 1148 * Return: number available remaining dwords to read (0 if empty) 1149 * or a negative error code on failure 1150 */ 1151 static int ct_receive(struct intel_guc_ct *ct) 1152 { 1153 struct ct_incoming_msg *msg = NULL; 1154 unsigned long flags; 1155 int ret; 1156 1157 spin_lock_irqsave(&ct->ctbs.recv.lock, flags); 1158 ret = ct_read(ct, &msg); 1159 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags); 1160 if (ret < 0) 1161 return ret; 1162 1163 if (msg) 1164 ct_handle_msg(ct, msg); 1165 1166 return ret; 1167 } 1168 1169 static void ct_try_receive_message(struct intel_guc_ct *ct) 1170 { 1171 int ret; 1172 1173 if (GEM_WARN_ON(!ct->enabled)) 1174 return; 1175 1176 ret = ct_receive(ct); 1177 if (ret > 0) 1178 tasklet_hi_schedule(&ct->receive_tasklet); 1179 } 1180 1181 static void ct_receive_tasklet_func(struct tasklet_struct *t) 1182 { 1183 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet); 1184 1185 ct_try_receive_message(ct); 1186 } 1187 1188 /* 1189 * When we're communicating with the GuC over CT, GuC uses events 1190 * to notify us about new messages being posted on the RECV buffer. 1191 */ 1192 void intel_guc_ct_event_handler(struct intel_guc_ct *ct) 1193 { 1194 if (unlikely(!ct->enabled)) { 1195 WARN(1, "Unexpected GuC event received while CT disabled!\n"); 1196 return; 1197 } 1198 1199 ct_try_receive_message(ct); 1200 } 1201 1202 void intel_guc_ct_print_info(struct intel_guc_ct *ct, 1203 struct drm_printer *p) 1204 { 1205 drm_printf(p, "CT %s\n", enableddisabled(ct->enabled)); 1206 1207 if (!ct->enabled) 1208 return; 1209 1210 drm_printf(p, "H2G Space: %u\n", 1211 atomic_read(&ct->ctbs.send.space) * 4); 1212 drm_printf(p, "Head: %u\n", 1213 ct->ctbs.send.desc->head); 1214 drm_printf(p, "Tail: %u\n", 1215 ct->ctbs.send.desc->tail); 1216 drm_printf(p, "G2H Space: %u\n", 1217 atomic_read(&ct->ctbs.recv.space) * 4); 1218 drm_printf(p, "Head: %u\n", 1219 ct->ctbs.recv.desc->head); 1220 drm_printf(p, "Tail: %u\n", 1221 ct->ctbs.recv.desc->tail); 1222 } 1223