1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2017 Linaro Ltd. 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "hfi_cmds.h" 17 #include "hfi_msgs.h" 18 #include "hfi_venus.h" 19 #include "hfi_venus_io.h" 20 #include "firmware.h" 21 22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000 23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000 24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00 25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff 26 27 #define HFI_HOST_TO_CTRL_CMD_Q 0 28 #define HFI_CTRL_TO_HOST_MSG_Q 1 29 #define HFI_CTRL_TO_HOST_DBG_Q 2 30 #define HFI_MASK_QHDR_STATUS 0x000000ff 31 32 #define IFACEQ_NUM 3 33 #define IFACEQ_CMD_IDX 0 34 #define IFACEQ_MSG_IDX 1 35 #define IFACEQ_DBG_IDX 2 36 #define IFACEQ_MAX_BUF_COUNT 50 37 #define IFACEQ_MAX_PARALLEL_CLNTS 16 38 #define IFACEQ_DFLT_QHDR 0x01010000 39 40 #define POLL_INTERVAL_US 50 41 42 #define IFACEQ_MAX_PKT_SIZE 1024 43 #define IFACEQ_MED_PKT_SIZE 768 44 #define IFACEQ_MIN_PKT_SIZE 8 45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100 46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512 47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12) 48 49 struct hfi_queue_table_header { 50 u32 version; 51 u32 size; 52 u32 qhdr0_offset; 53 u32 qhdr_size; 54 u32 num_q; 55 u32 num_active_q; 56 }; 57 58 struct hfi_queue_header { 59 u32 status; 60 u32 start_addr; 61 u32 type; 62 u32 q_size; 63 u32 pkt_size; 64 u32 pkt_drop_cnt; 65 u32 rx_wm; 66 u32 tx_wm; 67 u32 rx_req; 68 u32 tx_req; 69 u32 rx_irq_status; 70 u32 tx_irq_status; 71 u32 read_idx; 72 u32 write_idx; 73 }; 74 75 #define IFACEQ_TABLE_SIZE \ 76 (sizeof(struct hfi_queue_table_header) + \ 77 sizeof(struct hfi_queue_header) * IFACEQ_NUM) 78 79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \ 80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS) 81 82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \ 83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \ 84 ((i) * sizeof(struct hfi_queue_header))) 85 86 #define QDSS_SIZE SZ_4K 87 #define SFR_SIZE SZ_4K 88 #define QUEUE_SIZE \ 89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM)) 90 91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K) 92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K) 93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K) 94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \ 95 ALIGNED_QDSS_SIZE, SZ_1M) 96 97 struct mem_desc { 98 dma_addr_t da; /* device address */ 99 void *kva; /* kernel virtual address */ 100 u32 size; 101 unsigned long attrs; 102 }; 103 104 struct iface_queue { 105 struct hfi_queue_header *qhdr; 106 struct mem_desc qmem; 107 }; 108 109 enum venus_state { 110 VENUS_STATE_DEINIT = 1, 111 VENUS_STATE_INIT, 112 }; 113 114 struct venus_hfi_device { 115 struct venus_core *core; 116 u32 irq_status; 117 u32 last_packet_type; 118 bool power_enabled; 119 bool suspended; 120 enum venus_state state; 121 /* serialize read / write to the shared memory */ 122 struct mutex lock; 123 struct completion pwr_collapse_prep; 124 struct completion release_resource; 125 struct mem_desc ifaceq_table; 126 struct mem_desc sfr; 127 struct iface_queue queues[IFACEQ_NUM]; 128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 130 }; 131 132 static bool venus_pkt_debug; 133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL; 134 static bool venus_sys_idle_indicator; 135 static bool venus_fw_low_power_mode = true; 136 static int venus_hw_rsp_timeout = 1000; 137 static bool venus_fw_coverage; 138 139 static void venus_set_state(struct venus_hfi_device *hdev, 140 enum venus_state state) 141 { 142 mutex_lock(&hdev->lock); 143 hdev->state = state; 144 mutex_unlock(&hdev->lock); 145 } 146 147 static bool venus_is_valid_state(struct venus_hfi_device *hdev) 148 { 149 return hdev->state != VENUS_STATE_DEINIT; 150 } 151 152 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet) 153 { 154 size_t pkt_size = *(u32 *)packet; 155 156 if (!venus_pkt_debug) 157 return; 158 159 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet, 160 pkt_size, true); 161 } 162 163 static int venus_write_queue(struct venus_hfi_device *hdev, 164 struct iface_queue *queue, 165 void *packet, u32 *rx_req) 166 { 167 struct hfi_queue_header *qhdr; 168 u32 dwords, new_wr_idx; 169 u32 empty_space, rd_idx, wr_idx, qsize; 170 u32 *wr_ptr; 171 172 if (!queue->qmem.kva) 173 return -EINVAL; 174 175 qhdr = queue->qhdr; 176 if (!qhdr) 177 return -EINVAL; 178 179 venus_dump_packet(hdev, packet); 180 181 dwords = (*(u32 *)packet) >> 2; 182 if (!dwords) 183 return -EINVAL; 184 185 rd_idx = qhdr->read_idx; 186 wr_idx = qhdr->write_idx; 187 qsize = qhdr->q_size; 188 /* ensure rd/wr indices's are read from memory */ 189 rmb(); 190 191 if (wr_idx >= rd_idx) 192 empty_space = qsize - (wr_idx - rd_idx); 193 else 194 empty_space = rd_idx - wr_idx; 195 196 if (empty_space <= dwords) { 197 qhdr->tx_req = 1; 198 /* ensure tx_req is updated in memory */ 199 wmb(); 200 return -ENOSPC; 201 } 202 203 qhdr->tx_req = 0; 204 /* ensure tx_req is updated in memory */ 205 wmb(); 206 207 new_wr_idx = wr_idx + dwords; 208 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2)); 209 if (new_wr_idx < qsize) { 210 memcpy(wr_ptr, packet, dwords << 2); 211 } else { 212 size_t len; 213 214 new_wr_idx -= qsize; 215 len = (dwords - new_wr_idx) << 2; 216 memcpy(wr_ptr, packet, len); 217 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2); 218 } 219 220 /* make sure packet is written before updating the write index */ 221 wmb(); 222 223 qhdr->write_idx = new_wr_idx; 224 *rx_req = qhdr->rx_req ? 1 : 0; 225 226 /* make sure write index is updated before an interrupt is raised */ 227 mb(); 228 229 return 0; 230 } 231 232 static int venus_read_queue(struct venus_hfi_device *hdev, 233 struct iface_queue *queue, void *pkt, u32 *tx_req) 234 { 235 struct hfi_queue_header *qhdr; 236 u32 dwords, new_rd_idx; 237 u32 rd_idx, wr_idx, type, qsize; 238 u32 *rd_ptr; 239 u32 recv_request = 0; 240 int ret = 0; 241 242 if (!queue->qmem.kva) 243 return -EINVAL; 244 245 qhdr = queue->qhdr; 246 if (!qhdr) 247 return -EINVAL; 248 249 type = qhdr->type; 250 rd_idx = qhdr->read_idx; 251 wr_idx = qhdr->write_idx; 252 qsize = qhdr->q_size; 253 254 /* make sure data is valid before using it */ 255 rmb(); 256 257 /* 258 * Do not set receive request for debug queue, if set, Venus generates 259 * interrupt for debug messages even when there is no response message 260 * available. In general debug queue will not become full as it is being 261 * emptied out for every interrupt from Venus. Venus will anyway 262 * generates interrupt if it is full. 263 */ 264 if (type & HFI_CTRL_TO_HOST_MSG_Q) 265 recv_request = 1; 266 267 if (rd_idx == wr_idx) { 268 qhdr->rx_req = recv_request; 269 *tx_req = 0; 270 /* update rx_req field in memory */ 271 wmb(); 272 return -ENODATA; 273 } 274 275 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2)); 276 dwords = *rd_ptr >> 2; 277 if (!dwords) 278 return -EINVAL; 279 280 new_rd_idx = rd_idx + dwords; 281 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) { 282 if (new_rd_idx < qsize) { 283 memcpy(pkt, rd_ptr, dwords << 2); 284 } else { 285 size_t len; 286 287 new_rd_idx -= qsize; 288 len = (dwords - new_rd_idx) << 2; 289 memcpy(pkt, rd_ptr, len); 290 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2); 291 } 292 } else { 293 /* bad packet received, dropping */ 294 new_rd_idx = qhdr->write_idx; 295 ret = -EBADMSG; 296 } 297 298 /* ensure the packet is read before updating read index */ 299 rmb(); 300 301 qhdr->read_idx = new_rd_idx; 302 /* ensure updating read index */ 303 wmb(); 304 305 rd_idx = qhdr->read_idx; 306 wr_idx = qhdr->write_idx; 307 /* ensure rd/wr indices are read from memory */ 308 rmb(); 309 310 if (rd_idx != wr_idx) 311 qhdr->rx_req = 0; 312 else 313 qhdr->rx_req = recv_request; 314 315 *tx_req = qhdr->tx_req ? 1 : 0; 316 317 /* ensure rx_req is stored to memory and tx_req is loaded from memory */ 318 mb(); 319 320 venus_dump_packet(hdev, pkt); 321 322 return ret; 323 } 324 325 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc, 326 u32 size) 327 { 328 struct device *dev = hdev->core->dev; 329 330 desc->attrs = DMA_ATTR_WRITE_COMBINE; 331 desc->size = ALIGN(size, SZ_4K); 332 333 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL, 334 desc->attrs); 335 if (!desc->kva) 336 return -ENOMEM; 337 338 return 0; 339 } 340 341 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem) 342 { 343 struct device *dev = hdev->core->dev; 344 345 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs); 346 } 347 348 static void venus_writel(struct venus_hfi_device *hdev, u32 reg, u32 value) 349 { 350 writel(value, hdev->core->base + reg); 351 } 352 353 static u32 venus_readl(struct venus_hfi_device *hdev, u32 reg) 354 { 355 return readl(hdev->core->base + reg); 356 } 357 358 static void venus_set_registers(struct venus_hfi_device *hdev) 359 { 360 const struct venus_resources *res = hdev->core->res; 361 const struct reg_val *tbl = res->reg_tbl; 362 unsigned int count = res->reg_tbl_size; 363 unsigned int i; 364 365 for (i = 0; i < count; i++) 366 venus_writel(hdev, tbl[i].reg, tbl[i].value); 367 } 368 369 static void venus_soft_int(struct venus_hfi_device *hdev) 370 { 371 venus_writel(hdev, CPU_IC_SOFTINT, BIT(CPU_IC_SOFTINT_H2A_SHIFT)); 372 } 373 374 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev, 375 void *pkt, bool sync) 376 { 377 struct device *dev = hdev->core->dev; 378 struct hfi_pkt_hdr *cmd_packet; 379 struct iface_queue *queue; 380 u32 rx_req; 381 int ret; 382 383 if (!venus_is_valid_state(hdev)) 384 return -EINVAL; 385 386 cmd_packet = (struct hfi_pkt_hdr *)pkt; 387 hdev->last_packet_type = cmd_packet->pkt_type; 388 389 queue = &hdev->queues[IFACEQ_CMD_IDX]; 390 391 ret = venus_write_queue(hdev, queue, pkt, &rx_req); 392 if (ret) { 393 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret); 394 return ret; 395 } 396 397 if (sync) { 398 /* 399 * Inform video hardware to raise interrupt for synchronous 400 * commands 401 */ 402 queue = &hdev->queues[IFACEQ_MSG_IDX]; 403 queue->qhdr->rx_req = 1; 404 /* ensure rx_req is updated in memory */ 405 wmb(); 406 } 407 408 if (rx_req) 409 venus_soft_int(hdev); 410 411 return 0; 412 } 413 414 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync) 415 { 416 int ret; 417 418 mutex_lock(&hdev->lock); 419 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync); 420 mutex_unlock(&hdev->lock); 421 422 return ret; 423 } 424 425 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id, 426 u32 size, u32 addr, void *cookie) 427 { 428 struct venus_hfi_device *hdev = to_hfi_priv(core); 429 struct hfi_sys_set_resource_pkt *pkt; 430 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 431 int ret; 432 433 if (id == VIDC_RESOURCE_NONE) 434 return 0; 435 436 pkt = (struct hfi_sys_set_resource_pkt *)packet; 437 438 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie); 439 if (ret) 440 return ret; 441 442 ret = venus_iface_cmdq_write(hdev, pkt, false); 443 if (ret) 444 return ret; 445 446 return 0; 447 } 448 449 static int venus_boot_core(struct venus_hfi_device *hdev) 450 { 451 struct device *dev = hdev->core->dev; 452 static const unsigned int max_tries = 100; 453 u32 ctrl_status = 0; 454 unsigned int count = 0; 455 int ret = 0; 456 457 venus_writel(hdev, VIDC_CTRL_INIT, BIT(VIDC_CTRL_INIT_CTRL_SHIFT)); 458 venus_writel(hdev, WRAPPER_INTR_MASK, WRAPPER_INTR_MASK_A2HVCODEC_MASK); 459 venus_writel(hdev, CPU_CS_SCIACMDARG3, 1); 460 461 while (!ctrl_status && count < max_tries) { 462 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0); 463 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) { 464 dev_err(dev, "invalid setting for UC_REGION\n"); 465 ret = -EINVAL; 466 break; 467 } 468 469 usleep_range(500, 1000); 470 count++; 471 } 472 473 if (count >= max_tries) 474 ret = -ETIMEDOUT; 475 476 return ret; 477 } 478 479 static u32 venus_hwversion(struct venus_hfi_device *hdev) 480 { 481 struct device *dev = hdev->core->dev; 482 u32 ver = venus_readl(hdev, WRAPPER_HW_VERSION); 483 u32 major, minor, step; 484 485 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK; 486 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT; 487 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK; 488 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT; 489 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK; 490 491 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step); 492 493 return major; 494 } 495 496 static int venus_run(struct venus_hfi_device *hdev) 497 { 498 struct device *dev = hdev->core->dev; 499 int ret; 500 501 /* 502 * Re-program all of the registers that get reset as a result of 503 * regulator_disable() and _enable() 504 */ 505 venus_set_registers(hdev); 506 507 venus_writel(hdev, UC_REGION_ADDR, hdev->ifaceq_table.da); 508 venus_writel(hdev, UC_REGION_SIZE, SHARED_QSIZE); 509 venus_writel(hdev, CPU_CS_SCIACMDARG2, hdev->ifaceq_table.da); 510 venus_writel(hdev, CPU_CS_SCIACMDARG1, 0x01); 511 if (hdev->sfr.da) 512 venus_writel(hdev, SFR_ADDR, hdev->sfr.da); 513 514 ret = venus_boot_core(hdev); 515 if (ret) { 516 dev_err(dev, "failed to reset venus core\n"); 517 return ret; 518 } 519 520 venus_hwversion(hdev); 521 522 return 0; 523 } 524 525 static int venus_halt_axi(struct venus_hfi_device *hdev) 526 { 527 void __iomem *base = hdev->core->base; 528 struct device *dev = hdev->core->dev; 529 u32 val; 530 int ret; 531 532 if (IS_V4(hdev->core)) { 533 val = venus_readl(hdev, WRAPPER_CPU_AXI_HALT); 534 val |= WRAPPER_CPU_AXI_HALT_HALT; 535 venus_writel(hdev, WRAPPER_CPU_AXI_HALT, val); 536 537 ret = readl_poll_timeout(base + WRAPPER_CPU_AXI_HALT_STATUS, 538 val, 539 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE, 540 POLL_INTERVAL_US, 541 VBIF_AXI_HALT_ACK_TIMEOUT_US); 542 if (ret) { 543 dev_err(dev, "AXI bus port halt timeout\n"); 544 return ret; 545 } 546 547 return 0; 548 } 549 550 /* Halt AXI and AXI IMEM VBIF Access */ 551 val = venus_readl(hdev, VBIF_AXI_HALT_CTRL0); 552 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ; 553 venus_writel(hdev, VBIF_AXI_HALT_CTRL0, val); 554 555 /* Request for AXI bus port halt */ 556 ret = readl_poll_timeout(base + VBIF_AXI_HALT_CTRL1, val, 557 val & VBIF_AXI_HALT_CTRL1_HALT_ACK, 558 POLL_INTERVAL_US, 559 VBIF_AXI_HALT_ACK_TIMEOUT_US); 560 if (ret) { 561 dev_err(dev, "AXI bus port halt timeout\n"); 562 return ret; 563 } 564 565 return 0; 566 } 567 568 static int venus_power_off(struct venus_hfi_device *hdev) 569 { 570 int ret; 571 572 if (!hdev->power_enabled) 573 return 0; 574 575 ret = venus_set_hw_state_suspend(hdev->core); 576 if (ret) 577 return ret; 578 579 ret = venus_halt_axi(hdev); 580 if (ret) 581 return ret; 582 583 hdev->power_enabled = false; 584 585 return 0; 586 } 587 588 static int venus_power_on(struct venus_hfi_device *hdev) 589 { 590 int ret; 591 592 if (hdev->power_enabled) 593 return 0; 594 595 ret = venus_set_hw_state_resume(hdev->core); 596 if (ret) 597 goto err; 598 599 ret = venus_run(hdev); 600 if (ret) 601 goto err_suspend; 602 603 hdev->power_enabled = true; 604 605 return 0; 606 607 err_suspend: 608 venus_set_hw_state_suspend(hdev->core); 609 err: 610 hdev->power_enabled = false; 611 return ret; 612 } 613 614 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev, 615 void *pkt) 616 { 617 struct iface_queue *queue; 618 u32 tx_req; 619 int ret; 620 621 if (!venus_is_valid_state(hdev)) 622 return -EINVAL; 623 624 queue = &hdev->queues[IFACEQ_MSG_IDX]; 625 626 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 627 if (ret) 628 return ret; 629 630 if (tx_req) 631 venus_soft_int(hdev); 632 633 return 0; 634 } 635 636 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt) 637 { 638 int ret; 639 640 mutex_lock(&hdev->lock); 641 ret = venus_iface_msgq_read_nolock(hdev, pkt); 642 mutex_unlock(&hdev->lock); 643 644 return ret; 645 } 646 647 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev, 648 void *pkt) 649 { 650 struct iface_queue *queue; 651 u32 tx_req; 652 int ret; 653 654 ret = venus_is_valid_state(hdev); 655 if (!ret) 656 return -EINVAL; 657 658 queue = &hdev->queues[IFACEQ_DBG_IDX]; 659 660 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 661 if (ret) 662 return ret; 663 664 if (tx_req) 665 venus_soft_int(hdev); 666 667 return 0; 668 } 669 670 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt) 671 { 672 int ret; 673 674 if (!pkt) 675 return -EINVAL; 676 677 mutex_lock(&hdev->lock); 678 ret = venus_iface_dbgq_read_nolock(hdev, pkt); 679 mutex_unlock(&hdev->lock); 680 681 return ret; 682 } 683 684 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr) 685 { 686 qhdr->status = 1; 687 qhdr->type = IFACEQ_DFLT_QHDR; 688 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4; 689 qhdr->pkt_size = 0; 690 qhdr->rx_wm = 1; 691 qhdr->tx_wm = 1; 692 qhdr->rx_req = 1; 693 qhdr->tx_req = 0; 694 qhdr->rx_irq_status = 0; 695 qhdr->tx_irq_status = 0; 696 qhdr->read_idx = 0; 697 qhdr->write_idx = 0; 698 } 699 700 static void venus_interface_queues_release(struct venus_hfi_device *hdev) 701 { 702 mutex_lock(&hdev->lock); 703 704 venus_free(hdev, &hdev->ifaceq_table); 705 venus_free(hdev, &hdev->sfr); 706 707 memset(hdev->queues, 0, sizeof(hdev->queues)); 708 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table)); 709 memset(&hdev->sfr, 0, sizeof(hdev->sfr)); 710 711 mutex_unlock(&hdev->lock); 712 } 713 714 static int venus_interface_queues_init(struct venus_hfi_device *hdev) 715 { 716 struct hfi_queue_table_header *tbl_hdr; 717 struct iface_queue *queue; 718 struct hfi_sfr *sfr; 719 struct mem_desc desc = {0}; 720 unsigned int offset; 721 unsigned int i; 722 int ret; 723 724 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE); 725 if (ret) 726 return ret; 727 728 hdev->ifaceq_table = desc; 729 offset = IFACEQ_TABLE_SIZE; 730 731 for (i = 0; i < IFACEQ_NUM; i++) { 732 queue = &hdev->queues[i]; 733 queue->qmem.da = desc.da + offset; 734 queue->qmem.kva = desc.kva + offset; 735 queue->qmem.size = IFACEQ_QUEUE_SIZE; 736 offset += queue->qmem.size; 737 queue->qhdr = 738 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 739 740 venus_set_qhdr_defaults(queue->qhdr); 741 742 queue->qhdr->start_addr = queue->qmem.da; 743 744 if (i == IFACEQ_CMD_IDX) 745 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 746 else if (i == IFACEQ_MSG_IDX) 747 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 748 else if (i == IFACEQ_DBG_IDX) 749 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 750 } 751 752 tbl_hdr = hdev->ifaceq_table.kva; 753 tbl_hdr->version = 0; 754 tbl_hdr->size = IFACEQ_TABLE_SIZE; 755 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 756 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 757 tbl_hdr->num_q = IFACEQ_NUM; 758 tbl_hdr->num_active_q = IFACEQ_NUM; 759 760 /* 761 * Set receive request to zero on debug queue as there is no 762 * need of interrupt from video hardware for debug messages 763 */ 764 queue = &hdev->queues[IFACEQ_DBG_IDX]; 765 queue->qhdr->rx_req = 0; 766 767 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE); 768 if (ret) { 769 hdev->sfr.da = 0; 770 } else { 771 hdev->sfr = desc; 772 sfr = hdev->sfr.kva; 773 sfr->buf_size = ALIGNED_SFR_SIZE; 774 } 775 776 /* ensure table and queue header structs are settled in memory */ 777 wmb(); 778 779 return 0; 780 } 781 782 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug) 783 { 784 struct hfi_sys_set_property_pkt *pkt; 785 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 786 int ret; 787 788 pkt = (struct hfi_sys_set_property_pkt *)packet; 789 790 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug); 791 792 ret = venus_iface_cmdq_write(hdev, pkt, false); 793 if (ret) 794 return ret; 795 796 return 0; 797 } 798 799 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode) 800 { 801 struct hfi_sys_set_property_pkt *pkt; 802 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 803 int ret; 804 805 pkt = (struct hfi_sys_set_property_pkt *)packet; 806 807 pkt_sys_coverage_config(pkt, mode); 808 809 ret = venus_iface_cmdq_write(hdev, pkt, false); 810 if (ret) 811 return ret; 812 813 return 0; 814 } 815 816 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev, 817 bool enable) 818 { 819 struct hfi_sys_set_property_pkt *pkt; 820 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 821 int ret; 822 823 if (!enable) 824 return 0; 825 826 pkt = (struct hfi_sys_set_property_pkt *)packet; 827 828 pkt_sys_idle_indicator(pkt, enable); 829 830 ret = venus_iface_cmdq_write(hdev, pkt, false); 831 if (ret) 832 return ret; 833 834 return 0; 835 } 836 837 static int venus_sys_set_power_control(struct venus_hfi_device *hdev, 838 bool enable) 839 { 840 struct hfi_sys_set_property_pkt *pkt; 841 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 842 int ret; 843 844 pkt = (struct hfi_sys_set_property_pkt *)packet; 845 846 pkt_sys_power_control(pkt, enable); 847 848 ret = venus_iface_cmdq_write(hdev, pkt, false); 849 if (ret) 850 return ret; 851 852 return 0; 853 } 854 855 static int venus_get_queue_size(struct venus_hfi_device *hdev, 856 unsigned int index) 857 { 858 struct hfi_queue_header *qhdr; 859 860 if (index >= IFACEQ_NUM) 861 return -EINVAL; 862 863 qhdr = hdev->queues[index].qhdr; 864 if (!qhdr) 865 return -EINVAL; 866 867 return abs(qhdr->read_idx - qhdr->write_idx); 868 } 869 870 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev) 871 { 872 struct device *dev = hdev->core->dev; 873 int ret; 874 875 ret = venus_sys_set_debug(hdev, venus_fw_debug); 876 if (ret) 877 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret); 878 879 /* 880 * Idle indicator is disabled by default on some 4xx firmware versions, 881 * enable it explicitly in order to make suspend functional by checking 882 * WFI (wait-for-interrupt) bit. 883 */ 884 if (IS_V4(hdev->core)) 885 venus_sys_idle_indicator = true; 886 887 ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator); 888 if (ret) 889 dev_warn(dev, "setting idle response ON failed (%d)\n", ret); 890 891 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode); 892 if (ret) 893 dev_warn(dev, "setting hw power collapse ON failed (%d)\n", 894 ret); 895 896 return ret; 897 } 898 899 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync) 900 { 901 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 902 struct hfi_session_pkt pkt; 903 904 pkt_session_cmd(&pkt, pkt_type, inst); 905 906 return venus_iface_cmdq_write(hdev, &pkt, sync); 907 } 908 909 static void venus_flush_debug_queue(struct venus_hfi_device *hdev) 910 { 911 struct device *dev = hdev->core->dev; 912 void *packet = hdev->dbg_buf; 913 914 while (!venus_iface_dbgq_read(hdev, packet)) { 915 struct hfi_msg_sys_coverage_pkt *pkt = packet; 916 917 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) { 918 struct hfi_msg_sys_debug_pkt *pkt = packet; 919 920 dev_dbg(dev, VDBGFW "%s", pkt->msg_data); 921 } 922 } 923 } 924 925 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev, 926 bool wait) 927 { 928 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout); 929 struct hfi_sys_pc_prep_pkt pkt; 930 int ret; 931 932 init_completion(&hdev->pwr_collapse_prep); 933 934 pkt_sys_pc_prep(&pkt); 935 936 ret = venus_iface_cmdq_write(hdev, &pkt, false); 937 if (ret) 938 return ret; 939 940 if (!wait) 941 return 0; 942 943 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout); 944 if (!ret) { 945 venus_flush_debug_queue(hdev); 946 return -ETIMEDOUT; 947 } 948 949 return 0; 950 } 951 952 static int venus_are_queues_empty(struct venus_hfi_device *hdev) 953 { 954 int ret1, ret2; 955 956 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX); 957 if (ret1 < 0) 958 return ret1; 959 960 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX); 961 if (ret2 < 0) 962 return ret2; 963 964 if (!ret1 && !ret2) 965 return 1; 966 967 return 0; 968 } 969 970 static void venus_sfr_print(struct venus_hfi_device *hdev) 971 { 972 struct device *dev = hdev->core->dev; 973 struct hfi_sfr *sfr = hdev->sfr.kva; 974 void *p; 975 976 if (!sfr) 977 return; 978 979 p = memchr(sfr->data, '\0', sfr->buf_size); 980 /* 981 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates 982 * that Venus is in the process of crashing. 983 */ 984 if (!p) 985 sfr->data[sfr->buf_size - 1] = '\0'; 986 987 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data); 988 } 989 990 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev, 991 void *packet) 992 { 993 struct hfi_msg_event_notify_pkt *event_pkt = packet; 994 995 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR) 996 return; 997 998 venus_set_state(hdev, VENUS_STATE_DEINIT); 999 1000 venus_sfr_print(hdev); 1001 } 1002 1003 static irqreturn_t venus_isr_thread(struct venus_core *core) 1004 { 1005 struct venus_hfi_device *hdev = to_hfi_priv(core); 1006 const struct venus_resources *res; 1007 void *pkt; 1008 u32 msg_ret; 1009 1010 if (!hdev) 1011 return IRQ_NONE; 1012 1013 res = hdev->core->res; 1014 pkt = hdev->pkt_buf; 1015 1016 1017 while (!venus_iface_msgq_read(hdev, pkt)) { 1018 msg_ret = hfi_process_msg_packet(core, pkt); 1019 switch (msg_ret) { 1020 case HFI_MSG_EVENT_NOTIFY: 1021 venus_process_msg_sys_error(hdev, pkt); 1022 break; 1023 case HFI_MSG_SYS_INIT: 1024 venus_hfi_core_set_resource(core, res->vmem_id, 1025 res->vmem_size, 1026 res->vmem_addr, 1027 hdev); 1028 break; 1029 case HFI_MSG_SYS_RELEASE_RESOURCE: 1030 complete(&hdev->release_resource); 1031 break; 1032 case HFI_MSG_SYS_PC_PREP: 1033 complete(&hdev->pwr_collapse_prep); 1034 break; 1035 default: 1036 break; 1037 } 1038 } 1039 1040 venus_flush_debug_queue(hdev); 1041 1042 return IRQ_HANDLED; 1043 } 1044 1045 static irqreturn_t venus_isr(struct venus_core *core) 1046 { 1047 struct venus_hfi_device *hdev = to_hfi_priv(core); 1048 u32 status; 1049 1050 if (!hdev) 1051 return IRQ_NONE; 1052 1053 status = venus_readl(hdev, WRAPPER_INTR_STATUS); 1054 1055 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1056 status & WRAPPER_INTR_STATUS_A2HWD_MASK || 1057 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1058 hdev->irq_status = status; 1059 1060 venus_writel(hdev, CPU_CS_A2HSOFTINTCLR, 1); 1061 venus_writel(hdev, WRAPPER_INTR_CLEAR, status); 1062 1063 return IRQ_WAKE_THREAD; 1064 } 1065 1066 static int venus_core_init(struct venus_core *core) 1067 { 1068 struct venus_hfi_device *hdev = to_hfi_priv(core); 1069 struct device *dev = core->dev; 1070 struct hfi_sys_get_property_pkt version_pkt; 1071 struct hfi_sys_init_pkt pkt; 1072 int ret; 1073 1074 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX); 1075 1076 venus_set_state(hdev, VENUS_STATE_INIT); 1077 1078 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1079 if (ret) 1080 return ret; 1081 1082 pkt_sys_image_version(&version_pkt); 1083 1084 ret = venus_iface_cmdq_write(hdev, &version_pkt, false); 1085 if (ret) 1086 dev_warn(dev, "failed to send image version pkt to fw\n"); 1087 1088 ret = venus_sys_set_default_properties(hdev); 1089 if (ret) 1090 return ret; 1091 1092 return 0; 1093 } 1094 1095 static int venus_core_deinit(struct venus_core *core) 1096 { 1097 struct venus_hfi_device *hdev = to_hfi_priv(core); 1098 1099 venus_set_state(hdev, VENUS_STATE_DEINIT); 1100 hdev->suspended = true; 1101 hdev->power_enabled = false; 1102 1103 return 0; 1104 } 1105 1106 static int venus_core_ping(struct venus_core *core, u32 cookie) 1107 { 1108 struct venus_hfi_device *hdev = to_hfi_priv(core); 1109 struct hfi_sys_ping_pkt pkt; 1110 1111 pkt_sys_ping(&pkt, cookie); 1112 1113 return venus_iface_cmdq_write(hdev, &pkt, false); 1114 } 1115 1116 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type) 1117 { 1118 struct venus_hfi_device *hdev = to_hfi_priv(core); 1119 struct hfi_sys_test_ssr_pkt pkt; 1120 int ret; 1121 1122 ret = pkt_sys_ssr_cmd(&pkt, trigger_type); 1123 if (ret) 1124 return ret; 1125 1126 return venus_iface_cmdq_write(hdev, &pkt, false); 1127 } 1128 1129 static int venus_session_init(struct venus_inst *inst, u32 session_type, 1130 u32 codec) 1131 { 1132 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1133 struct hfi_session_init_pkt pkt; 1134 int ret; 1135 1136 ret = venus_sys_set_debug(hdev, venus_fw_debug); 1137 if (ret) 1138 goto err; 1139 1140 ret = pkt_session_init(&pkt, inst, session_type, codec); 1141 if (ret) 1142 goto err; 1143 1144 ret = venus_iface_cmdq_write(hdev, &pkt, true); 1145 if (ret) 1146 goto err; 1147 1148 return 0; 1149 1150 err: 1151 venus_flush_debug_queue(hdev); 1152 return ret; 1153 } 1154 1155 static int venus_session_end(struct venus_inst *inst) 1156 { 1157 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1158 struct device *dev = hdev->core->dev; 1159 1160 if (venus_fw_coverage) { 1161 if (venus_sys_set_coverage(hdev, venus_fw_coverage)) 1162 dev_warn(dev, "fw coverage msg ON failed\n"); 1163 } 1164 1165 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true); 1166 } 1167 1168 static int venus_session_abort(struct venus_inst *inst) 1169 { 1170 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1171 1172 venus_flush_debug_queue(hdev); 1173 1174 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true); 1175 } 1176 1177 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode) 1178 { 1179 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1180 struct hfi_session_flush_pkt pkt; 1181 int ret; 1182 1183 ret = pkt_session_flush(&pkt, inst, flush_mode); 1184 if (ret) 1185 return ret; 1186 1187 return venus_iface_cmdq_write(hdev, &pkt, true); 1188 } 1189 1190 static int venus_session_start(struct venus_inst *inst) 1191 { 1192 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true); 1193 } 1194 1195 static int venus_session_stop(struct venus_inst *inst) 1196 { 1197 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true); 1198 } 1199 1200 static int venus_session_continue(struct venus_inst *inst) 1201 { 1202 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false); 1203 } 1204 1205 static int venus_session_etb(struct venus_inst *inst, 1206 struct hfi_frame_data *in_frame) 1207 { 1208 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1209 u32 session_type = inst->session_type; 1210 int ret; 1211 1212 if (session_type == VIDC_SESSION_TYPE_DEC) { 1213 struct hfi_session_empty_buffer_compressed_pkt pkt; 1214 1215 ret = pkt_session_etb_decoder(&pkt, inst, in_frame); 1216 if (ret) 1217 return ret; 1218 1219 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1220 } else if (session_type == VIDC_SESSION_TYPE_ENC) { 1221 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt; 1222 1223 ret = pkt_session_etb_encoder(&pkt, inst, in_frame); 1224 if (ret) 1225 return ret; 1226 1227 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1228 } else { 1229 ret = -EINVAL; 1230 } 1231 1232 return ret; 1233 } 1234 1235 static int venus_session_ftb(struct venus_inst *inst, 1236 struct hfi_frame_data *out_frame) 1237 { 1238 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1239 struct hfi_session_fill_buffer_pkt pkt; 1240 int ret; 1241 1242 ret = pkt_session_ftb(&pkt, inst, out_frame); 1243 if (ret) 1244 return ret; 1245 1246 return venus_iface_cmdq_write(hdev, &pkt, false); 1247 } 1248 1249 static int venus_session_set_buffers(struct venus_inst *inst, 1250 struct hfi_buffer_desc *bd) 1251 { 1252 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1253 struct hfi_session_set_buffers_pkt *pkt; 1254 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1255 int ret; 1256 1257 if (bd->buffer_type == HFI_BUFFER_INPUT) 1258 return 0; 1259 1260 pkt = (struct hfi_session_set_buffers_pkt *)packet; 1261 1262 ret = pkt_session_set_buffers(pkt, inst, bd); 1263 if (ret) 1264 return ret; 1265 1266 return venus_iface_cmdq_write(hdev, pkt, false); 1267 } 1268 1269 static int venus_session_unset_buffers(struct venus_inst *inst, 1270 struct hfi_buffer_desc *bd) 1271 { 1272 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1273 struct hfi_session_release_buffer_pkt *pkt; 1274 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1275 int ret; 1276 1277 if (bd->buffer_type == HFI_BUFFER_INPUT) 1278 return 0; 1279 1280 pkt = (struct hfi_session_release_buffer_pkt *)packet; 1281 1282 ret = pkt_session_unset_buffers(pkt, inst, bd); 1283 if (ret) 1284 return ret; 1285 1286 return venus_iface_cmdq_write(hdev, pkt, true); 1287 } 1288 1289 static int venus_session_load_res(struct venus_inst *inst) 1290 { 1291 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true); 1292 } 1293 1294 static int venus_session_release_res(struct venus_inst *inst) 1295 { 1296 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true); 1297 } 1298 1299 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1300 u32 seq_hdr_len) 1301 { 1302 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1303 struct hfi_session_parse_sequence_header_pkt *pkt; 1304 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1305 int ret; 1306 1307 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet; 1308 1309 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len); 1310 if (ret) 1311 return ret; 1312 1313 ret = venus_iface_cmdq_write(hdev, pkt, false); 1314 if (ret) 1315 return ret; 1316 1317 return 0; 1318 } 1319 1320 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1321 u32 seq_hdr_len) 1322 { 1323 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1324 struct hfi_session_get_sequence_header_pkt *pkt; 1325 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1326 int ret; 1327 1328 pkt = (struct hfi_session_get_sequence_header_pkt *)packet; 1329 1330 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len); 1331 if (ret) 1332 return ret; 1333 1334 return venus_iface_cmdq_write(hdev, pkt, false); 1335 } 1336 1337 static int venus_session_set_property(struct venus_inst *inst, u32 ptype, 1338 void *pdata) 1339 { 1340 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1341 struct hfi_session_set_property_pkt *pkt; 1342 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1343 int ret; 1344 1345 pkt = (struct hfi_session_set_property_pkt *)packet; 1346 1347 ret = pkt_session_set_property(pkt, inst, ptype, pdata); 1348 if (ret == -ENOTSUPP) 1349 return 0; 1350 if (ret) 1351 return ret; 1352 1353 return venus_iface_cmdq_write(hdev, pkt, false); 1354 } 1355 1356 static int venus_session_get_property(struct venus_inst *inst, u32 ptype) 1357 { 1358 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1359 struct hfi_session_get_property_pkt pkt; 1360 int ret; 1361 1362 ret = pkt_session_get_property(&pkt, inst, ptype); 1363 if (ret) 1364 return ret; 1365 1366 return venus_iface_cmdq_write(hdev, &pkt, true); 1367 } 1368 1369 static int venus_resume(struct venus_core *core) 1370 { 1371 struct venus_hfi_device *hdev = to_hfi_priv(core); 1372 int ret = 0; 1373 1374 mutex_lock(&hdev->lock); 1375 1376 if (!hdev->suspended) 1377 goto unlock; 1378 1379 ret = venus_power_on(hdev); 1380 1381 unlock: 1382 if (!ret) 1383 hdev->suspended = false; 1384 1385 mutex_unlock(&hdev->lock); 1386 1387 return ret; 1388 } 1389 1390 static int venus_suspend_1xx(struct venus_core *core) 1391 { 1392 struct venus_hfi_device *hdev = to_hfi_priv(core); 1393 struct device *dev = core->dev; 1394 u32 ctrl_status; 1395 int ret; 1396 1397 if (!hdev->power_enabled || hdev->suspended) 1398 return 0; 1399 1400 mutex_lock(&hdev->lock); 1401 ret = venus_is_valid_state(hdev); 1402 mutex_unlock(&hdev->lock); 1403 1404 if (!ret) { 1405 dev_err(dev, "bad state, cannot suspend\n"); 1406 return -EINVAL; 1407 } 1408 1409 ret = venus_prepare_power_collapse(hdev, true); 1410 if (ret) { 1411 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1412 return ret; 1413 } 1414 1415 mutex_lock(&hdev->lock); 1416 1417 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) { 1418 mutex_unlock(&hdev->lock); 1419 return -EINVAL; 1420 } 1421 1422 ret = venus_are_queues_empty(hdev); 1423 if (ret < 0 || !ret) { 1424 mutex_unlock(&hdev->lock); 1425 return -EINVAL; 1426 } 1427 1428 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0); 1429 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) { 1430 mutex_unlock(&hdev->lock); 1431 return -EINVAL; 1432 } 1433 1434 ret = venus_power_off(hdev); 1435 if (ret) { 1436 mutex_unlock(&hdev->lock); 1437 return ret; 1438 } 1439 1440 hdev->suspended = true; 1441 1442 mutex_unlock(&hdev->lock); 1443 1444 return 0; 1445 } 1446 1447 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev) 1448 { 1449 u32 ctrl_status, cpu_status; 1450 1451 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS); 1452 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0); 1453 1454 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1455 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1456 return true; 1457 1458 return false; 1459 } 1460 1461 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev) 1462 { 1463 u32 ctrl_status, cpu_status; 1464 1465 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS); 1466 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0); 1467 1468 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1469 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1470 return true; 1471 1472 return false; 1473 } 1474 1475 static int venus_suspend_3xx(struct venus_core *core) 1476 { 1477 struct venus_hfi_device *hdev = to_hfi_priv(core); 1478 struct device *dev = core->dev; 1479 u32 ctrl_status; 1480 bool val; 1481 int ret; 1482 1483 if (!hdev->power_enabled || hdev->suspended) 1484 return 0; 1485 1486 mutex_lock(&hdev->lock); 1487 ret = venus_is_valid_state(hdev); 1488 mutex_unlock(&hdev->lock); 1489 1490 if (!ret) { 1491 dev_err(dev, "bad state, cannot suspend\n"); 1492 return -EINVAL; 1493 } 1494 1495 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0); 1496 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1497 goto power_off; 1498 1499 /* 1500 * Power collapse sequence for Venus 3xx and 4xx versions: 1501 * 1. Check for ARM9 and video core to be idle by checking WFI bit 1502 * (bit 0) in CPU status register and by checking Idle (bit 30) in 1503 * Control status register for video core. 1504 * 2. Send a command to prepare for power collapse. 1505 * 3. Check for WFI and PC_READY bits. 1506 */ 1507 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val, 1508 1500, 100 * 1500); 1509 if (ret) 1510 return ret; 1511 1512 ret = venus_prepare_power_collapse(hdev, false); 1513 if (ret) { 1514 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1515 return ret; 1516 } 1517 1518 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val, 1519 1500, 100 * 1500); 1520 if (ret) 1521 return ret; 1522 1523 power_off: 1524 mutex_lock(&hdev->lock); 1525 1526 ret = venus_power_off(hdev); 1527 if (ret) { 1528 dev_err(dev, "venus_power_off (%d)\n", ret); 1529 mutex_unlock(&hdev->lock); 1530 return ret; 1531 } 1532 1533 hdev->suspended = true; 1534 1535 mutex_unlock(&hdev->lock); 1536 1537 return 0; 1538 } 1539 1540 static int venus_suspend(struct venus_core *core) 1541 { 1542 if (IS_V3(core) || IS_V4(core)) 1543 return venus_suspend_3xx(core); 1544 1545 return venus_suspend_1xx(core); 1546 } 1547 1548 static const struct hfi_ops venus_hfi_ops = { 1549 .core_init = venus_core_init, 1550 .core_deinit = venus_core_deinit, 1551 .core_ping = venus_core_ping, 1552 .core_trigger_ssr = venus_core_trigger_ssr, 1553 1554 .session_init = venus_session_init, 1555 .session_end = venus_session_end, 1556 .session_abort = venus_session_abort, 1557 .session_flush = venus_session_flush, 1558 .session_start = venus_session_start, 1559 .session_stop = venus_session_stop, 1560 .session_continue = venus_session_continue, 1561 .session_etb = venus_session_etb, 1562 .session_ftb = venus_session_ftb, 1563 .session_set_buffers = venus_session_set_buffers, 1564 .session_unset_buffers = venus_session_unset_buffers, 1565 .session_load_res = venus_session_load_res, 1566 .session_release_res = venus_session_release_res, 1567 .session_parse_seq_hdr = venus_session_parse_seq_hdr, 1568 .session_get_seq_hdr = venus_session_get_seq_hdr, 1569 .session_set_property = venus_session_set_property, 1570 .session_get_property = venus_session_get_property, 1571 1572 .resume = venus_resume, 1573 .suspend = venus_suspend, 1574 1575 .isr = venus_isr, 1576 .isr_thread = venus_isr_thread, 1577 }; 1578 1579 void venus_hfi_destroy(struct venus_core *core) 1580 { 1581 struct venus_hfi_device *hdev = to_hfi_priv(core); 1582 1583 venus_interface_queues_release(hdev); 1584 mutex_destroy(&hdev->lock); 1585 kfree(hdev); 1586 core->priv = NULL; 1587 core->ops = NULL; 1588 } 1589 1590 int venus_hfi_create(struct venus_core *core) 1591 { 1592 struct venus_hfi_device *hdev; 1593 int ret; 1594 1595 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL); 1596 if (!hdev) 1597 return -ENOMEM; 1598 1599 mutex_init(&hdev->lock); 1600 1601 hdev->core = core; 1602 hdev->suspended = true; 1603 core->priv = hdev; 1604 core->ops = &venus_hfi_ops; 1605 1606 ret = venus_interface_queues_init(hdev); 1607 if (ret) 1608 goto err_kfree; 1609 1610 return 0; 1611 1612 err_kfree: 1613 kfree(hdev); 1614 core->priv = NULL; 1615 core->ops = NULL; 1616 return ret; 1617 } 1618 1619 void venus_hfi_queues_reinit(struct venus_core *core) 1620 { 1621 struct venus_hfi_device *hdev = to_hfi_priv(core); 1622 struct hfi_queue_table_header *tbl_hdr; 1623 struct iface_queue *queue; 1624 struct hfi_sfr *sfr; 1625 unsigned int i; 1626 1627 mutex_lock(&hdev->lock); 1628 1629 for (i = 0; i < IFACEQ_NUM; i++) { 1630 queue = &hdev->queues[i]; 1631 queue->qhdr = 1632 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 1633 1634 venus_set_qhdr_defaults(queue->qhdr); 1635 1636 queue->qhdr->start_addr = queue->qmem.da; 1637 1638 if (i == IFACEQ_CMD_IDX) 1639 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 1640 else if (i == IFACEQ_MSG_IDX) 1641 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 1642 else if (i == IFACEQ_DBG_IDX) 1643 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 1644 } 1645 1646 tbl_hdr = hdev->ifaceq_table.kva; 1647 tbl_hdr->version = 0; 1648 tbl_hdr->size = IFACEQ_TABLE_SIZE; 1649 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 1650 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 1651 tbl_hdr->num_q = IFACEQ_NUM; 1652 tbl_hdr->num_active_q = IFACEQ_NUM; 1653 1654 /* 1655 * Set receive request to zero on debug queue as there is no 1656 * need of interrupt from video hardware for debug messages 1657 */ 1658 queue = &hdev->queues[IFACEQ_DBG_IDX]; 1659 queue->qhdr->rx_req = 0; 1660 1661 sfr = hdev->sfr.kva; 1662 sfr->buf_size = ALIGNED_SFR_SIZE; 1663 1664 /* ensure table and queue header structs are settled in memory */ 1665 wmb(); 1666 1667 mutex_unlock(&hdev->lock); 1668 } 1669