1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 - 2018 Intel Corporation 4 * Copyright 2017 Google LLC 5 * 6 * Based on Intel IPU4 driver. 7 * 8 */ 9 10 #include <linux/delay.h> 11 #include <linux/interrupt.h> 12 #include <linux/module.h> 13 #include <linux/pm_runtime.h> 14 15 #include "ipu3.h" 16 #include "ipu3-dmamap.h" 17 #include "ipu3-mmu.h" 18 19 #define IMGU_PCI_ID 0x1919 20 #define IMGU_PCI_BAR 0 21 #define IMGU_DMA_MASK DMA_BIT_MASK(39) 22 #define IMGU_MAX_QUEUE_DEPTH (2 + 2) 23 24 /* 25 * pre-allocated buffer size for IMGU dummy buffers. Those 26 * values should be tuned to big enough to avoid buffer 27 * re-allocation when streaming to lower streaming latency. 28 */ 29 #define CSS_QUEUE_IN_BUF_SIZE 0 30 #define CSS_QUEUE_PARAMS_BUF_SIZE 0 31 #define CSS_QUEUE_OUT_BUF_SIZE (4160 * 3120 * 12 / 8) 32 #define CSS_QUEUE_VF_BUF_SIZE (1920 * 1080 * 12 / 8) 33 #define CSS_QUEUE_STAT_3A_BUF_SIZE sizeof(struct ipu3_uapi_stats_3a) 34 35 static const size_t css_queue_buf_size_map[IPU3_CSS_QUEUES] = { 36 [IPU3_CSS_QUEUE_IN] = CSS_QUEUE_IN_BUF_SIZE, 37 [IPU3_CSS_QUEUE_PARAMS] = CSS_QUEUE_PARAMS_BUF_SIZE, 38 [IPU3_CSS_QUEUE_OUT] = CSS_QUEUE_OUT_BUF_SIZE, 39 [IPU3_CSS_QUEUE_VF] = CSS_QUEUE_VF_BUF_SIZE, 40 [IPU3_CSS_QUEUE_STAT_3A] = CSS_QUEUE_STAT_3A_BUF_SIZE, 41 }; 42 43 static const struct imgu_node_mapping imgu_node_map[IMGU_NODE_NUM] = { 44 [IMGU_NODE_IN] = {IPU3_CSS_QUEUE_IN, "input"}, 45 [IMGU_NODE_PARAMS] = {IPU3_CSS_QUEUE_PARAMS, "parameters"}, 46 [IMGU_NODE_OUT] = {IPU3_CSS_QUEUE_OUT, "output"}, 47 [IMGU_NODE_VF] = {IPU3_CSS_QUEUE_VF, "viewfinder"}, 48 [IMGU_NODE_STAT_3A] = {IPU3_CSS_QUEUE_STAT_3A, "3a stat"}, 49 }; 50 51 unsigned int imgu_node_to_queue(unsigned int node) 52 { 53 return imgu_node_map[node].css_queue; 54 } 55 56 unsigned int imgu_map_node(struct imgu_device *imgu, unsigned int css_queue) 57 { 58 unsigned int i; 59 60 for (i = 0; i < IMGU_NODE_NUM; i++) 61 if (imgu_node_map[i].css_queue == css_queue) 62 break; 63 64 return i; 65 } 66 67 /**************** Dummy buffers ****************/ 68 69 static void imgu_dummybufs_cleanup(struct imgu_device *imgu, unsigned int pipe) 70 { 71 unsigned int i; 72 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 73 74 for (i = 0; i < IPU3_CSS_QUEUES; i++) 75 imgu_dmamap_free(imgu, 76 &imgu_pipe->queues[i].dmap); 77 } 78 79 static int imgu_dummybufs_preallocate(struct imgu_device *imgu, 80 unsigned int pipe) 81 { 82 unsigned int i; 83 size_t size; 84 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 85 86 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 87 size = css_queue_buf_size_map[i]; 88 /* 89 * Do not enable dummy buffers for master queue, 90 * always require that real buffers from user are 91 * available. 92 */ 93 if (i == IMGU_QUEUE_MASTER || size == 0) 94 continue; 95 96 if (!imgu_dmamap_alloc(imgu, 97 &imgu_pipe->queues[i].dmap, size)) { 98 imgu_dummybufs_cleanup(imgu, pipe); 99 return -ENOMEM; 100 } 101 } 102 103 return 0; 104 } 105 106 static int imgu_dummybufs_init(struct imgu_device *imgu, unsigned int pipe) 107 { 108 const struct v4l2_pix_format_mplane *mpix; 109 const struct v4l2_meta_format *meta; 110 unsigned int i, k, node; 111 size_t size; 112 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 113 114 /* Allocate a dummy buffer for each queue where buffer is optional */ 115 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 116 node = imgu_map_node(imgu, i); 117 if (!imgu_pipe->queue_enabled[node] || i == IMGU_QUEUE_MASTER) 118 continue; 119 120 if (!imgu_pipe->nodes[IMGU_NODE_VF].enabled && 121 i == IPU3_CSS_QUEUE_VF) 122 /* 123 * Do not enable dummy buffers for VF if it is not 124 * requested by the user. 125 */ 126 continue; 127 128 meta = &imgu_pipe->nodes[node].vdev_fmt.fmt.meta; 129 mpix = &imgu_pipe->nodes[node].vdev_fmt.fmt.pix_mp; 130 131 if (node == IMGU_NODE_STAT_3A || node == IMGU_NODE_PARAMS) 132 size = meta->buffersize; 133 else 134 size = mpix->plane_fmt[0].sizeimage; 135 136 if (imgu_css_dma_buffer_resize(imgu, 137 &imgu_pipe->queues[i].dmap, 138 size)) { 139 imgu_dummybufs_cleanup(imgu, pipe); 140 return -ENOMEM; 141 } 142 143 for (k = 0; k < IMGU_MAX_QUEUE_DEPTH; k++) 144 imgu_css_buf_init(&imgu_pipe->queues[i].dummybufs[k], i, 145 imgu_pipe->queues[i].dmap.daddr); 146 } 147 148 return 0; 149 } 150 151 /* May be called from atomic context */ 152 static struct imgu_css_buffer *imgu_dummybufs_get(struct imgu_device *imgu, 153 int queue, unsigned int pipe) 154 { 155 unsigned int i; 156 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 157 158 /* dummybufs are not allocated for master q */ 159 if (queue == IPU3_CSS_QUEUE_IN) 160 return NULL; 161 162 if (WARN_ON(!imgu_pipe->queues[queue].dmap.vaddr)) 163 /* Buffer should not be allocated here */ 164 return NULL; 165 166 for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++) 167 if (imgu_css_buf_state(&imgu_pipe->queues[queue].dummybufs[i]) != 168 IPU3_CSS_BUFFER_QUEUED) 169 break; 170 171 if (i == IMGU_MAX_QUEUE_DEPTH) 172 return NULL; 173 174 imgu_css_buf_init(&imgu_pipe->queues[queue].dummybufs[i], queue, 175 imgu_pipe->queues[queue].dmap.daddr); 176 177 return &imgu_pipe->queues[queue].dummybufs[i]; 178 } 179 180 /* Check if given buffer is a dummy buffer */ 181 static bool imgu_dummybufs_check(struct imgu_device *imgu, 182 struct imgu_css_buffer *buf, 183 unsigned int pipe) 184 { 185 unsigned int i; 186 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 187 188 for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++) 189 if (buf == &imgu_pipe->queues[buf->queue].dummybufs[i]) 190 break; 191 192 return i < IMGU_MAX_QUEUE_DEPTH; 193 } 194 195 static void imgu_buffer_done(struct imgu_device *imgu, struct vb2_buffer *vb, 196 enum vb2_buffer_state state) 197 { 198 mutex_lock(&imgu->lock); 199 imgu_v4l2_buffer_done(vb, state); 200 mutex_unlock(&imgu->lock); 201 } 202 203 static struct imgu_css_buffer *imgu_queue_getbuf(struct imgu_device *imgu, 204 unsigned int node, 205 unsigned int pipe) 206 { 207 struct imgu_buffer *buf; 208 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 209 210 if (WARN_ON(node >= IMGU_NODE_NUM)) 211 return NULL; 212 213 /* Find first free buffer from the node */ 214 list_for_each_entry(buf, &imgu_pipe->nodes[node].buffers, vid_buf.list) { 215 if (imgu_css_buf_state(&buf->css_buf) == IPU3_CSS_BUFFER_NEW) 216 return &buf->css_buf; 217 } 218 219 /* There were no free buffers, try to return a dummy buffer */ 220 return imgu_dummybufs_get(imgu, imgu_node_map[node].css_queue, pipe); 221 } 222 223 /* 224 * Queue as many buffers to CSS as possible. If all buffers don't fit into 225 * CSS buffer queues, they remain unqueued and will be queued later. 226 */ 227 int imgu_queue_buffers(struct imgu_device *imgu, bool initial, unsigned int pipe) 228 { 229 unsigned int node; 230 int r = 0; 231 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe]; 232 233 if (!imgu_css_is_streaming(&imgu->css)) 234 return 0; 235 236 dev_dbg(&imgu->pci_dev->dev, "Queue buffers to pipe %d", pipe); 237 mutex_lock(&imgu->lock); 238 239 if (!imgu_css_pipe_queue_empty(&imgu->css, pipe)) { 240 mutex_unlock(&imgu->lock); 241 return 0; 242 } 243 244 /* Buffer set is queued to FW only when input buffer is ready */ 245 for (node = IMGU_NODE_NUM - 1; 246 imgu_queue_getbuf(imgu, IMGU_NODE_IN, pipe); 247 node = node ? node - 1 : IMGU_NODE_NUM - 1) { 248 if (node == IMGU_NODE_VF && 249 !imgu_pipe->nodes[IMGU_NODE_VF].enabled) { 250 dev_warn(&imgu->pci_dev->dev, 251 "Vf not enabled, ignore queue"); 252 continue; 253 } else if (node == IMGU_NODE_PARAMS && 254 imgu_pipe->nodes[node].enabled) { 255 struct vb2_buffer *vb; 256 struct imgu_vb2_buffer *ivb; 257 258 /* No parameters for this frame */ 259 if (list_empty(&imgu_pipe->nodes[node].buffers)) 260 continue; 261 262 ivb = list_first_entry(&imgu_pipe->nodes[node].buffers, 263 struct imgu_vb2_buffer, list); 264 list_del(&ivb->list); 265 vb = &ivb->vbb.vb2_buf; 266 r = imgu_css_set_parameters(&imgu->css, pipe, 267 vb2_plane_vaddr(vb, 0)); 268 if (r) { 269 vb2_buffer_done(vb, VB2_BUF_STATE_ERROR); 270 dev_warn(&imgu->pci_dev->dev, 271 "set parameters failed."); 272 continue; 273 } 274 275 vb2_buffer_done(vb, VB2_BUF_STATE_DONE); 276 dev_dbg(&imgu->pci_dev->dev, 277 "queue user parameters %d to css.", vb->index); 278 } else if (imgu_pipe->queue_enabled[node]) { 279 struct imgu_css_buffer *buf = 280 imgu_queue_getbuf(imgu, node, pipe); 281 struct imgu_buffer *ibuf = NULL; 282 bool dummy; 283 284 if (!buf) 285 break; 286 287 r = imgu_css_buf_queue(&imgu->css, pipe, buf); 288 if (r) 289 break; 290 dummy = imgu_dummybufs_check(imgu, buf, pipe); 291 if (!dummy) 292 ibuf = container_of(buf, struct imgu_buffer, 293 css_buf); 294 dev_dbg(&imgu->pci_dev->dev, 295 "queue %s %s buffer %u to css da: 0x%08x\n", 296 dummy ? "dummy" : "user", 297 imgu_node_map[node].name, 298 dummy ? 0 : ibuf->vid_buf.vbb.vb2_buf.index, 299 (u32)buf->daddr); 300 } 301 } 302 mutex_unlock(&imgu->lock); 303 304 if (r && r != -EBUSY) 305 goto failed; 306 307 return 0; 308 309 failed: 310 /* 311 * On error, mark all buffers as failed which are not 312 * yet queued to CSS 313 */ 314 dev_err(&imgu->pci_dev->dev, 315 "failed to queue buffer to CSS on queue %i (%d)\n", 316 node, r); 317 318 if (initial) 319 /* If we were called from streamon(), no need to finish bufs */ 320 return r; 321 322 for (node = 0; node < IMGU_NODE_NUM; node++) { 323 struct imgu_buffer *buf, *buf0; 324 325 if (!imgu_pipe->queue_enabled[node]) 326 continue; /* Skip disabled queues */ 327 328 mutex_lock(&imgu->lock); 329 list_for_each_entry_safe(buf, buf0, 330 &imgu_pipe->nodes[node].buffers, 331 vid_buf.list) { 332 if (imgu_css_buf_state(&buf->css_buf) == 333 IPU3_CSS_BUFFER_QUEUED) 334 continue; /* Was already queued, skip */ 335 336 imgu_v4l2_buffer_done(&buf->vid_buf.vbb.vb2_buf, 337 VB2_BUF_STATE_ERROR); 338 } 339 mutex_unlock(&imgu->lock); 340 } 341 342 return r; 343 } 344 345 static int imgu_powerup(struct imgu_device *imgu) 346 { 347 int r; 348 unsigned int pipe; 349 unsigned int freq = 200; 350 struct v4l2_mbus_framefmt *fmt; 351 352 /* input larger than 2048*1152, ask imgu to work on high freq */ 353 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) { 354 fmt = &imgu->imgu_pipe[pipe].nodes[IMGU_NODE_IN].pad_fmt; 355 dev_dbg(&imgu->pci_dev->dev, "pipe %u input format = %ux%u", 356 pipe, fmt->width, fmt->height); 357 if ((fmt->width * fmt->height) >= (2048 * 1152)) 358 freq = 450; 359 } 360 361 r = imgu_css_set_powerup(&imgu->pci_dev->dev, imgu->base, freq); 362 if (r) 363 return r; 364 365 imgu_mmu_resume(imgu->mmu); 366 return 0; 367 } 368 369 static void imgu_powerdown(struct imgu_device *imgu) 370 { 371 imgu_mmu_suspend(imgu->mmu); 372 imgu_css_set_powerdown(&imgu->pci_dev->dev, imgu->base); 373 } 374 375 int imgu_s_stream(struct imgu_device *imgu, int enable) 376 { 377 struct device *dev = &imgu->pci_dev->dev; 378 int r, pipe; 379 380 if (!enable) { 381 /* Stop streaming */ 382 dev_dbg(dev, "stream off\n"); 383 /* Block new buffers to be queued to CSS. */ 384 atomic_set(&imgu->qbuf_barrier, 1); 385 imgu_css_stop_streaming(&imgu->css); 386 synchronize_irq(imgu->pci_dev->irq); 387 atomic_set(&imgu->qbuf_barrier, 0); 388 imgu_powerdown(imgu); 389 pm_runtime_put(&imgu->pci_dev->dev); 390 391 return 0; 392 } 393 394 /* Set Power */ 395 r = pm_runtime_get_sync(dev); 396 if (r < 0) { 397 dev_err(dev, "failed to set imgu power\n"); 398 pm_runtime_put(dev); 399 return r; 400 } 401 402 r = imgu_powerup(imgu); 403 if (r) { 404 dev_err(dev, "failed to power up imgu\n"); 405 pm_runtime_put(dev); 406 return r; 407 } 408 409 /* Start CSS streaming */ 410 r = imgu_css_start_streaming(&imgu->css); 411 if (r) { 412 dev_err(dev, "failed to start css streaming (%d)", r); 413 goto fail_start_streaming; 414 } 415 416 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) { 417 /* Initialize dummy buffers */ 418 r = imgu_dummybufs_init(imgu, pipe); 419 if (r) { 420 dev_err(dev, "failed to initialize dummy buffers (%d)", r); 421 goto fail_dummybufs; 422 } 423 424 /* Queue as many buffers from queue as possible */ 425 r = imgu_queue_buffers(imgu, true, pipe); 426 if (r) { 427 dev_err(dev, "failed to queue initial buffers (%d)", r); 428 goto fail_queueing; 429 } 430 } 431 432 return 0; 433 fail_queueing: 434 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) 435 imgu_dummybufs_cleanup(imgu, pipe); 436 fail_dummybufs: 437 imgu_css_stop_streaming(&imgu->css); 438 fail_start_streaming: 439 pm_runtime_put(dev); 440 441 return r; 442 } 443 444 static int imgu_video_nodes_init(struct imgu_device *imgu) 445 { 446 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES] = { NULL }; 447 struct v4l2_rect *rects[IPU3_CSS_RECTS] = { NULL }; 448 struct imgu_media_pipe *imgu_pipe; 449 unsigned int i, j; 450 int r; 451 452 imgu->buf_struct_size = sizeof(struct imgu_buffer); 453 454 for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) { 455 imgu_pipe = &imgu->imgu_pipe[j]; 456 457 for (i = 0; i < IMGU_NODE_NUM; i++) { 458 imgu_pipe->nodes[i].name = imgu_node_map[i].name; 459 imgu_pipe->nodes[i].output = i < IMGU_QUEUE_FIRST_INPUT; 460 imgu_pipe->nodes[i].enabled = false; 461 462 if (i != IMGU_NODE_PARAMS && i != IMGU_NODE_STAT_3A) 463 fmts[imgu_node_map[i].css_queue] = 464 &imgu_pipe->nodes[i].vdev_fmt.fmt.pix_mp; 465 atomic_set(&imgu_pipe->nodes[i].sequence, 0); 466 } 467 } 468 469 r = imgu_v4l2_register(imgu); 470 if (r) 471 return r; 472 473 /* Set initial formats and initialize formats of video nodes */ 474 for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) { 475 imgu_pipe = &imgu->imgu_pipe[j]; 476 477 rects[IPU3_CSS_RECT_EFFECTIVE] = &imgu_pipe->imgu_sd.rect.eff; 478 rects[IPU3_CSS_RECT_BDS] = &imgu_pipe->imgu_sd.rect.bds; 479 imgu_css_fmt_set(&imgu->css, fmts, rects, j); 480 481 /* Pre-allocate dummy buffers */ 482 r = imgu_dummybufs_preallocate(imgu, j); 483 if (r) { 484 dev_err(&imgu->pci_dev->dev, 485 "failed to pre-allocate dummy buffers (%d)", r); 486 goto out_cleanup; 487 } 488 } 489 490 return 0; 491 492 out_cleanup: 493 for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) 494 imgu_dummybufs_cleanup(imgu, j); 495 496 imgu_v4l2_unregister(imgu); 497 498 return r; 499 } 500 501 static void imgu_video_nodes_exit(struct imgu_device *imgu) 502 { 503 int i; 504 505 for (i = 0; i < IMGU_MAX_PIPE_NUM; i++) 506 imgu_dummybufs_cleanup(imgu, i); 507 508 imgu_v4l2_unregister(imgu); 509 } 510 511 /**************** PCI interface ****************/ 512 513 static irqreturn_t imgu_isr_threaded(int irq, void *imgu_ptr) 514 { 515 struct imgu_device *imgu = imgu_ptr; 516 struct imgu_media_pipe *imgu_pipe; 517 int p; 518 519 /* Dequeue / queue buffers */ 520 do { 521 u64 ns = ktime_get_ns(); 522 struct imgu_css_buffer *b; 523 struct imgu_buffer *buf = NULL; 524 unsigned int node, pipe; 525 bool dummy; 526 527 do { 528 mutex_lock(&imgu->lock); 529 b = imgu_css_buf_dequeue(&imgu->css); 530 mutex_unlock(&imgu->lock); 531 } while (PTR_ERR(b) == -EAGAIN); 532 533 if (IS_ERR(b)) { 534 if (PTR_ERR(b) != -EBUSY) /* All done */ 535 dev_err(&imgu->pci_dev->dev, 536 "failed to dequeue buffers (%ld)\n", 537 PTR_ERR(b)); 538 break; 539 } 540 541 node = imgu_map_node(imgu, b->queue); 542 pipe = b->pipe; 543 dummy = imgu_dummybufs_check(imgu, b, pipe); 544 if (!dummy) 545 buf = container_of(b, struct imgu_buffer, css_buf); 546 dev_dbg(&imgu->pci_dev->dev, 547 "dequeue %s %s buffer %d daddr 0x%x from css\n", 548 dummy ? "dummy" : "user", 549 imgu_node_map[node].name, 550 dummy ? 0 : buf->vid_buf.vbb.vb2_buf.index, 551 (u32)b->daddr); 552 553 if (dummy) 554 /* It was a dummy buffer, skip it */ 555 continue; 556 557 /* Fill vb2 buffer entries and tell it's ready */ 558 imgu_pipe = &imgu->imgu_pipe[pipe]; 559 if (!imgu_pipe->nodes[node].output) { 560 buf->vid_buf.vbb.vb2_buf.timestamp = ns; 561 buf->vid_buf.vbb.field = V4L2_FIELD_NONE; 562 buf->vid_buf.vbb.sequence = 563 atomic_inc_return( 564 &imgu_pipe->nodes[node].sequence); 565 dev_dbg(&imgu->pci_dev->dev, "vb2 buffer sequence %d", 566 buf->vid_buf.vbb.sequence); 567 } 568 imgu_buffer_done(imgu, &buf->vid_buf.vbb.vb2_buf, 569 imgu_css_buf_state(&buf->css_buf) == 570 IPU3_CSS_BUFFER_DONE ? 571 VB2_BUF_STATE_DONE : 572 VB2_BUF_STATE_ERROR); 573 mutex_lock(&imgu->lock); 574 if (imgu_css_queue_empty(&imgu->css)) 575 wake_up_all(&imgu->buf_drain_wq); 576 mutex_unlock(&imgu->lock); 577 } while (1); 578 579 /* 580 * Try to queue more buffers for CSS. 581 * qbuf_barrier is used to disable new buffers 582 * to be queued to CSS. 583 */ 584 if (!atomic_read(&imgu->qbuf_barrier)) 585 for_each_set_bit(p, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) 586 imgu_queue_buffers(imgu, false, p); 587 588 return IRQ_HANDLED; 589 } 590 591 static irqreturn_t imgu_isr(int irq, void *imgu_ptr) 592 { 593 struct imgu_device *imgu = imgu_ptr; 594 595 /* acknowledge interruption */ 596 if (imgu_css_irq_ack(&imgu->css) < 0) 597 return IRQ_NONE; 598 599 return IRQ_WAKE_THREAD; 600 } 601 602 static int imgu_pci_config_setup(struct pci_dev *dev) 603 { 604 u16 pci_command; 605 int r = pci_enable_msi(dev); 606 607 if (r) { 608 dev_err(&dev->dev, "failed to enable MSI (%d)\n", r); 609 return r; 610 } 611 612 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 613 pci_command |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | 614 PCI_COMMAND_INTX_DISABLE; 615 pci_write_config_word(dev, PCI_COMMAND, pci_command); 616 617 return 0; 618 } 619 620 static int imgu_pci_probe(struct pci_dev *pci_dev, 621 const struct pci_device_id *id) 622 { 623 struct imgu_device *imgu; 624 phys_addr_t phys; 625 unsigned long phys_len; 626 void __iomem *const *iomap; 627 int r; 628 629 imgu = devm_kzalloc(&pci_dev->dev, sizeof(*imgu), GFP_KERNEL); 630 if (!imgu) 631 return -ENOMEM; 632 633 imgu->pci_dev = pci_dev; 634 635 r = pcim_enable_device(pci_dev); 636 if (r) { 637 dev_err(&pci_dev->dev, "failed to enable device (%d)\n", r); 638 return r; 639 } 640 641 dev_info(&pci_dev->dev, "device 0x%x (rev: 0x%x)\n", 642 pci_dev->device, pci_dev->revision); 643 644 phys = pci_resource_start(pci_dev, IMGU_PCI_BAR); 645 phys_len = pci_resource_len(pci_dev, IMGU_PCI_BAR); 646 647 r = pcim_iomap_regions(pci_dev, 1 << IMGU_PCI_BAR, pci_name(pci_dev)); 648 if (r) { 649 dev_err(&pci_dev->dev, "failed to remap I/O memory (%d)\n", r); 650 return r; 651 } 652 dev_info(&pci_dev->dev, "physical base address %pap, %lu bytes\n", 653 &phys, phys_len); 654 655 iomap = pcim_iomap_table(pci_dev); 656 if (!iomap) { 657 dev_err(&pci_dev->dev, "failed to iomap table\n"); 658 return -ENODEV; 659 } 660 661 imgu->base = iomap[IMGU_PCI_BAR]; 662 663 pci_set_drvdata(pci_dev, imgu); 664 665 pci_set_master(pci_dev); 666 667 r = dma_coerce_mask_and_coherent(&pci_dev->dev, IMGU_DMA_MASK); 668 if (r) { 669 dev_err(&pci_dev->dev, "failed to set DMA mask (%d)\n", r); 670 return -ENODEV; 671 } 672 673 r = imgu_pci_config_setup(pci_dev); 674 if (r) 675 return r; 676 677 mutex_init(&imgu->lock); 678 mutex_init(&imgu->streaming_lock); 679 atomic_set(&imgu->qbuf_barrier, 0); 680 init_waitqueue_head(&imgu->buf_drain_wq); 681 682 r = imgu_css_set_powerup(&pci_dev->dev, imgu->base, 200); 683 if (r) { 684 dev_err(&pci_dev->dev, 685 "failed to power up CSS (%d)\n", r); 686 goto out_mutex_destroy; 687 } 688 689 imgu->mmu = imgu_mmu_init(&pci_dev->dev, imgu->base); 690 if (IS_ERR(imgu->mmu)) { 691 r = PTR_ERR(imgu->mmu); 692 dev_err(&pci_dev->dev, "failed to initialize MMU (%d)\n", r); 693 goto out_css_powerdown; 694 } 695 696 r = imgu_dmamap_init(imgu); 697 if (r) { 698 dev_err(&pci_dev->dev, 699 "failed to initialize DMA mapping (%d)\n", r); 700 goto out_mmu_exit; 701 } 702 703 /* ISP programming */ 704 r = imgu_css_init(&pci_dev->dev, &imgu->css, imgu->base, phys_len); 705 if (r) { 706 dev_err(&pci_dev->dev, "failed to initialize CSS (%d)\n", r); 707 goto out_dmamap_exit; 708 } 709 710 /* v4l2 sub-device registration */ 711 r = imgu_video_nodes_init(imgu); 712 if (r) { 713 dev_err(&pci_dev->dev, "failed to create V4L2 devices (%d)\n", 714 r); 715 goto out_css_cleanup; 716 } 717 718 r = devm_request_threaded_irq(&pci_dev->dev, pci_dev->irq, 719 imgu_isr, imgu_isr_threaded, 720 IRQF_SHARED, IMGU_NAME, imgu); 721 if (r) { 722 dev_err(&pci_dev->dev, "failed to request IRQ (%d)\n", r); 723 goto out_video_exit; 724 } 725 726 pm_runtime_put_noidle(&pci_dev->dev); 727 pm_runtime_allow(&pci_dev->dev); 728 729 return 0; 730 731 out_video_exit: 732 imgu_video_nodes_exit(imgu); 733 out_css_cleanup: 734 imgu_css_cleanup(&imgu->css); 735 out_dmamap_exit: 736 imgu_dmamap_exit(imgu); 737 out_mmu_exit: 738 imgu_mmu_exit(imgu->mmu); 739 out_css_powerdown: 740 imgu_css_set_powerdown(&pci_dev->dev, imgu->base); 741 out_mutex_destroy: 742 mutex_destroy(&imgu->streaming_lock); 743 mutex_destroy(&imgu->lock); 744 745 return r; 746 } 747 748 static void imgu_pci_remove(struct pci_dev *pci_dev) 749 { 750 struct imgu_device *imgu = pci_get_drvdata(pci_dev); 751 752 pm_runtime_forbid(&pci_dev->dev); 753 pm_runtime_get_noresume(&pci_dev->dev); 754 755 imgu_video_nodes_exit(imgu); 756 imgu_css_cleanup(&imgu->css); 757 imgu_css_set_powerdown(&pci_dev->dev, imgu->base); 758 imgu_dmamap_exit(imgu); 759 imgu_mmu_exit(imgu->mmu); 760 mutex_destroy(&imgu->streaming_lock); 761 mutex_destroy(&imgu->lock); 762 } 763 764 static int __maybe_unused imgu_suspend(struct device *dev) 765 { 766 struct pci_dev *pci_dev = to_pci_dev(dev); 767 struct imgu_device *imgu = pci_get_drvdata(pci_dev); 768 769 dev_dbg(dev, "enter %s\n", __func__); 770 imgu->suspend_in_stream = imgu_css_is_streaming(&imgu->css); 771 if (!imgu->suspend_in_stream) 772 goto out; 773 /* Block new buffers to be queued to CSS. */ 774 atomic_set(&imgu->qbuf_barrier, 1); 775 /* 776 * Wait for currently running irq handler to be done so that 777 * no new buffers will be queued to fw later. 778 */ 779 synchronize_irq(pci_dev->irq); 780 /* Wait until all buffers in CSS are done. */ 781 if (!wait_event_timeout(imgu->buf_drain_wq, 782 imgu_css_queue_empty(&imgu->css), msecs_to_jiffies(1000))) 783 dev_err(dev, "wait buffer drain timeout.\n"); 784 785 imgu_css_stop_streaming(&imgu->css); 786 atomic_set(&imgu->qbuf_barrier, 0); 787 imgu_powerdown(imgu); 788 pm_runtime_force_suspend(dev); 789 out: 790 dev_dbg(dev, "leave %s\n", __func__); 791 return 0; 792 } 793 794 static int __maybe_unused imgu_resume(struct device *dev) 795 { 796 struct imgu_device *imgu = dev_get_drvdata(dev); 797 int r = 0; 798 unsigned int pipe; 799 800 dev_dbg(dev, "enter %s\n", __func__); 801 802 if (!imgu->suspend_in_stream) 803 goto out; 804 805 pm_runtime_force_resume(dev); 806 807 r = imgu_powerup(imgu); 808 if (r) { 809 dev_err(dev, "failed to power up imgu\n"); 810 goto out; 811 } 812 813 /* Start CSS streaming */ 814 r = imgu_css_start_streaming(&imgu->css); 815 if (r) { 816 dev_err(dev, "failed to resume css streaming (%d)", r); 817 goto out; 818 } 819 820 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) { 821 r = imgu_queue_buffers(imgu, true, pipe); 822 if (r) 823 dev_err(dev, "failed to queue buffers to pipe %d (%d)", 824 pipe, r); 825 } 826 827 out: 828 dev_dbg(dev, "leave %s\n", __func__); 829 830 return r; 831 } 832 833 /* 834 * PCI rpm framework checks the existence of driver rpm callbacks. 835 * Place a dummy callback here to avoid rpm going into error state. 836 */ 837 static __maybe_unused int imgu_rpm_dummy_cb(struct device *dev) 838 { 839 return 0; 840 } 841 842 static const struct dev_pm_ops imgu_pm_ops = { 843 SET_RUNTIME_PM_OPS(&imgu_rpm_dummy_cb, &imgu_rpm_dummy_cb, NULL) 844 SET_SYSTEM_SLEEP_PM_OPS(&imgu_suspend, &imgu_resume) 845 }; 846 847 static const struct pci_device_id imgu_pci_tbl[] = { 848 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, IMGU_PCI_ID) }, 849 { 0, } 850 }; 851 852 MODULE_DEVICE_TABLE(pci, imgu_pci_tbl); 853 854 static struct pci_driver imgu_pci_driver = { 855 .name = IMGU_NAME, 856 .id_table = imgu_pci_tbl, 857 .probe = imgu_pci_probe, 858 .remove = imgu_pci_remove, 859 .driver = { 860 .pm = &imgu_pm_ops, 861 }, 862 }; 863 864 module_pci_driver(imgu_pci_driver); 865 866 MODULE_AUTHOR("Tuukka Toivonen <tuukka.toivonen@intel.com>"); 867 MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>"); 868 MODULE_AUTHOR("Jian Xu Zheng <jian.xu.zheng@intel.com>"); 869 MODULE_AUTHOR("Yuning Pu <yuning.pu@intel.com>"); 870 MODULE_AUTHOR("Yong Zhi <yong.zhi@intel.com>"); 871 MODULE_LICENSE("GPL v2"); 872 MODULE_DESCRIPTION("Intel ipu3_imgu PCI driver"); 873