1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Command DMA 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 9 #include <asm/cacheflush.h> 10 #include <linux/device.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/host1x.h> 13 #include <linux/interrupt.h> 14 #include <linux/kernel.h> 15 #include <linux/kfifo.h> 16 #include <linux/slab.h> 17 #include <trace/events/host1x.h> 18 19 #include "cdma.h" 20 #include "channel.h" 21 #include "dev.h" 22 #include "debug.h" 23 #include "job.h" 24 25 /* 26 * push_buffer 27 * 28 * The push buffer is a circular array of words to be fetched by command DMA. 29 * Note that it works slightly differently to the sync queue; fence == pos 30 * means that the push buffer is full, not empty. 31 */ 32 33 /* 34 * Typically the commands written into the push buffer are a pair of words. We 35 * use slots to represent each of these pairs and to simplify things. Note the 36 * strange number of slots allocated here. 512 slots will fit exactly within a 37 * single memory page. We also need one additional word at the end of the push 38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to 39 * the beginning of the push buffer. With 512 slots, this means that we'll use 40 * 2 memory pages and waste 4092 bytes of the second page that will never be 41 * used. 42 */ 43 #define HOST1X_PUSHBUFFER_SLOTS 511 44 45 /* 46 * Clean up push buffer resources 47 */ 48 static void host1x_pushbuffer_destroy(struct push_buffer *pb) 49 { 50 struct host1x_cdma *cdma = pb_to_cdma(pb); 51 struct host1x *host1x = cdma_to_host1x(cdma); 52 53 if (!pb->mapped) 54 return; 55 56 if (host1x->domain) { 57 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); 58 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); 59 } 60 61 dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); 62 63 pb->mapped = NULL; 64 pb->phys = 0; 65 } 66 67 /* 68 * Init push buffer resources 69 */ 70 static int host1x_pushbuffer_init(struct push_buffer *pb) 71 { 72 struct host1x_cdma *cdma = pb_to_cdma(pb); 73 struct host1x *host1x = cdma_to_host1x(cdma); 74 struct iova *alloc; 75 u32 size; 76 int err; 77 78 pb->mapped = NULL; 79 pb->phys = 0; 80 pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; 81 82 size = pb->size + 4; 83 84 /* initialize buffer pointers */ 85 pb->fence = pb->size - 8; 86 pb->pos = 0; 87 88 if (host1x->domain) { 89 unsigned long shift; 90 91 size = iova_align(&host1x->iova, size); 92 93 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 94 GFP_KERNEL); 95 if (!pb->mapped) 96 return -ENOMEM; 97 98 shift = iova_shift(&host1x->iova); 99 alloc = alloc_iova(&host1x->iova, size >> shift, 100 host1x->iova_end >> shift, true); 101 if (!alloc) { 102 err = -ENOMEM; 103 goto iommu_free_mem; 104 } 105 106 pb->dma = iova_dma_addr(&host1x->iova, alloc); 107 err = iommu_map(host1x->domain, pb->dma, pb->phys, size, 108 IOMMU_READ); 109 if (err) 110 goto iommu_free_iova; 111 } else { 112 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 113 GFP_KERNEL); 114 if (!pb->mapped) 115 return -ENOMEM; 116 117 pb->dma = pb->phys; 118 } 119 120 pb->alloc_size = size; 121 122 host1x_hw_pushbuffer_init(host1x, pb); 123 124 return 0; 125 126 iommu_free_iova: 127 __free_iova(&host1x->iova, alloc); 128 iommu_free_mem: 129 dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); 130 131 return err; 132 } 133 134 /* 135 * Push two words to the push buffer 136 * Caller must ensure push buffer is not full 137 */ 138 static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) 139 { 140 u32 *p = (u32 *)((void *)pb->mapped + pb->pos); 141 142 WARN_ON(pb->pos == pb->fence); 143 *(p++) = op1; 144 *(p++) = op2; 145 pb->pos += 8; 146 147 if (pb->pos >= pb->size) 148 pb->pos -= pb->size; 149 } 150 151 /* 152 * Pop a number of two word slots from the push buffer 153 * Caller must ensure push buffer is not empty 154 */ 155 static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) 156 { 157 /* Advance the next write position */ 158 pb->fence += slots * 8; 159 160 if (pb->fence >= pb->size) 161 pb->fence -= pb->size; 162 } 163 164 /* 165 * Return the number of two word slots free in the push buffer 166 */ 167 static u32 host1x_pushbuffer_space(struct push_buffer *pb) 168 { 169 unsigned int fence = pb->fence; 170 171 if (pb->fence < pb->pos) 172 fence += pb->size; 173 174 return (fence - pb->pos) / 8; 175 } 176 177 /* 178 * Sleep (if necessary) until the requested event happens 179 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. 180 * - Returns 1 181 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer 182 * - Return the amount of space (> 0) 183 * Must be called with the cdma lock held. 184 */ 185 unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, 186 enum cdma_event event) 187 { 188 for (;;) { 189 struct push_buffer *pb = &cdma->push_buffer; 190 unsigned int space; 191 192 switch (event) { 193 case CDMA_EVENT_SYNC_QUEUE_EMPTY: 194 space = list_empty(&cdma->sync_queue) ? 1 : 0; 195 break; 196 197 case CDMA_EVENT_PUSH_BUFFER_SPACE: 198 space = host1x_pushbuffer_space(pb); 199 break; 200 201 default: 202 WARN_ON(1); 203 return -EINVAL; 204 } 205 206 if (space) 207 return space; 208 209 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 210 event); 211 212 /* If somebody has managed to already start waiting, yield */ 213 if (cdma->event != CDMA_EVENT_NONE) { 214 mutex_unlock(&cdma->lock); 215 schedule(); 216 mutex_lock(&cdma->lock); 217 continue; 218 } 219 220 cdma->event = event; 221 222 mutex_unlock(&cdma->lock); 223 wait_for_completion(&cdma->complete); 224 mutex_lock(&cdma->lock); 225 } 226 227 return 0; 228 } 229 230 /* 231 * Sleep (if necessary) until the push buffer has enough free space. 232 * 233 * Must be called with the cdma lock held. 234 */ 235 static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x, 236 struct host1x_cdma *cdma, 237 unsigned int needed) 238 { 239 while (true) { 240 struct push_buffer *pb = &cdma->push_buffer; 241 unsigned int space; 242 243 space = host1x_pushbuffer_space(pb); 244 if (space >= needed) 245 break; 246 247 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 248 CDMA_EVENT_PUSH_BUFFER_SPACE); 249 250 host1x_hw_cdma_flush(host1x, cdma); 251 252 /* If somebody has managed to already start waiting, yield */ 253 if (cdma->event != CDMA_EVENT_NONE) { 254 mutex_unlock(&cdma->lock); 255 schedule(); 256 mutex_lock(&cdma->lock); 257 continue; 258 } 259 260 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE; 261 262 mutex_unlock(&cdma->lock); 263 wait_for_completion(&cdma->complete); 264 mutex_lock(&cdma->lock); 265 } 266 267 return 0; 268 } 269 /* 270 * Start timer that tracks the time spent by the job. 271 * Must be called with the cdma lock held. 272 */ 273 static void cdma_start_timer_locked(struct host1x_cdma *cdma, 274 struct host1x_job *job) 275 { 276 if (cdma->timeout.client) { 277 /* timer already started */ 278 return; 279 } 280 281 cdma->timeout.client = job->client; 282 cdma->timeout.syncpt = job->syncpt; 283 cdma->timeout.syncpt_val = job->syncpt_end; 284 cdma->timeout.start_ktime = ktime_get(); 285 286 schedule_delayed_work(&cdma->timeout.wq, 287 msecs_to_jiffies(job->timeout)); 288 } 289 290 /* 291 * Stop timer when a buffer submission completes. 292 * Must be called with the cdma lock held. 293 */ 294 static void stop_cdma_timer_locked(struct host1x_cdma *cdma) 295 { 296 cancel_delayed_work(&cdma->timeout.wq); 297 cdma->timeout.client = NULL; 298 } 299 300 /* 301 * For all sync queue entries that have already finished according to the 302 * current sync point registers: 303 * - unpin & unref their mems 304 * - pop their push buffer slots 305 * - remove them from the sync queue 306 * This is normally called from the host code's worker thread, but can be 307 * called manually if necessary. 308 * Must be called with the cdma lock held. 309 */ 310 static void update_cdma_locked(struct host1x_cdma *cdma) 311 { 312 bool signal = false; 313 struct host1x_job *job, *n; 314 315 /* 316 * Walk the sync queue, reading the sync point registers as necessary, 317 * to consume as many sync queue entries as possible without blocking 318 */ 319 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { 320 struct host1x_syncpt *sp = job->syncpt; 321 322 /* Check whether this syncpt has completed, and bail if not */ 323 if (!host1x_syncpt_is_expired(sp, job->syncpt_end) && 324 !job->cancelled) { 325 /* Start timer on next pending syncpt */ 326 if (job->timeout) 327 cdma_start_timer_locked(cdma, job); 328 329 break; 330 } 331 332 /* Cancel timeout, when a buffer completes */ 333 if (cdma->timeout.client) 334 stop_cdma_timer_locked(cdma); 335 336 /* Unpin the memory */ 337 host1x_job_unpin(job); 338 339 /* Pop push buffer slots */ 340 if (job->num_slots) { 341 struct push_buffer *pb = &cdma->push_buffer; 342 343 host1x_pushbuffer_pop(pb, job->num_slots); 344 345 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) 346 signal = true; 347 } 348 349 list_del(&job->list); 350 host1x_job_put(job); 351 } 352 353 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && 354 list_empty(&cdma->sync_queue)) 355 signal = true; 356 357 if (signal) { 358 cdma->event = CDMA_EVENT_NONE; 359 complete(&cdma->complete); 360 } 361 } 362 363 void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, 364 struct device *dev) 365 { 366 struct host1x *host1x = cdma_to_host1x(cdma); 367 u32 restart_addr, syncpt_incrs, syncpt_val; 368 struct host1x_job *job, *next_job = NULL; 369 370 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 371 372 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", 373 __func__, syncpt_val); 374 375 /* 376 * Move the sync_queue read pointer to the first entry that hasn't 377 * completed based on the current HW syncpt value. It's likely there 378 * won't be any (i.e. we're still at the head), but covers the case 379 * where a syncpt incr happens just prior/during the teardown. 380 */ 381 382 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", 383 __func__); 384 385 list_for_each_entry(job, &cdma->sync_queue, list) { 386 if (syncpt_val < job->syncpt_end) { 387 388 if (!list_is_last(&job->list, &cdma->sync_queue)) 389 next_job = list_next_entry(job, list); 390 391 goto syncpt_incr; 392 } 393 394 host1x_job_dump(dev, job); 395 } 396 397 /* all jobs have been completed */ 398 job = NULL; 399 400 syncpt_incr: 401 402 /* 403 * Increment with CPU the remaining syncpts of a partially executed job. 404 * 405 * CDMA will continue execution starting with the next job or will get 406 * into idle state. 407 */ 408 if (next_job) 409 restart_addr = next_job->first_get; 410 else 411 restart_addr = cdma->last_pos; 412 413 if (!job) 414 goto resume; 415 416 /* do CPU increments for the remaining syncpts */ 417 if (job->syncpt_recovery) { 418 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n", 419 __func__); 420 421 /* won't need a timeout when replayed */ 422 job->timeout = 0; 423 424 syncpt_incrs = job->syncpt_end - syncpt_val; 425 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); 426 427 host1x_job_dump(dev, job); 428 429 /* safe to use CPU to incr syncpts */ 430 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, 431 syncpt_incrs, job->syncpt_end, 432 job->num_slots); 433 434 dev_dbg(dev, "%s: finished sync_queue modification\n", 435 __func__); 436 } else { 437 struct host1x_job *failed_job = job; 438 439 host1x_job_dump(dev, job); 440 441 host1x_syncpt_set_locked(job->syncpt); 442 failed_job->cancelled = true; 443 444 list_for_each_entry_continue(job, &cdma->sync_queue, list) { 445 unsigned int i; 446 447 if (job->syncpt != failed_job->syncpt) 448 continue; 449 450 for (i = 0; i < job->num_slots; i++) { 451 unsigned int slot = (job->first_get/8 + i) % 452 HOST1X_PUSHBUFFER_SLOTS; 453 u32 *mapped = cdma->push_buffer.mapped; 454 455 /* 456 * Overwrite opcodes with 0 word writes 457 * to offset 0xbad. This does nothing but 458 * has a easily detected signature in debug 459 * traces. 460 * 461 * On systems with MLOCK enforcement enabled, 462 * the above 0 word writes would fall foul of 463 * the enforcement. As such, in the first slot 464 * put a RESTART_W opcode to the beginning 465 * of the next job. We don't use this for older 466 * chips since those only support the RESTART 467 * opcode with inconvenient alignment requirements. 468 */ 469 if (i == 0 && host1x->info->has_wide_gather) { 470 unsigned int next_job = (job->first_get/8 + job->num_slots) 471 % HOST1X_PUSHBUFFER_SLOTS; 472 mapped[2*slot+0] = (0xd << 28) | (next_job * 2); 473 mapped[2*slot+1] = 0x0; 474 } else { 475 mapped[2*slot+0] = 0x1bad0000; 476 mapped[2*slot+1] = 0x1bad0000; 477 } 478 } 479 480 job->cancelled = true; 481 } 482 483 wmb(); 484 485 update_cdma_locked(cdma); 486 } 487 488 resume: 489 /* roll back DMAGET and start up channel again */ 490 host1x_hw_cdma_resume(host1x, cdma, restart_addr); 491 } 492 493 /* 494 * Create a cdma 495 */ 496 int host1x_cdma_init(struct host1x_cdma *cdma) 497 { 498 int err; 499 500 mutex_init(&cdma->lock); 501 init_completion(&cdma->complete); 502 503 INIT_LIST_HEAD(&cdma->sync_queue); 504 505 cdma->event = CDMA_EVENT_NONE; 506 cdma->running = false; 507 cdma->torndown = false; 508 509 err = host1x_pushbuffer_init(&cdma->push_buffer); 510 if (err) 511 return err; 512 513 return 0; 514 } 515 516 /* 517 * Destroy a cdma 518 */ 519 int host1x_cdma_deinit(struct host1x_cdma *cdma) 520 { 521 struct push_buffer *pb = &cdma->push_buffer; 522 struct host1x *host1x = cdma_to_host1x(cdma); 523 524 if (cdma->running) { 525 pr_warn("%s: CDMA still running\n", __func__); 526 return -EBUSY; 527 } 528 529 host1x_pushbuffer_destroy(pb); 530 host1x_hw_cdma_timeout_destroy(host1x, cdma); 531 532 return 0; 533 } 534 535 /* 536 * Begin a cdma submit 537 */ 538 int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) 539 { 540 struct host1x *host1x = cdma_to_host1x(cdma); 541 542 mutex_lock(&cdma->lock); 543 544 /* 545 * Check if syncpoint was locked due to previous job timeout. 546 * This needs to be done within the cdma lock to avoid a race 547 * with the timeout handler. 548 */ 549 if (job->syncpt->locked) { 550 mutex_unlock(&cdma->lock); 551 return -EPERM; 552 } 553 554 if (job->timeout) { 555 /* init state on first submit with timeout value */ 556 if (!cdma->timeout.initialized) { 557 int err; 558 559 err = host1x_hw_cdma_timeout_init(host1x, cdma); 560 if (err) { 561 mutex_unlock(&cdma->lock); 562 return err; 563 } 564 } 565 } 566 567 if (!cdma->running) 568 host1x_hw_cdma_start(host1x, cdma); 569 570 cdma->slots_free = 0; 571 cdma->slots_used = 0; 572 cdma->first_get = cdma->push_buffer.pos; 573 574 trace_host1x_cdma_begin(dev_name(job->channel->dev)); 575 return 0; 576 } 577 578 /* 579 * Push two words into a push buffer slot 580 * Blocks as necessary if the push buffer is full. 581 */ 582 void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) 583 { 584 struct host1x *host1x = cdma_to_host1x(cdma); 585 struct push_buffer *pb = &cdma->push_buffer; 586 u32 slots_free = cdma->slots_free; 587 588 if (host1x_debug_trace_cmdbuf) 589 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), 590 op1, op2); 591 592 if (slots_free == 0) { 593 host1x_hw_cdma_flush(host1x, cdma); 594 slots_free = host1x_cdma_wait_locked(cdma, 595 CDMA_EVENT_PUSH_BUFFER_SPACE); 596 } 597 598 cdma->slots_free = slots_free - 1; 599 cdma->slots_used++; 600 host1x_pushbuffer_push(pb, op1, op2); 601 } 602 603 /* 604 * Push four words into two consecutive push buffer slots. Note that extra 605 * care needs to be taken not to split the two slots across the end of the 606 * push buffer. Otherwise the RESTART opcode at the end of the push buffer 607 * that ensures processing will restart at the beginning will break up the 608 * four words. 609 * 610 * Blocks as necessary if the push buffer is full. 611 */ 612 void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2, 613 u32 op3, u32 op4) 614 { 615 struct host1x_channel *channel = cdma_to_channel(cdma); 616 struct host1x *host1x = cdma_to_host1x(cdma); 617 struct push_buffer *pb = &cdma->push_buffer; 618 unsigned int space = cdma->slots_free; 619 unsigned int needed = 2, extra = 0; 620 621 if (host1x_debug_trace_cmdbuf) 622 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2, 623 op3, op4); 624 625 /* compute number of extra slots needed for padding */ 626 if (pb->pos + 16 > pb->size) { 627 extra = (pb->size - pb->pos) / 8; 628 needed += extra; 629 } 630 631 host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed); 632 space = host1x_pushbuffer_space(pb); 633 634 cdma->slots_free = space - needed; 635 cdma->slots_used += needed; 636 637 if (extra > 0) { 638 /* 639 * If there isn't enough space at the tail of the pushbuffer, 640 * insert a RESTART(0) here to go back to the beginning. 641 * The code above adjusted the indexes appropriately. 642 */ 643 host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000); 644 } 645 646 host1x_pushbuffer_push(pb, op1, op2); 647 host1x_pushbuffer_push(pb, op3, op4); 648 } 649 650 /* 651 * End a cdma submit 652 * Kick off DMA, add job to the sync queue, and a number of slots to be freed 653 * from the pushbuffer. The handles for a submit must all be pinned at the same 654 * time, but they can be unpinned in smaller chunks. 655 */ 656 void host1x_cdma_end(struct host1x_cdma *cdma, 657 struct host1x_job *job) 658 { 659 struct host1x *host1x = cdma_to_host1x(cdma); 660 bool idle = list_empty(&cdma->sync_queue); 661 662 host1x_hw_cdma_flush(host1x, cdma); 663 664 job->first_get = cdma->first_get; 665 job->num_slots = cdma->slots_used; 666 host1x_job_get(job); 667 list_add_tail(&job->list, &cdma->sync_queue); 668 669 /* start timer on idle -> active transitions */ 670 if (job->timeout && idle) 671 cdma_start_timer_locked(cdma, job); 672 673 trace_host1x_cdma_end(dev_name(job->channel->dev)); 674 mutex_unlock(&cdma->lock); 675 } 676 677 /* 678 * Update cdma state according to current sync point values 679 */ 680 void host1x_cdma_update(struct host1x_cdma *cdma) 681 { 682 mutex_lock(&cdma->lock); 683 update_cdma_locked(cdma); 684 mutex_unlock(&cdma->lock); 685 } 686