1 /* 2 * Tegra host1x Job 3 * 4 * Copyright (c) 2010-2015, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/dma-mapping.h> 20 #include <linux/err.h> 21 #include <linux/host1x.h> 22 #include <linux/kref.h> 23 #include <linux/module.h> 24 #include <linux/scatterlist.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 #include <trace/events/host1x.h> 28 29 #include "channel.h" 30 #include "dev.h" 31 #include "job.h" 32 #include "syncpt.h" 33 34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 35 u32 num_cmdbufs, u32 num_relocs, 36 u32 num_waitchks) 37 { 38 struct host1x_job *job = NULL; 39 unsigned int num_unpins = num_cmdbufs + num_relocs; 40 u64 total; 41 void *mem; 42 43 /* Check that we're not going to overflow */ 44 total = sizeof(struct host1x_job) + 45 (u64)num_relocs * sizeof(struct host1x_reloc) + 46 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) + 47 (u64)num_waitchks * sizeof(struct host1x_waitchk) + 48 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) + 49 (u64)num_unpins * sizeof(dma_addr_t) + 50 (u64)num_unpins * sizeof(u32 *); 51 if (total > ULONG_MAX) 52 return NULL; 53 54 mem = job = kzalloc(total, GFP_KERNEL); 55 if (!job) 56 return NULL; 57 58 kref_init(&job->ref); 59 job->channel = ch; 60 61 /* Redistribute memory to the structs */ 62 mem += sizeof(struct host1x_job); 63 job->relocarray = num_relocs ? mem : NULL; 64 mem += num_relocs * sizeof(struct host1x_reloc); 65 job->unpins = num_unpins ? mem : NULL; 66 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 67 job->waitchk = num_waitchks ? mem : NULL; 68 mem += num_waitchks * sizeof(struct host1x_waitchk); 69 job->gathers = num_cmdbufs ? mem : NULL; 70 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 71 job->addr_phys = num_unpins ? mem : NULL; 72 73 job->reloc_addr_phys = job->addr_phys; 74 job->gather_addr_phys = &job->addr_phys[num_relocs]; 75 76 return job; 77 } 78 EXPORT_SYMBOL(host1x_job_alloc); 79 80 struct host1x_job *host1x_job_get(struct host1x_job *job) 81 { 82 kref_get(&job->ref); 83 return job; 84 } 85 EXPORT_SYMBOL(host1x_job_get); 86 87 static void job_free(struct kref *ref) 88 { 89 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 90 91 kfree(job); 92 } 93 94 void host1x_job_put(struct host1x_job *job) 95 { 96 kref_put(&job->ref, job_free); 97 } 98 EXPORT_SYMBOL(host1x_job_put); 99 100 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 101 u32 words, u32 offset) 102 { 103 struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers]; 104 105 cur_gather->words = words; 106 cur_gather->bo = bo; 107 cur_gather->offset = offset; 108 job->num_gathers++; 109 } 110 EXPORT_SYMBOL(host1x_job_add_gather); 111 112 /* 113 * NULL an already satisfied WAIT_SYNCPT host method, by patching its 114 * args in the command stream. The method data is changed to reference 115 * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt 116 * with a matching threshold value of 0, so is guaranteed to be popped 117 * by the host HW. 118 */ 119 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp, 120 struct host1x_bo *h, u32 offset) 121 { 122 void *patch_addr = NULL; 123 124 /* patch the wait */ 125 patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT); 126 if (patch_addr) { 127 host1x_syncpt_patch_wait(sp, 128 patch_addr + (offset & ~PAGE_MASK)); 129 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr); 130 } else 131 pr_err("Could not map cmdbuf for wait check\n"); 132 } 133 134 /* 135 * Check driver supplied waitchk structs for syncpt thresholds 136 * that have already been satisfied and NULL the comparison (to 137 * avoid a wrap condition in the HW). 138 */ 139 static int do_waitchks(struct host1x_job *job, struct host1x *host, 140 struct host1x_job_gather *g) 141 { 142 struct host1x_bo *patch = g->bo; 143 int i; 144 145 /* compare syncpt vs wait threshold */ 146 for (i = 0; i < job->num_waitchk; i++) { 147 struct host1x_waitchk *wait = &job->waitchk[i]; 148 struct host1x_syncpt *sp = 149 host1x_syncpt_get(host, wait->syncpt_id); 150 151 /* validate syncpt id */ 152 if (wait->syncpt_id > host1x_syncpt_nb_pts(host)) 153 continue; 154 155 /* skip all other gathers */ 156 if (patch != wait->bo) 157 continue; 158 159 trace_host1x_syncpt_wait_check(wait->bo, wait->offset, 160 wait->syncpt_id, wait->thresh, 161 host1x_syncpt_read_min(sp)); 162 163 if (host1x_syncpt_is_expired(sp, wait->thresh)) { 164 dev_dbg(host->dev, 165 "drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n", 166 wait->syncpt_id, sp->name, wait->thresh, 167 host1x_syncpt_read_min(sp)); 168 169 host1x_syncpt_patch_offset(sp, patch, 170 g->offset + wait->offset); 171 } 172 173 wait->bo = NULL; 174 } 175 176 return 0; 177 } 178 179 static unsigned int pin_job(struct host1x *host, struct host1x_job *job) 180 { 181 unsigned int i; 182 int err; 183 184 job->num_unpins = 0; 185 186 for (i = 0; i < job->num_relocs; i++) { 187 struct host1x_reloc *reloc = &job->relocarray[i]; 188 struct sg_table *sgt; 189 dma_addr_t phys_addr; 190 191 reloc->target.bo = host1x_bo_get(reloc->target.bo); 192 if (!reloc->target.bo) { 193 err = -EINVAL; 194 goto unpin; 195 } 196 197 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt); 198 if (!phys_addr) { 199 err = -EINVAL; 200 goto unpin; 201 } 202 203 job->addr_phys[job->num_unpins] = phys_addr; 204 job->unpins[job->num_unpins].bo = reloc->target.bo; 205 job->unpins[job->num_unpins].sgt = sgt; 206 job->num_unpins++; 207 } 208 209 for (i = 0; i < job->num_gathers; i++) { 210 struct host1x_job_gather *g = &job->gathers[i]; 211 size_t gather_size = 0; 212 struct scatterlist *sg; 213 struct sg_table *sgt; 214 dma_addr_t phys_addr; 215 unsigned long shift; 216 struct iova *alloc; 217 unsigned int j; 218 219 g->bo = host1x_bo_get(g->bo); 220 if (!g->bo) { 221 err = -EINVAL; 222 goto unpin; 223 } 224 225 phys_addr = host1x_bo_pin(g->bo, &sgt); 226 if (!phys_addr) { 227 err = -EINVAL; 228 goto unpin; 229 } 230 231 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { 232 for_each_sg(sgt->sgl, sg, sgt->nents, j) 233 gather_size += sg->length; 234 gather_size = iova_align(&host->iova, gather_size); 235 236 shift = iova_shift(&host->iova); 237 alloc = alloc_iova(&host->iova, gather_size >> shift, 238 host->iova_end >> shift, true); 239 if (!alloc) { 240 err = -ENOMEM; 241 goto unpin; 242 } 243 244 err = iommu_map_sg(host->domain, 245 iova_dma_addr(&host->iova, alloc), 246 sgt->sgl, sgt->nents, IOMMU_READ); 247 if (err == 0) { 248 __free_iova(&host->iova, alloc); 249 err = -EINVAL; 250 goto unpin; 251 } 252 253 job->addr_phys[job->num_unpins] = 254 iova_dma_addr(&host->iova, alloc); 255 job->unpins[job->num_unpins].size = gather_size; 256 } else { 257 job->addr_phys[job->num_unpins] = phys_addr; 258 } 259 260 job->gather_addr_phys[i] = job->addr_phys[job->num_unpins]; 261 262 job->unpins[job->num_unpins].bo = g->bo; 263 job->unpins[job->num_unpins].sgt = sgt; 264 job->num_unpins++; 265 } 266 267 return 0; 268 269 unpin: 270 host1x_job_unpin(job); 271 return err; 272 } 273 274 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g) 275 { 276 int i = 0; 277 u32 last_page = ~0; 278 void *cmdbuf_page_addr = NULL; 279 struct host1x_bo *cmdbuf = g->bo; 280 281 /* pin & patch the relocs for one gather */ 282 for (i = 0; i < job->num_relocs; i++) { 283 struct host1x_reloc *reloc = &job->relocarray[i]; 284 u32 reloc_addr = (job->reloc_addr_phys[i] + 285 reloc->target.offset) >> reloc->shift; 286 u32 *target; 287 288 /* skip all other gathers */ 289 if (cmdbuf != reloc->cmdbuf.bo) 290 continue; 291 292 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 293 target = (u32 *)job->gather_copy_mapped + 294 reloc->cmdbuf.offset / sizeof(u32) + 295 g->offset / sizeof(u32); 296 goto patch_reloc; 297 } 298 299 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) { 300 if (cmdbuf_page_addr) 301 host1x_bo_kunmap(cmdbuf, last_page, 302 cmdbuf_page_addr); 303 304 cmdbuf_page_addr = host1x_bo_kmap(cmdbuf, 305 reloc->cmdbuf.offset >> PAGE_SHIFT); 306 last_page = reloc->cmdbuf.offset >> PAGE_SHIFT; 307 308 if (unlikely(!cmdbuf_page_addr)) { 309 pr_err("Could not map cmdbuf for relocation\n"); 310 return -ENOMEM; 311 } 312 } 313 314 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK); 315 patch_reloc: 316 *target = reloc_addr; 317 } 318 319 if (cmdbuf_page_addr) 320 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr); 321 322 return 0; 323 } 324 325 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 326 unsigned int offset) 327 { 328 offset *= sizeof(u32); 329 330 if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset) 331 return false; 332 333 /* relocation shift value validation isn't implemented yet */ 334 if (reloc->shift) 335 return false; 336 337 return true; 338 } 339 340 struct host1x_firewall { 341 struct host1x_job *job; 342 struct device *dev; 343 344 unsigned int num_relocs; 345 struct host1x_reloc *reloc; 346 347 struct host1x_bo *cmdbuf; 348 unsigned int offset; 349 350 u32 words; 351 u32 class; 352 u32 reg; 353 u32 mask; 354 u32 count; 355 }; 356 357 static int check_register(struct host1x_firewall *fw, unsigned long offset) 358 { 359 if (!fw->job->is_addr_reg) 360 return 0; 361 362 if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) { 363 if (!fw->num_relocs) 364 return -EINVAL; 365 366 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset)) 367 return -EINVAL; 368 369 fw->num_relocs--; 370 fw->reloc++; 371 } 372 373 return 0; 374 } 375 376 static int check_class(struct host1x_firewall *fw, u32 class) 377 { 378 if (!fw->job->is_valid_class) { 379 if (fw->class != class) 380 return -EINVAL; 381 } else { 382 if (!fw->job->is_valid_class(fw->class)) 383 return -EINVAL; 384 } 385 386 return 0; 387 } 388 389 static int check_mask(struct host1x_firewall *fw) 390 { 391 u32 mask = fw->mask; 392 u32 reg = fw->reg; 393 int ret; 394 395 while (mask) { 396 if (fw->words == 0) 397 return -EINVAL; 398 399 if (mask & 1) { 400 ret = check_register(fw, reg); 401 if (ret < 0) 402 return ret; 403 404 fw->words--; 405 fw->offset++; 406 } 407 mask >>= 1; 408 reg++; 409 } 410 411 return 0; 412 } 413 414 static int check_incr(struct host1x_firewall *fw) 415 { 416 u32 count = fw->count; 417 u32 reg = fw->reg; 418 int ret; 419 420 while (count) { 421 if (fw->words == 0) 422 return -EINVAL; 423 424 ret = check_register(fw, reg); 425 if (ret < 0) 426 return ret; 427 428 reg++; 429 fw->words--; 430 fw->offset++; 431 count--; 432 } 433 434 return 0; 435 } 436 437 static int check_nonincr(struct host1x_firewall *fw) 438 { 439 u32 count = fw->count; 440 int ret; 441 442 while (count) { 443 if (fw->words == 0) 444 return -EINVAL; 445 446 ret = check_register(fw, fw->reg); 447 if (ret < 0) 448 return ret; 449 450 fw->words--; 451 fw->offset++; 452 count--; 453 } 454 455 return 0; 456 } 457 458 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g) 459 { 460 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped + 461 (g->offset / sizeof(u32)); 462 u32 job_class = fw->class; 463 int err = 0; 464 465 fw->words = g->words; 466 fw->cmdbuf = g->bo; 467 fw->offset = 0; 468 469 while (fw->words && !err) { 470 u32 word = cmdbuf_base[fw->offset]; 471 u32 opcode = (word & 0xf0000000) >> 28; 472 473 fw->mask = 0; 474 fw->reg = 0; 475 fw->count = 0; 476 fw->words--; 477 fw->offset++; 478 479 switch (opcode) { 480 case 0: 481 fw->class = word >> 6 & 0x3ff; 482 fw->mask = word & 0x3f; 483 fw->reg = word >> 16 & 0xfff; 484 err = check_class(fw, job_class); 485 if (!err) 486 err = check_mask(fw); 487 if (err) 488 goto out; 489 break; 490 case 1: 491 fw->reg = word >> 16 & 0xfff; 492 fw->count = word & 0xffff; 493 err = check_incr(fw); 494 if (err) 495 goto out; 496 break; 497 498 case 2: 499 fw->reg = word >> 16 & 0xfff; 500 fw->count = word & 0xffff; 501 err = check_nonincr(fw); 502 if (err) 503 goto out; 504 break; 505 506 case 3: 507 fw->mask = word & 0xffff; 508 fw->reg = word >> 16 & 0xfff; 509 err = check_mask(fw); 510 if (err) 511 goto out; 512 break; 513 case 4: 514 case 14: 515 break; 516 default: 517 err = -EINVAL; 518 break; 519 } 520 } 521 522 out: 523 return err; 524 } 525 526 static inline int copy_gathers(struct host1x_job *job, struct device *dev) 527 { 528 struct host1x_firewall fw; 529 size_t size = 0; 530 size_t offset = 0; 531 int i; 532 533 fw.job = job; 534 fw.dev = dev; 535 fw.reloc = job->relocarray; 536 fw.num_relocs = job->num_relocs; 537 fw.class = job->class; 538 539 for (i = 0; i < job->num_gathers; i++) { 540 struct host1x_job_gather *g = &job->gathers[i]; 541 542 size += g->words * sizeof(u32); 543 } 544 545 job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy, 546 GFP_KERNEL); 547 if (!job->gather_copy_mapped) { 548 job->gather_copy_mapped = NULL; 549 return -ENOMEM; 550 } 551 552 job->gather_copy_size = size; 553 554 for (i = 0; i < job->num_gathers; i++) { 555 struct host1x_job_gather *g = &job->gathers[i]; 556 void *gather; 557 558 /* Copy the gather */ 559 gather = host1x_bo_mmap(g->bo); 560 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 561 g->words * sizeof(u32)); 562 host1x_bo_munmap(g->bo, gather); 563 564 /* Store the location in the buffer */ 565 g->base = job->gather_copy; 566 g->offset = offset; 567 568 /* Validate the job */ 569 if (validate(&fw, g)) 570 return -EINVAL; 571 572 offset += g->words * sizeof(u32); 573 } 574 575 /* No relocs should remain at this point */ 576 if (fw.num_relocs) 577 return -EINVAL; 578 579 return 0; 580 } 581 582 int host1x_job_pin(struct host1x_job *job, struct device *dev) 583 { 584 int err; 585 unsigned int i, j; 586 struct host1x *host = dev_get_drvdata(dev->parent); 587 DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host)); 588 589 bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host)); 590 for (i = 0; i < job->num_waitchk; i++) { 591 u32 syncpt_id = job->waitchk[i].syncpt_id; 592 593 if (syncpt_id < host1x_syncpt_nb_pts(host)) 594 set_bit(syncpt_id, waitchk_mask); 595 } 596 597 /* get current syncpt values for waitchk */ 598 for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host)) 599 host1x_syncpt_load(host->syncpt + i); 600 601 /* pin memory */ 602 err = pin_job(host, job); 603 if (err) 604 goto out; 605 606 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 607 err = copy_gathers(job, dev); 608 if (err) 609 goto out; 610 } 611 612 /* patch gathers */ 613 for (i = 0; i < job->num_gathers; i++) { 614 struct host1x_job_gather *g = &job->gathers[i]; 615 616 /* process each gather mem only once */ 617 if (g->handled) 618 continue; 619 620 /* copy_gathers() sets gathers base if firewall is enabled */ 621 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 622 g->base = job->gather_addr_phys[i]; 623 624 for (j = i + 1; j < job->num_gathers; j++) { 625 if (job->gathers[j].bo == g->bo) { 626 job->gathers[j].handled = true; 627 job->gathers[j].base = g->base; 628 } 629 } 630 631 err = do_relocs(job, g); 632 if (err) 633 break; 634 635 err = do_waitchks(job, host, g); 636 if (err) 637 break; 638 } 639 640 out: 641 if (err) 642 host1x_job_unpin(job); 643 wmb(); 644 645 return err; 646 } 647 EXPORT_SYMBOL(host1x_job_pin); 648 649 void host1x_job_unpin(struct host1x_job *job) 650 { 651 struct host1x *host = dev_get_drvdata(job->channel->dev->parent); 652 unsigned int i; 653 654 for (i = 0; i < job->num_unpins; i++) { 655 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 656 657 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { 658 iommu_unmap(host->domain, job->addr_phys[i], 659 unpin->size); 660 free_iova(&host->iova, 661 iova_pfn(&host->iova, job->addr_phys[i])); 662 } 663 664 host1x_bo_unpin(unpin->bo, unpin->sgt); 665 host1x_bo_put(unpin->bo); 666 } 667 668 job->num_unpins = 0; 669 670 if (job->gather_copy_size) 671 dma_free_wc(job->channel->dev, job->gather_copy_size, 672 job->gather_copy_mapped, job->gather_copy); 673 } 674 EXPORT_SYMBOL(host1x_job_unpin); 675 676 /* 677 * Debug routine used to dump job entries 678 */ 679 void host1x_job_dump(struct device *dev, struct host1x_job *job) 680 { 681 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 682 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 683 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 684 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 685 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 686 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 687 } 688