1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Job 4 * 5 * Copyright (c) 2010-2015, NVIDIA Corporation. 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/err.h> 10 #include <linux/host1x.h> 11 #include <linux/kref.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 #include <linux/vmalloc.h> 16 #include <trace/events/host1x.h> 17 18 #include "channel.h" 19 #include "dev.h" 20 #include "job.h" 21 #include "syncpt.h" 22 23 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8 24 25 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 26 u32 num_cmdbufs, u32 num_relocs) 27 { 28 struct host1x_job *job = NULL; 29 unsigned int num_unpins = num_cmdbufs + num_relocs; 30 u64 total; 31 void *mem; 32 33 /* Check that we're not going to overflow */ 34 total = sizeof(struct host1x_job) + 35 (u64)num_relocs * sizeof(struct host1x_reloc) + 36 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) + 37 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) + 38 (u64)num_unpins * sizeof(dma_addr_t) + 39 (u64)num_unpins * sizeof(u32 *); 40 if (total > ULONG_MAX) 41 return NULL; 42 43 mem = job = kzalloc(total, GFP_KERNEL); 44 if (!job) 45 return NULL; 46 47 kref_init(&job->ref); 48 job->channel = ch; 49 50 /* Redistribute memory to the structs */ 51 mem += sizeof(struct host1x_job); 52 job->relocs = num_relocs ? mem : NULL; 53 mem += num_relocs * sizeof(struct host1x_reloc); 54 job->unpins = num_unpins ? mem : NULL; 55 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 56 job->gathers = num_cmdbufs ? mem : NULL; 57 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 58 job->addr_phys = num_unpins ? mem : NULL; 59 60 job->reloc_addr_phys = job->addr_phys; 61 job->gather_addr_phys = &job->addr_phys[num_relocs]; 62 63 return job; 64 } 65 EXPORT_SYMBOL(host1x_job_alloc); 66 67 struct host1x_job *host1x_job_get(struct host1x_job *job) 68 { 69 kref_get(&job->ref); 70 return job; 71 } 72 EXPORT_SYMBOL(host1x_job_get); 73 74 static void job_free(struct kref *ref) 75 { 76 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 77 78 kfree(job); 79 } 80 81 void host1x_job_put(struct host1x_job *job) 82 { 83 kref_put(&job->ref, job_free); 84 } 85 EXPORT_SYMBOL(host1x_job_put); 86 87 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 88 unsigned int words, unsigned int offset) 89 { 90 struct host1x_job_gather *gather = &job->gathers[job->num_gathers]; 91 92 gather->words = words; 93 gather->bo = bo; 94 gather->offset = offset; 95 96 job->num_gathers++; 97 } 98 EXPORT_SYMBOL(host1x_job_add_gather); 99 100 static unsigned int pin_job(struct host1x *host, struct host1x_job *job) 101 { 102 struct host1x_client *client = job->client; 103 struct device *dev = client->dev; 104 unsigned int i; 105 int err; 106 107 job->num_unpins = 0; 108 109 for (i = 0; i < job->num_relocs; i++) { 110 struct host1x_reloc *reloc = &job->relocs[i]; 111 dma_addr_t phys_addr, *phys; 112 struct sg_table *sgt; 113 114 reloc->target.bo = host1x_bo_get(reloc->target.bo); 115 if (!reloc->target.bo) { 116 err = -EINVAL; 117 goto unpin; 118 } 119 120 if (client->group) 121 phys = &phys_addr; 122 else 123 phys = NULL; 124 125 sgt = host1x_bo_pin(dev, reloc->target.bo, phys); 126 if (IS_ERR(sgt)) { 127 err = PTR_ERR(sgt); 128 goto unpin; 129 } 130 131 if (sgt) { 132 unsigned long mask = HOST1X_RELOC_READ | 133 HOST1X_RELOC_WRITE; 134 enum dma_data_direction dir; 135 136 switch (reloc->flags & mask) { 137 case HOST1X_RELOC_READ: 138 dir = DMA_TO_DEVICE; 139 break; 140 141 case HOST1X_RELOC_WRITE: 142 dir = DMA_FROM_DEVICE; 143 break; 144 145 case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: 146 dir = DMA_BIDIRECTIONAL; 147 break; 148 149 default: 150 err = -EINVAL; 151 goto unpin; 152 } 153 154 err = dma_map_sg(dev, sgt->sgl, sgt->nents, dir); 155 if (!err) { 156 err = -ENOMEM; 157 goto unpin; 158 } 159 160 job->unpins[job->num_unpins].dev = dev; 161 job->unpins[job->num_unpins].dir = dir; 162 phys_addr = sg_dma_address(sgt->sgl); 163 } 164 165 job->addr_phys[job->num_unpins] = phys_addr; 166 job->unpins[job->num_unpins].bo = reloc->target.bo; 167 job->unpins[job->num_unpins].sgt = sgt; 168 job->num_unpins++; 169 } 170 171 for (i = 0; i < job->num_gathers; i++) { 172 struct host1x_job_gather *g = &job->gathers[i]; 173 size_t gather_size = 0; 174 struct scatterlist *sg; 175 struct sg_table *sgt; 176 dma_addr_t phys_addr; 177 unsigned long shift; 178 struct iova *alloc; 179 unsigned int j; 180 181 g->bo = host1x_bo_get(g->bo); 182 if (!g->bo) { 183 err = -EINVAL; 184 goto unpin; 185 } 186 187 sgt = host1x_bo_pin(host->dev, g->bo, NULL); 188 if (IS_ERR(sgt)) { 189 err = PTR_ERR(sgt); 190 goto unpin; 191 } 192 193 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { 194 for_each_sg(sgt->sgl, sg, sgt->nents, j) 195 gather_size += sg->length; 196 gather_size = iova_align(&host->iova, gather_size); 197 198 shift = iova_shift(&host->iova); 199 alloc = alloc_iova(&host->iova, gather_size >> shift, 200 host->iova_end >> shift, true); 201 if (!alloc) { 202 err = -ENOMEM; 203 goto unpin; 204 } 205 206 err = iommu_map_sg(host->domain, 207 iova_dma_addr(&host->iova, alloc), 208 sgt->sgl, sgt->nents, IOMMU_READ); 209 if (err == 0) { 210 __free_iova(&host->iova, alloc); 211 err = -EINVAL; 212 goto unpin; 213 } 214 215 job->unpins[job->num_unpins].size = gather_size; 216 phys_addr = iova_dma_addr(&host->iova, alloc); 217 } else { 218 err = dma_map_sg(host->dev, sgt->sgl, sgt->nents, 219 DMA_TO_DEVICE); 220 if (!err) { 221 err = -ENOMEM; 222 goto unpin; 223 } 224 225 job->unpins[job->num_unpins].dev = host->dev; 226 phys_addr = sg_dma_address(sgt->sgl); 227 } 228 229 job->addr_phys[job->num_unpins] = phys_addr; 230 job->gather_addr_phys[i] = phys_addr; 231 232 job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; 233 job->unpins[job->num_unpins].bo = g->bo; 234 job->unpins[job->num_unpins].sgt = sgt; 235 job->num_unpins++; 236 } 237 238 return 0; 239 240 unpin: 241 host1x_job_unpin(job); 242 return err; 243 } 244 245 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g) 246 { 247 u32 last_page = ~0; 248 void *cmdbuf_page_addr = NULL; 249 struct host1x_bo *cmdbuf = g->bo; 250 unsigned int i; 251 252 /* pin & patch the relocs for one gather */ 253 for (i = 0; i < job->num_relocs; i++) { 254 struct host1x_reloc *reloc = &job->relocs[i]; 255 u32 reloc_addr = (job->reloc_addr_phys[i] + 256 reloc->target.offset) >> reloc->shift; 257 u32 *target; 258 259 /* skip all other gathers */ 260 if (cmdbuf != reloc->cmdbuf.bo) 261 continue; 262 263 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 264 target = (u32 *)job->gather_copy_mapped + 265 reloc->cmdbuf.offset / sizeof(u32) + 266 g->offset / sizeof(u32); 267 goto patch_reloc; 268 } 269 270 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) { 271 if (cmdbuf_page_addr) 272 host1x_bo_kunmap(cmdbuf, last_page, 273 cmdbuf_page_addr); 274 275 cmdbuf_page_addr = host1x_bo_kmap(cmdbuf, 276 reloc->cmdbuf.offset >> PAGE_SHIFT); 277 last_page = reloc->cmdbuf.offset >> PAGE_SHIFT; 278 279 if (unlikely(!cmdbuf_page_addr)) { 280 pr_err("Could not map cmdbuf for relocation\n"); 281 return -ENOMEM; 282 } 283 } 284 285 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK); 286 patch_reloc: 287 *target = reloc_addr; 288 } 289 290 if (cmdbuf_page_addr) 291 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr); 292 293 return 0; 294 } 295 296 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 297 unsigned int offset) 298 { 299 offset *= sizeof(u32); 300 301 if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset) 302 return false; 303 304 /* relocation shift value validation isn't implemented yet */ 305 if (reloc->shift) 306 return false; 307 308 return true; 309 } 310 311 struct host1x_firewall { 312 struct host1x_job *job; 313 struct device *dev; 314 315 unsigned int num_relocs; 316 struct host1x_reloc *reloc; 317 318 struct host1x_bo *cmdbuf; 319 unsigned int offset; 320 321 u32 words; 322 u32 class; 323 u32 reg; 324 u32 mask; 325 u32 count; 326 }; 327 328 static int check_register(struct host1x_firewall *fw, unsigned long offset) 329 { 330 if (!fw->job->is_addr_reg) 331 return 0; 332 333 if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) { 334 if (!fw->num_relocs) 335 return -EINVAL; 336 337 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset)) 338 return -EINVAL; 339 340 fw->num_relocs--; 341 fw->reloc++; 342 } 343 344 return 0; 345 } 346 347 static int check_class(struct host1x_firewall *fw, u32 class) 348 { 349 if (!fw->job->is_valid_class) { 350 if (fw->class != class) 351 return -EINVAL; 352 } else { 353 if (!fw->job->is_valid_class(fw->class)) 354 return -EINVAL; 355 } 356 357 return 0; 358 } 359 360 static int check_mask(struct host1x_firewall *fw) 361 { 362 u32 mask = fw->mask; 363 u32 reg = fw->reg; 364 int ret; 365 366 while (mask) { 367 if (fw->words == 0) 368 return -EINVAL; 369 370 if (mask & 1) { 371 ret = check_register(fw, reg); 372 if (ret < 0) 373 return ret; 374 375 fw->words--; 376 fw->offset++; 377 } 378 mask >>= 1; 379 reg++; 380 } 381 382 return 0; 383 } 384 385 static int check_incr(struct host1x_firewall *fw) 386 { 387 u32 count = fw->count; 388 u32 reg = fw->reg; 389 int ret; 390 391 while (count) { 392 if (fw->words == 0) 393 return -EINVAL; 394 395 ret = check_register(fw, reg); 396 if (ret < 0) 397 return ret; 398 399 reg++; 400 fw->words--; 401 fw->offset++; 402 count--; 403 } 404 405 return 0; 406 } 407 408 static int check_nonincr(struct host1x_firewall *fw) 409 { 410 u32 count = fw->count; 411 int ret; 412 413 while (count) { 414 if (fw->words == 0) 415 return -EINVAL; 416 417 ret = check_register(fw, fw->reg); 418 if (ret < 0) 419 return ret; 420 421 fw->words--; 422 fw->offset++; 423 count--; 424 } 425 426 return 0; 427 } 428 429 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g) 430 { 431 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped + 432 (g->offset / sizeof(u32)); 433 u32 job_class = fw->class; 434 int err = 0; 435 436 fw->words = g->words; 437 fw->cmdbuf = g->bo; 438 fw->offset = 0; 439 440 while (fw->words && !err) { 441 u32 word = cmdbuf_base[fw->offset]; 442 u32 opcode = (word & 0xf0000000) >> 28; 443 444 fw->mask = 0; 445 fw->reg = 0; 446 fw->count = 0; 447 fw->words--; 448 fw->offset++; 449 450 switch (opcode) { 451 case 0: 452 fw->class = word >> 6 & 0x3ff; 453 fw->mask = word & 0x3f; 454 fw->reg = word >> 16 & 0xfff; 455 err = check_class(fw, job_class); 456 if (!err) 457 err = check_mask(fw); 458 if (err) 459 goto out; 460 break; 461 case 1: 462 fw->reg = word >> 16 & 0xfff; 463 fw->count = word & 0xffff; 464 err = check_incr(fw); 465 if (err) 466 goto out; 467 break; 468 469 case 2: 470 fw->reg = word >> 16 & 0xfff; 471 fw->count = word & 0xffff; 472 err = check_nonincr(fw); 473 if (err) 474 goto out; 475 break; 476 477 case 3: 478 fw->mask = word & 0xffff; 479 fw->reg = word >> 16 & 0xfff; 480 err = check_mask(fw); 481 if (err) 482 goto out; 483 break; 484 case 4: 485 case 14: 486 break; 487 default: 488 err = -EINVAL; 489 break; 490 } 491 } 492 493 out: 494 return err; 495 } 496 497 static inline int copy_gathers(struct device *host, struct host1x_job *job, 498 struct device *dev) 499 { 500 struct host1x_firewall fw; 501 size_t size = 0; 502 size_t offset = 0; 503 unsigned int i; 504 505 fw.job = job; 506 fw.dev = dev; 507 fw.reloc = job->relocs; 508 fw.num_relocs = job->num_relocs; 509 fw.class = job->class; 510 511 for (i = 0; i < job->num_gathers; i++) { 512 struct host1x_job_gather *g = &job->gathers[i]; 513 514 size += g->words * sizeof(u32); 515 } 516 517 /* 518 * Try a non-blocking allocation from a higher priority pools first, 519 * as awaiting for the allocation here is a major performance hit. 520 */ 521 job->gather_copy_mapped = dma_alloc_wc(host, size, &job->gather_copy, 522 GFP_NOWAIT); 523 524 /* the higher priority allocation failed, try the generic-blocking */ 525 if (!job->gather_copy_mapped) 526 job->gather_copy_mapped = dma_alloc_wc(host, size, 527 &job->gather_copy, 528 GFP_KERNEL); 529 if (!job->gather_copy_mapped) 530 return -ENOMEM; 531 532 job->gather_copy_size = size; 533 534 for (i = 0; i < job->num_gathers; i++) { 535 struct host1x_job_gather *g = &job->gathers[i]; 536 void *gather; 537 538 /* Copy the gather */ 539 gather = host1x_bo_mmap(g->bo); 540 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 541 g->words * sizeof(u32)); 542 host1x_bo_munmap(g->bo, gather); 543 544 /* Store the location in the buffer */ 545 g->base = job->gather_copy; 546 g->offset = offset; 547 548 /* Validate the job */ 549 if (validate(&fw, g)) 550 return -EINVAL; 551 552 offset += g->words * sizeof(u32); 553 } 554 555 /* No relocs should remain at this point */ 556 if (fw.num_relocs) 557 return -EINVAL; 558 559 return 0; 560 } 561 562 int host1x_job_pin(struct host1x_job *job, struct device *dev) 563 { 564 int err; 565 unsigned int i, j; 566 struct host1x *host = dev_get_drvdata(dev->parent); 567 568 /* pin memory */ 569 err = pin_job(host, job); 570 if (err) 571 goto out; 572 573 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 574 err = copy_gathers(host->dev, job, dev); 575 if (err) 576 goto out; 577 } 578 579 /* patch gathers */ 580 for (i = 0; i < job->num_gathers; i++) { 581 struct host1x_job_gather *g = &job->gathers[i]; 582 583 /* process each gather mem only once */ 584 if (g->handled) 585 continue; 586 587 /* copy_gathers() sets gathers base if firewall is enabled */ 588 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 589 g->base = job->gather_addr_phys[i]; 590 591 for (j = i + 1; j < job->num_gathers; j++) { 592 if (job->gathers[j].bo == g->bo) { 593 job->gathers[j].handled = true; 594 job->gathers[j].base = g->base; 595 } 596 } 597 598 err = do_relocs(job, g); 599 if (err) 600 break; 601 } 602 603 out: 604 if (err) 605 host1x_job_unpin(job); 606 wmb(); 607 608 return err; 609 } 610 EXPORT_SYMBOL(host1x_job_pin); 611 612 void host1x_job_unpin(struct host1x_job *job) 613 { 614 struct host1x *host = dev_get_drvdata(job->channel->dev->parent); 615 unsigned int i; 616 617 for (i = 0; i < job->num_unpins; i++) { 618 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 619 struct device *dev = unpin->dev ?: host->dev; 620 struct sg_table *sgt = unpin->sgt; 621 622 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && 623 unpin->size && host->domain) { 624 iommu_unmap(host->domain, job->addr_phys[i], 625 unpin->size); 626 free_iova(&host->iova, 627 iova_pfn(&host->iova, job->addr_phys[i])); 628 } 629 630 if (unpin->dev && sgt) 631 dma_unmap_sg(unpin->dev, sgt->sgl, sgt->nents, 632 unpin->dir); 633 634 host1x_bo_unpin(dev, unpin->bo, sgt); 635 host1x_bo_put(unpin->bo); 636 } 637 638 job->num_unpins = 0; 639 640 if (job->gather_copy_size) 641 dma_free_wc(host->dev, job->gather_copy_size, 642 job->gather_copy_mapped, job->gather_copy); 643 } 644 EXPORT_SYMBOL(host1x_job_unpin); 645 646 /* 647 * Debug routine used to dump job entries 648 */ 649 void host1x_job_dump(struct device *dev, struct host1x_job *job) 650 { 651 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 652 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 653 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 654 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 655 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 656 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 657 } 658