1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Job 4 * 5 * Copyright (c) 2010-2015, NVIDIA Corporation. 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/err.h> 10 #include <linux/host1x.h> 11 #include <linux/kref.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 #include <linux/vmalloc.h> 16 #include <trace/events/host1x.h> 17 18 #include "channel.h" 19 #include "dev.h" 20 #include "job.h" 21 #include "syncpt.h" 22 23 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8 24 25 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 26 u32 num_cmdbufs, u32 num_relocs) 27 { 28 struct host1x_job *job = NULL; 29 unsigned int num_unpins = num_cmdbufs + num_relocs; 30 u64 total; 31 void *mem; 32 33 /* Check that we're not going to overflow */ 34 total = sizeof(struct host1x_job) + 35 (u64)num_relocs * sizeof(struct host1x_reloc) + 36 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) + 37 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) + 38 (u64)num_unpins * sizeof(dma_addr_t) + 39 (u64)num_unpins * sizeof(u32 *); 40 if (total > ULONG_MAX) 41 return NULL; 42 43 mem = job = kzalloc(total, GFP_KERNEL); 44 if (!job) 45 return NULL; 46 47 kref_init(&job->ref); 48 job->channel = ch; 49 50 /* Redistribute memory to the structs */ 51 mem += sizeof(struct host1x_job); 52 job->relocs = num_relocs ? mem : NULL; 53 mem += num_relocs * sizeof(struct host1x_reloc); 54 job->unpins = num_unpins ? mem : NULL; 55 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 56 job->gathers = num_cmdbufs ? mem : NULL; 57 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 58 job->addr_phys = num_unpins ? mem : NULL; 59 60 job->reloc_addr_phys = job->addr_phys; 61 job->gather_addr_phys = &job->addr_phys[num_relocs]; 62 63 return job; 64 } 65 EXPORT_SYMBOL(host1x_job_alloc); 66 67 struct host1x_job *host1x_job_get(struct host1x_job *job) 68 { 69 kref_get(&job->ref); 70 return job; 71 } 72 EXPORT_SYMBOL(host1x_job_get); 73 74 static void job_free(struct kref *ref) 75 { 76 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 77 78 kfree(job); 79 } 80 81 void host1x_job_put(struct host1x_job *job) 82 { 83 kref_put(&job->ref, job_free); 84 } 85 EXPORT_SYMBOL(host1x_job_put); 86 87 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 88 unsigned int words, unsigned int offset) 89 { 90 struct host1x_job_gather *gather = &job->gathers[job->num_gathers]; 91 92 gather->words = words; 93 gather->bo = bo; 94 gather->offset = offset; 95 96 job->num_gathers++; 97 } 98 EXPORT_SYMBOL(host1x_job_add_gather); 99 100 static unsigned int pin_job(struct host1x *host, struct host1x_job *job) 101 { 102 struct host1x_client *client = job->client; 103 struct device *dev = client->dev; 104 unsigned int i; 105 int err; 106 107 job->num_unpins = 0; 108 109 for (i = 0; i < job->num_relocs; i++) { 110 struct host1x_reloc *reloc = &job->relocs[i]; 111 dma_addr_t phys_addr, *phys; 112 struct sg_table *sgt; 113 114 reloc->target.bo = host1x_bo_get(reloc->target.bo); 115 if (!reloc->target.bo) { 116 err = -EINVAL; 117 goto unpin; 118 } 119 120 if (client->group) 121 phys = &phys_addr; 122 else 123 phys = NULL; 124 125 sgt = host1x_bo_pin(dev, reloc->target.bo, phys); 126 if (IS_ERR(sgt)) { 127 err = PTR_ERR(sgt); 128 goto unpin; 129 } 130 131 if (sgt) { 132 unsigned long mask = HOST1X_RELOC_READ | 133 HOST1X_RELOC_WRITE; 134 enum dma_data_direction dir; 135 136 switch (reloc->flags & mask) { 137 case HOST1X_RELOC_READ: 138 dir = DMA_TO_DEVICE; 139 break; 140 141 case HOST1X_RELOC_WRITE: 142 dir = DMA_FROM_DEVICE; 143 break; 144 145 case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: 146 dir = DMA_BIDIRECTIONAL; 147 break; 148 149 default: 150 err = -EINVAL; 151 goto unpin; 152 } 153 154 err = dma_map_sg(dev, sgt->sgl, sgt->nents, dir); 155 if (!err) { 156 err = -ENOMEM; 157 goto unpin; 158 } 159 160 job->unpins[job->num_unpins].dev = dev; 161 job->unpins[job->num_unpins].dir = dir; 162 phys_addr = sg_dma_address(sgt->sgl); 163 } 164 165 job->addr_phys[job->num_unpins] = phys_addr; 166 job->unpins[job->num_unpins].bo = reloc->target.bo; 167 job->unpins[job->num_unpins].sgt = sgt; 168 job->num_unpins++; 169 } 170 171 for (i = 0; i < job->num_gathers; i++) { 172 struct host1x_job_gather *g = &job->gathers[i]; 173 size_t gather_size = 0; 174 struct scatterlist *sg; 175 struct sg_table *sgt; 176 dma_addr_t phys_addr; 177 unsigned long shift; 178 struct iova *alloc; 179 unsigned int j; 180 181 g->bo = host1x_bo_get(g->bo); 182 if (!g->bo) { 183 err = -EINVAL; 184 goto unpin; 185 } 186 187 sgt = host1x_bo_pin(host->dev, g->bo, NULL); 188 if (IS_ERR(sgt)) { 189 err = PTR_ERR(sgt); 190 goto unpin; 191 } 192 193 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { 194 for_each_sg(sgt->sgl, sg, sgt->nents, j) 195 gather_size += sg->length; 196 gather_size = iova_align(&host->iova, gather_size); 197 198 shift = iova_shift(&host->iova); 199 alloc = alloc_iova(&host->iova, gather_size >> shift, 200 host->iova_end >> shift, true); 201 if (!alloc) { 202 err = -ENOMEM; 203 goto unpin; 204 } 205 206 err = iommu_map_sg(host->domain, 207 iova_dma_addr(&host->iova, alloc), 208 sgt->sgl, sgt->nents, IOMMU_READ); 209 if (err == 0) { 210 __free_iova(&host->iova, alloc); 211 err = -EINVAL; 212 goto unpin; 213 } 214 215 job->unpins[job->num_unpins].size = gather_size; 216 phys_addr = iova_dma_addr(&host->iova, alloc); 217 } else { 218 err = dma_map_sg(host->dev, sgt->sgl, sgt->nents, 219 DMA_TO_DEVICE); 220 if (!err) { 221 err = -ENOMEM; 222 goto unpin; 223 } 224 225 job->unpins[job->num_unpins].dev = host->dev; 226 phys_addr = sg_dma_address(sgt->sgl); 227 } 228 229 job->addr_phys[job->num_unpins] = phys_addr; 230 job->gather_addr_phys[i] = phys_addr; 231 232 job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; 233 job->unpins[job->num_unpins].bo = g->bo; 234 job->unpins[job->num_unpins].sgt = sgt; 235 job->num_unpins++; 236 } 237 238 return 0; 239 240 unpin: 241 host1x_job_unpin(job); 242 return err; 243 } 244 245 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g) 246 { 247 void *cmdbuf_addr = NULL; 248 struct host1x_bo *cmdbuf = g->bo; 249 unsigned int i; 250 251 /* pin & patch the relocs for one gather */ 252 for (i = 0; i < job->num_relocs; i++) { 253 struct host1x_reloc *reloc = &job->relocs[i]; 254 u32 reloc_addr = (job->reloc_addr_phys[i] + 255 reloc->target.offset) >> reloc->shift; 256 u32 *target; 257 258 /* skip all other gathers */ 259 if (cmdbuf != reloc->cmdbuf.bo) 260 continue; 261 262 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 263 target = (u32 *)job->gather_copy_mapped + 264 reloc->cmdbuf.offset / sizeof(u32) + 265 g->offset / sizeof(u32); 266 goto patch_reloc; 267 } 268 269 if (!cmdbuf_addr) { 270 cmdbuf_addr = host1x_bo_mmap(cmdbuf); 271 272 if (unlikely(!cmdbuf_addr)) { 273 pr_err("Could not map cmdbuf for relocation\n"); 274 return -ENOMEM; 275 } 276 } 277 278 target = cmdbuf_addr + reloc->cmdbuf.offset; 279 patch_reloc: 280 *target = reloc_addr; 281 } 282 283 if (cmdbuf_addr) 284 host1x_bo_munmap(cmdbuf, cmdbuf_addr); 285 286 return 0; 287 } 288 289 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 290 unsigned int offset) 291 { 292 offset *= sizeof(u32); 293 294 if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset) 295 return false; 296 297 /* relocation shift value validation isn't implemented yet */ 298 if (reloc->shift) 299 return false; 300 301 return true; 302 } 303 304 struct host1x_firewall { 305 struct host1x_job *job; 306 struct device *dev; 307 308 unsigned int num_relocs; 309 struct host1x_reloc *reloc; 310 311 struct host1x_bo *cmdbuf; 312 unsigned int offset; 313 314 u32 words; 315 u32 class; 316 u32 reg; 317 u32 mask; 318 u32 count; 319 }; 320 321 static int check_register(struct host1x_firewall *fw, unsigned long offset) 322 { 323 if (!fw->job->is_addr_reg) 324 return 0; 325 326 if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) { 327 if (!fw->num_relocs) 328 return -EINVAL; 329 330 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset)) 331 return -EINVAL; 332 333 fw->num_relocs--; 334 fw->reloc++; 335 } 336 337 return 0; 338 } 339 340 static int check_class(struct host1x_firewall *fw, u32 class) 341 { 342 if (!fw->job->is_valid_class) { 343 if (fw->class != class) 344 return -EINVAL; 345 } else { 346 if (!fw->job->is_valid_class(fw->class)) 347 return -EINVAL; 348 } 349 350 return 0; 351 } 352 353 static int check_mask(struct host1x_firewall *fw) 354 { 355 u32 mask = fw->mask; 356 u32 reg = fw->reg; 357 int ret; 358 359 while (mask) { 360 if (fw->words == 0) 361 return -EINVAL; 362 363 if (mask & 1) { 364 ret = check_register(fw, reg); 365 if (ret < 0) 366 return ret; 367 368 fw->words--; 369 fw->offset++; 370 } 371 mask >>= 1; 372 reg++; 373 } 374 375 return 0; 376 } 377 378 static int check_incr(struct host1x_firewall *fw) 379 { 380 u32 count = fw->count; 381 u32 reg = fw->reg; 382 int ret; 383 384 while (count) { 385 if (fw->words == 0) 386 return -EINVAL; 387 388 ret = check_register(fw, reg); 389 if (ret < 0) 390 return ret; 391 392 reg++; 393 fw->words--; 394 fw->offset++; 395 count--; 396 } 397 398 return 0; 399 } 400 401 static int check_nonincr(struct host1x_firewall *fw) 402 { 403 u32 count = fw->count; 404 int ret; 405 406 while (count) { 407 if (fw->words == 0) 408 return -EINVAL; 409 410 ret = check_register(fw, fw->reg); 411 if (ret < 0) 412 return ret; 413 414 fw->words--; 415 fw->offset++; 416 count--; 417 } 418 419 return 0; 420 } 421 422 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g) 423 { 424 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped + 425 (g->offset / sizeof(u32)); 426 u32 job_class = fw->class; 427 int err = 0; 428 429 fw->words = g->words; 430 fw->cmdbuf = g->bo; 431 fw->offset = 0; 432 433 while (fw->words && !err) { 434 u32 word = cmdbuf_base[fw->offset]; 435 u32 opcode = (word & 0xf0000000) >> 28; 436 437 fw->mask = 0; 438 fw->reg = 0; 439 fw->count = 0; 440 fw->words--; 441 fw->offset++; 442 443 switch (opcode) { 444 case 0: 445 fw->class = word >> 6 & 0x3ff; 446 fw->mask = word & 0x3f; 447 fw->reg = word >> 16 & 0xfff; 448 err = check_class(fw, job_class); 449 if (!err) 450 err = check_mask(fw); 451 if (err) 452 goto out; 453 break; 454 case 1: 455 fw->reg = word >> 16 & 0xfff; 456 fw->count = word & 0xffff; 457 err = check_incr(fw); 458 if (err) 459 goto out; 460 break; 461 462 case 2: 463 fw->reg = word >> 16 & 0xfff; 464 fw->count = word & 0xffff; 465 err = check_nonincr(fw); 466 if (err) 467 goto out; 468 break; 469 470 case 3: 471 fw->mask = word & 0xffff; 472 fw->reg = word >> 16 & 0xfff; 473 err = check_mask(fw); 474 if (err) 475 goto out; 476 break; 477 case 4: 478 case 14: 479 break; 480 default: 481 err = -EINVAL; 482 break; 483 } 484 } 485 486 out: 487 return err; 488 } 489 490 static inline int copy_gathers(struct device *host, struct host1x_job *job, 491 struct device *dev) 492 { 493 struct host1x_firewall fw; 494 size_t size = 0; 495 size_t offset = 0; 496 unsigned int i; 497 498 fw.job = job; 499 fw.dev = dev; 500 fw.reloc = job->relocs; 501 fw.num_relocs = job->num_relocs; 502 fw.class = job->class; 503 504 for (i = 0; i < job->num_gathers; i++) { 505 struct host1x_job_gather *g = &job->gathers[i]; 506 507 size += g->words * sizeof(u32); 508 } 509 510 /* 511 * Try a non-blocking allocation from a higher priority pools first, 512 * as awaiting for the allocation here is a major performance hit. 513 */ 514 job->gather_copy_mapped = dma_alloc_wc(host, size, &job->gather_copy, 515 GFP_NOWAIT); 516 517 /* the higher priority allocation failed, try the generic-blocking */ 518 if (!job->gather_copy_mapped) 519 job->gather_copy_mapped = dma_alloc_wc(host, size, 520 &job->gather_copy, 521 GFP_KERNEL); 522 if (!job->gather_copy_mapped) 523 return -ENOMEM; 524 525 job->gather_copy_size = size; 526 527 for (i = 0; i < job->num_gathers; i++) { 528 struct host1x_job_gather *g = &job->gathers[i]; 529 void *gather; 530 531 /* Copy the gather */ 532 gather = host1x_bo_mmap(g->bo); 533 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 534 g->words * sizeof(u32)); 535 host1x_bo_munmap(g->bo, gather); 536 537 /* Store the location in the buffer */ 538 g->base = job->gather_copy; 539 g->offset = offset; 540 541 /* Validate the job */ 542 if (validate(&fw, g)) 543 return -EINVAL; 544 545 offset += g->words * sizeof(u32); 546 } 547 548 /* No relocs should remain at this point */ 549 if (fw.num_relocs) 550 return -EINVAL; 551 552 return 0; 553 } 554 555 int host1x_job_pin(struct host1x_job *job, struct device *dev) 556 { 557 int err; 558 unsigned int i, j; 559 struct host1x *host = dev_get_drvdata(dev->parent); 560 561 /* pin memory */ 562 err = pin_job(host, job); 563 if (err) 564 goto out; 565 566 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 567 err = copy_gathers(host->dev, job, dev); 568 if (err) 569 goto out; 570 } 571 572 /* patch gathers */ 573 for (i = 0; i < job->num_gathers; i++) { 574 struct host1x_job_gather *g = &job->gathers[i]; 575 576 /* process each gather mem only once */ 577 if (g->handled) 578 continue; 579 580 /* copy_gathers() sets gathers base if firewall is enabled */ 581 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 582 g->base = job->gather_addr_phys[i]; 583 584 for (j = i + 1; j < job->num_gathers; j++) { 585 if (job->gathers[j].bo == g->bo) { 586 job->gathers[j].handled = true; 587 job->gathers[j].base = g->base; 588 } 589 } 590 591 err = do_relocs(job, g); 592 if (err) 593 break; 594 } 595 596 out: 597 if (err) 598 host1x_job_unpin(job); 599 wmb(); 600 601 return err; 602 } 603 EXPORT_SYMBOL(host1x_job_pin); 604 605 void host1x_job_unpin(struct host1x_job *job) 606 { 607 struct host1x *host = dev_get_drvdata(job->channel->dev->parent); 608 unsigned int i; 609 610 for (i = 0; i < job->num_unpins; i++) { 611 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 612 struct device *dev = unpin->dev ?: host->dev; 613 struct sg_table *sgt = unpin->sgt; 614 615 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && 616 unpin->size && host->domain) { 617 iommu_unmap(host->domain, job->addr_phys[i], 618 unpin->size); 619 free_iova(&host->iova, 620 iova_pfn(&host->iova, job->addr_phys[i])); 621 } 622 623 if (unpin->dev && sgt) 624 dma_unmap_sg(unpin->dev, sgt->sgl, sgt->nents, 625 unpin->dir); 626 627 host1x_bo_unpin(dev, unpin->bo, sgt); 628 host1x_bo_put(unpin->bo); 629 } 630 631 job->num_unpins = 0; 632 633 if (job->gather_copy_size) 634 dma_free_wc(host->dev, job->gather_copy_size, 635 job->gather_copy_mapped, job->gather_copy); 636 } 637 EXPORT_SYMBOL(host1x_job_unpin); 638 639 /* 640 * Debug routine used to dump job entries 641 */ 642 void host1x_job_dump(struct device *dev, struct host1x_job *job) 643 { 644 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 645 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 646 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 647 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 648 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 649 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 650 } 651