1 /* 2 * Tegra host1x Job 3 * 4 * Copyright (c) 2010-2015, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/dma-mapping.h> 20 #include <linux/err.h> 21 #include <linux/host1x.h> 22 #include <linux/kref.h> 23 #include <linux/module.h> 24 #include <linux/scatterlist.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 #include <trace/events/host1x.h> 28 29 #include "channel.h" 30 #include "dev.h" 31 #include "job.h" 32 #include "syncpt.h" 33 34 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8 35 36 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 37 u32 num_cmdbufs, u32 num_relocs) 38 { 39 struct host1x_job *job = NULL; 40 unsigned int num_unpins = num_cmdbufs + num_relocs; 41 u64 total; 42 void *mem; 43 44 /* Check that we're not going to overflow */ 45 total = sizeof(struct host1x_job) + 46 (u64)num_relocs * sizeof(struct host1x_reloc) + 47 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) + 48 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) + 49 (u64)num_unpins * sizeof(dma_addr_t) + 50 (u64)num_unpins * sizeof(u32 *); 51 if (total > ULONG_MAX) 52 return NULL; 53 54 mem = job = kzalloc(total, GFP_KERNEL); 55 if (!job) 56 return NULL; 57 58 kref_init(&job->ref); 59 job->channel = ch; 60 61 /* Redistribute memory to the structs */ 62 mem += sizeof(struct host1x_job); 63 job->relocs = num_relocs ? mem : NULL; 64 mem += num_relocs * sizeof(struct host1x_reloc); 65 job->unpins = num_unpins ? mem : NULL; 66 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 67 job->gathers = num_cmdbufs ? mem : NULL; 68 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 69 job->addr_phys = num_unpins ? mem : NULL; 70 71 job->reloc_addr_phys = job->addr_phys; 72 job->gather_addr_phys = &job->addr_phys[num_relocs]; 73 74 return job; 75 } 76 EXPORT_SYMBOL(host1x_job_alloc); 77 78 struct host1x_job *host1x_job_get(struct host1x_job *job) 79 { 80 kref_get(&job->ref); 81 return job; 82 } 83 EXPORT_SYMBOL(host1x_job_get); 84 85 static void job_free(struct kref *ref) 86 { 87 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 88 89 kfree(job); 90 } 91 92 void host1x_job_put(struct host1x_job *job) 93 { 94 kref_put(&job->ref, job_free); 95 } 96 EXPORT_SYMBOL(host1x_job_put); 97 98 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 99 unsigned int words, unsigned int offset) 100 { 101 struct host1x_job_gather *gather = &job->gathers[job->num_gathers]; 102 103 gather->words = words; 104 gather->bo = bo; 105 gather->offset = offset; 106 107 job->num_gathers++; 108 } 109 EXPORT_SYMBOL(host1x_job_add_gather); 110 111 static unsigned int pin_job(struct host1x *host, struct host1x_job *job) 112 { 113 unsigned int i; 114 int err; 115 116 job->num_unpins = 0; 117 118 for (i = 0; i < job->num_relocs; i++) { 119 struct host1x_reloc *reloc = &job->relocs[i]; 120 struct sg_table *sgt; 121 dma_addr_t phys_addr; 122 123 reloc->target.bo = host1x_bo_get(reloc->target.bo); 124 if (!reloc->target.bo) { 125 err = -EINVAL; 126 goto unpin; 127 } 128 129 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt); 130 131 job->addr_phys[job->num_unpins] = phys_addr; 132 job->unpins[job->num_unpins].bo = reloc->target.bo; 133 job->unpins[job->num_unpins].sgt = sgt; 134 job->num_unpins++; 135 } 136 137 for (i = 0; i < job->num_gathers; i++) { 138 struct host1x_job_gather *g = &job->gathers[i]; 139 size_t gather_size = 0; 140 struct scatterlist *sg; 141 struct sg_table *sgt; 142 dma_addr_t phys_addr; 143 unsigned long shift; 144 struct iova *alloc; 145 unsigned int j; 146 147 g->bo = host1x_bo_get(g->bo); 148 if (!g->bo) { 149 err = -EINVAL; 150 goto unpin; 151 } 152 153 phys_addr = host1x_bo_pin(g->bo, &sgt); 154 155 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { 156 for_each_sg(sgt->sgl, sg, sgt->nents, j) 157 gather_size += sg->length; 158 gather_size = iova_align(&host->iova, gather_size); 159 160 shift = iova_shift(&host->iova); 161 alloc = alloc_iova(&host->iova, gather_size >> shift, 162 host->iova_end >> shift, true); 163 if (!alloc) { 164 err = -ENOMEM; 165 goto unpin; 166 } 167 168 err = iommu_map_sg(host->domain, 169 iova_dma_addr(&host->iova, alloc), 170 sgt->sgl, sgt->nents, IOMMU_READ); 171 if (err == 0) { 172 __free_iova(&host->iova, alloc); 173 err = -EINVAL; 174 goto unpin; 175 } 176 177 job->addr_phys[job->num_unpins] = 178 iova_dma_addr(&host->iova, alloc); 179 job->unpins[job->num_unpins].size = gather_size; 180 } else { 181 job->addr_phys[job->num_unpins] = phys_addr; 182 } 183 184 job->gather_addr_phys[i] = job->addr_phys[job->num_unpins]; 185 186 job->unpins[job->num_unpins].bo = g->bo; 187 job->unpins[job->num_unpins].sgt = sgt; 188 job->num_unpins++; 189 } 190 191 return 0; 192 193 unpin: 194 host1x_job_unpin(job); 195 return err; 196 } 197 198 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g) 199 { 200 u32 last_page = ~0; 201 void *cmdbuf_page_addr = NULL; 202 struct host1x_bo *cmdbuf = g->bo; 203 unsigned int i; 204 205 /* pin & patch the relocs for one gather */ 206 for (i = 0; i < job->num_relocs; i++) { 207 struct host1x_reloc *reloc = &job->relocs[i]; 208 u32 reloc_addr = (job->reloc_addr_phys[i] + 209 reloc->target.offset) >> reloc->shift; 210 u32 *target; 211 212 /* skip all other gathers */ 213 if (cmdbuf != reloc->cmdbuf.bo) 214 continue; 215 216 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 217 target = (u32 *)job->gather_copy_mapped + 218 reloc->cmdbuf.offset / sizeof(u32) + 219 g->offset / sizeof(u32); 220 goto patch_reloc; 221 } 222 223 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) { 224 if (cmdbuf_page_addr) 225 host1x_bo_kunmap(cmdbuf, last_page, 226 cmdbuf_page_addr); 227 228 cmdbuf_page_addr = host1x_bo_kmap(cmdbuf, 229 reloc->cmdbuf.offset >> PAGE_SHIFT); 230 last_page = reloc->cmdbuf.offset >> PAGE_SHIFT; 231 232 if (unlikely(!cmdbuf_page_addr)) { 233 pr_err("Could not map cmdbuf for relocation\n"); 234 return -ENOMEM; 235 } 236 } 237 238 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK); 239 patch_reloc: 240 *target = reloc_addr; 241 } 242 243 if (cmdbuf_page_addr) 244 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr); 245 246 return 0; 247 } 248 249 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 250 unsigned int offset) 251 { 252 offset *= sizeof(u32); 253 254 if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset) 255 return false; 256 257 /* relocation shift value validation isn't implemented yet */ 258 if (reloc->shift) 259 return false; 260 261 return true; 262 } 263 264 struct host1x_firewall { 265 struct host1x_job *job; 266 struct device *dev; 267 268 unsigned int num_relocs; 269 struct host1x_reloc *reloc; 270 271 struct host1x_bo *cmdbuf; 272 unsigned int offset; 273 274 u32 words; 275 u32 class; 276 u32 reg; 277 u32 mask; 278 u32 count; 279 }; 280 281 static int check_register(struct host1x_firewall *fw, unsigned long offset) 282 { 283 if (!fw->job->is_addr_reg) 284 return 0; 285 286 if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) { 287 if (!fw->num_relocs) 288 return -EINVAL; 289 290 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset)) 291 return -EINVAL; 292 293 fw->num_relocs--; 294 fw->reloc++; 295 } 296 297 return 0; 298 } 299 300 static int check_class(struct host1x_firewall *fw, u32 class) 301 { 302 if (!fw->job->is_valid_class) { 303 if (fw->class != class) 304 return -EINVAL; 305 } else { 306 if (!fw->job->is_valid_class(fw->class)) 307 return -EINVAL; 308 } 309 310 return 0; 311 } 312 313 static int check_mask(struct host1x_firewall *fw) 314 { 315 u32 mask = fw->mask; 316 u32 reg = fw->reg; 317 int ret; 318 319 while (mask) { 320 if (fw->words == 0) 321 return -EINVAL; 322 323 if (mask & 1) { 324 ret = check_register(fw, reg); 325 if (ret < 0) 326 return ret; 327 328 fw->words--; 329 fw->offset++; 330 } 331 mask >>= 1; 332 reg++; 333 } 334 335 return 0; 336 } 337 338 static int check_incr(struct host1x_firewall *fw) 339 { 340 u32 count = fw->count; 341 u32 reg = fw->reg; 342 int ret; 343 344 while (count) { 345 if (fw->words == 0) 346 return -EINVAL; 347 348 ret = check_register(fw, reg); 349 if (ret < 0) 350 return ret; 351 352 reg++; 353 fw->words--; 354 fw->offset++; 355 count--; 356 } 357 358 return 0; 359 } 360 361 static int check_nonincr(struct host1x_firewall *fw) 362 { 363 u32 count = fw->count; 364 int ret; 365 366 while (count) { 367 if (fw->words == 0) 368 return -EINVAL; 369 370 ret = check_register(fw, fw->reg); 371 if (ret < 0) 372 return ret; 373 374 fw->words--; 375 fw->offset++; 376 count--; 377 } 378 379 return 0; 380 } 381 382 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g) 383 { 384 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped + 385 (g->offset / sizeof(u32)); 386 u32 job_class = fw->class; 387 int err = 0; 388 389 fw->words = g->words; 390 fw->cmdbuf = g->bo; 391 fw->offset = 0; 392 393 while (fw->words && !err) { 394 u32 word = cmdbuf_base[fw->offset]; 395 u32 opcode = (word & 0xf0000000) >> 28; 396 397 fw->mask = 0; 398 fw->reg = 0; 399 fw->count = 0; 400 fw->words--; 401 fw->offset++; 402 403 switch (opcode) { 404 case 0: 405 fw->class = word >> 6 & 0x3ff; 406 fw->mask = word & 0x3f; 407 fw->reg = word >> 16 & 0xfff; 408 err = check_class(fw, job_class); 409 if (!err) 410 err = check_mask(fw); 411 if (err) 412 goto out; 413 break; 414 case 1: 415 fw->reg = word >> 16 & 0xfff; 416 fw->count = word & 0xffff; 417 err = check_incr(fw); 418 if (err) 419 goto out; 420 break; 421 422 case 2: 423 fw->reg = word >> 16 & 0xfff; 424 fw->count = word & 0xffff; 425 err = check_nonincr(fw); 426 if (err) 427 goto out; 428 break; 429 430 case 3: 431 fw->mask = word & 0xffff; 432 fw->reg = word >> 16 & 0xfff; 433 err = check_mask(fw); 434 if (err) 435 goto out; 436 break; 437 case 4: 438 case 14: 439 break; 440 default: 441 err = -EINVAL; 442 break; 443 } 444 } 445 446 out: 447 return err; 448 } 449 450 static inline int copy_gathers(struct host1x_job *job, struct device *dev) 451 { 452 struct host1x_firewall fw; 453 size_t size = 0; 454 size_t offset = 0; 455 unsigned int i; 456 457 fw.job = job; 458 fw.dev = dev; 459 fw.reloc = job->relocs; 460 fw.num_relocs = job->num_relocs; 461 fw.class = job->class; 462 463 for (i = 0; i < job->num_gathers; i++) { 464 struct host1x_job_gather *g = &job->gathers[i]; 465 466 size += g->words * sizeof(u32); 467 } 468 469 /* 470 * Try a non-blocking allocation from a higher priority pools first, 471 * as awaiting for the allocation here is a major performance hit. 472 */ 473 job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy, 474 GFP_NOWAIT); 475 476 /* the higher priority allocation failed, try the generic-blocking */ 477 if (!job->gather_copy_mapped) 478 job->gather_copy_mapped = dma_alloc_wc(dev, size, 479 &job->gather_copy, 480 GFP_KERNEL); 481 if (!job->gather_copy_mapped) 482 return -ENOMEM; 483 484 job->gather_copy_size = size; 485 486 for (i = 0; i < job->num_gathers; i++) { 487 struct host1x_job_gather *g = &job->gathers[i]; 488 void *gather; 489 490 /* Copy the gather */ 491 gather = host1x_bo_mmap(g->bo); 492 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 493 g->words * sizeof(u32)); 494 host1x_bo_munmap(g->bo, gather); 495 496 /* Store the location in the buffer */ 497 g->base = job->gather_copy; 498 g->offset = offset; 499 500 /* Validate the job */ 501 if (validate(&fw, g)) 502 return -EINVAL; 503 504 offset += g->words * sizeof(u32); 505 } 506 507 /* No relocs should remain at this point */ 508 if (fw.num_relocs) 509 return -EINVAL; 510 511 return 0; 512 } 513 514 int host1x_job_pin(struct host1x_job *job, struct device *dev) 515 { 516 int err; 517 unsigned int i, j; 518 struct host1x *host = dev_get_drvdata(dev->parent); 519 520 /* pin memory */ 521 err = pin_job(host, job); 522 if (err) 523 goto out; 524 525 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) { 526 err = copy_gathers(job, dev); 527 if (err) 528 goto out; 529 } 530 531 /* patch gathers */ 532 for (i = 0; i < job->num_gathers; i++) { 533 struct host1x_job_gather *g = &job->gathers[i]; 534 535 /* process each gather mem only once */ 536 if (g->handled) 537 continue; 538 539 /* copy_gathers() sets gathers base if firewall is enabled */ 540 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 541 g->base = job->gather_addr_phys[i]; 542 543 for (j = i + 1; j < job->num_gathers; j++) { 544 if (job->gathers[j].bo == g->bo) { 545 job->gathers[j].handled = true; 546 job->gathers[j].base = g->base; 547 } 548 } 549 550 err = do_relocs(job, g); 551 if (err) 552 break; 553 } 554 555 out: 556 if (err) 557 host1x_job_unpin(job); 558 wmb(); 559 560 return err; 561 } 562 EXPORT_SYMBOL(host1x_job_pin); 563 564 void host1x_job_unpin(struct host1x_job *job) 565 { 566 struct host1x *host = dev_get_drvdata(job->channel->dev->parent); 567 unsigned int i; 568 569 for (i = 0; i < job->num_unpins; i++) { 570 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 571 572 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && 573 unpin->size && host->domain) { 574 iommu_unmap(host->domain, job->addr_phys[i], 575 unpin->size); 576 free_iova(&host->iova, 577 iova_pfn(&host->iova, job->addr_phys[i])); 578 } 579 580 host1x_bo_unpin(unpin->bo, unpin->sgt); 581 host1x_bo_put(unpin->bo); 582 } 583 584 job->num_unpins = 0; 585 586 if (job->gather_copy_size) 587 dma_free_wc(job->channel->dev, job->gather_copy_size, 588 job->gather_copy_mapped, job->gather_copy); 589 } 590 EXPORT_SYMBOL(host1x_job_unpin); 591 592 /* 593 * Debug routine used to dump job entries 594 */ 595 void host1x_job_dump(struct device *dev, struct host1x_job *job) 596 { 597 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 598 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 599 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 600 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 601 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 602 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 603 } 604