1 /* 2 * Tegra host1x Job 3 * 4 * Copyright (c) 2010-2015, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/dma-mapping.h> 20 #include <linux/err.h> 21 #include <linux/host1x.h> 22 #include <linux/kref.h> 23 #include <linux/module.h> 24 #include <linux/scatterlist.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 #include <trace/events/host1x.h> 28 29 #include "channel.h" 30 #include "dev.h" 31 #include "job.h" 32 #include "syncpt.h" 33 34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 35 u32 num_cmdbufs, u32 num_relocs, 36 u32 num_waitchks) 37 { 38 struct host1x_job *job = NULL; 39 unsigned int num_unpins = num_cmdbufs + num_relocs; 40 u64 total; 41 void *mem; 42 43 /* Check that we're not going to overflow */ 44 total = sizeof(struct host1x_job) + 45 (u64)num_relocs * sizeof(struct host1x_reloc) + 46 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) + 47 (u64)num_waitchks * sizeof(struct host1x_waitchk) + 48 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) + 49 (u64)num_unpins * sizeof(dma_addr_t) + 50 (u64)num_unpins * sizeof(u32 *); 51 if (total > ULONG_MAX) 52 return NULL; 53 54 mem = job = kzalloc(total, GFP_KERNEL); 55 if (!job) 56 return NULL; 57 58 kref_init(&job->ref); 59 job->channel = ch; 60 61 /* Redistribute memory to the structs */ 62 mem += sizeof(struct host1x_job); 63 job->relocarray = num_relocs ? mem : NULL; 64 mem += num_relocs * sizeof(struct host1x_reloc); 65 job->unpins = num_unpins ? mem : NULL; 66 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 67 job->waitchk = num_waitchks ? mem : NULL; 68 mem += num_waitchks * sizeof(struct host1x_waitchk); 69 job->gathers = num_cmdbufs ? mem : NULL; 70 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 71 job->addr_phys = num_unpins ? mem : NULL; 72 73 job->reloc_addr_phys = job->addr_phys; 74 job->gather_addr_phys = &job->addr_phys[num_relocs]; 75 76 return job; 77 } 78 EXPORT_SYMBOL(host1x_job_alloc); 79 80 struct host1x_job *host1x_job_get(struct host1x_job *job) 81 { 82 kref_get(&job->ref); 83 return job; 84 } 85 EXPORT_SYMBOL(host1x_job_get); 86 87 static void job_free(struct kref *ref) 88 { 89 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 90 91 kfree(job); 92 } 93 94 void host1x_job_put(struct host1x_job *job) 95 { 96 kref_put(&job->ref, job_free); 97 } 98 EXPORT_SYMBOL(host1x_job_put); 99 100 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 101 u32 words, u32 offset) 102 { 103 struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers]; 104 105 cur_gather->words = words; 106 cur_gather->bo = bo; 107 cur_gather->offset = offset; 108 job->num_gathers++; 109 } 110 EXPORT_SYMBOL(host1x_job_add_gather); 111 112 /* 113 * NULL an already satisfied WAIT_SYNCPT host method, by patching its 114 * args in the command stream. The method data is changed to reference 115 * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt 116 * with a matching threshold value of 0, so is guaranteed to be popped 117 * by the host HW. 118 */ 119 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp, 120 struct host1x_bo *h, u32 offset) 121 { 122 void *patch_addr = NULL; 123 124 /* patch the wait */ 125 patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT); 126 if (patch_addr) { 127 host1x_syncpt_patch_wait(sp, 128 patch_addr + (offset & ~PAGE_MASK)); 129 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr); 130 } else 131 pr_err("Could not map cmdbuf for wait check\n"); 132 } 133 134 /* 135 * Check driver supplied waitchk structs for syncpt thresholds 136 * that have already been satisfied and NULL the comparison (to 137 * avoid a wrap condition in the HW). 138 */ 139 static int do_waitchks(struct host1x_job *job, struct host1x *host, 140 struct host1x_bo *patch) 141 { 142 int i; 143 144 /* compare syncpt vs wait threshold */ 145 for (i = 0; i < job->num_waitchk; i++) { 146 struct host1x_waitchk *wait = &job->waitchk[i]; 147 struct host1x_syncpt *sp = 148 host1x_syncpt_get(host, wait->syncpt_id); 149 150 /* validate syncpt id */ 151 if (wait->syncpt_id > host1x_syncpt_nb_pts(host)) 152 continue; 153 154 /* skip all other gathers */ 155 if (patch != wait->bo) 156 continue; 157 158 trace_host1x_syncpt_wait_check(wait->bo, wait->offset, 159 wait->syncpt_id, wait->thresh, 160 host1x_syncpt_read_min(sp)); 161 162 if (host1x_syncpt_is_expired(sp, wait->thresh)) { 163 dev_dbg(host->dev, 164 "drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n", 165 wait->syncpt_id, sp->name, wait->thresh, 166 host1x_syncpt_read_min(sp)); 167 168 host1x_syncpt_patch_offset(sp, patch, wait->offset); 169 } 170 171 wait->bo = NULL; 172 } 173 174 return 0; 175 } 176 177 static unsigned int pin_job(struct host1x_job *job) 178 { 179 unsigned int i; 180 181 job->num_unpins = 0; 182 183 for (i = 0; i < job->num_relocs; i++) { 184 struct host1x_reloc *reloc = &job->relocarray[i]; 185 struct sg_table *sgt; 186 dma_addr_t phys_addr; 187 188 reloc->target.bo = host1x_bo_get(reloc->target.bo); 189 if (!reloc->target.bo) 190 goto unpin; 191 192 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt); 193 if (!phys_addr) 194 goto unpin; 195 196 job->addr_phys[job->num_unpins] = phys_addr; 197 job->unpins[job->num_unpins].bo = reloc->target.bo; 198 job->unpins[job->num_unpins].sgt = sgt; 199 job->num_unpins++; 200 } 201 202 for (i = 0; i < job->num_gathers; i++) { 203 struct host1x_job_gather *g = &job->gathers[i]; 204 struct sg_table *sgt; 205 dma_addr_t phys_addr; 206 207 g->bo = host1x_bo_get(g->bo); 208 if (!g->bo) 209 goto unpin; 210 211 phys_addr = host1x_bo_pin(g->bo, &sgt); 212 if (!phys_addr) 213 goto unpin; 214 215 job->addr_phys[job->num_unpins] = phys_addr; 216 job->unpins[job->num_unpins].bo = g->bo; 217 job->unpins[job->num_unpins].sgt = sgt; 218 job->num_unpins++; 219 } 220 221 return job->num_unpins; 222 223 unpin: 224 host1x_job_unpin(job); 225 return 0; 226 } 227 228 static int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf) 229 { 230 int i = 0; 231 u32 last_page = ~0; 232 void *cmdbuf_page_addr = NULL; 233 234 /* pin & patch the relocs for one gather */ 235 for (i = 0; i < job->num_relocs; i++) { 236 struct host1x_reloc *reloc = &job->relocarray[i]; 237 u32 reloc_addr = (job->reloc_addr_phys[i] + 238 reloc->target.offset) >> reloc->shift; 239 u32 *target; 240 241 /* skip all other gathers */ 242 if (cmdbuf != reloc->cmdbuf.bo) 243 continue; 244 245 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) { 246 if (cmdbuf_page_addr) 247 host1x_bo_kunmap(cmdbuf, last_page, 248 cmdbuf_page_addr); 249 250 cmdbuf_page_addr = host1x_bo_kmap(cmdbuf, 251 reloc->cmdbuf.offset >> PAGE_SHIFT); 252 last_page = reloc->cmdbuf.offset >> PAGE_SHIFT; 253 254 if (unlikely(!cmdbuf_page_addr)) { 255 pr_err("Could not map cmdbuf for relocation\n"); 256 return -ENOMEM; 257 } 258 } 259 260 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK); 261 *target = reloc_addr; 262 } 263 264 if (cmdbuf_page_addr) 265 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr); 266 267 return 0; 268 } 269 270 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 271 unsigned int offset) 272 { 273 offset *= sizeof(u32); 274 275 if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset) 276 return false; 277 278 return true; 279 } 280 281 struct host1x_firewall { 282 struct host1x_job *job; 283 struct device *dev; 284 285 unsigned int num_relocs; 286 struct host1x_reloc *reloc; 287 288 struct host1x_bo *cmdbuf; 289 unsigned int offset; 290 291 u32 words; 292 u32 class; 293 u32 reg; 294 u32 mask; 295 u32 count; 296 }; 297 298 static int check_register(struct host1x_firewall *fw, unsigned long offset) 299 { 300 if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) { 301 if (!fw->num_relocs) 302 return -EINVAL; 303 304 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset)) 305 return -EINVAL; 306 307 fw->num_relocs--; 308 fw->reloc++; 309 } 310 311 return 0; 312 } 313 314 static int check_mask(struct host1x_firewall *fw) 315 { 316 u32 mask = fw->mask; 317 u32 reg = fw->reg; 318 int ret; 319 320 while (mask) { 321 if (fw->words == 0) 322 return -EINVAL; 323 324 if (mask & 1) { 325 ret = check_register(fw, reg); 326 if (ret < 0) 327 return ret; 328 329 fw->words--; 330 fw->offset++; 331 } 332 mask >>= 1; 333 reg++; 334 } 335 336 return 0; 337 } 338 339 static int check_incr(struct host1x_firewall *fw) 340 { 341 u32 count = fw->count; 342 u32 reg = fw->reg; 343 int ret; 344 345 while (count) { 346 if (fw->words == 0) 347 return -EINVAL; 348 349 ret = check_register(fw, reg); 350 if (ret < 0) 351 return ret; 352 353 reg++; 354 fw->words--; 355 fw->offset++; 356 count--; 357 } 358 359 return 0; 360 } 361 362 static int check_nonincr(struct host1x_firewall *fw) 363 { 364 u32 count = fw->count; 365 int ret; 366 367 while (count) { 368 if (fw->words == 0) 369 return -EINVAL; 370 371 ret = check_register(fw, fw->reg); 372 if (ret < 0) 373 return ret; 374 375 fw->words--; 376 fw->offset++; 377 count--; 378 } 379 380 return 0; 381 } 382 383 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g) 384 { 385 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped + 386 (g->offset / sizeof(u32)); 387 int err = 0; 388 389 if (!fw->job->is_addr_reg) 390 return 0; 391 392 fw->words = g->words; 393 fw->cmdbuf = g->bo; 394 fw->offset = 0; 395 396 while (fw->words && !err) { 397 u32 word = cmdbuf_base[fw->offset]; 398 u32 opcode = (word & 0xf0000000) >> 28; 399 400 fw->mask = 0; 401 fw->reg = 0; 402 fw->count = 0; 403 fw->words--; 404 fw->offset++; 405 406 switch (opcode) { 407 case 0: 408 fw->class = word >> 6 & 0x3ff; 409 fw->mask = word & 0x3f; 410 fw->reg = word >> 16 & 0xfff; 411 err = check_mask(fw); 412 if (err) 413 goto out; 414 break; 415 case 1: 416 fw->reg = word >> 16 & 0xfff; 417 fw->count = word & 0xffff; 418 err = check_incr(fw); 419 if (err) 420 goto out; 421 break; 422 423 case 2: 424 fw->reg = word >> 16 & 0xfff; 425 fw->count = word & 0xffff; 426 err = check_nonincr(fw); 427 if (err) 428 goto out; 429 break; 430 431 case 3: 432 fw->mask = word & 0xffff; 433 fw->reg = word >> 16 & 0xfff; 434 err = check_mask(fw); 435 if (err) 436 goto out; 437 break; 438 case 4: 439 case 5: 440 case 14: 441 break; 442 default: 443 err = -EINVAL; 444 break; 445 } 446 } 447 448 out: 449 return err; 450 } 451 452 static inline int copy_gathers(struct host1x_job *job, struct device *dev) 453 { 454 struct host1x_firewall fw; 455 size_t size = 0; 456 size_t offset = 0; 457 int i; 458 459 fw.job = job; 460 fw.dev = dev; 461 fw.reloc = job->relocarray; 462 fw.num_relocs = job->num_relocs; 463 fw.class = 0; 464 465 for (i = 0; i < job->num_gathers; i++) { 466 struct host1x_job_gather *g = &job->gathers[i]; 467 468 size += g->words * sizeof(u32); 469 } 470 471 job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy, 472 GFP_KERNEL); 473 if (!job->gather_copy_mapped) { 474 job->gather_copy_mapped = NULL; 475 return -ENOMEM; 476 } 477 478 job->gather_copy_size = size; 479 480 for (i = 0; i < job->num_gathers; i++) { 481 struct host1x_job_gather *g = &job->gathers[i]; 482 void *gather; 483 484 /* Copy the gather */ 485 gather = host1x_bo_mmap(g->bo); 486 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 487 g->words * sizeof(u32)); 488 host1x_bo_munmap(g->bo, gather); 489 490 /* Store the location in the buffer */ 491 g->base = job->gather_copy; 492 g->offset = offset; 493 494 /* Validate the job */ 495 if (validate(&fw, g)) 496 return -EINVAL; 497 498 offset += g->words * sizeof(u32); 499 } 500 501 /* No relocs should remain at this point */ 502 if (fw.num_relocs) 503 return -EINVAL; 504 505 return 0; 506 } 507 508 int host1x_job_pin(struct host1x_job *job, struct device *dev) 509 { 510 int err; 511 unsigned int i, j; 512 struct host1x *host = dev_get_drvdata(dev->parent); 513 DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host)); 514 515 bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host)); 516 for (i = 0; i < job->num_waitchk; i++) { 517 u32 syncpt_id = job->waitchk[i].syncpt_id; 518 519 if (syncpt_id < host1x_syncpt_nb_pts(host)) 520 set_bit(syncpt_id, waitchk_mask); 521 } 522 523 /* get current syncpt values for waitchk */ 524 for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host)) 525 host1x_syncpt_load(host->syncpt + i); 526 527 /* pin memory */ 528 err = pin_job(job); 529 if (!err) 530 goto out; 531 532 /* patch gathers */ 533 for (i = 0; i < job->num_gathers; i++) { 534 struct host1x_job_gather *g = &job->gathers[i]; 535 536 /* process each gather mem only once */ 537 if (g->handled) 538 continue; 539 540 g->base = job->gather_addr_phys[i]; 541 542 for (j = i + 1; j < job->num_gathers; j++) { 543 if (job->gathers[j].bo == g->bo) { 544 job->gathers[j].handled = true; 545 job->gathers[j].base = g->base; 546 } 547 } 548 549 err = do_relocs(job, g->bo); 550 if (err) 551 break; 552 553 err = do_waitchks(job, host, g->bo); 554 if (err) 555 break; 556 } 557 558 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) { 559 err = copy_gathers(job, dev); 560 if (err) { 561 host1x_job_unpin(job); 562 return err; 563 } 564 } 565 566 out: 567 wmb(); 568 569 return err; 570 } 571 EXPORT_SYMBOL(host1x_job_pin); 572 573 void host1x_job_unpin(struct host1x_job *job) 574 { 575 unsigned int i; 576 577 for (i = 0; i < job->num_unpins; i++) { 578 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 579 580 host1x_bo_unpin(unpin->bo, unpin->sgt); 581 host1x_bo_put(unpin->bo); 582 } 583 584 job->num_unpins = 0; 585 586 if (job->gather_copy_size) 587 dma_free_wc(job->channel->dev, job->gather_copy_size, 588 job->gather_copy_mapped, job->gather_copy); 589 } 590 EXPORT_SYMBOL(host1x_job_unpin); 591 592 /* 593 * Debug routine used to dump job entries 594 */ 595 void host1x_job_dump(struct device *dev, struct host1x_job *job) 596 { 597 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 598 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 599 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 600 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 601 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 602 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 603 } 604