1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include <nvif/push006c.h> 25 26 #include <nvif/class.h> 27 #include <nvif/cl0002.h> 28 #include <nvif/cl006b.h> 29 #include <nvif/cl506f.h> 30 #include <nvif/cl906f.h> 31 #include <nvif/cla06f.h> 32 #include <nvif/clc36f.h> 33 #include <nvif/if0020.h> 34 #include <nvif/ioctl.h> 35 36 #include "nouveau_drv.h" 37 #include "nouveau_dma.h" 38 #include "nouveau_bo.h" 39 #include "nouveau_chan.h" 40 #include "nouveau_fence.h" 41 #include "nouveau_abi16.h" 42 #include "nouveau_vmm.h" 43 #include "nouveau_svm.h" 44 45 MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM"); 46 int nouveau_vram_pushbuf; 47 module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400); 48 49 static int 50 nouveau_channel_killed(struct nvif_event *event, void *repv, u32 repc) 51 { 52 struct nouveau_channel *chan = container_of(event, typeof(*chan), kill); 53 struct nouveau_cli *cli = (void *)chan->user.client; 54 55 NV_PRINTK(warn, cli, "channel %d killed!\n", chan->chid); 56 atomic_set(&chan->killed, 1); 57 if (chan->fence) 58 nouveau_fence_context_kill(chan->fence, -ENODEV); 59 60 return NVIF_EVENT_DROP; 61 } 62 63 int 64 nouveau_channel_idle(struct nouveau_channel *chan) 65 { 66 if (likely(chan && chan->fence && !atomic_read(&chan->killed))) { 67 struct nouveau_cli *cli = (void *)chan->user.client; 68 struct nouveau_fence *fence = NULL; 69 int ret; 70 71 ret = nouveau_fence_new(chan, false, &fence); 72 if (!ret) { 73 ret = nouveau_fence_wait(fence, false, false); 74 nouveau_fence_unref(&fence); 75 } 76 77 if (ret) { 78 NV_PRINTK(err, cli, "failed to idle channel %d [%s]\n", 79 chan->chid, nvxx_client(&cli->base)->name); 80 return ret; 81 } 82 } 83 return 0; 84 } 85 86 void 87 nouveau_channel_del(struct nouveau_channel **pchan) 88 { 89 struct nouveau_channel *chan = *pchan; 90 if (chan) { 91 struct nouveau_cli *cli = (void *)chan->user.client; 92 93 if (chan->fence) 94 nouveau_fence(chan->drm)->context_del(chan); 95 96 if (cli) 97 nouveau_svmm_part(chan->vmm->svmm, chan->inst); 98 99 nvif_object_dtor(&chan->nvsw); 100 nvif_object_dtor(&chan->gart); 101 nvif_object_dtor(&chan->vram); 102 nvif_event_dtor(&chan->kill); 103 nvif_object_dtor(&chan->user); 104 nvif_object_dtor(&chan->push.ctxdma); 105 nouveau_vma_del(&chan->push.vma); 106 nouveau_bo_unmap(chan->push.buffer); 107 if (chan->push.buffer && chan->push.buffer->bo.pin_count) 108 nouveau_bo_unpin(chan->push.buffer); 109 nouveau_bo_ref(NULL, &chan->push.buffer); 110 kfree(chan); 111 } 112 *pchan = NULL; 113 } 114 115 static void 116 nouveau_channel_kick(struct nvif_push *push) 117 { 118 struct nouveau_channel *chan = container_of(push, typeof(*chan), chan._push); 119 chan->dma.cur = chan->dma.cur + (chan->chan._push.cur - chan->chan._push.bgn); 120 FIRE_RING(chan); 121 chan->chan._push.bgn = chan->chan._push.cur; 122 } 123 124 static int 125 nouveau_channel_wait(struct nvif_push *push, u32 size) 126 { 127 struct nouveau_channel *chan = container_of(push, typeof(*chan), chan._push); 128 int ret; 129 chan->dma.cur = chan->dma.cur + (chan->chan._push.cur - chan->chan._push.bgn); 130 ret = RING_SPACE(chan, size); 131 if (ret == 0) { 132 chan->chan._push.bgn = chan->chan._push.mem.object.map.ptr; 133 chan->chan._push.bgn = chan->chan._push.bgn + chan->dma.cur; 134 chan->chan._push.cur = chan->chan._push.bgn; 135 chan->chan._push.end = chan->chan._push.bgn + size; 136 } 137 return ret; 138 } 139 140 static int 141 nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, 142 u32 size, struct nouveau_channel **pchan) 143 { 144 struct nouveau_cli *cli = (void *)device->object.client; 145 struct nv_dma_v0 args = {}; 146 struct nouveau_channel *chan; 147 u32 target; 148 int ret; 149 150 chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL); 151 if (!chan) 152 return -ENOMEM; 153 154 chan->device = device; 155 chan->drm = drm; 156 chan->vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 157 atomic_set(&chan->killed, 0); 158 159 /* allocate memory for dma push buffer */ 160 target = NOUVEAU_GEM_DOMAIN_GART | NOUVEAU_GEM_DOMAIN_COHERENT; 161 if (nouveau_vram_pushbuf) 162 target = NOUVEAU_GEM_DOMAIN_VRAM; 163 164 ret = nouveau_bo_new(cli, size, 0, target, 0, 0, NULL, NULL, 165 &chan->push.buffer); 166 if (ret == 0) { 167 ret = nouveau_bo_pin(chan->push.buffer, target, false); 168 if (ret == 0) 169 ret = nouveau_bo_map(chan->push.buffer); 170 } 171 172 if (ret) { 173 nouveau_channel_del(pchan); 174 return ret; 175 } 176 177 chan->chan._push.mem.object.parent = cli->base.object.parent; 178 chan->chan._push.mem.object.client = &cli->base; 179 chan->chan._push.mem.object.name = "chanPush"; 180 chan->chan._push.mem.object.map.ptr = chan->push.buffer->kmap.virtual; 181 chan->chan._push.wait = nouveau_channel_wait; 182 chan->chan._push.kick = nouveau_channel_kick; 183 chan->chan.push = &chan->chan._push; 184 185 /* create dma object covering the *entire* memory space that the 186 * pushbuf lives in, this is because the GEM code requires that 187 * we be able to call out to other (indirect) push buffers 188 */ 189 chan->push.addr = chan->push.buffer->offset; 190 191 if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 192 ret = nouveau_vma_new(chan->push.buffer, chan->vmm, 193 &chan->push.vma); 194 if (ret) { 195 nouveau_channel_del(pchan); 196 return ret; 197 } 198 199 chan->push.addr = chan->push.vma->addr; 200 201 if (device->info.family >= NV_DEVICE_INFO_V0_FERMI) 202 return 0; 203 204 args.target = NV_DMA_V0_TARGET_VM; 205 args.access = NV_DMA_V0_ACCESS_VM; 206 args.start = 0; 207 args.limit = chan->vmm->vmm.limit - 1; 208 } else 209 if (chan->push.buffer->bo.resource->mem_type == TTM_PL_VRAM) { 210 if (device->info.family == NV_DEVICE_INFO_V0_TNT) { 211 /* nv04 vram pushbuf hack, retarget to its location in 212 * the framebuffer bar rather than direct vram access.. 213 * nfi why this exists, it came from the -nv ddx. 214 */ 215 args.target = NV_DMA_V0_TARGET_PCI; 216 args.access = NV_DMA_V0_ACCESS_RDWR; 217 args.start = nvxx_device(device)->func-> 218 resource_addr(nvxx_device(device), 1); 219 args.limit = args.start + device->info.ram_user - 1; 220 } else { 221 args.target = NV_DMA_V0_TARGET_VRAM; 222 args.access = NV_DMA_V0_ACCESS_RDWR; 223 args.start = 0; 224 args.limit = device->info.ram_user - 1; 225 } 226 } else { 227 if (chan->drm->agp.bridge) { 228 args.target = NV_DMA_V0_TARGET_AGP; 229 args.access = NV_DMA_V0_ACCESS_RDWR; 230 args.start = chan->drm->agp.base; 231 args.limit = chan->drm->agp.base + 232 chan->drm->agp.size - 1; 233 } else { 234 args.target = NV_DMA_V0_TARGET_VM; 235 args.access = NV_DMA_V0_ACCESS_RDWR; 236 args.start = 0; 237 args.limit = chan->vmm->vmm.limit - 1; 238 } 239 } 240 241 ret = nvif_object_ctor(&device->object, "abi16PushCtxDma", 0, 242 NV_DMA_FROM_MEMORY, &args, sizeof(args), 243 &chan->push.ctxdma); 244 if (ret) { 245 nouveau_channel_del(pchan); 246 return ret; 247 } 248 249 return 0; 250 } 251 252 static int 253 nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, 254 u64 runlist, bool priv, struct nouveau_channel **pchan) 255 { 256 static const u16 oclasses[] = { AMPERE_CHANNEL_GPFIFO_B, 257 TURING_CHANNEL_GPFIFO_A, 258 VOLTA_CHANNEL_GPFIFO_A, 259 PASCAL_CHANNEL_GPFIFO_A, 260 MAXWELL_CHANNEL_GPFIFO_A, 261 KEPLER_CHANNEL_GPFIFO_B, 262 KEPLER_CHANNEL_GPFIFO_A, 263 FERMI_CHANNEL_GPFIFO, 264 G82_CHANNEL_GPFIFO, 265 NV50_CHANNEL_GPFIFO, 266 0 }; 267 const u16 *oclass = oclasses; 268 union { 269 struct nv50_channel_gpfifo_v0 nv50; 270 struct fermi_channel_gpfifo_v0 fermi; 271 struct kepler_channel_gpfifo_a_v0 kepler; 272 struct volta_channel_gpfifo_a_v0 volta; 273 } args; 274 struct nouveau_channel *chan; 275 u32 size; 276 int ret; 277 278 /* allocate dma push buffer */ 279 ret = nouveau_channel_prep(drm, device, 0x12000, &chan); 280 *pchan = chan; 281 if (ret) 282 return ret; 283 284 /* create channel object */ 285 do { 286 if (oclass[0] >= VOLTA_CHANNEL_GPFIFO_A) { 287 args.volta.version = 0; 288 args.volta.ilength = 0x02000; 289 args.volta.ioffset = 0x10000 + chan->push.addr; 290 args.volta.runlist = runlist; 291 args.volta.vmm = nvif_handle(&chan->vmm->vmm.object); 292 args.volta.priv = priv; 293 size = sizeof(args.volta); 294 } else 295 if (oclass[0] >= KEPLER_CHANNEL_GPFIFO_A) { 296 args.kepler.version = 0; 297 args.kepler.ilength = 0x02000; 298 args.kepler.ioffset = 0x10000 + chan->push.addr; 299 args.kepler.runlist = runlist; 300 args.kepler.vmm = nvif_handle(&chan->vmm->vmm.object); 301 args.kepler.priv = priv; 302 size = sizeof(args.kepler); 303 } else 304 if (oclass[0] >= FERMI_CHANNEL_GPFIFO) { 305 args.fermi.version = 0; 306 args.fermi.ilength = 0x02000; 307 args.fermi.ioffset = 0x10000 + chan->push.addr; 308 args.fermi.vmm = nvif_handle(&chan->vmm->vmm.object); 309 size = sizeof(args.fermi); 310 } else { 311 args.nv50.version = 0; 312 args.nv50.ilength = 0x02000; 313 args.nv50.ioffset = 0x10000 + chan->push.addr; 314 args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); 315 args.nv50.vmm = nvif_handle(&chan->vmm->vmm.object); 316 size = sizeof(args.nv50); 317 } 318 319 ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, 320 *oclass++, &args, size, &chan->user); 321 if (ret == 0) { 322 if (chan->user.oclass >= VOLTA_CHANNEL_GPFIFO_A) { 323 chan->chid = args.volta.chid; 324 chan->inst = args.volta.inst; 325 chan->token = args.volta.token; 326 } else 327 if (chan->user.oclass >= KEPLER_CHANNEL_GPFIFO_A) { 328 chan->chid = args.kepler.chid; 329 chan->inst = args.kepler.inst; 330 } else 331 if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { 332 chan->chid = args.fermi.chid; 333 } else { 334 chan->chid = args.nv50.chid; 335 } 336 return ret; 337 } 338 } while (*oclass); 339 340 nouveau_channel_del(pchan); 341 return ret; 342 } 343 344 static int 345 nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device, 346 struct nouveau_channel **pchan) 347 { 348 static const u16 oclasses[] = { NV40_CHANNEL_DMA, 349 NV17_CHANNEL_DMA, 350 NV10_CHANNEL_DMA, 351 NV03_CHANNEL_DMA, 352 0 }; 353 const u16 *oclass = oclasses; 354 struct nv03_channel_dma_v0 args; 355 struct nouveau_channel *chan; 356 int ret; 357 358 /* allocate dma push buffer */ 359 ret = nouveau_channel_prep(drm, device, 0x10000, &chan); 360 *pchan = chan; 361 if (ret) 362 return ret; 363 364 /* create channel object */ 365 args.version = 0; 366 args.pushbuf = nvif_handle(&chan->push.ctxdma); 367 args.offset = chan->push.addr; 368 369 do { 370 ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, 371 *oclass++, &args, sizeof(args), 372 &chan->user); 373 if (ret == 0) { 374 chan->chid = args.chid; 375 return ret; 376 } 377 } while (ret && *oclass); 378 379 nouveau_channel_del(pchan); 380 return ret; 381 } 382 383 static int 384 nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) 385 { 386 struct nvif_device *device = chan->device; 387 struct nouveau_drm *drm = chan->drm; 388 struct nv_dma_v0 args = {}; 389 int ret, i; 390 391 ret = nvif_object_map(&chan->user, NULL, 0); 392 if (ret) 393 return ret; 394 395 if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO && 396 chan->user.oclass < AMPERE_CHANNEL_GPFIFO_B) { 397 struct { 398 struct nvif_event_v0 base; 399 struct nvif_chan_event_v0 host; 400 } args; 401 402 args.host.version = 0; 403 args.host.type = NVIF_CHAN_EVENT_V0_KILLED; 404 405 ret = nvif_event_ctor(&chan->user, "abi16ChanKilled", chan->chid, 406 nouveau_channel_killed, false, 407 &args.base, sizeof(args), &chan->kill); 408 if (ret == 0) 409 ret = nvif_event_allow(&chan->kill); 410 if (ret) { 411 NV_ERROR(drm, "Failed to request channel kill " 412 "notification: %d\n", ret); 413 return ret; 414 } 415 } 416 417 /* allocate dma objects to cover all allowed vram, and gart */ 418 if (device->info.family < NV_DEVICE_INFO_V0_FERMI) { 419 if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 420 args.target = NV_DMA_V0_TARGET_VM; 421 args.access = NV_DMA_V0_ACCESS_VM; 422 args.start = 0; 423 args.limit = chan->vmm->vmm.limit - 1; 424 } else { 425 args.target = NV_DMA_V0_TARGET_VRAM; 426 args.access = NV_DMA_V0_ACCESS_RDWR; 427 args.start = 0; 428 args.limit = device->info.ram_user - 1; 429 } 430 431 ret = nvif_object_ctor(&chan->user, "abi16ChanVramCtxDma", vram, 432 NV_DMA_IN_MEMORY, &args, sizeof(args), 433 &chan->vram); 434 if (ret) 435 return ret; 436 437 if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 438 args.target = NV_DMA_V0_TARGET_VM; 439 args.access = NV_DMA_V0_ACCESS_VM; 440 args.start = 0; 441 args.limit = chan->vmm->vmm.limit - 1; 442 } else 443 if (chan->drm->agp.bridge) { 444 args.target = NV_DMA_V0_TARGET_AGP; 445 args.access = NV_DMA_V0_ACCESS_RDWR; 446 args.start = chan->drm->agp.base; 447 args.limit = chan->drm->agp.base + 448 chan->drm->agp.size - 1; 449 } else { 450 args.target = NV_DMA_V0_TARGET_VM; 451 args.access = NV_DMA_V0_ACCESS_RDWR; 452 args.start = 0; 453 args.limit = chan->vmm->vmm.limit - 1; 454 } 455 456 ret = nvif_object_ctor(&chan->user, "abi16ChanGartCtxDma", gart, 457 NV_DMA_IN_MEMORY, &args, sizeof(args), 458 &chan->gart); 459 if (ret) 460 return ret; 461 } 462 463 /* initialise dma tracking parameters */ 464 switch (chan->user.oclass & 0x00ff) { 465 case 0x006b: 466 case 0x006e: 467 chan->user_put = 0x40; 468 chan->user_get = 0x44; 469 chan->dma.max = (0x10000 / 4) - 2; 470 break; 471 default: 472 chan->user_put = 0x40; 473 chan->user_get = 0x44; 474 chan->user_get_hi = 0x60; 475 chan->dma.ib_base = 0x10000 / 4; 476 chan->dma.ib_max = (0x02000 / 8) - 1; 477 chan->dma.ib_put = 0; 478 chan->dma.ib_free = chan->dma.ib_max - chan->dma.ib_put; 479 chan->dma.max = chan->dma.ib_base; 480 break; 481 } 482 483 chan->dma.put = 0; 484 chan->dma.cur = chan->dma.put; 485 chan->dma.free = chan->dma.max - chan->dma.cur; 486 487 ret = PUSH_WAIT(chan->chan.push, NOUVEAU_DMA_SKIPS); 488 if (ret) 489 return ret; 490 491 for (i = 0; i < NOUVEAU_DMA_SKIPS; i++) 492 PUSH_DATA(chan->chan.push, 0x00000000); 493 494 /* allocate software object class (used for fences on <= nv05) */ 495 if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) { 496 ret = nvif_object_ctor(&chan->user, "abi16NvswFence", 0x006e, 497 NVIF_CLASS_SW_NV04, 498 NULL, 0, &chan->nvsw); 499 if (ret) 500 return ret; 501 502 ret = PUSH_WAIT(chan->chan.push, 2); 503 if (ret) 504 return ret; 505 506 PUSH_NVSQ(chan->chan.push, NV_SW, 0x0000, chan->nvsw.handle); 507 PUSH_KICK(chan->chan.push); 508 } 509 510 /* initialise synchronisation */ 511 return nouveau_fence(chan->drm)->context_new(chan); 512 } 513 514 int 515 nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, 516 bool priv, u64 runm, u32 vram, u32 gart, struct nouveau_channel **pchan) 517 { 518 struct nouveau_cli *cli = (void *)device->object.client; 519 int ret; 520 521 /* hack until fencenv50 is fixed, and agp access relaxed */ 522 ret = nouveau_channel_ind(drm, device, runm, priv, pchan); 523 if (ret) { 524 NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret); 525 ret = nouveau_channel_dma(drm, device, pchan); 526 if (ret) { 527 NV_PRINTK(dbg, cli, "dma channel create, %d\n", ret); 528 return ret; 529 } 530 } 531 532 ret = nouveau_channel_init(*pchan, vram, gart); 533 if (ret) { 534 NV_PRINTK(err, cli, "channel failed to initialise, %d\n", ret); 535 nouveau_channel_del(pchan); 536 return ret; 537 } 538 539 ret = nouveau_svmm_join((*pchan)->vmm->svmm, (*pchan)->inst); 540 if (ret) 541 nouveau_channel_del(pchan); 542 543 return ret; 544 } 545 546 void 547 nouveau_channels_fini(struct nouveau_drm *drm) 548 { 549 kfree(drm->runl); 550 } 551 552 int 553 nouveau_channels_init(struct nouveau_drm *drm) 554 { 555 struct { 556 struct nv_device_info_v1 m; 557 struct { 558 struct nv_device_info_v1_data channels; 559 struct nv_device_info_v1_data runlists; 560 } v; 561 } args = { 562 .m.version = 1, 563 .m.count = sizeof(args.v) / sizeof(args.v.channels), 564 .v.channels.mthd = NV_DEVICE_HOST_CHANNELS, 565 .v.runlists.mthd = NV_DEVICE_HOST_RUNLISTS, 566 }; 567 struct nvif_object *device = &drm->client.device.object; 568 int ret, i; 569 570 ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args)); 571 if (ret || 572 args.v.runlists.mthd == NV_DEVICE_INFO_INVALID || !args.v.runlists.data || 573 args.v.channels.mthd == NV_DEVICE_INFO_INVALID) 574 return -ENODEV; 575 576 drm->chan_nr = drm->chan_total = args.v.channels.data; 577 drm->runl_nr = fls64(args.v.runlists.data); 578 drm->runl = kcalloc(drm->runl_nr, sizeof(*drm->runl), GFP_KERNEL); 579 if (!drm->runl) 580 return -ENOMEM; 581 582 if (drm->chan_nr == 0) { 583 for (i = 0; i < drm->runl_nr; i++) { 584 if (!(args.v.runlists.data & BIT(i))) 585 continue; 586 587 args.v.channels.mthd = NV_DEVICE_HOST_RUNLIST_CHANNELS; 588 args.v.channels.data = i; 589 590 ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args)); 591 if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID) 592 return -ENODEV; 593 594 drm->runl[i].chan_nr = args.v.channels.data; 595 drm->runl[i].chan_id_base = drm->chan_total; 596 drm->runl[i].context_base = dma_fence_context_alloc(drm->runl[i].chan_nr); 597 598 drm->chan_total += drm->runl[i].chan_nr; 599 } 600 } else { 601 drm->runl[0].context_base = dma_fence_context_alloc(drm->chan_nr); 602 for (i = 1; i < drm->runl_nr; i++) 603 drm->runl[i].context_base = drm->runl[0].context_base; 604 605 } 606 607 return 0; 608 } 609