1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "nv50.h" 25 #include "nv04.h" 26 27 #include <core/client.h> 28 #include <core/engctx.h> 29 #include <core/ramht.h> 30 #include <subdev/bar.h> 31 #include <subdev/mmu.h> 32 #include <subdev/timer.h> 33 34 #include <nvif/class.h> 35 #include <nvif/unpack.h> 36 37 /******************************************************************************* 38 * FIFO channel objects 39 ******************************************************************************/ 40 41 static void 42 nv50_fifo_playlist_update_locked(struct nv50_fifo *fifo) 43 { 44 struct nvkm_device *device = fifo->base.engine.subdev.device; 45 struct nvkm_bar *bar = device->bar; 46 struct nvkm_gpuobj *cur; 47 int i, p; 48 49 cur = fifo->playlist[fifo->cur_playlist]; 50 fifo->cur_playlist = !fifo->cur_playlist; 51 52 for (i = fifo->base.min, p = 0; i < fifo->base.max; i++) { 53 if (nvkm_rd32(device, 0x002600 + (i * 4)) & 0x80000000) 54 nv_wo32(cur, p++ * 4, i); 55 } 56 57 bar->flush(bar); 58 59 nvkm_wr32(device, 0x0032f4, cur->addr >> 12); 60 nvkm_wr32(device, 0x0032ec, p); 61 nvkm_wr32(device, 0x002500, 0x00000101); 62 } 63 64 void 65 nv50_fifo_playlist_update(struct nv50_fifo *fifo) 66 { 67 mutex_lock(&nv_subdev(fifo)->mutex); 68 nv50_fifo_playlist_update_locked(fifo); 69 mutex_unlock(&nv_subdev(fifo)->mutex); 70 } 71 72 static int 73 nv50_fifo_context_attach(struct nvkm_object *parent, struct nvkm_object *object) 74 { 75 struct nvkm_bar *bar = nvkm_bar(parent); 76 struct nv50_fifo_base *base = (void *)parent->parent; 77 struct nvkm_gpuobj *ectx = (void *)object; 78 u64 limit = ectx->addr + ectx->size - 1; 79 u64 start = ectx->addr; 80 u32 addr; 81 82 switch (nv_engidx(object->engine)) { 83 case NVDEV_ENGINE_SW : return 0; 84 case NVDEV_ENGINE_GR : addr = 0x0000; break; 85 case NVDEV_ENGINE_MPEG : addr = 0x0060; break; 86 default: 87 return -EINVAL; 88 } 89 90 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; 91 nv_wo32(base->eng, addr + 0x00, 0x00190000); 92 nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit)); 93 nv_wo32(base->eng, addr + 0x08, lower_32_bits(start)); 94 nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 | 95 upper_32_bits(start)); 96 nv_wo32(base->eng, addr + 0x10, 0x00000000); 97 nv_wo32(base->eng, addr + 0x14, 0x00000000); 98 bar->flush(bar); 99 return 0; 100 } 101 102 static int 103 nv50_fifo_context_detach(struct nvkm_object *parent, bool suspend, 104 struct nvkm_object *object) 105 { 106 struct nv50_fifo *fifo = (void *)parent->engine; 107 struct nv50_fifo_base *base = (void *)parent->parent; 108 struct nv50_fifo_chan *chan = (void *)parent; 109 struct nvkm_device *device = fifo->base.engine.subdev.device; 110 struct nvkm_bar *bar = device->bar; 111 u32 addr, me; 112 int ret = 0; 113 114 switch (nv_engidx(object->engine)) { 115 case NVDEV_ENGINE_SW : return 0; 116 case NVDEV_ENGINE_GR : addr = 0x0000; break; 117 case NVDEV_ENGINE_MPEG : addr = 0x0060; break; 118 default: 119 return -EINVAL; 120 } 121 122 /* HW bug workaround: 123 * 124 * PFIFO will hang forever if the connected engines don't report 125 * that they've processed the context switch request. 126 * 127 * In order for the kickoff to work, we need to ensure all the 128 * connected engines are in a state where they can answer. 129 * 130 * Newer chipsets don't seem to suffer from this issue, and well, 131 * there's also a "ignore these engines" bitmask reg we can use 132 * if we hit the issue there.. 133 */ 134 me = nvkm_mask(device, 0x00b860, 0x00000001, 0x00000001); 135 136 /* do the kickoff... */ 137 nvkm_wr32(device, 0x0032fc, nv_gpuobj(base)->addr >> 12); 138 if (nvkm_msec(device, 2000, 139 if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) 140 break; 141 ) < 0) { 142 nv_error(fifo, "channel %d [%s] unload timeout\n", 143 chan->base.chid, nvkm_client_name(chan)); 144 if (suspend) 145 ret = -EBUSY; 146 } 147 nvkm_wr32(device, 0x00b860, me); 148 149 if (ret == 0) { 150 nv_wo32(base->eng, addr + 0x00, 0x00000000); 151 nv_wo32(base->eng, addr + 0x04, 0x00000000); 152 nv_wo32(base->eng, addr + 0x08, 0x00000000); 153 nv_wo32(base->eng, addr + 0x0c, 0x00000000); 154 nv_wo32(base->eng, addr + 0x10, 0x00000000); 155 nv_wo32(base->eng, addr + 0x14, 0x00000000); 156 bar->flush(bar); 157 } 158 159 return ret; 160 } 161 162 static int 163 nv50_fifo_object_attach(struct nvkm_object *parent, 164 struct nvkm_object *object, u32 handle) 165 { 166 struct nv50_fifo_chan *chan = (void *)parent; 167 u32 context; 168 169 if (nv_iclass(object, NV_GPUOBJ_CLASS)) 170 context = nv_gpuobj(object)->node->offset >> 4; 171 else 172 context = 0x00000004; /* just non-zero */ 173 174 switch (nv_engidx(object->engine)) { 175 case NVDEV_ENGINE_DMAOBJ: 176 case NVDEV_ENGINE_SW : context |= 0x00000000; break; 177 case NVDEV_ENGINE_GR : context |= 0x00100000; break; 178 case NVDEV_ENGINE_MPEG : context |= 0x00200000; break; 179 default: 180 return -EINVAL; 181 } 182 183 return nvkm_ramht_insert(chan->ramht, 0, handle, context); 184 } 185 186 void 187 nv50_fifo_object_detach(struct nvkm_object *parent, int cookie) 188 { 189 struct nv50_fifo_chan *chan = (void *)parent; 190 nvkm_ramht_remove(chan->ramht, cookie); 191 } 192 193 static int 194 nv50_fifo_chan_ctor_dma(struct nvkm_object *parent, struct nvkm_object *engine, 195 struct nvkm_oclass *oclass, void *data, u32 size, 196 struct nvkm_object **pobject) 197 { 198 union { 199 struct nv03_channel_dma_v0 v0; 200 } *args = data; 201 struct nvkm_bar *bar = nvkm_bar(parent); 202 struct nv50_fifo_base *base = (void *)parent; 203 struct nv50_fifo_chan *chan; 204 int ret; 205 206 nv_ioctl(parent, "create channel dma size %d\n", size); 207 if (nvif_unpack(args->v0, 0, 0, false)) { 208 nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " 209 "offset %016llx\n", args->v0.version, 210 args->v0.pushbuf, args->v0.offset); 211 } else 212 return ret; 213 214 ret = nvkm_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, 215 0x2000, args->v0.pushbuf, 216 (1ULL << NVDEV_ENGINE_DMAOBJ) | 217 (1ULL << NVDEV_ENGINE_SW) | 218 (1ULL << NVDEV_ENGINE_GR) | 219 (1ULL << NVDEV_ENGINE_MPEG), &chan); 220 *pobject = nv_object(chan); 221 if (ret) 222 return ret; 223 224 args->v0.chid = chan->base.chid; 225 226 nv_parent(chan)->context_attach = nv50_fifo_context_attach; 227 nv_parent(chan)->context_detach = nv50_fifo_context_detach; 228 nv_parent(chan)->object_attach = nv50_fifo_object_attach; 229 nv_parent(chan)->object_detach = nv50_fifo_object_detach; 230 231 ret = nvkm_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, 232 &chan->ramht); 233 if (ret) 234 return ret; 235 236 nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset)); 237 nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset)); 238 nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset)); 239 nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset)); 240 nv_wo32(base->ramfc, 0x3c, 0x003f6078); 241 nv_wo32(base->ramfc, 0x44, 0x01003fff); 242 nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); 243 nv_wo32(base->ramfc, 0x4c, 0xffffffff); 244 nv_wo32(base->ramfc, 0x60, 0x7fffffff); 245 nv_wo32(base->ramfc, 0x78, 0x00000000); 246 nv_wo32(base->ramfc, 0x7c, 0x30000001); 247 nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | 248 (4 << 24) /* SEARCH_FULL */ | 249 (chan->ramht->gpuobj.node->offset >> 4)); 250 bar->flush(bar); 251 return 0; 252 } 253 254 static int 255 nv50_fifo_chan_ctor_ind(struct nvkm_object *parent, struct nvkm_object *engine, 256 struct nvkm_oclass *oclass, void *data, u32 size, 257 struct nvkm_object **pobject) 258 { 259 union { 260 struct nv50_channel_gpfifo_v0 v0; 261 } *args = data; 262 struct nvkm_bar *bar = nvkm_bar(parent); 263 struct nv50_fifo_base *base = (void *)parent; 264 struct nv50_fifo_chan *chan; 265 u64 ioffset, ilength; 266 int ret; 267 268 nv_ioctl(parent, "create channel gpfifo size %d\n", size); 269 if (nvif_unpack(args->v0, 0, 0, false)) { 270 nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " 271 "ioffset %016llx ilength %08x\n", 272 args->v0.version, args->v0.pushbuf, args->v0.ioffset, 273 args->v0.ilength); 274 } else 275 return ret; 276 277 ret = nvkm_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, 278 0x2000, args->v0.pushbuf, 279 (1ULL << NVDEV_ENGINE_DMAOBJ) | 280 (1ULL << NVDEV_ENGINE_SW) | 281 (1ULL << NVDEV_ENGINE_GR) | 282 (1ULL << NVDEV_ENGINE_MPEG), &chan); 283 *pobject = nv_object(chan); 284 if (ret) 285 return ret; 286 287 args->v0.chid = chan->base.chid; 288 289 nv_parent(chan)->context_attach = nv50_fifo_context_attach; 290 nv_parent(chan)->context_detach = nv50_fifo_context_detach; 291 nv_parent(chan)->object_attach = nv50_fifo_object_attach; 292 nv_parent(chan)->object_detach = nv50_fifo_object_detach; 293 294 ret = nvkm_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, 295 &chan->ramht); 296 if (ret) 297 return ret; 298 299 ioffset = args->v0.ioffset; 300 ilength = order_base_2(args->v0.ilength / 8); 301 302 nv_wo32(base->ramfc, 0x3c, 0x403f6078); 303 nv_wo32(base->ramfc, 0x44, 0x01003fff); 304 nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); 305 nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset)); 306 nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); 307 nv_wo32(base->ramfc, 0x60, 0x7fffffff); 308 nv_wo32(base->ramfc, 0x78, 0x00000000); 309 nv_wo32(base->ramfc, 0x7c, 0x30000001); 310 nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | 311 (4 << 24) /* SEARCH_FULL */ | 312 (chan->ramht->gpuobj.node->offset >> 4)); 313 bar->flush(bar); 314 return 0; 315 } 316 317 void 318 nv50_fifo_chan_dtor(struct nvkm_object *object) 319 { 320 struct nv50_fifo_chan *chan = (void *)object; 321 nvkm_ramht_ref(NULL, &chan->ramht); 322 nvkm_fifo_channel_destroy(&chan->base); 323 } 324 325 static int 326 nv50_fifo_chan_init(struct nvkm_object *object) 327 { 328 struct nv50_fifo *fifo = (void *)object->engine; 329 struct nv50_fifo_base *base = (void *)object->parent; 330 struct nv50_fifo_chan *chan = (void *)object; 331 struct nvkm_gpuobj *ramfc = base->ramfc; 332 struct nvkm_device *device = fifo->base.engine.subdev.device; 333 u32 chid = chan->base.chid; 334 int ret; 335 336 ret = nvkm_fifo_channel_init(&chan->base); 337 if (ret) 338 return ret; 339 340 nvkm_wr32(device, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 12); 341 nv50_fifo_playlist_update(fifo); 342 return 0; 343 } 344 345 int 346 nv50_fifo_chan_fini(struct nvkm_object *object, bool suspend) 347 { 348 struct nv50_fifo *fifo = (void *)object->engine; 349 struct nv50_fifo_chan *chan = (void *)object; 350 struct nvkm_device *device = fifo->base.engine.subdev.device; 351 u32 chid = chan->base.chid; 352 353 /* remove channel from playlist, fifo will unload context */ 354 nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x00000000); 355 nv50_fifo_playlist_update(fifo); 356 nvkm_wr32(device, 0x002600 + (chid * 4), 0x00000000); 357 358 return nvkm_fifo_channel_fini(&chan->base, suspend); 359 } 360 361 static struct nvkm_ofuncs 362 nv50_fifo_ofuncs_dma = { 363 .ctor = nv50_fifo_chan_ctor_dma, 364 .dtor = nv50_fifo_chan_dtor, 365 .init = nv50_fifo_chan_init, 366 .fini = nv50_fifo_chan_fini, 367 .map = _nvkm_fifo_channel_map, 368 .rd32 = _nvkm_fifo_channel_rd32, 369 .wr32 = _nvkm_fifo_channel_wr32, 370 .ntfy = _nvkm_fifo_channel_ntfy 371 }; 372 373 static struct nvkm_ofuncs 374 nv50_fifo_ofuncs_ind = { 375 .ctor = nv50_fifo_chan_ctor_ind, 376 .dtor = nv50_fifo_chan_dtor, 377 .init = nv50_fifo_chan_init, 378 .fini = nv50_fifo_chan_fini, 379 .map = _nvkm_fifo_channel_map, 380 .rd32 = _nvkm_fifo_channel_rd32, 381 .wr32 = _nvkm_fifo_channel_wr32, 382 .ntfy = _nvkm_fifo_channel_ntfy 383 }; 384 385 static struct nvkm_oclass 386 nv50_fifo_sclass[] = { 387 { NV50_CHANNEL_DMA, &nv50_fifo_ofuncs_dma }, 388 { NV50_CHANNEL_GPFIFO, &nv50_fifo_ofuncs_ind }, 389 {} 390 }; 391 392 /******************************************************************************* 393 * FIFO context - basically just the instmem reserved for the channel 394 ******************************************************************************/ 395 396 static int 397 nv50_fifo_context_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 398 struct nvkm_oclass *oclass, void *data, u32 size, 399 struct nvkm_object **pobject) 400 { 401 struct nv50_fifo_base *base; 402 int ret; 403 404 ret = nvkm_fifo_context_create(parent, engine, oclass, NULL, 0x10000, 405 0x1000, NVOBJ_FLAG_HEAP, &base); 406 *pobject = nv_object(base); 407 if (ret) 408 return ret; 409 410 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x0200, 411 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc); 412 if (ret) 413 return ret; 414 415 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x1200, 0, 416 NVOBJ_FLAG_ZERO_ALLOC, &base->eng); 417 if (ret) 418 return ret; 419 420 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x4000, 0, 0, 421 &base->pgd); 422 if (ret) 423 return ret; 424 425 ret = nvkm_vm_ref(nvkm_client(parent)->vm, &base->vm, base->pgd); 426 if (ret) 427 return ret; 428 429 return 0; 430 } 431 432 void 433 nv50_fifo_context_dtor(struct nvkm_object *object) 434 { 435 struct nv50_fifo_base *base = (void *)object; 436 nvkm_vm_ref(NULL, &base->vm, base->pgd); 437 nvkm_gpuobj_ref(NULL, &base->pgd); 438 nvkm_gpuobj_ref(NULL, &base->eng); 439 nvkm_gpuobj_ref(NULL, &base->ramfc); 440 nvkm_gpuobj_ref(NULL, &base->cache); 441 nvkm_fifo_context_destroy(&base->base); 442 } 443 444 static struct nvkm_oclass 445 nv50_fifo_cclass = { 446 .handle = NV_ENGCTX(FIFO, 0x50), 447 .ofuncs = &(struct nvkm_ofuncs) { 448 .ctor = nv50_fifo_context_ctor, 449 .dtor = nv50_fifo_context_dtor, 450 .init = _nvkm_fifo_context_init, 451 .fini = _nvkm_fifo_context_fini, 452 .rd32 = _nvkm_fifo_context_rd32, 453 .wr32 = _nvkm_fifo_context_wr32, 454 }, 455 }; 456 457 /******************************************************************************* 458 * PFIFO engine 459 ******************************************************************************/ 460 461 static int 462 nv50_fifo_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 463 struct nvkm_oclass *oclass, void *data, u32 size, 464 struct nvkm_object **pobject) 465 { 466 struct nv50_fifo *fifo; 467 int ret; 468 469 ret = nvkm_fifo_create(parent, engine, oclass, 1, 127, &fifo); 470 *pobject = nv_object(fifo); 471 if (ret) 472 return ret; 473 474 ret = nvkm_gpuobj_new(nv_object(fifo), NULL, 128 * 4, 0x1000, 0, 475 &fifo->playlist[0]); 476 if (ret) 477 return ret; 478 479 ret = nvkm_gpuobj_new(nv_object(fifo), NULL, 128 * 4, 0x1000, 0, 480 &fifo->playlist[1]); 481 if (ret) 482 return ret; 483 484 nv_subdev(fifo)->unit = 0x00000100; 485 nv_subdev(fifo)->intr = nv04_fifo_intr; 486 nv_engine(fifo)->cclass = &nv50_fifo_cclass; 487 nv_engine(fifo)->sclass = nv50_fifo_sclass; 488 fifo->base.pause = nv04_fifo_pause; 489 fifo->base.start = nv04_fifo_start; 490 return 0; 491 } 492 493 void 494 nv50_fifo_dtor(struct nvkm_object *object) 495 { 496 struct nv50_fifo *fifo = (void *)object; 497 498 nvkm_gpuobj_ref(NULL, &fifo->playlist[1]); 499 nvkm_gpuobj_ref(NULL, &fifo->playlist[0]); 500 501 nvkm_fifo_destroy(&fifo->base); 502 } 503 504 int 505 nv50_fifo_init(struct nvkm_object *object) 506 { 507 struct nv50_fifo *fifo = (void *)object; 508 struct nvkm_device *device = fifo->base.engine.subdev.device; 509 int ret, i; 510 511 ret = nvkm_fifo_init(&fifo->base); 512 if (ret) 513 return ret; 514 515 nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); 516 nvkm_mask(device, 0x000200, 0x00000100, 0x00000100); 517 nvkm_wr32(device, 0x00250c, 0x6f3cfc34); 518 nvkm_wr32(device, 0x002044, 0x01003fff); 519 520 nvkm_wr32(device, 0x002100, 0xffffffff); 521 nvkm_wr32(device, 0x002140, 0xbfffffff); 522 523 for (i = 0; i < 128; i++) 524 nvkm_wr32(device, 0x002600 + (i * 4), 0x00000000); 525 nv50_fifo_playlist_update_locked(fifo); 526 527 nvkm_wr32(device, 0x003200, 0x00000001); 528 nvkm_wr32(device, 0x003250, 0x00000001); 529 nvkm_wr32(device, 0x002500, 0x00000001); 530 return 0; 531 } 532 533 struct nvkm_oclass * 534 nv50_fifo_oclass = &(struct nvkm_oclass) { 535 .handle = NV_ENGINE(FIFO, 0x50), 536 .ofuncs = &(struct nvkm_ofuncs) { 537 .ctor = nv50_fifo_ctor, 538 .dtor = nv50_fifo_dtor, 539 .init = nv50_fifo_init, 540 .fini = _nvkm_fifo_fini, 541 }, 542 }; 543