1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "nv50.h" 25 #include "nv04.h" 26 27 #include <core/client.h> 28 #include <core/engctx.h> 29 #include <core/ramht.h> 30 #include <subdev/bar.h> 31 #include <subdev/mmu.h> 32 #include <subdev/timer.h> 33 34 #include <nvif/class.h> 35 #include <nvif/unpack.h> 36 37 /******************************************************************************* 38 * FIFO channel objects 39 ******************************************************************************/ 40 41 static void 42 nv50_fifo_playlist_update_locked(struct nv50_fifo_priv *priv) 43 { 44 struct nvkm_bar *bar = nvkm_bar(priv); 45 struct nvkm_gpuobj *cur; 46 int i, p; 47 48 cur = priv->playlist[priv->cur_playlist]; 49 priv->cur_playlist = !priv->cur_playlist; 50 51 for (i = priv->base.min, p = 0; i < priv->base.max; i++) { 52 if (nv_rd32(priv, 0x002600 + (i * 4)) & 0x80000000) 53 nv_wo32(cur, p++ * 4, i); 54 } 55 56 bar->flush(bar); 57 58 nv_wr32(priv, 0x0032f4, cur->addr >> 12); 59 nv_wr32(priv, 0x0032ec, p); 60 nv_wr32(priv, 0x002500, 0x00000101); 61 } 62 63 void 64 nv50_fifo_playlist_update(struct nv50_fifo_priv *priv) 65 { 66 mutex_lock(&nv_subdev(priv)->mutex); 67 nv50_fifo_playlist_update_locked(priv); 68 mutex_unlock(&nv_subdev(priv)->mutex); 69 } 70 71 static int 72 nv50_fifo_context_attach(struct nvkm_object *parent, struct nvkm_object *object) 73 { 74 struct nvkm_bar *bar = nvkm_bar(parent); 75 struct nv50_fifo_base *base = (void *)parent->parent; 76 struct nvkm_gpuobj *ectx = (void *)object; 77 u64 limit = ectx->addr + ectx->size - 1; 78 u64 start = ectx->addr; 79 u32 addr; 80 81 switch (nv_engidx(object->engine)) { 82 case NVDEV_ENGINE_SW : return 0; 83 case NVDEV_ENGINE_GR : addr = 0x0000; break; 84 case NVDEV_ENGINE_MPEG : addr = 0x0060; break; 85 default: 86 return -EINVAL; 87 } 88 89 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; 90 nv_wo32(base->eng, addr + 0x00, 0x00190000); 91 nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit)); 92 nv_wo32(base->eng, addr + 0x08, lower_32_bits(start)); 93 nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 | 94 upper_32_bits(start)); 95 nv_wo32(base->eng, addr + 0x10, 0x00000000); 96 nv_wo32(base->eng, addr + 0x14, 0x00000000); 97 bar->flush(bar); 98 return 0; 99 } 100 101 static int 102 nv50_fifo_context_detach(struct nvkm_object *parent, bool suspend, 103 struct nvkm_object *object) 104 { 105 struct nvkm_bar *bar = nvkm_bar(parent); 106 struct nv50_fifo_priv *priv = (void *)parent->engine; 107 struct nv50_fifo_base *base = (void *)parent->parent; 108 struct nv50_fifo_chan *chan = (void *)parent; 109 u32 addr, me; 110 int ret = 0; 111 112 switch (nv_engidx(object->engine)) { 113 case NVDEV_ENGINE_SW : return 0; 114 case NVDEV_ENGINE_GR : addr = 0x0000; break; 115 case NVDEV_ENGINE_MPEG : addr = 0x0060; break; 116 default: 117 return -EINVAL; 118 } 119 120 /* HW bug workaround: 121 * 122 * PFIFO will hang forever if the connected engines don't report 123 * that they've processed the context switch request. 124 * 125 * In order for the kickoff to work, we need to ensure all the 126 * connected engines are in a state where they can answer. 127 * 128 * Newer chipsets don't seem to suffer from this issue, and well, 129 * there's also a "ignore these engines" bitmask reg we can use 130 * if we hit the issue there.. 131 */ 132 me = nv_mask(priv, 0x00b860, 0x00000001, 0x00000001); 133 134 /* do the kickoff... */ 135 nv_wr32(priv, 0x0032fc, nv_gpuobj(base)->addr >> 12); 136 if (!nv_wait_ne(priv, 0x0032fc, 0xffffffff, 0xffffffff)) { 137 nv_error(priv, "channel %d [%s] unload timeout\n", 138 chan->base.chid, nvkm_client_name(chan)); 139 if (suspend) 140 ret = -EBUSY; 141 } 142 nv_wr32(priv, 0x00b860, me); 143 144 if (ret == 0) { 145 nv_wo32(base->eng, addr + 0x00, 0x00000000); 146 nv_wo32(base->eng, addr + 0x04, 0x00000000); 147 nv_wo32(base->eng, addr + 0x08, 0x00000000); 148 nv_wo32(base->eng, addr + 0x0c, 0x00000000); 149 nv_wo32(base->eng, addr + 0x10, 0x00000000); 150 nv_wo32(base->eng, addr + 0x14, 0x00000000); 151 bar->flush(bar); 152 } 153 154 return ret; 155 } 156 157 static int 158 nv50_fifo_object_attach(struct nvkm_object *parent, 159 struct nvkm_object *object, u32 handle) 160 { 161 struct nv50_fifo_chan *chan = (void *)parent; 162 u32 context; 163 164 if (nv_iclass(object, NV_GPUOBJ_CLASS)) 165 context = nv_gpuobj(object)->node->offset >> 4; 166 else 167 context = 0x00000004; /* just non-zero */ 168 169 switch (nv_engidx(object->engine)) { 170 case NVDEV_ENGINE_DMAOBJ: 171 case NVDEV_ENGINE_SW : context |= 0x00000000; break; 172 case NVDEV_ENGINE_GR : context |= 0x00100000; break; 173 case NVDEV_ENGINE_MPEG : context |= 0x00200000; break; 174 default: 175 return -EINVAL; 176 } 177 178 return nvkm_ramht_insert(chan->ramht, 0, handle, context); 179 } 180 181 void 182 nv50_fifo_object_detach(struct nvkm_object *parent, int cookie) 183 { 184 struct nv50_fifo_chan *chan = (void *)parent; 185 nvkm_ramht_remove(chan->ramht, cookie); 186 } 187 188 static int 189 nv50_fifo_chan_ctor_dma(struct nvkm_object *parent, struct nvkm_object *engine, 190 struct nvkm_oclass *oclass, void *data, u32 size, 191 struct nvkm_object **pobject) 192 { 193 union { 194 struct nv03_channel_dma_v0 v0; 195 } *args = data; 196 struct nvkm_bar *bar = nvkm_bar(parent); 197 struct nv50_fifo_base *base = (void *)parent; 198 struct nv50_fifo_chan *chan; 199 int ret; 200 201 nv_ioctl(parent, "create channel dma size %d\n", size); 202 if (nvif_unpack(args->v0, 0, 0, false)) { 203 nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " 204 "offset %016llx\n", args->v0.version, 205 args->v0.pushbuf, args->v0.offset); 206 } else 207 return ret; 208 209 ret = nvkm_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, 210 0x2000, args->v0.pushbuf, 211 (1ULL << NVDEV_ENGINE_DMAOBJ) | 212 (1ULL << NVDEV_ENGINE_SW) | 213 (1ULL << NVDEV_ENGINE_GR) | 214 (1ULL << NVDEV_ENGINE_MPEG), &chan); 215 *pobject = nv_object(chan); 216 if (ret) 217 return ret; 218 219 args->v0.chid = chan->base.chid; 220 221 nv_parent(chan)->context_attach = nv50_fifo_context_attach; 222 nv_parent(chan)->context_detach = nv50_fifo_context_detach; 223 nv_parent(chan)->object_attach = nv50_fifo_object_attach; 224 nv_parent(chan)->object_detach = nv50_fifo_object_detach; 225 226 ret = nvkm_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, 227 &chan->ramht); 228 if (ret) 229 return ret; 230 231 nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset)); 232 nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset)); 233 nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset)); 234 nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset)); 235 nv_wo32(base->ramfc, 0x3c, 0x003f6078); 236 nv_wo32(base->ramfc, 0x44, 0x01003fff); 237 nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); 238 nv_wo32(base->ramfc, 0x4c, 0xffffffff); 239 nv_wo32(base->ramfc, 0x60, 0x7fffffff); 240 nv_wo32(base->ramfc, 0x78, 0x00000000); 241 nv_wo32(base->ramfc, 0x7c, 0x30000001); 242 nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | 243 (4 << 24) /* SEARCH_FULL */ | 244 (chan->ramht->gpuobj.node->offset >> 4)); 245 bar->flush(bar); 246 return 0; 247 } 248 249 static int 250 nv50_fifo_chan_ctor_ind(struct nvkm_object *parent, struct nvkm_object *engine, 251 struct nvkm_oclass *oclass, void *data, u32 size, 252 struct nvkm_object **pobject) 253 { 254 union { 255 struct nv50_channel_gpfifo_v0 v0; 256 } *args = data; 257 struct nvkm_bar *bar = nvkm_bar(parent); 258 struct nv50_fifo_base *base = (void *)parent; 259 struct nv50_fifo_chan *chan; 260 u64 ioffset, ilength; 261 int ret; 262 263 nv_ioctl(parent, "create channel gpfifo size %d\n", size); 264 if (nvif_unpack(args->v0, 0, 0, false)) { 265 nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " 266 "ioffset %016llx ilength %08x\n", 267 args->v0.version, args->v0.pushbuf, args->v0.ioffset, 268 args->v0.ilength); 269 } else 270 return ret; 271 272 ret = nvkm_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, 273 0x2000, args->v0.pushbuf, 274 (1ULL << NVDEV_ENGINE_DMAOBJ) | 275 (1ULL << NVDEV_ENGINE_SW) | 276 (1ULL << NVDEV_ENGINE_GR) | 277 (1ULL << NVDEV_ENGINE_MPEG), &chan); 278 *pobject = nv_object(chan); 279 if (ret) 280 return ret; 281 282 args->v0.chid = chan->base.chid; 283 284 nv_parent(chan)->context_attach = nv50_fifo_context_attach; 285 nv_parent(chan)->context_detach = nv50_fifo_context_detach; 286 nv_parent(chan)->object_attach = nv50_fifo_object_attach; 287 nv_parent(chan)->object_detach = nv50_fifo_object_detach; 288 289 ret = nvkm_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, 290 &chan->ramht); 291 if (ret) 292 return ret; 293 294 ioffset = args->v0.ioffset; 295 ilength = order_base_2(args->v0.ilength / 8); 296 297 nv_wo32(base->ramfc, 0x3c, 0x403f6078); 298 nv_wo32(base->ramfc, 0x44, 0x01003fff); 299 nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); 300 nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset)); 301 nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); 302 nv_wo32(base->ramfc, 0x60, 0x7fffffff); 303 nv_wo32(base->ramfc, 0x78, 0x00000000); 304 nv_wo32(base->ramfc, 0x7c, 0x30000001); 305 nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | 306 (4 << 24) /* SEARCH_FULL */ | 307 (chan->ramht->gpuobj.node->offset >> 4)); 308 bar->flush(bar); 309 return 0; 310 } 311 312 void 313 nv50_fifo_chan_dtor(struct nvkm_object *object) 314 { 315 struct nv50_fifo_chan *chan = (void *)object; 316 nvkm_ramht_ref(NULL, &chan->ramht); 317 nvkm_fifo_channel_destroy(&chan->base); 318 } 319 320 static int 321 nv50_fifo_chan_init(struct nvkm_object *object) 322 { 323 struct nv50_fifo_priv *priv = (void *)object->engine; 324 struct nv50_fifo_base *base = (void *)object->parent; 325 struct nv50_fifo_chan *chan = (void *)object; 326 struct nvkm_gpuobj *ramfc = base->ramfc; 327 u32 chid = chan->base.chid; 328 int ret; 329 330 ret = nvkm_fifo_channel_init(&chan->base); 331 if (ret) 332 return ret; 333 334 nv_wr32(priv, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 12); 335 nv50_fifo_playlist_update(priv); 336 return 0; 337 } 338 339 int 340 nv50_fifo_chan_fini(struct nvkm_object *object, bool suspend) 341 { 342 struct nv50_fifo_priv *priv = (void *)object->engine; 343 struct nv50_fifo_chan *chan = (void *)object; 344 u32 chid = chan->base.chid; 345 346 /* remove channel from playlist, fifo will unload context */ 347 nv_mask(priv, 0x002600 + (chid * 4), 0x80000000, 0x00000000); 348 nv50_fifo_playlist_update(priv); 349 nv_wr32(priv, 0x002600 + (chid * 4), 0x00000000); 350 351 return nvkm_fifo_channel_fini(&chan->base, suspend); 352 } 353 354 static struct nvkm_ofuncs 355 nv50_fifo_ofuncs_dma = { 356 .ctor = nv50_fifo_chan_ctor_dma, 357 .dtor = nv50_fifo_chan_dtor, 358 .init = nv50_fifo_chan_init, 359 .fini = nv50_fifo_chan_fini, 360 .map = _nvkm_fifo_channel_map, 361 .rd32 = _nvkm_fifo_channel_rd32, 362 .wr32 = _nvkm_fifo_channel_wr32, 363 .ntfy = _nvkm_fifo_channel_ntfy 364 }; 365 366 static struct nvkm_ofuncs 367 nv50_fifo_ofuncs_ind = { 368 .ctor = nv50_fifo_chan_ctor_ind, 369 .dtor = nv50_fifo_chan_dtor, 370 .init = nv50_fifo_chan_init, 371 .fini = nv50_fifo_chan_fini, 372 .map = _nvkm_fifo_channel_map, 373 .rd32 = _nvkm_fifo_channel_rd32, 374 .wr32 = _nvkm_fifo_channel_wr32, 375 .ntfy = _nvkm_fifo_channel_ntfy 376 }; 377 378 static struct nvkm_oclass 379 nv50_fifo_sclass[] = { 380 { NV50_CHANNEL_DMA, &nv50_fifo_ofuncs_dma }, 381 { NV50_CHANNEL_GPFIFO, &nv50_fifo_ofuncs_ind }, 382 {} 383 }; 384 385 /******************************************************************************* 386 * FIFO context - basically just the instmem reserved for the channel 387 ******************************************************************************/ 388 389 static int 390 nv50_fifo_context_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 391 struct nvkm_oclass *oclass, void *data, u32 size, 392 struct nvkm_object **pobject) 393 { 394 struct nv50_fifo_base *base; 395 int ret; 396 397 ret = nvkm_fifo_context_create(parent, engine, oclass, NULL, 0x10000, 398 0x1000, NVOBJ_FLAG_HEAP, &base); 399 *pobject = nv_object(base); 400 if (ret) 401 return ret; 402 403 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x0200, 404 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc); 405 if (ret) 406 return ret; 407 408 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x1200, 0, 409 NVOBJ_FLAG_ZERO_ALLOC, &base->eng); 410 if (ret) 411 return ret; 412 413 ret = nvkm_gpuobj_new(nv_object(base), nv_object(base), 0x4000, 0, 0, 414 &base->pgd); 415 if (ret) 416 return ret; 417 418 ret = nvkm_vm_ref(nvkm_client(parent)->vm, &base->vm, base->pgd); 419 if (ret) 420 return ret; 421 422 return 0; 423 } 424 425 void 426 nv50_fifo_context_dtor(struct nvkm_object *object) 427 { 428 struct nv50_fifo_base *base = (void *)object; 429 nvkm_vm_ref(NULL, &base->vm, base->pgd); 430 nvkm_gpuobj_ref(NULL, &base->pgd); 431 nvkm_gpuobj_ref(NULL, &base->eng); 432 nvkm_gpuobj_ref(NULL, &base->ramfc); 433 nvkm_gpuobj_ref(NULL, &base->cache); 434 nvkm_fifo_context_destroy(&base->base); 435 } 436 437 static struct nvkm_oclass 438 nv50_fifo_cclass = { 439 .handle = NV_ENGCTX(FIFO, 0x50), 440 .ofuncs = &(struct nvkm_ofuncs) { 441 .ctor = nv50_fifo_context_ctor, 442 .dtor = nv50_fifo_context_dtor, 443 .init = _nvkm_fifo_context_init, 444 .fini = _nvkm_fifo_context_fini, 445 .rd32 = _nvkm_fifo_context_rd32, 446 .wr32 = _nvkm_fifo_context_wr32, 447 }, 448 }; 449 450 /******************************************************************************* 451 * PFIFO engine 452 ******************************************************************************/ 453 454 static int 455 nv50_fifo_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 456 struct nvkm_oclass *oclass, void *data, u32 size, 457 struct nvkm_object **pobject) 458 { 459 struct nv50_fifo_priv *priv; 460 int ret; 461 462 ret = nvkm_fifo_create(parent, engine, oclass, 1, 127, &priv); 463 *pobject = nv_object(priv); 464 if (ret) 465 return ret; 466 467 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, 468 &priv->playlist[0]); 469 if (ret) 470 return ret; 471 472 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, 473 &priv->playlist[1]); 474 if (ret) 475 return ret; 476 477 nv_subdev(priv)->unit = 0x00000100; 478 nv_subdev(priv)->intr = nv04_fifo_intr; 479 nv_engine(priv)->cclass = &nv50_fifo_cclass; 480 nv_engine(priv)->sclass = nv50_fifo_sclass; 481 priv->base.pause = nv04_fifo_pause; 482 priv->base.start = nv04_fifo_start; 483 return 0; 484 } 485 486 void 487 nv50_fifo_dtor(struct nvkm_object *object) 488 { 489 struct nv50_fifo_priv *priv = (void *)object; 490 491 nvkm_gpuobj_ref(NULL, &priv->playlist[1]); 492 nvkm_gpuobj_ref(NULL, &priv->playlist[0]); 493 494 nvkm_fifo_destroy(&priv->base); 495 } 496 497 int 498 nv50_fifo_init(struct nvkm_object *object) 499 { 500 struct nv50_fifo_priv *priv = (void *)object; 501 int ret, i; 502 503 ret = nvkm_fifo_init(&priv->base); 504 if (ret) 505 return ret; 506 507 nv_mask(priv, 0x000200, 0x00000100, 0x00000000); 508 nv_mask(priv, 0x000200, 0x00000100, 0x00000100); 509 nv_wr32(priv, 0x00250c, 0x6f3cfc34); 510 nv_wr32(priv, 0x002044, 0x01003fff); 511 512 nv_wr32(priv, 0x002100, 0xffffffff); 513 nv_wr32(priv, 0x002140, 0xbfffffff); 514 515 for (i = 0; i < 128; i++) 516 nv_wr32(priv, 0x002600 + (i * 4), 0x00000000); 517 nv50_fifo_playlist_update_locked(priv); 518 519 nv_wr32(priv, 0x003200, 0x00000001); 520 nv_wr32(priv, 0x003250, 0x00000001); 521 nv_wr32(priv, 0x002500, 0x00000001); 522 return 0; 523 } 524 525 struct nvkm_oclass * 526 nv50_fifo_oclass = &(struct nvkm_oclass) { 527 .handle = NV_ENGINE(FIFO, 0x50), 528 .ofuncs = &(struct nvkm_ofuncs) { 529 .ctor = nv50_fifo_ctor, 530 .dtor = nv50_fifo_dtor, 531 .init = nv50_fifo_init, 532 .fini = _nvkm_fifo_fini, 533 }, 534 }; 535