1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include <engine/fifo.h> 25 26 #include <core/client.h> 27 #include <core/engctx.h> 28 #include <core/enum.h> 29 #include <core/handle.h> 30 #include <subdev/bar.h> 31 #include <subdev/fb.h> 32 #include <subdev/mmu.h> 33 #include <subdev/timer.h> 34 35 #include <nvif/class.h> 36 #include <nvif/unpack.h> 37 38 struct gf100_fifo_priv { 39 struct nvkm_fifo base; 40 41 struct work_struct fault; 42 u64 mask; 43 44 struct { 45 struct nvkm_gpuobj *mem[2]; 46 int active; 47 wait_queue_head_t wait; 48 } runlist; 49 50 struct { 51 struct nvkm_gpuobj *mem; 52 struct nvkm_vma bar; 53 } user; 54 int spoon_nr; 55 }; 56 57 struct gf100_fifo_base { 58 struct nvkm_fifo_base base; 59 struct nvkm_gpuobj *pgd; 60 struct nvkm_vm *vm; 61 }; 62 63 struct gf100_fifo_chan { 64 struct nvkm_fifo_chan base; 65 enum { 66 STOPPED, 67 RUNNING, 68 KILLED 69 } state; 70 }; 71 72 /******************************************************************************* 73 * FIFO channel objects 74 ******************************************************************************/ 75 76 static void 77 gf100_fifo_runlist_update(struct gf100_fifo_priv *priv) 78 { 79 struct nvkm_bar *bar = nvkm_bar(priv); 80 struct nvkm_gpuobj *cur; 81 int i, p; 82 83 mutex_lock(&nv_subdev(priv)->mutex); 84 cur = priv->runlist.mem[priv->runlist.active]; 85 priv->runlist.active = !priv->runlist.active; 86 87 for (i = 0, p = 0; i < 128; i++) { 88 struct gf100_fifo_chan *chan = (void *)priv->base.channel[i]; 89 if (chan && chan->state == RUNNING) { 90 nv_wo32(cur, p + 0, i); 91 nv_wo32(cur, p + 4, 0x00000004); 92 p += 8; 93 } 94 } 95 bar->flush(bar); 96 97 nv_wr32(priv, 0x002270, cur->addr >> 12); 98 nv_wr32(priv, 0x002274, 0x01f00000 | (p >> 3)); 99 100 if (wait_event_timeout(priv->runlist.wait, 101 !(nv_rd32(priv, 0x00227c) & 0x00100000), 102 msecs_to_jiffies(2000)) == 0) 103 nv_error(priv, "runlist update timeout\n"); 104 mutex_unlock(&nv_subdev(priv)->mutex); 105 } 106 107 static int 108 gf100_fifo_context_attach(struct nvkm_object *parent, 109 struct nvkm_object *object) 110 { 111 struct nvkm_bar *bar = nvkm_bar(parent); 112 struct gf100_fifo_base *base = (void *)parent->parent; 113 struct nvkm_engctx *ectx = (void *)object; 114 u32 addr; 115 int ret; 116 117 switch (nv_engidx(object->engine)) { 118 case NVDEV_ENGINE_SW : return 0; 119 case NVDEV_ENGINE_GR : addr = 0x0210; break; 120 case NVDEV_ENGINE_CE0 : addr = 0x0230; break; 121 case NVDEV_ENGINE_CE1 : addr = 0x0240; break; 122 case NVDEV_ENGINE_MSVLD : addr = 0x0270; break; 123 case NVDEV_ENGINE_MSPDEC: addr = 0x0250; break; 124 case NVDEV_ENGINE_MSPPP : addr = 0x0260; break; 125 default: 126 return -EINVAL; 127 } 128 129 if (!ectx->vma.node) { 130 ret = nvkm_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, 131 NV_MEM_ACCESS_RW, &ectx->vma); 132 if (ret) 133 return ret; 134 135 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; 136 } 137 138 nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); 139 nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); 140 bar->flush(bar); 141 return 0; 142 } 143 144 static int 145 gf100_fifo_context_detach(struct nvkm_object *parent, bool suspend, 146 struct nvkm_object *object) 147 { 148 struct nvkm_bar *bar = nvkm_bar(parent); 149 struct gf100_fifo_priv *priv = (void *)parent->engine; 150 struct gf100_fifo_base *base = (void *)parent->parent; 151 struct gf100_fifo_chan *chan = (void *)parent; 152 u32 addr; 153 154 switch (nv_engidx(object->engine)) { 155 case NVDEV_ENGINE_SW : return 0; 156 case NVDEV_ENGINE_GR : addr = 0x0210; break; 157 case NVDEV_ENGINE_CE0 : addr = 0x0230; break; 158 case NVDEV_ENGINE_CE1 : addr = 0x0240; break; 159 case NVDEV_ENGINE_MSVLD : addr = 0x0270; break; 160 case NVDEV_ENGINE_MSPDEC: addr = 0x0250; break; 161 case NVDEV_ENGINE_MSPPP : addr = 0x0260; break; 162 default: 163 return -EINVAL; 164 } 165 166 nv_wr32(priv, 0x002634, chan->base.chid); 167 if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { 168 nv_error(priv, "channel %d [%s] kick timeout\n", 169 chan->base.chid, nvkm_client_name(chan)); 170 if (suspend) 171 return -EBUSY; 172 } 173 174 nv_wo32(base, addr + 0x00, 0x00000000); 175 nv_wo32(base, addr + 0x04, 0x00000000); 176 bar->flush(bar); 177 return 0; 178 } 179 180 static int 181 gf100_fifo_chan_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 182 struct nvkm_oclass *oclass, void *data, u32 size, 183 struct nvkm_object **pobject) 184 { 185 union { 186 struct nv50_channel_gpfifo_v0 v0; 187 } *args = data; 188 struct nvkm_bar *bar = nvkm_bar(parent); 189 struct gf100_fifo_priv *priv = (void *)engine; 190 struct gf100_fifo_base *base = (void *)parent; 191 struct gf100_fifo_chan *chan; 192 u64 usermem, ioffset, ilength; 193 int ret, i; 194 195 nv_ioctl(parent, "create channel gpfifo size %d\n", size); 196 if (nvif_unpack(args->v0, 0, 0, false)) { 197 nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " 198 "ioffset %016llx ilength %08x\n", 199 args->v0.version, args->v0.pushbuf, args->v0.ioffset, 200 args->v0.ilength); 201 } else 202 return ret; 203 204 ret = nvkm_fifo_channel_create(parent, engine, oclass, 1, 205 priv->user.bar.offset, 0x1000, 206 args->v0.pushbuf, 207 (1ULL << NVDEV_ENGINE_SW) | 208 (1ULL << NVDEV_ENGINE_GR) | 209 (1ULL << NVDEV_ENGINE_CE0) | 210 (1ULL << NVDEV_ENGINE_CE1) | 211 (1ULL << NVDEV_ENGINE_MSVLD) | 212 (1ULL << NVDEV_ENGINE_MSPDEC) | 213 (1ULL << NVDEV_ENGINE_MSPPP), &chan); 214 *pobject = nv_object(chan); 215 if (ret) 216 return ret; 217 218 args->v0.chid = chan->base.chid; 219 220 nv_parent(chan)->context_attach = gf100_fifo_context_attach; 221 nv_parent(chan)->context_detach = gf100_fifo_context_detach; 222 223 usermem = chan->base.chid * 0x1000; 224 ioffset = args->v0.ioffset; 225 ilength = order_base_2(args->v0.ilength / 8); 226 227 for (i = 0; i < 0x1000; i += 4) 228 nv_wo32(priv->user.mem, usermem + i, 0x00000000); 229 230 nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); 231 nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); 232 nv_wo32(base, 0x10, 0x0000face); 233 nv_wo32(base, 0x30, 0xfffff902); 234 nv_wo32(base, 0x48, lower_32_bits(ioffset)); 235 nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); 236 nv_wo32(base, 0x54, 0x00000002); 237 nv_wo32(base, 0x84, 0x20400000); 238 nv_wo32(base, 0x94, 0x30000001); 239 nv_wo32(base, 0x9c, 0x00000100); 240 nv_wo32(base, 0xa4, 0x1f1f1f1f); 241 nv_wo32(base, 0xa8, 0x1f1f1f1f); 242 nv_wo32(base, 0xac, 0x0000001f); 243 nv_wo32(base, 0xb8, 0xf8000000); 244 nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ 245 nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ 246 bar->flush(bar); 247 return 0; 248 } 249 250 static int 251 gf100_fifo_chan_init(struct nvkm_object *object) 252 { 253 struct nvkm_gpuobj *base = nv_gpuobj(object->parent); 254 struct gf100_fifo_priv *priv = (void *)object->engine; 255 struct gf100_fifo_chan *chan = (void *)object; 256 u32 chid = chan->base.chid; 257 int ret; 258 259 ret = nvkm_fifo_channel_init(&chan->base); 260 if (ret) 261 return ret; 262 263 nv_wr32(priv, 0x003000 + (chid * 8), 0xc0000000 | base->addr >> 12); 264 265 if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { 266 nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001); 267 gf100_fifo_runlist_update(priv); 268 } 269 270 return 0; 271 } 272 273 static void gf100_fifo_intr_engine(struct gf100_fifo_priv *priv); 274 275 static int 276 gf100_fifo_chan_fini(struct nvkm_object *object, bool suspend) 277 { 278 struct gf100_fifo_priv *priv = (void *)object->engine; 279 struct gf100_fifo_chan *chan = (void *)object; 280 u32 chid = chan->base.chid; 281 282 if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { 283 nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000); 284 gf100_fifo_runlist_update(priv); 285 } 286 287 gf100_fifo_intr_engine(priv); 288 289 nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000); 290 return nvkm_fifo_channel_fini(&chan->base, suspend); 291 } 292 293 static struct nvkm_ofuncs 294 gf100_fifo_ofuncs = { 295 .ctor = gf100_fifo_chan_ctor, 296 .dtor = _nvkm_fifo_channel_dtor, 297 .init = gf100_fifo_chan_init, 298 .fini = gf100_fifo_chan_fini, 299 .map = _nvkm_fifo_channel_map, 300 .rd32 = _nvkm_fifo_channel_rd32, 301 .wr32 = _nvkm_fifo_channel_wr32, 302 .ntfy = _nvkm_fifo_channel_ntfy 303 }; 304 305 static struct nvkm_oclass 306 gf100_fifo_sclass[] = { 307 { FERMI_CHANNEL_GPFIFO, &gf100_fifo_ofuncs }, 308 {} 309 }; 310 311 /******************************************************************************* 312 * FIFO context - instmem heap and vm setup 313 ******************************************************************************/ 314 315 static int 316 gf100_fifo_context_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 317 struct nvkm_oclass *oclass, void *data, u32 size, 318 struct nvkm_object **pobject) 319 { 320 struct gf100_fifo_base *base; 321 int ret; 322 323 ret = nvkm_fifo_context_create(parent, engine, oclass, NULL, 0x1000, 324 0x1000, NVOBJ_FLAG_ZERO_ALLOC | 325 NVOBJ_FLAG_HEAP, &base); 326 *pobject = nv_object(base); 327 if (ret) 328 return ret; 329 330 ret = nvkm_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, 331 &base->pgd); 332 if (ret) 333 return ret; 334 335 nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); 336 nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); 337 nv_wo32(base, 0x0208, 0xffffffff); 338 nv_wo32(base, 0x020c, 0x000000ff); 339 340 ret = nvkm_vm_ref(nvkm_client(parent)->vm, &base->vm, base->pgd); 341 if (ret) 342 return ret; 343 344 return 0; 345 } 346 347 static void 348 gf100_fifo_context_dtor(struct nvkm_object *object) 349 { 350 struct gf100_fifo_base *base = (void *)object; 351 nvkm_vm_ref(NULL, &base->vm, base->pgd); 352 nvkm_gpuobj_ref(NULL, &base->pgd); 353 nvkm_fifo_context_destroy(&base->base); 354 } 355 356 static struct nvkm_oclass 357 gf100_fifo_cclass = { 358 .handle = NV_ENGCTX(FIFO, 0xc0), 359 .ofuncs = &(struct nvkm_ofuncs) { 360 .ctor = gf100_fifo_context_ctor, 361 .dtor = gf100_fifo_context_dtor, 362 .init = _nvkm_fifo_context_init, 363 .fini = _nvkm_fifo_context_fini, 364 .rd32 = _nvkm_fifo_context_rd32, 365 .wr32 = _nvkm_fifo_context_wr32, 366 }, 367 }; 368 369 /******************************************************************************* 370 * PFIFO engine 371 ******************************************************************************/ 372 373 static inline int 374 gf100_fifo_engidx(struct gf100_fifo_priv *priv, u32 engn) 375 { 376 switch (engn) { 377 case NVDEV_ENGINE_GR : engn = 0; break; 378 case NVDEV_ENGINE_MSVLD : engn = 1; break; 379 case NVDEV_ENGINE_MSPPP : engn = 2; break; 380 case NVDEV_ENGINE_MSPDEC: engn = 3; break; 381 case NVDEV_ENGINE_CE0 : engn = 4; break; 382 case NVDEV_ENGINE_CE1 : engn = 5; break; 383 default: 384 return -1; 385 } 386 387 return engn; 388 } 389 390 static inline struct nvkm_engine * 391 gf100_fifo_engine(struct gf100_fifo_priv *priv, u32 engn) 392 { 393 switch (engn) { 394 case 0: engn = NVDEV_ENGINE_GR; break; 395 case 1: engn = NVDEV_ENGINE_MSVLD; break; 396 case 2: engn = NVDEV_ENGINE_MSPPP; break; 397 case 3: engn = NVDEV_ENGINE_MSPDEC; break; 398 case 4: engn = NVDEV_ENGINE_CE0; break; 399 case 5: engn = NVDEV_ENGINE_CE1; break; 400 default: 401 return NULL; 402 } 403 404 return nvkm_engine(priv, engn); 405 } 406 407 static void 408 gf100_fifo_recover_work(struct work_struct *work) 409 { 410 struct gf100_fifo_priv *priv = container_of(work, typeof(*priv), fault); 411 struct nvkm_object *engine; 412 unsigned long flags; 413 u32 engn, engm = 0; 414 u64 mask, todo; 415 416 spin_lock_irqsave(&priv->base.lock, flags); 417 mask = priv->mask; 418 priv->mask = 0ULL; 419 spin_unlock_irqrestore(&priv->base.lock, flags); 420 421 for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) 422 engm |= 1 << gf100_fifo_engidx(priv, engn); 423 nv_mask(priv, 0x002630, engm, engm); 424 425 for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { 426 if ((engine = (void *)nvkm_engine(priv, engn))) { 427 nv_ofuncs(engine)->fini(engine, false); 428 WARN_ON(nv_ofuncs(engine)->init(engine)); 429 } 430 } 431 432 gf100_fifo_runlist_update(priv); 433 nv_wr32(priv, 0x00262c, engm); 434 nv_mask(priv, 0x002630, engm, 0x00000000); 435 } 436 437 static void 438 gf100_fifo_recover(struct gf100_fifo_priv *priv, struct nvkm_engine *engine, 439 struct gf100_fifo_chan *chan) 440 { 441 u32 chid = chan->base.chid; 442 unsigned long flags; 443 444 nv_error(priv, "%s engine fault on channel %d, recovering...\n", 445 nv_subdev(engine)->name, chid); 446 447 nv_mask(priv, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000); 448 chan->state = KILLED; 449 450 spin_lock_irqsave(&priv->base.lock, flags); 451 priv->mask |= 1ULL << nv_engidx(engine); 452 spin_unlock_irqrestore(&priv->base.lock, flags); 453 schedule_work(&priv->fault); 454 } 455 456 static int 457 gf100_fifo_swmthd(struct gf100_fifo_priv *priv, u32 chid, u32 mthd, u32 data) 458 { 459 struct gf100_fifo_chan *chan = NULL; 460 struct nvkm_handle *bind; 461 unsigned long flags; 462 int ret = -EINVAL; 463 464 spin_lock_irqsave(&priv->base.lock, flags); 465 if (likely(chid >= priv->base.min && chid <= priv->base.max)) 466 chan = (void *)priv->base.channel[chid]; 467 if (unlikely(!chan)) 468 goto out; 469 470 bind = nvkm_namedb_get_class(nv_namedb(chan), 0x906e); 471 if (likely(bind)) { 472 if (!mthd || !nv_call(bind->object, mthd, data)) 473 ret = 0; 474 nvkm_namedb_put(bind); 475 } 476 477 out: 478 spin_unlock_irqrestore(&priv->base.lock, flags); 479 return ret; 480 } 481 482 static const struct nvkm_enum 483 gf100_fifo_sched_reason[] = { 484 { 0x0a, "CTXSW_TIMEOUT" }, 485 {} 486 }; 487 488 static void 489 gf100_fifo_intr_sched_ctxsw(struct gf100_fifo_priv *priv) 490 { 491 struct nvkm_engine *engine; 492 struct gf100_fifo_chan *chan; 493 u32 engn; 494 495 for (engn = 0; engn < 6; engn++) { 496 u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); 497 u32 busy = (stat & 0x80000000); 498 u32 save = (stat & 0x00100000); /* maybe? */ 499 u32 unk0 = (stat & 0x00040000); 500 u32 unk1 = (stat & 0x00001000); 501 u32 chid = (stat & 0x0000007f); 502 (void)save; 503 504 if (busy && unk0 && unk1) { 505 if (!(chan = (void *)priv->base.channel[chid])) 506 continue; 507 if (!(engine = gf100_fifo_engine(priv, engn))) 508 continue; 509 gf100_fifo_recover(priv, engine, chan); 510 } 511 } 512 } 513 514 static void 515 gf100_fifo_intr_sched(struct gf100_fifo_priv *priv) 516 { 517 u32 intr = nv_rd32(priv, 0x00254c); 518 u32 code = intr & 0x000000ff; 519 const struct nvkm_enum *en; 520 char enunk[6] = ""; 521 522 en = nvkm_enum_find(gf100_fifo_sched_reason, code); 523 if (!en) 524 snprintf(enunk, sizeof(enunk), "UNK%02x", code); 525 526 nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); 527 528 switch (code) { 529 case 0x0a: 530 gf100_fifo_intr_sched_ctxsw(priv); 531 break; 532 default: 533 break; 534 } 535 } 536 537 static const struct nvkm_enum 538 gf100_fifo_fault_engine[] = { 539 { 0x00, "PGRAPH", NULL, NVDEV_ENGINE_GR }, 540 { 0x03, "PEEPHOLE", NULL, NVDEV_ENGINE_IFB }, 541 { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, 542 { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, 543 { 0x07, "PFIFO", NULL, NVDEV_ENGINE_FIFO }, 544 { 0x10, "PMSVLD", NULL, NVDEV_ENGINE_MSVLD }, 545 { 0x11, "PMSPPP", NULL, NVDEV_ENGINE_MSPPP }, 546 { 0x13, "PCOUNTER" }, 547 { 0x14, "PMSPDEC", NULL, NVDEV_ENGINE_MSPDEC }, 548 { 0x15, "PCE0", NULL, NVDEV_ENGINE_CE0 }, 549 { 0x16, "PCE1", NULL, NVDEV_ENGINE_CE1 }, 550 { 0x17, "PDAEMON" }, 551 {} 552 }; 553 554 static const struct nvkm_enum 555 gf100_fifo_fault_reason[] = { 556 { 0x00, "PT_NOT_PRESENT" }, 557 { 0x01, "PT_TOO_SHORT" }, 558 { 0x02, "PAGE_NOT_PRESENT" }, 559 { 0x03, "VM_LIMIT_EXCEEDED" }, 560 { 0x04, "NO_CHANNEL" }, 561 { 0x05, "PAGE_SYSTEM_ONLY" }, 562 { 0x06, "PAGE_READ_ONLY" }, 563 { 0x0a, "COMPRESSED_SYSRAM" }, 564 { 0x0c, "INVALID_STORAGE_TYPE" }, 565 {} 566 }; 567 568 static const struct nvkm_enum 569 gf100_fifo_fault_hubclient[] = { 570 { 0x01, "PCOPY0" }, 571 { 0x02, "PCOPY1" }, 572 { 0x04, "DISPATCH" }, 573 { 0x05, "CTXCTL" }, 574 { 0x06, "PFIFO" }, 575 { 0x07, "BAR_READ" }, 576 { 0x08, "BAR_WRITE" }, 577 { 0x0b, "PVP" }, 578 { 0x0c, "PMSPPP" }, 579 { 0x0d, "PMSVLD" }, 580 { 0x11, "PCOUNTER" }, 581 { 0x12, "PDAEMON" }, 582 { 0x14, "CCACHE" }, 583 { 0x15, "CCACHE_POST" }, 584 {} 585 }; 586 587 static const struct nvkm_enum 588 gf100_fifo_fault_gpcclient[] = { 589 { 0x01, "TEX" }, 590 { 0x0c, "ESETUP" }, 591 { 0x0e, "CTXCTL" }, 592 { 0x0f, "PROP" }, 593 {} 594 }; 595 596 static void 597 gf100_fifo_intr_fault(struct gf100_fifo_priv *priv, int unit) 598 { 599 u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); 600 u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); 601 u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); 602 u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); 603 u32 gpc = (stat & 0x1f000000) >> 24; 604 u32 client = (stat & 0x00001f00) >> 8; 605 u32 write = (stat & 0x00000080); 606 u32 hub = (stat & 0x00000040); 607 u32 reason = (stat & 0x0000000f); 608 struct nvkm_object *engctx = NULL, *object; 609 struct nvkm_engine *engine = NULL; 610 const struct nvkm_enum *er, *eu, *ec; 611 char erunk[6] = ""; 612 char euunk[6] = ""; 613 char ecunk[6] = ""; 614 char gpcid[3] = ""; 615 616 er = nvkm_enum_find(gf100_fifo_fault_reason, reason); 617 if (!er) 618 snprintf(erunk, sizeof(erunk), "UNK%02X", reason); 619 620 eu = nvkm_enum_find(gf100_fifo_fault_engine, unit); 621 if (eu) { 622 switch (eu->data2) { 623 case NVDEV_SUBDEV_BAR: 624 nv_mask(priv, 0x001704, 0x00000000, 0x00000000); 625 break; 626 case NVDEV_SUBDEV_INSTMEM: 627 nv_mask(priv, 0x001714, 0x00000000, 0x00000000); 628 break; 629 case NVDEV_ENGINE_IFB: 630 nv_mask(priv, 0x001718, 0x00000000, 0x00000000); 631 break; 632 default: 633 engine = nvkm_engine(priv, eu->data2); 634 if (engine) 635 engctx = nvkm_engctx_get(engine, inst); 636 break; 637 } 638 } else { 639 snprintf(euunk, sizeof(euunk), "UNK%02x", unit); 640 } 641 642 if (hub) { 643 ec = nvkm_enum_find(gf100_fifo_fault_hubclient, client); 644 } else { 645 ec = nvkm_enum_find(gf100_fifo_fault_gpcclient, client); 646 snprintf(gpcid, sizeof(gpcid), "%d", gpc); 647 } 648 649 if (!ec) 650 snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); 651 652 nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " 653 "channel 0x%010llx [%s]\n", write ? "write" : "read", 654 (u64)vahi << 32 | valo, er ? er->name : erunk, 655 eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", 656 ec ? ec->name : ecunk, (u64)inst << 12, 657 nvkm_client_name(engctx)); 658 659 object = engctx; 660 while (object) { 661 switch (nv_mclass(object)) { 662 case FERMI_CHANNEL_GPFIFO: 663 gf100_fifo_recover(priv, engine, (void *)object); 664 break; 665 } 666 object = object->parent; 667 } 668 669 nvkm_engctx_put(engctx); 670 } 671 672 static const struct nvkm_bitfield 673 gf100_fifo_pbdma_intr[] = { 674 /* { 0x00008000, "" } seen with null ib push */ 675 { 0x00200000, "ILLEGAL_MTHD" }, 676 { 0x00800000, "EMPTY_SUBC" }, 677 {} 678 }; 679 680 static void 681 gf100_fifo_intr_pbdma(struct gf100_fifo_priv *priv, int unit) 682 { 683 u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)); 684 u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); 685 u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); 686 u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0x7f; 687 u32 subc = (addr & 0x00070000) >> 16; 688 u32 mthd = (addr & 0x00003ffc); 689 u32 show = stat; 690 691 if (stat & 0x00800000) { 692 if (!gf100_fifo_swmthd(priv, chid, mthd, data)) 693 show &= ~0x00800000; 694 } 695 696 if (show) { 697 nv_error(priv, "PBDMA%d:", unit); 698 nvkm_bitfield_print(gf100_fifo_pbdma_intr, show); 699 pr_cont("\n"); 700 nv_error(priv, 701 "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", 702 unit, chid, 703 nvkm_client_name_for_fifo_chid(&priv->base, chid), 704 subc, mthd, data); 705 } 706 707 nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); 708 nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); 709 } 710 711 static void 712 gf100_fifo_intr_runlist(struct gf100_fifo_priv *priv) 713 { 714 u32 intr = nv_rd32(priv, 0x002a00); 715 716 if (intr & 0x10000000) { 717 wake_up(&priv->runlist.wait); 718 nv_wr32(priv, 0x002a00, 0x10000000); 719 intr &= ~0x10000000; 720 } 721 722 if (intr) { 723 nv_error(priv, "RUNLIST 0x%08x\n", intr); 724 nv_wr32(priv, 0x002a00, intr); 725 } 726 } 727 728 static void 729 gf100_fifo_intr_engine_unit(struct gf100_fifo_priv *priv, int engn) 730 { 731 u32 intr = nv_rd32(priv, 0x0025a8 + (engn * 0x04)); 732 u32 inte = nv_rd32(priv, 0x002628); 733 u32 unkn; 734 735 nv_wr32(priv, 0x0025a8 + (engn * 0x04), intr); 736 737 for (unkn = 0; unkn < 8; unkn++) { 738 u32 ints = (intr >> (unkn * 0x04)) & inte; 739 if (ints & 0x1) { 740 nvkm_fifo_uevent(&priv->base); 741 ints &= ~1; 742 } 743 if (ints) { 744 nv_error(priv, "ENGINE %d %d %01x", engn, unkn, ints); 745 nv_mask(priv, 0x002628, ints, 0); 746 } 747 } 748 } 749 750 static void 751 gf100_fifo_intr_engine(struct gf100_fifo_priv *priv) 752 { 753 u32 mask = nv_rd32(priv, 0x0025a4); 754 while (mask) { 755 u32 unit = __ffs(mask); 756 gf100_fifo_intr_engine_unit(priv, unit); 757 mask &= ~(1 << unit); 758 } 759 } 760 761 static void 762 gf100_fifo_intr(struct nvkm_subdev *subdev) 763 { 764 struct gf100_fifo_priv *priv = (void *)subdev; 765 u32 mask = nv_rd32(priv, 0x002140); 766 u32 stat = nv_rd32(priv, 0x002100) & mask; 767 768 if (stat & 0x00000001) { 769 u32 intr = nv_rd32(priv, 0x00252c); 770 nv_warn(priv, "INTR 0x00000001: 0x%08x\n", intr); 771 nv_wr32(priv, 0x002100, 0x00000001); 772 stat &= ~0x00000001; 773 } 774 775 if (stat & 0x00000100) { 776 gf100_fifo_intr_sched(priv); 777 nv_wr32(priv, 0x002100, 0x00000100); 778 stat &= ~0x00000100; 779 } 780 781 if (stat & 0x00010000) { 782 u32 intr = nv_rd32(priv, 0x00256c); 783 nv_warn(priv, "INTR 0x00010000: 0x%08x\n", intr); 784 nv_wr32(priv, 0x002100, 0x00010000); 785 stat &= ~0x00010000; 786 } 787 788 if (stat & 0x01000000) { 789 u32 intr = nv_rd32(priv, 0x00258c); 790 nv_warn(priv, "INTR 0x01000000: 0x%08x\n", intr); 791 nv_wr32(priv, 0x002100, 0x01000000); 792 stat &= ~0x01000000; 793 } 794 795 if (stat & 0x10000000) { 796 u32 mask = nv_rd32(priv, 0x00259c); 797 while (mask) { 798 u32 unit = __ffs(mask); 799 gf100_fifo_intr_fault(priv, unit); 800 nv_wr32(priv, 0x00259c, (1 << unit)); 801 mask &= ~(1 << unit); 802 } 803 stat &= ~0x10000000; 804 } 805 806 if (stat & 0x20000000) { 807 u32 mask = nv_rd32(priv, 0x0025a0); 808 while (mask) { 809 u32 unit = __ffs(mask); 810 gf100_fifo_intr_pbdma(priv, unit); 811 nv_wr32(priv, 0x0025a0, (1 << unit)); 812 mask &= ~(1 << unit); 813 } 814 stat &= ~0x20000000; 815 } 816 817 if (stat & 0x40000000) { 818 gf100_fifo_intr_runlist(priv); 819 stat &= ~0x40000000; 820 } 821 822 if (stat & 0x80000000) { 823 gf100_fifo_intr_engine(priv); 824 stat &= ~0x80000000; 825 } 826 827 if (stat) { 828 nv_error(priv, "INTR 0x%08x\n", stat); 829 nv_mask(priv, 0x002140, stat, 0x00000000); 830 nv_wr32(priv, 0x002100, stat); 831 } 832 } 833 834 static void 835 gf100_fifo_uevent_init(struct nvkm_event *event, int type, int index) 836 { 837 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent); 838 nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); 839 } 840 841 static void 842 gf100_fifo_uevent_fini(struct nvkm_event *event, int type, int index) 843 { 844 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent); 845 nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); 846 } 847 848 static const struct nvkm_event_func 849 gf100_fifo_uevent_func = { 850 .ctor = nvkm_fifo_uevent_ctor, 851 .init = gf100_fifo_uevent_init, 852 .fini = gf100_fifo_uevent_fini, 853 }; 854 855 static int 856 gf100_fifo_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 857 struct nvkm_oclass *oclass, void *data, u32 size, 858 struct nvkm_object **pobject) 859 { 860 struct gf100_fifo_priv *priv; 861 int ret; 862 863 ret = nvkm_fifo_create(parent, engine, oclass, 0, 127, &priv); 864 *pobject = nv_object(priv); 865 if (ret) 866 return ret; 867 868 INIT_WORK(&priv->fault, gf100_fifo_recover_work); 869 870 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, 871 &priv->runlist.mem[0]); 872 if (ret) 873 return ret; 874 875 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, 876 &priv->runlist.mem[1]); 877 if (ret) 878 return ret; 879 880 init_waitqueue_head(&priv->runlist.wait); 881 882 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 128 * 0x1000, 0x1000, 0, 883 &priv->user.mem); 884 if (ret) 885 return ret; 886 887 ret = nvkm_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, 888 &priv->user.bar); 889 if (ret) 890 return ret; 891 892 ret = nvkm_event_init(&gf100_fifo_uevent_func, 1, 1, &priv->base.uevent); 893 if (ret) 894 return ret; 895 896 nv_subdev(priv)->unit = 0x00000100; 897 nv_subdev(priv)->intr = gf100_fifo_intr; 898 nv_engine(priv)->cclass = &gf100_fifo_cclass; 899 nv_engine(priv)->sclass = gf100_fifo_sclass; 900 return 0; 901 } 902 903 static void 904 gf100_fifo_dtor(struct nvkm_object *object) 905 { 906 struct gf100_fifo_priv *priv = (void *)object; 907 908 nvkm_gpuobj_unmap(&priv->user.bar); 909 nvkm_gpuobj_ref(NULL, &priv->user.mem); 910 nvkm_gpuobj_ref(NULL, &priv->runlist.mem[0]); 911 nvkm_gpuobj_ref(NULL, &priv->runlist.mem[1]); 912 913 nvkm_fifo_destroy(&priv->base); 914 } 915 916 static int 917 gf100_fifo_init(struct nvkm_object *object) 918 { 919 struct gf100_fifo_priv *priv = (void *)object; 920 int ret, i; 921 922 ret = nvkm_fifo_init(&priv->base); 923 if (ret) 924 return ret; 925 926 nv_wr32(priv, 0x000204, 0xffffffff); 927 nv_wr32(priv, 0x002204, 0xffffffff); 928 929 priv->spoon_nr = hweight32(nv_rd32(priv, 0x002204)); 930 nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); 931 932 /* assign engines to PBDMAs */ 933 if (priv->spoon_nr >= 3) { 934 nv_wr32(priv, 0x002208, ~(1 << 0)); /* PGRAPH */ 935 nv_wr32(priv, 0x00220c, ~(1 << 1)); /* PVP */ 936 nv_wr32(priv, 0x002210, ~(1 << 1)); /* PMSPP */ 937 nv_wr32(priv, 0x002214, ~(1 << 1)); /* PMSVLD */ 938 nv_wr32(priv, 0x002218, ~(1 << 2)); /* PCE0 */ 939 nv_wr32(priv, 0x00221c, ~(1 << 1)); /* PCE1 */ 940 } 941 942 /* PBDMA[n] */ 943 for (i = 0; i < priv->spoon_nr; i++) { 944 nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); 945 nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ 946 nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ 947 } 948 949 nv_mask(priv, 0x002200, 0x00000001, 0x00000001); 950 nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); 951 952 nv_wr32(priv, 0x002100, 0xffffffff); 953 nv_wr32(priv, 0x002140, 0x7fffffff); 954 nv_wr32(priv, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ 955 return 0; 956 } 957 958 struct nvkm_oclass * 959 gf100_fifo_oclass = &(struct nvkm_oclass) { 960 .handle = NV_ENGINE(FIFO, 0xc0), 961 .ofuncs = &(struct nvkm_ofuncs) { 962 .ctor = gf100_fifo_ctor, 963 .dtor = gf100_fifo_dtor, 964 .init = gf100_fifo_init, 965 .fini = _nvkm_fifo_fini, 966 }, 967 }; 968