1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "cgrp.h" 25 #include "chan.h" 26 #include "chid.h" 27 #include "runl.h" 28 29 #include "nv04.h" 30 #include "channv04.h" 31 #include "regsnv04.h" 32 33 #include <core/ramht.h> 34 #include <subdev/instmem.h> 35 #include <subdev/mc.h> 36 #include <subdev/timer.h> 37 #include <engine/sw.h> 38 39 #include <nvif/class.h> 40 41 static const struct nv04_fifo_ramfc 42 nv04_fifo_ramfc[] = { 43 { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 44 { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 45 { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 46 { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 47 { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 48 { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 49 { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 50 { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 51 {} 52 }; 53 54 void 55 nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) 56 { 57 struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 58 struct nv04_fifo *fifo = chan->fifo; 59 struct nvkm_device *device = fifo->base.engine.subdev.device; 60 struct nvkm_memory *fctx = device->imem->ramfc; 61 const struct nv04_fifo_ramfc *c; 62 unsigned long flags; 63 u32 data = chan->ramfc; 64 u32 chid; 65 66 /* prevent fifo context switches */ 67 spin_lock_irqsave(&fifo->base.lock, flags); 68 nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 69 70 /* if this channel is active, replace it with a null context */ 71 chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; 72 if (chid == chan->base.chid) { 73 nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); 74 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); 75 nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); 76 77 c = fifo->ramfc; 78 nvkm_kmap(fctx); 79 do { 80 u32 rm = ((1ULL << c->bits) - 1) << c->regs; 81 u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; 82 u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; 83 u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); 84 nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); 85 } while ((++c)->bits); 86 nvkm_done(fctx); 87 88 c = fifo->ramfc; 89 do { 90 nvkm_wr32(device, c->regp, 0x00000000); 91 } while ((++c)->bits); 92 93 nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); 94 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); 95 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.chid->mask); 96 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 97 nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 98 } 99 100 /* restore normal operation, after disabling dma mode */ 101 nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); 102 nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 103 spin_unlock_irqrestore(&fifo->base.lock, flags); 104 } 105 106 void 107 nv04_fifo_dma_init(struct nvkm_fifo_chan *base) 108 { 109 struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 110 struct nv04_fifo *fifo = chan->fifo; 111 struct nvkm_device *device = fifo->base.engine.subdev.device; 112 u32 mask = 1 << chan->base.chid; 113 unsigned long flags; 114 spin_lock_irqsave(&fifo->base.lock, flags); 115 nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); 116 spin_unlock_irqrestore(&fifo->base.lock, flags); 117 } 118 119 static const struct nvkm_chan_func 120 nv04_chan = { 121 }; 122 123 const struct nvkm_cgrp_func 124 nv04_cgrp = { 125 }; 126 127 const struct nvkm_engn_func 128 nv04_engn = { 129 }; 130 131 void 132 nv04_fifo_pause(struct nvkm_fifo *fifo, unsigned long *pflags) 133 __acquires(fifo->lock) 134 { 135 struct nvkm_device *device = fifo->engine.subdev.device; 136 unsigned long flags; 137 138 spin_lock_irqsave(&fifo->lock, flags); 139 *pflags = flags; 140 141 nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 142 nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 143 144 /* in some cases the puller may be left in an inconsistent state 145 * if you try to stop it while it's busy translating handles. 146 * sometimes you get a CACHE_ERROR, sometimes it just fails 147 * silently; sending incorrect instance offsets to PGRAPH after 148 * it's started up again. 149 * 150 * to avoid this, we invalidate the most recently calculated 151 * instance. 152 */ 153 nvkm_msec(device, 2000, 154 u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0); 155 if (!(tmp & NV04_PFIFO_CACHE1_PULL0_HASH_BUSY)) 156 break; 157 ); 158 159 if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 160 NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 161 nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 162 163 nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 164 } 165 166 void 167 nv04_fifo_start(struct nvkm_fifo *fifo, unsigned long *pflags) 168 __releases(fifo->lock) 169 { 170 struct nvkm_device *device = fifo->engine.subdev.device; 171 unsigned long flags = *pflags; 172 173 nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 174 nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 175 176 spin_unlock_irqrestore(&fifo->lock, flags); 177 } 178 179 const struct nvkm_runl_func 180 nv04_runl = { 181 }; 182 183 int 184 nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) 185 { 186 switch (engine->subdev.type) { 187 case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; 188 case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; 189 case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; 190 case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; 191 default: 192 WARN_ON(1); 193 return 0; 194 } 195 } 196 197 static const char * 198 nv_dma_state_err(u32 state) 199 { 200 static const char * const desc[] = { 201 "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 202 "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 203 }; 204 return desc[(state >> 29) & 0x7]; 205 } 206 207 static bool 208 nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) 209 { 210 struct nvkm_sw *sw = device->sw; 211 const int subc = (addr & 0x0000e000) >> 13; 212 const int mthd = (addr & 0x00001ffc); 213 const u32 mask = 0x0000000f << (subc * 4); 214 u32 engine = nvkm_rd32(device, 0x003280); 215 bool handled = false; 216 217 switch (mthd) { 218 case 0x0000 ... 0x0000: /* subchannel's engine -> software */ 219 nvkm_wr32(device, 0x003280, (engine &= ~mask)); 220 fallthrough; 221 case 0x0180 ... 0x01fc: /* handle -> instance */ 222 data = nvkm_rd32(device, 0x003258) & 0x0000ffff; 223 fallthrough; 224 case 0x0100 ... 0x017c: 225 case 0x0200 ... 0x1ffc: /* pass method down to sw */ 226 if (!(engine & mask) && sw) 227 handled = nvkm_sw_mthd(sw, chid, subc, mthd, data); 228 break; 229 default: 230 break; 231 } 232 233 return handled; 234 } 235 236 static void 237 nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) 238 { 239 struct nvkm_subdev *subdev = &fifo->engine.subdev; 240 struct nvkm_device *device = subdev->device; 241 struct nvkm_chan *chan; 242 unsigned long flags; 243 u32 pull0 = nvkm_rd32(device, 0x003250); 244 u32 mthd, data; 245 int ptr; 246 247 /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 248 * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 249 * show that it wraps around to the start at GET=0x800.. No clue as to 250 * why.. 251 */ 252 ptr = (get & 0x7ff) >> 2; 253 254 if (device->card_type < NV_40) { 255 mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 256 data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 257 } else { 258 mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 259 data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 260 } 261 262 if (!(pull0 & 0x00000100) || 263 !nv04_fifo_swmthd(device, chid, mthd, data)) { 264 chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); 265 nvkm_error(subdev, "CACHE_ERROR - " 266 "ch %d [%s] subc %d mthd %04x data %08x\n", 267 chid, chan ? chan->name : "unknown", 268 (mthd >> 13) & 7, mthd & 0x1ffc, data); 269 nvkm_chan_put(&chan, flags); 270 } 271 272 nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 273 nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 274 275 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 276 nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 277 nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 278 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 279 nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 280 nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 281 282 nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 283 nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 284 nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 285 } 286 287 static void 288 nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) 289 { 290 struct nvkm_subdev *subdev = &fifo->engine.subdev; 291 struct nvkm_device *device = subdev->device; 292 u32 dma_get = nvkm_rd32(device, 0x003244); 293 u32 dma_put = nvkm_rd32(device, 0x003240); 294 u32 push = nvkm_rd32(device, 0x003220); 295 u32 state = nvkm_rd32(device, 0x003228); 296 struct nvkm_chan *chan; 297 unsigned long flags; 298 const char *name; 299 300 chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); 301 name = chan ? chan->name : "unknown"; 302 if (device->card_type == NV_50) { 303 u32 ho_get = nvkm_rd32(device, 0x003328); 304 u32 ho_put = nvkm_rd32(device, 0x003320); 305 u32 ib_get = nvkm_rd32(device, 0x003334); 306 u32 ib_put = nvkm_rd32(device, 0x003330); 307 308 nvkm_error(subdev, "DMA_PUSHER - " 309 "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x " 310 "ib_put %08x state %08x (err: %s) push %08x\n", 311 chid, name, ho_get, dma_get, ho_put, dma_put, 312 ib_get, ib_put, state, nv_dma_state_err(state), 313 push); 314 315 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 316 nvkm_wr32(device, 0x003364, 0x00000000); 317 if (dma_get != dma_put || ho_get != ho_put) { 318 nvkm_wr32(device, 0x003244, dma_put); 319 nvkm_wr32(device, 0x003328, ho_put); 320 } else 321 if (ib_get != ib_put) 322 nvkm_wr32(device, 0x003334, ib_put); 323 } else { 324 nvkm_error(subdev, "DMA_PUSHER - ch %d [%s] get %08x put %08x " 325 "state %08x (err: %s) push %08x\n", 326 chid, name, dma_get, dma_put, state, 327 nv_dma_state_err(state), push); 328 329 if (dma_get != dma_put) 330 nvkm_wr32(device, 0x003244, dma_put); 331 } 332 nvkm_chan_put(&chan, flags); 333 334 nvkm_wr32(device, 0x003228, 0x00000000); 335 nvkm_wr32(device, 0x003220, 0x00000001); 336 nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 337 } 338 339 irqreturn_t 340 nv04_fifo_intr(struct nvkm_inth *inth) 341 { 342 struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); 343 struct nvkm_subdev *subdev = &fifo->engine.subdev; 344 struct nvkm_device *device = subdev->device; 345 u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 346 u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 347 u32 reassign, chid, get, sem; 348 349 reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 350 nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 351 352 chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; 353 get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 354 355 if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 356 nv04_fifo_intr_cache_error(fifo, chid, get); 357 stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 358 } 359 360 if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 361 nv04_fifo_intr_dma_pusher(fifo, chid); 362 stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 363 } 364 365 if (stat & NV_PFIFO_INTR_SEMAPHORE) { 366 stat &= ~NV_PFIFO_INTR_SEMAPHORE; 367 nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 368 369 sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 370 nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 371 372 nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 373 nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 374 } 375 376 if (device->card_type == NV_50) { 377 if (stat & 0x00000010) { 378 stat &= ~0x00000010; 379 nvkm_wr32(device, 0x002100, 0x00000010); 380 } 381 382 if (stat & 0x40000000) { 383 nvkm_wr32(device, 0x002100, 0x40000000); 384 nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); 385 stat &= ~0x40000000; 386 } 387 } 388 389 if (stat) { 390 nvkm_warn(subdev, "intr %08x\n", stat); 391 nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 392 nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 393 } 394 395 nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 396 return IRQ_HANDLED; 397 } 398 399 void 400 nv04_fifo_init(struct nvkm_fifo *fifo) 401 { 402 struct nvkm_device *device = fifo->engine.subdev.device; 403 struct nvkm_instmem *imem = device->imem; 404 struct nvkm_ramht *ramht = imem->ramht; 405 struct nvkm_memory *ramro = imem->ramro; 406 struct nvkm_memory *ramfc = imem->ramfc; 407 408 nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 409 nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 410 411 nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 412 ((ramht->bits - 9) << 16) | 413 (ramht->gpuobj->addr >> 8)); 414 nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); 415 nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); 416 417 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); 418 419 nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 420 nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 421 422 nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 423 nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 424 nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 425 } 426 427 int 428 nv04_fifo_runl_ctor(struct nvkm_fifo *fifo) 429 { 430 struct nvkm_runl *runl; 431 432 runl = nvkm_runl_new(fifo, 0, 0, 0); 433 if (IS_ERR(runl)) 434 return PTR_ERR(runl); 435 436 nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); 437 nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); 438 nvkm_runl_add(runl, 1, fifo->func->engn , NVKM_ENGINE_GR, 0); 439 nvkm_runl_add(runl, 2, fifo->func->engn , NVKM_ENGINE_MPEG, 0); /* NV31- */ 440 return 0; 441 } 442 443 int 444 nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) 445 { 446 /* The last CHID is reserved by HW as a "channel invalid" marker. */ 447 return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr - 1, &fifo->chid); 448 } 449 450 static int 451 nv04_fifo_chid_nr(struct nvkm_fifo *fifo) 452 { 453 return 16; 454 } 455 456 int 457 nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 458 enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, 459 struct nvkm_fifo **pfifo) 460 { 461 struct nv04_fifo *fifo; 462 int ret; 463 464 if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 465 return -ENOMEM; 466 fifo->ramfc = ramfc; 467 *pfifo = &fifo->base; 468 469 ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); 470 if (ret) 471 return ret; 472 473 return 0; 474 } 475 476 static const struct nvkm_fifo_func 477 nv04_fifo = { 478 .chid_nr = nv04_fifo_chid_nr, 479 .chid_ctor = nv04_fifo_chid_ctor, 480 .runl_ctor = nv04_fifo_runl_ctor, 481 .init = nv04_fifo_init, 482 .intr = nv04_fifo_intr, 483 .engine_id = nv04_fifo_engine_id, 484 .pause = nv04_fifo_pause, 485 .start = nv04_fifo_start, 486 .runl = &nv04_runl, 487 .engn = &nv04_engn, 488 .engn_sw = &nv04_engn, 489 .cgrp = {{ }, &nv04_cgrp }, 490 .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, 491 }; 492 493 int 494 nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 495 struct nvkm_fifo **pfifo) 496 { 497 return nv04_fifo_new_(&nv04_fifo, device, type, inst, 16, nv04_fifo_ramfc, pfifo); 498 } 499