1c39f472eSBen Skeggs /* 2c39f472eSBen Skeggs * Copyright 2012 Red Hat Inc. 3c39f472eSBen Skeggs * 4c39f472eSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5c39f472eSBen Skeggs * copy of this software and associated documentation files (the "Software"), 6c39f472eSBen Skeggs * to deal in the Software without restriction, including without limitation 7c39f472eSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8c39f472eSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9c39f472eSBen Skeggs * Software is furnished to do so, subject to the following conditions: 10c39f472eSBen Skeggs * 11c39f472eSBen Skeggs * The above copyright notice and this permission notice shall be included in 12c39f472eSBen Skeggs * all copies or substantial portions of the Software. 13c39f472eSBen Skeggs * 14c39f472eSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15c39f472eSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16c39f472eSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17c39f472eSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18c39f472eSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19c39f472eSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20c39f472eSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21c39f472eSBen Skeggs * 22c39f472eSBen Skeggs * Authors: Ben Skeggs 23c39f472eSBen Skeggs */ 24f5e45689SBen Skeggs #include "cgrp.h" 25f5e45689SBen Skeggs #include "chan.h" 26*800ac1f8SBen Skeggs #include "chid.h" 27f5e45689SBen Skeggs 2805c7145dSBen Skeggs #include "nv04.h" 299a65a38cSBen Skeggs #include "channv04.h" 309a65a38cSBen Skeggs #include "regsnv04.h" 31c39f472eSBen Skeggs 328f0649b5SBen Skeggs #include <core/client.h> 33c39f472eSBen Skeggs #include <core/ramht.h> 34d8e83994SBen Skeggs #include <subdev/instmem.h> 35c39f472eSBen Skeggs #include <subdev/timer.h> 3661570911SBen Skeggs #include <engine/sw.h> 37c39f472eSBen Skeggs 38f5e45689SBen Skeggs #include <nvif/class.h> 39f5e45689SBen Skeggs 4013de7f46SBen Skeggs static const struct nv04_fifo_ramfc 4113de7f46SBen Skeggs nv04_fifo_ramfc[] = { 42c39f472eSBen Skeggs { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 43c39f472eSBen Skeggs { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 44c39f472eSBen Skeggs { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 45c39f472eSBen Skeggs { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 46c39f472eSBen Skeggs { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 47c39f472eSBen Skeggs { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 48c39f472eSBen Skeggs { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 49c39f472eSBen Skeggs { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 50c39f472eSBen Skeggs {} 51c39f472eSBen Skeggs }; 52c39f472eSBen Skeggs 53fd67738aSBen Skeggs void 54fd67738aSBen Skeggs nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) 55fd67738aSBen Skeggs { 56fd67738aSBen Skeggs struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 57fd67738aSBen Skeggs struct nv04_fifo *fifo = chan->fifo; 58fd67738aSBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 59fd67738aSBen Skeggs struct nvkm_memory *fctx = device->imem->ramfc; 60fd67738aSBen Skeggs const struct nv04_fifo_ramfc *c; 61fd67738aSBen Skeggs unsigned long flags; 62fd67738aSBen Skeggs u32 data = chan->ramfc; 63fd67738aSBen Skeggs u32 chid; 64fd67738aSBen Skeggs 65fd67738aSBen Skeggs /* prevent fifo context switches */ 66fd67738aSBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 67fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 68fd67738aSBen Skeggs 69fd67738aSBen Skeggs /* if this channel is active, replace it with a null context */ 70*800ac1f8SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; 71fd67738aSBen Skeggs if (chid == chan->base.chid) { 72fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); 73fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); 74fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); 75fd67738aSBen Skeggs 76fd67738aSBen Skeggs c = fifo->ramfc; 77fd67738aSBen Skeggs nvkm_kmap(fctx); 78fd67738aSBen Skeggs do { 79fd67738aSBen Skeggs u32 rm = ((1ULL << c->bits) - 1) << c->regs; 80fd67738aSBen Skeggs u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; 81fd67738aSBen Skeggs u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; 82fd67738aSBen Skeggs u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); 83fd67738aSBen Skeggs nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); 84fd67738aSBen Skeggs } while ((++c)->bits); 85fd67738aSBen Skeggs nvkm_done(fctx); 86fd67738aSBen Skeggs 87fd67738aSBen Skeggs c = fifo->ramfc; 88fd67738aSBen Skeggs do { 89fd67738aSBen Skeggs nvkm_wr32(device, c->regp, 0x00000000); 90fd67738aSBen Skeggs } while ((++c)->bits); 91fd67738aSBen Skeggs 92fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); 93fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); 94*800ac1f8SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.chid->mask); 95fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 96fd67738aSBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 97fd67738aSBen Skeggs } 98fd67738aSBen Skeggs 99fd67738aSBen Skeggs /* restore normal operation, after disabling dma mode */ 100fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); 101fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 102fd67738aSBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 103fd67738aSBen Skeggs } 104fd67738aSBen Skeggs 105fd67738aSBen Skeggs void 106fd67738aSBen Skeggs nv04_fifo_dma_init(struct nvkm_fifo_chan *base) 107fd67738aSBen Skeggs { 108fd67738aSBen Skeggs struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 109fd67738aSBen Skeggs struct nv04_fifo *fifo = chan->fifo; 110fd67738aSBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 111fd67738aSBen Skeggs u32 mask = 1 << chan->base.chid; 112fd67738aSBen Skeggs unsigned long flags; 113fd67738aSBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 114fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); 115fd67738aSBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 116fd67738aSBen Skeggs } 117fd67738aSBen Skeggs 118f5e45689SBen Skeggs static const struct nvkm_chan_func 119f5e45689SBen Skeggs nv04_chan = { 120f5e45689SBen Skeggs }; 121f5e45689SBen Skeggs 122f5e45689SBen Skeggs const struct nvkm_cgrp_func 123f5e45689SBen Skeggs nv04_cgrp = { 124f5e45689SBen Skeggs }; 125f5e45689SBen Skeggs 126c39f472eSBen Skeggs void 12713de7f46SBen Skeggs nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) 1286189f1b0SBen Skeggs __acquires(fifo->base.lock) 129c39f472eSBen Skeggs { 13013de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 13187744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 132c39f472eSBen Skeggs unsigned long flags; 133c39f472eSBen Skeggs 1346189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 135c39f472eSBen Skeggs *pflags = flags; 136c39f472eSBen Skeggs 13787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 13887744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 139c39f472eSBen Skeggs 140c39f472eSBen Skeggs /* in some cases the puller may be left in an inconsistent state 141c39f472eSBen Skeggs * if you try to stop it while it's busy translating handles. 142c39f472eSBen Skeggs * sometimes you get a CACHE_ERROR, sometimes it just fails 143c39f472eSBen Skeggs * silently; sending incorrect instance offsets to PGRAPH after 144c39f472eSBen Skeggs * it's started up again. 145c39f472eSBen Skeggs * 146c39f472eSBen Skeggs * to avoid this, we invalidate the most recently calculated 147c39f472eSBen Skeggs * instance. 148c39f472eSBen Skeggs */ 149af3082b3SBen Skeggs nvkm_msec(device, 2000, 150af3082b3SBen Skeggs u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0); 151af3082b3SBen Skeggs if (!(tmp & NV04_PFIFO_CACHE1_PULL0_HASH_BUSY)) 152af3082b3SBen Skeggs break; 153af3082b3SBen Skeggs ); 154c39f472eSBen Skeggs 15587744403SBen Skeggs if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 156c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 15787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 158c39f472eSBen Skeggs 15987744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 160c39f472eSBen Skeggs } 161c39f472eSBen Skeggs 162c39f472eSBen Skeggs void 16313de7f46SBen Skeggs nv04_fifo_start(struct nvkm_fifo *base, unsigned long *pflags) 1646189f1b0SBen Skeggs __releases(fifo->base.lock) 165c39f472eSBen Skeggs { 16613de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 16787744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 168c39f472eSBen Skeggs unsigned long flags = *pflags; 169c39f472eSBen Skeggs 17087744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 17187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 172c39f472eSBen Skeggs 1736189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 174c39f472eSBen Skeggs } 175c39f472eSBen Skeggs 17649616203SBen Skeggs struct nvkm_engine * 17749616203SBen Skeggs nv04_fifo_id_engine(struct nvkm_fifo *fifo, int engi) 17849616203SBen Skeggs { 17949616203SBen Skeggs enum nvkm_subdev_type type; 18049616203SBen Skeggs 18149616203SBen Skeggs switch (engi) { 18249616203SBen Skeggs case NV04_FIFO_ENGN_SW : type = NVKM_ENGINE_SW; break; 18349616203SBen Skeggs case NV04_FIFO_ENGN_GR : type = NVKM_ENGINE_GR; break; 18449616203SBen Skeggs case NV04_FIFO_ENGN_MPEG: type = NVKM_ENGINE_MPEG; break; 18549616203SBen Skeggs case NV04_FIFO_ENGN_DMA : type = NVKM_ENGINE_DMAOBJ; break; 18649616203SBen Skeggs default: 18749616203SBen Skeggs WARN_ON(1); 18849616203SBen Skeggs return NULL; 18949616203SBen Skeggs } 19049616203SBen Skeggs 19149616203SBen Skeggs return nvkm_device_engine(fifo->engine.subdev.device, type, 0); 19249616203SBen Skeggs } 19349616203SBen Skeggs 19464f7c698SBen Skeggs int 19564f7c698SBen Skeggs nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) 19664f7c698SBen Skeggs { 19764f7c698SBen Skeggs switch (engine->subdev.type) { 19864f7c698SBen Skeggs case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; 19964f7c698SBen Skeggs case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; 20064f7c698SBen Skeggs case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; 20164f7c698SBen Skeggs case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; 20264f7c698SBen Skeggs default: 20364f7c698SBen Skeggs WARN_ON(1); 20464f7c698SBen Skeggs return 0; 20564f7c698SBen Skeggs } 20664f7c698SBen Skeggs } 20764f7c698SBen Skeggs 208c39f472eSBen Skeggs static const char * 209c39f472eSBen Skeggs nv_dma_state_err(u32 state) 210c39f472eSBen Skeggs { 211c39f472eSBen Skeggs static const char * const desc[] = { 212c39f472eSBen Skeggs "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 213c39f472eSBen Skeggs "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 214c39f472eSBen Skeggs }; 215c39f472eSBen Skeggs return desc[(state >> 29) & 0x7]; 216c39f472eSBen Skeggs } 217c39f472eSBen Skeggs 218c39f472eSBen Skeggs static bool 21961570911SBen Skeggs nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) 220c39f472eSBen Skeggs { 22161570911SBen Skeggs struct nvkm_sw *sw = device->sw; 22261570911SBen Skeggs const int subc = (addr & 0x0000e000) >> 13; 22361570911SBen Skeggs const int mthd = (addr & 0x00001ffc); 22461570911SBen Skeggs const u32 mask = 0x0000000f << (subc * 4); 22561570911SBen Skeggs u32 engine = nvkm_rd32(device, 0x003280); 226c39f472eSBen Skeggs bool handled = false; 227c39f472eSBen Skeggs 228c39f472eSBen Skeggs switch (mthd) { 22961570911SBen Skeggs case 0x0000 ... 0x0000: /* subchannel's engine -> software */ 23061570911SBen Skeggs nvkm_wr32(device, 0x003280, (engine &= ~mask)); 231f6e7393eSGustavo A. R. Silva fallthrough; 23261570911SBen Skeggs case 0x0180 ... 0x01fc: /* handle -> instance */ 23361570911SBen Skeggs data = nvkm_rd32(device, 0x003258) & 0x0000ffff; 234f6e7393eSGustavo A. R. Silva fallthrough; 23561570911SBen Skeggs case 0x0100 ... 0x017c: 23661570911SBen Skeggs case 0x0200 ... 0x1ffc: /* pass method down to sw */ 23761570911SBen Skeggs if (!(engine & mask) && sw) 23861570911SBen Skeggs handled = nvkm_sw_mthd(sw, chid, subc, mthd, data); 239c39f472eSBen Skeggs break; 240c39f472eSBen Skeggs default: 241c39f472eSBen Skeggs break; 242c39f472eSBen Skeggs } 243c39f472eSBen Skeggs 244c39f472eSBen Skeggs return handled; 245c39f472eSBen Skeggs } 246c39f472eSBen Skeggs 247c39f472eSBen Skeggs static void 248e5c5e4f5SBen Skeggs nv04_fifo_cache_error(struct nv04_fifo *fifo, u32 chid, u32 get) 249c39f472eSBen Skeggs { 250e5c5e4f5SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 251e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 2528f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 2538f0649b5SBen Skeggs unsigned long flags; 25461570911SBen Skeggs u32 pull0 = nvkm_rd32(device, 0x003250); 255c39f472eSBen Skeggs u32 mthd, data; 256c39f472eSBen Skeggs int ptr; 257c39f472eSBen Skeggs 258c39f472eSBen Skeggs /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 259c39f472eSBen Skeggs * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 260c39f472eSBen Skeggs * show that it wraps around to the start at GET=0x800.. No clue as to 261c39f472eSBen Skeggs * why.. 262c39f472eSBen Skeggs */ 263c39f472eSBen Skeggs ptr = (get & 0x7ff) >> 2; 264c39f472eSBen Skeggs 265c39f472eSBen Skeggs if (device->card_type < NV_40) { 26687744403SBen Skeggs mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 26787744403SBen Skeggs data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 268c39f472eSBen Skeggs } else { 26987744403SBen Skeggs mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 27087744403SBen Skeggs data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 271c39f472eSBen Skeggs } 272c39f472eSBen Skeggs 27361570911SBen Skeggs if (!(pull0 & 0x00000100) || 27461570911SBen Skeggs !nv04_fifo_swmthd(device, chid, mthd, data)) { 2758f0649b5SBen Skeggs chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); 276e5c5e4f5SBen Skeggs nvkm_error(subdev, "CACHE_ERROR - " 277e5c5e4f5SBen Skeggs "ch %d [%s] subc %d mthd %04x data %08x\n", 2788f0649b5SBen Skeggs chid, chan ? chan->object.client->name : "unknown", 2798f0649b5SBen Skeggs (mthd >> 13) & 7, mthd & 0x1ffc, data); 2808f0649b5SBen Skeggs nvkm_fifo_chan_put(&fifo->base, flags, &chan); 281c39f472eSBen Skeggs } 282c39f472eSBen Skeggs 28387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 28487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 285c39f472eSBen Skeggs 28687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 28787744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 28887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 28987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 29087744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 29187744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 292c39f472eSBen Skeggs 29387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 29487744403SBen Skeggs nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 29587744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 296c39f472eSBen Skeggs } 297c39f472eSBen Skeggs 298c39f472eSBen Skeggs static void 299e5c5e4f5SBen Skeggs nv04_fifo_dma_pusher(struct nv04_fifo *fifo, u32 chid) 300c39f472eSBen Skeggs { 301e5c5e4f5SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 302e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 30387744403SBen Skeggs u32 dma_get = nvkm_rd32(device, 0x003244); 30487744403SBen Skeggs u32 dma_put = nvkm_rd32(device, 0x003240); 30587744403SBen Skeggs u32 push = nvkm_rd32(device, 0x003220); 30687744403SBen Skeggs u32 state = nvkm_rd32(device, 0x003228); 3078f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 3088f0649b5SBen Skeggs unsigned long flags; 3098f0649b5SBen Skeggs const char *name; 310c39f472eSBen Skeggs 3118f0649b5SBen Skeggs chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); 3128f0649b5SBen Skeggs name = chan ? chan->object.client->name : "unknown"; 313c39f472eSBen Skeggs if (device->card_type == NV_50) { 31487744403SBen Skeggs u32 ho_get = nvkm_rd32(device, 0x003328); 31587744403SBen Skeggs u32 ho_put = nvkm_rd32(device, 0x003320); 31687744403SBen Skeggs u32 ib_get = nvkm_rd32(device, 0x003334); 31787744403SBen Skeggs u32 ib_put = nvkm_rd32(device, 0x003330); 318c39f472eSBen Skeggs 319e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - " 320e5c5e4f5SBen Skeggs "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x " 321e5c5e4f5SBen Skeggs "ib_put %08x state %08x (err: %s) push %08x\n", 3228f0649b5SBen Skeggs chid, name, ho_get, dma_get, ho_put, dma_put, 323e5c5e4f5SBen Skeggs ib_get, ib_put, state, nv_dma_state_err(state), 324e5c5e4f5SBen Skeggs push); 325c39f472eSBen Skeggs 326c39f472eSBen Skeggs /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 32787744403SBen Skeggs nvkm_wr32(device, 0x003364, 0x00000000); 328c39f472eSBen Skeggs if (dma_get != dma_put || ho_get != ho_put) { 32987744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 33087744403SBen Skeggs nvkm_wr32(device, 0x003328, ho_put); 331c39f472eSBen Skeggs } else 332c39f472eSBen Skeggs if (ib_get != ib_put) 33387744403SBen Skeggs nvkm_wr32(device, 0x003334, ib_put); 334c39f472eSBen Skeggs } else { 335e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - ch %d [%s] get %08x put %08x " 336e5c5e4f5SBen Skeggs "state %08x (err: %s) push %08x\n", 3378f0649b5SBen Skeggs chid, name, dma_get, dma_put, state, 338c39f472eSBen Skeggs nv_dma_state_err(state), push); 339c39f472eSBen Skeggs 340c39f472eSBen Skeggs if (dma_get != dma_put) 34187744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 342c39f472eSBen Skeggs } 3438f0649b5SBen Skeggs nvkm_fifo_chan_put(&fifo->base, flags, &chan); 344c39f472eSBen Skeggs 34587744403SBen Skeggs nvkm_wr32(device, 0x003228, 0x00000000); 34687744403SBen Skeggs nvkm_wr32(device, 0x003220, 0x00000001); 34787744403SBen Skeggs nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 348c39f472eSBen Skeggs } 349c39f472eSBen Skeggs 350c39f472eSBen Skeggs void 35113de7f46SBen Skeggs nv04_fifo_intr(struct nvkm_fifo *base) 352c39f472eSBen Skeggs { 35313de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 35413de7f46SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 355e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 35687744403SBen Skeggs u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 35787744403SBen Skeggs u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 358adc346b1SBen Skeggs u32 reassign, chid, get, sem; 359c39f472eSBen Skeggs 36087744403SBen Skeggs reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 36187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 362c39f472eSBen Skeggs 363*800ac1f8SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; 36487744403SBen Skeggs get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 365c39f472eSBen Skeggs 366adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 367e5c5e4f5SBen Skeggs nv04_fifo_cache_error(fifo, chid, get); 368adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 369c39f472eSBen Skeggs } 370c39f472eSBen Skeggs 371adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 372e5c5e4f5SBen Skeggs nv04_fifo_dma_pusher(fifo, chid); 373adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 374c39f472eSBen Skeggs } 375c39f472eSBen Skeggs 376adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_SEMAPHORE) { 377adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_SEMAPHORE; 37887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 379c39f472eSBen Skeggs 38087744403SBen Skeggs sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 38187744403SBen Skeggs nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 382c39f472eSBen Skeggs 38387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 38487744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 385c39f472eSBen Skeggs } 386c39f472eSBen Skeggs 387c39f472eSBen Skeggs if (device->card_type == NV_50) { 388adc346b1SBen Skeggs if (stat & 0x00000010) { 389adc346b1SBen Skeggs stat &= ~0x00000010; 39087744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x00000010); 391c39f472eSBen Skeggs } 392c39f472eSBen Skeggs 393adc346b1SBen Skeggs if (stat & 0x40000000) { 39487744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x40000000); 3956189f1b0SBen Skeggs nvkm_fifo_uevent(&fifo->base); 396adc346b1SBen Skeggs stat &= ~0x40000000; 397c39f472eSBen Skeggs } 398c39f472eSBen Skeggs } 399c39f472eSBen Skeggs 400adc346b1SBen Skeggs if (stat) { 401e5c5e4f5SBen Skeggs nvkm_warn(subdev, "intr %08x\n", stat); 40287744403SBen Skeggs nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 40387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 404c39f472eSBen Skeggs } 405c39f472eSBen Skeggs 40687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 407c39f472eSBen Skeggs } 408c39f472eSBen Skeggs 40913de7f46SBen Skeggs void 410*800ac1f8SBen Skeggs nv04_fifo_init(struct nvkm_fifo *fifo) 411c39f472eSBen Skeggs { 412*800ac1f8SBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 4135b1ab0c2SBen Skeggs struct nvkm_instmem *imem = device->imem; 4145b1ab0c2SBen Skeggs struct nvkm_ramht *ramht = imem->ramht; 4155b1ab0c2SBen Skeggs struct nvkm_memory *ramro = imem->ramro; 4165b1ab0c2SBen Skeggs struct nvkm_memory *ramfc = imem->ramfc; 417c39f472eSBen Skeggs 41887744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 41987744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 420c39f472eSBen Skeggs 42187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 4225b1ab0c2SBen Skeggs ((ramht->bits - 9) << 16) | 4231d2a1e53SBen Skeggs (ramht->gpuobj->addr >> 8)); 4245b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); 4255b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); 426c39f472eSBen Skeggs 427*800ac1f8SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); 428c39f472eSBen Skeggs 42987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 43087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 431c39f472eSBen Skeggs 43287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 43387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 43487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 43513de7f46SBen Skeggs } 43613de7f46SBen Skeggs 437*800ac1f8SBen Skeggs int 438*800ac1f8SBen Skeggs nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) 439*800ac1f8SBen Skeggs { 440*800ac1f8SBen Skeggs /* The last CHID is reserved by HW as a "channel invalid" marker. */ 441*800ac1f8SBen Skeggs return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr - 1, &fifo->chid); 442*800ac1f8SBen Skeggs } 443*800ac1f8SBen Skeggs 4448c18138cSBen Skeggs static int 4458c18138cSBen Skeggs nv04_fifo_chid_nr(struct nvkm_fifo *fifo) 4468c18138cSBen Skeggs { 4478c18138cSBen Skeggs return 16; 4488c18138cSBen Skeggs } 4498c18138cSBen Skeggs 45013de7f46SBen Skeggs int 45113de7f46SBen Skeggs nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 452ab0db2bdSBen Skeggs enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, 45313de7f46SBen Skeggs struct nvkm_fifo **pfifo) 45413de7f46SBen Skeggs { 45513de7f46SBen Skeggs struct nv04_fifo *fifo; 45613de7f46SBen Skeggs int ret; 45713de7f46SBen Skeggs 45813de7f46SBen Skeggs if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 45913de7f46SBen Skeggs return -ENOMEM; 46013de7f46SBen Skeggs fifo->ramfc = ramfc; 46113de7f46SBen Skeggs *pfifo = &fifo->base; 46213de7f46SBen Skeggs 4638c18138cSBen Skeggs ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); 46413de7f46SBen Skeggs if (ret) 46513de7f46SBen Skeggs return ret; 46613de7f46SBen Skeggs 46713de7f46SBen Skeggs set_bit(nr - 1, fifo->base.mask); /* inactive channel */ 468c39f472eSBen Skeggs return 0; 469c39f472eSBen Skeggs } 470c39f472eSBen Skeggs 4718f0649b5SBen Skeggs static const struct nvkm_fifo_func 47213de7f46SBen Skeggs nv04_fifo = { 4738c18138cSBen Skeggs .chid_nr = nv04_fifo_chid_nr, 474*800ac1f8SBen Skeggs .chid_ctor = nv04_fifo_chid_ctor, 47513de7f46SBen Skeggs .init = nv04_fifo_init, 47613de7f46SBen Skeggs .intr = nv04_fifo_intr, 47764f7c698SBen Skeggs .engine_id = nv04_fifo_engine_id, 47849616203SBen Skeggs .id_engine = nv04_fifo_id_engine, 47913de7f46SBen Skeggs .pause = nv04_fifo_pause, 48013de7f46SBen Skeggs .start = nv04_fifo_start, 481f5e45689SBen Skeggs .cgrp = {{ }, &nv04_cgrp }, 482f5e45689SBen Skeggs .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, 4838f0649b5SBen Skeggs }; 4848f0649b5SBen Skeggs 48513de7f46SBen Skeggs int 486ab0db2bdSBen Skeggs nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 487ab0db2bdSBen Skeggs struct nvkm_fifo **pfifo) 4889a65a38cSBen Skeggs { 489ab0db2bdSBen Skeggs return nv04_fifo_new_(&nv04_fifo, device, type, inst, 16, nv04_fifo_ramfc, pfifo); 4909a65a38cSBen Skeggs } 491