1c39f472eSBen Skeggs /* 2c39f472eSBen Skeggs * Copyright 2012 Red Hat Inc. 3c39f472eSBen Skeggs * 4c39f472eSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5c39f472eSBen Skeggs * copy of this software and associated documentation files (the "Software"), 6c39f472eSBen Skeggs * to deal in the Software without restriction, including without limitation 7c39f472eSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8c39f472eSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9c39f472eSBen Skeggs * Software is furnished to do so, subject to the following conditions: 10c39f472eSBen Skeggs * 11c39f472eSBen Skeggs * The above copyright notice and this permission notice shall be included in 12c39f472eSBen Skeggs * all copies or substantial portions of the Software. 13c39f472eSBen Skeggs * 14c39f472eSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15c39f472eSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16c39f472eSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17c39f472eSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18c39f472eSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19c39f472eSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20c39f472eSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21c39f472eSBen Skeggs * 22c39f472eSBen Skeggs * Authors: Ben Skeggs 23c39f472eSBen Skeggs */ 24f5e45689SBen Skeggs #include "cgrp.h" 25f5e45689SBen Skeggs #include "chan.h" 26800ac1f8SBen Skeggs #include "chid.h" 27d94470e9SBen Skeggs #include "runl.h" 28f5e45689SBen Skeggs 2905c7145dSBen Skeggs #include "nv04.h" 309a65a38cSBen Skeggs #include "channv04.h" 319a65a38cSBen Skeggs #include "regsnv04.h" 32c39f472eSBen Skeggs 338f0649b5SBen Skeggs #include <core/client.h> 34c39f472eSBen Skeggs #include <core/ramht.h> 35d8e83994SBen Skeggs #include <subdev/instmem.h> 362fc71a05SBen Skeggs #include <subdev/mc.h> 37c39f472eSBen Skeggs #include <subdev/timer.h> 3861570911SBen Skeggs #include <engine/sw.h> 39c39f472eSBen Skeggs 40f5e45689SBen Skeggs #include <nvif/class.h> 41f5e45689SBen Skeggs 4213de7f46SBen Skeggs static const struct nv04_fifo_ramfc 4313de7f46SBen Skeggs nv04_fifo_ramfc[] = { 44c39f472eSBen Skeggs { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 45c39f472eSBen Skeggs { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 46c39f472eSBen Skeggs { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 47c39f472eSBen Skeggs { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 48c39f472eSBen Skeggs { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 49c39f472eSBen Skeggs { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 50c39f472eSBen Skeggs { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 51c39f472eSBen Skeggs { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 52c39f472eSBen Skeggs {} 53c39f472eSBen Skeggs }; 54c39f472eSBen Skeggs 55fd67738aSBen Skeggs void 56fd67738aSBen Skeggs nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) 57fd67738aSBen Skeggs { 58fd67738aSBen Skeggs struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 59fd67738aSBen Skeggs struct nv04_fifo *fifo = chan->fifo; 60fd67738aSBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 61fd67738aSBen Skeggs struct nvkm_memory *fctx = device->imem->ramfc; 62fd67738aSBen Skeggs const struct nv04_fifo_ramfc *c; 63fd67738aSBen Skeggs unsigned long flags; 64fd67738aSBen Skeggs u32 data = chan->ramfc; 65fd67738aSBen Skeggs u32 chid; 66fd67738aSBen Skeggs 67fd67738aSBen Skeggs /* prevent fifo context switches */ 68fd67738aSBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 69fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 70fd67738aSBen Skeggs 71fd67738aSBen Skeggs /* if this channel is active, replace it with a null context */ 72800ac1f8SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; 73fd67738aSBen Skeggs if (chid == chan->base.chid) { 74fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); 75fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); 76fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); 77fd67738aSBen Skeggs 78fd67738aSBen Skeggs c = fifo->ramfc; 79fd67738aSBen Skeggs nvkm_kmap(fctx); 80fd67738aSBen Skeggs do { 81fd67738aSBen Skeggs u32 rm = ((1ULL << c->bits) - 1) << c->regs; 82fd67738aSBen Skeggs u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; 83fd67738aSBen Skeggs u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; 84fd67738aSBen Skeggs u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); 85fd67738aSBen Skeggs nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); 86fd67738aSBen Skeggs } while ((++c)->bits); 87fd67738aSBen Skeggs nvkm_done(fctx); 88fd67738aSBen Skeggs 89fd67738aSBen Skeggs c = fifo->ramfc; 90fd67738aSBen Skeggs do { 91fd67738aSBen Skeggs nvkm_wr32(device, c->regp, 0x00000000); 92fd67738aSBen Skeggs } while ((++c)->bits); 93fd67738aSBen Skeggs 94fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); 95fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); 96800ac1f8SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.chid->mask); 97fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 98fd67738aSBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 99fd67738aSBen Skeggs } 100fd67738aSBen Skeggs 101fd67738aSBen Skeggs /* restore normal operation, after disabling dma mode */ 102fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); 103fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 104fd67738aSBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 105fd67738aSBen Skeggs } 106fd67738aSBen Skeggs 107fd67738aSBen Skeggs void 108fd67738aSBen Skeggs nv04_fifo_dma_init(struct nvkm_fifo_chan *base) 109fd67738aSBen Skeggs { 110fd67738aSBen Skeggs struct nv04_fifo_chan *chan = nv04_fifo_chan(base); 111fd67738aSBen Skeggs struct nv04_fifo *fifo = chan->fifo; 112fd67738aSBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 113fd67738aSBen Skeggs u32 mask = 1 << chan->base.chid; 114fd67738aSBen Skeggs unsigned long flags; 115fd67738aSBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 116fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); 117fd67738aSBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 118fd67738aSBen Skeggs } 119fd67738aSBen Skeggs 120f5e45689SBen Skeggs static const struct nvkm_chan_func 121f5e45689SBen Skeggs nv04_chan = { 122f5e45689SBen Skeggs }; 123f5e45689SBen Skeggs 124f5e45689SBen Skeggs const struct nvkm_cgrp_func 125f5e45689SBen Skeggs nv04_cgrp = { 126f5e45689SBen Skeggs }; 127f5e45689SBen Skeggs 128d94470e9SBen Skeggs const struct nvkm_engn_func 129d94470e9SBen Skeggs nv04_engn = { 130d94470e9SBen Skeggs }; 131d94470e9SBen Skeggs 132c39f472eSBen Skeggs void 13313de7f46SBen Skeggs nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) 1346189f1b0SBen Skeggs __acquires(fifo->base.lock) 135c39f472eSBen Skeggs { 13613de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 13787744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 138c39f472eSBen Skeggs unsigned long flags; 139c39f472eSBen Skeggs 1406189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 141c39f472eSBen Skeggs *pflags = flags; 142c39f472eSBen Skeggs 14387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 14487744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 145c39f472eSBen Skeggs 146c39f472eSBen Skeggs /* in some cases the puller may be left in an inconsistent state 147c39f472eSBen Skeggs * if you try to stop it while it's busy translating handles. 148c39f472eSBen Skeggs * sometimes you get a CACHE_ERROR, sometimes it just fails 149c39f472eSBen Skeggs * silently; sending incorrect instance offsets to PGRAPH after 150c39f472eSBen Skeggs * it's started up again. 151c39f472eSBen Skeggs * 152c39f472eSBen Skeggs * to avoid this, we invalidate the most recently calculated 153c39f472eSBen Skeggs * instance. 154c39f472eSBen Skeggs */ 155af3082b3SBen Skeggs nvkm_msec(device, 2000, 156af3082b3SBen Skeggs u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0); 157af3082b3SBen Skeggs if (!(tmp & NV04_PFIFO_CACHE1_PULL0_HASH_BUSY)) 158af3082b3SBen Skeggs break; 159af3082b3SBen Skeggs ); 160c39f472eSBen Skeggs 16187744403SBen Skeggs if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 162c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 16387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 164c39f472eSBen Skeggs 16587744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 166c39f472eSBen Skeggs } 167c39f472eSBen Skeggs 168c39f472eSBen Skeggs void 16913de7f46SBen Skeggs nv04_fifo_start(struct nvkm_fifo *base, unsigned long *pflags) 1706189f1b0SBen Skeggs __releases(fifo->base.lock) 171c39f472eSBen Skeggs { 17213de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 17387744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 174c39f472eSBen Skeggs unsigned long flags = *pflags; 175c39f472eSBen Skeggs 17687744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 17787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 178c39f472eSBen Skeggs 1796189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 180c39f472eSBen Skeggs } 181c39f472eSBen Skeggs 182d94470e9SBen Skeggs const struct nvkm_runl_func 183d94470e9SBen Skeggs nv04_runl = { 184d94470e9SBen Skeggs }; 185d94470e9SBen Skeggs 18664f7c698SBen Skeggs int 18764f7c698SBen Skeggs nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) 18864f7c698SBen Skeggs { 18964f7c698SBen Skeggs switch (engine->subdev.type) { 19064f7c698SBen Skeggs case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; 19164f7c698SBen Skeggs case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; 19264f7c698SBen Skeggs case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; 19364f7c698SBen Skeggs case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; 19464f7c698SBen Skeggs default: 19564f7c698SBen Skeggs WARN_ON(1); 19664f7c698SBen Skeggs return 0; 19764f7c698SBen Skeggs } 19864f7c698SBen Skeggs } 19964f7c698SBen Skeggs 200c39f472eSBen Skeggs static const char * 201c39f472eSBen Skeggs nv_dma_state_err(u32 state) 202c39f472eSBen Skeggs { 203c39f472eSBen Skeggs static const char * const desc[] = { 204c39f472eSBen Skeggs "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 205c39f472eSBen Skeggs "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 206c39f472eSBen Skeggs }; 207c39f472eSBen Skeggs return desc[(state >> 29) & 0x7]; 208c39f472eSBen Skeggs } 209c39f472eSBen Skeggs 210c39f472eSBen Skeggs static bool 21161570911SBen Skeggs nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) 212c39f472eSBen Skeggs { 21361570911SBen Skeggs struct nvkm_sw *sw = device->sw; 21461570911SBen Skeggs const int subc = (addr & 0x0000e000) >> 13; 21561570911SBen Skeggs const int mthd = (addr & 0x00001ffc); 21661570911SBen Skeggs const u32 mask = 0x0000000f << (subc * 4); 21761570911SBen Skeggs u32 engine = nvkm_rd32(device, 0x003280); 218c39f472eSBen Skeggs bool handled = false; 219c39f472eSBen Skeggs 220c39f472eSBen Skeggs switch (mthd) { 22161570911SBen Skeggs case 0x0000 ... 0x0000: /* subchannel's engine -> software */ 22261570911SBen Skeggs nvkm_wr32(device, 0x003280, (engine &= ~mask)); 223f6e7393eSGustavo A. R. Silva fallthrough; 22461570911SBen Skeggs case 0x0180 ... 0x01fc: /* handle -> instance */ 22561570911SBen Skeggs data = nvkm_rd32(device, 0x003258) & 0x0000ffff; 226f6e7393eSGustavo A. R. Silva fallthrough; 22761570911SBen Skeggs case 0x0100 ... 0x017c: 22861570911SBen Skeggs case 0x0200 ... 0x1ffc: /* pass method down to sw */ 22961570911SBen Skeggs if (!(engine & mask) && sw) 23061570911SBen Skeggs handled = nvkm_sw_mthd(sw, chid, subc, mthd, data); 231c39f472eSBen Skeggs break; 232c39f472eSBen Skeggs default: 233c39f472eSBen Skeggs break; 234c39f472eSBen Skeggs } 235c39f472eSBen Skeggs 236c39f472eSBen Skeggs return handled; 237c39f472eSBen Skeggs } 238c39f472eSBen Skeggs 239c39f472eSBen Skeggs static void 2402fc71a05SBen Skeggs nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) 241c39f472eSBen Skeggs { 2422fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 243e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 2448f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 2458f0649b5SBen Skeggs unsigned long flags; 24661570911SBen Skeggs u32 pull0 = nvkm_rd32(device, 0x003250); 247c39f472eSBen Skeggs u32 mthd, data; 248c39f472eSBen Skeggs int ptr; 249c39f472eSBen Skeggs 250c39f472eSBen Skeggs /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 251c39f472eSBen Skeggs * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 252c39f472eSBen Skeggs * show that it wraps around to the start at GET=0x800.. No clue as to 253c39f472eSBen Skeggs * why.. 254c39f472eSBen Skeggs */ 255c39f472eSBen Skeggs ptr = (get & 0x7ff) >> 2; 256c39f472eSBen Skeggs 257c39f472eSBen Skeggs if (device->card_type < NV_40) { 25887744403SBen Skeggs mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 25987744403SBen Skeggs data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 260c39f472eSBen Skeggs } else { 26187744403SBen Skeggs mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 26287744403SBen Skeggs data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 263c39f472eSBen Skeggs } 264c39f472eSBen Skeggs 26561570911SBen Skeggs if (!(pull0 & 0x00000100) || 26661570911SBen Skeggs !nv04_fifo_swmthd(device, chid, mthd, data)) { 2672fc71a05SBen Skeggs chan = nvkm_fifo_chan_chid(fifo, chid, &flags); 268e5c5e4f5SBen Skeggs nvkm_error(subdev, "CACHE_ERROR - " 269e5c5e4f5SBen Skeggs "ch %d [%s] subc %d mthd %04x data %08x\n", 2708f0649b5SBen Skeggs chid, chan ? chan->object.client->name : "unknown", 2718f0649b5SBen Skeggs (mthd >> 13) & 7, mthd & 0x1ffc, data); 2722fc71a05SBen Skeggs nvkm_fifo_chan_put(fifo, flags, &chan); 273c39f472eSBen Skeggs } 274c39f472eSBen Skeggs 27587744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 27687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 277c39f472eSBen Skeggs 27887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 27987744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 28087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 28187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 28287744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 28387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 284c39f472eSBen Skeggs 28587744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 28687744403SBen Skeggs nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 28787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 288c39f472eSBen Skeggs } 289c39f472eSBen Skeggs 290c39f472eSBen Skeggs static void 2912fc71a05SBen Skeggs nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) 292c39f472eSBen Skeggs { 2932fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 294e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 29587744403SBen Skeggs u32 dma_get = nvkm_rd32(device, 0x003244); 29687744403SBen Skeggs u32 dma_put = nvkm_rd32(device, 0x003240); 29787744403SBen Skeggs u32 push = nvkm_rd32(device, 0x003220); 29887744403SBen Skeggs u32 state = nvkm_rd32(device, 0x003228); 2998f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 3008f0649b5SBen Skeggs unsigned long flags; 3018f0649b5SBen Skeggs const char *name; 302c39f472eSBen Skeggs 3032fc71a05SBen Skeggs chan = nvkm_fifo_chan_chid(fifo, chid, &flags); 3048f0649b5SBen Skeggs name = chan ? chan->object.client->name : "unknown"; 305c39f472eSBen Skeggs if (device->card_type == NV_50) { 30687744403SBen Skeggs u32 ho_get = nvkm_rd32(device, 0x003328); 30787744403SBen Skeggs u32 ho_put = nvkm_rd32(device, 0x003320); 30887744403SBen Skeggs u32 ib_get = nvkm_rd32(device, 0x003334); 30987744403SBen Skeggs u32 ib_put = nvkm_rd32(device, 0x003330); 310c39f472eSBen Skeggs 311e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - " 312e5c5e4f5SBen Skeggs "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x " 313e5c5e4f5SBen Skeggs "ib_put %08x state %08x (err: %s) push %08x\n", 3148f0649b5SBen Skeggs chid, name, ho_get, dma_get, ho_put, dma_put, 315e5c5e4f5SBen Skeggs ib_get, ib_put, state, nv_dma_state_err(state), 316e5c5e4f5SBen Skeggs push); 317c39f472eSBen Skeggs 318c39f472eSBen Skeggs /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 31987744403SBen Skeggs nvkm_wr32(device, 0x003364, 0x00000000); 320c39f472eSBen Skeggs if (dma_get != dma_put || ho_get != ho_put) { 32187744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 32287744403SBen Skeggs nvkm_wr32(device, 0x003328, ho_put); 323c39f472eSBen Skeggs } else 324c39f472eSBen Skeggs if (ib_get != ib_put) 32587744403SBen Skeggs nvkm_wr32(device, 0x003334, ib_put); 326c39f472eSBen Skeggs } else { 327e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - ch %d [%s] get %08x put %08x " 328e5c5e4f5SBen Skeggs "state %08x (err: %s) push %08x\n", 3298f0649b5SBen Skeggs chid, name, dma_get, dma_put, state, 330c39f472eSBen Skeggs nv_dma_state_err(state), push); 331c39f472eSBen Skeggs 332c39f472eSBen Skeggs if (dma_get != dma_put) 33387744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 334c39f472eSBen Skeggs } 3352fc71a05SBen Skeggs nvkm_fifo_chan_put(fifo, flags, &chan); 336c39f472eSBen Skeggs 33787744403SBen Skeggs nvkm_wr32(device, 0x003228, 0x00000000); 33887744403SBen Skeggs nvkm_wr32(device, 0x003220, 0x00000001); 33987744403SBen Skeggs nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 340c39f472eSBen Skeggs } 341c39f472eSBen Skeggs 3422fc71a05SBen Skeggs irqreturn_t 3432fc71a05SBen Skeggs nv04_fifo_intr(struct nvkm_inth *inth) 344c39f472eSBen Skeggs { 3452fc71a05SBen Skeggs struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); 3462fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 347e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 34887744403SBen Skeggs u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 34987744403SBen Skeggs u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 350adc346b1SBen Skeggs u32 reassign, chid, get, sem; 351c39f472eSBen Skeggs 35287744403SBen Skeggs reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 35387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 354c39f472eSBen Skeggs 3552fc71a05SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; 35687744403SBen Skeggs get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 357c39f472eSBen Skeggs 358adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 3592fc71a05SBen Skeggs nv04_fifo_intr_cache_error(fifo, chid, get); 360adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 361c39f472eSBen Skeggs } 362c39f472eSBen Skeggs 363adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 3642fc71a05SBen Skeggs nv04_fifo_intr_dma_pusher(fifo, chid); 365adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 366c39f472eSBen Skeggs } 367c39f472eSBen Skeggs 368adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_SEMAPHORE) { 369adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_SEMAPHORE; 37087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 371c39f472eSBen Skeggs 37287744403SBen Skeggs sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 37387744403SBen Skeggs nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 374c39f472eSBen Skeggs 37587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 37687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 377c39f472eSBen Skeggs } 378c39f472eSBen Skeggs 379c39f472eSBen Skeggs if (device->card_type == NV_50) { 380adc346b1SBen Skeggs if (stat & 0x00000010) { 381adc346b1SBen Skeggs stat &= ~0x00000010; 38287744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x00000010); 383c39f472eSBen Skeggs } 384c39f472eSBen Skeggs 385adc346b1SBen Skeggs if (stat & 0x40000000) { 38687744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x40000000); 387*d67f3b96SBen Skeggs nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); 388adc346b1SBen Skeggs stat &= ~0x40000000; 389c39f472eSBen Skeggs } 390c39f472eSBen Skeggs } 391c39f472eSBen Skeggs 392adc346b1SBen Skeggs if (stat) { 393e5c5e4f5SBen Skeggs nvkm_warn(subdev, "intr %08x\n", stat); 39487744403SBen Skeggs nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 39587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 396c39f472eSBen Skeggs } 397c39f472eSBen Skeggs 39887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 3992fc71a05SBen Skeggs return IRQ_HANDLED; 400c39f472eSBen Skeggs } 401c39f472eSBen Skeggs 40213de7f46SBen Skeggs void 403800ac1f8SBen Skeggs nv04_fifo_init(struct nvkm_fifo *fifo) 404c39f472eSBen Skeggs { 405800ac1f8SBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 4065b1ab0c2SBen Skeggs struct nvkm_instmem *imem = device->imem; 4075b1ab0c2SBen Skeggs struct nvkm_ramht *ramht = imem->ramht; 4085b1ab0c2SBen Skeggs struct nvkm_memory *ramro = imem->ramro; 4095b1ab0c2SBen Skeggs struct nvkm_memory *ramfc = imem->ramfc; 410c39f472eSBen Skeggs 41187744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 41287744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 413c39f472eSBen Skeggs 41487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 4155b1ab0c2SBen Skeggs ((ramht->bits - 9) << 16) | 4161d2a1e53SBen Skeggs (ramht->gpuobj->addr >> 8)); 4175b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); 4185b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); 419c39f472eSBen Skeggs 420800ac1f8SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); 421c39f472eSBen Skeggs 42287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 42387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 424c39f472eSBen Skeggs 42587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 42687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 42787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 42813de7f46SBen Skeggs } 42913de7f46SBen Skeggs 430800ac1f8SBen Skeggs int 431d94470e9SBen Skeggs nv04_fifo_runl_ctor(struct nvkm_fifo *fifo) 432d94470e9SBen Skeggs { 433d94470e9SBen Skeggs struct nvkm_runl *runl; 434d94470e9SBen Skeggs 435d94470e9SBen Skeggs runl = nvkm_runl_new(fifo, 0, 0, 0); 436d94470e9SBen Skeggs if (IS_ERR(runl)) 437d94470e9SBen Skeggs return PTR_ERR(runl); 438d94470e9SBen Skeggs 439d94470e9SBen Skeggs nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); 440d94470e9SBen Skeggs nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); 441d94470e9SBen Skeggs nvkm_runl_add(runl, 1, fifo->func->engn , NVKM_ENGINE_GR, 0); 442d94470e9SBen Skeggs nvkm_runl_add(runl, 2, fifo->func->engn , NVKM_ENGINE_MPEG, 0); /* NV31- */ 443d94470e9SBen Skeggs return 0; 444d94470e9SBen Skeggs } 445d94470e9SBen Skeggs 446d94470e9SBen Skeggs int 447800ac1f8SBen Skeggs nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) 448800ac1f8SBen Skeggs { 449800ac1f8SBen Skeggs /* The last CHID is reserved by HW as a "channel invalid" marker. */ 450800ac1f8SBen Skeggs return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr - 1, &fifo->chid); 451800ac1f8SBen Skeggs } 452800ac1f8SBen Skeggs 4538c18138cSBen Skeggs static int 4548c18138cSBen Skeggs nv04_fifo_chid_nr(struct nvkm_fifo *fifo) 4558c18138cSBen Skeggs { 4568c18138cSBen Skeggs return 16; 4578c18138cSBen Skeggs } 4588c18138cSBen Skeggs 45913de7f46SBen Skeggs int 46013de7f46SBen Skeggs nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 461ab0db2bdSBen Skeggs enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, 46213de7f46SBen Skeggs struct nvkm_fifo **pfifo) 46313de7f46SBen Skeggs { 46413de7f46SBen Skeggs struct nv04_fifo *fifo; 46513de7f46SBen Skeggs int ret; 46613de7f46SBen Skeggs 46713de7f46SBen Skeggs if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 46813de7f46SBen Skeggs return -ENOMEM; 46913de7f46SBen Skeggs fifo->ramfc = ramfc; 47013de7f46SBen Skeggs *pfifo = &fifo->base; 47113de7f46SBen Skeggs 4728c18138cSBen Skeggs ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); 47313de7f46SBen Skeggs if (ret) 47413de7f46SBen Skeggs return ret; 47513de7f46SBen Skeggs 476c39f472eSBen Skeggs return 0; 477c39f472eSBen Skeggs } 478c39f472eSBen Skeggs 4798f0649b5SBen Skeggs static const struct nvkm_fifo_func 48013de7f46SBen Skeggs nv04_fifo = { 4818c18138cSBen Skeggs .chid_nr = nv04_fifo_chid_nr, 482800ac1f8SBen Skeggs .chid_ctor = nv04_fifo_chid_ctor, 483d94470e9SBen Skeggs .runl_ctor = nv04_fifo_runl_ctor, 48413de7f46SBen Skeggs .init = nv04_fifo_init, 48513de7f46SBen Skeggs .intr = nv04_fifo_intr, 48664f7c698SBen Skeggs .engine_id = nv04_fifo_engine_id, 48713de7f46SBen Skeggs .pause = nv04_fifo_pause, 48813de7f46SBen Skeggs .start = nv04_fifo_start, 489d94470e9SBen Skeggs .runl = &nv04_runl, 490d94470e9SBen Skeggs .engn = &nv04_engn, 491d94470e9SBen Skeggs .engn_sw = &nv04_engn, 492f5e45689SBen Skeggs .cgrp = {{ }, &nv04_cgrp }, 493f5e45689SBen Skeggs .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, 4948f0649b5SBen Skeggs }; 4958f0649b5SBen Skeggs 49613de7f46SBen Skeggs int 497ab0db2bdSBen Skeggs nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 498ab0db2bdSBen Skeggs struct nvkm_fifo **pfifo) 4999a65a38cSBen Skeggs { 500ab0db2bdSBen Skeggs return nv04_fifo_new_(&nv04_fifo, device, type, inst, 16, nv04_fifo_ramfc, pfifo); 5019a65a38cSBen Skeggs } 502