1c39f472eSBen Skeggs /* 2c39f472eSBen Skeggs * Copyright 2012 Red Hat Inc. 3c39f472eSBen Skeggs * 4c39f472eSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5c39f472eSBen Skeggs * copy of this software and associated documentation files (the "Software"), 6c39f472eSBen Skeggs * to deal in the Software without restriction, including without limitation 7c39f472eSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8c39f472eSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9c39f472eSBen Skeggs * Software is furnished to do so, subject to the following conditions: 10c39f472eSBen Skeggs * 11c39f472eSBen Skeggs * The above copyright notice and this permission notice shall be included in 12c39f472eSBen Skeggs * all copies or substantial portions of the Software. 13c39f472eSBen Skeggs * 14c39f472eSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15c39f472eSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16c39f472eSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17c39f472eSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18c39f472eSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19c39f472eSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20c39f472eSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21c39f472eSBen Skeggs * 22c39f472eSBen Skeggs * Authors: Ben Skeggs 23c39f472eSBen Skeggs */ 24f5e45689SBen Skeggs #include "cgrp.h" 25f5e45689SBen Skeggs #include "chan.h" 26800ac1f8SBen Skeggs #include "chid.h" 27d94470e9SBen Skeggs #include "runl.h" 28f5e45689SBen Skeggs 2905c7145dSBen Skeggs #include "nv04.h" 309a65a38cSBen Skeggs #include "channv04.h" 319a65a38cSBen Skeggs #include "regsnv04.h" 32c39f472eSBen Skeggs 33c39f472eSBen Skeggs #include <core/ramht.h> 34d8e83994SBen Skeggs #include <subdev/instmem.h> 352fc71a05SBen Skeggs #include <subdev/mc.h> 36c39f472eSBen Skeggs #include <subdev/timer.h> 3761570911SBen Skeggs #include <engine/sw.h> 38c39f472eSBen Skeggs 39f5e45689SBen Skeggs #include <nvif/class.h> 40f5e45689SBen Skeggs 41fd67738aSBen Skeggs void 4267059b9fSBen Skeggs nv04_chan_stop(struct nvkm_chan *chan) 43fd67738aSBen Skeggs { 44*3647c53bSBen Skeggs struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; 45*3647c53bSBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 46fd67738aSBen Skeggs struct nvkm_memory *fctx = device->imem->ramfc; 47*3647c53bSBen Skeggs const struct nvkm_ramfc_layout *c; 48fd67738aSBen Skeggs unsigned long flags; 49*3647c53bSBen Skeggs u32 data = chan->ramfc_offset; 50fd67738aSBen Skeggs u32 chid; 51fd67738aSBen Skeggs 52fd67738aSBen Skeggs /* prevent fifo context switches */ 53*3647c53bSBen Skeggs spin_lock_irqsave(&fifo->lock, flags); 54fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 55fd67738aSBen Skeggs 56fd67738aSBen Skeggs /* if this channel is active, replace it with a null context */ 57*3647c53bSBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; 5867059b9fSBen Skeggs if (chid == chan->id) { 59fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); 60fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); 61fd67738aSBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); 62fd67738aSBen Skeggs 63*3647c53bSBen Skeggs c = chan->func->ramfc->layout; 64fd67738aSBen Skeggs nvkm_kmap(fctx); 65fd67738aSBen Skeggs do { 66fd67738aSBen Skeggs u32 rm = ((1ULL << c->bits) - 1) << c->regs; 67fd67738aSBen Skeggs u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; 68fd67738aSBen Skeggs u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; 69fd67738aSBen Skeggs u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); 70fd67738aSBen Skeggs nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); 71fd67738aSBen Skeggs } while ((++c)->bits); 72fd67738aSBen Skeggs nvkm_done(fctx); 73fd67738aSBen Skeggs 74*3647c53bSBen Skeggs c = chan->func->ramfc->layout; 75fd67738aSBen Skeggs do { 76fd67738aSBen Skeggs nvkm_wr32(device, c->regp, 0x00000000); 77fd67738aSBen Skeggs } while ((++c)->bits); 78fd67738aSBen Skeggs 79fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); 80fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); 81*3647c53bSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); 82fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 83fd67738aSBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 84fd67738aSBen Skeggs } 85fd67738aSBen Skeggs 86fd67738aSBen Skeggs /* restore normal operation, after disabling dma mode */ 8767059b9fSBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, BIT(chan->id), 0); 88fd67738aSBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 89*3647c53bSBen Skeggs spin_unlock_irqrestore(&fifo->lock, flags); 90fd67738aSBen Skeggs } 91fd67738aSBen Skeggs 92fd67738aSBen Skeggs void 9367059b9fSBen Skeggs nv04_chan_start(struct nvkm_chan *chan) 94fd67738aSBen Skeggs { 9567059b9fSBen Skeggs struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; 96fd67738aSBen Skeggs unsigned long flags; 9767059b9fSBen Skeggs 9867059b9fSBen Skeggs spin_lock_irqsave(&fifo->lock, flags); 9967059b9fSBen Skeggs nvkm_mask(fifo->engine.subdev.device, NV04_PFIFO_MODE, BIT(chan->id), BIT(chan->id)); 10067059b9fSBen Skeggs spin_unlock_irqrestore(&fifo->lock, flags); 101fd67738aSBen Skeggs } 102fd67738aSBen Skeggs 103*3647c53bSBen Skeggs void 104*3647c53bSBen Skeggs nv04_chan_ramfc_clear(struct nvkm_chan *chan) 105*3647c53bSBen Skeggs { 106*3647c53bSBen Skeggs struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; 107*3647c53bSBen Skeggs const struct nvkm_ramfc_layout *c = chan->func->ramfc->layout; 108*3647c53bSBen Skeggs 109*3647c53bSBen Skeggs nvkm_kmap(ramfc); 110*3647c53bSBen Skeggs do { 111*3647c53bSBen Skeggs nvkm_wo32(ramfc, chan->ramfc_offset + c->ctxp, 0x00000000); 112*3647c53bSBen Skeggs } while ((++c)->bits); 113*3647c53bSBen Skeggs nvkm_done(ramfc); 114*3647c53bSBen Skeggs } 115*3647c53bSBen Skeggs 116*3647c53bSBen Skeggs static int 117*3647c53bSBen Skeggs nv04_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) 118*3647c53bSBen Skeggs { 119*3647c53bSBen Skeggs struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; 120*3647c53bSBen Skeggs const u32 base = chan->id * 32; 121*3647c53bSBen Skeggs 122*3647c53bSBen Skeggs chan->ramfc_offset = base; 123*3647c53bSBen Skeggs 124*3647c53bSBen Skeggs nvkm_kmap(ramfc); 125*3647c53bSBen Skeggs nvkm_wo32(ramfc, base + 0x00, offset); 126*3647c53bSBen Skeggs nvkm_wo32(ramfc, base + 0x04, offset); 127*3647c53bSBen Skeggs nvkm_wo32(ramfc, base + 0x08, chan->push->addr >> 4); 128*3647c53bSBen Skeggs nvkm_wo32(ramfc, base + 0x10, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | 129*3647c53bSBen Skeggs NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | 130*3647c53bSBen Skeggs #ifdef __BIG_ENDIAN 131*3647c53bSBen Skeggs NV_PFIFO_CACHE1_BIG_ENDIAN | 132*3647c53bSBen Skeggs #endif 133*3647c53bSBen Skeggs NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); 134*3647c53bSBen Skeggs nvkm_done(ramfc); 135*3647c53bSBen Skeggs return 0; 136*3647c53bSBen Skeggs } 137*3647c53bSBen Skeggs 138*3647c53bSBen Skeggs static const struct nvkm_chan_func_ramfc 139*3647c53bSBen Skeggs nv04_chan_ramfc = { 140*3647c53bSBen Skeggs .layout = (const struct nvkm_ramfc_layout[]) { 141*3647c53bSBen Skeggs { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 142*3647c53bSBen Skeggs { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 143*3647c53bSBen Skeggs { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 144*3647c53bSBen Skeggs { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 145*3647c53bSBen Skeggs { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 146*3647c53bSBen Skeggs { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 147*3647c53bSBen Skeggs { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 148*3647c53bSBen Skeggs { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 149*3647c53bSBen Skeggs {} 150*3647c53bSBen Skeggs }, 151*3647c53bSBen Skeggs .write = nv04_chan_ramfc_write, 152*3647c53bSBen Skeggs .clear = nv04_chan_ramfc_clear, 153*3647c53bSBen Skeggs .ctxdma = true, 154*3647c53bSBen Skeggs }; 155*3647c53bSBen Skeggs 156fbe9f433SBen Skeggs const struct nvkm_chan_func_userd 157fbe9f433SBen Skeggs nv04_chan_userd = { 158fbe9f433SBen Skeggs .bar = 0, 159fbe9f433SBen Skeggs .base = 0x800000, 160fbe9f433SBen Skeggs .size = 0x010000, 161fbe9f433SBen Skeggs }; 162fbe9f433SBen Skeggs 163d3e7a439SBen Skeggs const struct nvkm_chan_func_inst 164d3e7a439SBen Skeggs nv04_chan_inst = { 165d3e7a439SBen Skeggs .size = 0x1000, 166d3e7a439SBen Skeggs }; 167d3e7a439SBen Skeggs 168f5e45689SBen Skeggs static const struct nvkm_chan_func 169f5e45689SBen Skeggs nv04_chan = { 170d3e7a439SBen Skeggs .inst = &nv04_chan_inst, 171fbe9f433SBen Skeggs .userd = &nv04_chan_userd, 172*3647c53bSBen Skeggs .ramfc = &nv04_chan_ramfc, 17367059b9fSBen Skeggs .start = nv04_chan_start, 17467059b9fSBen Skeggs .stop = nv04_chan_stop, 175f5e45689SBen Skeggs }; 176f5e45689SBen Skeggs 177f5e45689SBen Skeggs const struct nvkm_cgrp_func 178f5e45689SBen Skeggs nv04_cgrp = { 179f5e45689SBen Skeggs }; 180f5e45689SBen Skeggs 181d94470e9SBen Skeggs const struct nvkm_engn_func 182d94470e9SBen Skeggs nv04_engn = { 183d94470e9SBen Skeggs }; 184d94470e9SBen Skeggs 185c39f472eSBen Skeggs void 1863a6bc9c2SBen Skeggs nv04_fifo_pause(struct nvkm_fifo *fifo, unsigned long *pflags) 1873a6bc9c2SBen Skeggs __acquires(fifo->lock) 188c39f472eSBen Skeggs { 1893a6bc9c2SBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 190c39f472eSBen Skeggs unsigned long flags; 191c39f472eSBen Skeggs 1923a6bc9c2SBen Skeggs spin_lock_irqsave(&fifo->lock, flags); 193c39f472eSBen Skeggs *pflags = flags; 194c39f472eSBen Skeggs 19587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 19687744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 197c39f472eSBen Skeggs 198c39f472eSBen Skeggs /* in some cases the puller may be left in an inconsistent state 199c39f472eSBen Skeggs * if you try to stop it while it's busy translating handles. 200c39f472eSBen Skeggs * sometimes you get a CACHE_ERROR, sometimes it just fails 201c39f472eSBen Skeggs * silently; sending incorrect instance offsets to PGRAPH after 202c39f472eSBen Skeggs * it's started up again. 203c39f472eSBen Skeggs * 204c39f472eSBen Skeggs * to avoid this, we invalidate the most recently calculated 205c39f472eSBen Skeggs * instance. 206c39f472eSBen Skeggs */ 207af3082b3SBen Skeggs nvkm_msec(device, 2000, 208af3082b3SBen Skeggs u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0); 209af3082b3SBen Skeggs if (!(tmp & NV04_PFIFO_CACHE1_PULL0_HASH_BUSY)) 210af3082b3SBen Skeggs break; 211af3082b3SBen Skeggs ); 212c39f472eSBen Skeggs 21387744403SBen Skeggs if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 214c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 21587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 216c39f472eSBen Skeggs 21787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 218c39f472eSBen Skeggs } 219c39f472eSBen Skeggs 220c39f472eSBen Skeggs void 2213a6bc9c2SBen Skeggs nv04_fifo_start(struct nvkm_fifo *fifo, unsigned long *pflags) 2223a6bc9c2SBen Skeggs __releases(fifo->lock) 223c39f472eSBen Skeggs { 2243a6bc9c2SBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 225c39f472eSBen Skeggs unsigned long flags = *pflags; 226c39f472eSBen Skeggs 22787744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 22887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 229c39f472eSBen Skeggs 2303a6bc9c2SBen Skeggs spin_unlock_irqrestore(&fifo->lock, flags); 231c39f472eSBen Skeggs } 232c39f472eSBen Skeggs 233d94470e9SBen Skeggs const struct nvkm_runl_func 234d94470e9SBen Skeggs nv04_runl = { 235d94470e9SBen Skeggs }; 236d94470e9SBen Skeggs 23764f7c698SBen Skeggs int 23864f7c698SBen Skeggs nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) 23964f7c698SBen Skeggs { 24064f7c698SBen Skeggs switch (engine->subdev.type) { 24164f7c698SBen Skeggs case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; 24264f7c698SBen Skeggs case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; 24364f7c698SBen Skeggs case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; 24464f7c698SBen Skeggs case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; 24564f7c698SBen Skeggs default: 24664f7c698SBen Skeggs WARN_ON(1); 24764f7c698SBen Skeggs return 0; 24864f7c698SBen Skeggs } 24964f7c698SBen Skeggs } 25064f7c698SBen Skeggs 251c39f472eSBen Skeggs static const char * 252c39f472eSBen Skeggs nv_dma_state_err(u32 state) 253c39f472eSBen Skeggs { 254c39f472eSBen Skeggs static const char * const desc[] = { 255c39f472eSBen Skeggs "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 256c39f472eSBen Skeggs "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 257c39f472eSBen Skeggs }; 258c39f472eSBen Skeggs return desc[(state >> 29) & 0x7]; 259c39f472eSBen Skeggs } 260c39f472eSBen Skeggs 261c39f472eSBen Skeggs static bool 26261570911SBen Skeggs nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) 263c39f472eSBen Skeggs { 26461570911SBen Skeggs struct nvkm_sw *sw = device->sw; 26561570911SBen Skeggs const int subc = (addr & 0x0000e000) >> 13; 26661570911SBen Skeggs const int mthd = (addr & 0x00001ffc); 26761570911SBen Skeggs const u32 mask = 0x0000000f << (subc * 4); 26861570911SBen Skeggs u32 engine = nvkm_rd32(device, 0x003280); 269c39f472eSBen Skeggs bool handled = false; 270c39f472eSBen Skeggs 271c39f472eSBen Skeggs switch (mthd) { 27261570911SBen Skeggs case 0x0000 ... 0x0000: /* subchannel's engine -> software */ 27361570911SBen Skeggs nvkm_wr32(device, 0x003280, (engine &= ~mask)); 274f6e7393eSGustavo A. R. Silva fallthrough; 27561570911SBen Skeggs case 0x0180 ... 0x01fc: /* handle -> instance */ 27661570911SBen Skeggs data = nvkm_rd32(device, 0x003258) & 0x0000ffff; 277f6e7393eSGustavo A. R. Silva fallthrough; 27861570911SBen Skeggs case 0x0100 ... 0x017c: 27961570911SBen Skeggs case 0x0200 ... 0x1ffc: /* pass method down to sw */ 28061570911SBen Skeggs if (!(engine & mask) && sw) 28161570911SBen Skeggs handled = nvkm_sw_mthd(sw, chid, subc, mthd, data); 282c39f472eSBen Skeggs break; 283c39f472eSBen Skeggs default: 284c39f472eSBen Skeggs break; 285c39f472eSBen Skeggs } 286c39f472eSBen Skeggs 287c39f472eSBen Skeggs return handled; 288c39f472eSBen Skeggs } 289c39f472eSBen Skeggs 290c39f472eSBen Skeggs static void 2912fc71a05SBen Skeggs nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) 292c39f472eSBen Skeggs { 2932fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 294e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 295c358f538SBen Skeggs struct nvkm_chan *chan; 2968f0649b5SBen Skeggs unsigned long flags; 29761570911SBen Skeggs u32 pull0 = nvkm_rd32(device, 0x003250); 298c39f472eSBen Skeggs u32 mthd, data; 299c39f472eSBen Skeggs int ptr; 300c39f472eSBen Skeggs 301c39f472eSBen Skeggs /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 302c39f472eSBen Skeggs * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 303c39f472eSBen Skeggs * show that it wraps around to the start at GET=0x800.. No clue as to 304c39f472eSBen Skeggs * why.. 305c39f472eSBen Skeggs */ 306c39f472eSBen Skeggs ptr = (get & 0x7ff) >> 2; 307c39f472eSBen Skeggs 308c39f472eSBen Skeggs if (device->card_type < NV_40) { 30987744403SBen Skeggs mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 31087744403SBen Skeggs data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 311c39f472eSBen Skeggs } else { 31287744403SBen Skeggs mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 31387744403SBen Skeggs data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 314c39f472eSBen Skeggs } 315c39f472eSBen Skeggs 31661570911SBen Skeggs if (!(pull0 & 0x00000100) || 31761570911SBen Skeggs !nv04_fifo_swmthd(device, chid, mthd, data)) { 318c358f538SBen Skeggs chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); 319e5c5e4f5SBen Skeggs nvkm_error(subdev, "CACHE_ERROR - " 320e5c5e4f5SBen Skeggs "ch %d [%s] subc %d mthd %04x data %08x\n", 321c358f538SBen Skeggs chid, chan ? chan->name : "unknown", 3228f0649b5SBen Skeggs (mthd >> 13) & 7, mthd & 0x1ffc, data); 323c358f538SBen Skeggs nvkm_chan_put(&chan, flags); 324c39f472eSBen Skeggs } 325c39f472eSBen Skeggs 32687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 32787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 328c39f472eSBen Skeggs 32987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 33087744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 33187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 33287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 33387744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 33487744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 335c39f472eSBen Skeggs 33687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 33787744403SBen Skeggs nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 33887744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 339c39f472eSBen Skeggs } 340c39f472eSBen Skeggs 341c39f472eSBen Skeggs static void 3422fc71a05SBen Skeggs nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) 343c39f472eSBen Skeggs { 3442fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 345e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 34687744403SBen Skeggs u32 dma_get = nvkm_rd32(device, 0x003244); 34787744403SBen Skeggs u32 dma_put = nvkm_rd32(device, 0x003240); 34887744403SBen Skeggs u32 push = nvkm_rd32(device, 0x003220); 34987744403SBen Skeggs u32 state = nvkm_rd32(device, 0x003228); 350c358f538SBen Skeggs struct nvkm_chan *chan; 3518f0649b5SBen Skeggs unsigned long flags; 3528f0649b5SBen Skeggs const char *name; 353c39f472eSBen Skeggs 354c358f538SBen Skeggs chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); 355c358f538SBen Skeggs name = chan ? chan->name : "unknown"; 356c39f472eSBen Skeggs if (device->card_type == NV_50) { 35787744403SBen Skeggs u32 ho_get = nvkm_rd32(device, 0x003328); 35887744403SBen Skeggs u32 ho_put = nvkm_rd32(device, 0x003320); 35987744403SBen Skeggs u32 ib_get = nvkm_rd32(device, 0x003334); 36087744403SBen Skeggs u32 ib_put = nvkm_rd32(device, 0x003330); 361c39f472eSBen Skeggs 362e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - " 363e5c5e4f5SBen Skeggs "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x " 364e5c5e4f5SBen Skeggs "ib_put %08x state %08x (err: %s) push %08x\n", 3658f0649b5SBen Skeggs chid, name, ho_get, dma_get, ho_put, dma_put, 366e5c5e4f5SBen Skeggs ib_get, ib_put, state, nv_dma_state_err(state), 367e5c5e4f5SBen Skeggs push); 368c39f472eSBen Skeggs 369c39f472eSBen Skeggs /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 37087744403SBen Skeggs nvkm_wr32(device, 0x003364, 0x00000000); 371c39f472eSBen Skeggs if (dma_get != dma_put || ho_get != ho_put) { 37287744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 37387744403SBen Skeggs nvkm_wr32(device, 0x003328, ho_put); 374c39f472eSBen Skeggs } else 375c39f472eSBen Skeggs if (ib_get != ib_put) 37687744403SBen Skeggs nvkm_wr32(device, 0x003334, ib_put); 377c39f472eSBen Skeggs } else { 378e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - ch %d [%s] get %08x put %08x " 379e5c5e4f5SBen Skeggs "state %08x (err: %s) push %08x\n", 3808f0649b5SBen Skeggs chid, name, dma_get, dma_put, state, 381c39f472eSBen Skeggs nv_dma_state_err(state), push); 382c39f472eSBen Skeggs 383c39f472eSBen Skeggs if (dma_get != dma_put) 38487744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 385c39f472eSBen Skeggs } 386c358f538SBen Skeggs nvkm_chan_put(&chan, flags); 387c39f472eSBen Skeggs 38887744403SBen Skeggs nvkm_wr32(device, 0x003228, 0x00000000); 38987744403SBen Skeggs nvkm_wr32(device, 0x003220, 0x00000001); 39087744403SBen Skeggs nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 391c39f472eSBen Skeggs } 392c39f472eSBen Skeggs 3932fc71a05SBen Skeggs irqreturn_t 3942fc71a05SBen Skeggs nv04_fifo_intr(struct nvkm_inth *inth) 395c39f472eSBen Skeggs { 3962fc71a05SBen Skeggs struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); 3972fc71a05SBen Skeggs struct nvkm_subdev *subdev = &fifo->engine.subdev; 398e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 39987744403SBen Skeggs u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 40087744403SBen Skeggs u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 401adc346b1SBen Skeggs u32 reassign, chid, get, sem; 402c39f472eSBen Skeggs 40387744403SBen Skeggs reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 40487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 405c39f472eSBen Skeggs 4062fc71a05SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; 40787744403SBen Skeggs get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 408c39f472eSBen Skeggs 409adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 4102fc71a05SBen Skeggs nv04_fifo_intr_cache_error(fifo, chid, get); 411adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 412c39f472eSBen Skeggs } 413c39f472eSBen Skeggs 414adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 4152fc71a05SBen Skeggs nv04_fifo_intr_dma_pusher(fifo, chid); 416adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 417c39f472eSBen Skeggs } 418c39f472eSBen Skeggs 419adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_SEMAPHORE) { 420adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_SEMAPHORE; 42187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 422c39f472eSBen Skeggs 42387744403SBen Skeggs sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 42487744403SBen Skeggs nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 425c39f472eSBen Skeggs 42687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 42787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 428c39f472eSBen Skeggs } 429c39f472eSBen Skeggs 430c39f472eSBen Skeggs if (device->card_type == NV_50) { 431adc346b1SBen Skeggs if (stat & 0x00000010) { 432adc346b1SBen Skeggs stat &= ~0x00000010; 43387744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x00000010); 434c39f472eSBen Skeggs } 435c39f472eSBen Skeggs 436adc346b1SBen Skeggs if (stat & 0x40000000) { 43787744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x40000000); 438d67f3b96SBen Skeggs nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); 439adc346b1SBen Skeggs stat &= ~0x40000000; 440c39f472eSBen Skeggs } 441c39f472eSBen Skeggs } 442c39f472eSBen Skeggs 443adc346b1SBen Skeggs if (stat) { 444e5c5e4f5SBen Skeggs nvkm_warn(subdev, "intr %08x\n", stat); 44587744403SBen Skeggs nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 44687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 447c39f472eSBen Skeggs } 448c39f472eSBen Skeggs 44987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 4502fc71a05SBen Skeggs return IRQ_HANDLED; 451c39f472eSBen Skeggs } 452c39f472eSBen Skeggs 45313de7f46SBen Skeggs void 454800ac1f8SBen Skeggs nv04_fifo_init(struct nvkm_fifo *fifo) 455c39f472eSBen Skeggs { 456800ac1f8SBen Skeggs struct nvkm_device *device = fifo->engine.subdev.device; 4575b1ab0c2SBen Skeggs struct nvkm_instmem *imem = device->imem; 4585b1ab0c2SBen Skeggs struct nvkm_ramht *ramht = imem->ramht; 4595b1ab0c2SBen Skeggs struct nvkm_memory *ramro = imem->ramro; 4605b1ab0c2SBen Skeggs struct nvkm_memory *ramfc = imem->ramfc; 461c39f472eSBen Skeggs 46287744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 46387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 464c39f472eSBen Skeggs 46587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 4665b1ab0c2SBen Skeggs ((ramht->bits - 9) << 16) | 4671d2a1e53SBen Skeggs (ramht->gpuobj->addr >> 8)); 4685b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); 4695b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); 470c39f472eSBen Skeggs 471800ac1f8SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); 472c39f472eSBen Skeggs 47387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 47487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 475c39f472eSBen Skeggs 47687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 47787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 47887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 47913de7f46SBen Skeggs } 48013de7f46SBen Skeggs 481800ac1f8SBen Skeggs int 482d94470e9SBen Skeggs nv04_fifo_runl_ctor(struct nvkm_fifo *fifo) 483d94470e9SBen Skeggs { 484d94470e9SBen Skeggs struct nvkm_runl *runl; 485d94470e9SBen Skeggs 486d94470e9SBen Skeggs runl = nvkm_runl_new(fifo, 0, 0, 0); 487d94470e9SBen Skeggs if (IS_ERR(runl)) 488d94470e9SBen Skeggs return PTR_ERR(runl); 489d94470e9SBen Skeggs 490d94470e9SBen Skeggs nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); 491d94470e9SBen Skeggs nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); 492d94470e9SBen Skeggs nvkm_runl_add(runl, 1, fifo->func->engn , NVKM_ENGINE_GR, 0); 493d94470e9SBen Skeggs nvkm_runl_add(runl, 2, fifo->func->engn , NVKM_ENGINE_MPEG, 0); /* NV31- */ 494d94470e9SBen Skeggs return 0; 495d94470e9SBen Skeggs } 496d94470e9SBen Skeggs 497d94470e9SBen Skeggs int 498800ac1f8SBen Skeggs nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) 499800ac1f8SBen Skeggs { 500800ac1f8SBen Skeggs /* The last CHID is reserved by HW as a "channel invalid" marker. */ 501800ac1f8SBen Skeggs return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr - 1, &fifo->chid); 502800ac1f8SBen Skeggs } 503800ac1f8SBen Skeggs 5048c18138cSBen Skeggs static int 5058c18138cSBen Skeggs nv04_fifo_chid_nr(struct nvkm_fifo *fifo) 5068c18138cSBen Skeggs { 5078c18138cSBen Skeggs return 16; 5088c18138cSBen Skeggs } 5098c18138cSBen Skeggs 51013de7f46SBen Skeggs int 51113de7f46SBen Skeggs nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 512ab0db2bdSBen Skeggs enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, 51313de7f46SBen Skeggs struct nvkm_fifo **pfifo) 51413de7f46SBen Skeggs { 51513de7f46SBen Skeggs struct nv04_fifo *fifo; 51613de7f46SBen Skeggs int ret; 51713de7f46SBen Skeggs 51813de7f46SBen Skeggs if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 51913de7f46SBen Skeggs return -ENOMEM; 52013de7f46SBen Skeggs *pfifo = &fifo->base; 52113de7f46SBen Skeggs 5228c18138cSBen Skeggs ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); 52313de7f46SBen Skeggs if (ret) 52413de7f46SBen Skeggs return ret; 52513de7f46SBen Skeggs 526c39f472eSBen Skeggs return 0; 527c39f472eSBen Skeggs } 528c39f472eSBen Skeggs 5298f0649b5SBen Skeggs static const struct nvkm_fifo_func 53013de7f46SBen Skeggs nv04_fifo = { 5318c18138cSBen Skeggs .chid_nr = nv04_fifo_chid_nr, 532800ac1f8SBen Skeggs .chid_ctor = nv04_fifo_chid_ctor, 533d94470e9SBen Skeggs .runl_ctor = nv04_fifo_runl_ctor, 53413de7f46SBen Skeggs .init = nv04_fifo_init, 53513de7f46SBen Skeggs .intr = nv04_fifo_intr, 53664f7c698SBen Skeggs .engine_id = nv04_fifo_engine_id, 53713de7f46SBen Skeggs .pause = nv04_fifo_pause, 53813de7f46SBen Skeggs .start = nv04_fifo_start, 539d94470e9SBen Skeggs .runl = &nv04_runl, 540d94470e9SBen Skeggs .engn = &nv04_engn, 541d94470e9SBen Skeggs .engn_sw = &nv04_engn, 542f5e45689SBen Skeggs .cgrp = {{ }, &nv04_cgrp }, 543f5e45689SBen Skeggs .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, 5448f0649b5SBen Skeggs }; 5458f0649b5SBen Skeggs 54613de7f46SBen Skeggs int 547ab0db2bdSBen Skeggs nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 548ab0db2bdSBen Skeggs struct nvkm_fifo **pfifo) 5499a65a38cSBen Skeggs { 550*3647c53bSBen Skeggs return nv04_fifo_new_(&nv04_fifo, device, type, inst, 0, NULL, pfifo); 5519a65a38cSBen Skeggs } 552