1c39f472eSBen Skeggs /* 2c39f472eSBen Skeggs * Copyright 2012 Red Hat Inc. 3c39f472eSBen Skeggs * 4c39f472eSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5c39f472eSBen Skeggs * copy of this software and associated documentation files (the "Software"), 6c39f472eSBen Skeggs * to deal in the Software without restriction, including without limitation 7c39f472eSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8c39f472eSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9c39f472eSBen Skeggs * Software is furnished to do so, subject to the following conditions: 10c39f472eSBen Skeggs * 11c39f472eSBen Skeggs * The above copyright notice and this permission notice shall be included in 12c39f472eSBen Skeggs * all copies or substantial portions of the Software. 13c39f472eSBen Skeggs * 14c39f472eSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15c39f472eSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16c39f472eSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17c39f472eSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18c39f472eSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19c39f472eSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20c39f472eSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21c39f472eSBen Skeggs * 22c39f472eSBen Skeggs * Authors: Ben Skeggs 23c39f472eSBen Skeggs */ 2405c7145dSBen Skeggs #include "nv04.h" 259a65a38cSBen Skeggs #include "channv04.h" 269a65a38cSBen Skeggs #include "regsnv04.h" 27c39f472eSBen Skeggs 288f0649b5SBen Skeggs #include <core/client.h> 29c39f472eSBen Skeggs #include <core/ramht.h> 30d8e83994SBen Skeggs #include <subdev/instmem.h> 31c39f472eSBen Skeggs #include <subdev/timer.h> 3261570911SBen Skeggs #include <engine/sw.h> 33c39f472eSBen Skeggs 3413de7f46SBen Skeggs static const struct nv04_fifo_ramfc 3513de7f46SBen Skeggs nv04_fifo_ramfc[] = { 36c39f472eSBen Skeggs { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 37c39f472eSBen Skeggs { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 38c39f472eSBen Skeggs { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 39c39f472eSBen Skeggs { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 40c39f472eSBen Skeggs { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 41c39f472eSBen Skeggs { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 42c39f472eSBen Skeggs { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 43c39f472eSBen Skeggs { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 44c39f472eSBen Skeggs {} 45c39f472eSBen Skeggs }; 46c39f472eSBen Skeggs 47c39f472eSBen Skeggs void 4813de7f46SBen Skeggs nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) 496189f1b0SBen Skeggs __acquires(fifo->base.lock) 50c39f472eSBen Skeggs { 5113de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 5287744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 53c39f472eSBen Skeggs unsigned long flags; 54c39f472eSBen Skeggs 556189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 56c39f472eSBen Skeggs *pflags = flags; 57c39f472eSBen Skeggs 5887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 5987744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 60c39f472eSBen Skeggs 61c39f472eSBen Skeggs /* in some cases the puller may be left in an inconsistent state 62c39f472eSBen Skeggs * if you try to stop it while it's busy translating handles. 63c39f472eSBen Skeggs * sometimes you get a CACHE_ERROR, sometimes it just fails 64c39f472eSBen Skeggs * silently; sending incorrect instance offsets to PGRAPH after 65c39f472eSBen Skeggs * it's started up again. 66c39f472eSBen Skeggs * 67c39f472eSBen Skeggs * to avoid this, we invalidate the most recently calculated 68c39f472eSBen Skeggs * instance. 69c39f472eSBen Skeggs */ 70af3082b3SBen Skeggs nvkm_msec(device, 2000, 71af3082b3SBen Skeggs u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0); 72af3082b3SBen Skeggs if (!(tmp & NV04_PFIFO_CACHE1_PULL0_HASH_BUSY)) 73af3082b3SBen Skeggs break; 74af3082b3SBen Skeggs ); 75c39f472eSBen Skeggs 7687744403SBen Skeggs if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 77c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 7887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 79c39f472eSBen Skeggs 8087744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 81c39f472eSBen Skeggs } 82c39f472eSBen Skeggs 83c39f472eSBen Skeggs void 8413de7f46SBen Skeggs nv04_fifo_start(struct nvkm_fifo *base, unsigned long *pflags) 856189f1b0SBen Skeggs __releases(fifo->base.lock) 86c39f472eSBen Skeggs { 8713de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 8887744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 89c39f472eSBen Skeggs unsigned long flags = *pflags; 90c39f472eSBen Skeggs 9187744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 9287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 93c39f472eSBen Skeggs 946189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 95c39f472eSBen Skeggs } 96c39f472eSBen Skeggs 97*64f7c698SBen Skeggs int 98*64f7c698SBen Skeggs nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) 99*64f7c698SBen Skeggs { 100*64f7c698SBen Skeggs switch (engine->subdev.type) { 101*64f7c698SBen Skeggs case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; 102*64f7c698SBen Skeggs case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; 103*64f7c698SBen Skeggs case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; 104*64f7c698SBen Skeggs case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; 105*64f7c698SBen Skeggs default: 106*64f7c698SBen Skeggs WARN_ON(1); 107*64f7c698SBen Skeggs return 0; 108*64f7c698SBen Skeggs } 109*64f7c698SBen Skeggs } 110*64f7c698SBen Skeggs 111c39f472eSBen Skeggs static const char * 112c39f472eSBen Skeggs nv_dma_state_err(u32 state) 113c39f472eSBen Skeggs { 114c39f472eSBen Skeggs static const char * const desc[] = { 115c39f472eSBen Skeggs "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 116c39f472eSBen Skeggs "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 117c39f472eSBen Skeggs }; 118c39f472eSBen Skeggs return desc[(state >> 29) & 0x7]; 119c39f472eSBen Skeggs } 120c39f472eSBen Skeggs 121c39f472eSBen Skeggs static bool 12261570911SBen Skeggs nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) 123c39f472eSBen Skeggs { 12461570911SBen Skeggs struct nvkm_sw *sw = device->sw; 12561570911SBen Skeggs const int subc = (addr & 0x0000e000) >> 13; 12661570911SBen Skeggs const int mthd = (addr & 0x00001ffc); 12761570911SBen Skeggs const u32 mask = 0x0000000f << (subc * 4); 12861570911SBen Skeggs u32 engine = nvkm_rd32(device, 0x003280); 129c39f472eSBen Skeggs bool handled = false; 130c39f472eSBen Skeggs 131c39f472eSBen Skeggs switch (mthd) { 13261570911SBen Skeggs case 0x0000 ... 0x0000: /* subchannel's engine -> software */ 13361570911SBen Skeggs nvkm_wr32(device, 0x003280, (engine &= ~mask)); 134f6e7393eSGustavo A. R. Silva fallthrough; 13561570911SBen Skeggs case 0x0180 ... 0x01fc: /* handle -> instance */ 13661570911SBen Skeggs data = nvkm_rd32(device, 0x003258) & 0x0000ffff; 137f6e7393eSGustavo A. R. Silva fallthrough; 13861570911SBen Skeggs case 0x0100 ... 0x017c: 13961570911SBen Skeggs case 0x0200 ... 0x1ffc: /* pass method down to sw */ 14061570911SBen Skeggs if (!(engine & mask) && sw) 14161570911SBen Skeggs handled = nvkm_sw_mthd(sw, chid, subc, mthd, data); 142c39f472eSBen Skeggs break; 143c39f472eSBen Skeggs default: 144c39f472eSBen Skeggs break; 145c39f472eSBen Skeggs } 146c39f472eSBen Skeggs 147c39f472eSBen Skeggs return handled; 148c39f472eSBen Skeggs } 149c39f472eSBen Skeggs 150c39f472eSBen Skeggs static void 151e5c5e4f5SBen Skeggs nv04_fifo_cache_error(struct nv04_fifo *fifo, u32 chid, u32 get) 152c39f472eSBen Skeggs { 153e5c5e4f5SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 154e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 1558f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 1568f0649b5SBen Skeggs unsigned long flags; 15761570911SBen Skeggs u32 pull0 = nvkm_rd32(device, 0x003250); 158c39f472eSBen Skeggs u32 mthd, data; 159c39f472eSBen Skeggs int ptr; 160c39f472eSBen Skeggs 161c39f472eSBen Skeggs /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 162c39f472eSBen Skeggs * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 163c39f472eSBen Skeggs * show that it wraps around to the start at GET=0x800.. No clue as to 164c39f472eSBen Skeggs * why.. 165c39f472eSBen Skeggs */ 166c39f472eSBen Skeggs ptr = (get & 0x7ff) >> 2; 167c39f472eSBen Skeggs 168c39f472eSBen Skeggs if (device->card_type < NV_40) { 16987744403SBen Skeggs mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 17087744403SBen Skeggs data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 171c39f472eSBen Skeggs } else { 17287744403SBen Skeggs mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 17387744403SBen Skeggs data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 174c39f472eSBen Skeggs } 175c39f472eSBen Skeggs 17661570911SBen Skeggs if (!(pull0 & 0x00000100) || 17761570911SBen Skeggs !nv04_fifo_swmthd(device, chid, mthd, data)) { 1788f0649b5SBen Skeggs chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); 179e5c5e4f5SBen Skeggs nvkm_error(subdev, "CACHE_ERROR - " 180e5c5e4f5SBen Skeggs "ch %d [%s] subc %d mthd %04x data %08x\n", 1818f0649b5SBen Skeggs chid, chan ? chan->object.client->name : "unknown", 1828f0649b5SBen Skeggs (mthd >> 13) & 7, mthd & 0x1ffc, data); 1838f0649b5SBen Skeggs nvkm_fifo_chan_put(&fifo->base, flags, &chan); 184c39f472eSBen Skeggs } 185c39f472eSBen Skeggs 18687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 18787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 188c39f472eSBen Skeggs 18987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 19087744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 19187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 19287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 19387744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 19487744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 195c39f472eSBen Skeggs 19687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 19787744403SBen Skeggs nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 19887744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 199c39f472eSBen Skeggs } 200c39f472eSBen Skeggs 201c39f472eSBen Skeggs static void 202e5c5e4f5SBen Skeggs nv04_fifo_dma_pusher(struct nv04_fifo *fifo, u32 chid) 203c39f472eSBen Skeggs { 204e5c5e4f5SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 205e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 20687744403SBen Skeggs u32 dma_get = nvkm_rd32(device, 0x003244); 20787744403SBen Skeggs u32 dma_put = nvkm_rd32(device, 0x003240); 20887744403SBen Skeggs u32 push = nvkm_rd32(device, 0x003220); 20987744403SBen Skeggs u32 state = nvkm_rd32(device, 0x003228); 2108f0649b5SBen Skeggs struct nvkm_fifo_chan *chan; 2118f0649b5SBen Skeggs unsigned long flags; 2128f0649b5SBen Skeggs const char *name; 213c39f472eSBen Skeggs 2148f0649b5SBen Skeggs chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); 2158f0649b5SBen Skeggs name = chan ? chan->object.client->name : "unknown"; 216c39f472eSBen Skeggs if (device->card_type == NV_50) { 21787744403SBen Skeggs u32 ho_get = nvkm_rd32(device, 0x003328); 21887744403SBen Skeggs u32 ho_put = nvkm_rd32(device, 0x003320); 21987744403SBen Skeggs u32 ib_get = nvkm_rd32(device, 0x003334); 22087744403SBen Skeggs u32 ib_put = nvkm_rd32(device, 0x003330); 221c39f472eSBen Skeggs 222e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - " 223e5c5e4f5SBen Skeggs "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x " 224e5c5e4f5SBen Skeggs "ib_put %08x state %08x (err: %s) push %08x\n", 2258f0649b5SBen Skeggs chid, name, ho_get, dma_get, ho_put, dma_put, 226e5c5e4f5SBen Skeggs ib_get, ib_put, state, nv_dma_state_err(state), 227e5c5e4f5SBen Skeggs push); 228c39f472eSBen Skeggs 229c39f472eSBen Skeggs /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 23087744403SBen Skeggs nvkm_wr32(device, 0x003364, 0x00000000); 231c39f472eSBen Skeggs if (dma_get != dma_put || ho_get != ho_put) { 23287744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 23387744403SBen Skeggs nvkm_wr32(device, 0x003328, ho_put); 234c39f472eSBen Skeggs } else 235c39f472eSBen Skeggs if (ib_get != ib_put) 23687744403SBen Skeggs nvkm_wr32(device, 0x003334, ib_put); 237c39f472eSBen Skeggs } else { 238e5c5e4f5SBen Skeggs nvkm_error(subdev, "DMA_PUSHER - ch %d [%s] get %08x put %08x " 239e5c5e4f5SBen Skeggs "state %08x (err: %s) push %08x\n", 2408f0649b5SBen Skeggs chid, name, dma_get, dma_put, state, 241c39f472eSBen Skeggs nv_dma_state_err(state), push); 242c39f472eSBen Skeggs 243c39f472eSBen Skeggs if (dma_get != dma_put) 24487744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 245c39f472eSBen Skeggs } 2468f0649b5SBen Skeggs nvkm_fifo_chan_put(&fifo->base, flags, &chan); 247c39f472eSBen Skeggs 24887744403SBen Skeggs nvkm_wr32(device, 0x003228, 0x00000000); 24987744403SBen Skeggs nvkm_wr32(device, 0x003220, 0x00000001); 25087744403SBen Skeggs nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 251c39f472eSBen Skeggs } 252c39f472eSBen Skeggs 253c39f472eSBen Skeggs void 25413de7f46SBen Skeggs nv04_fifo_intr(struct nvkm_fifo *base) 255c39f472eSBen Skeggs { 25613de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 25713de7f46SBen Skeggs struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 258e5c5e4f5SBen Skeggs struct nvkm_device *device = subdev->device; 25987744403SBen Skeggs u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 26087744403SBen Skeggs u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 261adc346b1SBen Skeggs u32 reassign, chid, get, sem; 262c39f472eSBen Skeggs 26387744403SBen Skeggs reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 26487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 265c39f472eSBen Skeggs 2668f0649b5SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & (fifo->base.nr - 1); 26787744403SBen Skeggs get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 268c39f472eSBen Skeggs 269adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 270e5c5e4f5SBen Skeggs nv04_fifo_cache_error(fifo, chid, get); 271adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 272c39f472eSBen Skeggs } 273c39f472eSBen Skeggs 274adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 275e5c5e4f5SBen Skeggs nv04_fifo_dma_pusher(fifo, chid); 276adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 277c39f472eSBen Skeggs } 278c39f472eSBen Skeggs 279adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_SEMAPHORE) { 280adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_SEMAPHORE; 28187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 282c39f472eSBen Skeggs 28387744403SBen Skeggs sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 28487744403SBen Skeggs nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 285c39f472eSBen Skeggs 28687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 28787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 288c39f472eSBen Skeggs } 289c39f472eSBen Skeggs 290c39f472eSBen Skeggs if (device->card_type == NV_50) { 291adc346b1SBen Skeggs if (stat & 0x00000010) { 292adc346b1SBen Skeggs stat &= ~0x00000010; 29387744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x00000010); 294c39f472eSBen Skeggs } 295c39f472eSBen Skeggs 296adc346b1SBen Skeggs if (stat & 0x40000000) { 29787744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x40000000); 2986189f1b0SBen Skeggs nvkm_fifo_uevent(&fifo->base); 299adc346b1SBen Skeggs stat &= ~0x40000000; 300c39f472eSBen Skeggs } 301c39f472eSBen Skeggs } 302c39f472eSBen Skeggs 303adc346b1SBen Skeggs if (stat) { 304e5c5e4f5SBen Skeggs nvkm_warn(subdev, "intr %08x\n", stat); 30587744403SBen Skeggs nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 30687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 307c39f472eSBen Skeggs } 308c39f472eSBen Skeggs 30987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 310c39f472eSBen Skeggs } 311c39f472eSBen Skeggs 31213de7f46SBen Skeggs void 31313de7f46SBen Skeggs nv04_fifo_init(struct nvkm_fifo *base) 314c39f472eSBen Skeggs { 31513de7f46SBen Skeggs struct nv04_fifo *fifo = nv04_fifo(base); 31687744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 3175b1ab0c2SBen Skeggs struct nvkm_instmem *imem = device->imem; 3185b1ab0c2SBen Skeggs struct nvkm_ramht *ramht = imem->ramht; 3195b1ab0c2SBen Skeggs struct nvkm_memory *ramro = imem->ramro; 3205b1ab0c2SBen Skeggs struct nvkm_memory *ramfc = imem->ramfc; 321c39f472eSBen Skeggs 32287744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 32387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 324c39f472eSBen Skeggs 32587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 3265b1ab0c2SBen Skeggs ((ramht->bits - 9) << 16) | 3271d2a1e53SBen Skeggs (ramht->gpuobj->addr >> 8)); 3285b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); 3295b1ab0c2SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); 330c39f472eSBen Skeggs 3318f0649b5SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.nr - 1); 332c39f472eSBen Skeggs 33387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 33487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 335c39f472eSBen Skeggs 33687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 33787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 33887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 33913de7f46SBen Skeggs } 34013de7f46SBen Skeggs 34113de7f46SBen Skeggs int 34213de7f46SBen Skeggs nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 343ab0db2bdSBen Skeggs enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, 34413de7f46SBen Skeggs struct nvkm_fifo **pfifo) 34513de7f46SBen Skeggs { 34613de7f46SBen Skeggs struct nv04_fifo *fifo; 34713de7f46SBen Skeggs int ret; 34813de7f46SBen Skeggs 34913de7f46SBen Skeggs if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 35013de7f46SBen Skeggs return -ENOMEM; 35113de7f46SBen Skeggs fifo->ramfc = ramfc; 35213de7f46SBen Skeggs *pfifo = &fifo->base; 35313de7f46SBen Skeggs 354ab0db2bdSBen Skeggs ret = nvkm_fifo_ctor(func, device, type, inst, nr, &fifo->base); 35513de7f46SBen Skeggs if (ret) 35613de7f46SBen Skeggs return ret; 35713de7f46SBen Skeggs 35813de7f46SBen Skeggs set_bit(nr - 1, fifo->base.mask); /* inactive channel */ 359c39f472eSBen Skeggs return 0; 360c39f472eSBen Skeggs } 361c39f472eSBen Skeggs 3628f0649b5SBen Skeggs static const struct nvkm_fifo_func 36313de7f46SBen Skeggs nv04_fifo = { 36413de7f46SBen Skeggs .init = nv04_fifo_init, 36513de7f46SBen Skeggs .intr = nv04_fifo_intr, 366*64f7c698SBen Skeggs .engine_id = nv04_fifo_engine_id, 36713de7f46SBen Skeggs .pause = nv04_fifo_pause, 36813de7f46SBen Skeggs .start = nv04_fifo_start, 3698f0649b5SBen Skeggs .chan = { 3708f0649b5SBen Skeggs &nv04_fifo_dma_oclass, 3718f0649b5SBen Skeggs NULL 3728f0649b5SBen Skeggs }, 3738f0649b5SBen Skeggs }; 3748f0649b5SBen Skeggs 37513de7f46SBen Skeggs int 376ab0db2bdSBen Skeggs nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 377ab0db2bdSBen Skeggs struct nvkm_fifo **pfifo) 3789a65a38cSBen Skeggs { 379ab0db2bdSBen Skeggs return nv04_fifo_new_(&nv04_fifo, device, type, inst, 16, nv04_fifo_ramfc, pfifo); 3809a65a38cSBen Skeggs } 381