1c39f472eSBen Skeggs /* 2c39f472eSBen Skeggs * Copyright 2012 Red Hat Inc. 3c39f472eSBen Skeggs * 4c39f472eSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5c39f472eSBen Skeggs * copy of this software and associated documentation files (the "Software"), 6c39f472eSBen Skeggs * to deal in the Software without restriction, including without limitation 7c39f472eSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8c39f472eSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9c39f472eSBen Skeggs * Software is furnished to do so, subject to the following conditions: 10c39f472eSBen Skeggs * 11c39f472eSBen Skeggs * The above copyright notice and this permission notice shall be included in 12c39f472eSBen Skeggs * all copies or substantial portions of the Software. 13c39f472eSBen Skeggs * 14c39f472eSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15c39f472eSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16c39f472eSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17c39f472eSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18c39f472eSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19c39f472eSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20c39f472eSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21c39f472eSBen Skeggs * 22c39f472eSBen Skeggs * Authors: Ben Skeggs 23c39f472eSBen Skeggs */ 2405c7145dSBen Skeggs #include "nv04.h" 25c39f472eSBen Skeggs 26c39f472eSBen Skeggs #include <core/client.h> 27c39f472eSBen Skeggs #include <core/engctx.h> 28c39f472eSBen Skeggs #include <core/handle.h> 29c39f472eSBen Skeggs #include <core/ramht.h> 30c39f472eSBen Skeggs #include <subdev/instmem/nv04.h> 31c39f472eSBen Skeggs #include <subdev/timer.h> 32c39f472eSBen Skeggs 3305c7145dSBen Skeggs #include <nvif/class.h> 3405c7145dSBen Skeggs #include <nvif/unpack.h> 35c39f472eSBen Skeggs 36c39f472eSBen Skeggs static struct ramfc_desc 37c39f472eSBen Skeggs nv04_ramfc[] = { 38c39f472eSBen Skeggs { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, 39c39f472eSBen Skeggs { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, 40c39f472eSBen Skeggs { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, 41c39f472eSBen Skeggs { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, 42c39f472eSBen Skeggs { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, 43c39f472eSBen Skeggs { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, 44c39f472eSBen Skeggs { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, 45c39f472eSBen Skeggs { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, 46c39f472eSBen Skeggs {} 47c39f472eSBen Skeggs }; 48c39f472eSBen Skeggs 49c39f472eSBen Skeggs /******************************************************************************* 50c39f472eSBen Skeggs * FIFO channel objects 51c39f472eSBen Skeggs ******************************************************************************/ 52c39f472eSBen Skeggs 53c39f472eSBen Skeggs int 5405c7145dSBen Skeggs nv04_fifo_object_attach(struct nvkm_object *parent, 5505c7145dSBen Skeggs struct nvkm_object *object, u32 handle) 56c39f472eSBen Skeggs { 576189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)parent->engine; 58c39f472eSBen Skeggs struct nv04_fifo_chan *chan = (void *)parent; 59c39f472eSBen Skeggs u32 context, chid = chan->base.chid; 60c39f472eSBen Skeggs int ret; 61c39f472eSBen Skeggs 62c39f472eSBen Skeggs if (nv_iclass(object, NV_GPUOBJ_CLASS)) 63c39f472eSBen Skeggs context = nv_gpuobj(object)->addr >> 4; 64c39f472eSBen Skeggs else 65c39f472eSBen Skeggs context = 0x00000004; /* just non-zero */ 66c39f472eSBen Skeggs 67c39f472eSBen Skeggs switch (nv_engidx(object->engine)) { 68c39f472eSBen Skeggs case NVDEV_ENGINE_DMAOBJ: 69c39f472eSBen Skeggs case NVDEV_ENGINE_SW: 70c39f472eSBen Skeggs context |= 0x00000000; 71c39f472eSBen Skeggs break; 72c39f472eSBen Skeggs case NVDEV_ENGINE_GR: 73c39f472eSBen Skeggs context |= 0x00010000; 74c39f472eSBen Skeggs break; 75c39f472eSBen Skeggs case NVDEV_ENGINE_MPEG: 76c39f472eSBen Skeggs context |= 0x00020000; 77c39f472eSBen Skeggs break; 78c39f472eSBen Skeggs default: 79c39f472eSBen Skeggs return -EINVAL; 80c39f472eSBen Skeggs } 81c39f472eSBen Skeggs 82c39f472eSBen Skeggs context |= 0x80000000; /* valid */ 83c39f472eSBen Skeggs context |= chid << 24; 84c39f472eSBen Skeggs 856189f1b0SBen Skeggs mutex_lock(&nv_subdev(fifo)->mutex); 866189f1b0SBen Skeggs ret = nvkm_ramht_insert(fifo->ramht, chid, handle, context); 876189f1b0SBen Skeggs mutex_unlock(&nv_subdev(fifo)->mutex); 88c39f472eSBen Skeggs return ret; 89c39f472eSBen Skeggs } 90c39f472eSBen Skeggs 91c39f472eSBen Skeggs void 9205c7145dSBen Skeggs nv04_fifo_object_detach(struct nvkm_object *parent, int cookie) 93c39f472eSBen Skeggs { 946189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)parent->engine; 956189f1b0SBen Skeggs mutex_lock(&nv_subdev(fifo)->mutex); 966189f1b0SBen Skeggs nvkm_ramht_remove(fifo->ramht, cookie); 976189f1b0SBen Skeggs mutex_unlock(&nv_subdev(fifo)->mutex); 98c39f472eSBen Skeggs } 99c39f472eSBen Skeggs 100c39f472eSBen Skeggs int 10105c7145dSBen Skeggs nv04_fifo_context_attach(struct nvkm_object *parent, 10205c7145dSBen Skeggs struct nvkm_object *object) 103c39f472eSBen Skeggs { 10405c7145dSBen Skeggs nv_engctx(object)->addr = nvkm_fifo_chan(parent)->chid; 105c39f472eSBen Skeggs return 0; 106c39f472eSBen Skeggs } 107c39f472eSBen Skeggs 108c39f472eSBen Skeggs static int 10905c7145dSBen Skeggs nv04_fifo_chan_ctor(struct nvkm_object *parent, 11005c7145dSBen Skeggs struct nvkm_object *engine, 11105c7145dSBen Skeggs struct nvkm_oclass *oclass, void *data, u32 size, 11205c7145dSBen Skeggs struct nvkm_object **pobject) 113c39f472eSBen Skeggs { 114c39f472eSBen Skeggs union { 115c39f472eSBen Skeggs struct nv03_channel_dma_v0 v0; 116c39f472eSBen Skeggs } *args = data; 1176189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)engine; 118c39f472eSBen Skeggs struct nv04_fifo_chan *chan; 119c39f472eSBen Skeggs int ret; 120c39f472eSBen Skeggs 121c39f472eSBen Skeggs nv_ioctl(parent, "create channel dma size %d\n", size); 122c39f472eSBen Skeggs if (nvif_unpack(args->v0, 0, 0, false)) { 123c39f472eSBen Skeggs nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " 124c39f472eSBen Skeggs "offset %016llx\n", args->v0.version, 125c39f472eSBen Skeggs args->v0.pushbuf, args->v0.offset); 126c39f472eSBen Skeggs } else 127c39f472eSBen Skeggs return ret; 128c39f472eSBen Skeggs 12905c7145dSBen Skeggs ret = nvkm_fifo_channel_create(parent, engine, oclass, 0, 0x800000, 130c39f472eSBen Skeggs 0x10000, args->v0.pushbuf, 131c39f472eSBen Skeggs (1ULL << NVDEV_ENGINE_DMAOBJ) | 132c39f472eSBen Skeggs (1ULL << NVDEV_ENGINE_SW) | 133c39f472eSBen Skeggs (1ULL << NVDEV_ENGINE_GR), &chan); 134c39f472eSBen Skeggs *pobject = nv_object(chan); 135c39f472eSBen Skeggs if (ret) 136c39f472eSBen Skeggs return ret; 137c39f472eSBen Skeggs 138c39f472eSBen Skeggs args->v0.chid = chan->base.chid; 139c39f472eSBen Skeggs 140c39f472eSBen Skeggs nv_parent(chan)->object_attach = nv04_fifo_object_attach; 141c39f472eSBen Skeggs nv_parent(chan)->object_detach = nv04_fifo_object_detach; 142c39f472eSBen Skeggs nv_parent(chan)->context_attach = nv04_fifo_context_attach; 143c39f472eSBen Skeggs chan->ramfc = chan->base.chid * 32; 144c39f472eSBen Skeggs 1456189f1b0SBen Skeggs nv_wo32(fifo->ramfc, chan->ramfc + 0x00, args->v0.offset); 1466189f1b0SBen Skeggs nv_wo32(fifo->ramfc, chan->ramfc + 0x04, args->v0.offset); 1476189f1b0SBen Skeggs nv_wo32(fifo->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4); 1486189f1b0SBen Skeggs nv_wo32(fifo->ramfc, chan->ramfc + 0x10, 149c39f472eSBen Skeggs NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | 150c39f472eSBen Skeggs NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | 151c39f472eSBen Skeggs #ifdef __BIG_ENDIAN 152c39f472eSBen Skeggs NV_PFIFO_CACHE1_BIG_ENDIAN | 153c39f472eSBen Skeggs #endif 154c39f472eSBen Skeggs NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); 155c39f472eSBen Skeggs return 0; 156c39f472eSBen Skeggs } 157c39f472eSBen Skeggs 158c39f472eSBen Skeggs void 15905c7145dSBen Skeggs nv04_fifo_chan_dtor(struct nvkm_object *object) 160c39f472eSBen Skeggs { 1616189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)object->engine; 162c39f472eSBen Skeggs struct nv04_fifo_chan *chan = (void *)object; 1636189f1b0SBen Skeggs struct ramfc_desc *c = fifo->ramfc_desc; 164c39f472eSBen Skeggs 165c39f472eSBen Skeggs do { 1666189f1b0SBen Skeggs nv_wo32(fifo->ramfc, chan->ramfc + c->ctxp, 0x00000000); 167c39f472eSBen Skeggs } while ((++c)->bits); 168c39f472eSBen Skeggs 16905c7145dSBen Skeggs nvkm_fifo_channel_destroy(&chan->base); 170c39f472eSBen Skeggs } 171c39f472eSBen Skeggs 172c39f472eSBen Skeggs int 17305c7145dSBen Skeggs nv04_fifo_chan_init(struct nvkm_object *object) 174c39f472eSBen Skeggs { 1756189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)object->engine; 176c39f472eSBen Skeggs struct nv04_fifo_chan *chan = (void *)object; 17787744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 178c39f472eSBen Skeggs u32 mask = 1 << chan->base.chid; 179c39f472eSBen Skeggs unsigned long flags; 180c39f472eSBen Skeggs int ret; 181c39f472eSBen Skeggs 18205c7145dSBen Skeggs ret = nvkm_fifo_channel_init(&chan->base); 183c39f472eSBen Skeggs if (ret) 184c39f472eSBen Skeggs return ret; 185c39f472eSBen Skeggs 1866189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 18787744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); 1886189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 189c39f472eSBen Skeggs return 0; 190c39f472eSBen Skeggs } 191c39f472eSBen Skeggs 192c39f472eSBen Skeggs int 19305c7145dSBen Skeggs nv04_fifo_chan_fini(struct nvkm_object *object, bool suspend) 194c39f472eSBen Skeggs { 1956189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)object->engine; 196c39f472eSBen Skeggs struct nv04_fifo_chan *chan = (void *)object; 1976189f1b0SBen Skeggs struct nvkm_gpuobj *fctx = fifo->ramfc; 19887744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 199c39f472eSBen Skeggs struct ramfc_desc *c; 200c39f472eSBen Skeggs unsigned long flags; 201c39f472eSBen Skeggs u32 data = chan->ramfc; 202c39f472eSBen Skeggs u32 chid; 203c39f472eSBen Skeggs 204c39f472eSBen Skeggs /* prevent fifo context switches */ 2056189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 20687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 207c39f472eSBen Skeggs 208c39f472eSBen Skeggs /* if this channel is active, replace it with a null context */ 20987744403SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.max; 210c39f472eSBen Skeggs if (chid == chan->base.chid) { 21187744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); 21287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); 21387744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); 214c39f472eSBen Skeggs 2156189f1b0SBen Skeggs c = fifo->ramfc_desc; 216c39f472eSBen Skeggs do { 217c39f472eSBen Skeggs u32 rm = ((1ULL << c->bits) - 1) << c->regs; 218c39f472eSBen Skeggs u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; 21987744403SBen Skeggs u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; 220c39f472eSBen Skeggs u32 cv = (nv_ro32(fctx, c->ctxp + data) & ~cm); 221c39f472eSBen Skeggs nv_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); 222c39f472eSBen Skeggs } while ((++c)->bits); 223c39f472eSBen Skeggs 2246189f1b0SBen Skeggs c = fifo->ramfc_desc; 225c39f472eSBen Skeggs do { 22687744403SBen Skeggs nvkm_wr32(device, c->regp, 0x00000000); 227c39f472eSBen Skeggs } while ((++c)->bits); 228c39f472eSBen Skeggs 22987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); 23087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); 23187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.max); 23287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 23387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 234c39f472eSBen Skeggs } 235c39f472eSBen Skeggs 236c39f472eSBen Skeggs /* restore normal operation, after disabling dma mode */ 23787744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); 23887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 2396189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 240c39f472eSBen Skeggs 24105c7145dSBen Skeggs return nvkm_fifo_channel_fini(&chan->base, suspend); 242c39f472eSBen Skeggs } 243c39f472eSBen Skeggs 24405c7145dSBen Skeggs static struct nvkm_ofuncs 245c39f472eSBen Skeggs nv04_fifo_ofuncs = { 246c39f472eSBen Skeggs .ctor = nv04_fifo_chan_ctor, 247c39f472eSBen Skeggs .dtor = nv04_fifo_chan_dtor, 248c39f472eSBen Skeggs .init = nv04_fifo_chan_init, 249c39f472eSBen Skeggs .fini = nv04_fifo_chan_fini, 25005c7145dSBen Skeggs .map = _nvkm_fifo_channel_map, 25105c7145dSBen Skeggs .rd32 = _nvkm_fifo_channel_rd32, 25205c7145dSBen Skeggs .wr32 = _nvkm_fifo_channel_wr32, 25305c7145dSBen Skeggs .ntfy = _nvkm_fifo_channel_ntfy 254c39f472eSBen Skeggs }; 255c39f472eSBen Skeggs 25605c7145dSBen Skeggs static struct nvkm_oclass 257c39f472eSBen Skeggs nv04_fifo_sclass[] = { 258c39f472eSBen Skeggs { NV03_CHANNEL_DMA, &nv04_fifo_ofuncs }, 259c39f472eSBen Skeggs {} 260c39f472eSBen Skeggs }; 261c39f472eSBen Skeggs 262c39f472eSBen Skeggs /******************************************************************************* 263c39f472eSBen Skeggs * FIFO context - basically just the instmem reserved for the channel 264c39f472eSBen Skeggs ******************************************************************************/ 265c39f472eSBen Skeggs 266c39f472eSBen Skeggs int 26705c7145dSBen Skeggs nv04_fifo_context_ctor(struct nvkm_object *parent, 26805c7145dSBen Skeggs struct nvkm_object *engine, 26905c7145dSBen Skeggs struct nvkm_oclass *oclass, void *data, u32 size, 27005c7145dSBen Skeggs struct nvkm_object **pobject) 271c39f472eSBen Skeggs { 272c39f472eSBen Skeggs struct nv04_fifo_base *base; 273c39f472eSBen Skeggs int ret; 274c39f472eSBen Skeggs 27505c7145dSBen Skeggs ret = nvkm_fifo_context_create(parent, engine, oclass, NULL, 0x1000, 276c39f472eSBen Skeggs 0x1000, NVOBJ_FLAG_HEAP, &base); 277c39f472eSBen Skeggs *pobject = nv_object(base); 278c39f472eSBen Skeggs if (ret) 279c39f472eSBen Skeggs return ret; 280c39f472eSBen Skeggs 281c39f472eSBen Skeggs return 0; 282c39f472eSBen Skeggs } 283c39f472eSBen Skeggs 28405c7145dSBen Skeggs static struct nvkm_oclass 285c39f472eSBen Skeggs nv04_fifo_cclass = { 286c39f472eSBen Skeggs .handle = NV_ENGCTX(FIFO, 0x04), 28705c7145dSBen Skeggs .ofuncs = &(struct nvkm_ofuncs) { 288c39f472eSBen Skeggs .ctor = nv04_fifo_context_ctor, 28905c7145dSBen Skeggs .dtor = _nvkm_fifo_context_dtor, 29005c7145dSBen Skeggs .init = _nvkm_fifo_context_init, 29105c7145dSBen Skeggs .fini = _nvkm_fifo_context_fini, 29205c7145dSBen Skeggs .rd32 = _nvkm_fifo_context_rd32, 29305c7145dSBen Skeggs .wr32 = _nvkm_fifo_context_wr32, 294c39f472eSBen Skeggs }, 295c39f472eSBen Skeggs }; 296c39f472eSBen Skeggs 297c39f472eSBen Skeggs /******************************************************************************* 298c39f472eSBen Skeggs * PFIFO engine 299c39f472eSBen Skeggs ******************************************************************************/ 300c39f472eSBen Skeggs 301c39f472eSBen Skeggs void 3026189f1b0SBen Skeggs nv04_fifo_pause(struct nvkm_fifo *obj, unsigned long *pflags) 3036189f1b0SBen Skeggs __acquires(fifo->base.lock) 304c39f472eSBen Skeggs { 3056189f1b0SBen Skeggs struct nv04_fifo *fifo = container_of(obj, typeof(*fifo), base); 30687744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 307c39f472eSBen Skeggs unsigned long flags; 308c39f472eSBen Skeggs 3096189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 310c39f472eSBen Skeggs *pflags = flags; 311c39f472eSBen Skeggs 31287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); 31387744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); 314c39f472eSBen Skeggs 315c39f472eSBen Skeggs /* in some cases the puller may be left in an inconsistent state 316c39f472eSBen Skeggs * if you try to stop it while it's busy translating handles. 317c39f472eSBen Skeggs * sometimes you get a CACHE_ERROR, sometimes it just fails 318c39f472eSBen Skeggs * silently; sending incorrect instance offsets to PGRAPH after 319c39f472eSBen Skeggs * it's started up again. 320c39f472eSBen Skeggs * 321c39f472eSBen Skeggs * to avoid this, we invalidate the most recently calculated 322c39f472eSBen Skeggs * instance. 323c39f472eSBen Skeggs */ 3246189f1b0SBen Skeggs if (!nv_wait(fifo, NV04_PFIFO_CACHE1_PULL0, 325c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0x00000000)) 3266189f1b0SBen Skeggs nv_warn(fifo, "timeout idling puller\n"); 327c39f472eSBen Skeggs 32887744403SBen Skeggs if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) & 329c39f472eSBen Skeggs NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) 33087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 331c39f472eSBen Skeggs 33287744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000); 333c39f472eSBen Skeggs } 334c39f472eSBen Skeggs 335c39f472eSBen Skeggs void 3366189f1b0SBen Skeggs nv04_fifo_start(struct nvkm_fifo *obj, unsigned long *pflags) 3376189f1b0SBen Skeggs __releases(fifo->base.lock) 338c39f472eSBen Skeggs { 3396189f1b0SBen Skeggs struct nv04_fifo *fifo = container_of(obj, typeof(*fifo), base); 34087744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 341c39f472eSBen Skeggs unsigned long flags = *pflags; 342c39f472eSBen Skeggs 34387744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); 34487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); 345c39f472eSBen Skeggs 3466189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 347c39f472eSBen Skeggs } 348c39f472eSBen Skeggs 349c39f472eSBen Skeggs static const char * 350c39f472eSBen Skeggs nv_dma_state_err(u32 state) 351c39f472eSBen Skeggs { 352c39f472eSBen Skeggs static const char * const desc[] = { 353c39f472eSBen Skeggs "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", 354c39f472eSBen Skeggs "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" 355c39f472eSBen Skeggs }; 356c39f472eSBen Skeggs return desc[(state >> 29) & 0x7]; 357c39f472eSBen Skeggs } 358c39f472eSBen Skeggs 359c39f472eSBen Skeggs static bool 3606189f1b0SBen Skeggs nv04_fifo_swmthd(struct nv04_fifo *fifo, u32 chid, u32 addr, u32 data) 361c39f472eSBen Skeggs { 36287744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 363c39f472eSBen Skeggs struct nv04_fifo_chan *chan = NULL; 36405c7145dSBen Skeggs struct nvkm_handle *bind; 365c39f472eSBen Skeggs const int subc = (addr >> 13) & 0x7; 366c39f472eSBen Skeggs const int mthd = addr & 0x1ffc; 367c39f472eSBen Skeggs bool handled = false; 368c39f472eSBen Skeggs unsigned long flags; 369c39f472eSBen Skeggs u32 engine; 370c39f472eSBen Skeggs 3716189f1b0SBen Skeggs spin_lock_irqsave(&fifo->base.lock, flags); 3726189f1b0SBen Skeggs if (likely(chid >= fifo->base.min && chid <= fifo->base.max)) 3736189f1b0SBen Skeggs chan = (void *)fifo->base.channel[chid]; 374c39f472eSBen Skeggs if (unlikely(!chan)) 375c39f472eSBen Skeggs goto out; 376c39f472eSBen Skeggs 377c39f472eSBen Skeggs switch (mthd) { 378c39f472eSBen Skeggs case 0x0000: 37905c7145dSBen Skeggs bind = nvkm_namedb_get(nv_namedb(chan), data); 380c39f472eSBen Skeggs if (unlikely(!bind)) 381c39f472eSBen Skeggs break; 382c39f472eSBen Skeggs 383c39f472eSBen Skeggs if (nv_engidx(bind->object->engine) == NVDEV_ENGINE_SW) { 384c39f472eSBen Skeggs engine = 0x0000000f << (subc * 4); 385c39f472eSBen Skeggs chan->subc[subc] = data; 386c39f472eSBen Skeggs handled = true; 387c39f472eSBen Skeggs 38887744403SBen Skeggs nvkm_mask(device, NV04_PFIFO_CACHE1_ENGINE, engine, 0); 389c39f472eSBen Skeggs } 390c39f472eSBen Skeggs 39105c7145dSBen Skeggs nvkm_namedb_put(bind); 392c39f472eSBen Skeggs break; 393c39f472eSBen Skeggs default: 39487744403SBen Skeggs engine = nvkm_rd32(device, NV04_PFIFO_CACHE1_ENGINE); 395c39f472eSBen Skeggs if (unlikely(((engine >> (subc * 4)) & 0xf) != 0)) 396c39f472eSBen Skeggs break; 397c39f472eSBen Skeggs 39805c7145dSBen Skeggs bind = nvkm_namedb_get(nv_namedb(chan), chan->subc[subc]); 399c39f472eSBen Skeggs if (likely(bind)) { 400c39f472eSBen Skeggs if (!nv_call(bind->object, mthd, data)) 401c39f472eSBen Skeggs handled = true; 40205c7145dSBen Skeggs nvkm_namedb_put(bind); 403c39f472eSBen Skeggs } 404c39f472eSBen Skeggs break; 405c39f472eSBen Skeggs } 406c39f472eSBen Skeggs 407c39f472eSBen Skeggs out: 4086189f1b0SBen Skeggs spin_unlock_irqrestore(&fifo->base.lock, flags); 409c39f472eSBen Skeggs return handled; 410c39f472eSBen Skeggs } 411c39f472eSBen Skeggs 412c39f472eSBen Skeggs static void 41305c7145dSBen Skeggs nv04_fifo_cache_error(struct nvkm_device *device, 4146189f1b0SBen Skeggs struct nv04_fifo *fifo, u32 chid, u32 get) 415c39f472eSBen Skeggs { 416c39f472eSBen Skeggs u32 mthd, data; 417c39f472eSBen Skeggs int ptr; 418c39f472eSBen Skeggs 419c39f472eSBen Skeggs /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my 420c39f472eSBen Skeggs * G80 chips, but CACHE1 isn't big enough for this much data.. Tests 421c39f472eSBen Skeggs * show that it wraps around to the start at GET=0x800.. No clue as to 422c39f472eSBen Skeggs * why.. 423c39f472eSBen Skeggs */ 424c39f472eSBen Skeggs ptr = (get & 0x7ff) >> 2; 425c39f472eSBen Skeggs 426c39f472eSBen Skeggs if (device->card_type < NV_40) { 42787744403SBen Skeggs mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr)); 42887744403SBen Skeggs data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr)); 429c39f472eSBen Skeggs } else { 43087744403SBen Skeggs mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr)); 43187744403SBen Skeggs data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr)); 432c39f472eSBen Skeggs } 433c39f472eSBen Skeggs 4346189f1b0SBen Skeggs if (!nv04_fifo_swmthd(fifo, chid, mthd, data)) { 435c39f472eSBen Skeggs const char *client_name = 4366189f1b0SBen Skeggs nvkm_client_name_for_fifo_chid(&fifo->base, chid); 4376189f1b0SBen Skeggs nv_error(fifo, 438c39f472eSBen Skeggs "CACHE_ERROR - ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", 439c39f472eSBen Skeggs chid, client_name, (mthd >> 13) & 7, mthd & 0x1ffc, 440c39f472eSBen Skeggs data); 441c39f472eSBen Skeggs } 442c39f472eSBen Skeggs 44387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); 44487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); 445c39f472eSBen Skeggs 44687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 44787744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1); 44887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 44987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 45087744403SBen Skeggs nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1); 45187744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0); 452c39f472eSBen Skeggs 45387744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 45487744403SBen Skeggs nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); 45587744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 456c39f472eSBen Skeggs } 457c39f472eSBen Skeggs 458c39f472eSBen Skeggs static void 45905c7145dSBen Skeggs nv04_fifo_dma_pusher(struct nvkm_device *device, 4606189f1b0SBen Skeggs struct nv04_fifo *fifo, u32 chid) 461c39f472eSBen Skeggs { 462c39f472eSBen Skeggs const char *client_name; 46387744403SBen Skeggs u32 dma_get = nvkm_rd32(device, 0x003244); 46487744403SBen Skeggs u32 dma_put = nvkm_rd32(device, 0x003240); 46587744403SBen Skeggs u32 push = nvkm_rd32(device, 0x003220); 46687744403SBen Skeggs u32 state = nvkm_rd32(device, 0x003228); 467c39f472eSBen Skeggs 4686189f1b0SBen Skeggs client_name = nvkm_client_name_for_fifo_chid(&fifo->base, chid); 469c39f472eSBen Skeggs 470c39f472eSBen Skeggs if (device->card_type == NV_50) { 47187744403SBen Skeggs u32 ho_get = nvkm_rd32(device, 0x003328); 47287744403SBen Skeggs u32 ho_put = nvkm_rd32(device, 0x003320); 47387744403SBen Skeggs u32 ib_get = nvkm_rd32(device, 0x003334); 47487744403SBen Skeggs u32 ib_put = nvkm_rd32(device, 0x003330); 475c39f472eSBen Skeggs 4766189f1b0SBen Skeggs nv_error(fifo, 477c39f472eSBen Skeggs "DMA_PUSHER - ch %d [%s] get 0x%02x%08x put 0x%02x%08x ib_get 0x%08x ib_put 0x%08x state 0x%08x (err: %s) push 0x%08x\n", 478c39f472eSBen Skeggs chid, client_name, ho_get, dma_get, ho_put, dma_put, 479c39f472eSBen Skeggs ib_get, ib_put, state, nv_dma_state_err(state), push); 480c39f472eSBen Skeggs 481c39f472eSBen Skeggs /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ 48287744403SBen Skeggs nvkm_wr32(device, 0x003364, 0x00000000); 483c39f472eSBen Skeggs if (dma_get != dma_put || ho_get != ho_put) { 48487744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 48587744403SBen Skeggs nvkm_wr32(device, 0x003328, ho_put); 486c39f472eSBen Skeggs } else 487c39f472eSBen Skeggs if (ib_get != ib_put) 48887744403SBen Skeggs nvkm_wr32(device, 0x003334, ib_put); 489c39f472eSBen Skeggs } else { 4906189f1b0SBen Skeggs nv_error(fifo, 491c39f472eSBen Skeggs "DMA_PUSHER - ch %d [%s] get 0x%08x put 0x%08x state 0x%08x (err: %s) push 0x%08x\n", 492c39f472eSBen Skeggs chid, client_name, dma_get, dma_put, state, 493c39f472eSBen Skeggs nv_dma_state_err(state), push); 494c39f472eSBen Skeggs 495c39f472eSBen Skeggs if (dma_get != dma_put) 49687744403SBen Skeggs nvkm_wr32(device, 0x003244, dma_put); 497c39f472eSBen Skeggs } 498c39f472eSBen Skeggs 49987744403SBen Skeggs nvkm_wr32(device, 0x003228, 0x00000000); 50087744403SBen Skeggs nvkm_wr32(device, 0x003220, 0x00000001); 50187744403SBen Skeggs nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); 502c39f472eSBen Skeggs } 503c39f472eSBen Skeggs 504c39f472eSBen Skeggs void 50505c7145dSBen Skeggs nv04_fifo_intr(struct nvkm_subdev *subdev) 506c39f472eSBen Skeggs { 50705c7145dSBen Skeggs struct nvkm_device *device = nv_device(subdev); 5086189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)subdev; 50987744403SBen Skeggs u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); 51087744403SBen Skeggs u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; 511adc346b1SBen Skeggs u32 reassign, chid, get, sem; 512c39f472eSBen Skeggs 51387744403SBen Skeggs reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; 51487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 0); 515c39f472eSBen Skeggs 51687744403SBen Skeggs chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.max; 51787744403SBen Skeggs get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); 518c39f472eSBen Skeggs 519adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_CACHE_ERROR) { 5206189f1b0SBen Skeggs nv04_fifo_cache_error(device, fifo, chid, get); 521adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_CACHE_ERROR; 522c39f472eSBen Skeggs } 523c39f472eSBen Skeggs 524adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_DMA_PUSHER) { 5256189f1b0SBen Skeggs nv04_fifo_dma_pusher(device, fifo, chid); 526adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_DMA_PUSHER; 527c39f472eSBen Skeggs } 528c39f472eSBen Skeggs 529adc346b1SBen Skeggs if (stat & NV_PFIFO_INTR_SEMAPHORE) { 530adc346b1SBen Skeggs stat &= ~NV_PFIFO_INTR_SEMAPHORE; 53187744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE); 532c39f472eSBen Skeggs 53387744403SBen Skeggs sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE); 53487744403SBen Skeggs nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); 535c39f472eSBen Skeggs 53687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4); 53787744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 538c39f472eSBen Skeggs } 539c39f472eSBen Skeggs 540c39f472eSBen Skeggs if (device->card_type == NV_50) { 541adc346b1SBen Skeggs if (stat & 0x00000010) { 542adc346b1SBen Skeggs stat &= ~0x00000010; 54387744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x00000010); 544c39f472eSBen Skeggs } 545c39f472eSBen Skeggs 546adc346b1SBen Skeggs if (stat & 0x40000000) { 54787744403SBen Skeggs nvkm_wr32(device, 0x002100, 0x40000000); 5486189f1b0SBen Skeggs nvkm_fifo_uevent(&fifo->base); 549adc346b1SBen Skeggs stat &= ~0x40000000; 550c39f472eSBen Skeggs } 551c39f472eSBen Skeggs } 552c39f472eSBen Skeggs 553adc346b1SBen Skeggs if (stat) { 5546189f1b0SBen Skeggs nv_warn(fifo, "unknown intr 0x%08x\n", stat); 55587744403SBen Skeggs nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000); 55687744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, stat); 557c39f472eSBen Skeggs } 558c39f472eSBen Skeggs 55987744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); 560c39f472eSBen Skeggs } 561c39f472eSBen Skeggs 562c39f472eSBen Skeggs static int 56305c7145dSBen Skeggs nv04_fifo_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 56405c7145dSBen Skeggs struct nvkm_oclass *oclass, void *data, u32 size, 56505c7145dSBen Skeggs struct nvkm_object **pobject) 566c39f472eSBen Skeggs { 567c44c06aeSBen Skeggs struct nv04_instmem *imem = nv04_instmem(parent); 5686189f1b0SBen Skeggs struct nv04_fifo *fifo; 569c39f472eSBen Skeggs int ret; 570c39f472eSBen Skeggs 5716189f1b0SBen Skeggs ret = nvkm_fifo_create(parent, engine, oclass, 0, 15, &fifo); 5726189f1b0SBen Skeggs *pobject = nv_object(fifo); 573c39f472eSBen Skeggs if (ret) 574c39f472eSBen Skeggs return ret; 575c39f472eSBen Skeggs 5766189f1b0SBen Skeggs nvkm_ramht_ref(imem->ramht, &fifo->ramht); 5776189f1b0SBen Skeggs nvkm_gpuobj_ref(imem->ramro, &fifo->ramro); 5786189f1b0SBen Skeggs nvkm_gpuobj_ref(imem->ramfc, &fifo->ramfc); 579c39f472eSBen Skeggs 5806189f1b0SBen Skeggs nv_subdev(fifo)->unit = 0x00000100; 5816189f1b0SBen Skeggs nv_subdev(fifo)->intr = nv04_fifo_intr; 5826189f1b0SBen Skeggs nv_engine(fifo)->cclass = &nv04_fifo_cclass; 5836189f1b0SBen Skeggs nv_engine(fifo)->sclass = nv04_fifo_sclass; 5846189f1b0SBen Skeggs fifo->base.pause = nv04_fifo_pause; 5856189f1b0SBen Skeggs fifo->base.start = nv04_fifo_start; 5866189f1b0SBen Skeggs fifo->ramfc_desc = nv04_ramfc; 587c39f472eSBen Skeggs return 0; 588c39f472eSBen Skeggs } 589c39f472eSBen Skeggs 590c39f472eSBen Skeggs void 59105c7145dSBen Skeggs nv04_fifo_dtor(struct nvkm_object *object) 592c39f472eSBen Skeggs { 5936189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)object; 5946189f1b0SBen Skeggs nvkm_gpuobj_ref(NULL, &fifo->ramfc); 5956189f1b0SBen Skeggs nvkm_gpuobj_ref(NULL, &fifo->ramro); 5966189f1b0SBen Skeggs nvkm_ramht_ref(NULL, &fifo->ramht); 5976189f1b0SBen Skeggs nvkm_fifo_destroy(&fifo->base); 598c39f472eSBen Skeggs } 599c39f472eSBen Skeggs 600c39f472eSBen Skeggs int 60105c7145dSBen Skeggs nv04_fifo_init(struct nvkm_object *object) 602c39f472eSBen Skeggs { 6036189f1b0SBen Skeggs struct nv04_fifo *fifo = (void *)object; 60487744403SBen Skeggs struct nvkm_device *device = fifo->base.engine.subdev.device; 605c39f472eSBen Skeggs int ret; 606c39f472eSBen Skeggs 6076189f1b0SBen Skeggs ret = nvkm_fifo_init(&fifo->base); 608c39f472eSBen Skeggs if (ret) 609c39f472eSBen Skeggs return ret; 610c39f472eSBen Skeggs 61187744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff); 61287744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); 613c39f472eSBen Skeggs 61487744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | 6156189f1b0SBen Skeggs ((fifo->ramht->bits - 9) << 16) | 6166189f1b0SBen Skeggs (fifo->ramht->gpuobj.addr >> 8)); 61787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMRO, fifo->ramro->addr >> 8); 61887744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_RAMFC, fifo->ramfc->addr >> 8); 619c39f472eSBen Skeggs 62087744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.max); 621c39f472eSBen Skeggs 62287744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); 62387744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); 624c39f472eSBen Skeggs 62587744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); 62687744403SBen Skeggs nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); 62787744403SBen Skeggs nvkm_wr32(device, NV03_PFIFO_CACHES, 1); 628c39f472eSBen Skeggs return 0; 629c39f472eSBen Skeggs } 630c39f472eSBen Skeggs 63105c7145dSBen Skeggs struct nvkm_oclass * 63205c7145dSBen Skeggs nv04_fifo_oclass = &(struct nvkm_oclass) { 633c39f472eSBen Skeggs .handle = NV_ENGINE(FIFO, 0x04), 63405c7145dSBen Skeggs .ofuncs = &(struct nvkm_ofuncs) { 635c39f472eSBen Skeggs .ctor = nv04_fifo_ctor, 636c39f472eSBen Skeggs .dtor = nv04_fifo_dtor, 637c39f472eSBen Skeggs .init = nv04_fifo_init, 63805c7145dSBen Skeggs .fini = _nvkm_fifo_fini, 639c39f472eSBen Skeggs }, 640c39f472eSBen Skeggs }; 641