1 /*
2  * Copyright 2012 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 
25 #include <core/client.h>
26 #include <core/device.h>
27 #include <nvif/unpack.h>
28 #include <nvif/class.h>
29 #include <core/engctx.h>
30 #include <core/namedb.h>
31 #include <core/handle.h>
32 #include <core/ramht.h>
33 #include <core/event.h>
34 
35 #include <subdev/instmem.h>
36 #include <subdev/instmem/nv04.h>
37 #include <subdev/timer.h>
38 #include <subdev/fb.h>
39 
40 #include <engine/fifo.h>
41 
42 #include "nv04.h"
43 
44 static struct ramfc_desc
45 nv04_ramfc[] = {
46 	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
47 	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
48 	{ 16,  0, 0x08,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
49 	{ 16, 16, 0x08,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
50 	{ 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_STATE },
51 	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
52 	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_ENGINE },
53 	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_PULL1 },
54 	{}
55 };
56 
57 /*******************************************************************************
58  * FIFO channel objects
59  ******************************************************************************/
60 
61 int
62 nv04_fifo_object_attach(struct nouveau_object *parent,
63 			struct nouveau_object *object, u32 handle)
64 {
65 	struct nv04_fifo_priv *priv = (void *)parent->engine;
66 	struct nv04_fifo_chan *chan = (void *)parent;
67 	u32 context, chid = chan->base.chid;
68 	int ret;
69 
70 	if (nv_iclass(object, NV_GPUOBJ_CLASS))
71 		context = nv_gpuobj(object)->addr >> 4;
72 	else
73 		context = 0x00000004; /* just non-zero */
74 
75 	switch (nv_engidx(object->engine)) {
76 	case NVDEV_ENGINE_DMAOBJ:
77 	case NVDEV_ENGINE_SW:
78 		context |= 0x00000000;
79 		break;
80 	case NVDEV_ENGINE_GR:
81 		context |= 0x00010000;
82 		break;
83 	case NVDEV_ENGINE_MPEG:
84 		context |= 0x00020000;
85 		break;
86 	default:
87 		return -EINVAL;
88 	}
89 
90 	context |= 0x80000000; /* valid */
91 	context |= chid << 24;
92 
93 	mutex_lock(&nv_subdev(priv)->mutex);
94 	ret = nouveau_ramht_insert(priv->ramht, chid, handle, context);
95 	mutex_unlock(&nv_subdev(priv)->mutex);
96 	return ret;
97 }
98 
99 void
100 nv04_fifo_object_detach(struct nouveau_object *parent, int cookie)
101 {
102 	struct nv04_fifo_priv *priv = (void *)parent->engine;
103 	mutex_lock(&nv_subdev(priv)->mutex);
104 	nouveau_ramht_remove(priv->ramht, cookie);
105 	mutex_unlock(&nv_subdev(priv)->mutex);
106 }
107 
108 int
109 nv04_fifo_context_attach(struct nouveau_object *parent,
110 			 struct nouveau_object *object)
111 {
112 	nv_engctx(object)->addr = nouveau_fifo_chan(parent)->chid;
113 	return 0;
114 }
115 
116 static int
117 nv04_fifo_chan_ctor(struct nouveau_object *parent,
118 		    struct nouveau_object *engine,
119 		    struct nouveau_oclass *oclass, void *data, u32 size,
120 		    struct nouveau_object **pobject)
121 {
122 	union {
123 		struct nv03_channel_dma_v0 v0;
124 	} *args = data;
125 	struct nv04_fifo_priv *priv = (void *)engine;
126 	struct nv04_fifo_chan *chan;
127 	int ret;
128 
129 	nv_ioctl(parent, "create channel dma size %d\n", size);
130 	if (nvif_unpack(args->v0, 0, 0, false)) {
131 		nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
132 				 "offset %016llx\n", args->v0.version,
133 			 args->v0.pushbuf, args->v0.offset);
134 	} else
135 		return ret;
136 
137 	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
138 					  0x10000, args->v0.pushbuf,
139 					  (1ULL << NVDEV_ENGINE_DMAOBJ) |
140 					  (1ULL << NVDEV_ENGINE_SW) |
141 					  (1ULL << NVDEV_ENGINE_GR), &chan);
142 	*pobject = nv_object(chan);
143 	if (ret)
144 		return ret;
145 
146 	args->v0.chid = chan->base.chid;
147 
148 	nv_parent(chan)->object_attach = nv04_fifo_object_attach;
149 	nv_parent(chan)->object_detach = nv04_fifo_object_detach;
150 	nv_parent(chan)->context_attach = nv04_fifo_context_attach;
151 	chan->ramfc = chan->base.chid * 32;
152 
153 	nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
154 	nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
155 	nv_wo32(priv->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4);
156 	nv_wo32(priv->ramfc, chan->ramfc + 0x10,
157 			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
158 			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
159 #ifdef __BIG_ENDIAN
160 			     NV_PFIFO_CACHE1_BIG_ENDIAN |
161 #endif
162 			     NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
163 	return 0;
164 }
165 
166 void
167 nv04_fifo_chan_dtor(struct nouveau_object *object)
168 {
169 	struct nv04_fifo_priv *priv = (void *)object->engine;
170 	struct nv04_fifo_chan *chan = (void *)object;
171 	struct ramfc_desc *c = priv->ramfc_desc;
172 
173 	do {
174 		nv_wo32(priv->ramfc, chan->ramfc + c->ctxp, 0x00000000);
175 	} while ((++c)->bits);
176 
177 	nouveau_fifo_channel_destroy(&chan->base);
178 }
179 
180 int
181 nv04_fifo_chan_init(struct nouveau_object *object)
182 {
183 	struct nv04_fifo_priv *priv = (void *)object->engine;
184 	struct nv04_fifo_chan *chan = (void *)object;
185 	u32 mask = 1 << chan->base.chid;
186 	unsigned long flags;
187 	int ret;
188 
189 	ret = nouveau_fifo_channel_init(&chan->base);
190 	if (ret)
191 		return ret;
192 
193 	spin_lock_irqsave(&priv->base.lock, flags);
194 	nv_mask(priv, NV04_PFIFO_MODE, mask, mask);
195 	spin_unlock_irqrestore(&priv->base.lock, flags);
196 	return 0;
197 }
198 
199 int
200 nv04_fifo_chan_fini(struct nouveau_object *object, bool suspend)
201 {
202 	struct nv04_fifo_priv *priv = (void *)object->engine;
203 	struct nv04_fifo_chan *chan = (void *)object;
204 	struct nouveau_gpuobj *fctx = priv->ramfc;
205 	struct ramfc_desc *c;
206 	unsigned long flags;
207 	u32 data = chan->ramfc;
208 	u32 chid;
209 
210 	/* prevent fifo context switches */
211 	spin_lock_irqsave(&priv->base.lock, flags);
212 	nv_wr32(priv, NV03_PFIFO_CACHES, 0);
213 
214 	/* if this channel is active, replace it with a null context */
215 	chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
216 	if (chid == chan->base.chid) {
217 		nv_mask(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
218 		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 0);
219 		nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
220 
221 		c = priv->ramfc_desc;
222 		do {
223 			u32 rm = ((1ULL << c->bits) - 1) << c->regs;
224 			u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
225 			u32 rv = (nv_rd32(priv, c->regp) &  rm) >> c->regs;
226 			u32 cv = (nv_ro32(fctx, c->ctxp + data) & ~cm);
227 			nv_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
228 		} while ((++c)->bits);
229 
230 		c = priv->ramfc_desc;
231 		do {
232 			nv_wr32(priv, c->regp, 0x00000000);
233 		} while ((++c)->bits);
234 
235 		nv_wr32(priv, NV03_PFIFO_CACHE1_GET, 0);
236 		nv_wr32(priv, NV03_PFIFO_CACHE1_PUT, 0);
237 		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
238 		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
239 		nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
240 	}
241 
242 	/* restore normal operation, after disabling dma mode */
243 	nv_mask(priv, NV04_PFIFO_MODE, 1 << chan->base.chid, 0);
244 	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
245 	spin_unlock_irqrestore(&priv->base.lock, flags);
246 
247 	return nouveau_fifo_channel_fini(&chan->base, suspend);
248 }
249 
250 static struct nouveau_ofuncs
251 nv04_fifo_ofuncs = {
252 	.ctor = nv04_fifo_chan_ctor,
253 	.dtor = nv04_fifo_chan_dtor,
254 	.init = nv04_fifo_chan_init,
255 	.fini = nv04_fifo_chan_fini,
256 	.map  = _nouveau_fifo_channel_map,
257 	.rd32 = _nouveau_fifo_channel_rd32,
258 	.wr32 = _nouveau_fifo_channel_wr32,
259 	.ntfy = _nouveau_fifo_channel_ntfy
260 };
261 
262 static struct nouveau_oclass
263 nv04_fifo_sclass[] = {
264 	{ NV03_CHANNEL_DMA, &nv04_fifo_ofuncs },
265 	{}
266 };
267 
268 /*******************************************************************************
269  * FIFO context - basically just the instmem reserved for the channel
270  ******************************************************************************/
271 
272 int
273 nv04_fifo_context_ctor(struct nouveau_object *parent,
274 		       struct nouveau_object *engine,
275 		       struct nouveau_oclass *oclass, void *data, u32 size,
276 		       struct nouveau_object **pobject)
277 {
278 	struct nv04_fifo_base *base;
279 	int ret;
280 
281 	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
282 				          0x1000, NVOBJ_FLAG_HEAP, &base);
283 	*pobject = nv_object(base);
284 	if (ret)
285 		return ret;
286 
287 	return 0;
288 }
289 
290 static struct nouveau_oclass
291 nv04_fifo_cclass = {
292 	.handle = NV_ENGCTX(FIFO, 0x04),
293 	.ofuncs = &(struct nouveau_ofuncs) {
294 		.ctor = nv04_fifo_context_ctor,
295 		.dtor = _nouveau_fifo_context_dtor,
296 		.init = _nouveau_fifo_context_init,
297 		.fini = _nouveau_fifo_context_fini,
298 		.rd32 = _nouveau_fifo_context_rd32,
299 		.wr32 = _nouveau_fifo_context_wr32,
300 	},
301 };
302 
303 /*******************************************************************************
304  * PFIFO engine
305  ******************************************************************************/
306 
307 void
308 nv04_fifo_pause(struct nouveau_fifo *pfifo, unsigned long *pflags)
309 __acquires(priv->base.lock)
310 {
311 	struct nv04_fifo_priv *priv = (void *)pfifo;
312 	unsigned long flags;
313 
314 	spin_lock_irqsave(&priv->base.lock, flags);
315 	*pflags = flags;
316 
317 	nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000000);
318 	nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000);
319 
320 	/* in some cases the puller may be left in an inconsistent state
321 	 * if you try to stop it while it's busy translating handles.
322 	 * sometimes you get a CACHE_ERROR, sometimes it just fails
323 	 * silently; sending incorrect instance offsets to PGRAPH after
324 	 * it's started up again.
325 	 *
326 	 * to avoid this, we invalidate the most recently calculated
327 	 * instance.
328 	 */
329 	if (!nv_wait(priv, NV04_PFIFO_CACHE1_PULL0,
330 			   NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0x00000000))
331 		nv_warn(priv, "timeout idling puller\n");
332 
333 	if (nv_rd32(priv, NV04_PFIFO_CACHE1_PULL0) &
334 			  NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
335 		nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);
336 
337 	nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0x00000000);
338 }
339 
340 void
341 nv04_fifo_start(struct nouveau_fifo *pfifo, unsigned long *pflags)
342 __releases(priv->base.lock)
343 {
344 	struct nv04_fifo_priv *priv = (void *)pfifo;
345 	unsigned long flags = *pflags;
346 
347 	nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001);
348 	nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000001);
349 
350 	spin_unlock_irqrestore(&priv->base.lock, flags);
351 }
352 
353 static const char *
354 nv_dma_state_err(u32 state)
355 {
356 	static const char * const desc[] = {
357 		"NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
358 		"INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
359 	};
360 	return desc[(state >> 29) & 0x7];
361 }
362 
363 static bool
364 nv04_fifo_swmthd(struct nv04_fifo_priv *priv, u32 chid, u32 addr, u32 data)
365 {
366 	struct nv04_fifo_chan *chan = NULL;
367 	struct nouveau_handle *bind;
368 	const int subc = (addr >> 13) & 0x7;
369 	const int mthd = addr & 0x1ffc;
370 	bool handled = false;
371 	unsigned long flags;
372 	u32 engine;
373 
374 	spin_lock_irqsave(&priv->base.lock, flags);
375 	if (likely(chid >= priv->base.min && chid <= priv->base.max))
376 		chan = (void *)priv->base.channel[chid];
377 	if (unlikely(!chan))
378 		goto out;
379 
380 	switch (mthd) {
381 	case 0x0000:
382 		bind = nouveau_namedb_get(nv_namedb(chan), data);
383 		if (unlikely(!bind))
384 			break;
385 
386 		if (nv_engidx(bind->object->engine) == NVDEV_ENGINE_SW) {
387 			engine = 0x0000000f << (subc * 4);
388 			chan->subc[subc] = data;
389 			handled = true;
390 
391 			nv_mask(priv, NV04_PFIFO_CACHE1_ENGINE, engine, 0);
392 		}
393 
394 		nouveau_namedb_put(bind);
395 		break;
396 	default:
397 		engine = nv_rd32(priv, NV04_PFIFO_CACHE1_ENGINE);
398 		if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
399 			break;
400 
401 		bind = nouveau_namedb_get(nv_namedb(chan), chan->subc[subc]);
402 		if (likely(bind)) {
403 			if (!nv_call(bind->object, mthd, data))
404 				handled = true;
405 			nouveau_namedb_put(bind);
406 		}
407 		break;
408 	}
409 
410 out:
411 	spin_unlock_irqrestore(&priv->base.lock, flags);
412 	return handled;
413 }
414 
415 static void
416 nv04_fifo_cache_error(struct nouveau_device *device,
417 		struct nv04_fifo_priv *priv, u32 chid, u32 get)
418 {
419 	u32 mthd, data;
420 	int ptr;
421 
422 	/* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my
423 	 * G80 chips, but CACHE1 isn't big enough for this much data.. Tests
424 	 * show that it wraps around to the start at GET=0x800.. No clue as to
425 	 * why..
426 	 */
427 	ptr = (get & 0x7ff) >> 2;
428 
429 	if (device->card_type < NV_40) {
430 		mthd = nv_rd32(priv, NV04_PFIFO_CACHE1_METHOD(ptr));
431 		data = nv_rd32(priv, NV04_PFIFO_CACHE1_DATA(ptr));
432 	} else {
433 		mthd = nv_rd32(priv, NV40_PFIFO_CACHE1_METHOD(ptr));
434 		data = nv_rd32(priv, NV40_PFIFO_CACHE1_DATA(ptr));
435 	}
436 
437 	if (!nv04_fifo_swmthd(priv, chid, mthd, data)) {
438 		const char *client_name =
439 			nouveau_client_name_for_fifo_chid(&priv->base, chid);
440 		nv_error(priv,
441 			 "CACHE_ERROR - ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
442 			 chid, client_name, (mthd >> 13) & 7, mthd & 0x1ffc,
443 			 data);
444 	}
445 
446 	nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
447 	nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);
448 
449 	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
450 		nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) & ~1);
451 	nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
452 	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
453 		nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) | 1);
454 	nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0);
455 
456 	nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH,
457 		nv_rd32(priv, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
458 	nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
459 }
460 
461 static void
462 nv04_fifo_dma_pusher(struct nouveau_device *device, struct nv04_fifo_priv *priv,
463 		u32 chid)
464 {
465 	const char *client_name;
466 	u32 dma_get = nv_rd32(priv, 0x003244);
467 	u32 dma_put = nv_rd32(priv, 0x003240);
468 	u32 push = nv_rd32(priv, 0x003220);
469 	u32 state = nv_rd32(priv, 0x003228);
470 
471 	client_name = nouveau_client_name_for_fifo_chid(&priv->base, chid);
472 
473 	if (device->card_type == NV_50) {
474 		u32 ho_get = nv_rd32(priv, 0x003328);
475 		u32 ho_put = nv_rd32(priv, 0x003320);
476 		u32 ib_get = nv_rd32(priv, 0x003334);
477 		u32 ib_put = nv_rd32(priv, 0x003330);
478 
479 		nv_error(priv,
480 			 "DMA_PUSHER - ch %d [%s] get 0x%02x%08x put 0x%02x%08x ib_get 0x%08x ib_put 0x%08x state 0x%08x (err: %s) push 0x%08x\n",
481 			 chid, client_name, ho_get, dma_get, ho_put, dma_put,
482 			 ib_get, ib_put, state, nv_dma_state_err(state), push);
483 
484 		/* METHOD_COUNT, in DMA_STATE on earlier chipsets */
485 		nv_wr32(priv, 0x003364, 0x00000000);
486 		if (dma_get != dma_put || ho_get != ho_put) {
487 			nv_wr32(priv, 0x003244, dma_put);
488 			nv_wr32(priv, 0x003328, ho_put);
489 		} else
490 		if (ib_get != ib_put)
491 			nv_wr32(priv, 0x003334, ib_put);
492 	} else {
493 		nv_error(priv,
494 			 "DMA_PUSHER - ch %d [%s] get 0x%08x put 0x%08x state 0x%08x (err: %s) push 0x%08x\n",
495 			 chid, client_name, dma_get, dma_put, state,
496 			 nv_dma_state_err(state), push);
497 
498 		if (dma_get != dma_put)
499 			nv_wr32(priv, 0x003244, dma_put);
500 	}
501 
502 	nv_wr32(priv, 0x003228, 0x00000000);
503 	nv_wr32(priv, 0x003220, 0x00000001);
504 	nv_wr32(priv, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
505 }
506 
507 void
508 nv04_fifo_intr(struct nouveau_subdev *subdev)
509 {
510 	struct nouveau_device *device = nv_device(subdev);
511 	struct nv04_fifo_priv *priv = (void *)subdev;
512 	uint32_t status, reassign;
513 	int cnt = 0;
514 
515 	reassign = nv_rd32(priv, NV03_PFIFO_CACHES) & 1;
516 	while ((status = nv_rd32(priv, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
517 		uint32_t chid, get;
518 
519 		nv_wr32(priv, NV03_PFIFO_CACHES, 0);
520 
521 		chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
522 		get  = nv_rd32(priv, NV03_PFIFO_CACHE1_GET);
523 
524 		if (status & NV_PFIFO_INTR_CACHE_ERROR) {
525 			nv04_fifo_cache_error(device, priv, chid, get);
526 			status &= ~NV_PFIFO_INTR_CACHE_ERROR;
527 		}
528 
529 		if (status & NV_PFIFO_INTR_DMA_PUSHER) {
530 			nv04_fifo_dma_pusher(device, priv, chid);
531 			status &= ~NV_PFIFO_INTR_DMA_PUSHER;
532 		}
533 
534 		if (status & NV_PFIFO_INTR_SEMAPHORE) {
535 			uint32_t sem;
536 
537 			status &= ~NV_PFIFO_INTR_SEMAPHORE;
538 			nv_wr32(priv, NV03_PFIFO_INTR_0,
539 				NV_PFIFO_INTR_SEMAPHORE);
540 
541 			sem = nv_rd32(priv, NV10_PFIFO_CACHE1_SEMAPHORE);
542 			nv_wr32(priv, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
543 
544 			nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
545 			nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
546 		}
547 
548 		if (device->card_type == NV_50) {
549 			if (status & 0x00000010) {
550 				status &= ~0x00000010;
551 				nv_wr32(priv, 0x002100, 0x00000010);
552 			}
553 
554 			if (status & 0x40000000) {
555 				nv_wr32(priv, 0x002100, 0x40000000);
556 				nouveau_fifo_uevent(&priv->base);
557 				status &= ~0x40000000;
558 			}
559 		}
560 
561 		if (status) {
562 			nv_warn(priv, "unknown intr 0x%08x, ch %d\n",
563 				status, chid);
564 			nv_wr32(priv, NV03_PFIFO_INTR_0, status);
565 			status = 0;
566 		}
567 
568 		nv_wr32(priv, NV03_PFIFO_CACHES, reassign);
569 	}
570 
571 	if (status) {
572 		nv_error(priv, "still angry after %d spins, halt\n", cnt);
573 		nv_wr32(priv, 0x002140, 0);
574 		nv_wr32(priv, 0x000140, 0);
575 	}
576 
577 	nv_wr32(priv, 0x000100, 0x00000100);
578 }
579 
580 static int
581 nv04_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
582 	       struct nouveau_oclass *oclass, void *data, u32 size,
583 	       struct nouveau_object **pobject)
584 {
585 	struct nv04_instmem_priv *imem = nv04_instmem(parent);
586 	struct nv04_fifo_priv *priv;
587 	int ret;
588 
589 	ret = nouveau_fifo_create(parent, engine, oclass, 0, 15, &priv);
590 	*pobject = nv_object(priv);
591 	if (ret)
592 		return ret;
593 
594 	nouveau_ramht_ref(imem->ramht, &priv->ramht);
595 	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
596 	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
597 
598 	nv_subdev(priv)->unit = 0x00000100;
599 	nv_subdev(priv)->intr = nv04_fifo_intr;
600 	nv_engine(priv)->cclass = &nv04_fifo_cclass;
601 	nv_engine(priv)->sclass = nv04_fifo_sclass;
602 	priv->base.pause = nv04_fifo_pause;
603 	priv->base.start = nv04_fifo_start;
604 	priv->ramfc_desc = nv04_ramfc;
605 	return 0;
606 }
607 
608 void
609 nv04_fifo_dtor(struct nouveau_object *object)
610 {
611 	struct nv04_fifo_priv *priv = (void *)object;
612 	nouveau_gpuobj_ref(NULL, &priv->ramfc);
613 	nouveau_gpuobj_ref(NULL, &priv->ramro);
614 	nouveau_ramht_ref(NULL, &priv->ramht);
615 	nouveau_fifo_destroy(&priv->base);
616 }
617 
618 int
619 nv04_fifo_init(struct nouveau_object *object)
620 {
621 	struct nv04_fifo_priv *priv = (void *)object;
622 	int ret;
623 
624 	ret = nouveau_fifo_init(&priv->base);
625 	if (ret)
626 		return ret;
627 
628 	nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff);
629 	nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
630 
631 	nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
632 				       ((priv->ramht->bits - 9) << 16) |
633 				        (priv->ramht->gpuobj.addr >> 8));
634 	nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
635 	nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8);
636 
637 	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
638 
639 	nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff);
640 	nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff);
641 
642 	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
643 	nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
644 	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
645 	return 0;
646 }
647 
648 struct nouveau_oclass *
649 nv04_fifo_oclass = &(struct nouveau_oclass) {
650 	.handle = NV_ENGINE(FIFO, 0x04),
651 	.ofuncs = &(struct nouveau_ofuncs) {
652 		.ctor = nv04_fifo_ctor,
653 		.dtor = nv04_fifo_dtor,
654 		.init = nv04_fifo_init,
655 		.fini = _nouveau_fifo_fini,
656 	},
657 };
658