1 /*
2  * DMM IOMMU driver support functions for TI OMAP processors.
3  *
4  * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/
5  * Author: Rob Clark <rob@ti.com>
6  *         Andy Gross <andy.gross@ti.com>
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License as
10  * published by the Free Software Foundation version 2.
11  *
12  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
13  * kind, whether express or implied; without even the implied warranty
14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  */
17 
18 #include <linux/completion.h>
19 #include <linux/delay.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/dmaengine.h>
22 #include <linux/errno.h>
23 #include <linux/init.h>
24 #include <linux/interrupt.h>
25 #include <linux/list.h>
26 #include <linux/mm.h>
27 #include <linux/module.h>
28 #include <linux/platform_device.h> /* platform_device() */
29 #include <linux/sched.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/time.h>
33 #include <linux/vmalloc.h>
34 #include <linux/wait.h>
35 
36 #include "omap_dmm_tiler.h"
37 #include "omap_dmm_priv.h"
38 
39 #define DMM_DRIVER_NAME "dmm"
40 
41 /* mappings for associating views to luts */
42 static struct tcm *containers[TILFMT_NFORMATS];
43 static struct dmm *omap_dmm;
44 
45 #if defined(CONFIG_OF)
46 static const struct of_device_id dmm_of_match[];
47 #endif
48 
49 /* global spinlock for protecting lists */
50 static DEFINE_SPINLOCK(list_lock);
51 
52 /* Geometry table */
53 #define GEOM(xshift, yshift, bytes_per_pixel) { \
54 		.x_shft = (xshift), \
55 		.y_shft = (yshift), \
56 		.cpp    = (bytes_per_pixel), \
57 		.slot_w = 1 << (SLOT_WIDTH_BITS - (xshift)), \
58 		.slot_h = 1 << (SLOT_HEIGHT_BITS - (yshift)), \
59 	}
60 
61 static const struct {
62 	u32 x_shft;	/* unused X-bits (as part of bpp) */
63 	u32 y_shft;	/* unused Y-bits (as part of bpp) */
64 	u32 cpp;		/* bytes/chars per pixel */
65 	u32 slot_w;	/* width of each slot (in pixels) */
66 	u32 slot_h;	/* height of each slot (in pixels) */
67 } geom[TILFMT_NFORMATS] = {
68 	[TILFMT_8BIT]  = GEOM(0, 0, 1),
69 	[TILFMT_16BIT] = GEOM(0, 1, 2),
70 	[TILFMT_32BIT] = GEOM(1, 1, 4),
71 	[TILFMT_PAGE]  = GEOM(SLOT_WIDTH_BITS, SLOT_HEIGHT_BITS, 1),
72 };
73 
74 
75 /* lookup table for registers w/ per-engine instances */
76 static const u32 reg[][4] = {
77 	[PAT_STATUS] = {DMM_PAT_STATUS__0, DMM_PAT_STATUS__1,
78 			DMM_PAT_STATUS__2, DMM_PAT_STATUS__3},
79 	[PAT_DESCR]  = {DMM_PAT_DESCR__0, DMM_PAT_DESCR__1,
80 			DMM_PAT_DESCR__2, DMM_PAT_DESCR__3},
81 };
82 
83 static int dmm_dma_copy(struct dmm *dmm, dma_addr_t src, dma_addr_t dst)
84 {
85 	struct dma_async_tx_descriptor *tx;
86 	enum dma_status status;
87 	dma_cookie_t cookie;
88 
89 	tx = dmaengine_prep_dma_memcpy(dmm->wa_dma_chan, dst, src, 4, 0);
90 	if (!tx) {
91 		dev_err(dmm->dev, "Failed to prepare DMA memcpy\n");
92 		return -EIO;
93 	}
94 
95 	cookie = tx->tx_submit(tx);
96 	if (dma_submit_error(cookie)) {
97 		dev_err(dmm->dev, "Failed to do DMA tx_submit\n");
98 		return -EIO;
99 	}
100 
101 	status = dma_sync_wait(dmm->wa_dma_chan, cookie);
102 	if (status != DMA_COMPLETE)
103 		dev_err(dmm->dev, "i878 wa DMA copy failure\n");
104 
105 	dmaengine_terminate_all(dmm->wa_dma_chan);
106 	return 0;
107 }
108 
109 static u32 dmm_read_wa(struct dmm *dmm, u32 reg)
110 {
111 	dma_addr_t src, dst;
112 	int r;
113 
114 	src = dmm->phys_base + reg;
115 	dst = dmm->wa_dma_handle;
116 
117 	r = dmm_dma_copy(dmm, src, dst);
118 	if (r) {
119 		dev_err(dmm->dev, "sDMA read transfer timeout\n");
120 		return readl(dmm->base + reg);
121 	}
122 
123 	/*
124 	 * As per i878 workaround, the DMA is used to access the DMM registers.
125 	 * Make sure that the readl is not moved by the compiler or the CPU
126 	 * earlier than the DMA finished writing the value to memory.
127 	 */
128 	rmb();
129 	return readl(dmm->wa_dma_data);
130 }
131 
132 static void dmm_write_wa(struct dmm *dmm, u32 val, u32 reg)
133 {
134 	dma_addr_t src, dst;
135 	int r;
136 
137 	writel(val, dmm->wa_dma_data);
138 	/*
139 	 * As per i878 workaround, the DMA is used to access the DMM registers.
140 	 * Make sure that the writel is not moved by the compiler or the CPU, so
141 	 * the data will be in place before we start the DMA to do the actual
142 	 * register write.
143 	 */
144 	wmb();
145 
146 	src = dmm->wa_dma_handle;
147 	dst = dmm->phys_base + reg;
148 
149 	r = dmm_dma_copy(dmm, src, dst);
150 	if (r) {
151 		dev_err(dmm->dev, "sDMA write transfer timeout\n");
152 		writel(val, dmm->base + reg);
153 	}
154 }
155 
156 static u32 dmm_read(struct dmm *dmm, u32 reg)
157 {
158 	if (dmm->dmm_workaround) {
159 		u32 v;
160 		unsigned long flags;
161 
162 		spin_lock_irqsave(&dmm->wa_lock, flags);
163 		v = dmm_read_wa(dmm, reg);
164 		spin_unlock_irqrestore(&dmm->wa_lock, flags);
165 
166 		return v;
167 	} else {
168 		return readl(dmm->base + reg);
169 	}
170 }
171 
172 static void dmm_write(struct dmm *dmm, u32 val, u32 reg)
173 {
174 	if (dmm->dmm_workaround) {
175 		unsigned long flags;
176 
177 		spin_lock_irqsave(&dmm->wa_lock, flags);
178 		dmm_write_wa(dmm, val, reg);
179 		spin_unlock_irqrestore(&dmm->wa_lock, flags);
180 	} else {
181 		writel(val, dmm->base + reg);
182 	}
183 }
184 
185 static int dmm_workaround_init(struct dmm *dmm)
186 {
187 	dma_cap_mask_t mask;
188 
189 	spin_lock_init(&dmm->wa_lock);
190 
191 	dmm->wa_dma_data = dma_alloc_coherent(dmm->dev,  sizeof(u32),
192 					      &dmm->wa_dma_handle, GFP_KERNEL);
193 	if (!dmm->wa_dma_data)
194 		return -ENOMEM;
195 
196 	dma_cap_zero(mask);
197 	dma_cap_set(DMA_MEMCPY, mask);
198 
199 	dmm->wa_dma_chan = dma_request_channel(mask, NULL, NULL);
200 	if (!dmm->wa_dma_chan) {
201 		dma_free_coherent(dmm->dev, 4, dmm->wa_dma_data, dmm->wa_dma_handle);
202 		return -ENODEV;
203 	}
204 
205 	return 0;
206 }
207 
208 static void dmm_workaround_uninit(struct dmm *dmm)
209 {
210 	dma_release_channel(dmm->wa_dma_chan);
211 
212 	dma_free_coherent(dmm->dev, 4, dmm->wa_dma_data, dmm->wa_dma_handle);
213 }
214 
215 /* simple allocator to grab next 16 byte aligned memory from txn */
216 static void *alloc_dma(struct dmm_txn *txn, size_t sz, dma_addr_t *pa)
217 {
218 	void *ptr;
219 	struct refill_engine *engine = txn->engine_handle;
220 
221 	/* dmm programming requires 16 byte aligned addresses */
222 	txn->current_pa = round_up(txn->current_pa, 16);
223 	txn->current_va = (void *)round_up((long)txn->current_va, 16);
224 
225 	ptr = txn->current_va;
226 	*pa = txn->current_pa;
227 
228 	txn->current_pa += sz;
229 	txn->current_va += sz;
230 
231 	BUG_ON((txn->current_va - engine->refill_va) > REFILL_BUFFER_SIZE);
232 
233 	return ptr;
234 }
235 
236 /* check status and spin until wait_mask comes true */
237 static int wait_status(struct refill_engine *engine, u32 wait_mask)
238 {
239 	struct dmm *dmm = engine->dmm;
240 	u32 r = 0, err, i;
241 
242 	i = DMM_FIXED_RETRY_COUNT;
243 	while (true) {
244 		r = dmm_read(dmm, reg[PAT_STATUS][engine->id]);
245 		err = r & DMM_PATSTATUS_ERR;
246 		if (err) {
247 			dev_err(dmm->dev,
248 				"%s: error (engine%d). PAT_STATUS: 0x%08x\n",
249 				__func__, engine->id, r);
250 			return -EFAULT;
251 		}
252 
253 		if ((r & wait_mask) == wait_mask)
254 			break;
255 
256 		if (--i == 0) {
257 			dev_err(dmm->dev,
258 				"%s: timeout (engine%d). PAT_STATUS: 0x%08x\n",
259 				__func__, engine->id, r);
260 			return -ETIMEDOUT;
261 		}
262 
263 		udelay(1);
264 	}
265 
266 	return 0;
267 }
268 
269 static void release_engine(struct refill_engine *engine)
270 {
271 	unsigned long flags;
272 
273 	spin_lock_irqsave(&list_lock, flags);
274 	list_add(&engine->idle_node, &omap_dmm->idle_head);
275 	spin_unlock_irqrestore(&list_lock, flags);
276 
277 	atomic_inc(&omap_dmm->engine_counter);
278 	wake_up_interruptible(&omap_dmm->engine_queue);
279 }
280 
281 static irqreturn_t omap_dmm_irq_handler(int irq, void *arg)
282 {
283 	struct dmm *dmm = arg;
284 	u32 status = dmm_read(dmm, DMM_PAT_IRQSTATUS);
285 	int i;
286 
287 	/* ack IRQ */
288 	dmm_write(dmm, status, DMM_PAT_IRQSTATUS);
289 
290 	for (i = 0; i < dmm->num_engines; i++) {
291 		if (status & DMM_IRQSTAT_ERR_MASK)
292 			dev_err(dmm->dev,
293 				"irq error(engine%d): IRQSTAT 0x%02x\n",
294 				i, status & 0xff);
295 
296 		if (status & DMM_IRQSTAT_LST) {
297 			if (dmm->engines[i].async)
298 				release_engine(&dmm->engines[i]);
299 
300 			complete(&dmm->engines[i].compl);
301 		}
302 
303 		status >>= 8;
304 	}
305 
306 	return IRQ_HANDLED;
307 }
308 
309 /*
310  * Get a handle for a DMM transaction
311  */
312 static struct dmm_txn *dmm_txn_init(struct dmm *dmm, struct tcm *tcm)
313 {
314 	struct dmm_txn *txn = NULL;
315 	struct refill_engine *engine = NULL;
316 	int ret;
317 	unsigned long flags;
318 
319 
320 	/* wait until an engine is available */
321 	ret = wait_event_interruptible(omap_dmm->engine_queue,
322 		atomic_add_unless(&omap_dmm->engine_counter, -1, 0));
323 	if (ret)
324 		return ERR_PTR(ret);
325 
326 	/* grab an idle engine */
327 	spin_lock_irqsave(&list_lock, flags);
328 	if (!list_empty(&dmm->idle_head)) {
329 		engine = list_entry(dmm->idle_head.next, struct refill_engine,
330 					idle_node);
331 		list_del(&engine->idle_node);
332 	}
333 	spin_unlock_irqrestore(&list_lock, flags);
334 
335 	BUG_ON(!engine);
336 
337 	txn = &engine->txn;
338 	engine->tcm = tcm;
339 	txn->engine_handle = engine;
340 	txn->last_pat = NULL;
341 	txn->current_va = engine->refill_va;
342 	txn->current_pa = engine->refill_pa;
343 
344 	return txn;
345 }
346 
347 /*
348  * Add region to DMM transaction.  If pages or pages[i] is NULL, then the
349  * corresponding slot is cleared (ie. dummy_pa is programmed)
350  */
351 static void dmm_txn_append(struct dmm_txn *txn, struct pat_area *area,
352 		struct page **pages, u32 npages, u32 roll)
353 {
354 	dma_addr_t pat_pa = 0, data_pa = 0;
355 	u32 *data;
356 	struct pat *pat;
357 	struct refill_engine *engine = txn->engine_handle;
358 	int columns = (1 + area->x1 - area->x0);
359 	int rows = (1 + area->y1 - area->y0);
360 	int i = columns*rows;
361 
362 	pat = alloc_dma(txn, sizeof(*pat), &pat_pa);
363 
364 	if (txn->last_pat)
365 		txn->last_pat->next_pa = (u32)pat_pa;
366 
367 	pat->area = *area;
368 
369 	/* adjust Y coordinates based off of container parameters */
370 	pat->area.y0 += engine->tcm->y_offset;
371 	pat->area.y1 += engine->tcm->y_offset;
372 
373 	pat->ctrl = (struct pat_ctrl){
374 			.start = 1,
375 			.lut_id = engine->tcm->lut_id,
376 		};
377 
378 	data = alloc_dma(txn, 4*i, &data_pa);
379 	/* FIXME: what if data_pa is more than 32-bit ? */
380 	pat->data_pa = data_pa;
381 
382 	while (i--) {
383 		int n = i + roll;
384 		if (n >= npages)
385 			n -= npages;
386 		data[i] = (pages && pages[n]) ?
387 			page_to_phys(pages[n]) : engine->dmm->dummy_pa;
388 	}
389 
390 	txn->last_pat = pat;
391 
392 	return;
393 }
394 
395 /*
396  * Commit the DMM transaction.
397  */
398 static int dmm_txn_commit(struct dmm_txn *txn, bool wait)
399 {
400 	int ret = 0;
401 	struct refill_engine *engine = txn->engine_handle;
402 	struct dmm *dmm = engine->dmm;
403 
404 	if (!txn->last_pat) {
405 		dev_err(engine->dmm->dev, "need at least one txn\n");
406 		ret = -EINVAL;
407 		goto cleanup;
408 	}
409 
410 	txn->last_pat->next_pa = 0;
411 	/* ensure that the written descriptors are visible to DMM */
412 	wmb();
413 
414 	/*
415 	 * NOTE: the wmb() above should be enough, but there seems to be a bug
416 	 * in OMAP's memory barrier implementation, which in some rare cases may
417 	 * cause the writes not to be observable after wmb().
418 	 */
419 
420 	/* read back to ensure the data is in RAM */
421 	readl(&txn->last_pat->next_pa);
422 
423 	/* write to PAT_DESCR to clear out any pending transaction */
424 	dmm_write(dmm, 0x0, reg[PAT_DESCR][engine->id]);
425 
426 	/* wait for engine ready: */
427 	ret = wait_status(engine, DMM_PATSTATUS_READY);
428 	if (ret) {
429 		ret = -EFAULT;
430 		goto cleanup;
431 	}
432 
433 	/* mark whether it is async to denote list management in IRQ handler */
434 	engine->async = wait ? false : true;
435 	reinit_completion(&engine->compl);
436 	/* verify that the irq handler sees the 'async' and completion value */
437 	smp_mb();
438 
439 	/* kick reload */
440 	dmm_write(dmm, engine->refill_pa, reg[PAT_DESCR][engine->id]);
441 
442 	if (wait) {
443 		if (!wait_for_completion_timeout(&engine->compl,
444 				msecs_to_jiffies(100))) {
445 			dev_err(dmm->dev, "timed out waiting for done\n");
446 			ret = -ETIMEDOUT;
447 			goto cleanup;
448 		}
449 
450 		/* Check the engine status before continue */
451 		ret = wait_status(engine, DMM_PATSTATUS_READY |
452 				  DMM_PATSTATUS_VALID | DMM_PATSTATUS_DONE);
453 	}
454 
455 cleanup:
456 	/* only place engine back on list if we are done with it */
457 	if (ret || wait)
458 		release_engine(engine);
459 
460 	return ret;
461 }
462 
463 /*
464  * DMM programming
465  */
466 static int fill(struct tcm_area *area, struct page **pages,
467 		u32 npages, u32 roll, bool wait)
468 {
469 	int ret = 0;
470 	struct tcm_area slice, area_s;
471 	struct dmm_txn *txn;
472 
473 	/*
474 	 * FIXME
475 	 *
476 	 * Asynchronous fill does not work reliably, as the driver does not
477 	 * handle errors in the async code paths. The fill operation may
478 	 * silently fail, leading to leaking DMM engines, which may eventually
479 	 * lead to deadlock if we run out of DMM engines.
480 	 *
481 	 * For now, always set 'wait' so that we only use sync fills. Async
482 	 * fills should be fixed, or alternatively we could decide to only
483 	 * support sync fills and so the whole async code path could be removed.
484 	 */
485 
486 	wait = true;
487 
488 	txn = dmm_txn_init(omap_dmm, area->tcm);
489 	if (IS_ERR_OR_NULL(txn))
490 		return -ENOMEM;
491 
492 	tcm_for_each_slice(slice, *area, area_s) {
493 		struct pat_area p_area = {
494 				.x0 = slice.p0.x,  .y0 = slice.p0.y,
495 				.x1 = slice.p1.x,  .y1 = slice.p1.y,
496 		};
497 
498 		dmm_txn_append(txn, &p_area, pages, npages, roll);
499 
500 		roll += tcm_sizeof(slice);
501 	}
502 
503 	ret = dmm_txn_commit(txn, wait);
504 
505 	return ret;
506 }
507 
508 /*
509  * Pin/unpin
510  */
511 
512 /* note: slots for which pages[i] == NULL are filled w/ dummy page
513  */
514 int tiler_pin(struct tiler_block *block, struct page **pages,
515 		u32 npages, u32 roll, bool wait)
516 {
517 	int ret;
518 
519 	ret = fill(&block->area, pages, npages, roll, wait);
520 
521 	if (ret)
522 		tiler_unpin(block);
523 
524 	return ret;
525 }
526 
527 int tiler_unpin(struct tiler_block *block)
528 {
529 	return fill(&block->area, NULL, 0, 0, false);
530 }
531 
532 /*
533  * Reserve/release
534  */
535 struct tiler_block *tiler_reserve_2d(enum tiler_fmt fmt, u16 w,
536 		u16 h, u16 align)
537 {
538 	struct tiler_block *block;
539 	u32 min_align = 128;
540 	int ret;
541 	unsigned long flags;
542 	u32 slot_bytes;
543 
544 	block = kzalloc(sizeof(*block), GFP_KERNEL);
545 	if (!block)
546 		return ERR_PTR(-ENOMEM);
547 
548 	BUG_ON(!validfmt(fmt));
549 
550 	/* convert width/height to slots */
551 	w = DIV_ROUND_UP(w, geom[fmt].slot_w);
552 	h = DIV_ROUND_UP(h, geom[fmt].slot_h);
553 
554 	/* convert alignment to slots */
555 	slot_bytes = geom[fmt].slot_w * geom[fmt].cpp;
556 	min_align = max(min_align, slot_bytes);
557 	align = (align > min_align) ? ALIGN(align, min_align) : min_align;
558 	align /= slot_bytes;
559 
560 	block->fmt = fmt;
561 
562 	ret = tcm_reserve_2d(containers[fmt], w, h, align, -1, slot_bytes,
563 			&block->area);
564 	if (ret) {
565 		kfree(block);
566 		return ERR_PTR(-ENOMEM);
567 	}
568 
569 	/* add to allocation list */
570 	spin_lock_irqsave(&list_lock, flags);
571 	list_add(&block->alloc_node, &omap_dmm->alloc_head);
572 	spin_unlock_irqrestore(&list_lock, flags);
573 
574 	return block;
575 }
576 
577 struct tiler_block *tiler_reserve_1d(size_t size)
578 {
579 	struct tiler_block *block = kzalloc(sizeof(*block), GFP_KERNEL);
580 	int num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
581 	unsigned long flags;
582 
583 	if (!block)
584 		return ERR_PTR(-ENOMEM);
585 
586 	block->fmt = TILFMT_PAGE;
587 
588 	if (tcm_reserve_1d(containers[TILFMT_PAGE], num_pages,
589 				&block->area)) {
590 		kfree(block);
591 		return ERR_PTR(-ENOMEM);
592 	}
593 
594 	spin_lock_irqsave(&list_lock, flags);
595 	list_add(&block->alloc_node, &omap_dmm->alloc_head);
596 	spin_unlock_irqrestore(&list_lock, flags);
597 
598 	return block;
599 }
600 
601 /* note: if you have pin'd pages, you should have already unpin'd first! */
602 int tiler_release(struct tiler_block *block)
603 {
604 	int ret = tcm_free(&block->area);
605 	unsigned long flags;
606 
607 	if (block->area.tcm)
608 		dev_err(omap_dmm->dev, "failed to release block\n");
609 
610 	spin_lock_irqsave(&list_lock, flags);
611 	list_del(&block->alloc_node);
612 	spin_unlock_irqrestore(&list_lock, flags);
613 
614 	kfree(block);
615 	return ret;
616 }
617 
618 /*
619  * Utils
620  */
621 
622 /* calculate the tiler space address of a pixel in a view orientation...
623  * below description copied from the display subsystem section of TRM:
624  *
625  * When the TILER is addressed, the bits:
626  *   [28:27] = 0x0 for 8-bit tiled
627  *             0x1 for 16-bit tiled
628  *             0x2 for 32-bit tiled
629  *             0x3 for page mode
630  *   [31:29] = 0x0 for 0-degree view
631  *             0x1 for 180-degree view + mirroring
632  *             0x2 for 0-degree view + mirroring
633  *             0x3 for 180-degree view
634  *             0x4 for 270-degree view + mirroring
635  *             0x5 for 270-degree view
636  *             0x6 for 90-degree view
637  *             0x7 for 90-degree view + mirroring
638  * Otherwise the bits indicated the corresponding bit address to access
639  * the SDRAM.
640  */
641 static u32 tiler_get_address(enum tiler_fmt fmt, u32 orient, u32 x, u32 y)
642 {
643 	u32 x_bits, y_bits, tmp, x_mask, y_mask, alignment;
644 
645 	x_bits = CONT_WIDTH_BITS - geom[fmt].x_shft;
646 	y_bits = CONT_HEIGHT_BITS - geom[fmt].y_shft;
647 	alignment = geom[fmt].x_shft + geom[fmt].y_shft;
648 
649 	/* validate coordinate */
650 	x_mask = MASK(x_bits);
651 	y_mask = MASK(y_bits);
652 
653 	if (x < 0 || x > x_mask || y < 0 || y > y_mask) {
654 		DBG("invalid coords: %u < 0 || %u > %u || %u < 0 || %u > %u",
655 				x, x, x_mask, y, y, y_mask);
656 		return 0;
657 	}
658 
659 	/* account for mirroring */
660 	if (orient & MASK_X_INVERT)
661 		x ^= x_mask;
662 	if (orient & MASK_Y_INVERT)
663 		y ^= y_mask;
664 
665 	/* get coordinate address */
666 	if (orient & MASK_XY_FLIP)
667 		tmp = ((x << y_bits) + y);
668 	else
669 		tmp = ((y << x_bits) + x);
670 
671 	return TIL_ADDR((tmp << alignment), orient, fmt);
672 }
673 
674 dma_addr_t tiler_ssptr(struct tiler_block *block)
675 {
676 	BUG_ON(!validfmt(block->fmt));
677 
678 	return TILVIEW_8BIT + tiler_get_address(block->fmt, 0,
679 			block->area.p0.x * geom[block->fmt].slot_w,
680 			block->area.p0.y * geom[block->fmt].slot_h);
681 }
682 
683 dma_addr_t tiler_tsptr(struct tiler_block *block, u32 orient,
684 		u32 x, u32 y)
685 {
686 	struct tcm_pt *p = &block->area.p0;
687 	BUG_ON(!validfmt(block->fmt));
688 
689 	return tiler_get_address(block->fmt, orient,
690 			(p->x * geom[block->fmt].slot_w) + x,
691 			(p->y * geom[block->fmt].slot_h) + y);
692 }
693 
694 void tiler_align(enum tiler_fmt fmt, u16 *w, u16 *h)
695 {
696 	BUG_ON(!validfmt(fmt));
697 	*w = round_up(*w, geom[fmt].slot_w);
698 	*h = round_up(*h, geom[fmt].slot_h);
699 }
700 
701 u32 tiler_stride(enum tiler_fmt fmt, u32 orient)
702 {
703 	BUG_ON(!validfmt(fmt));
704 
705 	if (orient & MASK_XY_FLIP)
706 		return 1 << (CONT_HEIGHT_BITS + geom[fmt].x_shft);
707 	else
708 		return 1 << (CONT_WIDTH_BITS + geom[fmt].y_shft);
709 }
710 
711 size_t tiler_size(enum tiler_fmt fmt, u16 w, u16 h)
712 {
713 	tiler_align(fmt, &w, &h);
714 	return geom[fmt].cpp * w * h;
715 }
716 
717 size_t tiler_vsize(enum tiler_fmt fmt, u16 w, u16 h)
718 {
719 	BUG_ON(!validfmt(fmt));
720 	return round_up(geom[fmt].cpp * w, PAGE_SIZE) * h;
721 }
722 
723 u32 tiler_get_cpu_cache_flags(void)
724 {
725 	return omap_dmm->plat_data->cpu_cache_flags;
726 }
727 
728 bool dmm_is_available(void)
729 {
730 	return omap_dmm ? true : false;
731 }
732 
733 static int omap_dmm_remove(struct platform_device *dev)
734 {
735 	struct tiler_block *block, *_block;
736 	int i;
737 	unsigned long flags;
738 
739 	if (omap_dmm) {
740 		/* Disable all enabled interrupts */
741 		dmm_write(omap_dmm, 0x7e7e7e7e, DMM_PAT_IRQENABLE_CLR);
742 		free_irq(omap_dmm->irq, omap_dmm);
743 
744 		/* free all area regions */
745 		spin_lock_irqsave(&list_lock, flags);
746 		list_for_each_entry_safe(block, _block, &omap_dmm->alloc_head,
747 					alloc_node) {
748 			list_del(&block->alloc_node);
749 			kfree(block);
750 		}
751 		spin_unlock_irqrestore(&list_lock, flags);
752 
753 		for (i = 0; i < omap_dmm->num_lut; i++)
754 			if (omap_dmm->tcm && omap_dmm->tcm[i])
755 				omap_dmm->tcm[i]->deinit(omap_dmm->tcm[i]);
756 		kfree(omap_dmm->tcm);
757 
758 		kfree(omap_dmm->engines);
759 		if (omap_dmm->refill_va)
760 			dma_free_wc(omap_dmm->dev,
761 				    REFILL_BUFFER_SIZE * omap_dmm->num_engines,
762 				    omap_dmm->refill_va, omap_dmm->refill_pa);
763 		if (omap_dmm->dummy_page)
764 			__free_page(omap_dmm->dummy_page);
765 
766 		if (omap_dmm->dmm_workaround)
767 			dmm_workaround_uninit(omap_dmm);
768 
769 		iounmap(omap_dmm->base);
770 		kfree(omap_dmm);
771 		omap_dmm = NULL;
772 	}
773 
774 	return 0;
775 }
776 
777 static int omap_dmm_probe(struct platform_device *dev)
778 {
779 	int ret = -EFAULT, i;
780 	struct tcm_area area = {0};
781 	u32 hwinfo, pat_geom;
782 	struct resource *mem;
783 
784 	omap_dmm = kzalloc(sizeof(*omap_dmm), GFP_KERNEL);
785 	if (!omap_dmm)
786 		goto fail;
787 
788 	/* initialize lists */
789 	INIT_LIST_HEAD(&omap_dmm->alloc_head);
790 	INIT_LIST_HEAD(&omap_dmm->idle_head);
791 
792 	init_waitqueue_head(&omap_dmm->engine_queue);
793 
794 	if (dev->dev.of_node) {
795 		const struct of_device_id *match;
796 
797 		match = of_match_node(dmm_of_match, dev->dev.of_node);
798 		if (!match) {
799 			dev_err(&dev->dev, "failed to find matching device node\n");
800 			ret = -ENODEV;
801 			goto fail;
802 		}
803 
804 		omap_dmm->plat_data = match->data;
805 	}
806 
807 	/* lookup hwmod data - base address and irq */
808 	mem = platform_get_resource(dev, IORESOURCE_MEM, 0);
809 	if (!mem) {
810 		dev_err(&dev->dev, "failed to get base address resource\n");
811 		goto fail;
812 	}
813 
814 	omap_dmm->phys_base = mem->start;
815 	omap_dmm->base = ioremap(mem->start, SZ_2K);
816 
817 	if (!omap_dmm->base) {
818 		dev_err(&dev->dev, "failed to get dmm base address\n");
819 		goto fail;
820 	}
821 
822 	omap_dmm->irq = platform_get_irq(dev, 0);
823 	if (omap_dmm->irq < 0) {
824 		dev_err(&dev->dev, "failed to get IRQ resource\n");
825 		goto fail;
826 	}
827 
828 	omap_dmm->dev = &dev->dev;
829 
830 	if (of_machine_is_compatible("ti,dra7")) {
831 		/*
832 		 * DRA7 Errata i878 says that MPU should not be used to access
833 		 * RAM and DMM at the same time. As it's not possible to prevent
834 		 * MPU accessing RAM, we need to access DMM via a proxy.
835 		 */
836 		if (!dmm_workaround_init(omap_dmm)) {
837 			omap_dmm->dmm_workaround = true;
838 			dev_info(&dev->dev,
839 				"workaround for errata i878 in use\n");
840 		} else {
841 			dev_warn(&dev->dev,
842 				 "failed to initialize work-around for i878\n");
843 		}
844 	}
845 
846 	hwinfo = dmm_read(omap_dmm, DMM_PAT_HWINFO);
847 	omap_dmm->num_engines = (hwinfo >> 24) & 0x1F;
848 	omap_dmm->num_lut = (hwinfo >> 16) & 0x1F;
849 	omap_dmm->container_width = 256;
850 	omap_dmm->container_height = 128;
851 
852 	atomic_set(&omap_dmm->engine_counter, omap_dmm->num_engines);
853 
854 	/* read out actual LUT width and height */
855 	pat_geom = dmm_read(omap_dmm, DMM_PAT_GEOMETRY);
856 	omap_dmm->lut_width = ((pat_geom >> 16) & 0xF) << 5;
857 	omap_dmm->lut_height = ((pat_geom >> 24) & 0xF) << 5;
858 
859 	/* increment LUT by one if on OMAP5 */
860 	/* LUT has twice the height, and is split into a separate container */
861 	if (omap_dmm->lut_height != omap_dmm->container_height)
862 		omap_dmm->num_lut++;
863 
864 	/* initialize DMM registers */
865 	dmm_write(omap_dmm, 0x88888888, DMM_PAT_VIEW__0);
866 	dmm_write(omap_dmm, 0x88888888, DMM_PAT_VIEW__1);
867 	dmm_write(omap_dmm, 0x80808080, DMM_PAT_VIEW_MAP__0);
868 	dmm_write(omap_dmm, 0x80000000, DMM_PAT_VIEW_MAP_BASE);
869 	dmm_write(omap_dmm, 0x88888888, DMM_TILER_OR__0);
870 	dmm_write(omap_dmm, 0x88888888, DMM_TILER_OR__1);
871 
872 	omap_dmm->dummy_page = alloc_page(GFP_KERNEL | __GFP_DMA32);
873 	if (!omap_dmm->dummy_page) {
874 		dev_err(&dev->dev, "could not allocate dummy page\n");
875 		ret = -ENOMEM;
876 		goto fail;
877 	}
878 
879 	/* set dma mask for device */
880 	ret = dma_set_coherent_mask(&dev->dev, DMA_BIT_MASK(32));
881 	if (ret)
882 		goto fail;
883 
884 	omap_dmm->dummy_pa = page_to_phys(omap_dmm->dummy_page);
885 
886 	/* alloc refill memory */
887 	omap_dmm->refill_va = dma_alloc_wc(&dev->dev,
888 					   REFILL_BUFFER_SIZE * omap_dmm->num_engines,
889 					   &omap_dmm->refill_pa, GFP_KERNEL);
890 	if (!omap_dmm->refill_va) {
891 		dev_err(&dev->dev, "could not allocate refill memory\n");
892 		ret = -ENOMEM;
893 		goto fail;
894 	}
895 
896 	/* alloc engines */
897 	omap_dmm->engines = kcalloc(omap_dmm->num_engines,
898 				    sizeof(*omap_dmm->engines), GFP_KERNEL);
899 	if (!omap_dmm->engines) {
900 		ret = -ENOMEM;
901 		goto fail;
902 	}
903 
904 	for (i = 0; i < omap_dmm->num_engines; i++) {
905 		omap_dmm->engines[i].id = i;
906 		omap_dmm->engines[i].dmm = omap_dmm;
907 		omap_dmm->engines[i].refill_va = omap_dmm->refill_va +
908 						(REFILL_BUFFER_SIZE * i);
909 		omap_dmm->engines[i].refill_pa = omap_dmm->refill_pa +
910 						(REFILL_BUFFER_SIZE * i);
911 		init_completion(&omap_dmm->engines[i].compl);
912 
913 		list_add(&omap_dmm->engines[i].idle_node, &omap_dmm->idle_head);
914 	}
915 
916 	omap_dmm->tcm = kcalloc(omap_dmm->num_lut, sizeof(*omap_dmm->tcm),
917 				GFP_KERNEL);
918 	if (!omap_dmm->tcm) {
919 		ret = -ENOMEM;
920 		goto fail;
921 	}
922 
923 	/* init containers */
924 	/* Each LUT is associated with a TCM (container manager).  We use the
925 	   lut_id to denote the lut_id used to identify the correct LUT for
926 	   programming during reill operations */
927 	for (i = 0; i < omap_dmm->num_lut; i++) {
928 		omap_dmm->tcm[i] = sita_init(omap_dmm->container_width,
929 						omap_dmm->container_height);
930 
931 		if (!omap_dmm->tcm[i]) {
932 			dev_err(&dev->dev, "failed to allocate container\n");
933 			ret = -ENOMEM;
934 			goto fail;
935 		}
936 
937 		omap_dmm->tcm[i]->lut_id = i;
938 	}
939 
940 	/* assign access mode containers to applicable tcm container */
941 	/* OMAP 4 has 1 container for all 4 views */
942 	/* OMAP 5 has 2 containers, 1 for 2D and 1 for 1D */
943 	containers[TILFMT_8BIT] = omap_dmm->tcm[0];
944 	containers[TILFMT_16BIT] = omap_dmm->tcm[0];
945 	containers[TILFMT_32BIT] = omap_dmm->tcm[0];
946 
947 	if (omap_dmm->container_height != omap_dmm->lut_height) {
948 		/* second LUT is used for PAGE mode.  Programming must use
949 		   y offset that is added to all y coordinates.  LUT id is still
950 		   0, because it is the same LUT, just the upper 128 lines */
951 		containers[TILFMT_PAGE] = omap_dmm->tcm[1];
952 		omap_dmm->tcm[1]->y_offset = OMAP5_LUT_OFFSET;
953 		omap_dmm->tcm[1]->lut_id = 0;
954 	} else {
955 		containers[TILFMT_PAGE] = omap_dmm->tcm[0];
956 	}
957 
958 	area = (struct tcm_area) {
959 		.tcm = NULL,
960 		.p1.x = omap_dmm->container_width - 1,
961 		.p1.y = omap_dmm->container_height - 1,
962 	};
963 
964 	ret = request_irq(omap_dmm->irq, omap_dmm_irq_handler, IRQF_SHARED,
965 				"omap_dmm_irq_handler", omap_dmm);
966 
967 	if (ret) {
968 		dev_err(&dev->dev, "couldn't register IRQ %d, error %d\n",
969 			omap_dmm->irq, ret);
970 		omap_dmm->irq = -1;
971 		goto fail;
972 	}
973 
974 	/* Enable all interrupts for each refill engine except
975 	 * ERR_LUT_MISS<n> (which is just advisory, and we don't care
976 	 * about because we want to be able to refill live scanout
977 	 * buffers for accelerated pan/scroll) and FILL_DSC<n> which
978 	 * we just generally don't care about.
979 	 */
980 	dmm_write(omap_dmm, 0x7e7e7e7e, DMM_PAT_IRQENABLE_SET);
981 
982 	/* initialize all LUTs to dummy page entries */
983 	for (i = 0; i < omap_dmm->num_lut; i++) {
984 		area.tcm = omap_dmm->tcm[i];
985 		if (fill(&area, NULL, 0, 0, true))
986 			dev_err(omap_dmm->dev, "refill failed");
987 	}
988 
989 	dev_info(omap_dmm->dev, "initialized all PAT entries\n");
990 
991 	return 0;
992 
993 fail:
994 	if (omap_dmm_remove(dev))
995 		dev_err(&dev->dev, "cleanup failed\n");
996 	return ret;
997 }
998 
999 /*
1000  * debugfs support
1001  */
1002 
1003 #ifdef CONFIG_DEBUG_FS
1004 
1005 static const char *alphabet = "abcdefghijklmnopqrstuvwxyz"
1006 				"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
1007 static const char *special = ".,:;'\"`~!^-+";
1008 
1009 static void fill_map(char **map, int xdiv, int ydiv, struct tcm_area *a,
1010 							char c, bool ovw)
1011 {
1012 	int x, y;
1013 	for (y = a->p0.y / ydiv; y <= a->p1.y / ydiv; y++)
1014 		for (x = a->p0.x / xdiv; x <= a->p1.x / xdiv; x++)
1015 			if (map[y][x] == ' ' || ovw)
1016 				map[y][x] = c;
1017 }
1018 
1019 static void fill_map_pt(char **map, int xdiv, int ydiv, struct tcm_pt *p,
1020 									char c)
1021 {
1022 	map[p->y / ydiv][p->x / xdiv] = c;
1023 }
1024 
1025 static char read_map_pt(char **map, int xdiv, int ydiv, struct tcm_pt *p)
1026 {
1027 	return map[p->y / ydiv][p->x / xdiv];
1028 }
1029 
1030 static int map_width(int xdiv, int x0, int x1)
1031 {
1032 	return (x1 / xdiv) - (x0 / xdiv) + 1;
1033 }
1034 
1035 static void text_map(char **map, int xdiv, char *nice, int yd, int x0, int x1)
1036 {
1037 	char *p = map[yd] + (x0 / xdiv);
1038 	int w = (map_width(xdiv, x0, x1) - strlen(nice)) / 2;
1039 	if (w >= 0) {
1040 		p += w;
1041 		while (*nice)
1042 			*p++ = *nice++;
1043 	}
1044 }
1045 
1046 static void map_1d_info(char **map, int xdiv, int ydiv, char *nice,
1047 							struct tcm_area *a)
1048 {
1049 	sprintf(nice, "%dK", tcm_sizeof(*a) * 4);
1050 	if (a->p0.y + 1 < a->p1.y) {
1051 		text_map(map, xdiv, nice, (a->p0.y + a->p1.y) / 2 / ydiv, 0,
1052 							256 - 1);
1053 	} else if (a->p0.y < a->p1.y) {
1054 		if (strlen(nice) < map_width(xdiv, a->p0.x, 256 - 1))
1055 			text_map(map, xdiv, nice, a->p0.y / ydiv,
1056 					a->p0.x + xdiv,	256 - 1);
1057 		else if (strlen(nice) < map_width(xdiv, 0, a->p1.x))
1058 			text_map(map, xdiv, nice, a->p1.y / ydiv,
1059 					0, a->p1.y - xdiv);
1060 	} else if (strlen(nice) + 1 < map_width(xdiv, a->p0.x, a->p1.x)) {
1061 		text_map(map, xdiv, nice, a->p0.y / ydiv, a->p0.x, a->p1.x);
1062 	}
1063 }
1064 
1065 static void map_2d_info(char **map, int xdiv, int ydiv, char *nice,
1066 							struct tcm_area *a)
1067 {
1068 	sprintf(nice, "(%d*%d)", tcm_awidth(*a), tcm_aheight(*a));
1069 	if (strlen(nice) + 1 < map_width(xdiv, a->p0.x, a->p1.x))
1070 		text_map(map, xdiv, nice, (a->p0.y + a->p1.y) / 2 / ydiv,
1071 							a->p0.x, a->p1.x);
1072 }
1073 
1074 int tiler_map_show(struct seq_file *s, void *arg)
1075 {
1076 	int xdiv = 2, ydiv = 1;
1077 	char **map = NULL, *global_map;
1078 	struct tiler_block *block;
1079 	struct tcm_area a, p;
1080 	int i;
1081 	const char *m2d = alphabet;
1082 	const char *a2d = special;
1083 	const char *m2dp = m2d, *a2dp = a2d;
1084 	char nice[128];
1085 	int h_adj;
1086 	int w_adj;
1087 	unsigned long flags;
1088 	int lut_idx;
1089 
1090 
1091 	if (!omap_dmm) {
1092 		/* early return if dmm/tiler device is not initialized */
1093 		return 0;
1094 	}
1095 
1096 	h_adj = omap_dmm->container_height / ydiv;
1097 	w_adj = omap_dmm->container_width / xdiv;
1098 
1099 	map = kmalloc_array(h_adj, sizeof(*map), GFP_KERNEL);
1100 	global_map = kmalloc_array(w_adj + 1, h_adj, GFP_KERNEL);
1101 
1102 	if (!map || !global_map)
1103 		goto error;
1104 
1105 	for (lut_idx = 0; lut_idx < omap_dmm->num_lut; lut_idx++) {
1106 		memset(map, 0, h_adj * sizeof(*map));
1107 		memset(global_map, ' ', (w_adj + 1) * h_adj);
1108 
1109 		for (i = 0; i < omap_dmm->container_height; i++) {
1110 			map[i] = global_map + i * (w_adj + 1);
1111 			map[i][w_adj] = 0;
1112 		}
1113 
1114 		spin_lock_irqsave(&list_lock, flags);
1115 
1116 		list_for_each_entry(block, &omap_dmm->alloc_head, alloc_node) {
1117 			if (block->area.tcm == omap_dmm->tcm[lut_idx]) {
1118 				if (block->fmt != TILFMT_PAGE) {
1119 					fill_map(map, xdiv, ydiv, &block->area,
1120 						*m2dp, true);
1121 					if (!*++a2dp)
1122 						a2dp = a2d;
1123 					if (!*++m2dp)
1124 						m2dp = m2d;
1125 					map_2d_info(map, xdiv, ydiv, nice,
1126 							&block->area);
1127 				} else {
1128 					bool start = read_map_pt(map, xdiv,
1129 						ydiv, &block->area.p0) == ' ';
1130 					bool end = read_map_pt(map, xdiv, ydiv,
1131 							&block->area.p1) == ' ';
1132 
1133 					tcm_for_each_slice(a, block->area, p)
1134 						fill_map(map, xdiv, ydiv, &a,
1135 							'=', true);
1136 					fill_map_pt(map, xdiv, ydiv,
1137 							&block->area.p0,
1138 							start ? '<' : 'X');
1139 					fill_map_pt(map, xdiv, ydiv,
1140 							&block->area.p1,
1141 							end ? '>' : 'X');
1142 					map_1d_info(map, xdiv, ydiv, nice,
1143 							&block->area);
1144 				}
1145 			}
1146 		}
1147 
1148 		spin_unlock_irqrestore(&list_lock, flags);
1149 
1150 		if (s) {
1151 			seq_printf(s, "CONTAINER %d DUMP BEGIN\n", lut_idx);
1152 			for (i = 0; i < 128; i++)
1153 				seq_printf(s, "%03d:%s\n", i, map[i]);
1154 			seq_printf(s, "CONTAINER %d DUMP END\n", lut_idx);
1155 		} else {
1156 			dev_dbg(omap_dmm->dev, "CONTAINER %d DUMP BEGIN\n",
1157 				lut_idx);
1158 			for (i = 0; i < 128; i++)
1159 				dev_dbg(omap_dmm->dev, "%03d:%s\n", i, map[i]);
1160 			dev_dbg(omap_dmm->dev, "CONTAINER %d DUMP END\n",
1161 				lut_idx);
1162 		}
1163 	}
1164 
1165 error:
1166 	kfree(map);
1167 	kfree(global_map);
1168 
1169 	return 0;
1170 }
1171 #endif
1172 
1173 #ifdef CONFIG_PM_SLEEP
1174 static int omap_dmm_resume(struct device *dev)
1175 {
1176 	struct tcm_area area;
1177 	int i;
1178 
1179 	if (!omap_dmm)
1180 		return -ENODEV;
1181 
1182 	area = (struct tcm_area) {
1183 		.tcm = NULL,
1184 		.p1.x = omap_dmm->container_width - 1,
1185 		.p1.y = omap_dmm->container_height - 1,
1186 	};
1187 
1188 	/* initialize all LUTs to dummy page entries */
1189 	for (i = 0; i < omap_dmm->num_lut; i++) {
1190 		area.tcm = omap_dmm->tcm[i];
1191 		if (fill(&area, NULL, 0, 0, true))
1192 			dev_err(dev, "refill failed");
1193 	}
1194 
1195 	return 0;
1196 }
1197 #endif
1198 
1199 static SIMPLE_DEV_PM_OPS(omap_dmm_pm_ops, NULL, omap_dmm_resume);
1200 
1201 #if defined(CONFIG_OF)
1202 static const struct dmm_platform_data dmm_omap4_platform_data = {
1203 	.cpu_cache_flags = OMAP_BO_WC,
1204 };
1205 
1206 static const struct dmm_platform_data dmm_omap5_platform_data = {
1207 	.cpu_cache_flags = OMAP_BO_UNCACHED,
1208 };
1209 
1210 static const struct of_device_id dmm_of_match[] = {
1211 	{
1212 		.compatible = "ti,omap4-dmm",
1213 		.data = &dmm_omap4_platform_data,
1214 	},
1215 	{
1216 		.compatible = "ti,omap5-dmm",
1217 		.data = &dmm_omap5_platform_data,
1218 	},
1219 	{},
1220 };
1221 #endif
1222 
1223 struct platform_driver omap_dmm_driver = {
1224 	.probe = omap_dmm_probe,
1225 	.remove = omap_dmm_remove,
1226 	.driver = {
1227 		.owner = THIS_MODULE,
1228 		.name = DMM_DRIVER_NAME,
1229 		.of_match_table = of_match_ptr(dmm_of_match),
1230 		.pm = &omap_dmm_pm_ops,
1231 	},
1232 };
1233 
1234 MODULE_LICENSE("GPL v2");
1235 MODULE_AUTHOR("Andy Gross <andy.gross@ti.com>");
1236 MODULE_DESCRIPTION("OMAP DMM/Tiler Driver");
1237