xref: /openbmc/linux/drivers/memory/tegra/tegra20.c (revision 0b483871)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/mutex.h>
10 #include <linux/of.h>
11 #include <linux/slab.h>
12 #include <linux/string.h>
13 
14 #include <dt-bindings/memory/tegra20-mc.h>
15 
16 #include "mc.h"
17 
18 #define MC_STAT_CONTROL				0x90
19 #define MC_STAT_EMC_CLOCK_LIMIT			0xa0
20 #define MC_STAT_EMC_CLOCKS			0xa4
21 #define MC_STAT_EMC_CONTROL_0			0xa8
22 #define MC_STAT_EMC_CONTROL_1			0xac
23 #define MC_STAT_EMC_COUNT_0			0xb8
24 #define MC_STAT_EMC_COUNT_1			0xbc
25 
26 #define MC_STAT_CONTROL_CLIENT_ID		GENMASK(13,  8)
27 #define MC_STAT_CONTROL_EVENT			GENMASK(23, 16)
28 #define MC_STAT_CONTROL_PRI_EVENT		GENMASK(25, 24)
29 #define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE	GENMASK(26, 26)
30 #define MC_STAT_CONTROL_FILTER_PRI		GENMASK(29, 28)
31 
32 #define MC_STAT_CONTROL_PRI_EVENT_HP		0
33 #define MC_STAT_CONTROL_PRI_EVENT_TM		1
34 #define MC_STAT_CONTROL_PRI_EVENT_BW		2
35 
36 #define MC_STAT_CONTROL_FILTER_PRI_DISABLE	0
37 #define MC_STAT_CONTROL_FILTER_PRI_NO		1
38 #define MC_STAT_CONTROL_FILTER_PRI_YES		2
39 
40 #define MC_STAT_CONTROL_EVENT_QUALIFIED		0
41 #define MC_STAT_CONTROL_EVENT_ANY_READ		1
42 #define MC_STAT_CONTROL_EVENT_ANY_WRITE		2
43 #define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE	3
44 #define MC_STAT_CONTROL_EVENT_SUCCESSIVE	4
45 #define MC_STAT_CONTROL_EVENT_ARB_BANK_AA	5
46 #define MC_STAT_CONTROL_EVENT_ARB_BANK_BB	6
47 #define MC_STAT_CONTROL_EVENT_PAGE_MISS		7
48 #define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE	8
49 
50 #define EMC_GATHER_RST				(0 << 8)
51 #define EMC_GATHER_CLEAR			(1 << 8)
52 #define EMC_GATHER_DISABLE			(2 << 8)
53 #define EMC_GATHER_ENABLE			(3 << 8)
54 
55 #define MC_STAT_SAMPLE_TIME_USEC		16000
56 
57 /* we store collected statistics as a fixed point values */
58 #define MC_FX_FRAC_SCALE			100
59 
60 static DEFINE_MUTEX(tegra20_mc_stat_lock);
61 
62 struct tegra20_mc_stat_gather {
63 	unsigned int pri_filter;
64 	unsigned int pri_event;
65 	unsigned int result;
66 	unsigned int client;
67 	unsigned int event;
68 	bool client_enb;
69 };
70 
71 struct tegra20_mc_stat {
72 	struct tegra20_mc_stat_gather gather0;
73 	struct tegra20_mc_stat_gather gather1;
74 	unsigned int sample_time_usec;
75 	const struct tegra_mc *mc;
76 };
77 
78 struct tegra20_mc_client_stat {
79 	unsigned int events;
80 	unsigned int arb_high_prio;
81 	unsigned int arb_timeout;
82 	unsigned int arb_bandwidth;
83 	unsigned int rd_wr_change;
84 	unsigned int successive;
85 	unsigned int page_miss;
86 	unsigned int auto_precharge;
87 	unsigned int arb_bank_aa;
88 	unsigned int arb_bank_bb;
89 };
90 
91 static const struct tegra_mc_client tegra20_mc_clients[] = {
92 	{
93 		.id = 0x00,
94 		.name = "display0a",
95 	}, {
96 		.id = 0x01,
97 		.name = "display0ab",
98 	}, {
99 		.id = 0x02,
100 		.name = "display0b",
101 	}, {
102 		.id = 0x03,
103 		.name = "display0bb",
104 	}, {
105 		.id = 0x04,
106 		.name = "display0c",
107 	}, {
108 		.id = 0x05,
109 		.name = "display0cb",
110 	}, {
111 		.id = 0x06,
112 		.name = "display1b",
113 	}, {
114 		.id = 0x07,
115 		.name = "display1bb",
116 	}, {
117 		.id = 0x08,
118 		.name = "eppup",
119 	}, {
120 		.id = 0x09,
121 		.name = "g2pr",
122 	}, {
123 		.id = 0x0a,
124 		.name = "g2sr",
125 	}, {
126 		.id = 0x0b,
127 		.name = "mpeunifbr",
128 	}, {
129 		.id = 0x0c,
130 		.name = "viruv",
131 	}, {
132 		.id = 0x0d,
133 		.name = "avpcarm7r",
134 	}, {
135 		.id = 0x0e,
136 		.name = "displayhc",
137 	}, {
138 		.id = 0x0f,
139 		.name = "displayhcb",
140 	}, {
141 		.id = 0x10,
142 		.name = "fdcdrd",
143 	}, {
144 		.id = 0x11,
145 		.name = "g2dr",
146 	}, {
147 		.id = 0x12,
148 		.name = "host1xdmar",
149 	}, {
150 		.id = 0x13,
151 		.name = "host1xr",
152 	}, {
153 		.id = 0x14,
154 		.name = "idxsrd",
155 	}, {
156 		.id = 0x15,
157 		.name = "mpcorer",
158 	}, {
159 		.id = 0x16,
160 		.name = "mpe_ipred",
161 	}, {
162 		.id = 0x17,
163 		.name = "mpeamemrd",
164 	}, {
165 		.id = 0x18,
166 		.name = "mpecsrd",
167 	}, {
168 		.id = 0x19,
169 		.name = "ppcsahbdmar",
170 	}, {
171 		.id = 0x1a,
172 		.name = "ppcsahbslvr",
173 	}, {
174 		.id = 0x1b,
175 		.name = "texsrd",
176 	}, {
177 		.id = 0x1c,
178 		.name = "vdebsevr",
179 	}, {
180 		.id = 0x1d,
181 		.name = "vdember",
182 	}, {
183 		.id = 0x1e,
184 		.name = "vdemcer",
185 	}, {
186 		.id = 0x1f,
187 		.name = "vdetper",
188 	}, {
189 		.id = 0x20,
190 		.name = "eppu",
191 	}, {
192 		.id = 0x21,
193 		.name = "eppv",
194 	}, {
195 		.id = 0x22,
196 		.name = "eppy",
197 	}, {
198 		.id = 0x23,
199 		.name = "mpeunifbw",
200 	}, {
201 		.id = 0x24,
202 		.name = "viwsb",
203 	}, {
204 		.id = 0x25,
205 		.name = "viwu",
206 	}, {
207 		.id = 0x26,
208 		.name = "viwv",
209 	}, {
210 		.id = 0x27,
211 		.name = "viwy",
212 	}, {
213 		.id = 0x28,
214 		.name = "g2dw",
215 	}, {
216 		.id = 0x29,
217 		.name = "avpcarm7w",
218 	}, {
219 		.id = 0x2a,
220 		.name = "fdcdwr",
221 	}, {
222 		.id = 0x2b,
223 		.name = "host1xw",
224 	}, {
225 		.id = 0x2c,
226 		.name = "ispw",
227 	}, {
228 		.id = 0x2d,
229 		.name = "mpcorew",
230 	}, {
231 		.id = 0x2e,
232 		.name = "mpecswr",
233 	}, {
234 		.id = 0x2f,
235 		.name = "ppcsahbdmaw",
236 	}, {
237 		.id = 0x30,
238 		.name = "ppcsahbslvw",
239 	}, {
240 		.id = 0x31,
241 		.name = "vdebsevw",
242 	}, {
243 		.id = 0x32,
244 		.name = "vdembew",
245 	}, {
246 		.id = 0x33,
247 		.name = "vdetpmw",
248 	},
249 };
250 
251 #define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit)	\
252 	{								\
253 		.name = #_name,						\
254 		.id = TEGRA20_MC_RESET_##_name,				\
255 		.control = _control,					\
256 		.status = _status,					\
257 		.reset = _reset,					\
258 		.bit = _bit,						\
259 	}
260 
261 static const struct tegra_mc_reset tegra20_mc_resets[] = {
262 	TEGRA20_MC_RESET(AVPC,   0x100, 0x140, 0x104,  0),
263 	TEGRA20_MC_RESET(DC,     0x100, 0x144, 0x104,  1),
264 	TEGRA20_MC_RESET(DCB,    0x100, 0x148, 0x104,  2),
265 	TEGRA20_MC_RESET(EPP,    0x100, 0x14c, 0x104,  3),
266 	TEGRA20_MC_RESET(2D,     0x100, 0x150, 0x104,  4),
267 	TEGRA20_MC_RESET(HC,     0x100, 0x154, 0x104,  5),
268 	TEGRA20_MC_RESET(ISP,    0x100, 0x158, 0x104,  6),
269 	TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104,  7),
270 	TEGRA20_MC_RESET(MPEA,   0x100, 0x160, 0x104,  8),
271 	TEGRA20_MC_RESET(MPEB,   0x100, 0x164, 0x104,  9),
272 	TEGRA20_MC_RESET(MPEC,   0x100, 0x168, 0x104, 10),
273 	TEGRA20_MC_RESET(3D,     0x100, 0x16c, 0x104, 11),
274 	TEGRA20_MC_RESET(PPCS,   0x100, 0x170, 0x104, 12),
275 	TEGRA20_MC_RESET(VDE,    0x100, 0x174, 0x104, 13),
276 	TEGRA20_MC_RESET(VI,     0x100, 0x178, 0x104, 14),
277 };
278 
tegra20_mc_hotreset_assert(struct tegra_mc * mc,const struct tegra_mc_reset * rst)279 static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
280 				      const struct tegra_mc_reset *rst)
281 {
282 	unsigned long flags;
283 	u32 value;
284 
285 	spin_lock_irqsave(&mc->lock, flags);
286 
287 	value = mc_readl(mc, rst->reset);
288 	mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
289 
290 	spin_unlock_irqrestore(&mc->lock, flags);
291 
292 	return 0;
293 }
294 
tegra20_mc_hotreset_deassert(struct tegra_mc * mc,const struct tegra_mc_reset * rst)295 static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
296 					const struct tegra_mc_reset *rst)
297 {
298 	unsigned long flags;
299 	u32 value;
300 
301 	spin_lock_irqsave(&mc->lock, flags);
302 
303 	value = mc_readl(mc, rst->reset);
304 	mc_writel(mc, value | BIT(rst->bit), rst->reset);
305 
306 	spin_unlock_irqrestore(&mc->lock, flags);
307 
308 	return 0;
309 }
310 
tegra20_mc_block_dma(struct tegra_mc * mc,const struct tegra_mc_reset * rst)311 static int tegra20_mc_block_dma(struct tegra_mc *mc,
312 				const struct tegra_mc_reset *rst)
313 {
314 	unsigned long flags;
315 	u32 value;
316 
317 	spin_lock_irqsave(&mc->lock, flags);
318 
319 	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
320 	mc_writel(mc, value, rst->control);
321 
322 	spin_unlock_irqrestore(&mc->lock, flags);
323 
324 	return 0;
325 }
326 
tegra20_mc_dma_idling(struct tegra_mc * mc,const struct tegra_mc_reset * rst)327 static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
328 				  const struct tegra_mc_reset *rst)
329 {
330 	return mc_readl(mc, rst->status) == 0;
331 }
332 
tegra20_mc_reset_status(struct tegra_mc * mc,const struct tegra_mc_reset * rst)333 static int tegra20_mc_reset_status(struct tegra_mc *mc,
334 				   const struct tegra_mc_reset *rst)
335 {
336 	return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
337 }
338 
tegra20_mc_unblock_dma(struct tegra_mc * mc,const struct tegra_mc_reset * rst)339 static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
340 				  const struct tegra_mc_reset *rst)
341 {
342 	unsigned long flags;
343 	u32 value;
344 
345 	spin_lock_irqsave(&mc->lock, flags);
346 
347 	value = mc_readl(mc, rst->control) | BIT(rst->bit);
348 	mc_writel(mc, value, rst->control);
349 
350 	spin_unlock_irqrestore(&mc->lock, flags);
351 
352 	return 0;
353 }
354 
355 static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
356 	.hotreset_assert = tegra20_mc_hotreset_assert,
357 	.hotreset_deassert = tegra20_mc_hotreset_deassert,
358 	.block_dma = tegra20_mc_block_dma,
359 	.dma_idling = tegra20_mc_dma_idling,
360 	.unblock_dma = tegra20_mc_unblock_dma,
361 	.reset_status = tegra20_mc_reset_status,
362 };
363 
tegra20_mc_icc_set(struct icc_node * src,struct icc_node * dst)364 static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
365 {
366 	/*
367 	 * It should be possible to tune arbitration knobs here, but the
368 	 * default values are known to work well on all devices. Hence
369 	 * nothing to do here so far.
370 	 */
371 	return 0;
372 }
373 
tegra20_mc_icc_aggreate(struct icc_node * node,u32 tag,u32 avg_bw,u32 peak_bw,u32 * agg_avg,u32 * agg_peak)374 static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
375 				   u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
376 {
377 	/*
378 	 * ISO clients need to reserve extra bandwidth up-front because
379 	 * there could be high bandwidth pressure during initial filling
380 	 * of the client's FIFO buffers.  Secondly, we need to take into
381 	 * account impurities of the memory subsystem.
382 	 */
383 	if (tag & TEGRA_MC_ICC_TAG_ISO)
384 		peak_bw = tegra_mc_scale_percents(peak_bw, 300);
385 
386 	*agg_avg += avg_bw;
387 	*agg_peak = max(*agg_peak, peak_bw);
388 
389 	return 0;
390 }
391 
392 static struct icc_node_data *
tegra20_mc_of_icc_xlate_extended(struct of_phandle_args * spec,void * data)393 tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
394 {
395 	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
396 	unsigned int i, idx = spec->args[0];
397 	struct icc_node_data *ndata;
398 	struct icc_node *node;
399 
400 	list_for_each_entry(node, &mc->provider.nodes, node_list) {
401 		if (node->id != idx)
402 			continue;
403 
404 		ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
405 		if (!ndata)
406 			return ERR_PTR(-ENOMEM);
407 
408 		ndata->node = node;
409 
410 		/* these clients are isochronous by default */
411 		if (strstarts(node->name, "display") ||
412 		    strstarts(node->name, "vi"))
413 			ndata->tag = TEGRA_MC_ICC_TAG_ISO;
414 		else
415 			ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
416 
417 		return ndata;
418 	}
419 
420 	for (i = 0; i < mc->soc->num_clients; i++) {
421 		if (mc->soc->clients[i].id == idx)
422 			return ERR_PTR(-EPROBE_DEFER);
423 	}
424 
425 	dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
426 
427 	return ERR_PTR(-EINVAL);
428 }
429 
430 static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
431 	.xlate_extended = tegra20_mc_of_icc_xlate_extended,
432 	.aggregate = tegra20_mc_icc_aggreate,
433 	.set = tegra20_mc_icc_set,
434 };
435 
tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather * g)436 static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
437 {
438 	u32 control;
439 
440 	control  = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
441 	control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
442 	control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
443 	control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
444 	control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
445 
446 	return control;
447 }
448 
tegra20_mc_stat_gather(struct tegra20_mc_stat * stat)449 static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
450 {
451 	u32 clocks, count0, count1, control_0, control_1;
452 	const struct tegra_mc *mc = stat->mc;
453 
454 	control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
455 	control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
456 
457 	/*
458 	 * Reset statistic gathers state, select statistics collection mode
459 	 * and set clocks counter saturation limit to maximum.
460 	 */
461 	mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
462 	mc_writel(mc,  control_0, MC_STAT_EMC_CONTROL_0);
463 	mc_writel(mc,  control_1, MC_STAT_EMC_CONTROL_1);
464 	mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
465 
466 	mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
467 	fsleep(stat->sample_time_usec);
468 	mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
469 
470 	count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
471 	count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
472 	clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
473 	clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
474 
475 	stat->gather0.result = DIV_ROUND_UP(count0, clocks);
476 	stat->gather1.result = DIV_ROUND_UP(count1, clocks);
477 }
478 
tegra20_mc_stat_events(const struct tegra_mc * mc,const struct tegra_mc_client * client0,const struct tegra_mc_client * client1,unsigned int pri_filter,unsigned int pri_event,unsigned int event,unsigned int * result0,unsigned int * result1)479 static void tegra20_mc_stat_events(const struct tegra_mc *mc,
480 				   const struct tegra_mc_client *client0,
481 				   const struct tegra_mc_client *client1,
482 				   unsigned int pri_filter,
483 				   unsigned int pri_event,
484 				   unsigned int event,
485 				   unsigned int *result0,
486 				   unsigned int *result1)
487 {
488 	struct tegra20_mc_stat stat = {};
489 
490 	stat.gather0.client = client0 ? client0->id : 0;
491 	stat.gather0.pri_filter = pri_filter;
492 	stat.gather0.client_enb = !!client0;
493 	stat.gather0.pri_event = pri_event;
494 	stat.gather0.event = event;
495 
496 	stat.gather1.client = client1 ? client1->id : 0;
497 	stat.gather1.pri_filter = pri_filter;
498 	stat.gather1.client_enb = !!client1;
499 	stat.gather1.pri_event = pri_event;
500 	stat.gather1.event = event;
501 
502 	stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
503 	stat.mc = mc;
504 
505 	tegra20_mc_stat_gather(&stat);
506 
507 	*result0 = stat.gather0.result;
508 	*result1 = stat.gather1.result;
509 }
510 
tegra20_mc_collect_stats(const struct tegra_mc * mc,struct tegra20_mc_client_stat * stats)511 static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
512 				     struct tegra20_mc_client_stat *stats)
513 {
514 	const struct tegra_mc_client *client0, *client1;
515 	unsigned int i;
516 
517 	/* collect memory controller utilization percent for each client */
518 	for (i = 0; i < mc->soc->num_clients; i += 2) {
519 		client0 = &mc->soc->clients[i];
520 		client1 = &mc->soc->clients[i + 1];
521 
522 		if (i + 1 == mc->soc->num_clients)
523 			client1 = NULL;
524 
525 		tegra20_mc_stat_events(mc, client0, client1,
526 				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
527 				       MC_STAT_CONTROL_PRI_EVENT_HP,
528 				       MC_STAT_CONTROL_EVENT_QUALIFIED,
529 				       &stats[i + 0].events,
530 				       &stats[i + 1].events);
531 	}
532 
533 	/* collect more info from active clients */
534 	for (i = 0; i < mc->soc->num_clients; i++) {
535 		unsigned int clienta, clientb = mc->soc->num_clients;
536 
537 		for (client0 = NULL; i < mc->soc->num_clients; i++) {
538 			if (stats[i].events) {
539 				client0 = &mc->soc->clients[i];
540 				clienta = i++;
541 				break;
542 			}
543 		}
544 
545 		for (client1 = NULL; i < mc->soc->num_clients; i++) {
546 			if (stats[i].events) {
547 				client1 = &mc->soc->clients[i];
548 				clientb = i;
549 				break;
550 			}
551 		}
552 
553 		if (!client0 && !client1)
554 			break;
555 
556 		tegra20_mc_stat_events(mc, client0, client1,
557 				       MC_STAT_CONTROL_FILTER_PRI_YES,
558 				       MC_STAT_CONTROL_PRI_EVENT_HP,
559 				       MC_STAT_CONTROL_EVENT_QUALIFIED,
560 				       &stats[clienta].arb_high_prio,
561 				       &stats[clientb].arb_high_prio);
562 
563 		tegra20_mc_stat_events(mc, client0, client1,
564 				       MC_STAT_CONTROL_FILTER_PRI_YES,
565 				       MC_STAT_CONTROL_PRI_EVENT_TM,
566 				       MC_STAT_CONTROL_EVENT_QUALIFIED,
567 				       &stats[clienta].arb_timeout,
568 				       &stats[clientb].arb_timeout);
569 
570 		tegra20_mc_stat_events(mc, client0, client1,
571 				       MC_STAT_CONTROL_FILTER_PRI_YES,
572 				       MC_STAT_CONTROL_PRI_EVENT_BW,
573 				       MC_STAT_CONTROL_EVENT_QUALIFIED,
574 				       &stats[clienta].arb_bandwidth,
575 				       &stats[clientb].arb_bandwidth);
576 
577 		tegra20_mc_stat_events(mc, client0, client1,
578 				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
579 				       MC_STAT_CONTROL_PRI_EVENT_HP,
580 				       MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
581 				       &stats[clienta].rd_wr_change,
582 				       &stats[clientb].rd_wr_change);
583 
584 		tegra20_mc_stat_events(mc, client0, client1,
585 				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
586 				       MC_STAT_CONTROL_PRI_EVENT_HP,
587 				       MC_STAT_CONTROL_EVENT_SUCCESSIVE,
588 				       &stats[clienta].successive,
589 				       &stats[clientb].successive);
590 
591 		tegra20_mc_stat_events(mc, client0, client1,
592 				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
593 				       MC_STAT_CONTROL_PRI_EVENT_HP,
594 				       MC_STAT_CONTROL_EVENT_PAGE_MISS,
595 				       &stats[clienta].page_miss,
596 				       &stats[clientb].page_miss);
597 	}
598 }
599 
tegra20_mc_printf_percents(struct seq_file * s,const char * fmt,unsigned int percents_fx)600 static void tegra20_mc_printf_percents(struct seq_file *s,
601 				       const char *fmt,
602 				       unsigned int percents_fx)
603 {
604 	char percents_str[8];
605 
606 	snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
607 		 percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
608 
609 	seq_printf(s, fmt, percents_str);
610 }
611 
tegra20_mc_stats_show(struct seq_file * s,void * unused)612 static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
613 {
614 	const struct tegra_mc *mc = dev_get_drvdata(s->private);
615 	struct tegra20_mc_client_stat *stats;
616 	unsigned int i;
617 
618 	stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
619 	if (!stats)
620 		return -ENOMEM;
621 
622 	mutex_lock(&tegra20_mc_stat_lock);
623 
624 	tegra20_mc_collect_stats(mc, stats);
625 
626 	mutex_unlock(&tegra20_mc_stat_lock);
627 
628 	seq_puts(s, "Memory client   Events   Timeout   High priority   Bandwidth ARB   RW change   Successive   Page miss\n");
629 	seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
630 
631 	for (i = 0; i < mc->soc->num_clients; i++) {
632 		seq_printf(s, "%-14s  ", mc->soc->clients[i].name);
633 
634 		/* An event is generated when client performs R/W request. */
635 		tegra20_mc_printf_percents(s,  "%-9s", stats[i].events);
636 
637 		/*
638 		 * An event is generated based on the timeout (TM) signal
639 		 * accompanying a request for arbitration.
640 		 */
641 		tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
642 
643 		/*
644 		 * An event is generated based on the high-priority (HP) signal
645 		 * accompanying a request for arbitration.
646 		 */
647 		tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
648 
649 		/*
650 		 * An event is generated based on the bandwidth (BW) signal
651 		 * accompanying a request for arbitration.
652 		 */
653 		tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
654 
655 		/*
656 		 * An event is generated when the memory controller switches
657 		 * between making a read request to making a write request.
658 		 */
659 		tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
660 
661 		/*
662 		 * An even generated when the chosen client has wins arbitration
663 		 * when it was also the winner at the previous request.  If a
664 		 * client makes N requests in a row that are honored, SUCCESSIVE
665 		 * will be counted (N-1) times.  Large values for this event
666 		 * imply that if we were patient enough, all of those requests
667 		 * could have been coalesced.
668 		 */
669 		tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
670 
671 		/*
672 		 * An event is generated when the memory controller detects a
673 		 * page miss for the current request.
674 		 */
675 		tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
676 	}
677 
678 	kfree(stats);
679 
680 	return 0;
681 }
682 
tegra20_mc_probe(struct tegra_mc * mc)683 static int tegra20_mc_probe(struct tegra_mc *mc)
684 {
685 	debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
686 				    tegra20_mc_stats_show);
687 
688 	return 0;
689 }
690 
tegra20_mc_suspend(struct tegra_mc * mc)691 static int tegra20_mc_suspend(struct tegra_mc *mc)
692 {
693 	int err;
694 
695 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
696 		err = tegra_gart_suspend(mc->gart);
697 		if (err < 0)
698 			return err;
699 	}
700 
701 	return 0;
702 }
703 
tegra20_mc_resume(struct tegra_mc * mc)704 static int tegra20_mc_resume(struct tegra_mc *mc)
705 {
706 	int err;
707 
708 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
709 		err = tegra_gart_resume(mc->gart);
710 		if (err < 0)
711 			return err;
712 	}
713 
714 	return 0;
715 }
716 
tegra20_mc_handle_irq(int irq,void * data)717 static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
718 {
719 	struct tegra_mc *mc = data;
720 	unsigned long status;
721 	unsigned int bit;
722 
723 	/* mask all interrupts to avoid flooding */
724 	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
725 	if (!status)
726 		return IRQ_NONE;
727 
728 	for_each_set_bit(bit, &status, 32) {
729 		const char *error = tegra_mc_status_names[bit];
730 		const char *direction = "read", *secure = "";
731 		const char *client, *desc;
732 		phys_addr_t addr;
733 		u32 value, reg;
734 		u8 id, type;
735 
736 		switch (BIT(bit)) {
737 		case MC_INT_DECERR_EMEM:
738 			reg = MC_DECERR_EMEM_OTHERS_STATUS;
739 			value = mc_readl(mc, reg);
740 
741 			id = value & mc->soc->client_id_mask;
742 			desc = tegra_mc_error_names[2];
743 
744 			if (value & BIT(31))
745 				direction = "write";
746 			break;
747 
748 		case MC_INT_INVALID_GART_PAGE:
749 			reg = MC_GART_ERROR_REQ;
750 			value = mc_readl(mc, reg);
751 
752 			id = (value >> 1) & mc->soc->client_id_mask;
753 			desc = tegra_mc_error_names[2];
754 
755 			if (value & BIT(0))
756 				direction = "write";
757 			break;
758 
759 		case MC_INT_SECURITY_VIOLATION:
760 			reg = MC_SECURITY_VIOLATION_STATUS;
761 			value = mc_readl(mc, reg);
762 
763 			id = value & mc->soc->client_id_mask;
764 			type = (value & BIT(30)) ? 4 : 3;
765 			desc = tegra_mc_error_names[type];
766 			secure = "secure ";
767 
768 			if (value & BIT(31))
769 				direction = "write";
770 			break;
771 
772 		default:
773 			continue;
774 		}
775 
776 		client = mc->soc->clients[id].name;
777 		addr = mc_readl(mc, reg + sizeof(u32));
778 
779 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
780 				    client, secure, direction, &addr, error,
781 				    desc);
782 	}
783 
784 	/* clear interrupts */
785 	mc_writel(mc, status, MC_INTSTATUS);
786 
787 	return IRQ_HANDLED;
788 }
789 
790 static const struct tegra_mc_ops tegra20_mc_ops = {
791 	.probe = tegra20_mc_probe,
792 	.suspend = tegra20_mc_suspend,
793 	.resume = tegra20_mc_resume,
794 	.handle_irq = tegra20_mc_handle_irq,
795 };
796 
797 const struct tegra_mc_soc tegra20_mc_soc = {
798 	.clients = tegra20_mc_clients,
799 	.num_clients = ARRAY_SIZE(tegra20_mc_clients),
800 	.num_address_bits = 32,
801 	.client_id_mask = 0x3f,
802 	.intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
803 		   MC_INT_DECERR_EMEM,
804 	.reset_ops = &tegra20_mc_reset_ops,
805 	.resets = tegra20_mc_resets,
806 	.num_resets = ARRAY_SIZE(tegra20_mc_resets),
807 	.icc_ops = &tegra20_mc_icc_ops,
808 	.ops = &tegra20_mc_ops,
809 };
810