1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4  * Author: Lukasz Luba <l.luba@partner.samsung.com>
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/devfreq.h>
9 #include <linux/devfreq-event.h>
10 #include <linux/device.h>
11 #include <linux/interrupt.h>
12 #include <linux/io.h>
13 #include <linux/mfd/syscon.h>
14 #include <linux/module.h>
15 #include <linux/moduleparam.h>
16 #include <linux/of.h>
17 #include <linux/pm_opp.h>
18 #include <linux/platform_device.h>
19 #include <linux/regmap.h>
20 #include <linux/regulator/consumer.h>
21 #include <linux/slab.h>
22 #include "../jedec_ddr.h"
23 #include "../of_memory.h"
24 
25 static int irqmode;
26 module_param(irqmode, int, 0644);
27 MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
28 
29 #define EXYNOS5_DREXI_TIMINGAREF		(0x0030)
30 #define EXYNOS5_DREXI_TIMINGROW0		(0x0034)
31 #define EXYNOS5_DREXI_TIMINGDATA0		(0x0038)
32 #define EXYNOS5_DREXI_TIMINGPOWER0		(0x003C)
33 #define EXYNOS5_DREXI_TIMINGROW1		(0x00E4)
34 #define EXYNOS5_DREXI_TIMINGDATA1		(0x00E8)
35 #define EXYNOS5_DREXI_TIMINGPOWER1		(0x00EC)
36 #define CDREX_PAUSE				(0x2091c)
37 #define CDREX_LPDDR3PHY_CON3			(0x20a20)
38 #define CDREX_LPDDR3PHY_CLKM_SRC		(0x20700)
39 #define EXYNOS5_TIMING_SET_SWI			BIT(28)
40 #define USE_MX_MSPLL_TIMINGS			(1)
41 #define USE_BPLL_TIMINGS			(0)
42 #define EXYNOS5_AREF_NORMAL			(0x2e)
43 
44 #define DREX_PPCCLKCON		(0x0130)
45 #define DREX_PEREV2CONFIG	(0x013c)
46 #define DREX_PMNC_PPC		(0xE000)
47 #define DREX_CNTENS_PPC		(0xE010)
48 #define DREX_CNTENC_PPC		(0xE020)
49 #define DREX_INTENS_PPC		(0xE030)
50 #define DREX_INTENC_PPC		(0xE040)
51 #define DREX_FLAG_PPC		(0xE050)
52 #define DREX_PMCNT2_PPC		(0xE130)
53 
54 /*
55  * A value for register DREX_PMNC_PPC which should be written to reset
56  * the cycle counter CCNT (a reference wall clock). It sets zero to the
57  * CCNT counter.
58  */
59 #define CC_RESET		BIT(2)
60 
61 /*
62  * A value for register DREX_PMNC_PPC which does the reset of all performance
63  * counters to zero.
64  */
65 #define PPC_COUNTER_RESET	BIT(1)
66 
67 /*
68  * Enables all configured counters (including cycle counter). The value should
69  * be written to the register DREX_PMNC_PPC.
70  */
71 #define PPC_ENABLE		BIT(0)
72 
73 /* A value for register DREX_PPCCLKCON which enables performance events clock.
74  * Must be written before first access to the performance counters register
75  * set, otherwise it could crash.
76  */
77 #define PEREV_CLK_EN		BIT(0)
78 
79 /*
80  * Values which are used to enable counters, interrupts or configure flags of
81  * the performance counters. They configure counter 2 and cycle counter.
82  */
83 #define PERF_CNT2		BIT(2)
84 #define PERF_CCNT		BIT(31)
85 
86 /*
87  * Performance event types which are used for setting the preferred event
88  * to track in the counters.
89  * There is a set of different types, the values are from range 0 to 0x6f.
90  * These settings should be written to the configuration register which manages
91  * the type of the event (register DREX_PEREV2CONFIG).
92  */
93 #define READ_TRANSFER_CH0	(0x6d)
94 #define READ_TRANSFER_CH1	(0x6f)
95 
96 #define PERF_COUNTER_START_VALUE 0xff000000
97 #define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
98 
99 /**
100  * struct dmc_opp_table - Operating level desciption
101  * @freq_hz:		target frequency in Hz
102  * @volt_uv:		target voltage in uV
103  *
104  * Covers frequency and voltage settings of the DMC operating mode.
105  */
106 struct dmc_opp_table {
107 	u32 freq_hz;
108 	u32 volt_uv;
109 };
110 
111 /**
112  * struct exynos5_dmc - main structure describing DMC device
113  * @dev:		DMC device
114  * @df:			devfreq device structure returned by devfreq framework
115  * @gov_data:		configuration of devfreq governor
116  * @base_drexi0:	DREX0 registers mapping
117  * @base_drexi1:	DREX1 registers mapping
118  * @clk_regmap:		regmap for clock controller registers
119  * @lock:		protects curr_rate and frequency/voltage setting section
120  * @curr_rate:		current frequency
121  * @curr_volt:		current voltage
122  * @opp:		OPP table
123  * @opp_count:		number of 'opp' elements
124  * @timings_arr_size:	number of 'timings' elements
125  * @timing_row:		values for timing row register, for each OPP
126  * @timing_data:	values for timing data register, for each OPP
127  * @timing_power:	balues for timing power register, for each OPP
128  * @timings:		DDR memory timings, from device tree
129  * @min_tck:		DDR memory minimum timing values, from device tree
130  * @bypass_timing_row:	value for timing row register for bypass timings
131  * @bypass_timing_data:	value for timing data register for bypass timings
132  * @bypass_timing_power:	value for timing power register for bypass
133  *				timings
134  * @vdd_mif:		Memory interface regulator
135  * @fout_spll:		clock: SPLL
136  * @fout_bpll:		clock: BPLL
137  * @mout_spll:		clock: mux SPLL
138  * @mout_bpll:		clock: mux BPLL
139  * @mout_mclk_cdrex:	clock: mux mclk_cdrex
140  * @mout_mx_mspll_ccore:	clock: mux mx_mspll_ccore
141  * @counter:		devfreq events
142  * @num_counters:	number of 'counter' elements
143  * @last_overflow_ts:	time (in ns) of last overflow of each DREX
144  * @load:		utilization in percents
145  * @total:		total time between devfreq events
146  * @in_irq_mode:	whether running in interrupt mode (true)
147  *			or polling (false)
148  *
149  * The main structure for the Dynamic Memory Controller which covers clocks,
150  * memory regions, HW information, parameters and current operating mode.
151  */
152 struct exynos5_dmc {
153 	struct device *dev;
154 	struct devfreq *df;
155 	struct devfreq_simple_ondemand_data gov_data;
156 	void __iomem *base_drexi0;
157 	void __iomem *base_drexi1;
158 	struct regmap *clk_regmap;
159 	/* Protects curr_rate and frequency/voltage setting section */
160 	struct mutex lock;
161 	unsigned long curr_rate;
162 	unsigned long curr_volt;
163 	struct dmc_opp_table *opp;
164 	int opp_count;
165 	u32 timings_arr_size;
166 	u32 *timing_row;
167 	u32 *timing_data;
168 	u32 *timing_power;
169 	const struct lpddr3_timings *timings;
170 	const struct lpddr3_min_tck *min_tck;
171 	u32 bypass_timing_row;
172 	u32 bypass_timing_data;
173 	u32 bypass_timing_power;
174 	struct regulator *vdd_mif;
175 	struct clk *fout_spll;
176 	struct clk *fout_bpll;
177 	struct clk *mout_spll;
178 	struct clk *mout_bpll;
179 	struct clk *mout_mclk_cdrex;
180 	struct clk *mout_mx_mspll_ccore;
181 	struct devfreq_event_dev **counter;
182 	int num_counters;
183 	u64 last_overflow_ts[2];
184 	unsigned long load;
185 	unsigned long total;
186 	bool in_irq_mode;
187 };
188 
189 #define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
190 	{ .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
191 
192 #define TIMING_VAL2REG(timing, t_val)			\
193 ({							\
194 		u32 __val;				\
195 		__val = (t_val) << (timing)->bit_beg;	\
196 		__val;					\
197 })
198 
199 struct timing_reg {
200 	char *name;
201 	int bit_beg;
202 	int bit_end;
203 	unsigned int val;
204 };
205 
206 static const struct timing_reg timing_row_reg_fields[] = {
207 	TIMING_FIELD("tRFC", 24, 31),
208 	TIMING_FIELD("tRRD", 20, 23),
209 	TIMING_FIELD("tRP", 16, 19),
210 	TIMING_FIELD("tRCD", 12, 15),
211 	TIMING_FIELD("tRC", 6, 11),
212 	TIMING_FIELD("tRAS", 0, 5),
213 };
214 
215 static const struct timing_reg timing_data_reg_fields[] = {
216 	TIMING_FIELD("tWTR", 28, 31),
217 	TIMING_FIELD("tWR", 24, 27),
218 	TIMING_FIELD("tRTP", 20, 23),
219 	TIMING_FIELD("tW2W-C2C", 14, 14),
220 	TIMING_FIELD("tR2R-C2C", 12, 12),
221 	TIMING_FIELD("WL", 8, 11),
222 	TIMING_FIELD("tDQSCK", 4, 7),
223 	TIMING_FIELD("RL", 0, 3),
224 };
225 
226 static const struct timing_reg timing_power_reg_fields[] = {
227 	TIMING_FIELD("tFAW", 26, 31),
228 	TIMING_FIELD("tXSR", 16, 25),
229 	TIMING_FIELD("tXP", 8, 15),
230 	TIMING_FIELD("tCKE", 4, 7),
231 	TIMING_FIELD("tMRD", 0, 3),
232 };
233 
234 #define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \
235 		      ARRAY_SIZE(timing_data_reg_fields) + \
236 		      ARRAY_SIZE(timing_power_reg_fields))
237 
238 static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
239 {
240 	int i, ret;
241 
242 	for (i = 0; i < dmc->num_counters; i++) {
243 		if (!dmc->counter[i])
244 			continue;
245 		ret = devfreq_event_set_event(dmc->counter[i]);
246 		if (ret < 0)
247 			return ret;
248 	}
249 	return 0;
250 }
251 
252 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
253 {
254 	int i, ret;
255 
256 	for (i = 0; i < dmc->num_counters; i++) {
257 		if (!dmc->counter[i])
258 			continue;
259 		ret = devfreq_event_enable_edev(dmc->counter[i]);
260 		if (ret < 0)
261 			return ret;
262 	}
263 	return 0;
264 }
265 
266 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
267 {
268 	int i, ret;
269 
270 	for (i = 0; i < dmc->num_counters; i++) {
271 		if (!dmc->counter[i])
272 			continue;
273 		ret = devfreq_event_disable_edev(dmc->counter[i]);
274 		if (ret < 0)
275 			return ret;
276 	}
277 	return 0;
278 }
279 
280 /**
281  * find_target_freq_idx() - Finds requested frequency in local DMC configuration
282  * @dmc:	device for which the information is checked
283  * @target_rate:	requested frequency in KHz
284  *
285  * Seeks in the local DMC driver structure for the requested frequency value
286  * and returns index or error value.
287  */
288 static int find_target_freq_idx(struct exynos5_dmc *dmc,
289 				unsigned long target_rate)
290 {
291 	int i;
292 
293 	for (i = dmc->opp_count - 1; i >= 0; i--)
294 		if (dmc->opp[i].freq_hz <= target_rate)
295 			return i;
296 
297 	return -EINVAL;
298 }
299 
300 /**
301  * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
302  * @dmc:	device for which the new settings is going to be applied
303  * @set:	boolean variable passing set value
304  *
305  * Changes the register set, which holds timing parameters.
306  * There is two register sets: 0 and 1. The register set 0
307  * is used in normal operation when the clock is provided from main PLL.
308  * The bank register set 1 is used when the main PLL frequency is going to be
309  * changed and the clock is taken from alternative, stable source.
310  * This function switches between these banks according to the
311  * currently used clock source.
312  */
313 static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
314 {
315 	unsigned int reg;
316 	int ret;
317 
318 	ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, &reg);
319 	if (ret)
320 		return ret;
321 
322 	if (set)
323 		reg |= EXYNOS5_TIMING_SET_SWI;
324 	else
325 		reg &= ~EXYNOS5_TIMING_SET_SWI;
326 
327 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
328 
329 	return 0;
330 }
331 
332 /**
333  * exynos5_init_freq_table() - Initialized PM OPP framework
334  * @dmc:	DMC device for which the frequencies are used for OPP init
335  * @profile:	devfreq device's profile
336  *
337  * Populate the devfreq device's OPP table based on current frequency, voltage.
338  */
339 static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
340 				   struct devfreq_dev_profile *profile)
341 {
342 	int i, ret;
343 	int idx;
344 	unsigned long freq;
345 
346 	ret = devm_pm_opp_of_add_table(dmc->dev);
347 	if (ret < 0) {
348 		dev_err(dmc->dev, "Failed to get OPP table\n");
349 		return ret;
350 	}
351 
352 	dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
353 
354 	dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
355 				      sizeof(struct dmc_opp_table), GFP_KERNEL);
356 	if (!dmc->opp)
357 		return -ENOMEM;
358 
359 	idx = dmc->opp_count - 1;
360 	for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
361 		struct dev_pm_opp *opp;
362 
363 		opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
364 		if (IS_ERR(opp))
365 			return PTR_ERR(opp);
366 
367 		dmc->opp[idx - i].freq_hz = freq;
368 		dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
369 
370 		dev_pm_opp_put(opp);
371 	}
372 
373 	return 0;
374 }
375 
376 /**
377  * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
378  * @dmc:	device for which the new settings is going to be applied
379  *
380  * Low-level function for changing timings for DRAM memory clocking from
381  * 'bypass' clock source (fixed frequency @400MHz).
382  * It uses timing bank registers set 1.
383  */
384 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
385 {
386 	writel(EXYNOS5_AREF_NORMAL,
387 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
388 
389 	writel(dmc->bypass_timing_row,
390 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
391 	writel(dmc->bypass_timing_row,
392 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
393 	writel(dmc->bypass_timing_data,
394 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
395 	writel(dmc->bypass_timing_data,
396 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
397 	writel(dmc->bypass_timing_power,
398 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
399 	writel(dmc->bypass_timing_power,
400 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
401 }
402 
403 /**
404  * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
405  * @dmc:	device for which the new settings is going to be applied
406  * @target_rate:	target frequency of the DMC
407  *
408  * Low-level function for changing timings for DRAM memory operating from main
409  * clock source (BPLL), which can have different frequencies. Thus, each
410  * frequency must have corresponding timings register values in order to keep
411  * the needed delays.
412  * It uses timing bank registers set 0.
413  */
414 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
415 				       unsigned long target_rate)
416 {
417 	int idx;
418 
419 	for (idx = dmc->opp_count - 1; idx >= 0; idx--)
420 		if (dmc->opp[idx].freq_hz <= target_rate)
421 			break;
422 
423 	if (idx < 0)
424 		return -EINVAL;
425 
426 	writel(EXYNOS5_AREF_NORMAL,
427 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
428 
429 	writel(dmc->timing_row[idx],
430 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
431 	writel(dmc->timing_row[idx],
432 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
433 	writel(dmc->timing_data[idx],
434 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
435 	writel(dmc->timing_data[idx],
436 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
437 	writel(dmc->timing_power[idx],
438 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
439 	writel(dmc->timing_power[idx],
440 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
441 
442 	return 0;
443 }
444 
445 /**
446  * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
447  * @dmc:	device for which it is going to be set
448  * @target_volt:	new voltage which is chosen to be final
449  *
450  * Function tries to align voltage to the safe level for 'normal' mode.
451  * It checks the need of higher voltage and changes the value. The target
452  * voltage might be lower that currently set and still the system will be
453  * stable.
454  */
455 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
456 					    unsigned long target_volt)
457 {
458 	int ret = 0;
459 
460 	if (dmc->curr_volt <= target_volt)
461 		return 0;
462 
463 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
464 				    target_volt);
465 	if (!ret)
466 		dmc->curr_volt = target_volt;
467 
468 	return ret;
469 }
470 
471 /**
472  * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
473  * @dmc:	device for which it is going to be set
474  * @target_volt:	new voltage which is chosen to be final
475  *
476  * Function tries to align voltage to the safe level for the 'bypass' mode.
477  * It checks the need of higher voltage and changes the value.
478  * The target voltage must not be less than currently needed, because
479  * for current frequency the device might become unstable.
480  */
481 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
482 					    unsigned long target_volt)
483 {
484 	int ret = 0;
485 
486 	if (dmc->curr_volt >= target_volt)
487 		return 0;
488 
489 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
490 				    target_volt);
491 	if (!ret)
492 		dmc->curr_volt = target_volt;
493 
494 	return ret;
495 }
496 
497 /**
498  * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
499  * @dmc:	device for which it is going to be set
500  * @target_rate:	new frequency which is chosen to be final
501  *
502  * Function changes the DRAM timings for the temporary 'bypass' mode.
503  */
504 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
505 						 unsigned long target_rate)
506 {
507 	int idx = find_target_freq_idx(dmc, target_rate);
508 
509 	if (idx < 0)
510 		return -EINVAL;
511 
512 	exynos5_set_bypass_dram_timings(dmc);
513 
514 	return 0;
515 }
516 
517 /**
518  * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
519  * @dmc:	DMC device for which the switching is going to happen
520  * @target_rate:	new frequency which is going to be set as a final
521  * @target_volt:	new voltage which is going to be set as a final
522  *
523  * Function configures DMC and clocks for operating in temporary 'bypass' mode.
524  * This mode is used only temporary but if required, changes voltage and timings
525  * for DRAM chips. It switches the main clock to stable clock source for the
526  * period of the main PLL reconfiguration.
527  */
528 static int
529 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
530 					   unsigned long target_rate,
531 					   unsigned long target_volt)
532 {
533 	int ret;
534 
535 	/*
536 	 * Having higher voltage for a particular frequency does not harm
537 	 * the chip. Use it for the temporary frequency change when one
538 	 * voltage manipulation might be avoided.
539 	 */
540 	ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
541 	if (ret)
542 		return ret;
543 
544 	/*
545 	 * Longer delays for DRAM does not cause crash, the opposite does.
546 	 */
547 	ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
548 	if (ret)
549 		return ret;
550 
551 	/*
552 	 * Delays are long enough, so use them for the new coming clock.
553 	 */
554 	ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
555 
556 	return ret;
557 }
558 
559 /**
560  * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
561  * using safe procedure
562  * @dmc:	device for which the frequency is going to be changed
563  * @target_rate:	requested new frequency
564  * @target_volt:	requested voltage which corresponds to the new frequency
565  *
566  * The DMC frequency change procedure requires a few steps.
567  * The main requirement is to change the clock source in the clk mux
568  * for the time of main clock PLL locking. The assumption is that the
569  * alternative clock source set as parent is stable.
570  * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
571  * clock. This requires alignment in DRAM timing parameters for the new
572  * T-period. There is two bank sets for keeping DRAM
573  * timings: set 0 and set 1. The set 0 is used when main clock source is
574  * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
575  * the two bank sets is part of the process.
576  * The voltage must also be aligned to the minimum required level. There is
577  * this intermediate step with switching to 'bypass' parent clock source.
578  * if the old voltage is lower, it requires an increase of the voltage level.
579  * The complexity of the voltage manipulation is hidden in low level function.
580  * In this function there is last alignment of the voltage level at the end.
581  */
582 static int
583 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
584 				 unsigned long target_rate,
585 				 unsigned long target_volt)
586 {
587 	int ret;
588 
589 	ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
590 							 target_volt);
591 	if (ret)
592 		return ret;
593 
594 	/*
595 	 * Voltage is set at least to a level needed for this frequency,
596 	 * so switching clock source is safe now.
597 	 */
598 	clk_prepare_enable(dmc->fout_spll);
599 	clk_prepare_enable(dmc->mout_spll);
600 	clk_prepare_enable(dmc->mout_mx_mspll_ccore);
601 
602 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
603 	if (ret)
604 		goto disable_clocks;
605 
606 	/*
607 	 * We are safe to increase the timings for current bypass frequency.
608 	 * Thanks to this the settings will be ready for the upcoming clock
609 	 * source change.
610 	 */
611 	exynos5_dram_change_timings(dmc, target_rate);
612 
613 	clk_set_rate(dmc->fout_bpll, target_rate);
614 
615 	ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
616 	if (ret)
617 		goto disable_clocks;
618 
619 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
620 	if (ret)
621 		goto disable_clocks;
622 
623 	/*
624 	 * Make sure if the voltage is not from 'bypass' settings and align to
625 	 * the right level for power efficiency.
626 	 */
627 	ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
628 
629 disable_clocks:
630 	clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
631 	clk_disable_unprepare(dmc->mout_spll);
632 	clk_disable_unprepare(dmc->fout_spll);
633 
634 	return ret;
635 }
636 
637 /**
638  * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
639  * table.
640  * @dmc:	device for which the frequency is going to be changed
641  * @freq:       requested frequency in KHz
642  * @target_rate:	returned frequency which is the same or lower than
643  *			requested
644  * @target_volt:	returned voltage which corresponds to the returned
645  *			frequency
646  * @flags:	devfreq flags provided for this frequency change request
647  *
648  * Function gets requested frequency and checks OPP framework for needed
649  * frequency and voltage. It populates the values 'target_rate' and
650  * 'target_volt' or returns error value when OPP framework fails.
651  */
652 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
653 				     unsigned long *freq,
654 				     unsigned long *target_rate,
655 				     unsigned long *target_volt, u32 flags)
656 {
657 	struct dev_pm_opp *opp;
658 
659 	opp = devfreq_recommended_opp(dmc->dev, freq, flags);
660 	if (IS_ERR(opp))
661 		return PTR_ERR(opp);
662 
663 	*target_rate = dev_pm_opp_get_freq(opp);
664 	*target_volt = dev_pm_opp_get_voltage(opp);
665 	dev_pm_opp_put(opp);
666 
667 	return 0;
668 }
669 
670 /**
671  * exynos5_dmc_target() - Function responsible for changing frequency of DMC
672  * @dev:	device for which the frequency is going to be changed
673  * @freq:	requested frequency in KHz
674  * @flags:	flags provided for this frequency change request
675  *
676  * An entry function provided to the devfreq framework which provides frequency
677  * change of the DMC. The function gets the possible rate from OPP table based
678  * on requested frequency. It calls the next function responsible for the
679  * frequency and voltage change. In case of failure, does not set 'curr_rate'
680  * and returns error value to the framework.
681  */
682 static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
683 			      u32 flags)
684 {
685 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
686 	unsigned long target_rate = 0;
687 	unsigned long target_volt = 0;
688 	int ret;
689 
690 	ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
691 					flags);
692 
693 	if (ret)
694 		return ret;
695 
696 	if (target_rate == dmc->curr_rate)
697 		return 0;
698 
699 	mutex_lock(&dmc->lock);
700 
701 	ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
702 
703 	if (ret) {
704 		mutex_unlock(&dmc->lock);
705 		return ret;
706 	}
707 
708 	dmc->curr_rate = target_rate;
709 
710 	mutex_unlock(&dmc->lock);
711 	return 0;
712 }
713 
714 /**
715  * exynos5_counters_get() - Gets the performance counters values.
716  * @dmc:	device for which the counters are going to be checked
717  * @load_count:	variable which is populated with counter value
718  * @total_count:	variable which is used as 'wall clock' reference
719  *
720  * Function which provides performance counters values. It sums up counters for
721  * two DMC channels. The 'total_count' is used as a reference and max value.
722  * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
723  */
724 static int exynos5_counters_get(struct exynos5_dmc *dmc,
725 				unsigned long *load_count,
726 				unsigned long *total_count)
727 {
728 	unsigned long total = 0;
729 	struct devfreq_event_data event;
730 	int ret, i;
731 
732 	*load_count = 0;
733 
734 	/* Take into account only read+write counters, but stop all */
735 	for (i = 0; i < dmc->num_counters; i++) {
736 		if (!dmc->counter[i])
737 			continue;
738 
739 		ret = devfreq_event_get_event(dmc->counter[i], &event);
740 		if (ret < 0)
741 			return ret;
742 
743 		*load_count += event.load_count;
744 
745 		if (total < event.total_count)
746 			total = event.total_count;
747 	}
748 
749 	*total_count = total;
750 
751 	return 0;
752 }
753 
754 /**
755  * exynos5_dmc_start_perf_events() - Setup and start performance event counters
756  * @dmc:	device for which the counters are going to be checked
757  * @beg_value:	initial value for the counter
758  *
759  * Function which enables needed counters, interrupts and sets initial values
760  * then starts the counters.
761  */
762 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
763 					  u32 beg_value)
764 {
765 	/* Enable interrupts for counter 2 */
766 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
767 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
768 
769 	/* Enable counter 2 and CCNT  */
770 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
771 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
772 
773 	/* Clear overflow flag for all counters */
774 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
775 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
776 
777 	/* Reset all counters */
778 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
779 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
780 
781 	/*
782 	 * Set start value for the counters, the number of samples that
783 	 * will be gathered is calculated as: 0xffffffff - beg_value
784 	 */
785 	writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
786 	writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
787 
788 	/* Start all counters */
789 	writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
790 	writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
791 }
792 
793 /**
794  * exynos5_dmc_perf_events_calc() - Calculate utilization
795  * @dmc:	device for which the counters are going to be checked
796  * @diff_ts:	time between last interrupt and current one
797  *
798  * Function which calculates needed utilization for the devfreq governor.
799  * It prepares values for 'busy_time' and 'total_time' based on elapsed time
800  * between interrupts, which approximates utilization.
801  */
802 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
803 {
804 	/*
805 	 * This is a simple algorithm for managing traffic on DMC.
806 	 * When there is almost no load the counters overflow every 4s,
807 	 * no mater the DMC frequency.
808 	 * The high load might be approximated using linear function.
809 	 * Knowing that, simple calculation can provide 'busy_time' and
810 	 * 'total_time' to the devfreq governor which picks up target
811 	 * frequency.
812 	 * We want a fast ramp up and slow decay in frequency change function.
813 	 */
814 	if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
815 		/*
816 		 * Set higher utilization for the simple_ondemand governor.
817 		 * The governor should increase the frequency of the DMC.
818 		 */
819 		dmc->load = 70;
820 		dmc->total = 100;
821 	} else {
822 		/*
823 		 * Set low utilization for the simple_ondemand governor.
824 		 * The governor should decrease the frequency of the DMC.
825 		 */
826 		dmc->load = 35;
827 		dmc->total = 100;
828 	}
829 
830 	dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
831 }
832 
833 /**
834  * exynos5_dmc_perf_events_check() - Checks the status of the counters
835  * @dmc:	device for which the counters are going to be checked
836  *
837  * Function which is called from threaded IRQ to check the counters state
838  * and to call approximation for the needed utilization.
839  */
840 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
841 {
842 	u32 val;
843 	u64 diff_ts, ts;
844 
845 	ts = ktime_get_ns();
846 
847 	/* Stop all counters */
848 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
849 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
850 
851 	/* Check the source in interrupt flag registers (which channel) */
852 	val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
853 	if (val) {
854 		diff_ts = ts - dmc->last_overflow_ts[0];
855 		dmc->last_overflow_ts[0] = ts;
856 		dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n",  val);
857 	} else {
858 		val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
859 		diff_ts = ts - dmc->last_overflow_ts[1];
860 		dmc->last_overflow_ts[1] = ts;
861 		dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n",  val);
862 	}
863 
864 	exynos5_dmc_perf_events_calc(dmc, diff_ts);
865 
866 	exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
867 }
868 
869 /**
870  * exynos5_dmc_enable_perf_events() - Enable performance events
871  * @dmc:	device for which the counters are going to be checked
872  *
873  * Function which is setup needed environment and enables counters.
874  */
875 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
876 {
877 	u64 ts;
878 
879 	/* Enable Performance Event Clock */
880 	writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
881 	writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
882 
883 	/* Select read transfers as performance event2 */
884 	writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
885 	writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
886 
887 	ts = ktime_get_ns();
888 	dmc->last_overflow_ts[0] = ts;
889 	dmc->last_overflow_ts[1] = ts;
890 
891 	/* Devfreq shouldn't be faster than initialization, play safe though. */
892 	dmc->load = 99;
893 	dmc->total = 100;
894 }
895 
896 /**
897  * exynos5_dmc_disable_perf_events() - Disable performance events
898  * @dmc:	device for which the counters are going to be checked
899  *
900  * Function which stops, disables performance event counters and interrupts.
901  */
902 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
903 {
904 	/* Stop all counters */
905 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
906 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
907 
908 	/* Disable interrupts for counter 2 */
909 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
910 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
911 
912 	/* Disable counter 2 and CCNT  */
913 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
914 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
915 
916 	/* Clear overflow flag for all counters */
917 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
918 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
919 }
920 
921 /**
922  * exynos5_dmc_get_status() - Read current DMC performance statistics.
923  * @dev:	device for which the statistics are requested
924  * @stat:	structure which has statistic fields
925  *
926  * Function reads the DMC performance counters and calculates 'busy_time'
927  * and 'total_time'. To protect from overflow, the values are shifted right
928  * by 10. After read out the counters are setup to count again.
929  */
930 static int exynos5_dmc_get_status(struct device *dev,
931 				  struct devfreq_dev_status *stat)
932 {
933 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
934 	unsigned long load, total;
935 	int ret;
936 
937 	if (dmc->in_irq_mode) {
938 		mutex_lock(&dmc->lock);
939 		stat->current_frequency = dmc->curr_rate;
940 		mutex_unlock(&dmc->lock);
941 
942 		stat->busy_time = dmc->load;
943 		stat->total_time = dmc->total;
944 	} else {
945 		ret = exynos5_counters_get(dmc, &load, &total);
946 		if (ret < 0)
947 			return -EINVAL;
948 
949 		/* To protect from overflow, divide by 1024 */
950 		stat->busy_time = load >> 10;
951 		stat->total_time = total >> 10;
952 
953 		ret = exynos5_counters_set_event(dmc);
954 		if (ret < 0) {
955 			dev_err(dev, "could not set event counter\n");
956 			return ret;
957 		}
958 	}
959 
960 	return 0;
961 }
962 
963 /**
964  * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
965  * @dev:	device for which the framework checks operating frequency
966  * @freq:	returned frequency value
967  *
968  * It returns the currently used frequency of the DMC. The real operating
969  * frequency might be lower when the clock source value could not be divided
970  * to the requested value.
971  */
972 static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
973 {
974 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
975 
976 	mutex_lock(&dmc->lock);
977 	*freq = dmc->curr_rate;
978 	mutex_unlock(&dmc->lock);
979 
980 	return 0;
981 }
982 
983 /*
984  * exynos5_dmc_df_profile - Devfreq governor's profile structure
985  *
986  * It provides to the devfreq framework needed functions and polling period.
987  */
988 static struct devfreq_dev_profile exynos5_dmc_df_profile = {
989 	.timer = DEVFREQ_TIMER_DELAYED,
990 	.target = exynos5_dmc_target,
991 	.get_dev_status = exynos5_dmc_get_status,
992 	.get_cur_freq = exynos5_dmc_get_cur_freq,
993 };
994 
995 /**
996  * exynos5_dmc_align_init_freq() - Align initial frequency value
997  * @dmc:	device for which the frequency is going to be set
998  * @bootloader_init_freq:	initial frequency set by the bootloader in KHz
999  *
1000  * The initial bootloader frequency, which is present during boot, might be
1001  * different that supported frequency values in the driver. It is possible
1002  * due to different PLL settings or used PLL as a source.
1003  * This function provides the 'initial_freq' for the devfreq framework
1004  * statistics engine which supports only registered values. Thus, some alignment
1005  * must be made.
1006  */
1007 static unsigned long
1008 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
1009 			    unsigned long bootloader_init_freq)
1010 {
1011 	unsigned long aligned_freq;
1012 	int idx;
1013 
1014 	idx = find_target_freq_idx(dmc, bootloader_init_freq);
1015 	if (idx >= 0)
1016 		aligned_freq = dmc->opp[idx].freq_hz;
1017 	else
1018 		aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
1019 
1020 	return aligned_freq;
1021 }
1022 
1023 /**
1024  * create_timings_aligned() - Create register values and align with standard
1025  * @dmc:	device for which the frequency is going to be set
1026  * @reg_timing_row:	array to fill with values for timing row register
1027  * @reg_timing_data:	array to fill with values for timing data register
1028  * @reg_timing_power:	array to fill with values for timing power register
1029  * @clk_period_ps:	the period of the clock, known as tCK
1030  *
1031  * The function calculates timings and creates a register value ready for
1032  * a frequency transition. The register contains a few timings. They are
1033  * shifted by a known offset. The timing value is calculated based on memory
1034  * specyfication: minimal time required and minimal cycles required.
1035  */
1036 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1037 				  u32 *reg_timing_data, u32 *reg_timing_power,
1038 				  u32 clk_period_ps)
1039 {
1040 	u32 val;
1041 	const struct timing_reg *reg;
1042 
1043 	if (clk_period_ps == 0)
1044 		return -EINVAL;
1045 
1046 	*reg_timing_row = 0;
1047 	*reg_timing_data = 0;
1048 	*reg_timing_power = 0;
1049 
1050 	val = dmc->timings->tRFC / clk_period_ps;
1051 	val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1052 	val = max(val, dmc->min_tck->tRFC);
1053 	reg = &timing_row_reg_fields[0];
1054 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1055 
1056 	val = dmc->timings->tRRD / clk_period_ps;
1057 	val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1058 	val = max(val, dmc->min_tck->tRRD);
1059 	reg = &timing_row_reg_fields[1];
1060 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1061 
1062 	val = dmc->timings->tRPab / clk_period_ps;
1063 	val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1064 	val = max(val, dmc->min_tck->tRPab);
1065 	reg = &timing_row_reg_fields[2];
1066 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1067 
1068 	val = dmc->timings->tRCD / clk_period_ps;
1069 	val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1070 	val = max(val, dmc->min_tck->tRCD);
1071 	reg = &timing_row_reg_fields[3];
1072 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1073 
1074 	val = dmc->timings->tRC / clk_period_ps;
1075 	val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1076 	val = max(val, dmc->min_tck->tRC);
1077 	reg = &timing_row_reg_fields[4];
1078 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1079 
1080 	val = dmc->timings->tRAS / clk_period_ps;
1081 	val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1082 	val = max(val, dmc->min_tck->tRAS);
1083 	reg = &timing_row_reg_fields[5];
1084 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1085 
1086 	/* data related timings */
1087 	val = dmc->timings->tWTR / clk_period_ps;
1088 	val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1089 	val = max(val, dmc->min_tck->tWTR);
1090 	reg = &timing_data_reg_fields[0];
1091 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1092 
1093 	val = dmc->timings->tWR / clk_period_ps;
1094 	val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1095 	val = max(val, dmc->min_tck->tWR);
1096 	reg = &timing_data_reg_fields[1];
1097 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1098 
1099 	val = dmc->timings->tRTP / clk_period_ps;
1100 	val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1101 	val = max(val, dmc->min_tck->tRTP);
1102 	reg = &timing_data_reg_fields[2];
1103 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1104 
1105 	val = dmc->timings->tW2W_C2C / clk_period_ps;
1106 	val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1107 	val = max(val, dmc->min_tck->tW2W_C2C);
1108 	reg = &timing_data_reg_fields[3];
1109 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1110 
1111 	val = dmc->timings->tR2R_C2C / clk_period_ps;
1112 	val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1113 	val = max(val, dmc->min_tck->tR2R_C2C);
1114 	reg = &timing_data_reg_fields[4];
1115 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1116 
1117 	val = dmc->timings->tWL / clk_period_ps;
1118 	val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1119 	val = max(val, dmc->min_tck->tWL);
1120 	reg = &timing_data_reg_fields[5];
1121 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1122 
1123 	val = dmc->timings->tDQSCK / clk_period_ps;
1124 	val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1125 	val = max(val, dmc->min_tck->tDQSCK);
1126 	reg = &timing_data_reg_fields[6];
1127 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1128 
1129 	val = dmc->timings->tRL / clk_period_ps;
1130 	val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1131 	val = max(val, dmc->min_tck->tRL);
1132 	reg = &timing_data_reg_fields[7];
1133 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1134 
1135 	/* power related timings */
1136 	val = dmc->timings->tFAW / clk_period_ps;
1137 	val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1138 	val = max(val, dmc->min_tck->tFAW);
1139 	reg = &timing_power_reg_fields[0];
1140 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1141 
1142 	val = dmc->timings->tXSR / clk_period_ps;
1143 	val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1144 	val = max(val, dmc->min_tck->tXSR);
1145 	reg = &timing_power_reg_fields[1];
1146 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1147 
1148 	val = dmc->timings->tXP / clk_period_ps;
1149 	val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1150 	val = max(val, dmc->min_tck->tXP);
1151 	reg = &timing_power_reg_fields[2];
1152 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1153 
1154 	val = dmc->timings->tCKE / clk_period_ps;
1155 	val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1156 	val = max(val, dmc->min_tck->tCKE);
1157 	reg = &timing_power_reg_fields[3];
1158 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1159 
1160 	val = dmc->timings->tMRD / clk_period_ps;
1161 	val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1162 	val = max(val, dmc->min_tck->tMRD);
1163 	reg = &timing_power_reg_fields[4];
1164 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1165 
1166 	return 0;
1167 }
1168 
1169 /**
1170  * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1171  * @dmc:        device for which the frequency is going to be set
1172  *
1173  * The function parses DT entries with DRAM information.
1174  */
1175 static int of_get_dram_timings(struct exynos5_dmc *dmc)
1176 {
1177 	int ret = 0;
1178 	int idx;
1179 	struct device_node *np_ddr;
1180 	u32 freq_mhz, clk_period_ps;
1181 
1182 	np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1183 	if (!np_ddr) {
1184 		dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1185 		return -EINVAL;
1186 	}
1187 
1188 	dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1189 					     sizeof(u32), GFP_KERNEL);
1190 	if (!dmc->timing_row) {
1191 		ret = -ENOMEM;
1192 		goto put_node;
1193 	}
1194 
1195 	dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1196 					      sizeof(u32), GFP_KERNEL);
1197 	if (!dmc->timing_data) {
1198 		ret = -ENOMEM;
1199 		goto put_node;
1200 	}
1201 
1202 	dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1203 					       sizeof(u32), GFP_KERNEL);
1204 	if (!dmc->timing_power) {
1205 		ret = -ENOMEM;
1206 		goto put_node;
1207 	}
1208 
1209 	dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1210 						 DDR_TYPE_LPDDR3,
1211 						 &dmc->timings_arr_size);
1212 	if (!dmc->timings) {
1213 		dev_warn(dmc->dev, "could not get timings from DT\n");
1214 		ret = -EINVAL;
1215 		goto put_node;
1216 	}
1217 
1218 	dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1219 	if (!dmc->min_tck) {
1220 		dev_warn(dmc->dev, "could not get tck from DT\n");
1221 		ret = -EINVAL;
1222 		goto put_node;
1223 	}
1224 
1225 	/* Sorted array of OPPs with frequency ascending */
1226 	for (idx = 0; idx < dmc->opp_count; idx++) {
1227 		freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1228 		clk_period_ps = 1000000 / freq_mhz;
1229 
1230 		ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1231 					     &dmc->timing_data[idx],
1232 					     &dmc->timing_power[idx],
1233 					     clk_period_ps);
1234 	}
1235 
1236 
1237 	/* Take the highest frequency's timings as 'bypass' */
1238 	dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1239 	dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1240 	dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1241 
1242 put_node:
1243 	of_node_put(np_ddr);
1244 	return ret;
1245 }
1246 
1247 /**
1248  * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1249  * @dmc:	DMC structure containing needed fields
1250  *
1251  * Get the needed clocks defined in DT device, enable and set the right parents.
1252  * Read current frequency and initialize the initial rate for governor.
1253  */
1254 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1255 {
1256 	int ret;
1257 	unsigned long target_volt = 0;
1258 	unsigned long target_rate = 0;
1259 	unsigned int tmp;
1260 
1261 	dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1262 	if (IS_ERR(dmc->fout_spll))
1263 		return PTR_ERR(dmc->fout_spll);
1264 
1265 	dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1266 	if (IS_ERR(dmc->fout_bpll))
1267 		return PTR_ERR(dmc->fout_bpll);
1268 
1269 	dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1270 	if (IS_ERR(dmc->mout_mclk_cdrex))
1271 		return PTR_ERR(dmc->mout_mclk_cdrex);
1272 
1273 	dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1274 	if (IS_ERR(dmc->mout_bpll))
1275 		return PTR_ERR(dmc->mout_bpll);
1276 
1277 	dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1278 						"mout_mx_mspll_ccore");
1279 	if (IS_ERR(dmc->mout_mx_mspll_ccore))
1280 		return PTR_ERR(dmc->mout_mx_mspll_ccore);
1281 
1282 	dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1283 	if (IS_ERR(dmc->mout_spll)) {
1284 		dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1285 		if (IS_ERR(dmc->mout_spll))
1286 			return PTR_ERR(dmc->mout_spll);
1287 	}
1288 
1289 	/*
1290 	 * Convert frequency to KHz values and set it for the governor.
1291 	 */
1292 	dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1293 	dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1294 	exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1295 
1296 	ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1297 					&target_volt, 0);
1298 	if (ret)
1299 		return ret;
1300 
1301 	dmc->curr_volt = target_volt;
1302 
1303 	ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1304 	if (ret)
1305 		return ret;
1306 
1307 	clk_prepare_enable(dmc->fout_bpll);
1308 	clk_prepare_enable(dmc->mout_bpll);
1309 
1310 	/*
1311 	 * Some bootloaders do not set clock routes correctly.
1312 	 * Stop one path in clocks to PHY.
1313 	 */
1314 	regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1315 	tmp &= ~(BIT(1) | BIT(0));
1316 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1317 
1318 	return 0;
1319 }
1320 
1321 /**
1322  * exynos5_performance_counters_init() - Initializes performance DMC's counters
1323  * @dmc:	DMC for which it does the setup
1324  *
1325  * Initialization of performance counters in DMC for estimating usage.
1326  * The counter's values are used for calculation of a memory bandwidth and based
1327  * on that the governor changes the frequency.
1328  * The counters are not used when the governor is GOVERNOR_USERSPACE.
1329  */
1330 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1331 {
1332 	int ret, i;
1333 
1334 	dmc->num_counters = devfreq_event_get_edev_count(dmc->dev,
1335 							"devfreq-events");
1336 	if (dmc->num_counters < 0) {
1337 		dev_err(dmc->dev, "could not get devfreq-event counters\n");
1338 		return dmc->num_counters;
1339 	}
1340 
1341 	dmc->counter = devm_kcalloc(dmc->dev, dmc->num_counters,
1342 				    sizeof(*dmc->counter), GFP_KERNEL);
1343 	if (!dmc->counter)
1344 		return -ENOMEM;
1345 
1346 	for (i = 0; i < dmc->num_counters; i++) {
1347 		dmc->counter[i] =
1348 			devfreq_event_get_edev_by_phandle(dmc->dev,
1349 						"devfreq-events", i);
1350 		if (IS_ERR_OR_NULL(dmc->counter[i]))
1351 			return -EPROBE_DEFER;
1352 	}
1353 
1354 	ret = exynos5_counters_enable_edev(dmc);
1355 	if (ret < 0) {
1356 		dev_err(dmc->dev, "could not enable event counter\n");
1357 		return ret;
1358 	}
1359 
1360 	ret = exynos5_counters_set_event(dmc);
1361 	if (ret < 0) {
1362 		exynos5_counters_disable_edev(dmc);
1363 		dev_err(dmc->dev, "could not set event counter\n");
1364 		return ret;
1365 	}
1366 
1367 	return 0;
1368 }
1369 
1370 /**
1371  * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1372  * @dmc:	device which is used for changing this feature
1373  *
1374  * There is a need of pausing DREX DMC when divider or MUX in clock tree
1375  * changes its configuration. In such situation access to the memory is blocked
1376  * in DMC automatically. This feature is used when clock frequency change
1377  * request appears and touches clock tree.
1378  */
1379 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1380 {
1381 	unsigned int val;
1382 	int ret;
1383 
1384 	ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1385 	if (ret)
1386 		return ret;
1387 
1388 	val |= 1UL;
1389 	regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1390 
1391 	return 0;
1392 }
1393 
1394 static irqreturn_t dmc_irq_thread(int irq, void *priv)
1395 {
1396 	int res;
1397 	struct exynos5_dmc *dmc = priv;
1398 
1399 	mutex_lock(&dmc->df->lock);
1400 	exynos5_dmc_perf_events_check(dmc);
1401 	res = update_devfreq(dmc->df);
1402 	mutex_unlock(&dmc->df->lock);
1403 
1404 	if (res)
1405 		dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1406 
1407 	return IRQ_HANDLED;
1408 }
1409 
1410 /**
1411  * exynos5_dmc_probe() - Probe function for the DMC driver
1412  * @pdev:	platform device for which the driver is going to be initialized
1413  *
1414  * Initialize basic components: clocks, regulators, performance counters, etc.
1415  * Read out product version and based on the information setup
1416  * internal structures for the controller (frequency and voltage) and for DRAM
1417  * memory parameters: timings for each operating frequency.
1418  * Register new devfreq device for controlling DVFS of the DMC.
1419  */
1420 static int exynos5_dmc_probe(struct platform_device *pdev)
1421 {
1422 	int ret = 0;
1423 	struct device *dev = &pdev->dev;
1424 	struct device_node *np = dev->of_node;
1425 	struct exynos5_dmc *dmc;
1426 	int irq[2];
1427 
1428 	dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1429 	if (!dmc)
1430 		return -ENOMEM;
1431 
1432 	mutex_init(&dmc->lock);
1433 
1434 	dmc->dev = dev;
1435 	platform_set_drvdata(pdev, dmc);
1436 
1437 	dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1438 	if (IS_ERR(dmc->base_drexi0))
1439 		return PTR_ERR(dmc->base_drexi0);
1440 
1441 	dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1442 	if (IS_ERR(dmc->base_drexi1))
1443 		return PTR_ERR(dmc->base_drexi1);
1444 
1445 	dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1446 							  "samsung,syscon-clk");
1447 	if (IS_ERR(dmc->clk_regmap))
1448 		return PTR_ERR(dmc->clk_regmap);
1449 
1450 	ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1451 	if (ret) {
1452 		dev_warn(dev, "couldn't initialize frequency settings\n");
1453 		return ret;
1454 	}
1455 
1456 	dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1457 	if (IS_ERR(dmc->vdd_mif)) {
1458 		ret = PTR_ERR(dmc->vdd_mif);
1459 		return ret;
1460 	}
1461 
1462 	ret = exynos5_dmc_init_clks(dmc);
1463 	if (ret)
1464 		return ret;
1465 
1466 	ret = of_get_dram_timings(dmc);
1467 	if (ret) {
1468 		dev_warn(dev, "couldn't initialize timings settings\n");
1469 		goto remove_clocks;
1470 	}
1471 
1472 	ret = exynos5_dmc_set_pause_on_switching(dmc);
1473 	if (ret) {
1474 		dev_warn(dev, "couldn't get access to PAUSE register\n");
1475 		goto remove_clocks;
1476 	}
1477 
1478 	/* There is two modes in which the driver works: polling or IRQ */
1479 	irq[0] = platform_get_irq_byname(pdev, "drex_0");
1480 	irq[1] = platform_get_irq_byname(pdev, "drex_1");
1481 	if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1482 		ret = devm_request_threaded_irq(dev, irq[0], NULL,
1483 						dmc_irq_thread, IRQF_ONESHOT,
1484 						dev_name(dev), dmc);
1485 		if (ret) {
1486 			dev_err(dev, "couldn't grab IRQ\n");
1487 			goto remove_clocks;
1488 		}
1489 
1490 		ret = devm_request_threaded_irq(dev, irq[1], NULL,
1491 						dmc_irq_thread, IRQF_ONESHOT,
1492 						dev_name(dev), dmc);
1493 		if (ret) {
1494 			dev_err(dev, "couldn't grab IRQ\n");
1495 			goto remove_clocks;
1496 		}
1497 
1498 		/*
1499 		 * Setup default thresholds for the devfreq governor.
1500 		 * The values are chosen based on experiments.
1501 		 */
1502 		dmc->gov_data.upthreshold = 55;
1503 		dmc->gov_data.downdifferential = 5;
1504 
1505 		exynos5_dmc_enable_perf_events(dmc);
1506 
1507 		dmc->in_irq_mode = 1;
1508 	} else {
1509 		ret = exynos5_performance_counters_init(dmc);
1510 		if (ret) {
1511 			dev_warn(dev, "couldn't probe performance counters\n");
1512 			goto remove_clocks;
1513 		}
1514 
1515 		/*
1516 		 * Setup default thresholds for the devfreq governor.
1517 		 * The values are chosen based on experiments.
1518 		 */
1519 		dmc->gov_data.upthreshold = 10;
1520 		dmc->gov_data.downdifferential = 5;
1521 
1522 		exynos5_dmc_df_profile.polling_ms = 100;
1523 	}
1524 
1525 	dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1526 					  DEVFREQ_GOV_SIMPLE_ONDEMAND,
1527 					  &dmc->gov_data);
1528 
1529 	if (IS_ERR(dmc->df)) {
1530 		ret = PTR_ERR(dmc->df);
1531 		goto err_devfreq_add;
1532 	}
1533 
1534 	if (dmc->in_irq_mode)
1535 		exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1536 
1537 	dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1538 
1539 	return 0;
1540 
1541 err_devfreq_add:
1542 	if (dmc->in_irq_mode)
1543 		exynos5_dmc_disable_perf_events(dmc);
1544 	else
1545 		exynos5_counters_disable_edev(dmc);
1546 remove_clocks:
1547 	clk_disable_unprepare(dmc->mout_bpll);
1548 	clk_disable_unprepare(dmc->fout_bpll);
1549 
1550 	return ret;
1551 }
1552 
1553 /**
1554  * exynos5_dmc_remove() - Remove function for the platform device
1555  * @pdev:	platform device which is going to be removed
1556  *
1557  * The function relies on 'devm' framework function which automatically
1558  * clean the device's resources. It just calls explicitly disable function for
1559  * the performance counters.
1560  */
1561 static int exynos5_dmc_remove(struct platform_device *pdev)
1562 {
1563 	struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1564 
1565 	if (dmc->in_irq_mode)
1566 		exynos5_dmc_disable_perf_events(dmc);
1567 	else
1568 		exynos5_counters_disable_edev(dmc);
1569 
1570 	clk_disable_unprepare(dmc->mout_bpll);
1571 	clk_disable_unprepare(dmc->fout_bpll);
1572 
1573 	return 0;
1574 }
1575 
1576 static const struct of_device_id exynos5_dmc_of_match[] = {
1577 	{ .compatible = "samsung,exynos5422-dmc", },
1578 	{ },
1579 };
1580 MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1581 
1582 static struct platform_driver exynos5_dmc_platdrv = {
1583 	.probe	= exynos5_dmc_probe,
1584 	.remove = exynos5_dmc_remove,
1585 	.driver = {
1586 		.name	= "exynos5-dmc",
1587 		.of_match_table = exynos5_dmc_of_match,
1588 	},
1589 };
1590 module_platform_driver(exynos5_dmc_platdrv);
1591 MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1592 MODULE_LICENSE("GPL v2");
1593 MODULE_AUTHOR("Lukasz Luba");
1594