1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4  * Author: Lukasz Luba <l.luba@partner.samsung.com>
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/devfreq.h>
9 #include <linux/devfreq-event.h>
10 #include <linux/device.h>
11 #include <linux/interrupt.h>
12 #include <linux/io.h>
13 #include <linux/mfd/syscon.h>
14 #include <linux/module.h>
15 #include <linux/moduleparam.h>
16 #include <linux/of_device.h>
17 #include <linux/pm_opp.h>
18 #include <linux/platform_device.h>
19 #include <linux/regmap.h>
20 #include <linux/regulator/consumer.h>
21 #include <linux/slab.h>
22 #include "../jedec_ddr.h"
23 #include "../of_memory.h"
24 
25 static int irqmode;
26 module_param(irqmode, int, 0644);
27 MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
28 
29 #define EXYNOS5_DREXI_TIMINGAREF		(0x0030)
30 #define EXYNOS5_DREXI_TIMINGROW0		(0x0034)
31 #define EXYNOS5_DREXI_TIMINGDATA0		(0x0038)
32 #define EXYNOS5_DREXI_TIMINGPOWER0		(0x003C)
33 #define EXYNOS5_DREXI_TIMINGROW1		(0x00E4)
34 #define EXYNOS5_DREXI_TIMINGDATA1		(0x00E8)
35 #define EXYNOS5_DREXI_TIMINGPOWER1		(0x00EC)
36 #define CDREX_PAUSE				(0x2091c)
37 #define CDREX_LPDDR3PHY_CON3			(0x20a20)
38 #define CDREX_LPDDR3PHY_CLKM_SRC		(0x20700)
39 #define EXYNOS5_TIMING_SET_SWI			BIT(28)
40 #define USE_MX_MSPLL_TIMINGS			(1)
41 #define USE_BPLL_TIMINGS			(0)
42 #define EXYNOS5_AREF_NORMAL			(0x2e)
43 
44 #define DREX_PPCCLKCON		(0x0130)
45 #define DREX_PEREV2CONFIG	(0x013c)
46 #define DREX_PMNC_PPC		(0xE000)
47 #define DREX_CNTENS_PPC		(0xE010)
48 #define DREX_CNTENC_PPC		(0xE020)
49 #define DREX_INTENS_PPC		(0xE030)
50 #define DREX_INTENC_PPC		(0xE040)
51 #define DREX_FLAG_PPC		(0xE050)
52 #define DREX_PMCNT2_PPC		(0xE130)
53 
54 /*
55  * A value for register DREX_PMNC_PPC which should be written to reset
56  * the cycle counter CCNT (a reference wall clock). It sets zero to the
57  * CCNT counter.
58  */
59 #define CC_RESET		BIT(2)
60 
61 /*
62  * A value for register DREX_PMNC_PPC which does the reset of all performance
63  * counters to zero.
64  */
65 #define PPC_COUNTER_RESET	BIT(1)
66 
67 /*
68  * Enables all configured counters (including cycle counter). The value should
69  * be written to the register DREX_PMNC_PPC.
70  */
71 #define PPC_ENABLE		BIT(0)
72 
73 /* A value for register DREX_PPCCLKCON which enables performance events clock.
74  * Must be written before first access to the performance counters register
75  * set, otherwise it could crash.
76  */
77 #define PEREV_CLK_EN		BIT(0)
78 
79 /*
80  * Values which are used to enable counters, interrupts or configure flags of
81  * the performance counters. They configure counter 2 and cycle counter.
82  */
83 #define PERF_CNT2		BIT(2)
84 #define PERF_CCNT		BIT(31)
85 
86 /*
87  * Performance event types which are used for setting the preferred event
88  * to track in the counters.
89  * There is a set of different types, the values are from range 0 to 0x6f.
90  * These settings should be written to the configuration register which manages
91  * the type of the event (register DREX_PEREV2CONFIG).
92  */
93 #define READ_TRANSFER_CH0	(0x6d)
94 #define READ_TRANSFER_CH1	(0x6f)
95 
96 #define PERF_COUNTER_START_VALUE 0xff000000
97 #define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
98 
99 /**
100  * struct dmc_opp_table - Operating level desciption
101  * @freq_hz:		target frequency in Hz
102  * @volt_uv:		target voltage in uV
103  *
104  * Covers frequency and voltage settings of the DMC operating mode.
105  */
106 struct dmc_opp_table {
107 	u32 freq_hz;
108 	u32 volt_uv;
109 };
110 
111 /**
112  * struct exynos5_dmc - main structure describing DMC device
113  * @dev:		DMC device
114  * @df:			devfreq device structure returned by devfreq framework
115  * @gov_data:		configuration of devfreq governor
116  * @base_drexi0:	DREX0 registers mapping
117  * @base_drexi1:	DREX1 registers mapping
118  * @clk_regmap:		regmap for clock controller registers
119  * @lock:		protects curr_rate and frequency/voltage setting section
120  * @curr_rate:		current frequency
121  * @curr_volt:		current voltage
122  * @opp:		OPP table
123  * @opp_count:		number of 'opp' elements
124  * @timings_arr_size:	number of 'timings' elements
125  * @timing_row:		values for timing row register, for each OPP
126  * @timing_data:	values for timing data register, for each OPP
127  * @timing_power:	balues for timing power register, for each OPP
128  * @timings:		DDR memory timings, from device tree
129  * @min_tck:		DDR memory minimum timing values, from device tree
130  * @bypass_timing_row:	value for timing row register for bypass timings
131  * @bypass_timing_data:	value for timing data register for bypass timings
132  * @bypass_timing_power:	value for timing power register for bypass
133  *				timings
134  * @vdd_mif:		Memory interface regulator
135  * @fout_spll:		clock: SPLL
136  * @fout_bpll:		clock: BPLL
137  * @mout_spll:		clock: mux SPLL
138  * @mout_bpll:		clock: mux BPLL
139  * @mout_mclk_cdrex:	clock: mux mclk_cdrex
140  * @mout_mx_mspll_ccore:	clock: mux mx_mspll_ccore
141  * @counter:		devfreq events
142  * @num_counters:	number of 'counter' elements
143  * @last_overflow_ts:	time (in ns) of last overflow of each DREX
144  * @load:		utilization in percents
145  * @total:		total time between devfreq events
146  * @in_irq_mode:	whether running in interrupt mode (true)
147  *			or polling (false)
148  *
149  * The main structure for the Dynamic Memory Controller which covers clocks,
150  * memory regions, HW information, parameters and current operating mode.
151  */
152 struct exynos5_dmc {
153 	struct device *dev;
154 	struct devfreq *df;
155 	struct devfreq_simple_ondemand_data gov_data;
156 	void __iomem *base_drexi0;
157 	void __iomem *base_drexi1;
158 	struct regmap *clk_regmap;
159 	/* Protects curr_rate and frequency/voltage setting section */
160 	struct mutex lock;
161 	unsigned long curr_rate;
162 	unsigned long curr_volt;
163 	struct dmc_opp_table *opp;
164 	int opp_count;
165 	u32 timings_arr_size;
166 	u32 *timing_row;
167 	u32 *timing_data;
168 	u32 *timing_power;
169 	const struct lpddr3_timings *timings;
170 	const struct lpddr3_min_tck *min_tck;
171 	u32 bypass_timing_row;
172 	u32 bypass_timing_data;
173 	u32 bypass_timing_power;
174 	struct regulator *vdd_mif;
175 	struct clk *fout_spll;
176 	struct clk *fout_bpll;
177 	struct clk *mout_spll;
178 	struct clk *mout_bpll;
179 	struct clk *mout_mclk_cdrex;
180 	struct clk *mout_mx_mspll_ccore;
181 	struct devfreq_event_dev **counter;
182 	int num_counters;
183 	u64 last_overflow_ts[2];
184 	unsigned long load;
185 	unsigned long total;
186 	bool in_irq_mode;
187 };
188 
189 #define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
190 	{ .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
191 
192 #define TIMING_VAL2REG(timing, t_val)			\
193 ({							\
194 		u32 __val;				\
195 		__val = (t_val) << (timing)->bit_beg;	\
196 		__val;					\
197 })
198 
199 struct timing_reg {
200 	char *name;
201 	int bit_beg;
202 	int bit_end;
203 	unsigned int val;
204 };
205 
206 static const struct timing_reg timing_row_reg_fields[] = {
207 	TIMING_FIELD("tRFC", 24, 31),
208 	TIMING_FIELD("tRRD", 20, 23),
209 	TIMING_FIELD("tRP", 16, 19),
210 	TIMING_FIELD("tRCD", 12, 15),
211 	TIMING_FIELD("tRC", 6, 11),
212 	TIMING_FIELD("tRAS", 0, 5),
213 };
214 
215 static const struct timing_reg timing_data_reg_fields[] = {
216 	TIMING_FIELD("tWTR", 28, 31),
217 	TIMING_FIELD("tWR", 24, 27),
218 	TIMING_FIELD("tRTP", 20, 23),
219 	TIMING_FIELD("tW2W-C2C", 14, 14),
220 	TIMING_FIELD("tR2R-C2C", 12, 12),
221 	TIMING_FIELD("WL", 8, 11),
222 	TIMING_FIELD("tDQSCK", 4, 7),
223 	TIMING_FIELD("RL", 0, 3),
224 };
225 
226 static const struct timing_reg timing_power_reg_fields[] = {
227 	TIMING_FIELD("tFAW", 26, 31),
228 	TIMING_FIELD("tXSR", 16, 25),
229 	TIMING_FIELD("tXP", 8, 15),
230 	TIMING_FIELD("tCKE", 4, 7),
231 	TIMING_FIELD("tMRD", 0, 3),
232 };
233 
234 #define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \
235 		      ARRAY_SIZE(timing_data_reg_fields) + \
236 		      ARRAY_SIZE(timing_power_reg_fields))
237 
238 static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
239 {
240 	int i, ret;
241 
242 	for (i = 0; i < dmc->num_counters; i++) {
243 		if (!dmc->counter[i])
244 			continue;
245 		ret = devfreq_event_set_event(dmc->counter[i]);
246 		if (ret < 0)
247 			return ret;
248 	}
249 	return 0;
250 }
251 
252 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
253 {
254 	int i, ret;
255 
256 	for (i = 0; i < dmc->num_counters; i++) {
257 		if (!dmc->counter[i])
258 			continue;
259 		ret = devfreq_event_enable_edev(dmc->counter[i]);
260 		if (ret < 0)
261 			return ret;
262 	}
263 	return 0;
264 }
265 
266 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
267 {
268 	int i, ret;
269 
270 	for (i = 0; i < dmc->num_counters; i++) {
271 		if (!dmc->counter[i])
272 			continue;
273 		ret = devfreq_event_disable_edev(dmc->counter[i]);
274 		if (ret < 0)
275 			return ret;
276 	}
277 	return 0;
278 }
279 
280 /**
281  * find_target_freq_id() - Finds requested frequency in local DMC configuration
282  * @dmc:	device for which the information is checked
283  * @target_rate:	requested frequency in KHz
284  *
285  * Seeks in the local DMC driver structure for the requested frequency value
286  * and returns index or error value.
287  */
288 static int find_target_freq_idx(struct exynos5_dmc *dmc,
289 				unsigned long target_rate)
290 {
291 	int i;
292 
293 	for (i = dmc->opp_count - 1; i >= 0; i--)
294 		if (dmc->opp[i].freq_hz <= target_rate)
295 			return i;
296 
297 	return -EINVAL;
298 }
299 
300 /**
301  * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
302  * @dmc:	device for which the new settings is going to be applied
303  * @set:	boolean variable passing set value
304  *
305  * Changes the register set, which holds timing parameters.
306  * There is two register sets: 0 and 1. The register set 0
307  * is used in normal operation when the clock is provided from main PLL.
308  * The bank register set 1 is used when the main PLL frequency is going to be
309  * changed and the clock is taken from alternative, stable source.
310  * This function switches between these banks according to the
311  * currently used clock source.
312  */
313 static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
314 {
315 	unsigned int reg;
316 	int ret;
317 
318 	ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, &reg);
319 	if (ret)
320 		return ret;
321 
322 	if (set)
323 		reg |= EXYNOS5_TIMING_SET_SWI;
324 	else
325 		reg &= ~EXYNOS5_TIMING_SET_SWI;
326 
327 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
328 
329 	return 0;
330 }
331 
332 /**
333  * exynos5_init_freq_table() - Initialized PM OPP framework
334  * @dmc:	DMC device for which the frequencies are used for OPP init
335  * @profile:	devfreq device's profile
336  *
337  * Populate the devfreq device's OPP table based on current frequency, voltage.
338  */
339 static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
340 				   struct devfreq_dev_profile *profile)
341 {
342 	int i, ret;
343 	int idx;
344 	unsigned long freq;
345 
346 	ret = dev_pm_opp_of_add_table(dmc->dev);
347 	if (ret < 0) {
348 		dev_err(dmc->dev, "Failed to get OPP table\n");
349 		return ret;
350 	}
351 
352 	dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
353 
354 	dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
355 				      sizeof(struct dmc_opp_table), GFP_KERNEL);
356 	if (!dmc->opp)
357 		goto err_opp;
358 
359 	idx = dmc->opp_count - 1;
360 	for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
361 		struct dev_pm_opp *opp;
362 
363 		opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
364 		if (IS_ERR(opp))
365 			goto err_opp;
366 
367 		dmc->opp[idx - i].freq_hz = freq;
368 		dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
369 
370 		dev_pm_opp_put(opp);
371 	}
372 
373 	return 0;
374 
375 err_opp:
376 	dev_pm_opp_of_remove_table(dmc->dev);
377 
378 	return -EINVAL;
379 }
380 
381 /**
382  * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
383  * @dmc:	device for which the new settings is going to be applied
384  *
385  * Low-level function for changing timings for DRAM memory clocking from
386  * 'bypass' clock source (fixed frequency @400MHz).
387  * It uses timing bank registers set 1.
388  */
389 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
390 {
391 	writel(EXYNOS5_AREF_NORMAL,
392 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
393 
394 	writel(dmc->bypass_timing_row,
395 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
396 	writel(dmc->bypass_timing_row,
397 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
398 	writel(dmc->bypass_timing_data,
399 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
400 	writel(dmc->bypass_timing_data,
401 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
402 	writel(dmc->bypass_timing_power,
403 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
404 	writel(dmc->bypass_timing_power,
405 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
406 }
407 
408 /**
409  * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
410  * @dmc:	device for which the new settings is going to be applied
411  * @target_rate:	target frequency of the DMC
412  *
413  * Low-level function for changing timings for DRAM memory operating from main
414  * clock source (BPLL), which can have different frequencies. Thus, each
415  * frequency must have corresponding timings register values in order to keep
416  * the needed delays.
417  * It uses timing bank registers set 0.
418  */
419 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
420 				       unsigned long target_rate)
421 {
422 	int idx;
423 
424 	for (idx = dmc->opp_count - 1; idx >= 0; idx--)
425 		if (dmc->opp[idx].freq_hz <= target_rate)
426 			break;
427 
428 	if (idx < 0)
429 		return -EINVAL;
430 
431 	writel(EXYNOS5_AREF_NORMAL,
432 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
433 
434 	writel(dmc->timing_row[idx],
435 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
436 	writel(dmc->timing_row[idx],
437 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
438 	writel(dmc->timing_data[idx],
439 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
440 	writel(dmc->timing_data[idx],
441 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
442 	writel(dmc->timing_power[idx],
443 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
444 	writel(dmc->timing_power[idx],
445 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
446 
447 	return 0;
448 }
449 
450 /**
451  * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
452  * @dmc:	device for which it is going to be set
453  * @target_volt:	new voltage which is chosen to be final
454  *
455  * Function tries to align voltage to the safe level for 'normal' mode.
456  * It checks the need of higher voltage and changes the value. The target
457  * voltage might be lower that currently set and still the system will be
458  * stable.
459  */
460 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
461 					    unsigned long target_volt)
462 {
463 	int ret = 0;
464 
465 	if (dmc->curr_volt <= target_volt)
466 		return 0;
467 
468 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
469 				    target_volt);
470 	if (!ret)
471 		dmc->curr_volt = target_volt;
472 
473 	return ret;
474 }
475 
476 /**
477  * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
478  * @dmc:	device for which it is going to be set
479  * @target_volt:	new voltage which is chosen to be final
480  *
481  * Function tries to align voltage to the safe level for the 'bypass' mode.
482  * It checks the need of higher voltage and changes the value.
483  * The target voltage must not be less than currently needed, because
484  * for current frequency the device might become unstable.
485  */
486 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
487 					    unsigned long target_volt)
488 {
489 	int ret = 0;
490 
491 	if (dmc->curr_volt >= target_volt)
492 		return 0;
493 
494 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
495 				    target_volt);
496 	if (!ret)
497 		dmc->curr_volt = target_volt;
498 
499 	return ret;
500 }
501 
502 /**
503  * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
504  * @dmc:	device for which it is going to be set
505  * @target_rate:	new frequency which is chosen to be final
506  *
507  * Function changes the DRAM timings for the temporary 'bypass' mode.
508  */
509 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
510 						 unsigned long target_rate)
511 {
512 	int idx = find_target_freq_idx(dmc, target_rate);
513 
514 	if (idx < 0)
515 		return -EINVAL;
516 
517 	exynos5_set_bypass_dram_timings(dmc);
518 
519 	return 0;
520 }
521 
522 /**
523  * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
524  * @dmc:	DMC device for which the switching is going to happen
525  * @target_rate:	new frequency which is going to be set as a final
526  * @target_volt:	new voltage which is going to be set as a final
527  *
528  * Function configures DMC and clocks for operating in temporary 'bypass' mode.
529  * This mode is used only temporary but if required, changes voltage and timings
530  * for DRAM chips. It switches the main clock to stable clock source for the
531  * period of the main PLL reconfiguration.
532  */
533 static int
534 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
535 					   unsigned long target_rate,
536 					   unsigned long target_volt)
537 {
538 	int ret;
539 
540 	/*
541 	 * Having higher voltage for a particular frequency does not harm
542 	 * the chip. Use it for the temporary frequency change when one
543 	 * voltage manipulation might be avoided.
544 	 */
545 	ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
546 	if (ret)
547 		return ret;
548 
549 	/*
550 	 * Longer delays for DRAM does not cause crash, the opposite does.
551 	 */
552 	ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
553 	if (ret)
554 		return ret;
555 
556 	/*
557 	 * Delays are long enough, so use them for the new coming clock.
558 	 */
559 	ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
560 
561 	return ret;
562 }
563 
564 /**
565  * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
566  * using safe procedure
567  * @dmc:	device for which the frequency is going to be changed
568  * @target_rate:	requested new frequency
569  * @target_volt:	requested voltage which corresponds to the new frequency
570  *
571  * The DMC frequency change procedure requires a few steps.
572  * The main requirement is to change the clock source in the clk mux
573  * for the time of main clock PLL locking. The assumption is that the
574  * alternative clock source set as parent is stable.
575  * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
576  * clock. This requires alignment in DRAM timing parameters for the new
577  * T-period. There is two bank sets for keeping DRAM
578  * timings: set 0 and set 1. The set 0 is used when main clock source is
579  * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
580  * the two bank sets is part of the process.
581  * The voltage must also be aligned to the minimum required level. There is
582  * this intermediate step with switching to 'bypass' parent clock source.
583  * if the old voltage is lower, it requires an increase of the voltage level.
584  * The complexity of the voltage manipulation is hidden in low level function.
585  * In this function there is last alignment of the voltage level at the end.
586  */
587 static int
588 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
589 				 unsigned long target_rate,
590 				 unsigned long target_volt)
591 {
592 	int ret;
593 
594 	ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
595 							 target_volt);
596 	if (ret)
597 		return ret;
598 
599 	/*
600 	 * Voltage is set at least to a level needed for this frequency,
601 	 * so switching clock source is safe now.
602 	 */
603 	clk_prepare_enable(dmc->fout_spll);
604 	clk_prepare_enable(dmc->mout_spll);
605 	clk_prepare_enable(dmc->mout_mx_mspll_ccore);
606 
607 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
608 	if (ret)
609 		goto disable_clocks;
610 
611 	/*
612 	 * We are safe to increase the timings for current bypass frequency.
613 	 * Thanks to this the settings will be ready for the upcoming clock
614 	 * source change.
615 	 */
616 	exynos5_dram_change_timings(dmc, target_rate);
617 
618 	clk_set_rate(dmc->fout_bpll, target_rate);
619 
620 	ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
621 	if (ret)
622 		goto disable_clocks;
623 
624 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
625 	if (ret)
626 		goto disable_clocks;
627 
628 	/*
629 	 * Make sure if the voltage is not from 'bypass' settings and align to
630 	 * the right level for power efficiency.
631 	 */
632 	ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
633 
634 disable_clocks:
635 	clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
636 	clk_disable_unprepare(dmc->mout_spll);
637 	clk_disable_unprepare(dmc->fout_spll);
638 
639 	return ret;
640 }
641 
642 /**
643  * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
644  * table.
645  * @dmc:	device for which the frequency is going to be changed
646  * @freq:       requested frequency in KHz
647  * @target_rate:	returned frequency which is the same or lower than
648  *			requested
649  * @target_volt:	returned voltage which corresponds to the returned
650  *			frequency
651  * @flags:	devfreq flags provided for this frequency change request
652  *
653  * Function gets requested frequency and checks OPP framework for needed
654  * frequency and voltage. It populates the values 'target_rate' and
655  * 'target_volt' or returns error value when OPP framework fails.
656  */
657 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
658 				     unsigned long *freq,
659 				     unsigned long *target_rate,
660 				     unsigned long *target_volt, u32 flags)
661 {
662 	struct dev_pm_opp *opp;
663 
664 	opp = devfreq_recommended_opp(dmc->dev, freq, flags);
665 	if (IS_ERR(opp))
666 		return PTR_ERR(opp);
667 
668 	*target_rate = dev_pm_opp_get_freq(opp);
669 	*target_volt = dev_pm_opp_get_voltage(opp);
670 	dev_pm_opp_put(opp);
671 
672 	return 0;
673 }
674 
675 /**
676  * exynos5_dmc_target() - Function responsible for changing frequency of DMC
677  * @dev:	device for which the frequency is going to be changed
678  * @freq:	requested frequency in KHz
679  * @flags:	flags provided for this frequency change request
680  *
681  * An entry function provided to the devfreq framework which provides frequency
682  * change of the DMC. The function gets the possible rate from OPP table based
683  * on requested frequency. It calls the next function responsible for the
684  * frequency and voltage change. In case of failure, does not set 'curr_rate'
685  * and returns error value to the framework.
686  */
687 static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
688 			      u32 flags)
689 {
690 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
691 	unsigned long target_rate = 0;
692 	unsigned long target_volt = 0;
693 	int ret;
694 
695 	ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
696 					flags);
697 
698 	if (ret)
699 		return ret;
700 
701 	if (target_rate == dmc->curr_rate)
702 		return 0;
703 
704 	mutex_lock(&dmc->lock);
705 
706 	ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
707 
708 	if (ret) {
709 		mutex_unlock(&dmc->lock);
710 		return ret;
711 	}
712 
713 	dmc->curr_rate = target_rate;
714 
715 	mutex_unlock(&dmc->lock);
716 	return 0;
717 }
718 
719 /**
720  * exynos5_counters_get() - Gets the performance counters values.
721  * @dmc:	device for which the counters are going to be checked
722  * @load_count:	variable which is populated with counter value
723  * @total_count:	variable which is used as 'wall clock' reference
724  *
725  * Function which provides performance counters values. It sums up counters for
726  * two DMC channels. The 'total_count' is used as a reference and max value.
727  * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
728  */
729 static int exynos5_counters_get(struct exynos5_dmc *dmc,
730 				unsigned long *load_count,
731 				unsigned long *total_count)
732 {
733 	unsigned long total = 0;
734 	struct devfreq_event_data event;
735 	int ret, i;
736 
737 	*load_count = 0;
738 
739 	/* Take into account only read+write counters, but stop all */
740 	for (i = 0; i < dmc->num_counters; i++) {
741 		if (!dmc->counter[i])
742 			continue;
743 
744 		ret = devfreq_event_get_event(dmc->counter[i], &event);
745 		if (ret < 0)
746 			return ret;
747 
748 		*load_count += event.load_count;
749 
750 		if (total < event.total_count)
751 			total = event.total_count;
752 	}
753 
754 	*total_count = total;
755 
756 	return 0;
757 }
758 
759 /**
760  * exynos5_dmc_start_perf_events() - Setup and start performance event counters
761  * @dmc:	device for which the counters are going to be checked
762  * @beg_value:	initial value for the counter
763  *
764  * Function which enables needed counters, interrupts and sets initial values
765  * then starts the counters.
766  */
767 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
768 					  u32 beg_value)
769 {
770 	/* Enable interrupts for counter 2 */
771 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
772 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
773 
774 	/* Enable counter 2 and CCNT  */
775 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
776 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
777 
778 	/* Clear overflow flag for all counters */
779 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
780 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
781 
782 	/* Reset all counters */
783 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
784 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
785 
786 	/*
787 	 * Set start value for the counters, the number of samples that
788 	 * will be gathered is calculated as: 0xffffffff - beg_value
789 	 */
790 	writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
791 	writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
792 
793 	/* Start all counters */
794 	writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
795 	writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
796 }
797 
798 /**
799  * exynos5_dmc_perf_events_calc() - Calculate utilization
800  * @dmc:	device for which the counters are going to be checked
801  * @diff_ts:	time between last interrupt and current one
802  *
803  * Function which calculates needed utilization for the devfreq governor.
804  * It prepares values for 'busy_time' and 'total_time' based on elapsed time
805  * between interrupts, which approximates utilization.
806  */
807 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
808 {
809 	/*
810 	 * This is a simple algorithm for managing traffic on DMC.
811 	 * When there is almost no load the counters overflow every 4s,
812 	 * no mater the DMC frequency.
813 	 * The high load might be approximated using linear function.
814 	 * Knowing that, simple calculation can provide 'busy_time' and
815 	 * 'total_time' to the devfreq governor which picks up target
816 	 * frequency.
817 	 * We want a fast ramp up and slow decay in frequency change function.
818 	 */
819 	if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
820 		/*
821 		 * Set higher utilization for the simple_ondemand governor.
822 		 * The governor should increase the frequency of the DMC.
823 		 */
824 		dmc->load = 70;
825 		dmc->total = 100;
826 	} else {
827 		/*
828 		 * Set low utilization for the simple_ondemand governor.
829 		 * The governor should decrease the frequency of the DMC.
830 		 */
831 		dmc->load = 35;
832 		dmc->total = 100;
833 	}
834 
835 	dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
836 }
837 
838 /**
839  * exynos5_dmc_perf_events_check() - Checks the status of the counters
840  * @dmc:	device for which the counters are going to be checked
841  *
842  * Function which is called from threaded IRQ to check the counters state
843  * and to call approximation for the needed utilization.
844  */
845 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
846 {
847 	u32 val;
848 	u64 diff_ts, ts;
849 
850 	ts = ktime_get_ns();
851 
852 	/* Stop all counters */
853 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
854 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
855 
856 	/* Check the source in interrupt flag registers (which channel) */
857 	val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
858 	if (val) {
859 		diff_ts = ts - dmc->last_overflow_ts[0];
860 		dmc->last_overflow_ts[0] = ts;
861 		dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n",  val);
862 	} else {
863 		val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
864 		diff_ts = ts - dmc->last_overflow_ts[1];
865 		dmc->last_overflow_ts[1] = ts;
866 		dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n",  val);
867 	}
868 
869 	exynos5_dmc_perf_events_calc(dmc, diff_ts);
870 
871 	exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
872 }
873 
874 /**
875  * exynos5_dmc_enable_perf_events() - Enable performance events
876  * @dmc:	device for which the counters are going to be checked
877  *
878  * Function which is setup needed environment and enables counters.
879  */
880 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
881 {
882 	u64 ts;
883 
884 	/* Enable Performance Event Clock */
885 	writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
886 	writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
887 
888 	/* Select read transfers as performance event2 */
889 	writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
890 	writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
891 
892 	ts = ktime_get_ns();
893 	dmc->last_overflow_ts[0] = ts;
894 	dmc->last_overflow_ts[1] = ts;
895 
896 	/* Devfreq shouldn't be faster than initialization, play safe though. */
897 	dmc->load = 99;
898 	dmc->total = 100;
899 }
900 
901 /**
902  * exynos5_dmc_disable_perf_events() - Disable performance events
903  * @dmc:	device for which the counters are going to be checked
904  *
905  * Function which stops, disables performance event counters and interrupts.
906  */
907 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
908 {
909 	/* Stop all counters */
910 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
911 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
912 
913 	/* Disable interrupts for counter 2 */
914 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
915 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
916 
917 	/* Disable counter 2 and CCNT  */
918 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
919 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
920 
921 	/* Clear overflow flag for all counters */
922 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
923 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
924 }
925 
926 /**
927  * exynos5_dmc_get_status() - Read current DMC performance statistics.
928  * @dev:	device for which the statistics are requested
929  * @stat:	structure which has statistic fields
930  *
931  * Function reads the DMC performance counters and calculates 'busy_time'
932  * and 'total_time'. To protect from overflow, the values are shifted right
933  * by 10. After read out the counters are setup to count again.
934  */
935 static int exynos5_dmc_get_status(struct device *dev,
936 				  struct devfreq_dev_status *stat)
937 {
938 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
939 	unsigned long load, total;
940 	int ret;
941 
942 	if (dmc->in_irq_mode) {
943 		mutex_lock(&dmc->lock);
944 		stat->current_frequency = dmc->curr_rate;
945 		mutex_unlock(&dmc->lock);
946 
947 		stat->busy_time = dmc->load;
948 		stat->total_time = dmc->total;
949 	} else {
950 		ret = exynos5_counters_get(dmc, &load, &total);
951 		if (ret < 0)
952 			return -EINVAL;
953 
954 		/* To protect from overflow, divide by 1024 */
955 		stat->busy_time = load >> 10;
956 		stat->total_time = total >> 10;
957 
958 		ret = exynos5_counters_set_event(dmc);
959 		if (ret < 0) {
960 			dev_err(dev, "could not set event counter\n");
961 			return ret;
962 		}
963 	}
964 
965 	return 0;
966 }
967 
968 /**
969  * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
970  * @dev:	device for which the framework checks operating frequency
971  * @freq:	returned frequency value
972  *
973  * It returns the currently used frequency of the DMC. The real operating
974  * frequency might be lower when the clock source value could not be divided
975  * to the requested value.
976  */
977 static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
978 {
979 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);
980 
981 	mutex_lock(&dmc->lock);
982 	*freq = dmc->curr_rate;
983 	mutex_unlock(&dmc->lock);
984 
985 	return 0;
986 }
987 
988 /*
989  * exynos5_dmc_df_profile - Devfreq governor's profile structure
990  *
991  * It provides to the devfreq framework needed functions and polling period.
992  */
993 static struct devfreq_dev_profile exynos5_dmc_df_profile = {
994 	.timer = DEVFREQ_TIMER_DELAYED,
995 	.target = exynos5_dmc_target,
996 	.get_dev_status = exynos5_dmc_get_status,
997 	.get_cur_freq = exynos5_dmc_get_cur_freq,
998 };
999 
1000 /**
1001  * exynos5_dmc_align_initial_frequency() - Align initial frequency value
1002  * @dmc:	device for which the frequency is going to be set
1003  * @bootloader_init_freq:	initial frequency set by the bootloader in KHz
1004  *
1005  * The initial bootloader frequency, which is present during boot, might be
1006  * different that supported frequency values in the driver. It is possible
1007  * due to different PLL settings or used PLL as a source.
1008  * This function provides the 'initial_freq' for the devfreq framework
1009  * statistics engine which supports only registered values. Thus, some alignment
1010  * must be made.
1011  */
1012 static unsigned long
1013 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
1014 			    unsigned long bootloader_init_freq)
1015 {
1016 	unsigned long aligned_freq;
1017 	int idx;
1018 
1019 	idx = find_target_freq_idx(dmc, bootloader_init_freq);
1020 	if (idx >= 0)
1021 		aligned_freq = dmc->opp[idx].freq_hz;
1022 	else
1023 		aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
1024 
1025 	return aligned_freq;
1026 }
1027 
1028 /**
1029  * create_timings_aligned() - Create register values and align with standard
1030  * @dmc:	device for which the frequency is going to be set
1031  * @reg_timing_row:	array to fill with values for timing row register
1032  * @reg_timing_data:	array to fill with values for timing data register
1033  * @reg_timing_power:	array to fill with values for timing power register
1034  * @clk_period_ps:	the period of the clock, known as tCK
1035  *
1036  * The function calculates timings and creates a register value ready for
1037  * a frequency transition. The register contains a few timings. They are
1038  * shifted by a known offset. The timing value is calculated based on memory
1039  * specyfication: minimal time required and minimal cycles required.
1040  */
1041 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1042 				  u32 *reg_timing_data, u32 *reg_timing_power,
1043 				  u32 clk_period_ps)
1044 {
1045 	u32 val;
1046 	const struct timing_reg *reg;
1047 
1048 	if (clk_period_ps == 0)
1049 		return -EINVAL;
1050 
1051 	*reg_timing_row = 0;
1052 	*reg_timing_data = 0;
1053 	*reg_timing_power = 0;
1054 
1055 	val = dmc->timings->tRFC / clk_period_ps;
1056 	val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1057 	val = max(val, dmc->min_tck->tRFC);
1058 	reg = &timing_row_reg_fields[0];
1059 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1060 
1061 	val = dmc->timings->tRRD / clk_period_ps;
1062 	val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1063 	val = max(val, dmc->min_tck->tRRD);
1064 	reg = &timing_row_reg_fields[1];
1065 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1066 
1067 	val = dmc->timings->tRPab / clk_period_ps;
1068 	val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1069 	val = max(val, dmc->min_tck->tRPab);
1070 	reg = &timing_row_reg_fields[2];
1071 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1072 
1073 	val = dmc->timings->tRCD / clk_period_ps;
1074 	val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1075 	val = max(val, dmc->min_tck->tRCD);
1076 	reg = &timing_row_reg_fields[3];
1077 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1078 
1079 	val = dmc->timings->tRC / clk_period_ps;
1080 	val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1081 	val = max(val, dmc->min_tck->tRC);
1082 	reg = &timing_row_reg_fields[4];
1083 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1084 
1085 	val = dmc->timings->tRAS / clk_period_ps;
1086 	val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1087 	val = max(val, dmc->min_tck->tRAS);
1088 	reg = &timing_row_reg_fields[5];
1089 	*reg_timing_row |= TIMING_VAL2REG(reg, val);
1090 
1091 	/* data related timings */
1092 	val = dmc->timings->tWTR / clk_period_ps;
1093 	val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1094 	val = max(val, dmc->min_tck->tWTR);
1095 	reg = &timing_data_reg_fields[0];
1096 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1097 
1098 	val = dmc->timings->tWR / clk_period_ps;
1099 	val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1100 	val = max(val, dmc->min_tck->tWR);
1101 	reg = &timing_data_reg_fields[1];
1102 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1103 
1104 	val = dmc->timings->tRTP / clk_period_ps;
1105 	val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1106 	val = max(val, dmc->min_tck->tRTP);
1107 	reg = &timing_data_reg_fields[2];
1108 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1109 
1110 	val = dmc->timings->tW2W_C2C / clk_period_ps;
1111 	val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1112 	val = max(val, dmc->min_tck->tW2W_C2C);
1113 	reg = &timing_data_reg_fields[3];
1114 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1115 
1116 	val = dmc->timings->tR2R_C2C / clk_period_ps;
1117 	val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1118 	val = max(val, dmc->min_tck->tR2R_C2C);
1119 	reg = &timing_data_reg_fields[4];
1120 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1121 
1122 	val = dmc->timings->tWL / clk_period_ps;
1123 	val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1124 	val = max(val, dmc->min_tck->tWL);
1125 	reg = &timing_data_reg_fields[5];
1126 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1127 
1128 	val = dmc->timings->tDQSCK / clk_period_ps;
1129 	val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1130 	val = max(val, dmc->min_tck->tDQSCK);
1131 	reg = &timing_data_reg_fields[6];
1132 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1133 
1134 	val = dmc->timings->tRL / clk_period_ps;
1135 	val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1136 	val = max(val, dmc->min_tck->tRL);
1137 	reg = &timing_data_reg_fields[7];
1138 	*reg_timing_data |= TIMING_VAL2REG(reg, val);
1139 
1140 	/* power related timings */
1141 	val = dmc->timings->tFAW / clk_period_ps;
1142 	val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1143 	val = max(val, dmc->min_tck->tFAW);
1144 	reg = &timing_power_reg_fields[0];
1145 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1146 
1147 	val = dmc->timings->tXSR / clk_period_ps;
1148 	val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1149 	val = max(val, dmc->min_tck->tXSR);
1150 	reg = &timing_power_reg_fields[1];
1151 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1152 
1153 	val = dmc->timings->tXP / clk_period_ps;
1154 	val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1155 	val = max(val, dmc->min_tck->tXP);
1156 	reg = &timing_power_reg_fields[2];
1157 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1158 
1159 	val = dmc->timings->tCKE / clk_period_ps;
1160 	val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1161 	val = max(val, dmc->min_tck->tCKE);
1162 	reg = &timing_power_reg_fields[3];
1163 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1164 
1165 	val = dmc->timings->tMRD / clk_period_ps;
1166 	val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1167 	val = max(val, dmc->min_tck->tMRD);
1168 	reg = &timing_power_reg_fields[4];
1169 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
1170 
1171 	return 0;
1172 }
1173 
1174 /**
1175  * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1176  * @dmc:        device for which the frequency is going to be set
1177  *
1178  * The function parses DT entries with DRAM information.
1179  */
1180 static int of_get_dram_timings(struct exynos5_dmc *dmc)
1181 {
1182 	int ret = 0;
1183 	int idx;
1184 	struct device_node *np_ddr;
1185 	u32 freq_mhz, clk_period_ps;
1186 
1187 	np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1188 	if (!np_ddr) {
1189 		dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1190 		return -EINVAL;
1191 	}
1192 
1193 	dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1194 					     sizeof(u32), GFP_KERNEL);
1195 	if (!dmc->timing_row)
1196 		return -ENOMEM;
1197 
1198 	dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1199 					      sizeof(u32), GFP_KERNEL);
1200 	if (!dmc->timing_data)
1201 		return -ENOMEM;
1202 
1203 	dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1204 					       sizeof(u32), GFP_KERNEL);
1205 	if (!dmc->timing_power)
1206 		return -ENOMEM;
1207 
1208 	dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1209 						 DDR_TYPE_LPDDR3,
1210 						 &dmc->timings_arr_size);
1211 	if (!dmc->timings) {
1212 		of_node_put(np_ddr);
1213 		dev_warn(dmc->dev, "could not get timings from DT\n");
1214 		return -EINVAL;
1215 	}
1216 
1217 	dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1218 	if (!dmc->min_tck) {
1219 		of_node_put(np_ddr);
1220 		dev_warn(dmc->dev, "could not get tck from DT\n");
1221 		return -EINVAL;
1222 	}
1223 
1224 	/* Sorted array of OPPs with frequency ascending */
1225 	for (idx = 0; idx < dmc->opp_count; idx++) {
1226 		freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1227 		clk_period_ps = 1000000 / freq_mhz;
1228 
1229 		ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1230 					     &dmc->timing_data[idx],
1231 					     &dmc->timing_power[idx],
1232 					     clk_period_ps);
1233 	}
1234 
1235 	of_node_put(np_ddr);
1236 
1237 	/* Take the highest frequency's timings as 'bypass' */
1238 	dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1239 	dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1240 	dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1241 
1242 	return ret;
1243 }
1244 
1245 /**
1246  * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1247  * @dmc:	DMC structure containing needed fields
1248  *
1249  * Get the needed clocks defined in DT device, enable and set the right parents.
1250  * Read current frequency and initialize the initial rate for governor.
1251  */
1252 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1253 {
1254 	int ret;
1255 	unsigned long target_volt = 0;
1256 	unsigned long target_rate = 0;
1257 	unsigned int tmp;
1258 
1259 	dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1260 	if (IS_ERR(dmc->fout_spll))
1261 		return PTR_ERR(dmc->fout_spll);
1262 
1263 	dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1264 	if (IS_ERR(dmc->fout_bpll))
1265 		return PTR_ERR(dmc->fout_bpll);
1266 
1267 	dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1268 	if (IS_ERR(dmc->mout_mclk_cdrex))
1269 		return PTR_ERR(dmc->mout_mclk_cdrex);
1270 
1271 	dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1272 	if (IS_ERR(dmc->mout_bpll))
1273 		return PTR_ERR(dmc->mout_bpll);
1274 
1275 	dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1276 						"mout_mx_mspll_ccore");
1277 	if (IS_ERR(dmc->mout_mx_mspll_ccore))
1278 		return PTR_ERR(dmc->mout_mx_mspll_ccore);
1279 
1280 	dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1281 	if (IS_ERR(dmc->mout_spll)) {
1282 		dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1283 		if (IS_ERR(dmc->mout_spll))
1284 			return PTR_ERR(dmc->mout_spll);
1285 	}
1286 
1287 	/*
1288 	 * Convert frequency to KHz values and set it for the governor.
1289 	 */
1290 	dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1291 	dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1292 	exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1293 
1294 	ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1295 					&target_volt, 0);
1296 	if (ret)
1297 		return ret;
1298 
1299 	dmc->curr_volt = target_volt;
1300 
1301 	clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1302 
1303 	clk_prepare_enable(dmc->fout_bpll);
1304 	clk_prepare_enable(dmc->mout_bpll);
1305 
1306 	/*
1307 	 * Some bootloaders do not set clock routes correctly.
1308 	 * Stop one path in clocks to PHY.
1309 	 */
1310 	regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1311 	tmp &= ~(BIT(1) | BIT(0));
1312 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1313 
1314 	return 0;
1315 }
1316 
1317 /**
1318  * exynos5_performance_counters_init() - Initializes performance DMC's counters
1319  * @dmc:	DMC for which it does the setup
1320  *
1321  * Initialization of performance counters in DMC for estimating usage.
1322  * The counter's values are used for calculation of a memory bandwidth and based
1323  * on that the governor changes the frequency.
1324  * The counters are not used when the governor is GOVERNOR_USERSPACE.
1325  */
1326 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1327 {
1328 	int counters_size;
1329 	int ret, i;
1330 
1331 	dmc->num_counters = devfreq_event_get_edev_count(dmc->dev,
1332 							"devfreq-events");
1333 	if (dmc->num_counters < 0) {
1334 		dev_err(dmc->dev, "could not get devfreq-event counters\n");
1335 		return dmc->num_counters;
1336 	}
1337 
1338 	counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters;
1339 	dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL);
1340 	if (!dmc->counter)
1341 		return -ENOMEM;
1342 
1343 	for (i = 0; i < dmc->num_counters; i++) {
1344 		dmc->counter[i] =
1345 			devfreq_event_get_edev_by_phandle(dmc->dev,
1346 						"devfreq-events", i);
1347 		if (IS_ERR_OR_NULL(dmc->counter[i]))
1348 			return -EPROBE_DEFER;
1349 	}
1350 
1351 	ret = exynos5_counters_enable_edev(dmc);
1352 	if (ret < 0) {
1353 		dev_err(dmc->dev, "could not enable event counter\n");
1354 		return ret;
1355 	}
1356 
1357 	ret = exynos5_counters_set_event(dmc);
1358 	if (ret < 0) {
1359 		exynos5_counters_disable_edev(dmc);
1360 		dev_err(dmc->dev, "could not set event counter\n");
1361 		return ret;
1362 	}
1363 
1364 	return 0;
1365 }
1366 
1367 /**
1368  * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1369  * @dmc:	device which is used for changing this feature
1370  *
1371  * There is a need of pausing DREX DMC when divider or MUX in clock tree
1372  * changes its configuration. In such situation access to the memory is blocked
1373  * in DMC automatically. This feature is used when clock frequency change
1374  * request appears and touches clock tree.
1375  */
1376 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1377 {
1378 	unsigned int val;
1379 	int ret;
1380 
1381 	ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1382 	if (ret)
1383 		return ret;
1384 
1385 	val |= 1UL;
1386 	regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1387 
1388 	return 0;
1389 }
1390 
1391 static irqreturn_t dmc_irq_thread(int irq, void *priv)
1392 {
1393 	int res;
1394 	struct exynos5_dmc *dmc = priv;
1395 
1396 	mutex_lock(&dmc->df->lock);
1397 	exynos5_dmc_perf_events_check(dmc);
1398 	res = update_devfreq(dmc->df);
1399 	mutex_unlock(&dmc->df->lock);
1400 
1401 	if (res)
1402 		dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1403 
1404 	return IRQ_HANDLED;
1405 }
1406 
1407 /**
1408  * exynos5_dmc_probe() - Probe function for the DMC driver
1409  * @pdev:	platform device for which the driver is going to be initialized
1410  *
1411  * Initialize basic components: clocks, regulators, performance counters, etc.
1412  * Read out product version and based on the information setup
1413  * internal structures for the controller (frequency and voltage) and for DRAM
1414  * memory parameters: timings for each operating frequency.
1415  * Register new devfreq device for controlling DVFS of the DMC.
1416  */
1417 static int exynos5_dmc_probe(struct platform_device *pdev)
1418 {
1419 	int ret = 0;
1420 	struct device *dev = &pdev->dev;
1421 	struct device_node *np = dev->of_node;
1422 	struct exynos5_dmc *dmc;
1423 	int irq[2];
1424 
1425 	dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1426 	if (!dmc)
1427 		return -ENOMEM;
1428 
1429 	mutex_init(&dmc->lock);
1430 
1431 	dmc->dev = dev;
1432 	platform_set_drvdata(pdev, dmc);
1433 
1434 	dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1435 	if (IS_ERR(dmc->base_drexi0))
1436 		return PTR_ERR(dmc->base_drexi0);
1437 
1438 	dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1439 	if (IS_ERR(dmc->base_drexi1))
1440 		return PTR_ERR(dmc->base_drexi1);
1441 
1442 	dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1443 							  "samsung,syscon-clk");
1444 	if (IS_ERR(dmc->clk_regmap))
1445 		return PTR_ERR(dmc->clk_regmap);
1446 
1447 	ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1448 	if (ret) {
1449 		dev_warn(dev, "couldn't initialize frequency settings\n");
1450 		return ret;
1451 	}
1452 
1453 	dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1454 	if (IS_ERR(dmc->vdd_mif)) {
1455 		ret = PTR_ERR(dmc->vdd_mif);
1456 		return ret;
1457 	}
1458 
1459 	ret = exynos5_dmc_init_clks(dmc);
1460 	if (ret)
1461 		return ret;
1462 
1463 	ret = of_get_dram_timings(dmc);
1464 	if (ret) {
1465 		dev_warn(dev, "couldn't initialize timings settings\n");
1466 		goto remove_clocks;
1467 	}
1468 
1469 	ret = exynos5_dmc_set_pause_on_switching(dmc);
1470 	if (ret) {
1471 		dev_warn(dev, "couldn't get access to PAUSE register\n");
1472 		goto remove_clocks;
1473 	}
1474 
1475 	/* There is two modes in which the driver works: polling or IRQ */
1476 	irq[0] = platform_get_irq_byname(pdev, "drex_0");
1477 	irq[1] = platform_get_irq_byname(pdev, "drex_1");
1478 	if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1479 		ret = devm_request_threaded_irq(dev, irq[0], NULL,
1480 						dmc_irq_thread, IRQF_ONESHOT,
1481 						dev_name(dev), dmc);
1482 		if (ret) {
1483 			dev_err(dev, "couldn't grab IRQ\n");
1484 			goto remove_clocks;
1485 		}
1486 
1487 		ret = devm_request_threaded_irq(dev, irq[1], NULL,
1488 						dmc_irq_thread, IRQF_ONESHOT,
1489 						dev_name(dev), dmc);
1490 		if (ret) {
1491 			dev_err(dev, "couldn't grab IRQ\n");
1492 			goto remove_clocks;
1493 		}
1494 
1495 		/*
1496 		 * Setup default thresholds for the devfreq governor.
1497 		 * The values are chosen based on experiments.
1498 		 */
1499 		dmc->gov_data.upthreshold = 55;
1500 		dmc->gov_data.downdifferential = 5;
1501 
1502 		exynos5_dmc_enable_perf_events(dmc);
1503 
1504 		dmc->in_irq_mode = 1;
1505 	} else {
1506 		ret = exynos5_performance_counters_init(dmc);
1507 		if (ret) {
1508 			dev_warn(dev, "couldn't probe performance counters\n");
1509 			goto remove_clocks;
1510 		}
1511 
1512 		/*
1513 		 * Setup default thresholds for the devfreq governor.
1514 		 * The values are chosen based on experiments.
1515 		 */
1516 		dmc->gov_data.upthreshold = 10;
1517 		dmc->gov_data.downdifferential = 5;
1518 
1519 		exynos5_dmc_df_profile.polling_ms = 100;
1520 	}
1521 
1522 	dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1523 					  DEVFREQ_GOV_SIMPLE_ONDEMAND,
1524 					  &dmc->gov_data);
1525 
1526 	if (IS_ERR(dmc->df)) {
1527 		ret = PTR_ERR(dmc->df);
1528 		goto err_devfreq_add;
1529 	}
1530 
1531 	if (dmc->in_irq_mode)
1532 		exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1533 
1534 	dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1535 
1536 	return 0;
1537 
1538 err_devfreq_add:
1539 	if (dmc->in_irq_mode)
1540 		exynos5_dmc_disable_perf_events(dmc);
1541 	else
1542 		exynos5_counters_disable_edev(dmc);
1543 remove_clocks:
1544 	clk_disable_unprepare(dmc->mout_bpll);
1545 	clk_disable_unprepare(dmc->fout_bpll);
1546 
1547 	return ret;
1548 }
1549 
1550 /**
1551  * exynos5_dmc_remove() - Remove function for the platform device
1552  * @pdev:	platform device which is going to be removed
1553  *
1554  * The function relies on 'devm' framework function which automatically
1555  * clean the device's resources. It just calls explicitly disable function for
1556  * the performance counters.
1557  */
1558 static int exynos5_dmc_remove(struct platform_device *pdev)
1559 {
1560 	struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1561 
1562 	if (dmc->in_irq_mode)
1563 		exynos5_dmc_disable_perf_events(dmc);
1564 	else
1565 		exynos5_counters_disable_edev(dmc);
1566 
1567 	clk_disable_unprepare(dmc->mout_bpll);
1568 	clk_disable_unprepare(dmc->fout_bpll);
1569 
1570 	dev_pm_opp_remove_table(dmc->dev);
1571 
1572 	return 0;
1573 }
1574 
1575 static const struct of_device_id exynos5_dmc_of_match[] = {
1576 	{ .compatible = "samsung,exynos5422-dmc", },
1577 	{ },
1578 };
1579 MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1580 
1581 static struct platform_driver exynos5_dmc_platdrv = {
1582 	.probe	= exynos5_dmc_probe,
1583 	.remove = exynos5_dmc_remove,
1584 	.driver = {
1585 		.name	= "exynos5-dmc",
1586 		.of_match_table = exynos5_dmc_of_match,
1587 	},
1588 };
1589 module_platform_driver(exynos5_dmc_platdrv);
1590 MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1591 MODULE_LICENSE("GPL v2");
1592 MODULE_AUTHOR("Lukasz Luba");
1593