1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Marvell Armada 37xx SoC Peripheral clocks
4  *
5  * Copyright (C) 2016 Marvell
6  *
7  * Gregory CLEMENT <gregory.clement@free-electrons.com>
8  *
9  * Most of the peripheral clocks can be modelled like this:
10  *             _____    _______    _______
11  * TBG-A-P  --|     |  |       |  |       |   ______
12  * TBG-B-P  --| Mux |--| /div1 |--| /div2 |--| Gate |--> perip_clk
13  * TBG-A-S  --|     |  |       |  |       |  |______|
14  * TBG-B-S  --|_____|  |_______|  |_______|
15  *
16  * However some clocks may use only one or two block or and use the
17  * xtal clock as parent.
18  */
19 
20 #include <linux/clk-provider.h>
21 #include <linux/io.h>
22 #include <linux/mfd/syscon.h>
23 #include <linux/of.h>
24 #include <linux/of_device.h>
25 #include <linux/platform_device.h>
26 #include <linux/regmap.h>
27 #include <linux/slab.h>
28 #include <linux/jiffies.h>
29 
30 #define TBG_SEL		0x0
31 #define DIV_SEL0	0x4
32 #define DIV_SEL1	0x8
33 #define DIV_SEL2	0xC
34 #define CLK_SEL		0x10
35 #define CLK_DIS		0x14
36 
37 #define  ARMADA_37XX_DVFS_LOAD_1 1
38 #define LOAD_LEVEL_NR	4
39 
40 #define ARMADA_37XX_NB_L0L1	0x18
41 #define ARMADA_37XX_NB_L2L3	0x1C
42 #define		ARMADA_37XX_NB_TBG_DIV_OFF	13
43 #define		ARMADA_37XX_NB_TBG_DIV_MASK	0x7
44 #define		ARMADA_37XX_NB_CLK_SEL_OFF	11
45 #define		ARMADA_37XX_NB_CLK_SEL_MASK	0x1
46 #define		ARMADA_37XX_NB_TBG_SEL_OFF	9
47 #define		ARMADA_37XX_NB_TBG_SEL_MASK	0x3
48 #define		ARMADA_37XX_NB_CONFIG_SHIFT	16
49 #define ARMADA_37XX_NB_DYN_MOD	0x24
50 #define		ARMADA_37XX_NB_DFS_EN	31
51 #define ARMADA_37XX_NB_CPU_LOAD	0x30
52 #define		ARMADA_37XX_NB_CPU_LOAD_MASK	0x3
53 #define		ARMADA_37XX_DVFS_LOAD_0		0
54 #define		ARMADA_37XX_DVFS_LOAD_1		1
55 #define		ARMADA_37XX_DVFS_LOAD_2		2
56 #define		ARMADA_37XX_DVFS_LOAD_3		3
57 
58 struct clk_periph_driver_data {
59 	struct clk_hw_onecell_data *hw_data;
60 	spinlock_t lock;
61 	void __iomem *reg;
62 
63 	/* Storage registers for suspend/resume operations */
64 	u32 tbg_sel;
65 	u32 div_sel0;
66 	u32 div_sel1;
67 	u32 div_sel2;
68 	u32 clk_sel;
69 	u32 clk_dis;
70 };
71 
72 struct clk_double_div {
73 	struct clk_hw hw;
74 	void __iomem *reg1;
75 	u8 shift1;
76 	void __iomem *reg2;
77 	u8 shift2;
78 };
79 
80 struct clk_pm_cpu {
81 	struct clk_hw hw;
82 	void __iomem *reg_mux;
83 	u8 shift_mux;
84 	u32 mask_mux;
85 	void __iomem *reg_div;
86 	u8 shift_div;
87 	struct regmap *nb_pm_base;
88 	unsigned long l1_expiration;
89 };
90 
91 #define to_clk_double_div(_hw) container_of(_hw, struct clk_double_div, hw)
92 #define to_clk_pm_cpu(_hw) container_of(_hw, struct clk_pm_cpu, hw)
93 
94 struct clk_periph_data {
95 	const char *name;
96 	const char * const *parent_names;
97 	int num_parents;
98 	struct clk_hw *mux_hw;
99 	struct clk_hw *rate_hw;
100 	struct clk_hw *gate_hw;
101 	struct clk_hw *muxrate_hw;
102 	bool is_double_div;
103 };
104 
105 static const struct clk_div_table clk_table6[] = {
106 	{ .val = 1, .div = 1, },
107 	{ .val = 2, .div = 2, },
108 	{ .val = 3, .div = 3, },
109 	{ .val = 4, .div = 4, },
110 	{ .val = 5, .div = 5, },
111 	{ .val = 6, .div = 6, },
112 	{ .val = 0, .div = 0, }, /* last entry */
113 };
114 
115 static const struct clk_div_table clk_table1[] = {
116 	{ .val = 0, .div = 1, },
117 	{ .val = 1, .div = 2, },
118 	{ .val = 0, .div = 0, }, /* last entry */
119 };
120 
121 static const struct clk_div_table clk_table2[] = {
122 	{ .val = 0, .div = 2, },
123 	{ .val = 1, .div = 4, },
124 	{ .val = 0, .div = 0, }, /* last entry */
125 };
126 
127 static const struct clk_ops clk_double_div_ops;
128 static const struct clk_ops clk_pm_cpu_ops;
129 
130 #define PERIPH_GATE(_name, _bit)		\
131 struct clk_gate gate_##_name = {		\
132 	.reg = (void *)CLK_DIS,			\
133 	.bit_idx = _bit,			\
134 	.hw.init = &(struct clk_init_data){	\
135 		.ops =  &clk_gate_ops,		\
136 	}					\
137 };
138 
139 #define PERIPH_MUX(_name, _shift)		\
140 struct clk_mux mux_##_name = {			\
141 	.reg = (void *)TBG_SEL,			\
142 	.shift = _shift,			\
143 	.mask = 3,				\
144 	.hw.init = &(struct clk_init_data){	\
145 		.ops =  &clk_mux_ro_ops,	\
146 	}					\
147 };
148 
149 #define PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2)	\
150 struct clk_double_div rate_##_name = {		\
151 	.reg1 = (void *)_reg1,			\
152 	.reg2 = (void *)_reg2,			\
153 	.shift1 = _shift1,			\
154 	.shift2 = _shift2,			\
155 	.hw.init = &(struct clk_init_data){	\
156 		.ops =  &clk_double_div_ops,	\
157 	}					\
158 };
159 
160 #define PERIPH_DIV(_name, _reg, _shift, _table)	\
161 struct clk_divider rate_##_name = {		\
162 	.reg = (void *)_reg,			\
163 	.table = _table,			\
164 	.shift = _shift,			\
165 	.hw.init = &(struct clk_init_data){	\
166 		.ops =  &clk_divider_ro_ops,	\
167 	}					\
168 };
169 
170 #define PERIPH_PM_CPU(_name, _shift1, _reg, _shift2)	\
171 struct clk_pm_cpu muxrate_##_name = {		\
172 	.reg_mux = (void *)TBG_SEL,		\
173 	.mask_mux = 3,				\
174 	.shift_mux = _shift1,			\
175 	.reg_div = (void *)_reg,		\
176 	.shift_div = _shift2,			\
177 	.hw.init = &(struct clk_init_data){	\
178 		.ops =  &clk_pm_cpu_ops,	\
179 	}					\
180 };
181 
182 #define PERIPH_CLK_FULL_DD(_name, _bit, _shift, _reg1, _reg2, _shift1, _shift2)\
183 static PERIPH_GATE(_name, _bit);			    \
184 static PERIPH_MUX(_name, _shift);			    \
185 static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
186 
187 #define PERIPH_CLK_FULL(_name, _bit, _shift, _reg, _shift1, _table)	\
188 static PERIPH_GATE(_name, _bit);			    \
189 static PERIPH_MUX(_name, _shift);			    \
190 static PERIPH_DIV(_name, _reg, _shift1, _table);
191 
192 #define PERIPH_CLK_GATE_DIV(_name, _bit,  _reg, _shift, _table)	\
193 static PERIPH_GATE(_name, _bit);			\
194 static PERIPH_DIV(_name, _reg, _shift, _table);
195 
196 #define PERIPH_CLK_MUX_DD(_name, _shift, _reg1, _reg2, _shift1, _shift2)\
197 static PERIPH_MUX(_name, _shift);			    \
198 static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
199 
200 #define REF_CLK_FULL(_name)				\
201 	{ .name = #_name,				\
202 	  .parent_names = (const char *[]){ "TBG-A-P",	\
203 	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
204 	  .num_parents = 4,				\
205 	  .mux_hw = &mux_##_name.hw,			\
206 	  .gate_hw = &gate_##_name.hw,			\
207 	  .rate_hw = &rate_##_name.hw,			\
208 	}
209 
210 #define REF_CLK_FULL_DD(_name)				\
211 	{ .name = #_name,				\
212 	  .parent_names = (const char *[]){ "TBG-A-P",	\
213 	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
214 	  .num_parents = 4,				\
215 	  .mux_hw = &mux_##_name.hw,			\
216 	  .gate_hw = &gate_##_name.hw,			\
217 	  .rate_hw = &rate_##_name.hw,			\
218 	  .is_double_div = true,			\
219 	}
220 
221 #define REF_CLK_GATE(_name, _parent_name)			\
222 	{ .name = #_name,					\
223 	  .parent_names = (const char *[]){ _parent_name},	\
224 	  .num_parents = 1,					\
225 	  .gate_hw = &gate_##_name.hw,				\
226 	}
227 
228 #define REF_CLK_GATE_DIV(_name, _parent_name)			\
229 	{ .name = #_name,					\
230 	  .parent_names = (const char *[]){ _parent_name},	\
231 	  .num_parents = 1,					\
232 	  .gate_hw = &gate_##_name.hw,				\
233 	  .rate_hw = &rate_##_name.hw,				\
234 	}
235 
236 #define REF_CLK_PM_CPU(_name)				\
237 	{ .name = #_name,				\
238 	  .parent_names = (const char *[]){ "TBG-A-P",	\
239 	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
240 	  .num_parents = 4,				\
241 	  .muxrate_hw = &muxrate_##_name.hw,		\
242 	}
243 
244 #define REF_CLK_MUX_DD(_name)				\
245 	{ .name = #_name,				\
246 	  .parent_names = (const char *[]){ "TBG-A-P",	\
247 	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
248 	  .num_parents = 4,				\
249 	  .mux_hw = &mux_##_name.hw,			\
250 	  .rate_hw = &rate_##_name.hw,			\
251 	  .is_double_div = true,			\
252 	}
253 
254 /* NB periph clocks */
255 PERIPH_CLK_FULL_DD(mmc, 2, 0, DIV_SEL2, DIV_SEL2, 16, 13);
256 PERIPH_CLK_FULL_DD(sata_host, 3, 2, DIV_SEL2, DIV_SEL2, 10, 7);
257 PERIPH_CLK_FULL_DD(sec_at, 6, 4, DIV_SEL1, DIV_SEL1, 3, 0);
258 PERIPH_CLK_FULL_DD(sec_dap, 7, 6, DIV_SEL1, DIV_SEL1, 9, 6);
259 PERIPH_CLK_FULL_DD(tscem, 8, 8, DIV_SEL1, DIV_SEL1, 15, 12);
260 PERIPH_CLK_FULL(tscem_tmx, 10, 10, DIV_SEL1, 18, clk_table6);
261 static PERIPH_GATE(avs, 11);
262 PERIPH_CLK_FULL_DD(pwm, 13, 14, DIV_SEL0, DIV_SEL0, 3, 0);
263 PERIPH_CLK_FULL_DD(sqf, 12, 12, DIV_SEL1, DIV_SEL1, 27, 24);
264 static PERIPH_GATE(i2c_2, 16);
265 static PERIPH_GATE(i2c_1, 17);
266 PERIPH_CLK_GATE_DIV(ddr_phy, 19, DIV_SEL0, 18, clk_table2);
267 PERIPH_CLK_FULL_DD(ddr_fclk, 21, 16, DIV_SEL0, DIV_SEL0, 15, 12);
268 PERIPH_CLK_FULL(trace, 22, 18, DIV_SEL0, 20, clk_table6);
269 PERIPH_CLK_FULL(counter, 23, 20, DIV_SEL0, 23, clk_table6);
270 PERIPH_CLK_FULL_DD(eip97, 24, 24, DIV_SEL2, DIV_SEL2, 22, 19);
271 static PERIPH_PM_CPU(cpu, 22, DIV_SEL0, 28);
272 
273 static struct clk_periph_data data_nb[] = {
274 	REF_CLK_FULL_DD(mmc),
275 	REF_CLK_FULL_DD(sata_host),
276 	REF_CLK_FULL_DD(sec_at),
277 	REF_CLK_FULL_DD(sec_dap),
278 	REF_CLK_FULL_DD(tscem),
279 	REF_CLK_FULL(tscem_tmx),
280 	REF_CLK_GATE(avs, "xtal"),
281 	REF_CLK_FULL_DD(sqf),
282 	REF_CLK_FULL_DD(pwm),
283 	REF_CLK_GATE(i2c_2, "xtal"),
284 	REF_CLK_GATE(i2c_1, "xtal"),
285 	REF_CLK_GATE_DIV(ddr_phy, "TBG-A-S"),
286 	REF_CLK_FULL_DD(ddr_fclk),
287 	REF_CLK_FULL(trace),
288 	REF_CLK_FULL(counter),
289 	REF_CLK_FULL_DD(eip97),
290 	REF_CLK_PM_CPU(cpu),
291 	{ },
292 };
293 
294 /* SB periph clocks */
295 PERIPH_CLK_MUX_DD(gbe_50, 6, DIV_SEL2, DIV_SEL2, 6, 9);
296 PERIPH_CLK_MUX_DD(gbe_core, 8, DIV_SEL1, DIV_SEL1, 18, 21);
297 PERIPH_CLK_MUX_DD(gbe_125, 10, DIV_SEL1, DIV_SEL1, 6, 9);
298 static PERIPH_GATE(gbe1_50, 0);
299 static PERIPH_GATE(gbe0_50, 1);
300 static PERIPH_GATE(gbe1_125, 2);
301 static PERIPH_GATE(gbe0_125, 3);
302 PERIPH_CLK_GATE_DIV(gbe1_core, 4, DIV_SEL1, 13, clk_table1);
303 PERIPH_CLK_GATE_DIV(gbe0_core, 5, DIV_SEL1, 14, clk_table1);
304 PERIPH_CLK_GATE_DIV(gbe_bm, 12, DIV_SEL1, 0, clk_table1);
305 PERIPH_CLK_FULL_DD(sdio, 11, 14, DIV_SEL0, DIV_SEL0, 3, 6);
306 PERIPH_CLK_FULL_DD(usb32_usb2_sys, 16, 16, DIV_SEL0, DIV_SEL0, 9, 12);
307 PERIPH_CLK_FULL_DD(usb32_ss_sys, 17, 18, DIV_SEL0, DIV_SEL0, 15, 18);
308 static PERIPH_GATE(pcie, 14);
309 
310 static struct clk_periph_data data_sb[] = {
311 	REF_CLK_MUX_DD(gbe_50),
312 	REF_CLK_MUX_DD(gbe_core),
313 	REF_CLK_MUX_DD(gbe_125),
314 	REF_CLK_GATE(gbe1_50, "gbe_50"),
315 	REF_CLK_GATE(gbe0_50, "gbe_50"),
316 	REF_CLK_GATE(gbe1_125, "gbe_125"),
317 	REF_CLK_GATE(gbe0_125, "gbe_125"),
318 	REF_CLK_GATE_DIV(gbe1_core, "gbe_core"),
319 	REF_CLK_GATE_DIV(gbe0_core, "gbe_core"),
320 	REF_CLK_GATE_DIV(gbe_bm, "gbe_core"),
321 	REF_CLK_FULL_DD(sdio),
322 	REF_CLK_FULL_DD(usb32_usb2_sys),
323 	REF_CLK_FULL_DD(usb32_ss_sys),
324 	REF_CLK_GATE(pcie, "gbe_core"),
325 	{ },
326 };
327 
328 static unsigned int get_div(void __iomem *reg, int shift)
329 {
330 	u32 val;
331 
332 	val = (readl(reg) >> shift) & 0x7;
333 	if (val > 6)
334 		return 0;
335 	return val;
336 }
337 
338 static unsigned long clk_double_div_recalc_rate(struct clk_hw *hw,
339 						unsigned long parent_rate)
340 {
341 	struct clk_double_div *double_div = to_clk_double_div(hw);
342 	unsigned int div;
343 
344 	div = get_div(double_div->reg1, double_div->shift1);
345 	div *= get_div(double_div->reg2, double_div->shift2);
346 
347 	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
348 }
349 
350 static const struct clk_ops clk_double_div_ops = {
351 	.recalc_rate = clk_double_div_recalc_rate,
352 };
353 
354 static void armada_3700_pm_dvfs_update_regs(unsigned int load_level,
355 					    unsigned int *reg,
356 					    unsigned int *offset)
357 {
358 	if (load_level <= ARMADA_37XX_DVFS_LOAD_1)
359 		*reg = ARMADA_37XX_NB_L0L1;
360 	else
361 		*reg = ARMADA_37XX_NB_L2L3;
362 
363 	if (load_level == ARMADA_37XX_DVFS_LOAD_0 ||
364 	    load_level ==  ARMADA_37XX_DVFS_LOAD_2)
365 		*offset += ARMADA_37XX_NB_CONFIG_SHIFT;
366 }
367 
368 static bool armada_3700_pm_dvfs_is_enabled(struct regmap *base)
369 {
370 	unsigned int val, reg = ARMADA_37XX_NB_DYN_MOD;
371 
372 	if (IS_ERR(base))
373 		return false;
374 
375 	regmap_read(base, reg, &val);
376 
377 	return !!(val & BIT(ARMADA_37XX_NB_DFS_EN));
378 }
379 
380 static unsigned int armada_3700_pm_dvfs_get_cpu_div(struct regmap *base)
381 {
382 	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
383 	unsigned int offset = ARMADA_37XX_NB_TBG_DIV_OFF;
384 	unsigned int load_level, div;
385 
386 	/*
387 	 * This function is always called after the function
388 	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
389 	 * if the base is valid.
390 	 */
391 	regmap_read(base, reg, &load_level);
392 
393 	/*
394 	 * The register and the offset inside this register accessed to
395 	 * read the current divider depend on the load level
396 	 */
397 	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
398 	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
399 
400 	regmap_read(base, reg, &div);
401 
402 	return (div >> offset) & ARMADA_37XX_NB_TBG_DIV_MASK;
403 }
404 
405 static unsigned int armada_3700_pm_dvfs_get_cpu_parent(struct regmap *base)
406 {
407 	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
408 	unsigned int offset = ARMADA_37XX_NB_TBG_SEL_OFF;
409 	unsigned int load_level, sel;
410 
411 	/*
412 	 * This function is always called after the function
413 	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
414 	 * if the base is valid
415 	 */
416 	regmap_read(base, reg, &load_level);
417 
418 	/*
419 	 * The register and the offset inside this register accessed to
420 	 * read the current divider depend on the load level
421 	 */
422 	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
423 	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
424 
425 	regmap_read(base, reg, &sel);
426 
427 	return (sel >> offset) & ARMADA_37XX_NB_TBG_SEL_MASK;
428 }
429 
430 static u8 clk_pm_cpu_get_parent(struct clk_hw *hw)
431 {
432 	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
433 	u32 val;
434 
435 	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base)) {
436 		val = armada_3700_pm_dvfs_get_cpu_parent(pm_cpu->nb_pm_base);
437 	} else {
438 		val = readl(pm_cpu->reg_mux) >> pm_cpu->shift_mux;
439 		val &= pm_cpu->mask_mux;
440 	}
441 
442 	return val;
443 }
444 
445 static unsigned long clk_pm_cpu_recalc_rate(struct clk_hw *hw,
446 					    unsigned long parent_rate)
447 {
448 	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
449 	unsigned int div;
450 
451 	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base))
452 		div = armada_3700_pm_dvfs_get_cpu_div(pm_cpu->nb_pm_base);
453 	else
454 		div = get_div(pm_cpu->reg_div, pm_cpu->shift_div);
455 	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
456 }
457 
458 static long clk_pm_cpu_round_rate(struct clk_hw *hw, unsigned long rate,
459 				  unsigned long *parent_rate)
460 {
461 	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
462 	struct regmap *base = pm_cpu->nb_pm_base;
463 	unsigned int div = *parent_rate / rate;
464 	unsigned int load_level;
465 	/* only available when DVFS is enabled */
466 	if (!armada_3700_pm_dvfs_is_enabled(base))
467 		return -EINVAL;
468 
469 	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
470 		unsigned int reg, val, offset = ARMADA_37XX_NB_TBG_DIV_OFF;
471 
472 		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
473 
474 		regmap_read(base, reg, &val);
475 
476 		val >>= offset;
477 		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
478 		if (val == div)
479 			/*
480 			 * We found a load level matching the target
481 			 * divider, switch to this load level and
482 			 * return.
483 			 */
484 			return *parent_rate / div;
485 	}
486 
487 	/* We didn't find any valid divider */
488 	return -EINVAL;
489 }
490 
491 /*
492  * Workaround when base CPU frequnecy is 1000 or 1200 MHz
493  *
494  * Switching the CPU from the L2 or L3 frequencies (250/300 or 200 MHz
495  * respectively) to L0 frequency (1/1.2 GHz) requires a significant
496  * amount of time to let VDD stabilize to the appropriate
497  * voltage. This amount of time is large enough that it cannot be
498  * covered by the hardware countdown register. Due to this, the CPU
499  * might start operating at L0 before the voltage is stabilized,
500  * leading to CPU stalls.
501  *
502  * To work around this problem, we prevent switching directly from the
503  * L2/L3 frequencies to the L0 frequency, and instead switch to the L1
504  * frequency in-between. The sequence therefore becomes:
505  * 1. First switch from L2/L3 (200/250/300 MHz) to L1 (500/600 MHz)
506  * 2. Sleep 20ms for stabling VDD voltage
507  * 3. Then switch from L1 (500/600 MHz) to L0 (1000/1200 MHz).
508  */
509 static void clk_pm_cpu_set_rate_wa(struct clk_pm_cpu *pm_cpu,
510 				   unsigned int new_level, unsigned long rate,
511 				   struct regmap *base)
512 {
513 	unsigned int cur_level;
514 
515 	regmap_read(base, ARMADA_37XX_NB_CPU_LOAD, &cur_level);
516 	cur_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
517 
518 	if (cur_level == new_level)
519 		return;
520 
521 	/*
522 	 * System wants to go to L1 on its own. If we are going from L2/L3,
523 	 * remember when 20ms will expire. If from L0, set the value so that
524 	 * next switch to L0 won't have to wait.
525 	 */
526 	if (new_level == ARMADA_37XX_DVFS_LOAD_1) {
527 		if (cur_level == ARMADA_37XX_DVFS_LOAD_0)
528 			pm_cpu->l1_expiration = jiffies;
529 		else
530 			pm_cpu->l1_expiration = jiffies + msecs_to_jiffies(20);
531 		return;
532 	}
533 
534 	/*
535 	 * If we are setting to L2/L3, just invalidate L1 expiration time,
536 	 * sleeping is not needed.
537 	 */
538 	if (rate < 1000*1000*1000)
539 		goto invalidate_l1_exp;
540 
541 	/*
542 	 * We are going to L0 with rate >= 1GHz. Check whether we have been at
543 	 * L1 for long enough time. If not, go to L1 for 20ms.
544 	 */
545 	if (pm_cpu->l1_expiration && time_is_before_eq_jiffies(pm_cpu->l1_expiration))
546 		goto invalidate_l1_exp;
547 
548 	regmap_update_bits(base, ARMADA_37XX_NB_CPU_LOAD,
549 			   ARMADA_37XX_NB_CPU_LOAD_MASK,
550 			   ARMADA_37XX_DVFS_LOAD_1);
551 	msleep(20);
552 
553 invalidate_l1_exp:
554 	pm_cpu->l1_expiration = 0;
555 }
556 
557 static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate,
558 			       unsigned long parent_rate)
559 {
560 	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
561 	struct regmap *base = pm_cpu->nb_pm_base;
562 	unsigned int div = parent_rate / rate;
563 	unsigned int load_level;
564 
565 	/* only available when DVFS is enabled */
566 	if (!armada_3700_pm_dvfs_is_enabled(base))
567 		return -EINVAL;
568 
569 	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
570 		unsigned int reg, mask, val,
571 			offset = ARMADA_37XX_NB_TBG_DIV_OFF;
572 
573 		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
574 
575 		regmap_read(base, reg, &val);
576 		val >>= offset;
577 		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
578 
579 		if (val == div) {
580 			/*
581 			 * We found a load level matching the target
582 			 * divider, switch to this load level and
583 			 * return.
584 			 */
585 			reg = ARMADA_37XX_NB_CPU_LOAD;
586 			mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
587 
588 			/* Apply workaround when base CPU frequency is 1000 or 1200 MHz */
589 			if (parent_rate >= 1000*1000*1000)
590 				clk_pm_cpu_set_rate_wa(pm_cpu, load_level, rate, base);
591 
592 			regmap_update_bits(base, reg, mask, load_level);
593 
594 			return rate;
595 		}
596 	}
597 
598 	/* We didn't find any valid divider */
599 	return -EINVAL;
600 }
601 
602 static const struct clk_ops clk_pm_cpu_ops = {
603 	.get_parent = clk_pm_cpu_get_parent,
604 	.round_rate = clk_pm_cpu_round_rate,
605 	.set_rate = clk_pm_cpu_set_rate,
606 	.recalc_rate = clk_pm_cpu_recalc_rate,
607 };
608 
609 static const struct of_device_id armada_3700_periph_clock_of_match[] = {
610 	{ .compatible = "marvell,armada-3700-periph-clock-nb",
611 	  .data = data_nb, },
612 	{ .compatible = "marvell,armada-3700-periph-clock-sb",
613 	.data = data_sb, },
614 	{ }
615 };
616 
617 static int armada_3700_add_composite_clk(const struct clk_periph_data *data,
618 					 void __iomem *reg, spinlock_t *lock,
619 					 struct device *dev, struct clk_hw **hw)
620 {
621 	const struct clk_ops *mux_ops = NULL, *gate_ops = NULL,
622 		*rate_ops = NULL;
623 	struct clk_hw *mux_hw = NULL, *gate_hw = NULL, *rate_hw = NULL;
624 
625 	if (data->mux_hw) {
626 		struct clk_mux *mux;
627 
628 		mux_hw = data->mux_hw;
629 		mux = to_clk_mux(mux_hw);
630 		mux->lock = lock;
631 		mux_ops = mux_hw->init->ops;
632 		mux->reg = reg + (u64)mux->reg;
633 	}
634 
635 	if (data->gate_hw) {
636 		struct clk_gate *gate;
637 
638 		gate_hw = data->gate_hw;
639 		gate = to_clk_gate(gate_hw);
640 		gate->lock = lock;
641 		gate_ops = gate_hw->init->ops;
642 		gate->reg = reg + (u64)gate->reg;
643 		gate->flags = CLK_GATE_SET_TO_DISABLE;
644 	}
645 
646 	if (data->rate_hw) {
647 		rate_hw = data->rate_hw;
648 		rate_ops = rate_hw->init->ops;
649 		if (data->is_double_div) {
650 			struct clk_double_div *rate;
651 
652 			rate =  to_clk_double_div(rate_hw);
653 			rate->reg1 = reg + (u64)rate->reg1;
654 			rate->reg2 = reg + (u64)rate->reg2;
655 		} else {
656 			struct clk_divider *rate = to_clk_divider(rate_hw);
657 			const struct clk_div_table *clkt;
658 			int table_size = 0;
659 
660 			rate->reg = reg + (u64)rate->reg;
661 			for (clkt = rate->table; clkt->div; clkt++)
662 				table_size++;
663 			rate->width = order_base_2(table_size);
664 			rate->lock = lock;
665 		}
666 	}
667 
668 	if (data->muxrate_hw) {
669 		struct clk_pm_cpu *pmcpu_clk;
670 		struct clk_hw *muxrate_hw = data->muxrate_hw;
671 		struct regmap *map;
672 
673 		pmcpu_clk =  to_clk_pm_cpu(muxrate_hw);
674 		pmcpu_clk->reg_mux = reg + (u64)pmcpu_clk->reg_mux;
675 		pmcpu_clk->reg_div = reg + (u64)pmcpu_clk->reg_div;
676 
677 		mux_hw = muxrate_hw;
678 		rate_hw = muxrate_hw;
679 		mux_ops = muxrate_hw->init->ops;
680 		rate_ops = muxrate_hw->init->ops;
681 
682 		map = syscon_regmap_lookup_by_compatible(
683 				"marvell,armada-3700-nb-pm");
684 		pmcpu_clk->nb_pm_base = map;
685 	}
686 
687 	*hw = clk_hw_register_composite(dev, data->name, data->parent_names,
688 					data->num_parents, mux_hw,
689 					mux_ops, rate_hw, rate_ops,
690 					gate_hw, gate_ops, CLK_IGNORE_UNUSED);
691 
692 	return PTR_ERR_OR_ZERO(*hw);
693 }
694 
695 static int __maybe_unused armada_3700_periph_clock_suspend(struct device *dev)
696 {
697 	struct clk_periph_driver_data *data = dev_get_drvdata(dev);
698 
699 	data->tbg_sel = readl(data->reg + TBG_SEL);
700 	data->div_sel0 = readl(data->reg + DIV_SEL0);
701 	data->div_sel1 = readl(data->reg + DIV_SEL1);
702 	data->div_sel2 = readl(data->reg + DIV_SEL2);
703 	data->clk_sel = readl(data->reg + CLK_SEL);
704 	data->clk_dis = readl(data->reg + CLK_DIS);
705 
706 	return 0;
707 }
708 
709 static int __maybe_unused armada_3700_periph_clock_resume(struct device *dev)
710 {
711 	struct clk_periph_driver_data *data = dev_get_drvdata(dev);
712 
713 	/* Follow the same order than what the Cortex-M3 does (ATF code) */
714 	writel(data->clk_dis, data->reg + CLK_DIS);
715 	writel(data->div_sel0, data->reg + DIV_SEL0);
716 	writel(data->div_sel1, data->reg + DIV_SEL1);
717 	writel(data->div_sel2, data->reg + DIV_SEL2);
718 	writel(data->tbg_sel, data->reg + TBG_SEL);
719 	writel(data->clk_sel, data->reg + CLK_SEL);
720 
721 	return 0;
722 }
723 
724 static const struct dev_pm_ops armada_3700_periph_clock_pm_ops = {
725 	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(armada_3700_periph_clock_suspend,
726 				      armada_3700_periph_clock_resume)
727 };
728 
729 static int armada_3700_periph_clock_probe(struct platform_device *pdev)
730 {
731 	struct clk_periph_driver_data *driver_data;
732 	struct device_node *np = pdev->dev.of_node;
733 	const struct clk_periph_data *data;
734 	struct device *dev = &pdev->dev;
735 	int num_periph = 0, i, ret;
736 	struct resource *res;
737 
738 	data = of_device_get_match_data(dev);
739 	if (!data)
740 		return -ENODEV;
741 
742 	while (data[num_periph].name)
743 		num_periph++;
744 
745 	driver_data = devm_kzalloc(dev, sizeof(*driver_data), GFP_KERNEL);
746 	if (!driver_data)
747 		return -ENOMEM;
748 
749 	driver_data->hw_data = devm_kzalloc(dev,
750 					    struct_size(driver_data->hw_data,
751 							hws, num_periph),
752 					    GFP_KERNEL);
753 	if (!driver_data->hw_data)
754 		return -ENOMEM;
755 	driver_data->hw_data->num = num_periph;
756 
757 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
758 	driver_data->reg = devm_ioremap_resource(dev, res);
759 	if (IS_ERR(driver_data->reg))
760 		return PTR_ERR(driver_data->reg);
761 
762 	spin_lock_init(&driver_data->lock);
763 
764 	for (i = 0; i < num_periph; i++) {
765 		struct clk_hw **hw = &driver_data->hw_data->hws[i];
766 		if (armada_3700_add_composite_clk(&data[i], driver_data->reg,
767 						  &driver_data->lock, dev, hw))
768 			dev_err(dev, "Can't register periph clock %s\n",
769 				data[i].name);
770 	}
771 
772 	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
773 				     driver_data->hw_data);
774 	if (ret) {
775 		for (i = 0; i < num_periph; i++)
776 			clk_hw_unregister(driver_data->hw_data->hws[i]);
777 		return ret;
778 	}
779 
780 	platform_set_drvdata(pdev, driver_data);
781 	return 0;
782 }
783 
784 static int armada_3700_periph_clock_remove(struct platform_device *pdev)
785 {
786 	struct clk_periph_driver_data *data = platform_get_drvdata(pdev);
787 	struct clk_hw_onecell_data *hw_data = data->hw_data;
788 	int i;
789 
790 	of_clk_del_provider(pdev->dev.of_node);
791 
792 	for (i = 0; i < hw_data->num; i++)
793 		clk_hw_unregister(hw_data->hws[i]);
794 
795 	return 0;
796 }
797 
798 static struct platform_driver armada_3700_periph_clock_driver = {
799 	.probe = armada_3700_periph_clock_probe,
800 	.remove = armada_3700_periph_clock_remove,
801 	.driver		= {
802 		.name	= "marvell-armada-3700-periph-clock",
803 		.of_match_table = armada_3700_periph_clock_of_match,
804 		.pm	= &armada_3700_periph_clock_pm_ops,
805 	},
806 };
807 
808 builtin_platform_driver(armada_3700_periph_clock_driver);
809