1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * A devfreq driver for NVIDIA Tegra SoCs
4  *
5  * Copyright (c) 2014 NVIDIA CORPORATION. All rights reserved.
6  * Copyright (C) 2014 Google, Inc
7  */
8 
9 #include <linux/clk.h>
10 #include <linux/cpufreq.h>
11 #include <linux/devfreq.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/irq.h>
15 #include <linux/module.h>
16 #include <linux/of_device.h>
17 #include <linux/platform_device.h>
18 #include <linux/pm_opp.h>
19 #include <linux/reset.h>
20 #include <linux/workqueue.h>
21 
22 #include <soc/tegra/fuse.h>
23 
24 #include "governor.h"
25 
26 #define ACTMON_GLB_STATUS					0x0
27 #define ACTMON_GLB_PERIOD_CTRL					0x4
28 
29 #define ACTMON_DEV_CTRL						0x0
30 #define ACTMON_DEV_CTRL_K_VAL_SHIFT				10
31 #define ACTMON_DEV_CTRL_ENB_PERIODIC				BIT(18)
32 #define ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN			BIT(20)
33 #define ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN			BIT(21)
34 #define ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT	23
35 #define ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT	26
36 #define ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN		BIT(29)
37 #define ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN		BIT(30)
38 #define ACTMON_DEV_CTRL_ENB					BIT(31)
39 
40 #define ACTMON_DEV_CTRL_STOP					0x00000000
41 
42 #define ACTMON_DEV_UPPER_WMARK					0x4
43 #define ACTMON_DEV_LOWER_WMARK					0x8
44 #define ACTMON_DEV_INIT_AVG					0xc
45 #define ACTMON_DEV_AVG_UPPER_WMARK				0x10
46 #define ACTMON_DEV_AVG_LOWER_WMARK				0x14
47 #define ACTMON_DEV_COUNT_WEIGHT					0x18
48 #define ACTMON_DEV_AVG_COUNT					0x20
49 #define ACTMON_DEV_INTR_STATUS					0x24
50 
51 #define ACTMON_INTR_STATUS_CLEAR				0xffffffff
52 
53 #define ACTMON_DEV_INTR_CONSECUTIVE_UPPER			BIT(31)
54 #define ACTMON_DEV_INTR_CONSECUTIVE_LOWER			BIT(30)
55 
56 #define ACTMON_ABOVE_WMARK_WINDOW				1
57 #define ACTMON_BELOW_WMARK_WINDOW				3
58 #define ACTMON_BOOST_FREQ_STEP					16000
59 
60 /*
61  * ACTMON_AVERAGE_WINDOW_LOG2: default value for @DEV_CTRL_K_VAL, which
62  * translates to 2 ^ (K_VAL + 1). ex: 2 ^ (6 + 1) = 128
63  */
64 #define ACTMON_AVERAGE_WINDOW_LOG2			6
65 #define ACTMON_SAMPLING_PERIOD				12 /* ms */
66 #define ACTMON_DEFAULT_AVG_BAND				6  /* 1/10 of % */
67 
68 #define KHZ							1000
69 
70 #define KHZ_MAX						(ULONG_MAX / KHZ)
71 
72 /* Assume that the bus is saturated if the utilization is 25% */
73 #define BUS_SATURATION_RATIO					25
74 
75 /**
76  * struct tegra_devfreq_device_config - configuration specific to an ACTMON
77  * device
78  *
79  * Coefficients and thresholds are percentages unless otherwise noted
80  */
81 struct tegra_devfreq_device_config {
82 	u32		offset;
83 	u32		irq_mask;
84 
85 	/* Factors applied to boost_freq every consecutive watermark breach */
86 	unsigned int	boost_up_coeff;
87 	unsigned int	boost_down_coeff;
88 
89 	/* Define the watermark bounds when applied to the current avg */
90 	unsigned int	boost_up_threshold;
91 	unsigned int	boost_down_threshold;
92 
93 	/*
94 	 * Threshold of activity (cycles translated to kHz) below which the
95 	 * CPU frequency isn't to be taken into account. This is to avoid
96 	 * increasing the EMC frequency when the CPU is very busy but not
97 	 * accessing the bus often.
98 	 */
99 	u32		avg_dependency_threshold;
100 };
101 
102 enum tegra_actmon_device {
103 	MCALL = 0,
104 	MCCPU,
105 };
106 
107 static const struct tegra_devfreq_device_config tegra124_device_configs[] = {
108 	{
109 		/* MCALL: All memory accesses (including from the CPUs) */
110 		.offset = 0x1c0,
111 		.irq_mask = 1 << 26,
112 		.boost_up_coeff = 200,
113 		.boost_down_coeff = 50,
114 		.boost_up_threshold = 60,
115 		.boost_down_threshold = 40,
116 	},
117 	{
118 		/* MCCPU: memory accesses from the CPUs */
119 		.offset = 0x200,
120 		.irq_mask = 1 << 25,
121 		.boost_up_coeff = 800,
122 		.boost_down_coeff = 40,
123 		.boost_up_threshold = 27,
124 		.boost_down_threshold = 10,
125 		.avg_dependency_threshold = 16000, /* 16MHz in kHz units */
126 	},
127 };
128 
129 static const struct tegra_devfreq_device_config tegra30_device_configs[] = {
130 	{
131 		/* MCALL: All memory accesses (including from the CPUs) */
132 		.offset = 0x1c0,
133 		.irq_mask = 1 << 26,
134 		.boost_up_coeff = 200,
135 		.boost_down_coeff = 50,
136 		.boost_up_threshold = 20,
137 		.boost_down_threshold = 10,
138 	},
139 	{
140 		/* MCCPU: memory accesses from the CPUs */
141 		.offset = 0x200,
142 		.irq_mask = 1 << 25,
143 		.boost_up_coeff = 800,
144 		.boost_down_coeff = 40,
145 		.boost_up_threshold = 27,
146 		.boost_down_threshold = 10,
147 		.avg_dependency_threshold = 16000, /* 16MHz in kHz units */
148 	},
149 };
150 
151 /**
152  * struct tegra_devfreq_device - state specific to an ACTMON device
153  *
154  * Frequencies are in kHz.
155  */
156 struct tegra_devfreq_device {
157 	const struct tegra_devfreq_device_config *config;
158 	void __iomem *regs;
159 
160 	/* Average event count sampled in the last interrupt */
161 	u32 avg_count;
162 
163 	/*
164 	 * Extra frequency to increase the target by due to consecutive
165 	 * watermark breaches.
166 	 */
167 	unsigned long boost_freq;
168 
169 	/* Optimal frequency calculated from the stats for this device */
170 	unsigned long target_freq;
171 };
172 
173 struct tegra_devfreq_soc_data {
174 	const struct tegra_devfreq_device_config *configs;
175 	/* Weight value for count measurements */
176 	unsigned int count_weight;
177 };
178 
179 struct tegra_devfreq {
180 	struct devfreq		*devfreq;
181 
182 	struct reset_control	*reset;
183 	struct clk		*clock;
184 	void __iomem		*regs;
185 
186 	struct clk		*emc_clock;
187 	unsigned long		max_freq;
188 	unsigned long		cur_freq;
189 	struct notifier_block	clk_rate_change_nb;
190 
191 	struct delayed_work	cpufreq_update_work;
192 	struct notifier_block	cpu_rate_change_nb;
193 
194 	struct tegra_devfreq_device devices[2];
195 
196 	unsigned int		irq;
197 
198 	bool			started;
199 
200 	const struct tegra_devfreq_soc_data *soc;
201 };
202 
203 struct tegra_actmon_emc_ratio {
204 	unsigned long cpu_freq;
205 	unsigned long emc_freq;
206 };
207 
208 static const struct tegra_actmon_emc_ratio actmon_emc_ratios[] = {
209 	{ 1400000,    KHZ_MAX },
210 	{ 1200000,    750000 },
211 	{ 1100000,    600000 },
212 	{ 1000000,    500000 },
213 	{  800000,    375000 },
214 	{  500000,    200000 },
215 	{  250000,    100000 },
216 };
217 
218 static u32 actmon_readl(struct tegra_devfreq *tegra, u32 offset)
219 {
220 	return readl_relaxed(tegra->regs + offset);
221 }
222 
223 static void actmon_writel(struct tegra_devfreq *tegra, u32 val, u32 offset)
224 {
225 	writel_relaxed(val, tegra->regs + offset);
226 }
227 
228 static u32 device_readl(struct tegra_devfreq_device *dev, u32 offset)
229 {
230 	return readl_relaxed(dev->regs + offset);
231 }
232 
233 static void device_writel(struct tegra_devfreq_device *dev, u32 val,
234 			  u32 offset)
235 {
236 	writel_relaxed(val, dev->regs + offset);
237 }
238 
239 static unsigned long do_percent(unsigned long long val, unsigned int pct)
240 {
241 	val = val * pct;
242 	do_div(val, 100);
243 
244 	/*
245 	 * High freq + high boosting percent + large polling interval are
246 	 * resulting in integer overflow when watermarks are calculated.
247 	 */
248 	return min_t(u64, val, U32_MAX);
249 }
250 
251 static void tegra_devfreq_update_avg_wmark(struct tegra_devfreq *tegra,
252 					   struct tegra_devfreq_device *dev)
253 {
254 	u32 avg_band_freq = tegra->max_freq * ACTMON_DEFAULT_AVG_BAND / KHZ;
255 	u32 band = avg_band_freq * tegra->devfreq->profile->polling_ms;
256 	u32 avg;
257 
258 	avg = min(dev->avg_count, U32_MAX - band);
259 	device_writel(dev, avg + band, ACTMON_DEV_AVG_UPPER_WMARK);
260 
261 	avg = max(dev->avg_count, band);
262 	device_writel(dev, avg - band, ACTMON_DEV_AVG_LOWER_WMARK);
263 }
264 
265 static void tegra_devfreq_update_wmark(struct tegra_devfreq *tegra,
266 				       struct tegra_devfreq_device *dev)
267 {
268 	u32 val = tegra->cur_freq * tegra->devfreq->profile->polling_ms;
269 
270 	device_writel(dev, do_percent(val, dev->config->boost_up_threshold),
271 		      ACTMON_DEV_UPPER_WMARK);
272 
273 	device_writel(dev, do_percent(val, dev->config->boost_down_threshold),
274 		      ACTMON_DEV_LOWER_WMARK);
275 }
276 
277 static void actmon_isr_device(struct tegra_devfreq *tegra,
278 			      struct tegra_devfreq_device *dev)
279 {
280 	u32 intr_status, dev_ctrl;
281 
282 	dev->avg_count = device_readl(dev, ACTMON_DEV_AVG_COUNT);
283 	tegra_devfreq_update_avg_wmark(tegra, dev);
284 
285 	intr_status = device_readl(dev, ACTMON_DEV_INTR_STATUS);
286 	dev_ctrl = device_readl(dev, ACTMON_DEV_CTRL);
287 
288 	if (intr_status & ACTMON_DEV_INTR_CONSECUTIVE_UPPER) {
289 		/*
290 		 * new_boost = min(old_boost * up_coef + step, max_freq)
291 		 */
292 		dev->boost_freq = do_percent(dev->boost_freq,
293 					     dev->config->boost_up_coeff);
294 		dev->boost_freq += ACTMON_BOOST_FREQ_STEP;
295 
296 		dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
297 
298 		if (dev->boost_freq >= tegra->max_freq) {
299 			dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
300 			dev->boost_freq = tegra->max_freq;
301 		}
302 	} else if (intr_status & ACTMON_DEV_INTR_CONSECUTIVE_LOWER) {
303 		/*
304 		 * new_boost = old_boost * down_coef
305 		 * or 0 if (old_boost * down_coef < step / 2)
306 		 */
307 		dev->boost_freq = do_percent(dev->boost_freq,
308 					     dev->config->boost_down_coeff);
309 
310 		dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
311 
312 		if (dev->boost_freq < (ACTMON_BOOST_FREQ_STEP >> 1)) {
313 			dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
314 			dev->boost_freq = 0;
315 		}
316 	}
317 
318 	device_writel(dev, dev_ctrl, ACTMON_DEV_CTRL);
319 
320 	device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS);
321 }
322 
323 static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra,
324 					    unsigned long cpu_freq)
325 {
326 	unsigned int i;
327 	const struct tegra_actmon_emc_ratio *ratio = actmon_emc_ratios;
328 
329 	for (i = 0; i < ARRAY_SIZE(actmon_emc_ratios); i++, ratio++) {
330 		if (cpu_freq >= ratio->cpu_freq) {
331 			if (ratio->emc_freq >= tegra->max_freq)
332 				return tegra->max_freq;
333 			else
334 				return ratio->emc_freq;
335 		}
336 	}
337 
338 	return 0;
339 }
340 
341 static unsigned long actmon_device_target_freq(struct tegra_devfreq *tegra,
342 					       struct tegra_devfreq_device *dev)
343 {
344 	unsigned int avg_sustain_coef;
345 	unsigned long target_freq;
346 
347 	target_freq = dev->avg_count / tegra->devfreq->profile->polling_ms;
348 	avg_sustain_coef = 100 * 100 / dev->config->boost_up_threshold;
349 	target_freq = do_percent(target_freq, avg_sustain_coef);
350 
351 	return target_freq;
352 }
353 
354 static void actmon_update_target(struct tegra_devfreq *tegra,
355 				 struct tegra_devfreq_device *dev)
356 {
357 	unsigned long cpu_freq = 0;
358 	unsigned long static_cpu_emc_freq = 0;
359 
360 	dev->target_freq = actmon_device_target_freq(tegra, dev);
361 
362 	if (dev->config->avg_dependency_threshold &&
363 	    dev->config->avg_dependency_threshold <= dev->target_freq) {
364 		cpu_freq = cpufreq_quick_get(0);
365 		static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq);
366 
367 		dev->target_freq += dev->boost_freq;
368 		dev->target_freq = max(dev->target_freq, static_cpu_emc_freq);
369 	} else {
370 		dev->target_freq += dev->boost_freq;
371 	}
372 }
373 
374 static irqreturn_t actmon_thread_isr(int irq, void *data)
375 {
376 	struct tegra_devfreq *tegra = data;
377 	bool handled = false;
378 	unsigned int i;
379 	u32 val;
380 
381 	mutex_lock(&tegra->devfreq->lock);
382 
383 	val = actmon_readl(tegra, ACTMON_GLB_STATUS);
384 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
385 		if (val & tegra->devices[i].config->irq_mask) {
386 			actmon_isr_device(tegra, tegra->devices + i);
387 			handled = true;
388 		}
389 	}
390 
391 	if (handled)
392 		update_devfreq(tegra->devfreq);
393 
394 	mutex_unlock(&tegra->devfreq->lock);
395 
396 	return handled ? IRQ_HANDLED : IRQ_NONE;
397 }
398 
399 static int tegra_actmon_clk_notify_cb(struct notifier_block *nb,
400 				      unsigned long action, void *ptr)
401 {
402 	struct clk_notifier_data *data = ptr;
403 	struct tegra_devfreq *tegra;
404 	struct tegra_devfreq_device *dev;
405 	unsigned int i;
406 
407 	if (action != POST_RATE_CHANGE)
408 		return NOTIFY_OK;
409 
410 	tegra = container_of(nb, struct tegra_devfreq, clk_rate_change_nb);
411 
412 	tegra->cur_freq = data->new_rate / KHZ;
413 
414 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
415 		dev = &tegra->devices[i];
416 
417 		tegra_devfreq_update_wmark(tegra, dev);
418 	}
419 
420 	return NOTIFY_OK;
421 }
422 
423 static void tegra_actmon_delayed_update(struct work_struct *work)
424 {
425 	struct tegra_devfreq *tegra = container_of(work, struct tegra_devfreq,
426 						   cpufreq_update_work.work);
427 
428 	mutex_lock(&tegra->devfreq->lock);
429 	update_devfreq(tegra->devfreq);
430 	mutex_unlock(&tegra->devfreq->lock);
431 }
432 
433 static unsigned long
434 tegra_actmon_cpufreq_contribution(struct tegra_devfreq *tegra,
435 				  unsigned int cpu_freq)
436 {
437 	struct tegra_devfreq_device *actmon_dev = &tegra->devices[MCCPU];
438 	unsigned long static_cpu_emc_freq, dev_freq;
439 
440 	dev_freq = actmon_device_target_freq(tegra, actmon_dev);
441 
442 	/* check whether CPU's freq is taken into account at all */
443 	if (dev_freq < actmon_dev->config->avg_dependency_threshold)
444 		return 0;
445 
446 	static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq);
447 
448 	if (dev_freq + actmon_dev->boost_freq >= static_cpu_emc_freq)
449 		return 0;
450 
451 	return static_cpu_emc_freq;
452 }
453 
454 static int tegra_actmon_cpu_notify_cb(struct notifier_block *nb,
455 				      unsigned long action, void *ptr)
456 {
457 	struct cpufreq_freqs *freqs = ptr;
458 	struct tegra_devfreq *tegra;
459 	unsigned long old, new, delay;
460 
461 	if (action != CPUFREQ_POSTCHANGE)
462 		return NOTIFY_OK;
463 
464 	tegra = container_of(nb, struct tegra_devfreq, cpu_rate_change_nb);
465 
466 	/*
467 	 * Quickly check whether CPU frequency should be taken into account
468 	 * at all, without blocking CPUFreq's core.
469 	 */
470 	if (mutex_trylock(&tegra->devfreq->lock)) {
471 		old = tegra_actmon_cpufreq_contribution(tegra, freqs->old);
472 		new = tegra_actmon_cpufreq_contribution(tegra, freqs->new);
473 		mutex_unlock(&tegra->devfreq->lock);
474 
475 		/*
476 		 * If CPU's frequency shouldn't be taken into account at
477 		 * the moment, then there is no need to update the devfreq's
478 		 * state because ISR will re-check CPU's frequency on the
479 		 * next interrupt.
480 		 */
481 		if (old == new)
482 			return NOTIFY_OK;
483 	}
484 
485 	/*
486 	 * CPUFreq driver should support CPUFREQ_ASYNC_NOTIFICATION in order
487 	 * to allow asynchronous notifications. This means we can't block
488 	 * here for too long, otherwise CPUFreq's core will complain with a
489 	 * warning splat.
490 	 */
491 	delay = msecs_to_jiffies(ACTMON_SAMPLING_PERIOD);
492 	schedule_delayed_work(&tegra->cpufreq_update_work, delay);
493 
494 	return NOTIFY_OK;
495 }
496 
497 static void tegra_actmon_configure_device(struct tegra_devfreq *tegra,
498 					  struct tegra_devfreq_device *dev)
499 {
500 	u32 val = 0;
501 
502 	/* reset boosting on governor's restart */
503 	dev->boost_freq = 0;
504 
505 	dev->target_freq = tegra->cur_freq;
506 
507 	dev->avg_count = tegra->cur_freq * tegra->devfreq->profile->polling_ms;
508 	device_writel(dev, dev->avg_count, ACTMON_DEV_INIT_AVG);
509 
510 	tegra_devfreq_update_avg_wmark(tegra, dev);
511 	tegra_devfreq_update_wmark(tegra, dev);
512 
513 	device_writel(dev, tegra->soc->count_weight, ACTMON_DEV_COUNT_WEIGHT);
514 	device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS);
515 
516 	val |= ACTMON_DEV_CTRL_ENB_PERIODIC;
517 	val |= (ACTMON_AVERAGE_WINDOW_LOG2 - 1)
518 		<< ACTMON_DEV_CTRL_K_VAL_SHIFT;
519 	val |= (ACTMON_BELOW_WMARK_WINDOW - 1)
520 		<< ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT;
521 	val |= (ACTMON_ABOVE_WMARK_WINDOW - 1)
522 		<< ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT;
523 	val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN;
524 	val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN;
525 	val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
526 	val |= ACTMON_DEV_CTRL_ENB;
527 
528 	device_writel(dev, val, ACTMON_DEV_CTRL);
529 }
530 
531 static void tegra_actmon_stop_devices(struct tegra_devfreq *tegra)
532 {
533 	struct tegra_devfreq_device *dev = tegra->devices;
534 	unsigned int i;
535 
536 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++, dev++) {
537 		device_writel(dev, ACTMON_DEV_CTRL_STOP, ACTMON_DEV_CTRL);
538 		device_writel(dev, ACTMON_INTR_STATUS_CLEAR,
539 			      ACTMON_DEV_INTR_STATUS);
540 	}
541 }
542 
543 static int tegra_actmon_resume(struct tegra_devfreq *tegra)
544 {
545 	unsigned int i;
546 	int err;
547 
548 	if (!tegra->devfreq->profile->polling_ms || !tegra->started)
549 		return 0;
550 
551 	actmon_writel(tegra, tegra->devfreq->profile->polling_ms - 1,
552 		      ACTMON_GLB_PERIOD_CTRL);
553 
554 	/*
555 	 * CLK notifications are needed in order to reconfigure the upper
556 	 * consecutive watermark in accordance to the actual clock rate
557 	 * to avoid unnecessary upper interrupts.
558 	 */
559 	err = clk_notifier_register(tegra->emc_clock,
560 				    &tegra->clk_rate_change_nb);
561 	if (err) {
562 		dev_err(tegra->devfreq->dev.parent,
563 			"Failed to register rate change notifier\n");
564 		return err;
565 	}
566 
567 	tegra->cur_freq = clk_get_rate(tegra->emc_clock) / KHZ;
568 
569 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++)
570 		tegra_actmon_configure_device(tegra, &tegra->devices[i]);
571 
572 	/*
573 	 * We are estimating CPU's memory bandwidth requirement based on
574 	 * amount of memory accesses and system's load, judging by CPU's
575 	 * frequency. We also don't want to receive events about CPU's
576 	 * frequency transaction when governor is stopped, hence notifier
577 	 * is registered dynamically.
578 	 */
579 	err = cpufreq_register_notifier(&tegra->cpu_rate_change_nb,
580 					CPUFREQ_TRANSITION_NOTIFIER);
581 	if (err) {
582 		dev_err(tegra->devfreq->dev.parent,
583 			"Failed to register rate change notifier: %d\n", err);
584 		goto err_stop;
585 	}
586 
587 	enable_irq(tegra->irq);
588 
589 	return 0;
590 
591 err_stop:
592 	tegra_actmon_stop_devices(tegra);
593 
594 	clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb);
595 
596 	return err;
597 }
598 
599 static int tegra_actmon_start(struct tegra_devfreq *tegra)
600 {
601 	int ret = 0;
602 
603 	if (!tegra->started) {
604 		tegra->started = true;
605 
606 		ret = tegra_actmon_resume(tegra);
607 		if (ret)
608 			tegra->started = false;
609 	}
610 
611 	return ret;
612 }
613 
614 static void tegra_actmon_pause(struct tegra_devfreq *tegra)
615 {
616 	if (!tegra->devfreq->profile->polling_ms || !tegra->started)
617 		return;
618 
619 	disable_irq(tegra->irq);
620 
621 	cpufreq_unregister_notifier(&tegra->cpu_rate_change_nb,
622 				    CPUFREQ_TRANSITION_NOTIFIER);
623 
624 	cancel_delayed_work_sync(&tegra->cpufreq_update_work);
625 
626 	tegra_actmon_stop_devices(tegra);
627 
628 	clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb);
629 }
630 
631 static void tegra_actmon_stop(struct tegra_devfreq *tegra)
632 {
633 	tegra_actmon_pause(tegra);
634 	tegra->started = false;
635 }
636 
637 static int tegra_devfreq_target(struct device *dev, unsigned long *freq,
638 				u32 flags)
639 {
640 	struct dev_pm_opp *opp;
641 	int ret;
642 
643 	opp = devfreq_recommended_opp(dev, freq, flags);
644 	if (IS_ERR(opp)) {
645 		dev_err(dev, "Failed to find opp for %lu Hz\n", *freq);
646 		return PTR_ERR(opp);
647 	}
648 
649 	ret = dev_pm_opp_set_opp(dev, opp);
650 	dev_pm_opp_put(opp);
651 
652 	return ret;
653 }
654 
655 static int tegra_devfreq_get_dev_status(struct device *dev,
656 					struct devfreq_dev_status *stat)
657 {
658 	struct tegra_devfreq *tegra = dev_get_drvdata(dev);
659 	struct tegra_devfreq_device *actmon_dev;
660 	unsigned long cur_freq;
661 
662 	cur_freq = READ_ONCE(tegra->cur_freq);
663 
664 	/* To be used by the tegra governor */
665 	stat->private_data = tegra;
666 
667 	/* The below are to be used by the other governors */
668 	stat->current_frequency = cur_freq * KHZ;
669 
670 	actmon_dev = &tegra->devices[MCALL];
671 
672 	/* Number of cycles spent on memory access */
673 	stat->busy_time = device_readl(actmon_dev, ACTMON_DEV_AVG_COUNT);
674 
675 	/* The bus can be considered to be saturated way before 100% */
676 	stat->busy_time *= 100 / BUS_SATURATION_RATIO;
677 
678 	/* Number of cycles in a sampling period */
679 	stat->total_time = tegra->devfreq->profile->polling_ms * cur_freq;
680 
681 	stat->busy_time = min(stat->busy_time, stat->total_time);
682 
683 	return 0;
684 }
685 
686 static struct devfreq_dev_profile tegra_devfreq_profile = {
687 	.polling_ms	= ACTMON_SAMPLING_PERIOD,
688 	.target		= tegra_devfreq_target,
689 	.get_dev_status	= tegra_devfreq_get_dev_status,
690 	.is_cooling_device = true,
691 };
692 
693 static int tegra_governor_get_target(struct devfreq *devfreq,
694 				     unsigned long *freq)
695 {
696 	struct devfreq_dev_status *stat;
697 	struct tegra_devfreq *tegra;
698 	struct tegra_devfreq_device *dev;
699 	unsigned long target_freq = 0;
700 	unsigned int i;
701 	int err;
702 
703 	err = devfreq_update_stats(devfreq);
704 	if (err)
705 		return err;
706 
707 	stat = &devfreq->last_status;
708 
709 	tegra = stat->private_data;
710 
711 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
712 		dev = &tegra->devices[i];
713 
714 		actmon_update_target(tegra, dev);
715 
716 		target_freq = max(target_freq, dev->target_freq);
717 	}
718 
719 	/*
720 	 * tegra-devfreq driver operates with KHz units, while OPP table
721 	 * entries use Hz units. Hence we need to convert the units for the
722 	 * devfreq core.
723 	 */
724 	*freq = target_freq * KHZ;
725 
726 	return 0;
727 }
728 
729 static int tegra_governor_event_handler(struct devfreq *devfreq,
730 					unsigned int event, void *data)
731 {
732 	struct tegra_devfreq *tegra = dev_get_drvdata(devfreq->dev.parent);
733 	unsigned int *new_delay = data;
734 	int ret = 0;
735 
736 	/*
737 	 * Couple devfreq-device with the governor early because it is
738 	 * needed at the moment of governor's start (used by ISR).
739 	 */
740 	tegra->devfreq = devfreq;
741 
742 	switch (event) {
743 	case DEVFREQ_GOV_START:
744 		devfreq_monitor_start(devfreq);
745 		ret = tegra_actmon_start(tegra);
746 		break;
747 
748 	case DEVFREQ_GOV_STOP:
749 		tegra_actmon_stop(tegra);
750 		devfreq_monitor_stop(devfreq);
751 		break;
752 
753 	case DEVFREQ_GOV_UPDATE_INTERVAL:
754 		/*
755 		 * ACTMON hardware supports up to 256 milliseconds for the
756 		 * sampling period.
757 		 */
758 		if (*new_delay > 256) {
759 			ret = -EINVAL;
760 			break;
761 		}
762 
763 		tegra_actmon_pause(tegra);
764 		devfreq_update_interval(devfreq, new_delay);
765 		ret = tegra_actmon_resume(tegra);
766 		break;
767 
768 	case DEVFREQ_GOV_SUSPEND:
769 		tegra_actmon_stop(tegra);
770 		devfreq_monitor_suspend(devfreq);
771 		break;
772 
773 	case DEVFREQ_GOV_RESUME:
774 		devfreq_monitor_resume(devfreq);
775 		ret = tegra_actmon_start(tegra);
776 		break;
777 	}
778 
779 	return ret;
780 }
781 
782 static struct devfreq_governor tegra_devfreq_governor = {
783 	.name = "tegra_actmon",
784 	.attrs = DEVFREQ_GOV_ATTR_POLLING_INTERVAL,
785 	.flags = DEVFREQ_GOV_FLAG_IMMUTABLE
786 		| DEVFREQ_GOV_FLAG_IRQ_DRIVEN,
787 	.get_target_freq = tegra_governor_get_target,
788 	.event_handler = tegra_governor_event_handler,
789 };
790 
791 static void devm_tegra_devfreq_deinit_hw(void *data)
792 {
793 	struct tegra_devfreq *tegra = data;
794 
795 	reset_control_reset(tegra->reset);
796 	clk_disable_unprepare(tegra->clock);
797 }
798 
799 static int devm_tegra_devfreq_init_hw(struct device *dev,
800 				      struct tegra_devfreq *tegra)
801 {
802 	int err;
803 
804 	err = clk_prepare_enable(tegra->clock);
805 	if (err) {
806 		dev_err(dev, "Failed to prepare and enable ACTMON clock\n");
807 		return err;
808 	}
809 
810 	err = devm_add_action_or_reset(dev, devm_tegra_devfreq_deinit_hw,
811 				       tegra);
812 	if (err)
813 		return err;
814 
815 	err = reset_control_reset(tegra->reset);
816 	if (err) {
817 		dev_err(dev, "Failed to reset hardware: %d\n", err);
818 		return err;
819 	}
820 
821 	return err;
822 }
823 
824 static int tegra_devfreq_probe(struct platform_device *pdev)
825 {
826 	u32 hw_version = BIT(tegra_sku_info.soc_speedo_id);
827 	struct tegra_devfreq_device *dev;
828 	struct tegra_devfreq *tegra;
829 	struct devfreq *devfreq;
830 	unsigned int i;
831 	long rate;
832 	int err;
833 
834 	tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL);
835 	if (!tegra)
836 		return -ENOMEM;
837 
838 	tegra->soc = of_device_get_match_data(&pdev->dev);
839 
840 	tegra->regs = devm_platform_ioremap_resource(pdev, 0);
841 	if (IS_ERR(tegra->regs))
842 		return PTR_ERR(tegra->regs);
843 
844 	tegra->reset = devm_reset_control_get(&pdev->dev, "actmon");
845 	if (IS_ERR(tegra->reset)) {
846 		dev_err(&pdev->dev, "Failed to get reset\n");
847 		return PTR_ERR(tegra->reset);
848 	}
849 
850 	tegra->clock = devm_clk_get(&pdev->dev, "actmon");
851 	if (IS_ERR(tegra->clock)) {
852 		dev_err(&pdev->dev, "Failed to get actmon clock\n");
853 		return PTR_ERR(tegra->clock);
854 	}
855 
856 	tegra->emc_clock = devm_clk_get(&pdev->dev, "emc");
857 	if (IS_ERR(tegra->emc_clock))
858 		return dev_err_probe(&pdev->dev, PTR_ERR(tegra->emc_clock),
859 				     "Failed to get emc clock\n");
860 
861 	err = platform_get_irq(pdev, 0);
862 	if (err < 0)
863 		return err;
864 
865 	tegra->irq = err;
866 
867 	irq_set_status_flags(tegra->irq, IRQ_NOAUTOEN);
868 
869 	err = devm_request_threaded_irq(&pdev->dev, tegra->irq, NULL,
870 					actmon_thread_isr, IRQF_ONESHOT,
871 					"tegra-devfreq", tegra);
872 	if (err) {
873 		dev_err(&pdev->dev, "Interrupt request failed: %d\n", err);
874 		return err;
875 	}
876 
877 	err = devm_pm_opp_set_supported_hw(&pdev->dev, &hw_version, 1);
878 	if (err) {
879 		dev_err(&pdev->dev, "Failed to set supported HW: %d\n", err);
880 		return err;
881 	}
882 
883 	err = devm_pm_opp_of_add_table_noclk(&pdev->dev, 0);
884 	if (err) {
885 		dev_err(&pdev->dev, "Failed to add OPP table: %d\n", err);
886 		return err;
887 	}
888 
889 	err = devm_tegra_devfreq_init_hw(&pdev->dev, tegra);
890 	if (err)
891 		return err;
892 
893 	rate = clk_round_rate(tegra->emc_clock, ULONG_MAX);
894 	if (rate <= 0) {
895 		dev_err(&pdev->dev, "Failed to round clock rate: %ld\n", rate);
896 		return rate ?: -EINVAL;
897 	}
898 
899 	tegra->max_freq = rate / KHZ;
900 
901 	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
902 		dev = tegra->devices + i;
903 		dev->config = tegra->soc->configs + i;
904 		dev->regs = tegra->regs + dev->config->offset;
905 	}
906 
907 	platform_set_drvdata(pdev, tegra);
908 
909 	tegra->clk_rate_change_nb.notifier_call = tegra_actmon_clk_notify_cb;
910 	tegra->cpu_rate_change_nb.notifier_call = tegra_actmon_cpu_notify_cb;
911 
912 	INIT_DELAYED_WORK(&tegra->cpufreq_update_work,
913 			  tegra_actmon_delayed_update);
914 
915 	err = devm_devfreq_add_governor(&pdev->dev, &tegra_devfreq_governor);
916 	if (err) {
917 		dev_err(&pdev->dev, "Failed to add governor: %d\n", err);
918 		return err;
919 	}
920 
921 	tegra_devfreq_profile.initial_freq = clk_get_rate(tegra->emc_clock);
922 
923 	devfreq = devm_devfreq_add_device(&pdev->dev, &tegra_devfreq_profile,
924 					  "tegra_actmon", NULL);
925 	if (IS_ERR(devfreq))
926 		return PTR_ERR(devfreq);
927 
928 	return 0;
929 }
930 
931 static const struct tegra_devfreq_soc_data tegra124_soc = {
932 	.configs = tegra124_device_configs,
933 
934 	/*
935 	 * Activity counter is incremented every 256 memory transactions,
936 	 * and each transaction takes 4 EMC clocks.
937 	 */
938 	.count_weight = 4 * 256,
939 };
940 
941 static const struct tegra_devfreq_soc_data tegra30_soc = {
942 	.configs = tegra30_device_configs,
943 	.count_weight = 2 * 256,
944 };
945 
946 static const struct of_device_id tegra_devfreq_of_match[] = {
947 	{ .compatible = "nvidia,tegra30-actmon",  .data = &tegra30_soc, },
948 	{ .compatible = "nvidia,tegra124-actmon", .data = &tegra124_soc, },
949 	{ },
950 };
951 
952 MODULE_DEVICE_TABLE(of, tegra_devfreq_of_match);
953 
954 static struct platform_driver tegra_devfreq_driver = {
955 	.probe	= tegra_devfreq_probe,
956 	.driver = {
957 		.name = "tegra-devfreq",
958 		.of_match_table = tegra_devfreq_of_match,
959 	},
960 };
961 module_platform_driver(tegra_devfreq_driver);
962 
963 MODULE_LICENSE("GPL v2");
964 MODULE_DESCRIPTION("Tegra devfreq driver");
965 MODULE_AUTHOR("Tomeu Vizoso <tomeu.vizoso@collabora.com>");
966