1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * RISC-V SBI CPU idle driver.
4  *
5  * Copyright (c) 2021 Western Digital Corporation or its affiliates.
6  * Copyright (c) 2022 Ventana Micro Systems Inc.
7  */
8 
9 #define pr_fmt(fmt) "cpuidle-riscv-sbi: " fmt
10 
11 #include <linux/cleanup.h>
12 #include <linux/cpuhotplug.h>
13 #include <linux/cpuidle.h>
14 #include <linux/cpumask.h>
15 #include <linux/cpu_pm.h>
16 #include <linux/cpu_cooling.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/slab.h>
21 #include <linux/platform_device.h>
22 #include <linux/pm_domain.h>
23 #include <linux/pm_runtime.h>
24 #include <asm/cpuidle.h>
25 #include <asm/sbi.h>
26 #include <asm/smp.h>
27 #include <asm/suspend.h>
28 
29 #include "dt_idle_states.h"
30 #include "dt_idle_genpd.h"
31 
32 struct sbi_cpuidle_data {
33 	u32 *states;
34 	struct device *dev;
35 };
36 
37 struct sbi_domain_state {
38 	bool available;
39 	u32 state;
40 };
41 
42 static DEFINE_PER_CPU_READ_MOSTLY(struct sbi_cpuidle_data, sbi_cpuidle_data);
43 static DEFINE_PER_CPU(struct sbi_domain_state, domain_state);
44 static bool sbi_cpuidle_use_osi;
45 static bool sbi_cpuidle_use_cpuhp;
46 static bool sbi_cpuidle_pd_allow_domain_state;
47 
48 static inline void sbi_set_domain_state(u32 state)
49 {
50 	struct sbi_domain_state *data = this_cpu_ptr(&domain_state);
51 
52 	data->available = true;
53 	data->state = state;
54 }
55 
56 static inline u32 sbi_get_domain_state(void)
57 {
58 	struct sbi_domain_state *data = this_cpu_ptr(&domain_state);
59 
60 	return data->state;
61 }
62 
63 static inline void sbi_clear_domain_state(void)
64 {
65 	struct sbi_domain_state *data = this_cpu_ptr(&domain_state);
66 
67 	data->available = false;
68 }
69 
70 static inline bool sbi_is_domain_state_available(void)
71 {
72 	struct sbi_domain_state *data = this_cpu_ptr(&domain_state);
73 
74 	return data->available;
75 }
76 
77 static int sbi_suspend_finisher(unsigned long suspend_type,
78 				unsigned long resume_addr,
79 				unsigned long opaque)
80 {
81 	struct sbiret ret;
82 
83 	ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND,
84 			suspend_type, resume_addr, opaque, 0, 0, 0);
85 
86 	return (ret.error) ? sbi_err_map_linux_errno(ret.error) : 0;
87 }
88 
89 static int sbi_suspend(u32 state)
90 {
91 	if (state & SBI_HSM_SUSP_NON_RET_BIT)
92 		return cpu_suspend(state, sbi_suspend_finisher);
93 	else
94 		return sbi_suspend_finisher(state, 0, 0);
95 }
96 
97 static __cpuidle int sbi_cpuidle_enter_state(struct cpuidle_device *dev,
98 					     struct cpuidle_driver *drv, int idx)
99 {
100 	u32 *states = __this_cpu_read(sbi_cpuidle_data.states);
101 	u32 state = states[idx];
102 
103 	if (state & SBI_HSM_SUSP_NON_RET_BIT)
104 		return CPU_PM_CPU_IDLE_ENTER_PARAM(sbi_suspend, idx, state);
105 	else
106 		return CPU_PM_CPU_IDLE_ENTER_RETENTION_PARAM(sbi_suspend,
107 							     idx, state);
108 }
109 
110 static __cpuidle int __sbi_enter_domain_idle_state(struct cpuidle_device *dev,
111 						   struct cpuidle_driver *drv, int idx,
112 						   bool s2idle)
113 {
114 	struct sbi_cpuidle_data *data = this_cpu_ptr(&sbi_cpuidle_data);
115 	u32 *states = data->states;
116 	struct device *pd_dev = data->dev;
117 	u32 state;
118 	int ret;
119 
120 	ret = cpu_pm_enter();
121 	if (ret)
122 		return -1;
123 
124 	/* Do runtime PM to manage a hierarchical CPU toplogy. */
125 	if (s2idle)
126 		dev_pm_genpd_suspend(pd_dev);
127 	else
128 		pm_runtime_put_sync_suspend(pd_dev);
129 
130 	ct_cpuidle_enter();
131 
132 	if (sbi_is_domain_state_available())
133 		state = sbi_get_domain_state();
134 	else
135 		state = states[idx];
136 
137 	ret = sbi_suspend(state) ? -1 : idx;
138 
139 	ct_cpuidle_exit();
140 
141 	if (s2idle)
142 		dev_pm_genpd_resume(pd_dev);
143 	else
144 		pm_runtime_get_sync(pd_dev);
145 
146 	cpu_pm_exit();
147 
148 	/* Clear the domain state to start fresh when back from idle. */
149 	sbi_clear_domain_state();
150 	return ret;
151 }
152 
153 static int sbi_enter_domain_idle_state(struct cpuidle_device *dev,
154 				       struct cpuidle_driver *drv, int idx)
155 {
156 	return __sbi_enter_domain_idle_state(dev, drv, idx, false);
157 }
158 
159 static int sbi_enter_s2idle_domain_idle_state(struct cpuidle_device *dev,
160 					      struct cpuidle_driver *drv,
161 					      int idx)
162 {
163 	return __sbi_enter_domain_idle_state(dev, drv, idx, true);
164 }
165 
166 static int sbi_cpuidle_cpuhp_up(unsigned int cpu)
167 {
168 	struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev);
169 
170 	if (pd_dev)
171 		pm_runtime_get_sync(pd_dev);
172 
173 	return 0;
174 }
175 
176 static int sbi_cpuidle_cpuhp_down(unsigned int cpu)
177 {
178 	struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev);
179 
180 	if (pd_dev) {
181 		pm_runtime_put_sync(pd_dev);
182 		/* Clear domain state to start fresh at next online. */
183 		sbi_clear_domain_state();
184 	}
185 
186 	return 0;
187 }
188 
189 static void sbi_idle_init_cpuhp(void)
190 {
191 	int err;
192 
193 	if (!sbi_cpuidle_use_cpuhp)
194 		return;
195 
196 	err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING,
197 					"cpuidle/sbi:online",
198 					sbi_cpuidle_cpuhp_up,
199 					sbi_cpuidle_cpuhp_down);
200 	if (err)
201 		pr_warn("Failed %d while setup cpuhp state\n", err);
202 }
203 
204 static const struct of_device_id sbi_cpuidle_state_match[] = {
205 	{ .compatible = "riscv,idle-state",
206 	  .data = sbi_cpuidle_enter_state },
207 	{ },
208 };
209 
210 static bool sbi_suspend_state_is_valid(u32 state)
211 {
212 	if (state > SBI_HSM_SUSPEND_RET_DEFAULT &&
213 	    state < SBI_HSM_SUSPEND_RET_PLATFORM)
214 		return false;
215 	if (state > SBI_HSM_SUSPEND_NON_RET_DEFAULT &&
216 	    state < SBI_HSM_SUSPEND_NON_RET_PLATFORM)
217 		return false;
218 	return true;
219 }
220 
221 static int sbi_dt_parse_state_node(struct device_node *np, u32 *state)
222 {
223 	int err = of_property_read_u32(np, "riscv,sbi-suspend-param", state);
224 
225 	if (err) {
226 		pr_warn("%pOF missing riscv,sbi-suspend-param property\n", np);
227 		return err;
228 	}
229 
230 	if (!sbi_suspend_state_is_valid(*state)) {
231 		pr_warn("Invalid SBI suspend state %#x\n", *state);
232 		return -EINVAL;
233 	}
234 
235 	return 0;
236 }
237 
238 static int sbi_dt_cpu_init_topology(struct cpuidle_driver *drv,
239 				     struct sbi_cpuidle_data *data,
240 				     unsigned int state_count, int cpu)
241 {
242 	/* Currently limit the hierarchical topology to be used in OSI mode. */
243 	if (!sbi_cpuidle_use_osi)
244 		return 0;
245 
246 	data->dev = dt_idle_attach_cpu(cpu, "sbi");
247 	if (IS_ERR_OR_NULL(data->dev))
248 		return PTR_ERR_OR_ZERO(data->dev);
249 
250 	/*
251 	 * Using the deepest state for the CPU to trigger a potential selection
252 	 * of a shared state for the domain, assumes the domain states are all
253 	 * deeper states.
254 	 */
255 	drv->states[state_count - 1].flags |= CPUIDLE_FLAG_RCU_IDLE;
256 	drv->states[state_count - 1].enter = sbi_enter_domain_idle_state;
257 	drv->states[state_count - 1].enter_s2idle =
258 					sbi_enter_s2idle_domain_idle_state;
259 	sbi_cpuidle_use_cpuhp = true;
260 
261 	return 0;
262 }
263 
264 static int sbi_cpuidle_dt_init_states(struct device *dev,
265 					struct cpuidle_driver *drv,
266 					unsigned int cpu,
267 					unsigned int state_count)
268 {
269 	struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu);
270 	struct device_node *state_node;
271 	u32 *states;
272 	int i, ret;
273 
274 	struct device_node *cpu_node __free(device_node) = of_cpu_device_node_get(cpu);
275 	if (!cpu_node)
276 		return -ENODEV;
277 
278 	states = devm_kcalloc(dev, state_count, sizeof(*states), GFP_KERNEL);
279 	if (!states)
280 		return -ENOMEM;
281 
282 	/* Parse SBI specific details from state DT nodes */
283 	for (i = 1; i < state_count; i++) {
284 		state_node = of_get_cpu_state_node(cpu_node, i - 1);
285 		if (!state_node)
286 			break;
287 
288 		ret = sbi_dt_parse_state_node(state_node, &states[i]);
289 		of_node_put(state_node);
290 
291 		if (ret)
292 			return ret;
293 
294 		pr_debug("sbi-state %#x index %d\n", states[i], i);
295 	}
296 	if (i != state_count)
297 		return -ENODEV;
298 
299 	/* Initialize optional data, used for the hierarchical topology. */
300 	ret = sbi_dt_cpu_init_topology(drv, data, state_count, cpu);
301 	if (ret < 0)
302 		return ret;
303 
304 	/* Store states in the per-cpu struct. */
305 	data->states = states;
306 
307 	return 0;
308 }
309 
310 static void sbi_cpuidle_deinit_cpu(int cpu)
311 {
312 	struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu);
313 
314 	dt_idle_detach_cpu(data->dev);
315 	sbi_cpuidle_use_cpuhp = false;
316 }
317 
318 static int sbi_cpuidle_init_cpu(struct device *dev, int cpu)
319 {
320 	struct cpuidle_driver *drv;
321 	unsigned int state_count = 0;
322 	int ret = 0;
323 
324 	drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
325 	if (!drv)
326 		return -ENOMEM;
327 
328 	drv->name = "sbi_cpuidle";
329 	drv->owner = THIS_MODULE;
330 	drv->cpumask = (struct cpumask *)cpumask_of(cpu);
331 
332 	/* RISC-V architectural WFI to be represented as state index 0. */
333 	drv->states[0].enter = sbi_cpuidle_enter_state;
334 	drv->states[0].exit_latency = 1;
335 	drv->states[0].target_residency = 1;
336 	drv->states[0].power_usage = UINT_MAX;
337 	strcpy(drv->states[0].name, "WFI");
338 	strcpy(drv->states[0].desc, "RISC-V WFI");
339 
340 	/*
341 	 * If no DT idle states are detected (ret == 0) let the driver
342 	 * initialization fail accordingly since there is no reason to
343 	 * initialize the idle driver if only wfi is supported, the
344 	 * default archictectural back-end already executes wfi
345 	 * on idle entry.
346 	 */
347 	ret = dt_init_idle_driver(drv, sbi_cpuidle_state_match, 1);
348 	if (ret <= 0) {
349 		pr_debug("HART%ld: failed to parse DT idle states\n",
350 			 cpuid_to_hartid_map(cpu));
351 		return ret ? : -ENODEV;
352 	}
353 	state_count = ret + 1; /* Include WFI state as well */
354 
355 	/* Initialize idle states from DT. */
356 	ret = sbi_cpuidle_dt_init_states(dev, drv, cpu, state_count);
357 	if (ret) {
358 		pr_err("HART%ld: failed to init idle states\n",
359 		       cpuid_to_hartid_map(cpu));
360 		return ret;
361 	}
362 
363 	ret = cpuidle_register(drv, NULL);
364 	if (ret)
365 		goto deinit;
366 
367 	cpuidle_cooling_register(drv);
368 
369 	return 0;
370 deinit:
371 	sbi_cpuidle_deinit_cpu(cpu);
372 	return ret;
373 }
374 
375 static void sbi_cpuidle_domain_sync_state(struct device *dev)
376 {
377 	/*
378 	 * All devices have now been attached/probed to the PM domain
379 	 * topology, hence it's fine to allow domain states to be picked.
380 	 */
381 	sbi_cpuidle_pd_allow_domain_state = true;
382 }
383 
384 #ifdef CONFIG_DT_IDLE_GENPD
385 
386 static int sbi_cpuidle_pd_power_off(struct generic_pm_domain *pd)
387 {
388 	struct genpd_power_state *state = &pd->states[pd->state_idx];
389 	u32 *pd_state;
390 
391 	if (!state->data)
392 		return 0;
393 
394 	if (!sbi_cpuidle_pd_allow_domain_state)
395 		return -EBUSY;
396 
397 	/* OSI mode is enabled, set the corresponding domain state. */
398 	pd_state = state->data;
399 	sbi_set_domain_state(*pd_state);
400 
401 	return 0;
402 }
403 
404 struct sbi_pd_provider {
405 	struct list_head link;
406 	struct device_node *node;
407 };
408 
409 static LIST_HEAD(sbi_pd_providers);
410 
411 static int sbi_pd_init(struct device_node *np)
412 {
413 	struct generic_pm_domain *pd;
414 	struct sbi_pd_provider *pd_provider;
415 	struct dev_power_governor *pd_gov;
416 	int ret = -ENOMEM;
417 
418 	pd = dt_idle_pd_alloc(np, sbi_dt_parse_state_node);
419 	if (!pd)
420 		goto out;
421 
422 	pd_provider = kzalloc(sizeof(*pd_provider), GFP_KERNEL);
423 	if (!pd_provider)
424 		goto free_pd;
425 
426 	pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN;
427 
428 	/* Allow power off when OSI is available. */
429 	if (sbi_cpuidle_use_osi)
430 		pd->power_off = sbi_cpuidle_pd_power_off;
431 	else
432 		pd->flags |= GENPD_FLAG_ALWAYS_ON;
433 
434 	/* Use governor for CPU PM domains if it has some states to manage. */
435 	pd_gov = pd->states ? &pm_domain_cpu_gov : NULL;
436 
437 	ret = pm_genpd_init(pd, pd_gov, false);
438 	if (ret)
439 		goto free_pd_prov;
440 
441 	ret = of_genpd_add_provider_simple(np, pd);
442 	if (ret)
443 		goto remove_pd;
444 
445 	pd_provider->node = of_node_get(np);
446 	list_add(&pd_provider->link, &sbi_pd_providers);
447 
448 	pr_debug("init PM domain %s\n", pd->name);
449 	return 0;
450 
451 remove_pd:
452 	pm_genpd_remove(pd);
453 free_pd_prov:
454 	kfree(pd_provider);
455 free_pd:
456 	dt_idle_pd_free(pd);
457 out:
458 	pr_err("failed to init PM domain ret=%d %pOF\n", ret, np);
459 	return ret;
460 }
461 
462 static void sbi_pd_remove(void)
463 {
464 	struct sbi_pd_provider *pd_provider, *it;
465 	struct generic_pm_domain *genpd;
466 
467 	list_for_each_entry_safe(pd_provider, it, &sbi_pd_providers, link) {
468 		of_genpd_del_provider(pd_provider->node);
469 
470 		genpd = of_genpd_remove_last(pd_provider->node);
471 		if (!IS_ERR(genpd))
472 			kfree(genpd);
473 
474 		of_node_put(pd_provider->node);
475 		list_del(&pd_provider->link);
476 		kfree(pd_provider);
477 	}
478 }
479 
480 static int sbi_genpd_probe(struct device_node *np)
481 {
482 	struct device_node *node;
483 	int ret = 0, pd_count = 0;
484 
485 	if (!np)
486 		return -ENODEV;
487 
488 	/*
489 	 * Parse child nodes for the "#power-domain-cells" property and
490 	 * initialize a genpd/genpd-of-provider pair when it's found.
491 	 */
492 	for_each_child_of_node(np, node) {
493 		if (!of_property_present(node, "#power-domain-cells"))
494 			continue;
495 
496 		ret = sbi_pd_init(node);
497 		if (ret)
498 			goto put_node;
499 
500 		pd_count++;
501 	}
502 
503 	/* Bail out if not using the hierarchical CPU topology. */
504 	if (!pd_count)
505 		goto no_pd;
506 
507 	/* Link genpd masters/subdomains to model the CPU topology. */
508 	ret = dt_idle_pd_init_topology(np);
509 	if (ret)
510 		goto remove_pd;
511 
512 	return 0;
513 
514 put_node:
515 	of_node_put(node);
516 remove_pd:
517 	sbi_pd_remove();
518 	pr_err("failed to create CPU PM domains ret=%d\n", ret);
519 no_pd:
520 	return ret;
521 }
522 
523 #else
524 
525 static inline int sbi_genpd_probe(struct device_node *np)
526 {
527 	return 0;
528 }
529 
530 #endif
531 
532 static int sbi_cpuidle_probe(struct platform_device *pdev)
533 {
534 	int cpu, ret;
535 	struct cpuidle_driver *drv;
536 	struct cpuidle_device *dev;
537 	struct device_node *np, *pds_node;
538 
539 	/* Detect OSI support based on CPU DT nodes */
540 	sbi_cpuidle_use_osi = true;
541 	for_each_possible_cpu(cpu) {
542 		np = of_cpu_device_node_get(cpu);
543 		if (np &&
544 		    of_property_present(np, "power-domains") &&
545 		    of_property_present(np, "power-domain-names")) {
546 			continue;
547 		} else {
548 			sbi_cpuidle_use_osi = false;
549 			break;
550 		}
551 	}
552 
553 	/* Populate generic power domains from DT nodes */
554 	pds_node = of_find_node_by_path("/cpus/power-domains");
555 	if (pds_node) {
556 		ret = sbi_genpd_probe(pds_node);
557 		of_node_put(pds_node);
558 		if (ret)
559 			return ret;
560 	}
561 
562 	/* Initialize CPU idle driver for each CPU */
563 	for_each_possible_cpu(cpu) {
564 		ret = sbi_cpuidle_init_cpu(&pdev->dev, cpu);
565 		if (ret) {
566 			pr_debug("HART%ld: idle driver init failed\n",
567 				 cpuid_to_hartid_map(cpu));
568 			goto out_fail;
569 		}
570 	}
571 
572 	/* Setup CPU hotplut notifiers */
573 	sbi_idle_init_cpuhp();
574 
575 	pr_info("idle driver registered for all CPUs\n");
576 
577 	return 0;
578 
579 out_fail:
580 	while (--cpu >= 0) {
581 		dev = per_cpu(cpuidle_devices, cpu);
582 		drv = cpuidle_get_cpu_driver(dev);
583 		cpuidle_unregister(drv);
584 		sbi_cpuidle_deinit_cpu(cpu);
585 	}
586 
587 	return ret;
588 }
589 
590 static struct platform_driver sbi_cpuidle_driver = {
591 	.probe = sbi_cpuidle_probe,
592 	.driver = {
593 		.name = "sbi-cpuidle",
594 		.sync_state = sbi_cpuidle_domain_sync_state,
595 	},
596 };
597 
598 static int __init sbi_cpuidle_init(void)
599 {
600 	int ret;
601 	struct platform_device *pdev;
602 
603 	/*
604 	 * The SBI HSM suspend function is only available when:
605 	 * 1) SBI version is 0.3 or higher
606 	 * 2) SBI HSM extension is available
607 	 */
608 	if ((sbi_spec_version < sbi_mk_version(0, 3)) ||
609 	    !sbi_probe_extension(SBI_EXT_HSM)) {
610 		pr_info("HSM suspend not available\n");
611 		return 0;
612 	}
613 
614 	ret = platform_driver_register(&sbi_cpuidle_driver);
615 	if (ret)
616 		return ret;
617 
618 	pdev = platform_device_register_simple("sbi-cpuidle",
619 						-1, NULL, 0);
620 	if (IS_ERR(pdev)) {
621 		platform_driver_unregister(&sbi_cpuidle_driver);
622 		return PTR_ERR(pdev);
623 	}
624 
625 	return 0;
626 }
627 device_initcall(sbi_cpuidle_init);
628