1 /*
2  * PowerNV cpuidle code
3  *
4  * Copyright 2015 IBM Corp.
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11 
12 #include <linux/types.h>
13 #include <linux/mm.h>
14 #include <linux/slab.h>
15 #include <linux/of.h>
16 #include <linux/device.h>
17 #include <linux/cpu.h>
18 
19 #include <asm/firmware.h>
20 #include <asm/machdep.h>
21 #include <asm/opal.h>
22 #include <asm/cputhreads.h>
23 #include <asm/cpuidle.h>
24 #include <asm/code-patching.h>
25 #include <asm/smp.h>
26 
27 #include "powernv.h"
28 #include "subcore.h"
29 
30 static u32 supported_cpuidle_states;
31 
32 int pnv_save_sprs_for_winkle(void)
33 {
34 	int cpu;
35 	int rc;
36 
37 	/*
38 	 * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric accross
39 	 * all cpus at boot. Get these reg values of current cpu and use the
40 	 * same accross all cpus.
41 	 */
42 	uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1;
43 	uint64_t hid0_val = mfspr(SPRN_HID0);
44 	uint64_t hid1_val = mfspr(SPRN_HID1);
45 	uint64_t hid4_val = mfspr(SPRN_HID4);
46 	uint64_t hid5_val = mfspr(SPRN_HID5);
47 	uint64_t hmeer_val = mfspr(SPRN_HMEER);
48 
49 	for_each_possible_cpu(cpu) {
50 		uint64_t pir = get_hard_smp_processor_id(cpu);
51 		uint64_t hsprg0_val = (uint64_t)&paca[cpu];
52 
53 		/*
54 		 * HSPRG0 is used to store the cpu's pointer to paca. Hence last
55 		 * 3 bits are guaranteed to be 0. Program slw to restore HSPRG0
56 		 * with 63rd bit set, so that when a thread wakes up at 0x100 we
57 		 * can use this bit to distinguish between fastsleep and
58 		 * deep winkle.
59 		 */
60 		hsprg0_val |= 1;
61 
62 		rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val);
63 		if (rc != 0)
64 			return rc;
65 
66 		rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
67 		if (rc != 0)
68 			return rc;
69 
70 		/* HIDs are per core registers */
71 		if (cpu_thread_in_core(cpu) == 0) {
72 
73 			rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val);
74 			if (rc != 0)
75 				return rc;
76 
77 			rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val);
78 			if (rc != 0)
79 				return rc;
80 
81 			rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
82 			if (rc != 0)
83 				return rc;
84 
85 			rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val);
86 			if (rc != 0)
87 				return rc;
88 
89 			rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val);
90 			if (rc != 0)
91 				return rc;
92 		}
93 	}
94 
95 	return 0;
96 }
97 
98 static void pnv_alloc_idle_core_states(void)
99 {
100 	int i, j;
101 	int nr_cores = cpu_nr_cores();
102 	u32 *core_idle_state;
103 
104 	/*
105 	 * core_idle_state - First 8 bits track the idle state of each thread
106 	 * of the core. The 8th bit is the lock bit. Initially all thread bits
107 	 * are set. They are cleared when the thread enters deep idle state
108 	 * like sleep and winkle. Initially the lock bit is cleared.
109 	 * The lock bit has 2 purposes
110 	 * a. While the first thread is restoring core state, it prevents
111 	 * other threads in the core from switching to process context.
112 	 * b. While the last thread in the core is saving the core state, it
113 	 * prevents a different thread from waking up.
114 	 */
115 	for (i = 0; i < nr_cores; i++) {
116 		int first_cpu = i * threads_per_core;
117 		int node = cpu_to_node(first_cpu);
118 
119 		core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
120 		*core_idle_state = PNV_CORE_IDLE_THREAD_BITS;
121 
122 		for (j = 0; j < threads_per_core; j++) {
123 			int cpu = first_cpu + j;
124 
125 			paca[cpu].core_idle_state_ptr = core_idle_state;
126 			paca[cpu].thread_idle_state = PNV_THREAD_RUNNING;
127 			paca[cpu].thread_mask = 1 << j;
128 		}
129 	}
130 
131 	update_subcore_sibling_mask();
132 
133 	if (supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED)
134 		pnv_save_sprs_for_winkle();
135 }
136 
137 u32 pnv_get_supported_cpuidle_states(void)
138 {
139 	return supported_cpuidle_states;
140 }
141 EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states);
142 
143 
144 static void pnv_fastsleep_workaround_apply(void *info)
145 
146 {
147 	int rc;
148 	int *err = info;
149 
150 	rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
151 					OPAL_CONFIG_IDLE_APPLY);
152 	if (rc)
153 		*err = 1;
154 }
155 
156 /*
157  * Used to store fastsleep workaround state
158  * 0 - Workaround applied/undone at fastsleep entry/exit path (Default)
159  * 1 - Workaround applied once, never undone.
160  */
161 static u8 fastsleep_workaround_applyonce;
162 
163 static ssize_t show_fastsleep_workaround_applyonce(struct device *dev,
164 		struct device_attribute *attr, char *buf)
165 {
166 	return sprintf(buf, "%u\n", fastsleep_workaround_applyonce);
167 }
168 
169 static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
170 		struct device_attribute *attr, const char *buf,
171 		size_t count)
172 {
173 	cpumask_t primary_thread_mask;
174 	int err;
175 	u8 val;
176 
177 	if (kstrtou8(buf, 0, &val) || val != 1)
178 		return -EINVAL;
179 
180 	if (fastsleep_workaround_applyonce == 1)
181 		return count;
182 
183 	/*
184 	 * fastsleep_workaround_applyonce = 1 implies
185 	 * fastsleep workaround needs to be left in 'applied' state on all
186 	 * the cores. Do this by-
187 	 * 1. Patching out the call to 'undo' workaround in fastsleep exit path
188 	 * 2. Sending ipi to all the cores which have atleast one online thread
189 	 * 3. Patching out the call to 'apply' workaround in fastsleep entry
190 	 * path
191 	 * There is no need to send ipi to cores which have all threads
192 	 * offlined, as last thread of the core entering fastsleep or deeper
193 	 * state would have applied workaround.
194 	 */
195 	err = patch_instruction(
196 		(unsigned int *)pnv_fastsleep_workaround_at_exit,
197 		PPC_INST_NOP);
198 	if (err) {
199 		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit");
200 		goto fail;
201 	}
202 
203 	get_online_cpus();
204 	primary_thread_mask = cpu_online_cores_map();
205 	on_each_cpu_mask(&primary_thread_mask,
206 				pnv_fastsleep_workaround_apply,
207 				&err, 1);
208 	put_online_cpus();
209 	if (err) {
210 		pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply");
211 		goto fail;
212 	}
213 
214 	err = patch_instruction(
215 		(unsigned int *)pnv_fastsleep_workaround_at_entry,
216 		PPC_INST_NOP);
217 	if (err) {
218 		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry");
219 		goto fail;
220 	}
221 
222 	fastsleep_workaround_applyonce = 1;
223 
224 	return count;
225 fail:
226 	return -EIO;
227 }
228 
229 static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
230 			show_fastsleep_workaround_applyonce,
231 			store_fastsleep_workaround_applyonce);
232 
233 static int __init pnv_init_idle_states(void)
234 {
235 	struct device_node *power_mgt;
236 	int dt_idle_states;
237 	u32 *flags;
238 	int i;
239 
240 	supported_cpuidle_states = 0;
241 
242 	if (cpuidle_disable != IDLE_NO_OVERRIDE)
243 		goto out;
244 
245 	if (!firmware_has_feature(FW_FEATURE_OPALv3))
246 		goto out;
247 
248 	power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
249 	if (!power_mgt) {
250 		pr_warn("opal: PowerMgmt Node not found\n");
251 		goto out;
252 	}
253 	dt_idle_states = of_property_count_u32_elems(power_mgt,
254 			"ibm,cpu-idle-state-flags");
255 	if (dt_idle_states < 0) {
256 		pr_warn("cpuidle-powernv: no idle states found in the DT\n");
257 		goto out;
258 	}
259 
260 	flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL);
261 	if (of_property_read_u32_array(power_mgt,
262 			"ibm,cpu-idle-state-flags", flags, dt_idle_states)) {
263 		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n");
264 		goto out_free;
265 	}
266 
267 	for (i = 0; i < dt_idle_states; i++)
268 		supported_cpuidle_states |= flags[i];
269 
270 	if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
271 		patch_instruction(
272 			(unsigned int *)pnv_fastsleep_workaround_at_entry,
273 			PPC_INST_NOP);
274 		patch_instruction(
275 			(unsigned int *)pnv_fastsleep_workaround_at_exit,
276 			PPC_INST_NOP);
277 	} else {
278 		/*
279 		 * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
280 		 * workaround is needed to use fastsleep. Provide sysfs
281 		 * control to choose how this workaround has to be applied.
282 		 */
283 		device_create_file(cpu_subsys.dev_root,
284 				&dev_attr_fastsleep_workaround_applyonce);
285 	}
286 
287 	pnv_alloc_idle_core_states();
288 out_free:
289 	kfree(flags);
290 out:
291 	return 0;
292 }
293 machine_subsys_initcall(powernv, pnv_init_idle_states);
294