xref: /openbmc/linux/arch/x86/lib/msr-smp.c (revision 396a66aa)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/export.h>
3 #include <linux/preempt.h>
4 #include <linux/smp.h>
5 #include <linux/completion.h>
6 #include <asm/msr.h>
7 
__rdmsr_on_cpu(void * info)8 static void __rdmsr_on_cpu(void *info)
9 {
10 	struct msr_info *rv = info;
11 	struct msr *reg;
12 	int this_cpu = raw_smp_processor_id();
13 
14 	if (rv->msrs)
15 		reg = per_cpu_ptr(rv->msrs, this_cpu);
16 	else
17 		reg = &rv->reg;
18 
19 	rdmsr(rv->msr_no, reg->l, reg->h);
20 }
21 
__wrmsr_on_cpu(void * info)22 static void __wrmsr_on_cpu(void *info)
23 {
24 	struct msr_info *rv = info;
25 	struct msr *reg;
26 	int this_cpu = raw_smp_processor_id();
27 
28 	if (rv->msrs)
29 		reg = per_cpu_ptr(rv->msrs, this_cpu);
30 	else
31 		reg = &rv->reg;
32 
33 	wrmsr(rv->msr_no, reg->l, reg->h);
34 }
35 
rdmsr_on_cpu(unsigned int cpu,u32 msr_no,u32 * l,u32 * h)36 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
37 {
38 	int err;
39 	struct msr_info rv;
40 
41 	memset(&rv, 0, sizeof(rv));
42 
43 	rv.msr_no = msr_no;
44 	err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
45 	*l = rv.reg.l;
46 	*h = rv.reg.h;
47 
48 	return err;
49 }
50 EXPORT_SYMBOL(rdmsr_on_cpu);
51 
rdmsrl_on_cpu(unsigned int cpu,u32 msr_no,u64 * q)52 int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
53 {
54 	int err;
55 	struct msr_info rv;
56 
57 	memset(&rv, 0, sizeof(rv));
58 
59 	rv.msr_no = msr_no;
60 	err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
61 	*q = rv.reg.q;
62 
63 	return err;
64 }
65 EXPORT_SYMBOL(rdmsrl_on_cpu);
66 
wrmsr_on_cpu(unsigned int cpu,u32 msr_no,u32 l,u32 h)67 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
68 {
69 	int err;
70 	struct msr_info rv;
71 
72 	memset(&rv, 0, sizeof(rv));
73 
74 	rv.msr_no = msr_no;
75 	rv.reg.l = l;
76 	rv.reg.h = h;
77 	err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
78 
79 	return err;
80 }
81 EXPORT_SYMBOL(wrmsr_on_cpu);
82 
wrmsrl_on_cpu(unsigned int cpu,u32 msr_no,u64 q)83 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
84 {
85 	int err;
86 	struct msr_info rv;
87 
88 	memset(&rv, 0, sizeof(rv));
89 
90 	rv.msr_no = msr_no;
91 	rv.reg.q = q;
92 
93 	err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
94 
95 	return err;
96 }
97 EXPORT_SYMBOL(wrmsrl_on_cpu);
98 
__rwmsr_on_cpus(const struct cpumask * mask,u32 msr_no,struct msr * msrs,void (* msr_func)(void * info))99 static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no,
100 			    struct msr *msrs,
101 			    void (*msr_func) (void *info))
102 {
103 	struct msr_info rv;
104 	int this_cpu;
105 
106 	memset(&rv, 0, sizeof(rv));
107 
108 	rv.msrs	  = msrs;
109 	rv.msr_no = msr_no;
110 
111 	this_cpu = get_cpu();
112 
113 	if (cpumask_test_cpu(this_cpu, mask))
114 		msr_func(&rv);
115 
116 	smp_call_function_many(mask, msr_func, &rv, 1);
117 	put_cpu();
118 }
119 
120 /* rdmsr on a bunch of CPUs
121  *
122  * @mask:       which CPUs
123  * @msr_no:     which MSR
124  * @msrs:       array of MSR values
125  *
126  */
rdmsr_on_cpus(const struct cpumask * mask,u32 msr_no,struct msr * msrs)127 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
128 {
129 	__rwmsr_on_cpus(mask, msr_no, msrs, __rdmsr_on_cpu);
130 }
131 EXPORT_SYMBOL(rdmsr_on_cpus);
132 
133 /*
134  * wrmsr on a bunch of CPUs
135  *
136  * @mask:       which CPUs
137  * @msr_no:     which MSR
138  * @msrs:       array of MSR values
139  *
140  */
wrmsr_on_cpus(const struct cpumask * mask,u32 msr_no,struct msr * msrs)141 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
142 {
143 	__rwmsr_on_cpus(mask, msr_no, msrs, __wrmsr_on_cpu);
144 }
145 EXPORT_SYMBOL(wrmsr_on_cpus);
146 
147 struct msr_info_completion {
148 	struct msr_info		msr;
149 	struct completion	done;
150 };
151 
152 /* These "safe" variants are slower and should be used when the target MSR
153    may not actually exist. */
__rdmsr_safe_on_cpu(void * info)154 static void __rdmsr_safe_on_cpu(void *info)
155 {
156 	struct msr_info_completion *rv = info;
157 
158 	rv->msr.err = rdmsr_safe(rv->msr.msr_no, &rv->msr.reg.l, &rv->msr.reg.h);
159 	complete(&rv->done);
160 }
161 
__wrmsr_safe_on_cpu(void * info)162 static void __wrmsr_safe_on_cpu(void *info)
163 {
164 	struct msr_info *rv = info;
165 
166 	rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h);
167 }
168 
rdmsr_safe_on_cpu(unsigned int cpu,u32 msr_no,u32 * l,u32 * h)169 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
170 {
171 	struct msr_info_completion rv;
172 	call_single_data_t csd;
173 	int err;
174 
175 	INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv);
176 
177 	memset(&rv, 0, sizeof(rv));
178 	init_completion(&rv.done);
179 	rv.msr.msr_no = msr_no;
180 
181 	err = smp_call_function_single_async(cpu, &csd);
182 	if (!err) {
183 		wait_for_completion(&rv.done);
184 		err = rv.msr.err;
185 	}
186 	*l = rv.msr.reg.l;
187 	*h = rv.msr.reg.h;
188 
189 	return err;
190 }
191 EXPORT_SYMBOL(rdmsr_safe_on_cpu);
192 
wrmsr_safe_on_cpu(unsigned int cpu,u32 msr_no,u32 l,u32 h)193 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
194 {
195 	int err;
196 	struct msr_info rv;
197 
198 	memset(&rv, 0, sizeof(rv));
199 
200 	rv.msr_no = msr_no;
201 	rv.reg.l = l;
202 	rv.reg.h = h;
203 	err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
204 
205 	return err ? err : rv.err;
206 }
207 EXPORT_SYMBOL(wrmsr_safe_on_cpu);
208 
wrmsrl_safe_on_cpu(unsigned int cpu,u32 msr_no,u64 q)209 int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
210 {
211 	int err;
212 	struct msr_info rv;
213 
214 	memset(&rv, 0, sizeof(rv));
215 
216 	rv.msr_no = msr_no;
217 	rv.reg.q = q;
218 
219 	err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
220 
221 	return err ? err : rv.err;
222 }
223 EXPORT_SYMBOL(wrmsrl_safe_on_cpu);
224 
rdmsrl_safe_on_cpu(unsigned int cpu,u32 msr_no,u64 * q)225 int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
226 {
227 	u32 low, high;
228 	int err;
229 
230 	err = rdmsr_safe_on_cpu(cpu, msr_no, &low, &high);
231 	*q = (u64)high << 32 | low;
232 
233 	return err;
234 }
235 EXPORT_SYMBOL(rdmsrl_safe_on_cpu);
236 
237 /*
238  * These variants are significantly slower, but allows control over
239  * the entire 32-bit GPR set.
240  */
__rdmsr_safe_regs_on_cpu(void * info)241 static void __rdmsr_safe_regs_on_cpu(void *info)
242 {
243 	struct msr_regs_info *rv = info;
244 
245 	rv->err = rdmsr_safe_regs(rv->regs);
246 }
247 
__wrmsr_safe_regs_on_cpu(void * info)248 static void __wrmsr_safe_regs_on_cpu(void *info)
249 {
250 	struct msr_regs_info *rv = info;
251 
252 	rv->err = wrmsr_safe_regs(rv->regs);
253 }
254 
rdmsr_safe_regs_on_cpu(unsigned int cpu,u32 regs[8])255 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
256 {
257 	int err;
258 	struct msr_regs_info rv;
259 
260 	rv.regs   = regs;
261 	rv.err    = -EIO;
262 	err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1);
263 
264 	return err ? err : rv.err;
265 }
266 EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu);
267 
wrmsr_safe_regs_on_cpu(unsigned int cpu,u32 regs[8])268 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
269 {
270 	int err;
271 	struct msr_regs_info rv;
272 
273 	rv.regs = regs;
274 	rv.err  = -EIO;
275 	err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1);
276 
277 	return err ? err : rv.err;
278 }
279 EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu);
280