xref: /openbmc/linux/drivers/firmware/arm_sdei.c (revision ae0be8de) 
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2017 Arm Ltd.
3 #define pr_fmt(fmt) "sdei: " fmt
4 
5 #include <acpi/ghes.h>
6 #include <linux/acpi.h>
7 #include <linux/arm_sdei.h>
8 #include <linux/arm-smccc.h>
9 #include <linux/atomic.h>
10 #include <linux/bitops.h>
11 #include <linux/compiler.h>
12 #include <linux/cpuhotplug.h>
13 #include <linux/cpu.h>
14 #include <linux/cpu_pm.h>
15 #include <linux/errno.h>
16 #include <linux/hardirq.h>
17 #include <linux/kernel.h>
18 #include <linux/kprobes.h>
19 #include <linux/kvm_host.h>
20 #include <linux/list.h>
21 #include <linux/mutex.h>
22 #include <linux/notifier.h>
23 #include <linux/of.h>
24 #include <linux/of_platform.h>
25 #include <linux/percpu.h>
26 #include <linux/platform_device.h>
27 #include <linux/pm.h>
28 #include <linux/ptrace.h>
29 #include <linux/preempt.h>
30 #include <linux/reboot.h>
31 #include <linux/slab.h>
32 #include <linux/smp.h>
33 #include <linux/spinlock.h>
34 #include <linux/uaccess.h>
35 
36 /*
37  * The call to use to reach the firmware.
38  */
39 static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
40 		      unsigned long arg0, unsigned long arg1,
41 		      unsigned long arg2, unsigned long arg3,
42 		      unsigned long arg4, struct arm_smccc_res *res);
43 
44 /* entry point from firmware to arch asm code */
45 static unsigned long sdei_entry_point;
46 
47 struct sdei_event {
48 	/* These three are protected by the sdei_list_lock */
49 	struct list_head	list;
50 	bool			reregister;
51 	bool			reenable;
52 
53 	u32			event_num;
54 	u8			type;
55 	u8			priority;
56 
57 	/* This pointer is handed to firmware as the event argument. */
58 	union {
59 		/* Shared events */
60 		struct sdei_registered_event *registered;
61 
62 		/* CPU private events */
63 		struct sdei_registered_event __percpu *private_registered;
64 	};
65 };
66 
67 /* Take the mutex for any API call or modification. Take the mutex first. */
68 static DEFINE_MUTEX(sdei_events_lock);
69 
70 /* and then hold this when modifying the list */
71 static DEFINE_SPINLOCK(sdei_list_lock);
72 static LIST_HEAD(sdei_list);
73 
74 /* Private events are registered/enabled via IPI passing one of these */
75 struct sdei_crosscall_args {
76 	struct sdei_event *event;
77 	atomic_t errors;
78 	int first_error;
79 };
80 
81 #define CROSSCALL_INIT(arg, event)	(arg.event = event, \
82 					 arg.first_error = 0, \
83 					 atomic_set(&arg.errors, 0))
84 
85 static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
86 {
87 	struct sdei_crosscall_args arg;
88 
89 	CROSSCALL_INIT(arg, event);
90 	on_each_cpu(fn, &arg, true);
91 
92 	return arg.first_error;
93 }
94 
95 static inline void
96 sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
97 {
98 	if (err && (atomic_inc_return(&arg->errors) == 1))
99 		arg->first_error = err;
100 }
101 
102 static int sdei_to_linux_errno(unsigned long sdei_err)
103 {
104 	switch (sdei_err) {
105 	case SDEI_NOT_SUPPORTED:
106 		return -EOPNOTSUPP;
107 	case SDEI_INVALID_PARAMETERS:
108 		return -EINVAL;
109 	case SDEI_DENIED:
110 		return -EPERM;
111 	case SDEI_PENDING:
112 		return -EINPROGRESS;
113 	case SDEI_OUT_OF_RESOURCE:
114 		return -ENOMEM;
115 	}
116 
117 	/* Not an error value ... */
118 	return sdei_err;
119 }
120 
121 /*
122  * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
123  * to translate.
124  */
125 static int sdei_is_err(struct arm_smccc_res *res)
126 {
127 	switch (res->a0) {
128 	case SDEI_NOT_SUPPORTED:
129 	case SDEI_INVALID_PARAMETERS:
130 	case SDEI_DENIED:
131 	case SDEI_PENDING:
132 	case SDEI_OUT_OF_RESOURCE:
133 		return true;
134 	}
135 
136 	return false;
137 }
138 
139 static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
140 			  unsigned long arg1, unsigned long arg2,
141 			  unsigned long arg3, unsigned long arg4,
142 			  u64 *result)
143 {
144 	int err = 0;
145 	struct arm_smccc_res res;
146 
147 	if (sdei_firmware_call) {
148 		sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
149 				   &res);
150 		if (sdei_is_err(&res))
151 			err = sdei_to_linux_errno(res.a0);
152 	} else {
153 		/*
154 		 * !sdei_firmware_call means we failed to probe or called
155 		 * sdei_mark_interface_broken(). -EIO is not an error returned
156 		 * by sdei_to_linux_errno() and is used to suppress messages
157 		 * from this driver.
158 		 */
159 		err = -EIO;
160 		res.a0 = SDEI_NOT_SUPPORTED;
161 	}
162 
163 	if (result)
164 		*result = res.a0;
165 
166 	return err;
167 }
168 
169 static struct sdei_event *sdei_event_find(u32 event_num)
170 {
171 	struct sdei_event *e, *found = NULL;
172 
173 	lockdep_assert_held(&sdei_events_lock);
174 
175 	spin_lock(&sdei_list_lock);
176 	list_for_each_entry(e, &sdei_list, list) {
177 		if (e->event_num == event_num) {
178 			found = e;
179 			break;
180 		}
181 	}
182 	spin_unlock(&sdei_list_lock);
183 
184 	return found;
185 }
186 
187 int sdei_api_event_context(u32 query, u64 *result)
188 {
189 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
190 			      result);
191 }
192 NOKPROBE_SYMBOL(sdei_api_event_context);
193 
194 static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
195 {
196 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
197 			      0, 0, result);
198 }
199 
200 static struct sdei_event *sdei_event_create(u32 event_num,
201 					    sdei_event_callback *cb,
202 					    void *cb_arg)
203 {
204 	int err;
205 	u64 result;
206 	struct sdei_event *event;
207 	struct sdei_registered_event *reg;
208 
209 	lockdep_assert_held(&sdei_events_lock);
210 
211 	event = kzalloc(sizeof(*event), GFP_KERNEL);
212 	if (!event)
213 		return ERR_PTR(-ENOMEM);
214 
215 	INIT_LIST_HEAD(&event->list);
216 	event->event_num = event_num;
217 
218 	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
219 				      &result);
220 	if (err) {
221 		kfree(event);
222 		return ERR_PTR(err);
223 	}
224 	event->priority = result;
225 
226 	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
227 				      &result);
228 	if (err) {
229 		kfree(event);
230 		return ERR_PTR(err);
231 	}
232 	event->type = result;
233 
234 	if (event->type == SDEI_EVENT_TYPE_SHARED) {
235 		reg = kzalloc(sizeof(*reg), GFP_KERNEL);
236 		if (!reg) {
237 			kfree(event);
238 			return ERR_PTR(-ENOMEM);
239 		}
240 
241 		reg->event_num = event_num;
242 		reg->priority = event->priority;
243 
244 		reg->callback = cb;
245 		reg->callback_arg = cb_arg;
246 		event->registered = reg;
247 	} else {
248 		int cpu;
249 		struct sdei_registered_event __percpu *regs;
250 
251 		regs = alloc_percpu(struct sdei_registered_event);
252 		if (!regs) {
253 			kfree(event);
254 			return ERR_PTR(-ENOMEM);
255 		}
256 
257 		for_each_possible_cpu(cpu) {
258 			reg = per_cpu_ptr(regs, cpu);
259 
260 			reg->event_num = event->event_num;
261 			reg->priority = event->priority;
262 			reg->callback = cb;
263 			reg->callback_arg = cb_arg;
264 		}
265 
266 		event->private_registered = regs;
267 	}
268 
269 	if (sdei_event_find(event_num)) {
270 		kfree(event->registered);
271 		kfree(event);
272 		event = ERR_PTR(-EBUSY);
273 	} else {
274 		spin_lock(&sdei_list_lock);
275 		list_add(&event->list, &sdei_list);
276 		spin_unlock(&sdei_list_lock);
277 	}
278 
279 	return event;
280 }
281 
282 static void sdei_event_destroy(struct sdei_event *event)
283 {
284 	lockdep_assert_held(&sdei_events_lock);
285 
286 	spin_lock(&sdei_list_lock);
287 	list_del(&event->list);
288 	spin_unlock(&sdei_list_lock);
289 
290 	if (event->type == SDEI_EVENT_TYPE_SHARED)
291 		kfree(event->registered);
292 	else
293 		free_percpu(event->private_registered);
294 
295 	kfree(event);
296 }
297 
298 static int sdei_api_get_version(u64 *version)
299 {
300 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
301 }
302 
303 int sdei_mask_local_cpu(void)
304 {
305 	int err;
306 
307 	WARN_ON_ONCE(preemptible());
308 
309 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
310 	if (err && err != -EIO) {
311 		pr_warn_once("failed to mask CPU[%u]: %d\n",
312 			      smp_processor_id(), err);
313 		return err;
314 	}
315 
316 	return 0;
317 }
318 
319 static void _ipi_mask_cpu(void *ignored)
320 {
321 	sdei_mask_local_cpu();
322 }
323 
324 int sdei_unmask_local_cpu(void)
325 {
326 	int err;
327 
328 	WARN_ON_ONCE(preemptible());
329 
330 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
331 	if (err && err != -EIO) {
332 		pr_warn_once("failed to unmask CPU[%u]: %d\n",
333 			     smp_processor_id(), err);
334 		return err;
335 	}
336 
337 	return 0;
338 }
339 
340 static void _ipi_unmask_cpu(void *ignored)
341 {
342 	sdei_unmask_local_cpu();
343 }
344 
345 static void _ipi_private_reset(void *ignored)
346 {
347 	int err;
348 
349 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
350 			     NULL);
351 	if (err && err != -EIO)
352 		pr_warn_once("failed to reset CPU[%u]: %d\n",
353 			     smp_processor_id(), err);
354 }
355 
356 static int sdei_api_shared_reset(void)
357 {
358 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
359 			      NULL);
360 }
361 
362 static void sdei_mark_interface_broken(void)
363 {
364 	pr_err("disabling SDEI firmware interface\n");
365 	on_each_cpu(&_ipi_mask_cpu, NULL, true);
366 	sdei_firmware_call = NULL;
367 }
368 
369 static int sdei_platform_reset(void)
370 {
371 	int err;
372 
373 	on_each_cpu(&_ipi_private_reset, NULL, true);
374 	err = sdei_api_shared_reset();
375 	if (err) {
376 		pr_err("Failed to reset platform: %d\n", err);
377 		sdei_mark_interface_broken();
378 	}
379 
380 	return err;
381 }
382 
383 static int sdei_api_event_enable(u32 event_num)
384 {
385 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
386 			      0, NULL);
387 }
388 
389 /* Called directly by the hotplug callbacks */
390 static void _local_event_enable(void *data)
391 {
392 	int err;
393 	struct sdei_crosscall_args *arg = data;
394 
395 	WARN_ON_ONCE(preemptible());
396 
397 	err = sdei_api_event_enable(arg->event->event_num);
398 
399 	sdei_cross_call_return(arg, err);
400 }
401 
402 int sdei_event_enable(u32 event_num)
403 {
404 	int err = -EINVAL;
405 	struct sdei_event *event;
406 
407 	mutex_lock(&sdei_events_lock);
408 	event = sdei_event_find(event_num);
409 	if (!event) {
410 		mutex_unlock(&sdei_events_lock);
411 		return -ENOENT;
412 	}
413 
414 	spin_lock(&sdei_list_lock);
415 	event->reenable = true;
416 	spin_unlock(&sdei_list_lock);
417 
418 	if (event->type == SDEI_EVENT_TYPE_SHARED)
419 		err = sdei_api_event_enable(event->event_num);
420 	else
421 		err = sdei_do_cross_call(_local_event_enable, event);
422 	mutex_unlock(&sdei_events_lock);
423 
424 	return err;
425 }
426 EXPORT_SYMBOL(sdei_event_enable);
427 
428 static int sdei_api_event_disable(u32 event_num)
429 {
430 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
431 			      0, 0, NULL);
432 }
433 
434 static void _ipi_event_disable(void *data)
435 {
436 	int err;
437 	struct sdei_crosscall_args *arg = data;
438 
439 	err = sdei_api_event_disable(arg->event->event_num);
440 
441 	sdei_cross_call_return(arg, err);
442 }
443 
444 int sdei_event_disable(u32 event_num)
445 {
446 	int err = -EINVAL;
447 	struct sdei_event *event;
448 
449 	mutex_lock(&sdei_events_lock);
450 	event = sdei_event_find(event_num);
451 	if (!event) {
452 		mutex_unlock(&sdei_events_lock);
453 		return -ENOENT;
454 	}
455 
456 	spin_lock(&sdei_list_lock);
457 	event->reenable = false;
458 	spin_unlock(&sdei_list_lock);
459 
460 	if (event->type == SDEI_EVENT_TYPE_SHARED)
461 		err = sdei_api_event_disable(event->event_num);
462 	else
463 		err = sdei_do_cross_call(_ipi_event_disable, event);
464 	mutex_unlock(&sdei_events_lock);
465 
466 	return err;
467 }
468 EXPORT_SYMBOL(sdei_event_disable);
469 
470 static int sdei_api_event_unregister(u32 event_num)
471 {
472 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
473 			      0, 0, 0, NULL);
474 }
475 
476 /* Called directly by the hotplug callbacks */
477 static void _local_event_unregister(void *data)
478 {
479 	int err;
480 	struct sdei_crosscall_args *arg = data;
481 
482 	WARN_ON_ONCE(preemptible());
483 
484 	err = sdei_api_event_unregister(arg->event->event_num);
485 
486 	sdei_cross_call_return(arg, err);
487 }
488 
489 static int _sdei_event_unregister(struct sdei_event *event)
490 {
491 	lockdep_assert_held(&sdei_events_lock);
492 
493 	spin_lock(&sdei_list_lock);
494 	event->reregister = false;
495 	event->reenable = false;
496 	spin_unlock(&sdei_list_lock);
497 
498 	if (event->type == SDEI_EVENT_TYPE_SHARED)
499 		return sdei_api_event_unregister(event->event_num);
500 
501 	return sdei_do_cross_call(_local_event_unregister, event);
502 }
503 
504 int sdei_event_unregister(u32 event_num)
505 {
506 	int err;
507 	struct sdei_event *event;
508 
509 	WARN_ON(in_nmi());
510 
511 	mutex_lock(&sdei_events_lock);
512 	event = sdei_event_find(event_num);
513 	do {
514 		if (!event) {
515 			pr_warn("Event %u not registered\n", event_num);
516 			err = -ENOENT;
517 			break;
518 		}
519 
520 		err = _sdei_event_unregister(event);
521 		if (err)
522 			break;
523 
524 		sdei_event_destroy(event);
525 	} while (0);
526 	mutex_unlock(&sdei_events_lock);
527 
528 	return err;
529 }
530 EXPORT_SYMBOL(sdei_event_unregister);
531 
532 /*
533  * unregister events, but don't destroy them as they are re-registered by
534  * sdei_reregister_shared().
535  */
536 static int sdei_unregister_shared(void)
537 {
538 	int err = 0;
539 	struct sdei_event *event;
540 
541 	mutex_lock(&sdei_events_lock);
542 	spin_lock(&sdei_list_lock);
543 	list_for_each_entry(event, &sdei_list, list) {
544 		if (event->type != SDEI_EVENT_TYPE_SHARED)
545 			continue;
546 
547 		err = _sdei_event_unregister(event);
548 		if (err)
549 			break;
550 	}
551 	spin_unlock(&sdei_list_lock);
552 	mutex_unlock(&sdei_events_lock);
553 
554 	return err;
555 }
556 
557 static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
558 				   void *arg, u64 flags, u64 affinity)
559 {
560 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
561 			      (unsigned long)entry_point, (unsigned long)arg,
562 			      flags, affinity, NULL);
563 }
564 
565 /* Called directly by the hotplug callbacks */
566 static void _local_event_register(void *data)
567 {
568 	int err;
569 	struct sdei_registered_event *reg;
570 	struct sdei_crosscall_args *arg = data;
571 
572 	WARN_ON(preemptible());
573 
574 	reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
575 	err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
576 				      reg, 0, 0);
577 
578 	sdei_cross_call_return(arg, err);
579 }
580 
581 static int _sdei_event_register(struct sdei_event *event)
582 {
583 	int err;
584 
585 	lockdep_assert_held(&sdei_events_lock);
586 
587 	spin_lock(&sdei_list_lock);
588 	event->reregister = true;
589 	spin_unlock(&sdei_list_lock);
590 
591 	if (event->type == SDEI_EVENT_TYPE_SHARED)
592 		return sdei_api_event_register(event->event_num,
593 					       sdei_entry_point,
594 					       event->registered,
595 					       SDEI_EVENT_REGISTER_RM_ANY, 0);
596 
597 
598 	err = sdei_do_cross_call(_local_event_register, event);
599 	if (err) {
600 		spin_lock(&sdei_list_lock);
601 		event->reregister = false;
602 		event->reenable = false;
603 		spin_unlock(&sdei_list_lock);
604 
605 		sdei_do_cross_call(_local_event_unregister, event);
606 	}
607 
608 	return err;
609 }
610 
611 int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
612 {
613 	int err;
614 	struct sdei_event *event;
615 
616 	WARN_ON(in_nmi());
617 
618 	mutex_lock(&sdei_events_lock);
619 	do {
620 		if (sdei_event_find(event_num)) {
621 			pr_warn("Event %u already registered\n", event_num);
622 			err = -EBUSY;
623 			break;
624 		}
625 
626 		event = sdei_event_create(event_num, cb, arg);
627 		if (IS_ERR(event)) {
628 			err = PTR_ERR(event);
629 			pr_warn("Failed to create event %u: %d\n", event_num,
630 				err);
631 			break;
632 		}
633 
634 		err = _sdei_event_register(event);
635 		if (err) {
636 			sdei_event_destroy(event);
637 			pr_warn("Failed to register event %u: %d\n", event_num,
638 				err);
639 		}
640 	} while (0);
641 	mutex_unlock(&sdei_events_lock);
642 
643 	return err;
644 }
645 EXPORT_SYMBOL(sdei_event_register);
646 
647 static int sdei_reregister_event(struct sdei_event *event)
648 {
649 	int err;
650 
651 	lockdep_assert_held(&sdei_events_lock);
652 
653 	err = _sdei_event_register(event);
654 	if (err) {
655 		pr_err("Failed to re-register event %u\n", event->event_num);
656 		sdei_event_destroy(event);
657 		return err;
658 	}
659 
660 	if (event->reenable) {
661 		if (event->type == SDEI_EVENT_TYPE_SHARED)
662 			err = sdei_api_event_enable(event->event_num);
663 		else
664 			err = sdei_do_cross_call(_local_event_enable, event);
665 	}
666 
667 	if (err)
668 		pr_err("Failed to re-enable event %u\n", event->event_num);
669 
670 	return err;
671 }
672 
673 static int sdei_reregister_shared(void)
674 {
675 	int err = 0;
676 	struct sdei_event *event;
677 
678 	mutex_lock(&sdei_events_lock);
679 	spin_lock(&sdei_list_lock);
680 	list_for_each_entry(event, &sdei_list, list) {
681 		if (event->type != SDEI_EVENT_TYPE_SHARED)
682 			continue;
683 
684 		if (event->reregister) {
685 			err = sdei_reregister_event(event);
686 			if (err)
687 				break;
688 		}
689 	}
690 	spin_unlock(&sdei_list_lock);
691 	mutex_unlock(&sdei_events_lock);
692 
693 	return err;
694 }
695 
696 static int sdei_cpuhp_down(unsigned int cpu)
697 {
698 	struct sdei_event *event;
699 	struct sdei_crosscall_args arg;
700 
701 	/* un-register private events */
702 	spin_lock(&sdei_list_lock);
703 	list_for_each_entry(event, &sdei_list, list) {
704 		if (event->type == SDEI_EVENT_TYPE_SHARED)
705 			continue;
706 
707 		CROSSCALL_INIT(arg, event);
708 		/* call the cross-call function locally... */
709 		_local_event_unregister(&arg);
710 		if (arg.first_error)
711 			pr_err("Failed to unregister event %u: %d\n",
712 			       event->event_num, arg.first_error);
713 	}
714 	spin_unlock(&sdei_list_lock);
715 
716 	return sdei_mask_local_cpu();
717 }
718 
719 static int sdei_cpuhp_up(unsigned int cpu)
720 {
721 	struct sdei_event *event;
722 	struct sdei_crosscall_args arg;
723 
724 	/* re-register/enable private events */
725 	spin_lock(&sdei_list_lock);
726 	list_for_each_entry(event, &sdei_list, list) {
727 		if (event->type == SDEI_EVENT_TYPE_SHARED)
728 			continue;
729 
730 		if (event->reregister) {
731 			CROSSCALL_INIT(arg, event);
732 			/* call the cross-call function locally... */
733 			_local_event_register(&arg);
734 			if (arg.first_error)
735 				pr_err("Failed to re-register event %u: %d\n",
736 				       event->event_num, arg.first_error);
737 		}
738 
739 		if (event->reenable) {
740 			CROSSCALL_INIT(arg, event);
741 			_local_event_enable(&arg);
742 			if (arg.first_error)
743 				pr_err("Failed to re-enable event %u: %d\n",
744 				       event->event_num, arg.first_error);
745 		}
746 	}
747 	spin_unlock(&sdei_list_lock);
748 
749 	return sdei_unmask_local_cpu();
750 }
751 
752 /* When entering idle, mask/unmask events for this cpu */
753 static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
754 			    void *data)
755 {
756 	int rv;
757 
758 	switch (action) {
759 	case CPU_PM_ENTER:
760 		rv = sdei_mask_local_cpu();
761 		break;
762 	case CPU_PM_EXIT:
763 	case CPU_PM_ENTER_FAILED:
764 		rv = sdei_unmask_local_cpu();
765 		break;
766 	default:
767 		return NOTIFY_DONE;
768 	}
769 
770 	if (rv)
771 		return notifier_from_errno(rv);
772 
773 	return NOTIFY_OK;
774 }
775 
776 static struct notifier_block sdei_pm_nb = {
777 	.notifier_call = sdei_pm_notifier,
778 };
779 
780 static int sdei_device_suspend(struct device *dev)
781 {
782 	on_each_cpu(_ipi_mask_cpu, NULL, true);
783 
784 	return 0;
785 }
786 
787 static int sdei_device_resume(struct device *dev)
788 {
789 	on_each_cpu(_ipi_unmask_cpu, NULL, true);
790 
791 	return 0;
792 }
793 
794 /*
795  * We need all events to be reregistered when we resume from hibernate.
796  *
797  * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
798  * events during freeze, then re-register and re-enable them during thaw
799  * and restore.
800  */
801 static int sdei_device_freeze(struct device *dev)
802 {
803 	int err;
804 
805 	/* unregister private events */
806 	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
807 
808 	err = sdei_unregister_shared();
809 	if (err)
810 		return err;
811 
812 	return 0;
813 }
814 
815 static int sdei_device_thaw(struct device *dev)
816 {
817 	int err;
818 
819 	/* re-register shared events */
820 	err = sdei_reregister_shared();
821 	if (err) {
822 		pr_warn("Failed to re-register shared events...\n");
823 		sdei_mark_interface_broken();
824 		return err;
825 	}
826 
827 	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
828 				&sdei_cpuhp_up, &sdei_cpuhp_down);
829 	if (err)
830 		pr_warn("Failed to re-register CPU hotplug notifier...\n");
831 
832 	return err;
833 }
834 
835 static int sdei_device_restore(struct device *dev)
836 {
837 	int err;
838 
839 	err = sdei_platform_reset();
840 	if (err)
841 		return err;
842 
843 	return sdei_device_thaw(dev);
844 }
845 
846 static const struct dev_pm_ops sdei_pm_ops = {
847 	.suspend = sdei_device_suspend,
848 	.resume = sdei_device_resume,
849 	.freeze = sdei_device_freeze,
850 	.thaw = sdei_device_thaw,
851 	.restore = sdei_device_restore,
852 };
853 
854 /*
855  * Mask all CPUs and unregister all events on panic, reboot or kexec.
856  */
857 static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
858 				void *data)
859 {
860 	/*
861 	 * We are going to reset the interface, after this there is no point
862 	 * doing work when we take CPUs offline.
863 	 */
864 	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
865 
866 	sdei_platform_reset();
867 
868 	return NOTIFY_OK;
869 }
870 
871 static struct notifier_block sdei_reboot_nb = {
872 	.notifier_call = sdei_reboot_notifier,
873 };
874 
875 static void sdei_smccc_smc(unsigned long function_id,
876 			   unsigned long arg0, unsigned long arg1,
877 			   unsigned long arg2, unsigned long arg3,
878 			   unsigned long arg4, struct arm_smccc_res *res)
879 {
880 	arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
881 }
882 
883 static void sdei_smccc_hvc(unsigned long function_id,
884 			   unsigned long arg0, unsigned long arg1,
885 			   unsigned long arg2, unsigned long arg3,
886 			   unsigned long arg4, struct arm_smccc_res *res)
887 {
888 	arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
889 }
890 
891 int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb,
892 		       sdei_event_callback *critical_cb)
893 {
894 	int err;
895 	u64 result;
896 	u32 event_num;
897 	sdei_event_callback *cb;
898 
899 	if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
900 		return -EOPNOTSUPP;
901 
902 	event_num = ghes->generic->notify.vector;
903 	if (event_num == 0) {
904 		/*
905 		 * Event 0 is reserved by the specification for
906 		 * SDEI_EVENT_SIGNAL.
907 		 */
908 		return -EINVAL;
909 	}
910 
911 	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
912 				      &result);
913 	if (err)
914 		return err;
915 
916 	if (result == SDEI_EVENT_PRIORITY_CRITICAL)
917 		cb = critical_cb;
918 	else
919 		cb = normal_cb;
920 
921 	err = sdei_event_register(event_num, cb, ghes);
922 	if (!err)
923 		err = sdei_event_enable(event_num);
924 
925 	return err;
926 }
927 
928 int sdei_unregister_ghes(struct ghes *ghes)
929 {
930 	int i;
931 	int err;
932 	u32 event_num = ghes->generic->notify.vector;
933 
934 	might_sleep();
935 
936 	if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
937 		return -EOPNOTSUPP;
938 
939 	/*
940 	 * The event may be running on another CPU. Disable it
941 	 * to stop new events, then try to unregister a few times.
942 	 */
943 	err = sdei_event_disable(event_num);
944 	if (err)
945 		return err;
946 
947 	for (i = 0; i < 3; i++) {
948 		err = sdei_event_unregister(event_num);
949 		if (err != -EINPROGRESS)
950 			break;
951 
952 		schedule();
953 	}
954 
955 	return err;
956 }
957 
958 static int sdei_get_conduit(struct platform_device *pdev)
959 {
960 	const char *method;
961 	struct device_node *np = pdev->dev.of_node;
962 
963 	sdei_firmware_call = NULL;
964 	if (np) {
965 		if (of_property_read_string(np, "method", &method)) {
966 			pr_warn("missing \"method\" property\n");
967 			return CONDUIT_INVALID;
968 		}
969 
970 		if (!strcmp("hvc", method)) {
971 			sdei_firmware_call = &sdei_smccc_hvc;
972 			return CONDUIT_HVC;
973 		} else if (!strcmp("smc", method)) {
974 			sdei_firmware_call = &sdei_smccc_smc;
975 			return CONDUIT_SMC;
976 		}
977 
978 		pr_warn("invalid \"method\" property: %s\n", method);
979 	} else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
980 		if (acpi_psci_use_hvc()) {
981 			sdei_firmware_call = &sdei_smccc_hvc;
982 			return CONDUIT_HVC;
983 		} else {
984 			sdei_firmware_call = &sdei_smccc_smc;
985 			return CONDUIT_SMC;
986 		}
987 	}
988 
989 	return CONDUIT_INVALID;
990 }
991 
992 static int sdei_probe(struct platform_device *pdev)
993 {
994 	int err;
995 	u64 ver = 0;
996 	int conduit;
997 
998 	conduit = sdei_get_conduit(pdev);
999 	if (!sdei_firmware_call)
1000 		return 0;
1001 
1002 	err = sdei_api_get_version(&ver);
1003 	if (err == -EOPNOTSUPP)
1004 		pr_err("advertised but not implemented in platform firmware\n");
1005 	if (err) {
1006 		pr_err("Failed to get SDEI version: %d\n", err);
1007 		sdei_mark_interface_broken();
1008 		return err;
1009 	}
1010 
1011 	pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
1012 		(int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
1013 		(int)SDEI_VERSION_VENDOR(ver));
1014 
1015 	if (SDEI_VERSION_MAJOR(ver) != 1) {
1016 		pr_warn("Conflicting SDEI version detected.\n");
1017 		sdei_mark_interface_broken();
1018 		return -EINVAL;
1019 	}
1020 
1021 	err = sdei_platform_reset();
1022 	if (err)
1023 		return err;
1024 
1025 	sdei_entry_point = sdei_arch_get_entry_point(conduit);
1026 	if (!sdei_entry_point) {
1027 		/* Not supported due to hardware or boot configuration */
1028 		sdei_mark_interface_broken();
1029 		return 0;
1030 	}
1031 
1032 	err = cpu_pm_register_notifier(&sdei_pm_nb);
1033 	if (err) {
1034 		pr_warn("Failed to register CPU PM notifier...\n");
1035 		goto error;
1036 	}
1037 
1038 	err = register_reboot_notifier(&sdei_reboot_nb);
1039 	if (err) {
1040 		pr_warn("Failed to register reboot notifier...\n");
1041 		goto remove_cpupm;
1042 	}
1043 
1044 	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
1045 				&sdei_cpuhp_up, &sdei_cpuhp_down);
1046 	if (err) {
1047 		pr_warn("Failed to register CPU hotplug notifier...\n");
1048 		goto remove_reboot;
1049 	}
1050 
1051 	return 0;
1052 
1053 remove_reboot:
1054 	unregister_reboot_notifier(&sdei_reboot_nb);
1055 
1056 remove_cpupm:
1057 	cpu_pm_unregister_notifier(&sdei_pm_nb);
1058 
1059 error:
1060 	sdei_mark_interface_broken();
1061 	return err;
1062 }
1063 
1064 static const struct of_device_id sdei_of_match[] = {
1065 	{ .compatible = "arm,sdei-1.0" },
1066 	{}
1067 };
1068 
1069 static struct platform_driver sdei_driver = {
1070 	.driver		= {
1071 		.name			= "sdei",
1072 		.pm			= &sdei_pm_ops,
1073 		.of_match_table		= sdei_of_match,
1074 	},
1075 	.probe		= sdei_probe,
1076 };
1077 
1078 static bool __init sdei_present_dt(void)
1079 {
1080 	struct device_node *np, *fw_np;
1081 
1082 	fw_np = of_find_node_by_name(NULL, "firmware");
1083 	if (!fw_np)
1084 		return false;
1085 
1086 	np = of_find_matching_node(fw_np, sdei_of_match);
1087 	if (!np)
1088 		return false;
1089 	of_node_put(np);
1090 
1091 	return true;
1092 }
1093 
1094 static bool __init sdei_present_acpi(void)
1095 {
1096 	acpi_status status;
1097 	struct platform_device *pdev;
1098 	struct acpi_table_header *sdei_table_header;
1099 
1100 	if (acpi_disabled)
1101 		return false;
1102 
1103 	status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1104 	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1105 		const char *msg = acpi_format_exception(status);
1106 
1107 		pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1108 	}
1109 	if (ACPI_FAILURE(status))
1110 		return false;
1111 
1112 	pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
1113 					       0);
1114 	if (IS_ERR(pdev))
1115 		return false;
1116 
1117 	return true;
1118 }
1119 
1120 static int __init sdei_init(void)
1121 {
1122 	if (sdei_present_dt() || sdei_present_acpi())
1123 		platform_driver_register(&sdei_driver);
1124 
1125 	return 0;
1126 }
1127 
1128 /*
1129  * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1130  * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1131  * by device_initcall(). We want to be called in the middle.
1132  */
1133 subsys_initcall_sync(sdei_init);
1134 
1135 int sdei_event_handler(struct pt_regs *regs,
1136 		       struct sdei_registered_event *arg)
1137 {
1138 	int err;
1139 	mm_segment_t orig_addr_limit;
1140 	u32 event_num = arg->event_num;
1141 
1142 	orig_addr_limit = get_fs();
1143 	set_fs(USER_DS);
1144 
1145 	err = arg->callback(event_num, regs, arg->callback_arg);
1146 	if (err)
1147 		pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1148 				   event_num, smp_processor_id(), err);
1149 
1150 	set_fs(orig_addr_limit);
1151 
1152 	return err;
1153 }
1154 NOKPROBE_SYMBOL(sdei_event_handler);
1155