1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * System Control and Management Interface (SCMI) Notification support
4  *
5  * Copyright (C) 2020 ARM Ltd.
6  */
7 /**
8  * DOC: Theory of operation
9  *
10  * SCMI Protocol specification allows the platform to signal events to
11  * interested agents via notification messages: this is an implementation
12  * of the dispatch and delivery of such notifications to the interested users
13  * inside the Linux kernel.
14  *
15  * An SCMI Notification core instance is initialized for each active platform
16  * instance identified by the means of the usual &struct scmi_handle.
17  *
18  * Each SCMI Protocol implementation, during its initialization, registers with
19  * this core its set of supported events using scmi_register_protocol_events():
20  * all the needed descriptors are stored in the &struct registered_protocols and
21  * &struct registered_events arrays.
22  *
23  * Kernel users interested in some specific event can register their callbacks
24  * providing the usual notifier_block descriptor, since this core implements
25  * events' delivery using the standard Kernel notification chains machinery.
26  *
27  * Given the number of possible events defined by SCMI and the extensibility
28  * of the SCMI Protocol itself, the underlying notification chains are created
29  * and destroyed dynamically on demand depending on the number of users
30  * effectively registered for an event, so that no support structures or chains
31  * are allocated until at least one user has registered a notifier_block for
32  * such event. Similarly, events' generation itself is enabled at the platform
33  * level only after at least one user has registered, and it is shutdown after
34  * the last user for that event has gone.
35  *
36  * All users provided callbacks and allocated notification-chains are stored in
37  * the @registered_events_handlers hashtable. Callbacks' registration requests
38  * for still to be registered events are instead kept in the dedicated common
39  * hashtable @pending_events_handlers.
40  *
41  * An event is identified univocally by the tuple (proto_id, evt_id, src_id)
42  * and is served by its own dedicated notification chain; information contained
43  * in such tuples is used, in a few different ways, to generate the needed
44  * hash-keys.
45  *
46  * Here proto_id and evt_id are simply the protocol_id and message_id numbers
47  * as described in the SCMI Protocol specification, while src_id represents an
48  * optional, protocol dependent, source identifier (like domain_id, perf_id
49  * or sensor_id and so forth).
50  *
51  * Upon reception of a notification message from the platform the SCMI RX ISR
52  * passes the received message payload and some ancillary information (including
53  * an arrival timestamp in nanoseconds) to the core via @scmi_notify() which
54  * pushes the event-data itself on a protocol-dedicated kfifo queue for further
55  * deferred processing as specified in @scmi_events_dispatcher().
56  *
57  * Each protocol has it own dedicated work_struct and worker which, once kicked
58  * by the ISR, takes care to empty its own dedicated queue, deliverying the
59  * queued items into the proper notification-chain: notifications processing can
60  * proceed concurrently on distinct workers only between events belonging to
61  * different protocols while delivery of events within the same protocol is
62  * still strictly sequentially ordered by time of arrival.
63  *
64  * Events' information is then extracted from the SCMI Notification messages and
65  * conveyed, converted into a custom per-event report struct, as the void *data
66  * param to the user callback provided by the registered notifier_block, so that
67  * from the user perspective his callback will look invoked like:
68  *
69  * int user_cb(struct notifier_block *nb, unsigned long event_id, void *report)
70  *
71  */
72 
73 #define dev_fmt(fmt) "SCMI Notifications - " fmt
74 #define pr_fmt(fmt) "SCMI Notifications - " fmt
75 
76 #include <linux/bitfield.h>
77 #include <linux/bug.h>
78 #include <linux/compiler.h>
79 #include <linux/device.h>
80 #include <linux/err.h>
81 #include <linux/hashtable.h>
82 #include <linux/kernel.h>
83 #include <linux/ktime.h>
84 #include <linux/kfifo.h>
85 #include <linux/list.h>
86 #include <linux/mutex.h>
87 #include <linux/notifier.h>
88 #include <linux/refcount.h>
89 #include <linux/scmi_protocol.h>
90 #include <linux/slab.h>
91 #include <linux/types.h>
92 #include <linux/workqueue.h>
93 
94 #include "notify.h"
95 
96 #define SCMI_MAX_PROTO		256
97 
98 #define PROTO_ID_MASK		GENMASK(31, 24)
99 #define EVT_ID_MASK		GENMASK(23, 16)
100 #define SRC_ID_MASK		GENMASK(15, 0)
101 
102 /*
103  * Builds an unsigned 32bit key from the given input tuple to be used
104  * as a key in hashtables.
105  */
106 #define MAKE_HASH_KEY(p, e, s)			\
107 	(FIELD_PREP(PROTO_ID_MASK, (p)) |	\
108 	   FIELD_PREP(EVT_ID_MASK, (e)) |	\
109 	   FIELD_PREP(SRC_ID_MASK, (s)))
110 
111 #define MAKE_ALL_SRCS_KEY(p, e)		MAKE_HASH_KEY((p), (e), SRC_ID_MASK)
112 
113 /*
114  * Assumes that the stored obj includes its own hash-key in a field named 'key':
115  * with this simplification this macro can be equally used for all the objects'
116  * types hashed by this implementation.
117  *
118  * @__ht: The hashtable name
119  * @__obj: A pointer to the object type to be retrieved from the hashtable;
120  *	   it will be used as a cursor while scanning the hastable and it will
121  *	   be possibly left as NULL when @__k is not found
122  * @__k: The key to search for
123  */
124 #define KEY_FIND(__ht, __obj, __k)				\
125 ({								\
126 	typeof(__k) k_ = __k;					\
127 	typeof(__obj) obj_;					\
128 								\
129 	hash_for_each_possible((__ht), obj_, hash, k_)		\
130 		if (obj_->key == k_)				\
131 			break;					\
132 	__obj = obj_;						\
133 })
134 
135 #define KEY_XTRACT_PROTO_ID(key)	FIELD_GET(PROTO_ID_MASK, (key))
136 #define KEY_XTRACT_EVT_ID(key)		FIELD_GET(EVT_ID_MASK, (key))
137 #define KEY_XTRACT_SRC_ID(key)		FIELD_GET(SRC_ID_MASK, (key))
138 
139 /*
140  * A set of macros used to access safely @registered_protocols and
141  * @registered_events arrays; these are fixed in size and each entry is possibly
142  * populated at protocols' registration time and then only read but NEVER
143  * modified or removed.
144  */
145 #define SCMI_GET_PROTO(__ni, __pid)					\
146 ({									\
147 	typeof(__ni) ni_ = __ni;					\
148 	struct scmi_registered_events_desc *__pd = NULL;		\
149 									\
150 	if (ni_)							\
151 		__pd = READ_ONCE(ni_->registered_protocols[(__pid)]);	\
152 	__pd;								\
153 })
154 
155 #define SCMI_GET_REVT_FROM_PD(__pd, __eid)				\
156 ({									\
157 	typeof(__pd) pd_ = __pd;					\
158 	typeof(__eid) eid_ = __eid;					\
159 	struct scmi_registered_event *__revt = NULL;			\
160 									\
161 	if (pd_ && eid_ < pd_->num_events)				\
162 		__revt = READ_ONCE(pd_->registered_events[eid_]);	\
163 	__revt;								\
164 })
165 
166 #define SCMI_GET_REVT(__ni, __pid, __eid)				\
167 ({									\
168 	struct scmi_registered_event *__revt;				\
169 	struct scmi_registered_events_desc *__pd;			\
170 									\
171 	__pd = SCMI_GET_PROTO((__ni), (__pid));				\
172 	__revt = SCMI_GET_REVT_FROM_PD(__pd, (__eid));			\
173 	__revt;								\
174 })
175 
176 /* A couple of utility macros to limit cruft when calling protocols' helpers */
177 #define REVT_NOTIFY_SET_STATUS(revt, eid, sid, state)		\
178 ({								\
179 	typeof(revt) r = revt;					\
180 	r->proto->ops->set_notify_enabled(r->proto->ni->handle,	\
181 					(eid), (sid), (state));	\
182 })
183 
184 #define REVT_NOTIFY_ENABLE(revt, eid, sid)			\
185 	REVT_NOTIFY_SET_STATUS((revt), (eid), (sid), true)
186 
187 #define REVT_NOTIFY_DISABLE(revt, eid, sid)			\
188 	REVT_NOTIFY_SET_STATUS((revt), (eid), (sid), false)
189 
190 #define REVT_FILL_REPORT(revt, ...)				\
191 ({								\
192 	typeof(revt) r = revt;					\
193 	r->proto->ops->fill_custom_report(r->proto->ni->handle,	\
194 					  __VA_ARGS__);		\
195 })
196 
197 #define SCMI_PENDING_HASH_SZ		4
198 #define SCMI_REGISTERED_HASH_SZ		6
199 
200 struct scmi_registered_events_desc;
201 
202 /**
203  * struct scmi_notify_instance  - Represents an instance of the notification
204  * core
205  * @gid: GroupID used for devres
206  * @handle: A reference to the platform instance
207  * @init_work: A work item to perform final initializations of pending handlers
208  * @notify_wq: A reference to the allocated Kernel cmwq
209  * @pending_mtx: A mutex to protect @pending_events_handlers
210  * @registered_protocols: A statically allocated array containing pointers to
211  *			  all the registered protocol-level specific information
212  *			  related to events' handling
213  * @pending_events_handlers: An hashtable containing all pending events'
214  *			     handlers descriptors
215  *
216  * Each platform instance, represented by a handle, has its own instance of
217  * the notification subsystem represented by this structure.
218  */
219 struct scmi_notify_instance {
220 	void			*gid;
221 	struct scmi_handle	*handle;
222 	struct work_struct	init_work;
223 	struct workqueue_struct	*notify_wq;
224 	/* lock to protect pending_events_handlers */
225 	struct mutex		pending_mtx;
226 	struct scmi_registered_events_desc	**registered_protocols;
227 	DECLARE_HASHTABLE(pending_events_handlers, SCMI_PENDING_HASH_SZ);
228 };
229 
230 /**
231  * struct events_queue  - Describes a queue and its associated worker
232  * @sz: Size in bytes of the related kfifo
233  * @kfifo: A dedicated Kernel kfifo descriptor
234  * @notify_work: A custom work item bound to this queue
235  * @wq: A reference to the associated workqueue
236  *
237  * Each protocol has its own dedicated events_queue descriptor.
238  */
239 struct events_queue {
240 	size_t			sz;
241 	struct kfifo		kfifo;
242 	struct work_struct	notify_work;
243 	struct workqueue_struct	*wq;
244 };
245 
246 /**
247  * struct scmi_event_header  - A utility header
248  * @timestamp: The timestamp, in nanoseconds (boottime), which was associated
249  *	       to this event as soon as it entered the SCMI RX ISR
250  * @payld_sz: Effective size of the embedded message payload which follows
251  * @evt_id: Event ID (corresponds to the Event MsgID for this Protocol)
252  * @payld: A reference to the embedded event payload
253  *
254  * This header is prepended to each received event message payload before
255  * queueing it on the related &struct events_queue.
256  */
257 struct scmi_event_header {
258 	ktime_t timestamp;
259 	size_t payld_sz;
260 	unsigned char evt_id;
261 	unsigned char payld[];
262 };
263 
264 struct scmi_registered_event;
265 
266 /**
267  * struct scmi_registered_events_desc  - Protocol Specific information
268  * @id: Protocol ID
269  * @ops: Protocol specific and event-related operations
270  * @equeue: The embedded per-protocol events_queue
271  * @ni: A reference to the initialized instance descriptor
272  * @eh: A reference to pre-allocated buffer to be used as a scratch area by the
273  *	deferred worker when fetching data from the kfifo
274  * @eh_sz: Size of the pre-allocated buffer @eh
275  * @in_flight: A reference to an in flight &struct scmi_registered_event
276  * @num_events: Number of events in @registered_events
277  * @registered_events: A dynamically allocated array holding all the registered
278  *		       events' descriptors, whose fixed-size is determined at
279  *		       compile time.
280  * @registered_mtx: A mutex to protect @registered_events_handlers
281  * @registered_events_handlers: An hashtable containing all events' handlers
282  *				descriptors registered for this protocol
283  *
284  * All protocols that register at least one event have their protocol-specific
285  * information stored here, together with the embedded allocated events_queue.
286  * These descriptors are stored in the @registered_protocols array at protocol
287  * registration time.
288  *
289  * Once these descriptors are successfully registered, they are NEVER again
290  * removed or modified since protocols do not unregister ever, so that, once
291  * we safely grab a NON-NULL reference from the array we can keep it and use it.
292  */
293 struct scmi_registered_events_desc {
294 	u8				id;
295 	const struct scmi_event_ops	*ops;
296 	struct events_queue		equeue;
297 	struct scmi_notify_instance	*ni;
298 	struct scmi_event_header	*eh;
299 	size_t				eh_sz;
300 	void				*in_flight;
301 	int				num_events;
302 	struct scmi_registered_event	**registered_events;
303 	/* mutex to protect registered_events_handlers */
304 	struct mutex			registered_mtx;
305 	DECLARE_HASHTABLE(registered_events_handlers, SCMI_REGISTERED_HASH_SZ);
306 };
307 
308 /**
309  * struct scmi_registered_event  - Event Specific Information
310  * @proto: A reference to the associated protocol descriptor
311  * @evt: A reference to the associated event descriptor (as provided at
312  *       registration time)
313  * @report: A pre-allocated buffer used by the deferred worker to fill a
314  *	    customized event report
315  * @num_sources: The number of possible sources for this event as stated at
316  *		 events' registration time
317  * @sources: A reference to a dynamically allocated array used to refcount the
318  *	     events' enable requests for all the existing sources
319  * @sources_mtx: A mutex to serialize the access to @sources
320  *
321  * All registered events are represented by one of these structures that are
322  * stored in the @registered_events array at protocol registration time.
323  *
324  * Once these descriptors are successfully registered, they are NEVER again
325  * removed or modified since protocols do not unregister ever, so that once we
326  * safely grab a NON-NULL reference from the table we can keep it and use it.
327  */
328 struct scmi_registered_event {
329 	struct scmi_registered_events_desc *proto;
330 	const struct scmi_event	*evt;
331 	void		*report;
332 	u32		num_sources;
333 	refcount_t	*sources;
334 	/* locking to serialize the access to sources */
335 	struct mutex	sources_mtx;
336 };
337 
338 /**
339  * struct scmi_event_handler  - Event handler information
340  * @key: The used hashkey
341  * @users: A reference count for number of active users for this handler
342  * @r_evt: A reference to the associated registered event; when this is NULL
343  *	   this handler is pending, which means that identifies a set of
344  *	   callbacks intended to be attached to an event which is still not
345  *	   known nor registered by any protocol at that point in time
346  * @chain: The notification chain dedicated to this specific event tuple
347  * @hash: The hlist_node used for collision handling
348  * @enabled: A boolean which records if event's generation has been already
349  *	     enabled for this handler as a whole
350  *
351  * This structure collects all the information needed to process a received
352  * event identified by the tuple (proto_id, evt_id, src_id).
353  * These descriptors are stored in a per-protocol @registered_events_handlers
354  * table using as a key a value derived from that tuple.
355  */
356 struct scmi_event_handler {
357 	u32				key;
358 	refcount_t			users;
359 	struct scmi_registered_event	*r_evt;
360 	struct blocking_notifier_head	chain;
361 	struct hlist_node		hash;
362 	bool				enabled;
363 };
364 
365 #define IS_HNDL_PENDING(hndl)	(!(hndl)->r_evt)
366 
367 static struct scmi_event_handler *
368 scmi_get_active_handler(struct scmi_notify_instance *ni, u32 evt_key);
369 static void scmi_put_active_handler(struct scmi_notify_instance *ni,
370 				    struct scmi_event_handler *hndl);
371 static void scmi_put_handler_unlocked(struct scmi_notify_instance *ni,
372 				      struct scmi_event_handler *hndl);
373 
374 /**
375  * scmi_lookup_and_call_event_chain()  - Lookup the proper chain and call it
376  * @ni: A reference to the notification instance to use
377  * @evt_key: The key to use to lookup the related notification chain
378  * @report: The customized event-specific report to pass down to the callbacks
379  *	    as their *data parameter.
380  */
381 static inline void
382 scmi_lookup_and_call_event_chain(struct scmi_notify_instance *ni,
383 				 u32 evt_key, void *report)
384 {
385 	int ret;
386 	struct scmi_event_handler *hndl;
387 
388 	/*
389 	 * Here ensure the event handler cannot vanish while using it.
390 	 * It is legitimate, though, for an handler not to be found at all here,
391 	 * e.g. when it has been unregistered by the user after some events had
392 	 * already been queued.
393 	 */
394 	hndl = scmi_get_active_handler(ni, evt_key);
395 	if (!hndl)
396 		return;
397 
398 	ret = blocking_notifier_call_chain(&hndl->chain,
399 					   KEY_XTRACT_EVT_ID(evt_key),
400 					   report);
401 	/* Notifiers are NOT supposed to cut the chain ... */
402 	WARN_ON_ONCE(ret & NOTIFY_STOP_MASK);
403 
404 	scmi_put_active_handler(ni, hndl);
405 }
406 
407 /**
408  * scmi_process_event_header()  - Dequeue and process an event header
409  * @eq: The queue to use
410  * @pd: The protocol descriptor to use
411  *
412  * Read an event header from the protocol queue into the dedicated scratch
413  * buffer and looks for a matching registered event; in case an anomalously
414  * sized read is detected just flush the queue.
415  *
416  * Return:
417  * * a reference to the matching registered event when found
418  * * ERR_PTR(-EINVAL) when NO registered event could be found
419  * * NULL when the queue is empty
420  */
421 static inline struct scmi_registered_event *
422 scmi_process_event_header(struct events_queue *eq,
423 			  struct scmi_registered_events_desc *pd)
424 {
425 	unsigned int outs;
426 	struct scmi_registered_event *r_evt;
427 
428 	outs = kfifo_out(&eq->kfifo, pd->eh,
429 			 sizeof(struct scmi_event_header));
430 	if (!outs)
431 		return NULL;
432 	if (outs != sizeof(struct scmi_event_header)) {
433 		dev_err(pd->ni->handle->dev, "corrupted EVT header. Flush.\n");
434 		kfifo_reset_out(&eq->kfifo);
435 		return NULL;
436 	}
437 
438 	r_evt = SCMI_GET_REVT_FROM_PD(pd, pd->eh->evt_id);
439 	if (!r_evt)
440 		r_evt = ERR_PTR(-EINVAL);
441 
442 	return r_evt;
443 }
444 
445 /**
446  * scmi_process_event_payload()  - Dequeue and process an event payload
447  * @eq: The queue to use
448  * @pd: The protocol descriptor to use
449  * @r_evt: The registered event descriptor to use
450  *
451  * Read an event payload from the protocol queue into the dedicated scratch
452  * buffer, fills a custom report and then look for matching event handlers and
453  * call them; skip any unknown event (as marked by scmi_process_event_header())
454  * and in case an anomalously sized read is detected just flush the queue.
455  *
456  * Return: False when the queue is empty
457  */
458 static inline bool
459 scmi_process_event_payload(struct events_queue *eq,
460 			   struct scmi_registered_events_desc *pd,
461 			   struct scmi_registered_event *r_evt)
462 {
463 	u32 src_id, key;
464 	unsigned int outs;
465 	void *report = NULL;
466 
467 	outs = kfifo_out(&eq->kfifo, pd->eh->payld, pd->eh->payld_sz);
468 	if (!outs)
469 		return false;
470 
471 	/* Any in-flight event has now been officially processed */
472 	pd->in_flight = NULL;
473 
474 	if (outs != pd->eh->payld_sz) {
475 		dev_err(pd->ni->handle->dev, "corrupted EVT Payload. Flush.\n");
476 		kfifo_reset_out(&eq->kfifo);
477 		return false;
478 	}
479 
480 	if (IS_ERR(r_evt)) {
481 		dev_warn(pd->ni->handle->dev,
482 			 "SKIP UNKNOWN EVT - proto:%X  evt:%d\n",
483 			 pd->id, pd->eh->evt_id);
484 		return true;
485 	}
486 
487 	report = REVT_FILL_REPORT(r_evt, pd->eh->evt_id, pd->eh->timestamp,
488 				  pd->eh->payld, pd->eh->payld_sz,
489 				  r_evt->report, &src_id);
490 	if (!report) {
491 		dev_err(pd->ni->handle->dev,
492 			"report not available - proto:%X  evt:%d\n",
493 			pd->id, pd->eh->evt_id);
494 		return true;
495 	}
496 
497 	/* At first search for a generic ALL src_ids handler... */
498 	key = MAKE_ALL_SRCS_KEY(pd->id, pd->eh->evt_id);
499 	scmi_lookup_and_call_event_chain(pd->ni, key, report);
500 
501 	/* ...then search for any specific src_id */
502 	key = MAKE_HASH_KEY(pd->id, pd->eh->evt_id, src_id);
503 	scmi_lookup_and_call_event_chain(pd->ni, key, report);
504 
505 	return true;
506 }
507 
508 /**
509  * scmi_events_dispatcher()  - Common worker logic for all work items.
510  * @work: The work item to use, which is associated to a dedicated events_queue
511  *
512  * Logic:
513  *  1. dequeue one pending RX notification (queued in SCMI RX ISR context)
514  *  2. generate a custom event report from the received event message
515  *  3. lookup for any registered ALL_SRC_IDs handler:
516  *    - > call the related notification chain passing in the report
517  *  4. lookup for any registered specific SRC_ID handler:
518  *    - > call the related notification chain passing in the report
519  *
520  * Note that:
521  * * a dedicated per-protocol kfifo queue is used: in this way an anomalous
522  *   flood of events cannot saturate other protocols' queues.
523  * * each per-protocol queue is associated to a distinct work_item, which
524  *   means, in turn, that:
525  *   + all protocols can process their dedicated queues concurrently
526  *     (since notify_wq:max_active != 1)
527  *   + anyway at most one worker instance is allowed to run on the same queue
528  *     concurrently: this ensures that we can have only one concurrent
529  *     reader/writer on the associated kfifo, so that we can use it lock-less
530  *
531  * Context: Process context.
532  */
533 static void scmi_events_dispatcher(struct work_struct *work)
534 {
535 	struct events_queue *eq;
536 	struct scmi_registered_events_desc *pd;
537 	struct scmi_registered_event *r_evt;
538 
539 	eq = container_of(work, struct events_queue, notify_work);
540 	pd = container_of(eq, struct scmi_registered_events_desc, equeue);
541 	/*
542 	 * In order to keep the queue lock-less and the number of memcopies
543 	 * to the bare minimum needed, the dispatcher accounts for the
544 	 * possibility of per-protocol in-flight events: i.e. an event whose
545 	 * reception could end up being split across two subsequent runs of this
546 	 * worker, first the header, then the payload.
547 	 */
548 	do {
549 		if (!pd->in_flight) {
550 			r_evt = scmi_process_event_header(eq, pd);
551 			if (!r_evt)
552 				break;
553 			pd->in_flight = r_evt;
554 		} else {
555 			r_evt = pd->in_flight;
556 		}
557 	} while (scmi_process_event_payload(eq, pd, r_evt));
558 }
559 
560 /**
561  * scmi_notify()  - Queues a notification for further deferred processing
562  * @handle: The handle identifying the platform instance from which the
563  *	    dispatched event is generated
564  * @proto_id: Protocol ID
565  * @evt_id: Event ID (msgID)
566  * @buf: Event Message Payload (without the header)
567  * @len: Event Message Payload size
568  * @ts: RX Timestamp in nanoseconds (boottime)
569  *
570  * Context: Called in interrupt context to queue a received event for
571  * deferred processing.
572  *
573  * Return: 0 on Success
574  */
575 int scmi_notify(const struct scmi_handle *handle, u8 proto_id, u8 evt_id,
576 		const void *buf, size_t len, ktime_t ts)
577 {
578 	struct scmi_registered_event *r_evt;
579 	struct scmi_event_header eh;
580 	struct scmi_notify_instance *ni;
581 
582 	/* Ensure notify_priv is updated */
583 	smp_rmb();
584 	if (!handle->notify_priv)
585 		return 0;
586 	ni = handle->notify_priv;
587 
588 	r_evt = SCMI_GET_REVT(ni, proto_id, evt_id);
589 	if (!r_evt)
590 		return -EINVAL;
591 
592 	if (len > r_evt->evt->max_payld_sz) {
593 		dev_err(handle->dev, "discard badly sized message\n");
594 		return -EINVAL;
595 	}
596 	if (kfifo_avail(&r_evt->proto->equeue.kfifo) < sizeof(eh) + len) {
597 		dev_warn(handle->dev,
598 			 "queue full, dropping proto_id:%d  evt_id:%d  ts:%lld\n",
599 			 proto_id, evt_id, ktime_to_ns(ts));
600 		return -ENOMEM;
601 	}
602 
603 	eh.timestamp = ts;
604 	eh.evt_id = evt_id;
605 	eh.payld_sz = len;
606 	/*
607 	 * Header and payload are enqueued with two distinct kfifo_in() (so non
608 	 * atomic), but this situation is handled properly on the consumer side
609 	 * with in-flight events tracking.
610 	 */
611 	kfifo_in(&r_evt->proto->equeue.kfifo, &eh, sizeof(eh));
612 	kfifo_in(&r_evt->proto->equeue.kfifo, buf, len);
613 	/*
614 	 * Don't care about return value here since we just want to ensure that
615 	 * a work is queued all the times whenever some items have been pushed
616 	 * on the kfifo:
617 	 * - if work was already queued it will simply fail to queue a new one
618 	 *   since it is not needed
619 	 * - if work was not queued already it will be now, even in case work
620 	 *   was in fact already running: this behavior avoids any possible race
621 	 *   when this function pushes new items onto the kfifos after the
622 	 *   related executing worker had already determined the kfifo to be
623 	 *   empty and it was terminating.
624 	 */
625 	queue_work(r_evt->proto->equeue.wq,
626 		   &r_evt->proto->equeue.notify_work);
627 
628 	return 0;
629 }
630 
631 /**
632  * scmi_kfifo_free()  - Devres action helper to free the kfifo
633  * @kfifo: The kfifo to free
634  */
635 static void scmi_kfifo_free(void *kfifo)
636 {
637 	kfifo_free((struct kfifo *)kfifo);
638 }
639 
640 /**
641  * scmi_initialize_events_queue()  - Allocate/Initialize a kfifo buffer
642  * @ni: A reference to the notification instance to use
643  * @equeue: The events_queue to initialize
644  * @sz: Size of the kfifo buffer to allocate
645  *
646  * Allocate a buffer for the kfifo and initialize it.
647  *
648  * Return: 0 on Success
649  */
650 static int scmi_initialize_events_queue(struct scmi_notify_instance *ni,
651 					struct events_queue *equeue, size_t sz)
652 {
653 	int ret;
654 
655 	if (kfifo_alloc(&equeue->kfifo, sz, GFP_KERNEL))
656 		return -ENOMEM;
657 	/* Size could have been roundup to power-of-two */
658 	equeue->sz = kfifo_size(&equeue->kfifo);
659 
660 	ret = devm_add_action_or_reset(ni->handle->dev, scmi_kfifo_free,
661 				       &equeue->kfifo);
662 	if (ret)
663 		return ret;
664 
665 	INIT_WORK(&equeue->notify_work, scmi_events_dispatcher);
666 	equeue->wq = ni->notify_wq;
667 
668 	return ret;
669 }
670 
671 /**
672  * scmi_allocate_registered_events_desc()  - Allocate a registered events'
673  * descriptor
674  * @ni: A reference to the &struct scmi_notify_instance notification instance
675  *	to use
676  * @proto_id: Protocol ID
677  * @queue_sz: Size of the associated queue to allocate
678  * @eh_sz: Size of the event header scratch area to pre-allocate
679  * @num_events: Number of events to support (size of @registered_events)
680  * @ops: Pointer to a struct holding references to protocol specific helpers
681  *	 needed during events handling
682  *
683  * It is supposed to be called only once for each protocol at protocol
684  * initialization time, so it warns if the requested protocol is found already
685  * registered.
686  *
687  * Return: The allocated and registered descriptor on Success
688  */
689 static struct scmi_registered_events_desc *
690 scmi_allocate_registered_events_desc(struct scmi_notify_instance *ni,
691 				     u8 proto_id, size_t queue_sz, size_t eh_sz,
692 				     int num_events,
693 				     const struct scmi_event_ops *ops)
694 {
695 	int ret;
696 	struct scmi_registered_events_desc *pd;
697 
698 	/* Ensure protocols are up to date */
699 	smp_rmb();
700 	if (WARN_ON(ni->registered_protocols[proto_id]))
701 		return ERR_PTR(-EINVAL);
702 
703 	pd = devm_kzalloc(ni->handle->dev, sizeof(*pd), GFP_KERNEL);
704 	if (!pd)
705 		return ERR_PTR(-ENOMEM);
706 	pd->id = proto_id;
707 	pd->ops = ops;
708 	pd->ni = ni;
709 
710 	ret = scmi_initialize_events_queue(ni, &pd->equeue, queue_sz);
711 	if (ret)
712 		return ERR_PTR(ret);
713 
714 	pd->eh = devm_kzalloc(ni->handle->dev, eh_sz, GFP_KERNEL);
715 	if (!pd->eh)
716 		return ERR_PTR(-ENOMEM);
717 	pd->eh_sz = eh_sz;
718 
719 	pd->registered_events = devm_kcalloc(ni->handle->dev, num_events,
720 					     sizeof(char *), GFP_KERNEL);
721 	if (!pd->registered_events)
722 		return ERR_PTR(-ENOMEM);
723 	pd->num_events = num_events;
724 
725 	/* Initialize per protocol handlers table */
726 	mutex_init(&pd->registered_mtx);
727 	hash_init(pd->registered_events_handlers);
728 
729 	return pd;
730 }
731 
732 /**
733  * scmi_register_protocol_events()  - Register Protocol Events with the core
734  * @handle: The handle identifying the platform instance against which the
735  *	    the protocol's events are registered
736  * @proto_id: Protocol ID
737  * @queue_sz: Size in bytes of the associated queue to be allocated
738  * @ops: Protocol specific event-related operations
739  * @evt: Event descriptor array
740  * @num_events: Number of events in @evt array
741  * @num_sources: Number of possible sources for this protocol on this
742  *		 platform.
743  *
744  * Used by SCMI Protocols initialization code to register with the notification
745  * core the list of supported events and their descriptors: takes care to
746  * pre-allocate and store all needed descriptors, scratch buffers and event
747  * queues.
748  *
749  * Return: 0 on Success
750  */
751 int scmi_register_protocol_events(const struct scmi_handle *handle,
752 				  u8 proto_id, size_t queue_sz,
753 				  const struct scmi_event_ops *ops,
754 				  const struct scmi_event *evt, int num_events,
755 				  int num_sources)
756 {
757 	int i;
758 	size_t payld_sz = 0;
759 	struct scmi_registered_events_desc *pd;
760 	struct scmi_notify_instance *ni;
761 
762 	if (!ops || !evt)
763 		return -EINVAL;
764 
765 	/* Ensure notify_priv is updated */
766 	smp_rmb();
767 	if (!handle->notify_priv)
768 		return -ENOMEM;
769 	ni = handle->notify_priv;
770 
771 	/* Attach to the notification main devres group */
772 	if (!devres_open_group(ni->handle->dev, ni->gid, GFP_KERNEL))
773 		return -ENOMEM;
774 
775 	for (i = 0; i < num_events; i++)
776 		payld_sz = max_t(size_t, payld_sz, evt[i].max_payld_sz);
777 	payld_sz += sizeof(struct scmi_event_header);
778 
779 	pd = scmi_allocate_registered_events_desc(ni, proto_id, queue_sz,
780 						  payld_sz, num_events, ops);
781 	if (IS_ERR(pd))
782 		goto err;
783 
784 	for (i = 0; i < num_events; i++, evt++) {
785 		struct scmi_registered_event *r_evt;
786 
787 		r_evt = devm_kzalloc(ni->handle->dev, sizeof(*r_evt),
788 				     GFP_KERNEL);
789 		if (!r_evt)
790 			goto err;
791 		r_evt->proto = pd;
792 		r_evt->evt = evt;
793 
794 		r_evt->sources = devm_kcalloc(ni->handle->dev, num_sources,
795 					      sizeof(refcount_t), GFP_KERNEL);
796 		if (!r_evt->sources)
797 			goto err;
798 		r_evt->num_sources = num_sources;
799 		mutex_init(&r_evt->sources_mtx);
800 
801 		r_evt->report = devm_kzalloc(ni->handle->dev,
802 					     evt->max_report_sz, GFP_KERNEL);
803 		if (!r_evt->report)
804 			goto err;
805 
806 		pd->registered_events[i] = r_evt;
807 		/* Ensure events are updated */
808 		smp_wmb();
809 		dev_dbg(handle->dev, "registered event - %lX\n",
810 			MAKE_ALL_SRCS_KEY(r_evt->proto->id, r_evt->evt->id));
811 	}
812 
813 	/* Register protocol and events...it will never be removed */
814 	ni->registered_protocols[proto_id] = pd;
815 	/* Ensure protocols are updated */
816 	smp_wmb();
817 
818 	devres_close_group(ni->handle->dev, ni->gid);
819 
820 	/*
821 	 * Finalize any pending events' handler which could have been waiting
822 	 * for this protocol's events registration.
823 	 */
824 	schedule_work(&ni->init_work);
825 
826 	return 0;
827 
828 err:
829 	dev_warn(handle->dev, "Proto:%X - Registration Failed !\n", proto_id);
830 	/* A failing protocol registration does not trigger full failure */
831 	devres_close_group(ni->handle->dev, ni->gid);
832 
833 	return -ENOMEM;
834 }
835 
836 /**
837  * scmi_allocate_event_handler()  - Allocate Event handler
838  * @ni: A reference to the notification instance to use
839  * @evt_key: 32bit key uniquely bind to the event identified by the tuple
840  *	     (proto_id, evt_id, src_id)
841  *
842  * Allocate an event handler and related notification chain associated with
843  * the provided event handler key.
844  * Note that, at this point, a related registered_event is still to be
845  * associated to this handler descriptor (hndl->r_evt == NULL), so the handler
846  * is initialized as pending.
847  *
848  * Context: Assumes to be called with @pending_mtx already acquired.
849  * Return: the freshly allocated structure on Success
850  */
851 static struct scmi_event_handler *
852 scmi_allocate_event_handler(struct scmi_notify_instance *ni, u32 evt_key)
853 {
854 	struct scmi_event_handler *hndl;
855 
856 	hndl = kzalloc(sizeof(*hndl), GFP_KERNEL);
857 	if (!hndl)
858 		return NULL;
859 	hndl->key = evt_key;
860 	BLOCKING_INIT_NOTIFIER_HEAD(&hndl->chain);
861 	refcount_set(&hndl->users, 1);
862 	/* New handlers are created pending */
863 	hash_add(ni->pending_events_handlers, &hndl->hash, hndl->key);
864 
865 	return hndl;
866 }
867 
868 /**
869  * scmi_free_event_handler()  - Free the provided Event handler
870  * @hndl: The event handler structure to free
871  *
872  * Context: Assumes to be called with proper locking acquired depending
873  *	    on the situation.
874  */
875 static void scmi_free_event_handler(struct scmi_event_handler *hndl)
876 {
877 	hash_del(&hndl->hash);
878 	kfree(hndl);
879 }
880 
881 /**
882  * scmi_bind_event_handler()  - Helper to attempt binding an handler to an event
883  * @ni: A reference to the notification instance to use
884  * @hndl: The event handler to bind
885  *
886  * If an associated registered event is found, move the handler from the pending
887  * into the registered table.
888  *
889  * Context: Assumes to be called with @pending_mtx already acquired.
890  *
891  * Return: 0 on Success
892  */
893 static inline int scmi_bind_event_handler(struct scmi_notify_instance *ni,
894 					  struct scmi_event_handler *hndl)
895 {
896 	struct scmi_registered_event *r_evt;
897 
898 	r_evt = SCMI_GET_REVT(ni, KEY_XTRACT_PROTO_ID(hndl->key),
899 			      KEY_XTRACT_EVT_ID(hndl->key));
900 	if (!r_evt)
901 		return -EINVAL;
902 
903 	/* Remove from pending and insert into registered */
904 	hash_del(&hndl->hash);
905 	hndl->r_evt = r_evt;
906 	mutex_lock(&r_evt->proto->registered_mtx);
907 	hash_add(r_evt->proto->registered_events_handlers,
908 		 &hndl->hash, hndl->key);
909 	mutex_unlock(&r_evt->proto->registered_mtx);
910 
911 	return 0;
912 }
913 
914 /**
915  * scmi_valid_pending_handler()  - Helper to check pending status of handlers
916  * @ni: A reference to the notification instance to use
917  * @hndl: The event handler to check
918  *
919  * An handler is considered pending when its r_evt == NULL, because the related
920  * event was still unknown at handler's registration time; anyway, since all
921  * protocols register their supported events once for all at protocols'
922  * initialization time, a pending handler cannot be considered valid anymore if
923  * the underlying event (which it is waiting for), belongs to an already
924  * initialized and registered protocol.
925  *
926  * Return: 0 on Success
927  */
928 static inline int scmi_valid_pending_handler(struct scmi_notify_instance *ni,
929 					     struct scmi_event_handler *hndl)
930 {
931 	struct scmi_registered_events_desc *pd;
932 
933 	if (!IS_HNDL_PENDING(hndl))
934 		return -EINVAL;
935 
936 	pd = SCMI_GET_PROTO(ni, KEY_XTRACT_PROTO_ID(hndl->key));
937 	if (pd)
938 		return -EINVAL;
939 
940 	return 0;
941 }
942 
943 /**
944  * scmi_register_event_handler()  - Register whenever possible an Event handler
945  * @ni: A reference to the notification instance to use
946  * @hndl: The event handler to register
947  *
948  * At first try to bind an event handler to its associated event, then check if
949  * it was at least a valid pending handler: if it was not bound nor valid return
950  * false.
951  *
952  * Valid pending incomplete bindings will be periodically retried by a dedicated
953  * worker which is kicked each time a new protocol completes its own
954  * registration phase.
955  *
956  * Context: Assumes to be called with @pending_mtx acquired.
957  *
958  * Return: 0 on Success
959  */
960 static int scmi_register_event_handler(struct scmi_notify_instance *ni,
961 				       struct scmi_event_handler *hndl)
962 {
963 	int ret;
964 
965 	ret = scmi_bind_event_handler(ni, hndl);
966 	if (!ret) {
967 		dev_dbg(ni->handle->dev, "registered NEW handler - key:%X\n",
968 			hndl->key);
969 	} else {
970 		ret = scmi_valid_pending_handler(ni, hndl);
971 		if (!ret)
972 			dev_dbg(ni->handle->dev,
973 				"registered PENDING handler - key:%X\n",
974 				hndl->key);
975 	}
976 
977 	return ret;
978 }
979 
980 /**
981  * __scmi_event_handler_get_ops()  - Utility to get or create an event handler
982  * @ni: A reference to the notification instance to use
983  * @evt_key: The event key to use
984  * @create: A boolean flag to specify if a handler must be created when
985  *	    not already existent
986  *
987  * Search for the desired handler matching the key in both the per-protocol
988  * registered table and the common pending table:
989  * * if found adjust users refcount
990  * * if not found and @create is true, create and register the new handler:
991  *   handler could end up being registered as pending if no matching event
992  *   could be found.
993  *
994  * An handler is guaranteed to reside in one and only one of the tables at
995  * any one time; to ensure this the whole search and create is performed
996  * holding the @pending_mtx lock, with @registered_mtx additionally acquired
997  * if needed.
998  *
999  * Note that when a nested acquisition of these mutexes is needed the locking
1000  * order is always (same as in @init_work):
1001  * 1. pending_mtx
1002  * 2. registered_mtx
1003  *
1004  * Events generation is NOT enabled right after creation within this routine
1005  * since at creation time we usually want to have all setup and ready before
1006  * events really start flowing.
1007  *
1008  * Return: A properly refcounted handler on Success, NULL on Failure
1009  */
1010 static inline struct scmi_event_handler *
1011 __scmi_event_handler_get_ops(struct scmi_notify_instance *ni,
1012 			     u32 evt_key, bool create)
1013 {
1014 	struct scmi_registered_event *r_evt;
1015 	struct scmi_event_handler *hndl = NULL;
1016 
1017 	r_evt = SCMI_GET_REVT(ni, KEY_XTRACT_PROTO_ID(evt_key),
1018 			      KEY_XTRACT_EVT_ID(evt_key));
1019 
1020 	mutex_lock(&ni->pending_mtx);
1021 	/* Search registered events at first ... if possible at all */
1022 	if (r_evt) {
1023 		mutex_lock(&r_evt->proto->registered_mtx);
1024 		hndl = KEY_FIND(r_evt->proto->registered_events_handlers,
1025 				hndl, evt_key);
1026 		if (hndl)
1027 			refcount_inc(&hndl->users);
1028 		mutex_unlock(&r_evt->proto->registered_mtx);
1029 	}
1030 
1031 	/* ...then amongst pending. */
1032 	if (!hndl) {
1033 		hndl = KEY_FIND(ni->pending_events_handlers, hndl, evt_key);
1034 		if (hndl)
1035 			refcount_inc(&hndl->users);
1036 	}
1037 
1038 	/* Create if still not found and required */
1039 	if (!hndl && create) {
1040 		hndl = scmi_allocate_event_handler(ni, evt_key);
1041 		if (hndl && scmi_register_event_handler(ni, hndl)) {
1042 			dev_dbg(ni->handle->dev,
1043 				"purging UNKNOWN handler - key:%X\n",
1044 				hndl->key);
1045 			/* this hndl can be only a pending one */
1046 			scmi_put_handler_unlocked(ni, hndl);
1047 			hndl = NULL;
1048 		}
1049 	}
1050 	mutex_unlock(&ni->pending_mtx);
1051 
1052 	return hndl;
1053 }
1054 
1055 static struct scmi_event_handler *
1056 scmi_get_handler(struct scmi_notify_instance *ni, u32 evt_key)
1057 {
1058 	return __scmi_event_handler_get_ops(ni, evt_key, false);
1059 }
1060 
1061 static struct scmi_event_handler *
1062 scmi_get_or_create_handler(struct scmi_notify_instance *ni, u32 evt_key)
1063 {
1064 	return __scmi_event_handler_get_ops(ni, evt_key, true);
1065 }
1066 
1067 /**
1068  * scmi_get_active_handler()  - Helper to get active handlers only
1069  * @ni: A reference to the notification instance to use
1070  * @evt_key: The event key to use
1071  *
1072  * Search for the desired handler matching the key only in the per-protocol
1073  * table of registered handlers: this is called only from the dispatching path
1074  * so want to be as quick as possible and do not care about pending.
1075  *
1076  * Return: A properly refcounted active handler
1077  */
1078 static struct scmi_event_handler *
1079 scmi_get_active_handler(struct scmi_notify_instance *ni, u32 evt_key)
1080 {
1081 	struct scmi_registered_event *r_evt;
1082 	struct scmi_event_handler *hndl = NULL;
1083 
1084 	r_evt = SCMI_GET_REVT(ni, KEY_XTRACT_PROTO_ID(evt_key),
1085 			      KEY_XTRACT_EVT_ID(evt_key));
1086 	if (r_evt) {
1087 		mutex_lock(&r_evt->proto->registered_mtx);
1088 		hndl = KEY_FIND(r_evt->proto->registered_events_handlers,
1089 				hndl, evt_key);
1090 		if (hndl)
1091 			refcount_inc(&hndl->users);
1092 		mutex_unlock(&r_evt->proto->registered_mtx);
1093 	}
1094 
1095 	return hndl;
1096 }
1097 
1098 /**
1099  * __scmi_enable_evt()  - Enable/disable events generation
1100  * @r_evt: The registered event to act upon
1101  * @src_id: The src_id to act upon
1102  * @enable: The action to perform: true->Enable, false->Disable
1103  *
1104  * Takes care of proper refcounting while performing enable/disable: handles
1105  * the special case of ALL sources requests by itself.
1106  * Returns successfully if at least one of the required src_id has been
1107  * successfully enabled/disabled.
1108  *
1109  * Return: 0 on Success
1110  */
1111 static inline int __scmi_enable_evt(struct scmi_registered_event *r_evt,
1112 				    u32 src_id, bool enable)
1113 {
1114 	int retvals = 0;
1115 	u32 num_sources;
1116 	refcount_t *sid;
1117 
1118 	if (src_id == SRC_ID_MASK) {
1119 		src_id = 0;
1120 		num_sources = r_evt->num_sources;
1121 	} else if (src_id < r_evt->num_sources) {
1122 		num_sources = 1;
1123 	} else {
1124 		return -EINVAL;
1125 	}
1126 
1127 	mutex_lock(&r_evt->sources_mtx);
1128 	if (enable) {
1129 		for (; num_sources; src_id++, num_sources--) {
1130 			int ret = 0;
1131 
1132 			sid = &r_evt->sources[src_id];
1133 			if (refcount_read(sid) == 0) {
1134 				ret = REVT_NOTIFY_ENABLE(r_evt, r_evt->evt->id,
1135 							 src_id);
1136 				if (!ret)
1137 					refcount_set(sid, 1);
1138 			} else {
1139 				refcount_inc(sid);
1140 			}
1141 			retvals += !ret;
1142 		}
1143 	} else {
1144 		for (; num_sources; src_id++, num_sources--) {
1145 			sid = &r_evt->sources[src_id];
1146 			if (refcount_dec_and_test(sid))
1147 				REVT_NOTIFY_DISABLE(r_evt,
1148 						    r_evt->evt->id, src_id);
1149 		}
1150 		retvals = 1;
1151 	}
1152 	mutex_unlock(&r_evt->sources_mtx);
1153 
1154 	return retvals ? 0 : -EINVAL;
1155 }
1156 
1157 static int scmi_enable_events(struct scmi_event_handler *hndl)
1158 {
1159 	int ret = 0;
1160 
1161 	if (!hndl->enabled) {
1162 		ret = __scmi_enable_evt(hndl->r_evt,
1163 					KEY_XTRACT_SRC_ID(hndl->key), true);
1164 		if (!ret)
1165 			hndl->enabled = true;
1166 	}
1167 
1168 	return ret;
1169 }
1170 
1171 static int scmi_disable_events(struct scmi_event_handler *hndl)
1172 {
1173 	int ret = 0;
1174 
1175 	if (hndl->enabled) {
1176 		ret = __scmi_enable_evt(hndl->r_evt,
1177 					KEY_XTRACT_SRC_ID(hndl->key), false);
1178 		if (!ret)
1179 			hndl->enabled = false;
1180 	}
1181 
1182 	return ret;
1183 }
1184 
1185 /**
1186  * scmi_put_handler_unlocked()  - Put an event handler
1187  * @ni: A reference to the notification instance to use
1188  * @hndl: The event handler to act upon
1189  *
1190  * After having got exclusive access to the registered handlers hashtable,
1191  * update the refcount and if @hndl is no more in use by anyone:
1192  * * ask for events' generation disabling
1193  * * unregister and free the handler itself
1194  *
1195  * Context: Assumes all the proper locking has been managed by the caller.
1196  */
1197 static void scmi_put_handler_unlocked(struct scmi_notify_instance *ni,
1198 				      struct scmi_event_handler *hndl)
1199 {
1200 	if (refcount_dec_and_test(&hndl->users)) {
1201 		if (!IS_HNDL_PENDING(hndl))
1202 			scmi_disable_events(hndl);
1203 		scmi_free_event_handler(hndl);
1204 	}
1205 }
1206 
1207 static void scmi_put_handler(struct scmi_notify_instance *ni,
1208 			     struct scmi_event_handler *hndl)
1209 {
1210 	struct scmi_registered_event *r_evt = hndl->r_evt;
1211 
1212 	mutex_lock(&ni->pending_mtx);
1213 	if (r_evt)
1214 		mutex_lock(&r_evt->proto->registered_mtx);
1215 
1216 	scmi_put_handler_unlocked(ni, hndl);
1217 
1218 	if (r_evt)
1219 		mutex_unlock(&r_evt->proto->registered_mtx);
1220 	mutex_unlock(&ni->pending_mtx);
1221 }
1222 
1223 static void scmi_put_active_handler(struct scmi_notify_instance *ni,
1224 				    struct scmi_event_handler *hndl)
1225 {
1226 	struct scmi_registered_event *r_evt = hndl->r_evt;
1227 
1228 	mutex_lock(&r_evt->proto->registered_mtx);
1229 	scmi_put_handler_unlocked(ni, hndl);
1230 	mutex_unlock(&r_evt->proto->registered_mtx);
1231 }
1232 
1233 /**
1234  * scmi_event_handler_enable_events()  - Enable events associated to an handler
1235  * @hndl: The Event handler to act upon
1236  *
1237  * Return: 0 on Success
1238  */
1239 static int scmi_event_handler_enable_events(struct scmi_event_handler *hndl)
1240 {
1241 	if (scmi_enable_events(hndl)) {
1242 		pr_err("Failed to ENABLE events for key:%X !\n", hndl->key);
1243 		return -EINVAL;
1244 	}
1245 
1246 	return 0;
1247 }
1248 
1249 /**
1250  * scmi_register_notifier()  - Register a notifier_block for an event
1251  * @handle: The handle identifying the platform instance against which the
1252  *	    callback is registered
1253  * @proto_id: Protocol ID
1254  * @evt_id: Event ID
1255  * @src_id: Source ID, when NULL register for events coming form ALL possible
1256  *	    sources
1257  * @nb: A standard notifier block to register for the specified event
1258  *
1259  * Generic helper to register a notifier_block against a protocol event.
1260  *
1261  * A notifier_block @nb will be registered for each distinct event identified
1262  * by the tuple (proto_id, evt_id, src_id) on a dedicated notification chain
1263  * so that:
1264  *
1265  *	(proto_X, evt_Y, src_Z) --> chain_X_Y_Z
1266  *
1267  * @src_id meaning is protocol specific and identifies the origin of the event
1268  * (like domain_id, sensor_id and so forth).
1269  *
1270  * @src_id can be NULL to signify that the caller is interested in receiving
1271  * notifications from ALL the available sources for that protocol OR simply that
1272  * the protocol does not support distinct sources.
1273  *
1274  * As soon as one user for the specified tuple appears, an handler is created,
1275  * and that specific event's generation is enabled at the platform level, unless
1276  * an associated registered event is found missing, meaning that the needed
1277  * protocol is still to be initialized and the handler has just been registered
1278  * as still pending.
1279  *
1280  * Return: 0 on Success
1281  */
1282 static int scmi_register_notifier(const struct scmi_handle *handle,
1283 				  u8 proto_id, u8 evt_id, u32 *src_id,
1284 				  struct notifier_block *nb)
1285 {
1286 	int ret = 0;
1287 	u32 evt_key;
1288 	struct scmi_event_handler *hndl;
1289 	struct scmi_notify_instance *ni;
1290 
1291 	/* Ensure notify_priv is updated */
1292 	smp_rmb();
1293 	if (!handle->notify_priv)
1294 		return -ENODEV;
1295 	ni = handle->notify_priv;
1296 
1297 	evt_key = MAKE_HASH_KEY(proto_id, evt_id,
1298 				src_id ? *src_id : SRC_ID_MASK);
1299 	hndl = scmi_get_or_create_handler(ni, evt_key);
1300 	if (!hndl)
1301 		return -EINVAL;
1302 
1303 	blocking_notifier_chain_register(&hndl->chain, nb);
1304 
1305 	/* Enable events for not pending handlers */
1306 	if (!IS_HNDL_PENDING(hndl)) {
1307 		ret = scmi_event_handler_enable_events(hndl);
1308 		if (ret)
1309 			scmi_put_handler(ni, hndl);
1310 	}
1311 
1312 	return ret;
1313 }
1314 
1315 /**
1316  * scmi_unregister_notifier()  - Unregister a notifier_block for an event
1317  * @handle: The handle identifying the platform instance against which the
1318  *	    callback is unregistered
1319  * @proto_id: Protocol ID
1320  * @evt_id: Event ID
1321  * @src_id: Source ID
1322  * @nb: The notifier_block to unregister
1323  *
1324  * Takes care to unregister the provided @nb from the notification chain
1325  * associated to the specified event and, if there are no more users for the
1326  * event handler, frees also the associated event handler structures.
1327  * (this could possibly cause disabling of event's generation at platform level)
1328  *
1329  * Return: 0 on Success
1330  */
1331 static int scmi_unregister_notifier(const struct scmi_handle *handle,
1332 				    u8 proto_id, u8 evt_id, u32 *src_id,
1333 				    struct notifier_block *nb)
1334 {
1335 	u32 evt_key;
1336 	struct scmi_event_handler *hndl;
1337 	struct scmi_notify_instance *ni;
1338 
1339 	/* Ensure notify_priv is updated */
1340 	smp_rmb();
1341 	if (!handle->notify_priv)
1342 		return -ENODEV;
1343 	ni = handle->notify_priv;
1344 
1345 	evt_key = MAKE_HASH_KEY(proto_id, evt_id,
1346 				src_id ? *src_id : SRC_ID_MASK);
1347 	hndl = scmi_get_handler(ni, evt_key);
1348 	if (!hndl)
1349 		return -EINVAL;
1350 
1351 	/*
1352 	 * Note that this chain unregistration call is safe on its own
1353 	 * being internally protected by an rwsem.
1354 	 */
1355 	blocking_notifier_chain_unregister(&hndl->chain, nb);
1356 	scmi_put_handler(ni, hndl);
1357 
1358 	/*
1359 	 * This balances the initial get issued in @scmi_register_notifier.
1360 	 * If this notifier_block happened to be the last known user callback
1361 	 * for this event, the handler is here freed and the event's generation
1362 	 * stopped.
1363 	 *
1364 	 * Note that, an ongoing concurrent lookup on the delivery workqueue
1365 	 * path could still hold the refcount to 1 even after this routine
1366 	 * completes: in such a case it will be the final put on the delivery
1367 	 * path which will finally free this unused handler.
1368 	 */
1369 	scmi_put_handler(ni, hndl);
1370 
1371 	return 0;
1372 }
1373 
1374 /**
1375  * scmi_protocols_late_init()  - Worker for late initialization
1376  * @work: The work item to use associated to the proper SCMI instance
1377  *
1378  * This kicks in whenever a new protocol has completed its own registration via
1379  * scmi_register_protocol_events(): it is in charge of scanning the table of
1380  * pending handlers (registered by users while the related protocol was still
1381  * not initialized) and finalizing their initialization whenever possible;
1382  * invalid pending handlers are purged at this point in time.
1383  */
1384 static void scmi_protocols_late_init(struct work_struct *work)
1385 {
1386 	int bkt;
1387 	struct scmi_event_handler *hndl;
1388 	struct scmi_notify_instance *ni;
1389 	struct hlist_node *tmp;
1390 
1391 	ni = container_of(work, struct scmi_notify_instance, init_work);
1392 
1393 	/* Ensure protocols and events are up to date */
1394 	smp_rmb();
1395 
1396 	mutex_lock(&ni->pending_mtx);
1397 	hash_for_each_safe(ni->pending_events_handlers, bkt, tmp, hndl, hash) {
1398 		int ret;
1399 
1400 		ret = scmi_bind_event_handler(ni, hndl);
1401 		if (!ret) {
1402 			dev_dbg(ni->handle->dev,
1403 				"finalized PENDING handler - key:%X\n",
1404 				hndl->key);
1405 			ret = scmi_event_handler_enable_events(hndl);
1406 			if (ret) {
1407 				dev_dbg(ni->handle->dev,
1408 					"purging INVALID handler - key:%X\n",
1409 					hndl->key);
1410 				scmi_put_active_handler(ni, hndl);
1411 			}
1412 		} else {
1413 			ret = scmi_valid_pending_handler(ni, hndl);
1414 			if (ret) {
1415 				dev_dbg(ni->handle->dev,
1416 					"purging PENDING handler - key:%X\n",
1417 					hndl->key);
1418 				/* this hndl can be only a pending one */
1419 				scmi_put_handler_unlocked(ni, hndl);
1420 			}
1421 		}
1422 	}
1423 	mutex_unlock(&ni->pending_mtx);
1424 }
1425 
1426 /*
1427  * notify_ops are attached to the handle so that can be accessed
1428  * directly from an scmi_driver to register its own notifiers.
1429  */
1430 static const struct scmi_notify_ops notify_ops = {
1431 	.register_event_notifier = scmi_register_notifier,
1432 	.unregister_event_notifier = scmi_unregister_notifier,
1433 };
1434 
1435 /**
1436  * scmi_notification_init()  - Initializes Notification Core Support
1437  * @handle: The handle identifying the platform instance to initialize
1438  *
1439  * This function lays out all the basic resources needed by the notification
1440  * core instance identified by the provided handle: once done, all of the
1441  * SCMI Protocols can register their events with the core during their own
1442  * initializations.
1443  *
1444  * Note that failing to initialize the core notifications support does not
1445  * cause the whole SCMI Protocols stack to fail its initialization.
1446  *
1447  * SCMI Notification Initialization happens in 2 steps:
1448  * * initialization: basic common allocations (this function)
1449  * * registration: protocols asynchronously come into life and registers their
1450  *		   own supported list of events with the core; this causes
1451  *		   further per-protocol allocations
1452  *
1453  * Any user's callback registration attempt, referring a still not registered
1454  * event, will be registered as pending and finalized later (if possible)
1455  * by scmi_protocols_late_init() work.
1456  * This allows for lazy initialization of SCMI Protocols due to late (or
1457  * missing) SCMI drivers' modules loading.
1458  *
1459  * Return: 0 on Success
1460  */
1461 int scmi_notification_init(struct scmi_handle *handle)
1462 {
1463 	void *gid;
1464 	struct scmi_notify_instance *ni;
1465 
1466 	gid = devres_open_group(handle->dev, NULL, GFP_KERNEL);
1467 	if (!gid)
1468 		return -ENOMEM;
1469 
1470 	ni = devm_kzalloc(handle->dev, sizeof(*ni), GFP_KERNEL);
1471 	if (!ni)
1472 		goto err;
1473 
1474 	ni->gid = gid;
1475 	ni->handle = handle;
1476 
1477 	ni->notify_wq = alloc_workqueue(dev_name(handle->dev),
1478 					WQ_UNBOUND | WQ_FREEZABLE | WQ_SYSFS,
1479 					0);
1480 	if (!ni->notify_wq)
1481 		goto err;
1482 
1483 	ni->registered_protocols = devm_kcalloc(handle->dev, SCMI_MAX_PROTO,
1484 						sizeof(char *), GFP_KERNEL);
1485 	if (!ni->registered_protocols)
1486 		goto err;
1487 
1488 	mutex_init(&ni->pending_mtx);
1489 	hash_init(ni->pending_events_handlers);
1490 
1491 	INIT_WORK(&ni->init_work, scmi_protocols_late_init);
1492 
1493 	handle->notify_ops = &notify_ops;
1494 	handle->notify_priv = ni;
1495 	/* Ensure handle is up to date */
1496 	smp_wmb();
1497 
1498 	dev_info(handle->dev, "Core Enabled.\n");
1499 
1500 	devres_close_group(handle->dev, ni->gid);
1501 
1502 	return 0;
1503 
1504 err:
1505 	dev_warn(handle->dev, "Initialization Failed.\n");
1506 	devres_release_group(handle->dev, NULL);
1507 	return -ENOMEM;
1508 }
1509 
1510 /**
1511  * scmi_notification_exit()  - Shutdown and clean Notification core
1512  * @handle: The handle identifying the platform instance to shutdown
1513  */
1514 void scmi_notification_exit(struct scmi_handle *handle)
1515 {
1516 	struct scmi_notify_instance *ni;
1517 
1518 	/* Ensure notify_priv is updated */
1519 	smp_rmb();
1520 	if (!handle->notify_priv)
1521 		return;
1522 	ni = handle->notify_priv;
1523 
1524 	handle->notify_priv = NULL;
1525 	/* Ensure handle is up to date */
1526 	smp_wmb();
1527 
1528 	/* Destroy while letting pending work complete */
1529 	destroy_workqueue(ni->notify_wq);
1530 
1531 	devres_release_group(ni->handle->dev, ni->gid);
1532 }
1533