1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * System Control and Management Interface (SCMI) Raw mode support
4  *
5  * Copyright (C) 2022 ARM Ltd.
6  */
7 /**
8  * DOC: Theory of operation
9  *
10  * When enabled the SCMI Raw mode support exposes a userspace API which allows
11  * to send and receive SCMI commands, replies and notifications from a user
12  * application through injection and snooping of bare SCMI messages in binary
13  * little-endian format.
14  *
15  * Such injected SCMI transactions will then be routed through the SCMI core
16  * stack towards the SCMI backend server using whatever SCMI transport is
17  * currently configured on the system under test.
18  *
19  * It is meant to help in running any sort of SCMI backend server testing, no
20  * matter where the server is placed, as long as it is normally reachable via
21  * the transport configured on the system.
22  *
23  * It is activated by a Kernel configuration option since it is NOT meant to
24  * be used in production but only during development and in CI deployments.
25  *
26  * In order to avoid possible interferences between the SCMI Raw transactions
27  * originated from a test-suite and the normal operations of the SCMI drivers,
28  * when Raw mode is enabled, by default, all the regular SCMI drivers are
29  * inhibited, unless CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX is enabled: in this
30  * latter case the regular SCMI stack drivers will be loaded as usual and it is
31  * up to the user of this interface to take care of manually inhibiting the
32  * regular SCMI drivers in order to avoid interferences during the test runs.
33  *
34  * The exposed API is as follows.
35  *
36  * All SCMI Raw entries are rooted under a common top /raw debugfs top directory
37  * which in turn is rooted under the corresponding underlying  SCMI instance.
38  *
39  * /sys/kernel/debug/scmi/
40  * `-- 0
41  *     |-- atomic_threshold_us
42  *     |-- instance_name
43  *     |-- raw
44  *     |   |-- channels
45  *     |   |   |-- 0x10
46  *     |   |   |   |-- message
47  *     |   |   |   `-- message_async
48  *     |   |   `-- 0x13
49  *     |   |       |-- message
50  *     |   |       `-- message_async
51  *     |   |-- errors
52  *     |   |-- message
53  *     |   |-- message_async
54  *     |   |-- notification
55  *     |   `-- reset
56  *     `-- transport
57  *         |-- is_atomic
58  *         |-- max_msg_size
59  *         |-- max_rx_timeout_ms
60  *         |-- rx_max_msg
61  *         |-- tx_max_msg
62  *         `-- type
63  *
64  * where:
65  *
66  *  - errors: used to read back timed-out and unexpected replies
67  *  - message*: used to send sync/async commands and read back immediate and
68  *		delayed reponses (if any)
69  *  - notification: used to read any notification being emitted by the system
70  *		    (if previously enabled by the user app)
71  *  - reset: used to flush the queues of messages (of any kind) still pending
72  *	     to be read; this is useful at test-suite start/stop to get
73  *	     rid of any unread messages from the previous run.
74  *
75  * with the per-channel entries rooted at /channels being present only on a
76  * system where multiple transport channels have been configured.
77  *
78  * Such per-channel entries can be used to explicitly choose a specific channel
79  * for SCMI bare message injection, in contrast with the general entries above
80  * where, instead, the selection of the proper channel to use is automatically
81  * performed based the protocol embedded in the injected message and on how the
82  * transport is configured on the system.
83  *
84  * Note that other common general entries are available under transport/ to let
85  * the user applications properly make up their expectations in terms of
86  * timeouts and message characteristics.
87  *
88  * Each write to the message* entries causes one command request to be built
89  * and sent while the replies or delayed response are read back from those same
90  * entries one message at time (receiving an EOF at each message boundary).
91  *
92  * The user application running the test is in charge of handling timeouts
93  * on replies and properly choosing SCMI sequence numbers for the outgoing
94  * requests (using the same sequence number is supported but discouraged).
95  *
96  * Injection of multiple in-flight requests is supported as long as the user
97  * application uses properly distinct sequence numbers for concurrent requests
98  * and takes care to properly manage all the related issues about concurrency
99  * and command/reply pairing. Keep in mind that, anyway, the real level of
100  * parallelism attainable in such scenario is dependent on the characteristics
101  * of the underlying transport being used.
102  *
103  * Since the SCMI core regular stack is partially used to deliver and collect
104  * the messages, late replies arrived after timeouts and any other sort of
105  * unexpected message can be identified by the SCMI core as usual and they will
106  * be reported as messages under "errors" for later analysis.
107  */
108 
109 #include <linux/bitmap.h>
110 #include <linux/debugfs.h>
111 #include <linux/delay.h>
112 #include <linux/device.h>
113 #include <linux/export.h>
114 #include <linux/io.h>
115 #include <linux/kernel.h>
116 #include <linux/fs.h>
117 #include <linux/list.h>
118 #include <linux/module.h>
119 #include <linux/poll.h>
120 #include <linux/of.h>
121 #include <linux/slab.h>
122 #include <linux/xarray.h>
123 
124 #include "common.h"
125 
126 #include "raw_mode.h"
127 
128 #include <trace/events/scmi.h>
129 
130 #define SCMI_XFER_RAW_MAX_RETRIES	10
131 
132 /**
133  * struct scmi_raw_queue  - Generic Raw queue descriptor
134  *
135  * @free_bufs: A freelists listhead used to keep unused raw buffers
136  * @free_bufs_lock: Spinlock used to protect access to @free_bufs
137  * @msg_q: A listhead to a queue of snooped messages waiting to be read out
138  * @msg_q_lock: Spinlock used to protect access to @msg_q
139  * @wq: A waitqueue used to wait and poll on related @msg_q
140  */
141 struct scmi_raw_queue {
142 	struct list_head free_bufs;
143 	/* Protect free_bufs[] lists */
144 	spinlock_t free_bufs_lock;
145 	struct list_head msg_q;
146 	/* Protect msg_q[] lists */
147 	spinlock_t msg_q_lock;
148 	wait_queue_head_t wq;
149 };
150 
151 /**
152  * struct scmi_raw_mode_info  - Structure holding SCMI Raw instance data
153  *
154  * @id: Sequential Raw instance ID.
155  * @handle: Pointer to SCMI entity handle to use
156  * @desc: Pointer to the transport descriptor to use
157  * @tx_max_msg: Maximum number of concurrent TX in-flight messages
158  * @q: An array of Raw queue descriptors
159  * @chans_q: An XArray mapping optional additional per-channel queues
160  * @free_waiters: Head of freelist for unused waiters
161  * @free_mtx: A mutex to protect the waiters freelist
162  * @active_waiters: Head of list for currently active and used waiters
163  * @active_mtx: A mutex to protect the active waiters list
164  * @waiters_work: A work descriptor to be used with the workqueue machinery
165  * @wait_wq: A workqueue reference to the created workqueue
166  * @dentry: Top debugfs root dentry for SCMI Raw
167  * @gid: A group ID used for devres accounting
168  *
169  * Note that this descriptor is passed back to the core after SCMI Raw is
170  * initialized as an opaque handle to use by subsequent SCMI Raw call hooks.
171  *
172  */
173 struct scmi_raw_mode_info {
174 	unsigned int id;
175 	const struct scmi_handle *handle;
176 	const struct scmi_desc *desc;
177 	int tx_max_msg;
178 	struct scmi_raw_queue *q[SCMI_RAW_MAX_QUEUE];
179 	struct xarray chans_q;
180 	struct list_head free_waiters;
181 	/* Protect free_waiters list */
182 	struct mutex free_mtx;
183 	struct list_head active_waiters;
184 	/* Protect active_waiters list */
185 	struct mutex active_mtx;
186 	struct work_struct waiters_work;
187 	struct workqueue_struct	*wait_wq;
188 	struct dentry *dentry;
189 	void *gid;
190 };
191 
192 /**
193  * struct scmi_xfer_raw_waiter  - Structure to describe an xfer to be waited for
194  *
195  * @start_jiffies: The timestamp in jiffies of when this structure was queued.
196  * @cinfo: A reference to the channel to use for this transaction
197  * @xfer: A reference to the xfer to be waited for
198  * @async_response: A completion to be, optionally, used for async waits: it
199  *		    will be setup by @scmi_do_xfer_raw_start, if needed, to be
200  *		    pointed at by xfer->async_done.
201  * @node: A list node.
202  */
203 struct scmi_xfer_raw_waiter {
204 	unsigned long start_jiffies;
205 	struct scmi_chan_info *cinfo;
206 	struct scmi_xfer *xfer;
207 	struct completion async_response;
208 	struct list_head node;
209 };
210 
211 /**
212  * struct scmi_raw_buffer  - Structure to hold a full SCMI message
213  *
214  * @max_len: The maximum allowed message size (header included) that can be
215  *	     stored into @msg
216  * @msg: A message buffer used to collect a full message grabbed from an xfer.
217  * @node: A list node.
218  */
219 struct scmi_raw_buffer {
220 	size_t max_len;
221 	struct scmi_msg msg;
222 	struct list_head node;
223 };
224 
225 /**
226  * struct scmi_dbg_raw_data  - Structure holding data needed by the debugfs
227  * layer
228  *
229  * @chan_id: The preferred channel to use: if zero the channel is automatically
230  *	     selected based on protocol.
231  * @raw: A reference to the Raw instance.
232  * @tx: A message buffer used to collect TX message on write.
233  * @tx_size: The effective size of the TX message.
234  * @tx_req_size: The final expected size of the complete TX message.
235  * @rx: A message buffer to collect RX message on read.
236  * @rx_size: The effective size of the RX message.
237  */
238 struct scmi_dbg_raw_data {
239 	u8 chan_id;
240 	struct scmi_raw_mode_info *raw;
241 	struct scmi_msg tx;
242 	size_t tx_size;
243 	size_t tx_req_size;
244 	struct scmi_msg rx;
245 	size_t rx_size;
246 };
247 
248 static struct scmi_raw_queue *
249 scmi_raw_queue_select(struct scmi_raw_mode_info *raw, unsigned int idx,
250 		      unsigned int chan_id)
251 {
252 	if (!chan_id)
253 		return raw->q[idx];
254 
255 	return xa_load(&raw->chans_q, chan_id);
256 }
257 
258 static struct scmi_raw_buffer *scmi_raw_buffer_get(struct scmi_raw_queue *q)
259 {
260 	unsigned long flags;
261 	struct scmi_raw_buffer *rb = NULL;
262 	struct list_head *head = &q->free_bufs;
263 
264 	spin_lock_irqsave(&q->free_bufs_lock, flags);
265 	if (!list_empty(head)) {
266 		rb = list_first_entry(head, struct scmi_raw_buffer, node);
267 		list_del_init(&rb->node);
268 	}
269 	spin_unlock_irqrestore(&q->free_bufs_lock, flags);
270 
271 	return rb;
272 }
273 
274 static void scmi_raw_buffer_put(struct scmi_raw_queue *q,
275 				struct scmi_raw_buffer *rb)
276 {
277 	unsigned long flags;
278 
279 	/* Reset to full buffer length */
280 	rb->msg.len = rb->max_len;
281 
282 	spin_lock_irqsave(&q->free_bufs_lock, flags);
283 	list_add_tail(&rb->node, &q->free_bufs);
284 	spin_unlock_irqrestore(&q->free_bufs_lock, flags);
285 }
286 
287 static void scmi_raw_buffer_enqueue(struct scmi_raw_queue *q,
288 				    struct scmi_raw_buffer *rb)
289 {
290 	unsigned long flags;
291 
292 	spin_lock_irqsave(&q->msg_q_lock, flags);
293 	list_add_tail(&rb->node, &q->msg_q);
294 	spin_unlock_irqrestore(&q->msg_q_lock, flags);
295 
296 	wake_up_interruptible(&q->wq);
297 }
298 
299 static struct scmi_raw_buffer*
300 scmi_raw_buffer_dequeue_unlocked(struct scmi_raw_queue *q)
301 {
302 	struct scmi_raw_buffer *rb = NULL;
303 
304 	if (!list_empty(&q->msg_q)) {
305 		rb = list_first_entry(&q->msg_q, struct scmi_raw_buffer, node);
306 		list_del_init(&rb->node);
307 	}
308 
309 	return rb;
310 }
311 
312 static struct scmi_raw_buffer *scmi_raw_buffer_dequeue(struct scmi_raw_queue *q)
313 {
314 	unsigned long flags;
315 	struct scmi_raw_buffer *rb;
316 
317 	spin_lock_irqsave(&q->msg_q_lock, flags);
318 	rb = scmi_raw_buffer_dequeue_unlocked(q);
319 	spin_unlock_irqrestore(&q->msg_q_lock, flags);
320 
321 	return rb;
322 }
323 
324 static void scmi_raw_buffer_queue_flush(struct scmi_raw_queue *q)
325 {
326 	struct scmi_raw_buffer *rb;
327 
328 	do {
329 		rb = scmi_raw_buffer_dequeue(q);
330 		if (rb)
331 			scmi_raw_buffer_put(q, rb);
332 	} while (rb);
333 }
334 
335 static struct scmi_xfer_raw_waiter *
336 scmi_xfer_raw_waiter_get(struct scmi_raw_mode_info *raw, struct scmi_xfer *xfer,
337 			 struct scmi_chan_info *cinfo, bool async)
338 {
339 	struct scmi_xfer_raw_waiter *rw = NULL;
340 
341 	mutex_lock(&raw->free_mtx);
342 	if (!list_empty(&raw->free_waiters)) {
343 		rw = list_first_entry(&raw->free_waiters,
344 				      struct scmi_xfer_raw_waiter, node);
345 		list_del_init(&rw->node);
346 
347 		if (async) {
348 			reinit_completion(&rw->async_response);
349 			xfer->async_done = &rw->async_response;
350 		}
351 
352 		rw->cinfo = cinfo;
353 		rw->xfer = xfer;
354 	}
355 	mutex_unlock(&raw->free_mtx);
356 
357 	return rw;
358 }
359 
360 static void scmi_xfer_raw_waiter_put(struct scmi_raw_mode_info *raw,
361 				     struct scmi_xfer_raw_waiter *rw)
362 {
363 	if (rw->xfer) {
364 		rw->xfer->async_done = NULL;
365 		rw->xfer = NULL;
366 	}
367 
368 	mutex_lock(&raw->free_mtx);
369 	list_add_tail(&rw->node, &raw->free_waiters);
370 	mutex_unlock(&raw->free_mtx);
371 }
372 
373 static void scmi_xfer_raw_waiter_enqueue(struct scmi_raw_mode_info *raw,
374 					 struct scmi_xfer_raw_waiter *rw)
375 {
376 	/* A timestamp for the deferred worker to know how much this has aged */
377 	rw->start_jiffies = jiffies;
378 
379 	trace_scmi_xfer_response_wait(rw->xfer->transfer_id, rw->xfer->hdr.id,
380 				      rw->xfer->hdr.protocol_id,
381 				      rw->xfer->hdr.seq,
382 				      raw->desc->max_rx_timeout_ms,
383 				      rw->xfer->hdr.poll_completion);
384 
385 	mutex_lock(&raw->active_mtx);
386 	list_add_tail(&rw->node, &raw->active_waiters);
387 	mutex_unlock(&raw->active_mtx);
388 
389 	/* kick waiter work */
390 	queue_work(raw->wait_wq, &raw->waiters_work);
391 }
392 
393 static struct scmi_xfer_raw_waiter *
394 scmi_xfer_raw_waiter_dequeue(struct scmi_raw_mode_info *raw)
395 {
396 	struct scmi_xfer_raw_waiter *rw = NULL;
397 
398 	mutex_lock(&raw->active_mtx);
399 	if (!list_empty(&raw->active_waiters)) {
400 		rw = list_first_entry(&raw->active_waiters,
401 				      struct scmi_xfer_raw_waiter, node);
402 		list_del_init(&rw->node);
403 	}
404 	mutex_unlock(&raw->active_mtx);
405 
406 	return rw;
407 }
408 
409 /**
410  * scmi_xfer_raw_worker  - Work function to wait for Raw xfers completions
411  *
412  * @work: A reference to the work.
413  *
414  * In SCMI Raw mode, once a user-provided injected SCMI message is sent, we
415  * cannot wait to receive its response (if any) in the context of the injection
416  * routines so as not to leave the userspace write syscall, which delivered the
417  * SCMI message to send, pending till eventually a reply is received.
418  * Userspace should and will poll/wait instead on the read syscalls which will
419  * be in charge of reading a received reply (if any).
420  *
421  * Even though reply messages are collected and reported into the SCMI Raw layer
422  * on the RX path, nonetheless we have to properly wait for their completion as
423  * usual (and async_completion too if needed) in order to properly release the
424  * xfer structure at the end: to do this out of the context of the write/send
425  * these waiting jobs are delegated to this deferred worker.
426  *
427  * Any sent xfer, to be waited for, is timestamped and queued for later
428  * consumption by this worker: queue aging is accounted for while choosing a
429  * timeout for the completion, BUT we do not really care here if we end up
430  * accidentally waiting for a bit too long.
431  */
432 static void scmi_xfer_raw_worker(struct work_struct *work)
433 {
434 	struct scmi_raw_mode_info *raw;
435 	struct device *dev;
436 	unsigned long max_tmo;
437 
438 	raw = container_of(work, struct scmi_raw_mode_info, waiters_work);
439 	dev = raw->handle->dev;
440 	max_tmo = msecs_to_jiffies(raw->desc->max_rx_timeout_ms);
441 
442 	do {
443 		int ret = 0;
444 		unsigned int timeout_ms;
445 		unsigned long aging;
446 		struct scmi_xfer *xfer;
447 		struct scmi_xfer_raw_waiter *rw;
448 		struct scmi_chan_info *cinfo;
449 
450 		rw = scmi_xfer_raw_waiter_dequeue(raw);
451 		if (!rw)
452 			return;
453 
454 		cinfo = rw->cinfo;
455 		xfer = rw->xfer;
456 		/*
457 		 * Waiters are queued by wait-deadline at the end, so some of
458 		 * them could have been already expired when processed, BUT we
459 		 * have to check the completion status anyway just in case a
460 		 * virtually expired (aged) transaction was indeed completed
461 		 * fine and we'll have to wait for the asynchronous part (if
462 		 * any): for this reason a 1 ms timeout is used for already
463 		 * expired/aged xfers.
464 		 */
465 		aging = jiffies - rw->start_jiffies;
466 		timeout_ms = max_tmo > aging ?
467 			jiffies_to_msecs(max_tmo - aging) : 1;
468 
469 		ret = scmi_xfer_raw_wait_for_message_response(cinfo, xfer,
470 							      timeout_ms);
471 		if (!ret && xfer->hdr.status)
472 			ret = scmi_to_linux_errno(xfer->hdr.status);
473 
474 		if (raw->desc->ops->mark_txdone)
475 			raw->desc->ops->mark_txdone(rw->cinfo, ret, xfer);
476 
477 		trace_scmi_xfer_end(xfer->transfer_id, xfer->hdr.id,
478 				    xfer->hdr.protocol_id, xfer->hdr.seq, ret);
479 
480 		/* Wait also for an async delayed response if needed */
481 		if (!ret && xfer->async_done) {
482 			unsigned long tmo = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);
483 
484 			if (!wait_for_completion_timeout(xfer->async_done, tmo))
485 				dev_err(dev,
486 					"timed out in RAW delayed resp - HDR:%08X\n",
487 					pack_scmi_header(&xfer->hdr));
488 		}
489 
490 		/* Release waiter and xfer */
491 		scmi_xfer_raw_put(raw->handle, xfer);
492 		scmi_xfer_raw_waiter_put(raw, rw);
493 	} while (1);
494 }
495 
496 static void scmi_xfer_raw_reset(struct scmi_raw_mode_info *raw)
497 {
498 	int i;
499 
500 	dev_info(raw->handle->dev, "Resetting SCMI Raw stack.\n");
501 
502 	for (i = 0; i < SCMI_RAW_MAX_QUEUE; i++)
503 		scmi_raw_buffer_queue_flush(raw->q[i]);
504 }
505 
506 /**
507  * scmi_xfer_raw_get_init  - An helper to build a valid xfer from the provided
508  * bare SCMI message.
509  *
510  * @raw: A reference to the Raw instance.
511  * @buf: A buffer containing the whole SCMI message to send (including the
512  *	 header) in little-endian binary formmat.
513  * @len: Length of the message in @buf.
514  * @p: A pointer to return the initialized Raw xfer.
515  *
516  * After an xfer is picked from the TX pool and filled in with the message
517  * content, the xfer is registered as pending with the core in the usual way
518  * using the original sequence number provided by the user with the message.
519  *
520  * Note that, in case the testing user application is NOT using distinct
521  * sequence-numbers between successive SCMI messages such registration could
522  * fail temporarily if the previous message, using the same sequence number,
523  * had still not released; in such a case we just wait and retry.
524  *
525  * Return: 0 on Success
526  */
527 static int scmi_xfer_raw_get_init(struct scmi_raw_mode_info *raw, void *buf,
528 				  size_t len, struct scmi_xfer **p)
529 {
530 	u32 msg_hdr;
531 	size_t tx_size;
532 	struct scmi_xfer *xfer;
533 	int ret, retry = SCMI_XFER_RAW_MAX_RETRIES;
534 	struct device *dev = raw->handle->dev;
535 
536 	if (!buf || len < sizeof(u32))
537 		return -EINVAL;
538 
539 	tx_size = len - sizeof(u32);
540 	/* Ensure we have sane transfer sizes */
541 	if (tx_size > raw->desc->max_msg_size)
542 		return -ERANGE;
543 
544 	xfer = scmi_xfer_raw_get(raw->handle);
545 	if (IS_ERR(xfer)) {
546 		dev_warn(dev, "RAW - Cannot get a free RAW xfer !\n");
547 		return PTR_ERR(xfer);
548 	}
549 
550 	/* Build xfer from the provided SCMI bare LE message */
551 	msg_hdr = le32_to_cpu(*((__le32 *)buf));
552 	unpack_scmi_header(msg_hdr, &xfer->hdr);
553 	xfer->hdr.seq = (u16)MSG_XTRACT_TOKEN(msg_hdr);
554 	/* Polling not supported */
555 	xfer->hdr.poll_completion = false;
556 	xfer->hdr.status = SCMI_SUCCESS;
557 	xfer->tx.len = tx_size;
558 	xfer->rx.len = raw->desc->max_msg_size;
559 	/* Clear the whole TX buffer */
560 	memset(xfer->tx.buf, 0x00, raw->desc->max_msg_size);
561 	if (xfer->tx.len)
562 		memcpy(xfer->tx.buf, (u8 *)buf + sizeof(msg_hdr), xfer->tx.len);
563 	*p = xfer;
564 
565 	/*
566 	 * In flight registration can temporarily fail in case of Raw messages
567 	 * if the user injects messages without using monotonically increasing
568 	 * sequence numbers since, in Raw mode, the xfer (and the token) is
569 	 * finally released later by a deferred worker. Just retry for a while.
570 	 */
571 	do {
572 		ret = scmi_xfer_raw_inflight_register(raw->handle, xfer);
573 		if (ret) {
574 			dev_dbg(dev,
575 				"...retrying[%d] inflight registration\n",
576 				retry);
577 			msleep(raw->desc->max_rx_timeout_ms /
578 			       SCMI_XFER_RAW_MAX_RETRIES);
579 		}
580 	} while (ret && --retry);
581 
582 	if (ret) {
583 		dev_warn(dev,
584 			 "RAW - Could NOT register xfer %d in-flight HDR:0x%08X\n",
585 			 xfer->hdr.seq, msg_hdr);
586 		scmi_xfer_raw_put(raw->handle, xfer);
587 	}
588 
589 	return ret;
590 }
591 
592 /**
593  * scmi_do_xfer_raw_start  - An helper to send a valid raw xfer
594  *
595  * @raw: A reference to the Raw instance.
596  * @xfer: The xfer to send
597  * @chan_id: The channel ID to use, if zero the channels is automatically
598  *	     selected based on the protocol used.
599  * @async: A flag stating if an asynchronous command is required.
600  *
601  * This function send a previously built raw xfer using an appropriate channel
602  * and queues the related waiting work.
603  *
604  * Note that we need to know explicitly if the required command is meant to be
605  * asynchronous in kind since we have to properly setup the waiter.
606  * (and deducing this from the payload is weak and do not scale given there is
607  *  NOT a common header-flag stating if the command is asynchronous or not)
608  *
609  * Return: 0 on Success
610  */
611 static int scmi_do_xfer_raw_start(struct scmi_raw_mode_info *raw,
612 				  struct scmi_xfer *xfer, u8 chan_id,
613 				  bool async)
614 {
615 	int ret;
616 	struct scmi_chan_info *cinfo;
617 	struct scmi_xfer_raw_waiter *rw;
618 	struct device *dev = raw->handle->dev;
619 
620 	if (!chan_id)
621 		chan_id = xfer->hdr.protocol_id;
622 	else
623 		xfer->flags |= SCMI_XFER_FLAG_CHAN_SET;
624 
625 	cinfo = scmi_xfer_raw_channel_get(raw->handle, chan_id);
626 	if (IS_ERR(cinfo))
627 		return PTR_ERR(cinfo);
628 
629 	rw = scmi_xfer_raw_waiter_get(raw, xfer, cinfo, async);
630 	if (!rw) {
631 		dev_warn(dev, "RAW - Cannot get a free waiter !\n");
632 		return -ENOMEM;
633 	}
634 
635 	/* True ONLY if also supported by transport. */
636 	if (is_polling_enabled(cinfo, raw->desc))
637 		xfer->hdr.poll_completion = true;
638 
639 	reinit_completion(&xfer->done);
640 	/* Make sure xfer state update is visible before sending */
641 	smp_store_mb(xfer->state, SCMI_XFER_SENT_OK);
642 
643 	trace_scmi_xfer_begin(xfer->transfer_id, xfer->hdr.id,
644 			      xfer->hdr.protocol_id, xfer->hdr.seq,
645 			      xfer->hdr.poll_completion);
646 
647 	ret = raw->desc->ops->send_message(rw->cinfo, xfer);
648 	if (ret) {
649 		dev_err(dev, "Failed to send RAW message %d\n", ret);
650 		scmi_xfer_raw_waiter_put(raw, rw);
651 		return ret;
652 	}
653 
654 	trace_scmi_msg_dump(raw->id, cinfo->id, xfer->hdr.protocol_id,
655 			    xfer->hdr.id, "cmnd", xfer->hdr.seq,
656 			    xfer->hdr.status,
657 			    xfer->tx.buf, xfer->tx.len);
658 
659 	scmi_xfer_raw_waiter_enqueue(raw, rw);
660 
661 	return ret;
662 }
663 
664 /**
665  * scmi_raw_message_send  - An helper to build and send an SCMI command using
666  * the provided SCMI bare message buffer
667  *
668  * @raw: A reference to the Raw instance.
669  * @buf: A buffer containing the whole SCMI message to send (including the
670  *	 header) in little-endian binary format.
671  * @len: Length of the message in @buf.
672  * @chan_id: The channel ID to use.
673  * @async: A flag stating if an asynchronous command is required.
674  *
675  * Return: 0 on Success
676  */
677 static int scmi_raw_message_send(struct scmi_raw_mode_info *raw,
678 				 void *buf, size_t len, u8 chan_id, bool async)
679 {
680 	int ret;
681 	struct scmi_xfer *xfer;
682 
683 	ret = scmi_xfer_raw_get_init(raw, buf, len, &xfer);
684 	if (ret)
685 		return ret;
686 
687 	ret = scmi_do_xfer_raw_start(raw, xfer, chan_id, async);
688 	if (ret)
689 		scmi_xfer_raw_put(raw->handle, xfer);
690 
691 	return ret;
692 }
693 
694 static struct scmi_raw_buffer *
695 scmi_raw_message_dequeue(struct scmi_raw_queue *q, bool o_nonblock)
696 {
697 	unsigned long flags;
698 	struct scmi_raw_buffer *rb;
699 
700 	spin_lock_irqsave(&q->msg_q_lock, flags);
701 	while (list_empty(&q->msg_q)) {
702 		spin_unlock_irqrestore(&q->msg_q_lock, flags);
703 
704 		if (o_nonblock)
705 			return ERR_PTR(-EAGAIN);
706 
707 		if (wait_event_interruptible(q->wq, !list_empty(&q->msg_q)))
708 			return ERR_PTR(-ERESTARTSYS);
709 
710 		spin_lock_irqsave(&q->msg_q_lock, flags);
711 	}
712 
713 	rb = scmi_raw_buffer_dequeue_unlocked(q);
714 
715 	spin_unlock_irqrestore(&q->msg_q_lock, flags);
716 
717 	return rb;
718 }
719 
720 /**
721  * scmi_raw_message_receive  - An helper to dequeue and report the next
722  * available enqueued raw message payload that has been collected.
723  *
724  * @raw: A reference to the Raw instance.
725  * @buf: A buffer to get hold of the whole SCMI message received and represented
726  *	 in little-endian binary format.
727  * @len: Length of @buf.
728  * @size: The effective size of the message copied into @buf
729  * @idx: The index of the queue to pick the next queued message from.
730  * @chan_id: The channel ID to use.
731  * @o_nonblock: A flag to request a non-blocking message dequeue.
732  *
733  * Return: 0 on Success
734  */
735 static int scmi_raw_message_receive(struct scmi_raw_mode_info *raw,
736 				    void *buf, size_t len, size_t *size,
737 				    unsigned int idx, unsigned int chan_id,
738 				    bool o_nonblock)
739 {
740 	int ret = 0;
741 	struct scmi_raw_buffer *rb;
742 	struct scmi_raw_queue *q;
743 
744 	q = scmi_raw_queue_select(raw, idx, chan_id);
745 	if (!q)
746 		return -ENODEV;
747 
748 	rb = scmi_raw_message_dequeue(q, o_nonblock);
749 	if (IS_ERR(rb)) {
750 		dev_dbg(raw->handle->dev, "RAW - No message available!\n");
751 		return PTR_ERR(rb);
752 	}
753 
754 	if (rb->msg.len <= len) {
755 		memcpy(buf, rb->msg.buf, rb->msg.len);
756 		*size = rb->msg.len;
757 	} else {
758 		ret = -ENOSPC;
759 	}
760 
761 	scmi_raw_buffer_put(q, rb);
762 
763 	return ret;
764 }
765 
766 /* SCMI Raw debugfs helpers */
767 
768 static ssize_t scmi_dbg_raw_mode_common_read(struct file *filp,
769 					     char __user *buf,
770 					     size_t count, loff_t *ppos,
771 					     unsigned int idx)
772 {
773 	ssize_t cnt;
774 	struct scmi_dbg_raw_data *rd = filp->private_data;
775 
776 	if (!rd->rx_size) {
777 		int ret;
778 
779 		ret = scmi_raw_message_receive(rd->raw, rd->rx.buf, rd->rx.len,
780 					       &rd->rx_size, idx, rd->chan_id,
781 					       filp->f_flags & O_NONBLOCK);
782 		if (ret) {
783 			rd->rx_size = 0;
784 			return ret;
785 		}
786 
787 		/* Reset any previous filepos change, including writes */
788 		*ppos = 0;
789 	} else if (*ppos == rd->rx_size) {
790 		/* Return EOF once all the message has been read-out */
791 		rd->rx_size = 0;
792 		return 0;
793 	}
794 
795 	cnt = simple_read_from_buffer(buf, count, ppos,
796 				      rd->rx.buf, rd->rx_size);
797 
798 	return cnt;
799 }
800 
801 static ssize_t scmi_dbg_raw_mode_common_write(struct file *filp,
802 					      const char __user *buf,
803 					      size_t count, loff_t *ppos,
804 					      bool async)
805 {
806 	int ret;
807 	struct scmi_dbg_raw_data *rd = filp->private_data;
808 
809 	if (count > rd->tx.len - rd->tx_size)
810 		return -ENOSPC;
811 
812 	/* On first write attempt @count carries the total full message size. */
813 	if (!rd->tx_size)
814 		rd->tx_req_size = count;
815 
816 	/*
817 	 * Gather a full message, possibly across multiple interrupted wrrtes,
818 	 * before sending it with a single RAW xfer.
819 	 */
820 	if (rd->tx_size < rd->tx_req_size) {
821 		ssize_t cnt;
822 
823 		cnt = simple_write_to_buffer(rd->tx.buf, rd->tx.len, ppos,
824 					     buf, count);
825 		if (cnt < 0)
826 			return cnt;
827 
828 		rd->tx_size += cnt;
829 		if (cnt < count)
830 			return cnt;
831 	}
832 
833 	ret = scmi_raw_message_send(rd->raw, rd->tx.buf, rd->tx_size,
834 				    rd->chan_id, async);
835 
836 	/* Reset ppos for next message ... */
837 	rd->tx_size = 0;
838 	*ppos = 0;
839 
840 	return ret ?: count;
841 }
842 
843 static __poll_t scmi_test_dbg_raw_common_poll(struct file *filp,
844 					      struct poll_table_struct *wait,
845 					      unsigned int idx)
846 {
847 	unsigned long flags;
848 	struct scmi_dbg_raw_data *rd = filp->private_data;
849 	struct scmi_raw_queue *q;
850 	__poll_t mask = 0;
851 
852 	q = scmi_raw_queue_select(rd->raw, idx, rd->chan_id);
853 	if (!q)
854 		return mask;
855 
856 	poll_wait(filp, &q->wq, wait);
857 
858 	spin_lock_irqsave(&q->msg_q_lock, flags);
859 	if (!list_empty(&q->msg_q))
860 		mask = EPOLLIN | EPOLLRDNORM;
861 	spin_unlock_irqrestore(&q->msg_q_lock, flags);
862 
863 	return mask;
864 }
865 
866 static ssize_t scmi_dbg_raw_mode_message_read(struct file *filp,
867 					      char __user *buf,
868 					      size_t count, loff_t *ppos)
869 {
870 	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
871 					     SCMI_RAW_REPLY_QUEUE);
872 }
873 
874 static ssize_t scmi_dbg_raw_mode_message_write(struct file *filp,
875 					       const char __user *buf,
876 					       size_t count, loff_t *ppos)
877 {
878 	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, false);
879 }
880 
881 static __poll_t scmi_dbg_raw_mode_message_poll(struct file *filp,
882 					       struct poll_table_struct *wait)
883 {
884 	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_REPLY_QUEUE);
885 }
886 
887 static int scmi_dbg_raw_mode_open(struct inode *inode, struct file *filp)
888 {
889 	u8 id;
890 	struct scmi_raw_mode_info *raw;
891 	struct scmi_dbg_raw_data *rd;
892 	const char *id_str = filp->f_path.dentry->d_parent->d_name.name;
893 
894 	if (!inode->i_private)
895 		return -ENODEV;
896 
897 	raw = inode->i_private;
898 	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
899 	if (!rd)
900 		return -ENOMEM;
901 
902 	rd->rx.len = raw->desc->max_msg_size + sizeof(u32);
903 	rd->rx.buf = kzalloc(rd->rx.len, GFP_KERNEL);
904 	if (!rd->rx.buf) {
905 		kfree(rd);
906 		return -ENOMEM;
907 	}
908 
909 	rd->tx.len = raw->desc->max_msg_size + sizeof(u32);
910 	rd->tx.buf = kzalloc(rd->tx.len, GFP_KERNEL);
911 	if (!rd->tx.buf) {
912 		kfree(rd->rx.buf);
913 		kfree(rd);
914 		return -ENOMEM;
915 	}
916 
917 	/* Grab channel ID from debugfs entry naming if any */
918 	if (!kstrtou8(id_str, 16, &id))
919 		rd->chan_id = id;
920 
921 	rd->raw = raw;
922 	filp->private_data = rd;
923 
924 	return nonseekable_open(inode, filp);
925 }
926 
927 static int scmi_dbg_raw_mode_release(struct inode *inode, struct file *filp)
928 {
929 	struct scmi_dbg_raw_data *rd = filp->private_data;
930 
931 	kfree(rd->rx.buf);
932 	kfree(rd->tx.buf);
933 	kfree(rd);
934 
935 	return 0;
936 }
937 
938 static ssize_t scmi_dbg_raw_mode_reset_write(struct file *filp,
939 					     const char __user *buf,
940 					     size_t count, loff_t *ppos)
941 {
942 	struct scmi_dbg_raw_data *rd = filp->private_data;
943 
944 	scmi_xfer_raw_reset(rd->raw);
945 
946 	return count;
947 }
948 
949 static const struct file_operations scmi_dbg_raw_mode_reset_fops = {
950 	.open = scmi_dbg_raw_mode_open,
951 	.release = scmi_dbg_raw_mode_release,
952 	.write = scmi_dbg_raw_mode_reset_write,
953 	.llseek = no_llseek,
954 	.owner = THIS_MODULE,
955 };
956 
957 static const struct file_operations scmi_dbg_raw_mode_message_fops = {
958 	.open = scmi_dbg_raw_mode_open,
959 	.release = scmi_dbg_raw_mode_release,
960 	.read = scmi_dbg_raw_mode_message_read,
961 	.write = scmi_dbg_raw_mode_message_write,
962 	.poll = scmi_dbg_raw_mode_message_poll,
963 	.llseek = no_llseek,
964 	.owner = THIS_MODULE,
965 };
966 
967 static ssize_t scmi_dbg_raw_mode_message_async_write(struct file *filp,
968 						     const char __user *buf,
969 						     size_t count, loff_t *ppos)
970 {
971 	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, true);
972 }
973 
974 static const struct file_operations scmi_dbg_raw_mode_message_async_fops = {
975 	.open = scmi_dbg_raw_mode_open,
976 	.release = scmi_dbg_raw_mode_release,
977 	.read = scmi_dbg_raw_mode_message_read,
978 	.write = scmi_dbg_raw_mode_message_async_write,
979 	.poll = scmi_dbg_raw_mode_message_poll,
980 	.llseek = no_llseek,
981 	.owner = THIS_MODULE,
982 };
983 
984 static ssize_t scmi_test_dbg_raw_mode_notif_read(struct file *filp,
985 						 char __user *buf,
986 						 size_t count, loff_t *ppos)
987 {
988 	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
989 					     SCMI_RAW_NOTIF_QUEUE);
990 }
991 
992 static __poll_t
993 scmi_test_dbg_raw_mode_notif_poll(struct file *filp,
994 				  struct poll_table_struct *wait)
995 {
996 	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_NOTIF_QUEUE);
997 }
998 
999 static const struct file_operations scmi_dbg_raw_mode_notification_fops = {
1000 	.open = scmi_dbg_raw_mode_open,
1001 	.release = scmi_dbg_raw_mode_release,
1002 	.read = scmi_test_dbg_raw_mode_notif_read,
1003 	.poll = scmi_test_dbg_raw_mode_notif_poll,
1004 	.llseek = no_llseek,
1005 	.owner = THIS_MODULE,
1006 };
1007 
1008 static ssize_t scmi_test_dbg_raw_mode_errors_read(struct file *filp,
1009 						  char __user *buf,
1010 						  size_t count, loff_t *ppos)
1011 {
1012 	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
1013 					     SCMI_RAW_ERRS_QUEUE);
1014 }
1015 
1016 static __poll_t
1017 scmi_test_dbg_raw_mode_errors_poll(struct file *filp,
1018 				   struct poll_table_struct *wait)
1019 {
1020 	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_ERRS_QUEUE);
1021 }
1022 
1023 static const struct file_operations scmi_dbg_raw_mode_errors_fops = {
1024 	.open = scmi_dbg_raw_mode_open,
1025 	.release = scmi_dbg_raw_mode_release,
1026 	.read = scmi_test_dbg_raw_mode_errors_read,
1027 	.poll = scmi_test_dbg_raw_mode_errors_poll,
1028 	.llseek = no_llseek,
1029 	.owner = THIS_MODULE,
1030 };
1031 
1032 static struct scmi_raw_queue *
1033 scmi_raw_queue_init(struct scmi_raw_mode_info *raw)
1034 {
1035 	int i;
1036 	struct scmi_raw_buffer *rb;
1037 	struct device *dev = raw->handle->dev;
1038 	struct scmi_raw_queue *q;
1039 
1040 	q = devm_kzalloc(dev, sizeof(*q), GFP_KERNEL);
1041 	if (!q)
1042 		return ERR_PTR(-ENOMEM);
1043 
1044 	rb = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rb), GFP_KERNEL);
1045 	if (!rb)
1046 		return ERR_PTR(-ENOMEM);
1047 
1048 	spin_lock_init(&q->free_bufs_lock);
1049 	INIT_LIST_HEAD(&q->free_bufs);
1050 	for (i = 0; i < raw->tx_max_msg; i++, rb++) {
1051 		rb->max_len = raw->desc->max_msg_size + sizeof(u32);
1052 		rb->msg.buf = devm_kzalloc(dev, rb->max_len, GFP_KERNEL);
1053 		if (!rb->msg.buf)
1054 			return ERR_PTR(-ENOMEM);
1055 		scmi_raw_buffer_put(q, rb);
1056 	}
1057 
1058 	spin_lock_init(&q->msg_q_lock);
1059 	INIT_LIST_HEAD(&q->msg_q);
1060 	init_waitqueue_head(&q->wq);
1061 
1062 	return q;
1063 }
1064 
1065 static int scmi_xfer_raw_worker_init(struct scmi_raw_mode_info *raw)
1066 {
1067 	int i;
1068 	struct scmi_xfer_raw_waiter *rw;
1069 	struct device *dev = raw->handle->dev;
1070 
1071 	rw = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rw), GFP_KERNEL);
1072 	if (!rw)
1073 		return -ENOMEM;
1074 
1075 	raw->wait_wq = alloc_workqueue("scmi-raw-wait-wq-%d",
1076 				       WQ_UNBOUND | WQ_FREEZABLE |
1077 				       WQ_HIGHPRI | WQ_SYSFS, 0, raw->id);
1078 	if (!raw->wait_wq)
1079 		return -ENOMEM;
1080 
1081 	mutex_init(&raw->free_mtx);
1082 	INIT_LIST_HEAD(&raw->free_waiters);
1083 	mutex_init(&raw->active_mtx);
1084 	INIT_LIST_HEAD(&raw->active_waiters);
1085 
1086 	for (i = 0; i < raw->tx_max_msg; i++, rw++) {
1087 		init_completion(&rw->async_response);
1088 		scmi_xfer_raw_waiter_put(raw, rw);
1089 	}
1090 	INIT_WORK(&raw->waiters_work, scmi_xfer_raw_worker);
1091 
1092 	return 0;
1093 }
1094 
1095 static int scmi_raw_mode_setup(struct scmi_raw_mode_info *raw,
1096 			       u8 *channels, int num_chans)
1097 {
1098 	int ret, idx;
1099 	void *gid;
1100 	struct device *dev = raw->handle->dev;
1101 
1102 	gid = devres_open_group(dev, NULL, GFP_KERNEL);
1103 	if (!gid)
1104 		return -ENOMEM;
1105 
1106 	for (idx = 0; idx < SCMI_RAW_MAX_QUEUE; idx++) {
1107 		raw->q[idx] = scmi_raw_queue_init(raw);
1108 		if (IS_ERR(raw->q[idx])) {
1109 			ret = PTR_ERR(raw->q[idx]);
1110 			goto err;
1111 		}
1112 	}
1113 
1114 	xa_init(&raw->chans_q);
1115 	if (num_chans > 1) {
1116 		int i;
1117 
1118 		for (i = 0; i < num_chans; i++) {
1119 			struct scmi_raw_queue *q;
1120 
1121 			q = scmi_raw_queue_init(raw);
1122 			if (IS_ERR(q)) {
1123 				ret = PTR_ERR(q);
1124 				goto err_xa;
1125 			}
1126 
1127 			ret = xa_insert(&raw->chans_q, channels[i], q,
1128 					GFP_KERNEL);
1129 			if (ret) {
1130 				dev_err(dev,
1131 					"Fail to allocate Raw queue 0x%02X\n",
1132 					channels[i]);
1133 				goto err_xa;
1134 			}
1135 		}
1136 	}
1137 
1138 	ret = scmi_xfer_raw_worker_init(raw);
1139 	if (ret)
1140 		goto err_xa;
1141 
1142 	devres_close_group(dev, gid);
1143 	raw->gid = gid;
1144 
1145 	return 0;
1146 
1147 err_xa:
1148 	xa_destroy(&raw->chans_q);
1149 err:
1150 	devres_release_group(dev, gid);
1151 	return ret;
1152 }
1153 
1154 /**
1155  * scmi_raw_mode_init  - Function to initialize the SCMI Raw stack
1156  *
1157  * @handle: Pointer to SCMI entity handle
1158  * @top_dentry: A reference to the top Raw debugfs dentry
1159  * @instance_id: The ID of the underlying SCMI platform instance represented by
1160  *		 this Raw instance
1161  * @channels: The list of the existing channels
1162  * @num_chans: The number of entries in @channels
1163  * @desc: Reference to the transport operations
1164  * @tx_max_msg: Max number of in-flight messages allowed by the transport
1165  *
1166  * This function prepare the SCMI Raw stack and creates the debugfs API.
1167  *
1168  * Return: An opaque handle to the Raw instance on Success, an ERR_PTR otherwise
1169  */
1170 void *scmi_raw_mode_init(const struct scmi_handle *handle,
1171 			 struct dentry *top_dentry, int instance_id,
1172 			 u8 *channels, int num_chans,
1173 			 const struct scmi_desc *desc, int tx_max_msg)
1174 {
1175 	int ret;
1176 	struct scmi_raw_mode_info *raw;
1177 	struct device *dev;
1178 
1179 	if (!handle || !desc)
1180 		return ERR_PTR(-EINVAL);
1181 
1182 	dev = handle->dev;
1183 	raw = devm_kzalloc(dev, sizeof(*raw), GFP_KERNEL);
1184 	if (!raw)
1185 		return ERR_PTR(-ENOMEM);
1186 
1187 	raw->handle = handle;
1188 	raw->desc = desc;
1189 	raw->tx_max_msg = tx_max_msg;
1190 	raw->id = instance_id;
1191 
1192 	ret = scmi_raw_mode_setup(raw, channels, num_chans);
1193 	if (ret) {
1194 		devm_kfree(dev, raw);
1195 		return ERR_PTR(ret);
1196 	}
1197 
1198 	raw->dentry = debugfs_create_dir("raw", top_dentry);
1199 
1200 	debugfs_create_file("reset", 0200, raw->dentry, raw,
1201 			    &scmi_dbg_raw_mode_reset_fops);
1202 
1203 	debugfs_create_file("message", 0600, raw->dentry, raw,
1204 			    &scmi_dbg_raw_mode_message_fops);
1205 
1206 	debugfs_create_file("message_async", 0600, raw->dentry, raw,
1207 			    &scmi_dbg_raw_mode_message_async_fops);
1208 
1209 	debugfs_create_file("notification", 0400, raw->dentry, raw,
1210 			    &scmi_dbg_raw_mode_notification_fops);
1211 
1212 	debugfs_create_file("errors", 0400, raw->dentry, raw,
1213 			    &scmi_dbg_raw_mode_errors_fops);
1214 
1215 	/*
1216 	 * Expose per-channel entries if multiple channels available.
1217 	 * Just ignore errors while setting up these interfaces since we
1218 	 * have anyway already a working core Raw support.
1219 	 */
1220 	if (num_chans > 1) {
1221 		int i;
1222 		struct dentry *top_chans;
1223 
1224 		top_chans = debugfs_create_dir("channels", raw->dentry);
1225 
1226 		for (i = 0; i < num_chans; i++) {
1227 			char cdir[8];
1228 			struct dentry *chd;
1229 
1230 			snprintf(cdir, 8, "0x%02X", channels[i]);
1231 			chd = debugfs_create_dir(cdir, top_chans);
1232 
1233 			debugfs_create_file("message", 0600, chd, raw,
1234 					    &scmi_dbg_raw_mode_message_fops);
1235 
1236 			debugfs_create_file("message_async", 0600, chd, raw,
1237 					    &scmi_dbg_raw_mode_message_async_fops);
1238 		}
1239 	}
1240 
1241 	dev_info(dev, "SCMI RAW Mode initialized for instance %d\n", raw->id);
1242 
1243 	return raw;
1244 }
1245 
1246 /**
1247  * scmi_raw_mode_cleanup  - Function to cleanup the SCMI Raw stack
1248  *
1249  * @r: An opaque handle to an initialized SCMI Raw instance
1250  */
1251 void scmi_raw_mode_cleanup(void *r)
1252 {
1253 	struct scmi_raw_mode_info *raw = r;
1254 
1255 	if (!raw)
1256 		return;
1257 
1258 	debugfs_remove_recursive(raw->dentry);
1259 
1260 	cancel_work_sync(&raw->waiters_work);
1261 	destroy_workqueue(raw->wait_wq);
1262 	xa_destroy(&raw->chans_q);
1263 }
1264 
1265 static int scmi_xfer_raw_collect(void *msg, size_t *msg_len,
1266 				 struct scmi_xfer *xfer)
1267 {
1268 	__le32 *m;
1269 	size_t msg_size;
1270 
1271 	if (!xfer || !msg || !msg_len)
1272 		return -EINVAL;
1273 
1274 	/* Account for hdr ...*/
1275 	msg_size = xfer->rx.len + sizeof(u32);
1276 	/* ... and status if needed */
1277 	if (xfer->hdr.type != MSG_TYPE_NOTIFICATION)
1278 		msg_size += sizeof(u32);
1279 
1280 	if (msg_size > *msg_len)
1281 		return -ENOSPC;
1282 
1283 	m = msg;
1284 	*m = cpu_to_le32(pack_scmi_header(&xfer->hdr));
1285 	if (xfer->hdr.type != MSG_TYPE_NOTIFICATION)
1286 		*++m = cpu_to_le32(xfer->hdr.status);
1287 
1288 	memcpy(++m, xfer->rx.buf, xfer->rx.len);
1289 
1290 	*msg_len = msg_size;
1291 
1292 	return 0;
1293 }
1294 
1295 /**
1296  * scmi_raw_message_report  - Helper to report back valid reponses/notifications
1297  * to raw message requests.
1298  *
1299  * @r: An opaque reference to the raw instance configuration
1300  * @xfer: The xfer containing the message to be reported
1301  * @idx: The index of the queue.
1302  * @chan_id: The channel ID to use.
1303  *
1304  * If Raw mode is enabled, this is called from the SCMI core on the regular RX
1305  * path to save and enqueue the response/notification payload carried by this
1306  * xfer into a dedicated scmi_raw_buffer for later consumption by the user.
1307  *
1308  * This way the caller can free the related xfer immediately afterwards and the
1309  * user can read back the raw message payload at its own pace (if ever) without
1310  * holding an xfer for too long.
1311  */
1312 void scmi_raw_message_report(void *r, struct scmi_xfer *xfer,
1313 			     unsigned int idx, unsigned int chan_id)
1314 {
1315 	int ret;
1316 	unsigned long flags;
1317 	struct scmi_raw_buffer *rb;
1318 	struct device *dev;
1319 	struct scmi_raw_queue *q;
1320 	struct scmi_raw_mode_info *raw = r;
1321 
1322 	if (!raw || (idx == SCMI_RAW_REPLY_QUEUE && !SCMI_XFER_IS_RAW(xfer)))
1323 		return;
1324 
1325 	dev = raw->handle->dev;
1326 	q = scmi_raw_queue_select(raw, idx,
1327 				  SCMI_XFER_IS_CHAN_SET(xfer) ? chan_id : 0);
1328 	if (!q) {
1329 		dev_warn(dev,
1330 			 "RAW[%d] - NO queue for chan 0x%X. Dropping report.\n",
1331 			 idx, chan_id);
1332 		return;
1333 	}
1334 
1335 	/*
1336 	 * Grab the msg_q_lock upfront to avoid a possible race between
1337 	 * realizing the free list was empty and effectively picking the next
1338 	 * buffer to use from the oldest one enqueued and still unread on this
1339 	 * msg_q.
1340 	 *
1341 	 * Note that nowhere else these locks are taken together, so no risk of
1342 	 * deadlocks du eto inversion.
1343 	 */
1344 	spin_lock_irqsave(&q->msg_q_lock, flags);
1345 	rb = scmi_raw_buffer_get(q);
1346 	if (!rb) {
1347 		/*
1348 		 * Immediate and delayed replies to previously injected Raw
1349 		 * commands MUST be read back from userspace to free the buffers:
1350 		 * if this is not happening something is seriously broken and
1351 		 * must be fixed at the application level: complain loudly.
1352 		 */
1353 		if (idx == SCMI_RAW_REPLY_QUEUE) {
1354 			spin_unlock_irqrestore(&q->msg_q_lock, flags);
1355 			dev_warn(dev,
1356 				 "RAW[%d] - Buffers exhausted. Dropping report.\n",
1357 				 idx);
1358 			return;
1359 		}
1360 
1361 		/*
1362 		 * Notifications and errors queues are instead handled in a
1363 		 * circular manner: unread old buffers are just overwritten by
1364 		 * newer ones.
1365 		 *
1366 		 * The main reason for this is that notifications originated
1367 		 * by Raw requests cannot be distinguished from normal ones, so
1368 		 * your Raw buffers queues risk to be flooded and depleted by
1369 		 * notifications if you left it mistakenly enabled or when in
1370 		 * coexistence mode.
1371 		 */
1372 		rb = scmi_raw_buffer_dequeue_unlocked(q);
1373 		if (WARN_ON(!rb)) {
1374 			spin_unlock_irqrestore(&q->msg_q_lock, flags);
1375 			return;
1376 		}
1377 
1378 		/* Reset to full buffer length */
1379 		rb->msg.len = rb->max_len;
1380 
1381 		dev_warn_once(dev,
1382 			      "RAW[%d] - Buffers exhausted. Re-using oldest.\n",
1383 			      idx);
1384 	}
1385 	spin_unlock_irqrestore(&q->msg_q_lock, flags);
1386 
1387 	ret = scmi_xfer_raw_collect(rb->msg.buf, &rb->msg.len, xfer);
1388 	if (ret) {
1389 		dev_warn(dev, "RAW - Cannot collect xfer into buffer !\n");
1390 		scmi_raw_buffer_put(q, rb);
1391 		return;
1392 	}
1393 
1394 	scmi_raw_buffer_enqueue(q, rb);
1395 }
1396 
1397 static void scmi_xfer_raw_fill(struct scmi_raw_mode_info *raw,
1398 			       struct scmi_chan_info *cinfo,
1399 			       struct scmi_xfer *xfer, u32 msg_hdr)
1400 {
1401 	/* Unpack received HDR as it is */
1402 	unpack_scmi_header(msg_hdr, &xfer->hdr);
1403 	xfer->hdr.seq = MSG_XTRACT_TOKEN(msg_hdr);
1404 
1405 	memset(xfer->rx.buf, 0x00, xfer->rx.len);
1406 
1407 	raw->desc->ops->fetch_response(cinfo, xfer);
1408 }
1409 
1410 /**
1411  * scmi_raw_error_report  - Helper to report back timed-out or generally
1412  * unexpected replies.
1413  *
1414  * @r: An opaque reference to the raw instance configuration
1415  * @cinfo: A reference to the channel to use to retrieve the broken xfer
1416  * @msg_hdr: The SCMI message header of the message to fetch and report
1417  * @priv: Any private data related to the xfer.
1418  *
1419  * If Raw mode is enabled, this is called from the SCMI core on the RX path in
1420  * case of errors to save and enqueue the bad message payload carried by the
1421  * message that has just been received.
1422  *
1423  * Note that we have to manually fetch any available payload into a temporary
1424  * xfer to be able to save and enqueue the message, since the regular RX error
1425  * path which had called this would have not fetched the message payload having
1426  * classified it as an error.
1427  */
1428 void scmi_raw_error_report(void *r, struct scmi_chan_info *cinfo,
1429 			   u32 msg_hdr, void *priv)
1430 {
1431 	struct scmi_xfer xfer;
1432 	struct scmi_raw_mode_info *raw = r;
1433 
1434 	if (!raw)
1435 		return;
1436 
1437 	xfer.rx.len = raw->desc->max_msg_size;
1438 	xfer.rx.buf = kzalloc(xfer.rx.len, GFP_ATOMIC);
1439 	if (!xfer.rx.buf) {
1440 		dev_info(raw->handle->dev,
1441 			 "Cannot report Raw error for HDR:0x%X - ENOMEM\n",
1442 			 msg_hdr);
1443 		return;
1444 	}
1445 
1446 	/* Any transport-provided priv must be passed back down to transport */
1447 	if (priv)
1448 		/* Ensure priv is visible */
1449 		smp_store_mb(xfer.priv, priv);
1450 
1451 	scmi_xfer_raw_fill(raw, cinfo, &xfer, msg_hdr);
1452 	scmi_raw_message_report(raw, &xfer, SCMI_RAW_ERRS_QUEUE, 0);
1453 
1454 	kfree(xfer.rx.buf);
1455 }
1456