1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5 
6 #include <asm/byteorder.h>
7 #include <linux/completion.h>
8 #include <linux/crc32.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/kref.h>
12 #include <linux/list.h>
13 #include <linux/mhi.h>
14 #include <linux/mm.h>
15 #include <linux/moduleparam.h>
16 #include <linux/mutex.h>
17 #include <linux/overflow.h>
18 #include <linux/pci.h>
19 #include <linux/scatterlist.h>
20 #include <linux/types.h>
21 #include <linux/uaccess.h>
22 #include <linux/workqueue.h>
23 #include <linux/wait.h>
24 #include <drm/drm_device.h>
25 #include <drm/drm_file.h>
26 #include <uapi/drm/qaic_accel.h>
27 
28 #include "qaic.h"
29 
30 #define MANAGE_MAGIC_NUMBER		((__force __le32)0x43494151) /* "QAIC" in little endian */
31 #define QAIC_DBC_Q_GAP			SZ_256
32 #define QAIC_DBC_Q_BUF_ALIGN		SZ_4K
33 #define QAIC_MANAGE_EXT_MSG_LENGTH	SZ_64K /* Max DMA message length */
34 #define QAIC_WRAPPER_MAX_SIZE		SZ_4K
35 #define QAIC_MHI_RETRY_WAIT_MS		100
36 #define QAIC_MHI_RETRY_MAX		20
37 
38 static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
39 module_param(control_resp_timeout_s, uint, 0600);
40 MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
41 
42 struct manage_msg {
43 	u32 len;
44 	u32 count;
45 	u8 data[];
46 };
47 
48 /*
49  * wire encoding structures for the manage protocol.
50  * All fields are little endian on the wire
51  */
52 struct wire_msg_hdr {
53 	__le32 crc32; /* crc of everything following this field in the message */
54 	__le32 magic_number;
55 	__le32 sequence_number;
56 	__le32 len; /* length of this message */
57 	__le32 count; /* number of transactions in this message */
58 	__le32 handle; /* unique id to track the resources consumed */
59 	__le32 partition_id; /* partition id for the request (signed) */
60 	__le32 padding; /* must be 0 */
61 } __packed;
62 
63 struct wire_msg {
64 	struct wire_msg_hdr hdr;
65 	u8 data[];
66 } __packed;
67 
68 struct wire_trans_hdr {
69 	__le32 type;
70 	__le32 len;
71 } __packed;
72 
73 /* Each message sent from driver to device are organized in a list of wrapper_msg */
74 struct wrapper_msg {
75 	struct list_head list;
76 	struct kref ref_count;
77 	u32 len; /* length of data to transfer */
78 	struct wrapper_list *head;
79 	union {
80 		struct wire_msg msg;
81 		struct wire_trans_hdr trans;
82 	};
83 };
84 
85 struct wrapper_list {
86 	struct list_head list;
87 	spinlock_t lock; /* Protects the list state during additions and removals */
88 };
89 
90 struct wire_trans_passthrough {
91 	struct wire_trans_hdr hdr;
92 	u8 data[];
93 } __packed;
94 
95 struct wire_addr_size_pair {
96 	__le64 addr;
97 	__le64 size;
98 } __packed;
99 
100 struct wire_trans_dma_xfer {
101 	struct wire_trans_hdr hdr;
102 	__le32 tag;
103 	__le32 count;
104 	__le32 dma_chunk_id;
105 	__le32 padding;
106 	struct wire_addr_size_pair data[];
107 } __packed;
108 
109 /* Initiated by device to continue the DMA xfer of a large piece of data */
110 struct wire_trans_dma_xfer_cont {
111 	struct wire_trans_hdr hdr;
112 	__le32 dma_chunk_id;
113 	__le32 padding;
114 	__le64 xferred_size;
115 } __packed;
116 
117 struct wire_trans_activate_to_dev {
118 	struct wire_trans_hdr hdr;
119 	__le64 req_q_addr;
120 	__le64 rsp_q_addr;
121 	__le32 req_q_size;
122 	__le32 rsp_q_size;
123 	__le32 buf_len;
124 	__le32 options; /* unused, but BIT(16) has meaning to the device */
125 } __packed;
126 
127 struct wire_trans_activate_from_dev {
128 	struct wire_trans_hdr hdr;
129 	__le32 status;
130 	__le32 dbc_id;
131 	__le64 options; /* unused */
132 } __packed;
133 
134 struct wire_trans_deactivate_from_dev {
135 	struct wire_trans_hdr hdr;
136 	__le32 status;
137 	__le32 dbc_id;
138 } __packed;
139 
140 struct wire_trans_terminate_to_dev {
141 	struct wire_trans_hdr hdr;
142 	__le32 handle;
143 	__le32 padding;
144 } __packed;
145 
146 struct wire_trans_terminate_from_dev {
147 	struct wire_trans_hdr hdr;
148 	__le32 status;
149 	__le32 padding;
150 } __packed;
151 
152 struct wire_trans_status_to_dev {
153 	struct wire_trans_hdr hdr;
154 } __packed;
155 
156 struct wire_trans_status_from_dev {
157 	struct wire_trans_hdr hdr;
158 	__le16 major;
159 	__le16 minor;
160 	__le32 status;
161 	__le64 status_flags;
162 } __packed;
163 
164 struct wire_trans_validate_part_to_dev {
165 	struct wire_trans_hdr hdr;
166 	__le32 part_id;
167 	__le32 padding;
168 } __packed;
169 
170 struct wire_trans_validate_part_from_dev {
171 	struct wire_trans_hdr hdr;
172 	__le32 status;
173 	__le32 padding;
174 } __packed;
175 
176 struct xfer_queue_elem {
177 	/*
178 	 * Node in list of ongoing transfer request on control channel.
179 	 * Maintained by root device struct.
180 	 */
181 	struct list_head list;
182 	/* Sequence number of this transfer request */
183 	u32 seq_num;
184 	/* This is used to wait on until completion of transfer request */
185 	struct completion xfer_done;
186 	/* Received data from device */
187 	void *buf;
188 };
189 
190 struct dma_xfer {
191 	/* Node in list of DMA transfers which is used for cleanup */
192 	struct list_head list;
193 	/* SG table of memory used for DMA */
194 	struct sg_table *sgt;
195 	/* Array pages used for DMA */
196 	struct page **page_list;
197 	/* Number of pages used for DMA */
198 	unsigned long nr_pages;
199 };
200 
201 struct ioctl_resources {
202 	/* List of all DMA transfers which is used later for cleanup */
203 	struct list_head dma_xfers;
204 	/* Base address of request queue which belongs to a DBC */
205 	void *buf;
206 	/*
207 	 * Base bus address of request queue which belongs to a DBC. Response
208 	 * queue base bus address can be calculated by adding size of request
209 	 * queue to base bus address of request queue.
210 	 */
211 	dma_addr_t dma_addr;
212 	/* Total size of request queue and response queue in byte */
213 	u32 total_size;
214 	/* Total number of elements that can be queued in each of request and response queue */
215 	u32 nelem;
216 	/* Base address of response queue which belongs to a DBC */
217 	void *rsp_q_base;
218 	/* Status of the NNC message received */
219 	u32 status;
220 	/* DBC id of the DBC received from device */
221 	u32 dbc_id;
222 	/*
223 	 * DMA transfer request messages can be big in size and it may not be
224 	 * possible to send them in one shot. In such cases the messages are
225 	 * broken into chunks, this field stores ID of such chunks.
226 	 */
227 	u32 dma_chunk_id;
228 	/* Total number of bytes transferred for a DMA xfer request */
229 	u64 xferred_dma_size;
230 	/* Header of transaction message received from user. Used during DMA xfer request. */
231 	void *trans_hdr;
232 };
233 
234 struct resp_work {
235 	struct work_struct work;
236 	struct qaic_device *qdev;
237 	void *buf;
238 };
239 
240 /*
241  * Since we're working with little endian messages, its useful to be able to
242  * increment without filling a whole line with conversions back and forth just
243  * to add one(1) to a message count.
244  */
245 static __le32 incr_le32(__le32 val)
246 {
247 	return cpu_to_le32(le32_to_cpu(val) + 1);
248 }
249 
250 static u32 gen_crc(void *msg)
251 {
252 	struct wrapper_list *wrappers = msg;
253 	struct wrapper_msg *w;
254 	u32 crc = ~0;
255 
256 	list_for_each_entry(w, &wrappers->list, list)
257 		crc = crc32(crc, &w->msg, w->len);
258 
259 	return crc ^ ~0;
260 }
261 
262 static u32 gen_crc_stub(void *msg)
263 {
264 	return 0;
265 }
266 
267 static bool valid_crc(void *msg)
268 {
269 	struct wire_msg_hdr *hdr = msg;
270 	bool ret;
271 	u32 crc;
272 
273 	/*
274 	 * The output of this algorithm is always converted to the native
275 	 * endianness.
276 	 */
277 	crc = le32_to_cpu(hdr->crc32);
278 	hdr->crc32 = 0;
279 	ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
280 	hdr->crc32 = cpu_to_le32(crc);
281 	return ret;
282 }
283 
284 static bool valid_crc_stub(void *msg)
285 {
286 	return true;
287 }
288 
289 static void free_wrapper(struct kref *ref)
290 {
291 	struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
292 
293 	list_del(&wrapper->list);
294 	kfree(wrapper);
295 }
296 
297 static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
298 			 struct qaic_user *usr)
299 {
300 	u32 dbc_id = resources->dbc_id;
301 
302 	if (resources->buf) {
303 		wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
304 		qdev->dbc[dbc_id].req_q_base = resources->buf;
305 		qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
306 		qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
307 		qdev->dbc[dbc_id].total_size = resources->total_size;
308 		qdev->dbc[dbc_id].nelem = resources->nelem;
309 		enable_dbc(qdev, dbc_id, usr);
310 		qdev->dbc[dbc_id].in_use = true;
311 		resources->buf = NULL;
312 	}
313 }
314 
315 static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
316 {
317 	if (resources->buf)
318 		dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,
319 				  resources->dma_addr);
320 	resources->buf = NULL;
321 }
322 
323 static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
324 {
325 	struct dma_xfer *xfer;
326 	struct dma_xfer *x;
327 	int i;
328 
329 	list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
330 		dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
331 		sg_free_table(xfer->sgt);
332 		kfree(xfer->sgt);
333 		for (i = 0; i < xfer->nr_pages; ++i)
334 			put_page(xfer->page_list[i]);
335 		kfree(xfer->page_list);
336 		list_del(&xfer->list);
337 		kfree(xfer);
338 	}
339 }
340 
341 static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
342 {
343 	struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
344 
345 	if (!w)
346 		return NULL;
347 	list_add_tail(&w->list, &wrappers->list);
348 	kref_init(&w->ref_count);
349 	w->head = wrappers;
350 	return w;
351 }
352 
353 static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
354 			      u32 *user_len)
355 {
356 	struct qaic_manage_trans_passthrough *in_trans = trans;
357 	struct wire_trans_passthrough *out_trans;
358 	struct wrapper_msg *trans_wrapper;
359 	struct wrapper_msg *wrapper;
360 	struct wire_msg *msg;
361 	u32 msg_hdr_len;
362 
363 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
364 	msg = &wrapper->msg;
365 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
366 
367 	if (in_trans->hdr.len % 8 != 0)
368 		return -EINVAL;
369 
370 	if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_EXT_MSG_LENGTH)
371 		return -ENOSPC;
372 
373 	trans_wrapper = add_wrapper(wrappers,
374 				    offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
375 	if (!trans_wrapper)
376 		return -ENOMEM;
377 	trans_wrapper->len = in_trans->hdr.len;
378 	out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
379 
380 	memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
381 	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
382 	msg->hdr.count = incr_le32(msg->hdr.count);
383 	*user_len += in_trans->hdr.len;
384 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
385 	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
386 
387 	return 0;
388 }
389 
390 /* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
391 static int find_and_map_user_pages(struct qaic_device *qdev,
392 				   struct qaic_manage_trans_dma_xfer *in_trans,
393 				   struct ioctl_resources *resources, struct dma_xfer *xfer)
394 {
395 	unsigned long need_pages;
396 	struct page **page_list;
397 	unsigned long nr_pages;
398 	struct sg_table *sgt;
399 	u64 xfer_start_addr;
400 	int ret;
401 	int i;
402 
403 	xfer_start_addr = in_trans->addr + resources->xferred_dma_size;
404 
405 	need_pages = DIV_ROUND_UP(in_trans->size + offset_in_page(xfer_start_addr) -
406 				  resources->xferred_dma_size, PAGE_SIZE);
407 
408 	nr_pages = need_pages;
409 
410 	while (1) {
411 		page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
412 		if (!page_list) {
413 			nr_pages = nr_pages / 2;
414 			if (!nr_pages)
415 				return -ENOMEM;
416 		} else {
417 			break;
418 		}
419 	}
420 
421 	ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);
422 	if (ret < 0)
423 		goto free_page_list;
424 	if (ret != nr_pages) {
425 		nr_pages = ret;
426 		ret = -EFAULT;
427 		goto put_pages;
428 	}
429 
430 	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
431 	if (!sgt) {
432 		ret = -ENOMEM;
433 		goto put_pages;
434 	}
435 
436 	ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
437 					offset_in_page(xfer_start_addr),
438 					in_trans->size - resources->xferred_dma_size, GFP_KERNEL);
439 	if (ret) {
440 		ret = -ENOMEM;
441 		goto free_sgt;
442 	}
443 
444 	ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);
445 	if (ret)
446 		goto free_table;
447 
448 	xfer->sgt = sgt;
449 	xfer->page_list = page_list;
450 	xfer->nr_pages = nr_pages;
451 
452 	return need_pages > nr_pages ? 1 : 0;
453 
454 free_table:
455 	sg_free_table(sgt);
456 free_sgt:
457 	kfree(sgt);
458 put_pages:
459 	for (i = 0; i < nr_pages; ++i)
460 		put_page(page_list[i]);
461 free_page_list:
462 	kfree(page_list);
463 	return ret;
464 }
465 
466 /* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
467 static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
468 				  struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
469 				  struct wire_trans_dma_xfer **out_trans)
470 {
471 	struct wrapper_msg *trans_wrapper;
472 	struct sg_table *sgt = xfer->sgt;
473 	struct wire_addr_size_pair *asp;
474 	struct scatterlist *sg;
475 	struct wrapper_msg *w;
476 	unsigned int dma_len;
477 	u64 dma_chunk_len;
478 	void *boundary;
479 	int nents_dma;
480 	int nents;
481 	int i;
482 
483 	nents = sgt->nents;
484 	nents_dma = nents;
485 	*size = QAIC_MANAGE_EXT_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
486 	for_each_sgtable_sg(sgt, sg, i) {
487 		*size -= sizeof(*asp);
488 		/* Save 1K for possible follow-up transactions. */
489 		if (*size < SZ_1K) {
490 			nents_dma = i;
491 			break;
492 		}
493 	}
494 
495 	trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
496 	if (!trans_wrapper)
497 		return -ENOMEM;
498 	*out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
499 
500 	asp = (*out_trans)->data;
501 	boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
502 	*size = 0;
503 
504 	dma_len = 0;
505 	w = trans_wrapper;
506 	dma_chunk_len = 0;
507 	for_each_sg(sgt->sgl, sg, nents_dma, i) {
508 		asp->size = cpu_to_le64(dma_len);
509 		dma_chunk_len += dma_len;
510 		if (dma_len) {
511 			asp++;
512 			if ((void *)asp + sizeof(*asp) > boundary) {
513 				w->len = (void *)asp - (void *)&w->msg;
514 				*size += w->len;
515 				w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
516 				if (!w)
517 					return -ENOMEM;
518 				boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
519 				asp = (struct wire_addr_size_pair *)&w->msg;
520 			}
521 		}
522 		asp->addr = cpu_to_le64(sg_dma_address(sg));
523 		dma_len = sg_dma_len(sg);
524 	}
525 	/* finalize the last segment */
526 	asp->size = cpu_to_le64(dma_len);
527 	w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
528 	*size += w->len;
529 	dma_chunk_len += dma_len;
530 	resources->xferred_dma_size += dma_chunk_len;
531 
532 	return nents_dma < nents ? 1 : 0;
533 }
534 
535 static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
536 {
537 	int i;
538 
539 	dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
540 	sg_free_table(xfer->sgt);
541 	kfree(xfer->sgt);
542 	for (i = 0; i < xfer->nr_pages; ++i)
543 		put_page(xfer->page_list[i]);
544 	kfree(xfer->page_list);
545 }
546 
547 static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
548 		      u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
549 {
550 	struct qaic_manage_trans_dma_xfer *in_trans = trans;
551 	struct wire_trans_dma_xfer *out_trans;
552 	struct wrapper_msg *wrapper;
553 	struct dma_xfer *xfer;
554 	struct wire_msg *msg;
555 	bool need_cont_dma;
556 	u32 msg_hdr_len;
557 	u32 size;
558 	int ret;
559 
560 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
561 	msg = &wrapper->msg;
562 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
563 
564 	/* There should be enough space to hold at least one ASP entry. */
565 	if (size_add(msg_hdr_len, sizeof(*out_trans) + sizeof(struct wire_addr_size_pair)) >
566 	    QAIC_MANAGE_EXT_MSG_LENGTH)
567 		return -ENOMEM;
568 
569 	if (in_trans->addr + in_trans->size < in_trans->addr || !in_trans->size)
570 		return -EINVAL;
571 
572 	xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
573 	if (!xfer)
574 		return -ENOMEM;
575 
576 	ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
577 	if (ret < 0)
578 		goto free_xfer;
579 
580 	need_cont_dma = (bool)ret;
581 
582 	ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);
583 	if (ret < 0)
584 		goto cleanup_xfer;
585 
586 	need_cont_dma = need_cont_dma || (bool)ret;
587 
588 	msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
589 	msg->hdr.count = incr_le32(msg->hdr.count);
590 
591 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
592 	out_trans->hdr.len = cpu_to_le32(size);
593 	out_trans->tag = cpu_to_le32(in_trans->tag);
594 	out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
595 								sizeof(struct wire_addr_size_pair));
596 
597 	*user_len += in_trans->hdr.len;
598 
599 	if (resources->dma_chunk_id) {
600 		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
601 	} else if (need_cont_dma) {
602 		while (resources->dma_chunk_id == 0)
603 			resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);
604 
605 		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
606 	}
607 	resources->trans_hdr = trans;
608 
609 	list_add(&xfer->list, &resources->dma_xfers);
610 	return 0;
611 
612 cleanup_xfer:
613 	cleanup_xfer(qdev, xfer);
614 free_xfer:
615 	kfree(xfer);
616 	return ret;
617 }
618 
619 static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
620 			   u32 *user_len, struct ioctl_resources *resources)
621 {
622 	struct qaic_manage_trans_activate_to_dev *in_trans = trans;
623 	struct wire_trans_activate_to_dev *out_trans;
624 	struct wrapper_msg *trans_wrapper;
625 	struct wrapper_msg *wrapper;
626 	struct wire_msg *msg;
627 	dma_addr_t dma_addr;
628 	u32 msg_hdr_len;
629 	void *buf;
630 	u32 nelem;
631 	u32 size;
632 	int ret;
633 
634 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
635 	msg = &wrapper->msg;
636 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
637 
638 	if (size_add(msg_hdr_len, sizeof(*out_trans)) > QAIC_MANAGE_MAX_MSG_LENGTH)
639 		return -ENOSPC;
640 
641 	if (!in_trans->queue_size)
642 		return -EINVAL;
643 
644 	if (in_trans->pad)
645 		return -EINVAL;
646 
647 	nelem = in_trans->queue_size;
648 	size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;
649 	if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())
650 		return -EINVAL;
651 
652 	if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
653 		return -EINVAL;
654 
655 	size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
656 
657 	buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
658 	if (!buf)
659 		return -ENOMEM;
660 
661 	trans_wrapper = add_wrapper(wrappers,
662 				    offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
663 	if (!trans_wrapper) {
664 		ret = -ENOMEM;
665 		goto free_dma;
666 	}
667 	trans_wrapper->len = sizeof(*out_trans);
668 	out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
669 
670 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
671 	out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
672 	out_trans->buf_len = cpu_to_le32(size);
673 	out_trans->req_q_addr = cpu_to_le64(dma_addr);
674 	out_trans->req_q_size = cpu_to_le32(nelem);
675 	out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
676 	out_trans->rsp_q_size = cpu_to_le32(nelem);
677 	out_trans->options = cpu_to_le32(in_trans->options);
678 
679 	*user_len += in_trans->hdr.len;
680 	msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
681 	msg->hdr.count = incr_le32(msg->hdr.count);
682 
683 	resources->buf = buf;
684 	resources->dma_addr = dma_addr;
685 	resources->total_size = size;
686 	resources->nelem = nelem;
687 	resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
688 	return 0;
689 
690 free_dma:
691 	dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);
692 	return ret;
693 }
694 
695 static int encode_deactivate(struct qaic_device *qdev, void *trans,
696 			     u32 *user_len, struct qaic_user *usr)
697 {
698 	struct qaic_manage_trans_deactivate *in_trans = trans;
699 
700 	if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
701 		return -EINVAL;
702 
703 	*user_len += in_trans->hdr.len;
704 
705 	return disable_dbc(qdev, in_trans->dbc_id, usr);
706 }
707 
708 static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
709 			 u32 *user_len)
710 {
711 	struct qaic_manage_trans_status_to_dev *in_trans = trans;
712 	struct wire_trans_status_to_dev *out_trans;
713 	struct wrapper_msg *trans_wrapper;
714 	struct wrapper_msg *wrapper;
715 	struct wire_msg *msg;
716 	u32 msg_hdr_len;
717 
718 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
719 	msg = &wrapper->msg;
720 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
721 
722 	if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_MAX_MSG_LENGTH)
723 		return -ENOSPC;
724 
725 	trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));
726 	if (!trans_wrapper)
727 		return -ENOMEM;
728 
729 	trans_wrapper->len = sizeof(*out_trans);
730 	out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
731 
732 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
733 	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
734 	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
735 	msg->hdr.count = incr_le32(msg->hdr.count);
736 	*user_len += in_trans->hdr.len;
737 
738 	return 0;
739 }
740 
741 static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
742 			  struct wrapper_list *wrappers, struct ioctl_resources *resources,
743 			  struct qaic_user *usr)
744 {
745 	struct qaic_manage_trans_hdr *trans_hdr;
746 	struct wrapper_msg *wrapper;
747 	struct wire_msg *msg;
748 	u32 user_len = 0;
749 	int ret;
750 	int i;
751 
752 	if (!user_msg->count ||
753 	    user_msg->len < sizeof(*trans_hdr)) {
754 		ret = -EINVAL;
755 		goto out;
756 	}
757 
758 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
759 	msg = &wrapper->msg;
760 
761 	msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
762 
763 	if (resources->dma_chunk_id) {
764 		ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);
765 		msg->hdr.count = cpu_to_le32(1);
766 		goto out;
767 	}
768 
769 	for (i = 0; i < user_msg->count; ++i) {
770 		if (user_len > user_msg->len - sizeof(*trans_hdr)) {
771 			ret = -EINVAL;
772 			break;
773 		}
774 		trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
775 		if (trans_hdr->len < sizeof(trans_hdr) ||
776 		    size_add(user_len, trans_hdr->len) > user_msg->len) {
777 			ret = -EINVAL;
778 			break;
779 		}
780 
781 		switch (trans_hdr->type) {
782 		case QAIC_TRANS_PASSTHROUGH_FROM_USR:
783 			ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);
784 			break;
785 		case QAIC_TRANS_DMA_XFER_FROM_USR:
786 			ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);
787 			break;
788 		case QAIC_TRANS_ACTIVATE_FROM_USR:
789 			ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);
790 			break;
791 		case QAIC_TRANS_DEACTIVATE_FROM_USR:
792 			ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);
793 			break;
794 		case QAIC_TRANS_STATUS_FROM_USR:
795 			ret = encode_status(qdev, trans_hdr, wrappers, &user_len);
796 			break;
797 		default:
798 			ret = -EINVAL;
799 			break;
800 		}
801 
802 		if (ret)
803 			break;
804 	}
805 
806 	if (user_len != user_msg->len)
807 		ret = -EINVAL;
808 out:
809 	if (ret) {
810 		free_dma_xfers(qdev, resources);
811 		free_dbc_buf(qdev, resources);
812 		return ret;
813 	}
814 
815 	return 0;
816 }
817 
818 static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
819 			      u32 *msg_len)
820 {
821 	struct qaic_manage_trans_passthrough *out_trans;
822 	struct wire_trans_passthrough *in_trans = trans;
823 	u32 len;
824 
825 	out_trans = (void *)user_msg->data + user_msg->len;
826 
827 	len = le32_to_cpu(in_trans->hdr.len);
828 	if (len % 8 != 0)
829 		return -EINVAL;
830 
831 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
832 		return -ENOSPC;
833 
834 	memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
835 	user_msg->len += len;
836 	*msg_len += len;
837 	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
838 	out_trans->hdr.len = len;
839 
840 	return 0;
841 }
842 
843 static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
844 			   u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
845 {
846 	struct qaic_manage_trans_activate_from_dev *out_trans;
847 	struct wire_trans_activate_from_dev *in_trans = trans;
848 	u32 len;
849 
850 	out_trans = (void *)user_msg->data + user_msg->len;
851 
852 	len = le32_to_cpu(in_trans->hdr.len);
853 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
854 		return -ENOSPC;
855 
856 	user_msg->len += len;
857 	*msg_len += len;
858 	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
859 	out_trans->hdr.len = len;
860 	out_trans->status = le32_to_cpu(in_trans->status);
861 	out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
862 	out_trans->options = le64_to_cpu(in_trans->options);
863 
864 	if (!resources->buf)
865 		/* how did we get an activate response without a request? */
866 		return -EINVAL;
867 
868 	if (out_trans->dbc_id >= qdev->num_dbc)
869 		/*
870 		 * The device assigned an invalid resource, which should never
871 		 * happen. Return an error so the user can try to recover.
872 		 */
873 		return -ENODEV;
874 
875 	if (out_trans->status)
876 		/*
877 		 * Allocating resources failed on device side. This is not an
878 		 * expected behaviour, user is expected to handle this situation.
879 		 */
880 		return -ECANCELED;
881 
882 	resources->status = out_trans->status;
883 	resources->dbc_id = out_trans->dbc_id;
884 	save_dbc_buf(qdev, resources, usr);
885 
886 	return 0;
887 }
888 
889 static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
890 			     struct qaic_user *usr)
891 {
892 	struct wire_trans_deactivate_from_dev *in_trans = trans;
893 	u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
894 	u32 status = le32_to_cpu(in_trans->status);
895 
896 	if (dbc_id >= qdev->num_dbc)
897 		/*
898 		 * The device assigned an invalid resource, which should never
899 		 * happen. Inject an error so the user can try to recover.
900 		 */
901 		return -ENODEV;
902 
903 	if (status) {
904 		/*
905 		 * Releasing resources failed on the device side, which puts
906 		 * us in a bind since they may still be in use, so enable the
907 		 * dbc. User is expected to retry deactivation.
908 		 */
909 		enable_dbc(qdev, dbc_id, usr);
910 		return -ECANCELED;
911 	}
912 
913 	release_dbc(qdev, dbc_id);
914 	*msg_len += sizeof(*in_trans);
915 
916 	return 0;
917 }
918 
919 static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
920 			 u32 *user_len, struct wire_msg *msg)
921 {
922 	struct qaic_manage_trans_status_from_dev *out_trans;
923 	struct wire_trans_status_from_dev *in_trans = trans;
924 	u32 len;
925 
926 	out_trans = (void *)user_msg->data + user_msg->len;
927 
928 	len = le32_to_cpu(in_trans->hdr.len);
929 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
930 		return -ENOSPC;
931 
932 	out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
933 	out_trans->hdr.len = len;
934 	out_trans->major = le16_to_cpu(in_trans->major);
935 	out_trans->minor = le16_to_cpu(in_trans->minor);
936 	out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
937 	out_trans->status = le32_to_cpu(in_trans->status);
938 	*user_len += le32_to_cpu(in_trans->hdr.len);
939 	user_msg->len += len;
940 
941 	if (out_trans->status)
942 		return -ECANCELED;
943 	if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
944 		return -EPIPE;
945 
946 	return 0;
947 }
948 
949 static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
950 			  struct wire_msg *msg, struct ioctl_resources *resources,
951 			  struct qaic_user *usr)
952 {
953 	u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
954 	struct wire_trans_hdr *trans_hdr;
955 	u32 msg_len = 0;
956 	int ret;
957 	int i;
958 
959 	if (msg_hdr_len < sizeof(*trans_hdr) ||
960 	    msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
961 		return -EINVAL;
962 
963 	user_msg->len = 0;
964 	user_msg->count = le32_to_cpu(msg->hdr.count);
965 
966 	for (i = 0; i < user_msg->count; ++i) {
967 		u32 hdr_len;
968 
969 		if (msg_len > msg_hdr_len - sizeof(*trans_hdr))
970 			return -EINVAL;
971 
972 		trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
973 		hdr_len = le32_to_cpu(trans_hdr->len);
974 		if (hdr_len < sizeof(*trans_hdr) ||
975 		    size_add(msg_len, hdr_len) > msg_hdr_len)
976 			return -EINVAL;
977 
978 		switch (le32_to_cpu(trans_hdr->type)) {
979 		case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
980 			ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);
981 			break;
982 		case QAIC_TRANS_ACTIVATE_FROM_DEV:
983 			ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);
984 			break;
985 		case QAIC_TRANS_DEACTIVATE_FROM_DEV:
986 			ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);
987 			break;
988 		case QAIC_TRANS_STATUS_FROM_DEV:
989 			ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);
990 			break;
991 		default:
992 			return -EINVAL;
993 		}
994 
995 		if (ret)
996 			return ret;
997 	}
998 
999 	if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
1000 		return -EINVAL;
1001 
1002 	return 0;
1003 }
1004 
1005 static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
1006 		      bool ignore_signal)
1007 {
1008 	struct xfer_queue_elem elem;
1009 	struct wire_msg *out_buf;
1010 	struct wrapper_msg *w;
1011 	long ret = -EAGAIN;
1012 	int xfer_count = 0;
1013 	int retry_count;
1014 
1015 	if (qdev->in_reset) {
1016 		mutex_unlock(&qdev->cntl_mutex);
1017 		return ERR_PTR(-ENODEV);
1018 	}
1019 
1020 	/* Attempt to avoid a partial commit of a message */
1021 	list_for_each_entry(w, &wrappers->list, list)
1022 		xfer_count++;
1023 
1024 	for (retry_count = 0; retry_count < QAIC_MHI_RETRY_MAX; retry_count++) {
1025 		if (xfer_count <= mhi_get_free_desc_count(qdev->cntl_ch, DMA_TO_DEVICE)) {
1026 			ret = 0;
1027 			break;
1028 		}
1029 		msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
1030 		if (signal_pending(current))
1031 			break;
1032 	}
1033 
1034 	if (ret) {
1035 		mutex_unlock(&qdev->cntl_mutex);
1036 		return ERR_PTR(ret);
1037 	}
1038 
1039 	elem.seq_num = seq_num;
1040 	elem.buf = NULL;
1041 	init_completion(&elem.xfer_done);
1042 	if (likely(!qdev->cntl_lost_buf)) {
1043 		/*
1044 		 * The max size of request to device is QAIC_MANAGE_EXT_MSG_LENGTH.
1045 		 * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
1046 		 */
1047 		out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
1048 		if (!out_buf) {
1049 			mutex_unlock(&qdev->cntl_mutex);
1050 			return ERR_PTR(-ENOMEM);
1051 		}
1052 
1053 		ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,
1054 				    QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);
1055 		if (ret) {
1056 			mutex_unlock(&qdev->cntl_mutex);
1057 			return ERR_PTR(ret);
1058 		}
1059 	} else {
1060 		/*
1061 		 * we lost a buffer because we queued a recv buf, but then
1062 		 * queuing the corresponding tx buf failed. To try to avoid
1063 		 * a memory leak, lets reclaim it and use it for this
1064 		 * transaction.
1065 		 */
1066 		qdev->cntl_lost_buf = false;
1067 	}
1068 
1069 	list_for_each_entry(w, &wrappers->list, list) {
1070 		kref_get(&w->ref_count);
1071 		retry_count = 0;
1072 		ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,
1073 				    list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);
1074 		if (ret) {
1075 			qdev->cntl_lost_buf = true;
1076 			kref_put(&w->ref_count, free_wrapper);
1077 			mutex_unlock(&qdev->cntl_mutex);
1078 			return ERR_PTR(ret);
1079 		}
1080 	}
1081 
1082 	list_add_tail(&elem.list, &qdev->cntl_xfer_list);
1083 	mutex_unlock(&qdev->cntl_mutex);
1084 
1085 	if (ignore_signal)
1086 		ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);
1087 	else
1088 		ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
1089 								control_resp_timeout_s * HZ);
1090 	/*
1091 	 * not using _interruptable because we have to cleanup or we'll
1092 	 * likely cause memory corruption
1093 	 */
1094 	mutex_lock(&qdev->cntl_mutex);
1095 	if (!list_empty(&elem.list))
1096 		list_del(&elem.list);
1097 	if (!ret && !elem.buf)
1098 		ret = -ETIMEDOUT;
1099 	else if (ret > 0 && !elem.buf)
1100 		ret = -EIO;
1101 	mutex_unlock(&qdev->cntl_mutex);
1102 
1103 	if (ret < 0) {
1104 		kfree(elem.buf);
1105 		return ERR_PTR(ret);
1106 	} else if (!qdev->valid_crc(elem.buf)) {
1107 		kfree(elem.buf);
1108 		return ERR_PTR(-EPIPE);
1109 	}
1110 
1111 	return elem.buf;
1112 }
1113 
1114 /* Add a transaction to abort the outstanding DMA continuation */
1115 static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
1116 {
1117 	struct wire_trans_dma_xfer *out_trans;
1118 	u32 size = sizeof(*out_trans);
1119 	struct wrapper_msg *wrapper;
1120 	struct wrapper_msg *w;
1121 	struct wire_msg *msg;
1122 
1123 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
1124 	msg = &wrapper->msg;
1125 
1126 	/* Remove all but the first wrapper which has the msg header */
1127 	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1128 		if (!list_is_first(&wrapper->list, &wrappers->list))
1129 			kref_put(&wrapper->ref_count, free_wrapper);
1130 
1131 	wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
1132 
1133 	if (!wrapper)
1134 		return -ENOMEM;
1135 
1136 	out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
1137 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
1138 	out_trans->hdr.len = cpu_to_le32(size);
1139 	out_trans->tag = cpu_to_le32(0);
1140 	out_trans->count = cpu_to_le32(0);
1141 	out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
1142 
1143 	msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
1144 	msg->hdr.count = cpu_to_le32(1);
1145 	wrapper->len = size;
1146 
1147 	return 0;
1148 }
1149 
1150 static struct wrapper_list *alloc_wrapper_list(void)
1151 {
1152 	struct wrapper_list *wrappers;
1153 
1154 	wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
1155 	if (!wrappers)
1156 		return NULL;
1157 	INIT_LIST_HEAD(&wrappers->list);
1158 	spin_lock_init(&wrappers->lock);
1159 
1160 	return wrappers;
1161 }
1162 
1163 static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
1164 				struct manage_msg *user_msg, struct ioctl_resources *resources,
1165 				struct wire_msg **rsp)
1166 {
1167 	struct wrapper_list *wrappers;
1168 	struct wrapper_msg *wrapper;
1169 	struct wrapper_msg *w;
1170 	bool all_done = false;
1171 	struct wire_msg *msg;
1172 	int ret;
1173 
1174 	wrappers = alloc_wrapper_list();
1175 	if (!wrappers)
1176 		return -ENOMEM;
1177 
1178 	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
1179 	if (!wrapper) {
1180 		kfree(wrappers);
1181 		return -ENOMEM;
1182 	}
1183 
1184 	msg = &wrapper->msg;
1185 	wrapper->len = sizeof(*msg);
1186 
1187 	ret = encode_message(qdev, user_msg, wrappers, resources, usr);
1188 	if (ret && resources->dma_chunk_id)
1189 		ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);
1190 	if (ret)
1191 		goto encode_failed;
1192 
1193 	ret = mutex_lock_interruptible(&qdev->cntl_mutex);
1194 	if (ret)
1195 		goto lock_failed;
1196 
1197 	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1198 	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1199 
1200 	if (usr) {
1201 		msg->hdr.handle = cpu_to_le32(usr->handle);
1202 		msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
1203 	} else {
1204 		msg->hdr.handle = 0;
1205 		msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
1206 	}
1207 
1208 	msg->hdr.padding = cpu_to_le32(0);
1209 	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1210 
1211 	/* msg_xfer releases the mutex */
1212 	*rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);
1213 	if (IS_ERR(*rsp))
1214 		ret = PTR_ERR(*rsp);
1215 
1216 lock_failed:
1217 	free_dma_xfers(qdev, resources);
1218 encode_failed:
1219 	spin_lock(&wrappers->lock);
1220 	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1221 		kref_put(&wrapper->ref_count, free_wrapper);
1222 	all_done = list_empty(&wrappers->list);
1223 	spin_unlock(&wrappers->lock);
1224 	if (all_done)
1225 		kfree(wrappers);
1226 
1227 	return ret;
1228 }
1229 
1230 static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
1231 {
1232 	struct wire_trans_dma_xfer_cont *dma_cont = NULL;
1233 	struct ioctl_resources resources;
1234 	struct wire_msg *rsp = NULL;
1235 	int ret;
1236 
1237 	memset(&resources, 0, sizeof(struct ioctl_resources));
1238 
1239 	INIT_LIST_HEAD(&resources.dma_xfers);
1240 
1241 	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
1242 	    user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
1243 		return -EINVAL;
1244 
1245 dma_xfer_continue:
1246 	ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);
1247 	if (ret)
1248 		return ret;
1249 	/* dma_cont should be the only transaction if present */
1250 	if (le32_to_cpu(rsp->hdr.count) == 1) {
1251 		dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
1252 		if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
1253 			dma_cont = NULL;
1254 	}
1255 	if (dma_cont) {
1256 		if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
1257 		    le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
1258 			kfree(rsp);
1259 			goto dma_xfer_continue;
1260 		}
1261 
1262 		ret = -EINVAL;
1263 		goto dma_cont_failed;
1264 	}
1265 
1266 	ret = decode_message(qdev, user_msg, rsp, &resources, usr);
1267 
1268 dma_cont_failed:
1269 	free_dbc_buf(qdev, &resources);
1270 	kfree(rsp);
1271 	return ret;
1272 }
1273 
1274 int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1275 {
1276 	struct qaic_manage_msg *user_msg = data;
1277 	struct qaic_device *qdev;
1278 	struct manage_msg *msg;
1279 	struct qaic_user *usr;
1280 	u8 __user *user_data;
1281 	int qdev_rcu_id;
1282 	int usr_rcu_id;
1283 	int ret;
1284 
1285 	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH)
1286 		return -EINVAL;
1287 
1288 	usr = file_priv->driver_priv;
1289 
1290 	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1291 	if (!usr->qddev) {
1292 		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1293 		return -ENODEV;
1294 	}
1295 
1296 	qdev = usr->qddev->qdev;
1297 
1298 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1299 	if (qdev->in_reset) {
1300 		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1301 		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1302 		return -ENODEV;
1303 	}
1304 
1305 	msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
1306 	if (!msg) {
1307 		ret = -ENOMEM;
1308 		goto out;
1309 	}
1310 
1311 	msg->len = user_msg->len;
1312 	msg->count = user_msg->count;
1313 
1314 	user_data = u64_to_user_ptr(user_msg->data);
1315 
1316 	if (copy_from_user(msg->data, user_data, user_msg->len)) {
1317 		ret = -EFAULT;
1318 		goto free_msg;
1319 	}
1320 
1321 	ret = qaic_manage(qdev, usr, msg);
1322 
1323 	/*
1324 	 * If the qaic_manage() is successful then we copy the message onto
1325 	 * userspace memory but we have an exception for -ECANCELED.
1326 	 * For -ECANCELED, it means that device has NACKed the message with a
1327 	 * status error code which userspace would like to know.
1328 	 */
1329 	if (ret == -ECANCELED || !ret) {
1330 		if (copy_to_user(user_data, msg->data, msg->len)) {
1331 			ret = -EFAULT;
1332 		} else {
1333 			user_msg->len = msg->len;
1334 			user_msg->count = msg->count;
1335 		}
1336 	}
1337 
1338 free_msg:
1339 	kfree(msg);
1340 out:
1341 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1342 	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1343 	return ret;
1344 }
1345 
1346 int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
1347 {
1348 	struct qaic_manage_trans_status_from_dev *status_result;
1349 	struct qaic_manage_trans_status_to_dev *status_query;
1350 	struct manage_msg *user_msg;
1351 	int ret;
1352 
1353 	user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
1354 	if (!user_msg) {
1355 		ret = -ENOMEM;
1356 		goto out;
1357 	}
1358 	user_msg->len = sizeof(*status_query);
1359 	user_msg->count = 1;
1360 
1361 	status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
1362 	status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
1363 	status_query->hdr.len = sizeof(status_query->hdr);
1364 
1365 	ret = qaic_manage(qdev, usr, user_msg);
1366 	if (ret)
1367 		goto kfree_user_msg;
1368 	status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
1369 	*major = status_result->major;
1370 	*minor = status_result->minor;
1371 
1372 	if (status_result->status_flags & BIT(0)) { /* device is using CRC */
1373 		/* By default qdev->gen_crc is programmed to generate CRC */
1374 		qdev->valid_crc = valid_crc;
1375 	} else {
1376 		/* By default qdev->valid_crc is programmed to bypass CRC */
1377 		qdev->gen_crc = gen_crc_stub;
1378 	}
1379 
1380 kfree_user_msg:
1381 	kfree(user_msg);
1382 out:
1383 	return ret;
1384 }
1385 
1386 static void resp_worker(struct work_struct *work)
1387 {
1388 	struct resp_work *resp = container_of(work, struct resp_work, work);
1389 	struct qaic_device *qdev = resp->qdev;
1390 	struct wire_msg *msg = resp->buf;
1391 	struct xfer_queue_elem *elem;
1392 	struct xfer_queue_elem *i;
1393 	bool found = false;
1394 
1395 	mutex_lock(&qdev->cntl_mutex);
1396 	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1397 		if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
1398 			found = true;
1399 			list_del_init(&elem->list);
1400 			elem->buf = msg;
1401 			complete_all(&elem->xfer_done);
1402 			break;
1403 		}
1404 	}
1405 	mutex_unlock(&qdev->cntl_mutex);
1406 
1407 	if (!found)
1408 		/* request must have timed out, drop packet */
1409 		kfree(msg);
1410 
1411 	kfree(resp);
1412 }
1413 
1414 static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
1415 {
1416 	bool all_done = false;
1417 
1418 	spin_lock(&wrappers->lock);
1419 	kref_put(&wrapper->ref_count, free_wrapper);
1420 	all_done = list_empty(&wrappers->list);
1421 	spin_unlock(&wrappers->lock);
1422 
1423 	if (all_done)
1424 		kfree(wrappers);
1425 }
1426 
1427 void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1428 {
1429 	struct wire_msg *msg = mhi_result->buf_addr;
1430 	struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
1431 
1432 	free_wrapper_from_list(wrapper->head, wrapper);
1433 }
1434 
1435 void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1436 {
1437 	struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
1438 	struct wire_msg *msg = mhi_result->buf_addr;
1439 	struct resp_work *resp;
1440 
1441 	if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
1442 		kfree(msg);
1443 		return;
1444 	}
1445 
1446 	resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
1447 	if (!resp) {
1448 		kfree(msg);
1449 		return;
1450 	}
1451 
1452 	INIT_WORK(&resp->work, resp_worker);
1453 	resp->qdev = qdev;
1454 	resp->buf = msg;
1455 	queue_work(qdev->cntl_wq, &resp->work);
1456 }
1457 
1458 int qaic_control_open(struct qaic_device *qdev)
1459 {
1460 	if (!qdev->cntl_ch)
1461 		return -ENODEV;
1462 
1463 	qdev->cntl_lost_buf = false;
1464 	/*
1465 	 * By default qaic should assume that device has CRC enabled.
1466 	 * Qaic comes to know if device has CRC enabled or disabled during the
1467 	 * device status transaction, which is the first transaction performed
1468 	 * on control channel.
1469 	 *
1470 	 * So CRC validation of first device status transaction response is
1471 	 * ignored (by calling valid_crc_stub) and is done later during decoding
1472 	 * if device has CRC enabled.
1473 	 * Now that qaic knows whether device has CRC enabled or not it acts
1474 	 * accordingly.
1475 	 */
1476 	qdev->gen_crc = gen_crc;
1477 	qdev->valid_crc = valid_crc_stub;
1478 
1479 	return mhi_prepare_for_transfer(qdev->cntl_ch);
1480 }
1481 
1482 void qaic_control_close(struct qaic_device *qdev)
1483 {
1484 	mhi_unprepare_from_transfer(qdev->cntl_ch);
1485 }
1486 
1487 void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
1488 {
1489 	struct wire_trans_terminate_to_dev *trans;
1490 	struct wrapper_list *wrappers;
1491 	struct wrapper_msg *wrapper;
1492 	struct wire_msg *msg;
1493 	struct wire_msg *rsp;
1494 
1495 	wrappers = alloc_wrapper_list();
1496 	if (!wrappers)
1497 		return;
1498 
1499 	wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
1500 	if (!wrapper)
1501 		return;
1502 
1503 	msg = &wrapper->msg;
1504 
1505 	trans = (struct wire_trans_terminate_to_dev *)msg->data;
1506 
1507 	trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
1508 	trans->hdr.len = cpu_to_le32(sizeof(*trans));
1509 	trans->handle = cpu_to_le32(usr->handle);
1510 
1511 	mutex_lock(&qdev->cntl_mutex);
1512 	wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
1513 	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1514 	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1515 	msg->hdr.len = cpu_to_le32(wrapper->len);
1516 	msg->hdr.count = cpu_to_le32(1);
1517 	msg->hdr.handle = cpu_to_le32(usr->handle);
1518 	msg->hdr.padding = cpu_to_le32(0);
1519 	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1520 
1521 	/*
1522 	 * msg_xfer releases the mutex
1523 	 * We don't care about the return of msg_xfer since we will not do
1524 	 * anything different based on what happens.
1525 	 * We ignore pending signals since one will be set if the user is
1526 	 * killed, and we need give the device a chance to cleanup, otherwise
1527 	 * DMA may still be in progress when we return.
1528 	 */
1529 	rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);
1530 	if (!IS_ERR(rsp))
1531 		kfree(rsp);
1532 	free_wrapper_from_list(wrappers, wrapper);
1533 }
1534 
1535 void wake_all_cntl(struct qaic_device *qdev)
1536 {
1537 	struct xfer_queue_elem *elem;
1538 	struct xfer_queue_elem *i;
1539 
1540 	mutex_lock(&qdev->cntl_mutex);
1541 	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1542 		list_del_init(&elem->list);
1543 		complete_all(&elem->xfer_done);
1544 	}
1545 	mutex_unlock(&qdev->cntl_mutex);
1546 }
1547