1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5 
6 #include <asm/byteorder.h>
7 #include <linux/completion.h>
8 #include <linux/crc32.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/kref.h>
12 #include <linux/list.h>
13 #include <linux/mhi.h>
14 #include <linux/mm.h>
15 #include <linux/moduleparam.h>
16 #include <linux/mutex.h>
17 #include <linux/pci.h>
18 #include <linux/scatterlist.h>
19 #include <linux/types.h>
20 #include <linux/uaccess.h>
21 #include <linux/workqueue.h>
22 #include <linux/wait.h>
23 #include <drm/drm_device.h>
24 #include <drm/drm_file.h>
25 #include <uapi/drm/qaic_accel.h>
26 
27 #include "qaic.h"
28 
29 #define MANAGE_MAGIC_NUMBER		((__force __le32)0x43494151) /* "QAIC" in little endian */
30 #define QAIC_DBC_Q_GAP			SZ_256
31 #define QAIC_DBC_Q_BUF_ALIGN		SZ_4K
32 #define QAIC_MANAGE_EXT_MSG_LENGTH	SZ_64K /* Max DMA message length */
33 #define QAIC_WRAPPER_MAX_SIZE		SZ_4K
34 #define QAIC_MHI_RETRY_WAIT_MS		100
35 #define QAIC_MHI_RETRY_MAX		20
36 
37 static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
38 module_param(control_resp_timeout_s, uint, 0600);
39 MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
40 
41 struct manage_msg {
42 	u32 len;
43 	u32 count;
44 	u8 data[];
45 };
46 
47 /*
48  * wire encoding structures for the manage protocol.
49  * All fields are little endian on the wire
50  */
51 struct wire_msg_hdr {
52 	__le32 crc32; /* crc of everything following this field in the message */
53 	__le32 magic_number;
54 	__le32 sequence_number;
55 	__le32 len; /* length of this message */
56 	__le32 count; /* number of transactions in this message */
57 	__le32 handle; /* unique id to track the resources consumed */
58 	__le32 partition_id; /* partition id for the request (signed) */
59 	__le32 padding; /* must be 0 */
60 } __packed;
61 
62 struct wire_msg {
63 	struct wire_msg_hdr hdr;
64 	u8 data[];
65 } __packed;
66 
67 struct wire_trans_hdr {
68 	__le32 type;
69 	__le32 len;
70 } __packed;
71 
72 /* Each message sent from driver to device are organized in a list of wrapper_msg */
73 struct wrapper_msg {
74 	struct list_head list;
75 	struct kref ref_count;
76 	u32 len; /* length of data to transfer */
77 	struct wrapper_list *head;
78 	union {
79 		struct wire_msg msg;
80 		struct wire_trans_hdr trans;
81 	};
82 };
83 
84 struct wrapper_list {
85 	struct list_head list;
86 	spinlock_t lock; /* Protects the list state during additions and removals */
87 };
88 
89 struct wire_trans_passthrough {
90 	struct wire_trans_hdr hdr;
91 	u8 data[];
92 } __packed;
93 
94 struct wire_addr_size_pair {
95 	__le64 addr;
96 	__le64 size;
97 } __packed;
98 
99 struct wire_trans_dma_xfer {
100 	struct wire_trans_hdr hdr;
101 	__le32 tag;
102 	__le32 count;
103 	__le32 dma_chunk_id;
104 	__le32 padding;
105 	struct wire_addr_size_pair data[];
106 } __packed;
107 
108 /* Initiated by device to continue the DMA xfer of a large piece of data */
109 struct wire_trans_dma_xfer_cont {
110 	struct wire_trans_hdr hdr;
111 	__le32 dma_chunk_id;
112 	__le32 padding;
113 	__le64 xferred_size;
114 } __packed;
115 
116 struct wire_trans_activate_to_dev {
117 	struct wire_trans_hdr hdr;
118 	__le64 req_q_addr;
119 	__le64 rsp_q_addr;
120 	__le32 req_q_size;
121 	__le32 rsp_q_size;
122 	__le32 buf_len;
123 	__le32 options; /* unused, but BIT(16) has meaning to the device */
124 } __packed;
125 
126 struct wire_trans_activate_from_dev {
127 	struct wire_trans_hdr hdr;
128 	__le32 status;
129 	__le32 dbc_id;
130 	__le64 options; /* unused */
131 } __packed;
132 
133 struct wire_trans_deactivate_from_dev {
134 	struct wire_trans_hdr hdr;
135 	__le32 status;
136 	__le32 dbc_id;
137 } __packed;
138 
139 struct wire_trans_terminate_to_dev {
140 	struct wire_trans_hdr hdr;
141 	__le32 handle;
142 	__le32 padding;
143 } __packed;
144 
145 struct wire_trans_terminate_from_dev {
146 	struct wire_trans_hdr hdr;
147 	__le32 status;
148 	__le32 padding;
149 } __packed;
150 
151 struct wire_trans_status_to_dev {
152 	struct wire_trans_hdr hdr;
153 } __packed;
154 
155 struct wire_trans_status_from_dev {
156 	struct wire_trans_hdr hdr;
157 	__le16 major;
158 	__le16 minor;
159 	__le32 status;
160 	__le64 status_flags;
161 } __packed;
162 
163 struct wire_trans_validate_part_to_dev {
164 	struct wire_trans_hdr hdr;
165 	__le32 part_id;
166 	__le32 padding;
167 } __packed;
168 
169 struct wire_trans_validate_part_from_dev {
170 	struct wire_trans_hdr hdr;
171 	__le32 status;
172 	__le32 padding;
173 } __packed;
174 
175 struct xfer_queue_elem {
176 	/*
177 	 * Node in list of ongoing transfer request on control channel.
178 	 * Maintained by root device struct.
179 	 */
180 	struct list_head list;
181 	/* Sequence number of this transfer request */
182 	u32 seq_num;
183 	/* This is used to wait on until completion of transfer request */
184 	struct completion xfer_done;
185 	/* Received data from device */
186 	void *buf;
187 };
188 
189 struct dma_xfer {
190 	/* Node in list of DMA transfers which is used for cleanup */
191 	struct list_head list;
192 	/* SG table of memory used for DMA */
193 	struct sg_table *sgt;
194 	/* Array pages used for DMA */
195 	struct page **page_list;
196 	/* Number of pages used for DMA */
197 	unsigned long nr_pages;
198 };
199 
200 struct ioctl_resources {
201 	/* List of all DMA transfers which is used later for cleanup */
202 	struct list_head dma_xfers;
203 	/* Base address of request queue which belongs to a DBC */
204 	void *buf;
205 	/*
206 	 * Base bus address of request queue which belongs to a DBC. Response
207 	 * queue base bus address can be calculated by adding size of request
208 	 * queue to base bus address of request queue.
209 	 */
210 	dma_addr_t dma_addr;
211 	/* Total size of request queue and response queue in byte */
212 	u32 total_size;
213 	/* Total number of elements that can be queued in each of request and response queue */
214 	u32 nelem;
215 	/* Base address of response queue which belongs to a DBC */
216 	void *rsp_q_base;
217 	/* Status of the NNC message received */
218 	u32 status;
219 	/* DBC id of the DBC received from device */
220 	u32 dbc_id;
221 	/*
222 	 * DMA transfer request messages can be big in size and it may not be
223 	 * possible to send them in one shot. In such cases the messages are
224 	 * broken into chunks, this field stores ID of such chunks.
225 	 */
226 	u32 dma_chunk_id;
227 	/* Total number of bytes transferred for a DMA xfer request */
228 	u64 xferred_dma_size;
229 	/* Header of transaction message received from user. Used during DMA xfer request. */
230 	void *trans_hdr;
231 };
232 
233 struct resp_work {
234 	struct work_struct work;
235 	struct qaic_device *qdev;
236 	void *buf;
237 };
238 
239 /*
240  * Since we're working with little endian messages, its useful to be able to
241  * increment without filling a whole line with conversions back and forth just
242  * to add one(1) to a message count.
243  */
244 static __le32 incr_le32(__le32 val)
245 {
246 	return cpu_to_le32(le32_to_cpu(val) + 1);
247 }
248 
249 static u32 gen_crc(void *msg)
250 {
251 	struct wrapper_list *wrappers = msg;
252 	struct wrapper_msg *w;
253 	u32 crc = ~0;
254 
255 	list_for_each_entry(w, &wrappers->list, list)
256 		crc = crc32(crc, &w->msg, w->len);
257 
258 	return crc ^ ~0;
259 }
260 
261 static u32 gen_crc_stub(void *msg)
262 {
263 	return 0;
264 }
265 
266 static bool valid_crc(void *msg)
267 {
268 	struct wire_msg_hdr *hdr = msg;
269 	bool ret;
270 	u32 crc;
271 
272 	/*
273 	 * The output of this algorithm is always converted to the native
274 	 * endianness.
275 	 */
276 	crc = le32_to_cpu(hdr->crc32);
277 	hdr->crc32 = 0;
278 	ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
279 	hdr->crc32 = cpu_to_le32(crc);
280 	return ret;
281 }
282 
283 static bool valid_crc_stub(void *msg)
284 {
285 	return true;
286 }
287 
288 static void free_wrapper(struct kref *ref)
289 {
290 	struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
291 
292 	list_del(&wrapper->list);
293 	kfree(wrapper);
294 }
295 
296 static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
297 			 struct qaic_user *usr)
298 {
299 	u32 dbc_id = resources->dbc_id;
300 
301 	if (resources->buf) {
302 		wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
303 		qdev->dbc[dbc_id].req_q_base = resources->buf;
304 		qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
305 		qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
306 		qdev->dbc[dbc_id].total_size = resources->total_size;
307 		qdev->dbc[dbc_id].nelem = resources->nelem;
308 		enable_dbc(qdev, dbc_id, usr);
309 		qdev->dbc[dbc_id].in_use = true;
310 		resources->buf = NULL;
311 	}
312 }
313 
314 static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
315 {
316 	if (resources->buf)
317 		dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,
318 				  resources->dma_addr);
319 	resources->buf = NULL;
320 }
321 
322 static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
323 {
324 	struct dma_xfer *xfer;
325 	struct dma_xfer *x;
326 	int i;
327 
328 	list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
329 		dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
330 		sg_free_table(xfer->sgt);
331 		kfree(xfer->sgt);
332 		for (i = 0; i < xfer->nr_pages; ++i)
333 			put_page(xfer->page_list[i]);
334 		kfree(xfer->page_list);
335 		list_del(&xfer->list);
336 		kfree(xfer);
337 	}
338 }
339 
340 static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
341 {
342 	struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
343 
344 	if (!w)
345 		return NULL;
346 	list_add_tail(&w->list, &wrappers->list);
347 	kref_init(&w->ref_count);
348 	w->head = wrappers;
349 	return w;
350 }
351 
352 static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
353 			      u32 *user_len)
354 {
355 	struct qaic_manage_trans_passthrough *in_trans = trans;
356 	struct wire_trans_passthrough *out_trans;
357 	struct wrapper_msg *trans_wrapper;
358 	struct wrapper_msg *wrapper;
359 	struct wire_msg *msg;
360 	u32 msg_hdr_len;
361 
362 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
363 	msg = &wrapper->msg;
364 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
365 
366 	if (in_trans->hdr.len % 8 != 0)
367 		return -EINVAL;
368 
369 	if (msg_hdr_len + in_trans->hdr.len > QAIC_MANAGE_EXT_MSG_LENGTH)
370 		return -ENOSPC;
371 
372 	trans_wrapper = add_wrapper(wrappers,
373 				    offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
374 	if (!trans_wrapper)
375 		return -ENOMEM;
376 	trans_wrapper->len = in_trans->hdr.len;
377 	out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
378 
379 	memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
380 	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
381 	msg->hdr.count = incr_le32(msg->hdr.count);
382 	*user_len += in_trans->hdr.len;
383 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
384 	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
385 
386 	return 0;
387 }
388 
389 /* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
390 static int find_and_map_user_pages(struct qaic_device *qdev,
391 				   struct qaic_manage_trans_dma_xfer *in_trans,
392 				   struct ioctl_resources *resources, struct dma_xfer *xfer)
393 {
394 	unsigned long need_pages;
395 	struct page **page_list;
396 	unsigned long nr_pages;
397 	struct sg_table *sgt;
398 	u64 xfer_start_addr;
399 	int ret;
400 	int i;
401 
402 	xfer_start_addr = in_trans->addr + resources->xferred_dma_size;
403 
404 	need_pages = DIV_ROUND_UP(in_trans->size + offset_in_page(xfer_start_addr) -
405 				  resources->xferred_dma_size, PAGE_SIZE);
406 
407 	nr_pages = need_pages;
408 
409 	while (1) {
410 		page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
411 		if (!page_list) {
412 			nr_pages = nr_pages / 2;
413 			if (!nr_pages)
414 				return -ENOMEM;
415 		} else {
416 			break;
417 		}
418 	}
419 
420 	ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);
421 	if (ret < 0 || ret != nr_pages) {
422 		ret = -EFAULT;
423 		goto free_page_list;
424 	}
425 
426 	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
427 	if (!sgt) {
428 		ret = -ENOMEM;
429 		goto put_pages;
430 	}
431 
432 	ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
433 					offset_in_page(xfer_start_addr),
434 					in_trans->size - resources->xferred_dma_size, GFP_KERNEL);
435 	if (ret) {
436 		ret = -ENOMEM;
437 		goto free_sgt;
438 	}
439 
440 	ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);
441 	if (ret)
442 		goto free_table;
443 
444 	xfer->sgt = sgt;
445 	xfer->page_list = page_list;
446 	xfer->nr_pages = nr_pages;
447 
448 	return need_pages > nr_pages ? 1 : 0;
449 
450 free_table:
451 	sg_free_table(sgt);
452 free_sgt:
453 	kfree(sgt);
454 put_pages:
455 	for (i = 0; i < nr_pages; ++i)
456 		put_page(page_list[i]);
457 free_page_list:
458 	kfree(page_list);
459 	return ret;
460 }
461 
462 /* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
463 static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
464 				  struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
465 				  struct wire_trans_dma_xfer **out_trans)
466 {
467 	struct wrapper_msg *trans_wrapper;
468 	struct sg_table *sgt = xfer->sgt;
469 	struct wire_addr_size_pair *asp;
470 	struct scatterlist *sg;
471 	struct wrapper_msg *w;
472 	unsigned int dma_len;
473 	u64 dma_chunk_len;
474 	void *boundary;
475 	int nents_dma;
476 	int nents;
477 	int i;
478 
479 	nents = sgt->nents;
480 	nents_dma = nents;
481 	*size = QAIC_MANAGE_EXT_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
482 	for_each_sgtable_sg(sgt, sg, i) {
483 		*size -= sizeof(*asp);
484 		/* Save 1K for possible follow-up transactions. */
485 		if (*size < SZ_1K) {
486 			nents_dma = i;
487 			break;
488 		}
489 	}
490 
491 	trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
492 	if (!trans_wrapper)
493 		return -ENOMEM;
494 	*out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
495 
496 	asp = (*out_trans)->data;
497 	boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
498 	*size = 0;
499 
500 	dma_len = 0;
501 	w = trans_wrapper;
502 	dma_chunk_len = 0;
503 	for_each_sg(sgt->sgl, sg, nents_dma, i) {
504 		asp->size = cpu_to_le64(dma_len);
505 		dma_chunk_len += dma_len;
506 		if (dma_len) {
507 			asp++;
508 			if ((void *)asp + sizeof(*asp) > boundary) {
509 				w->len = (void *)asp - (void *)&w->msg;
510 				*size += w->len;
511 				w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
512 				if (!w)
513 					return -ENOMEM;
514 				boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
515 				asp = (struct wire_addr_size_pair *)&w->msg;
516 			}
517 		}
518 		asp->addr = cpu_to_le64(sg_dma_address(sg));
519 		dma_len = sg_dma_len(sg);
520 	}
521 	/* finalize the last segment */
522 	asp->size = cpu_to_le64(dma_len);
523 	w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
524 	*size += w->len;
525 	dma_chunk_len += dma_len;
526 	resources->xferred_dma_size += dma_chunk_len;
527 
528 	return nents_dma < nents ? 1 : 0;
529 }
530 
531 static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
532 {
533 	int i;
534 
535 	dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
536 	sg_free_table(xfer->sgt);
537 	kfree(xfer->sgt);
538 	for (i = 0; i < xfer->nr_pages; ++i)
539 		put_page(xfer->page_list[i]);
540 	kfree(xfer->page_list);
541 }
542 
543 static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
544 		      u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
545 {
546 	struct qaic_manage_trans_dma_xfer *in_trans = trans;
547 	struct wire_trans_dma_xfer *out_trans;
548 	struct wrapper_msg *wrapper;
549 	struct dma_xfer *xfer;
550 	struct wire_msg *msg;
551 	bool need_cont_dma;
552 	u32 msg_hdr_len;
553 	u32 size;
554 	int ret;
555 
556 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
557 	msg = &wrapper->msg;
558 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
559 
560 	if (msg_hdr_len > (UINT_MAX - QAIC_MANAGE_EXT_MSG_LENGTH))
561 		return -EINVAL;
562 
563 	/* There should be enough space to hold at least one ASP entry. */
564 	if (msg_hdr_len + sizeof(*out_trans) + sizeof(struct wire_addr_size_pair) >
565 	    QAIC_MANAGE_EXT_MSG_LENGTH)
566 		return -ENOMEM;
567 
568 	if (in_trans->addr + in_trans->size < in_trans->addr || !in_trans->size)
569 		return -EINVAL;
570 
571 	xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
572 	if (!xfer)
573 		return -ENOMEM;
574 
575 	ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
576 	if (ret < 0)
577 		goto free_xfer;
578 
579 	need_cont_dma = (bool)ret;
580 
581 	ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);
582 	if (ret < 0)
583 		goto cleanup_xfer;
584 
585 	need_cont_dma = need_cont_dma || (bool)ret;
586 
587 	msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
588 	msg->hdr.count = incr_le32(msg->hdr.count);
589 
590 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
591 	out_trans->hdr.len = cpu_to_le32(size);
592 	out_trans->tag = cpu_to_le32(in_trans->tag);
593 	out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
594 								sizeof(struct wire_addr_size_pair));
595 
596 	*user_len += in_trans->hdr.len;
597 
598 	if (resources->dma_chunk_id) {
599 		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
600 	} else if (need_cont_dma) {
601 		while (resources->dma_chunk_id == 0)
602 			resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);
603 
604 		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
605 	}
606 	resources->trans_hdr = trans;
607 
608 	list_add(&xfer->list, &resources->dma_xfers);
609 	return 0;
610 
611 cleanup_xfer:
612 	cleanup_xfer(qdev, xfer);
613 free_xfer:
614 	kfree(xfer);
615 	return ret;
616 }
617 
618 static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
619 			   u32 *user_len, struct ioctl_resources *resources)
620 {
621 	struct qaic_manage_trans_activate_to_dev *in_trans = trans;
622 	struct wire_trans_activate_to_dev *out_trans;
623 	struct wrapper_msg *trans_wrapper;
624 	struct wrapper_msg *wrapper;
625 	struct wire_msg *msg;
626 	dma_addr_t dma_addr;
627 	u32 msg_hdr_len;
628 	void *buf;
629 	u32 nelem;
630 	u32 size;
631 	int ret;
632 
633 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
634 	msg = &wrapper->msg;
635 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
636 
637 	if (msg_hdr_len + sizeof(*out_trans) > QAIC_MANAGE_MAX_MSG_LENGTH)
638 		return -ENOSPC;
639 
640 	if (!in_trans->queue_size)
641 		return -EINVAL;
642 
643 	if (in_trans->pad)
644 		return -EINVAL;
645 
646 	nelem = in_trans->queue_size;
647 	size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;
648 	if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())
649 		return -EINVAL;
650 
651 	if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
652 		return -EINVAL;
653 
654 	size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
655 
656 	buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
657 	if (!buf)
658 		return -ENOMEM;
659 
660 	trans_wrapper = add_wrapper(wrappers,
661 				    offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
662 	if (!trans_wrapper) {
663 		ret = -ENOMEM;
664 		goto free_dma;
665 	}
666 	trans_wrapper->len = sizeof(*out_trans);
667 	out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
668 
669 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
670 	out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
671 	out_trans->buf_len = cpu_to_le32(size);
672 	out_trans->req_q_addr = cpu_to_le64(dma_addr);
673 	out_trans->req_q_size = cpu_to_le32(nelem);
674 	out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
675 	out_trans->rsp_q_size = cpu_to_le32(nelem);
676 	out_trans->options = cpu_to_le32(in_trans->options);
677 
678 	*user_len += in_trans->hdr.len;
679 	msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
680 	msg->hdr.count = incr_le32(msg->hdr.count);
681 
682 	resources->buf = buf;
683 	resources->dma_addr = dma_addr;
684 	resources->total_size = size;
685 	resources->nelem = nelem;
686 	resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
687 	return 0;
688 
689 free_dma:
690 	dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);
691 	return ret;
692 }
693 
694 static int encode_deactivate(struct qaic_device *qdev, void *trans,
695 			     u32 *user_len, struct qaic_user *usr)
696 {
697 	struct qaic_manage_trans_deactivate *in_trans = trans;
698 
699 	if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
700 		return -EINVAL;
701 
702 	*user_len += in_trans->hdr.len;
703 
704 	return disable_dbc(qdev, in_trans->dbc_id, usr);
705 }
706 
707 static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
708 			 u32 *user_len)
709 {
710 	struct qaic_manage_trans_status_to_dev *in_trans = trans;
711 	struct wire_trans_status_to_dev *out_trans;
712 	struct wrapper_msg *trans_wrapper;
713 	struct wrapper_msg *wrapper;
714 	struct wire_msg *msg;
715 	u32 msg_hdr_len;
716 
717 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
718 	msg = &wrapper->msg;
719 	msg_hdr_len = le32_to_cpu(msg->hdr.len);
720 
721 	if (msg_hdr_len + in_trans->hdr.len > QAIC_MANAGE_MAX_MSG_LENGTH)
722 		return -ENOSPC;
723 
724 	trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));
725 	if (!trans_wrapper)
726 		return -ENOMEM;
727 
728 	trans_wrapper->len = sizeof(*out_trans);
729 	out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
730 
731 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
732 	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
733 	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
734 	msg->hdr.count = incr_le32(msg->hdr.count);
735 	*user_len += in_trans->hdr.len;
736 
737 	return 0;
738 }
739 
740 static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
741 			  struct wrapper_list *wrappers, struct ioctl_resources *resources,
742 			  struct qaic_user *usr)
743 {
744 	struct qaic_manage_trans_hdr *trans_hdr;
745 	struct wrapper_msg *wrapper;
746 	struct wire_msg *msg;
747 	u32 user_len = 0;
748 	int ret;
749 	int i;
750 
751 	if (!user_msg->count) {
752 		ret = -EINVAL;
753 		goto out;
754 	}
755 
756 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
757 	msg = &wrapper->msg;
758 
759 	msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
760 
761 	if (resources->dma_chunk_id) {
762 		ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);
763 		msg->hdr.count = cpu_to_le32(1);
764 		goto out;
765 	}
766 
767 	for (i = 0; i < user_msg->count; ++i) {
768 		if (user_len >= user_msg->len) {
769 			ret = -EINVAL;
770 			break;
771 		}
772 		trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
773 		if (user_len + trans_hdr->len > user_msg->len) {
774 			ret = -EINVAL;
775 			break;
776 		}
777 
778 		switch (trans_hdr->type) {
779 		case QAIC_TRANS_PASSTHROUGH_FROM_USR:
780 			ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);
781 			break;
782 		case QAIC_TRANS_DMA_XFER_FROM_USR:
783 			ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);
784 			break;
785 		case QAIC_TRANS_ACTIVATE_FROM_USR:
786 			ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);
787 			break;
788 		case QAIC_TRANS_DEACTIVATE_FROM_USR:
789 			ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);
790 			break;
791 		case QAIC_TRANS_STATUS_FROM_USR:
792 			ret = encode_status(qdev, trans_hdr, wrappers, &user_len);
793 			break;
794 		default:
795 			ret = -EINVAL;
796 			break;
797 		}
798 
799 		if (ret)
800 			break;
801 	}
802 
803 	if (user_len != user_msg->len)
804 		ret = -EINVAL;
805 out:
806 	if (ret) {
807 		free_dma_xfers(qdev, resources);
808 		free_dbc_buf(qdev, resources);
809 		return ret;
810 	}
811 
812 	return 0;
813 }
814 
815 static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
816 			      u32 *msg_len)
817 {
818 	struct qaic_manage_trans_passthrough *out_trans;
819 	struct wire_trans_passthrough *in_trans = trans;
820 	u32 len;
821 
822 	out_trans = (void *)user_msg->data + user_msg->len;
823 
824 	len = le32_to_cpu(in_trans->hdr.len);
825 	if (len % 8 != 0)
826 		return -EINVAL;
827 
828 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
829 		return -ENOSPC;
830 
831 	memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
832 	user_msg->len += len;
833 	*msg_len += len;
834 	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
835 	out_trans->hdr.len = len;
836 
837 	return 0;
838 }
839 
840 static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
841 			   u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
842 {
843 	struct qaic_manage_trans_activate_from_dev *out_trans;
844 	struct wire_trans_activate_from_dev *in_trans = trans;
845 	u32 len;
846 
847 	out_trans = (void *)user_msg->data + user_msg->len;
848 
849 	len = le32_to_cpu(in_trans->hdr.len);
850 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
851 		return -ENOSPC;
852 
853 	user_msg->len += len;
854 	*msg_len += len;
855 	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
856 	out_trans->hdr.len = len;
857 	out_trans->status = le32_to_cpu(in_trans->status);
858 	out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
859 	out_trans->options = le64_to_cpu(in_trans->options);
860 
861 	if (!resources->buf)
862 		/* how did we get an activate response without a request? */
863 		return -EINVAL;
864 
865 	if (out_trans->dbc_id >= qdev->num_dbc)
866 		/*
867 		 * The device assigned an invalid resource, which should never
868 		 * happen. Return an error so the user can try to recover.
869 		 */
870 		return -ENODEV;
871 
872 	if (out_trans->status)
873 		/*
874 		 * Allocating resources failed on device side. This is not an
875 		 * expected behaviour, user is expected to handle this situation.
876 		 */
877 		return -ECANCELED;
878 
879 	resources->status = out_trans->status;
880 	resources->dbc_id = out_trans->dbc_id;
881 	save_dbc_buf(qdev, resources, usr);
882 
883 	return 0;
884 }
885 
886 static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
887 			     struct qaic_user *usr)
888 {
889 	struct wire_trans_deactivate_from_dev *in_trans = trans;
890 	u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
891 	u32 status = le32_to_cpu(in_trans->status);
892 
893 	if (dbc_id >= qdev->num_dbc)
894 		/*
895 		 * The device assigned an invalid resource, which should never
896 		 * happen. Inject an error so the user can try to recover.
897 		 */
898 		return -ENODEV;
899 
900 	if (status) {
901 		/*
902 		 * Releasing resources failed on the device side, which puts
903 		 * us in a bind since they may still be in use, so enable the
904 		 * dbc. User is expected to retry deactivation.
905 		 */
906 		enable_dbc(qdev, dbc_id, usr);
907 		return -ECANCELED;
908 	}
909 
910 	release_dbc(qdev, dbc_id);
911 	*msg_len += sizeof(*in_trans);
912 
913 	return 0;
914 }
915 
916 static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
917 			 u32 *user_len, struct wire_msg *msg)
918 {
919 	struct qaic_manage_trans_status_from_dev *out_trans;
920 	struct wire_trans_status_from_dev *in_trans = trans;
921 	u32 len;
922 
923 	out_trans = (void *)user_msg->data + user_msg->len;
924 
925 	len = le32_to_cpu(in_trans->hdr.len);
926 	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
927 		return -ENOSPC;
928 
929 	out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
930 	out_trans->hdr.len = len;
931 	out_trans->major = le16_to_cpu(in_trans->major);
932 	out_trans->minor = le16_to_cpu(in_trans->minor);
933 	out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
934 	out_trans->status = le32_to_cpu(in_trans->status);
935 	*user_len += le32_to_cpu(in_trans->hdr.len);
936 	user_msg->len += len;
937 
938 	if (out_trans->status)
939 		return -ECANCELED;
940 	if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
941 		return -EPIPE;
942 
943 	return 0;
944 }
945 
946 static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
947 			  struct wire_msg *msg, struct ioctl_resources *resources,
948 			  struct qaic_user *usr)
949 {
950 	u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
951 	struct wire_trans_hdr *trans_hdr;
952 	u32 msg_len = 0;
953 	int ret;
954 	int i;
955 
956 	if (msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
957 		return -EINVAL;
958 
959 	user_msg->len = 0;
960 	user_msg->count = le32_to_cpu(msg->hdr.count);
961 
962 	for (i = 0; i < user_msg->count; ++i) {
963 		trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
964 		if (msg_len + le32_to_cpu(trans_hdr->len) > msg_hdr_len)
965 			return -EINVAL;
966 
967 		switch (le32_to_cpu(trans_hdr->type)) {
968 		case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
969 			ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);
970 			break;
971 		case QAIC_TRANS_ACTIVATE_FROM_DEV:
972 			ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);
973 			break;
974 		case QAIC_TRANS_DEACTIVATE_FROM_DEV:
975 			ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);
976 			break;
977 		case QAIC_TRANS_STATUS_FROM_DEV:
978 			ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);
979 			break;
980 		default:
981 			return -EINVAL;
982 		}
983 
984 		if (ret)
985 			return ret;
986 	}
987 
988 	if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
989 		return -EINVAL;
990 
991 	return 0;
992 }
993 
994 static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
995 		      bool ignore_signal)
996 {
997 	struct xfer_queue_elem elem;
998 	struct wire_msg *out_buf;
999 	struct wrapper_msg *w;
1000 	int retry_count;
1001 	long ret;
1002 
1003 	if (qdev->in_reset) {
1004 		mutex_unlock(&qdev->cntl_mutex);
1005 		return ERR_PTR(-ENODEV);
1006 	}
1007 
1008 	elem.seq_num = seq_num;
1009 	elem.buf = NULL;
1010 	init_completion(&elem.xfer_done);
1011 	if (likely(!qdev->cntl_lost_buf)) {
1012 		/*
1013 		 * The max size of request to device is QAIC_MANAGE_EXT_MSG_LENGTH.
1014 		 * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
1015 		 */
1016 		out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
1017 		if (!out_buf) {
1018 			mutex_unlock(&qdev->cntl_mutex);
1019 			return ERR_PTR(-ENOMEM);
1020 		}
1021 
1022 		ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,
1023 				    QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);
1024 		if (ret) {
1025 			mutex_unlock(&qdev->cntl_mutex);
1026 			return ERR_PTR(ret);
1027 		}
1028 	} else {
1029 		/*
1030 		 * we lost a buffer because we queued a recv buf, but then
1031 		 * queuing the corresponding tx buf failed. To try to avoid
1032 		 * a memory leak, lets reclaim it and use it for this
1033 		 * transaction.
1034 		 */
1035 		qdev->cntl_lost_buf = false;
1036 	}
1037 
1038 	list_for_each_entry(w, &wrappers->list, list) {
1039 		kref_get(&w->ref_count);
1040 		retry_count = 0;
1041 retry:
1042 		ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,
1043 				    list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);
1044 		if (ret) {
1045 			if (ret == -EAGAIN && retry_count++ < QAIC_MHI_RETRY_MAX) {
1046 				msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
1047 				if (!signal_pending(current))
1048 					goto retry;
1049 			}
1050 
1051 			qdev->cntl_lost_buf = true;
1052 			kref_put(&w->ref_count, free_wrapper);
1053 			mutex_unlock(&qdev->cntl_mutex);
1054 			return ERR_PTR(ret);
1055 		}
1056 	}
1057 
1058 	list_add_tail(&elem.list, &qdev->cntl_xfer_list);
1059 	mutex_unlock(&qdev->cntl_mutex);
1060 
1061 	if (ignore_signal)
1062 		ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);
1063 	else
1064 		ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
1065 								control_resp_timeout_s * HZ);
1066 	/*
1067 	 * not using _interruptable because we have to cleanup or we'll
1068 	 * likely cause memory corruption
1069 	 */
1070 	mutex_lock(&qdev->cntl_mutex);
1071 	if (!list_empty(&elem.list))
1072 		list_del(&elem.list);
1073 	if (!ret && !elem.buf)
1074 		ret = -ETIMEDOUT;
1075 	else if (ret > 0 && !elem.buf)
1076 		ret = -EIO;
1077 	mutex_unlock(&qdev->cntl_mutex);
1078 
1079 	if (ret < 0) {
1080 		kfree(elem.buf);
1081 		return ERR_PTR(ret);
1082 	} else if (!qdev->valid_crc(elem.buf)) {
1083 		kfree(elem.buf);
1084 		return ERR_PTR(-EPIPE);
1085 	}
1086 
1087 	return elem.buf;
1088 }
1089 
1090 /* Add a transaction to abort the outstanding DMA continuation */
1091 static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
1092 {
1093 	struct wire_trans_dma_xfer *out_trans;
1094 	u32 size = sizeof(*out_trans);
1095 	struct wrapper_msg *wrapper;
1096 	struct wrapper_msg *w;
1097 	struct wire_msg *msg;
1098 
1099 	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
1100 	msg = &wrapper->msg;
1101 
1102 	/* Remove all but the first wrapper which has the msg header */
1103 	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1104 		if (!list_is_first(&wrapper->list, &wrappers->list))
1105 			kref_put(&wrapper->ref_count, free_wrapper);
1106 
1107 	wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
1108 
1109 	if (!wrapper)
1110 		return -ENOMEM;
1111 
1112 	out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
1113 	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
1114 	out_trans->hdr.len = cpu_to_le32(size);
1115 	out_trans->tag = cpu_to_le32(0);
1116 	out_trans->count = cpu_to_le32(0);
1117 	out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
1118 
1119 	msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
1120 	msg->hdr.count = cpu_to_le32(1);
1121 	wrapper->len = size;
1122 
1123 	return 0;
1124 }
1125 
1126 static struct wrapper_list *alloc_wrapper_list(void)
1127 {
1128 	struct wrapper_list *wrappers;
1129 
1130 	wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
1131 	if (!wrappers)
1132 		return NULL;
1133 	INIT_LIST_HEAD(&wrappers->list);
1134 	spin_lock_init(&wrappers->lock);
1135 
1136 	return wrappers;
1137 }
1138 
1139 static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
1140 				struct manage_msg *user_msg, struct ioctl_resources *resources,
1141 				struct wire_msg **rsp)
1142 {
1143 	struct wrapper_list *wrappers;
1144 	struct wrapper_msg *wrapper;
1145 	struct wrapper_msg *w;
1146 	bool all_done = false;
1147 	struct wire_msg *msg;
1148 	int ret;
1149 
1150 	wrappers = alloc_wrapper_list();
1151 	if (!wrappers)
1152 		return -ENOMEM;
1153 
1154 	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
1155 	if (!wrapper) {
1156 		kfree(wrappers);
1157 		return -ENOMEM;
1158 	}
1159 
1160 	msg = &wrapper->msg;
1161 	wrapper->len = sizeof(*msg);
1162 
1163 	ret = encode_message(qdev, user_msg, wrappers, resources, usr);
1164 	if (ret && resources->dma_chunk_id)
1165 		ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);
1166 	if (ret)
1167 		goto encode_failed;
1168 
1169 	ret = mutex_lock_interruptible(&qdev->cntl_mutex);
1170 	if (ret)
1171 		goto lock_failed;
1172 
1173 	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1174 	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1175 
1176 	if (usr) {
1177 		msg->hdr.handle = cpu_to_le32(usr->handle);
1178 		msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
1179 	} else {
1180 		msg->hdr.handle = 0;
1181 		msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
1182 	}
1183 
1184 	msg->hdr.padding = cpu_to_le32(0);
1185 	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1186 
1187 	/* msg_xfer releases the mutex */
1188 	*rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);
1189 	if (IS_ERR(*rsp))
1190 		ret = PTR_ERR(*rsp);
1191 
1192 lock_failed:
1193 	free_dma_xfers(qdev, resources);
1194 encode_failed:
1195 	spin_lock(&wrappers->lock);
1196 	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1197 		kref_put(&wrapper->ref_count, free_wrapper);
1198 	all_done = list_empty(&wrappers->list);
1199 	spin_unlock(&wrappers->lock);
1200 	if (all_done)
1201 		kfree(wrappers);
1202 
1203 	return ret;
1204 }
1205 
1206 static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
1207 {
1208 	struct wire_trans_dma_xfer_cont *dma_cont = NULL;
1209 	struct ioctl_resources resources;
1210 	struct wire_msg *rsp = NULL;
1211 	int ret;
1212 
1213 	memset(&resources, 0, sizeof(struct ioctl_resources));
1214 
1215 	INIT_LIST_HEAD(&resources.dma_xfers);
1216 
1217 	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
1218 	    user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
1219 		return -EINVAL;
1220 
1221 dma_xfer_continue:
1222 	ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);
1223 	if (ret)
1224 		return ret;
1225 	/* dma_cont should be the only transaction if present */
1226 	if (le32_to_cpu(rsp->hdr.count) == 1) {
1227 		dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
1228 		if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
1229 			dma_cont = NULL;
1230 	}
1231 	if (dma_cont) {
1232 		if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
1233 		    le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
1234 			kfree(rsp);
1235 			goto dma_xfer_continue;
1236 		}
1237 
1238 		ret = -EINVAL;
1239 		goto dma_cont_failed;
1240 	}
1241 
1242 	ret = decode_message(qdev, user_msg, rsp, &resources, usr);
1243 
1244 dma_cont_failed:
1245 	free_dbc_buf(qdev, &resources);
1246 	kfree(rsp);
1247 	return ret;
1248 }
1249 
1250 int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1251 {
1252 	struct qaic_manage_msg *user_msg;
1253 	struct qaic_device *qdev;
1254 	struct manage_msg *msg;
1255 	struct qaic_user *usr;
1256 	u8 __user *user_data;
1257 	int qdev_rcu_id;
1258 	int usr_rcu_id;
1259 	int ret;
1260 
1261 	usr = file_priv->driver_priv;
1262 
1263 	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1264 	if (!usr->qddev) {
1265 		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1266 		return -ENODEV;
1267 	}
1268 
1269 	qdev = usr->qddev->qdev;
1270 
1271 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1272 	if (qdev->in_reset) {
1273 		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1274 		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1275 		return -ENODEV;
1276 	}
1277 
1278 	user_msg = data;
1279 
1280 	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH) {
1281 		ret = -EINVAL;
1282 		goto out;
1283 	}
1284 
1285 	msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
1286 	if (!msg) {
1287 		ret = -ENOMEM;
1288 		goto out;
1289 	}
1290 
1291 	msg->len = user_msg->len;
1292 	msg->count = user_msg->count;
1293 
1294 	user_data = u64_to_user_ptr(user_msg->data);
1295 
1296 	if (copy_from_user(msg->data, user_data, user_msg->len)) {
1297 		ret = -EFAULT;
1298 		goto free_msg;
1299 	}
1300 
1301 	ret = qaic_manage(qdev, usr, msg);
1302 
1303 	/*
1304 	 * If the qaic_manage() is successful then we copy the message onto
1305 	 * userspace memory but we have an exception for -ECANCELED.
1306 	 * For -ECANCELED, it means that device has NACKed the message with a
1307 	 * status error code which userspace would like to know.
1308 	 */
1309 	if (ret == -ECANCELED || !ret) {
1310 		if (copy_to_user(user_data, msg->data, msg->len)) {
1311 			ret = -EFAULT;
1312 		} else {
1313 			user_msg->len = msg->len;
1314 			user_msg->count = msg->count;
1315 		}
1316 	}
1317 
1318 free_msg:
1319 	kfree(msg);
1320 out:
1321 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1322 	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1323 	return ret;
1324 }
1325 
1326 int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
1327 {
1328 	struct qaic_manage_trans_status_from_dev *status_result;
1329 	struct qaic_manage_trans_status_to_dev *status_query;
1330 	struct manage_msg *user_msg;
1331 	int ret;
1332 
1333 	user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
1334 	if (!user_msg) {
1335 		ret = -ENOMEM;
1336 		goto out;
1337 	}
1338 	user_msg->len = sizeof(*status_query);
1339 	user_msg->count = 1;
1340 
1341 	status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
1342 	status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
1343 	status_query->hdr.len = sizeof(status_query->hdr);
1344 
1345 	ret = qaic_manage(qdev, usr, user_msg);
1346 	if (ret)
1347 		goto kfree_user_msg;
1348 	status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
1349 	*major = status_result->major;
1350 	*minor = status_result->minor;
1351 
1352 	if (status_result->status_flags & BIT(0)) { /* device is using CRC */
1353 		/* By default qdev->gen_crc is programmed to generate CRC */
1354 		qdev->valid_crc = valid_crc;
1355 	} else {
1356 		/* By default qdev->valid_crc is programmed to bypass CRC */
1357 		qdev->gen_crc = gen_crc_stub;
1358 	}
1359 
1360 kfree_user_msg:
1361 	kfree(user_msg);
1362 out:
1363 	return ret;
1364 }
1365 
1366 static void resp_worker(struct work_struct *work)
1367 {
1368 	struct resp_work *resp = container_of(work, struct resp_work, work);
1369 	struct qaic_device *qdev = resp->qdev;
1370 	struct wire_msg *msg = resp->buf;
1371 	struct xfer_queue_elem *elem;
1372 	struct xfer_queue_elem *i;
1373 	bool found = false;
1374 
1375 	mutex_lock(&qdev->cntl_mutex);
1376 	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1377 		if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
1378 			found = true;
1379 			list_del_init(&elem->list);
1380 			elem->buf = msg;
1381 			complete_all(&elem->xfer_done);
1382 			break;
1383 		}
1384 	}
1385 	mutex_unlock(&qdev->cntl_mutex);
1386 
1387 	if (!found)
1388 		/* request must have timed out, drop packet */
1389 		kfree(msg);
1390 
1391 	kfree(resp);
1392 }
1393 
1394 static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
1395 {
1396 	bool all_done = false;
1397 
1398 	spin_lock(&wrappers->lock);
1399 	kref_put(&wrapper->ref_count, free_wrapper);
1400 	all_done = list_empty(&wrappers->list);
1401 	spin_unlock(&wrappers->lock);
1402 
1403 	if (all_done)
1404 		kfree(wrappers);
1405 }
1406 
1407 void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1408 {
1409 	struct wire_msg *msg = mhi_result->buf_addr;
1410 	struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
1411 
1412 	free_wrapper_from_list(wrapper->head, wrapper);
1413 }
1414 
1415 void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1416 {
1417 	struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
1418 	struct wire_msg *msg = mhi_result->buf_addr;
1419 	struct resp_work *resp;
1420 
1421 	if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
1422 		kfree(msg);
1423 		return;
1424 	}
1425 
1426 	resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
1427 	if (!resp) {
1428 		kfree(msg);
1429 		return;
1430 	}
1431 
1432 	INIT_WORK(&resp->work, resp_worker);
1433 	resp->qdev = qdev;
1434 	resp->buf = msg;
1435 	queue_work(qdev->cntl_wq, &resp->work);
1436 }
1437 
1438 int qaic_control_open(struct qaic_device *qdev)
1439 {
1440 	if (!qdev->cntl_ch)
1441 		return -ENODEV;
1442 
1443 	qdev->cntl_lost_buf = false;
1444 	/*
1445 	 * By default qaic should assume that device has CRC enabled.
1446 	 * Qaic comes to know if device has CRC enabled or disabled during the
1447 	 * device status transaction, which is the first transaction performed
1448 	 * on control channel.
1449 	 *
1450 	 * So CRC validation of first device status transaction response is
1451 	 * ignored (by calling valid_crc_stub) and is done later during decoding
1452 	 * if device has CRC enabled.
1453 	 * Now that qaic knows whether device has CRC enabled or not it acts
1454 	 * accordingly.
1455 	 */
1456 	qdev->gen_crc = gen_crc;
1457 	qdev->valid_crc = valid_crc_stub;
1458 
1459 	return mhi_prepare_for_transfer(qdev->cntl_ch);
1460 }
1461 
1462 void qaic_control_close(struct qaic_device *qdev)
1463 {
1464 	mhi_unprepare_from_transfer(qdev->cntl_ch);
1465 }
1466 
1467 void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
1468 {
1469 	struct wire_trans_terminate_to_dev *trans;
1470 	struct wrapper_list *wrappers;
1471 	struct wrapper_msg *wrapper;
1472 	struct wire_msg *msg;
1473 	struct wire_msg *rsp;
1474 
1475 	wrappers = alloc_wrapper_list();
1476 	if (!wrappers)
1477 		return;
1478 
1479 	wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
1480 	if (!wrapper)
1481 		return;
1482 
1483 	msg = &wrapper->msg;
1484 
1485 	trans = (struct wire_trans_terminate_to_dev *)msg->data;
1486 
1487 	trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
1488 	trans->hdr.len = cpu_to_le32(sizeof(*trans));
1489 	trans->handle = cpu_to_le32(usr->handle);
1490 
1491 	mutex_lock(&qdev->cntl_mutex);
1492 	wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
1493 	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1494 	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1495 	msg->hdr.len = cpu_to_le32(wrapper->len);
1496 	msg->hdr.count = cpu_to_le32(1);
1497 	msg->hdr.handle = cpu_to_le32(usr->handle);
1498 	msg->hdr.padding = cpu_to_le32(0);
1499 	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1500 
1501 	/*
1502 	 * msg_xfer releases the mutex
1503 	 * We don't care about the return of msg_xfer since we will not do
1504 	 * anything different based on what happens.
1505 	 * We ignore pending signals since one will be set if the user is
1506 	 * killed, and we need give the device a chance to cleanup, otherwise
1507 	 * DMA may still be in progress when we return.
1508 	 */
1509 	rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);
1510 	if (!IS_ERR(rsp))
1511 		kfree(rsp);
1512 	free_wrapper_from_list(wrappers, wrapper);
1513 }
1514 
1515 void wake_all_cntl(struct qaic_device *qdev)
1516 {
1517 	struct xfer_queue_elem *elem;
1518 	struct xfer_queue_elem *i;
1519 
1520 	mutex_lock(&qdev->cntl_mutex);
1521 	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1522 		list_del_init(&elem->list);
1523 		complete_all(&elem->xfer_done);
1524 	}
1525 	mutex_unlock(&qdev->cntl_mutex);
1526 }
1527