1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * ISH-TP client driver for ISH firmware loading
4  *
5  * Copyright (c) 2019, Intel Corporation.
6  */
7 
8 #include <linux/firmware.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/intel-ish-client-if.h>
12 #include <linux/property.h>
13 #include <asm/cacheflush.h>
14 
15 /* Number of times we attempt to load the firmware before giving up */
16 #define MAX_LOAD_ATTEMPTS			3
17 
18 /* ISH TX/RX ring buffer pool size */
19 #define LOADER_CL_RX_RING_SIZE			1
20 #define LOADER_CL_TX_RING_SIZE			1
21 
22 /*
23  * ISH Shim firmware loader reserves 4 Kb buffer in SRAM. The buffer is
24  * used to temporarily hold the data transferred from host to Shim
25  * firmware loader. Reason for the odd size of 3968 bytes? Each IPC
26  * transfer is 128 bytes (= 4 bytes header + 124 bytes payload). So the
27  * 4 Kb buffer can hold maximum of 32 IPC transfers, which means we can
28  * have a max payload of 3968 bytes (= 32 x 124 payload).
29  */
30 #define LOADER_SHIM_IPC_BUF_SIZE		3968
31 
32 /**
33  * enum ish_loader_commands -	ISH loader host commands.
34  * LOADER_CMD_XFER_QUERY	Query the Shim firmware loader for
35  *				capabilities
36  * LOADER_CMD_XFER_FRAGMENT	Transfer one firmware image fragment at a
37  *				time. The command may be executed
38  *				multiple times until the entire firmware
39  *				image is downloaded to SRAM.
40  * LOADER_CMD_START		Start executing the main firmware.
41  */
42 enum ish_loader_commands {
43 	LOADER_CMD_XFER_QUERY = 0,
44 	LOADER_CMD_XFER_FRAGMENT,
45 	LOADER_CMD_START,
46 };
47 
48 /* Command bit mask */
49 #define	CMD_MASK				GENMASK(6, 0)
50 #define	IS_RESPONSE				BIT(7)
51 
52 /*
53  * ISH firmware max delay for one transmit failure is 1 Hz,
54  * and firmware will retry 2 times, so 3 Hz is used for timeout.
55  */
56 #define ISHTP_SEND_TIMEOUT			(3 * HZ)
57 
58 /*
59  * Loader transfer modes:
60  *
61  * LOADER_XFER_MODE_ISHTP mode uses the existing ISH-TP mechanism to
62  * transfer data. This may use IPC or DMA if supported in firmware.
63  * The buffer size is limited to 4 Kb by the IPC/ISH-TP protocol for
64  * both IPC & DMA (legacy).
65  *
66  * LOADER_XFER_MODE_DIRECT_DMA - firmware loading is a bit different
67  * from the sensor data streaming. Here we download a large (300+ Kb)
68  * image directly to ISH SRAM memory. There is limited benefit of
69  * DMA'ing 300 Kb image in 4 Kb chucks limit. Hence, we introduce
70  * this "direct dma" mode, where we do not use ISH-TP for DMA, but
71  * instead manage the DMA directly in kernel driver and Shim firmware
72  * loader (allocate buffer, break in chucks and transfer). This allows
73  * to overcome 4 Kb limit, and optimize the data flow path in firmware.
74  */
75 #define LOADER_XFER_MODE_DIRECT_DMA		BIT(0)
76 #define LOADER_XFER_MODE_ISHTP			BIT(1)
77 
78 /* ISH Transport Loader client unique GUID */
79 static const guid_t loader_ishtp_guid =
80 	GUID_INIT(0xc804d06a, 0x55bd, 0x4ea7,
81 		  0xad, 0xed, 0x1e, 0x31, 0x22, 0x8c, 0x76, 0xdc);
82 
83 #define FILENAME_SIZE				256
84 
85 /*
86  * The firmware loading latency will be minimum if we can DMA the
87  * entire ISH firmware image in one go. This requires that we allocate
88  * a large DMA buffer in kernel, which could be problematic on some
89  * platforms. So here we limit the DMA buffer size via a module_param.
90  * We default to 4 pages, but a customer can set it to higher limit if
91  * deemed appropriate for his platform.
92  */
93 static int dma_buf_size_limit = 4 * PAGE_SIZE;
94 
95 /**
96  * struct loader_msg_hdr - Header for ISH Loader commands.
97  * @command:		LOADER_CMD* commands. Bit 7 is the response.
98  * @status:		Command response status. Non 0, is error
99  *			condition.
100  *
101  * This structure is used as header for every command/data sent/received
102  * between Host driver and ISH Shim firmware loader.
103  */
104 struct loader_msg_hdr {
105 	u8 command;
106 	u8 reserved[2];
107 	u8 status;
108 } __packed;
109 
110 struct loader_xfer_query {
111 	struct loader_msg_hdr hdr;
112 	u32 image_size;
113 } __packed;
114 
115 struct ish_fw_version {
116 	u16 major;
117 	u16 minor;
118 	u16 hotfix;
119 	u16 build;
120 } __packed;
121 
122 union loader_version {
123 	u32 value;
124 	struct {
125 		u8 major;
126 		u8 minor;
127 		u8 hotfix;
128 		u8 build;
129 	};
130 } __packed;
131 
132 struct loader_capability {
133 	u32 max_fw_image_size;
134 	u32 xfer_mode;
135 	u32 max_dma_buf_size; /* only for dma mode, multiples of cacheline */
136 } __packed;
137 
138 struct shim_fw_info {
139 	struct ish_fw_version ish_fw_version;
140 	u32 protocol_version;
141 	union loader_version ldr_version;
142 	struct loader_capability ldr_capability;
143 } __packed;
144 
145 struct loader_xfer_query_response {
146 	struct loader_msg_hdr hdr;
147 	struct shim_fw_info fw_info;
148 } __packed;
149 
150 struct loader_xfer_fragment {
151 	struct loader_msg_hdr hdr;
152 	u32 xfer_mode;
153 	u32 offset;
154 	u32 size;
155 	u32 is_last;
156 } __packed;
157 
158 struct loader_xfer_ipc_fragment {
159 	struct loader_xfer_fragment fragment;
160 	u8 data[] ____cacheline_aligned; /* variable length payload here */
161 } __packed;
162 
163 struct loader_xfer_dma_fragment {
164 	struct loader_xfer_fragment fragment;
165 	u64 ddr_phys_addr;
166 } __packed;
167 
168 struct loader_start {
169 	struct loader_msg_hdr hdr;
170 } __packed;
171 
172 /**
173  * struct response_info - Encapsulate firmware response related
174  *			information for passing between function
175  *			loader_cl_send() and process_recv() callback.
176  * @data		Copy the data received from firmware here.
177  * @max_size		Max size allocated for the @data buffer. If the
178  *			received data exceeds this value, we log an
179  *			error.
180  * @size		Actual size of data received from firmware.
181  * @error		Returns 0 for success, negative error code for a
182  *			failure in function process_recv().
183  * @received		Set to true on receiving a valid firmware
184  *			response to host command
185  * @wait_queue		Wait queue for Host firmware loading where the
186  *			client sends message to ISH firmware and waits
187  *			for response
188  */
189 struct response_info {
190 	void *data;
191 	size_t max_size;
192 	size_t size;
193 	int error;
194 	bool received;
195 	wait_queue_head_t wait_queue;
196 };
197 
198 /**
199  * struct ishtp_cl_data - Encapsulate per ISH-TP Client Data.
200  * @work_ishtp_reset:	Work queue for reset handling.
201  * @work_fw_load:	Work queue for host firmware loading.
202  * @flag_retry		Flag for indicating host firmware loading should
203  *			be retried.
204  * @retry_count		Count the number of retries.
205  *
206  * This structure is used to store data per client.
207  */
208 struct ishtp_cl_data {
209 	struct ishtp_cl *loader_ishtp_cl;
210 	struct ishtp_cl_device *cl_device;
211 
212 	/*
213 	 * Used for passing firmware response information between
214 	 * loader_cl_send() and process_recv() callback.
215 	 */
216 	struct response_info response;
217 
218 	struct work_struct work_ishtp_reset;
219 	struct work_struct work_fw_load;
220 
221 	/*
222 	 * In certain failure scenrios, it makes sense to reset the ISH
223 	 * subsystem and retry Host firmware loading (e.g. bad message
224 	 * packet, ENOMEM, etc.). On the other hand, failures due to
225 	 * protocol mismatch, etc., are not recoverable. We do not
226 	 * retry them.
227 	 *
228 	 * If set, the flag indicates that we should re-try the
229 	 * particular failure.
230 	 */
231 	bool flag_retry;
232 	int retry_count;
233 };
234 
235 #define IPC_FRAGMENT_DATA_PREAMBLE				\
236 	offsetof(struct loader_xfer_ipc_fragment, data)
237 
238 #define cl_data_to_dev(client_data) ishtp_device((client_data)->cl_device)
239 
240 /**
241  * get_firmware_variant() - Gets the filename of firmware image to be
242  *			loaded based on platform variant.
243  * @client_data		Client data instance.
244  * @filename		Returns firmware filename.
245  *
246  * Queries the firmware-name device property string.
247  *
248  * Return: 0 for success, negative error code for failure.
249  */
250 static int get_firmware_variant(struct ishtp_cl_data *client_data,
251 				char *filename)
252 {
253 	int rv;
254 	const char *val;
255 	struct device *devc = ishtp_get_pci_device(client_data->cl_device);
256 
257 	rv = device_property_read_string(devc, "firmware-name", &val);
258 	if (rv < 0) {
259 		dev_err(devc,
260 			"Error: ISH firmware-name device property required\n");
261 		return rv;
262 	}
263 	return snprintf(filename, FILENAME_SIZE, "intel/%s", val);
264 }
265 
266 /**
267  * loader_cl_send()	Send message from host to firmware
268  * @client_data:	Client data instance
269  * @out_msg		Message buffer to be sent to firmware
270  * @out_size		Size of out going message
271  * @in_msg		Message buffer where the incoming data copied.
272  *			This buffer is allocated by calling
273  * @in_size		Max size of incoming message
274  *
275  * Return: Number of bytes copied in the in_msg on success, negative
276  * error code on failure.
277  */
278 static int loader_cl_send(struct ishtp_cl_data *client_data,
279 			  u8 *out_msg, size_t out_size,
280 			  u8 *in_msg, size_t in_size)
281 {
282 	int rv;
283 	struct loader_msg_hdr *out_hdr = (struct loader_msg_hdr *)out_msg;
284 	struct ishtp_cl *loader_ishtp_cl = client_data->loader_ishtp_cl;
285 
286 	dev_dbg(cl_data_to_dev(client_data),
287 		"%s: command=%02lx is_response=%u status=%02x\n",
288 		__func__,
289 		out_hdr->command & CMD_MASK,
290 		out_hdr->command & IS_RESPONSE ? 1 : 0,
291 		out_hdr->status);
292 
293 	/* Setup in coming buffer & size */
294 	client_data->response.data = in_msg;
295 	client_data->response.max_size = in_size;
296 	client_data->response.error = 0;
297 	client_data->response.received = false;
298 
299 	rv = ishtp_cl_send(loader_ishtp_cl, out_msg, out_size);
300 	if (rv < 0) {
301 		dev_err(cl_data_to_dev(client_data),
302 			"ishtp_cl_send error %d\n", rv);
303 		return rv;
304 	}
305 
306 	wait_event_interruptible_timeout(client_data->response.wait_queue,
307 					 client_data->response.received,
308 					 ISHTP_SEND_TIMEOUT);
309 	if (!client_data->response.received) {
310 		dev_err(cl_data_to_dev(client_data),
311 			"Timed out for response to command=%02lx",
312 			out_hdr->command & CMD_MASK);
313 		return -ETIMEDOUT;
314 	}
315 
316 	if (client_data->response.error < 0)
317 		return client_data->response.error;
318 
319 	return client_data->response.size;
320 }
321 
322 /**
323  * process_recv() -	Receive and parse incoming packet
324  * @loader_ishtp_cl:	Client instance to get stats
325  * @rb_in_proc:		ISH received message buffer
326  *
327  * Parse the incoming packet. If it is a response packet then it will
328  * update received and wake up the caller waiting to for the response.
329  */
330 static void process_recv(struct ishtp_cl *loader_ishtp_cl,
331 			 struct ishtp_cl_rb *rb_in_proc)
332 {
333 	struct loader_msg_hdr *hdr;
334 	size_t data_len = rb_in_proc->buf_idx;
335 	struct ishtp_cl_data *client_data =
336 		ishtp_get_client_data(loader_ishtp_cl);
337 
338 	/* Sanity check */
339 	if (!client_data->response.data) {
340 		dev_err(cl_data_to_dev(client_data),
341 			"Receiving buffer is null. Should be allocated by calling function\n");
342 		client_data->response.error = -EINVAL;
343 		goto end;
344 	}
345 
346 	if (client_data->response.received) {
347 		dev_err(cl_data_to_dev(client_data),
348 			"Previous firmware message not yet processed\n");
349 		client_data->response.error = -EINVAL;
350 		goto end;
351 	}
352 	/*
353 	 * All firmware messages have a header. Check buffer size
354 	 * before accessing elements inside.
355 	 */
356 	if (!rb_in_proc->buffer.data) {
357 		dev_warn(cl_data_to_dev(client_data),
358 			 "rb_in_proc->buffer.data returned null");
359 		client_data->response.error = -EBADMSG;
360 		goto end;
361 	}
362 
363 	if (data_len < sizeof(struct loader_msg_hdr)) {
364 		dev_err(cl_data_to_dev(client_data),
365 			"data size %zu is less than header %zu\n",
366 			data_len, sizeof(struct loader_msg_hdr));
367 		client_data->response.error = -EMSGSIZE;
368 		goto end;
369 	}
370 
371 	hdr = (struct loader_msg_hdr *)rb_in_proc->buffer.data;
372 
373 	dev_dbg(cl_data_to_dev(client_data),
374 		"%s: command=%02lx is_response=%u status=%02x\n",
375 		__func__,
376 		hdr->command & CMD_MASK,
377 		hdr->command & IS_RESPONSE ? 1 : 0,
378 		hdr->status);
379 
380 	if (((hdr->command & CMD_MASK) != LOADER_CMD_XFER_QUERY) &&
381 	    ((hdr->command & CMD_MASK) != LOADER_CMD_XFER_FRAGMENT) &&
382 	    ((hdr->command & CMD_MASK) != LOADER_CMD_START)) {
383 		dev_err(cl_data_to_dev(client_data),
384 			"Invalid command=%02lx\n",
385 			hdr->command & CMD_MASK);
386 		client_data->response.error = -EPROTO;
387 		goto end;
388 	}
389 
390 	if (data_len > client_data->response.max_size) {
391 		dev_err(cl_data_to_dev(client_data),
392 			"Received buffer size %zu is larger than allocated buffer %zu\n",
393 			data_len, client_data->response.max_size);
394 		client_data->response.error = -EMSGSIZE;
395 		goto end;
396 	}
397 
398 	/* We expect only "response" messages from firmware */
399 	if (!(hdr->command & IS_RESPONSE)) {
400 		dev_err(cl_data_to_dev(client_data),
401 			"Invalid response to command\n");
402 		client_data->response.error = -EIO;
403 		goto end;
404 	}
405 
406 	if (hdr->status) {
407 		dev_err(cl_data_to_dev(client_data),
408 			"Loader returned status %d\n",
409 			hdr->status);
410 		client_data->response.error = -EIO;
411 		goto end;
412 	}
413 
414 	/* Update the actual received buffer size */
415 	client_data->response.size = data_len;
416 
417 	/*
418 	 * Copy the buffer received in firmware response for the
419 	 * calling thread.
420 	 */
421 	memcpy(client_data->response.data,
422 	       rb_in_proc->buffer.data, data_len);
423 
424 	/* Set flag before waking up the caller */
425 	client_data->response.received = true;
426 
427 end:
428 	/* Free the buffer */
429 	ishtp_cl_io_rb_recycle(rb_in_proc);
430 	rb_in_proc = NULL;
431 
432 	/* Wake the calling thread */
433 	wake_up_interruptible(&client_data->response.wait_queue);
434 }
435 
436 /**
437  * loader_cl_event_cb() - bus driver callback for incoming message
438  * @device:		Pointer to the ishtp client device for which this
439  *			message is targeted
440  *
441  * Remove the packet from the list and process the message by calling
442  * process_recv
443  */
444 static void loader_cl_event_cb(struct ishtp_cl_device *cl_device)
445 {
446 	struct ishtp_cl_rb *rb_in_proc;
447 	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);
448 
449 	while ((rb_in_proc = ishtp_cl_rx_get_rb(loader_ishtp_cl)) != NULL) {
450 		/* Process the data packet from firmware */
451 		process_recv(loader_ishtp_cl, rb_in_proc);
452 	}
453 }
454 
455 /**
456  * ish_query_loader_prop() -  Query ISH Shim firmware loader
457  * @client_data:	Client data instance
458  * @fw:			Poiner to firmware data struct in host memory
459  * @fw_info:		Loader firmware properties
460  *
461  * This function queries the ISH Shim firmware loader for capabilities.
462  *
463  * Return: 0 for success, negative error code for failure.
464  */
465 static int ish_query_loader_prop(struct ishtp_cl_data *client_data,
466 				 const struct firmware *fw,
467 				 struct shim_fw_info *fw_info)
468 {
469 	int rv;
470 	struct loader_xfer_query ldr_xfer_query;
471 	struct loader_xfer_query_response ldr_xfer_query_resp;
472 
473 	memset(&ldr_xfer_query, 0, sizeof(ldr_xfer_query));
474 	ldr_xfer_query.hdr.command = LOADER_CMD_XFER_QUERY;
475 	ldr_xfer_query.image_size = fw->size;
476 	rv = loader_cl_send(client_data,
477 			    (u8 *)&ldr_xfer_query,
478 			    sizeof(ldr_xfer_query),
479 			    (u8 *)&ldr_xfer_query_resp,
480 			    sizeof(ldr_xfer_query_resp));
481 	if (rv < 0) {
482 		client_data->flag_retry = true;
483 		return rv;
484 	}
485 
486 	/* On success, the return value is the received buffer size */
487 	if (rv != sizeof(struct loader_xfer_query_response)) {
488 		dev_err(cl_data_to_dev(client_data),
489 			"data size %d is not equal to size of loader_xfer_query_response %zu\n",
490 			rv, sizeof(struct loader_xfer_query_response));
491 		client_data->flag_retry = true;
492 		return -EMSGSIZE;
493 	}
494 
495 	/* Save fw_info for use outside this function */
496 	*fw_info = ldr_xfer_query_resp.fw_info;
497 
498 	/* Loader firmware properties */
499 	dev_dbg(cl_data_to_dev(client_data),
500 		"ish_fw_version: major=%d minor=%d hotfix=%d build=%d protocol_version=0x%x loader_version=%d\n",
501 		fw_info->ish_fw_version.major,
502 		fw_info->ish_fw_version.minor,
503 		fw_info->ish_fw_version.hotfix,
504 		fw_info->ish_fw_version.build,
505 		fw_info->protocol_version,
506 		fw_info->ldr_version.value);
507 
508 	dev_dbg(cl_data_to_dev(client_data),
509 		"loader_capability: max_fw_image_size=0x%x xfer_mode=%d max_dma_buf_size=0x%x dma_buf_size_limit=0x%x\n",
510 		fw_info->ldr_capability.max_fw_image_size,
511 		fw_info->ldr_capability.xfer_mode,
512 		fw_info->ldr_capability.max_dma_buf_size,
513 		dma_buf_size_limit);
514 
515 	/* Sanity checks */
516 	if (fw_info->ldr_capability.max_fw_image_size < fw->size) {
517 		dev_err(cl_data_to_dev(client_data),
518 			"ISH firmware size %zu is greater than Shim firmware loader max supported %d\n",
519 			fw->size,
520 			fw_info->ldr_capability.max_fw_image_size);
521 		return -ENOSPC;
522 	}
523 
524 	/* For DMA the buffer size should be multiple of cacheline size */
525 	if ((fw_info->ldr_capability.xfer_mode & LOADER_XFER_MODE_DIRECT_DMA) &&
526 	    (fw_info->ldr_capability.max_dma_buf_size % L1_CACHE_BYTES)) {
527 		dev_err(cl_data_to_dev(client_data),
528 			"Shim firmware loader buffer size %d should be multiple of cacheline\n",
529 			fw_info->ldr_capability.max_dma_buf_size);
530 		return -EINVAL;
531 	}
532 
533 	return 0;
534 }
535 
536 /**
537  * ish_fw_xfer_ishtp()	Loads ISH firmware using ishtp interface
538  * @client_data:	Client data instance
539  * @fw:			Pointer to firmware data struct in host memory
540  *
541  * This function uses ISH-TP to transfer ISH firmware from host to
542  * ISH SRAM. Lower layers may use IPC or DMA depending on firmware
543  * support.
544  *
545  * Return: 0 for success, negative error code for failure.
546  */
547 static int ish_fw_xfer_ishtp(struct ishtp_cl_data *client_data,
548 			     const struct firmware *fw)
549 {
550 	int rv;
551 	u32 fragment_offset, fragment_size, payload_max_size;
552 	struct loader_xfer_ipc_fragment *ldr_xfer_ipc_frag;
553 	struct loader_msg_hdr ldr_xfer_ipc_ack;
554 
555 	payload_max_size =
556 		LOADER_SHIM_IPC_BUF_SIZE - IPC_FRAGMENT_DATA_PREAMBLE;
557 
558 	ldr_xfer_ipc_frag = kzalloc(LOADER_SHIM_IPC_BUF_SIZE, GFP_KERNEL);
559 	if (!ldr_xfer_ipc_frag) {
560 		client_data->flag_retry = true;
561 		return -ENOMEM;
562 	}
563 
564 	ldr_xfer_ipc_frag->fragment.hdr.command = LOADER_CMD_XFER_FRAGMENT;
565 	ldr_xfer_ipc_frag->fragment.xfer_mode = LOADER_XFER_MODE_ISHTP;
566 
567 	/* Break the firmware image into fragments and send as ISH-TP payload */
568 	fragment_offset = 0;
569 	while (fragment_offset < fw->size) {
570 		if (fragment_offset + payload_max_size < fw->size) {
571 			fragment_size = payload_max_size;
572 			ldr_xfer_ipc_frag->fragment.is_last = 0;
573 		} else {
574 			fragment_size = fw->size - fragment_offset;
575 			ldr_xfer_ipc_frag->fragment.is_last = 1;
576 		}
577 
578 		ldr_xfer_ipc_frag->fragment.offset = fragment_offset;
579 		ldr_xfer_ipc_frag->fragment.size = fragment_size;
580 		memcpy(ldr_xfer_ipc_frag->data,
581 		       &fw->data[fragment_offset],
582 		       fragment_size);
583 
584 		dev_dbg(cl_data_to_dev(client_data),
585 			"xfer_mode=ipc offset=0x%08x size=0x%08x is_last=%d\n",
586 			ldr_xfer_ipc_frag->fragment.offset,
587 			ldr_xfer_ipc_frag->fragment.size,
588 			ldr_xfer_ipc_frag->fragment.is_last);
589 
590 		rv = loader_cl_send(client_data,
591 				    (u8 *)ldr_xfer_ipc_frag,
592 				    IPC_FRAGMENT_DATA_PREAMBLE + fragment_size,
593 				    (u8 *)&ldr_xfer_ipc_ack,
594 				    sizeof(ldr_xfer_ipc_ack));
595 		if (rv < 0) {
596 			client_data->flag_retry = true;
597 			goto end_err_resp_buf_release;
598 		}
599 
600 		fragment_offset += fragment_size;
601 	}
602 
603 	kfree(ldr_xfer_ipc_frag);
604 	return 0;
605 
606 end_err_resp_buf_release:
607 	/* Free ISH buffer if not done already, in error case */
608 	kfree(ldr_xfer_ipc_frag);
609 	return rv;
610 }
611 
612 /**
613  * ish_fw_xfer_direct_dma() - Loads ISH firmware using direct dma
614  * @client_data:	Client data instance
615  * @fw:			Pointer to firmware data struct in host memory
616  * @fw_info:		Loader firmware properties
617  *
618  * Host firmware load is a unique case where we need to download
619  * a large firmware image (200+ Kb). This function implements
620  * direct DMA transfer in kernel and ISH firmware. This allows
621  * us to overcome the ISH-TP 4 Kb limit, and allows us to DMA
622  * directly to ISH UMA at location of choice.
623  * Function depends on corresponding support in ISH firmware.
624  *
625  * Return: 0 for success, negative error code for failure.
626  */
627 static int ish_fw_xfer_direct_dma(struct ishtp_cl_data *client_data,
628 				  const struct firmware *fw,
629 				  const struct shim_fw_info fw_info)
630 {
631 	int rv;
632 	void *dma_buf;
633 	dma_addr_t dma_buf_phy;
634 	u32 fragment_offset, fragment_size, payload_max_size;
635 	struct loader_msg_hdr ldr_xfer_dma_frag_ack;
636 	struct loader_xfer_dma_fragment ldr_xfer_dma_frag;
637 	struct device *devc = ishtp_get_pci_device(client_data->cl_device);
638 	u32 shim_fw_buf_size =
639 		fw_info.ldr_capability.max_dma_buf_size;
640 
641 	/*
642 	 * payload_max_size should be set to minimum of
643 	 *  (1) Size of firmware to be loaded,
644 	 *  (2) Max DMA buffer size supported by Shim firmware,
645 	 *  (3) DMA buffer size limit set by boot_param dma_buf_size_limit.
646 	 */
647 	payload_max_size = min3(fw->size,
648 				(size_t)shim_fw_buf_size,
649 				(size_t)dma_buf_size_limit);
650 
651 	/*
652 	 * Buffer size should be multiple of cacheline size
653 	 * if it's not, select the previous cacheline boundary.
654 	 */
655 	payload_max_size &= ~(L1_CACHE_BYTES - 1);
656 
657 	dma_buf = kmalloc(payload_max_size, GFP_KERNEL | GFP_DMA32);
658 	if (!dma_buf) {
659 		client_data->flag_retry = true;
660 		return -ENOMEM;
661 	}
662 
663 	dma_buf_phy = dma_map_single(devc, dma_buf, payload_max_size,
664 				     DMA_TO_DEVICE);
665 	if (dma_mapping_error(devc, dma_buf_phy)) {
666 		dev_err(cl_data_to_dev(client_data), "DMA map failed\n");
667 		client_data->flag_retry = true;
668 		rv = -ENOMEM;
669 		goto end_err_dma_buf_release;
670 	}
671 
672 	ldr_xfer_dma_frag.fragment.hdr.command = LOADER_CMD_XFER_FRAGMENT;
673 	ldr_xfer_dma_frag.fragment.xfer_mode = LOADER_XFER_MODE_DIRECT_DMA;
674 	ldr_xfer_dma_frag.ddr_phys_addr = (u64)dma_buf_phy;
675 
676 	/* Send the firmware image in chucks of payload_max_size */
677 	fragment_offset = 0;
678 	while (fragment_offset < fw->size) {
679 		if (fragment_offset + payload_max_size < fw->size) {
680 			fragment_size = payload_max_size;
681 			ldr_xfer_dma_frag.fragment.is_last = 0;
682 		} else {
683 			fragment_size = fw->size - fragment_offset;
684 			ldr_xfer_dma_frag.fragment.is_last = 1;
685 		}
686 
687 		ldr_xfer_dma_frag.fragment.offset = fragment_offset;
688 		ldr_xfer_dma_frag.fragment.size = fragment_size;
689 		memcpy(dma_buf, &fw->data[fragment_offset], fragment_size);
690 
691 		dma_sync_single_for_device(devc, dma_buf_phy,
692 					   payload_max_size,
693 					   DMA_TO_DEVICE);
694 
695 		/*
696 		 * Flush cache here because the dma_sync_single_for_device()
697 		 * does not do for x86.
698 		 */
699 		clflush_cache_range(dma_buf, payload_max_size);
700 
701 		dev_dbg(cl_data_to_dev(client_data),
702 			"xfer_mode=dma offset=0x%08x size=0x%x is_last=%d ddr_phys_addr=0x%016llx\n",
703 			ldr_xfer_dma_frag.fragment.offset,
704 			ldr_xfer_dma_frag.fragment.size,
705 			ldr_xfer_dma_frag.fragment.is_last,
706 			ldr_xfer_dma_frag.ddr_phys_addr);
707 
708 		rv = loader_cl_send(client_data,
709 				    (u8 *)&ldr_xfer_dma_frag,
710 				    sizeof(ldr_xfer_dma_frag),
711 				    (u8 *)&ldr_xfer_dma_frag_ack,
712 				    sizeof(ldr_xfer_dma_frag_ack));
713 		if (rv < 0) {
714 			client_data->flag_retry = true;
715 			goto end_err_resp_buf_release;
716 		}
717 
718 		fragment_offset += fragment_size;
719 	}
720 
721 	dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
722 	kfree(dma_buf);
723 	return 0;
724 
725 end_err_resp_buf_release:
726 	/* Free ISH buffer if not done already, in error case */
727 	dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
728 end_err_dma_buf_release:
729 	kfree(dma_buf);
730 	return rv;
731 }
732 
733 /**
734  * ish_fw_start()	Start executing ISH main firmware
735  * @client_data:	client data instance
736  *
737  * This function sends message to Shim firmware loader to start
738  * the execution of ISH main firmware.
739  *
740  * Return: 0 for success, negative error code for failure.
741  */
742 static int ish_fw_start(struct ishtp_cl_data *client_data)
743 {
744 	struct loader_start ldr_start;
745 	struct loader_msg_hdr ldr_start_ack;
746 
747 	memset(&ldr_start, 0, sizeof(ldr_start));
748 	ldr_start.hdr.command = LOADER_CMD_START;
749 	return loader_cl_send(client_data,
750 			    (u8 *)&ldr_start,
751 			    sizeof(ldr_start),
752 			    (u8 *)&ldr_start_ack,
753 			    sizeof(ldr_start_ack));
754 }
755 
756 /**
757  * load_fw_from_host()	Loads ISH firmware from host
758  * @client_data:	Client data instance
759  *
760  * This function loads the ISH firmware to ISH SRAM and starts execution
761  *
762  * Return: 0 for success, negative error code for failure.
763  */
764 static int load_fw_from_host(struct ishtp_cl_data *client_data)
765 {
766 	int rv;
767 	u32 xfer_mode;
768 	char *filename;
769 	const struct firmware *fw;
770 	struct shim_fw_info fw_info;
771 	struct ishtp_cl *loader_ishtp_cl = client_data->loader_ishtp_cl;
772 
773 	client_data->flag_retry = false;
774 
775 	filename = kzalloc(FILENAME_SIZE, GFP_KERNEL);
776 	if (!filename) {
777 		client_data->flag_retry = true;
778 		rv = -ENOMEM;
779 		goto end_error;
780 	}
781 
782 	/* Get filename of the ISH firmware to be loaded */
783 	rv = get_firmware_variant(client_data, filename);
784 	if (rv < 0)
785 		goto end_err_filename_buf_release;
786 
787 	rv = request_firmware(&fw, filename, cl_data_to_dev(client_data));
788 	if (rv < 0)
789 		goto end_err_filename_buf_release;
790 
791 	/* Step 1: Query Shim firmware loader properties */
792 
793 	rv = ish_query_loader_prop(client_data, fw, &fw_info);
794 	if (rv < 0)
795 		goto end_err_fw_release;
796 
797 	/* Step 2: Send the main firmware image to be loaded, to ISH SRAM */
798 
799 	xfer_mode = fw_info.ldr_capability.xfer_mode;
800 	if (xfer_mode & LOADER_XFER_MODE_DIRECT_DMA) {
801 		rv = ish_fw_xfer_direct_dma(client_data, fw, fw_info);
802 	} else if (xfer_mode & LOADER_XFER_MODE_ISHTP) {
803 		rv = ish_fw_xfer_ishtp(client_data, fw);
804 	} else {
805 		dev_err(cl_data_to_dev(client_data),
806 			"No transfer mode selected in firmware\n");
807 		rv = -EINVAL;
808 	}
809 	if (rv < 0)
810 		goto end_err_fw_release;
811 
812 	/* Step 3: Start ISH main firmware exeuction */
813 
814 	rv = ish_fw_start(client_data);
815 	if (rv < 0)
816 		goto end_err_fw_release;
817 
818 	release_firmware(fw);
819 	dev_info(cl_data_to_dev(client_data), "ISH firmware %s loaded\n",
820 		 filename);
821 	kfree(filename);
822 	return 0;
823 
824 end_err_fw_release:
825 	release_firmware(fw);
826 end_err_filename_buf_release:
827 	kfree(filename);
828 end_error:
829 	/* Keep a count of retries, and give up after 3 attempts */
830 	if (client_data->flag_retry &&
831 	    client_data->retry_count++ < MAX_LOAD_ATTEMPTS) {
832 		dev_warn(cl_data_to_dev(client_data),
833 			 "ISH host firmware load failed %d. Resetting ISH, and trying again..\n",
834 			 rv);
835 		ish_hw_reset(ishtp_get_ishtp_device(loader_ishtp_cl));
836 	} else {
837 		dev_err(cl_data_to_dev(client_data),
838 			"ISH host firmware load failed %d\n", rv);
839 	}
840 	return rv;
841 }
842 
843 static void load_fw_from_host_handler(struct work_struct *work)
844 {
845 	struct ishtp_cl_data *client_data;
846 
847 	client_data = container_of(work, struct ishtp_cl_data,
848 				   work_fw_load);
849 	load_fw_from_host(client_data);
850 }
851 
852 /**
853  * loader_init() -	Init function for ISH-TP client
854  * @loader_ishtp_cl:	ISH-TP client instance
855  * @reset:		true if called for init after reset
856  *
857  * Return: 0 for success, negative error code for failure
858  */
859 static int loader_init(struct ishtp_cl *loader_ishtp_cl, int reset)
860 {
861 	int rv;
862 	struct ishtp_fw_client *fw_client;
863 	struct ishtp_cl_data *client_data =
864 		ishtp_get_client_data(loader_ishtp_cl);
865 
866 	dev_dbg(cl_data_to_dev(client_data), "reset flag: %d\n", reset);
867 
868 	rv = ishtp_cl_link(loader_ishtp_cl);
869 	if (rv < 0) {
870 		dev_err(cl_data_to_dev(client_data), "ishtp_cl_link failed\n");
871 		return rv;
872 	}
873 
874 	/* Connect to firmware client */
875 	ishtp_set_tx_ring_size(loader_ishtp_cl, LOADER_CL_TX_RING_SIZE);
876 	ishtp_set_rx_ring_size(loader_ishtp_cl, LOADER_CL_RX_RING_SIZE);
877 
878 	fw_client =
879 		ishtp_fw_cl_get_client(ishtp_get_ishtp_device(loader_ishtp_cl),
880 				       &loader_ishtp_guid);
881 	if (!fw_client) {
882 		dev_err(cl_data_to_dev(client_data),
883 			"ISH client uuid not found\n");
884 		rv = -ENOENT;
885 		goto err_cl_unlink;
886 	}
887 
888 	ishtp_cl_set_fw_client_id(loader_ishtp_cl,
889 				  ishtp_get_fw_client_id(fw_client));
890 	ishtp_set_connection_state(loader_ishtp_cl, ISHTP_CL_CONNECTING);
891 
892 	rv = ishtp_cl_connect(loader_ishtp_cl);
893 	if (rv < 0) {
894 		dev_err(cl_data_to_dev(client_data), "Client connect fail\n");
895 		goto err_cl_unlink;
896 	}
897 
898 	dev_dbg(cl_data_to_dev(client_data), "Client connected\n");
899 
900 	ishtp_register_event_cb(client_data->cl_device, loader_cl_event_cb);
901 
902 	return 0;
903 
904 err_cl_unlink:
905 	ishtp_cl_unlink(loader_ishtp_cl);
906 	return rv;
907 }
908 
909 static void loader_deinit(struct ishtp_cl *loader_ishtp_cl)
910 {
911 	ishtp_set_connection_state(loader_ishtp_cl, ISHTP_CL_DISCONNECTING);
912 	ishtp_cl_disconnect(loader_ishtp_cl);
913 	ishtp_cl_unlink(loader_ishtp_cl);
914 	ishtp_cl_flush_queues(loader_ishtp_cl);
915 
916 	/* Disband and free all Tx and Rx client-level rings */
917 	ishtp_cl_free(loader_ishtp_cl);
918 }
919 
920 static void reset_handler(struct work_struct *work)
921 {
922 	int rv;
923 	struct ishtp_cl_data *client_data;
924 	struct ishtp_cl *loader_ishtp_cl;
925 	struct ishtp_cl_device *cl_device;
926 
927 	client_data = container_of(work, struct ishtp_cl_data,
928 				   work_ishtp_reset);
929 
930 	loader_ishtp_cl = client_data->loader_ishtp_cl;
931 	cl_device = client_data->cl_device;
932 
933 	/* Unlink, flush queues & start again */
934 	ishtp_cl_unlink(loader_ishtp_cl);
935 	ishtp_cl_flush_queues(loader_ishtp_cl);
936 	ishtp_cl_free(loader_ishtp_cl);
937 
938 	loader_ishtp_cl = ishtp_cl_allocate(cl_device);
939 	if (!loader_ishtp_cl)
940 		return;
941 
942 	ishtp_set_drvdata(cl_device, loader_ishtp_cl);
943 	ishtp_set_client_data(loader_ishtp_cl, client_data);
944 	client_data->loader_ishtp_cl = loader_ishtp_cl;
945 	client_data->cl_device = cl_device;
946 
947 	rv = loader_init(loader_ishtp_cl, 1);
948 	if (rv < 0) {
949 		dev_err(ishtp_device(cl_device), "Reset Failed\n");
950 		return;
951 	}
952 
953 	/* ISH firmware loading from host */
954 	load_fw_from_host(client_data);
955 }
956 
957 /**
958  * loader_ishtp_cl_probe() - ISH-TP client driver probe
959  * @cl_device:		ISH-TP client device instance
960  *
961  * This function gets called on device create on ISH-TP bus
962  *
963  * Return: 0 for success, negative error code for failure
964  */
965 static int loader_ishtp_cl_probe(struct ishtp_cl_device *cl_device)
966 {
967 	struct ishtp_cl *loader_ishtp_cl;
968 	struct ishtp_cl_data *client_data;
969 	int rv;
970 
971 	client_data = devm_kzalloc(ishtp_device(cl_device),
972 				   sizeof(*client_data),
973 				   GFP_KERNEL);
974 	if (!client_data)
975 		return -ENOMEM;
976 
977 	loader_ishtp_cl = ishtp_cl_allocate(cl_device);
978 	if (!loader_ishtp_cl)
979 		return -ENOMEM;
980 
981 	ishtp_set_drvdata(cl_device, loader_ishtp_cl);
982 	ishtp_set_client_data(loader_ishtp_cl, client_data);
983 	client_data->loader_ishtp_cl = loader_ishtp_cl;
984 	client_data->cl_device = cl_device;
985 
986 	init_waitqueue_head(&client_data->response.wait_queue);
987 
988 	INIT_WORK(&client_data->work_ishtp_reset,
989 		  reset_handler);
990 	INIT_WORK(&client_data->work_fw_load,
991 		  load_fw_from_host_handler);
992 
993 	rv = loader_init(loader_ishtp_cl, 0);
994 	if (rv < 0) {
995 		ishtp_cl_free(loader_ishtp_cl);
996 		return rv;
997 	}
998 	ishtp_get_device(cl_device);
999 
1000 	client_data->retry_count = 0;
1001 
1002 	/* ISH firmware loading from host */
1003 	schedule_work(&client_data->work_fw_load);
1004 
1005 	return 0;
1006 }
1007 
1008 /**
1009  * loader_ishtp_cl_remove() - ISH-TP client driver remove
1010  * @cl_device:		ISH-TP client device instance
1011  *
1012  * This function gets called on device remove on ISH-TP bus
1013  *
1014  * Return: 0
1015  */
1016 static int loader_ishtp_cl_remove(struct ishtp_cl_device *cl_device)
1017 {
1018 	struct ishtp_cl_data *client_data;
1019 	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);
1020 
1021 	client_data = ishtp_get_client_data(loader_ishtp_cl);
1022 
1023 	/*
1024 	 * The sequence of the following two cancel_work_sync() is
1025 	 * important. The work_fw_load can in turn schedue
1026 	 * work_ishtp_reset, so first cancel work_fw_load then
1027 	 * cancel work_ishtp_reset.
1028 	 */
1029 	cancel_work_sync(&client_data->work_fw_load);
1030 	cancel_work_sync(&client_data->work_ishtp_reset);
1031 	loader_deinit(loader_ishtp_cl);
1032 	ishtp_put_device(cl_device);
1033 
1034 	return 0;
1035 }
1036 
1037 /**
1038  * loader_ishtp_cl_reset() - ISH-TP client driver reset
1039  * @cl_device:		ISH-TP client device instance
1040  *
1041  * This function gets called on device reset on ISH-TP bus
1042  *
1043  * Return: 0
1044  */
1045 static int loader_ishtp_cl_reset(struct ishtp_cl_device *cl_device)
1046 {
1047 	struct ishtp_cl_data *client_data;
1048 	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);
1049 
1050 	client_data = ishtp_get_client_data(loader_ishtp_cl);
1051 
1052 	schedule_work(&client_data->work_ishtp_reset);
1053 
1054 	return 0;
1055 }
1056 
1057 static struct ishtp_cl_driver	loader_ishtp_cl_driver = {
1058 	.name = "ish-loader",
1059 	.guid = &loader_ishtp_guid,
1060 	.probe = loader_ishtp_cl_probe,
1061 	.remove = loader_ishtp_cl_remove,
1062 	.reset = loader_ishtp_cl_reset,
1063 };
1064 
1065 static int __init ish_loader_init(void)
1066 {
1067 	return ishtp_cl_driver_register(&loader_ishtp_cl_driver, THIS_MODULE);
1068 }
1069 
1070 static void __exit ish_loader_exit(void)
1071 {
1072 	ishtp_cl_driver_unregister(&loader_ishtp_cl_driver);
1073 }
1074 
1075 late_initcall(ish_loader_init);
1076 module_exit(ish_loader_exit);
1077 
1078 module_param(dma_buf_size_limit, int, 0644);
1079 MODULE_PARM_DESC(dma_buf_size_limit, "Limit the DMA buf size to this value in bytes");
1080 
1081 MODULE_DESCRIPTION("ISH ISH-TP Host firmware Loader Client Driver");
1082 MODULE_AUTHOR("Rushikesh S Kadam <rushikesh.s.kadam@intel.com>");
1083 
1084 MODULE_LICENSE("GPL v2");
1085 MODULE_ALIAS("ishtp:*");
1086