1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/slab.h>
4 #include <linux/ctype.h>
5 #include <linux/kernel.h>
6 #include <linux/delay.h>
7 #include <linux/pci_ids.h>
8 #include "adf_accel_devices.h"
9 #include "adf_common_drv.h"
10 #include "icp_qat_uclo.h"
11 #include "icp_qat_hal.h"
12 #include "icp_qat_fw_loader_handle.h"
13 
14 #define UWORD_CPYBUF_SIZE 1024
15 #define INVLD_UWORD 0xffffffffffull
16 #define PID_MINOR_REV 0xf
17 #define PID_MAJOR_REV (0xf << 4)
18 
19 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
20 				 unsigned int ae, unsigned int image_num)
21 {
22 	struct icp_qat_uclo_aedata *ae_data;
23 	struct icp_qat_uclo_encapme *encap_image;
24 	struct icp_qat_uclo_page *page = NULL;
25 	struct icp_qat_uclo_aeslice *ae_slice = NULL;
26 
27 	ae_data = &obj_handle->ae_data[ae];
28 	encap_image = &obj_handle->ae_uimage[image_num];
29 	ae_slice = &ae_data->ae_slices[ae_data->slice_num];
30 	ae_slice->encap_image = encap_image;
31 
32 	if (encap_image->img_ptr) {
33 		ae_slice->ctx_mask_assigned =
34 					encap_image->img_ptr->ctx_assigned;
35 		ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
36 	} else {
37 		ae_slice->ctx_mask_assigned = 0;
38 	}
39 	ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL);
40 	if (!ae_slice->region)
41 		return -ENOMEM;
42 	ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL);
43 	if (!ae_slice->page)
44 		goto out_err;
45 	page = ae_slice->page;
46 	page->encap_page = encap_image->page;
47 	ae_slice->page->region = ae_slice->region;
48 	ae_data->slice_num++;
49 	return 0;
50 out_err:
51 	kfree(ae_slice->region);
52 	ae_slice->region = NULL;
53 	return -ENOMEM;
54 }
55 
56 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
57 {
58 	unsigned int i;
59 
60 	if (!ae_data) {
61 		pr_err("QAT: bad argument, ae_data is NULL\n ");
62 		return -EINVAL;
63 	}
64 
65 	for (i = 0; i < ae_data->slice_num; i++) {
66 		kfree(ae_data->ae_slices[i].region);
67 		ae_data->ae_slices[i].region = NULL;
68 		kfree(ae_data->ae_slices[i].page);
69 		ae_data->ae_slices[i].page = NULL;
70 	}
71 	return 0;
72 }
73 
74 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
75 				 unsigned int str_offset)
76 {
77 	if (!str_table->table_len || str_offset > str_table->table_len)
78 		return NULL;
79 	return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
80 }
81 
82 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
83 {
84 	int maj = hdr->maj_ver & 0xff;
85 	int min = hdr->min_ver & 0xff;
86 
87 	if (hdr->file_id != ICP_QAT_UOF_FID) {
88 		pr_err("QAT: Invalid header 0x%x\n", hdr->file_id);
89 		return -EINVAL;
90 	}
91 	if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
92 		pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n",
93 		       maj, min);
94 		return -EINVAL;
95 	}
96 	return 0;
97 }
98 
99 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr)
100 {
101 	int maj = suof_hdr->maj_ver & 0xff;
102 	int min = suof_hdr->min_ver & 0xff;
103 
104 	if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
105 		pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id);
106 		return -EINVAL;
107 	}
108 	if (suof_hdr->fw_type != 0) {
109 		pr_err("QAT: unsupported firmware type\n");
110 		return -EINVAL;
111 	}
112 	if (suof_hdr->num_chunks <= 0x1) {
113 		pr_err("QAT: SUOF chunk amount is incorrect\n");
114 		return -EINVAL;
115 	}
116 	if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
117 		pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n",
118 		       maj, min);
119 		return -EINVAL;
120 	}
121 	return 0;
122 }
123 
124 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
125 				      unsigned int addr, unsigned int *val,
126 				      unsigned int num_in_bytes)
127 {
128 	unsigned int outval;
129 	unsigned char *ptr = (unsigned char *)val;
130 
131 	while (num_in_bytes) {
132 		memcpy(&outval, ptr, 4);
133 		SRAM_WRITE(handle, addr, outval);
134 		num_in_bytes -= 4;
135 		ptr += 4;
136 		addr += 4;
137 	}
138 }
139 
140 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
141 				      unsigned char ae, unsigned int addr,
142 				      unsigned int *val,
143 				      unsigned int num_in_bytes)
144 {
145 	unsigned int outval;
146 	unsigned char *ptr = (unsigned char *)val;
147 
148 	addr >>= 0x2; /* convert to uword address */
149 
150 	while (num_in_bytes) {
151 		memcpy(&outval, ptr, 4);
152 		qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
153 		num_in_bytes -= 4;
154 		ptr += 4;
155 	}
156 }
157 
158 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
159 				   unsigned char ae,
160 				   struct icp_qat_uof_batch_init
161 				   *umem_init_header)
162 {
163 	struct icp_qat_uof_batch_init *umem_init;
164 
165 	if (!umem_init_header)
166 		return;
167 	umem_init = umem_init_header->next;
168 	while (umem_init) {
169 		unsigned int addr, *value, size;
170 
171 		ae = umem_init->ae;
172 		addr = umem_init->addr;
173 		value = umem_init->value;
174 		size = umem_init->size;
175 		qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
176 		umem_init = umem_init->next;
177 	}
178 }
179 
180 static void
181 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
182 				 struct icp_qat_uof_batch_init **base)
183 {
184 	struct icp_qat_uof_batch_init *umem_init;
185 
186 	umem_init = *base;
187 	while (umem_init) {
188 		struct icp_qat_uof_batch_init *pre;
189 
190 		pre = umem_init;
191 		umem_init = umem_init->next;
192 		kfree(pre);
193 	}
194 	*base = NULL;
195 }
196 
197 static int qat_uclo_parse_num(char *str, unsigned int *num)
198 {
199 	char buf[16] = {0};
200 	unsigned long ae = 0;
201 	int i;
202 
203 	strncpy(buf, str, 15);
204 	for (i = 0; i < 16; i++) {
205 		if (!isdigit(buf[i])) {
206 			buf[i] = '\0';
207 			break;
208 		}
209 	}
210 	if ((kstrtoul(buf, 10, &ae)))
211 		return -EFAULT;
212 
213 	*num = (unsigned int)ae;
214 	return 0;
215 }
216 
217 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
218 				     struct icp_qat_uof_initmem *init_mem,
219 				     unsigned int size_range, unsigned int *ae)
220 {
221 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
222 	char *str;
223 
224 	if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
225 		pr_err("QAT: initmem is out of range");
226 		return -EINVAL;
227 	}
228 	if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
229 		pr_err("QAT: Memory scope for init_mem error\n");
230 		return -EINVAL;
231 	}
232 	str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
233 	if (!str) {
234 		pr_err("QAT: AE name assigned in UOF init table is NULL\n");
235 		return -EINVAL;
236 	}
237 	if (qat_uclo_parse_num(str, ae)) {
238 		pr_err("QAT: Parse num for AE number failed\n");
239 		return -EINVAL;
240 	}
241 	if (*ae >= ICP_QAT_UCLO_MAX_AE) {
242 		pr_err("QAT: ae %d out of range\n", *ae);
243 		return -EINVAL;
244 	}
245 	return 0;
246 }
247 
248 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle
249 					   *handle, struct icp_qat_uof_initmem
250 					   *init_mem, unsigned int ae,
251 					   struct icp_qat_uof_batch_init
252 					   **init_tab_base)
253 {
254 	struct icp_qat_uof_batch_init *init_header, *tail;
255 	struct icp_qat_uof_batch_init *mem_init, *tail_old;
256 	struct icp_qat_uof_memvar_attr *mem_val_attr;
257 	unsigned int i, flag = 0;
258 
259 	mem_val_attr =
260 		(struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
261 		sizeof(struct icp_qat_uof_initmem));
262 
263 	init_header = *init_tab_base;
264 	if (!init_header) {
265 		init_header = kzalloc(sizeof(*init_header), GFP_KERNEL);
266 		if (!init_header)
267 			return -ENOMEM;
268 		init_header->size = 1;
269 		*init_tab_base = init_header;
270 		flag = 1;
271 	}
272 	tail_old = init_header;
273 	while (tail_old->next)
274 		tail_old = tail_old->next;
275 	tail = tail_old;
276 	for (i = 0; i < init_mem->val_attr_num; i++) {
277 		mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL);
278 		if (!mem_init)
279 			goto out_err;
280 		mem_init->ae = ae;
281 		mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
282 		mem_init->value = &mem_val_attr->value;
283 		mem_init->size = 4;
284 		mem_init->next = NULL;
285 		tail->next = mem_init;
286 		tail = mem_init;
287 		init_header->size += qat_hal_get_ins_num();
288 		mem_val_attr++;
289 	}
290 	return 0;
291 out_err:
292 	/* Do not free the list head unless we allocated it. */
293 	tail_old = tail_old->next;
294 	if (flag) {
295 		kfree(*init_tab_base);
296 		*init_tab_base = NULL;
297 	}
298 
299 	while (tail_old) {
300 		mem_init = tail_old->next;
301 		kfree(tail_old);
302 		tail_old = mem_init;
303 	}
304 	return -ENOMEM;
305 }
306 
307 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
308 				  struct icp_qat_uof_initmem *init_mem)
309 {
310 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
311 	unsigned int ae;
312 
313 	if (qat_uclo_fetch_initmem_ae(handle, init_mem,
314 				      handle->chip_info->lm_size, &ae))
315 		return -EINVAL;
316 	if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
317 					    &obj_handle->lm_init_tab[ae]))
318 		return -EINVAL;
319 	return 0;
320 }
321 
322 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
323 				  struct icp_qat_uof_initmem *init_mem)
324 {
325 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
326 	unsigned int ae, ustore_size, uaddr, i;
327 	struct icp_qat_uclo_aedata *aed;
328 
329 	ustore_size = obj_handle->ustore_phy_size;
330 	if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
331 		return -EINVAL;
332 	if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
333 					    &obj_handle->umem_init_tab[ae]))
334 		return -EINVAL;
335 	/* set the highest ustore address referenced */
336 	uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
337 	aed = &obj_handle->ae_data[ae];
338 	for (i = 0; i < aed->slice_num; i++) {
339 		if (aed->ae_slices[i].encap_image->uwords_num < uaddr)
340 			aed->ae_slices[i].encap_image->uwords_num = uaddr;
341 	}
342 	return 0;
343 }
344 
345 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
346 				   struct icp_qat_uof_initmem *init_mem)
347 {
348 	switch (init_mem->region) {
349 	case ICP_QAT_UOF_LMEM_REGION:
350 		if (qat_uclo_init_lmem_seg(handle, init_mem))
351 			return -EINVAL;
352 		break;
353 	case ICP_QAT_UOF_UMEM_REGION:
354 		if (qat_uclo_init_umem_seg(handle, init_mem))
355 			return -EINVAL;
356 		break;
357 	default:
358 		pr_err("QAT: initmem region error. region type=0x%x\n",
359 		       init_mem->region);
360 		return -EINVAL;
361 	}
362 	return 0;
363 }
364 
365 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
366 				struct icp_qat_uclo_encapme *image)
367 {
368 	unsigned int i;
369 	struct icp_qat_uclo_encap_page *page;
370 	struct icp_qat_uof_image *uof_image;
371 	unsigned char ae;
372 	unsigned int ustore_size;
373 	unsigned int patt_pos;
374 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
375 	unsigned long ae_mask = handle->hal_handle->ae_mask;
376 	unsigned long cfg_ae_mask = handle->cfg_ae_mask;
377 	u64 *fill_data;
378 
379 	uof_image = image->img_ptr;
380 	fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64),
381 			    GFP_KERNEL);
382 	if (!fill_data)
383 		return -ENOMEM;
384 	for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++)
385 		memcpy(&fill_data[i], &uof_image->fill_pattern,
386 		       sizeof(u64));
387 	page = image->page;
388 
389 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
390 		unsigned long ae_assigned = uof_image->ae_assigned;
391 
392 		if (!test_bit(ae, &ae_assigned))
393 			continue;
394 
395 		if (!test_bit(ae, &cfg_ae_mask))
396 			continue;
397 
398 		ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
399 		patt_pos = page->beg_addr_p + page->micro_words_num;
400 
401 		qat_hal_wr_uwords(handle, (unsigned char)ae, 0,
402 				  page->beg_addr_p, &fill_data[0]);
403 		qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos,
404 				  ustore_size - patt_pos + 1,
405 				  &fill_data[page->beg_addr_p]);
406 	}
407 	kfree(fill_data);
408 	return 0;
409 }
410 
411 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
412 {
413 	int i, ae;
414 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
415 	struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
416 	unsigned long ae_mask = handle->hal_handle->ae_mask;
417 
418 	for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
419 		if (initmem->num_in_bytes) {
420 			if (qat_uclo_init_ae_memory(handle, initmem))
421 				return -EINVAL;
422 		}
423 		initmem = (struct icp_qat_uof_initmem *)((uintptr_t)(
424 			(uintptr_t)initmem +
425 			sizeof(struct icp_qat_uof_initmem)) +
426 			(sizeof(struct icp_qat_uof_memvar_attr) *
427 			initmem->val_attr_num));
428 	}
429 
430 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
431 		if (qat_hal_batch_wr_lm(handle, ae,
432 					obj_handle->lm_init_tab[ae])) {
433 			pr_err("QAT: fail to batch init lmem for AE %d\n", ae);
434 			return -EINVAL;
435 		}
436 		qat_uclo_cleanup_batch_init_list(handle,
437 						 &obj_handle->lm_init_tab[ae]);
438 		qat_uclo_batch_wr_umem(handle, ae,
439 				       obj_handle->umem_init_tab[ae]);
440 		qat_uclo_cleanup_batch_init_list(handle,
441 						 &obj_handle->
442 						 umem_init_tab[ae]);
443 	}
444 	return 0;
445 }
446 
447 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
448 				 char *chunk_id, void *cur)
449 {
450 	int i;
451 	struct icp_qat_uof_chunkhdr *chunk_hdr =
452 	    (struct icp_qat_uof_chunkhdr *)
453 	    ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr));
454 
455 	for (i = 0; i < obj_hdr->num_chunks; i++) {
456 		if ((cur < (void *)&chunk_hdr[i]) &&
457 		    !strncmp(chunk_hdr[i].chunk_id, chunk_id,
458 			     ICP_QAT_UOF_OBJID_LEN)) {
459 			return &chunk_hdr[i];
460 		}
461 	}
462 	return NULL;
463 }
464 
465 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch)
466 {
467 	int i;
468 	unsigned int topbit = 1 << 0xF;
469 	unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
470 
471 	reg ^= inbyte << 0x8;
472 	for (i = 0; i < 0x8; i++) {
473 		if (reg & topbit)
474 			reg = (reg << 1) ^ 0x1021;
475 		else
476 			reg <<= 1;
477 	}
478 	return reg & 0xFFFF;
479 }
480 
481 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num)
482 {
483 	unsigned int chksum = 0;
484 
485 	if (ptr)
486 		while (num--)
487 			chksum = qat_uclo_calc_checksum(chksum, *ptr++);
488 	return chksum;
489 }
490 
491 static struct icp_qat_uclo_objhdr *
492 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr,
493 		   char *chunk_id)
494 {
495 	struct icp_qat_uof_filechunkhdr *file_chunk;
496 	struct icp_qat_uclo_objhdr *obj_hdr;
497 	char *chunk;
498 	int i;
499 
500 	file_chunk = (struct icp_qat_uof_filechunkhdr *)
501 		(buf + sizeof(struct icp_qat_uof_filehdr));
502 	for (i = 0; i < file_hdr->num_chunks; i++) {
503 		if (!strncmp(file_chunk->chunk_id, chunk_id,
504 			     ICP_QAT_UOF_OBJID_LEN)) {
505 			chunk = buf + file_chunk->offset;
506 			if (file_chunk->checksum != qat_uclo_calc_str_checksum(
507 				chunk, file_chunk->size))
508 				break;
509 			obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL);
510 			if (!obj_hdr)
511 				break;
512 			obj_hdr->file_buff = chunk;
513 			obj_hdr->checksum = file_chunk->checksum;
514 			obj_hdr->size = file_chunk->size;
515 			return obj_hdr;
516 		}
517 		file_chunk++;
518 	}
519 	return NULL;
520 }
521 
522 static int
523 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
524 			    struct icp_qat_uof_image *image)
525 {
526 	struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
527 	struct icp_qat_uof_objtable *neigh_reg_tab;
528 	struct icp_qat_uof_code_page *code_page;
529 
530 	code_page = (struct icp_qat_uof_code_page *)
531 			((char *)image + sizeof(struct icp_qat_uof_image));
532 	uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
533 		     code_page->uc_var_tab_offset);
534 	imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
535 		      code_page->imp_var_tab_offset);
536 	imp_expr_tab = (struct icp_qat_uof_objtable *)
537 		       (encap_uof_obj->beg_uof +
538 		       code_page->imp_expr_tab_offset);
539 	if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
540 	    imp_expr_tab->entry_num) {
541 		pr_err("QAT: UOF can't contain imported variable to be parsed\n");
542 		return -EINVAL;
543 	}
544 	neigh_reg_tab = (struct icp_qat_uof_objtable *)
545 			(encap_uof_obj->beg_uof +
546 			code_page->neigh_reg_tab_offset);
547 	if (neigh_reg_tab->entry_num) {
548 		pr_err("QAT: UOF can't contain neighbor register table\n");
549 		return -EINVAL;
550 	}
551 	if (image->numpages > 1) {
552 		pr_err("QAT: UOF can't contain multiple pages\n");
553 		return -EINVAL;
554 	}
555 	if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) {
556 		pr_err("QAT: UOF can't use shared control store feature\n");
557 		return -EFAULT;
558 	}
559 	if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
560 		pr_err("QAT: UOF can't use reloadable feature\n");
561 		return -EFAULT;
562 	}
563 	return 0;
564 }
565 
566 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj
567 				     *encap_uof_obj,
568 				     struct icp_qat_uof_image *img,
569 				     struct icp_qat_uclo_encap_page *page)
570 {
571 	struct icp_qat_uof_code_page *code_page;
572 	struct icp_qat_uof_code_area *code_area;
573 	struct icp_qat_uof_objtable *uword_block_tab;
574 	struct icp_qat_uof_uword_block *uwblock;
575 	int i;
576 
577 	code_page = (struct icp_qat_uof_code_page *)
578 			((char *)img + sizeof(struct icp_qat_uof_image));
579 	page->def_page = code_page->def_page;
580 	page->page_region = code_page->page_region;
581 	page->beg_addr_v = code_page->beg_addr_v;
582 	page->beg_addr_p = code_page->beg_addr_p;
583 	code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
584 						code_page->code_area_offset);
585 	page->micro_words_num = code_area->micro_words_num;
586 	uword_block_tab = (struct icp_qat_uof_objtable *)
587 			  (encap_uof_obj->beg_uof +
588 			  code_area->uword_block_tab);
589 	page->uwblock_num = uword_block_tab->entry_num;
590 	uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab +
591 			sizeof(struct icp_qat_uof_objtable));
592 	page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
593 	for (i = 0; i < uword_block_tab->entry_num; i++)
594 		page->uwblock[i].micro_words =
595 		(uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
596 }
597 
598 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
599 			       struct icp_qat_uclo_encapme *ae_uimage,
600 			       int max_image)
601 {
602 	int i, j;
603 	struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
604 	struct icp_qat_uof_image *image;
605 	struct icp_qat_uof_objtable *ae_regtab;
606 	struct icp_qat_uof_objtable *init_reg_sym_tab;
607 	struct icp_qat_uof_objtable *sbreak_tab;
608 	struct icp_qat_uof_encap_obj *encap_uof_obj =
609 					&obj_handle->encap_uof_obj;
610 
611 	for (j = 0; j < max_image; j++) {
612 		chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
613 						ICP_QAT_UOF_IMAG, chunk_hdr);
614 		if (!chunk_hdr)
615 			break;
616 		image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
617 						     chunk_hdr->offset);
618 		ae_regtab = (struct icp_qat_uof_objtable *)
619 			   (image->reg_tab_offset +
620 			   obj_handle->obj_hdr->file_buff);
621 		ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
622 		ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)
623 			(((char *)ae_regtab) +
624 			sizeof(struct icp_qat_uof_objtable));
625 		init_reg_sym_tab = (struct icp_qat_uof_objtable *)
626 				   (image->init_reg_sym_tab +
627 				   obj_handle->obj_hdr->file_buff);
628 		ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
629 		ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)
630 			(((char *)init_reg_sym_tab) +
631 			sizeof(struct icp_qat_uof_objtable));
632 		sbreak_tab = (struct icp_qat_uof_objtable *)
633 			(image->sbreak_tab + obj_handle->obj_hdr->file_buff);
634 		ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
635 		ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)
636 				      (((char *)sbreak_tab) +
637 				      sizeof(struct icp_qat_uof_objtable));
638 		ae_uimage[j].img_ptr = image;
639 		if (qat_uclo_check_image_compat(encap_uof_obj, image))
640 			goto out_err;
641 		ae_uimage[j].page =
642 			kzalloc(sizeof(struct icp_qat_uclo_encap_page),
643 				GFP_KERNEL);
644 		if (!ae_uimage[j].page)
645 			goto out_err;
646 		qat_uclo_map_image_page(encap_uof_obj, image,
647 					ae_uimage[j].page);
648 	}
649 	return j;
650 out_err:
651 	for (i = 0; i < j; i++)
652 		kfree(ae_uimage[i].page);
653 	return 0;
654 }
655 
656 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
657 {
658 	int i, ae;
659 	int mflag = 0;
660 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
661 	unsigned long ae_mask = handle->hal_handle->ae_mask;
662 	unsigned long cfg_ae_mask = handle->cfg_ae_mask;
663 
664 	for_each_set_bit(ae, &ae_mask, max_ae) {
665 		if (!test_bit(ae, &cfg_ae_mask))
666 			continue;
667 
668 		for (i = 0; i < obj_handle->uimage_num; i++) {
669 			unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned;
670 
671 			if (!test_bit(ae, &ae_assigned))
672 				continue;
673 			mflag = 1;
674 			if (qat_uclo_init_ae_data(obj_handle, ae, i))
675 				return -EINVAL;
676 		}
677 	}
678 	if (!mflag) {
679 		pr_err("QAT: uimage uses AE not set\n");
680 		return -EINVAL;
681 	}
682 	return 0;
683 }
684 
685 static struct icp_qat_uof_strtable *
686 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
687 		       char *tab_name, struct icp_qat_uof_strtable *str_table)
688 {
689 	struct icp_qat_uof_chunkhdr *chunk_hdr;
690 
691 	chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)
692 					obj_hdr->file_buff, tab_name, NULL);
693 	if (chunk_hdr) {
694 		int hdr_size;
695 
696 		memcpy(&str_table->table_len, obj_hdr->file_buff +
697 		       chunk_hdr->offset, sizeof(str_table->table_len));
698 		hdr_size = (char *)&str_table->strings - (char *)str_table;
699 		str_table->strings = (uintptr_t)obj_hdr->file_buff +
700 					chunk_hdr->offset + hdr_size;
701 		return str_table;
702 	}
703 	return NULL;
704 }
705 
706 static void
707 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
708 			   struct icp_qat_uclo_init_mem_table *init_mem_tab)
709 {
710 	struct icp_qat_uof_chunkhdr *chunk_hdr;
711 
712 	chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
713 					ICP_QAT_UOF_IMEM, NULL);
714 	if (chunk_hdr) {
715 		memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof +
716 			chunk_hdr->offset, sizeof(unsigned int));
717 		init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)
718 		(encap_uof_obj->beg_uof + chunk_hdr->offset +
719 		sizeof(unsigned int));
720 	}
721 }
722 
723 static unsigned int
724 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
725 {
726 	switch (handle->pci_dev->device) {
727 	case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
728 		return ICP_QAT_AC_895XCC_DEV_TYPE;
729 	case PCI_DEVICE_ID_INTEL_QAT_C62X:
730 		return ICP_QAT_AC_C62X_DEV_TYPE;
731 	case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
732 		return ICP_QAT_AC_C3XXX_DEV_TYPE;
733 	case ADF_4XXX_PCI_DEVICE_ID:
734 	case ADF_401XX_PCI_DEVICE_ID:
735 	case ADF_402XX_PCI_DEVICE_ID:
736 		return ICP_QAT_AC_4XXX_A_DEV_TYPE;
737 	default:
738 		pr_err("QAT: unsupported device 0x%x\n",
739 		       handle->pci_dev->device);
740 		return 0;
741 	}
742 }
743 
744 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
745 {
746 	unsigned int maj_ver, prod_type = obj_handle->prod_type;
747 
748 	if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
749 		pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n",
750 		       obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
751 		       prod_type);
752 		return -EINVAL;
753 	}
754 	maj_ver = obj_handle->prod_rev & 0xff;
755 	if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver ||
756 	    obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) {
757 		pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver);
758 		return -EINVAL;
759 	}
760 	return 0;
761 }
762 
763 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
764 			     unsigned char ae, unsigned char ctx_mask,
765 			     enum icp_qat_uof_regtype reg_type,
766 			     unsigned short reg_addr, unsigned int value)
767 {
768 	switch (reg_type) {
769 	case ICP_GPA_ABS:
770 	case ICP_GPB_ABS:
771 		ctx_mask = 0;
772 		fallthrough;
773 	case ICP_GPA_REL:
774 	case ICP_GPB_REL:
775 		return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type,
776 					reg_addr, value);
777 	case ICP_SR_ABS:
778 	case ICP_DR_ABS:
779 	case ICP_SR_RD_ABS:
780 	case ICP_DR_RD_ABS:
781 		ctx_mask = 0;
782 		fallthrough;
783 	case ICP_SR_REL:
784 	case ICP_DR_REL:
785 	case ICP_SR_RD_REL:
786 	case ICP_DR_RD_REL:
787 		return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type,
788 					    reg_addr, value);
789 	case ICP_SR_WR_ABS:
790 	case ICP_DR_WR_ABS:
791 		ctx_mask = 0;
792 		fallthrough;
793 	case ICP_SR_WR_REL:
794 	case ICP_DR_WR_REL:
795 		return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type,
796 					    reg_addr, value);
797 	case ICP_NEIGH_REL:
798 		return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
799 	default:
800 		pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type);
801 		return -EFAULT;
802 	}
803 	return 0;
804 }
805 
806 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
807 				 unsigned int ae,
808 				 struct icp_qat_uclo_encapme *encap_ae)
809 {
810 	unsigned int i;
811 	unsigned char ctx_mask;
812 	struct icp_qat_uof_init_regsym *init_regsym;
813 
814 	if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
815 	    ICP_QAT_UCLO_MAX_CTX)
816 		ctx_mask = 0xff;
817 	else
818 		ctx_mask = 0x55;
819 
820 	for (i = 0; i < encap_ae->init_regsym_num; i++) {
821 		unsigned int exp_res;
822 
823 		init_regsym = &encap_ae->init_regsym[i];
824 		exp_res = init_regsym->value;
825 		switch (init_regsym->init_type) {
826 		case ICP_QAT_UOF_INIT_REG:
827 			qat_uclo_init_reg(handle, ae, ctx_mask,
828 					  (enum icp_qat_uof_regtype)
829 					  init_regsym->reg_type,
830 					  (unsigned short)init_regsym->reg_addr,
831 					  exp_res);
832 			break;
833 		case ICP_QAT_UOF_INIT_REG_CTX:
834 			/* check if ctx is appropriate for the ctxMode */
835 			if (!((1 << init_regsym->ctx) & ctx_mask)) {
836 				pr_err("QAT: invalid ctx num = 0x%x\n",
837 				       init_regsym->ctx);
838 				return -EINVAL;
839 			}
840 			qat_uclo_init_reg(handle, ae,
841 					  (unsigned char)
842 					  (1 << init_regsym->ctx),
843 					  (enum icp_qat_uof_regtype)
844 					  init_regsym->reg_type,
845 					  (unsigned short)init_regsym->reg_addr,
846 					  exp_res);
847 			break;
848 		case ICP_QAT_UOF_INIT_EXPR:
849 			pr_err("QAT: INIT_EXPR feature not supported\n");
850 			return -EINVAL;
851 		case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
852 			pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n");
853 			return -EINVAL;
854 		default:
855 			break;
856 		}
857 	}
858 	return 0;
859 }
860 
861 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
862 {
863 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
864 	unsigned long ae_mask = handle->hal_handle->ae_mask;
865 	struct icp_qat_uclo_aedata *aed;
866 	unsigned int s, ae;
867 
868 	if (obj_handle->global_inited)
869 		return 0;
870 	if (obj_handle->init_mem_tab.entry_num) {
871 		if (qat_uclo_init_memory(handle)) {
872 			pr_err("QAT: initialize memory failed\n");
873 			return -EINVAL;
874 		}
875 	}
876 
877 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
878 		aed = &obj_handle->ae_data[ae];
879 		for (s = 0; s < aed->slice_num; s++) {
880 			if (!aed->ae_slices[s].encap_image)
881 				continue;
882 			if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image))
883 				return -EINVAL;
884 		}
885 	}
886 	obj_handle->global_inited = 1;
887 	return 0;
888 }
889 
890 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle,
891 			     struct icp_qat_uclo_objhandle *obj_handle,
892 			     unsigned char ae,
893 			     struct icp_qat_uof_image *uof_image)
894 {
895 	unsigned char mode;
896 	int ret;
897 
898 	mode = ICP_QAT_CTX_MODE(uof_image->ae_mode);
899 	ret = qat_hal_set_ae_ctx_mode(handle, ae, mode);
900 	if (ret) {
901 		pr_err("QAT: qat_hal_set_ae_ctx_mode error\n");
902 		return ret;
903 	}
904 	if (handle->chip_info->nn) {
905 		mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
906 		ret = qat_hal_set_ae_nn_mode(handle, ae, mode);
907 		if (ret) {
908 			pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
909 			return ret;
910 		}
911 	}
912 	mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode);
913 	ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode);
914 	if (ret) {
915 		pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n");
916 		return ret;
917 	}
918 	mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode);
919 	ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode);
920 	if (ret) {
921 		pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
922 		return ret;
923 	}
924 	if (handle->chip_info->lm2lm3) {
925 		mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode);
926 		ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode);
927 		if (ret) {
928 			pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n");
929 			return ret;
930 		}
931 		mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode);
932 		ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode);
933 		if (ret) {
934 			pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n");
935 			return ret;
936 		}
937 		mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode);
938 		qat_hal_set_ae_tindex_mode(handle, ae, mode);
939 	}
940 	return 0;
941 }
942 
943 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
944 {
945 	struct icp_qat_uof_image *uof_image;
946 	struct icp_qat_uclo_aedata *ae_data;
947 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
948 	unsigned long ae_mask = handle->hal_handle->ae_mask;
949 	unsigned long cfg_ae_mask = handle->cfg_ae_mask;
950 	unsigned char ae, s;
951 	int error;
952 
953 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
954 		if (!test_bit(ae, &cfg_ae_mask))
955 			continue;
956 
957 		ae_data = &obj_handle->ae_data[ae];
958 		for (s = 0; s < min_t(unsigned int, ae_data->slice_num,
959 				      ICP_QAT_UCLO_MAX_CTX); s++) {
960 			if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
961 				continue;
962 			uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
963 			error = qat_hal_set_modes(handle, obj_handle, ae,
964 						  uof_image);
965 			if (error)
966 				return error;
967 		}
968 	}
969 	return 0;
970 }
971 
972 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
973 {
974 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
975 	struct icp_qat_uclo_encapme *image;
976 	int a;
977 
978 	for (a = 0; a < obj_handle->uimage_num; a++) {
979 		image = &obj_handle->ae_uimage[a];
980 		image->uwords_num = image->page->beg_addr_p +
981 					image->page->micro_words_num;
982 	}
983 }
984 
985 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
986 {
987 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
988 	unsigned int ae;
989 
990 	obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
991 	obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
992 					     obj_handle->obj_hdr->file_buff;
993 	obj_handle->uword_in_bytes = 6;
994 	obj_handle->prod_type = qat_uclo_get_dev_type(handle);
995 	obj_handle->prod_rev = PID_MAJOR_REV |
996 			(PID_MINOR_REV & handle->hal_handle->revision_id);
997 	if (qat_uclo_check_uof_compat(obj_handle)) {
998 		pr_err("QAT: UOF incompatible\n");
999 		return -EINVAL;
1000 	}
1001 	obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64),
1002 					GFP_KERNEL);
1003 	if (!obj_handle->uword_buf)
1004 		return -ENOMEM;
1005 	obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
1006 	if (!obj_handle->obj_hdr->file_buff ||
1007 	    !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
1008 				    &obj_handle->str_table)) {
1009 		pr_err("QAT: UOF doesn't have effective images\n");
1010 		goto out_err;
1011 	}
1012 	obj_handle->uimage_num =
1013 		qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage,
1014 				    ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
1015 	if (!obj_handle->uimage_num)
1016 		goto out_err;
1017 	if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
1018 		pr_err("QAT: Bad object\n");
1019 		goto out_check_uof_aemask_err;
1020 	}
1021 	qat_uclo_init_uword_num(handle);
1022 	qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
1023 				   &obj_handle->init_mem_tab);
1024 	if (qat_uclo_set_ae_mode(handle))
1025 		goto out_check_uof_aemask_err;
1026 	return 0;
1027 out_check_uof_aemask_err:
1028 	for (ae = 0; ae < obj_handle->uimage_num; ae++)
1029 		kfree(obj_handle->ae_uimage[ae].page);
1030 out_err:
1031 	kfree(obj_handle->uword_buf);
1032 	return -EFAULT;
1033 }
1034 
1035 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1036 				      struct icp_qat_suof_filehdr *suof_ptr,
1037 				      int suof_size)
1038 {
1039 	unsigned int check_sum = 0;
1040 	unsigned int min_ver_offset = 0;
1041 	struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1042 
1043 	suof_handle->file_id = ICP_QAT_SUOF_FID;
1044 	suof_handle->suof_buf = (char *)suof_ptr;
1045 	suof_handle->suof_size = suof_size;
1046 	min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr,
1047 					      min_ver);
1048 	check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver,
1049 					       min_ver_offset);
1050 	if (check_sum != suof_ptr->check_sum) {
1051 		pr_err("QAT: incorrect SUOF checksum\n");
1052 		return -EINVAL;
1053 	}
1054 	suof_handle->check_sum = suof_ptr->check_sum;
1055 	suof_handle->min_ver = suof_ptr->min_ver;
1056 	suof_handle->maj_ver = suof_ptr->maj_ver;
1057 	suof_handle->fw_type = suof_ptr->fw_type;
1058 	return 0;
1059 }
1060 
1061 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle,
1062 			      struct icp_qat_suof_img_hdr *suof_img_hdr,
1063 			      struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1064 {
1065 	struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1066 	struct icp_qat_simg_ae_mode *ae_mode;
1067 	struct icp_qat_suof_objhdr *suof_objhdr;
1068 
1069 	suof_img_hdr->simg_buf  = (suof_handle->suof_buf +
1070 				   suof_chunk_hdr->offset +
1071 				   sizeof(*suof_objhdr));
1072 	suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)
1073 				  (suof_handle->suof_buf +
1074 				   suof_chunk_hdr->offset))->img_length;
1075 
1076 	suof_img_hdr->css_header = suof_img_hdr->simg_buf;
1077 	suof_img_hdr->css_key = (suof_img_hdr->css_header +
1078 				 sizeof(struct icp_qat_css_hdr));
1079 	suof_img_hdr->css_signature = suof_img_hdr->css_key +
1080 				      ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1081 				      ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle);
1082 	suof_img_hdr->css_simg = suof_img_hdr->css_signature +
1083 				 ICP_QAT_CSS_SIGNATURE_LEN(handle);
1084 
1085 	ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
1086 	suof_img_hdr->ae_mask = ae_mode->ae_mask;
1087 	suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
1088 	suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
1089 	suof_img_hdr->fw_type = ae_mode->fw_type;
1090 }
1091 
1092 static void
1093 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
1094 			  struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1095 {
1096 	char **sym_str = (char **)&suof_handle->sym_str;
1097 	unsigned int *sym_size = &suof_handle->sym_size;
1098 	struct icp_qat_suof_strtable *str_table_obj;
1099 
1100 	*sym_size = *(unsigned int *)(uintptr_t)
1101 		   (suof_chunk_hdr->offset + suof_handle->suof_buf);
1102 	*sym_str = (char *)(uintptr_t)
1103 		   (suof_handle->suof_buf + suof_chunk_hdr->offset +
1104 		   sizeof(str_table_obj->tab_length));
1105 }
1106 
1107 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
1108 				      struct icp_qat_suof_img_hdr *img_hdr)
1109 {
1110 	struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
1111 	unsigned int prod_rev, maj_ver, prod_type;
1112 
1113 	prod_type = qat_uclo_get_dev_type(handle);
1114 	img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
1115 	prod_rev = PID_MAJOR_REV |
1116 			 (PID_MINOR_REV & handle->hal_handle->revision_id);
1117 	if (img_ae_mode->dev_type != prod_type) {
1118 		pr_err("QAT: incompatible product type %x\n",
1119 		       img_ae_mode->dev_type);
1120 		return -EINVAL;
1121 	}
1122 	maj_ver = prod_rev & 0xff;
1123 	if (maj_ver > img_ae_mode->devmax_ver ||
1124 	    maj_ver < img_ae_mode->devmin_ver) {
1125 		pr_err("QAT: incompatible device majver 0x%x\n", maj_ver);
1126 		return -EINVAL;
1127 	}
1128 	return 0;
1129 }
1130 
1131 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
1132 {
1133 	struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1134 
1135 	kfree(sobj_handle->img_table.simg_hdr);
1136 	sobj_handle->img_table.simg_hdr = NULL;
1137 	kfree(handle->sobj_handle);
1138 	handle->sobj_handle = NULL;
1139 }
1140 
1141 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
1142 			      unsigned int img_id, unsigned int num_simgs)
1143 {
1144 	struct icp_qat_suof_img_hdr img_header;
1145 
1146 	if (img_id != num_simgs - 1) {
1147 		memcpy(&img_header, &suof_img_hdr[num_simgs - 1],
1148 		       sizeof(*suof_img_hdr));
1149 		memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id],
1150 		       sizeof(*suof_img_hdr));
1151 		memcpy(&suof_img_hdr[img_id], &img_header,
1152 		       sizeof(*suof_img_hdr));
1153 	}
1154 }
1155 
1156 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
1157 			     struct icp_qat_suof_filehdr *suof_ptr,
1158 			     int suof_size)
1159 {
1160 	struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1161 	struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
1162 	struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
1163 	int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
1164 	unsigned int i = 0;
1165 	struct icp_qat_suof_img_hdr img_header;
1166 
1167 	if (!suof_ptr || suof_size == 0) {
1168 		pr_err("QAT: input parameter SUOF pointer/size is NULL\n");
1169 		return -EINVAL;
1170 	}
1171 	if (qat_uclo_check_suof_format(suof_ptr))
1172 		return -EINVAL;
1173 	ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
1174 	if (ret)
1175 		return ret;
1176 	suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)
1177 			 ((uintptr_t)suof_ptr + sizeof(*suof_ptr));
1178 
1179 	qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
1180 	suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
1181 
1182 	if (suof_handle->img_table.num_simgs != 0) {
1183 		suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs,
1184 				       sizeof(img_header),
1185 				       GFP_KERNEL);
1186 		if (!suof_img_hdr)
1187 			return -ENOMEM;
1188 		suof_handle->img_table.simg_hdr = suof_img_hdr;
1189 
1190 		for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
1191 			qat_uclo_map_simg(handle, &suof_img_hdr[i],
1192 					  &suof_chunk_hdr[1 + i]);
1193 			ret = qat_uclo_check_simg_compat(handle,
1194 							 &suof_img_hdr[i]);
1195 			if (ret)
1196 				return ret;
1197 			suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask;
1198 			if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
1199 				ae0_img = i;
1200 		}
1201 
1202 		if (!handle->chip_info->tgroup_share_ustore) {
1203 			qat_uclo_tail_img(suof_img_hdr, ae0_img,
1204 					  suof_handle->img_table.num_simgs);
1205 		}
1206 	}
1207 	return 0;
1208 }
1209 
1210 #define ADD_ADDR(high, low)  ((((u64)high) << 32) + low)
1211 #define BITS_IN_DWORD 32
1212 
1213 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
1214 			    struct icp_qat_fw_auth_desc *desc)
1215 {
1216 	u32 fcu_sts, retry = 0;
1217 	u32 fcu_ctl_csr, fcu_sts_csr;
1218 	u32 fcu_dram_hi_csr, fcu_dram_lo_csr;
1219 	u64 bus_addr;
1220 
1221 	bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low)
1222 			   - sizeof(struct icp_qat_auth_chunk);
1223 
1224 	fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1225 	fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1226 	fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi;
1227 	fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo;
1228 
1229 	SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD));
1230 	SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr);
1231 	SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH);
1232 
1233 	do {
1234 		msleep(FW_AUTH_WAIT_PERIOD);
1235 		fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1236 		if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
1237 			goto auth_fail;
1238 		if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
1239 			if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
1240 				return 0;
1241 	} while (retry++ < FW_AUTH_MAX_RETRY);
1242 auth_fail:
1243 	pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n",
1244 	       fcu_sts & FCU_AUTH_STS_MASK, retry);
1245 	return -EINVAL;
1246 }
1247 
1248 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle,
1249 				  int imgid)
1250 {
1251 	struct icp_qat_suof_handle *sobj_handle;
1252 
1253 	if (!handle->chip_info->tgroup_share_ustore)
1254 		return false;
1255 
1256 	sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
1257 	if (handle->hal_handle->admin_ae_mask &
1258 	    sobj_handle->img_table.simg_hdr[imgid].ae_mask)
1259 		return false;
1260 
1261 	return true;
1262 }
1263 
1264 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
1265 				      struct icp_qat_fw_auth_desc *desc)
1266 {
1267 	unsigned long ae_mask = handle->hal_handle->ae_mask;
1268 	unsigned long desc_ae_mask = desc->ae_mask;
1269 	u32 fcu_sts, ae_broadcast_mask = 0;
1270 	u32 fcu_loaded_csr, ae_loaded;
1271 	u32 fcu_sts_csr, fcu_ctl_csr;
1272 	unsigned int ae, retry = 0;
1273 
1274 	if (handle->chip_info->tgroup_share_ustore) {
1275 		fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1276 		fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1277 		fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1278 	} else {
1279 		pr_err("Chip 0x%x doesn't support broadcast load\n",
1280 		       handle->pci_dev->device);
1281 		return -EINVAL;
1282 	}
1283 
1284 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
1285 		if (qat_hal_check_ae_active(handle, (unsigned char)ae)) {
1286 			pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n");
1287 			return -EINVAL;
1288 		}
1289 
1290 		if (test_bit(ae, &desc_ae_mask))
1291 			ae_broadcast_mask |= 1 << ae;
1292 	}
1293 
1294 	if (ae_broadcast_mask) {
1295 		SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE,
1296 			    ae_broadcast_mask);
1297 
1298 		SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD);
1299 
1300 		do {
1301 			msleep(FW_AUTH_WAIT_PERIOD);
1302 			fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1303 			fcu_sts &= FCU_AUTH_STS_MASK;
1304 
1305 			if (fcu_sts == FCU_STS_LOAD_FAIL) {
1306 				pr_err("Broadcast load failed: 0x%x)\n", fcu_sts);
1307 				return -EINVAL;
1308 			} else if (fcu_sts == FCU_STS_LOAD_DONE) {
1309 				ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr);
1310 				ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos;
1311 
1312 				if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask)
1313 					break;
1314 			}
1315 		} while (retry++ < FW_AUTH_MAX_RETRY);
1316 
1317 		if (retry > FW_AUTH_MAX_RETRY) {
1318 			pr_err("QAT: broadcast load failed timeout %d\n", retry);
1319 			return -EINVAL;
1320 		}
1321 	}
1322 	return 0;
1323 }
1324 
1325 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
1326 			       struct icp_firml_dram_desc *dram_desc,
1327 			       unsigned int size)
1328 {
1329 	void *vptr;
1330 	dma_addr_t ptr;
1331 
1332 	vptr = dma_alloc_coherent(&handle->pci_dev->dev,
1333 				  size, &ptr, GFP_KERNEL);
1334 	if (!vptr)
1335 		return -ENOMEM;
1336 	dram_desc->dram_base_addr_v = vptr;
1337 	dram_desc->dram_bus_addr = ptr;
1338 	dram_desc->dram_size = size;
1339 	return 0;
1340 }
1341 
1342 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
1343 			       struct icp_firml_dram_desc *dram_desc)
1344 {
1345 	if (handle && dram_desc && dram_desc->dram_base_addr_v) {
1346 		dma_free_coherent(&handle->pci_dev->dev,
1347 				  (size_t)(dram_desc->dram_size),
1348 				  dram_desc->dram_base_addr_v,
1349 				  dram_desc->dram_bus_addr);
1350 	}
1351 
1352 	if (dram_desc)
1353 		memset(dram_desc, 0, sizeof(*dram_desc));
1354 }
1355 
1356 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle,
1357 				   struct icp_qat_fw_auth_desc **desc)
1358 {
1359 	struct icp_firml_dram_desc dram_desc;
1360 
1361 	if (*desc) {
1362 		dram_desc.dram_base_addr_v = *desc;
1363 		dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *)
1364 					   (*desc))->chunk_bus_addr;
1365 		dram_desc.dram_size = ((struct icp_qat_auth_chunk *)
1366 				       (*desc))->chunk_size;
1367 		qat_uclo_simg_free(handle, &dram_desc);
1368 	}
1369 }
1370 
1371 static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle,
1372 				char *image, unsigned int size,
1373 				unsigned int fw_type)
1374 {
1375 	char *fw_type_name = fw_type ? "MMP" : "AE";
1376 	unsigned int css_dword_size = sizeof(u32);
1377 
1378 	if (handle->chip_info->fw_auth) {
1379 		struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1380 		unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle);
1381 
1382 		if ((css_hdr->header_len * css_dword_size) != header_len)
1383 			goto err;
1384 		if ((css_hdr->size * css_dword_size) != size)
1385 			goto err;
1386 		if (fw_type != css_hdr->fw_type)
1387 			goto err;
1388 		if (size <= header_len)
1389 			goto err;
1390 		size -= header_len;
1391 	}
1392 
1393 	if (fw_type == CSS_AE_FIRMWARE) {
1394 		if (size < sizeof(struct icp_qat_simg_ae_mode *) +
1395 		    ICP_QAT_SIMG_AE_INIT_SEQ_LEN)
1396 			goto err;
1397 		if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)
1398 			goto err;
1399 	} else if (fw_type == CSS_MMP_FIRMWARE) {
1400 		if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN)
1401 			goto err;
1402 	} else {
1403 		pr_err("QAT: Unsupported firmware type\n");
1404 		return -EINVAL;
1405 	}
1406 	return 0;
1407 
1408 err:
1409 	pr_err("QAT: Invalid %s firmware image\n", fw_type_name);
1410 	return -EINVAL;
1411 }
1412 
1413 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
1414 				char *image, unsigned int size,
1415 				struct icp_qat_fw_auth_desc **desc)
1416 {
1417 	struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1418 	struct icp_qat_fw_auth_desc *auth_desc;
1419 	struct icp_qat_auth_chunk *auth_chunk;
1420 	u64 virt_addr,  bus_addr, virt_base;
1421 	unsigned int length, simg_offset = sizeof(*auth_chunk);
1422 	struct icp_qat_simg_ae_mode *simg_ae_mode;
1423 	struct icp_firml_dram_desc img_desc;
1424 
1425 	if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) {
1426 		pr_err("QAT: error, input image size overflow %d\n", size);
1427 		return -EINVAL;
1428 	}
1429 	length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
1430 		 ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset :
1431 		 size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset;
1432 	if (qat_uclo_simg_alloc(handle, &img_desc, length)) {
1433 		pr_err("QAT: error, allocate continuous dram fail\n");
1434 		return -ENOMEM;
1435 	}
1436 
1437 	auth_chunk = img_desc.dram_base_addr_v;
1438 	auth_chunk->chunk_size = img_desc.dram_size;
1439 	auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr;
1440 	virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset;
1441 	bus_addr  = img_desc.dram_bus_addr + simg_offset;
1442 	auth_desc = img_desc.dram_base_addr_v;
1443 	auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1444 	auth_desc->css_hdr_low = (unsigned int)bus_addr;
1445 	virt_addr = virt_base;
1446 
1447 	memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr));
1448 	/* pub key */
1449 	bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
1450 			   sizeof(*css_hdr);
1451 	virt_addr = virt_addr + sizeof(*css_hdr);
1452 
1453 	auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1454 	auth_desc->fwsk_pub_low = (unsigned int)bus_addr;
1455 
1456 	memcpy((void *)(uintptr_t)virt_addr,
1457 	       (void *)(image + sizeof(*css_hdr)),
1458 	       ICP_QAT_CSS_FWSK_MODULUS_LEN(handle));
1459 	/* padding */
1460 	memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)),
1461 	       0, ICP_QAT_CSS_FWSK_PAD_LEN(handle));
1462 
1463 	/* exponent */
1464 	memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1465 	       ICP_QAT_CSS_FWSK_PAD_LEN(handle)),
1466 	       (void *)(image + sizeof(*css_hdr) +
1467 			ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)),
1468 	       sizeof(unsigned int));
1469 
1470 	/* signature */
1471 	bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high,
1472 			    auth_desc->fwsk_pub_low) +
1473 		   ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1474 	virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1475 	auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1476 	auth_desc->signature_low = (unsigned int)bus_addr;
1477 
1478 	memcpy((void *)(uintptr_t)virt_addr,
1479 	       (void *)(image + sizeof(*css_hdr) +
1480 	       ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1481 	       ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)),
1482 	       ICP_QAT_CSS_SIGNATURE_LEN(handle));
1483 
1484 	bus_addr = ADD_ADDR(auth_desc->signature_high,
1485 			    auth_desc->signature_low) +
1486 		   ICP_QAT_CSS_SIGNATURE_LEN(handle);
1487 	virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1488 
1489 	auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1490 	auth_desc->img_low = (unsigned int)bus_addr;
1491 	auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle);
1492 	memcpy((void *)(uintptr_t)virt_addr,
1493 	       (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)),
1494 	       auth_desc->img_len);
1495 	virt_addr = virt_base;
1496 	/* AE firmware */
1497 	if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type ==
1498 	    CSS_AE_FIRMWARE) {
1499 		auth_desc->img_ae_mode_data_high = auth_desc->img_high;
1500 		auth_desc->img_ae_mode_data_low = auth_desc->img_low;
1501 		bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
1502 				    auth_desc->img_ae_mode_data_low) +
1503 			   sizeof(struct icp_qat_simg_ae_mode);
1504 
1505 		auth_desc->img_ae_init_data_high = (unsigned int)
1506 						 (bus_addr >> BITS_IN_DWORD);
1507 		auth_desc->img_ae_init_data_low = (unsigned int)bus_addr;
1508 		bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1509 		auth_desc->img_ae_insts_high = (unsigned int)
1510 					     (bus_addr >> BITS_IN_DWORD);
1511 		auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
1512 		virt_addr += sizeof(struct icp_qat_css_hdr);
1513 		virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1514 		virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1515 		simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr;
1516 		auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask;
1517 	} else {
1518 		auth_desc->img_ae_insts_high = auth_desc->img_high;
1519 		auth_desc->img_ae_insts_low = auth_desc->img_low;
1520 	}
1521 	*desc = auth_desc;
1522 	return 0;
1523 }
1524 
1525 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
1526 			    struct icp_qat_fw_auth_desc *desc)
1527 {
1528 	unsigned long ae_mask = handle->hal_handle->ae_mask;
1529 	u32 fcu_sts_csr, fcu_ctl_csr;
1530 	u32 loaded_aes, loaded_csr;
1531 	unsigned int i;
1532 	u32 fcu_sts;
1533 
1534 	fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1535 	fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1536 	loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1537 
1538 	for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) {
1539 		int retry = 0;
1540 
1541 		if (!((desc->ae_mask >> i) & 0x1))
1542 			continue;
1543 		if (qat_hal_check_ae_active(handle, i)) {
1544 			pr_err("QAT: AE %d is active\n", i);
1545 			return -EINVAL;
1546 		}
1547 		SET_CAP_CSR(handle, fcu_ctl_csr,
1548 			    (FCU_CTRL_CMD_LOAD |
1549 			    (1 << FCU_CTRL_BROADCAST_POS) |
1550 			    (i << FCU_CTRL_AE_POS)));
1551 
1552 		do {
1553 			msleep(FW_AUTH_WAIT_PERIOD);
1554 			fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1555 			if ((fcu_sts & FCU_AUTH_STS_MASK) ==
1556 			    FCU_STS_LOAD_DONE) {
1557 				loaded_aes = GET_CAP_CSR(handle, loaded_csr);
1558 				loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos;
1559 				if (loaded_aes & (1 << i))
1560 					break;
1561 			}
1562 		} while (retry++ < FW_AUTH_MAX_RETRY);
1563 		if (retry > FW_AUTH_MAX_RETRY) {
1564 			pr_err("QAT: firmware load failed timeout %x\n", retry);
1565 			return -EINVAL;
1566 		}
1567 	}
1568 	return 0;
1569 }
1570 
1571 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
1572 				 void *addr_ptr, int mem_size)
1573 {
1574 	struct icp_qat_suof_handle *suof_handle;
1575 
1576 	suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL);
1577 	if (!suof_handle)
1578 		return -ENOMEM;
1579 	handle->sobj_handle = suof_handle;
1580 	if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
1581 		qat_uclo_del_suof(handle);
1582 		pr_err("QAT: map SUOF failed\n");
1583 		return -EINVAL;
1584 	}
1585 	return 0;
1586 }
1587 
1588 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
1589 		       void *addr_ptr, int mem_size)
1590 {
1591 	struct icp_qat_fw_auth_desc *desc = NULL;
1592 	int status = 0;
1593 	int ret;
1594 
1595 	ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE);
1596 	if (ret)
1597 		return ret;
1598 
1599 	if (handle->chip_info->fw_auth) {
1600 		status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc);
1601 		if (!status)
1602 			status = qat_uclo_auth_fw(handle, desc);
1603 		qat_uclo_ummap_auth_fw(handle, &desc);
1604 	} else {
1605 		if (handle->chip_info->mmp_sram_size < mem_size) {
1606 			pr_err("QAT: MMP size is too large: 0x%x\n", mem_size);
1607 			return -EFBIG;
1608 		}
1609 		qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
1610 	}
1611 	return status;
1612 }
1613 
1614 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
1615 				void *addr_ptr, int mem_size)
1616 {
1617 	struct icp_qat_uof_filehdr *filehdr;
1618 	struct icp_qat_uclo_objhandle *objhdl;
1619 
1620 	objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL);
1621 	if (!objhdl)
1622 		return -ENOMEM;
1623 	objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL);
1624 	if (!objhdl->obj_buf)
1625 		goto out_objbuf_err;
1626 	filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
1627 	if (qat_uclo_check_uof_format(filehdr))
1628 		goto out_objhdr_err;
1629 	objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr,
1630 					     ICP_QAT_UOF_OBJS);
1631 	if (!objhdl->obj_hdr) {
1632 		pr_err("QAT: object file chunk is null\n");
1633 		goto out_objhdr_err;
1634 	}
1635 	handle->obj_handle = objhdl;
1636 	if (qat_uclo_parse_uof_obj(handle))
1637 		goto out_overlay_obj_err;
1638 	return 0;
1639 
1640 out_overlay_obj_err:
1641 	handle->obj_handle = NULL;
1642 	kfree(objhdl->obj_hdr);
1643 out_objhdr_err:
1644 	kfree(objhdl->obj_buf);
1645 out_objbuf_err:
1646 	kfree(objhdl);
1647 	return -ENOMEM;
1648 }
1649 
1650 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1651 				     struct icp_qat_mof_file_hdr *mof_ptr,
1652 				     u32 mof_size)
1653 {
1654 	struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1655 	unsigned int min_ver_offset;
1656 	unsigned int checksum;
1657 
1658 	mobj_handle->file_id = ICP_QAT_MOF_FID;
1659 	mobj_handle->mof_buf = (char *)mof_ptr;
1660 	mobj_handle->mof_size = mof_size;
1661 
1662 	min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr,
1663 					     min_ver);
1664 	checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver,
1665 					      min_ver_offset);
1666 	if (checksum != mof_ptr->checksum) {
1667 		pr_err("QAT: incorrect MOF checksum\n");
1668 		return -EINVAL;
1669 	}
1670 
1671 	mobj_handle->checksum = mof_ptr->checksum;
1672 	mobj_handle->min_ver = mof_ptr->min_ver;
1673 	mobj_handle->maj_ver = mof_ptr->maj_ver;
1674 	return 0;
1675 }
1676 
1677 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle)
1678 {
1679 	struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1680 
1681 	kfree(mobj_handle->obj_table.obj_hdr);
1682 	mobj_handle->obj_table.obj_hdr = NULL;
1683 	kfree(handle->mobj_handle);
1684 	handle->mobj_handle = NULL;
1685 }
1686 
1687 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle,
1688 					char *obj_name, char **obj_ptr,
1689 					unsigned int *obj_size)
1690 {
1691 	struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr;
1692 	unsigned int i;
1693 
1694 	for (i = 0; i < mobj_handle->obj_table.num_objs; i++) {
1695 		if (!strncmp(obj_hdr[i].obj_name, obj_name,
1696 			     ICP_QAT_SUOF_OBJ_NAME_LEN)) {
1697 			*obj_ptr  = obj_hdr[i].obj_buf;
1698 			*obj_size = obj_hdr[i].obj_size;
1699 			return 0;
1700 		}
1701 	}
1702 
1703 	pr_err("QAT: object %s is not found inside MOF\n", obj_name);
1704 	return -EINVAL;
1705 }
1706 
1707 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle,
1708 				     struct icp_qat_mof_objhdr *mobj_hdr,
1709 				     struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr)
1710 {
1711 	u8 *obj;
1712 
1713 	if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG,
1714 		     ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1715 		obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset;
1716 	} else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG,
1717 			    ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1718 		obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset;
1719 	} else {
1720 		pr_err("QAT: unsupported chunk id\n");
1721 		return -EINVAL;
1722 	}
1723 	mobj_hdr->obj_buf = obj;
1724 	mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size;
1725 	mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str;
1726 	return 0;
1727 }
1728 
1729 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle)
1730 {
1731 	struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr;
1732 	struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr;
1733 	struct icp_qat_mof_obj_hdr *uobj_hdr;
1734 	struct icp_qat_mof_obj_hdr *sobj_hdr;
1735 	struct icp_qat_mof_objhdr *mobj_hdr;
1736 	unsigned int uobj_chunk_num = 0;
1737 	unsigned int sobj_chunk_num = 0;
1738 	unsigned int *valid_chunk;
1739 	int ret, i;
1740 
1741 	uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr;
1742 	sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr;
1743 	if (uobj_hdr)
1744 		uobj_chunk_num = uobj_hdr->num_chunks;
1745 	if (sobj_hdr)
1746 		sobj_chunk_num = sobj_hdr->num_chunks;
1747 
1748 	mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) *
1749 			   sizeof(*mobj_hdr), GFP_KERNEL);
1750 	if (!mobj_hdr)
1751 		return -ENOMEM;
1752 
1753 	mobj_handle->obj_table.obj_hdr = mobj_hdr;
1754 	valid_chunk = &mobj_handle->obj_table.num_objs;
1755 	uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1756 			 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr));
1757 	sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1758 			((uintptr_t)sobj_hdr + sizeof(*sobj_hdr));
1759 
1760 	/* map uof objects */
1761 	for (i = 0; i < uobj_chunk_num; i++) {
1762 		ret = qat_uclo_map_obj_from_mof(mobj_handle,
1763 						&mobj_hdr[*valid_chunk],
1764 						&uobj_chunkhdr[i]);
1765 		if (ret)
1766 			return ret;
1767 		(*valid_chunk)++;
1768 	}
1769 
1770 	/* map suof objects */
1771 	for (i = 0; i < sobj_chunk_num; i++) {
1772 		ret = qat_uclo_map_obj_from_mof(mobj_handle,
1773 						&mobj_hdr[*valid_chunk],
1774 						&sobj_chunkhdr[i]);
1775 		if (ret)
1776 			return ret;
1777 		(*valid_chunk)++;
1778 	}
1779 
1780 	if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) {
1781 		pr_err("QAT: inconsistent UOF/SUOF chunk amount\n");
1782 		return -EINVAL;
1783 	}
1784 	return 0;
1785 }
1786 
1787 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle,
1788 				     struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1789 {
1790 	char **sym_str = (char **)&mobj_handle->sym_str;
1791 	unsigned int *sym_size = &mobj_handle->sym_size;
1792 	struct icp_qat_mof_str_table *str_table_obj;
1793 
1794 	*sym_size = *(unsigned int *)(uintptr_t)
1795 		    (mof_chunkhdr->offset + mobj_handle->mof_buf);
1796 	*sym_str = (char *)(uintptr_t)
1797 		   (mobj_handle->mof_buf + mof_chunkhdr->offset +
1798 		    sizeof(str_table_obj->tab_len));
1799 }
1800 
1801 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle,
1802 				   struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1803 {
1804 	char *chunk_id = mof_chunkhdr->chunk_id;
1805 
1806 	if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1807 		qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr);
1808 	else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1809 		mobj_handle->uobjs_hdr = mobj_handle->mof_buf +
1810 					 mof_chunkhdr->offset;
1811 	else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1812 		mobj_handle->sobjs_hdr = mobj_handle->mof_buf +
1813 					 mof_chunkhdr->offset;
1814 }
1815 
1816 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr)
1817 {
1818 	int maj = mof_hdr->maj_ver & 0xff;
1819 	int min = mof_hdr->min_ver & 0xff;
1820 
1821 	if (mof_hdr->file_id != ICP_QAT_MOF_FID) {
1822 		pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id);
1823 		return -EINVAL;
1824 	}
1825 
1826 	if (mof_hdr->num_chunks <= 0x1) {
1827 		pr_err("QAT: MOF chunk amount is incorrect\n");
1828 		return -EINVAL;
1829 	}
1830 	if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) {
1831 		pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n",
1832 		       maj, min);
1833 		return -EINVAL;
1834 	}
1835 	return 0;
1836 }
1837 
1838 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle,
1839 				struct icp_qat_mof_file_hdr *mof_ptr,
1840 				u32 mof_size, char *obj_name, char **obj_ptr,
1841 				unsigned int *obj_size)
1842 {
1843 	struct icp_qat_mof_chunkhdr *mof_chunkhdr;
1844 	unsigned int file_id = mof_ptr->file_id;
1845 	struct icp_qat_mof_handle *mobj_handle;
1846 	unsigned short chunks_num;
1847 	unsigned int i;
1848 	int ret;
1849 
1850 	if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) {
1851 		if (obj_ptr)
1852 			*obj_ptr = (char *)mof_ptr;
1853 		if (obj_size)
1854 			*obj_size = mof_size;
1855 		return 0;
1856 	}
1857 	if (qat_uclo_check_mof_format(mof_ptr))
1858 		return -EINVAL;
1859 
1860 	mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL);
1861 	if (!mobj_handle)
1862 		return -ENOMEM;
1863 
1864 	handle->mobj_handle = mobj_handle;
1865 	ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size);
1866 	if (ret)
1867 		return ret;
1868 
1869 	mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr);
1870 	chunks_num = mof_ptr->num_chunks;
1871 
1872 	/* Parse MOF file chunks */
1873 	for (i = 0; i < chunks_num; i++)
1874 		qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]);
1875 
1876 	/* All sym_objs uobjs and sobjs should be available */
1877 	if (!mobj_handle->sym_str ||
1878 	    (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr))
1879 		return -EINVAL;
1880 
1881 	ret = qat_uclo_map_objs_from_mof(mobj_handle);
1882 	if (ret)
1883 		return ret;
1884 
1885 	/* Seek specified uof object in MOF */
1886 	return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name,
1887 					    obj_ptr, obj_size);
1888 }
1889 
1890 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
1891 		     void *addr_ptr, u32 mem_size, char *obj_name)
1892 {
1893 	char *obj_addr;
1894 	u32 obj_size;
1895 	int ret;
1896 
1897 	BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >=
1898 		     (sizeof(handle->hal_handle->ae_mask) * 8));
1899 
1900 	if (!handle || !addr_ptr || mem_size < 24)
1901 		return -EINVAL;
1902 
1903 	if (obj_name) {
1904 		ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name,
1905 					   &obj_addr, &obj_size);
1906 		if (ret)
1907 			return ret;
1908 	} else {
1909 		obj_addr = addr_ptr;
1910 		obj_size = mem_size;
1911 	}
1912 
1913 	return (handle->chip_info->fw_auth) ?
1914 			qat_uclo_map_suof_obj(handle, obj_addr, obj_size) :
1915 			qat_uclo_map_uof_obj(handle, obj_addr, obj_size);
1916 }
1917 
1918 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle)
1919 {
1920 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1921 	unsigned int a;
1922 
1923 	if (handle->mobj_handle)
1924 		qat_uclo_del_mof(handle);
1925 	if (handle->sobj_handle)
1926 		qat_uclo_del_suof(handle);
1927 	if (!obj_handle)
1928 		return;
1929 
1930 	kfree(obj_handle->uword_buf);
1931 	for (a = 0; a < obj_handle->uimage_num; a++)
1932 		kfree(obj_handle->ae_uimage[a].page);
1933 
1934 	for (a = 0; a < handle->hal_handle->ae_max_num; a++)
1935 		qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
1936 
1937 	kfree(obj_handle->obj_hdr);
1938 	kfree(obj_handle->obj_buf);
1939 	kfree(obj_handle);
1940 	handle->obj_handle = NULL;
1941 }
1942 
1943 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
1944 				 struct icp_qat_uclo_encap_page *encap_page,
1945 				 u64 *uword, unsigned int addr_p,
1946 				 unsigned int raddr, u64 fill)
1947 {
1948 	unsigned int i, addr;
1949 	u64 uwrd = 0;
1950 
1951 	if (!encap_page) {
1952 		*uword = fill;
1953 		return;
1954 	}
1955 	addr = (encap_page->page_region) ? raddr : addr_p;
1956 	for (i = 0; i < encap_page->uwblock_num; i++) {
1957 		if (addr >= encap_page->uwblock[i].start_addr &&
1958 		    addr <= encap_page->uwblock[i].start_addr +
1959 		    encap_page->uwblock[i].words_num - 1) {
1960 			addr -= encap_page->uwblock[i].start_addr;
1961 			addr *= obj_handle->uword_in_bytes;
1962 			memcpy(&uwrd, (void *)(((uintptr_t)
1963 			       encap_page->uwblock[i].micro_words) + addr),
1964 			       obj_handle->uword_in_bytes);
1965 			uwrd = uwrd & GENMASK_ULL(43, 0);
1966 		}
1967 	}
1968 	*uword = uwrd;
1969 	if (*uword == INVLD_UWORD)
1970 		*uword = fill;
1971 }
1972 
1973 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
1974 					struct icp_qat_uclo_encap_page
1975 					*encap_page, unsigned int ae)
1976 {
1977 	unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
1978 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1979 	u64 fill_pat;
1980 
1981 	/* load the page starting at appropriate ustore address */
1982 	/* get fill-pattern from an image -- they are all the same */
1983 	memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern,
1984 	       sizeof(u64));
1985 	uw_physical_addr = encap_page->beg_addr_p;
1986 	uw_relative_addr = 0;
1987 	words_num = encap_page->micro_words_num;
1988 	while (words_num) {
1989 		if (words_num < UWORD_CPYBUF_SIZE)
1990 			cpylen = words_num;
1991 		else
1992 			cpylen = UWORD_CPYBUF_SIZE;
1993 
1994 		/* load the buffer */
1995 		for (i = 0; i < cpylen; i++)
1996 			qat_uclo_fill_uwords(obj_handle, encap_page,
1997 					     &obj_handle->uword_buf[i],
1998 					     uw_physical_addr + i,
1999 					     uw_relative_addr + i, fill_pat);
2000 
2001 		/* copy the buffer to ustore */
2002 		qat_hal_wr_uwords(handle, (unsigned char)ae,
2003 				  uw_physical_addr, cpylen,
2004 				  obj_handle->uword_buf);
2005 
2006 		uw_physical_addr += cpylen;
2007 		uw_relative_addr += cpylen;
2008 		words_num -= cpylen;
2009 	}
2010 }
2011 
2012 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
2013 				    struct icp_qat_uof_image *image)
2014 {
2015 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2016 	unsigned long ae_mask = handle->hal_handle->ae_mask;
2017 	unsigned long cfg_ae_mask = handle->cfg_ae_mask;
2018 	unsigned long ae_assigned = image->ae_assigned;
2019 	struct icp_qat_uclo_aedata *aed;
2020 	unsigned int ctx_mask, s;
2021 	struct icp_qat_uclo_page *page;
2022 	unsigned char ae;
2023 	int ctx;
2024 
2025 	if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
2026 		ctx_mask = 0xff;
2027 	else
2028 		ctx_mask = 0x55;
2029 	/* load the default page and set assigned CTX PC
2030 	 * to the entrypoint address */
2031 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
2032 		if (!test_bit(ae, &cfg_ae_mask))
2033 			continue;
2034 
2035 		if (!test_bit(ae, &ae_assigned))
2036 			continue;
2037 
2038 		aed = &obj_handle->ae_data[ae];
2039 		/* find the slice to which this image is assigned */
2040 		for (s = 0; s < aed->slice_num; s++) {
2041 			if (image->ctx_assigned &
2042 			    aed->ae_slices[s].ctx_mask_assigned)
2043 				break;
2044 		}
2045 		if (s >= aed->slice_num)
2046 			continue;
2047 		page = aed->ae_slices[s].page;
2048 		if (!page->encap_page->def_page)
2049 			continue;
2050 		qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
2051 
2052 		page = aed->ae_slices[s].page;
2053 		for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
2054 			aed->ae_slices[s].cur_page[ctx] =
2055 					(ctx_mask & (1 << ctx)) ? page : NULL;
2056 		qat_hal_set_live_ctx(handle, (unsigned char)ae,
2057 				     image->ctx_assigned);
2058 		qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned,
2059 			       image->entry_address);
2060 	}
2061 }
2062 
2063 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
2064 {
2065 	unsigned int i;
2066 	struct icp_qat_fw_auth_desc *desc = NULL;
2067 	struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
2068 	struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
2069 	int ret;
2070 
2071 	for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
2072 		ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf,
2073 					   simg_hdr[i].simg_len,
2074 					   CSS_AE_FIRMWARE);
2075 		if (ret)
2076 			return ret;
2077 
2078 		if (qat_uclo_map_auth_fw(handle,
2079 					 (char *)simg_hdr[i].simg_buf,
2080 					 (unsigned int)
2081 					 simg_hdr[i].simg_len,
2082 					 &desc))
2083 			goto wr_err;
2084 		if (qat_uclo_auth_fw(handle, desc))
2085 			goto wr_err;
2086 		if (qat_uclo_is_broadcast(handle, i)) {
2087 			if (qat_uclo_broadcast_load_fw(handle, desc))
2088 				goto wr_err;
2089 		} else {
2090 			if (qat_uclo_load_fw(handle, desc))
2091 				goto wr_err;
2092 		}
2093 		qat_uclo_ummap_auth_fw(handle, &desc);
2094 	}
2095 	return 0;
2096 wr_err:
2097 	qat_uclo_ummap_auth_fw(handle, &desc);
2098 	return -EINVAL;
2099 }
2100 
2101 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
2102 {
2103 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2104 	unsigned int i;
2105 
2106 	if (qat_uclo_init_globals(handle))
2107 		return -EINVAL;
2108 	for (i = 0; i < obj_handle->uimage_num; i++) {
2109 		if (!obj_handle->ae_uimage[i].img_ptr)
2110 			return -EINVAL;
2111 		if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
2112 			return -EINVAL;
2113 		qat_uclo_wr_uimage_page(handle,
2114 					obj_handle->ae_uimage[i].img_ptr);
2115 	}
2116 	return 0;
2117 }
2118 
2119 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
2120 {
2121 	return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) :
2122 				   qat_uclo_wr_uof_img(handle);
2123 }
2124 
2125 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle,
2126 			     unsigned int cfg_ae_mask)
2127 {
2128 	if (!cfg_ae_mask)
2129 		return -EINVAL;
2130 
2131 	handle->cfg_ae_mask = cfg_ae_mask;
2132 	return 0;
2133 }
2134