1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/slab.h> 4 #include <linux/ctype.h> 5 #include <linux/kernel.h> 6 #include <linux/delay.h> 7 #include <linux/pci_ids.h> 8 #include "adf_accel_devices.h" 9 #include "adf_common_drv.h" 10 #include "icp_qat_uclo.h" 11 #include "icp_qat_hal.h" 12 #include "icp_qat_fw_loader_handle.h" 13 14 #define UWORD_CPYBUF_SIZE 1024 15 #define INVLD_UWORD 0xffffffffffull 16 #define PID_MINOR_REV 0xf 17 #define PID_MAJOR_REV (0xf << 4) 18 19 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, 20 unsigned int ae, unsigned int image_num) 21 { 22 struct icp_qat_uclo_aedata *ae_data; 23 struct icp_qat_uclo_encapme *encap_image; 24 struct icp_qat_uclo_page *page = NULL; 25 struct icp_qat_uclo_aeslice *ae_slice = NULL; 26 27 ae_data = &obj_handle->ae_data[ae]; 28 encap_image = &obj_handle->ae_uimage[image_num]; 29 ae_slice = &ae_data->ae_slices[ae_data->slice_num]; 30 ae_slice->encap_image = encap_image; 31 32 if (encap_image->img_ptr) { 33 ae_slice->ctx_mask_assigned = 34 encap_image->img_ptr->ctx_assigned; 35 ae_data->eff_ustore_size = obj_handle->ustore_phy_size; 36 } else { 37 ae_slice->ctx_mask_assigned = 0; 38 } 39 ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL); 40 if (!ae_slice->region) 41 return -ENOMEM; 42 ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL); 43 if (!ae_slice->page) 44 goto out_err; 45 page = ae_slice->page; 46 page->encap_page = encap_image->page; 47 ae_slice->page->region = ae_slice->region; 48 ae_data->slice_num++; 49 return 0; 50 out_err: 51 kfree(ae_slice->region); 52 ae_slice->region = NULL; 53 return -ENOMEM; 54 } 55 56 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) 57 { 58 unsigned int i; 59 60 if (!ae_data) { 61 pr_err("QAT: bad argument, ae_data is NULL\n "); 62 return -EINVAL; 63 } 64 65 for (i = 0; i < ae_data->slice_num; i++) { 66 kfree(ae_data->ae_slices[i].region); 67 ae_data->ae_slices[i].region = NULL; 68 kfree(ae_data->ae_slices[i].page); 69 ae_data->ae_slices[i].page = NULL; 70 } 71 return 0; 72 } 73 74 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, 75 unsigned int str_offset) 76 { 77 if (!str_table->table_len || str_offset > str_table->table_len) 78 return NULL; 79 return (char *)(((uintptr_t)(str_table->strings)) + str_offset); 80 } 81 82 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) 83 { 84 int maj = hdr->maj_ver & 0xff; 85 int min = hdr->min_ver & 0xff; 86 87 if (hdr->file_id != ICP_QAT_UOF_FID) { 88 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); 89 return -EINVAL; 90 } 91 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { 92 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", 93 maj, min); 94 return -EINVAL; 95 } 96 return 0; 97 } 98 99 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) 100 { 101 int maj = suof_hdr->maj_ver & 0xff; 102 int min = suof_hdr->min_ver & 0xff; 103 104 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { 105 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); 106 return -EINVAL; 107 } 108 if (suof_hdr->fw_type != 0) { 109 pr_err("QAT: unsupported firmware type\n"); 110 return -EINVAL; 111 } 112 if (suof_hdr->num_chunks <= 0x1) { 113 pr_err("QAT: SUOF chunk amount is incorrect\n"); 114 return -EINVAL; 115 } 116 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { 117 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", 118 maj, min); 119 return -EINVAL; 120 } 121 return 0; 122 } 123 124 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, 125 unsigned int addr, unsigned int *val, 126 unsigned int num_in_bytes) 127 { 128 unsigned int outval; 129 unsigned char *ptr = (unsigned char *)val; 130 131 while (num_in_bytes) { 132 memcpy(&outval, ptr, 4); 133 SRAM_WRITE(handle, addr, outval); 134 num_in_bytes -= 4; 135 ptr += 4; 136 addr += 4; 137 } 138 } 139 140 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, 141 unsigned char ae, unsigned int addr, 142 unsigned int *val, 143 unsigned int num_in_bytes) 144 { 145 unsigned int outval; 146 unsigned char *ptr = (unsigned char *)val; 147 148 addr >>= 0x2; /* convert to uword address */ 149 150 while (num_in_bytes) { 151 memcpy(&outval, ptr, 4); 152 qat_hal_wr_umem(handle, ae, addr++, 1, &outval); 153 num_in_bytes -= 4; 154 ptr += 4; 155 } 156 } 157 158 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, 159 unsigned char ae, 160 struct icp_qat_uof_batch_init 161 *umem_init_header) 162 { 163 struct icp_qat_uof_batch_init *umem_init; 164 165 if (!umem_init_header) 166 return; 167 umem_init = umem_init_header->next; 168 while (umem_init) { 169 unsigned int addr, *value, size; 170 171 ae = umem_init->ae; 172 addr = umem_init->addr; 173 value = umem_init->value; 174 size = umem_init->size; 175 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); 176 umem_init = umem_init->next; 177 } 178 } 179 180 static void 181 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, 182 struct icp_qat_uof_batch_init **base) 183 { 184 struct icp_qat_uof_batch_init *umem_init; 185 186 umem_init = *base; 187 while (umem_init) { 188 struct icp_qat_uof_batch_init *pre; 189 190 pre = umem_init; 191 umem_init = umem_init->next; 192 kfree(pre); 193 } 194 *base = NULL; 195 } 196 197 static int qat_uclo_parse_num(char *str, unsigned int *num) 198 { 199 char buf[16] = {0}; 200 unsigned long ae = 0; 201 int i; 202 203 strncpy(buf, str, 15); 204 for (i = 0; i < 16; i++) { 205 if (!isdigit(buf[i])) { 206 buf[i] = '\0'; 207 break; 208 } 209 } 210 if ((kstrtoul(buf, 10, &ae))) 211 return -EFAULT; 212 213 *num = (unsigned int)ae; 214 return 0; 215 } 216 217 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, 218 struct icp_qat_uof_initmem *init_mem, 219 unsigned int size_range, unsigned int *ae) 220 { 221 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 222 char *str; 223 224 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { 225 pr_err("QAT: initmem is out of range"); 226 return -EINVAL; 227 } 228 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { 229 pr_err("QAT: Memory scope for init_mem error\n"); 230 return -EINVAL; 231 } 232 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); 233 if (!str) { 234 pr_err("QAT: AE name assigned in UOF init table is NULL\n"); 235 return -EINVAL; 236 } 237 if (qat_uclo_parse_num(str, ae)) { 238 pr_err("QAT: Parse num for AE number failed\n"); 239 return -EINVAL; 240 } 241 if (*ae >= ICP_QAT_UCLO_MAX_AE) { 242 pr_err("QAT: ae %d out of range\n", *ae); 243 return -EINVAL; 244 } 245 return 0; 246 } 247 248 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle 249 *handle, struct icp_qat_uof_initmem 250 *init_mem, unsigned int ae, 251 struct icp_qat_uof_batch_init 252 **init_tab_base) 253 { 254 struct icp_qat_uof_batch_init *init_header, *tail; 255 struct icp_qat_uof_batch_init *mem_init, *tail_old; 256 struct icp_qat_uof_memvar_attr *mem_val_attr; 257 unsigned int i, flag = 0; 258 259 mem_val_attr = 260 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + 261 sizeof(struct icp_qat_uof_initmem)); 262 263 init_header = *init_tab_base; 264 if (!init_header) { 265 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL); 266 if (!init_header) 267 return -ENOMEM; 268 init_header->size = 1; 269 *init_tab_base = init_header; 270 flag = 1; 271 } 272 tail_old = init_header; 273 while (tail_old->next) 274 tail_old = tail_old->next; 275 tail = tail_old; 276 for (i = 0; i < init_mem->val_attr_num; i++) { 277 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL); 278 if (!mem_init) 279 goto out_err; 280 mem_init->ae = ae; 281 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; 282 mem_init->value = &mem_val_attr->value; 283 mem_init->size = 4; 284 mem_init->next = NULL; 285 tail->next = mem_init; 286 tail = mem_init; 287 init_header->size += qat_hal_get_ins_num(); 288 mem_val_attr++; 289 } 290 return 0; 291 out_err: 292 /* Do not free the list head unless we allocated it. */ 293 tail_old = tail_old->next; 294 if (flag) { 295 kfree(*init_tab_base); 296 *init_tab_base = NULL; 297 } 298 299 while (tail_old) { 300 mem_init = tail_old->next; 301 kfree(tail_old); 302 tail_old = mem_init; 303 } 304 return -ENOMEM; 305 } 306 307 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, 308 struct icp_qat_uof_initmem *init_mem) 309 { 310 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 311 unsigned int ae; 312 313 if (qat_uclo_fetch_initmem_ae(handle, init_mem, 314 handle->chip_info->lm_size, &ae)) 315 return -EINVAL; 316 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 317 &obj_handle->lm_init_tab[ae])) 318 return -EINVAL; 319 return 0; 320 } 321 322 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, 323 struct icp_qat_uof_initmem *init_mem) 324 { 325 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 326 unsigned int ae, ustore_size, uaddr, i; 327 struct icp_qat_uclo_aedata *aed; 328 329 ustore_size = obj_handle->ustore_phy_size; 330 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) 331 return -EINVAL; 332 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 333 &obj_handle->umem_init_tab[ae])) 334 return -EINVAL; 335 /* set the highest ustore address referenced */ 336 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; 337 aed = &obj_handle->ae_data[ae]; 338 for (i = 0; i < aed->slice_num; i++) { 339 if (aed->ae_slices[i].encap_image->uwords_num < uaddr) 340 aed->ae_slices[i].encap_image->uwords_num = uaddr; 341 } 342 return 0; 343 } 344 345 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, 346 struct icp_qat_uof_initmem *init_mem) 347 { 348 switch (init_mem->region) { 349 case ICP_QAT_UOF_LMEM_REGION: 350 if (qat_uclo_init_lmem_seg(handle, init_mem)) 351 return -EINVAL; 352 break; 353 case ICP_QAT_UOF_UMEM_REGION: 354 if (qat_uclo_init_umem_seg(handle, init_mem)) 355 return -EINVAL; 356 break; 357 default: 358 pr_err("QAT: initmem region error. region type=0x%x\n", 359 init_mem->region); 360 return -EINVAL; 361 } 362 return 0; 363 } 364 365 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, 366 struct icp_qat_uclo_encapme *image) 367 { 368 unsigned int i; 369 struct icp_qat_uclo_encap_page *page; 370 struct icp_qat_uof_image *uof_image; 371 unsigned char ae; 372 unsigned int ustore_size; 373 unsigned int patt_pos; 374 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 375 unsigned long ae_mask = handle->hal_handle->ae_mask; 376 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 377 u64 *fill_data; 378 379 uof_image = image->img_ptr; 380 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64), 381 GFP_KERNEL); 382 if (!fill_data) 383 return -ENOMEM; 384 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) 385 memcpy(&fill_data[i], &uof_image->fill_pattern, 386 sizeof(u64)); 387 page = image->page; 388 389 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 390 unsigned long ae_assigned = uof_image->ae_assigned; 391 392 if (!test_bit(ae, &ae_assigned)) 393 continue; 394 395 if (!test_bit(ae, &cfg_ae_mask)) 396 continue; 397 398 ustore_size = obj_handle->ae_data[ae].eff_ustore_size; 399 patt_pos = page->beg_addr_p + page->micro_words_num; 400 401 qat_hal_wr_uwords(handle, (unsigned char)ae, 0, 402 page->beg_addr_p, &fill_data[0]); 403 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, 404 ustore_size - patt_pos + 1, 405 &fill_data[page->beg_addr_p]); 406 } 407 kfree(fill_data); 408 return 0; 409 } 410 411 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) 412 { 413 int i, ae; 414 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 415 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; 416 unsigned long ae_mask = handle->hal_handle->ae_mask; 417 418 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { 419 if (initmem->num_in_bytes) { 420 if (qat_uclo_init_ae_memory(handle, initmem)) 421 return -EINVAL; 422 } 423 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( 424 (uintptr_t)initmem + 425 sizeof(struct icp_qat_uof_initmem)) + 426 (sizeof(struct icp_qat_uof_memvar_attr) * 427 initmem->val_attr_num)); 428 } 429 430 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 431 if (qat_hal_batch_wr_lm(handle, ae, 432 obj_handle->lm_init_tab[ae])) { 433 pr_err("QAT: fail to batch init lmem for AE %d\n", ae); 434 return -EINVAL; 435 } 436 qat_uclo_cleanup_batch_init_list(handle, 437 &obj_handle->lm_init_tab[ae]); 438 qat_uclo_batch_wr_umem(handle, ae, 439 obj_handle->umem_init_tab[ae]); 440 qat_uclo_cleanup_batch_init_list(handle, 441 &obj_handle-> 442 umem_init_tab[ae]); 443 } 444 return 0; 445 } 446 447 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, 448 char *chunk_id, void *cur) 449 { 450 int i; 451 struct icp_qat_uof_chunkhdr *chunk_hdr = 452 (struct icp_qat_uof_chunkhdr *) 453 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); 454 455 for (i = 0; i < obj_hdr->num_chunks; i++) { 456 if ((cur < (void *)&chunk_hdr[i]) && 457 !strncmp(chunk_hdr[i].chunk_id, chunk_id, 458 ICP_QAT_UOF_OBJID_LEN)) { 459 return &chunk_hdr[i]; 460 } 461 } 462 return NULL; 463 } 464 465 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) 466 { 467 int i; 468 unsigned int topbit = 1 << 0xF; 469 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); 470 471 reg ^= inbyte << 0x8; 472 for (i = 0; i < 0x8; i++) { 473 if (reg & topbit) 474 reg = (reg << 1) ^ 0x1021; 475 else 476 reg <<= 1; 477 } 478 return reg & 0xFFFF; 479 } 480 481 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) 482 { 483 unsigned int chksum = 0; 484 485 if (ptr) 486 while (num--) 487 chksum = qat_uclo_calc_checksum(chksum, *ptr++); 488 return chksum; 489 } 490 491 static struct icp_qat_uclo_objhdr * 492 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, 493 char *chunk_id) 494 { 495 struct icp_qat_uof_filechunkhdr *file_chunk; 496 struct icp_qat_uclo_objhdr *obj_hdr; 497 char *chunk; 498 int i; 499 500 file_chunk = (struct icp_qat_uof_filechunkhdr *) 501 (buf + sizeof(struct icp_qat_uof_filehdr)); 502 for (i = 0; i < file_hdr->num_chunks; i++) { 503 if (!strncmp(file_chunk->chunk_id, chunk_id, 504 ICP_QAT_UOF_OBJID_LEN)) { 505 chunk = buf + file_chunk->offset; 506 if (file_chunk->checksum != qat_uclo_calc_str_checksum( 507 chunk, file_chunk->size)) 508 break; 509 obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL); 510 if (!obj_hdr) 511 break; 512 obj_hdr->file_buff = chunk; 513 obj_hdr->checksum = file_chunk->checksum; 514 obj_hdr->size = file_chunk->size; 515 return obj_hdr; 516 } 517 file_chunk++; 518 } 519 return NULL; 520 } 521 522 static int 523 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, 524 struct icp_qat_uof_image *image) 525 { 526 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; 527 struct icp_qat_uof_objtable *neigh_reg_tab; 528 struct icp_qat_uof_code_page *code_page; 529 530 code_page = (struct icp_qat_uof_code_page *) 531 ((char *)image + sizeof(struct icp_qat_uof_image)); 532 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 533 code_page->uc_var_tab_offset); 534 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 535 code_page->imp_var_tab_offset); 536 imp_expr_tab = (struct icp_qat_uof_objtable *) 537 (encap_uof_obj->beg_uof + 538 code_page->imp_expr_tab_offset); 539 if (uc_var_tab->entry_num || imp_var_tab->entry_num || 540 imp_expr_tab->entry_num) { 541 pr_err("QAT: UOF can't contain imported variable to be parsed\n"); 542 return -EINVAL; 543 } 544 neigh_reg_tab = (struct icp_qat_uof_objtable *) 545 (encap_uof_obj->beg_uof + 546 code_page->neigh_reg_tab_offset); 547 if (neigh_reg_tab->entry_num) { 548 pr_err("QAT: UOF can't contain neighbor register table\n"); 549 return -EINVAL; 550 } 551 if (image->numpages > 1) { 552 pr_err("QAT: UOF can't contain multiple pages\n"); 553 return -EINVAL; 554 } 555 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { 556 pr_err("QAT: UOF can't use shared control store feature\n"); 557 return -EFAULT; 558 } 559 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { 560 pr_err("QAT: UOF can't use reloadable feature\n"); 561 return -EFAULT; 562 } 563 return 0; 564 } 565 566 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj 567 *encap_uof_obj, 568 struct icp_qat_uof_image *img, 569 struct icp_qat_uclo_encap_page *page) 570 { 571 struct icp_qat_uof_code_page *code_page; 572 struct icp_qat_uof_code_area *code_area; 573 struct icp_qat_uof_objtable *uword_block_tab; 574 struct icp_qat_uof_uword_block *uwblock; 575 int i; 576 577 code_page = (struct icp_qat_uof_code_page *) 578 ((char *)img + sizeof(struct icp_qat_uof_image)); 579 page->def_page = code_page->def_page; 580 page->page_region = code_page->page_region; 581 page->beg_addr_v = code_page->beg_addr_v; 582 page->beg_addr_p = code_page->beg_addr_p; 583 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + 584 code_page->code_area_offset); 585 page->micro_words_num = code_area->micro_words_num; 586 uword_block_tab = (struct icp_qat_uof_objtable *) 587 (encap_uof_obj->beg_uof + 588 code_area->uword_block_tab); 589 page->uwblock_num = uword_block_tab->entry_num; 590 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + 591 sizeof(struct icp_qat_uof_objtable)); 592 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; 593 for (i = 0; i < uword_block_tab->entry_num; i++) 594 page->uwblock[i].micro_words = 595 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; 596 } 597 598 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, 599 struct icp_qat_uclo_encapme *ae_uimage, 600 int max_image) 601 { 602 int i, j; 603 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; 604 struct icp_qat_uof_image *image; 605 struct icp_qat_uof_objtable *ae_regtab; 606 struct icp_qat_uof_objtable *init_reg_sym_tab; 607 struct icp_qat_uof_objtable *sbreak_tab; 608 struct icp_qat_uof_encap_obj *encap_uof_obj = 609 &obj_handle->encap_uof_obj; 610 611 for (j = 0; j < max_image; j++) { 612 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 613 ICP_QAT_UOF_IMAG, chunk_hdr); 614 if (!chunk_hdr) 615 break; 616 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + 617 chunk_hdr->offset); 618 ae_regtab = (struct icp_qat_uof_objtable *) 619 (image->reg_tab_offset + 620 obj_handle->obj_hdr->file_buff); 621 ae_uimage[j].ae_reg_num = ae_regtab->entry_num; 622 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) 623 (((char *)ae_regtab) + 624 sizeof(struct icp_qat_uof_objtable)); 625 init_reg_sym_tab = (struct icp_qat_uof_objtable *) 626 (image->init_reg_sym_tab + 627 obj_handle->obj_hdr->file_buff); 628 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; 629 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) 630 (((char *)init_reg_sym_tab) + 631 sizeof(struct icp_qat_uof_objtable)); 632 sbreak_tab = (struct icp_qat_uof_objtable *) 633 (image->sbreak_tab + obj_handle->obj_hdr->file_buff); 634 ae_uimage[j].sbreak_num = sbreak_tab->entry_num; 635 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) 636 (((char *)sbreak_tab) + 637 sizeof(struct icp_qat_uof_objtable)); 638 ae_uimage[j].img_ptr = image; 639 if (qat_uclo_check_image_compat(encap_uof_obj, image)) 640 goto out_err; 641 ae_uimage[j].page = 642 kzalloc(sizeof(struct icp_qat_uclo_encap_page), 643 GFP_KERNEL); 644 if (!ae_uimage[j].page) 645 goto out_err; 646 qat_uclo_map_image_page(encap_uof_obj, image, 647 ae_uimage[j].page); 648 } 649 return j; 650 out_err: 651 for (i = 0; i < j; i++) 652 kfree(ae_uimage[i].page); 653 return 0; 654 } 655 656 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) 657 { 658 int i, ae; 659 int mflag = 0; 660 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 661 unsigned long ae_mask = handle->hal_handle->ae_mask; 662 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 663 664 for_each_set_bit(ae, &ae_mask, max_ae) { 665 if (!test_bit(ae, &cfg_ae_mask)) 666 continue; 667 668 for (i = 0; i < obj_handle->uimage_num; i++) { 669 unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; 670 671 if (!test_bit(ae, &ae_assigned)) 672 continue; 673 mflag = 1; 674 if (qat_uclo_init_ae_data(obj_handle, ae, i)) 675 return -EINVAL; 676 } 677 } 678 if (!mflag) { 679 pr_err("QAT: uimage uses AE not set\n"); 680 return -EINVAL; 681 } 682 return 0; 683 } 684 685 static struct icp_qat_uof_strtable * 686 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, 687 char *tab_name, struct icp_qat_uof_strtable *str_table) 688 { 689 struct icp_qat_uof_chunkhdr *chunk_hdr; 690 691 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) 692 obj_hdr->file_buff, tab_name, NULL); 693 if (chunk_hdr) { 694 int hdr_size; 695 696 memcpy(&str_table->table_len, obj_hdr->file_buff + 697 chunk_hdr->offset, sizeof(str_table->table_len)); 698 hdr_size = (char *)&str_table->strings - (char *)str_table; 699 str_table->strings = (uintptr_t)obj_hdr->file_buff + 700 chunk_hdr->offset + hdr_size; 701 return str_table; 702 } 703 return NULL; 704 } 705 706 static void 707 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, 708 struct icp_qat_uclo_init_mem_table *init_mem_tab) 709 { 710 struct icp_qat_uof_chunkhdr *chunk_hdr; 711 712 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 713 ICP_QAT_UOF_IMEM, NULL); 714 if (chunk_hdr) { 715 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + 716 chunk_hdr->offset, sizeof(unsigned int)); 717 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) 718 (encap_uof_obj->beg_uof + chunk_hdr->offset + 719 sizeof(unsigned int)); 720 } 721 } 722 723 static unsigned int 724 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) 725 { 726 switch (handle->pci_dev->device) { 727 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: 728 return ICP_QAT_AC_895XCC_DEV_TYPE; 729 case PCI_DEVICE_ID_INTEL_QAT_C62X: 730 return ICP_QAT_AC_C62X_DEV_TYPE; 731 case PCI_DEVICE_ID_INTEL_QAT_C3XXX: 732 return ICP_QAT_AC_C3XXX_DEV_TYPE; 733 case ADF_4XXX_PCI_DEVICE_ID: 734 case ADF_401XX_PCI_DEVICE_ID: 735 case ADF_402XX_PCI_DEVICE_ID: 736 return ICP_QAT_AC_4XXX_A_DEV_TYPE; 737 default: 738 pr_err("QAT: unsupported device 0x%x\n", 739 handle->pci_dev->device); 740 return 0; 741 } 742 } 743 744 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) 745 { 746 unsigned int maj_ver, prod_type = obj_handle->prod_type; 747 748 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { 749 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", 750 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, 751 prod_type); 752 return -EINVAL; 753 } 754 maj_ver = obj_handle->prod_rev & 0xff; 755 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || 756 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { 757 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); 758 return -EINVAL; 759 } 760 return 0; 761 } 762 763 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, 764 unsigned char ae, unsigned char ctx_mask, 765 enum icp_qat_uof_regtype reg_type, 766 unsigned short reg_addr, unsigned int value) 767 { 768 switch (reg_type) { 769 case ICP_GPA_ABS: 770 case ICP_GPB_ABS: 771 ctx_mask = 0; 772 fallthrough; 773 case ICP_GPA_REL: 774 case ICP_GPB_REL: 775 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, 776 reg_addr, value); 777 case ICP_SR_ABS: 778 case ICP_DR_ABS: 779 case ICP_SR_RD_ABS: 780 case ICP_DR_RD_ABS: 781 ctx_mask = 0; 782 fallthrough; 783 case ICP_SR_REL: 784 case ICP_DR_REL: 785 case ICP_SR_RD_REL: 786 case ICP_DR_RD_REL: 787 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, 788 reg_addr, value); 789 case ICP_SR_WR_ABS: 790 case ICP_DR_WR_ABS: 791 ctx_mask = 0; 792 fallthrough; 793 case ICP_SR_WR_REL: 794 case ICP_DR_WR_REL: 795 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, 796 reg_addr, value); 797 case ICP_NEIGH_REL: 798 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); 799 default: 800 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); 801 return -EFAULT; 802 } 803 return 0; 804 } 805 806 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, 807 unsigned int ae, 808 struct icp_qat_uclo_encapme *encap_ae) 809 { 810 unsigned int i; 811 unsigned char ctx_mask; 812 struct icp_qat_uof_init_regsym *init_regsym; 813 814 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == 815 ICP_QAT_UCLO_MAX_CTX) 816 ctx_mask = 0xff; 817 else 818 ctx_mask = 0x55; 819 820 for (i = 0; i < encap_ae->init_regsym_num; i++) { 821 unsigned int exp_res; 822 823 init_regsym = &encap_ae->init_regsym[i]; 824 exp_res = init_regsym->value; 825 switch (init_regsym->init_type) { 826 case ICP_QAT_UOF_INIT_REG: 827 qat_uclo_init_reg(handle, ae, ctx_mask, 828 (enum icp_qat_uof_regtype) 829 init_regsym->reg_type, 830 (unsigned short)init_regsym->reg_addr, 831 exp_res); 832 break; 833 case ICP_QAT_UOF_INIT_REG_CTX: 834 /* check if ctx is appropriate for the ctxMode */ 835 if (!((1 << init_regsym->ctx) & ctx_mask)) { 836 pr_err("QAT: invalid ctx num = 0x%x\n", 837 init_regsym->ctx); 838 return -EINVAL; 839 } 840 qat_uclo_init_reg(handle, ae, 841 (unsigned char) 842 (1 << init_regsym->ctx), 843 (enum icp_qat_uof_regtype) 844 init_regsym->reg_type, 845 (unsigned short)init_regsym->reg_addr, 846 exp_res); 847 break; 848 case ICP_QAT_UOF_INIT_EXPR: 849 pr_err("QAT: INIT_EXPR feature not supported\n"); 850 return -EINVAL; 851 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: 852 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); 853 return -EINVAL; 854 default: 855 break; 856 } 857 } 858 return 0; 859 } 860 861 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) 862 { 863 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 864 unsigned long ae_mask = handle->hal_handle->ae_mask; 865 struct icp_qat_uclo_aedata *aed; 866 unsigned int s, ae; 867 868 if (obj_handle->global_inited) 869 return 0; 870 if (obj_handle->init_mem_tab.entry_num) { 871 if (qat_uclo_init_memory(handle)) { 872 pr_err("QAT: initialize memory failed\n"); 873 return -EINVAL; 874 } 875 } 876 877 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 878 aed = &obj_handle->ae_data[ae]; 879 for (s = 0; s < aed->slice_num; s++) { 880 if (!aed->ae_slices[s].encap_image) 881 continue; 882 if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) 883 return -EINVAL; 884 } 885 } 886 obj_handle->global_inited = 1; 887 return 0; 888 } 889 890 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, 891 struct icp_qat_uclo_objhandle *obj_handle, 892 unsigned char ae, 893 struct icp_qat_uof_image *uof_image) 894 { 895 unsigned char mode; 896 int ret; 897 898 mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); 899 ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); 900 if (ret) { 901 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); 902 return ret; 903 } 904 if (handle->chip_info->nn) { 905 mode = ICP_QAT_NN_MODE(uof_image->ae_mode); 906 ret = qat_hal_set_ae_nn_mode(handle, ae, mode); 907 if (ret) { 908 pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); 909 return ret; 910 } 911 } 912 mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); 913 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); 914 if (ret) { 915 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); 916 return ret; 917 } 918 mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); 919 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); 920 if (ret) { 921 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); 922 return ret; 923 } 924 if (handle->chip_info->lm2lm3) { 925 mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); 926 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); 927 if (ret) { 928 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); 929 return ret; 930 } 931 mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); 932 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); 933 if (ret) { 934 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); 935 return ret; 936 } 937 mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); 938 qat_hal_set_ae_tindex_mode(handle, ae, mode); 939 } 940 return 0; 941 } 942 943 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) 944 { 945 struct icp_qat_uof_image *uof_image; 946 struct icp_qat_uclo_aedata *ae_data; 947 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 948 unsigned long ae_mask = handle->hal_handle->ae_mask; 949 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 950 unsigned char ae, s; 951 int error; 952 953 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 954 if (!test_bit(ae, &cfg_ae_mask)) 955 continue; 956 957 ae_data = &obj_handle->ae_data[ae]; 958 for (s = 0; s < min_t(unsigned int, ae_data->slice_num, 959 ICP_QAT_UCLO_MAX_CTX); s++) { 960 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) 961 continue; 962 uof_image = ae_data->ae_slices[s].encap_image->img_ptr; 963 error = qat_hal_set_modes(handle, obj_handle, ae, 964 uof_image); 965 if (error) 966 return error; 967 } 968 } 969 return 0; 970 } 971 972 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) 973 { 974 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 975 struct icp_qat_uclo_encapme *image; 976 int a; 977 978 for (a = 0; a < obj_handle->uimage_num; a++) { 979 image = &obj_handle->ae_uimage[a]; 980 image->uwords_num = image->page->beg_addr_p + 981 image->page->micro_words_num; 982 } 983 } 984 985 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) 986 { 987 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 988 unsigned int ae; 989 990 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; 991 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) 992 obj_handle->obj_hdr->file_buff; 993 obj_handle->uword_in_bytes = 6; 994 obj_handle->prod_type = qat_uclo_get_dev_type(handle); 995 obj_handle->prod_rev = PID_MAJOR_REV | 996 (PID_MINOR_REV & handle->hal_handle->revision_id); 997 if (qat_uclo_check_uof_compat(obj_handle)) { 998 pr_err("QAT: UOF incompatible\n"); 999 return -EINVAL; 1000 } 1001 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), 1002 GFP_KERNEL); 1003 if (!obj_handle->uword_buf) 1004 return -ENOMEM; 1005 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; 1006 if (!obj_handle->obj_hdr->file_buff || 1007 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, 1008 &obj_handle->str_table)) { 1009 pr_err("QAT: UOF doesn't have effective images\n"); 1010 goto out_err; 1011 } 1012 obj_handle->uimage_num = 1013 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, 1014 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); 1015 if (!obj_handle->uimage_num) 1016 goto out_err; 1017 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { 1018 pr_err("QAT: Bad object\n"); 1019 goto out_check_uof_aemask_err; 1020 } 1021 qat_uclo_init_uword_num(handle); 1022 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, 1023 &obj_handle->init_mem_tab); 1024 if (qat_uclo_set_ae_mode(handle)) 1025 goto out_check_uof_aemask_err; 1026 return 0; 1027 out_check_uof_aemask_err: 1028 for (ae = 0; ae < obj_handle->uimage_num; ae++) 1029 kfree(obj_handle->ae_uimage[ae].page); 1030 out_err: 1031 kfree(obj_handle->uword_buf); 1032 return -EFAULT; 1033 } 1034 1035 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1036 struct icp_qat_suof_filehdr *suof_ptr, 1037 int suof_size) 1038 { 1039 unsigned int check_sum = 0; 1040 unsigned int min_ver_offset = 0; 1041 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1042 1043 suof_handle->file_id = ICP_QAT_SUOF_FID; 1044 suof_handle->suof_buf = (char *)suof_ptr; 1045 suof_handle->suof_size = suof_size; 1046 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, 1047 min_ver); 1048 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, 1049 min_ver_offset); 1050 if (check_sum != suof_ptr->check_sum) { 1051 pr_err("QAT: incorrect SUOF checksum\n"); 1052 return -EINVAL; 1053 } 1054 suof_handle->check_sum = suof_ptr->check_sum; 1055 suof_handle->min_ver = suof_ptr->min_ver; 1056 suof_handle->maj_ver = suof_ptr->maj_ver; 1057 suof_handle->fw_type = suof_ptr->fw_type; 1058 return 0; 1059 } 1060 1061 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, 1062 struct icp_qat_suof_img_hdr *suof_img_hdr, 1063 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1064 { 1065 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1066 struct icp_qat_simg_ae_mode *ae_mode; 1067 struct icp_qat_suof_objhdr *suof_objhdr; 1068 1069 suof_img_hdr->simg_buf = (suof_handle->suof_buf + 1070 suof_chunk_hdr->offset + 1071 sizeof(*suof_objhdr)); 1072 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) 1073 (suof_handle->suof_buf + 1074 suof_chunk_hdr->offset))->img_length; 1075 1076 suof_img_hdr->css_header = suof_img_hdr->simg_buf; 1077 suof_img_hdr->css_key = (suof_img_hdr->css_header + 1078 sizeof(struct icp_qat_css_hdr)); 1079 suof_img_hdr->css_signature = suof_img_hdr->css_key + 1080 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1081 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); 1082 suof_img_hdr->css_simg = suof_img_hdr->css_signature + 1083 ICP_QAT_CSS_SIGNATURE_LEN(handle); 1084 1085 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); 1086 suof_img_hdr->ae_mask = ae_mode->ae_mask; 1087 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; 1088 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; 1089 suof_img_hdr->fw_type = ae_mode->fw_type; 1090 } 1091 1092 static void 1093 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, 1094 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1095 { 1096 char **sym_str = (char **)&suof_handle->sym_str; 1097 unsigned int *sym_size = &suof_handle->sym_size; 1098 struct icp_qat_suof_strtable *str_table_obj; 1099 1100 *sym_size = *(unsigned int *)(uintptr_t) 1101 (suof_chunk_hdr->offset + suof_handle->suof_buf); 1102 *sym_str = (char *)(uintptr_t) 1103 (suof_handle->suof_buf + suof_chunk_hdr->offset + 1104 sizeof(str_table_obj->tab_length)); 1105 } 1106 1107 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, 1108 struct icp_qat_suof_img_hdr *img_hdr) 1109 { 1110 struct icp_qat_simg_ae_mode *img_ae_mode = NULL; 1111 unsigned int prod_rev, maj_ver, prod_type; 1112 1113 prod_type = qat_uclo_get_dev_type(handle); 1114 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; 1115 prod_rev = PID_MAJOR_REV | 1116 (PID_MINOR_REV & handle->hal_handle->revision_id); 1117 if (img_ae_mode->dev_type != prod_type) { 1118 pr_err("QAT: incompatible product type %x\n", 1119 img_ae_mode->dev_type); 1120 return -EINVAL; 1121 } 1122 maj_ver = prod_rev & 0xff; 1123 if (maj_ver > img_ae_mode->devmax_ver || 1124 maj_ver < img_ae_mode->devmin_ver) { 1125 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); 1126 return -EINVAL; 1127 } 1128 return 0; 1129 } 1130 1131 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) 1132 { 1133 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 1134 1135 kfree(sobj_handle->img_table.simg_hdr); 1136 sobj_handle->img_table.simg_hdr = NULL; 1137 kfree(handle->sobj_handle); 1138 handle->sobj_handle = NULL; 1139 } 1140 1141 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, 1142 unsigned int img_id, unsigned int num_simgs) 1143 { 1144 struct icp_qat_suof_img_hdr img_header; 1145 1146 if (img_id != num_simgs - 1) { 1147 memcpy(&img_header, &suof_img_hdr[num_simgs - 1], 1148 sizeof(*suof_img_hdr)); 1149 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], 1150 sizeof(*suof_img_hdr)); 1151 memcpy(&suof_img_hdr[img_id], &img_header, 1152 sizeof(*suof_img_hdr)); 1153 } 1154 } 1155 1156 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, 1157 struct icp_qat_suof_filehdr *suof_ptr, 1158 int suof_size) 1159 { 1160 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1161 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; 1162 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; 1163 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; 1164 unsigned int i = 0; 1165 struct icp_qat_suof_img_hdr img_header; 1166 1167 if (!suof_ptr || suof_size == 0) { 1168 pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); 1169 return -EINVAL; 1170 } 1171 if (qat_uclo_check_suof_format(suof_ptr)) 1172 return -EINVAL; 1173 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); 1174 if (ret) 1175 return ret; 1176 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) 1177 ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); 1178 1179 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); 1180 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; 1181 1182 if (suof_handle->img_table.num_simgs != 0) { 1183 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs, 1184 sizeof(img_header), 1185 GFP_KERNEL); 1186 if (!suof_img_hdr) 1187 return -ENOMEM; 1188 suof_handle->img_table.simg_hdr = suof_img_hdr; 1189 1190 for (i = 0; i < suof_handle->img_table.num_simgs; i++) { 1191 qat_uclo_map_simg(handle, &suof_img_hdr[i], 1192 &suof_chunk_hdr[1 + i]); 1193 ret = qat_uclo_check_simg_compat(handle, 1194 &suof_img_hdr[i]); 1195 if (ret) 1196 return ret; 1197 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; 1198 if ((suof_img_hdr[i].ae_mask & 0x1) != 0) 1199 ae0_img = i; 1200 } 1201 1202 if (!handle->chip_info->tgroup_share_ustore) { 1203 qat_uclo_tail_img(suof_img_hdr, ae0_img, 1204 suof_handle->img_table.num_simgs); 1205 } 1206 } 1207 return 0; 1208 } 1209 1210 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) 1211 #define BITS_IN_DWORD 32 1212 1213 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, 1214 struct icp_qat_fw_auth_desc *desc) 1215 { 1216 u32 fcu_sts, retry = 0; 1217 u32 fcu_ctl_csr, fcu_sts_csr; 1218 u32 fcu_dram_hi_csr, fcu_dram_lo_csr; 1219 u64 bus_addr; 1220 1221 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) 1222 - sizeof(struct icp_qat_auth_chunk); 1223 1224 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1225 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1226 fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; 1227 fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; 1228 1229 SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); 1230 SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); 1231 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); 1232 1233 do { 1234 msleep(FW_AUTH_WAIT_PERIOD); 1235 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1236 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) 1237 goto auth_fail; 1238 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) 1239 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) 1240 return 0; 1241 } while (retry++ < FW_AUTH_MAX_RETRY); 1242 auth_fail: 1243 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", 1244 fcu_sts & FCU_AUTH_STS_MASK, retry); 1245 return -EINVAL; 1246 } 1247 1248 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, 1249 int imgid) 1250 { 1251 struct icp_qat_suof_handle *sobj_handle; 1252 1253 if (!handle->chip_info->tgroup_share_ustore) 1254 return false; 1255 1256 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; 1257 if (handle->hal_handle->admin_ae_mask & 1258 sobj_handle->img_table.simg_hdr[imgid].ae_mask) 1259 return false; 1260 1261 return true; 1262 } 1263 1264 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, 1265 struct icp_qat_fw_auth_desc *desc) 1266 { 1267 unsigned long ae_mask = handle->hal_handle->ae_mask; 1268 unsigned long desc_ae_mask = desc->ae_mask; 1269 u32 fcu_sts, ae_broadcast_mask = 0; 1270 u32 fcu_loaded_csr, ae_loaded; 1271 u32 fcu_sts_csr, fcu_ctl_csr; 1272 unsigned int ae, retry = 0; 1273 1274 if (handle->chip_info->tgroup_share_ustore) { 1275 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1276 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1277 fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1278 } else { 1279 pr_err("Chip 0x%x doesn't support broadcast load\n", 1280 handle->pci_dev->device); 1281 return -EINVAL; 1282 } 1283 1284 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 1285 if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { 1286 pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); 1287 return -EINVAL; 1288 } 1289 1290 if (test_bit(ae, &desc_ae_mask)) 1291 ae_broadcast_mask |= 1 << ae; 1292 } 1293 1294 if (ae_broadcast_mask) { 1295 SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, 1296 ae_broadcast_mask); 1297 1298 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); 1299 1300 do { 1301 msleep(FW_AUTH_WAIT_PERIOD); 1302 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1303 fcu_sts &= FCU_AUTH_STS_MASK; 1304 1305 if (fcu_sts == FCU_STS_LOAD_FAIL) { 1306 pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); 1307 return -EINVAL; 1308 } else if (fcu_sts == FCU_STS_LOAD_DONE) { 1309 ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); 1310 ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; 1311 1312 if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) 1313 break; 1314 } 1315 } while (retry++ < FW_AUTH_MAX_RETRY); 1316 1317 if (retry > FW_AUTH_MAX_RETRY) { 1318 pr_err("QAT: broadcast load failed timeout %d\n", retry); 1319 return -EINVAL; 1320 } 1321 } 1322 return 0; 1323 } 1324 1325 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, 1326 struct icp_firml_dram_desc *dram_desc, 1327 unsigned int size) 1328 { 1329 void *vptr; 1330 dma_addr_t ptr; 1331 1332 vptr = dma_alloc_coherent(&handle->pci_dev->dev, 1333 size, &ptr, GFP_KERNEL); 1334 if (!vptr) 1335 return -ENOMEM; 1336 dram_desc->dram_base_addr_v = vptr; 1337 dram_desc->dram_bus_addr = ptr; 1338 dram_desc->dram_size = size; 1339 return 0; 1340 } 1341 1342 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, 1343 struct icp_firml_dram_desc *dram_desc) 1344 { 1345 if (handle && dram_desc && dram_desc->dram_base_addr_v) { 1346 dma_free_coherent(&handle->pci_dev->dev, 1347 (size_t)(dram_desc->dram_size), 1348 dram_desc->dram_base_addr_v, 1349 dram_desc->dram_bus_addr); 1350 } 1351 1352 if (dram_desc) 1353 memset(dram_desc, 0, sizeof(*dram_desc)); 1354 } 1355 1356 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, 1357 struct icp_qat_fw_auth_desc **desc) 1358 { 1359 struct icp_firml_dram_desc dram_desc; 1360 1361 if (*desc) { 1362 dram_desc.dram_base_addr_v = *desc; 1363 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) 1364 (*desc))->chunk_bus_addr; 1365 dram_desc.dram_size = ((struct icp_qat_auth_chunk *) 1366 (*desc))->chunk_size; 1367 qat_uclo_simg_free(handle, &dram_desc); 1368 } 1369 } 1370 1371 static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, 1372 char *image, unsigned int size, 1373 unsigned int fw_type) 1374 { 1375 char *fw_type_name = fw_type ? "MMP" : "AE"; 1376 unsigned int css_dword_size = sizeof(u32); 1377 1378 if (handle->chip_info->fw_auth) { 1379 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1380 unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); 1381 1382 if ((css_hdr->header_len * css_dword_size) != header_len) 1383 goto err; 1384 if ((css_hdr->size * css_dword_size) != size) 1385 goto err; 1386 if (fw_type != css_hdr->fw_type) 1387 goto err; 1388 if (size <= header_len) 1389 goto err; 1390 size -= header_len; 1391 } 1392 1393 if (fw_type == CSS_AE_FIRMWARE) { 1394 if (size < sizeof(struct icp_qat_simg_ae_mode *) + 1395 ICP_QAT_SIMG_AE_INIT_SEQ_LEN) 1396 goto err; 1397 if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) 1398 goto err; 1399 } else if (fw_type == CSS_MMP_FIRMWARE) { 1400 if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) 1401 goto err; 1402 } else { 1403 pr_err("QAT: Unsupported firmware type\n"); 1404 return -EINVAL; 1405 } 1406 return 0; 1407 1408 err: 1409 pr_err("QAT: Invalid %s firmware image\n", fw_type_name); 1410 return -EINVAL; 1411 } 1412 1413 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, 1414 char *image, unsigned int size, 1415 struct icp_qat_fw_auth_desc **desc) 1416 { 1417 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1418 struct icp_qat_fw_auth_desc *auth_desc; 1419 struct icp_qat_auth_chunk *auth_chunk; 1420 u64 virt_addr, bus_addr, virt_base; 1421 unsigned int length, simg_offset = sizeof(*auth_chunk); 1422 struct icp_qat_simg_ae_mode *simg_ae_mode; 1423 struct icp_firml_dram_desc img_desc; 1424 1425 if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { 1426 pr_err("QAT: error, input image size overflow %d\n", size); 1427 return -EINVAL; 1428 } 1429 length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? 1430 ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : 1431 size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; 1432 if (qat_uclo_simg_alloc(handle, &img_desc, length)) { 1433 pr_err("QAT: error, allocate continuous dram fail\n"); 1434 return -ENOMEM; 1435 } 1436 1437 auth_chunk = img_desc.dram_base_addr_v; 1438 auth_chunk->chunk_size = img_desc.dram_size; 1439 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; 1440 virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; 1441 bus_addr = img_desc.dram_bus_addr + simg_offset; 1442 auth_desc = img_desc.dram_base_addr_v; 1443 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1444 auth_desc->css_hdr_low = (unsigned int)bus_addr; 1445 virt_addr = virt_base; 1446 1447 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); 1448 /* pub key */ 1449 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + 1450 sizeof(*css_hdr); 1451 virt_addr = virt_addr + sizeof(*css_hdr); 1452 1453 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1454 auth_desc->fwsk_pub_low = (unsigned int)bus_addr; 1455 1456 memcpy((void *)(uintptr_t)virt_addr, 1457 (void *)(image + sizeof(*css_hdr)), 1458 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); 1459 /* padding */ 1460 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1461 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); 1462 1463 /* exponent */ 1464 memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1465 ICP_QAT_CSS_FWSK_PAD_LEN(handle)), 1466 (void *)(image + sizeof(*css_hdr) + 1467 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1468 sizeof(unsigned int)); 1469 1470 /* signature */ 1471 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, 1472 auth_desc->fwsk_pub_low) + 1473 ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1474 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1475 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1476 auth_desc->signature_low = (unsigned int)bus_addr; 1477 1478 memcpy((void *)(uintptr_t)virt_addr, 1479 (void *)(image + sizeof(*css_hdr) + 1480 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1481 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), 1482 ICP_QAT_CSS_SIGNATURE_LEN(handle)); 1483 1484 bus_addr = ADD_ADDR(auth_desc->signature_high, 1485 auth_desc->signature_low) + 1486 ICP_QAT_CSS_SIGNATURE_LEN(handle); 1487 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1488 1489 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1490 auth_desc->img_low = (unsigned int)bus_addr; 1491 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); 1492 memcpy((void *)(uintptr_t)virt_addr, 1493 (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), 1494 auth_desc->img_len); 1495 virt_addr = virt_base; 1496 /* AE firmware */ 1497 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == 1498 CSS_AE_FIRMWARE) { 1499 auth_desc->img_ae_mode_data_high = auth_desc->img_high; 1500 auth_desc->img_ae_mode_data_low = auth_desc->img_low; 1501 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, 1502 auth_desc->img_ae_mode_data_low) + 1503 sizeof(struct icp_qat_simg_ae_mode); 1504 1505 auth_desc->img_ae_init_data_high = (unsigned int) 1506 (bus_addr >> BITS_IN_DWORD); 1507 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; 1508 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; 1509 auth_desc->img_ae_insts_high = (unsigned int) 1510 (bus_addr >> BITS_IN_DWORD); 1511 auth_desc->img_ae_insts_low = (unsigned int)bus_addr; 1512 virt_addr += sizeof(struct icp_qat_css_hdr); 1513 virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1514 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1515 simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; 1516 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; 1517 } else { 1518 auth_desc->img_ae_insts_high = auth_desc->img_high; 1519 auth_desc->img_ae_insts_low = auth_desc->img_low; 1520 } 1521 *desc = auth_desc; 1522 return 0; 1523 } 1524 1525 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, 1526 struct icp_qat_fw_auth_desc *desc) 1527 { 1528 unsigned long ae_mask = handle->hal_handle->ae_mask; 1529 u32 fcu_sts_csr, fcu_ctl_csr; 1530 u32 loaded_aes, loaded_csr; 1531 unsigned int i; 1532 u32 fcu_sts; 1533 1534 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1535 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1536 loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1537 1538 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { 1539 int retry = 0; 1540 1541 if (!((desc->ae_mask >> i) & 0x1)) 1542 continue; 1543 if (qat_hal_check_ae_active(handle, i)) { 1544 pr_err("QAT: AE %d is active\n", i); 1545 return -EINVAL; 1546 } 1547 SET_CAP_CSR(handle, fcu_ctl_csr, 1548 (FCU_CTRL_CMD_LOAD | 1549 (1 << FCU_CTRL_BROADCAST_POS) | 1550 (i << FCU_CTRL_AE_POS))); 1551 1552 do { 1553 msleep(FW_AUTH_WAIT_PERIOD); 1554 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1555 if ((fcu_sts & FCU_AUTH_STS_MASK) == 1556 FCU_STS_LOAD_DONE) { 1557 loaded_aes = GET_CAP_CSR(handle, loaded_csr); 1558 loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; 1559 if (loaded_aes & (1 << i)) 1560 break; 1561 } 1562 } while (retry++ < FW_AUTH_MAX_RETRY); 1563 if (retry > FW_AUTH_MAX_RETRY) { 1564 pr_err("QAT: firmware load failed timeout %x\n", retry); 1565 return -EINVAL; 1566 } 1567 } 1568 return 0; 1569 } 1570 1571 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, 1572 void *addr_ptr, int mem_size) 1573 { 1574 struct icp_qat_suof_handle *suof_handle; 1575 1576 suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL); 1577 if (!suof_handle) 1578 return -ENOMEM; 1579 handle->sobj_handle = suof_handle; 1580 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { 1581 qat_uclo_del_suof(handle); 1582 pr_err("QAT: map SUOF failed\n"); 1583 return -EINVAL; 1584 } 1585 return 0; 1586 } 1587 1588 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, 1589 void *addr_ptr, int mem_size) 1590 { 1591 struct icp_qat_fw_auth_desc *desc = NULL; 1592 int status = 0; 1593 int ret; 1594 1595 ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); 1596 if (ret) 1597 return ret; 1598 1599 if (handle->chip_info->fw_auth) { 1600 status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); 1601 if (!status) 1602 status = qat_uclo_auth_fw(handle, desc); 1603 qat_uclo_ummap_auth_fw(handle, &desc); 1604 } else { 1605 if (handle->chip_info->mmp_sram_size < mem_size) { 1606 pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); 1607 return -EFBIG; 1608 } 1609 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); 1610 } 1611 return status; 1612 } 1613 1614 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, 1615 void *addr_ptr, int mem_size) 1616 { 1617 struct icp_qat_uof_filehdr *filehdr; 1618 struct icp_qat_uclo_objhandle *objhdl; 1619 1620 objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); 1621 if (!objhdl) 1622 return -ENOMEM; 1623 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); 1624 if (!objhdl->obj_buf) 1625 goto out_objbuf_err; 1626 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; 1627 if (qat_uclo_check_uof_format(filehdr)) 1628 goto out_objhdr_err; 1629 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, 1630 ICP_QAT_UOF_OBJS); 1631 if (!objhdl->obj_hdr) { 1632 pr_err("QAT: object file chunk is null\n"); 1633 goto out_objhdr_err; 1634 } 1635 handle->obj_handle = objhdl; 1636 if (qat_uclo_parse_uof_obj(handle)) 1637 goto out_overlay_obj_err; 1638 return 0; 1639 1640 out_overlay_obj_err: 1641 handle->obj_handle = NULL; 1642 kfree(objhdl->obj_hdr); 1643 out_objhdr_err: 1644 kfree(objhdl->obj_buf); 1645 out_objbuf_err: 1646 kfree(objhdl); 1647 return -ENOMEM; 1648 } 1649 1650 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1651 struct icp_qat_mof_file_hdr *mof_ptr, 1652 u32 mof_size) 1653 { 1654 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1655 unsigned int min_ver_offset; 1656 unsigned int checksum; 1657 1658 mobj_handle->file_id = ICP_QAT_MOF_FID; 1659 mobj_handle->mof_buf = (char *)mof_ptr; 1660 mobj_handle->mof_size = mof_size; 1661 1662 min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, 1663 min_ver); 1664 checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, 1665 min_ver_offset); 1666 if (checksum != mof_ptr->checksum) { 1667 pr_err("QAT: incorrect MOF checksum\n"); 1668 return -EINVAL; 1669 } 1670 1671 mobj_handle->checksum = mof_ptr->checksum; 1672 mobj_handle->min_ver = mof_ptr->min_ver; 1673 mobj_handle->maj_ver = mof_ptr->maj_ver; 1674 return 0; 1675 } 1676 1677 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) 1678 { 1679 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1680 1681 kfree(mobj_handle->obj_table.obj_hdr); 1682 mobj_handle->obj_table.obj_hdr = NULL; 1683 kfree(handle->mobj_handle); 1684 handle->mobj_handle = NULL; 1685 } 1686 1687 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, 1688 const char *obj_name, char **obj_ptr, 1689 unsigned int *obj_size) 1690 { 1691 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; 1692 unsigned int i; 1693 1694 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { 1695 if (!strncmp(obj_hdr[i].obj_name, obj_name, 1696 ICP_QAT_SUOF_OBJ_NAME_LEN)) { 1697 *obj_ptr = obj_hdr[i].obj_buf; 1698 *obj_size = obj_hdr[i].obj_size; 1699 return 0; 1700 } 1701 } 1702 1703 pr_err("QAT: object %s is not found inside MOF\n", obj_name); 1704 return -EINVAL; 1705 } 1706 1707 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, 1708 struct icp_qat_mof_objhdr *mobj_hdr, 1709 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) 1710 { 1711 u8 *obj; 1712 1713 if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, 1714 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1715 obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; 1716 } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, 1717 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1718 obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; 1719 } else { 1720 pr_err("QAT: unsupported chunk id\n"); 1721 return -EINVAL; 1722 } 1723 mobj_hdr->obj_buf = obj; 1724 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; 1725 mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; 1726 return 0; 1727 } 1728 1729 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) 1730 { 1731 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; 1732 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; 1733 struct icp_qat_mof_obj_hdr *uobj_hdr; 1734 struct icp_qat_mof_obj_hdr *sobj_hdr; 1735 struct icp_qat_mof_objhdr *mobj_hdr; 1736 unsigned int uobj_chunk_num = 0; 1737 unsigned int sobj_chunk_num = 0; 1738 unsigned int *valid_chunk; 1739 int ret, i; 1740 1741 uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; 1742 sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; 1743 if (uobj_hdr) 1744 uobj_chunk_num = uobj_hdr->num_chunks; 1745 if (sobj_hdr) 1746 sobj_chunk_num = sobj_hdr->num_chunks; 1747 1748 mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * 1749 sizeof(*mobj_hdr), GFP_KERNEL); 1750 if (!mobj_hdr) 1751 return -ENOMEM; 1752 1753 mobj_handle->obj_table.obj_hdr = mobj_hdr; 1754 valid_chunk = &mobj_handle->obj_table.num_objs; 1755 uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1756 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); 1757 sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1758 ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); 1759 1760 /* map uof objects */ 1761 for (i = 0; i < uobj_chunk_num; i++) { 1762 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1763 &mobj_hdr[*valid_chunk], 1764 &uobj_chunkhdr[i]); 1765 if (ret) 1766 return ret; 1767 (*valid_chunk)++; 1768 } 1769 1770 /* map suof objects */ 1771 for (i = 0; i < sobj_chunk_num; i++) { 1772 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1773 &mobj_hdr[*valid_chunk], 1774 &sobj_chunkhdr[i]); 1775 if (ret) 1776 return ret; 1777 (*valid_chunk)++; 1778 } 1779 1780 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { 1781 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); 1782 return -EINVAL; 1783 } 1784 return 0; 1785 } 1786 1787 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, 1788 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1789 { 1790 char **sym_str = (char **)&mobj_handle->sym_str; 1791 unsigned int *sym_size = &mobj_handle->sym_size; 1792 struct icp_qat_mof_str_table *str_table_obj; 1793 1794 *sym_size = *(unsigned int *)(uintptr_t) 1795 (mof_chunkhdr->offset + mobj_handle->mof_buf); 1796 *sym_str = (char *)(uintptr_t) 1797 (mobj_handle->mof_buf + mof_chunkhdr->offset + 1798 sizeof(str_table_obj->tab_len)); 1799 } 1800 1801 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, 1802 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1803 { 1804 char *chunk_id = mof_chunkhdr->chunk_id; 1805 1806 if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1807 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); 1808 else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1809 mobj_handle->uobjs_hdr = mobj_handle->mof_buf + 1810 mof_chunkhdr->offset; 1811 else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1812 mobj_handle->sobjs_hdr = mobj_handle->mof_buf + 1813 mof_chunkhdr->offset; 1814 } 1815 1816 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) 1817 { 1818 int maj = mof_hdr->maj_ver & 0xff; 1819 int min = mof_hdr->min_ver & 0xff; 1820 1821 if (mof_hdr->file_id != ICP_QAT_MOF_FID) { 1822 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); 1823 return -EINVAL; 1824 } 1825 1826 if (mof_hdr->num_chunks <= 0x1) { 1827 pr_err("QAT: MOF chunk amount is incorrect\n"); 1828 return -EINVAL; 1829 } 1830 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { 1831 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", 1832 maj, min); 1833 return -EINVAL; 1834 } 1835 return 0; 1836 } 1837 1838 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, 1839 struct icp_qat_mof_file_hdr *mof_ptr, 1840 u32 mof_size, const char *obj_name, 1841 char **obj_ptr, unsigned int *obj_size) 1842 { 1843 struct icp_qat_mof_chunkhdr *mof_chunkhdr; 1844 unsigned int file_id = mof_ptr->file_id; 1845 struct icp_qat_mof_handle *mobj_handle; 1846 unsigned short chunks_num; 1847 unsigned int i; 1848 int ret; 1849 1850 if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { 1851 if (obj_ptr) 1852 *obj_ptr = (char *)mof_ptr; 1853 if (obj_size) 1854 *obj_size = mof_size; 1855 return 0; 1856 } 1857 if (qat_uclo_check_mof_format(mof_ptr)) 1858 return -EINVAL; 1859 1860 mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); 1861 if (!mobj_handle) 1862 return -ENOMEM; 1863 1864 handle->mobj_handle = mobj_handle; 1865 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); 1866 if (ret) 1867 return ret; 1868 1869 mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); 1870 chunks_num = mof_ptr->num_chunks; 1871 1872 /* Parse MOF file chunks */ 1873 for (i = 0; i < chunks_num; i++) 1874 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); 1875 1876 /* All sym_objs uobjs and sobjs should be available */ 1877 if (!mobj_handle->sym_str || 1878 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) 1879 return -EINVAL; 1880 1881 ret = qat_uclo_map_objs_from_mof(mobj_handle); 1882 if (ret) 1883 return ret; 1884 1885 /* Seek specified uof object in MOF */ 1886 return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, 1887 obj_ptr, obj_size); 1888 } 1889 1890 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, 1891 void *addr_ptr, u32 mem_size, const char *obj_name) 1892 { 1893 char *obj_addr; 1894 u32 obj_size; 1895 int ret; 1896 1897 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= 1898 (sizeof(handle->hal_handle->ae_mask) * 8)); 1899 1900 if (!handle || !addr_ptr || mem_size < 24) 1901 return -EINVAL; 1902 1903 if (obj_name) { 1904 ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, 1905 &obj_addr, &obj_size); 1906 if (ret) 1907 return ret; 1908 } else { 1909 obj_addr = addr_ptr; 1910 obj_size = mem_size; 1911 } 1912 1913 return (handle->chip_info->fw_auth) ? 1914 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : 1915 qat_uclo_map_uof_obj(handle, obj_addr, obj_size); 1916 } 1917 1918 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) 1919 { 1920 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1921 unsigned int a; 1922 1923 if (handle->mobj_handle) 1924 qat_uclo_del_mof(handle); 1925 if (handle->sobj_handle) 1926 qat_uclo_del_suof(handle); 1927 if (!obj_handle) 1928 return; 1929 1930 kfree(obj_handle->uword_buf); 1931 for (a = 0; a < obj_handle->uimage_num; a++) 1932 kfree(obj_handle->ae_uimage[a].page); 1933 1934 for (a = 0; a < handle->hal_handle->ae_max_num; a++) 1935 qat_uclo_free_ae_data(&obj_handle->ae_data[a]); 1936 1937 kfree(obj_handle->obj_hdr); 1938 kfree(obj_handle->obj_buf); 1939 kfree(obj_handle); 1940 handle->obj_handle = NULL; 1941 } 1942 1943 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, 1944 struct icp_qat_uclo_encap_page *encap_page, 1945 u64 *uword, unsigned int addr_p, 1946 unsigned int raddr, u64 fill) 1947 { 1948 unsigned int i, addr; 1949 u64 uwrd = 0; 1950 1951 if (!encap_page) { 1952 *uword = fill; 1953 return; 1954 } 1955 addr = (encap_page->page_region) ? raddr : addr_p; 1956 for (i = 0; i < encap_page->uwblock_num; i++) { 1957 if (addr >= encap_page->uwblock[i].start_addr && 1958 addr <= encap_page->uwblock[i].start_addr + 1959 encap_page->uwblock[i].words_num - 1) { 1960 addr -= encap_page->uwblock[i].start_addr; 1961 addr *= obj_handle->uword_in_bytes; 1962 memcpy(&uwrd, (void *)(((uintptr_t) 1963 encap_page->uwblock[i].micro_words) + addr), 1964 obj_handle->uword_in_bytes); 1965 uwrd = uwrd & GENMASK_ULL(43, 0); 1966 } 1967 } 1968 *uword = uwrd; 1969 if (*uword == INVLD_UWORD) 1970 *uword = fill; 1971 } 1972 1973 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, 1974 struct icp_qat_uclo_encap_page 1975 *encap_page, unsigned int ae) 1976 { 1977 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; 1978 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1979 u64 fill_pat; 1980 1981 /* load the page starting at appropriate ustore address */ 1982 /* get fill-pattern from an image -- they are all the same */ 1983 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, 1984 sizeof(u64)); 1985 uw_physical_addr = encap_page->beg_addr_p; 1986 uw_relative_addr = 0; 1987 words_num = encap_page->micro_words_num; 1988 while (words_num) { 1989 if (words_num < UWORD_CPYBUF_SIZE) 1990 cpylen = words_num; 1991 else 1992 cpylen = UWORD_CPYBUF_SIZE; 1993 1994 /* load the buffer */ 1995 for (i = 0; i < cpylen; i++) 1996 qat_uclo_fill_uwords(obj_handle, encap_page, 1997 &obj_handle->uword_buf[i], 1998 uw_physical_addr + i, 1999 uw_relative_addr + i, fill_pat); 2000 2001 /* copy the buffer to ustore */ 2002 qat_hal_wr_uwords(handle, (unsigned char)ae, 2003 uw_physical_addr, cpylen, 2004 obj_handle->uword_buf); 2005 2006 uw_physical_addr += cpylen; 2007 uw_relative_addr += cpylen; 2008 words_num -= cpylen; 2009 } 2010 } 2011 2012 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, 2013 struct icp_qat_uof_image *image) 2014 { 2015 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2016 unsigned long ae_mask = handle->hal_handle->ae_mask; 2017 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 2018 unsigned long ae_assigned = image->ae_assigned; 2019 struct icp_qat_uclo_aedata *aed; 2020 unsigned int ctx_mask, s; 2021 struct icp_qat_uclo_page *page; 2022 unsigned char ae; 2023 int ctx; 2024 2025 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) 2026 ctx_mask = 0xff; 2027 else 2028 ctx_mask = 0x55; 2029 /* load the default page and set assigned CTX PC 2030 * to the entrypoint address */ 2031 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 2032 if (!test_bit(ae, &cfg_ae_mask)) 2033 continue; 2034 2035 if (!test_bit(ae, &ae_assigned)) 2036 continue; 2037 2038 aed = &obj_handle->ae_data[ae]; 2039 /* find the slice to which this image is assigned */ 2040 for (s = 0; s < aed->slice_num; s++) { 2041 if (image->ctx_assigned & 2042 aed->ae_slices[s].ctx_mask_assigned) 2043 break; 2044 } 2045 if (s >= aed->slice_num) 2046 continue; 2047 page = aed->ae_slices[s].page; 2048 if (!page->encap_page->def_page) 2049 continue; 2050 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); 2051 2052 page = aed->ae_slices[s].page; 2053 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) 2054 aed->ae_slices[s].cur_page[ctx] = 2055 (ctx_mask & (1 << ctx)) ? page : NULL; 2056 qat_hal_set_live_ctx(handle, (unsigned char)ae, 2057 image->ctx_assigned); 2058 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, 2059 image->entry_address); 2060 } 2061 } 2062 2063 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) 2064 { 2065 unsigned int i; 2066 struct icp_qat_fw_auth_desc *desc = NULL; 2067 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 2068 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; 2069 int ret; 2070 2071 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { 2072 ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, 2073 simg_hdr[i].simg_len, 2074 CSS_AE_FIRMWARE); 2075 if (ret) 2076 return ret; 2077 2078 if (qat_uclo_map_auth_fw(handle, 2079 (char *)simg_hdr[i].simg_buf, 2080 (unsigned int) 2081 simg_hdr[i].simg_len, 2082 &desc)) 2083 goto wr_err; 2084 if (qat_uclo_auth_fw(handle, desc)) 2085 goto wr_err; 2086 if (qat_uclo_is_broadcast(handle, i)) { 2087 if (qat_uclo_broadcast_load_fw(handle, desc)) 2088 goto wr_err; 2089 } else { 2090 if (qat_uclo_load_fw(handle, desc)) 2091 goto wr_err; 2092 } 2093 qat_uclo_ummap_auth_fw(handle, &desc); 2094 } 2095 return 0; 2096 wr_err: 2097 qat_uclo_ummap_auth_fw(handle, &desc); 2098 return -EINVAL; 2099 } 2100 2101 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) 2102 { 2103 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2104 unsigned int i; 2105 2106 if (qat_uclo_init_globals(handle)) 2107 return -EINVAL; 2108 for (i = 0; i < obj_handle->uimage_num; i++) { 2109 if (!obj_handle->ae_uimage[i].img_ptr) 2110 return -EINVAL; 2111 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) 2112 return -EINVAL; 2113 qat_uclo_wr_uimage_page(handle, 2114 obj_handle->ae_uimage[i].img_ptr); 2115 } 2116 return 0; 2117 } 2118 2119 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) 2120 { 2121 return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : 2122 qat_uclo_wr_uof_img(handle); 2123 } 2124 2125 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, 2126 unsigned int cfg_ae_mask) 2127 { 2128 if (!cfg_ae_mask) 2129 return -EINVAL; 2130 2131 handle->cfg_ae_mask = cfg_ae_mask; 2132 return 0; 2133 } 2134