1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/slab.h> 4 #include <linux/delay.h> 5 #include <linux/pci_ids.h> 6 7 #include "adf_accel_devices.h" 8 #include "adf_common_drv.h" 9 #include "icp_qat_hal.h" 10 #include "icp_qat_uclo.h" 11 12 #define BAD_REGADDR 0xffff 13 #define MAX_RETRY_TIMES 10000 14 #define INIT_CTX_ARB_VALUE 0x0 15 #define INIT_CTX_ENABLE_VALUE 0x0 16 #define INIT_PC_VALUE 0x0 17 #define INIT_WAKEUP_EVENTS_VALUE 0x1 18 #define INIT_SIG_EVENTS_VALUE 0x1 19 #define INIT_CCENABLE_VALUE 0x2000 20 #define RST_CSR_QAT_LSB 20 21 #define RST_CSR_AE_LSB 0 22 #define MC_TIMESTAMP_ENABLE (0x1 << 7) 23 24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ 25 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ 26 (~(1 << CE_REG_PAR_ERR_BITPOS))) 27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \ 28 (inst = ((inst & 0xFFFF00C03FFull) | \ 29 ((((const_val) << 12) & 0x0FF00000ull) | \ 30 (((const_val) << 10) & 0x0003FC00ull)))) 31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \ 32 (inst = ((inst & 0xFFFF00FFF00ull) | \ 33 ((((const_val) << 12) & 0x0FF00000ull) | \ 34 (((const_val) << 0) & 0x000000FFull)))) 35 36 #define AE(handle, ae) ((handle)->hal_handle->aes[ae]) 37 38 static const u64 inst_4b[] = { 39 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, 40 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 41 0x0A021000000ull 42 }; 43 44 static const u64 inst[] = { 45 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, 46 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 47 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, 48 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 49 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, 50 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 51 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 52 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 53 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 54 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 55 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 56 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 57 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 58 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 59 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 60 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 61 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 62 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 63 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 64 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 65 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 66 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, 67 }; 68 qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, 70 unsigned char ae, unsigned int ctx_mask) 71 { 72 AE(handle, ae).live_ctx_mask = ctx_mask; 73 } 74 75 #define CSR_RETRY_TIMES 500 qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr)76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, 77 unsigned char ae, unsigned int csr) 78 { 79 unsigned int iterations = CSR_RETRY_TIMES; 80 int value; 81 82 do { 83 value = GET_AE_CSR(handle, ae, csr); 84 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 85 return value; 86 } while (iterations--); 87 88 pr_err("QAT: Read CSR timeout\n"); 89 return 0; 90 } 91 qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr,unsigned int value)92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, 93 unsigned char ae, unsigned int csr, 94 unsigned int value) 95 { 96 unsigned int iterations = CSR_RETRY_TIMES; 97 98 do { 99 SET_AE_CSR(handle, ae, csr, value); 100 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 101 return 0; 102 } while (iterations--); 103 104 pr_err("QAT: Write CSR Timeout\n"); 105 return -EFAULT; 106 } 107 qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int * events)108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, 109 unsigned char ae, unsigned char ctx, 110 unsigned int *events) 111 { 112 unsigned int cur_ctx; 113 114 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); 115 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 116 *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT); 117 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 118 } 119 qat_hal_wait_cycles(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int cycles,int chk_inactive)120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, 121 unsigned char ae, unsigned int cycles, 122 int chk_inactive) 123 { 124 unsigned int base_cnt = 0, cur_cnt = 0; 125 unsigned int csr = (1 << ACS_ABO_BITPOS); 126 int times = MAX_RETRY_TIMES; 127 int elapsed_cycles = 0; 128 129 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); 130 base_cnt &= 0xffff; 131 while ((int)cycles > elapsed_cycles && times--) { 132 if (chk_inactive) 133 csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); 134 135 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); 136 cur_cnt &= 0xffff; 137 elapsed_cycles = cur_cnt - base_cnt; 138 139 if (elapsed_cycles < 0) 140 elapsed_cycles += 0x10000; 141 142 /* ensure at least 8 time cycles elapsed in wait_cycles */ 143 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) 144 return 0; 145 } 146 if (times < 0) { 147 pr_err("QAT: wait_num_cycles time out\n"); 148 return -EFAULT; 149 } 150 return 0; 151 } 152 153 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) 154 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) 155 qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, 157 unsigned char ae, unsigned char mode) 158 { 159 unsigned int csr, new_csr; 160 161 if (mode != 4 && mode != 8) { 162 pr_err("QAT: bad ctx mode=%d\n", mode); 163 return -EINVAL; 164 } 165 166 /* Sets the accelaration engine context mode to either four or eight */ 167 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 168 csr = IGNORE_W1C_MASK & csr; 169 new_csr = (mode == 4) ? 170 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : 171 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); 172 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 173 return 0; 174 } 175 qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, 177 unsigned char ae, unsigned char mode) 178 { 179 unsigned int csr, new_csr; 180 181 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 182 csr &= IGNORE_W1C_MASK; 183 184 new_csr = (mode) ? 185 SET_BIT(csr, CE_NN_MODE_BITPOS) : 186 CLR_BIT(csr, CE_NN_MODE_BITPOS); 187 188 if (new_csr != csr) 189 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 190 191 return 0; 192 } 193 qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,enum icp_qat_uof_regtype lm_type,unsigned char mode)194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, 195 unsigned char ae, enum icp_qat_uof_regtype lm_type, 196 unsigned char mode) 197 { 198 unsigned int csr, new_csr; 199 200 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 201 csr &= IGNORE_W1C_MASK; 202 switch (lm_type) { 203 case ICP_LMEM0: 204 new_csr = (mode) ? 205 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : 206 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); 207 break; 208 case ICP_LMEM1: 209 new_csr = (mode) ? 210 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : 211 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); 212 break; 213 case ICP_LMEM2: 214 new_csr = (mode) ? 215 SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : 216 CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); 217 break; 218 case ICP_LMEM3: 219 new_csr = (mode) ? 220 SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : 221 CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); 222 break; 223 default: 224 pr_err("QAT: lmType = 0x%x\n", lm_type); 225 return -EINVAL; 226 } 227 228 if (new_csr != csr) 229 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 230 return 0; 231 } 232 qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)233 void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, 234 unsigned char ae, unsigned char mode) 235 { 236 unsigned int csr, new_csr; 237 238 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 239 csr &= IGNORE_W1C_MASK; 240 new_csr = (mode) ? 241 SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : 242 CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); 243 if (new_csr != csr) 244 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 245 } 246 qat_hal_get_reg_addr(unsigned int type,unsigned short reg_num)247 static unsigned short qat_hal_get_reg_addr(unsigned int type, 248 unsigned short reg_num) 249 { 250 unsigned short reg_addr; 251 252 switch (type) { 253 case ICP_GPA_ABS: 254 case ICP_GPB_ABS: 255 reg_addr = 0x80 | (reg_num & 0x7f); 256 break; 257 case ICP_GPA_REL: 258 case ICP_GPB_REL: 259 reg_addr = reg_num & 0x1f; 260 break; 261 case ICP_SR_RD_REL: 262 case ICP_SR_WR_REL: 263 case ICP_SR_REL: 264 reg_addr = 0x180 | (reg_num & 0x1f); 265 break; 266 case ICP_SR_ABS: 267 reg_addr = 0x140 | ((reg_num & 0x3) << 1); 268 break; 269 case ICP_DR_RD_REL: 270 case ICP_DR_WR_REL: 271 case ICP_DR_REL: 272 reg_addr = 0x1c0 | (reg_num & 0x1f); 273 break; 274 case ICP_DR_ABS: 275 reg_addr = 0x100 | ((reg_num & 0x3) << 1); 276 break; 277 case ICP_NEIGH_REL: 278 reg_addr = 0x280 | (reg_num & 0x1f); 279 break; 280 case ICP_LMEM0: 281 reg_addr = 0x200; 282 break; 283 case ICP_LMEM1: 284 reg_addr = 0x220; 285 break; 286 case ICP_LMEM2: 287 reg_addr = 0x2c0; 288 break; 289 case ICP_LMEM3: 290 reg_addr = 0x2e0; 291 break; 292 case ICP_NO_DEST: 293 reg_addr = 0x300 | (reg_num & 0xff); 294 break; 295 default: 296 reg_addr = BAD_REGADDR; 297 break; 298 } 299 return reg_addr; 300 } 301 qat_hal_reset(struct icp_qat_fw_loader_handle * handle)302 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) 303 { 304 unsigned int reset_mask = handle->chip_info->icp_rst_mask; 305 unsigned int reset_csr = handle->chip_info->icp_rst_csr; 306 unsigned int csr_val; 307 308 csr_val = GET_CAP_CSR(handle, reset_csr); 309 csr_val |= reset_mask; 310 SET_CAP_CSR(handle, reset_csr, csr_val); 311 } 312 qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int ae_csr,unsigned int csr_val)313 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, 314 unsigned char ae, unsigned int ctx_mask, 315 unsigned int ae_csr, unsigned int csr_val) 316 { 317 unsigned int ctx, cur_ctx; 318 319 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); 320 321 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 322 if (!(ctx_mask & (1 << ctx))) 323 continue; 324 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 325 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); 326 } 327 328 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 329 } 330 qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int ae_csr)331 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, 332 unsigned char ae, unsigned char ctx, 333 unsigned int ae_csr) 334 { 335 unsigned int cur_ctx, csr_val; 336 337 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); 338 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 339 csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr); 340 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 341 342 return csr_val; 343 } 344 qat_hal_put_sig_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)345 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, 346 unsigned char ae, unsigned int ctx_mask, 347 unsigned int events) 348 { 349 unsigned int ctx, cur_ctx; 350 351 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); 352 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 353 if (!(ctx_mask & (1 << ctx))) 354 continue; 355 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 356 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); 357 } 358 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 359 } 360 qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)361 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, 362 unsigned char ae, unsigned int ctx_mask, 363 unsigned int events) 364 { 365 unsigned int ctx, cur_ctx; 366 367 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); 368 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 369 if (!(ctx_mask & (1 << ctx))) 370 continue; 371 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 372 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, 373 events); 374 } 375 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 376 } 377 qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle * handle)378 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) 379 { 380 unsigned long ae_mask = handle->hal_handle->ae_mask; 381 unsigned int base_cnt, cur_cnt; 382 unsigned char ae; 383 int times = MAX_RETRY_TIMES; 384 385 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 386 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); 387 base_cnt &= 0xffff; 388 389 do { 390 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); 391 cur_cnt &= 0xffff; 392 } while (times-- && (cur_cnt == base_cnt)); 393 394 if (times < 0) { 395 pr_err("QAT: AE%d is inactive!!\n", ae); 396 return -EFAULT; 397 } 398 } 399 400 return 0; 401 } 402 qat_hal_check_ae_active(struct icp_qat_fw_loader_handle * handle,unsigned int ae)403 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, 404 unsigned int ae) 405 { 406 unsigned int enable = 0, active = 0; 407 408 enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 409 active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); 410 if ((enable & (0xff << CE_ENABLE_BITPOS)) || 411 (active & (1 << ACS_ABO_BITPOS))) 412 return 1; 413 else 414 return 0; 415 } 416 qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle * handle)417 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) 418 { 419 unsigned long ae_mask = handle->hal_handle->ae_mask; 420 unsigned int misc_ctl_csr, misc_ctl; 421 unsigned char ae; 422 423 misc_ctl_csr = handle->chip_info->misc_ctl_csr; 424 /* stop the timestamp timers */ 425 misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); 426 if (misc_ctl & MC_TIMESTAMP_ENABLE) 427 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & 428 (~MC_TIMESTAMP_ENABLE)); 429 430 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 431 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); 432 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); 433 } 434 /* start timestamp timers */ 435 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); 436 } 437 438 #define ESRAM_AUTO_TINIT BIT(2) 439 #define ESRAM_AUTO_TINIT_DONE BIT(3) 440 #define ESRAM_AUTO_INIT_USED_CYCLES (1640) 441 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C qat_hal_init_esram(struct icp_qat_fw_loader_handle * handle)442 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) 443 { 444 void __iomem *csr_addr = 445 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v + 446 ESRAM_AUTO_INIT_CSR_OFFSET); 447 unsigned int csr_val; 448 int times = 30; 449 450 if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC) 451 return 0; 452 453 csr_val = ADF_CSR_RD(csr_addr, 0); 454 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) 455 return 0; 456 457 csr_val = ADF_CSR_RD(csr_addr, 0); 458 csr_val |= ESRAM_AUTO_TINIT; 459 ADF_CSR_WR(csr_addr, 0, csr_val); 460 461 do { 462 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); 463 csr_val = ADF_CSR_RD(csr_addr, 0); 464 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); 465 if (times < 0) { 466 pr_err("QAT: Fail to init eSram!\n"); 467 return -EFAULT; 468 } 469 return 0; 470 } 471 472 #define SHRAM_INIT_CYCLES 2060 qat_hal_clr_reset(struct icp_qat_fw_loader_handle * handle)473 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) 474 { 475 unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; 476 unsigned int reset_mask = handle->chip_info->icp_rst_mask; 477 unsigned int reset_csr = handle->chip_info->icp_rst_csr; 478 unsigned long ae_mask = handle->hal_handle->ae_mask; 479 unsigned char ae = 0; 480 unsigned int times = 100; 481 unsigned int csr_val; 482 483 /* write to the reset csr */ 484 csr_val = GET_CAP_CSR(handle, reset_csr); 485 csr_val &= ~reset_mask; 486 do { 487 SET_CAP_CSR(handle, reset_csr, csr_val); 488 if (!(times--)) 489 goto out_err; 490 csr_val = GET_CAP_CSR(handle, reset_csr); 491 csr_val &= reset_mask; 492 } while (csr_val); 493 /* enable clock */ 494 csr_val = GET_CAP_CSR(handle, clk_csr); 495 csr_val |= reset_mask; 496 SET_CAP_CSR(handle, clk_csr, csr_val); 497 if (qat_hal_check_ae_alive(handle)) 498 goto out_err; 499 500 /* Set undefined power-up/reset states to reasonable default values */ 501 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 502 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, 503 INIT_CTX_ENABLE_VALUE); 504 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, 505 CTX_STS_INDIRECT, 506 handle->hal_handle->upc_mask & 507 INIT_PC_VALUE); 508 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 509 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 510 qat_hal_put_wakeup_event(handle, ae, 511 ICP_QAT_UCLO_AE_ALL_CTX, 512 INIT_WAKEUP_EVENTS_VALUE); 513 qat_hal_put_sig_event(handle, ae, 514 ICP_QAT_UCLO_AE_ALL_CTX, 515 INIT_SIG_EVENTS_VALUE); 516 } 517 if (qat_hal_init_esram(handle)) 518 goto out_err; 519 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) 520 goto out_err; 521 qat_hal_reset_timestamp(handle); 522 523 return 0; 524 out_err: 525 pr_err("QAT: failed to get device out of reset\n"); 526 return -EFAULT; 527 } 528 qat_hal_disable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)529 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, 530 unsigned char ae, unsigned int ctx_mask) 531 { 532 unsigned int ctx; 533 534 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 535 ctx &= IGNORE_W1C_MASK & 536 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); 537 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 538 } 539 qat_hal_parity_64bit(u64 word)540 static u64 qat_hal_parity_64bit(u64 word) 541 { 542 word ^= word >> 1; 543 word ^= word >> 2; 544 word ^= word >> 4; 545 word ^= word >> 8; 546 word ^= word >> 16; 547 word ^= word >> 32; 548 return word & 1; 549 } 550 qat_hal_set_uword_ecc(u64 uword)551 static u64 qat_hal_set_uword_ecc(u64 uword) 552 { 553 u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, 554 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, 555 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, 556 bit6_mask = 0xdaf69a46910ULL; 557 558 /* clear the ecc bits */ 559 uword &= ~(0x7fULL << 0x2C); 560 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; 561 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; 562 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; 563 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; 564 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; 565 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; 566 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; 567 return uword; 568 } 569 qat_hal_wr_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)570 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, 571 unsigned char ae, unsigned int uaddr, 572 unsigned int words_num, u64 *uword) 573 { 574 unsigned int ustore_addr; 575 unsigned int i; 576 577 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); 578 uaddr |= UA_ECS; 579 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 580 for (i = 0; i < words_num; i++) { 581 unsigned int uwrd_lo, uwrd_hi; 582 u64 tmp; 583 584 tmp = qat_hal_set_uword_ecc(uword[i]); 585 uwrd_lo = (unsigned int)(tmp & 0xffffffff); 586 uwrd_hi = (unsigned int)(tmp >> 0x20); 587 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 588 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 589 } 590 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 591 } 592 qat_hal_enable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)593 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, 594 unsigned char ae, unsigned int ctx_mask) 595 { 596 unsigned int ctx; 597 598 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 599 ctx &= IGNORE_W1C_MASK; 600 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; 601 ctx |= (ctx_mask << CE_ENABLE_BITPOS); 602 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 603 } 604 qat_hal_clear_xfer(struct icp_qat_fw_loader_handle * handle)605 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) 606 { 607 unsigned long ae_mask = handle->hal_handle->ae_mask; 608 unsigned char ae; 609 unsigned short reg; 610 611 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 612 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { 613 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, 614 reg, 0); 615 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, 616 reg, 0); 617 } 618 } 619 } 620 qat_hal_clear_gpr(struct icp_qat_fw_loader_handle * handle)621 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) 622 { 623 unsigned long ae_mask = handle->hal_handle->ae_mask; 624 unsigned char ae; 625 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; 626 int times = MAX_RETRY_TIMES; 627 unsigned int csr_val = 0; 628 unsigned int savctx = 0; 629 int ret = 0; 630 631 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 632 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); 633 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); 634 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); 635 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 636 csr_val &= IGNORE_W1C_MASK; 637 if (handle->chip_info->nn) 638 csr_val |= CE_NN_MODE; 639 640 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); 641 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), 642 (u64 *)inst); 643 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, 644 handle->hal_handle->upc_mask & 645 INIT_PC_VALUE); 646 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); 647 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); 648 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); 649 qat_hal_wr_indr_csr(handle, ae, ctx_mask, 650 CTX_SIG_EVENTS_INDIRECT, 0); 651 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 652 qat_hal_enable_ctx(handle, ae, ctx_mask); 653 } 654 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 655 /* wait for AE to finish */ 656 do { 657 ret = qat_hal_wait_cycles(handle, ae, 20, 1); 658 } while (ret && times--); 659 660 if (times < 0) { 661 pr_err("QAT: clear GPR of AE %d failed", ae); 662 return -EINVAL; 663 } 664 qat_hal_disable_ctx(handle, ae, ctx_mask); 665 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 666 savctx & ACS_ACNO); 667 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, 668 INIT_CTX_ENABLE_VALUE); 669 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, 670 handle->hal_handle->upc_mask & 671 INIT_PC_VALUE); 672 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 673 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 674 qat_hal_put_wakeup_event(handle, ae, ctx_mask, 675 INIT_WAKEUP_EVENTS_VALUE); 676 qat_hal_put_sig_event(handle, ae, ctx_mask, 677 INIT_SIG_EVENTS_VALUE); 678 } 679 return 0; 680 } 681 qat_hal_chip_init(struct icp_qat_fw_loader_handle * handle,struct adf_accel_dev * accel_dev)682 static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, 683 struct adf_accel_dev *accel_dev) 684 { 685 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; 686 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 687 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 688 unsigned int max_en_ae_id = 0; 689 struct adf_bar *sram_bar; 690 unsigned int csr_val = 0; 691 unsigned long ae_mask; 692 unsigned char ae = 0; 693 int ret = 0; 694 695 handle->pci_dev = pci_info->pci_dev; 696 switch (handle->pci_dev->device) { 697 case ADF_4XXX_PCI_DEVICE_ID: 698 case ADF_401XX_PCI_DEVICE_ID: 699 case ADF_402XX_PCI_DEVICE_ID: 700 handle->chip_info->mmp_sram_size = 0; 701 handle->chip_info->nn = false; 702 handle->chip_info->lm2lm3 = true; 703 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; 704 handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; 705 handle->chip_info->icp_rst_mask = 0x100015; 706 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; 707 handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; 708 handle->chip_info->wakeup_event_val = 0x80000000; 709 handle->chip_info->fw_auth = true; 710 handle->chip_info->css_3k = true; 711 handle->chip_info->tgroup_share_ustore = true; 712 handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; 713 handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; 714 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; 715 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; 716 handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; 717 handle->chip_info->fcu_loaded_ae_pos = 0; 718 719 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX; 720 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX; 721 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX; 722 handle->hal_cap_ae_local_csr_addr_v = 723 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v 724 + LOCAL_TO_XFER_REG_OFFSET); 725 break; 726 case PCI_DEVICE_ID_INTEL_QAT_C62X: 727 case PCI_DEVICE_ID_INTEL_QAT_C3XXX: 728 handle->chip_info->mmp_sram_size = 0; 729 handle->chip_info->nn = true; 730 handle->chip_info->lm2lm3 = false; 731 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; 732 handle->chip_info->icp_rst_csr = ICP_RESET; 733 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | 734 (hw_data->accel_mask << RST_CSR_QAT_LSB); 735 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; 736 handle->chip_info->misc_ctl_csr = MISC_CONTROL; 737 handle->chip_info->wakeup_event_val = WAKEUP_EVENT; 738 handle->chip_info->fw_auth = true; 739 handle->chip_info->css_3k = false; 740 handle->chip_info->tgroup_share_ustore = false; 741 handle->chip_info->fcu_ctl_csr = FCU_CONTROL; 742 handle->chip_info->fcu_sts_csr = FCU_STATUS; 743 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; 744 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; 745 handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; 746 handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; 747 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; 748 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; 749 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; 750 handle->hal_cap_ae_local_csr_addr_v = 751 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v 752 + LOCAL_TO_XFER_REG_OFFSET); 753 break; 754 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: 755 handle->chip_info->mmp_sram_size = 0x40000; 756 handle->chip_info->nn = true; 757 handle->chip_info->lm2lm3 = false; 758 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; 759 handle->chip_info->icp_rst_csr = ICP_RESET; 760 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | 761 (hw_data->accel_mask << RST_CSR_QAT_LSB); 762 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; 763 handle->chip_info->misc_ctl_csr = MISC_CONTROL; 764 handle->chip_info->wakeup_event_val = WAKEUP_EVENT; 765 handle->chip_info->fw_auth = false; 766 handle->chip_info->css_3k = false; 767 handle->chip_info->tgroup_share_ustore = false; 768 handle->chip_info->fcu_ctl_csr = 0; 769 handle->chip_info->fcu_sts_csr = 0; 770 handle->chip_info->fcu_dram_addr_hi = 0; 771 handle->chip_info->fcu_dram_addr_lo = 0; 772 handle->chip_info->fcu_loaded_ae_csr = 0; 773 handle->chip_info->fcu_loaded_ae_pos = 0; 774 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; 775 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; 776 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; 777 handle->hal_cap_ae_local_csr_addr_v = 778 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v 779 + LOCAL_TO_XFER_REG_OFFSET); 780 break; 781 default: 782 ret = -EINVAL; 783 goto out_err; 784 } 785 786 if (handle->chip_info->mmp_sram_size > 0) { 787 sram_bar = 788 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; 789 handle->hal_sram_addr_v = sram_bar->virt_addr; 790 } 791 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; 792 handle->hal_handle->ae_mask = hw_data->ae_mask; 793 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; 794 handle->hal_handle->slice_mask = hw_data->accel_mask; 795 handle->cfg_ae_mask = ALL_AE_MASK; 796 /* create AE objects */ 797 handle->hal_handle->upc_mask = 0x1ffff; 798 handle->hal_handle->max_ustore = 0x4000; 799 800 ae_mask = handle->hal_handle->ae_mask; 801 for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { 802 handle->hal_handle->aes[ae].free_addr = 0; 803 handle->hal_handle->aes[ae].free_size = 804 handle->hal_handle->max_ustore; 805 handle->hal_handle->aes[ae].ustore_size = 806 handle->hal_handle->max_ustore; 807 handle->hal_handle->aes[ae].live_ctx_mask = 808 ICP_QAT_UCLO_AE_ALL_CTX; 809 max_en_ae_id = ae; 810 } 811 handle->hal_handle->ae_max_num = max_en_ae_id + 1; 812 813 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 814 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 815 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); 816 csr_val |= 0x1; 817 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); 818 } 819 out_err: 820 return ret; 821 } 822 qat_hal_init(struct adf_accel_dev * accel_dev)823 int qat_hal_init(struct adf_accel_dev *accel_dev) 824 { 825 struct icp_qat_fw_loader_handle *handle; 826 int ret = 0; 827 828 handle = kzalloc(sizeof(*handle), GFP_KERNEL); 829 if (!handle) 830 return -ENOMEM; 831 832 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); 833 if (!handle->hal_handle) { 834 ret = -ENOMEM; 835 goto out_hal_handle; 836 } 837 838 handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL); 839 if (!handle->chip_info) { 840 ret = -ENOMEM; 841 goto out_chip_info; 842 } 843 844 ret = qat_hal_chip_init(handle, accel_dev); 845 if (ret) { 846 dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); 847 goto out_err; 848 } 849 850 /* take all AEs out of reset */ 851 ret = qat_hal_clr_reset(handle); 852 if (ret) { 853 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); 854 goto out_err; 855 } 856 857 qat_hal_clear_xfer(handle); 858 if (!handle->chip_info->fw_auth) { 859 ret = qat_hal_clear_gpr(handle); 860 if (ret) 861 goto out_err; 862 } 863 864 accel_dev->fw_loader->fw_loader = handle; 865 return 0; 866 867 out_err: 868 kfree(handle->chip_info); 869 out_chip_info: 870 kfree(handle->hal_handle); 871 out_hal_handle: 872 kfree(handle); 873 return ret; 874 } 875 qat_hal_deinit(struct icp_qat_fw_loader_handle * handle)876 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) 877 { 878 if (!handle) 879 return; 880 kfree(handle->chip_info); 881 kfree(handle->hal_handle); 882 kfree(handle); 883 } 884 qat_hal_start(struct icp_qat_fw_loader_handle * handle)885 int qat_hal_start(struct icp_qat_fw_loader_handle *handle) 886 { 887 unsigned long ae_mask = handle->hal_handle->ae_mask; 888 u32 wakeup_val = handle->chip_info->wakeup_event_val; 889 u32 fcu_ctl_csr, fcu_sts_csr; 890 unsigned int fcu_sts; 891 unsigned char ae; 892 u32 ae_ctr = 0; 893 int retry = 0; 894 895 if (handle->chip_info->fw_auth) { 896 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 897 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 898 ae_ctr = hweight32(ae_mask); 899 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); 900 do { 901 msleep(FW_AUTH_WAIT_PERIOD); 902 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 903 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) 904 return ae_ctr; 905 } while (retry++ < FW_AUTH_MAX_RETRY); 906 pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); 907 return 0; 908 } else { 909 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 910 qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); 911 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); 912 ae_ctr++; 913 } 914 return ae_ctr; 915 } 916 } 917 qat_hal_stop(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)918 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, 919 unsigned int ctx_mask) 920 { 921 if (!handle->chip_info->fw_auth) 922 qat_hal_disable_ctx(handle, ae, ctx_mask); 923 } 924 qat_hal_set_pc(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int upc)925 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, 926 unsigned char ae, unsigned int ctx_mask, unsigned int upc) 927 { 928 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, 929 handle->hal_handle->upc_mask & upc); 930 } 931 qat_hal_get_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)932 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, 933 unsigned char ae, unsigned int uaddr, 934 unsigned int words_num, u64 *uword) 935 { 936 unsigned int i, uwrd_lo, uwrd_hi; 937 unsigned int ustore_addr, misc_control; 938 939 misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); 940 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, 941 misc_control & 0xfffffffb); 942 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); 943 uaddr |= UA_ECS; 944 for (i = 0; i < words_num; i++) { 945 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 946 uaddr++; 947 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER); 948 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER); 949 uword[i] = uwrd_hi; 950 uword[i] = (uword[i] << 0x20) | uwrd_lo; 951 } 952 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); 953 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 954 } 955 qat_hal_wr_umem(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,unsigned int * data)956 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, 957 unsigned char ae, unsigned int uaddr, 958 unsigned int words_num, unsigned int *data) 959 { 960 unsigned int i, ustore_addr; 961 962 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); 963 uaddr |= UA_ECS; 964 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 965 for (i = 0; i < words_num; i++) { 966 unsigned int uwrd_lo, uwrd_hi, tmp; 967 968 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | 969 ((data[i] & 0xff00) << 2) | 970 (0x3 << 8) | (data[i] & 0xff); 971 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); 972 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8; 973 tmp = ((data[i] >> 0x10) & 0xffff); 974 uwrd_hi |= (hweight32(tmp) & 0x1) << 9; 975 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 976 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 977 } 978 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 979 } 980 981 #define MAX_EXEC_INST 100 qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,u64 * micro_inst,unsigned int inst_num,int code_off,unsigned int max_cycle,unsigned int * endpc)982 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, 983 unsigned char ae, unsigned char ctx, 984 u64 *micro_inst, unsigned int inst_num, 985 int code_off, unsigned int max_cycle, 986 unsigned int *endpc) 987 { 988 unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; 989 unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; 990 unsigned int ind_t_index = 0, ind_t_index_byte = 0; 991 unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; 992 unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; 993 u64 savuwords[MAX_EXEC_INST]; 994 unsigned int ind_cnt_sig; 995 unsigned int ind_sig, act_sig; 996 unsigned int csr_val = 0, newcsr_val; 997 unsigned int savctx; 998 unsigned int savcc, wakeup_events, savpc; 999 unsigned int ctxarb_ctl, ctx_enables; 1000 1001 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) { 1002 pr_err("QAT: invalid instruction num %d\n", inst_num); 1003 return -EINVAL; 1004 } 1005 /* save current context */ 1006 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT); 1007 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT); 1008 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx, 1009 INDIRECT_LM_ADDR_0_BYTE_INDEX); 1010 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, 1011 INDIRECT_LM_ADDR_1_BYTE_INDEX); 1012 if (handle->chip_info->lm2lm3) { 1013 ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, 1014 LM_ADDR_2_INDIRECT); 1015 ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, 1016 LM_ADDR_3_INDIRECT); 1017 ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, 1018 INDIRECT_LM_ADDR_2_BYTE_INDEX); 1019 ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, 1020 INDIRECT_LM_ADDR_3_BYTE_INDEX); 1021 ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, 1022 INDIRECT_T_INDEX); 1023 ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, 1024 INDIRECT_T_INDEX_BYTE_INDEX); 1025 } 1026 if (inst_num <= MAX_EXEC_INST) 1027 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); 1028 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); 1029 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT); 1030 savpc = (savpc & handle->hal_handle->upc_mask) >> 0; 1031 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1032 ctx_enables &= IGNORE_W1C_MASK; 1033 savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE); 1034 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); 1035 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); 1036 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx, 1037 FUTURE_COUNT_SIGNAL_INDIRECT); 1038 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx, 1039 CTX_SIG_EVENTS_INDIRECT); 1040 act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE); 1041 /* execute micro codes */ 1042 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1043 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); 1044 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); 1045 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); 1046 if (code_off) 1047 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); 1048 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); 1049 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); 1050 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1051 qat_hal_enable_ctx(handle, ae, (1 << ctx)); 1052 /* wait for micro codes to finish */ 1053 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) 1054 return -EFAULT; 1055 if (endpc) { 1056 unsigned int ctx_status; 1057 1058 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx, 1059 CTX_STS_INDIRECT); 1060 *endpc = ctx_status & handle->hal_handle->upc_mask; 1061 } 1062 /* retore to saved context */ 1063 qat_hal_disable_ctx(handle, ae, (1 << ctx)); 1064 if (inst_num <= MAX_EXEC_INST) 1065 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); 1066 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); 1067 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 1068 handle->hal_handle->upc_mask & savpc); 1069 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); 1070 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 1071 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 1072 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); 1073 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); 1074 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); 1075 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1076 LM_ADDR_0_INDIRECT, ind_lm_addr0); 1077 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1078 LM_ADDR_1_INDIRECT, ind_lm_addr1); 1079 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1080 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); 1081 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1082 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); 1083 if (handle->chip_info->lm2lm3) { 1084 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, 1085 ind_lm_addr2); 1086 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, 1087 ind_lm_addr3); 1088 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), 1089 INDIRECT_LM_ADDR_2_BYTE_INDEX, 1090 ind_lm_addr_byte2); 1091 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), 1092 INDIRECT_LM_ADDR_3_BYTE_INDEX, 1093 ind_lm_addr_byte3); 1094 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), 1095 INDIRECT_T_INDEX, ind_t_index); 1096 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), 1097 INDIRECT_T_INDEX_BYTE_INDEX, 1098 ind_t_index_byte); 1099 } 1100 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1101 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); 1102 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), 1103 CTX_SIG_EVENTS_INDIRECT, ind_sig); 1104 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); 1105 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1106 1107 return 0; 1108 } 1109 qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int * data)1110 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, 1111 unsigned char ae, unsigned char ctx, 1112 enum icp_qat_uof_regtype reg_type, 1113 unsigned short reg_num, unsigned int *data) 1114 { 1115 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; 1116 unsigned int ctxarb_cntl, ustore_addr, ctx_enables; 1117 unsigned short reg_addr; 1118 int status = 0; 1119 u64 insts, savuword; 1120 1121 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1122 if (reg_addr == BAD_REGADDR) { 1123 pr_err("QAT: bad regaddr=0x%x\n", reg_addr); 1124 return -EINVAL; 1125 } 1126 switch (reg_type) { 1127 case ICP_GPA_REL: 1128 insts = 0xA070000000ull | (reg_addr & 0x3ff); 1129 break; 1130 default: 1131 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10); 1132 break; 1133 } 1134 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); 1135 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); 1136 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1137 ctx_enables &= IGNORE_W1C_MASK; 1138 if (ctx != (savctx & ACS_ACNO)) 1139 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 1140 ctx & ACS_ACNO); 1141 qat_hal_get_uwords(handle, ae, 0, 1, &savuword); 1142 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1143 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); 1144 uaddr = UA_ECS; 1145 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1146 insts = qat_hal_set_uword_ecc(insts); 1147 uwrd_lo = (unsigned int)(insts & 0xffffffff); 1148 uwrd_hi = (unsigned int)(insts >> 0x20); 1149 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 1150 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 1151 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1152 /* delay for at least 8 cycles */ 1153 qat_hal_wait_cycles(handle, ae, 0x8, 0); 1154 /* 1155 * read ALU output 1156 * the instruction should have been executed 1157 * prior to clearing the ECS in putUwords 1158 */ 1159 *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT); 1160 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 1161 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); 1162 if (ctx != (savctx & ACS_ACNO)) 1163 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 1164 savctx & ACS_ACNO); 1165 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); 1166 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1167 1168 return status; 1169 } 1170 qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1171 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, 1172 unsigned char ae, unsigned char ctx, 1173 enum icp_qat_uof_regtype reg_type, 1174 unsigned short reg_num, unsigned int data) 1175 { 1176 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; 1177 u64 insts[] = { 1178 0x0F440000000ull, 1179 0x0F040000000ull, 1180 0x0F0000C0300ull, 1181 0x0E000010000ull 1182 }; 1183 const int num_inst = ARRAY_SIZE(insts), code_off = 1; 1184 const int imm_w1 = 0, imm_w0 = 1; 1185 1186 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1187 if (dest_addr == BAD_REGADDR) { 1188 pr_err("QAT: bad destAddr=0x%x\n", dest_addr); 1189 return -EINVAL; 1190 } 1191 1192 data16lo = 0xffff & data; 1193 data16hi = 0xffff & (data >> 0x10); 1194 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) 1195 (0xff & data16hi)); 1196 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) 1197 (0xff & data16lo)); 1198 switch (reg_type) { 1199 case ICP_GPA_REL: 1200 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 1201 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 1202 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 1203 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 1204 break; 1205 default: 1206 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 1207 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 1208 1209 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 1210 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 1211 break; 1212 } 1213 1214 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst, 1215 code_off, num_inst * 0x5, NULL); 1216 } 1217 qat_hal_get_ins_num(void)1218 int qat_hal_get_ins_num(void) 1219 { 1220 return ARRAY_SIZE(inst_4b); 1221 } 1222 qat_hal_concat_micro_code(u64 * micro_inst,unsigned int inst_num,unsigned int size,unsigned int addr,unsigned int * value)1223 static int qat_hal_concat_micro_code(u64 *micro_inst, 1224 unsigned int inst_num, unsigned int size, 1225 unsigned int addr, unsigned int *value) 1226 { 1227 int i; 1228 unsigned int cur_value; 1229 const u64 *inst_arr; 1230 int fixup_offset; 1231 int usize = 0; 1232 int orig_num; 1233 1234 orig_num = inst_num; 1235 cur_value = value[0]; 1236 inst_arr = inst_4b; 1237 usize = ARRAY_SIZE(inst_4b); 1238 fixup_offset = inst_num; 1239 for (i = 0; i < usize; i++) 1240 micro_inst[inst_num++] = inst_arr[i]; 1241 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); 1242 fixup_offset++; 1243 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); 1244 fixup_offset++; 1245 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); 1246 fixup_offset++; 1247 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); 1248 1249 return inst_num - orig_num; 1250 } 1251 qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,int * pfirst_exec,u64 * micro_inst,unsigned int inst_num)1252 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, 1253 unsigned char ae, unsigned char ctx, 1254 int *pfirst_exec, u64 *micro_inst, 1255 unsigned int inst_num) 1256 { 1257 int stat = 0; 1258 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; 1259 unsigned int gprb0 = 0, gprb1 = 0; 1260 1261 if (*pfirst_exec) { 1262 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); 1263 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); 1264 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); 1265 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); 1266 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); 1267 *pfirst_exec = 0; 1268 } 1269 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1, 1270 inst_num * 0x5, NULL); 1271 if (stat != 0) 1272 return -EFAULT; 1273 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); 1274 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); 1275 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); 1276 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); 1277 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); 1278 1279 return 0; 1280 } 1281 qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,struct icp_qat_uof_batch_init * lm_init_header)1282 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, 1283 unsigned char ae, 1284 struct icp_qat_uof_batch_init *lm_init_header) 1285 { 1286 struct icp_qat_uof_batch_init *plm_init; 1287 u64 *micro_inst_arry; 1288 int micro_inst_num; 1289 int alloc_inst_size; 1290 int first_exec = 1; 1291 int stat = 0; 1292 1293 plm_init = lm_init_header->next; 1294 alloc_inst_size = lm_init_header->size; 1295 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) 1296 alloc_inst_size = handle->hal_handle->max_ustore; 1297 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64), 1298 GFP_KERNEL); 1299 if (!micro_inst_arry) 1300 return -ENOMEM; 1301 micro_inst_num = 0; 1302 while (plm_init) { 1303 unsigned int addr, *value, size; 1304 1305 ae = plm_init->ae; 1306 addr = plm_init->addr; 1307 value = plm_init->value; 1308 size = plm_init->size; 1309 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry, 1310 micro_inst_num, 1311 size, addr, value); 1312 plm_init = plm_init->next; 1313 } 1314 /* exec micro codes */ 1315 if (micro_inst_arry && micro_inst_num > 0) { 1316 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; 1317 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, 1318 micro_inst_arry, 1319 micro_inst_num); 1320 } 1321 kfree(micro_inst_arry); 1322 return stat; 1323 } 1324 qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int val)1325 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, 1326 unsigned char ae, unsigned char ctx, 1327 enum icp_qat_uof_regtype reg_type, 1328 unsigned short reg_num, unsigned int val) 1329 { 1330 int status = 0; 1331 unsigned int reg_addr; 1332 unsigned int ctx_enables; 1333 unsigned short mask; 1334 unsigned short dr_offset = 0x10; 1335 1336 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1337 if (CE_INUSE_CONTEXTS & ctx_enables) { 1338 if (ctx & 0x1) { 1339 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); 1340 return -EINVAL; 1341 } 1342 mask = 0x1f; 1343 dr_offset = 0x20; 1344 } else { 1345 mask = 0x0f; 1346 } 1347 if (reg_num & ~mask) 1348 return -EINVAL; 1349 reg_addr = reg_num + (ctx << 0x5); 1350 switch (reg_type) { 1351 case ICP_SR_RD_REL: 1352 case ICP_SR_REL: 1353 SET_AE_XFER(handle, ae, reg_addr, val); 1354 break; 1355 case ICP_DR_RD_REL: 1356 case ICP_DR_REL: 1357 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); 1358 break; 1359 default: 1360 status = -EINVAL; 1361 break; 1362 } 1363 return status; 1364 } 1365 qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1366 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, 1367 unsigned char ae, unsigned char ctx, 1368 enum icp_qat_uof_regtype reg_type, 1369 unsigned short reg_num, unsigned int data) 1370 { 1371 unsigned int gprval, ctx_enables; 1372 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, 1373 data16low; 1374 unsigned short reg_mask; 1375 int status = 0; 1376 u64 micro_inst[] = { 1377 0x0F440000000ull, 1378 0x0F040000000ull, 1379 0x0A000000000ull, 1380 0x0F0000C0300ull, 1381 0x0E000010000ull 1382 }; 1383 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; 1384 const unsigned short gprnum = 0, dly = num_inst * 0x5; 1385 1386 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1387 if (CE_INUSE_CONTEXTS & ctx_enables) { 1388 if (ctx & 0x1) { 1389 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); 1390 return -EINVAL; 1391 } 1392 reg_mask = (unsigned short)~0x1f; 1393 } else { 1394 reg_mask = (unsigned short)~0xf; 1395 } 1396 if (reg_num & reg_mask) 1397 return -EINVAL; 1398 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1399 if (xfr_addr == BAD_REGADDR) { 1400 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); 1401 return -EINVAL; 1402 } 1403 status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); 1404 if (status) { 1405 pr_err("QAT: failed to read register"); 1406 return status; 1407 } 1408 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); 1409 data16low = 0xffff & data; 1410 data16hi = 0xffff & (data >> 0x10); 1411 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1412 (unsigned short)(0xff & data16hi)); 1413 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1414 (unsigned short)(0xff & data16low)); 1415 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | 1416 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 1417 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | 1418 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 1419 micro_inst[0x2] = micro_inst[0x2] | 1420 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10); 1421 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst, 1422 code_off, dly, NULL); 1423 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); 1424 return status; 1425 } 1426 qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned short nn,unsigned int val)1427 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, 1428 unsigned char ae, unsigned char ctx, 1429 unsigned short nn, unsigned int val) 1430 { 1431 unsigned int ctx_enables; 1432 int stat = 0; 1433 1434 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1435 ctx_enables &= IGNORE_W1C_MASK; 1436 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); 1437 1438 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); 1439 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1440 return stat; 1441 } 1442 qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned short absreg_num,unsigned short * relreg,unsigned char * ctx)1443 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle 1444 *handle, unsigned char ae, 1445 unsigned short absreg_num, 1446 unsigned short *relreg, 1447 unsigned char *ctx) 1448 { 1449 unsigned int ctx_enables; 1450 1451 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); 1452 if (ctx_enables & CE_INUSE_CONTEXTS) { 1453 /* 4-ctx mode */ 1454 *relreg = absreg_num & 0x1F; 1455 *ctx = (absreg_num >> 0x4) & 0x6; 1456 } else { 1457 /* 8-ctx mode */ 1458 *relreg = absreg_num & 0x0F; 1459 *ctx = (absreg_num >> 0x4) & 0x7; 1460 } 1461 return 0; 1462 } 1463 qat_hal_init_gpr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1464 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, 1465 unsigned char ae, unsigned long ctx_mask, 1466 enum icp_qat_uof_regtype reg_type, 1467 unsigned short reg_num, unsigned int regdata) 1468 { 1469 int stat = 0; 1470 unsigned short reg; 1471 unsigned char ctx = 0; 1472 enum icp_qat_uof_regtype type; 1473 1474 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) 1475 return -EINVAL; 1476 1477 do { 1478 if (ctx_mask == 0) { 1479 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, 1480 &ctx); 1481 type = reg_type - 1; 1482 } else { 1483 reg = reg_num; 1484 type = reg_type; 1485 if (!test_bit(ctx, &ctx_mask)) 1486 continue; 1487 } 1488 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); 1489 if (stat) { 1490 pr_err("QAT: write gpr fail\n"); 1491 return -EINVAL; 1492 } 1493 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1494 1495 return 0; 1496 } 1497 qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1498 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, 1499 unsigned char ae, unsigned long ctx_mask, 1500 enum icp_qat_uof_regtype reg_type, 1501 unsigned short reg_num, unsigned int regdata) 1502 { 1503 int stat = 0; 1504 unsigned short reg; 1505 unsigned char ctx = 0; 1506 enum icp_qat_uof_regtype type; 1507 1508 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 1509 return -EINVAL; 1510 1511 do { 1512 if (ctx_mask == 0) { 1513 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, 1514 &ctx); 1515 type = reg_type - 3; 1516 } else { 1517 reg = reg_num; 1518 type = reg_type; 1519 if (!test_bit(ctx, &ctx_mask)) 1520 continue; 1521 } 1522 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, 1523 regdata); 1524 if (stat) { 1525 pr_err("QAT: write wr xfer fail\n"); 1526 return -EINVAL; 1527 } 1528 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1529 1530 return 0; 1531 } 1532 qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1533 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, 1534 unsigned char ae, unsigned long ctx_mask, 1535 enum icp_qat_uof_regtype reg_type, 1536 unsigned short reg_num, unsigned int regdata) 1537 { 1538 int stat = 0; 1539 unsigned short reg; 1540 unsigned char ctx = 0; 1541 enum icp_qat_uof_regtype type; 1542 1543 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 1544 return -EINVAL; 1545 1546 do { 1547 if (ctx_mask == 0) { 1548 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, 1549 &ctx); 1550 type = reg_type - 3; 1551 } else { 1552 reg = reg_num; 1553 type = reg_type; 1554 if (!test_bit(ctx, &ctx_mask)) 1555 continue; 1556 } 1557 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, 1558 regdata); 1559 if (stat) { 1560 pr_err("QAT: write rd xfer fail\n"); 1561 return -EINVAL; 1562 } 1563 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1564 1565 return 0; 1566 } 1567 qat_hal_init_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,unsigned short reg_num,unsigned int regdata)1568 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, 1569 unsigned char ae, unsigned long ctx_mask, 1570 unsigned short reg_num, unsigned int regdata) 1571 { 1572 int stat = 0; 1573 unsigned char ctx; 1574 if (!handle->chip_info->nn) { 1575 dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", 1576 handle->pci_dev->device); 1577 return -EINVAL; 1578 } 1579 1580 if (ctx_mask == 0) 1581 return -EINVAL; 1582 1583 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 1584 if (!test_bit(ctx, &ctx_mask)) 1585 continue; 1586 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); 1587 if (stat) { 1588 pr_err("QAT: write neigh error\n"); 1589 return -EINVAL; 1590 } 1591 } 1592 1593 return 0; 1594 } 1595