1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 */ 5 6 #include <linux/types.h> 7 #include <linux/io.h> 8 #include <linux/delay.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/errno.h> 11 #include <linux/kernel.h> 12 #include <linux/list.h> 13 #include <linux/mutex.h> 14 #include <linux/pci.h> 15 #include <linux/slab.h> 16 #include <linux/spinlock.h> 17 #include <linux/string.h> 18 #include <linux/qed/qed_chain.h> 19 #include "qed.h" 20 #include "qed_hsi.h" 21 #include "qed_hw.h" 22 #include "qed_reg_addr.h" 23 #include "qed_sriov.h" 24 25 #define QED_BAR_ACQUIRE_TIMEOUT 1000 26 27 /* Invalid values */ 28 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 29 30 struct qed_ptt { 31 struct list_head list_entry; 32 unsigned int idx; 33 struct pxp_ptt_entry pxp; 34 u8 hwfn_id; 35 }; 36 37 struct qed_ptt_pool { 38 struct list_head free_list; 39 spinlock_t lock; /* ptt synchronized access */ 40 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 41 }; 42 43 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 44 { 45 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 46 int i; 47 48 if (!p_pool) 49 return -ENOMEM; 50 51 INIT_LIST_HEAD(&p_pool->free_list); 52 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 53 p_pool->ptts[i].idx = i; 54 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 55 p_pool->ptts[i].pxp.pretend.control = 0; 56 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 57 if (i >= RESERVED_PTT_MAX) 58 list_add(&p_pool->ptts[i].list_entry, 59 &p_pool->free_list); 60 } 61 62 p_hwfn->p_ptt_pool = p_pool; 63 spin_lock_init(&p_pool->lock); 64 65 return 0; 66 } 67 68 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) 69 { 70 struct qed_ptt *p_ptt; 71 int i; 72 73 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 74 p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; 75 p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; 76 } 77 } 78 79 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 80 { 81 kfree(p_hwfn->p_ptt_pool); 82 p_hwfn->p_ptt_pool = NULL; 83 } 84 85 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 86 { 87 struct qed_ptt *p_ptt; 88 unsigned int i; 89 90 /* Take the free PTT from the list */ 91 for (i = 0; i < QED_BAR_ACQUIRE_TIMEOUT; i++) { 92 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 93 94 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 95 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 96 struct qed_ptt, list_entry); 97 list_del(&p_ptt->list_entry); 98 99 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 100 101 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 102 "allocated ptt %d\n", p_ptt->idx); 103 return p_ptt; 104 } 105 106 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 107 usleep_range(1000, 2000); 108 } 109 110 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 111 return NULL; 112 } 113 114 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 115 { 116 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 117 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 118 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 119 } 120 121 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 122 { 123 /* The HW is using DWORDS and we need to translate it to Bytes */ 124 return le32_to_cpu(p_ptt->pxp.offset) << 2; 125 } 126 127 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 128 { 129 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 130 p_ptt->idx * sizeof(struct pxp_ptt_entry); 131 } 132 133 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 134 { 135 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 136 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 137 } 138 139 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 140 struct qed_ptt *p_ptt, u32 new_hw_addr) 141 { 142 u32 prev_hw_addr; 143 144 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 145 146 if (new_hw_addr == prev_hw_addr) 147 return; 148 149 /* Update PTT entery in admin window */ 150 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 151 "Updating PTT entry %d to offset 0x%x\n", 152 p_ptt->idx, new_hw_addr); 153 154 /* The HW is using DWORDS and the address is in Bytes */ 155 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 156 157 REG_WR(p_hwfn, 158 qed_ptt_config_addr(p_ptt) + 159 offsetof(struct pxp_ptt_entry, offset), 160 le32_to_cpu(p_ptt->pxp.offset)); 161 } 162 163 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 164 struct qed_ptt *p_ptt, u32 hw_addr) 165 { 166 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 167 u32 offset; 168 169 offset = hw_addr - win_hw_addr; 170 171 if (p_ptt->hwfn_id != p_hwfn->my_id) 172 DP_NOTICE(p_hwfn, 173 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 174 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 175 176 /* Verify the address is within the window */ 177 if (hw_addr < win_hw_addr || 178 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 179 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 180 offset = 0; 181 } 182 183 return qed_ptt_get_bar_addr(p_ptt) + offset; 184 } 185 186 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 187 enum reserved_ptts ptt_idx) 188 { 189 if (ptt_idx >= RESERVED_PTT_MAX) { 190 DP_NOTICE(p_hwfn, 191 "Requested PTT %d is out of range\n", ptt_idx); 192 return NULL; 193 } 194 195 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 196 } 197 198 void qed_wr(struct qed_hwfn *p_hwfn, 199 struct qed_ptt *p_ptt, 200 u32 hw_addr, u32 val) 201 { 202 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 203 204 REG_WR(p_hwfn, bar_addr, val); 205 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 206 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 207 bar_addr, hw_addr, val); 208 } 209 210 u32 qed_rd(struct qed_hwfn *p_hwfn, 211 struct qed_ptt *p_ptt, 212 u32 hw_addr) 213 { 214 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 215 u32 val = REG_RD(p_hwfn, bar_addr); 216 217 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 218 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 219 bar_addr, hw_addr, val); 220 221 return val; 222 } 223 224 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 225 struct qed_ptt *p_ptt, 226 void *addr, u32 hw_addr, size_t n, bool to_device) 227 { 228 u32 dw_count, *host_addr, hw_offset; 229 size_t quota, done = 0; 230 u32 __iomem *reg_addr; 231 232 while (done < n) { 233 quota = min_t(size_t, n - done, 234 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 235 236 if (IS_PF(p_hwfn->cdev)) { 237 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 238 hw_offset = qed_ptt_get_bar_addr(p_ptt); 239 } else { 240 hw_offset = hw_addr + done; 241 } 242 243 dw_count = quota / 4; 244 host_addr = (u32 *)((u8 *)addr + done); 245 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 246 if (to_device) 247 while (dw_count--) 248 DIRECT_REG_WR(reg_addr++, *host_addr++); 249 else 250 while (dw_count--) 251 *host_addr++ = DIRECT_REG_RD(reg_addr++); 252 253 done += quota; 254 } 255 } 256 257 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 258 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 259 { 260 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 261 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 262 hw_addr, dest, hw_addr, (unsigned long)n); 263 264 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 265 } 266 267 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 268 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 269 { 270 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 271 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 272 hw_addr, hw_addr, src, (unsigned long)n); 273 274 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 275 } 276 277 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 278 { 279 u16 control = 0; 280 281 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 282 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 283 284 /* Every pretend undos previous pretends, including 285 * previous port pretend. 286 */ 287 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 288 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 289 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 290 291 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 292 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 293 294 p_ptt->pxp.pretend.control = cpu_to_le16(control); 295 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 296 297 REG_WR(p_hwfn, 298 qed_ptt_config_addr(p_ptt) + 299 offsetof(struct pxp_ptt_entry, pretend), 300 *(u32 *)&p_ptt->pxp.pretend); 301 } 302 303 void qed_port_pretend(struct qed_hwfn *p_hwfn, 304 struct qed_ptt *p_ptt, u8 port_id) 305 { 306 u16 control = 0; 307 308 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 309 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 310 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 311 312 p_ptt->pxp.pretend.control = cpu_to_le16(control); 313 314 REG_WR(p_hwfn, 315 qed_ptt_config_addr(p_ptt) + 316 offsetof(struct pxp_ptt_entry, pretend), 317 *(u32 *)&p_ptt->pxp.pretend); 318 } 319 320 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 321 { 322 u16 control = 0; 323 324 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 325 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 326 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 327 328 p_ptt->pxp.pretend.control = cpu_to_le16(control); 329 330 REG_WR(p_hwfn, 331 qed_ptt_config_addr(p_ptt) + 332 offsetof(struct pxp_ptt_entry, pretend), 333 *(u32 *)&p_ptt->pxp.pretend); 334 } 335 336 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, 337 struct qed_ptt *p_ptt, u8 port_id, u16 fid) 338 { 339 u16 control = 0; 340 341 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 342 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 343 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 344 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 345 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 346 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 347 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 348 p_ptt->pxp.pretend.control = cpu_to_le16(control); 349 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 350 REG_WR(p_hwfn, 351 qed_ptt_config_addr(p_ptt) + 352 offsetof(struct pxp_ptt_entry, pretend), 353 *(u32 *)&p_ptt->pxp.pretend); 354 } 355 356 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 357 { 358 u32 concrete_fid = 0; 359 360 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 361 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 362 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 363 364 return concrete_fid; 365 } 366 367 /* DMAE */ 368 #define QED_DMAE_FLAGS_IS_SET(params, flag) \ 369 ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag)) 370 371 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 372 const u8 is_src_type_grc, 373 const u8 is_dst_type_grc, 374 struct qed_dmae_params *p_params) 375 { 376 u8 src_pfid, dst_pfid, port_id; 377 u16 opcode_b = 0; 378 u32 opcode = 0; 379 380 /* Whether the source is the PCIe or the GRC. 381 * 0- The source is the PCIe 382 * 1- The source is the GRC. 383 */ 384 SET_FIELD(opcode, DMAE_CMD_SRC, 385 (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie)); 386 src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ? 387 p_params->src_pfid : p_hwfn->rel_pf_id; 388 SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid); 389 390 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 391 SET_FIELD(opcode, DMAE_CMD_DST, 392 (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie)); 393 dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ? 394 p_params->dst_pfid : p_hwfn->rel_pf_id; 395 SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid); 396 397 398 /* Whether to write a completion word to the completion destination: 399 * 0-Do not write a completion word 400 * 1-Write the completion word 401 */ 402 SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1); 403 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 404 405 if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST)) 406 SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1); 407 408 /* swapping mode 3 - big endian */ 409 SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY); 410 411 port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ? 412 p_params->port_id : p_hwfn->port_id; 413 SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id); 414 415 /* reset source address in next go */ 416 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 417 418 /* reset dest address in next go */ 419 SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1); 420 421 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 422 if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) { 423 SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1); 424 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid); 425 } else { 426 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF); 427 } 428 if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) { 429 SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1); 430 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid); 431 } else { 432 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF); 433 } 434 435 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 436 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 437 } 438 439 u32 qed_dmae_idx_to_go_cmd(u8 idx) 440 { 441 /* All the DMAE 'go' registers form an array in internal memory */ 442 return DMAE_REG_GO_C0 + (idx << 2); 443 } 444 445 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 446 struct qed_ptt *p_ptt) 447 { 448 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 449 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 450 int qed_status = 0; 451 452 /* verify address is not NULL */ 453 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 454 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 455 DP_NOTICE(p_hwfn, 456 "source or destination address 0 idx_cmd=%d\n" 457 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 458 idx_cmd, 459 le32_to_cpu(p_command->opcode), 460 le16_to_cpu(p_command->opcode_b), 461 le16_to_cpu(p_command->length_dw), 462 le32_to_cpu(p_command->src_addr_hi), 463 le32_to_cpu(p_command->src_addr_lo), 464 le32_to_cpu(p_command->dst_addr_hi), 465 le32_to_cpu(p_command->dst_addr_lo)); 466 467 return -EINVAL; 468 } 469 470 DP_VERBOSE(p_hwfn, 471 NETIF_MSG_HW, 472 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 473 idx_cmd, 474 le32_to_cpu(p_command->opcode), 475 le16_to_cpu(p_command->opcode_b), 476 le16_to_cpu(p_command->length_dw), 477 le32_to_cpu(p_command->src_addr_hi), 478 le32_to_cpu(p_command->src_addr_lo), 479 le32_to_cpu(p_command->dst_addr_hi), 480 le32_to_cpu(p_command->dst_addr_lo)); 481 482 /* Copy the command to DMAE - need to do it before every call 483 * for source/dest address no reset. 484 * The first 9 DWs are the command registers, the 10 DW is the 485 * GO register, and the rest are result registers 486 * (which are read only by the client). 487 */ 488 for (i = 0; i < DMAE_CMD_SIZE; i++) { 489 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 490 *(((u32 *)p_command) + i) : 0; 491 492 qed_wr(p_hwfn, p_ptt, 493 DMAE_REG_CMD_MEM + 494 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 495 (i * sizeof(u32)), data); 496 } 497 498 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 499 500 return qed_status; 501 } 502 503 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 504 { 505 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 506 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 507 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 508 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 509 510 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 511 sizeof(u32), p_addr, GFP_KERNEL); 512 if (!*p_comp) 513 goto err; 514 515 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 516 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 517 sizeof(struct dmae_cmd), 518 p_addr, GFP_KERNEL); 519 if (!*p_cmd) 520 goto err; 521 522 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 523 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 524 sizeof(u32) * DMAE_MAX_RW_SIZE, 525 p_addr, GFP_KERNEL); 526 if (!*p_buff) 527 goto err; 528 529 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 530 531 return 0; 532 err: 533 qed_dmae_info_free(p_hwfn); 534 return -ENOMEM; 535 } 536 537 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 538 { 539 dma_addr_t p_phys; 540 541 /* Just make sure no one is in the middle */ 542 mutex_lock(&p_hwfn->dmae_info.mutex); 543 544 if (p_hwfn->dmae_info.p_completion_word) { 545 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 546 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 547 sizeof(u32), 548 p_hwfn->dmae_info.p_completion_word, p_phys); 549 p_hwfn->dmae_info.p_completion_word = NULL; 550 } 551 552 if (p_hwfn->dmae_info.p_dmae_cmd) { 553 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 554 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 555 sizeof(struct dmae_cmd), 556 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 557 p_hwfn->dmae_info.p_dmae_cmd = NULL; 558 } 559 560 if (p_hwfn->dmae_info.p_intermediate_buffer) { 561 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 562 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 563 sizeof(u32) * DMAE_MAX_RW_SIZE, 564 p_hwfn->dmae_info.p_intermediate_buffer, 565 p_phys); 566 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 567 } 568 569 mutex_unlock(&p_hwfn->dmae_info.mutex); 570 } 571 572 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 573 { 574 u32 wait_cnt_limit = 10000, wait_cnt = 0; 575 int qed_status = 0; 576 577 barrier(); 578 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 579 udelay(DMAE_MIN_WAIT_TIME); 580 if (++wait_cnt > wait_cnt_limit) { 581 DP_NOTICE(p_hwfn->cdev, 582 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 583 *p_hwfn->dmae_info.p_completion_word, 584 DMAE_COMPLETION_VAL); 585 qed_status = -EBUSY; 586 break; 587 } 588 589 /* to sync the completion_word since we are not 590 * using the volatile keyword for p_completion_word 591 */ 592 barrier(); 593 } 594 595 if (qed_status == 0) 596 *p_hwfn->dmae_info.p_completion_word = 0; 597 598 return qed_status; 599 } 600 601 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 602 struct qed_ptt *p_ptt, 603 u64 src_addr, 604 u64 dst_addr, 605 u8 src_type, 606 u8 dst_type, 607 u32 length_dw) 608 { 609 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 610 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 611 int qed_status = 0; 612 613 switch (src_type) { 614 case QED_DMAE_ADDRESS_GRC: 615 case QED_DMAE_ADDRESS_HOST_PHYS: 616 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 617 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 618 break; 619 /* for virtual source addresses we use the intermediate buffer. */ 620 case QED_DMAE_ADDRESS_HOST_VIRT: 621 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 622 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 623 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 624 (void *)(uintptr_t)src_addr, 625 length_dw * sizeof(u32)); 626 break; 627 default: 628 return -EINVAL; 629 } 630 631 switch (dst_type) { 632 case QED_DMAE_ADDRESS_GRC: 633 case QED_DMAE_ADDRESS_HOST_PHYS: 634 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 635 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 636 break; 637 /* for virtual source addresses we use the intermediate buffer. */ 638 case QED_DMAE_ADDRESS_HOST_VIRT: 639 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 640 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 641 break; 642 default: 643 return -EINVAL; 644 } 645 646 cmd->length_dw = cpu_to_le16((u16)length_dw); 647 648 qed_dmae_post_command(p_hwfn, p_ptt); 649 650 qed_status = qed_dmae_operation_wait(p_hwfn); 651 652 if (qed_status) { 653 DP_NOTICE(p_hwfn, 654 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 655 src_addr, dst_addr, length_dw); 656 return qed_status; 657 } 658 659 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 660 memcpy((void *)(uintptr_t)(dst_addr), 661 &p_hwfn->dmae_info.p_intermediate_buffer[0], 662 length_dw * sizeof(u32)); 663 664 return 0; 665 } 666 667 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 668 struct qed_ptt *p_ptt, 669 u64 src_addr, u64 dst_addr, 670 u8 src_type, u8 dst_type, 671 u32 size_in_dwords, 672 struct qed_dmae_params *p_params) 673 { 674 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 675 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 676 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 677 u64 src_addr_split = 0, dst_addr_split = 0; 678 u16 length_limit = DMAE_MAX_RW_SIZE; 679 int qed_status = 0; 680 u32 offset = 0; 681 682 if (p_hwfn->cdev->recov_in_prog) { 683 DP_VERBOSE(p_hwfn, 684 NETIF_MSG_HW, 685 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n", 686 src_addr, src_type, dst_addr, dst_type, 687 size_in_dwords); 688 689 /* Let the flow complete w/o any error handling */ 690 return 0; 691 } 692 693 qed_dmae_opcode(p_hwfn, 694 (src_type == QED_DMAE_ADDRESS_GRC), 695 (dst_type == QED_DMAE_ADDRESS_GRC), 696 p_params); 697 698 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 699 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 700 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 701 702 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 703 cnt_split = size_in_dwords / length_limit; 704 length_mod = size_in_dwords % length_limit; 705 706 src_addr_split = src_addr; 707 dst_addr_split = dst_addr; 708 709 for (i = 0; i <= cnt_split; i++) { 710 offset = length_limit * i; 711 712 if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) { 713 if (src_type == QED_DMAE_ADDRESS_GRC) 714 src_addr_split = src_addr + offset; 715 else 716 src_addr_split = src_addr + (offset * 4); 717 } 718 719 if (dst_type == QED_DMAE_ADDRESS_GRC) 720 dst_addr_split = dst_addr + offset; 721 else 722 dst_addr_split = dst_addr + (offset * 4); 723 724 length_cur = (cnt_split == i) ? length_mod : length_limit; 725 726 /* might be zero on last iteration */ 727 if (!length_cur) 728 continue; 729 730 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 731 p_ptt, 732 src_addr_split, 733 dst_addr_split, 734 src_type, 735 dst_type, 736 length_cur); 737 if (qed_status) { 738 qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL, 739 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 740 qed_status, src_addr, 741 dst_addr, length_cur); 742 break; 743 } 744 } 745 746 return qed_status; 747 } 748 749 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 750 struct qed_ptt *p_ptt, 751 u64 source_addr, u32 grc_addr, u32 size_in_dwords, 752 struct qed_dmae_params *p_params) 753 { 754 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 755 int rc; 756 757 758 mutex_lock(&p_hwfn->dmae_info.mutex); 759 760 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 761 grc_addr_in_dw, 762 QED_DMAE_ADDRESS_HOST_VIRT, 763 QED_DMAE_ADDRESS_GRC, 764 size_in_dwords, p_params); 765 766 mutex_unlock(&p_hwfn->dmae_info.mutex); 767 768 return rc; 769 } 770 771 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 772 struct qed_ptt *p_ptt, 773 u32 grc_addr, 774 dma_addr_t dest_addr, u32 size_in_dwords, 775 struct qed_dmae_params *p_params) 776 { 777 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 778 int rc; 779 780 781 mutex_lock(&p_hwfn->dmae_info.mutex); 782 783 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 784 dest_addr, QED_DMAE_ADDRESS_GRC, 785 QED_DMAE_ADDRESS_HOST_VIRT, 786 size_in_dwords, p_params); 787 788 mutex_unlock(&p_hwfn->dmae_info.mutex); 789 790 return rc; 791 } 792 793 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 794 struct qed_ptt *p_ptt, 795 dma_addr_t source_addr, 796 dma_addr_t dest_addr, 797 u32 size_in_dwords, struct qed_dmae_params *p_params) 798 { 799 int rc; 800 801 mutex_lock(&(p_hwfn->dmae_info.mutex)); 802 803 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 804 dest_addr, 805 QED_DMAE_ADDRESS_HOST_PHYS, 806 QED_DMAE_ADDRESS_HOST_PHYS, 807 size_in_dwords, p_params); 808 809 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 810 811 return rc; 812 } 813 814 void qed_hw_err_notify(struct qed_hwfn *p_hwfn, 815 struct qed_ptt *p_ptt, 816 enum qed_hw_err_type err_type, char *fmt, ...) 817 { 818 char buf[QED_HW_ERR_MAX_STR_SIZE]; 819 va_list vl; 820 int len; 821 822 if (fmt) { 823 va_start(vl, fmt); 824 len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl); 825 va_end(vl); 826 827 if (len > QED_HW_ERR_MAX_STR_SIZE - 1) 828 len = QED_HW_ERR_MAX_STR_SIZE - 1; 829 830 DP_NOTICE(p_hwfn, "%s", buf); 831 } 832 833 /* Fan failure cannot be masked by handling of another HW error */ 834 if (p_hwfn->cdev->recov_in_prog && 835 err_type != QED_HW_ERR_FAN_FAIL) { 836 DP_VERBOSE(p_hwfn, 837 NETIF_MSG_DRV, 838 "Recovery is in progress. Avoid notifying about HW error %d.\n", 839 err_type); 840 return; 841 } 842 843 qed_hw_error_occurred(p_hwfn, err_type); 844 845 if (fmt) 846 qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len); 847 } 848 849 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 850 struct qed_ptt *p_ptt, const char *phase) 851 { 852 u32 size = PAGE_SIZE / 2, val; 853 int rc = 0; 854 dma_addr_t p_phys; 855 void *p_virt; 856 u32 *p_tmp; 857 858 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 859 2 * size, &p_phys, GFP_KERNEL); 860 if (!p_virt) { 861 DP_NOTICE(p_hwfn, 862 "DMAE sanity [%s]: failed to allocate memory\n", 863 phase); 864 return -ENOMEM; 865 } 866 867 /* Fill the bottom half of the allocated memory with a known pattern */ 868 for (p_tmp = (u32 *)p_virt; 869 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 870 /* Save the address itself as the value */ 871 val = (u32)(uintptr_t)p_tmp; 872 *p_tmp = val; 873 } 874 875 /* Zero the top half of the allocated memory */ 876 memset((u8 *)p_virt + size, 0, size); 877 878 DP_VERBOSE(p_hwfn, 879 QED_MSG_SP, 880 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 881 phase, 882 (u64)p_phys, 883 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 884 885 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 886 size / 4, NULL); 887 if (rc) { 888 DP_NOTICE(p_hwfn, 889 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 890 phase, rc); 891 goto out; 892 } 893 894 /* Verify that the top half of the allocated memory has the pattern */ 895 for (p_tmp = (u32 *)((u8 *)p_virt + size); 896 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 897 /* The corresponding address in the bottom half */ 898 val = (u32)(uintptr_t)p_tmp - size; 899 900 if (*p_tmp != val) { 901 DP_NOTICE(p_hwfn, 902 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 903 phase, 904 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 905 p_tmp, *p_tmp, val); 906 rc = -EINVAL; 907 goto out; 908 } 909 } 910 911 out: 912 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 913 return rc; 914 } 915