1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <linux/io.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/kernel.h> 39 #include <linux/list.h> 40 #include <linux/mutex.h> 41 #include <linux/pci.h> 42 #include <linux/slab.h> 43 #include <linux/spinlock.h> 44 #include <linux/string.h> 45 #include <linux/qed/qed_chain.h> 46 #include "qed.h" 47 #include "qed_hsi.h" 48 #include "qed_hw.h" 49 #include "qed_reg_addr.h" 50 #include "qed_sriov.h" 51 52 #define QED_BAR_ACQUIRE_TIMEOUT 1000 53 54 /* Invalid values */ 55 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 56 57 struct qed_ptt { 58 struct list_head list_entry; 59 unsigned int idx; 60 struct pxp_ptt_entry pxp; 61 u8 hwfn_id; 62 }; 63 64 struct qed_ptt_pool { 65 struct list_head free_list; 66 spinlock_t lock; /* ptt synchronized access */ 67 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 68 }; 69 70 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 71 { 72 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 73 int i; 74 75 if (!p_pool) 76 return -ENOMEM; 77 78 INIT_LIST_HEAD(&p_pool->free_list); 79 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 80 p_pool->ptts[i].idx = i; 81 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 82 p_pool->ptts[i].pxp.pretend.control = 0; 83 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 84 if (i >= RESERVED_PTT_MAX) 85 list_add(&p_pool->ptts[i].list_entry, 86 &p_pool->free_list); 87 } 88 89 p_hwfn->p_ptt_pool = p_pool; 90 spin_lock_init(&p_pool->lock); 91 92 return 0; 93 } 94 95 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) 96 { 97 struct qed_ptt *p_ptt; 98 int i; 99 100 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 101 p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; 102 p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; 103 } 104 } 105 106 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 107 { 108 kfree(p_hwfn->p_ptt_pool); 109 p_hwfn->p_ptt_pool = NULL; 110 } 111 112 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 113 { 114 struct qed_ptt *p_ptt; 115 unsigned int i; 116 117 /* Take the free PTT from the list */ 118 for (i = 0; i < QED_BAR_ACQUIRE_TIMEOUT; i++) { 119 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 120 121 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 122 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 123 struct qed_ptt, list_entry); 124 list_del(&p_ptt->list_entry); 125 126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 127 128 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 129 "allocated ptt %d\n", p_ptt->idx); 130 return p_ptt; 131 } 132 133 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 134 usleep_range(1000, 2000); 135 } 136 137 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 138 return NULL; 139 } 140 141 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 142 { 143 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 144 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 145 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 146 } 147 148 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 149 { 150 /* The HW is using DWORDS and we need to translate it to Bytes */ 151 return le32_to_cpu(p_ptt->pxp.offset) << 2; 152 } 153 154 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 155 { 156 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 157 p_ptt->idx * sizeof(struct pxp_ptt_entry); 158 } 159 160 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 161 { 162 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 163 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 164 } 165 166 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 167 struct qed_ptt *p_ptt, u32 new_hw_addr) 168 { 169 u32 prev_hw_addr; 170 171 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 172 173 if (new_hw_addr == prev_hw_addr) 174 return; 175 176 /* Update PTT entery in admin window */ 177 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 178 "Updating PTT entry %d to offset 0x%x\n", 179 p_ptt->idx, new_hw_addr); 180 181 /* The HW is using DWORDS and the address is in Bytes */ 182 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 183 184 REG_WR(p_hwfn, 185 qed_ptt_config_addr(p_ptt) + 186 offsetof(struct pxp_ptt_entry, offset), 187 le32_to_cpu(p_ptt->pxp.offset)); 188 } 189 190 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 191 struct qed_ptt *p_ptt, u32 hw_addr) 192 { 193 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 194 u32 offset; 195 196 offset = hw_addr - win_hw_addr; 197 198 if (p_ptt->hwfn_id != p_hwfn->my_id) 199 DP_NOTICE(p_hwfn, 200 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 201 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 202 203 /* Verify the address is within the window */ 204 if (hw_addr < win_hw_addr || 205 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 206 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 207 offset = 0; 208 } 209 210 return qed_ptt_get_bar_addr(p_ptt) + offset; 211 } 212 213 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 214 enum reserved_ptts ptt_idx) 215 { 216 if (ptt_idx >= RESERVED_PTT_MAX) { 217 DP_NOTICE(p_hwfn, 218 "Requested PTT %d is out of range\n", ptt_idx); 219 return NULL; 220 } 221 222 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 223 } 224 225 void qed_wr(struct qed_hwfn *p_hwfn, 226 struct qed_ptt *p_ptt, 227 u32 hw_addr, u32 val) 228 { 229 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 230 231 REG_WR(p_hwfn, bar_addr, val); 232 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 233 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 234 bar_addr, hw_addr, val); 235 } 236 237 u32 qed_rd(struct qed_hwfn *p_hwfn, 238 struct qed_ptt *p_ptt, 239 u32 hw_addr) 240 { 241 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 242 u32 val = REG_RD(p_hwfn, bar_addr); 243 244 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 245 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 246 bar_addr, hw_addr, val); 247 248 return val; 249 } 250 251 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 252 struct qed_ptt *p_ptt, 253 void *addr, u32 hw_addr, size_t n, bool to_device) 254 { 255 u32 dw_count, *host_addr, hw_offset; 256 size_t quota, done = 0; 257 u32 __iomem *reg_addr; 258 259 while (done < n) { 260 quota = min_t(size_t, n - done, 261 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 262 263 if (IS_PF(p_hwfn->cdev)) { 264 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 265 hw_offset = qed_ptt_get_bar_addr(p_ptt); 266 } else { 267 hw_offset = hw_addr + done; 268 } 269 270 dw_count = quota / 4; 271 host_addr = (u32 *)((u8 *)addr + done); 272 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 273 if (to_device) 274 while (dw_count--) 275 DIRECT_REG_WR(reg_addr++, *host_addr++); 276 else 277 while (dw_count--) 278 *host_addr++ = DIRECT_REG_RD(reg_addr++); 279 280 done += quota; 281 } 282 } 283 284 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 285 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 286 { 287 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 288 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 289 hw_addr, dest, hw_addr, (unsigned long)n); 290 291 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 292 } 293 294 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 295 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 296 { 297 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 298 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 299 hw_addr, hw_addr, src, (unsigned long)n); 300 301 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 302 } 303 304 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 305 { 306 u16 control = 0; 307 308 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 309 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 310 311 /* Every pretend undos previous pretends, including 312 * previous port pretend. 313 */ 314 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 315 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 316 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 317 318 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 319 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 320 321 p_ptt->pxp.pretend.control = cpu_to_le16(control); 322 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 323 324 REG_WR(p_hwfn, 325 qed_ptt_config_addr(p_ptt) + 326 offsetof(struct pxp_ptt_entry, pretend), 327 *(u32 *)&p_ptt->pxp.pretend); 328 } 329 330 void qed_port_pretend(struct qed_hwfn *p_hwfn, 331 struct qed_ptt *p_ptt, u8 port_id) 332 { 333 u16 control = 0; 334 335 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 336 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 337 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 338 339 p_ptt->pxp.pretend.control = cpu_to_le16(control); 340 341 REG_WR(p_hwfn, 342 qed_ptt_config_addr(p_ptt) + 343 offsetof(struct pxp_ptt_entry, pretend), 344 *(u32 *)&p_ptt->pxp.pretend); 345 } 346 347 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 348 { 349 u16 control = 0; 350 351 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 352 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 354 355 p_ptt->pxp.pretend.control = cpu_to_le16(control); 356 357 REG_WR(p_hwfn, 358 qed_ptt_config_addr(p_ptt) + 359 offsetof(struct pxp_ptt_entry, pretend), 360 *(u32 *)&p_ptt->pxp.pretend); 361 } 362 363 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 364 { 365 u32 concrete_fid = 0; 366 367 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 368 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 369 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 370 371 return concrete_fid; 372 } 373 374 /* DMAE */ 375 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 376 const u8 is_src_type_grc, 377 const u8 is_dst_type_grc, 378 struct qed_dmae_params *p_params) 379 { 380 u16 opcode_b = 0; 381 u32 opcode = 0; 382 383 /* Whether the source is the PCIe or the GRC. 384 * 0- The source is the PCIe 385 * 1- The source is the GRC. 386 */ 387 opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC 388 : DMAE_CMD_SRC_MASK_PCIE) << 389 DMAE_CMD_SRC_SHIFT; 390 opcode |= ((p_hwfn->rel_pf_id & DMAE_CMD_SRC_PF_ID_MASK) << 391 DMAE_CMD_SRC_PF_ID_SHIFT); 392 393 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 394 opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC 395 : DMAE_CMD_DST_MASK_PCIE) << 396 DMAE_CMD_DST_SHIFT; 397 opcode |= ((p_hwfn->rel_pf_id & DMAE_CMD_DST_PF_ID_MASK) << 398 DMAE_CMD_DST_PF_ID_SHIFT); 399 400 /* Whether to write a completion word to the completion destination: 401 * 0-Do not write a completion word 402 * 1-Write the completion word 403 */ 404 opcode |= (DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT); 405 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 406 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 407 408 if (p_params->flags & QED_DMAE_FLAG_COMPLETION_DST) 409 opcode |= (1 << DMAE_CMD_COMP_FUNC_SHIFT); 410 411 opcode |= (DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT); 412 413 opcode |= ((p_hwfn->port_id) << DMAE_CMD_PORT_ID_SHIFT); 414 415 /* reset source address in next go */ 416 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 417 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 418 419 /* reset dest address in next go */ 420 opcode |= (DMAE_CMD_DST_ADDR_RESET_MASK << 421 DMAE_CMD_DST_ADDR_RESET_SHIFT); 422 423 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 424 if (p_params->flags & QED_DMAE_FLAG_VF_SRC) { 425 opcode |= 1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT; 426 opcode_b |= p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT; 427 } else { 428 opcode_b |= DMAE_CMD_SRC_VF_ID_MASK << 429 DMAE_CMD_SRC_VF_ID_SHIFT; 430 } 431 432 if (p_params->flags & QED_DMAE_FLAG_VF_DST) { 433 opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT; 434 opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT; 435 } else { 436 opcode_b |= DMAE_CMD_DST_VF_ID_MASK << DMAE_CMD_DST_VF_ID_SHIFT; 437 } 438 439 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 440 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 441 } 442 443 u32 qed_dmae_idx_to_go_cmd(u8 idx) 444 { 445 /* All the DMAE 'go' registers form an array in internal memory */ 446 return DMAE_REG_GO_C0 + (idx << 2); 447 } 448 449 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 450 struct qed_ptt *p_ptt) 451 { 452 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 453 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 454 int qed_status = 0; 455 456 /* verify address is not NULL */ 457 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 458 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 459 DP_NOTICE(p_hwfn, 460 "source or destination address 0 idx_cmd=%d\n" 461 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 462 idx_cmd, 463 le32_to_cpu(p_command->opcode), 464 le16_to_cpu(p_command->opcode_b), 465 le16_to_cpu(p_command->length_dw), 466 le32_to_cpu(p_command->src_addr_hi), 467 le32_to_cpu(p_command->src_addr_lo), 468 le32_to_cpu(p_command->dst_addr_hi), 469 le32_to_cpu(p_command->dst_addr_lo)); 470 471 return -EINVAL; 472 } 473 474 DP_VERBOSE(p_hwfn, 475 NETIF_MSG_HW, 476 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 477 idx_cmd, 478 le32_to_cpu(p_command->opcode), 479 le16_to_cpu(p_command->opcode_b), 480 le16_to_cpu(p_command->length_dw), 481 le32_to_cpu(p_command->src_addr_hi), 482 le32_to_cpu(p_command->src_addr_lo), 483 le32_to_cpu(p_command->dst_addr_hi), 484 le32_to_cpu(p_command->dst_addr_lo)); 485 486 /* Copy the command to DMAE - need to do it before every call 487 * for source/dest address no reset. 488 * The first 9 DWs are the command registers, the 10 DW is the 489 * GO register, and the rest are result registers 490 * (which are read only by the client). 491 */ 492 for (i = 0; i < DMAE_CMD_SIZE; i++) { 493 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 494 *(((u32 *)p_command) + i) : 0; 495 496 qed_wr(p_hwfn, p_ptt, 497 DMAE_REG_CMD_MEM + 498 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 499 (i * sizeof(u32)), data); 500 } 501 502 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 503 504 return qed_status; 505 } 506 507 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 508 { 509 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 510 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 511 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 512 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 513 514 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 515 sizeof(u32), p_addr, GFP_KERNEL); 516 if (!*p_comp) 517 goto err; 518 519 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 520 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 521 sizeof(struct dmae_cmd), 522 p_addr, GFP_KERNEL); 523 if (!*p_cmd) 524 goto err; 525 526 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 527 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 528 sizeof(u32) * DMAE_MAX_RW_SIZE, 529 p_addr, GFP_KERNEL); 530 if (!*p_buff) 531 goto err; 532 533 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 534 535 return 0; 536 err: 537 qed_dmae_info_free(p_hwfn); 538 return -ENOMEM; 539 } 540 541 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 542 { 543 dma_addr_t p_phys; 544 545 /* Just make sure no one is in the middle */ 546 mutex_lock(&p_hwfn->dmae_info.mutex); 547 548 if (p_hwfn->dmae_info.p_completion_word) { 549 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 550 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 551 sizeof(u32), 552 p_hwfn->dmae_info.p_completion_word, p_phys); 553 p_hwfn->dmae_info.p_completion_word = NULL; 554 } 555 556 if (p_hwfn->dmae_info.p_dmae_cmd) { 557 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 558 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 559 sizeof(struct dmae_cmd), 560 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 561 p_hwfn->dmae_info.p_dmae_cmd = NULL; 562 } 563 564 if (p_hwfn->dmae_info.p_intermediate_buffer) { 565 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 566 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 567 sizeof(u32) * DMAE_MAX_RW_SIZE, 568 p_hwfn->dmae_info.p_intermediate_buffer, 569 p_phys); 570 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 571 } 572 573 mutex_unlock(&p_hwfn->dmae_info.mutex); 574 } 575 576 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 577 { 578 u32 wait_cnt_limit = 10000, wait_cnt = 0; 579 int qed_status = 0; 580 581 barrier(); 582 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 583 udelay(DMAE_MIN_WAIT_TIME); 584 if (++wait_cnt > wait_cnt_limit) { 585 DP_NOTICE(p_hwfn->cdev, 586 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 587 *p_hwfn->dmae_info.p_completion_word, 588 DMAE_COMPLETION_VAL); 589 qed_status = -EBUSY; 590 break; 591 } 592 593 /* to sync the completion_word since we are not 594 * using the volatile keyword for p_completion_word 595 */ 596 barrier(); 597 } 598 599 if (qed_status == 0) 600 *p_hwfn->dmae_info.p_completion_word = 0; 601 602 return qed_status; 603 } 604 605 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 606 struct qed_ptt *p_ptt, 607 u64 src_addr, 608 u64 dst_addr, 609 u8 src_type, 610 u8 dst_type, 611 u32 length_dw) 612 { 613 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 614 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 615 int qed_status = 0; 616 617 switch (src_type) { 618 case QED_DMAE_ADDRESS_GRC: 619 case QED_DMAE_ADDRESS_HOST_PHYS: 620 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 621 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 622 break; 623 /* for virtual source addresses we use the intermediate buffer. */ 624 case QED_DMAE_ADDRESS_HOST_VIRT: 625 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 626 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 627 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 628 (void *)(uintptr_t)src_addr, 629 length_dw * sizeof(u32)); 630 break; 631 default: 632 return -EINVAL; 633 } 634 635 switch (dst_type) { 636 case QED_DMAE_ADDRESS_GRC: 637 case QED_DMAE_ADDRESS_HOST_PHYS: 638 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 639 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 640 break; 641 /* for virtual source addresses we use the intermediate buffer. */ 642 case QED_DMAE_ADDRESS_HOST_VIRT: 643 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 644 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 645 break; 646 default: 647 return -EINVAL; 648 } 649 650 cmd->length_dw = cpu_to_le16((u16)length_dw); 651 652 qed_dmae_post_command(p_hwfn, p_ptt); 653 654 qed_status = qed_dmae_operation_wait(p_hwfn); 655 656 if (qed_status) { 657 DP_NOTICE(p_hwfn, 658 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 659 src_addr, dst_addr, length_dw); 660 return qed_status; 661 } 662 663 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 664 memcpy((void *)(uintptr_t)(dst_addr), 665 &p_hwfn->dmae_info.p_intermediate_buffer[0], 666 length_dw * sizeof(u32)); 667 668 return 0; 669 } 670 671 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 672 struct qed_ptt *p_ptt, 673 u64 src_addr, u64 dst_addr, 674 u8 src_type, u8 dst_type, 675 u32 size_in_dwords, 676 struct qed_dmae_params *p_params) 677 { 678 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 679 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 680 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 681 u64 src_addr_split = 0, dst_addr_split = 0; 682 u16 length_limit = DMAE_MAX_RW_SIZE; 683 int qed_status = 0; 684 u32 offset = 0; 685 686 qed_dmae_opcode(p_hwfn, 687 (src_type == QED_DMAE_ADDRESS_GRC), 688 (dst_type == QED_DMAE_ADDRESS_GRC), 689 p_params); 690 691 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 692 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 693 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 694 695 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 696 cnt_split = size_in_dwords / length_limit; 697 length_mod = size_in_dwords % length_limit; 698 699 src_addr_split = src_addr; 700 dst_addr_split = dst_addr; 701 702 for (i = 0; i <= cnt_split; i++) { 703 offset = length_limit * i; 704 705 if (!(p_params->flags & QED_DMAE_FLAG_RW_REPL_SRC)) { 706 if (src_type == QED_DMAE_ADDRESS_GRC) 707 src_addr_split = src_addr + offset; 708 else 709 src_addr_split = src_addr + (offset * 4); 710 } 711 712 if (dst_type == QED_DMAE_ADDRESS_GRC) 713 dst_addr_split = dst_addr + offset; 714 else 715 dst_addr_split = dst_addr + (offset * 4); 716 717 length_cur = (cnt_split == i) ? length_mod : length_limit; 718 719 /* might be zero on last iteration */ 720 if (!length_cur) 721 continue; 722 723 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 724 p_ptt, 725 src_addr_split, 726 dst_addr_split, 727 src_type, 728 dst_type, 729 length_cur); 730 if (qed_status) { 731 DP_NOTICE(p_hwfn, 732 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 733 qed_status, src_addr, dst_addr, length_cur); 734 break; 735 } 736 } 737 738 return qed_status; 739 } 740 741 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 742 struct qed_ptt *p_ptt, 743 u64 source_addr, u32 grc_addr, u32 size_in_dwords, u32 flags) 744 { 745 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 746 struct qed_dmae_params params; 747 int rc; 748 749 memset(¶ms, 0, sizeof(struct qed_dmae_params)); 750 params.flags = flags; 751 752 mutex_lock(&p_hwfn->dmae_info.mutex); 753 754 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 755 grc_addr_in_dw, 756 QED_DMAE_ADDRESS_HOST_VIRT, 757 QED_DMAE_ADDRESS_GRC, 758 size_in_dwords, ¶ms); 759 760 mutex_unlock(&p_hwfn->dmae_info.mutex); 761 762 return rc; 763 } 764 765 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 766 struct qed_ptt *p_ptt, 767 u32 grc_addr, 768 dma_addr_t dest_addr, u32 size_in_dwords, u32 flags) 769 { 770 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 771 struct qed_dmae_params params; 772 int rc; 773 774 memset(¶ms, 0, sizeof(struct qed_dmae_params)); 775 params.flags = flags; 776 777 mutex_lock(&p_hwfn->dmae_info.mutex); 778 779 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 780 dest_addr, QED_DMAE_ADDRESS_GRC, 781 QED_DMAE_ADDRESS_HOST_VIRT, 782 size_in_dwords, ¶ms); 783 784 mutex_unlock(&p_hwfn->dmae_info.mutex); 785 786 return rc; 787 } 788 789 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 790 struct qed_ptt *p_ptt, 791 dma_addr_t source_addr, 792 dma_addr_t dest_addr, 793 u32 size_in_dwords, struct qed_dmae_params *p_params) 794 { 795 int rc; 796 797 mutex_lock(&(p_hwfn->dmae_info.mutex)); 798 799 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 800 dest_addr, 801 QED_DMAE_ADDRESS_HOST_PHYS, 802 QED_DMAE_ADDRESS_HOST_PHYS, 803 size_in_dwords, p_params); 804 805 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 806 807 return rc; 808 } 809 810 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 811 struct qed_ptt *p_ptt, const char *phase) 812 { 813 u32 size = PAGE_SIZE / 2, val; 814 struct qed_dmae_params params; 815 int rc = 0; 816 dma_addr_t p_phys; 817 void *p_virt; 818 u32 *p_tmp; 819 820 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 821 2 * size, &p_phys, GFP_KERNEL); 822 if (!p_virt) { 823 DP_NOTICE(p_hwfn, 824 "DMAE sanity [%s]: failed to allocate memory\n", 825 phase); 826 return -ENOMEM; 827 } 828 829 /* Fill the bottom half of the allocated memory with a known pattern */ 830 for (p_tmp = (u32 *)p_virt; 831 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 832 /* Save the address itself as the value */ 833 val = (u32)(uintptr_t)p_tmp; 834 *p_tmp = val; 835 } 836 837 /* Zero the top half of the allocated memory */ 838 memset((u8 *)p_virt + size, 0, size); 839 840 DP_VERBOSE(p_hwfn, 841 QED_MSG_SP, 842 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 843 phase, 844 (u64)p_phys, 845 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 846 847 memset(¶ms, 0, sizeof(params)); 848 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 849 size / 4 /* size_in_dwords */, ¶ms); 850 if (rc) { 851 DP_NOTICE(p_hwfn, 852 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 853 phase, rc); 854 goto out; 855 } 856 857 /* Verify that the top half of the allocated memory has the pattern */ 858 for (p_tmp = (u32 *)((u8 *)p_virt + size); 859 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 860 /* The corresponding address in the bottom half */ 861 val = (u32)(uintptr_t)p_tmp - size; 862 863 if (*p_tmp != val) { 864 DP_NOTICE(p_hwfn, 865 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 866 phase, 867 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 868 p_tmp, *p_tmp, val); 869 rc = -EINVAL; 870 goto out; 871 } 872 } 873 874 out: 875 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 876 return rc; 877 } 878