1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <linux/io.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/kernel.h> 39 #include <linux/list.h> 40 #include <linux/mutex.h> 41 #include <linux/pci.h> 42 #include <linux/slab.h> 43 #include <linux/spinlock.h> 44 #include <linux/string.h> 45 #include <linux/qed/qed_chain.h> 46 #include "qed.h" 47 #include "qed_hsi.h" 48 #include "qed_hw.h" 49 #include "qed_reg_addr.h" 50 #include "qed_sriov.h" 51 52 #define QED_BAR_ACQUIRE_TIMEOUT 1000 53 54 /* Invalid values */ 55 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 56 57 struct qed_ptt { 58 struct list_head list_entry; 59 unsigned int idx; 60 struct pxp_ptt_entry pxp; 61 u8 hwfn_id; 62 }; 63 64 struct qed_ptt_pool { 65 struct list_head free_list; 66 spinlock_t lock; /* ptt synchronized access */ 67 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 68 }; 69 70 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 71 { 72 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 73 int i; 74 75 if (!p_pool) 76 return -ENOMEM; 77 78 INIT_LIST_HEAD(&p_pool->free_list); 79 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 80 p_pool->ptts[i].idx = i; 81 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 82 p_pool->ptts[i].pxp.pretend.control = 0; 83 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 84 if (i >= RESERVED_PTT_MAX) 85 list_add(&p_pool->ptts[i].list_entry, 86 &p_pool->free_list); 87 } 88 89 p_hwfn->p_ptt_pool = p_pool; 90 spin_lock_init(&p_pool->lock); 91 92 return 0; 93 } 94 95 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) 96 { 97 struct qed_ptt *p_ptt; 98 int i; 99 100 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 101 p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; 102 p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; 103 } 104 } 105 106 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 107 { 108 kfree(p_hwfn->p_ptt_pool); 109 p_hwfn->p_ptt_pool = NULL; 110 } 111 112 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 113 { 114 struct qed_ptt *p_ptt; 115 unsigned int i; 116 117 /* Take the free PTT from the list */ 118 for (i = 0; i < QED_BAR_ACQUIRE_TIMEOUT; i++) { 119 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 120 121 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 122 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 123 struct qed_ptt, list_entry); 124 list_del(&p_ptt->list_entry); 125 126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 127 128 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 129 "allocated ptt %d\n", p_ptt->idx); 130 return p_ptt; 131 } 132 133 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 134 usleep_range(1000, 2000); 135 } 136 137 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 138 return NULL; 139 } 140 141 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 142 { 143 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 144 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 145 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 146 } 147 148 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 149 { 150 /* The HW is using DWORDS and we need to translate it to Bytes */ 151 return le32_to_cpu(p_ptt->pxp.offset) << 2; 152 } 153 154 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 155 { 156 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 157 p_ptt->idx * sizeof(struct pxp_ptt_entry); 158 } 159 160 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 161 { 162 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 163 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 164 } 165 166 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 167 struct qed_ptt *p_ptt, u32 new_hw_addr) 168 { 169 u32 prev_hw_addr; 170 171 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 172 173 if (new_hw_addr == prev_hw_addr) 174 return; 175 176 /* Update PTT entery in admin window */ 177 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 178 "Updating PTT entry %d to offset 0x%x\n", 179 p_ptt->idx, new_hw_addr); 180 181 /* The HW is using DWORDS and the address is in Bytes */ 182 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 183 184 REG_WR(p_hwfn, 185 qed_ptt_config_addr(p_ptt) + 186 offsetof(struct pxp_ptt_entry, offset), 187 le32_to_cpu(p_ptt->pxp.offset)); 188 } 189 190 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 191 struct qed_ptt *p_ptt, u32 hw_addr) 192 { 193 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 194 u32 offset; 195 196 offset = hw_addr - win_hw_addr; 197 198 if (p_ptt->hwfn_id != p_hwfn->my_id) 199 DP_NOTICE(p_hwfn, 200 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 201 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 202 203 /* Verify the address is within the window */ 204 if (hw_addr < win_hw_addr || 205 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 206 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 207 offset = 0; 208 } 209 210 return qed_ptt_get_bar_addr(p_ptt) + offset; 211 } 212 213 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 214 enum reserved_ptts ptt_idx) 215 { 216 if (ptt_idx >= RESERVED_PTT_MAX) { 217 DP_NOTICE(p_hwfn, 218 "Requested PTT %d is out of range\n", ptt_idx); 219 return NULL; 220 } 221 222 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 223 } 224 225 void qed_wr(struct qed_hwfn *p_hwfn, 226 struct qed_ptt *p_ptt, 227 u32 hw_addr, u32 val) 228 { 229 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 230 231 REG_WR(p_hwfn, bar_addr, val); 232 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 233 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 234 bar_addr, hw_addr, val); 235 } 236 237 u32 qed_rd(struct qed_hwfn *p_hwfn, 238 struct qed_ptt *p_ptt, 239 u32 hw_addr) 240 { 241 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 242 u32 val = REG_RD(p_hwfn, bar_addr); 243 244 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 245 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 246 bar_addr, hw_addr, val); 247 248 return val; 249 } 250 251 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 252 struct qed_ptt *p_ptt, 253 void *addr, u32 hw_addr, size_t n, bool to_device) 254 { 255 u32 dw_count, *host_addr, hw_offset; 256 size_t quota, done = 0; 257 u32 __iomem *reg_addr; 258 259 while (done < n) { 260 quota = min_t(size_t, n - done, 261 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 262 263 if (IS_PF(p_hwfn->cdev)) { 264 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 265 hw_offset = qed_ptt_get_bar_addr(p_ptt); 266 } else { 267 hw_offset = hw_addr + done; 268 } 269 270 dw_count = quota / 4; 271 host_addr = (u32 *)((u8 *)addr + done); 272 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 273 if (to_device) 274 while (dw_count--) 275 DIRECT_REG_WR(reg_addr++, *host_addr++); 276 else 277 while (dw_count--) 278 *host_addr++ = DIRECT_REG_RD(reg_addr++); 279 280 done += quota; 281 } 282 } 283 284 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 285 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 286 { 287 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 288 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 289 hw_addr, dest, hw_addr, (unsigned long)n); 290 291 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 292 } 293 294 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 295 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 296 { 297 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 298 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 299 hw_addr, hw_addr, src, (unsigned long)n); 300 301 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 302 } 303 304 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 305 { 306 u16 control = 0; 307 308 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 309 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 310 311 /* Every pretend undos previous pretends, including 312 * previous port pretend. 313 */ 314 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 315 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 316 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 317 318 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 319 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 320 321 p_ptt->pxp.pretend.control = cpu_to_le16(control); 322 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 323 324 REG_WR(p_hwfn, 325 qed_ptt_config_addr(p_ptt) + 326 offsetof(struct pxp_ptt_entry, pretend), 327 *(u32 *)&p_ptt->pxp.pretend); 328 } 329 330 void qed_port_pretend(struct qed_hwfn *p_hwfn, 331 struct qed_ptt *p_ptt, u8 port_id) 332 { 333 u16 control = 0; 334 335 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 336 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 337 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 338 339 p_ptt->pxp.pretend.control = cpu_to_le16(control); 340 341 REG_WR(p_hwfn, 342 qed_ptt_config_addr(p_ptt) + 343 offsetof(struct pxp_ptt_entry, pretend), 344 *(u32 *)&p_ptt->pxp.pretend); 345 } 346 347 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 348 { 349 u16 control = 0; 350 351 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 352 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 354 355 p_ptt->pxp.pretend.control = cpu_to_le16(control); 356 357 REG_WR(p_hwfn, 358 qed_ptt_config_addr(p_ptt) + 359 offsetof(struct pxp_ptt_entry, pretend), 360 *(u32 *)&p_ptt->pxp.pretend); 361 } 362 363 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, 364 struct qed_ptt *p_ptt, u8 port_id, u16 fid) 365 { 366 u16 control = 0; 367 368 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 369 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 370 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 371 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 372 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 373 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 374 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 375 p_ptt->pxp.pretend.control = cpu_to_le16(control); 376 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 377 REG_WR(p_hwfn, 378 qed_ptt_config_addr(p_ptt) + 379 offsetof(struct pxp_ptt_entry, pretend), 380 *(u32 *)&p_ptt->pxp.pretend); 381 } 382 383 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 384 { 385 u32 concrete_fid = 0; 386 387 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 388 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 389 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 390 391 return concrete_fid; 392 } 393 394 /* DMAE */ 395 #define QED_DMAE_FLAGS_IS_SET(params, flag) \ 396 ((params) != NULL && ((params)->flags & QED_DMAE_FLAG_##flag)) 397 398 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 399 const u8 is_src_type_grc, 400 const u8 is_dst_type_grc, 401 struct qed_dmae_params *p_params) 402 { 403 u8 src_pfid, dst_pfid, port_id; 404 u16 opcode_b = 0; 405 u32 opcode = 0; 406 407 /* Whether the source is the PCIe or the GRC. 408 * 0- The source is the PCIe 409 * 1- The source is the GRC. 410 */ 411 opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC 412 : DMAE_CMD_SRC_MASK_PCIE) << 413 DMAE_CMD_SRC_SHIFT; 414 src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, PF_SRC) ? 415 p_params->src_pfid : p_hwfn->rel_pf_id; 416 opcode |= ((src_pfid & DMAE_CMD_SRC_PF_ID_MASK) << 417 DMAE_CMD_SRC_PF_ID_SHIFT); 418 419 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 420 opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC 421 : DMAE_CMD_DST_MASK_PCIE) << 422 DMAE_CMD_DST_SHIFT; 423 dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, PF_DST) ? 424 p_params->dst_pfid : p_hwfn->rel_pf_id; 425 opcode |= ((dst_pfid & DMAE_CMD_DST_PF_ID_MASK) << 426 DMAE_CMD_DST_PF_ID_SHIFT); 427 428 /* Whether to write a completion word to the completion destination: 429 * 0-Do not write a completion word 430 * 1-Write the completion word 431 */ 432 opcode |= (DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT); 433 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 434 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 435 436 if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST)) 437 opcode |= (1 << DMAE_CMD_COMP_FUNC_SHIFT); 438 439 opcode |= (DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT); 440 441 port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT)) ? 442 p_params->port_id : p_hwfn->port_id; 443 opcode |= (port_id << DMAE_CMD_PORT_ID_SHIFT); 444 445 /* reset source address in next go */ 446 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 447 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 448 449 /* reset dest address in next go */ 450 opcode |= (DMAE_CMD_DST_ADDR_RESET_MASK << 451 DMAE_CMD_DST_ADDR_RESET_SHIFT); 452 453 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 454 if (QED_DMAE_FLAGS_IS_SET(p_params, VF_SRC)) { 455 opcode |= 1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT; 456 opcode_b |= p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT; 457 } else { 458 opcode_b |= DMAE_CMD_SRC_VF_ID_MASK << 459 DMAE_CMD_SRC_VF_ID_SHIFT; 460 } 461 462 if (QED_DMAE_FLAGS_IS_SET(p_params, VF_DST)) { 463 opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT; 464 opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT; 465 } else { 466 opcode_b |= DMAE_CMD_DST_VF_ID_MASK << DMAE_CMD_DST_VF_ID_SHIFT; 467 } 468 469 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 470 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 471 } 472 473 u32 qed_dmae_idx_to_go_cmd(u8 idx) 474 { 475 /* All the DMAE 'go' registers form an array in internal memory */ 476 return DMAE_REG_GO_C0 + (idx << 2); 477 } 478 479 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 480 struct qed_ptt *p_ptt) 481 { 482 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 483 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 484 int qed_status = 0; 485 486 /* verify address is not NULL */ 487 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 488 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 489 DP_NOTICE(p_hwfn, 490 "source or destination address 0 idx_cmd=%d\n" 491 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 492 idx_cmd, 493 le32_to_cpu(p_command->opcode), 494 le16_to_cpu(p_command->opcode_b), 495 le16_to_cpu(p_command->length_dw), 496 le32_to_cpu(p_command->src_addr_hi), 497 le32_to_cpu(p_command->src_addr_lo), 498 le32_to_cpu(p_command->dst_addr_hi), 499 le32_to_cpu(p_command->dst_addr_lo)); 500 501 return -EINVAL; 502 } 503 504 DP_VERBOSE(p_hwfn, 505 NETIF_MSG_HW, 506 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 507 idx_cmd, 508 le32_to_cpu(p_command->opcode), 509 le16_to_cpu(p_command->opcode_b), 510 le16_to_cpu(p_command->length_dw), 511 le32_to_cpu(p_command->src_addr_hi), 512 le32_to_cpu(p_command->src_addr_lo), 513 le32_to_cpu(p_command->dst_addr_hi), 514 le32_to_cpu(p_command->dst_addr_lo)); 515 516 /* Copy the command to DMAE - need to do it before every call 517 * for source/dest address no reset. 518 * The first 9 DWs are the command registers, the 10 DW is the 519 * GO register, and the rest are result registers 520 * (which are read only by the client). 521 */ 522 for (i = 0; i < DMAE_CMD_SIZE; i++) { 523 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 524 *(((u32 *)p_command) + i) : 0; 525 526 qed_wr(p_hwfn, p_ptt, 527 DMAE_REG_CMD_MEM + 528 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 529 (i * sizeof(u32)), data); 530 } 531 532 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 533 534 return qed_status; 535 } 536 537 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 538 { 539 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 540 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 541 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 542 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 543 544 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 545 sizeof(u32), p_addr, GFP_KERNEL); 546 if (!*p_comp) 547 goto err; 548 549 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 550 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 551 sizeof(struct dmae_cmd), 552 p_addr, GFP_KERNEL); 553 if (!*p_cmd) 554 goto err; 555 556 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 557 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 558 sizeof(u32) * DMAE_MAX_RW_SIZE, 559 p_addr, GFP_KERNEL); 560 if (!*p_buff) 561 goto err; 562 563 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 564 565 return 0; 566 err: 567 qed_dmae_info_free(p_hwfn); 568 return -ENOMEM; 569 } 570 571 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 572 { 573 dma_addr_t p_phys; 574 575 /* Just make sure no one is in the middle */ 576 mutex_lock(&p_hwfn->dmae_info.mutex); 577 578 if (p_hwfn->dmae_info.p_completion_word) { 579 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 580 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 581 sizeof(u32), 582 p_hwfn->dmae_info.p_completion_word, p_phys); 583 p_hwfn->dmae_info.p_completion_word = NULL; 584 } 585 586 if (p_hwfn->dmae_info.p_dmae_cmd) { 587 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 588 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 589 sizeof(struct dmae_cmd), 590 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 591 p_hwfn->dmae_info.p_dmae_cmd = NULL; 592 } 593 594 if (p_hwfn->dmae_info.p_intermediate_buffer) { 595 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 596 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 597 sizeof(u32) * DMAE_MAX_RW_SIZE, 598 p_hwfn->dmae_info.p_intermediate_buffer, 599 p_phys); 600 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 601 } 602 603 mutex_unlock(&p_hwfn->dmae_info.mutex); 604 } 605 606 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 607 { 608 u32 wait_cnt_limit = 10000, wait_cnt = 0; 609 int qed_status = 0; 610 611 barrier(); 612 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 613 udelay(DMAE_MIN_WAIT_TIME); 614 if (++wait_cnt > wait_cnt_limit) { 615 DP_NOTICE(p_hwfn->cdev, 616 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 617 *p_hwfn->dmae_info.p_completion_word, 618 DMAE_COMPLETION_VAL); 619 qed_status = -EBUSY; 620 break; 621 } 622 623 /* to sync the completion_word since we are not 624 * using the volatile keyword for p_completion_word 625 */ 626 barrier(); 627 } 628 629 if (qed_status == 0) 630 *p_hwfn->dmae_info.p_completion_word = 0; 631 632 return qed_status; 633 } 634 635 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 636 struct qed_ptt *p_ptt, 637 u64 src_addr, 638 u64 dst_addr, 639 u8 src_type, 640 u8 dst_type, 641 u32 length_dw) 642 { 643 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 644 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 645 int qed_status = 0; 646 647 switch (src_type) { 648 case QED_DMAE_ADDRESS_GRC: 649 case QED_DMAE_ADDRESS_HOST_PHYS: 650 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 651 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 652 break; 653 /* for virtual source addresses we use the intermediate buffer. */ 654 case QED_DMAE_ADDRESS_HOST_VIRT: 655 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 656 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 657 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 658 (void *)(uintptr_t)src_addr, 659 length_dw * sizeof(u32)); 660 break; 661 default: 662 return -EINVAL; 663 } 664 665 switch (dst_type) { 666 case QED_DMAE_ADDRESS_GRC: 667 case QED_DMAE_ADDRESS_HOST_PHYS: 668 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 669 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 670 break; 671 /* for virtual source addresses we use the intermediate buffer. */ 672 case QED_DMAE_ADDRESS_HOST_VIRT: 673 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 674 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 675 break; 676 default: 677 return -EINVAL; 678 } 679 680 cmd->length_dw = cpu_to_le16((u16)length_dw); 681 682 qed_dmae_post_command(p_hwfn, p_ptt); 683 684 qed_status = qed_dmae_operation_wait(p_hwfn); 685 686 if (qed_status) { 687 DP_NOTICE(p_hwfn, 688 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 689 src_addr, dst_addr, length_dw); 690 return qed_status; 691 } 692 693 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 694 memcpy((void *)(uintptr_t)(dst_addr), 695 &p_hwfn->dmae_info.p_intermediate_buffer[0], 696 length_dw * sizeof(u32)); 697 698 return 0; 699 } 700 701 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 702 struct qed_ptt *p_ptt, 703 u64 src_addr, u64 dst_addr, 704 u8 src_type, u8 dst_type, 705 u32 size_in_dwords, 706 struct qed_dmae_params *p_params) 707 { 708 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 709 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 710 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 711 u64 src_addr_split = 0, dst_addr_split = 0; 712 u16 length_limit = DMAE_MAX_RW_SIZE; 713 int qed_status = 0; 714 u32 offset = 0; 715 716 if (p_hwfn->cdev->recov_in_prog) { 717 DP_VERBOSE(p_hwfn, 718 NETIF_MSG_HW, 719 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n", 720 src_addr, src_type, dst_addr, dst_type, 721 size_in_dwords); 722 723 /* Let the flow complete w/o any error handling */ 724 return 0; 725 } 726 727 qed_dmae_opcode(p_hwfn, 728 (src_type == QED_DMAE_ADDRESS_GRC), 729 (dst_type == QED_DMAE_ADDRESS_GRC), 730 p_params); 731 732 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 733 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 734 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 735 736 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 737 cnt_split = size_in_dwords / length_limit; 738 length_mod = size_in_dwords % length_limit; 739 740 src_addr_split = src_addr; 741 dst_addr_split = dst_addr; 742 743 for (i = 0; i <= cnt_split; i++) { 744 offset = length_limit * i; 745 746 if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) { 747 if (src_type == QED_DMAE_ADDRESS_GRC) 748 src_addr_split = src_addr + offset; 749 else 750 src_addr_split = src_addr + (offset * 4); 751 } 752 753 if (dst_type == QED_DMAE_ADDRESS_GRC) 754 dst_addr_split = dst_addr + offset; 755 else 756 dst_addr_split = dst_addr + (offset * 4); 757 758 length_cur = (cnt_split == i) ? length_mod : length_limit; 759 760 /* might be zero on last iteration */ 761 if (!length_cur) 762 continue; 763 764 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 765 p_ptt, 766 src_addr_split, 767 dst_addr_split, 768 src_type, 769 dst_type, 770 length_cur); 771 if (qed_status) { 772 DP_NOTICE(p_hwfn, 773 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 774 qed_status, src_addr, dst_addr, length_cur); 775 break; 776 } 777 } 778 779 return qed_status; 780 } 781 782 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 783 struct qed_ptt *p_ptt, 784 u64 source_addr, u32 grc_addr, u32 size_in_dwords, 785 struct qed_dmae_params *p_params) 786 { 787 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 788 int rc; 789 790 791 mutex_lock(&p_hwfn->dmae_info.mutex); 792 793 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 794 grc_addr_in_dw, 795 QED_DMAE_ADDRESS_HOST_VIRT, 796 QED_DMAE_ADDRESS_GRC, 797 size_in_dwords, p_params); 798 799 mutex_unlock(&p_hwfn->dmae_info.mutex); 800 801 return rc; 802 } 803 804 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 805 struct qed_ptt *p_ptt, 806 u32 grc_addr, 807 dma_addr_t dest_addr, u32 size_in_dwords, 808 struct qed_dmae_params *p_params) 809 { 810 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 811 int rc; 812 813 814 mutex_lock(&p_hwfn->dmae_info.mutex); 815 816 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 817 dest_addr, QED_DMAE_ADDRESS_GRC, 818 QED_DMAE_ADDRESS_HOST_VIRT, 819 size_in_dwords, p_params); 820 821 mutex_unlock(&p_hwfn->dmae_info.mutex); 822 823 return rc; 824 } 825 826 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 827 struct qed_ptt *p_ptt, 828 dma_addr_t source_addr, 829 dma_addr_t dest_addr, 830 u32 size_in_dwords, struct qed_dmae_params *p_params) 831 { 832 int rc; 833 834 mutex_lock(&(p_hwfn->dmae_info.mutex)); 835 836 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 837 dest_addr, 838 QED_DMAE_ADDRESS_HOST_PHYS, 839 QED_DMAE_ADDRESS_HOST_PHYS, 840 size_in_dwords, p_params); 841 842 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 843 844 return rc; 845 } 846 847 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 848 struct qed_ptt *p_ptt, const char *phase) 849 { 850 u32 size = PAGE_SIZE / 2, val; 851 int rc = 0; 852 dma_addr_t p_phys; 853 void *p_virt; 854 u32 *p_tmp; 855 856 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 857 2 * size, &p_phys, GFP_KERNEL); 858 if (!p_virt) { 859 DP_NOTICE(p_hwfn, 860 "DMAE sanity [%s]: failed to allocate memory\n", 861 phase); 862 return -ENOMEM; 863 } 864 865 /* Fill the bottom half of the allocated memory with a known pattern */ 866 for (p_tmp = (u32 *)p_virt; 867 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 868 /* Save the address itself as the value */ 869 val = (u32)(uintptr_t)p_tmp; 870 *p_tmp = val; 871 } 872 873 /* Zero the top half of the allocated memory */ 874 memset((u8 *)p_virt + size, 0, size); 875 876 DP_VERBOSE(p_hwfn, 877 QED_MSG_SP, 878 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 879 phase, 880 (u64)p_phys, 881 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 882 883 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 884 size / 4, NULL); 885 if (rc) { 886 DP_NOTICE(p_hwfn, 887 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 888 phase, rc); 889 goto out; 890 } 891 892 /* Verify that the top half of the allocated memory has the pattern */ 893 for (p_tmp = (u32 *)((u8 *)p_virt + size); 894 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 895 /* The corresponding address in the bottom half */ 896 val = (u32)(uintptr_t)p_tmp - size; 897 898 if (*p_tmp != val) { 899 DP_NOTICE(p_hwfn, 900 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 901 phase, 902 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 903 p_tmp, *p_tmp, val); 904 rc = -EINVAL; 905 goto out; 906 } 907 } 908 909 out: 910 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 911 return rc; 912 } 913