1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <linux/io.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/kernel.h> 39 #include <linux/list.h> 40 #include <linux/mutex.h> 41 #include <linux/pci.h> 42 #include <linux/slab.h> 43 #include <linux/spinlock.h> 44 #include <linux/string.h> 45 #include <linux/qed/qed_chain.h> 46 #include "qed.h" 47 #include "qed_hsi.h" 48 #include "qed_hw.h" 49 #include "qed_reg_addr.h" 50 #include "qed_sriov.h" 51 52 #define QED_BAR_ACQUIRE_TIMEOUT 1000 53 54 /* Invalid values */ 55 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 56 57 struct qed_ptt { 58 struct list_head list_entry; 59 unsigned int idx; 60 struct pxp_ptt_entry pxp; 61 u8 hwfn_id; 62 }; 63 64 struct qed_ptt_pool { 65 struct list_head free_list; 66 spinlock_t lock; /* ptt synchronized access */ 67 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 68 }; 69 70 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 71 { 72 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 73 int i; 74 75 if (!p_pool) 76 return -ENOMEM; 77 78 INIT_LIST_HEAD(&p_pool->free_list); 79 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 80 p_pool->ptts[i].idx = i; 81 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 82 p_pool->ptts[i].pxp.pretend.control = 0; 83 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 84 if (i >= RESERVED_PTT_MAX) 85 list_add(&p_pool->ptts[i].list_entry, 86 &p_pool->free_list); 87 } 88 89 p_hwfn->p_ptt_pool = p_pool; 90 spin_lock_init(&p_pool->lock); 91 92 return 0; 93 } 94 95 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) 96 { 97 struct qed_ptt *p_ptt; 98 int i; 99 100 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 101 p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; 102 p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; 103 } 104 } 105 106 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 107 { 108 kfree(p_hwfn->p_ptt_pool); 109 p_hwfn->p_ptt_pool = NULL; 110 } 111 112 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 113 { 114 struct qed_ptt *p_ptt; 115 unsigned int i; 116 117 /* Take the free PTT from the list */ 118 for (i = 0; i < QED_BAR_ACQUIRE_TIMEOUT; i++) { 119 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 120 121 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 122 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 123 struct qed_ptt, list_entry); 124 list_del(&p_ptt->list_entry); 125 126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 127 128 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 129 "allocated ptt %d\n", p_ptt->idx); 130 return p_ptt; 131 } 132 133 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 134 usleep_range(1000, 2000); 135 } 136 137 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 138 return NULL; 139 } 140 141 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 142 { 143 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 144 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 145 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 146 } 147 148 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 149 { 150 /* The HW is using DWORDS and we need to translate it to Bytes */ 151 return le32_to_cpu(p_ptt->pxp.offset) << 2; 152 } 153 154 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 155 { 156 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 157 p_ptt->idx * sizeof(struct pxp_ptt_entry); 158 } 159 160 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 161 { 162 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 163 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 164 } 165 166 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 167 struct qed_ptt *p_ptt, u32 new_hw_addr) 168 { 169 u32 prev_hw_addr; 170 171 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 172 173 if (new_hw_addr == prev_hw_addr) 174 return; 175 176 /* Update PTT entery in admin window */ 177 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 178 "Updating PTT entry %d to offset 0x%x\n", 179 p_ptt->idx, new_hw_addr); 180 181 /* The HW is using DWORDS and the address is in Bytes */ 182 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 183 184 REG_WR(p_hwfn, 185 qed_ptt_config_addr(p_ptt) + 186 offsetof(struct pxp_ptt_entry, offset), 187 le32_to_cpu(p_ptt->pxp.offset)); 188 } 189 190 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 191 struct qed_ptt *p_ptt, u32 hw_addr) 192 { 193 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 194 u32 offset; 195 196 offset = hw_addr - win_hw_addr; 197 198 if (p_ptt->hwfn_id != p_hwfn->my_id) 199 DP_NOTICE(p_hwfn, 200 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 201 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 202 203 /* Verify the address is within the window */ 204 if (hw_addr < win_hw_addr || 205 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 206 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 207 offset = 0; 208 } 209 210 return qed_ptt_get_bar_addr(p_ptt) + offset; 211 } 212 213 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 214 enum reserved_ptts ptt_idx) 215 { 216 if (ptt_idx >= RESERVED_PTT_MAX) { 217 DP_NOTICE(p_hwfn, 218 "Requested PTT %d is out of range\n", ptt_idx); 219 return NULL; 220 } 221 222 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 223 } 224 225 void qed_wr(struct qed_hwfn *p_hwfn, 226 struct qed_ptt *p_ptt, 227 u32 hw_addr, u32 val) 228 { 229 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 230 231 REG_WR(p_hwfn, bar_addr, val); 232 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 233 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 234 bar_addr, hw_addr, val); 235 } 236 237 u32 qed_rd(struct qed_hwfn *p_hwfn, 238 struct qed_ptt *p_ptt, 239 u32 hw_addr) 240 { 241 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 242 u32 val = REG_RD(p_hwfn, bar_addr); 243 244 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 245 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 246 bar_addr, hw_addr, val); 247 248 return val; 249 } 250 251 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 252 struct qed_ptt *p_ptt, 253 void *addr, u32 hw_addr, size_t n, bool to_device) 254 { 255 u32 dw_count, *host_addr, hw_offset; 256 size_t quota, done = 0; 257 u32 __iomem *reg_addr; 258 259 while (done < n) { 260 quota = min_t(size_t, n - done, 261 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 262 263 if (IS_PF(p_hwfn->cdev)) { 264 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 265 hw_offset = qed_ptt_get_bar_addr(p_ptt); 266 } else { 267 hw_offset = hw_addr + done; 268 } 269 270 dw_count = quota / 4; 271 host_addr = (u32 *)((u8 *)addr + done); 272 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 273 if (to_device) 274 while (dw_count--) 275 DIRECT_REG_WR(reg_addr++, *host_addr++); 276 else 277 while (dw_count--) 278 *host_addr++ = DIRECT_REG_RD(reg_addr++); 279 280 done += quota; 281 } 282 } 283 284 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 285 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 286 { 287 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 288 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 289 hw_addr, dest, hw_addr, (unsigned long)n); 290 291 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 292 } 293 294 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 295 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 296 { 297 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 298 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 299 hw_addr, hw_addr, src, (unsigned long)n); 300 301 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 302 } 303 304 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 305 { 306 u16 control = 0; 307 308 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 309 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 310 311 /* Every pretend undos previous pretends, including 312 * previous port pretend. 313 */ 314 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 315 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 316 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 317 318 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 319 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 320 321 p_ptt->pxp.pretend.control = cpu_to_le16(control); 322 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 323 324 REG_WR(p_hwfn, 325 qed_ptt_config_addr(p_ptt) + 326 offsetof(struct pxp_ptt_entry, pretend), 327 *(u32 *)&p_ptt->pxp.pretend); 328 } 329 330 void qed_port_pretend(struct qed_hwfn *p_hwfn, 331 struct qed_ptt *p_ptt, u8 port_id) 332 { 333 u16 control = 0; 334 335 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 336 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 337 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 338 339 p_ptt->pxp.pretend.control = cpu_to_le16(control); 340 341 REG_WR(p_hwfn, 342 qed_ptt_config_addr(p_ptt) + 343 offsetof(struct pxp_ptt_entry, pretend), 344 *(u32 *)&p_ptt->pxp.pretend); 345 } 346 347 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 348 { 349 u16 control = 0; 350 351 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 352 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 354 355 p_ptt->pxp.pretend.control = cpu_to_le16(control); 356 357 REG_WR(p_hwfn, 358 qed_ptt_config_addr(p_ptt) + 359 offsetof(struct pxp_ptt_entry, pretend), 360 *(u32 *)&p_ptt->pxp.pretend); 361 } 362 363 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, 364 struct qed_ptt *p_ptt, u8 port_id, u16 fid) 365 { 366 u16 control = 0; 367 368 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 369 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 370 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 371 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 372 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 373 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 374 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 375 p_ptt->pxp.pretend.control = cpu_to_le16(control); 376 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 377 REG_WR(p_hwfn, 378 qed_ptt_config_addr(p_ptt) + 379 offsetof(struct pxp_ptt_entry, pretend), 380 *(u32 *)&p_ptt->pxp.pretend); 381 } 382 383 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 384 { 385 u32 concrete_fid = 0; 386 387 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 388 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 389 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 390 391 return concrete_fid; 392 } 393 394 /* DMAE */ 395 #define QED_DMAE_FLAGS_IS_SET(params, flag) \ 396 ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag)) 397 398 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 399 const u8 is_src_type_grc, 400 const u8 is_dst_type_grc, 401 struct qed_dmae_params *p_params) 402 { 403 u8 src_pfid, dst_pfid, port_id; 404 u16 opcode_b = 0; 405 u32 opcode = 0; 406 407 /* Whether the source is the PCIe or the GRC. 408 * 0- The source is the PCIe 409 * 1- The source is the GRC. 410 */ 411 SET_FIELD(opcode, DMAE_CMD_SRC, 412 (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie)); 413 src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ? 414 p_params->src_pfid : p_hwfn->rel_pf_id; 415 SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid); 416 417 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 418 SET_FIELD(opcode, DMAE_CMD_DST, 419 (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie)); 420 dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ? 421 p_params->dst_pfid : p_hwfn->rel_pf_id; 422 SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid); 423 424 425 /* Whether to write a completion word to the completion destination: 426 * 0-Do not write a completion word 427 * 1-Write the completion word 428 */ 429 SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1); 430 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 431 432 if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST)) 433 SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1); 434 435 /* swapping mode 3 - big endian */ 436 SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY); 437 438 port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ? 439 p_params->port_id : p_hwfn->port_id; 440 SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id); 441 442 /* reset source address in next go */ 443 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 444 445 /* reset dest address in next go */ 446 SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1); 447 448 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 449 if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) { 450 SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1); 451 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid); 452 } else { 453 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF); 454 } 455 if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) { 456 SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1); 457 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid); 458 } else { 459 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF); 460 } 461 462 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 463 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 464 } 465 466 u32 qed_dmae_idx_to_go_cmd(u8 idx) 467 { 468 /* All the DMAE 'go' registers form an array in internal memory */ 469 return DMAE_REG_GO_C0 + (idx << 2); 470 } 471 472 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 473 struct qed_ptt *p_ptt) 474 { 475 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 476 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 477 int qed_status = 0; 478 479 /* verify address is not NULL */ 480 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 481 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 482 DP_NOTICE(p_hwfn, 483 "source or destination address 0 idx_cmd=%d\n" 484 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 485 idx_cmd, 486 le32_to_cpu(p_command->opcode), 487 le16_to_cpu(p_command->opcode_b), 488 le16_to_cpu(p_command->length_dw), 489 le32_to_cpu(p_command->src_addr_hi), 490 le32_to_cpu(p_command->src_addr_lo), 491 le32_to_cpu(p_command->dst_addr_hi), 492 le32_to_cpu(p_command->dst_addr_lo)); 493 494 return -EINVAL; 495 } 496 497 DP_VERBOSE(p_hwfn, 498 NETIF_MSG_HW, 499 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 500 idx_cmd, 501 le32_to_cpu(p_command->opcode), 502 le16_to_cpu(p_command->opcode_b), 503 le16_to_cpu(p_command->length_dw), 504 le32_to_cpu(p_command->src_addr_hi), 505 le32_to_cpu(p_command->src_addr_lo), 506 le32_to_cpu(p_command->dst_addr_hi), 507 le32_to_cpu(p_command->dst_addr_lo)); 508 509 /* Copy the command to DMAE - need to do it before every call 510 * for source/dest address no reset. 511 * The first 9 DWs are the command registers, the 10 DW is the 512 * GO register, and the rest are result registers 513 * (which are read only by the client). 514 */ 515 for (i = 0; i < DMAE_CMD_SIZE; i++) { 516 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 517 *(((u32 *)p_command) + i) : 0; 518 519 qed_wr(p_hwfn, p_ptt, 520 DMAE_REG_CMD_MEM + 521 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 522 (i * sizeof(u32)), data); 523 } 524 525 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 526 527 return qed_status; 528 } 529 530 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 531 { 532 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 533 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 534 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 535 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 536 537 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 538 sizeof(u32), p_addr, GFP_KERNEL); 539 if (!*p_comp) 540 goto err; 541 542 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 543 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 544 sizeof(struct dmae_cmd), 545 p_addr, GFP_KERNEL); 546 if (!*p_cmd) 547 goto err; 548 549 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 550 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 551 sizeof(u32) * DMAE_MAX_RW_SIZE, 552 p_addr, GFP_KERNEL); 553 if (!*p_buff) 554 goto err; 555 556 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 557 558 return 0; 559 err: 560 qed_dmae_info_free(p_hwfn); 561 return -ENOMEM; 562 } 563 564 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 565 { 566 dma_addr_t p_phys; 567 568 /* Just make sure no one is in the middle */ 569 mutex_lock(&p_hwfn->dmae_info.mutex); 570 571 if (p_hwfn->dmae_info.p_completion_word) { 572 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 573 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 574 sizeof(u32), 575 p_hwfn->dmae_info.p_completion_word, p_phys); 576 p_hwfn->dmae_info.p_completion_word = NULL; 577 } 578 579 if (p_hwfn->dmae_info.p_dmae_cmd) { 580 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 581 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 582 sizeof(struct dmae_cmd), 583 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 584 p_hwfn->dmae_info.p_dmae_cmd = NULL; 585 } 586 587 if (p_hwfn->dmae_info.p_intermediate_buffer) { 588 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 589 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 590 sizeof(u32) * DMAE_MAX_RW_SIZE, 591 p_hwfn->dmae_info.p_intermediate_buffer, 592 p_phys); 593 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 594 } 595 596 mutex_unlock(&p_hwfn->dmae_info.mutex); 597 } 598 599 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 600 { 601 u32 wait_cnt_limit = 10000, wait_cnt = 0; 602 int qed_status = 0; 603 604 barrier(); 605 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 606 udelay(DMAE_MIN_WAIT_TIME); 607 if (++wait_cnt > wait_cnt_limit) { 608 DP_NOTICE(p_hwfn->cdev, 609 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 610 *p_hwfn->dmae_info.p_completion_word, 611 DMAE_COMPLETION_VAL); 612 qed_status = -EBUSY; 613 break; 614 } 615 616 /* to sync the completion_word since we are not 617 * using the volatile keyword for p_completion_word 618 */ 619 barrier(); 620 } 621 622 if (qed_status == 0) 623 *p_hwfn->dmae_info.p_completion_word = 0; 624 625 return qed_status; 626 } 627 628 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 629 struct qed_ptt *p_ptt, 630 u64 src_addr, 631 u64 dst_addr, 632 u8 src_type, 633 u8 dst_type, 634 u32 length_dw) 635 { 636 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 637 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 638 int qed_status = 0; 639 640 switch (src_type) { 641 case QED_DMAE_ADDRESS_GRC: 642 case QED_DMAE_ADDRESS_HOST_PHYS: 643 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 644 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 645 break; 646 /* for virtual source addresses we use the intermediate buffer. */ 647 case QED_DMAE_ADDRESS_HOST_VIRT: 648 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 649 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 650 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 651 (void *)(uintptr_t)src_addr, 652 length_dw * sizeof(u32)); 653 break; 654 default: 655 return -EINVAL; 656 } 657 658 switch (dst_type) { 659 case QED_DMAE_ADDRESS_GRC: 660 case QED_DMAE_ADDRESS_HOST_PHYS: 661 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 662 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 663 break; 664 /* for virtual source addresses we use the intermediate buffer. */ 665 case QED_DMAE_ADDRESS_HOST_VIRT: 666 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 667 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 668 break; 669 default: 670 return -EINVAL; 671 } 672 673 cmd->length_dw = cpu_to_le16((u16)length_dw); 674 675 qed_dmae_post_command(p_hwfn, p_ptt); 676 677 qed_status = qed_dmae_operation_wait(p_hwfn); 678 679 if (qed_status) { 680 DP_NOTICE(p_hwfn, 681 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 682 src_addr, dst_addr, length_dw); 683 return qed_status; 684 } 685 686 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 687 memcpy((void *)(uintptr_t)(dst_addr), 688 &p_hwfn->dmae_info.p_intermediate_buffer[0], 689 length_dw * sizeof(u32)); 690 691 return 0; 692 } 693 694 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 695 struct qed_ptt *p_ptt, 696 u64 src_addr, u64 dst_addr, 697 u8 src_type, u8 dst_type, 698 u32 size_in_dwords, 699 struct qed_dmae_params *p_params) 700 { 701 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 702 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 703 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 704 u64 src_addr_split = 0, dst_addr_split = 0; 705 u16 length_limit = DMAE_MAX_RW_SIZE; 706 int qed_status = 0; 707 u32 offset = 0; 708 709 if (p_hwfn->cdev->recov_in_prog) { 710 DP_VERBOSE(p_hwfn, 711 NETIF_MSG_HW, 712 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n", 713 src_addr, src_type, dst_addr, dst_type, 714 size_in_dwords); 715 716 /* Let the flow complete w/o any error handling */ 717 return 0; 718 } 719 720 qed_dmae_opcode(p_hwfn, 721 (src_type == QED_DMAE_ADDRESS_GRC), 722 (dst_type == QED_DMAE_ADDRESS_GRC), 723 p_params); 724 725 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 726 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 727 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 728 729 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 730 cnt_split = size_in_dwords / length_limit; 731 length_mod = size_in_dwords % length_limit; 732 733 src_addr_split = src_addr; 734 dst_addr_split = dst_addr; 735 736 for (i = 0; i <= cnt_split; i++) { 737 offset = length_limit * i; 738 739 if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) { 740 if (src_type == QED_DMAE_ADDRESS_GRC) 741 src_addr_split = src_addr + offset; 742 else 743 src_addr_split = src_addr + (offset * 4); 744 } 745 746 if (dst_type == QED_DMAE_ADDRESS_GRC) 747 dst_addr_split = dst_addr + offset; 748 else 749 dst_addr_split = dst_addr + (offset * 4); 750 751 length_cur = (cnt_split == i) ? length_mod : length_limit; 752 753 /* might be zero on last iteration */ 754 if (!length_cur) 755 continue; 756 757 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 758 p_ptt, 759 src_addr_split, 760 dst_addr_split, 761 src_type, 762 dst_type, 763 length_cur); 764 if (qed_status) { 765 DP_NOTICE(p_hwfn, 766 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 767 qed_status, src_addr, dst_addr, length_cur); 768 break; 769 } 770 } 771 772 return qed_status; 773 } 774 775 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 776 struct qed_ptt *p_ptt, 777 u64 source_addr, u32 grc_addr, u32 size_in_dwords, 778 struct qed_dmae_params *p_params) 779 { 780 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 781 int rc; 782 783 784 mutex_lock(&p_hwfn->dmae_info.mutex); 785 786 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 787 grc_addr_in_dw, 788 QED_DMAE_ADDRESS_HOST_VIRT, 789 QED_DMAE_ADDRESS_GRC, 790 size_in_dwords, p_params); 791 792 mutex_unlock(&p_hwfn->dmae_info.mutex); 793 794 return rc; 795 } 796 797 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 798 struct qed_ptt *p_ptt, 799 u32 grc_addr, 800 dma_addr_t dest_addr, u32 size_in_dwords, 801 struct qed_dmae_params *p_params) 802 { 803 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 804 int rc; 805 806 807 mutex_lock(&p_hwfn->dmae_info.mutex); 808 809 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 810 dest_addr, QED_DMAE_ADDRESS_GRC, 811 QED_DMAE_ADDRESS_HOST_VIRT, 812 size_in_dwords, p_params); 813 814 mutex_unlock(&p_hwfn->dmae_info.mutex); 815 816 return rc; 817 } 818 819 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 820 struct qed_ptt *p_ptt, 821 dma_addr_t source_addr, 822 dma_addr_t dest_addr, 823 u32 size_in_dwords, struct qed_dmae_params *p_params) 824 { 825 int rc; 826 827 mutex_lock(&(p_hwfn->dmae_info.mutex)); 828 829 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 830 dest_addr, 831 QED_DMAE_ADDRESS_HOST_PHYS, 832 QED_DMAE_ADDRESS_HOST_PHYS, 833 size_in_dwords, p_params); 834 835 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 836 837 return rc; 838 } 839 840 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 841 struct qed_ptt *p_ptt, const char *phase) 842 { 843 u32 size = PAGE_SIZE / 2, val; 844 int rc = 0; 845 dma_addr_t p_phys; 846 void *p_virt; 847 u32 *p_tmp; 848 849 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 850 2 * size, &p_phys, GFP_KERNEL); 851 if (!p_virt) { 852 DP_NOTICE(p_hwfn, 853 "DMAE sanity [%s]: failed to allocate memory\n", 854 phase); 855 return -ENOMEM; 856 } 857 858 /* Fill the bottom half of the allocated memory with a known pattern */ 859 for (p_tmp = (u32 *)p_virt; 860 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 861 /* Save the address itself as the value */ 862 val = (u32)(uintptr_t)p_tmp; 863 *p_tmp = val; 864 } 865 866 /* Zero the top half of the allocated memory */ 867 memset((u8 *)p_virt + size, 0, size); 868 869 DP_VERBOSE(p_hwfn, 870 QED_MSG_SP, 871 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 872 phase, 873 (u64)p_phys, 874 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 875 876 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 877 size / 4, NULL); 878 if (rc) { 879 DP_NOTICE(p_hwfn, 880 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 881 phase, rc); 882 goto out; 883 } 884 885 /* Verify that the top half of the allocated memory has the pattern */ 886 for (p_tmp = (u32 *)((u8 *)p_virt + size); 887 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 888 /* The corresponding address in the bottom half */ 889 val = (u32)(uintptr_t)p_tmp - size; 890 891 if (*p_tmp != val) { 892 DP_NOTICE(p_hwfn, 893 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 894 phase, 895 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 896 p_tmp, *p_tmp, val); 897 rc = -EINVAL; 898 goto out; 899 } 900 } 901 902 out: 903 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 904 return rc; 905 } 906