1 /* 2 * Copyright (C) 2016-2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #define pr_fmt(fmt) "NFP net bpf: " fmt 35 36 #include <linux/bug.h> 37 #include <linux/kernel.h> 38 #include <linux/bpf.h> 39 #include <linux/filter.h> 40 #include <linux/pkt_cls.h> 41 #include <linux/unistd.h> 42 43 #include "main.h" 44 #include "../nfp_asm.h" 45 46 /* --- NFP prog --- */ 47 /* Foreach "multiple" entries macros provide pos and next<n> pointers. 48 * It's safe to modify the next pointers (but not pos). 49 */ 50 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ 51 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 52 next = list_next_entry(pos, l); \ 53 &(nfp_prog)->insns != &pos->l && \ 54 &(nfp_prog)->insns != &next->l; \ 55 pos = nfp_meta_next(pos), \ 56 next = nfp_meta_next(pos)) 57 58 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ 59 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 60 next = list_next_entry(pos, l), \ 61 next2 = list_next_entry(next, l); \ 62 &(nfp_prog)->insns != &pos->l && \ 63 &(nfp_prog)->insns != &next->l && \ 64 &(nfp_prog)->insns != &next2->l; \ 65 pos = nfp_meta_next(pos), \ 66 next = nfp_meta_next(pos), \ 67 next2 = nfp_meta_next(next)) 68 69 static bool 70 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 71 { 72 return meta->l.prev != &nfp_prog->insns; 73 } 74 75 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) 76 { 77 if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) { 78 nfp_prog->error = -ENOSPC; 79 return; 80 } 81 82 nfp_prog->prog[nfp_prog->prog_len] = insn; 83 nfp_prog->prog_len++; 84 } 85 86 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) 87 { 88 return nfp_prog->prog_len; 89 } 90 91 static bool 92 nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off) 93 { 94 /* If there is a recorded error we may have dropped instructions; 95 * that doesn't have to be due to translator bug, and the translation 96 * will fail anyway, so just return OK. 97 */ 98 if (nfp_prog->error) 99 return true; 100 return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off); 101 } 102 103 /* --- Emitters --- */ 104 static void 105 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, 106 u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync, bool indir) 107 { 108 enum cmd_ctx_swap ctx; 109 u64 insn; 110 111 if (sync) 112 ctx = CMD_CTX_SWAP; 113 else 114 ctx = CMD_CTX_NO_SWAP; 115 116 insn = FIELD_PREP(OP_CMD_A_SRC, areg) | 117 FIELD_PREP(OP_CMD_CTX, ctx) | 118 FIELD_PREP(OP_CMD_B_SRC, breg) | 119 FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) | 120 FIELD_PREP(OP_CMD_XFER, xfer) | 121 FIELD_PREP(OP_CMD_CNT, size) | 122 FIELD_PREP(OP_CMD_SIG, sync) | 123 FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) | 124 FIELD_PREP(OP_CMD_INDIR, indir) | 125 FIELD_PREP(OP_CMD_MODE, mode); 126 127 nfp_prog_push(nfp_prog, insn); 128 } 129 130 static void 131 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 132 swreg lreg, swreg rreg, u8 size, bool sync, bool indir) 133 { 134 struct nfp_insn_re_regs reg; 135 int err; 136 137 err = swreg_to_restricted(reg_none(), lreg, rreg, ®, false); 138 if (err) { 139 nfp_prog->error = err; 140 return; 141 } 142 if (reg.swap) { 143 pr_err("cmd can't swap arguments\n"); 144 nfp_prog->error = -EFAULT; 145 return; 146 } 147 if (reg.dst_lmextn || reg.src_lmextn) { 148 pr_err("cmd can't use LMextn\n"); 149 nfp_prog->error = -EFAULT; 150 return; 151 } 152 153 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync, 154 indir); 155 } 156 157 static void 158 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 159 swreg lreg, swreg rreg, u8 size, bool sync) 160 { 161 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, false); 162 } 163 164 static void 165 emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 166 swreg lreg, swreg rreg, u8 size, bool sync) 167 { 168 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, true); 169 } 170 171 static void 172 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, 173 enum br_ctx_signal_state css, u16 addr, u8 defer) 174 { 175 u16 addr_lo, addr_hi; 176 u64 insn; 177 178 addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 179 addr_hi = addr != addr_lo; 180 181 insn = OP_BR_BASE | 182 FIELD_PREP(OP_BR_MASK, mask) | 183 FIELD_PREP(OP_BR_EV_PIP, ev_pip) | 184 FIELD_PREP(OP_BR_CSS, css) | 185 FIELD_PREP(OP_BR_DEFBR, defer) | 186 FIELD_PREP(OP_BR_ADDR_LO, addr_lo) | 187 FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 188 189 nfp_prog_push(nfp_prog, insn); 190 } 191 192 static void 193 emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer, 194 enum nfp_relo_type relo) 195 { 196 if (mask == BR_UNC && defer > 2) { 197 pr_err("BUG: branch defer out of bounds %d\n", defer); 198 nfp_prog->error = -EFAULT; 199 return; 200 } 201 202 __emit_br(nfp_prog, mask, 203 mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND, 204 BR_CSS_NONE, addr, defer); 205 206 nfp_prog->prog[nfp_prog->prog_len - 1] |= 207 FIELD_PREP(OP_RELO_TYPE, relo); 208 } 209 210 static void 211 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) 212 { 213 emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL); 214 } 215 216 static void 217 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, 218 enum immed_width width, bool invert, 219 enum immed_shift shift, bool wr_both, 220 bool dst_lmextn, bool src_lmextn) 221 { 222 u64 insn; 223 224 insn = OP_IMMED_BASE | 225 FIELD_PREP(OP_IMMED_A_SRC, areg) | 226 FIELD_PREP(OP_IMMED_B_SRC, breg) | 227 FIELD_PREP(OP_IMMED_IMM, imm_hi) | 228 FIELD_PREP(OP_IMMED_WIDTH, width) | 229 FIELD_PREP(OP_IMMED_INV, invert) | 230 FIELD_PREP(OP_IMMED_SHIFT, shift) | 231 FIELD_PREP(OP_IMMED_WR_AB, wr_both) | 232 FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) | 233 FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn); 234 235 nfp_prog_push(nfp_prog, insn); 236 } 237 238 static void 239 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, 240 enum immed_width width, bool invert, enum immed_shift shift) 241 { 242 struct nfp_insn_ur_regs reg; 243 int err; 244 245 if (swreg_type(dst) == NN_REG_IMM) { 246 nfp_prog->error = -EFAULT; 247 return; 248 } 249 250 err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), ®); 251 if (err) { 252 nfp_prog->error = err; 253 return; 254 } 255 256 /* Use reg.dst when destination is No-Dest. */ 257 __emit_immed(nfp_prog, 258 swreg_type(dst) == NN_REG_NONE ? reg.dst : reg.areg, 259 reg.breg, imm >> 8, width, invert, shift, 260 reg.wr_both, reg.dst_lmextn, reg.src_lmextn); 261 } 262 263 static void 264 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 265 enum shf_sc sc, u8 shift, 266 u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both, 267 bool dst_lmextn, bool src_lmextn) 268 { 269 u64 insn; 270 271 if (!FIELD_FIT(OP_SHF_SHIFT, shift)) { 272 nfp_prog->error = -EFAULT; 273 return; 274 } 275 276 if (sc == SHF_SC_L_SHF) 277 shift = 32 - shift; 278 279 insn = OP_SHF_BASE | 280 FIELD_PREP(OP_SHF_A_SRC, areg) | 281 FIELD_PREP(OP_SHF_SC, sc) | 282 FIELD_PREP(OP_SHF_B_SRC, breg) | 283 FIELD_PREP(OP_SHF_I8, i8) | 284 FIELD_PREP(OP_SHF_SW, sw) | 285 FIELD_PREP(OP_SHF_DST, dst) | 286 FIELD_PREP(OP_SHF_SHIFT, shift) | 287 FIELD_PREP(OP_SHF_OP, op) | 288 FIELD_PREP(OP_SHF_DST_AB, dst_ab) | 289 FIELD_PREP(OP_SHF_WR_AB, wr_both) | 290 FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) | 291 FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn); 292 293 nfp_prog_push(nfp_prog, insn); 294 } 295 296 static void 297 emit_shf(struct nfp_prog *nfp_prog, swreg dst, 298 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift) 299 { 300 struct nfp_insn_re_regs reg; 301 int err; 302 303 err = swreg_to_restricted(dst, lreg, rreg, ®, true); 304 if (err) { 305 nfp_prog->error = err; 306 return; 307 } 308 309 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, 310 reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both, 311 reg.dst_lmextn, reg.src_lmextn); 312 } 313 314 static void 315 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 316 u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both, 317 bool dst_lmextn, bool src_lmextn) 318 { 319 u64 insn; 320 321 insn = OP_ALU_BASE | 322 FIELD_PREP(OP_ALU_A_SRC, areg) | 323 FIELD_PREP(OP_ALU_B_SRC, breg) | 324 FIELD_PREP(OP_ALU_DST, dst) | 325 FIELD_PREP(OP_ALU_SW, swap) | 326 FIELD_PREP(OP_ALU_OP, op) | 327 FIELD_PREP(OP_ALU_DST_AB, dst_ab) | 328 FIELD_PREP(OP_ALU_WR_AB, wr_both) | 329 FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) | 330 FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn); 331 332 nfp_prog_push(nfp_prog, insn); 333 } 334 335 static void 336 emit_alu(struct nfp_prog *nfp_prog, swreg dst, 337 swreg lreg, enum alu_op op, swreg rreg) 338 { 339 struct nfp_insn_ur_regs reg; 340 int err; 341 342 err = swreg_to_unrestricted(dst, lreg, rreg, ®); 343 if (err) { 344 nfp_prog->error = err; 345 return; 346 } 347 348 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, 349 reg.areg, op, reg.breg, reg.swap, reg.wr_both, 350 reg.dst_lmextn, reg.src_lmextn); 351 } 352 353 static void 354 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, 355 u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8, 356 bool zero, bool swap, bool wr_both, 357 bool dst_lmextn, bool src_lmextn) 358 { 359 u64 insn; 360 361 insn = OP_LDF_BASE | 362 FIELD_PREP(OP_LDF_A_SRC, areg) | 363 FIELD_PREP(OP_LDF_SC, sc) | 364 FIELD_PREP(OP_LDF_B_SRC, breg) | 365 FIELD_PREP(OP_LDF_I8, imm8) | 366 FIELD_PREP(OP_LDF_SW, swap) | 367 FIELD_PREP(OP_LDF_ZF, zero) | 368 FIELD_PREP(OP_LDF_BMASK, bmask) | 369 FIELD_PREP(OP_LDF_SHF, shift) | 370 FIELD_PREP(OP_LDF_WR_AB, wr_both) | 371 FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) | 372 FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn); 373 374 nfp_prog_push(nfp_prog, insn); 375 } 376 377 static void 378 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 379 enum shf_sc sc, u8 shift, bool zero) 380 { 381 struct nfp_insn_re_regs reg; 382 int err; 383 384 /* Note: ld_field is special as it uses one of the src regs as dst */ 385 err = swreg_to_restricted(dst, dst, src, ®, true); 386 if (err) { 387 nfp_prog->error = err; 388 return; 389 } 390 391 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, 392 reg.i8, zero, reg.swap, reg.wr_both, 393 reg.dst_lmextn, reg.src_lmextn); 394 } 395 396 static void 397 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 398 enum shf_sc sc, u8 shift) 399 { 400 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); 401 } 402 403 static void 404 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, 405 bool dst_lmextn, bool src_lmextn) 406 { 407 u64 insn; 408 409 insn = OP_LCSR_BASE | 410 FIELD_PREP(OP_LCSR_A_SRC, areg) | 411 FIELD_PREP(OP_LCSR_B_SRC, breg) | 412 FIELD_PREP(OP_LCSR_WRITE, wr) | 413 FIELD_PREP(OP_LCSR_ADDR, addr) | 414 FIELD_PREP(OP_LCSR_SRC_LMEXTN, src_lmextn) | 415 FIELD_PREP(OP_LCSR_DST_LMEXTN, dst_lmextn); 416 417 nfp_prog_push(nfp_prog, insn); 418 } 419 420 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) 421 { 422 struct nfp_insn_ur_regs reg; 423 int err; 424 425 /* This instruction takes immeds instead of reg_none() for the ignored 426 * operand, but we can't encode 2 immeds in one instr with our normal 427 * swreg infra so if param is an immed, we encode as reg_none() and 428 * copy the immed to both operands. 429 */ 430 if (swreg_type(src) == NN_REG_IMM) { 431 err = swreg_to_unrestricted(reg_none(), src, reg_none(), ®); 432 reg.breg = reg.areg; 433 } else { 434 err = swreg_to_unrestricted(reg_none(), src, reg_imm(0), ®); 435 } 436 if (err) { 437 nfp_prog->error = err; 438 return; 439 } 440 441 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr / 4, 442 false, reg.src_lmextn); 443 } 444 445 static void emit_nop(struct nfp_prog *nfp_prog) 446 { 447 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); 448 } 449 450 /* --- Wrappers --- */ 451 static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift) 452 { 453 if (!(imm & 0xffff0000)) { 454 *val = imm; 455 *shift = IMMED_SHIFT_0B; 456 } else if (!(imm & 0xff0000ff)) { 457 *val = imm >> 8; 458 *shift = IMMED_SHIFT_1B; 459 } else if (!(imm & 0x0000ffff)) { 460 *val = imm >> 16; 461 *shift = IMMED_SHIFT_2B; 462 } else { 463 return false; 464 } 465 466 return true; 467 } 468 469 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) 470 { 471 enum immed_shift shift; 472 u16 val; 473 474 if (pack_immed(imm, &val, &shift)) { 475 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); 476 } else if (pack_immed(~imm, &val, &shift)) { 477 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); 478 } else { 479 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, 480 false, IMMED_SHIFT_0B); 481 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, 482 false, IMMED_SHIFT_2B); 483 } 484 } 485 486 static void 487 wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm, 488 enum nfp_relo_type relo) 489 { 490 if (imm > 0xffff) { 491 pr_err("relocation of a large immediate!\n"); 492 nfp_prog->error = -EFAULT; 493 return; 494 } 495 emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B); 496 497 nfp_prog->prog[nfp_prog->prog_len - 1] |= 498 FIELD_PREP(OP_RELO_TYPE, relo); 499 } 500 501 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted) 502 * If the @imm is small enough encode it directly in operand and return 503 * otherwise load @imm to a spare register and return its encoding. 504 */ 505 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 506 { 507 if (FIELD_FIT(UR_REG_IMM_MAX, imm)) 508 return reg_imm(imm); 509 510 wrp_immed(nfp_prog, tmp_reg, imm); 511 return tmp_reg; 512 } 513 514 /* re_load_imm_any() - encode immediate or use tmp register (restricted) 515 * If the @imm is small enough encode it directly in operand and return 516 * otherwise load @imm to a spare register and return its encoding. 517 */ 518 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 519 { 520 if (FIELD_FIT(RE_REG_IMM_MAX, imm)) 521 return reg_imm(imm); 522 523 wrp_immed(nfp_prog, tmp_reg, imm); 524 return tmp_reg; 525 } 526 527 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) 528 { 529 while (count--) 530 emit_nop(nfp_prog); 531 } 532 533 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) 534 { 535 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); 536 } 537 538 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) 539 { 540 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); 541 } 542 543 /* wrp_reg_subpart() - load @field_len bytes from @offset of @src, write the 544 * result to @dst from low end. 545 */ 546 static void 547 wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len, 548 u8 offset) 549 { 550 enum shf_sc sc = offset ? SHF_SC_R_SHF : SHF_SC_NONE; 551 u8 mask = (1 << field_len) - 1; 552 553 emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true); 554 } 555 556 /* NFP has Command Push Pull bus which supports bluk memory operations. */ 557 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 558 { 559 bool descending_seq = meta->ldst_gather_len < 0; 560 s16 len = abs(meta->ldst_gather_len); 561 swreg src_base, off; 562 unsigned int i; 563 u8 xfer_num; 564 565 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 566 src_base = reg_a(meta->insn.src_reg * 2); 567 xfer_num = round_up(len, 4) / 4; 568 569 /* Setup PREV_ALU fields to override memory read length. */ 570 if (len > 32) 571 wrp_immed(nfp_prog, reg_none(), 572 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1)); 573 574 /* Memory read from source addr into transfer-in registers. */ 575 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, 576 off, xfer_num - 1, true, len > 32); 577 578 /* Move from transfer-in to transfer-out. */ 579 for (i = 0; i < xfer_num; i++) 580 wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i)); 581 582 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); 583 584 if (len <= 8) { 585 /* Use single direct_ref write8. */ 586 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 587 reg_a(meta->paired_st->dst_reg * 2), off, len - 1, 588 true); 589 } else if (len <= 32 && IS_ALIGNED(len, 4)) { 590 /* Use single direct_ref write32. */ 591 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 592 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1, 593 true); 594 } else if (len <= 32) { 595 /* Use single indirect_ref write8. */ 596 wrp_immed(nfp_prog, reg_none(), 597 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, len - 1)); 598 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 599 reg_a(meta->paired_st->dst_reg * 2), off, 600 len - 1, true); 601 } else if (IS_ALIGNED(len, 4)) { 602 /* Use single indirect_ref write32. */ 603 wrp_immed(nfp_prog, reg_none(), 604 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1)); 605 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 606 reg_a(meta->paired_st->dst_reg * 2), off, 607 xfer_num - 1, true); 608 } else if (len <= 40) { 609 /* Use one direct_ref write32 to write the first 32-bytes, then 610 * another direct_ref write8 to write the remaining bytes. 611 */ 612 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 613 reg_a(meta->paired_st->dst_reg * 2), off, 7, 614 true); 615 616 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, 617 imm_b(nfp_prog)); 618 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8, 619 reg_a(meta->paired_st->dst_reg * 2), off, len - 33, 620 true); 621 } else { 622 /* Use one indirect_ref write32 to write 4-bytes aligned length, 623 * then another direct_ref write8 to write the remaining bytes. 624 */ 625 u8 new_off; 626 627 wrp_immed(nfp_prog, reg_none(), 628 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 2)); 629 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 630 reg_a(meta->paired_st->dst_reg * 2), off, 631 xfer_num - 2, true); 632 new_off = meta->paired_st->off + (xfer_num - 1) * 4; 633 off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog)); 634 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 635 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off, 636 (len & 0x3) - 1, true); 637 } 638 639 /* TODO: The following extra load is to make sure data flow be identical 640 * before and after we do memory copy optimization. 641 * 642 * The load destination register is not guaranteed to be dead, so we 643 * need to make sure it is loaded with the value the same as before 644 * this transformation. 645 * 646 * These extra loads could be removed once we have accurate register 647 * usage information. 648 */ 649 if (descending_seq) 650 xfer_num = 0; 651 else if (BPF_SIZE(meta->insn.code) != BPF_DW) 652 xfer_num = xfer_num - 1; 653 else 654 xfer_num = xfer_num - 2; 655 656 switch (BPF_SIZE(meta->insn.code)) { 657 case BPF_B: 658 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), 659 reg_xfer(xfer_num), 1, 660 IS_ALIGNED(len, 4) ? 3 : (len & 3) - 1); 661 break; 662 case BPF_H: 663 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), 664 reg_xfer(xfer_num), 2, (len & 3) ^ 2); 665 break; 666 case BPF_W: 667 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), 668 reg_xfer(0)); 669 break; 670 case BPF_DW: 671 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), 672 reg_xfer(xfer_num)); 673 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 674 reg_xfer(xfer_num + 1)); 675 break; 676 } 677 678 if (BPF_SIZE(meta->insn.code) != BPF_DW) 679 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 680 681 return 0; 682 } 683 684 static int 685 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size) 686 { 687 unsigned int i; 688 u16 shift, sz; 689 690 /* We load the value from the address indicated in @offset and then 691 * shift out the data we don't need. Note: this is big endian! 692 */ 693 sz = max(size, 4); 694 shift = size < 4 ? 4 - size : 0; 695 696 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, 697 pptr_reg(nfp_prog), offset, sz - 1, true); 698 699 i = 0; 700 if (shift) 701 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, 702 reg_xfer(0), SHF_SC_R_SHF, shift * 8); 703 else 704 for (; i * 4 < size; i++) 705 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 706 707 if (i < 2) 708 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 709 710 return 0; 711 } 712 713 static int 714 data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, 715 u8 dst_gpr, int size) 716 { 717 unsigned int i; 718 u8 mask, sz; 719 720 /* We load the value from the address indicated in @offset and then 721 * mask out the data we don't need. Note: this is little endian! 722 */ 723 sz = max(size, 4); 724 mask = size < 4 ? GENMASK(size - 1, 0) : 0; 725 726 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, 727 reg_a(src_gpr), offset, sz / 4 - 1, true); 728 729 i = 0; 730 if (mask) 731 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, 732 reg_xfer(0), SHF_SC_NONE, 0, true); 733 else 734 for (; i * 4 < size; i++) 735 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 736 737 if (i < 2) 738 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 739 740 return 0; 741 } 742 743 static int 744 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) 745 { 746 swreg tmp_reg; 747 748 /* Calculate the true offset (src_reg + imm) */ 749 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 750 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); 751 752 /* Check packet length (size guaranteed to fit b/c it's u8) */ 753 emit_alu(nfp_prog, imm_a(nfp_prog), 754 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); 755 emit_alu(nfp_prog, reg_none(), 756 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); 757 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); 758 759 /* Load data */ 760 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size); 761 } 762 763 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) 764 { 765 swreg tmp_reg; 766 767 /* Check packet length */ 768 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); 769 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); 770 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); 771 772 /* Load data */ 773 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 774 return data_ld(nfp_prog, tmp_reg, 0, size); 775 } 776 777 static int 778 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 779 u8 src_gpr, u8 size) 780 { 781 unsigned int i; 782 783 for (i = 0; i * 4 < size; i++) 784 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); 785 786 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 787 reg_a(dst_gpr), offset, size - 1, true); 788 789 return 0; 790 } 791 792 static int 793 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 794 u64 imm, u8 size) 795 { 796 wrp_immed(nfp_prog, reg_xfer(0), imm); 797 if (size == 8) 798 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); 799 800 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 801 reg_a(dst_gpr), offset, size - 1, true); 802 803 return 0; 804 } 805 806 typedef int 807 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off, 808 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 809 bool needs_inc); 810 811 static int 812 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, 813 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 814 bool needs_inc) 815 { 816 bool should_inc = needs_inc && new_gpr && !last; 817 u32 idx, src_byte; 818 enum shf_sc sc; 819 swreg reg; 820 int shf; 821 u8 mask; 822 823 if (WARN_ON_ONCE(dst_byte + size > 4 || off % 4 + size > 4)) 824 return -EOPNOTSUPP; 825 826 idx = off / 4; 827 828 /* Move the entire word */ 829 if (size == 4) { 830 wrp_mov(nfp_prog, reg_both(dst), 831 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx)); 832 return 0; 833 } 834 835 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 836 return -EOPNOTSUPP; 837 838 src_byte = off % 4; 839 840 mask = (1 << size) - 1; 841 mask <<= dst_byte; 842 843 if (WARN_ON_ONCE(mask > 0xf)) 844 return -EOPNOTSUPP; 845 846 shf = abs(src_byte - dst_byte) * 8; 847 if (src_byte == dst_byte) { 848 sc = SHF_SC_NONE; 849 } else if (src_byte < dst_byte) { 850 shf = 32 - shf; 851 sc = SHF_SC_L_SHF; 852 } else { 853 sc = SHF_SC_R_SHF; 854 } 855 856 /* ld_field can address fewer indexes, if offset too large do RMW. 857 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 858 */ 859 if (idx <= RE_REG_LM_IDX_MAX) { 860 reg = reg_lm(lm3 ? 3 : 0, idx); 861 } else { 862 reg = imm_a(nfp_prog); 863 /* If it's not the first part of the load and we start a new GPR 864 * that means we are loading a second part of the LMEM word into 865 * a new GPR. IOW we've already looked that LMEM word and 866 * therefore it has been loaded into imm_a(). 867 */ 868 if (first || !new_gpr) 869 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 870 } 871 872 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); 873 874 if (should_inc) 875 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 876 877 return 0; 878 } 879 880 static int 881 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, 882 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 883 bool needs_inc) 884 { 885 bool should_inc = needs_inc && new_gpr && !last; 886 u32 idx, dst_byte; 887 enum shf_sc sc; 888 swreg reg; 889 int shf; 890 u8 mask; 891 892 if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4)) 893 return -EOPNOTSUPP; 894 895 idx = off / 4; 896 897 /* Move the entire word */ 898 if (size == 4) { 899 wrp_mov(nfp_prog, 900 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx), 901 reg_b(src)); 902 return 0; 903 } 904 905 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 906 return -EOPNOTSUPP; 907 908 dst_byte = off % 4; 909 910 mask = (1 << size) - 1; 911 mask <<= dst_byte; 912 913 if (WARN_ON_ONCE(mask > 0xf)) 914 return -EOPNOTSUPP; 915 916 shf = abs(src_byte - dst_byte) * 8; 917 if (src_byte == dst_byte) { 918 sc = SHF_SC_NONE; 919 } else if (src_byte < dst_byte) { 920 shf = 32 - shf; 921 sc = SHF_SC_L_SHF; 922 } else { 923 sc = SHF_SC_R_SHF; 924 } 925 926 /* ld_field can address fewer indexes, if offset too large do RMW. 927 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 928 */ 929 if (idx <= RE_REG_LM_IDX_MAX) { 930 reg = reg_lm(lm3 ? 3 : 0, idx); 931 } else { 932 reg = imm_a(nfp_prog); 933 /* Only first and last LMEM locations are going to need RMW, 934 * the middle location will be overwritten fully. 935 */ 936 if (first || last) 937 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 938 } 939 940 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); 941 942 if (new_gpr || last) { 943 if (idx > RE_REG_LM_IDX_MAX) 944 wrp_mov(nfp_prog, reg_lm(0, idx), reg); 945 if (should_inc) 946 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 947 } 948 949 return 0; 950 } 951 952 static int 953 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 954 unsigned int size, unsigned int ptr_off, u8 gpr, u8 ptr_gpr, 955 bool clr_gpr, lmem_step step) 956 { 957 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off; 958 bool first = true, last; 959 bool needs_inc = false; 960 swreg stack_off_reg; 961 u8 prev_gpr = 255; 962 u32 gpr_byte = 0; 963 bool lm3 = true; 964 int ret; 965 966 if (meta->ptr_not_const) { 967 /* Use of the last encountered ptr_off is OK, they all have 968 * the same alignment. Depend on low bits of value being 969 * discarded when written to LMaddr register. 970 */ 971 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, 972 stack_imm(nfp_prog)); 973 974 emit_alu(nfp_prog, imm_b(nfp_prog), 975 reg_a(ptr_gpr), ALU_OP_ADD, stack_off_reg); 976 977 needs_inc = true; 978 } else if (off + size <= 64) { 979 /* We can reach bottom 64B with LMaddr0 */ 980 lm3 = false; 981 } else if (round_down(off, 32) == round_down(off + size - 1, 32)) { 982 /* We have to set up a new pointer. If we know the offset 983 * and the entire access falls into a single 32 byte aligned 984 * window we won't have to increment the LM pointer. 985 * The 32 byte alignment is imporant because offset is ORed in 986 * not added when doing *l$indexN[off]. 987 */ 988 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), 989 stack_imm(nfp_prog)); 990 emit_alu(nfp_prog, imm_b(nfp_prog), 991 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 992 993 off %= 32; 994 } else { 995 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), 996 stack_imm(nfp_prog)); 997 998 emit_alu(nfp_prog, imm_b(nfp_prog), 999 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 1000 1001 needs_inc = true; 1002 } 1003 if (lm3) { 1004 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); 1005 /* For size < 4 one slot will be filled by zeroing of upper. */ 1006 wrp_nops(nfp_prog, clr_gpr && size < 8 ? 2 : 3); 1007 } 1008 1009 if (clr_gpr && size < 8) 1010 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1011 1012 while (size) { 1013 u32 slice_end; 1014 u8 slice_size; 1015 1016 slice_size = min(size, 4 - gpr_byte); 1017 slice_end = min(off + slice_size, round_up(off + 1, 4)); 1018 slice_size = slice_end - off; 1019 1020 last = slice_size == size; 1021 1022 if (needs_inc) 1023 off %= 4; 1024 1025 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, 1026 first, gpr != prev_gpr, last, lm3, needs_inc); 1027 if (ret) 1028 return ret; 1029 1030 prev_gpr = gpr; 1031 first = false; 1032 1033 gpr_byte += slice_size; 1034 if (gpr_byte >= 4) { 1035 gpr_byte -= 4; 1036 gpr++; 1037 } 1038 1039 size -= slice_size; 1040 off += slice_size; 1041 } 1042 1043 return 0; 1044 } 1045 1046 static void 1047 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) 1048 { 1049 swreg tmp_reg; 1050 1051 if (alu_op == ALU_OP_AND) { 1052 if (!imm) 1053 wrp_immed(nfp_prog, reg_both(dst), 0); 1054 if (!imm || !~imm) 1055 return; 1056 } 1057 if (alu_op == ALU_OP_OR) { 1058 if (!~imm) 1059 wrp_immed(nfp_prog, reg_both(dst), ~0U); 1060 if (!imm || !~imm) 1061 return; 1062 } 1063 if (alu_op == ALU_OP_XOR) { 1064 if (!~imm) 1065 emit_alu(nfp_prog, reg_both(dst), reg_none(), 1066 ALU_OP_NOT, reg_b(dst)); 1067 if (!imm || !~imm) 1068 return; 1069 } 1070 1071 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); 1072 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); 1073 } 1074 1075 static int 1076 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1077 enum alu_op alu_op, bool skip) 1078 { 1079 const struct bpf_insn *insn = &meta->insn; 1080 u64 imm = insn->imm; /* sign extend */ 1081 1082 if (skip) { 1083 meta->skip = true; 1084 return 0; 1085 } 1086 1087 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); 1088 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); 1089 1090 return 0; 1091 } 1092 1093 static int 1094 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1095 enum alu_op alu_op) 1096 { 1097 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 1098 1099 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 1100 emit_alu(nfp_prog, reg_both(dst + 1), 1101 reg_a(dst + 1), alu_op, reg_b(src + 1)); 1102 1103 return 0; 1104 } 1105 1106 static int 1107 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1108 enum alu_op alu_op, bool skip) 1109 { 1110 const struct bpf_insn *insn = &meta->insn; 1111 1112 if (skip) { 1113 meta->skip = true; 1114 return 0; 1115 } 1116 1117 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm); 1118 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1119 1120 return 0; 1121 } 1122 1123 static int 1124 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1125 enum alu_op alu_op) 1126 { 1127 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 1128 1129 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 1130 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1131 1132 return 0; 1133 } 1134 1135 static void 1136 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, 1137 enum br_mask br_mask, u16 off) 1138 { 1139 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); 1140 emit_br(nfp_prog, br_mask, off, 0); 1141 } 1142 1143 static int 1144 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1145 enum alu_op alu_op, enum br_mask br_mask) 1146 { 1147 const struct bpf_insn *insn = &meta->insn; 1148 1149 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, 1150 insn->src_reg * 2, br_mask, insn->off); 1151 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, 1152 insn->src_reg * 2 + 1, br_mask, insn->off); 1153 1154 return 0; 1155 } 1156 1157 static int 1158 wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1159 enum br_mask br_mask, bool swap) 1160 { 1161 const struct bpf_insn *insn = &meta->insn; 1162 u64 imm = insn->imm; /* sign extend */ 1163 u8 reg = insn->dst_reg * 2; 1164 swreg tmp_reg; 1165 1166 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1167 if (!swap) 1168 emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg); 1169 else 1170 emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg)); 1171 1172 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1173 if (!swap) 1174 emit_alu(nfp_prog, reg_none(), 1175 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg); 1176 else 1177 emit_alu(nfp_prog, reg_none(), 1178 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1)); 1179 1180 emit_br(nfp_prog, br_mask, insn->off, 0); 1181 1182 return 0; 1183 } 1184 1185 static int 1186 wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1187 enum br_mask br_mask, bool swap) 1188 { 1189 const struct bpf_insn *insn = &meta->insn; 1190 u8 areg, breg; 1191 1192 areg = insn->dst_reg * 2; 1193 breg = insn->src_reg * 2; 1194 1195 if (swap) { 1196 areg ^= breg; 1197 breg ^= areg; 1198 areg ^= breg; 1199 } 1200 1201 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); 1202 emit_alu(nfp_prog, reg_none(), 1203 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1)); 1204 emit_br(nfp_prog, br_mask, insn->off, 0); 1205 1206 return 0; 1207 } 1208 1209 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) 1210 { 1211 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, 1212 SHF_SC_R_ROT, 8); 1213 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), 1214 SHF_SC_R_ROT, 16); 1215 } 1216 1217 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1218 { 1219 swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog); 1220 struct nfp_bpf_cap_adjust_head *adjust_head; 1221 u32 ret_einval, end; 1222 1223 adjust_head = &nfp_prog->bpf->adjust_head; 1224 1225 /* Optimized version - 5 vs 14 cycles */ 1226 if (nfp_prog->adjust_head_location != UINT_MAX) { 1227 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) 1228 return -EINVAL; 1229 1230 emit_alu(nfp_prog, pptr_reg(nfp_prog), 1231 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog)); 1232 emit_alu(nfp_prog, plen_reg(nfp_prog), 1233 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); 1234 emit_alu(nfp_prog, pv_len(nfp_prog), 1235 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); 1236 1237 wrp_immed(nfp_prog, reg_both(0), 0); 1238 wrp_immed(nfp_prog, reg_both(1), 0); 1239 1240 /* TODO: when adjust head is guaranteed to succeed we can 1241 * also eliminate the following if (r0 == 0) branch. 1242 */ 1243 1244 return 0; 1245 } 1246 1247 ret_einval = nfp_prog_current_offset(nfp_prog) + 14; 1248 end = ret_einval + 2; 1249 1250 /* We need to use a temp because offset is just a part of the pkt ptr */ 1251 emit_alu(nfp_prog, tmp, 1252 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog)); 1253 1254 /* Validate result will fit within FW datapath constraints */ 1255 emit_alu(nfp_prog, reg_none(), 1256 tmp, ALU_OP_SUB, reg_imm(adjust_head->off_min)); 1257 emit_br(nfp_prog, BR_BLO, ret_einval, 0); 1258 emit_alu(nfp_prog, reg_none(), 1259 reg_imm(adjust_head->off_max), ALU_OP_SUB, tmp); 1260 emit_br(nfp_prog, BR_BLO, ret_einval, 0); 1261 1262 /* Validate the length is at least ETH_HLEN */ 1263 emit_alu(nfp_prog, tmp_len, 1264 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); 1265 emit_alu(nfp_prog, reg_none(), 1266 tmp_len, ALU_OP_SUB, reg_imm(ETH_HLEN)); 1267 emit_br(nfp_prog, BR_BMI, ret_einval, 0); 1268 1269 /* Load the ret code */ 1270 wrp_immed(nfp_prog, reg_both(0), 0); 1271 wrp_immed(nfp_prog, reg_both(1), 0); 1272 1273 /* Modify the packet metadata */ 1274 emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0); 1275 1276 /* Skip over the -EINVAL ret code (defer 2) */ 1277 emit_br(nfp_prog, BR_UNC, end, 2); 1278 1279 emit_alu(nfp_prog, plen_reg(nfp_prog), 1280 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); 1281 emit_alu(nfp_prog, pv_len(nfp_prog), 1282 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); 1283 1284 /* return -EINVAL target */ 1285 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) 1286 return -EINVAL; 1287 1288 wrp_immed(nfp_prog, reg_both(0), -22); 1289 wrp_immed(nfp_prog, reg_both(1), ~0); 1290 1291 if (!nfp_prog_confirm_current_offset(nfp_prog, end)) 1292 return -EINVAL; 1293 1294 return 0; 1295 } 1296 1297 static int 1298 map_lookup_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1299 { 1300 struct bpf_offloaded_map *offmap; 1301 struct nfp_bpf_map *nfp_map; 1302 bool load_lm_ptr; 1303 u32 ret_tgt; 1304 s64 lm_off; 1305 swreg tid; 1306 1307 offmap = (struct bpf_offloaded_map *)meta->arg1.map_ptr; 1308 nfp_map = offmap->dev_priv; 1309 1310 /* We only have to reload LM0 if the key is not at start of stack */ 1311 lm_off = nfp_prog->stack_depth; 1312 lm_off += meta->arg2.var_off.value + meta->arg2.off; 1313 load_lm_ptr = meta->arg2_var_off || lm_off; 1314 1315 /* Set LM0 to start of key */ 1316 if (load_lm_ptr) 1317 emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0); 1318 1319 /* Load map ID into a register, it should actually fit as an immediate 1320 * but in case it doesn't deal with it here, not in the delay slots. 1321 */ 1322 tid = ur_load_imm_any(nfp_prog, nfp_map->tid, imm_a(nfp_prog)); 1323 1324 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + BPF_FUNC_map_lookup_elem, 1325 2, RELO_BR_HELPER); 1326 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; 1327 1328 /* Load map ID into A0 */ 1329 wrp_mov(nfp_prog, reg_a(0), tid); 1330 1331 /* Load the return address into B0 */ 1332 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); 1333 1334 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) 1335 return -EINVAL; 1336 1337 /* Reset the LM0 pointer */ 1338 if (!load_lm_ptr) 1339 return 0; 1340 1341 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); 1342 wrp_nops(nfp_prog, 3); 1343 1344 return 0; 1345 } 1346 1347 /* --- Callbacks --- */ 1348 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1349 { 1350 const struct bpf_insn *insn = &meta->insn; 1351 u8 dst = insn->dst_reg * 2; 1352 u8 src = insn->src_reg * 2; 1353 1354 if (insn->src_reg == BPF_REG_10) { 1355 swreg stack_depth_reg; 1356 1357 stack_depth_reg = ur_load_imm_any(nfp_prog, 1358 nfp_prog->stack_depth, 1359 stack_imm(nfp_prog)); 1360 emit_alu(nfp_prog, reg_both(dst), 1361 stack_reg(nfp_prog), ALU_OP_ADD, stack_depth_reg); 1362 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1363 } else { 1364 wrp_reg_mov(nfp_prog, dst, src); 1365 wrp_reg_mov(nfp_prog, dst + 1, src + 1); 1366 } 1367 1368 return 0; 1369 } 1370 1371 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1372 { 1373 u64 imm = meta->insn.imm; /* sign extend */ 1374 1375 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); 1376 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); 1377 1378 return 0; 1379 } 1380 1381 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1382 { 1383 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); 1384 } 1385 1386 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1387 { 1388 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); 1389 } 1390 1391 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1392 { 1393 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); 1394 } 1395 1396 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1397 { 1398 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1399 } 1400 1401 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1402 { 1403 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); 1404 } 1405 1406 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1407 { 1408 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1409 } 1410 1411 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1412 { 1413 const struct bpf_insn *insn = &meta->insn; 1414 1415 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1416 reg_a(insn->dst_reg * 2), ALU_OP_ADD, 1417 reg_b(insn->src_reg * 2)); 1418 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1419 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C, 1420 reg_b(insn->src_reg * 2 + 1)); 1421 1422 return 0; 1423 } 1424 1425 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1426 { 1427 const struct bpf_insn *insn = &meta->insn; 1428 u64 imm = insn->imm; /* sign extend */ 1429 1430 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); 1431 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); 1432 1433 return 0; 1434 } 1435 1436 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1437 { 1438 const struct bpf_insn *insn = &meta->insn; 1439 1440 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1441 reg_a(insn->dst_reg * 2), ALU_OP_SUB, 1442 reg_b(insn->src_reg * 2)); 1443 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1444 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C, 1445 reg_b(insn->src_reg * 2 + 1)); 1446 1447 return 0; 1448 } 1449 1450 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1451 { 1452 const struct bpf_insn *insn = &meta->insn; 1453 u64 imm = insn->imm; /* sign extend */ 1454 1455 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); 1456 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); 1457 1458 return 0; 1459 } 1460 1461 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1462 { 1463 const struct bpf_insn *insn = &meta->insn; 1464 1465 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), 1466 ALU_OP_SUB, reg_b(insn->dst_reg * 2)); 1467 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), 1468 ALU_OP_SUB_C, reg_b(insn->dst_reg * 2 + 1)); 1469 1470 return 0; 1471 } 1472 1473 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1474 { 1475 const struct bpf_insn *insn = &meta->insn; 1476 u8 dst = insn->dst_reg * 2; 1477 1478 if (insn->imm < 32) { 1479 emit_shf(nfp_prog, reg_both(dst + 1), 1480 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1481 SHF_SC_R_DSHF, 32 - insn->imm); 1482 emit_shf(nfp_prog, reg_both(dst), 1483 reg_none(), SHF_OP_NONE, reg_b(dst), 1484 SHF_SC_L_SHF, insn->imm); 1485 } else if (insn->imm == 32) { 1486 wrp_reg_mov(nfp_prog, dst + 1, dst); 1487 wrp_immed(nfp_prog, reg_both(dst), 0); 1488 } else if (insn->imm > 32) { 1489 emit_shf(nfp_prog, reg_both(dst + 1), 1490 reg_none(), SHF_OP_NONE, reg_b(dst), 1491 SHF_SC_L_SHF, insn->imm - 32); 1492 wrp_immed(nfp_prog, reg_both(dst), 0); 1493 } 1494 1495 return 0; 1496 } 1497 1498 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1499 { 1500 const struct bpf_insn *insn = &meta->insn; 1501 u8 dst = insn->dst_reg * 2; 1502 1503 if (insn->imm < 32) { 1504 emit_shf(nfp_prog, reg_both(dst), 1505 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1506 SHF_SC_R_DSHF, insn->imm); 1507 emit_shf(nfp_prog, reg_both(dst + 1), 1508 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1509 SHF_SC_R_SHF, insn->imm); 1510 } else if (insn->imm == 32) { 1511 wrp_reg_mov(nfp_prog, dst, dst + 1); 1512 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1513 } else if (insn->imm > 32) { 1514 emit_shf(nfp_prog, reg_both(dst), 1515 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1516 SHF_SC_R_SHF, insn->imm - 32); 1517 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1518 } 1519 1520 return 0; 1521 } 1522 1523 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1524 { 1525 const struct bpf_insn *insn = &meta->insn; 1526 1527 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); 1528 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1529 1530 return 0; 1531 } 1532 1533 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1534 { 1535 const struct bpf_insn *insn = &meta->insn; 1536 1537 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); 1538 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1539 1540 return 0; 1541 } 1542 1543 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1544 { 1545 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); 1546 } 1547 1548 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1549 { 1550 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); 1551 } 1552 1553 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1554 { 1555 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); 1556 } 1557 1558 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1559 { 1560 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1561 } 1562 1563 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1564 { 1565 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); 1566 } 1567 1568 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1569 { 1570 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1571 } 1572 1573 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1574 { 1575 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); 1576 } 1577 1578 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1579 { 1580 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); 1581 } 1582 1583 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1584 { 1585 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); 1586 } 1587 1588 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1589 { 1590 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); 1591 } 1592 1593 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1594 { 1595 u8 dst = meta->insn.dst_reg * 2; 1596 1597 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); 1598 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1599 1600 return 0; 1601 } 1602 1603 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1604 { 1605 const struct bpf_insn *insn = &meta->insn; 1606 1607 if (!insn->imm) 1608 return 1; /* TODO: zero shift means indirect */ 1609 1610 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2), 1611 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2), 1612 SHF_SC_L_SHF, insn->imm); 1613 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1614 1615 return 0; 1616 } 1617 1618 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1619 { 1620 const struct bpf_insn *insn = &meta->insn; 1621 u8 gpr = insn->dst_reg * 2; 1622 1623 switch (insn->imm) { 1624 case 16: 1625 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), 1626 SHF_SC_R_ROT, 8); 1627 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), 1628 SHF_SC_R_SHF, 16); 1629 1630 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1631 break; 1632 case 32: 1633 wrp_end32(nfp_prog, reg_a(gpr), gpr); 1634 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1635 break; 1636 case 64: 1637 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); 1638 1639 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); 1640 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); 1641 break; 1642 } 1643 1644 return 0; 1645 } 1646 1647 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1648 { 1649 struct nfp_insn_meta *prev = nfp_meta_prev(meta); 1650 u32 imm_lo, imm_hi; 1651 u8 dst; 1652 1653 dst = prev->insn.dst_reg * 2; 1654 imm_lo = prev->insn.imm; 1655 imm_hi = meta->insn.imm; 1656 1657 wrp_immed(nfp_prog, reg_both(dst), imm_lo); 1658 1659 /* mov is always 1 insn, load imm may be two, so try to use mov */ 1660 if (imm_hi == imm_lo) 1661 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); 1662 else 1663 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); 1664 1665 return 0; 1666 } 1667 1668 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1669 { 1670 meta->double_cb = imm_ld8_part2; 1671 return 0; 1672 } 1673 1674 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1675 { 1676 return construct_data_ld(nfp_prog, meta->insn.imm, 1); 1677 } 1678 1679 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1680 { 1681 return construct_data_ld(nfp_prog, meta->insn.imm, 2); 1682 } 1683 1684 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1685 { 1686 return construct_data_ld(nfp_prog, meta->insn.imm, 4); 1687 } 1688 1689 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1690 { 1691 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1692 meta->insn.src_reg * 2, 1); 1693 } 1694 1695 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1696 { 1697 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1698 meta->insn.src_reg * 2, 2); 1699 } 1700 1701 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1702 { 1703 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1704 meta->insn.src_reg * 2, 4); 1705 } 1706 1707 static int 1708 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1709 unsigned int size, unsigned int ptr_off) 1710 { 1711 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1712 meta->insn.dst_reg * 2, meta->insn.src_reg * 2, 1713 true, wrp_lmem_load); 1714 } 1715 1716 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1717 u8 size) 1718 { 1719 swreg dst = reg_both(meta->insn.dst_reg * 2); 1720 1721 switch (meta->insn.off) { 1722 case offsetof(struct __sk_buff, len): 1723 if (size != FIELD_SIZEOF(struct __sk_buff, len)) 1724 return -EOPNOTSUPP; 1725 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); 1726 break; 1727 case offsetof(struct __sk_buff, data): 1728 if (size != FIELD_SIZEOF(struct __sk_buff, data)) 1729 return -EOPNOTSUPP; 1730 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1731 break; 1732 case offsetof(struct __sk_buff, data_end): 1733 if (size != FIELD_SIZEOF(struct __sk_buff, data_end)) 1734 return -EOPNOTSUPP; 1735 emit_alu(nfp_prog, dst, 1736 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1737 break; 1738 default: 1739 return -EOPNOTSUPP; 1740 } 1741 1742 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1743 1744 return 0; 1745 } 1746 1747 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1748 u8 size) 1749 { 1750 swreg dst = reg_both(meta->insn.dst_reg * 2); 1751 1752 switch (meta->insn.off) { 1753 case offsetof(struct xdp_md, data): 1754 if (size != FIELD_SIZEOF(struct xdp_md, data)) 1755 return -EOPNOTSUPP; 1756 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1757 break; 1758 case offsetof(struct xdp_md, data_end): 1759 if (size != FIELD_SIZEOF(struct xdp_md, data_end)) 1760 return -EOPNOTSUPP; 1761 emit_alu(nfp_prog, dst, 1762 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1763 break; 1764 default: 1765 return -EOPNOTSUPP; 1766 } 1767 1768 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1769 1770 return 0; 1771 } 1772 1773 static int 1774 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1775 unsigned int size) 1776 { 1777 swreg tmp_reg; 1778 1779 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1780 1781 return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg, 1782 meta->insn.dst_reg * 2, size); 1783 } 1784 1785 static int 1786 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1787 unsigned int size) 1788 { 1789 if (meta->ldst_gather_len) 1790 return nfp_cpp_memcpy(nfp_prog, meta); 1791 1792 if (meta->ptr.type == PTR_TO_CTX) { 1793 if (nfp_prog->type == BPF_PROG_TYPE_XDP) 1794 return mem_ldx_xdp(nfp_prog, meta, size); 1795 else 1796 return mem_ldx_skb(nfp_prog, meta, size); 1797 } 1798 1799 if (meta->ptr.type == PTR_TO_PACKET) 1800 return mem_ldx_data(nfp_prog, meta, size); 1801 1802 if (meta->ptr.type == PTR_TO_STACK) 1803 return mem_ldx_stack(nfp_prog, meta, size, 1804 meta->ptr.off + meta->ptr.var_off.value); 1805 1806 return -EOPNOTSUPP; 1807 } 1808 1809 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1810 { 1811 return mem_ldx(nfp_prog, meta, 1); 1812 } 1813 1814 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1815 { 1816 return mem_ldx(nfp_prog, meta, 2); 1817 } 1818 1819 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1820 { 1821 return mem_ldx(nfp_prog, meta, 4); 1822 } 1823 1824 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1825 { 1826 return mem_ldx(nfp_prog, meta, 8); 1827 } 1828 1829 static int 1830 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1831 unsigned int size) 1832 { 1833 u64 imm = meta->insn.imm; /* sign extend */ 1834 swreg off_reg; 1835 1836 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1837 1838 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1839 imm, size); 1840 } 1841 1842 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1843 unsigned int size) 1844 { 1845 if (meta->ptr.type == PTR_TO_PACKET) 1846 return mem_st_data(nfp_prog, meta, size); 1847 1848 return -EOPNOTSUPP; 1849 } 1850 1851 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1852 { 1853 return mem_st(nfp_prog, meta, 1); 1854 } 1855 1856 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1857 { 1858 return mem_st(nfp_prog, meta, 2); 1859 } 1860 1861 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1862 { 1863 return mem_st(nfp_prog, meta, 4); 1864 } 1865 1866 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1867 { 1868 return mem_st(nfp_prog, meta, 8); 1869 } 1870 1871 static int 1872 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1873 unsigned int size) 1874 { 1875 swreg off_reg; 1876 1877 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1878 1879 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1880 meta->insn.src_reg * 2, size); 1881 } 1882 1883 static int 1884 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1885 unsigned int size, unsigned int ptr_off) 1886 { 1887 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1888 meta->insn.src_reg * 2, meta->insn.dst_reg * 2, 1889 false, wrp_lmem_store); 1890 } 1891 1892 static int 1893 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1894 unsigned int size) 1895 { 1896 if (meta->ptr.type == PTR_TO_PACKET) 1897 return mem_stx_data(nfp_prog, meta, size); 1898 1899 if (meta->ptr.type == PTR_TO_STACK) 1900 return mem_stx_stack(nfp_prog, meta, size, 1901 meta->ptr.off + meta->ptr.var_off.value); 1902 1903 return -EOPNOTSUPP; 1904 } 1905 1906 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1907 { 1908 return mem_stx(nfp_prog, meta, 1); 1909 } 1910 1911 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1912 { 1913 return mem_stx(nfp_prog, meta, 2); 1914 } 1915 1916 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1917 { 1918 return mem_stx(nfp_prog, meta, 4); 1919 } 1920 1921 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1922 { 1923 return mem_stx(nfp_prog, meta, 8); 1924 } 1925 1926 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1927 { 1928 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); 1929 1930 return 0; 1931 } 1932 1933 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1934 { 1935 const struct bpf_insn *insn = &meta->insn; 1936 u64 imm = insn->imm; /* sign extend */ 1937 swreg or1, or2, tmp_reg; 1938 1939 or1 = reg_a(insn->dst_reg * 2); 1940 or2 = reg_b(insn->dst_reg * 2 + 1); 1941 1942 if (imm & ~0U) { 1943 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1944 emit_alu(nfp_prog, imm_a(nfp_prog), 1945 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1946 or1 = imm_a(nfp_prog); 1947 } 1948 1949 if (imm >> 32) { 1950 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1951 emit_alu(nfp_prog, imm_b(nfp_prog), 1952 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1953 or2 = imm_b(nfp_prog); 1954 } 1955 1956 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); 1957 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1958 1959 return 0; 1960 } 1961 1962 static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1963 { 1964 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true); 1965 } 1966 1967 static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1968 { 1969 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false); 1970 } 1971 1972 static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1973 { 1974 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false); 1975 } 1976 1977 static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1978 { 1979 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true); 1980 } 1981 1982 static int jsgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1983 { 1984 return wrp_cmp_imm(nfp_prog, meta, BR_BLT, true); 1985 } 1986 1987 static int jsge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1988 { 1989 return wrp_cmp_imm(nfp_prog, meta, BR_BGE, false); 1990 } 1991 1992 static int jslt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1993 { 1994 return wrp_cmp_imm(nfp_prog, meta, BR_BLT, false); 1995 } 1996 1997 static int jsle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1998 { 1999 return wrp_cmp_imm(nfp_prog, meta, BR_BGE, true); 2000 } 2001 2002 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2003 { 2004 const struct bpf_insn *insn = &meta->insn; 2005 u64 imm = insn->imm; /* sign extend */ 2006 swreg tmp_reg; 2007 2008 if (!imm) { 2009 meta->skip = true; 2010 return 0; 2011 } 2012 2013 if (imm & ~0U) { 2014 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 2015 emit_alu(nfp_prog, reg_none(), 2016 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg); 2017 emit_br(nfp_prog, BR_BNE, insn->off, 0); 2018 } 2019 2020 if (imm >> 32) { 2021 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 2022 emit_alu(nfp_prog, reg_none(), 2023 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg); 2024 emit_br(nfp_prog, BR_BNE, insn->off, 0); 2025 } 2026 2027 return 0; 2028 } 2029 2030 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2031 { 2032 const struct bpf_insn *insn = &meta->insn; 2033 u64 imm = insn->imm; /* sign extend */ 2034 swreg tmp_reg; 2035 2036 if (!imm) { 2037 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), 2038 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1)); 2039 emit_br(nfp_prog, BR_BNE, insn->off, 0); 2040 return 0; 2041 } 2042 2043 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 2044 emit_alu(nfp_prog, reg_none(), 2045 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 2046 emit_br(nfp_prog, BR_BNE, insn->off, 0); 2047 2048 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 2049 emit_alu(nfp_prog, reg_none(), 2050 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 2051 emit_br(nfp_prog, BR_BNE, insn->off, 0); 2052 2053 return 0; 2054 } 2055 2056 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2057 { 2058 const struct bpf_insn *insn = &meta->insn; 2059 2060 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), 2061 ALU_OP_XOR, reg_b(insn->src_reg * 2)); 2062 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1), 2063 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1)); 2064 emit_alu(nfp_prog, reg_none(), 2065 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog)); 2066 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 2067 2068 return 0; 2069 } 2070 2071 static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2072 { 2073 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true); 2074 } 2075 2076 static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2077 { 2078 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false); 2079 } 2080 2081 static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2082 { 2083 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false); 2084 } 2085 2086 static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2087 { 2088 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true); 2089 } 2090 2091 static int jsgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2092 { 2093 return wrp_cmp_reg(nfp_prog, meta, BR_BLT, true); 2094 } 2095 2096 static int jsge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2097 { 2098 return wrp_cmp_reg(nfp_prog, meta, BR_BGE, false); 2099 } 2100 2101 static int jslt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2102 { 2103 return wrp_cmp_reg(nfp_prog, meta, BR_BLT, false); 2104 } 2105 2106 static int jsle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2107 { 2108 return wrp_cmp_reg(nfp_prog, meta, BR_BGE, true); 2109 } 2110 2111 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2112 { 2113 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); 2114 } 2115 2116 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2117 { 2118 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); 2119 } 2120 2121 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2122 { 2123 switch (meta->insn.imm) { 2124 case BPF_FUNC_xdp_adjust_head: 2125 return adjust_head(nfp_prog, meta); 2126 case BPF_FUNC_map_lookup_elem: 2127 return map_lookup_stack(nfp_prog, meta); 2128 default: 2129 WARN_ONCE(1, "verifier allowed unsupported function\n"); 2130 return -EOPNOTSUPP; 2131 } 2132 } 2133 2134 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 2135 { 2136 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT); 2137 2138 return 0; 2139 } 2140 2141 static const instr_cb_t instr_cb[256] = { 2142 [BPF_ALU64 | BPF_MOV | BPF_X] = mov_reg64, 2143 [BPF_ALU64 | BPF_MOV | BPF_K] = mov_imm64, 2144 [BPF_ALU64 | BPF_XOR | BPF_X] = xor_reg64, 2145 [BPF_ALU64 | BPF_XOR | BPF_K] = xor_imm64, 2146 [BPF_ALU64 | BPF_AND | BPF_X] = and_reg64, 2147 [BPF_ALU64 | BPF_AND | BPF_K] = and_imm64, 2148 [BPF_ALU64 | BPF_OR | BPF_X] = or_reg64, 2149 [BPF_ALU64 | BPF_OR | BPF_K] = or_imm64, 2150 [BPF_ALU64 | BPF_ADD | BPF_X] = add_reg64, 2151 [BPF_ALU64 | BPF_ADD | BPF_K] = add_imm64, 2152 [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64, 2153 [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64, 2154 [BPF_ALU64 | BPF_NEG] = neg_reg64, 2155 [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64, 2156 [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64, 2157 [BPF_ALU | BPF_MOV | BPF_X] = mov_reg, 2158 [BPF_ALU | BPF_MOV | BPF_K] = mov_imm, 2159 [BPF_ALU | BPF_XOR | BPF_X] = xor_reg, 2160 [BPF_ALU | BPF_XOR | BPF_K] = xor_imm, 2161 [BPF_ALU | BPF_AND | BPF_X] = and_reg, 2162 [BPF_ALU | BPF_AND | BPF_K] = and_imm, 2163 [BPF_ALU | BPF_OR | BPF_X] = or_reg, 2164 [BPF_ALU | BPF_OR | BPF_K] = or_imm, 2165 [BPF_ALU | BPF_ADD | BPF_X] = add_reg, 2166 [BPF_ALU | BPF_ADD | BPF_K] = add_imm, 2167 [BPF_ALU | BPF_SUB | BPF_X] = sub_reg, 2168 [BPF_ALU | BPF_SUB | BPF_K] = sub_imm, 2169 [BPF_ALU | BPF_NEG] = neg_reg, 2170 [BPF_ALU | BPF_LSH | BPF_K] = shl_imm, 2171 [BPF_ALU | BPF_END | BPF_X] = end_reg32, 2172 [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8, 2173 [BPF_LD | BPF_ABS | BPF_B] = data_ld1, 2174 [BPF_LD | BPF_ABS | BPF_H] = data_ld2, 2175 [BPF_LD | BPF_ABS | BPF_W] = data_ld4, 2176 [BPF_LD | BPF_IND | BPF_B] = data_ind_ld1, 2177 [BPF_LD | BPF_IND | BPF_H] = data_ind_ld2, 2178 [BPF_LD | BPF_IND | BPF_W] = data_ind_ld4, 2179 [BPF_LDX | BPF_MEM | BPF_B] = mem_ldx1, 2180 [BPF_LDX | BPF_MEM | BPF_H] = mem_ldx2, 2181 [BPF_LDX | BPF_MEM | BPF_W] = mem_ldx4, 2182 [BPF_LDX | BPF_MEM | BPF_DW] = mem_ldx8, 2183 [BPF_STX | BPF_MEM | BPF_B] = mem_stx1, 2184 [BPF_STX | BPF_MEM | BPF_H] = mem_stx2, 2185 [BPF_STX | BPF_MEM | BPF_W] = mem_stx4, 2186 [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8, 2187 [BPF_ST | BPF_MEM | BPF_B] = mem_st1, 2188 [BPF_ST | BPF_MEM | BPF_H] = mem_st2, 2189 [BPF_ST | BPF_MEM | BPF_W] = mem_st4, 2190 [BPF_ST | BPF_MEM | BPF_DW] = mem_st8, 2191 [BPF_JMP | BPF_JA | BPF_K] = jump, 2192 [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm, 2193 [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm, 2194 [BPF_JMP | BPF_JGE | BPF_K] = jge_imm, 2195 [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm, 2196 [BPF_JMP | BPF_JLE | BPF_K] = jle_imm, 2197 [BPF_JMP | BPF_JSGT | BPF_K] = jsgt_imm, 2198 [BPF_JMP | BPF_JSGE | BPF_K] = jsge_imm, 2199 [BPF_JMP | BPF_JSLT | BPF_K] = jslt_imm, 2200 [BPF_JMP | BPF_JSLE | BPF_K] = jsle_imm, 2201 [BPF_JMP | BPF_JSET | BPF_K] = jset_imm, 2202 [BPF_JMP | BPF_JNE | BPF_K] = jne_imm, 2203 [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg, 2204 [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg, 2205 [BPF_JMP | BPF_JGE | BPF_X] = jge_reg, 2206 [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg, 2207 [BPF_JMP | BPF_JLE | BPF_X] = jle_reg, 2208 [BPF_JMP | BPF_JSGT | BPF_X] = jsgt_reg, 2209 [BPF_JMP | BPF_JSGE | BPF_X] = jsge_reg, 2210 [BPF_JMP | BPF_JSLT | BPF_X] = jslt_reg, 2211 [BPF_JMP | BPF_JSLE | BPF_X] = jsle_reg, 2212 [BPF_JMP | BPF_JSET | BPF_X] = jset_reg, 2213 [BPF_JMP | BPF_JNE | BPF_X] = jne_reg, 2214 [BPF_JMP | BPF_CALL] = call, 2215 [BPF_JMP | BPF_EXIT] = goto_out, 2216 }; 2217 2218 /* --- Assembler logic --- */ 2219 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 2220 { 2221 struct nfp_insn_meta *meta, *jmp_dst; 2222 u32 idx, br_idx; 2223 2224 list_for_each_entry(meta, &nfp_prog->insns, l) { 2225 if (meta->skip) 2226 continue; 2227 if (meta->insn.code == (BPF_JMP | BPF_CALL)) 2228 continue; 2229 if (BPF_CLASS(meta->insn.code) != BPF_JMP) 2230 continue; 2231 2232 if (list_is_last(&meta->l, &nfp_prog->insns)) 2233 br_idx = nfp_prog->last_bpf_off; 2234 else 2235 br_idx = list_next_entry(meta, l)->off - 1; 2236 2237 if (!nfp_is_br(nfp_prog->prog[br_idx])) { 2238 pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n", 2239 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); 2240 return -ELOOP; 2241 } 2242 /* Leave special branches for later */ 2243 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != 2244 RELO_BR_REL) 2245 continue; 2246 2247 if (!meta->jmp_dst) { 2248 pr_err("Non-exit jump doesn't have destination info recorded!!\n"); 2249 return -ELOOP; 2250 } 2251 2252 jmp_dst = meta->jmp_dst; 2253 2254 if (jmp_dst->skip) { 2255 pr_err("Branch landing on removed instruction!!\n"); 2256 return -ELOOP; 2257 } 2258 2259 for (idx = meta->off; idx <= br_idx; idx++) { 2260 if (!nfp_is_br(nfp_prog->prog[idx])) 2261 continue; 2262 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); 2263 } 2264 } 2265 2266 return 0; 2267 } 2268 2269 static void nfp_intro(struct nfp_prog *nfp_prog) 2270 { 2271 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); 2272 emit_alu(nfp_prog, plen_reg(nfp_prog), 2273 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); 2274 } 2275 2276 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 2277 { 2278 /* TC direct-action mode: 2279 * 0,1 ok NOT SUPPORTED[1] 2280 * 2 drop 0x22 -> drop, count as stat1 2281 * 4,5 nuke 0x02 -> drop 2282 * 7 redir 0x44 -> redir, count as stat2 2283 * * unspec 0x11 -> pass, count as stat0 2284 * 2285 * [1] We can't support OK and RECLASSIFY because we can't tell TC 2286 * the exact decision made. We are forced to support UNSPEC 2287 * to handle aborts so that's the only one we handle for passing 2288 * packets up the stack. 2289 */ 2290 /* Target for aborts */ 2291 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 2292 2293 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); 2294 2295 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2296 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); 2297 2298 /* Target for normal exits */ 2299 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 2300 2301 /* if R0 > 7 jump to abort */ 2302 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); 2303 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 2304 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2305 2306 wrp_immed(nfp_prog, reg_b(2), 0x41221211); 2307 wrp_immed(nfp_prog, reg_b(3), 0x41001211); 2308 2309 emit_shf(nfp_prog, reg_a(1), 2310 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2); 2311 2312 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2313 emit_shf(nfp_prog, reg_a(2), 2314 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 2315 2316 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2317 emit_shf(nfp_prog, reg_b(2), 2318 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0); 2319 2320 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); 2321 2322 emit_shf(nfp_prog, reg_b(2), 2323 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4); 2324 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2325 } 2326 2327 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 2328 { 2329 /* XDP return codes: 2330 * 0 aborted 0x82 -> drop, count as stat3 2331 * 1 drop 0x22 -> drop, count as stat1 2332 * 2 pass 0x11 -> pass, count as stat0 2333 * 3 tx 0x44 -> redir, count as stat2 2334 * * unknown 0x82 -> drop, count as stat3 2335 */ 2336 /* Target for aborts */ 2337 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 2338 2339 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); 2340 2341 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2342 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); 2343 2344 /* Target for normal exits */ 2345 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 2346 2347 /* if R0 > 3 jump to abort */ 2348 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); 2349 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 2350 2351 wrp_immed(nfp_prog, reg_b(2), 0x44112282); 2352 2353 emit_shf(nfp_prog, reg_a(1), 2354 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3); 2355 2356 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2357 emit_shf(nfp_prog, reg_b(2), 2358 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 2359 2360 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); 2361 2362 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2363 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2364 } 2365 2366 static void nfp_outro(struct nfp_prog *nfp_prog) 2367 { 2368 switch (nfp_prog->type) { 2369 case BPF_PROG_TYPE_SCHED_CLS: 2370 nfp_outro_tc_da(nfp_prog); 2371 break; 2372 case BPF_PROG_TYPE_XDP: 2373 nfp_outro_xdp(nfp_prog); 2374 break; 2375 default: 2376 WARN_ON(1); 2377 } 2378 } 2379 2380 static int nfp_translate(struct nfp_prog *nfp_prog) 2381 { 2382 struct nfp_insn_meta *meta; 2383 int err; 2384 2385 nfp_intro(nfp_prog); 2386 if (nfp_prog->error) 2387 return nfp_prog->error; 2388 2389 list_for_each_entry(meta, &nfp_prog->insns, l) { 2390 instr_cb_t cb = instr_cb[meta->insn.code]; 2391 2392 meta->off = nfp_prog_current_offset(nfp_prog); 2393 2394 if (meta->skip) { 2395 nfp_prog->n_translated++; 2396 continue; 2397 } 2398 2399 if (nfp_meta_has_prev(nfp_prog, meta) && 2400 nfp_meta_prev(meta)->double_cb) 2401 cb = nfp_meta_prev(meta)->double_cb; 2402 if (!cb) 2403 return -ENOENT; 2404 err = cb(nfp_prog, meta); 2405 if (err) 2406 return err; 2407 2408 nfp_prog->n_translated++; 2409 } 2410 2411 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; 2412 2413 nfp_outro(nfp_prog); 2414 if (nfp_prog->error) 2415 return nfp_prog->error; 2416 2417 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); 2418 if (nfp_prog->error) 2419 return nfp_prog->error; 2420 2421 return nfp_fixup_branches(nfp_prog); 2422 } 2423 2424 /* --- Optimizations --- */ 2425 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) 2426 { 2427 struct nfp_insn_meta *meta; 2428 2429 list_for_each_entry(meta, &nfp_prog->insns, l) { 2430 struct bpf_insn insn = meta->insn; 2431 2432 /* Programs converted from cBPF start with register xoring */ 2433 if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) && 2434 insn.src_reg == insn.dst_reg) 2435 continue; 2436 2437 /* Programs start with R6 = R1 but we ignore the skb pointer */ 2438 if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) && 2439 insn.src_reg == 1 && insn.dst_reg == 6) 2440 meta->skip = true; 2441 2442 /* Return as soon as something doesn't match */ 2443 if (!meta->skip) 2444 return; 2445 } 2446 } 2447 2448 /* Remove masking after load since our load guarantees this is not needed */ 2449 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) 2450 { 2451 struct nfp_insn_meta *meta1, *meta2; 2452 const s32 exp_mask[] = { 2453 [BPF_B] = 0x000000ffU, 2454 [BPF_H] = 0x0000ffffU, 2455 [BPF_W] = 0xffffffffU, 2456 }; 2457 2458 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2459 struct bpf_insn insn, next; 2460 2461 insn = meta1->insn; 2462 next = meta2->insn; 2463 2464 if (BPF_CLASS(insn.code) != BPF_LD) 2465 continue; 2466 if (BPF_MODE(insn.code) != BPF_ABS && 2467 BPF_MODE(insn.code) != BPF_IND) 2468 continue; 2469 2470 if (next.code != (BPF_ALU64 | BPF_AND | BPF_K)) 2471 continue; 2472 2473 if (!exp_mask[BPF_SIZE(insn.code)]) 2474 continue; 2475 if (exp_mask[BPF_SIZE(insn.code)] != next.imm) 2476 continue; 2477 2478 if (next.src_reg || next.dst_reg) 2479 continue; 2480 2481 if (meta2->flags & FLAG_INSN_IS_JUMP_DST) 2482 continue; 2483 2484 meta2->skip = true; 2485 } 2486 } 2487 2488 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) 2489 { 2490 struct nfp_insn_meta *meta1, *meta2, *meta3; 2491 2492 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { 2493 struct bpf_insn insn, next1, next2; 2494 2495 insn = meta1->insn; 2496 next1 = meta2->insn; 2497 next2 = meta3->insn; 2498 2499 if (BPF_CLASS(insn.code) != BPF_LD) 2500 continue; 2501 if (BPF_MODE(insn.code) != BPF_ABS && 2502 BPF_MODE(insn.code) != BPF_IND) 2503 continue; 2504 if (BPF_SIZE(insn.code) != BPF_W) 2505 continue; 2506 2507 if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) && 2508 next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) && 2509 !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) && 2510 next2.code == (BPF_LSH | BPF_K | BPF_ALU64))) 2511 continue; 2512 2513 if (next1.src_reg || next1.dst_reg || 2514 next2.src_reg || next2.dst_reg) 2515 continue; 2516 2517 if (next1.imm != 0x20 || next2.imm != 0x20) 2518 continue; 2519 2520 if (meta2->flags & FLAG_INSN_IS_JUMP_DST || 2521 meta3->flags & FLAG_INSN_IS_JUMP_DST) 2522 continue; 2523 2524 meta2->skip = true; 2525 meta3->skip = true; 2526 } 2527 } 2528 2529 /* load/store pair that forms memory copy sould look like the following: 2530 * 2531 * ld_width R, [addr_src + offset_src] 2532 * st_width [addr_dest + offset_dest], R 2533 * 2534 * The destination register of load and source register of store should 2535 * be the same, load and store should also perform at the same width. 2536 * If either of addr_src or addr_dest is stack pointer, we don't do the 2537 * CPP optimization as stack is modelled by registers on NFP. 2538 */ 2539 static bool 2540 curr_pair_is_memcpy(struct nfp_insn_meta *ld_meta, 2541 struct nfp_insn_meta *st_meta) 2542 { 2543 struct bpf_insn *ld = &ld_meta->insn; 2544 struct bpf_insn *st = &st_meta->insn; 2545 2546 if (!is_mbpf_load(ld_meta) || !is_mbpf_store(st_meta)) 2547 return false; 2548 2549 if (ld_meta->ptr.type != PTR_TO_PACKET) 2550 return false; 2551 2552 if (st_meta->ptr.type != PTR_TO_PACKET) 2553 return false; 2554 2555 if (BPF_SIZE(ld->code) != BPF_SIZE(st->code)) 2556 return false; 2557 2558 if (ld->dst_reg != st->src_reg) 2559 return false; 2560 2561 /* There is jump to the store insn in this pair. */ 2562 if (st_meta->flags & FLAG_INSN_IS_JUMP_DST) 2563 return false; 2564 2565 return true; 2566 } 2567 2568 /* Currently, we only support chaining load/store pairs if: 2569 * 2570 * - Their address base registers are the same. 2571 * - Their address offsets are in the same order. 2572 * - They operate at the same memory width. 2573 * - There is no jump into the middle of them. 2574 */ 2575 static bool 2576 curr_pair_chain_with_previous(struct nfp_insn_meta *ld_meta, 2577 struct nfp_insn_meta *st_meta, 2578 struct bpf_insn *prev_ld, 2579 struct bpf_insn *prev_st) 2580 { 2581 u8 prev_size, curr_size, prev_ld_base, prev_st_base, prev_ld_dst; 2582 struct bpf_insn *ld = &ld_meta->insn; 2583 struct bpf_insn *st = &st_meta->insn; 2584 s16 prev_ld_off, prev_st_off; 2585 2586 /* This pair is the start pair. */ 2587 if (!prev_ld) 2588 return true; 2589 2590 prev_size = BPF_LDST_BYTES(prev_ld); 2591 curr_size = BPF_LDST_BYTES(ld); 2592 prev_ld_base = prev_ld->src_reg; 2593 prev_st_base = prev_st->dst_reg; 2594 prev_ld_dst = prev_ld->dst_reg; 2595 prev_ld_off = prev_ld->off; 2596 prev_st_off = prev_st->off; 2597 2598 if (ld->dst_reg != prev_ld_dst) 2599 return false; 2600 2601 if (ld->src_reg != prev_ld_base || st->dst_reg != prev_st_base) 2602 return false; 2603 2604 if (curr_size != prev_size) 2605 return false; 2606 2607 /* There is jump to the head of this pair. */ 2608 if (ld_meta->flags & FLAG_INSN_IS_JUMP_DST) 2609 return false; 2610 2611 /* Both in ascending order. */ 2612 if (prev_ld_off + prev_size == ld->off && 2613 prev_st_off + prev_size == st->off) 2614 return true; 2615 2616 /* Both in descending order. */ 2617 if (ld->off + curr_size == prev_ld_off && 2618 st->off + curr_size == prev_st_off) 2619 return true; 2620 2621 return false; 2622 } 2623 2624 /* Return TRUE if cross memory access happens. Cross memory access means 2625 * store area is overlapping with load area that a later load might load 2626 * the value from previous store, for this case we can't treat the sequence 2627 * as an memory copy. 2628 */ 2629 static bool 2630 cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta, 2631 struct nfp_insn_meta *head_st_meta) 2632 { 2633 s16 head_ld_off, head_st_off, ld_off; 2634 2635 /* Different pointer types does not overlap. */ 2636 if (head_ld_meta->ptr.type != head_st_meta->ptr.type) 2637 return false; 2638 2639 /* load and store are both PTR_TO_PACKET, check ID info. */ 2640 if (head_ld_meta->ptr.id != head_st_meta->ptr.id) 2641 return true; 2642 2643 /* Canonicalize the offsets. Turn all of them against the original 2644 * base register. 2645 */ 2646 head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off; 2647 head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off; 2648 ld_off = ld->off + head_ld_meta->ptr.off; 2649 2650 /* Ascending order cross. */ 2651 if (ld_off > head_ld_off && 2652 head_ld_off < head_st_off && ld_off >= head_st_off) 2653 return true; 2654 2655 /* Descending order cross. */ 2656 if (ld_off < head_ld_off && 2657 head_ld_off > head_st_off && ld_off <= head_st_off) 2658 return true; 2659 2660 return false; 2661 } 2662 2663 /* This pass try to identify the following instructoin sequences. 2664 * 2665 * load R, [regA + offA] 2666 * store [regB + offB], R 2667 * load R, [regA + offA + const_imm_A] 2668 * store [regB + offB + const_imm_A], R 2669 * load R, [regA + offA + 2 * const_imm_A] 2670 * store [regB + offB + 2 * const_imm_A], R 2671 * ... 2672 * 2673 * Above sequence is typically generated by compiler when lowering 2674 * memcpy. NFP prefer using CPP instructions to accelerate it. 2675 */ 2676 static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) 2677 { 2678 struct nfp_insn_meta *head_ld_meta = NULL; 2679 struct nfp_insn_meta *head_st_meta = NULL; 2680 struct nfp_insn_meta *meta1, *meta2; 2681 struct bpf_insn *prev_ld = NULL; 2682 struct bpf_insn *prev_st = NULL; 2683 u8 count = 0; 2684 2685 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2686 struct bpf_insn *ld = &meta1->insn; 2687 struct bpf_insn *st = &meta2->insn; 2688 2689 /* Reset record status if any of the following if true: 2690 * - The current insn pair is not load/store. 2691 * - The load/store pair doesn't chain with previous one. 2692 * - The chained load/store pair crossed with previous pair. 2693 * - The chained load/store pair has a total size of memory 2694 * copy beyond 128 bytes which is the maximum length a 2695 * single NFP CPP command can transfer. 2696 */ 2697 if (!curr_pair_is_memcpy(meta1, meta2) || 2698 !curr_pair_chain_with_previous(meta1, meta2, prev_ld, 2699 prev_st) || 2700 (head_ld_meta && (cross_mem_access(ld, head_ld_meta, 2701 head_st_meta) || 2702 head_ld_meta->ldst_gather_len >= 128))) { 2703 if (!count) 2704 continue; 2705 2706 if (count > 1) { 2707 s16 prev_ld_off = prev_ld->off; 2708 s16 prev_st_off = prev_st->off; 2709 s16 head_ld_off = head_ld_meta->insn.off; 2710 2711 if (prev_ld_off < head_ld_off) { 2712 head_ld_meta->insn.off = prev_ld_off; 2713 head_st_meta->insn.off = prev_st_off; 2714 head_ld_meta->ldst_gather_len = 2715 -head_ld_meta->ldst_gather_len; 2716 } 2717 2718 head_ld_meta->paired_st = &head_st_meta->insn; 2719 head_st_meta->skip = true; 2720 } else { 2721 head_ld_meta->ldst_gather_len = 0; 2722 } 2723 2724 /* If the chain is ended by an load/store pair then this 2725 * could serve as the new head of the the next chain. 2726 */ 2727 if (curr_pair_is_memcpy(meta1, meta2)) { 2728 head_ld_meta = meta1; 2729 head_st_meta = meta2; 2730 head_ld_meta->ldst_gather_len = 2731 BPF_LDST_BYTES(ld); 2732 meta1 = nfp_meta_next(meta1); 2733 meta2 = nfp_meta_next(meta2); 2734 prev_ld = ld; 2735 prev_st = st; 2736 count = 1; 2737 } else { 2738 head_ld_meta = NULL; 2739 head_st_meta = NULL; 2740 prev_ld = NULL; 2741 prev_st = NULL; 2742 count = 0; 2743 } 2744 2745 continue; 2746 } 2747 2748 if (!head_ld_meta) { 2749 head_ld_meta = meta1; 2750 head_st_meta = meta2; 2751 } else { 2752 meta1->skip = true; 2753 meta2->skip = true; 2754 } 2755 2756 head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld); 2757 meta1 = nfp_meta_next(meta1); 2758 meta2 = nfp_meta_next(meta2); 2759 prev_ld = ld; 2760 prev_st = st; 2761 count++; 2762 } 2763 } 2764 2765 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) 2766 { 2767 nfp_bpf_opt_reg_init(nfp_prog); 2768 2769 nfp_bpf_opt_ld_mask(nfp_prog); 2770 nfp_bpf_opt_ld_shift(nfp_prog); 2771 nfp_bpf_opt_ldst_gather(nfp_prog); 2772 2773 return 0; 2774 } 2775 2776 static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len) 2777 { 2778 __le64 *ustore = (__force __le64 *)prog; 2779 int i; 2780 2781 for (i = 0; i < len; i++) { 2782 int err; 2783 2784 err = nfp_ustore_check_valid_no_ecc(prog[i]); 2785 if (err) 2786 return err; 2787 2788 ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i])); 2789 } 2790 2791 return 0; 2792 } 2793 2794 static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog) 2795 { 2796 void *prog; 2797 2798 prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL); 2799 if (!prog) 2800 return; 2801 2802 nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64); 2803 memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len); 2804 kvfree(nfp_prog->prog); 2805 nfp_prog->prog = prog; 2806 } 2807 2808 int nfp_bpf_jit(struct nfp_prog *nfp_prog) 2809 { 2810 int ret; 2811 2812 ret = nfp_bpf_optimize(nfp_prog); 2813 if (ret) 2814 return ret; 2815 2816 ret = nfp_translate(nfp_prog); 2817 if (ret) { 2818 pr_err("Translation failed with error %d (translated: %u)\n", 2819 ret, nfp_prog->n_translated); 2820 return -EINVAL; 2821 } 2822 2823 nfp_bpf_prog_trim(nfp_prog); 2824 2825 return ret; 2826 } 2827 2828 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt) 2829 { 2830 struct nfp_insn_meta *meta; 2831 2832 /* Another pass to record jump information. */ 2833 list_for_each_entry(meta, &nfp_prog->insns, l) { 2834 u64 code = meta->insn.code; 2835 2836 if (BPF_CLASS(code) == BPF_JMP && BPF_OP(code) != BPF_EXIT && 2837 BPF_OP(code) != BPF_CALL) { 2838 struct nfp_insn_meta *dst_meta; 2839 unsigned short dst_indx; 2840 2841 dst_indx = meta->n + 1 + meta->insn.off; 2842 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx, 2843 cnt); 2844 2845 meta->jmp_dst = dst_meta; 2846 dst_meta->flags |= FLAG_INSN_IS_JUMP_DST; 2847 } 2848 } 2849 } 2850 2851 void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv) 2852 { 2853 unsigned int i; 2854 u64 *prog; 2855 int err; 2856 2857 prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64), 2858 GFP_KERNEL); 2859 if (!prog) 2860 return ERR_PTR(-ENOMEM); 2861 2862 for (i = 0; i < nfp_prog->prog_len; i++) { 2863 enum nfp_relo_type special; 2864 u32 val; 2865 2866 special = FIELD_GET(OP_RELO_TYPE, prog[i]); 2867 switch (special) { 2868 case RELO_NONE: 2869 continue; 2870 case RELO_BR_REL: 2871 br_add_offset(&prog[i], bv->start_off); 2872 break; 2873 case RELO_BR_GO_OUT: 2874 br_set_offset(&prog[i], 2875 nfp_prog->tgt_out + bv->start_off); 2876 break; 2877 case RELO_BR_GO_ABORT: 2878 br_set_offset(&prog[i], 2879 nfp_prog->tgt_abort + bv->start_off); 2880 break; 2881 case RELO_BR_NEXT_PKT: 2882 br_set_offset(&prog[i], bv->tgt_done); 2883 break; 2884 case RELO_BR_HELPER: 2885 val = br_get_offset(prog[i]); 2886 val -= BR_OFF_RELO; 2887 switch (val) { 2888 case BPF_FUNC_map_lookup_elem: 2889 val = nfp_prog->bpf->helpers.map_lookup; 2890 break; 2891 default: 2892 pr_err("relocation of unknown helper %d\n", 2893 val); 2894 err = -EINVAL; 2895 goto err_free_prog; 2896 } 2897 br_set_offset(&prog[i], val); 2898 break; 2899 case RELO_IMMED_REL: 2900 immed_add_value(&prog[i], bv->start_off); 2901 break; 2902 } 2903 2904 prog[i] &= ~OP_RELO_TYPE; 2905 } 2906 2907 err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len); 2908 if (err) 2909 goto err_free_prog; 2910 2911 return prog; 2912 2913 err_free_prog: 2914 kfree(prog); 2915 return ERR_PTR(err); 2916 } 2917