1 /* 2 * Copyright (C) 2016-2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #define pr_fmt(fmt) "NFP net bpf: " fmt 35 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/filter.h> 39 #include <linux/pkt_cls.h> 40 #include <linux/unistd.h> 41 42 #include "main.h" 43 #include "../nfp_asm.h" 44 45 /* --- NFP prog --- */ 46 /* Foreach "multiple" entries macros provide pos and next<n> pointers. 47 * It's safe to modify the next pointers (but not pos). 48 */ 49 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ 50 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 51 next = list_next_entry(pos, l); \ 52 &(nfp_prog)->insns != &pos->l && \ 53 &(nfp_prog)->insns != &next->l; \ 54 pos = nfp_meta_next(pos), \ 55 next = nfp_meta_next(pos)) 56 57 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ 58 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 59 next = list_next_entry(pos, l), \ 60 next2 = list_next_entry(next, l); \ 61 &(nfp_prog)->insns != &pos->l && \ 62 &(nfp_prog)->insns != &next->l && \ 63 &(nfp_prog)->insns != &next2->l; \ 64 pos = nfp_meta_next(pos), \ 65 next = nfp_meta_next(pos), \ 66 next2 = nfp_meta_next(next)) 67 68 static bool 69 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 70 { 71 return meta->l.prev != &nfp_prog->insns; 72 } 73 74 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) 75 { 76 if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) { 77 nfp_prog->error = -ENOSPC; 78 return; 79 } 80 81 nfp_prog->prog[nfp_prog->prog_len] = insn; 82 nfp_prog->prog_len++; 83 } 84 85 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) 86 { 87 return nfp_prog->start_off + nfp_prog->prog_len; 88 } 89 90 static unsigned int 91 nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset) 92 { 93 return offset - nfp_prog->start_off; 94 } 95 96 /* --- Emitters --- */ 97 static void 98 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, 99 u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync, bool indir) 100 { 101 enum cmd_ctx_swap ctx; 102 u64 insn; 103 104 if (sync) 105 ctx = CMD_CTX_SWAP; 106 else 107 ctx = CMD_CTX_NO_SWAP; 108 109 insn = FIELD_PREP(OP_CMD_A_SRC, areg) | 110 FIELD_PREP(OP_CMD_CTX, ctx) | 111 FIELD_PREP(OP_CMD_B_SRC, breg) | 112 FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) | 113 FIELD_PREP(OP_CMD_XFER, xfer) | 114 FIELD_PREP(OP_CMD_CNT, size) | 115 FIELD_PREP(OP_CMD_SIG, sync) | 116 FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) | 117 FIELD_PREP(OP_CMD_INDIR, indir) | 118 FIELD_PREP(OP_CMD_MODE, mode); 119 120 nfp_prog_push(nfp_prog, insn); 121 } 122 123 static void 124 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 125 swreg lreg, swreg rreg, u8 size, bool sync, bool indir) 126 { 127 struct nfp_insn_re_regs reg; 128 int err; 129 130 err = swreg_to_restricted(reg_none(), lreg, rreg, ®, false); 131 if (err) { 132 nfp_prog->error = err; 133 return; 134 } 135 if (reg.swap) { 136 pr_err("cmd can't swap arguments\n"); 137 nfp_prog->error = -EFAULT; 138 return; 139 } 140 if (reg.dst_lmextn || reg.src_lmextn) { 141 pr_err("cmd can't use LMextn\n"); 142 nfp_prog->error = -EFAULT; 143 return; 144 } 145 146 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync, 147 indir); 148 } 149 150 static void 151 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 152 swreg lreg, swreg rreg, u8 size, bool sync) 153 { 154 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, false); 155 } 156 157 static void 158 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, 159 enum br_ctx_signal_state css, u16 addr, u8 defer) 160 { 161 u16 addr_lo, addr_hi; 162 u64 insn; 163 164 addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 165 addr_hi = addr != addr_lo; 166 167 insn = OP_BR_BASE | 168 FIELD_PREP(OP_BR_MASK, mask) | 169 FIELD_PREP(OP_BR_EV_PIP, ev_pip) | 170 FIELD_PREP(OP_BR_CSS, css) | 171 FIELD_PREP(OP_BR_DEFBR, defer) | 172 FIELD_PREP(OP_BR_ADDR_LO, addr_lo) | 173 FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 174 175 nfp_prog_push(nfp_prog, insn); 176 } 177 178 static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer) 179 { 180 if (defer > 2) { 181 pr_err("BUG: branch defer out of bounds %d\n", defer); 182 nfp_prog->error = -EFAULT; 183 return; 184 } 185 __emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer); 186 } 187 188 static void 189 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) 190 { 191 __emit_br(nfp_prog, mask, 192 mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND, 193 BR_CSS_NONE, addr, defer); 194 } 195 196 static void 197 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, 198 enum immed_width width, bool invert, 199 enum immed_shift shift, bool wr_both, 200 bool dst_lmextn, bool src_lmextn) 201 { 202 u64 insn; 203 204 insn = OP_IMMED_BASE | 205 FIELD_PREP(OP_IMMED_A_SRC, areg) | 206 FIELD_PREP(OP_IMMED_B_SRC, breg) | 207 FIELD_PREP(OP_IMMED_IMM, imm_hi) | 208 FIELD_PREP(OP_IMMED_WIDTH, width) | 209 FIELD_PREP(OP_IMMED_INV, invert) | 210 FIELD_PREP(OP_IMMED_SHIFT, shift) | 211 FIELD_PREP(OP_IMMED_WR_AB, wr_both) | 212 FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) | 213 FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn); 214 215 nfp_prog_push(nfp_prog, insn); 216 } 217 218 static void 219 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, 220 enum immed_width width, bool invert, enum immed_shift shift) 221 { 222 struct nfp_insn_ur_regs reg; 223 int err; 224 225 if (swreg_type(dst) == NN_REG_IMM) { 226 nfp_prog->error = -EFAULT; 227 return; 228 } 229 230 err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), ®); 231 if (err) { 232 nfp_prog->error = err; 233 return; 234 } 235 236 /* Use reg.dst when destination is No-Dest. */ 237 __emit_immed(nfp_prog, 238 swreg_type(dst) == NN_REG_NONE ? reg.dst : reg.areg, 239 reg.breg, imm >> 8, width, invert, shift, 240 reg.wr_both, reg.dst_lmextn, reg.src_lmextn); 241 } 242 243 static void 244 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 245 enum shf_sc sc, u8 shift, 246 u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both, 247 bool dst_lmextn, bool src_lmextn) 248 { 249 u64 insn; 250 251 if (!FIELD_FIT(OP_SHF_SHIFT, shift)) { 252 nfp_prog->error = -EFAULT; 253 return; 254 } 255 256 if (sc == SHF_SC_L_SHF) 257 shift = 32 - shift; 258 259 insn = OP_SHF_BASE | 260 FIELD_PREP(OP_SHF_A_SRC, areg) | 261 FIELD_PREP(OP_SHF_SC, sc) | 262 FIELD_PREP(OP_SHF_B_SRC, breg) | 263 FIELD_PREP(OP_SHF_I8, i8) | 264 FIELD_PREP(OP_SHF_SW, sw) | 265 FIELD_PREP(OP_SHF_DST, dst) | 266 FIELD_PREP(OP_SHF_SHIFT, shift) | 267 FIELD_PREP(OP_SHF_OP, op) | 268 FIELD_PREP(OP_SHF_DST_AB, dst_ab) | 269 FIELD_PREP(OP_SHF_WR_AB, wr_both) | 270 FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) | 271 FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn); 272 273 nfp_prog_push(nfp_prog, insn); 274 } 275 276 static void 277 emit_shf(struct nfp_prog *nfp_prog, swreg dst, 278 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift) 279 { 280 struct nfp_insn_re_regs reg; 281 int err; 282 283 err = swreg_to_restricted(dst, lreg, rreg, ®, true); 284 if (err) { 285 nfp_prog->error = err; 286 return; 287 } 288 289 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, 290 reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both, 291 reg.dst_lmextn, reg.src_lmextn); 292 } 293 294 static void 295 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 296 u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both, 297 bool dst_lmextn, bool src_lmextn) 298 { 299 u64 insn; 300 301 insn = OP_ALU_BASE | 302 FIELD_PREP(OP_ALU_A_SRC, areg) | 303 FIELD_PREP(OP_ALU_B_SRC, breg) | 304 FIELD_PREP(OP_ALU_DST, dst) | 305 FIELD_PREP(OP_ALU_SW, swap) | 306 FIELD_PREP(OP_ALU_OP, op) | 307 FIELD_PREP(OP_ALU_DST_AB, dst_ab) | 308 FIELD_PREP(OP_ALU_WR_AB, wr_both) | 309 FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) | 310 FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn); 311 312 nfp_prog_push(nfp_prog, insn); 313 } 314 315 static void 316 emit_alu(struct nfp_prog *nfp_prog, swreg dst, 317 swreg lreg, enum alu_op op, swreg rreg) 318 { 319 struct nfp_insn_ur_regs reg; 320 int err; 321 322 err = swreg_to_unrestricted(dst, lreg, rreg, ®); 323 if (err) { 324 nfp_prog->error = err; 325 return; 326 } 327 328 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, 329 reg.areg, op, reg.breg, reg.swap, reg.wr_both, 330 reg.dst_lmextn, reg.src_lmextn); 331 } 332 333 static void 334 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, 335 u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8, 336 bool zero, bool swap, bool wr_both, 337 bool dst_lmextn, bool src_lmextn) 338 { 339 u64 insn; 340 341 insn = OP_LDF_BASE | 342 FIELD_PREP(OP_LDF_A_SRC, areg) | 343 FIELD_PREP(OP_LDF_SC, sc) | 344 FIELD_PREP(OP_LDF_B_SRC, breg) | 345 FIELD_PREP(OP_LDF_I8, imm8) | 346 FIELD_PREP(OP_LDF_SW, swap) | 347 FIELD_PREP(OP_LDF_ZF, zero) | 348 FIELD_PREP(OP_LDF_BMASK, bmask) | 349 FIELD_PREP(OP_LDF_SHF, shift) | 350 FIELD_PREP(OP_LDF_WR_AB, wr_both) | 351 FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) | 352 FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn); 353 354 nfp_prog_push(nfp_prog, insn); 355 } 356 357 static void 358 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 359 enum shf_sc sc, u8 shift, bool zero) 360 { 361 struct nfp_insn_re_regs reg; 362 int err; 363 364 /* Note: ld_field is special as it uses one of the src regs as dst */ 365 err = swreg_to_restricted(dst, dst, src, ®, true); 366 if (err) { 367 nfp_prog->error = err; 368 return; 369 } 370 371 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, 372 reg.i8, zero, reg.swap, reg.wr_both, 373 reg.dst_lmextn, reg.src_lmextn); 374 } 375 376 static void 377 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 378 enum shf_sc sc, u8 shift) 379 { 380 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); 381 } 382 383 static void 384 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, 385 bool dst_lmextn, bool src_lmextn) 386 { 387 u64 insn; 388 389 insn = OP_LCSR_BASE | 390 FIELD_PREP(OP_LCSR_A_SRC, areg) | 391 FIELD_PREP(OP_LCSR_B_SRC, breg) | 392 FIELD_PREP(OP_LCSR_WRITE, wr) | 393 FIELD_PREP(OP_LCSR_ADDR, addr) | 394 FIELD_PREP(OP_LCSR_SRC_LMEXTN, src_lmextn) | 395 FIELD_PREP(OP_LCSR_DST_LMEXTN, dst_lmextn); 396 397 nfp_prog_push(nfp_prog, insn); 398 } 399 400 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) 401 { 402 struct nfp_insn_ur_regs reg; 403 int err; 404 405 /* This instruction takes immeds instead of reg_none() for the ignored 406 * operand, but we can't encode 2 immeds in one instr with our normal 407 * swreg infra so if param is an immed, we encode as reg_none() and 408 * copy the immed to both operands. 409 */ 410 if (swreg_type(src) == NN_REG_IMM) { 411 err = swreg_to_unrestricted(reg_none(), src, reg_none(), ®); 412 reg.breg = reg.areg; 413 } else { 414 err = swreg_to_unrestricted(reg_none(), src, reg_imm(0), ®); 415 } 416 if (err) { 417 nfp_prog->error = err; 418 return; 419 } 420 421 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr / 4, 422 false, reg.src_lmextn); 423 } 424 425 static void emit_nop(struct nfp_prog *nfp_prog) 426 { 427 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); 428 } 429 430 /* --- Wrappers --- */ 431 static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift) 432 { 433 if (!(imm & 0xffff0000)) { 434 *val = imm; 435 *shift = IMMED_SHIFT_0B; 436 } else if (!(imm & 0xff0000ff)) { 437 *val = imm >> 8; 438 *shift = IMMED_SHIFT_1B; 439 } else if (!(imm & 0x0000ffff)) { 440 *val = imm >> 16; 441 *shift = IMMED_SHIFT_2B; 442 } else { 443 return false; 444 } 445 446 return true; 447 } 448 449 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) 450 { 451 enum immed_shift shift; 452 u16 val; 453 454 if (pack_immed(imm, &val, &shift)) { 455 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); 456 } else if (pack_immed(~imm, &val, &shift)) { 457 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); 458 } else { 459 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, 460 false, IMMED_SHIFT_0B); 461 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, 462 false, IMMED_SHIFT_2B); 463 } 464 } 465 466 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted) 467 * If the @imm is small enough encode it directly in operand and return 468 * otherwise load @imm to a spare register and return its encoding. 469 */ 470 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 471 { 472 if (FIELD_FIT(UR_REG_IMM_MAX, imm)) 473 return reg_imm(imm); 474 475 wrp_immed(nfp_prog, tmp_reg, imm); 476 return tmp_reg; 477 } 478 479 /* re_load_imm_any() - encode immediate or use tmp register (restricted) 480 * If the @imm is small enough encode it directly in operand and return 481 * otherwise load @imm to a spare register and return its encoding. 482 */ 483 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 484 { 485 if (FIELD_FIT(RE_REG_IMM_MAX, imm)) 486 return reg_imm(imm); 487 488 wrp_immed(nfp_prog, tmp_reg, imm); 489 return tmp_reg; 490 } 491 492 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) 493 { 494 while (count--) 495 emit_nop(nfp_prog); 496 } 497 498 static void 499 wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask, 500 enum br_special special) 501 { 502 emit_br(nfp_prog, mask, 0, 0); 503 504 nfp_prog->prog[nfp_prog->prog_len - 1] |= 505 FIELD_PREP(OP_BR_SPECIAL, special); 506 } 507 508 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) 509 { 510 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); 511 } 512 513 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) 514 { 515 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); 516 } 517 518 static int 519 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size) 520 { 521 unsigned int i; 522 u16 shift, sz; 523 524 /* We load the value from the address indicated in @offset and then 525 * shift out the data we don't need. Note: this is big endian! 526 */ 527 sz = max(size, 4); 528 shift = size < 4 ? 4 - size : 0; 529 530 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, 531 pptr_reg(nfp_prog), offset, sz - 1, true); 532 533 i = 0; 534 if (shift) 535 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, 536 reg_xfer(0), SHF_SC_R_SHF, shift * 8); 537 else 538 for (; i * 4 < size; i++) 539 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 540 541 if (i < 2) 542 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 543 544 return 0; 545 } 546 547 static int 548 data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, 549 u8 dst_gpr, int size) 550 { 551 unsigned int i; 552 u8 mask, sz; 553 554 /* We load the value from the address indicated in @offset and then 555 * mask out the data we don't need. Note: this is little endian! 556 */ 557 sz = max(size, 4); 558 mask = size < 4 ? GENMASK(size - 1, 0) : 0; 559 560 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, 561 reg_a(src_gpr), offset, sz / 4 - 1, true); 562 563 i = 0; 564 if (mask) 565 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, 566 reg_xfer(0), SHF_SC_NONE, 0, true); 567 else 568 for (; i * 4 < size; i++) 569 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 570 571 if (i < 2) 572 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 573 574 return 0; 575 } 576 577 static int 578 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) 579 { 580 swreg tmp_reg; 581 582 /* Calculate the true offset (src_reg + imm) */ 583 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 584 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); 585 586 /* Check packet length (size guaranteed to fit b/c it's u8) */ 587 emit_alu(nfp_prog, imm_a(nfp_prog), 588 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); 589 emit_alu(nfp_prog, reg_none(), 590 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); 591 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 592 593 /* Load data */ 594 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size); 595 } 596 597 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) 598 { 599 swreg tmp_reg; 600 601 /* Check packet length */ 602 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); 603 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); 604 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 605 606 /* Load data */ 607 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 608 return data_ld(nfp_prog, tmp_reg, 0, size); 609 } 610 611 static int 612 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 613 u8 src_gpr, u8 size) 614 { 615 unsigned int i; 616 617 for (i = 0; i * 4 < size; i++) 618 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); 619 620 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 621 reg_a(dst_gpr), offset, size - 1, true); 622 623 return 0; 624 } 625 626 static int 627 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 628 u64 imm, u8 size) 629 { 630 wrp_immed(nfp_prog, reg_xfer(0), imm); 631 if (size == 8) 632 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); 633 634 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 635 reg_a(dst_gpr), offset, size - 1, true); 636 637 return 0; 638 } 639 640 typedef int 641 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off, 642 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 643 bool needs_inc); 644 645 static int 646 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, 647 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 648 bool needs_inc) 649 { 650 bool should_inc = needs_inc && new_gpr && !last; 651 u32 idx, src_byte; 652 enum shf_sc sc; 653 swreg reg; 654 int shf; 655 u8 mask; 656 657 if (WARN_ON_ONCE(dst_byte + size > 4 || off % 4 + size > 4)) 658 return -EOPNOTSUPP; 659 660 idx = off / 4; 661 662 /* Move the entire word */ 663 if (size == 4) { 664 wrp_mov(nfp_prog, reg_both(dst), 665 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx)); 666 return 0; 667 } 668 669 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 670 return -EOPNOTSUPP; 671 672 src_byte = off % 4; 673 674 mask = (1 << size) - 1; 675 mask <<= dst_byte; 676 677 if (WARN_ON_ONCE(mask > 0xf)) 678 return -EOPNOTSUPP; 679 680 shf = abs(src_byte - dst_byte) * 8; 681 if (src_byte == dst_byte) { 682 sc = SHF_SC_NONE; 683 } else if (src_byte < dst_byte) { 684 shf = 32 - shf; 685 sc = SHF_SC_L_SHF; 686 } else { 687 sc = SHF_SC_R_SHF; 688 } 689 690 /* ld_field can address fewer indexes, if offset too large do RMW. 691 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 692 */ 693 if (idx <= RE_REG_LM_IDX_MAX) { 694 reg = reg_lm(lm3 ? 3 : 0, idx); 695 } else { 696 reg = imm_a(nfp_prog); 697 /* If it's not the first part of the load and we start a new GPR 698 * that means we are loading a second part of the LMEM word into 699 * a new GPR. IOW we've already looked that LMEM word and 700 * therefore it has been loaded into imm_a(). 701 */ 702 if (first || !new_gpr) 703 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 704 } 705 706 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); 707 708 if (should_inc) 709 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 710 711 return 0; 712 } 713 714 static int 715 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, 716 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 717 bool needs_inc) 718 { 719 bool should_inc = needs_inc && new_gpr && !last; 720 u32 idx, dst_byte; 721 enum shf_sc sc; 722 swreg reg; 723 int shf; 724 u8 mask; 725 726 if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4)) 727 return -EOPNOTSUPP; 728 729 idx = off / 4; 730 731 /* Move the entire word */ 732 if (size == 4) { 733 wrp_mov(nfp_prog, 734 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx), 735 reg_b(src)); 736 return 0; 737 } 738 739 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 740 return -EOPNOTSUPP; 741 742 dst_byte = off % 4; 743 744 mask = (1 << size) - 1; 745 mask <<= dst_byte; 746 747 if (WARN_ON_ONCE(mask > 0xf)) 748 return -EOPNOTSUPP; 749 750 shf = abs(src_byte - dst_byte) * 8; 751 if (src_byte == dst_byte) { 752 sc = SHF_SC_NONE; 753 } else if (src_byte < dst_byte) { 754 shf = 32 - shf; 755 sc = SHF_SC_L_SHF; 756 } else { 757 sc = SHF_SC_R_SHF; 758 } 759 760 /* ld_field can address fewer indexes, if offset too large do RMW. 761 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 762 */ 763 if (idx <= RE_REG_LM_IDX_MAX) { 764 reg = reg_lm(lm3 ? 3 : 0, idx); 765 } else { 766 reg = imm_a(nfp_prog); 767 /* Only first and last LMEM locations are going to need RMW, 768 * the middle location will be overwritten fully. 769 */ 770 if (first || last) 771 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 772 } 773 774 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); 775 776 if (new_gpr || last) { 777 if (idx > RE_REG_LM_IDX_MAX) 778 wrp_mov(nfp_prog, reg_lm(0, idx), reg); 779 if (should_inc) 780 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 781 } 782 783 return 0; 784 } 785 786 static int 787 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 788 unsigned int size, unsigned int ptr_off, u8 gpr, u8 ptr_gpr, 789 bool clr_gpr, lmem_step step) 790 { 791 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off; 792 bool first = true, last; 793 bool needs_inc = false; 794 swreg stack_off_reg; 795 u8 prev_gpr = 255; 796 u32 gpr_byte = 0; 797 bool lm3 = true; 798 int ret; 799 800 if (meta->ptr_not_const) { 801 /* Use of the last encountered ptr_off is OK, they all have 802 * the same alignment. Depend on low bits of value being 803 * discarded when written to LMaddr register. 804 */ 805 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, 806 stack_imm(nfp_prog)); 807 808 emit_alu(nfp_prog, imm_b(nfp_prog), 809 reg_a(ptr_gpr), ALU_OP_ADD, stack_off_reg); 810 811 needs_inc = true; 812 } else if (off + size <= 64) { 813 /* We can reach bottom 64B with LMaddr0 */ 814 lm3 = false; 815 } else if (round_down(off, 32) == round_down(off + size - 1, 32)) { 816 /* We have to set up a new pointer. If we know the offset 817 * and the entire access falls into a single 32 byte aligned 818 * window we won't have to increment the LM pointer. 819 * The 32 byte alignment is imporant because offset is ORed in 820 * not added when doing *l$indexN[off]. 821 */ 822 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), 823 stack_imm(nfp_prog)); 824 emit_alu(nfp_prog, imm_b(nfp_prog), 825 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 826 827 off %= 32; 828 } else { 829 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), 830 stack_imm(nfp_prog)); 831 832 emit_alu(nfp_prog, imm_b(nfp_prog), 833 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 834 835 needs_inc = true; 836 } 837 if (lm3) { 838 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); 839 /* For size < 4 one slot will be filled by zeroing of upper. */ 840 wrp_nops(nfp_prog, clr_gpr && size < 8 ? 2 : 3); 841 } 842 843 if (clr_gpr && size < 8) 844 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 845 846 while (size) { 847 u32 slice_end; 848 u8 slice_size; 849 850 slice_size = min(size, 4 - gpr_byte); 851 slice_end = min(off + slice_size, round_up(off + 1, 4)); 852 slice_size = slice_end - off; 853 854 last = slice_size == size; 855 856 if (needs_inc) 857 off %= 4; 858 859 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, 860 first, gpr != prev_gpr, last, lm3, needs_inc); 861 if (ret) 862 return ret; 863 864 prev_gpr = gpr; 865 first = false; 866 867 gpr_byte += slice_size; 868 if (gpr_byte >= 4) { 869 gpr_byte -= 4; 870 gpr++; 871 } 872 873 size -= slice_size; 874 off += slice_size; 875 } 876 877 return 0; 878 } 879 880 static void 881 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) 882 { 883 swreg tmp_reg; 884 885 if (alu_op == ALU_OP_AND) { 886 if (!imm) 887 wrp_immed(nfp_prog, reg_both(dst), 0); 888 if (!imm || !~imm) 889 return; 890 } 891 if (alu_op == ALU_OP_OR) { 892 if (!~imm) 893 wrp_immed(nfp_prog, reg_both(dst), ~0U); 894 if (!imm || !~imm) 895 return; 896 } 897 if (alu_op == ALU_OP_XOR) { 898 if (!~imm) 899 emit_alu(nfp_prog, reg_both(dst), reg_none(), 900 ALU_OP_NOT, reg_b(dst)); 901 if (!imm || !~imm) 902 return; 903 } 904 905 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); 906 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); 907 } 908 909 static int 910 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 911 enum alu_op alu_op, bool skip) 912 { 913 const struct bpf_insn *insn = &meta->insn; 914 u64 imm = insn->imm; /* sign extend */ 915 916 if (skip) { 917 meta->skip = true; 918 return 0; 919 } 920 921 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); 922 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); 923 924 return 0; 925 } 926 927 static int 928 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 929 enum alu_op alu_op) 930 { 931 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 932 933 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 934 emit_alu(nfp_prog, reg_both(dst + 1), 935 reg_a(dst + 1), alu_op, reg_b(src + 1)); 936 937 return 0; 938 } 939 940 static int 941 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 942 enum alu_op alu_op, bool skip) 943 { 944 const struct bpf_insn *insn = &meta->insn; 945 946 if (skip) { 947 meta->skip = true; 948 return 0; 949 } 950 951 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm); 952 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 953 954 return 0; 955 } 956 957 static int 958 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 959 enum alu_op alu_op) 960 { 961 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 962 963 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 964 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 965 966 return 0; 967 } 968 969 static void 970 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, 971 enum br_mask br_mask, u16 off) 972 { 973 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); 974 emit_br(nfp_prog, br_mask, off, 0); 975 } 976 977 static int 978 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 979 enum alu_op alu_op, enum br_mask br_mask) 980 { 981 const struct bpf_insn *insn = &meta->insn; 982 983 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, 984 insn->src_reg * 2, br_mask, insn->off); 985 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, 986 insn->src_reg * 2 + 1, br_mask, insn->off); 987 988 return 0; 989 } 990 991 static int 992 wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 993 enum br_mask br_mask, bool swap) 994 { 995 const struct bpf_insn *insn = &meta->insn; 996 u64 imm = insn->imm; /* sign extend */ 997 u8 reg = insn->dst_reg * 2; 998 swreg tmp_reg; 999 1000 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1001 if (!swap) 1002 emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg); 1003 else 1004 emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg)); 1005 1006 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1007 if (!swap) 1008 emit_alu(nfp_prog, reg_none(), 1009 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg); 1010 else 1011 emit_alu(nfp_prog, reg_none(), 1012 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1)); 1013 1014 emit_br(nfp_prog, br_mask, insn->off, 0); 1015 1016 return 0; 1017 } 1018 1019 static int 1020 wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1021 enum br_mask br_mask, bool swap) 1022 { 1023 const struct bpf_insn *insn = &meta->insn; 1024 u8 areg, breg; 1025 1026 areg = insn->dst_reg * 2; 1027 breg = insn->src_reg * 2; 1028 1029 if (swap) { 1030 areg ^= breg; 1031 breg ^= areg; 1032 areg ^= breg; 1033 } 1034 1035 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); 1036 emit_alu(nfp_prog, reg_none(), 1037 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1)); 1038 emit_br(nfp_prog, br_mask, insn->off, 0); 1039 1040 return 0; 1041 } 1042 1043 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) 1044 { 1045 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, 1046 SHF_SC_R_ROT, 8); 1047 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), 1048 SHF_SC_R_ROT, 16); 1049 } 1050 1051 /* --- Callbacks --- */ 1052 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1053 { 1054 const struct bpf_insn *insn = &meta->insn; 1055 u8 dst = insn->dst_reg * 2; 1056 u8 src = insn->src_reg * 2; 1057 1058 if (insn->src_reg == BPF_REG_10) { 1059 swreg stack_depth_reg; 1060 1061 stack_depth_reg = ur_load_imm_any(nfp_prog, 1062 nfp_prog->stack_depth, 1063 stack_imm(nfp_prog)); 1064 emit_alu(nfp_prog, reg_both(dst), 1065 stack_reg(nfp_prog), ALU_OP_ADD, stack_depth_reg); 1066 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1067 } else { 1068 wrp_reg_mov(nfp_prog, dst, src); 1069 wrp_reg_mov(nfp_prog, dst + 1, src + 1); 1070 } 1071 1072 return 0; 1073 } 1074 1075 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1076 { 1077 u64 imm = meta->insn.imm; /* sign extend */ 1078 1079 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); 1080 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); 1081 1082 return 0; 1083 } 1084 1085 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1086 { 1087 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); 1088 } 1089 1090 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1091 { 1092 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); 1093 } 1094 1095 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1096 { 1097 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); 1098 } 1099 1100 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1101 { 1102 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1103 } 1104 1105 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1106 { 1107 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); 1108 } 1109 1110 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1111 { 1112 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1113 } 1114 1115 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1116 { 1117 const struct bpf_insn *insn = &meta->insn; 1118 1119 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1120 reg_a(insn->dst_reg * 2), ALU_OP_ADD, 1121 reg_b(insn->src_reg * 2)); 1122 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1123 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C, 1124 reg_b(insn->src_reg * 2 + 1)); 1125 1126 return 0; 1127 } 1128 1129 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1130 { 1131 const struct bpf_insn *insn = &meta->insn; 1132 u64 imm = insn->imm; /* sign extend */ 1133 1134 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); 1135 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); 1136 1137 return 0; 1138 } 1139 1140 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1141 { 1142 const struct bpf_insn *insn = &meta->insn; 1143 1144 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1145 reg_a(insn->dst_reg * 2), ALU_OP_SUB, 1146 reg_b(insn->src_reg * 2)); 1147 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1148 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C, 1149 reg_b(insn->src_reg * 2 + 1)); 1150 1151 return 0; 1152 } 1153 1154 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1155 { 1156 const struct bpf_insn *insn = &meta->insn; 1157 u64 imm = insn->imm; /* sign extend */ 1158 1159 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); 1160 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); 1161 1162 return 0; 1163 } 1164 1165 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1166 { 1167 const struct bpf_insn *insn = &meta->insn; 1168 1169 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), 1170 ALU_OP_SUB, reg_b(insn->dst_reg * 2)); 1171 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), 1172 ALU_OP_SUB_C, reg_b(insn->dst_reg * 2 + 1)); 1173 1174 return 0; 1175 } 1176 1177 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1178 { 1179 const struct bpf_insn *insn = &meta->insn; 1180 u8 dst = insn->dst_reg * 2; 1181 1182 if (insn->imm < 32) { 1183 emit_shf(nfp_prog, reg_both(dst + 1), 1184 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1185 SHF_SC_R_DSHF, 32 - insn->imm); 1186 emit_shf(nfp_prog, reg_both(dst), 1187 reg_none(), SHF_OP_NONE, reg_b(dst), 1188 SHF_SC_L_SHF, insn->imm); 1189 } else if (insn->imm == 32) { 1190 wrp_reg_mov(nfp_prog, dst + 1, dst); 1191 wrp_immed(nfp_prog, reg_both(dst), 0); 1192 } else if (insn->imm > 32) { 1193 emit_shf(nfp_prog, reg_both(dst + 1), 1194 reg_none(), SHF_OP_NONE, reg_b(dst), 1195 SHF_SC_L_SHF, insn->imm - 32); 1196 wrp_immed(nfp_prog, reg_both(dst), 0); 1197 } 1198 1199 return 0; 1200 } 1201 1202 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1203 { 1204 const struct bpf_insn *insn = &meta->insn; 1205 u8 dst = insn->dst_reg * 2; 1206 1207 if (insn->imm < 32) { 1208 emit_shf(nfp_prog, reg_both(dst), 1209 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1210 SHF_SC_R_DSHF, insn->imm); 1211 emit_shf(nfp_prog, reg_both(dst + 1), 1212 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1213 SHF_SC_R_SHF, insn->imm); 1214 } else if (insn->imm == 32) { 1215 wrp_reg_mov(nfp_prog, dst, dst + 1); 1216 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1217 } else if (insn->imm > 32) { 1218 emit_shf(nfp_prog, reg_both(dst), 1219 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1220 SHF_SC_R_SHF, insn->imm - 32); 1221 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1222 } 1223 1224 return 0; 1225 } 1226 1227 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1228 { 1229 const struct bpf_insn *insn = &meta->insn; 1230 1231 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); 1232 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1233 1234 return 0; 1235 } 1236 1237 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1238 { 1239 const struct bpf_insn *insn = &meta->insn; 1240 1241 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); 1242 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1243 1244 return 0; 1245 } 1246 1247 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1248 { 1249 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); 1250 } 1251 1252 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1253 { 1254 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); 1255 } 1256 1257 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1258 { 1259 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); 1260 } 1261 1262 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1263 { 1264 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1265 } 1266 1267 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1268 { 1269 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); 1270 } 1271 1272 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1273 { 1274 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1275 } 1276 1277 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1278 { 1279 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); 1280 } 1281 1282 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1283 { 1284 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); 1285 } 1286 1287 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1288 { 1289 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); 1290 } 1291 1292 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1293 { 1294 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); 1295 } 1296 1297 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1298 { 1299 u8 dst = meta->insn.dst_reg * 2; 1300 1301 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); 1302 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1303 1304 return 0; 1305 } 1306 1307 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1308 { 1309 const struct bpf_insn *insn = &meta->insn; 1310 1311 if (!insn->imm) 1312 return 1; /* TODO: zero shift means indirect */ 1313 1314 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2), 1315 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2), 1316 SHF_SC_L_SHF, insn->imm); 1317 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1318 1319 return 0; 1320 } 1321 1322 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1323 { 1324 const struct bpf_insn *insn = &meta->insn; 1325 u8 gpr = insn->dst_reg * 2; 1326 1327 switch (insn->imm) { 1328 case 16: 1329 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), 1330 SHF_SC_R_ROT, 8); 1331 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), 1332 SHF_SC_R_SHF, 16); 1333 1334 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1335 break; 1336 case 32: 1337 wrp_end32(nfp_prog, reg_a(gpr), gpr); 1338 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1339 break; 1340 case 64: 1341 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); 1342 1343 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); 1344 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); 1345 break; 1346 } 1347 1348 return 0; 1349 } 1350 1351 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1352 { 1353 struct nfp_insn_meta *prev = nfp_meta_prev(meta); 1354 u32 imm_lo, imm_hi; 1355 u8 dst; 1356 1357 dst = prev->insn.dst_reg * 2; 1358 imm_lo = prev->insn.imm; 1359 imm_hi = meta->insn.imm; 1360 1361 wrp_immed(nfp_prog, reg_both(dst), imm_lo); 1362 1363 /* mov is always 1 insn, load imm may be two, so try to use mov */ 1364 if (imm_hi == imm_lo) 1365 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); 1366 else 1367 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); 1368 1369 return 0; 1370 } 1371 1372 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1373 { 1374 meta->double_cb = imm_ld8_part2; 1375 return 0; 1376 } 1377 1378 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1379 { 1380 return construct_data_ld(nfp_prog, meta->insn.imm, 1); 1381 } 1382 1383 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1384 { 1385 return construct_data_ld(nfp_prog, meta->insn.imm, 2); 1386 } 1387 1388 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1389 { 1390 return construct_data_ld(nfp_prog, meta->insn.imm, 4); 1391 } 1392 1393 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1394 { 1395 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1396 meta->insn.src_reg * 2, 1); 1397 } 1398 1399 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1400 { 1401 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1402 meta->insn.src_reg * 2, 2); 1403 } 1404 1405 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1406 { 1407 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1408 meta->insn.src_reg * 2, 4); 1409 } 1410 1411 static int 1412 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1413 unsigned int size, unsigned int ptr_off) 1414 { 1415 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1416 meta->insn.dst_reg * 2, meta->insn.src_reg * 2, 1417 true, wrp_lmem_load); 1418 } 1419 1420 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1421 u8 size) 1422 { 1423 swreg dst = reg_both(meta->insn.dst_reg * 2); 1424 1425 switch (meta->insn.off) { 1426 case offsetof(struct __sk_buff, len): 1427 if (size != FIELD_SIZEOF(struct __sk_buff, len)) 1428 return -EOPNOTSUPP; 1429 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); 1430 break; 1431 case offsetof(struct __sk_buff, data): 1432 if (size != FIELD_SIZEOF(struct __sk_buff, data)) 1433 return -EOPNOTSUPP; 1434 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1435 break; 1436 case offsetof(struct __sk_buff, data_end): 1437 if (size != FIELD_SIZEOF(struct __sk_buff, data_end)) 1438 return -EOPNOTSUPP; 1439 emit_alu(nfp_prog, dst, 1440 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1441 break; 1442 default: 1443 return -EOPNOTSUPP; 1444 } 1445 1446 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1447 1448 return 0; 1449 } 1450 1451 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1452 u8 size) 1453 { 1454 swreg dst = reg_both(meta->insn.dst_reg * 2); 1455 1456 switch (meta->insn.off) { 1457 case offsetof(struct xdp_md, data): 1458 if (size != FIELD_SIZEOF(struct xdp_md, data)) 1459 return -EOPNOTSUPP; 1460 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1461 break; 1462 case offsetof(struct xdp_md, data_end): 1463 if (size != FIELD_SIZEOF(struct xdp_md, data_end)) 1464 return -EOPNOTSUPP; 1465 emit_alu(nfp_prog, dst, 1466 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1467 break; 1468 default: 1469 return -EOPNOTSUPP; 1470 } 1471 1472 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1473 1474 return 0; 1475 } 1476 1477 static int 1478 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1479 unsigned int size) 1480 { 1481 swreg tmp_reg; 1482 1483 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1484 1485 return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg, 1486 meta->insn.dst_reg * 2, size); 1487 } 1488 1489 static int 1490 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1491 unsigned int size) 1492 { 1493 if (meta->ptr.type == PTR_TO_CTX) { 1494 if (nfp_prog->type == BPF_PROG_TYPE_XDP) 1495 return mem_ldx_xdp(nfp_prog, meta, size); 1496 else 1497 return mem_ldx_skb(nfp_prog, meta, size); 1498 } 1499 1500 if (meta->ptr.type == PTR_TO_PACKET) 1501 return mem_ldx_data(nfp_prog, meta, size); 1502 1503 if (meta->ptr.type == PTR_TO_STACK) 1504 return mem_ldx_stack(nfp_prog, meta, size, 1505 meta->ptr.off + meta->ptr.var_off.value); 1506 1507 return -EOPNOTSUPP; 1508 } 1509 1510 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1511 { 1512 return mem_ldx(nfp_prog, meta, 1); 1513 } 1514 1515 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1516 { 1517 return mem_ldx(nfp_prog, meta, 2); 1518 } 1519 1520 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1521 { 1522 return mem_ldx(nfp_prog, meta, 4); 1523 } 1524 1525 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1526 { 1527 return mem_ldx(nfp_prog, meta, 8); 1528 } 1529 1530 static int 1531 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1532 unsigned int size) 1533 { 1534 u64 imm = meta->insn.imm; /* sign extend */ 1535 swreg off_reg; 1536 1537 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1538 1539 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1540 imm, size); 1541 } 1542 1543 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1544 unsigned int size) 1545 { 1546 if (meta->ptr.type == PTR_TO_PACKET) 1547 return mem_st_data(nfp_prog, meta, size); 1548 1549 return -EOPNOTSUPP; 1550 } 1551 1552 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1553 { 1554 return mem_st(nfp_prog, meta, 1); 1555 } 1556 1557 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1558 { 1559 return mem_st(nfp_prog, meta, 2); 1560 } 1561 1562 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1563 { 1564 return mem_st(nfp_prog, meta, 4); 1565 } 1566 1567 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1568 { 1569 return mem_st(nfp_prog, meta, 8); 1570 } 1571 1572 static int 1573 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1574 unsigned int size) 1575 { 1576 swreg off_reg; 1577 1578 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1579 1580 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1581 meta->insn.src_reg * 2, size); 1582 } 1583 1584 static int 1585 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1586 unsigned int size, unsigned int ptr_off) 1587 { 1588 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1589 meta->insn.src_reg * 2, meta->insn.dst_reg * 2, 1590 false, wrp_lmem_store); 1591 } 1592 1593 static int 1594 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1595 unsigned int size) 1596 { 1597 if (meta->ptr.type == PTR_TO_PACKET) 1598 return mem_stx_data(nfp_prog, meta, size); 1599 1600 if (meta->ptr.type == PTR_TO_STACK) 1601 return mem_stx_stack(nfp_prog, meta, size, 1602 meta->ptr.off + meta->ptr.var_off.value); 1603 1604 return -EOPNOTSUPP; 1605 } 1606 1607 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1608 { 1609 return mem_stx(nfp_prog, meta, 1); 1610 } 1611 1612 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1613 { 1614 return mem_stx(nfp_prog, meta, 2); 1615 } 1616 1617 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1618 { 1619 return mem_stx(nfp_prog, meta, 4); 1620 } 1621 1622 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1623 { 1624 return mem_stx(nfp_prog, meta, 8); 1625 } 1626 1627 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1628 { 1629 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); 1630 1631 return 0; 1632 } 1633 1634 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1635 { 1636 const struct bpf_insn *insn = &meta->insn; 1637 u64 imm = insn->imm; /* sign extend */ 1638 swreg or1, or2, tmp_reg; 1639 1640 or1 = reg_a(insn->dst_reg * 2); 1641 or2 = reg_b(insn->dst_reg * 2 + 1); 1642 1643 if (imm & ~0U) { 1644 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1645 emit_alu(nfp_prog, imm_a(nfp_prog), 1646 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1647 or1 = imm_a(nfp_prog); 1648 } 1649 1650 if (imm >> 32) { 1651 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1652 emit_alu(nfp_prog, imm_b(nfp_prog), 1653 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1654 or2 = imm_b(nfp_prog); 1655 } 1656 1657 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); 1658 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1659 1660 return 0; 1661 } 1662 1663 static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1664 { 1665 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true); 1666 } 1667 1668 static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1669 { 1670 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false); 1671 } 1672 1673 static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1674 { 1675 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false); 1676 } 1677 1678 static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1679 { 1680 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true); 1681 } 1682 1683 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1684 { 1685 const struct bpf_insn *insn = &meta->insn; 1686 u64 imm = insn->imm; /* sign extend */ 1687 swreg tmp_reg; 1688 1689 if (!imm) { 1690 meta->skip = true; 1691 return 0; 1692 } 1693 1694 if (imm & ~0U) { 1695 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1696 emit_alu(nfp_prog, reg_none(), 1697 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg); 1698 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1699 } 1700 1701 if (imm >> 32) { 1702 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1703 emit_alu(nfp_prog, reg_none(), 1704 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg); 1705 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1706 } 1707 1708 return 0; 1709 } 1710 1711 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1712 { 1713 const struct bpf_insn *insn = &meta->insn; 1714 u64 imm = insn->imm; /* sign extend */ 1715 swreg tmp_reg; 1716 1717 if (!imm) { 1718 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), 1719 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1)); 1720 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1721 return 0; 1722 } 1723 1724 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1725 emit_alu(nfp_prog, reg_none(), 1726 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1727 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1728 1729 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1730 emit_alu(nfp_prog, reg_none(), 1731 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1732 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1733 1734 return 0; 1735 } 1736 1737 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1738 { 1739 const struct bpf_insn *insn = &meta->insn; 1740 1741 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), 1742 ALU_OP_XOR, reg_b(insn->src_reg * 2)); 1743 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1), 1744 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1)); 1745 emit_alu(nfp_prog, reg_none(), 1746 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog)); 1747 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1748 1749 return 0; 1750 } 1751 1752 static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1753 { 1754 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true); 1755 } 1756 1757 static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1758 { 1759 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false); 1760 } 1761 1762 static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1763 { 1764 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false); 1765 } 1766 1767 static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1768 { 1769 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true); 1770 } 1771 1772 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1773 { 1774 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); 1775 } 1776 1777 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1778 { 1779 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); 1780 } 1781 1782 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1783 { 1784 wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT); 1785 1786 return 0; 1787 } 1788 1789 static const instr_cb_t instr_cb[256] = { 1790 [BPF_ALU64 | BPF_MOV | BPF_X] = mov_reg64, 1791 [BPF_ALU64 | BPF_MOV | BPF_K] = mov_imm64, 1792 [BPF_ALU64 | BPF_XOR | BPF_X] = xor_reg64, 1793 [BPF_ALU64 | BPF_XOR | BPF_K] = xor_imm64, 1794 [BPF_ALU64 | BPF_AND | BPF_X] = and_reg64, 1795 [BPF_ALU64 | BPF_AND | BPF_K] = and_imm64, 1796 [BPF_ALU64 | BPF_OR | BPF_X] = or_reg64, 1797 [BPF_ALU64 | BPF_OR | BPF_K] = or_imm64, 1798 [BPF_ALU64 | BPF_ADD | BPF_X] = add_reg64, 1799 [BPF_ALU64 | BPF_ADD | BPF_K] = add_imm64, 1800 [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64, 1801 [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64, 1802 [BPF_ALU64 | BPF_NEG] = neg_reg64, 1803 [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64, 1804 [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64, 1805 [BPF_ALU | BPF_MOV | BPF_X] = mov_reg, 1806 [BPF_ALU | BPF_MOV | BPF_K] = mov_imm, 1807 [BPF_ALU | BPF_XOR | BPF_X] = xor_reg, 1808 [BPF_ALU | BPF_XOR | BPF_K] = xor_imm, 1809 [BPF_ALU | BPF_AND | BPF_X] = and_reg, 1810 [BPF_ALU | BPF_AND | BPF_K] = and_imm, 1811 [BPF_ALU | BPF_OR | BPF_X] = or_reg, 1812 [BPF_ALU | BPF_OR | BPF_K] = or_imm, 1813 [BPF_ALU | BPF_ADD | BPF_X] = add_reg, 1814 [BPF_ALU | BPF_ADD | BPF_K] = add_imm, 1815 [BPF_ALU | BPF_SUB | BPF_X] = sub_reg, 1816 [BPF_ALU | BPF_SUB | BPF_K] = sub_imm, 1817 [BPF_ALU | BPF_NEG] = neg_reg, 1818 [BPF_ALU | BPF_LSH | BPF_K] = shl_imm, 1819 [BPF_ALU | BPF_END | BPF_X] = end_reg32, 1820 [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8, 1821 [BPF_LD | BPF_ABS | BPF_B] = data_ld1, 1822 [BPF_LD | BPF_ABS | BPF_H] = data_ld2, 1823 [BPF_LD | BPF_ABS | BPF_W] = data_ld4, 1824 [BPF_LD | BPF_IND | BPF_B] = data_ind_ld1, 1825 [BPF_LD | BPF_IND | BPF_H] = data_ind_ld2, 1826 [BPF_LD | BPF_IND | BPF_W] = data_ind_ld4, 1827 [BPF_LDX | BPF_MEM | BPF_B] = mem_ldx1, 1828 [BPF_LDX | BPF_MEM | BPF_H] = mem_ldx2, 1829 [BPF_LDX | BPF_MEM | BPF_W] = mem_ldx4, 1830 [BPF_LDX | BPF_MEM | BPF_DW] = mem_ldx8, 1831 [BPF_STX | BPF_MEM | BPF_B] = mem_stx1, 1832 [BPF_STX | BPF_MEM | BPF_H] = mem_stx2, 1833 [BPF_STX | BPF_MEM | BPF_W] = mem_stx4, 1834 [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8, 1835 [BPF_ST | BPF_MEM | BPF_B] = mem_st1, 1836 [BPF_ST | BPF_MEM | BPF_H] = mem_st2, 1837 [BPF_ST | BPF_MEM | BPF_W] = mem_st4, 1838 [BPF_ST | BPF_MEM | BPF_DW] = mem_st8, 1839 [BPF_JMP | BPF_JA | BPF_K] = jump, 1840 [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm, 1841 [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm, 1842 [BPF_JMP | BPF_JGE | BPF_K] = jge_imm, 1843 [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm, 1844 [BPF_JMP | BPF_JLE | BPF_K] = jle_imm, 1845 [BPF_JMP | BPF_JSET | BPF_K] = jset_imm, 1846 [BPF_JMP | BPF_JNE | BPF_K] = jne_imm, 1847 [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg, 1848 [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg, 1849 [BPF_JMP | BPF_JGE | BPF_X] = jge_reg, 1850 [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg, 1851 [BPF_JMP | BPF_JLE | BPF_X] = jle_reg, 1852 [BPF_JMP | BPF_JSET | BPF_X] = jset_reg, 1853 [BPF_JMP | BPF_JNE | BPF_X] = jne_reg, 1854 [BPF_JMP | BPF_EXIT] = goto_out, 1855 }; 1856 1857 /* --- Misc code --- */ 1858 static void br_set_offset(u64 *instr, u16 offset) 1859 { 1860 u16 addr_lo, addr_hi; 1861 1862 addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 1863 addr_hi = offset != addr_lo; 1864 *instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO); 1865 *instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 1866 *instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo); 1867 } 1868 1869 /* --- Assembler logic --- */ 1870 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 1871 { 1872 struct nfp_insn_meta *meta, *jmp_dst; 1873 u32 idx, br_idx; 1874 1875 list_for_each_entry(meta, &nfp_prog->insns, l) { 1876 if (meta->skip) 1877 continue; 1878 if (BPF_CLASS(meta->insn.code) != BPF_JMP) 1879 continue; 1880 1881 if (list_is_last(&meta->l, &nfp_prog->insns)) 1882 idx = nfp_prog->last_bpf_off; 1883 else 1884 idx = list_next_entry(meta, l)->off - 1; 1885 1886 br_idx = nfp_prog_offset_to_index(nfp_prog, idx); 1887 1888 if (!nfp_is_br(nfp_prog->prog[br_idx])) { 1889 pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n", 1890 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); 1891 return -ELOOP; 1892 } 1893 /* Leave special branches for later */ 1894 if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx])) 1895 continue; 1896 1897 if (!meta->jmp_dst) { 1898 pr_err("Non-exit jump doesn't have destination info recorded!!\n"); 1899 return -ELOOP; 1900 } 1901 1902 jmp_dst = meta->jmp_dst; 1903 1904 if (jmp_dst->skip) { 1905 pr_err("Branch landing on removed instruction!!\n"); 1906 return -ELOOP; 1907 } 1908 1909 for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off); 1910 idx <= br_idx; idx++) { 1911 if (!nfp_is_br(nfp_prog->prog[idx])) 1912 continue; 1913 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); 1914 } 1915 } 1916 1917 /* Fixup 'goto out's separately, they can be scattered around */ 1918 for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) { 1919 enum br_special special; 1920 1921 if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE) 1922 continue; 1923 1924 special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]); 1925 switch (special) { 1926 case OP_BR_NORMAL: 1927 break; 1928 case OP_BR_GO_OUT: 1929 br_set_offset(&nfp_prog->prog[br_idx], 1930 nfp_prog->tgt_out); 1931 break; 1932 case OP_BR_GO_ABORT: 1933 br_set_offset(&nfp_prog->prog[br_idx], 1934 nfp_prog->tgt_abort); 1935 break; 1936 } 1937 1938 nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL; 1939 } 1940 1941 return 0; 1942 } 1943 1944 static void nfp_intro(struct nfp_prog *nfp_prog) 1945 { 1946 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); 1947 emit_alu(nfp_prog, plen_reg(nfp_prog), 1948 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); 1949 } 1950 1951 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 1952 { 1953 /* TC direct-action mode: 1954 * 0,1 ok NOT SUPPORTED[1] 1955 * 2 drop 0x22 -> drop, count as stat1 1956 * 4,5 nuke 0x02 -> drop 1957 * 7 redir 0x44 -> redir, count as stat2 1958 * * unspec 0x11 -> pass, count as stat0 1959 * 1960 * [1] We can't support OK and RECLASSIFY because we can't tell TC 1961 * the exact decision made. We are forced to support UNSPEC 1962 * to handle aborts so that's the only one we handle for passing 1963 * packets up the stack. 1964 */ 1965 /* Target for aborts */ 1966 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 1967 1968 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1969 1970 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1971 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); 1972 1973 /* Target for normal exits */ 1974 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 1975 1976 /* if R0 > 7 jump to abort */ 1977 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); 1978 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 1979 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1980 1981 wrp_immed(nfp_prog, reg_b(2), 0x41221211); 1982 wrp_immed(nfp_prog, reg_b(3), 0x41001211); 1983 1984 emit_shf(nfp_prog, reg_a(1), 1985 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2); 1986 1987 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 1988 emit_shf(nfp_prog, reg_a(2), 1989 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 1990 1991 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 1992 emit_shf(nfp_prog, reg_b(2), 1993 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0); 1994 1995 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1996 1997 emit_shf(nfp_prog, reg_b(2), 1998 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4); 1999 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2000 } 2001 2002 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 2003 { 2004 /* XDP return codes: 2005 * 0 aborted 0x82 -> drop, count as stat3 2006 * 1 drop 0x22 -> drop, count as stat1 2007 * 2 pass 0x11 -> pass, count as stat0 2008 * 3 tx 0x44 -> redir, count as stat2 2009 * * unknown 0x82 -> drop, count as stat3 2010 */ 2011 /* Target for aborts */ 2012 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 2013 2014 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2015 2016 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2017 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); 2018 2019 /* Target for normal exits */ 2020 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 2021 2022 /* if R0 > 3 jump to abort */ 2023 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); 2024 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 2025 2026 wrp_immed(nfp_prog, reg_b(2), 0x44112282); 2027 2028 emit_shf(nfp_prog, reg_a(1), 2029 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3); 2030 2031 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2032 emit_shf(nfp_prog, reg_b(2), 2033 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 2034 2035 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2036 2037 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2038 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2039 } 2040 2041 static void nfp_outro(struct nfp_prog *nfp_prog) 2042 { 2043 switch (nfp_prog->type) { 2044 case BPF_PROG_TYPE_SCHED_CLS: 2045 nfp_outro_tc_da(nfp_prog); 2046 break; 2047 case BPF_PROG_TYPE_XDP: 2048 nfp_outro_xdp(nfp_prog); 2049 break; 2050 default: 2051 WARN_ON(1); 2052 } 2053 } 2054 2055 static int nfp_translate(struct nfp_prog *nfp_prog) 2056 { 2057 struct nfp_insn_meta *meta; 2058 int err; 2059 2060 nfp_intro(nfp_prog); 2061 if (nfp_prog->error) 2062 return nfp_prog->error; 2063 2064 list_for_each_entry(meta, &nfp_prog->insns, l) { 2065 instr_cb_t cb = instr_cb[meta->insn.code]; 2066 2067 meta->off = nfp_prog_current_offset(nfp_prog); 2068 2069 if (meta->skip) { 2070 nfp_prog->n_translated++; 2071 continue; 2072 } 2073 2074 if (nfp_meta_has_prev(nfp_prog, meta) && 2075 nfp_meta_prev(meta)->double_cb) 2076 cb = nfp_meta_prev(meta)->double_cb; 2077 if (!cb) 2078 return -ENOENT; 2079 err = cb(nfp_prog, meta); 2080 if (err) 2081 return err; 2082 2083 nfp_prog->n_translated++; 2084 } 2085 2086 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; 2087 2088 nfp_outro(nfp_prog); 2089 if (nfp_prog->error) 2090 return nfp_prog->error; 2091 2092 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); 2093 if (nfp_prog->error) 2094 return nfp_prog->error; 2095 2096 return nfp_fixup_branches(nfp_prog); 2097 } 2098 2099 /* --- Optimizations --- */ 2100 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) 2101 { 2102 struct nfp_insn_meta *meta; 2103 2104 list_for_each_entry(meta, &nfp_prog->insns, l) { 2105 struct bpf_insn insn = meta->insn; 2106 2107 /* Programs converted from cBPF start with register xoring */ 2108 if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) && 2109 insn.src_reg == insn.dst_reg) 2110 continue; 2111 2112 /* Programs start with R6 = R1 but we ignore the skb pointer */ 2113 if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) && 2114 insn.src_reg == 1 && insn.dst_reg == 6) 2115 meta->skip = true; 2116 2117 /* Return as soon as something doesn't match */ 2118 if (!meta->skip) 2119 return; 2120 } 2121 } 2122 2123 /* Remove masking after load since our load guarantees this is not needed */ 2124 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) 2125 { 2126 struct nfp_insn_meta *meta1, *meta2; 2127 const s32 exp_mask[] = { 2128 [BPF_B] = 0x000000ffU, 2129 [BPF_H] = 0x0000ffffU, 2130 [BPF_W] = 0xffffffffU, 2131 }; 2132 2133 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2134 struct bpf_insn insn, next; 2135 2136 insn = meta1->insn; 2137 next = meta2->insn; 2138 2139 if (BPF_CLASS(insn.code) != BPF_LD) 2140 continue; 2141 if (BPF_MODE(insn.code) != BPF_ABS && 2142 BPF_MODE(insn.code) != BPF_IND) 2143 continue; 2144 2145 if (next.code != (BPF_ALU64 | BPF_AND | BPF_K)) 2146 continue; 2147 2148 if (!exp_mask[BPF_SIZE(insn.code)]) 2149 continue; 2150 if (exp_mask[BPF_SIZE(insn.code)] != next.imm) 2151 continue; 2152 2153 if (next.src_reg || next.dst_reg) 2154 continue; 2155 2156 if (meta2->flags & FLAG_INSN_IS_JUMP_DST) 2157 continue; 2158 2159 meta2->skip = true; 2160 } 2161 } 2162 2163 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) 2164 { 2165 struct nfp_insn_meta *meta1, *meta2, *meta3; 2166 2167 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { 2168 struct bpf_insn insn, next1, next2; 2169 2170 insn = meta1->insn; 2171 next1 = meta2->insn; 2172 next2 = meta3->insn; 2173 2174 if (BPF_CLASS(insn.code) != BPF_LD) 2175 continue; 2176 if (BPF_MODE(insn.code) != BPF_ABS && 2177 BPF_MODE(insn.code) != BPF_IND) 2178 continue; 2179 if (BPF_SIZE(insn.code) != BPF_W) 2180 continue; 2181 2182 if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) && 2183 next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) && 2184 !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) && 2185 next2.code == (BPF_LSH | BPF_K | BPF_ALU64))) 2186 continue; 2187 2188 if (next1.src_reg || next1.dst_reg || 2189 next2.src_reg || next2.dst_reg) 2190 continue; 2191 2192 if (next1.imm != 0x20 || next2.imm != 0x20) 2193 continue; 2194 2195 if (meta2->flags & FLAG_INSN_IS_JUMP_DST || 2196 meta3->flags & FLAG_INSN_IS_JUMP_DST) 2197 continue; 2198 2199 meta2->skip = true; 2200 meta3->skip = true; 2201 } 2202 } 2203 2204 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) 2205 { 2206 nfp_bpf_opt_reg_init(nfp_prog); 2207 2208 nfp_bpf_opt_ld_mask(nfp_prog); 2209 nfp_bpf_opt_ld_shift(nfp_prog); 2210 2211 return 0; 2212 } 2213 2214 static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore) 2215 { 2216 int i; 2217 2218 for (i = 0; i < nfp_prog->prog_len; i++) { 2219 int err; 2220 2221 err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]); 2222 if (err) 2223 return err; 2224 2225 nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]); 2226 2227 ustore[i] = cpu_to_le64(nfp_prog->prog[i]); 2228 } 2229 2230 return 0; 2231 } 2232 2233 int nfp_bpf_jit(struct nfp_prog *nfp_prog) 2234 { 2235 int ret; 2236 2237 ret = nfp_bpf_optimize(nfp_prog); 2238 if (ret) 2239 return ret; 2240 2241 ret = nfp_translate(nfp_prog); 2242 if (ret) { 2243 pr_err("Translation failed with error %d (translated: %u)\n", 2244 ret, nfp_prog->n_translated); 2245 return -EINVAL; 2246 } 2247 2248 return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog); 2249 } 2250