1 /* 2 * Copyright (C) 2016-2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #define pr_fmt(fmt) "NFP net bpf: " fmt 35 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/filter.h> 39 #include <linux/pkt_cls.h> 40 #include <linux/unistd.h> 41 42 #include "main.h" 43 #include "../nfp_asm.h" 44 45 /* --- NFP prog --- */ 46 /* Foreach "multiple" entries macros provide pos and next<n> pointers. 47 * It's safe to modify the next pointers (but not pos). 48 */ 49 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ 50 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 51 next = list_next_entry(pos, l); \ 52 &(nfp_prog)->insns != &pos->l && \ 53 &(nfp_prog)->insns != &next->l; \ 54 pos = nfp_meta_next(pos), \ 55 next = nfp_meta_next(pos)) 56 57 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ 58 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 59 next = list_next_entry(pos, l), \ 60 next2 = list_next_entry(next, l); \ 61 &(nfp_prog)->insns != &pos->l && \ 62 &(nfp_prog)->insns != &next->l && \ 63 &(nfp_prog)->insns != &next2->l; \ 64 pos = nfp_meta_next(pos), \ 65 next = nfp_meta_next(pos), \ 66 next2 = nfp_meta_next(next)) 67 68 static bool 69 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 70 { 71 return meta->l.prev != &nfp_prog->insns; 72 } 73 74 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) 75 { 76 if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) { 77 nfp_prog->error = -ENOSPC; 78 return; 79 } 80 81 nfp_prog->prog[nfp_prog->prog_len] = insn; 82 nfp_prog->prog_len++; 83 } 84 85 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) 86 { 87 return nfp_prog->start_off + nfp_prog->prog_len; 88 } 89 90 static unsigned int 91 nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset) 92 { 93 return offset - nfp_prog->start_off; 94 } 95 96 /* --- Emitters --- */ 97 static void 98 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, 99 u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync, bool indir) 100 { 101 enum cmd_ctx_swap ctx; 102 u64 insn; 103 104 if (sync) 105 ctx = CMD_CTX_SWAP; 106 else 107 ctx = CMD_CTX_NO_SWAP; 108 109 insn = FIELD_PREP(OP_CMD_A_SRC, areg) | 110 FIELD_PREP(OP_CMD_CTX, ctx) | 111 FIELD_PREP(OP_CMD_B_SRC, breg) | 112 FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) | 113 FIELD_PREP(OP_CMD_XFER, xfer) | 114 FIELD_PREP(OP_CMD_CNT, size) | 115 FIELD_PREP(OP_CMD_SIG, sync) | 116 FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) | 117 FIELD_PREP(OP_CMD_INDIR, indir) | 118 FIELD_PREP(OP_CMD_MODE, mode); 119 120 nfp_prog_push(nfp_prog, insn); 121 } 122 123 static void 124 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 125 swreg lreg, swreg rreg, u8 size, bool sync, bool indir) 126 { 127 struct nfp_insn_re_regs reg; 128 int err; 129 130 err = swreg_to_restricted(reg_none(), lreg, rreg, ®, false); 131 if (err) { 132 nfp_prog->error = err; 133 return; 134 } 135 if (reg.swap) { 136 pr_err("cmd can't swap arguments\n"); 137 nfp_prog->error = -EFAULT; 138 return; 139 } 140 if (reg.dst_lmextn || reg.src_lmextn) { 141 pr_err("cmd can't use LMextn\n"); 142 nfp_prog->error = -EFAULT; 143 return; 144 } 145 146 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync, 147 indir); 148 } 149 150 static void 151 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 152 swreg lreg, swreg rreg, u8 size, bool sync) 153 { 154 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, false); 155 } 156 157 static void 158 emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, 159 swreg lreg, swreg rreg, u8 size, bool sync) 160 { 161 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, true); 162 } 163 164 static void 165 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, 166 enum br_ctx_signal_state css, u16 addr, u8 defer) 167 { 168 u16 addr_lo, addr_hi; 169 u64 insn; 170 171 addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 172 addr_hi = addr != addr_lo; 173 174 insn = OP_BR_BASE | 175 FIELD_PREP(OP_BR_MASK, mask) | 176 FIELD_PREP(OP_BR_EV_PIP, ev_pip) | 177 FIELD_PREP(OP_BR_CSS, css) | 178 FIELD_PREP(OP_BR_DEFBR, defer) | 179 FIELD_PREP(OP_BR_ADDR_LO, addr_lo) | 180 FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 181 182 nfp_prog_push(nfp_prog, insn); 183 } 184 185 static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer) 186 { 187 if (defer > 2) { 188 pr_err("BUG: branch defer out of bounds %d\n", defer); 189 nfp_prog->error = -EFAULT; 190 return; 191 } 192 __emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer); 193 } 194 195 static void 196 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) 197 { 198 __emit_br(nfp_prog, mask, 199 mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND, 200 BR_CSS_NONE, addr, defer); 201 } 202 203 static void 204 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, 205 enum immed_width width, bool invert, 206 enum immed_shift shift, bool wr_both, 207 bool dst_lmextn, bool src_lmextn) 208 { 209 u64 insn; 210 211 insn = OP_IMMED_BASE | 212 FIELD_PREP(OP_IMMED_A_SRC, areg) | 213 FIELD_PREP(OP_IMMED_B_SRC, breg) | 214 FIELD_PREP(OP_IMMED_IMM, imm_hi) | 215 FIELD_PREP(OP_IMMED_WIDTH, width) | 216 FIELD_PREP(OP_IMMED_INV, invert) | 217 FIELD_PREP(OP_IMMED_SHIFT, shift) | 218 FIELD_PREP(OP_IMMED_WR_AB, wr_both) | 219 FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) | 220 FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn); 221 222 nfp_prog_push(nfp_prog, insn); 223 } 224 225 static void 226 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, 227 enum immed_width width, bool invert, enum immed_shift shift) 228 { 229 struct nfp_insn_ur_regs reg; 230 int err; 231 232 if (swreg_type(dst) == NN_REG_IMM) { 233 nfp_prog->error = -EFAULT; 234 return; 235 } 236 237 err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), ®); 238 if (err) { 239 nfp_prog->error = err; 240 return; 241 } 242 243 /* Use reg.dst when destination is No-Dest. */ 244 __emit_immed(nfp_prog, 245 swreg_type(dst) == NN_REG_NONE ? reg.dst : reg.areg, 246 reg.breg, imm >> 8, width, invert, shift, 247 reg.wr_both, reg.dst_lmextn, reg.src_lmextn); 248 } 249 250 static void 251 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 252 enum shf_sc sc, u8 shift, 253 u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both, 254 bool dst_lmextn, bool src_lmextn) 255 { 256 u64 insn; 257 258 if (!FIELD_FIT(OP_SHF_SHIFT, shift)) { 259 nfp_prog->error = -EFAULT; 260 return; 261 } 262 263 if (sc == SHF_SC_L_SHF) 264 shift = 32 - shift; 265 266 insn = OP_SHF_BASE | 267 FIELD_PREP(OP_SHF_A_SRC, areg) | 268 FIELD_PREP(OP_SHF_SC, sc) | 269 FIELD_PREP(OP_SHF_B_SRC, breg) | 270 FIELD_PREP(OP_SHF_I8, i8) | 271 FIELD_PREP(OP_SHF_SW, sw) | 272 FIELD_PREP(OP_SHF_DST, dst) | 273 FIELD_PREP(OP_SHF_SHIFT, shift) | 274 FIELD_PREP(OP_SHF_OP, op) | 275 FIELD_PREP(OP_SHF_DST_AB, dst_ab) | 276 FIELD_PREP(OP_SHF_WR_AB, wr_both) | 277 FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) | 278 FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn); 279 280 nfp_prog_push(nfp_prog, insn); 281 } 282 283 static void 284 emit_shf(struct nfp_prog *nfp_prog, swreg dst, 285 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift) 286 { 287 struct nfp_insn_re_regs reg; 288 int err; 289 290 err = swreg_to_restricted(dst, lreg, rreg, ®, true); 291 if (err) { 292 nfp_prog->error = err; 293 return; 294 } 295 296 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, 297 reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both, 298 reg.dst_lmextn, reg.src_lmextn); 299 } 300 301 static void 302 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 303 u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both, 304 bool dst_lmextn, bool src_lmextn) 305 { 306 u64 insn; 307 308 insn = OP_ALU_BASE | 309 FIELD_PREP(OP_ALU_A_SRC, areg) | 310 FIELD_PREP(OP_ALU_B_SRC, breg) | 311 FIELD_PREP(OP_ALU_DST, dst) | 312 FIELD_PREP(OP_ALU_SW, swap) | 313 FIELD_PREP(OP_ALU_OP, op) | 314 FIELD_PREP(OP_ALU_DST_AB, dst_ab) | 315 FIELD_PREP(OP_ALU_WR_AB, wr_both) | 316 FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) | 317 FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn); 318 319 nfp_prog_push(nfp_prog, insn); 320 } 321 322 static void 323 emit_alu(struct nfp_prog *nfp_prog, swreg dst, 324 swreg lreg, enum alu_op op, swreg rreg) 325 { 326 struct nfp_insn_ur_regs reg; 327 int err; 328 329 err = swreg_to_unrestricted(dst, lreg, rreg, ®); 330 if (err) { 331 nfp_prog->error = err; 332 return; 333 } 334 335 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, 336 reg.areg, op, reg.breg, reg.swap, reg.wr_both, 337 reg.dst_lmextn, reg.src_lmextn); 338 } 339 340 static void 341 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, 342 u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8, 343 bool zero, bool swap, bool wr_both, 344 bool dst_lmextn, bool src_lmextn) 345 { 346 u64 insn; 347 348 insn = OP_LDF_BASE | 349 FIELD_PREP(OP_LDF_A_SRC, areg) | 350 FIELD_PREP(OP_LDF_SC, sc) | 351 FIELD_PREP(OP_LDF_B_SRC, breg) | 352 FIELD_PREP(OP_LDF_I8, imm8) | 353 FIELD_PREP(OP_LDF_SW, swap) | 354 FIELD_PREP(OP_LDF_ZF, zero) | 355 FIELD_PREP(OP_LDF_BMASK, bmask) | 356 FIELD_PREP(OP_LDF_SHF, shift) | 357 FIELD_PREP(OP_LDF_WR_AB, wr_both) | 358 FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) | 359 FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn); 360 361 nfp_prog_push(nfp_prog, insn); 362 } 363 364 static void 365 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 366 enum shf_sc sc, u8 shift, bool zero) 367 { 368 struct nfp_insn_re_regs reg; 369 int err; 370 371 /* Note: ld_field is special as it uses one of the src regs as dst */ 372 err = swreg_to_restricted(dst, dst, src, ®, true); 373 if (err) { 374 nfp_prog->error = err; 375 return; 376 } 377 378 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, 379 reg.i8, zero, reg.swap, reg.wr_both, 380 reg.dst_lmextn, reg.src_lmextn); 381 } 382 383 static void 384 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 385 enum shf_sc sc, u8 shift) 386 { 387 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); 388 } 389 390 static void 391 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, 392 bool dst_lmextn, bool src_lmextn) 393 { 394 u64 insn; 395 396 insn = OP_LCSR_BASE | 397 FIELD_PREP(OP_LCSR_A_SRC, areg) | 398 FIELD_PREP(OP_LCSR_B_SRC, breg) | 399 FIELD_PREP(OP_LCSR_WRITE, wr) | 400 FIELD_PREP(OP_LCSR_ADDR, addr) | 401 FIELD_PREP(OP_LCSR_SRC_LMEXTN, src_lmextn) | 402 FIELD_PREP(OP_LCSR_DST_LMEXTN, dst_lmextn); 403 404 nfp_prog_push(nfp_prog, insn); 405 } 406 407 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) 408 { 409 struct nfp_insn_ur_regs reg; 410 int err; 411 412 /* This instruction takes immeds instead of reg_none() for the ignored 413 * operand, but we can't encode 2 immeds in one instr with our normal 414 * swreg infra so if param is an immed, we encode as reg_none() and 415 * copy the immed to both operands. 416 */ 417 if (swreg_type(src) == NN_REG_IMM) { 418 err = swreg_to_unrestricted(reg_none(), src, reg_none(), ®); 419 reg.breg = reg.areg; 420 } else { 421 err = swreg_to_unrestricted(reg_none(), src, reg_imm(0), ®); 422 } 423 if (err) { 424 nfp_prog->error = err; 425 return; 426 } 427 428 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr / 4, 429 false, reg.src_lmextn); 430 } 431 432 static void emit_nop(struct nfp_prog *nfp_prog) 433 { 434 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); 435 } 436 437 /* --- Wrappers --- */ 438 static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift) 439 { 440 if (!(imm & 0xffff0000)) { 441 *val = imm; 442 *shift = IMMED_SHIFT_0B; 443 } else if (!(imm & 0xff0000ff)) { 444 *val = imm >> 8; 445 *shift = IMMED_SHIFT_1B; 446 } else if (!(imm & 0x0000ffff)) { 447 *val = imm >> 16; 448 *shift = IMMED_SHIFT_2B; 449 } else { 450 return false; 451 } 452 453 return true; 454 } 455 456 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) 457 { 458 enum immed_shift shift; 459 u16 val; 460 461 if (pack_immed(imm, &val, &shift)) { 462 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); 463 } else if (pack_immed(~imm, &val, &shift)) { 464 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); 465 } else { 466 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, 467 false, IMMED_SHIFT_0B); 468 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, 469 false, IMMED_SHIFT_2B); 470 } 471 } 472 473 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted) 474 * If the @imm is small enough encode it directly in operand and return 475 * otherwise load @imm to a spare register and return its encoding. 476 */ 477 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 478 { 479 if (FIELD_FIT(UR_REG_IMM_MAX, imm)) 480 return reg_imm(imm); 481 482 wrp_immed(nfp_prog, tmp_reg, imm); 483 return tmp_reg; 484 } 485 486 /* re_load_imm_any() - encode immediate or use tmp register (restricted) 487 * If the @imm is small enough encode it directly in operand and return 488 * otherwise load @imm to a spare register and return its encoding. 489 */ 490 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 491 { 492 if (FIELD_FIT(RE_REG_IMM_MAX, imm)) 493 return reg_imm(imm); 494 495 wrp_immed(nfp_prog, tmp_reg, imm); 496 return tmp_reg; 497 } 498 499 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) 500 { 501 while (count--) 502 emit_nop(nfp_prog); 503 } 504 505 static void 506 wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask, 507 enum br_special special) 508 { 509 emit_br(nfp_prog, mask, 0, 0); 510 511 nfp_prog->prog[nfp_prog->prog_len - 1] |= 512 FIELD_PREP(OP_BR_SPECIAL, special); 513 } 514 515 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) 516 { 517 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); 518 } 519 520 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) 521 { 522 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); 523 } 524 525 /* wrp_reg_subpart() - load @field_len bytes from @offset of @src, write the 526 * result to @dst from low end. 527 */ 528 static void 529 wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len, 530 u8 offset) 531 { 532 enum shf_sc sc = offset ? SHF_SC_R_SHF : SHF_SC_NONE; 533 u8 mask = (1 << field_len) - 1; 534 535 emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true); 536 } 537 538 /* NFP has Command Push Pull bus which supports bluk memory operations. */ 539 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 540 { 541 bool descending_seq = meta->ldst_gather_len < 0; 542 s16 len = abs(meta->ldst_gather_len); 543 swreg src_base, off; 544 unsigned int i; 545 u8 xfer_num; 546 547 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 548 src_base = reg_a(meta->insn.src_reg * 2); 549 xfer_num = round_up(len, 4) / 4; 550 551 /* Setup PREV_ALU fields to override memory read length. */ 552 if (len > 32) 553 wrp_immed(nfp_prog, reg_none(), 554 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1)); 555 556 /* Memory read from source addr into transfer-in registers. */ 557 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, 558 off, xfer_num - 1, true, len > 32); 559 560 /* Move from transfer-in to transfer-out. */ 561 for (i = 0; i < xfer_num; i++) 562 wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i)); 563 564 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); 565 566 if (len <= 8) { 567 /* Use single direct_ref write8. */ 568 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 569 reg_a(meta->paired_st->dst_reg * 2), off, len - 1, 570 true); 571 } else if (len <= 32 && IS_ALIGNED(len, 4)) { 572 /* Use single direct_ref write32. */ 573 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 574 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1, 575 true); 576 } else if (len <= 32) { 577 /* Use single indirect_ref write8. */ 578 wrp_immed(nfp_prog, reg_none(), 579 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, len - 1)); 580 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 581 reg_a(meta->paired_st->dst_reg * 2), off, 582 len - 1, true); 583 } else if (IS_ALIGNED(len, 4)) { 584 /* Use single indirect_ref write32. */ 585 wrp_immed(nfp_prog, reg_none(), 586 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1)); 587 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 588 reg_a(meta->paired_st->dst_reg * 2), off, 589 xfer_num - 1, true); 590 } else if (len <= 40) { 591 /* Use one direct_ref write32 to write the first 32-bytes, then 592 * another direct_ref write8 to write the remaining bytes. 593 */ 594 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 595 reg_a(meta->paired_st->dst_reg * 2), off, 7, 596 true); 597 598 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, 599 imm_b(nfp_prog)); 600 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8, 601 reg_a(meta->paired_st->dst_reg * 2), off, len - 33, 602 true); 603 } else { 604 /* Use one indirect_ref write32 to write 4-bytes aligned length, 605 * then another direct_ref write8 to write the remaining bytes. 606 */ 607 u8 new_off; 608 609 wrp_immed(nfp_prog, reg_none(), 610 CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 2)); 611 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, 612 reg_a(meta->paired_st->dst_reg * 2), off, 613 xfer_num - 2, true); 614 new_off = meta->paired_st->off + (xfer_num - 1) * 4; 615 off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog)); 616 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 617 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off, 618 (len & 0x3) - 1, true); 619 } 620 621 /* TODO: The following extra load is to make sure data flow be identical 622 * before and after we do memory copy optimization. 623 * 624 * The load destination register is not guaranteed to be dead, so we 625 * need to make sure it is loaded with the value the same as before 626 * this transformation. 627 * 628 * These extra loads could be removed once we have accurate register 629 * usage information. 630 */ 631 if (descending_seq) 632 xfer_num = 0; 633 else if (BPF_SIZE(meta->insn.code) != BPF_DW) 634 xfer_num = xfer_num - 1; 635 else 636 xfer_num = xfer_num - 2; 637 638 switch (BPF_SIZE(meta->insn.code)) { 639 case BPF_B: 640 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), 641 reg_xfer(xfer_num), 1, 642 IS_ALIGNED(len, 4) ? 3 : (len & 3) - 1); 643 break; 644 case BPF_H: 645 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), 646 reg_xfer(xfer_num), 2, (len & 3) ^ 2); 647 break; 648 case BPF_W: 649 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), 650 reg_xfer(0)); 651 break; 652 case BPF_DW: 653 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), 654 reg_xfer(xfer_num)); 655 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 656 reg_xfer(xfer_num + 1)); 657 break; 658 } 659 660 if (BPF_SIZE(meta->insn.code) != BPF_DW) 661 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 662 663 return 0; 664 } 665 666 static int 667 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size) 668 { 669 unsigned int i; 670 u16 shift, sz; 671 672 /* We load the value from the address indicated in @offset and then 673 * shift out the data we don't need. Note: this is big endian! 674 */ 675 sz = max(size, 4); 676 shift = size < 4 ? 4 - size : 0; 677 678 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, 679 pptr_reg(nfp_prog), offset, sz - 1, true); 680 681 i = 0; 682 if (shift) 683 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, 684 reg_xfer(0), SHF_SC_R_SHF, shift * 8); 685 else 686 for (; i * 4 < size; i++) 687 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 688 689 if (i < 2) 690 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 691 692 return 0; 693 } 694 695 static int 696 data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, 697 u8 dst_gpr, int size) 698 { 699 unsigned int i; 700 u8 mask, sz; 701 702 /* We load the value from the address indicated in @offset and then 703 * mask out the data we don't need. Note: this is little endian! 704 */ 705 sz = max(size, 4); 706 mask = size < 4 ? GENMASK(size - 1, 0) : 0; 707 708 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, 709 reg_a(src_gpr), offset, sz / 4 - 1, true); 710 711 i = 0; 712 if (mask) 713 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, 714 reg_xfer(0), SHF_SC_NONE, 0, true); 715 else 716 for (; i * 4 < size; i++) 717 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 718 719 if (i < 2) 720 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 721 722 return 0; 723 } 724 725 static int 726 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) 727 { 728 swreg tmp_reg; 729 730 /* Calculate the true offset (src_reg + imm) */ 731 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 732 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); 733 734 /* Check packet length (size guaranteed to fit b/c it's u8) */ 735 emit_alu(nfp_prog, imm_a(nfp_prog), 736 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); 737 emit_alu(nfp_prog, reg_none(), 738 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); 739 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 740 741 /* Load data */ 742 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size); 743 } 744 745 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) 746 { 747 swreg tmp_reg; 748 749 /* Check packet length */ 750 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); 751 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); 752 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 753 754 /* Load data */ 755 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 756 return data_ld(nfp_prog, tmp_reg, 0, size); 757 } 758 759 static int 760 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 761 u8 src_gpr, u8 size) 762 { 763 unsigned int i; 764 765 for (i = 0; i * 4 < size; i++) 766 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); 767 768 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 769 reg_a(dst_gpr), offset, size - 1, true); 770 771 return 0; 772 } 773 774 static int 775 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 776 u64 imm, u8 size) 777 { 778 wrp_immed(nfp_prog, reg_xfer(0), imm); 779 if (size == 8) 780 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); 781 782 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 783 reg_a(dst_gpr), offset, size - 1, true); 784 785 return 0; 786 } 787 788 typedef int 789 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off, 790 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 791 bool needs_inc); 792 793 static int 794 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, 795 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 796 bool needs_inc) 797 { 798 bool should_inc = needs_inc && new_gpr && !last; 799 u32 idx, src_byte; 800 enum shf_sc sc; 801 swreg reg; 802 int shf; 803 u8 mask; 804 805 if (WARN_ON_ONCE(dst_byte + size > 4 || off % 4 + size > 4)) 806 return -EOPNOTSUPP; 807 808 idx = off / 4; 809 810 /* Move the entire word */ 811 if (size == 4) { 812 wrp_mov(nfp_prog, reg_both(dst), 813 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx)); 814 return 0; 815 } 816 817 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 818 return -EOPNOTSUPP; 819 820 src_byte = off % 4; 821 822 mask = (1 << size) - 1; 823 mask <<= dst_byte; 824 825 if (WARN_ON_ONCE(mask > 0xf)) 826 return -EOPNOTSUPP; 827 828 shf = abs(src_byte - dst_byte) * 8; 829 if (src_byte == dst_byte) { 830 sc = SHF_SC_NONE; 831 } else if (src_byte < dst_byte) { 832 shf = 32 - shf; 833 sc = SHF_SC_L_SHF; 834 } else { 835 sc = SHF_SC_R_SHF; 836 } 837 838 /* ld_field can address fewer indexes, if offset too large do RMW. 839 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 840 */ 841 if (idx <= RE_REG_LM_IDX_MAX) { 842 reg = reg_lm(lm3 ? 3 : 0, idx); 843 } else { 844 reg = imm_a(nfp_prog); 845 /* If it's not the first part of the load and we start a new GPR 846 * that means we are loading a second part of the LMEM word into 847 * a new GPR. IOW we've already looked that LMEM word and 848 * therefore it has been loaded into imm_a(). 849 */ 850 if (first || !new_gpr) 851 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 852 } 853 854 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); 855 856 if (should_inc) 857 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 858 859 return 0; 860 } 861 862 static int 863 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, 864 unsigned int size, bool first, bool new_gpr, bool last, bool lm3, 865 bool needs_inc) 866 { 867 bool should_inc = needs_inc && new_gpr && !last; 868 u32 idx, dst_byte; 869 enum shf_sc sc; 870 swreg reg; 871 int shf; 872 u8 mask; 873 874 if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4)) 875 return -EOPNOTSUPP; 876 877 idx = off / 4; 878 879 /* Move the entire word */ 880 if (size == 4) { 881 wrp_mov(nfp_prog, 882 should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx), 883 reg_b(src)); 884 return 0; 885 } 886 887 if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX)) 888 return -EOPNOTSUPP; 889 890 dst_byte = off % 4; 891 892 mask = (1 << size) - 1; 893 mask <<= dst_byte; 894 895 if (WARN_ON_ONCE(mask > 0xf)) 896 return -EOPNOTSUPP; 897 898 shf = abs(src_byte - dst_byte) * 8; 899 if (src_byte == dst_byte) { 900 sc = SHF_SC_NONE; 901 } else if (src_byte < dst_byte) { 902 shf = 32 - shf; 903 sc = SHF_SC_L_SHF; 904 } else { 905 sc = SHF_SC_R_SHF; 906 } 907 908 /* ld_field can address fewer indexes, if offset too large do RMW. 909 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 910 */ 911 if (idx <= RE_REG_LM_IDX_MAX) { 912 reg = reg_lm(lm3 ? 3 : 0, idx); 913 } else { 914 reg = imm_a(nfp_prog); 915 /* Only first and last LMEM locations are going to need RMW, 916 * the middle location will be overwritten fully. 917 */ 918 if (first || last) 919 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 920 } 921 922 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); 923 924 if (new_gpr || last) { 925 if (idx > RE_REG_LM_IDX_MAX) 926 wrp_mov(nfp_prog, reg_lm(0, idx), reg); 927 if (should_inc) 928 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); 929 } 930 931 return 0; 932 } 933 934 static int 935 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 936 unsigned int size, unsigned int ptr_off, u8 gpr, u8 ptr_gpr, 937 bool clr_gpr, lmem_step step) 938 { 939 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off; 940 bool first = true, last; 941 bool needs_inc = false; 942 swreg stack_off_reg; 943 u8 prev_gpr = 255; 944 u32 gpr_byte = 0; 945 bool lm3 = true; 946 int ret; 947 948 if (meta->ptr_not_const) { 949 /* Use of the last encountered ptr_off is OK, they all have 950 * the same alignment. Depend on low bits of value being 951 * discarded when written to LMaddr register. 952 */ 953 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, 954 stack_imm(nfp_prog)); 955 956 emit_alu(nfp_prog, imm_b(nfp_prog), 957 reg_a(ptr_gpr), ALU_OP_ADD, stack_off_reg); 958 959 needs_inc = true; 960 } else if (off + size <= 64) { 961 /* We can reach bottom 64B with LMaddr0 */ 962 lm3 = false; 963 } else if (round_down(off, 32) == round_down(off + size - 1, 32)) { 964 /* We have to set up a new pointer. If we know the offset 965 * and the entire access falls into a single 32 byte aligned 966 * window we won't have to increment the LM pointer. 967 * The 32 byte alignment is imporant because offset is ORed in 968 * not added when doing *l$indexN[off]. 969 */ 970 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), 971 stack_imm(nfp_prog)); 972 emit_alu(nfp_prog, imm_b(nfp_prog), 973 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 974 975 off %= 32; 976 } else { 977 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), 978 stack_imm(nfp_prog)); 979 980 emit_alu(nfp_prog, imm_b(nfp_prog), 981 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); 982 983 needs_inc = true; 984 } 985 if (lm3) { 986 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); 987 /* For size < 4 one slot will be filled by zeroing of upper. */ 988 wrp_nops(nfp_prog, clr_gpr && size < 8 ? 2 : 3); 989 } 990 991 if (clr_gpr && size < 8) 992 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 993 994 while (size) { 995 u32 slice_end; 996 u8 slice_size; 997 998 slice_size = min(size, 4 - gpr_byte); 999 slice_end = min(off + slice_size, round_up(off + 1, 4)); 1000 slice_size = slice_end - off; 1001 1002 last = slice_size == size; 1003 1004 if (needs_inc) 1005 off %= 4; 1006 1007 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, 1008 first, gpr != prev_gpr, last, lm3, needs_inc); 1009 if (ret) 1010 return ret; 1011 1012 prev_gpr = gpr; 1013 first = false; 1014 1015 gpr_byte += slice_size; 1016 if (gpr_byte >= 4) { 1017 gpr_byte -= 4; 1018 gpr++; 1019 } 1020 1021 size -= slice_size; 1022 off += slice_size; 1023 } 1024 1025 return 0; 1026 } 1027 1028 static void 1029 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) 1030 { 1031 swreg tmp_reg; 1032 1033 if (alu_op == ALU_OP_AND) { 1034 if (!imm) 1035 wrp_immed(nfp_prog, reg_both(dst), 0); 1036 if (!imm || !~imm) 1037 return; 1038 } 1039 if (alu_op == ALU_OP_OR) { 1040 if (!~imm) 1041 wrp_immed(nfp_prog, reg_both(dst), ~0U); 1042 if (!imm || !~imm) 1043 return; 1044 } 1045 if (alu_op == ALU_OP_XOR) { 1046 if (!~imm) 1047 emit_alu(nfp_prog, reg_both(dst), reg_none(), 1048 ALU_OP_NOT, reg_b(dst)); 1049 if (!imm || !~imm) 1050 return; 1051 } 1052 1053 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); 1054 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); 1055 } 1056 1057 static int 1058 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1059 enum alu_op alu_op, bool skip) 1060 { 1061 const struct bpf_insn *insn = &meta->insn; 1062 u64 imm = insn->imm; /* sign extend */ 1063 1064 if (skip) { 1065 meta->skip = true; 1066 return 0; 1067 } 1068 1069 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); 1070 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); 1071 1072 return 0; 1073 } 1074 1075 static int 1076 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1077 enum alu_op alu_op) 1078 { 1079 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 1080 1081 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 1082 emit_alu(nfp_prog, reg_both(dst + 1), 1083 reg_a(dst + 1), alu_op, reg_b(src + 1)); 1084 1085 return 0; 1086 } 1087 1088 static int 1089 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1090 enum alu_op alu_op, bool skip) 1091 { 1092 const struct bpf_insn *insn = &meta->insn; 1093 1094 if (skip) { 1095 meta->skip = true; 1096 return 0; 1097 } 1098 1099 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm); 1100 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1101 1102 return 0; 1103 } 1104 1105 static int 1106 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1107 enum alu_op alu_op) 1108 { 1109 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 1110 1111 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 1112 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1113 1114 return 0; 1115 } 1116 1117 static void 1118 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, 1119 enum br_mask br_mask, u16 off) 1120 { 1121 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); 1122 emit_br(nfp_prog, br_mask, off, 0); 1123 } 1124 1125 static int 1126 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1127 enum alu_op alu_op, enum br_mask br_mask) 1128 { 1129 const struct bpf_insn *insn = &meta->insn; 1130 1131 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, 1132 insn->src_reg * 2, br_mask, insn->off); 1133 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, 1134 insn->src_reg * 2 + 1, br_mask, insn->off); 1135 1136 return 0; 1137 } 1138 1139 static int 1140 wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1141 enum br_mask br_mask, bool swap) 1142 { 1143 const struct bpf_insn *insn = &meta->insn; 1144 u64 imm = insn->imm; /* sign extend */ 1145 u8 reg = insn->dst_reg * 2; 1146 swreg tmp_reg; 1147 1148 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1149 if (!swap) 1150 emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg); 1151 else 1152 emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg)); 1153 1154 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1155 if (!swap) 1156 emit_alu(nfp_prog, reg_none(), 1157 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg); 1158 else 1159 emit_alu(nfp_prog, reg_none(), 1160 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1)); 1161 1162 emit_br(nfp_prog, br_mask, insn->off, 0); 1163 1164 return 0; 1165 } 1166 1167 static int 1168 wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1169 enum br_mask br_mask, bool swap) 1170 { 1171 const struct bpf_insn *insn = &meta->insn; 1172 u8 areg, breg; 1173 1174 areg = insn->dst_reg * 2; 1175 breg = insn->src_reg * 2; 1176 1177 if (swap) { 1178 areg ^= breg; 1179 breg ^= areg; 1180 areg ^= breg; 1181 } 1182 1183 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); 1184 emit_alu(nfp_prog, reg_none(), 1185 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1)); 1186 emit_br(nfp_prog, br_mask, insn->off, 0); 1187 1188 return 0; 1189 } 1190 1191 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) 1192 { 1193 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, 1194 SHF_SC_R_ROT, 8); 1195 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), 1196 SHF_SC_R_ROT, 16); 1197 } 1198 1199 /* --- Callbacks --- */ 1200 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1201 { 1202 const struct bpf_insn *insn = &meta->insn; 1203 u8 dst = insn->dst_reg * 2; 1204 u8 src = insn->src_reg * 2; 1205 1206 if (insn->src_reg == BPF_REG_10) { 1207 swreg stack_depth_reg; 1208 1209 stack_depth_reg = ur_load_imm_any(nfp_prog, 1210 nfp_prog->stack_depth, 1211 stack_imm(nfp_prog)); 1212 emit_alu(nfp_prog, reg_both(dst), 1213 stack_reg(nfp_prog), ALU_OP_ADD, stack_depth_reg); 1214 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1215 } else { 1216 wrp_reg_mov(nfp_prog, dst, src); 1217 wrp_reg_mov(nfp_prog, dst + 1, src + 1); 1218 } 1219 1220 return 0; 1221 } 1222 1223 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1224 { 1225 u64 imm = meta->insn.imm; /* sign extend */ 1226 1227 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); 1228 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); 1229 1230 return 0; 1231 } 1232 1233 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1234 { 1235 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); 1236 } 1237 1238 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1239 { 1240 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); 1241 } 1242 1243 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1244 { 1245 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); 1246 } 1247 1248 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1249 { 1250 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1251 } 1252 1253 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1254 { 1255 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); 1256 } 1257 1258 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1259 { 1260 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1261 } 1262 1263 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1264 { 1265 const struct bpf_insn *insn = &meta->insn; 1266 1267 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1268 reg_a(insn->dst_reg * 2), ALU_OP_ADD, 1269 reg_b(insn->src_reg * 2)); 1270 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1271 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C, 1272 reg_b(insn->src_reg * 2 + 1)); 1273 1274 return 0; 1275 } 1276 1277 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1278 { 1279 const struct bpf_insn *insn = &meta->insn; 1280 u64 imm = insn->imm; /* sign extend */ 1281 1282 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); 1283 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); 1284 1285 return 0; 1286 } 1287 1288 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1289 { 1290 const struct bpf_insn *insn = &meta->insn; 1291 1292 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1293 reg_a(insn->dst_reg * 2), ALU_OP_SUB, 1294 reg_b(insn->src_reg * 2)); 1295 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1296 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C, 1297 reg_b(insn->src_reg * 2 + 1)); 1298 1299 return 0; 1300 } 1301 1302 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1303 { 1304 const struct bpf_insn *insn = &meta->insn; 1305 u64 imm = insn->imm; /* sign extend */ 1306 1307 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); 1308 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); 1309 1310 return 0; 1311 } 1312 1313 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1314 { 1315 const struct bpf_insn *insn = &meta->insn; 1316 1317 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), 1318 ALU_OP_SUB, reg_b(insn->dst_reg * 2)); 1319 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), 1320 ALU_OP_SUB_C, reg_b(insn->dst_reg * 2 + 1)); 1321 1322 return 0; 1323 } 1324 1325 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1326 { 1327 const struct bpf_insn *insn = &meta->insn; 1328 u8 dst = insn->dst_reg * 2; 1329 1330 if (insn->imm < 32) { 1331 emit_shf(nfp_prog, reg_both(dst + 1), 1332 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1333 SHF_SC_R_DSHF, 32 - insn->imm); 1334 emit_shf(nfp_prog, reg_both(dst), 1335 reg_none(), SHF_OP_NONE, reg_b(dst), 1336 SHF_SC_L_SHF, insn->imm); 1337 } else if (insn->imm == 32) { 1338 wrp_reg_mov(nfp_prog, dst + 1, dst); 1339 wrp_immed(nfp_prog, reg_both(dst), 0); 1340 } else if (insn->imm > 32) { 1341 emit_shf(nfp_prog, reg_both(dst + 1), 1342 reg_none(), SHF_OP_NONE, reg_b(dst), 1343 SHF_SC_L_SHF, insn->imm - 32); 1344 wrp_immed(nfp_prog, reg_both(dst), 0); 1345 } 1346 1347 return 0; 1348 } 1349 1350 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1351 { 1352 const struct bpf_insn *insn = &meta->insn; 1353 u8 dst = insn->dst_reg * 2; 1354 1355 if (insn->imm < 32) { 1356 emit_shf(nfp_prog, reg_both(dst), 1357 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1358 SHF_SC_R_DSHF, insn->imm); 1359 emit_shf(nfp_prog, reg_both(dst + 1), 1360 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1361 SHF_SC_R_SHF, insn->imm); 1362 } else if (insn->imm == 32) { 1363 wrp_reg_mov(nfp_prog, dst, dst + 1); 1364 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1365 } else if (insn->imm > 32) { 1366 emit_shf(nfp_prog, reg_both(dst), 1367 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1368 SHF_SC_R_SHF, insn->imm - 32); 1369 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1370 } 1371 1372 return 0; 1373 } 1374 1375 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1376 { 1377 const struct bpf_insn *insn = &meta->insn; 1378 1379 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); 1380 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1381 1382 return 0; 1383 } 1384 1385 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1386 { 1387 const struct bpf_insn *insn = &meta->insn; 1388 1389 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); 1390 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1391 1392 return 0; 1393 } 1394 1395 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1396 { 1397 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); 1398 } 1399 1400 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1401 { 1402 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); 1403 } 1404 1405 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1406 { 1407 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); 1408 } 1409 1410 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1411 { 1412 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1413 } 1414 1415 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1416 { 1417 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); 1418 } 1419 1420 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1421 { 1422 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1423 } 1424 1425 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1426 { 1427 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); 1428 } 1429 1430 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1431 { 1432 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); 1433 } 1434 1435 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1436 { 1437 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); 1438 } 1439 1440 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1441 { 1442 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); 1443 } 1444 1445 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1446 { 1447 u8 dst = meta->insn.dst_reg * 2; 1448 1449 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); 1450 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1451 1452 return 0; 1453 } 1454 1455 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1456 { 1457 const struct bpf_insn *insn = &meta->insn; 1458 1459 if (!insn->imm) 1460 return 1; /* TODO: zero shift means indirect */ 1461 1462 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2), 1463 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2), 1464 SHF_SC_L_SHF, insn->imm); 1465 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1466 1467 return 0; 1468 } 1469 1470 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1471 { 1472 const struct bpf_insn *insn = &meta->insn; 1473 u8 gpr = insn->dst_reg * 2; 1474 1475 switch (insn->imm) { 1476 case 16: 1477 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), 1478 SHF_SC_R_ROT, 8); 1479 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), 1480 SHF_SC_R_SHF, 16); 1481 1482 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1483 break; 1484 case 32: 1485 wrp_end32(nfp_prog, reg_a(gpr), gpr); 1486 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1487 break; 1488 case 64: 1489 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); 1490 1491 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); 1492 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); 1493 break; 1494 } 1495 1496 return 0; 1497 } 1498 1499 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1500 { 1501 struct nfp_insn_meta *prev = nfp_meta_prev(meta); 1502 u32 imm_lo, imm_hi; 1503 u8 dst; 1504 1505 dst = prev->insn.dst_reg * 2; 1506 imm_lo = prev->insn.imm; 1507 imm_hi = meta->insn.imm; 1508 1509 wrp_immed(nfp_prog, reg_both(dst), imm_lo); 1510 1511 /* mov is always 1 insn, load imm may be two, so try to use mov */ 1512 if (imm_hi == imm_lo) 1513 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); 1514 else 1515 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); 1516 1517 return 0; 1518 } 1519 1520 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1521 { 1522 meta->double_cb = imm_ld8_part2; 1523 return 0; 1524 } 1525 1526 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1527 { 1528 return construct_data_ld(nfp_prog, meta->insn.imm, 1); 1529 } 1530 1531 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1532 { 1533 return construct_data_ld(nfp_prog, meta->insn.imm, 2); 1534 } 1535 1536 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1537 { 1538 return construct_data_ld(nfp_prog, meta->insn.imm, 4); 1539 } 1540 1541 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1542 { 1543 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1544 meta->insn.src_reg * 2, 1); 1545 } 1546 1547 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1548 { 1549 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1550 meta->insn.src_reg * 2, 2); 1551 } 1552 1553 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1554 { 1555 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1556 meta->insn.src_reg * 2, 4); 1557 } 1558 1559 static int 1560 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1561 unsigned int size, unsigned int ptr_off) 1562 { 1563 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1564 meta->insn.dst_reg * 2, meta->insn.src_reg * 2, 1565 true, wrp_lmem_load); 1566 } 1567 1568 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1569 u8 size) 1570 { 1571 swreg dst = reg_both(meta->insn.dst_reg * 2); 1572 1573 switch (meta->insn.off) { 1574 case offsetof(struct __sk_buff, len): 1575 if (size != FIELD_SIZEOF(struct __sk_buff, len)) 1576 return -EOPNOTSUPP; 1577 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); 1578 break; 1579 case offsetof(struct __sk_buff, data): 1580 if (size != FIELD_SIZEOF(struct __sk_buff, data)) 1581 return -EOPNOTSUPP; 1582 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1583 break; 1584 case offsetof(struct __sk_buff, data_end): 1585 if (size != FIELD_SIZEOF(struct __sk_buff, data_end)) 1586 return -EOPNOTSUPP; 1587 emit_alu(nfp_prog, dst, 1588 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1589 break; 1590 default: 1591 return -EOPNOTSUPP; 1592 } 1593 1594 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1595 1596 return 0; 1597 } 1598 1599 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1600 u8 size) 1601 { 1602 swreg dst = reg_both(meta->insn.dst_reg * 2); 1603 1604 switch (meta->insn.off) { 1605 case offsetof(struct xdp_md, data): 1606 if (size != FIELD_SIZEOF(struct xdp_md, data)) 1607 return -EOPNOTSUPP; 1608 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1609 break; 1610 case offsetof(struct xdp_md, data_end): 1611 if (size != FIELD_SIZEOF(struct xdp_md, data_end)) 1612 return -EOPNOTSUPP; 1613 emit_alu(nfp_prog, dst, 1614 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1615 break; 1616 default: 1617 return -EOPNOTSUPP; 1618 } 1619 1620 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1621 1622 return 0; 1623 } 1624 1625 static int 1626 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1627 unsigned int size) 1628 { 1629 swreg tmp_reg; 1630 1631 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1632 1633 return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg, 1634 meta->insn.dst_reg * 2, size); 1635 } 1636 1637 static int 1638 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1639 unsigned int size) 1640 { 1641 if (meta->ldst_gather_len) 1642 return nfp_cpp_memcpy(nfp_prog, meta); 1643 1644 if (meta->ptr.type == PTR_TO_CTX) { 1645 if (nfp_prog->type == BPF_PROG_TYPE_XDP) 1646 return mem_ldx_xdp(nfp_prog, meta, size); 1647 else 1648 return mem_ldx_skb(nfp_prog, meta, size); 1649 } 1650 1651 if (meta->ptr.type == PTR_TO_PACKET) 1652 return mem_ldx_data(nfp_prog, meta, size); 1653 1654 if (meta->ptr.type == PTR_TO_STACK) 1655 return mem_ldx_stack(nfp_prog, meta, size, 1656 meta->ptr.off + meta->ptr.var_off.value); 1657 1658 return -EOPNOTSUPP; 1659 } 1660 1661 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1662 { 1663 return mem_ldx(nfp_prog, meta, 1); 1664 } 1665 1666 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1667 { 1668 return mem_ldx(nfp_prog, meta, 2); 1669 } 1670 1671 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1672 { 1673 return mem_ldx(nfp_prog, meta, 4); 1674 } 1675 1676 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1677 { 1678 return mem_ldx(nfp_prog, meta, 8); 1679 } 1680 1681 static int 1682 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1683 unsigned int size) 1684 { 1685 u64 imm = meta->insn.imm; /* sign extend */ 1686 swreg off_reg; 1687 1688 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1689 1690 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1691 imm, size); 1692 } 1693 1694 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1695 unsigned int size) 1696 { 1697 if (meta->ptr.type == PTR_TO_PACKET) 1698 return mem_st_data(nfp_prog, meta, size); 1699 1700 return -EOPNOTSUPP; 1701 } 1702 1703 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1704 { 1705 return mem_st(nfp_prog, meta, 1); 1706 } 1707 1708 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1709 { 1710 return mem_st(nfp_prog, meta, 2); 1711 } 1712 1713 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1714 { 1715 return mem_st(nfp_prog, meta, 4); 1716 } 1717 1718 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1719 { 1720 return mem_st(nfp_prog, meta, 8); 1721 } 1722 1723 static int 1724 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1725 unsigned int size) 1726 { 1727 swreg off_reg; 1728 1729 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1730 1731 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1732 meta->insn.src_reg * 2, size); 1733 } 1734 1735 static int 1736 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1737 unsigned int size, unsigned int ptr_off) 1738 { 1739 return mem_op_stack(nfp_prog, meta, size, ptr_off, 1740 meta->insn.src_reg * 2, meta->insn.dst_reg * 2, 1741 false, wrp_lmem_store); 1742 } 1743 1744 static int 1745 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1746 unsigned int size) 1747 { 1748 if (meta->ptr.type == PTR_TO_PACKET) 1749 return mem_stx_data(nfp_prog, meta, size); 1750 1751 if (meta->ptr.type == PTR_TO_STACK) 1752 return mem_stx_stack(nfp_prog, meta, size, 1753 meta->ptr.off + meta->ptr.var_off.value); 1754 1755 return -EOPNOTSUPP; 1756 } 1757 1758 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1759 { 1760 return mem_stx(nfp_prog, meta, 1); 1761 } 1762 1763 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1764 { 1765 return mem_stx(nfp_prog, meta, 2); 1766 } 1767 1768 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1769 { 1770 return mem_stx(nfp_prog, meta, 4); 1771 } 1772 1773 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1774 { 1775 return mem_stx(nfp_prog, meta, 8); 1776 } 1777 1778 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1779 { 1780 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); 1781 1782 return 0; 1783 } 1784 1785 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1786 { 1787 const struct bpf_insn *insn = &meta->insn; 1788 u64 imm = insn->imm; /* sign extend */ 1789 swreg or1, or2, tmp_reg; 1790 1791 or1 = reg_a(insn->dst_reg * 2); 1792 or2 = reg_b(insn->dst_reg * 2 + 1); 1793 1794 if (imm & ~0U) { 1795 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1796 emit_alu(nfp_prog, imm_a(nfp_prog), 1797 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1798 or1 = imm_a(nfp_prog); 1799 } 1800 1801 if (imm >> 32) { 1802 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1803 emit_alu(nfp_prog, imm_b(nfp_prog), 1804 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1805 or2 = imm_b(nfp_prog); 1806 } 1807 1808 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); 1809 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1810 1811 return 0; 1812 } 1813 1814 static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1815 { 1816 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true); 1817 } 1818 1819 static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1820 { 1821 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false); 1822 } 1823 1824 static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1825 { 1826 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false); 1827 } 1828 1829 static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1830 { 1831 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true); 1832 } 1833 1834 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1835 { 1836 const struct bpf_insn *insn = &meta->insn; 1837 u64 imm = insn->imm; /* sign extend */ 1838 swreg tmp_reg; 1839 1840 if (!imm) { 1841 meta->skip = true; 1842 return 0; 1843 } 1844 1845 if (imm & ~0U) { 1846 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1847 emit_alu(nfp_prog, reg_none(), 1848 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg); 1849 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1850 } 1851 1852 if (imm >> 32) { 1853 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1854 emit_alu(nfp_prog, reg_none(), 1855 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg); 1856 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1857 } 1858 1859 return 0; 1860 } 1861 1862 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1863 { 1864 const struct bpf_insn *insn = &meta->insn; 1865 u64 imm = insn->imm; /* sign extend */ 1866 swreg tmp_reg; 1867 1868 if (!imm) { 1869 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), 1870 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1)); 1871 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1872 return 0; 1873 } 1874 1875 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1876 emit_alu(nfp_prog, reg_none(), 1877 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1878 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1879 1880 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1881 emit_alu(nfp_prog, reg_none(), 1882 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1883 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1884 1885 return 0; 1886 } 1887 1888 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1889 { 1890 const struct bpf_insn *insn = &meta->insn; 1891 1892 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), 1893 ALU_OP_XOR, reg_b(insn->src_reg * 2)); 1894 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1), 1895 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1)); 1896 emit_alu(nfp_prog, reg_none(), 1897 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog)); 1898 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1899 1900 return 0; 1901 } 1902 1903 static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1904 { 1905 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true); 1906 } 1907 1908 static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1909 { 1910 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false); 1911 } 1912 1913 static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1914 { 1915 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false); 1916 } 1917 1918 static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1919 { 1920 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true); 1921 } 1922 1923 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1924 { 1925 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); 1926 } 1927 1928 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1929 { 1930 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); 1931 } 1932 1933 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1934 { 1935 switch (meta->insn.imm) { 1936 default: 1937 WARN_ONCE(1, "verifier allowed unsupported function\n"); 1938 return -EOPNOTSUPP; 1939 } 1940 } 1941 1942 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1943 { 1944 wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT); 1945 1946 return 0; 1947 } 1948 1949 static const instr_cb_t instr_cb[256] = { 1950 [BPF_ALU64 | BPF_MOV | BPF_X] = mov_reg64, 1951 [BPF_ALU64 | BPF_MOV | BPF_K] = mov_imm64, 1952 [BPF_ALU64 | BPF_XOR | BPF_X] = xor_reg64, 1953 [BPF_ALU64 | BPF_XOR | BPF_K] = xor_imm64, 1954 [BPF_ALU64 | BPF_AND | BPF_X] = and_reg64, 1955 [BPF_ALU64 | BPF_AND | BPF_K] = and_imm64, 1956 [BPF_ALU64 | BPF_OR | BPF_X] = or_reg64, 1957 [BPF_ALU64 | BPF_OR | BPF_K] = or_imm64, 1958 [BPF_ALU64 | BPF_ADD | BPF_X] = add_reg64, 1959 [BPF_ALU64 | BPF_ADD | BPF_K] = add_imm64, 1960 [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64, 1961 [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64, 1962 [BPF_ALU64 | BPF_NEG] = neg_reg64, 1963 [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64, 1964 [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64, 1965 [BPF_ALU | BPF_MOV | BPF_X] = mov_reg, 1966 [BPF_ALU | BPF_MOV | BPF_K] = mov_imm, 1967 [BPF_ALU | BPF_XOR | BPF_X] = xor_reg, 1968 [BPF_ALU | BPF_XOR | BPF_K] = xor_imm, 1969 [BPF_ALU | BPF_AND | BPF_X] = and_reg, 1970 [BPF_ALU | BPF_AND | BPF_K] = and_imm, 1971 [BPF_ALU | BPF_OR | BPF_X] = or_reg, 1972 [BPF_ALU | BPF_OR | BPF_K] = or_imm, 1973 [BPF_ALU | BPF_ADD | BPF_X] = add_reg, 1974 [BPF_ALU | BPF_ADD | BPF_K] = add_imm, 1975 [BPF_ALU | BPF_SUB | BPF_X] = sub_reg, 1976 [BPF_ALU | BPF_SUB | BPF_K] = sub_imm, 1977 [BPF_ALU | BPF_NEG] = neg_reg, 1978 [BPF_ALU | BPF_LSH | BPF_K] = shl_imm, 1979 [BPF_ALU | BPF_END | BPF_X] = end_reg32, 1980 [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8, 1981 [BPF_LD | BPF_ABS | BPF_B] = data_ld1, 1982 [BPF_LD | BPF_ABS | BPF_H] = data_ld2, 1983 [BPF_LD | BPF_ABS | BPF_W] = data_ld4, 1984 [BPF_LD | BPF_IND | BPF_B] = data_ind_ld1, 1985 [BPF_LD | BPF_IND | BPF_H] = data_ind_ld2, 1986 [BPF_LD | BPF_IND | BPF_W] = data_ind_ld4, 1987 [BPF_LDX | BPF_MEM | BPF_B] = mem_ldx1, 1988 [BPF_LDX | BPF_MEM | BPF_H] = mem_ldx2, 1989 [BPF_LDX | BPF_MEM | BPF_W] = mem_ldx4, 1990 [BPF_LDX | BPF_MEM | BPF_DW] = mem_ldx8, 1991 [BPF_STX | BPF_MEM | BPF_B] = mem_stx1, 1992 [BPF_STX | BPF_MEM | BPF_H] = mem_stx2, 1993 [BPF_STX | BPF_MEM | BPF_W] = mem_stx4, 1994 [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8, 1995 [BPF_ST | BPF_MEM | BPF_B] = mem_st1, 1996 [BPF_ST | BPF_MEM | BPF_H] = mem_st2, 1997 [BPF_ST | BPF_MEM | BPF_W] = mem_st4, 1998 [BPF_ST | BPF_MEM | BPF_DW] = mem_st8, 1999 [BPF_JMP | BPF_JA | BPF_K] = jump, 2000 [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm, 2001 [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm, 2002 [BPF_JMP | BPF_JGE | BPF_K] = jge_imm, 2003 [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm, 2004 [BPF_JMP | BPF_JLE | BPF_K] = jle_imm, 2005 [BPF_JMP | BPF_JSET | BPF_K] = jset_imm, 2006 [BPF_JMP | BPF_JNE | BPF_K] = jne_imm, 2007 [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg, 2008 [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg, 2009 [BPF_JMP | BPF_JGE | BPF_X] = jge_reg, 2010 [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg, 2011 [BPF_JMP | BPF_JLE | BPF_X] = jle_reg, 2012 [BPF_JMP | BPF_JSET | BPF_X] = jset_reg, 2013 [BPF_JMP | BPF_JNE | BPF_X] = jne_reg, 2014 [BPF_JMP | BPF_CALL] = call, 2015 [BPF_JMP | BPF_EXIT] = goto_out, 2016 }; 2017 2018 /* --- Misc code --- */ 2019 static void br_set_offset(u64 *instr, u16 offset) 2020 { 2021 u16 addr_lo, addr_hi; 2022 2023 addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 2024 addr_hi = offset != addr_lo; 2025 *instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO); 2026 *instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 2027 *instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo); 2028 } 2029 2030 /* --- Assembler logic --- */ 2031 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 2032 { 2033 struct nfp_insn_meta *meta, *jmp_dst; 2034 u32 idx, br_idx; 2035 2036 list_for_each_entry(meta, &nfp_prog->insns, l) { 2037 if (meta->skip) 2038 continue; 2039 if (meta->insn.code == (BPF_JMP | BPF_CALL)) 2040 continue; 2041 if (BPF_CLASS(meta->insn.code) != BPF_JMP) 2042 continue; 2043 2044 if (list_is_last(&meta->l, &nfp_prog->insns)) 2045 idx = nfp_prog->last_bpf_off; 2046 else 2047 idx = list_next_entry(meta, l)->off - 1; 2048 2049 br_idx = nfp_prog_offset_to_index(nfp_prog, idx); 2050 2051 if (!nfp_is_br(nfp_prog->prog[br_idx])) { 2052 pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n", 2053 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); 2054 return -ELOOP; 2055 } 2056 /* Leave special branches for later */ 2057 if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx])) 2058 continue; 2059 2060 if (!meta->jmp_dst) { 2061 pr_err("Non-exit jump doesn't have destination info recorded!!\n"); 2062 return -ELOOP; 2063 } 2064 2065 jmp_dst = meta->jmp_dst; 2066 2067 if (jmp_dst->skip) { 2068 pr_err("Branch landing on removed instruction!!\n"); 2069 return -ELOOP; 2070 } 2071 2072 for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off); 2073 idx <= br_idx; idx++) { 2074 if (!nfp_is_br(nfp_prog->prog[idx])) 2075 continue; 2076 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); 2077 } 2078 } 2079 2080 /* Fixup 'goto out's separately, they can be scattered around */ 2081 for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) { 2082 enum br_special special; 2083 2084 if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE) 2085 continue; 2086 2087 special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]); 2088 switch (special) { 2089 case OP_BR_NORMAL: 2090 break; 2091 case OP_BR_GO_OUT: 2092 br_set_offset(&nfp_prog->prog[br_idx], 2093 nfp_prog->tgt_out); 2094 break; 2095 case OP_BR_GO_ABORT: 2096 br_set_offset(&nfp_prog->prog[br_idx], 2097 nfp_prog->tgt_abort); 2098 break; 2099 } 2100 2101 nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL; 2102 } 2103 2104 return 0; 2105 } 2106 2107 static void nfp_intro(struct nfp_prog *nfp_prog) 2108 { 2109 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); 2110 emit_alu(nfp_prog, plen_reg(nfp_prog), 2111 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); 2112 } 2113 2114 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 2115 { 2116 /* TC direct-action mode: 2117 * 0,1 ok NOT SUPPORTED[1] 2118 * 2 drop 0x22 -> drop, count as stat1 2119 * 4,5 nuke 0x02 -> drop 2120 * 7 redir 0x44 -> redir, count as stat2 2121 * * unspec 0x11 -> pass, count as stat0 2122 * 2123 * [1] We can't support OK and RECLASSIFY because we can't tell TC 2124 * the exact decision made. We are forced to support UNSPEC 2125 * to handle aborts so that's the only one we handle for passing 2126 * packets up the stack. 2127 */ 2128 /* Target for aborts */ 2129 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 2130 2131 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2132 2133 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2134 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); 2135 2136 /* Target for normal exits */ 2137 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 2138 2139 /* if R0 > 7 jump to abort */ 2140 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); 2141 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 2142 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2143 2144 wrp_immed(nfp_prog, reg_b(2), 0x41221211); 2145 wrp_immed(nfp_prog, reg_b(3), 0x41001211); 2146 2147 emit_shf(nfp_prog, reg_a(1), 2148 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2); 2149 2150 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2151 emit_shf(nfp_prog, reg_a(2), 2152 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 2153 2154 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2155 emit_shf(nfp_prog, reg_b(2), 2156 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0); 2157 2158 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2159 2160 emit_shf(nfp_prog, reg_b(2), 2161 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4); 2162 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2163 } 2164 2165 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 2166 { 2167 /* XDP return codes: 2168 * 0 aborted 0x82 -> drop, count as stat3 2169 * 1 drop 0x22 -> drop, count as stat1 2170 * 2 pass 0x11 -> pass, count as stat0 2171 * 3 tx 0x44 -> redir, count as stat2 2172 * * unknown 0x82 -> drop, count as stat3 2173 */ 2174 /* Target for aborts */ 2175 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 2176 2177 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2178 2179 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2180 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); 2181 2182 /* Target for normal exits */ 2183 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 2184 2185 /* if R0 > 3 jump to abort */ 2186 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); 2187 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 2188 2189 wrp_immed(nfp_prog, reg_b(2), 0x44112282); 2190 2191 emit_shf(nfp_prog, reg_a(1), 2192 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3); 2193 2194 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 2195 emit_shf(nfp_prog, reg_b(2), 2196 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 2197 2198 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 2199 2200 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 2201 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 2202 } 2203 2204 static void nfp_outro(struct nfp_prog *nfp_prog) 2205 { 2206 switch (nfp_prog->type) { 2207 case BPF_PROG_TYPE_SCHED_CLS: 2208 nfp_outro_tc_da(nfp_prog); 2209 break; 2210 case BPF_PROG_TYPE_XDP: 2211 nfp_outro_xdp(nfp_prog); 2212 break; 2213 default: 2214 WARN_ON(1); 2215 } 2216 } 2217 2218 static int nfp_translate(struct nfp_prog *nfp_prog) 2219 { 2220 struct nfp_insn_meta *meta; 2221 int err; 2222 2223 nfp_intro(nfp_prog); 2224 if (nfp_prog->error) 2225 return nfp_prog->error; 2226 2227 list_for_each_entry(meta, &nfp_prog->insns, l) { 2228 instr_cb_t cb = instr_cb[meta->insn.code]; 2229 2230 meta->off = nfp_prog_current_offset(nfp_prog); 2231 2232 if (meta->skip) { 2233 nfp_prog->n_translated++; 2234 continue; 2235 } 2236 2237 if (nfp_meta_has_prev(nfp_prog, meta) && 2238 nfp_meta_prev(meta)->double_cb) 2239 cb = nfp_meta_prev(meta)->double_cb; 2240 if (!cb) 2241 return -ENOENT; 2242 err = cb(nfp_prog, meta); 2243 if (err) 2244 return err; 2245 2246 nfp_prog->n_translated++; 2247 } 2248 2249 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; 2250 2251 nfp_outro(nfp_prog); 2252 if (nfp_prog->error) 2253 return nfp_prog->error; 2254 2255 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); 2256 if (nfp_prog->error) 2257 return nfp_prog->error; 2258 2259 return nfp_fixup_branches(nfp_prog); 2260 } 2261 2262 /* --- Optimizations --- */ 2263 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) 2264 { 2265 struct nfp_insn_meta *meta; 2266 2267 list_for_each_entry(meta, &nfp_prog->insns, l) { 2268 struct bpf_insn insn = meta->insn; 2269 2270 /* Programs converted from cBPF start with register xoring */ 2271 if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) && 2272 insn.src_reg == insn.dst_reg) 2273 continue; 2274 2275 /* Programs start with R6 = R1 but we ignore the skb pointer */ 2276 if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) && 2277 insn.src_reg == 1 && insn.dst_reg == 6) 2278 meta->skip = true; 2279 2280 /* Return as soon as something doesn't match */ 2281 if (!meta->skip) 2282 return; 2283 } 2284 } 2285 2286 /* Remove masking after load since our load guarantees this is not needed */ 2287 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) 2288 { 2289 struct nfp_insn_meta *meta1, *meta2; 2290 const s32 exp_mask[] = { 2291 [BPF_B] = 0x000000ffU, 2292 [BPF_H] = 0x0000ffffU, 2293 [BPF_W] = 0xffffffffU, 2294 }; 2295 2296 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2297 struct bpf_insn insn, next; 2298 2299 insn = meta1->insn; 2300 next = meta2->insn; 2301 2302 if (BPF_CLASS(insn.code) != BPF_LD) 2303 continue; 2304 if (BPF_MODE(insn.code) != BPF_ABS && 2305 BPF_MODE(insn.code) != BPF_IND) 2306 continue; 2307 2308 if (next.code != (BPF_ALU64 | BPF_AND | BPF_K)) 2309 continue; 2310 2311 if (!exp_mask[BPF_SIZE(insn.code)]) 2312 continue; 2313 if (exp_mask[BPF_SIZE(insn.code)] != next.imm) 2314 continue; 2315 2316 if (next.src_reg || next.dst_reg) 2317 continue; 2318 2319 if (meta2->flags & FLAG_INSN_IS_JUMP_DST) 2320 continue; 2321 2322 meta2->skip = true; 2323 } 2324 } 2325 2326 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) 2327 { 2328 struct nfp_insn_meta *meta1, *meta2, *meta3; 2329 2330 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { 2331 struct bpf_insn insn, next1, next2; 2332 2333 insn = meta1->insn; 2334 next1 = meta2->insn; 2335 next2 = meta3->insn; 2336 2337 if (BPF_CLASS(insn.code) != BPF_LD) 2338 continue; 2339 if (BPF_MODE(insn.code) != BPF_ABS && 2340 BPF_MODE(insn.code) != BPF_IND) 2341 continue; 2342 if (BPF_SIZE(insn.code) != BPF_W) 2343 continue; 2344 2345 if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) && 2346 next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) && 2347 !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) && 2348 next2.code == (BPF_LSH | BPF_K | BPF_ALU64))) 2349 continue; 2350 2351 if (next1.src_reg || next1.dst_reg || 2352 next2.src_reg || next2.dst_reg) 2353 continue; 2354 2355 if (next1.imm != 0x20 || next2.imm != 0x20) 2356 continue; 2357 2358 if (meta2->flags & FLAG_INSN_IS_JUMP_DST || 2359 meta3->flags & FLAG_INSN_IS_JUMP_DST) 2360 continue; 2361 2362 meta2->skip = true; 2363 meta3->skip = true; 2364 } 2365 } 2366 2367 /* load/store pair that forms memory copy sould look like the following: 2368 * 2369 * ld_width R, [addr_src + offset_src] 2370 * st_width [addr_dest + offset_dest], R 2371 * 2372 * The destination register of load and source register of store should 2373 * be the same, load and store should also perform at the same width. 2374 * If either of addr_src or addr_dest is stack pointer, we don't do the 2375 * CPP optimization as stack is modelled by registers on NFP. 2376 */ 2377 static bool 2378 curr_pair_is_memcpy(struct nfp_insn_meta *ld_meta, 2379 struct nfp_insn_meta *st_meta) 2380 { 2381 struct bpf_insn *ld = &ld_meta->insn; 2382 struct bpf_insn *st = &st_meta->insn; 2383 2384 if (!is_mbpf_load(ld_meta) || !is_mbpf_store(st_meta)) 2385 return false; 2386 2387 if (ld_meta->ptr.type != PTR_TO_PACKET) 2388 return false; 2389 2390 if (st_meta->ptr.type != PTR_TO_PACKET) 2391 return false; 2392 2393 if (BPF_SIZE(ld->code) != BPF_SIZE(st->code)) 2394 return false; 2395 2396 if (ld->dst_reg != st->src_reg) 2397 return false; 2398 2399 /* There is jump to the store insn in this pair. */ 2400 if (st_meta->flags & FLAG_INSN_IS_JUMP_DST) 2401 return false; 2402 2403 return true; 2404 } 2405 2406 /* Currently, we only support chaining load/store pairs if: 2407 * 2408 * - Their address base registers are the same. 2409 * - Their address offsets are in the same order. 2410 * - They operate at the same memory width. 2411 * - There is no jump into the middle of them. 2412 */ 2413 static bool 2414 curr_pair_chain_with_previous(struct nfp_insn_meta *ld_meta, 2415 struct nfp_insn_meta *st_meta, 2416 struct bpf_insn *prev_ld, 2417 struct bpf_insn *prev_st) 2418 { 2419 u8 prev_size, curr_size, prev_ld_base, prev_st_base, prev_ld_dst; 2420 struct bpf_insn *ld = &ld_meta->insn; 2421 struct bpf_insn *st = &st_meta->insn; 2422 s16 prev_ld_off, prev_st_off; 2423 2424 /* This pair is the start pair. */ 2425 if (!prev_ld) 2426 return true; 2427 2428 prev_size = BPF_LDST_BYTES(prev_ld); 2429 curr_size = BPF_LDST_BYTES(ld); 2430 prev_ld_base = prev_ld->src_reg; 2431 prev_st_base = prev_st->dst_reg; 2432 prev_ld_dst = prev_ld->dst_reg; 2433 prev_ld_off = prev_ld->off; 2434 prev_st_off = prev_st->off; 2435 2436 if (ld->dst_reg != prev_ld_dst) 2437 return false; 2438 2439 if (ld->src_reg != prev_ld_base || st->dst_reg != prev_st_base) 2440 return false; 2441 2442 if (curr_size != prev_size) 2443 return false; 2444 2445 /* There is jump to the head of this pair. */ 2446 if (ld_meta->flags & FLAG_INSN_IS_JUMP_DST) 2447 return false; 2448 2449 /* Both in ascending order. */ 2450 if (prev_ld_off + prev_size == ld->off && 2451 prev_st_off + prev_size == st->off) 2452 return true; 2453 2454 /* Both in descending order. */ 2455 if (ld->off + curr_size == prev_ld_off && 2456 st->off + curr_size == prev_st_off) 2457 return true; 2458 2459 return false; 2460 } 2461 2462 /* Return TRUE if cross memory access happens. Cross memory access means 2463 * store area is overlapping with load area that a later load might load 2464 * the value from previous store, for this case we can't treat the sequence 2465 * as an memory copy. 2466 */ 2467 static bool 2468 cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta, 2469 struct nfp_insn_meta *head_st_meta) 2470 { 2471 s16 head_ld_off, head_st_off, ld_off; 2472 2473 /* Different pointer types does not overlap. */ 2474 if (head_ld_meta->ptr.type != head_st_meta->ptr.type) 2475 return false; 2476 2477 /* load and store are both PTR_TO_PACKET, check ID info. */ 2478 if (head_ld_meta->ptr.id != head_st_meta->ptr.id) 2479 return true; 2480 2481 /* Canonicalize the offsets. Turn all of them against the original 2482 * base register. 2483 */ 2484 head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off; 2485 head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off; 2486 ld_off = ld->off + head_ld_meta->ptr.off; 2487 2488 /* Ascending order cross. */ 2489 if (ld_off > head_ld_off && 2490 head_ld_off < head_st_off && ld_off >= head_st_off) 2491 return true; 2492 2493 /* Descending order cross. */ 2494 if (ld_off < head_ld_off && 2495 head_ld_off > head_st_off && ld_off <= head_st_off) 2496 return true; 2497 2498 return false; 2499 } 2500 2501 /* This pass try to identify the following instructoin sequences. 2502 * 2503 * load R, [regA + offA] 2504 * store [regB + offB], R 2505 * load R, [regA + offA + const_imm_A] 2506 * store [regB + offB + const_imm_A], R 2507 * load R, [regA + offA + 2 * const_imm_A] 2508 * store [regB + offB + 2 * const_imm_A], R 2509 * ... 2510 * 2511 * Above sequence is typically generated by compiler when lowering 2512 * memcpy. NFP prefer using CPP instructions to accelerate it. 2513 */ 2514 static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) 2515 { 2516 struct nfp_insn_meta *head_ld_meta = NULL; 2517 struct nfp_insn_meta *head_st_meta = NULL; 2518 struct nfp_insn_meta *meta1, *meta2; 2519 struct bpf_insn *prev_ld = NULL; 2520 struct bpf_insn *prev_st = NULL; 2521 u8 count = 0; 2522 2523 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2524 struct bpf_insn *ld = &meta1->insn; 2525 struct bpf_insn *st = &meta2->insn; 2526 2527 /* Reset record status if any of the following if true: 2528 * - The current insn pair is not load/store. 2529 * - The load/store pair doesn't chain with previous one. 2530 * - The chained load/store pair crossed with previous pair. 2531 * - The chained load/store pair has a total size of memory 2532 * copy beyond 128 bytes which is the maximum length a 2533 * single NFP CPP command can transfer. 2534 */ 2535 if (!curr_pair_is_memcpy(meta1, meta2) || 2536 !curr_pair_chain_with_previous(meta1, meta2, prev_ld, 2537 prev_st) || 2538 (head_ld_meta && (cross_mem_access(ld, head_ld_meta, 2539 head_st_meta) || 2540 head_ld_meta->ldst_gather_len >= 128))) { 2541 if (!count) 2542 continue; 2543 2544 if (count > 1) { 2545 s16 prev_ld_off = prev_ld->off; 2546 s16 prev_st_off = prev_st->off; 2547 s16 head_ld_off = head_ld_meta->insn.off; 2548 2549 if (prev_ld_off < head_ld_off) { 2550 head_ld_meta->insn.off = prev_ld_off; 2551 head_st_meta->insn.off = prev_st_off; 2552 head_ld_meta->ldst_gather_len = 2553 -head_ld_meta->ldst_gather_len; 2554 } 2555 2556 head_ld_meta->paired_st = &head_st_meta->insn; 2557 head_st_meta->skip = true; 2558 } else { 2559 head_ld_meta->ldst_gather_len = 0; 2560 } 2561 2562 /* If the chain is ended by an load/store pair then this 2563 * could serve as the new head of the the next chain. 2564 */ 2565 if (curr_pair_is_memcpy(meta1, meta2)) { 2566 head_ld_meta = meta1; 2567 head_st_meta = meta2; 2568 head_ld_meta->ldst_gather_len = 2569 BPF_LDST_BYTES(ld); 2570 meta1 = nfp_meta_next(meta1); 2571 meta2 = nfp_meta_next(meta2); 2572 prev_ld = ld; 2573 prev_st = st; 2574 count = 1; 2575 } else { 2576 head_ld_meta = NULL; 2577 head_st_meta = NULL; 2578 prev_ld = NULL; 2579 prev_st = NULL; 2580 count = 0; 2581 } 2582 2583 continue; 2584 } 2585 2586 if (!head_ld_meta) { 2587 head_ld_meta = meta1; 2588 head_st_meta = meta2; 2589 } else { 2590 meta1->skip = true; 2591 meta2->skip = true; 2592 } 2593 2594 head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld); 2595 meta1 = nfp_meta_next(meta1); 2596 meta2 = nfp_meta_next(meta2); 2597 prev_ld = ld; 2598 prev_st = st; 2599 count++; 2600 } 2601 } 2602 2603 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) 2604 { 2605 nfp_bpf_opt_reg_init(nfp_prog); 2606 2607 nfp_bpf_opt_ld_mask(nfp_prog); 2608 nfp_bpf_opt_ld_shift(nfp_prog); 2609 nfp_bpf_opt_ldst_gather(nfp_prog); 2610 2611 return 0; 2612 } 2613 2614 static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore) 2615 { 2616 int i; 2617 2618 for (i = 0; i < nfp_prog->prog_len; i++) { 2619 int err; 2620 2621 err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]); 2622 if (err) 2623 return err; 2624 2625 nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]); 2626 2627 ustore[i] = cpu_to_le64(nfp_prog->prog[i]); 2628 } 2629 2630 return 0; 2631 } 2632 2633 int nfp_bpf_jit(struct nfp_prog *nfp_prog) 2634 { 2635 int ret; 2636 2637 ret = nfp_bpf_optimize(nfp_prog); 2638 if (ret) 2639 return ret; 2640 2641 ret = nfp_translate(nfp_prog); 2642 if (ret) { 2643 pr_err("Translation failed with error %d (translated: %u)\n", 2644 ret, nfp_prog->n_translated); 2645 return -EINVAL; 2646 } 2647 2648 return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog); 2649 } 2650