1 /* 2 * Copyright (C) 2016 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #define pr_fmt(fmt) "NFP net bpf: " fmt 35 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/filter.h> 39 #include <linux/pkt_cls.h> 40 #include <linux/unistd.h> 41 42 #include "main.h" 43 #include "../nfp_asm.h" 44 45 /* --- NFP prog --- */ 46 /* Foreach "multiple" entries macros provide pos and next<n> pointers. 47 * It's safe to modify the next pointers (but not pos). 48 */ 49 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ 50 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 51 next = list_next_entry(pos, l); \ 52 &(nfp_prog)->insns != &pos->l && \ 53 &(nfp_prog)->insns != &next->l; \ 54 pos = nfp_meta_next(pos), \ 55 next = nfp_meta_next(pos)) 56 57 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ 58 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ 59 next = list_next_entry(pos, l), \ 60 next2 = list_next_entry(next, l); \ 61 &(nfp_prog)->insns != &pos->l && \ 62 &(nfp_prog)->insns != &next->l && \ 63 &(nfp_prog)->insns != &next2->l; \ 64 pos = nfp_meta_next(pos), \ 65 next = nfp_meta_next(pos), \ 66 next2 = nfp_meta_next(next)) 67 68 static bool 69 nfp_meta_has_next(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 70 { 71 return meta->l.next != &nfp_prog->insns; 72 } 73 74 static bool 75 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 76 { 77 return meta->l.prev != &nfp_prog->insns; 78 } 79 80 static void nfp_prog_free(struct nfp_prog *nfp_prog) 81 { 82 struct nfp_insn_meta *meta, *tmp; 83 84 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { 85 list_del(&meta->l); 86 kfree(meta); 87 } 88 kfree(nfp_prog); 89 } 90 91 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) 92 { 93 if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) { 94 nfp_prog->error = -ENOSPC; 95 return; 96 } 97 98 nfp_prog->prog[nfp_prog->prog_len] = insn; 99 nfp_prog->prog_len++; 100 } 101 102 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) 103 { 104 return nfp_prog->start_off + nfp_prog->prog_len; 105 } 106 107 static unsigned int 108 nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset) 109 { 110 return offset - nfp_prog->start_off; 111 } 112 113 /* --- Emitters --- */ 114 static void 115 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, 116 u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync) 117 { 118 enum cmd_ctx_swap ctx; 119 u64 insn; 120 121 if (sync) 122 ctx = CMD_CTX_SWAP; 123 else 124 ctx = CMD_CTX_NO_SWAP; 125 126 insn = FIELD_PREP(OP_CMD_A_SRC, areg) | 127 FIELD_PREP(OP_CMD_CTX, ctx) | 128 FIELD_PREP(OP_CMD_B_SRC, breg) | 129 FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) | 130 FIELD_PREP(OP_CMD_XFER, xfer) | 131 FIELD_PREP(OP_CMD_CNT, size) | 132 FIELD_PREP(OP_CMD_SIG, sync) | 133 FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) | 134 FIELD_PREP(OP_CMD_MODE, mode); 135 136 nfp_prog_push(nfp_prog, insn); 137 } 138 139 static void 140 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, 141 u8 mode, u8 xfer, swreg lreg, swreg rreg, u8 size, bool sync) 142 { 143 struct nfp_insn_re_regs reg; 144 int err; 145 146 err = swreg_to_restricted(reg_none(), lreg, rreg, ®, false); 147 if (err) { 148 nfp_prog->error = err; 149 return; 150 } 151 if (reg.swap) { 152 pr_err("cmd can't swap arguments\n"); 153 nfp_prog->error = -EFAULT; 154 return; 155 } 156 if (reg.dst_lmextn || reg.src_lmextn) { 157 pr_err("cmd can't use LMextn\n"); 158 nfp_prog->error = -EFAULT; 159 return; 160 } 161 162 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync); 163 } 164 165 static void 166 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, 167 enum br_ctx_signal_state css, u16 addr, u8 defer) 168 { 169 u16 addr_lo, addr_hi; 170 u64 insn; 171 172 addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 173 addr_hi = addr != addr_lo; 174 175 insn = OP_BR_BASE | 176 FIELD_PREP(OP_BR_MASK, mask) | 177 FIELD_PREP(OP_BR_EV_PIP, ev_pip) | 178 FIELD_PREP(OP_BR_CSS, css) | 179 FIELD_PREP(OP_BR_DEFBR, defer) | 180 FIELD_PREP(OP_BR_ADDR_LO, addr_lo) | 181 FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 182 183 nfp_prog_push(nfp_prog, insn); 184 } 185 186 static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer) 187 { 188 if (defer > 2) { 189 pr_err("BUG: branch defer out of bounds %d\n", defer); 190 nfp_prog->error = -EFAULT; 191 return; 192 } 193 __emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer); 194 } 195 196 static void 197 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) 198 { 199 __emit_br(nfp_prog, mask, 200 mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND, 201 BR_CSS_NONE, addr, defer); 202 } 203 204 static void 205 __emit_br_byte(struct nfp_prog *nfp_prog, u8 areg, u8 breg, bool imm8, 206 u8 byte, bool equal, u16 addr, u8 defer, bool src_lmextn) 207 { 208 u16 addr_lo, addr_hi; 209 u64 insn; 210 211 addr_lo = addr & (OP_BB_ADDR_LO >> __bf_shf(OP_BB_ADDR_LO)); 212 addr_hi = addr != addr_lo; 213 214 insn = OP_BBYTE_BASE | 215 FIELD_PREP(OP_BB_A_SRC, areg) | 216 FIELD_PREP(OP_BB_BYTE, byte) | 217 FIELD_PREP(OP_BB_B_SRC, breg) | 218 FIELD_PREP(OP_BB_I8, imm8) | 219 FIELD_PREP(OP_BB_EQ, equal) | 220 FIELD_PREP(OP_BB_DEFBR, defer) | 221 FIELD_PREP(OP_BB_ADDR_LO, addr_lo) | 222 FIELD_PREP(OP_BB_ADDR_HI, addr_hi) | 223 FIELD_PREP(OP_BB_SRC_LMEXTN, src_lmextn); 224 225 nfp_prog_push(nfp_prog, insn); 226 } 227 228 static void 229 emit_br_byte_neq(struct nfp_prog *nfp_prog, 230 swreg src, u8 imm, u8 byte, u16 addr, u8 defer) 231 { 232 struct nfp_insn_re_regs reg; 233 int err; 234 235 err = swreg_to_restricted(reg_none(), src, reg_imm(imm), ®, true); 236 if (err) { 237 nfp_prog->error = err; 238 return; 239 } 240 241 __emit_br_byte(nfp_prog, reg.areg, reg.breg, reg.i8, byte, false, addr, 242 defer, reg.src_lmextn); 243 } 244 245 static void 246 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, 247 enum immed_width width, bool invert, 248 enum immed_shift shift, bool wr_both, 249 bool dst_lmextn, bool src_lmextn) 250 { 251 u64 insn; 252 253 insn = OP_IMMED_BASE | 254 FIELD_PREP(OP_IMMED_A_SRC, areg) | 255 FIELD_PREP(OP_IMMED_B_SRC, breg) | 256 FIELD_PREP(OP_IMMED_IMM, imm_hi) | 257 FIELD_PREP(OP_IMMED_WIDTH, width) | 258 FIELD_PREP(OP_IMMED_INV, invert) | 259 FIELD_PREP(OP_IMMED_SHIFT, shift) | 260 FIELD_PREP(OP_IMMED_WR_AB, wr_both) | 261 FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) | 262 FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn); 263 264 nfp_prog_push(nfp_prog, insn); 265 } 266 267 static void 268 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, 269 enum immed_width width, bool invert, enum immed_shift shift) 270 { 271 struct nfp_insn_ur_regs reg; 272 int err; 273 274 if (swreg_type(dst) == NN_REG_IMM) { 275 nfp_prog->error = -EFAULT; 276 return; 277 } 278 279 err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), ®); 280 if (err) { 281 nfp_prog->error = err; 282 return; 283 } 284 285 __emit_immed(nfp_prog, reg.areg, reg.breg, imm >> 8, width, 286 invert, shift, reg.wr_both, 287 reg.dst_lmextn, reg.src_lmextn); 288 } 289 290 static void 291 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 292 enum shf_sc sc, u8 shift, 293 u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both, 294 bool dst_lmextn, bool src_lmextn) 295 { 296 u64 insn; 297 298 if (!FIELD_FIT(OP_SHF_SHIFT, shift)) { 299 nfp_prog->error = -EFAULT; 300 return; 301 } 302 303 if (sc == SHF_SC_L_SHF) 304 shift = 32 - shift; 305 306 insn = OP_SHF_BASE | 307 FIELD_PREP(OP_SHF_A_SRC, areg) | 308 FIELD_PREP(OP_SHF_SC, sc) | 309 FIELD_PREP(OP_SHF_B_SRC, breg) | 310 FIELD_PREP(OP_SHF_I8, i8) | 311 FIELD_PREP(OP_SHF_SW, sw) | 312 FIELD_PREP(OP_SHF_DST, dst) | 313 FIELD_PREP(OP_SHF_SHIFT, shift) | 314 FIELD_PREP(OP_SHF_OP, op) | 315 FIELD_PREP(OP_SHF_DST_AB, dst_ab) | 316 FIELD_PREP(OP_SHF_WR_AB, wr_both) | 317 FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) | 318 FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn); 319 320 nfp_prog_push(nfp_prog, insn); 321 } 322 323 static void 324 emit_shf(struct nfp_prog *nfp_prog, swreg dst, 325 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift) 326 { 327 struct nfp_insn_re_regs reg; 328 int err; 329 330 err = swreg_to_restricted(dst, lreg, rreg, ®, true); 331 if (err) { 332 nfp_prog->error = err; 333 return; 334 } 335 336 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, 337 reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both, 338 reg.dst_lmextn, reg.src_lmextn); 339 } 340 341 static void 342 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, 343 u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both, 344 bool dst_lmextn, bool src_lmextn) 345 { 346 u64 insn; 347 348 insn = OP_ALU_BASE | 349 FIELD_PREP(OP_ALU_A_SRC, areg) | 350 FIELD_PREP(OP_ALU_B_SRC, breg) | 351 FIELD_PREP(OP_ALU_DST, dst) | 352 FIELD_PREP(OP_ALU_SW, swap) | 353 FIELD_PREP(OP_ALU_OP, op) | 354 FIELD_PREP(OP_ALU_DST_AB, dst_ab) | 355 FIELD_PREP(OP_ALU_WR_AB, wr_both) | 356 FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) | 357 FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn); 358 359 nfp_prog_push(nfp_prog, insn); 360 } 361 362 static void 363 emit_alu(struct nfp_prog *nfp_prog, swreg dst, 364 swreg lreg, enum alu_op op, swreg rreg) 365 { 366 struct nfp_insn_ur_regs reg; 367 int err; 368 369 err = swreg_to_unrestricted(dst, lreg, rreg, ®); 370 if (err) { 371 nfp_prog->error = err; 372 return; 373 } 374 375 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, 376 reg.areg, op, reg.breg, reg.swap, reg.wr_both, 377 reg.dst_lmextn, reg.src_lmextn); 378 } 379 380 static void 381 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, 382 u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8, 383 bool zero, bool swap, bool wr_both, 384 bool dst_lmextn, bool src_lmextn) 385 { 386 u64 insn; 387 388 insn = OP_LDF_BASE | 389 FIELD_PREP(OP_LDF_A_SRC, areg) | 390 FIELD_PREP(OP_LDF_SC, sc) | 391 FIELD_PREP(OP_LDF_B_SRC, breg) | 392 FIELD_PREP(OP_LDF_I8, imm8) | 393 FIELD_PREP(OP_LDF_SW, swap) | 394 FIELD_PREP(OP_LDF_ZF, zero) | 395 FIELD_PREP(OP_LDF_BMASK, bmask) | 396 FIELD_PREP(OP_LDF_SHF, shift) | 397 FIELD_PREP(OP_LDF_WR_AB, wr_both) | 398 FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) | 399 FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn); 400 401 nfp_prog_push(nfp_prog, insn); 402 } 403 404 static void 405 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 406 enum shf_sc sc, u8 shift, bool zero) 407 { 408 struct nfp_insn_re_regs reg; 409 int err; 410 411 /* Note: ld_field is special as it uses one of the src regs as dst */ 412 err = swreg_to_restricted(dst, dst, src, ®, true); 413 if (err) { 414 nfp_prog->error = err; 415 return; 416 } 417 418 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, 419 reg.i8, zero, reg.swap, reg.wr_both, 420 reg.dst_lmextn, reg.src_lmextn); 421 } 422 423 static void 424 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, 425 enum shf_sc sc, u8 shift) 426 { 427 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); 428 } 429 430 static void emit_nop(struct nfp_prog *nfp_prog) 431 { 432 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); 433 } 434 435 /* --- Wrappers --- */ 436 static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift) 437 { 438 if (!(imm & 0xffff0000)) { 439 *val = imm; 440 *shift = IMMED_SHIFT_0B; 441 } else if (!(imm & 0xff0000ff)) { 442 *val = imm >> 8; 443 *shift = IMMED_SHIFT_1B; 444 } else if (!(imm & 0x0000ffff)) { 445 *val = imm >> 16; 446 *shift = IMMED_SHIFT_2B; 447 } else { 448 return false; 449 } 450 451 return true; 452 } 453 454 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) 455 { 456 enum immed_shift shift; 457 u16 val; 458 459 if (pack_immed(imm, &val, &shift)) { 460 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); 461 } else if (pack_immed(~imm, &val, &shift)) { 462 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); 463 } else { 464 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, 465 false, IMMED_SHIFT_0B); 466 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, 467 false, IMMED_SHIFT_2B); 468 } 469 } 470 471 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted) 472 * If the @imm is small enough encode it directly in operand and return 473 * otherwise load @imm to a spare register and return its encoding. 474 */ 475 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 476 { 477 if (FIELD_FIT(UR_REG_IMM_MAX, imm)) 478 return reg_imm(imm); 479 480 wrp_immed(nfp_prog, tmp_reg, imm); 481 return tmp_reg; 482 } 483 484 /* re_load_imm_any() - encode immediate or use tmp register (restricted) 485 * If the @imm is small enough encode it directly in operand and return 486 * otherwise load @imm to a spare register and return its encoding. 487 */ 488 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) 489 { 490 if (FIELD_FIT(RE_REG_IMM_MAX, imm)) 491 return reg_imm(imm); 492 493 wrp_immed(nfp_prog, tmp_reg, imm); 494 return tmp_reg; 495 } 496 497 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) 498 { 499 while (count--) 500 emit_nop(nfp_prog); 501 } 502 503 static void 504 wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask, 505 enum br_special special) 506 { 507 emit_br(nfp_prog, mask, 0, 0); 508 509 nfp_prog->prog[nfp_prog->prog_len - 1] |= 510 FIELD_PREP(OP_BR_SPECIAL, special); 511 } 512 513 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) 514 { 515 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); 516 } 517 518 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) 519 { 520 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); 521 } 522 523 static int 524 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size) 525 { 526 unsigned int i; 527 u16 shift, sz; 528 529 /* We load the value from the address indicated in @offset and then 530 * shift out the data we don't need. Note: this is big endian! 531 */ 532 sz = max(size, 4); 533 shift = size < 4 ? 4 - size : 0; 534 535 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, 536 pptr_reg(nfp_prog), offset, sz - 1, true); 537 538 i = 0; 539 if (shift) 540 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, 541 reg_xfer(0), SHF_SC_R_SHF, shift * 8); 542 else 543 for (; i * 4 < size; i++) 544 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 545 546 if (i < 2) 547 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 548 549 return 0; 550 } 551 552 static int 553 data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, 554 u8 dst_gpr, int size) 555 { 556 unsigned int i; 557 u8 mask, sz; 558 559 /* We load the value from the address indicated in @offset and then 560 * mask out the data we don't need. Note: this is little endian! 561 */ 562 sz = max(size, 4); 563 mask = size < 4 ? GENMASK(size - 1, 0) : 0; 564 565 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, 566 reg_a(src_gpr), offset, sz / 4 - 1, true); 567 568 i = 0; 569 if (mask) 570 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, 571 reg_xfer(0), SHF_SC_NONE, 0, true); 572 else 573 for (; i * 4 < size; i++) 574 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); 575 576 if (i < 2) 577 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0); 578 579 return 0; 580 } 581 582 static int 583 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) 584 { 585 swreg tmp_reg; 586 587 /* Calculate the true offset (src_reg + imm) */ 588 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 589 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); 590 591 /* Check packet length (size guaranteed to fit b/c it's u8) */ 592 emit_alu(nfp_prog, imm_a(nfp_prog), 593 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); 594 emit_alu(nfp_prog, reg_none(), 595 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); 596 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 597 598 /* Load data */ 599 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size); 600 } 601 602 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) 603 { 604 swreg tmp_reg; 605 606 /* Check packet length */ 607 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); 608 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); 609 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT); 610 611 /* Load data */ 612 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); 613 return data_ld(nfp_prog, tmp_reg, 0, size); 614 } 615 616 static int 617 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 618 u8 src_gpr, u8 size) 619 { 620 unsigned int i; 621 622 for (i = 0; i * 4 < size; i++) 623 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); 624 625 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 626 reg_a(dst_gpr), offset, size - 1, true); 627 628 return 0; 629 } 630 631 static int 632 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, 633 u64 imm, u8 size) 634 { 635 wrp_immed(nfp_prog, reg_xfer(0), imm); 636 if (size == 8) 637 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); 638 639 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, 640 reg_a(dst_gpr), offset, size - 1, true); 641 642 return 0; 643 } 644 645 typedef int 646 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off, 647 unsigned int size); 648 649 static int 650 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, 651 unsigned int size) 652 { 653 u32 idx, dst_byte; 654 enum shf_sc sc; 655 swreg reg; 656 int shf; 657 u8 mask; 658 659 if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4)) 660 return -EOPNOTSUPP; 661 662 idx = off / 4; 663 664 /* Move the entire word */ 665 if (size == 4) { 666 wrp_mov(nfp_prog, reg_lm(0, idx), reg_b(src)); 667 return 0; 668 } 669 670 dst_byte = off % 4; 671 672 mask = (1 << size) - 1; 673 mask <<= dst_byte; 674 675 if (WARN_ON_ONCE(mask > 0xf)) 676 return -EOPNOTSUPP; 677 678 shf = abs(src_byte - dst_byte) * 8; 679 if (src_byte == dst_byte) { 680 sc = SHF_SC_NONE; 681 } else if (src_byte < dst_byte) { 682 shf = 32 - shf; 683 sc = SHF_SC_L_SHF; 684 } else { 685 sc = SHF_SC_R_SHF; 686 } 687 688 /* ld_field can address fewer indexes, if offset too large do RMW. 689 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes. 690 */ 691 if (idx <= RE_REG_LM_IDX_MAX) { 692 reg = reg_lm(0, idx); 693 } else { 694 reg = imm_a(nfp_prog); 695 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); 696 } 697 698 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); 699 700 if (idx > RE_REG_LM_IDX_MAX) 701 wrp_mov(nfp_prog, reg_lm(0, idx), reg); 702 703 return 0; 704 } 705 706 static int 707 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 708 unsigned int size, u8 gpr, lmem_step step) 709 { 710 s32 off = nfp_prog->stack_depth + meta->insn.off; 711 u32 gpr_byte = 0; 712 int ret; 713 714 while (size) { 715 u32 slice_end; 716 u8 slice_size; 717 718 slice_size = min(size, 4 - gpr_byte); 719 slice_end = min(off + slice_size, round_up(off + 1, 4)); 720 slice_size = slice_end - off; 721 722 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size); 723 if (ret) 724 return ret; 725 726 gpr_byte += slice_size; 727 if (gpr_byte >= 4) { 728 gpr_byte -= 4; 729 gpr++; 730 } 731 732 size -= slice_size; 733 off += slice_size; 734 } 735 736 return 0; 737 } 738 739 static void 740 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) 741 { 742 swreg tmp_reg; 743 744 if (alu_op == ALU_OP_AND) { 745 if (!imm) 746 wrp_immed(nfp_prog, reg_both(dst), 0); 747 if (!imm || !~imm) 748 return; 749 } 750 if (alu_op == ALU_OP_OR) { 751 if (!~imm) 752 wrp_immed(nfp_prog, reg_both(dst), ~0U); 753 if (!imm || !~imm) 754 return; 755 } 756 if (alu_op == ALU_OP_XOR) { 757 if (!~imm) 758 emit_alu(nfp_prog, reg_both(dst), reg_none(), 759 ALU_OP_NEG, reg_b(dst)); 760 if (!imm || !~imm) 761 return; 762 } 763 764 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); 765 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); 766 } 767 768 static int 769 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 770 enum alu_op alu_op, bool skip) 771 { 772 const struct bpf_insn *insn = &meta->insn; 773 u64 imm = insn->imm; /* sign extend */ 774 775 if (skip) { 776 meta->skip = true; 777 return 0; 778 } 779 780 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); 781 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); 782 783 return 0; 784 } 785 786 static int 787 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 788 enum alu_op alu_op) 789 { 790 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 791 792 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 793 emit_alu(nfp_prog, reg_both(dst + 1), 794 reg_a(dst + 1), alu_op, reg_b(src + 1)); 795 796 return 0; 797 } 798 799 static int 800 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 801 enum alu_op alu_op, bool skip) 802 { 803 const struct bpf_insn *insn = &meta->insn; 804 805 if (skip) { 806 meta->skip = true; 807 return 0; 808 } 809 810 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm); 811 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 812 813 return 0; 814 } 815 816 static int 817 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 818 enum alu_op alu_op) 819 { 820 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2; 821 822 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); 823 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 824 825 return 0; 826 } 827 828 static void 829 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, 830 enum br_mask br_mask, u16 off) 831 { 832 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); 833 emit_br(nfp_prog, br_mask, off, 0); 834 } 835 836 static int 837 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 838 enum alu_op alu_op, enum br_mask br_mask) 839 { 840 const struct bpf_insn *insn = &meta->insn; 841 842 if (insn->off < 0) /* TODO */ 843 return -EOPNOTSUPP; 844 845 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, 846 insn->src_reg * 2, br_mask, insn->off); 847 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, 848 insn->src_reg * 2 + 1, br_mask, insn->off); 849 850 return 0; 851 } 852 853 static int 854 wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 855 enum br_mask br_mask, bool swap) 856 { 857 const struct bpf_insn *insn = &meta->insn; 858 u64 imm = insn->imm; /* sign extend */ 859 u8 reg = insn->dst_reg * 2; 860 swreg tmp_reg; 861 862 if (insn->off < 0) /* TODO */ 863 return -EOPNOTSUPP; 864 865 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 866 if (!swap) 867 emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg); 868 else 869 emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg)); 870 871 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 872 if (!swap) 873 emit_alu(nfp_prog, reg_none(), 874 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg); 875 else 876 emit_alu(nfp_prog, reg_none(), 877 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1)); 878 879 emit_br(nfp_prog, br_mask, insn->off, 0); 880 881 return 0; 882 } 883 884 static int 885 wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 886 enum br_mask br_mask, bool swap) 887 { 888 const struct bpf_insn *insn = &meta->insn; 889 u8 areg, breg; 890 891 areg = insn->dst_reg * 2; 892 breg = insn->src_reg * 2; 893 894 if (insn->off < 0) /* TODO */ 895 return -EOPNOTSUPP; 896 897 if (swap) { 898 areg ^= breg; 899 breg ^= areg; 900 areg ^= breg; 901 } 902 903 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); 904 emit_alu(nfp_prog, reg_none(), 905 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1)); 906 emit_br(nfp_prog, br_mask, insn->off, 0); 907 908 return 0; 909 } 910 911 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) 912 { 913 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, 914 SHF_SC_R_ROT, 8); 915 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), 916 SHF_SC_R_ROT, 16); 917 } 918 919 /* --- Callbacks --- */ 920 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 921 { 922 const struct bpf_insn *insn = &meta->insn; 923 924 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); 925 wrp_reg_mov(nfp_prog, insn->dst_reg * 2 + 1, insn->src_reg * 2 + 1); 926 927 return 0; 928 } 929 930 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 931 { 932 u64 imm = meta->insn.imm; /* sign extend */ 933 934 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); 935 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); 936 937 return 0; 938 } 939 940 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 941 { 942 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); 943 } 944 945 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 946 { 947 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); 948 } 949 950 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 951 { 952 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); 953 } 954 955 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 956 { 957 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 958 } 959 960 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 961 { 962 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); 963 } 964 965 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 966 { 967 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 968 } 969 970 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 971 { 972 const struct bpf_insn *insn = &meta->insn; 973 974 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 975 reg_a(insn->dst_reg * 2), ALU_OP_ADD, 976 reg_b(insn->src_reg * 2)); 977 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 978 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C, 979 reg_b(insn->src_reg * 2 + 1)); 980 981 return 0; 982 } 983 984 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 985 { 986 const struct bpf_insn *insn = &meta->insn; 987 u64 imm = insn->imm; /* sign extend */ 988 989 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); 990 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); 991 992 return 0; 993 } 994 995 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 996 { 997 const struct bpf_insn *insn = &meta->insn; 998 999 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), 1000 reg_a(insn->dst_reg * 2), ALU_OP_SUB, 1001 reg_b(insn->src_reg * 2)); 1002 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 1003 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C, 1004 reg_b(insn->src_reg * 2 + 1)); 1005 1006 return 0; 1007 } 1008 1009 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1010 { 1011 const struct bpf_insn *insn = &meta->insn; 1012 u64 imm = insn->imm; /* sign extend */ 1013 1014 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); 1015 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); 1016 1017 return 0; 1018 } 1019 1020 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1021 { 1022 const struct bpf_insn *insn = &meta->insn; 1023 u8 dst = insn->dst_reg * 2; 1024 1025 if (insn->imm < 32) { 1026 emit_shf(nfp_prog, reg_both(dst + 1), 1027 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1028 SHF_SC_R_DSHF, 32 - insn->imm); 1029 emit_shf(nfp_prog, reg_both(dst), 1030 reg_none(), SHF_OP_NONE, reg_b(dst), 1031 SHF_SC_L_SHF, insn->imm); 1032 } else if (insn->imm == 32) { 1033 wrp_reg_mov(nfp_prog, dst + 1, dst); 1034 wrp_immed(nfp_prog, reg_both(dst), 0); 1035 } else if (insn->imm > 32) { 1036 emit_shf(nfp_prog, reg_both(dst + 1), 1037 reg_none(), SHF_OP_NONE, reg_b(dst), 1038 SHF_SC_L_SHF, insn->imm - 32); 1039 wrp_immed(nfp_prog, reg_both(dst), 0); 1040 } 1041 1042 return 0; 1043 } 1044 1045 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1046 { 1047 const struct bpf_insn *insn = &meta->insn; 1048 u8 dst = insn->dst_reg * 2; 1049 1050 if (insn->imm < 32) { 1051 emit_shf(nfp_prog, reg_both(dst), 1052 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), 1053 SHF_SC_R_DSHF, insn->imm); 1054 emit_shf(nfp_prog, reg_both(dst + 1), 1055 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1056 SHF_SC_R_SHF, insn->imm); 1057 } else if (insn->imm == 32) { 1058 wrp_reg_mov(nfp_prog, dst, dst + 1); 1059 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1060 } else if (insn->imm > 32) { 1061 emit_shf(nfp_prog, reg_both(dst), 1062 reg_none(), SHF_OP_NONE, reg_b(dst + 1), 1063 SHF_SC_R_SHF, insn->imm - 32); 1064 wrp_immed(nfp_prog, reg_both(dst + 1), 0); 1065 } 1066 1067 return 0; 1068 } 1069 1070 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1071 { 1072 const struct bpf_insn *insn = &meta->insn; 1073 1074 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); 1075 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1076 1077 return 0; 1078 } 1079 1080 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1081 { 1082 const struct bpf_insn *insn = &meta->insn; 1083 1084 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); 1085 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1086 1087 return 0; 1088 } 1089 1090 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1091 { 1092 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); 1093 } 1094 1095 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1096 { 1097 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm); 1098 } 1099 1100 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1101 { 1102 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); 1103 } 1104 1105 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1106 { 1107 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); 1108 } 1109 1110 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1111 { 1112 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); 1113 } 1114 1115 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1116 { 1117 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); 1118 } 1119 1120 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1121 { 1122 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); 1123 } 1124 1125 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1126 { 1127 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm); 1128 } 1129 1130 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1131 { 1132 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); 1133 } 1134 1135 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1136 { 1137 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm); 1138 } 1139 1140 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1141 { 1142 const struct bpf_insn *insn = &meta->insn; 1143 1144 if (!insn->imm) 1145 return 1; /* TODO: zero shift means indirect */ 1146 1147 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2), 1148 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2), 1149 SHF_SC_L_SHF, insn->imm); 1150 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); 1151 1152 return 0; 1153 } 1154 1155 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1156 { 1157 const struct bpf_insn *insn = &meta->insn; 1158 u8 gpr = insn->dst_reg * 2; 1159 1160 switch (insn->imm) { 1161 case 16: 1162 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), 1163 SHF_SC_R_ROT, 8); 1164 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), 1165 SHF_SC_R_SHF, 16); 1166 1167 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1168 break; 1169 case 32: 1170 wrp_end32(nfp_prog, reg_a(gpr), gpr); 1171 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); 1172 break; 1173 case 64: 1174 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); 1175 1176 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); 1177 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); 1178 break; 1179 } 1180 1181 return 0; 1182 } 1183 1184 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1185 { 1186 wrp_immed(nfp_prog, reg_both(nfp_meta_prev(meta)->insn.dst_reg * 2 + 1), 1187 meta->insn.imm); 1188 1189 return 0; 1190 } 1191 1192 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1193 { 1194 const struct bpf_insn *insn = &meta->insn; 1195 1196 meta->double_cb = imm_ld8_part2; 1197 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); 1198 1199 return 0; 1200 } 1201 1202 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1203 { 1204 return construct_data_ld(nfp_prog, meta->insn.imm, 1); 1205 } 1206 1207 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1208 { 1209 return construct_data_ld(nfp_prog, meta->insn.imm, 2); 1210 } 1211 1212 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1213 { 1214 return construct_data_ld(nfp_prog, meta->insn.imm, 4); 1215 } 1216 1217 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1218 { 1219 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1220 meta->insn.src_reg * 2, 1); 1221 } 1222 1223 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1224 { 1225 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1226 meta->insn.src_reg * 2, 2); 1227 } 1228 1229 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1230 { 1231 return construct_data_ind_ld(nfp_prog, meta->insn.imm, 1232 meta->insn.src_reg * 2, 4); 1233 } 1234 1235 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1236 u8 size) 1237 { 1238 swreg dst = reg_both(meta->insn.dst_reg * 2); 1239 1240 switch (meta->insn.off) { 1241 case offsetof(struct sk_buff, len): 1242 if (size != FIELD_SIZEOF(struct sk_buff, len)) 1243 return -EOPNOTSUPP; 1244 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); 1245 break; 1246 case offsetof(struct sk_buff, data): 1247 if (size != sizeof(void *)) 1248 return -EOPNOTSUPP; 1249 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1250 break; 1251 case offsetof(struct sk_buff, cb) + 1252 offsetof(struct bpf_skb_data_end, data_end): 1253 if (size != sizeof(void *)) 1254 return -EOPNOTSUPP; 1255 emit_alu(nfp_prog, dst, 1256 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1257 break; 1258 default: 1259 return -EOPNOTSUPP; 1260 } 1261 1262 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1263 1264 return 0; 1265 } 1266 1267 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1268 u8 size) 1269 { 1270 swreg dst = reg_both(meta->insn.dst_reg * 2); 1271 1272 if (size != sizeof(void *)) 1273 return -EINVAL; 1274 1275 switch (meta->insn.off) { 1276 case offsetof(struct xdp_buff, data): 1277 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); 1278 break; 1279 case offsetof(struct xdp_buff, data_end): 1280 emit_alu(nfp_prog, dst, 1281 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); 1282 break; 1283 default: 1284 return -EOPNOTSUPP; 1285 } 1286 1287 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); 1288 1289 return 0; 1290 } 1291 1292 static int 1293 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1294 unsigned int size) 1295 { 1296 swreg tmp_reg; 1297 1298 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1299 1300 return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg, 1301 meta->insn.dst_reg * 2, size); 1302 } 1303 1304 static int 1305 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1306 unsigned int size) 1307 { 1308 if (meta->ptr.type == PTR_TO_CTX) { 1309 if (nfp_prog->act == NN_ACT_XDP) 1310 return mem_ldx_xdp(nfp_prog, meta, size); 1311 else 1312 return mem_ldx_skb(nfp_prog, meta, size); 1313 } 1314 1315 if (meta->ptr.type == PTR_TO_PACKET) 1316 return mem_ldx_data(nfp_prog, meta, size); 1317 1318 return -EOPNOTSUPP; 1319 } 1320 1321 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1322 { 1323 return mem_ldx(nfp_prog, meta, 1); 1324 } 1325 1326 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1327 { 1328 return mem_ldx(nfp_prog, meta, 2); 1329 } 1330 1331 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1332 { 1333 return mem_ldx(nfp_prog, meta, 4); 1334 } 1335 1336 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1337 { 1338 return mem_ldx(nfp_prog, meta, 8); 1339 } 1340 1341 static int 1342 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1343 unsigned int size) 1344 { 1345 u64 imm = meta->insn.imm; /* sign extend */ 1346 swreg off_reg; 1347 1348 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1349 1350 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1351 imm, size); 1352 } 1353 1354 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1355 unsigned int size) 1356 { 1357 if (meta->ptr.type == PTR_TO_PACKET) 1358 return mem_st_data(nfp_prog, meta, size); 1359 1360 return -EOPNOTSUPP; 1361 } 1362 1363 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1364 { 1365 return mem_st(nfp_prog, meta, 1); 1366 } 1367 1368 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1369 { 1370 return mem_st(nfp_prog, meta, 2); 1371 } 1372 1373 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1374 { 1375 return mem_st(nfp_prog, meta, 4); 1376 } 1377 1378 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1379 { 1380 return mem_st(nfp_prog, meta, 8); 1381 } 1382 1383 static int 1384 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1385 unsigned int size) 1386 { 1387 swreg off_reg; 1388 1389 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); 1390 1391 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, 1392 meta->insn.src_reg * 2, size); 1393 } 1394 1395 static int 1396 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1397 unsigned int size) 1398 { 1399 return mem_op_stack(nfp_prog, meta, size, meta->insn.src_reg * 2, 1400 wrp_lmem_store); 1401 } 1402 1403 static int 1404 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 1405 unsigned int size) 1406 { 1407 if (meta->ptr.type == PTR_TO_PACKET) 1408 return mem_stx_data(nfp_prog, meta, size); 1409 1410 if (meta->ptr.type == PTR_TO_STACK) 1411 return mem_stx_stack(nfp_prog, meta, size); 1412 1413 return -EOPNOTSUPP; 1414 } 1415 1416 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1417 { 1418 return mem_stx(nfp_prog, meta, 1); 1419 } 1420 1421 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1422 { 1423 return mem_stx(nfp_prog, meta, 2); 1424 } 1425 1426 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1427 { 1428 return mem_stx(nfp_prog, meta, 4); 1429 } 1430 1431 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1432 { 1433 return mem_stx(nfp_prog, meta, 8); 1434 } 1435 1436 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1437 { 1438 if (meta->insn.off < 0) /* TODO */ 1439 return -EOPNOTSUPP; 1440 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); 1441 1442 return 0; 1443 } 1444 1445 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1446 { 1447 const struct bpf_insn *insn = &meta->insn; 1448 u64 imm = insn->imm; /* sign extend */ 1449 swreg or1, or2, tmp_reg; 1450 1451 or1 = reg_a(insn->dst_reg * 2); 1452 or2 = reg_b(insn->dst_reg * 2 + 1); 1453 1454 if (insn->off < 0) /* TODO */ 1455 return -EOPNOTSUPP; 1456 1457 if (imm & ~0U) { 1458 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1459 emit_alu(nfp_prog, imm_a(nfp_prog), 1460 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1461 or1 = imm_a(nfp_prog); 1462 } 1463 1464 if (imm >> 32) { 1465 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1466 emit_alu(nfp_prog, imm_b(nfp_prog), 1467 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1468 or2 = imm_b(nfp_prog); 1469 } 1470 1471 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); 1472 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1473 1474 return 0; 1475 } 1476 1477 static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1478 { 1479 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true); 1480 } 1481 1482 static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1483 { 1484 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false); 1485 } 1486 1487 static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1488 { 1489 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false); 1490 } 1491 1492 static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1493 { 1494 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true); 1495 } 1496 1497 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1498 { 1499 const struct bpf_insn *insn = &meta->insn; 1500 u64 imm = insn->imm; /* sign extend */ 1501 swreg tmp_reg; 1502 1503 if (insn->off < 0) /* TODO */ 1504 return -EOPNOTSUPP; 1505 1506 if (!imm) { 1507 meta->skip = true; 1508 return 0; 1509 } 1510 1511 if (imm & ~0U) { 1512 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1513 emit_alu(nfp_prog, reg_none(), 1514 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg); 1515 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1516 } 1517 1518 if (imm >> 32) { 1519 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1520 emit_alu(nfp_prog, reg_none(), 1521 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg); 1522 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1523 } 1524 1525 return 0; 1526 } 1527 1528 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1529 { 1530 const struct bpf_insn *insn = &meta->insn; 1531 u64 imm = insn->imm; /* sign extend */ 1532 swreg tmp_reg; 1533 1534 if (insn->off < 0) /* TODO */ 1535 return -EOPNOTSUPP; 1536 1537 if (!imm) { 1538 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), 1539 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1)); 1540 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1541 return 0; 1542 } 1543 1544 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); 1545 emit_alu(nfp_prog, reg_none(), 1546 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg); 1547 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1548 1549 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); 1550 emit_alu(nfp_prog, reg_none(), 1551 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg); 1552 emit_br(nfp_prog, BR_BNE, insn->off, 0); 1553 1554 return 0; 1555 } 1556 1557 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1558 { 1559 const struct bpf_insn *insn = &meta->insn; 1560 1561 if (insn->off < 0) /* TODO */ 1562 return -EOPNOTSUPP; 1563 1564 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), 1565 ALU_OP_XOR, reg_b(insn->src_reg * 2)); 1566 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1), 1567 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1)); 1568 emit_alu(nfp_prog, reg_none(), 1569 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog)); 1570 emit_br(nfp_prog, BR_BEQ, insn->off, 0); 1571 1572 return 0; 1573 } 1574 1575 static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1576 { 1577 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true); 1578 } 1579 1580 static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1581 { 1582 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false); 1583 } 1584 1585 static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1586 { 1587 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false); 1588 } 1589 1590 static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1591 { 1592 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true); 1593 } 1594 1595 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1596 { 1597 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); 1598 } 1599 1600 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1601 { 1602 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); 1603 } 1604 1605 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 1606 { 1607 wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT); 1608 1609 return 0; 1610 } 1611 1612 static const instr_cb_t instr_cb[256] = { 1613 [BPF_ALU64 | BPF_MOV | BPF_X] = mov_reg64, 1614 [BPF_ALU64 | BPF_MOV | BPF_K] = mov_imm64, 1615 [BPF_ALU64 | BPF_XOR | BPF_X] = xor_reg64, 1616 [BPF_ALU64 | BPF_XOR | BPF_K] = xor_imm64, 1617 [BPF_ALU64 | BPF_AND | BPF_X] = and_reg64, 1618 [BPF_ALU64 | BPF_AND | BPF_K] = and_imm64, 1619 [BPF_ALU64 | BPF_OR | BPF_X] = or_reg64, 1620 [BPF_ALU64 | BPF_OR | BPF_K] = or_imm64, 1621 [BPF_ALU64 | BPF_ADD | BPF_X] = add_reg64, 1622 [BPF_ALU64 | BPF_ADD | BPF_K] = add_imm64, 1623 [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64, 1624 [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64, 1625 [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64, 1626 [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64, 1627 [BPF_ALU | BPF_MOV | BPF_X] = mov_reg, 1628 [BPF_ALU | BPF_MOV | BPF_K] = mov_imm, 1629 [BPF_ALU | BPF_XOR | BPF_X] = xor_reg, 1630 [BPF_ALU | BPF_XOR | BPF_K] = xor_imm, 1631 [BPF_ALU | BPF_AND | BPF_X] = and_reg, 1632 [BPF_ALU | BPF_AND | BPF_K] = and_imm, 1633 [BPF_ALU | BPF_OR | BPF_X] = or_reg, 1634 [BPF_ALU | BPF_OR | BPF_K] = or_imm, 1635 [BPF_ALU | BPF_ADD | BPF_X] = add_reg, 1636 [BPF_ALU | BPF_ADD | BPF_K] = add_imm, 1637 [BPF_ALU | BPF_SUB | BPF_X] = sub_reg, 1638 [BPF_ALU | BPF_SUB | BPF_K] = sub_imm, 1639 [BPF_ALU | BPF_LSH | BPF_K] = shl_imm, 1640 [BPF_ALU | BPF_END | BPF_X] = end_reg32, 1641 [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8, 1642 [BPF_LD | BPF_ABS | BPF_B] = data_ld1, 1643 [BPF_LD | BPF_ABS | BPF_H] = data_ld2, 1644 [BPF_LD | BPF_ABS | BPF_W] = data_ld4, 1645 [BPF_LD | BPF_IND | BPF_B] = data_ind_ld1, 1646 [BPF_LD | BPF_IND | BPF_H] = data_ind_ld2, 1647 [BPF_LD | BPF_IND | BPF_W] = data_ind_ld4, 1648 [BPF_LDX | BPF_MEM | BPF_B] = mem_ldx1, 1649 [BPF_LDX | BPF_MEM | BPF_H] = mem_ldx2, 1650 [BPF_LDX | BPF_MEM | BPF_W] = mem_ldx4, 1651 [BPF_LDX | BPF_MEM | BPF_DW] = mem_ldx8, 1652 [BPF_STX | BPF_MEM | BPF_B] = mem_stx1, 1653 [BPF_STX | BPF_MEM | BPF_H] = mem_stx2, 1654 [BPF_STX | BPF_MEM | BPF_W] = mem_stx4, 1655 [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8, 1656 [BPF_ST | BPF_MEM | BPF_B] = mem_st1, 1657 [BPF_ST | BPF_MEM | BPF_H] = mem_st2, 1658 [BPF_ST | BPF_MEM | BPF_W] = mem_st4, 1659 [BPF_ST | BPF_MEM | BPF_DW] = mem_st8, 1660 [BPF_JMP | BPF_JA | BPF_K] = jump, 1661 [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm, 1662 [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm, 1663 [BPF_JMP | BPF_JGE | BPF_K] = jge_imm, 1664 [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm, 1665 [BPF_JMP | BPF_JLE | BPF_K] = jle_imm, 1666 [BPF_JMP | BPF_JSET | BPF_K] = jset_imm, 1667 [BPF_JMP | BPF_JNE | BPF_K] = jne_imm, 1668 [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg, 1669 [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg, 1670 [BPF_JMP | BPF_JGE | BPF_X] = jge_reg, 1671 [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg, 1672 [BPF_JMP | BPF_JLE | BPF_X] = jle_reg, 1673 [BPF_JMP | BPF_JSET | BPF_X] = jset_reg, 1674 [BPF_JMP | BPF_JNE | BPF_X] = jne_reg, 1675 [BPF_JMP | BPF_EXIT] = goto_out, 1676 }; 1677 1678 /* --- Misc code --- */ 1679 static void br_set_offset(u64 *instr, u16 offset) 1680 { 1681 u16 addr_lo, addr_hi; 1682 1683 addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)); 1684 addr_hi = offset != addr_lo; 1685 *instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO); 1686 *instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi); 1687 *instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo); 1688 } 1689 1690 /* --- Assembler logic --- */ 1691 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 1692 { 1693 struct nfp_insn_meta *meta, *next; 1694 u32 off, br_idx; 1695 u32 idx; 1696 1697 nfp_for_each_insn_walk2(nfp_prog, meta, next) { 1698 if (meta->skip) 1699 continue; 1700 if (BPF_CLASS(meta->insn.code) != BPF_JMP) 1701 continue; 1702 1703 br_idx = nfp_prog_offset_to_index(nfp_prog, next->off) - 1; 1704 if (!nfp_is_br(nfp_prog->prog[br_idx])) { 1705 pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n", 1706 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); 1707 return -ELOOP; 1708 } 1709 /* Leave special branches for later */ 1710 if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx])) 1711 continue; 1712 1713 /* Find the target offset in assembler realm */ 1714 off = meta->insn.off; 1715 if (!off) { 1716 pr_err("Fixup found zero offset!!\n"); 1717 return -ELOOP; 1718 } 1719 1720 while (off && nfp_meta_has_next(nfp_prog, next)) { 1721 next = nfp_meta_next(next); 1722 off--; 1723 } 1724 if (off) { 1725 pr_err("Fixup found too large jump!! %d\n", off); 1726 return -ELOOP; 1727 } 1728 1729 if (next->skip) { 1730 pr_err("Branch landing on removed instruction!!\n"); 1731 return -ELOOP; 1732 } 1733 1734 for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off); 1735 idx <= br_idx; idx++) { 1736 if (!nfp_is_br(nfp_prog->prog[idx])) 1737 continue; 1738 br_set_offset(&nfp_prog->prog[idx], next->off); 1739 } 1740 } 1741 1742 /* Fixup 'goto out's separately, they can be scattered around */ 1743 for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) { 1744 enum br_special special; 1745 1746 if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE) 1747 continue; 1748 1749 special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]); 1750 switch (special) { 1751 case OP_BR_NORMAL: 1752 break; 1753 case OP_BR_GO_OUT: 1754 br_set_offset(&nfp_prog->prog[br_idx], 1755 nfp_prog->tgt_out); 1756 break; 1757 case OP_BR_GO_ABORT: 1758 br_set_offset(&nfp_prog->prog[br_idx], 1759 nfp_prog->tgt_abort); 1760 break; 1761 } 1762 1763 nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL; 1764 } 1765 1766 return 0; 1767 } 1768 1769 static void nfp_intro(struct nfp_prog *nfp_prog) 1770 { 1771 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); 1772 emit_alu(nfp_prog, plen_reg(nfp_prog), 1773 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); 1774 } 1775 1776 static void nfp_outro_tc_legacy(struct nfp_prog *nfp_prog) 1777 { 1778 const u8 act2code[] = { 1779 [NN_ACT_TC_DROP] = 0x22, 1780 [NN_ACT_TC_REDIR] = 0x24 1781 }; 1782 /* Target for aborts */ 1783 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 1784 wrp_immed(nfp_prog, reg_both(0), 0); 1785 1786 /* Target for normal exits */ 1787 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 1788 /* Legacy TC mode: 1789 * 0 0x11 -> pass, count as stat0 1790 * -1 drop 0x22 -> drop, count as stat1 1791 * redir 0x24 -> redir, count as stat1 1792 * ife mark 0x21 -> pass, count as stat1 1793 * ife + tx 0x24 -> redir, count as stat1 1794 */ 1795 emit_br_byte_neq(nfp_prog, reg_b(0), 0xff, 0, nfp_prog->tgt_done, 2); 1796 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1797 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); 1798 1799 emit_br(nfp_prog, BR_UNC, nfp_prog->tgt_done, 1); 1800 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(act2code[nfp_prog->act]), 1801 SHF_SC_L_SHF, 16); 1802 } 1803 1804 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 1805 { 1806 /* TC direct-action mode: 1807 * 0,1 ok NOT SUPPORTED[1] 1808 * 2 drop 0x22 -> drop, count as stat1 1809 * 4,5 nuke 0x02 -> drop 1810 * 7 redir 0x44 -> redir, count as stat2 1811 * * unspec 0x11 -> pass, count as stat0 1812 * 1813 * [1] We can't support OK and RECLASSIFY because we can't tell TC 1814 * the exact decision made. We are forced to support UNSPEC 1815 * to handle aborts so that's the only one we handle for passing 1816 * packets up the stack. 1817 */ 1818 /* Target for aborts */ 1819 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 1820 1821 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1822 1823 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1824 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); 1825 1826 /* Target for normal exits */ 1827 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 1828 1829 /* if R0 > 7 jump to abort */ 1830 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); 1831 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 1832 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1833 1834 wrp_immed(nfp_prog, reg_b(2), 0x41221211); 1835 wrp_immed(nfp_prog, reg_b(3), 0x41001211); 1836 1837 emit_shf(nfp_prog, reg_a(1), 1838 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2); 1839 1840 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 1841 emit_shf(nfp_prog, reg_a(2), 1842 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 1843 1844 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 1845 emit_shf(nfp_prog, reg_b(2), 1846 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0); 1847 1848 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1849 1850 emit_shf(nfp_prog, reg_b(2), 1851 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4); 1852 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 1853 } 1854 1855 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 1856 { 1857 /* XDP return codes: 1858 * 0 aborted 0x82 -> drop, count as stat3 1859 * 1 drop 0x22 -> drop, count as stat1 1860 * 2 pass 0x11 -> pass, count as stat0 1861 * 3 tx 0x44 -> redir, count as stat2 1862 * * unknown 0x82 -> drop, count as stat3 1863 */ 1864 /* Target for aborts */ 1865 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); 1866 1867 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1868 1869 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1870 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); 1871 1872 /* Target for normal exits */ 1873 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); 1874 1875 /* if R0 > 3 jump to abort */ 1876 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); 1877 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); 1878 1879 wrp_immed(nfp_prog, reg_b(2), 0x44112282); 1880 1881 emit_shf(nfp_prog, reg_a(1), 1882 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3); 1883 1884 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); 1885 emit_shf(nfp_prog, reg_b(2), 1886 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0); 1887 1888 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2); 1889 1890 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); 1891 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); 1892 } 1893 1894 static void nfp_outro(struct nfp_prog *nfp_prog) 1895 { 1896 switch (nfp_prog->act) { 1897 case NN_ACT_DIRECT: 1898 nfp_outro_tc_da(nfp_prog); 1899 break; 1900 case NN_ACT_TC_DROP: 1901 case NN_ACT_TC_REDIR: 1902 nfp_outro_tc_legacy(nfp_prog); 1903 break; 1904 case NN_ACT_XDP: 1905 nfp_outro_xdp(nfp_prog); 1906 break; 1907 } 1908 } 1909 1910 static int nfp_translate(struct nfp_prog *nfp_prog) 1911 { 1912 struct nfp_insn_meta *meta; 1913 int err; 1914 1915 nfp_intro(nfp_prog); 1916 if (nfp_prog->error) 1917 return nfp_prog->error; 1918 1919 list_for_each_entry(meta, &nfp_prog->insns, l) { 1920 instr_cb_t cb = instr_cb[meta->insn.code]; 1921 1922 meta->off = nfp_prog_current_offset(nfp_prog); 1923 1924 if (meta->skip) { 1925 nfp_prog->n_translated++; 1926 continue; 1927 } 1928 1929 if (nfp_meta_has_prev(nfp_prog, meta) && 1930 nfp_meta_prev(meta)->double_cb) 1931 cb = nfp_meta_prev(meta)->double_cb; 1932 if (!cb) 1933 return -ENOENT; 1934 err = cb(nfp_prog, meta); 1935 if (err) 1936 return err; 1937 1938 nfp_prog->n_translated++; 1939 } 1940 1941 nfp_outro(nfp_prog); 1942 if (nfp_prog->error) 1943 return nfp_prog->error; 1944 1945 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); 1946 if (nfp_prog->error) 1947 return nfp_prog->error; 1948 1949 return nfp_fixup_branches(nfp_prog); 1950 } 1951 1952 static int 1953 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 1954 unsigned int cnt) 1955 { 1956 unsigned int i; 1957 1958 for (i = 0; i < cnt; i++) { 1959 struct nfp_insn_meta *meta; 1960 1961 meta = kzalloc(sizeof(*meta), GFP_KERNEL); 1962 if (!meta) 1963 return -ENOMEM; 1964 1965 meta->insn = prog[i]; 1966 meta->n = i; 1967 1968 list_add_tail(&meta->l, &nfp_prog->insns); 1969 } 1970 1971 return 0; 1972 } 1973 1974 /* --- Optimizations --- */ 1975 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) 1976 { 1977 struct nfp_insn_meta *meta; 1978 1979 list_for_each_entry(meta, &nfp_prog->insns, l) { 1980 struct bpf_insn insn = meta->insn; 1981 1982 /* Programs converted from cBPF start with register xoring */ 1983 if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) && 1984 insn.src_reg == insn.dst_reg) 1985 continue; 1986 1987 /* Programs start with R6 = R1 but we ignore the skb pointer */ 1988 if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) && 1989 insn.src_reg == 1 && insn.dst_reg == 6) 1990 meta->skip = true; 1991 1992 /* Return as soon as something doesn't match */ 1993 if (!meta->skip) 1994 return; 1995 } 1996 } 1997 1998 /* Remove masking after load since our load guarantees this is not needed */ 1999 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) 2000 { 2001 struct nfp_insn_meta *meta1, *meta2; 2002 const s32 exp_mask[] = { 2003 [BPF_B] = 0x000000ffU, 2004 [BPF_H] = 0x0000ffffU, 2005 [BPF_W] = 0xffffffffU, 2006 }; 2007 2008 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { 2009 struct bpf_insn insn, next; 2010 2011 insn = meta1->insn; 2012 next = meta2->insn; 2013 2014 if (BPF_CLASS(insn.code) != BPF_LD) 2015 continue; 2016 if (BPF_MODE(insn.code) != BPF_ABS && 2017 BPF_MODE(insn.code) != BPF_IND) 2018 continue; 2019 2020 if (next.code != (BPF_ALU64 | BPF_AND | BPF_K)) 2021 continue; 2022 2023 if (!exp_mask[BPF_SIZE(insn.code)]) 2024 continue; 2025 if (exp_mask[BPF_SIZE(insn.code)] != next.imm) 2026 continue; 2027 2028 if (next.src_reg || next.dst_reg) 2029 continue; 2030 2031 meta2->skip = true; 2032 } 2033 } 2034 2035 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) 2036 { 2037 struct nfp_insn_meta *meta1, *meta2, *meta3; 2038 2039 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { 2040 struct bpf_insn insn, next1, next2; 2041 2042 insn = meta1->insn; 2043 next1 = meta2->insn; 2044 next2 = meta3->insn; 2045 2046 if (BPF_CLASS(insn.code) != BPF_LD) 2047 continue; 2048 if (BPF_MODE(insn.code) != BPF_ABS && 2049 BPF_MODE(insn.code) != BPF_IND) 2050 continue; 2051 if (BPF_SIZE(insn.code) != BPF_W) 2052 continue; 2053 2054 if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) && 2055 next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) && 2056 !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) && 2057 next2.code == (BPF_LSH | BPF_K | BPF_ALU64))) 2058 continue; 2059 2060 if (next1.src_reg || next1.dst_reg || 2061 next2.src_reg || next2.dst_reg) 2062 continue; 2063 2064 if (next1.imm != 0x20 || next2.imm != 0x20) 2065 continue; 2066 2067 meta2->skip = true; 2068 meta3->skip = true; 2069 } 2070 } 2071 2072 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) 2073 { 2074 nfp_bpf_opt_reg_init(nfp_prog); 2075 2076 nfp_bpf_opt_ld_mask(nfp_prog); 2077 nfp_bpf_opt_ld_shift(nfp_prog); 2078 2079 return 0; 2080 } 2081 2082 static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore) 2083 { 2084 int i; 2085 2086 for (i = 0; i < nfp_prog->prog_len; i++) { 2087 int err; 2088 2089 err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]); 2090 if (err) 2091 return err; 2092 2093 nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]); 2094 2095 ustore[i] = cpu_to_le64(nfp_prog->prog[i]); 2096 } 2097 2098 return 0; 2099 } 2100 2101 /** 2102 * nfp_bpf_jit() - translate BPF code into NFP assembly 2103 * @filter: kernel BPF filter struct 2104 * @prog_mem: memory to store assembler instructions 2105 * @act: action attached to this eBPF program 2106 * @prog_start: offset of the first instruction when loaded 2107 * @prog_done: where to jump on exit 2108 * @prog_sz: size of @prog_mem in instructions 2109 * @res: achieved parameters of translation results 2110 */ 2111 int 2112 nfp_bpf_jit(struct bpf_prog *filter, void *prog_mem, 2113 enum nfp_bpf_action_type act, 2114 unsigned int prog_start, unsigned int prog_done, 2115 unsigned int prog_sz, struct nfp_bpf_result *res) 2116 { 2117 struct nfp_prog *nfp_prog; 2118 int ret; 2119 2120 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); 2121 if (!nfp_prog) 2122 return -ENOMEM; 2123 2124 INIT_LIST_HEAD(&nfp_prog->insns); 2125 nfp_prog->act = act; 2126 nfp_prog->start_off = prog_start; 2127 nfp_prog->tgt_done = prog_done; 2128 2129 ret = nfp_prog_prepare(nfp_prog, filter->insnsi, filter->len); 2130 if (ret) 2131 goto out; 2132 2133 ret = nfp_prog_verify(nfp_prog, filter); 2134 if (ret) 2135 goto out; 2136 2137 ret = nfp_bpf_optimize(nfp_prog); 2138 if (ret) 2139 goto out; 2140 2141 nfp_prog->num_regs = MAX_BPF_REG; 2142 nfp_prog->regs_per_thread = 32; 2143 2144 nfp_prog->prog = prog_mem; 2145 nfp_prog->__prog_alloc_len = prog_sz; 2146 2147 ret = nfp_translate(nfp_prog); 2148 if (ret) { 2149 pr_err("Translation failed with error %d (translated: %u)\n", 2150 ret, nfp_prog->n_translated); 2151 ret = -EINVAL; 2152 goto out; 2153 } 2154 2155 ret = nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)prog_mem); 2156 2157 res->n_instr = nfp_prog->prog_len; 2158 res->dense_mode = false; 2159 out: 2160 nfp_prog_free(nfp_prog); 2161 2162 return ret; 2163 } 2164