Lines Matching refs:dst
174 static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) in emit_mov_se_i64() argument
176 emit_mov_i(ctx, lo(dst), imm); in emit_mov_se_i64()
178 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); in emit_mov_se_i64()
180 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_mov_se_i64()
181 clobber_reg64(ctx, dst); in emit_mov_se_i64()
185 static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) in emit_zext_ver() argument
188 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_zext_ver()
189 clobber_reg(ctx, hi(dst)); in emit_zext_ver()
202 const u8 dst[], s32 imm, u8 op) in emit_alu_i64() argument
228 emit(ctx, addu, lo(dst), lo(dst), src); in emit_alu_i64()
229 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); in emit_alu_i64()
230 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_i64()
232 emit(ctx, addiu, hi(dst), hi(dst), -1); in emit_alu_i64()
236 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); in emit_alu_i64()
237 emit(ctx, subu, lo(dst), lo(dst), src); in emit_alu_i64()
238 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_i64()
240 emit(ctx, addiu, hi(dst), hi(dst), 1); in emit_alu_i64()
244 emit(ctx, or, lo(dst), lo(dst), src); in emit_alu_i64()
246 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); in emit_alu_i64()
250 emit(ctx, and, lo(dst), lo(dst), src); in emit_alu_i64()
252 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_alu_i64()
256 emit(ctx, xor, lo(dst), lo(dst), src); in emit_alu_i64()
258 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); in emit_alu_i64()
259 emit(ctx, addiu, hi(dst), hi(dst), -1); in emit_alu_i64()
263 clobber_reg64(ctx, dst); in emit_alu_i64()
268 const u8 dst[], const u8 src[], u8 op) in emit_alu_r64() argument
273 if (src == dst) { in emit_alu_r64()
274 emit(ctx, srl, MIPS_R_T9, lo(dst), 31); in emit_alu_r64()
275 emit(ctx, addu, lo(dst), lo(dst), lo(dst)); in emit_alu_r64()
277 emit(ctx, addu, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
278 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); in emit_alu_r64()
280 emit(ctx, addu, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
281 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_r64()
285 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); in emit_alu_r64()
286 emit(ctx, subu, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
287 emit(ctx, subu, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
288 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_r64()
292 emit(ctx, or, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
293 emit(ctx, or, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
297 emit(ctx, and, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
298 emit(ctx, and, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
302 emit(ctx, xor, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
303 emit(ctx, xor, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
306 clobber_reg64(ctx, dst); in emit_alu_r64()
310 static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) in emit_neg_i64() argument
312 emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); in emit_neg_i64()
313 emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); in emit_neg_i64()
314 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); in emit_neg_i64()
315 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_neg_i64()
317 clobber_reg64(ctx, dst); in emit_neg_i64()
322 const u8 dst[], u32 imm, u8 op) in emit_shift_i64() argument
328 emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); in emit_shift_i64()
329 emit(ctx, sll, lo(dst), lo(dst), imm); in emit_shift_i64()
330 emit(ctx, sll, hi(dst), hi(dst), imm); in emit_shift_i64()
331 emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); in emit_shift_i64()
333 emit(ctx, sll, hi(dst), lo(dst), imm - 32); in emit_shift_i64()
334 emit(ctx, move, lo(dst), MIPS_R_ZERO); in emit_shift_i64()
340 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); in emit_shift_i64()
341 emit(ctx, srl, lo(dst), lo(dst), imm); in emit_shift_i64()
342 emit(ctx, srl, hi(dst), hi(dst), imm); in emit_shift_i64()
343 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); in emit_shift_i64()
345 emit(ctx, srl, lo(dst), hi(dst), imm - 32); in emit_shift_i64()
346 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_shift_i64()
352 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); in emit_shift_i64()
353 emit(ctx, srl, lo(dst), lo(dst), imm); in emit_shift_i64()
354 emit(ctx, sra, hi(dst), hi(dst), imm); in emit_shift_i64()
355 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); in emit_shift_i64()
357 emit(ctx, sra, lo(dst), hi(dst), imm - 32); in emit_shift_i64()
358 emit(ctx, sra, hi(dst), hi(dst), 31); in emit_shift_i64()
362 clobber_reg64(ctx, dst); in emit_shift_i64()
367 const u8 dst[], u8 src, u8 op) in emit_shift_r64() argument
380 emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ in emit_shift_r64()
381 emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ in emit_shift_r64()
384 emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ in emit_shift_r64()
386 emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ in emit_shift_r64()
387 emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ in emit_shift_r64()
388 emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ in emit_shift_r64()
393 emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ in emit_shift_r64()
394 emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ in emit_shift_r64()
397 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ in emit_shift_r64()
399 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ in emit_shift_r64()
400 emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ in emit_shift_r64()
401 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ in emit_shift_r64()
406 emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ in emit_shift_r64()
407 emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ in emit_shift_r64()
410 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ in emit_shift_r64()
412 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ in emit_shift_r64()
413 emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ in emit_shift_r64()
414 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ in emit_shift_r64()
419 clobber_reg64(ctx, dst); in emit_shift_r64()
423 static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) in emit_mul_i64() argument
434 emit_neg_i64(ctx, dst); in emit_mul_i64()
437 emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); in emit_mul_i64()
438 emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); in emit_mul_i64()
445 emit(ctx, mul, hi(dst), hi(dst), src); in emit_mul_i64()
447 emit(ctx, multu, hi(dst), src); in emit_mul_i64()
448 emit(ctx, mflo, hi(dst)); in emit_mul_i64()
453 emit(ctx, subu, hi(dst), hi(dst), lo(dst)); in emit_mul_i64()
458 emit(ctx, muhu, tmp, lo(dst), src); in emit_mul_i64()
459 emit(ctx, mulu, lo(dst), lo(dst), src); in emit_mul_i64()
461 emit(ctx, multu, lo(dst), src); in emit_mul_i64()
462 emit(ctx, mflo, lo(dst)); in emit_mul_i64()
467 emit(ctx, addu, hi(dst), hi(dst), tmp); in emit_mul_i64()
468 clobber_reg64(ctx, dst); in emit_mul_i64()
475 const u8 dst[], const u8 src[]) in emit_mul_r64() argument
482 emit(ctx, mul, acc, hi(dst), lo(src)); in emit_mul_r64()
484 emit(ctx, multu, hi(dst), lo(src)); in emit_mul_r64()
490 emit(ctx, mul, tmp, lo(dst), hi(src)); in emit_mul_r64()
492 emit(ctx, multu, lo(dst), hi(src)); in emit_mul_r64()
502 emit(ctx, muhu, tmp, lo(dst), lo(src)); in emit_mul_r64()
503 emit(ctx, mulu, lo(dst), lo(dst), lo(src)); in emit_mul_r64()
505 emit(ctx, multu, lo(dst), lo(src)); in emit_mul_r64()
506 emit(ctx, mflo, lo(dst)); in emit_mul_r64()
511 emit(ctx, addu, hi(dst), acc, tmp); in emit_mul_r64()
512 clobber_reg64(ctx, dst); in emit_mul_r64()
526 const u8 dst[], const u8 src[], u8 op) in emit_divmod_r64() argument
541 emit(ctx, move, r1[k], dst[k]); in emit_divmod_r64()
561 emit(ctx, move, dst[0], r0[0]); in emit_divmod_r64()
562 emit(ctx, move, dst[1], r0[1]); in emit_divmod_r64()
565 exclude = BIT(lo(dst)) | BIT(hi(dst)); in emit_divmod_r64()
570 clobber_reg64(ctx, dst); in emit_divmod_r64()
577 static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) in emit_swap8_r() argument
583 emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ in emit_swap8_r()
584 emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ in emit_swap8_r()
585 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_swap8_r()
589 static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) in emit_swap16_r() argument
594 emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ in emit_swap16_r()
595 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_swap16_r()
599 static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) in emit_bswap_r64() argument
607 emit(ctx, rotr, tmp, hi(dst), 16); in emit_bswap_r64()
608 emit(ctx, rotr, hi(dst), lo(dst), 16); in emit_bswap_r64()
609 emit(ctx, wsbh, lo(dst), tmp); in emit_bswap_r64()
610 emit(ctx, wsbh, hi(dst), hi(dst)); in emit_bswap_r64()
612 emit_swap16_r(ctx, tmp, lo(dst)); in emit_bswap_r64()
613 emit_swap16_r(ctx, lo(dst), hi(dst)); in emit_bswap_r64()
614 emit(ctx, move, hi(dst), tmp); in emit_bswap_r64()
618 emit_swap8_r(ctx, lo(dst), lo(dst), tmp); in emit_bswap_r64()
619 emit_swap8_r(ctx, hi(dst), hi(dst), tmp); in emit_bswap_r64()
626 emit_bswap_r(ctx, lo(dst), width); in emit_bswap_r64()
627 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_bswap_r64()
630 clobber_reg64(ctx, dst); in emit_bswap_r64()
634 static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) in emit_trunc_r64() argument
641 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_trunc_r64()
642 clobber_reg(ctx, hi(dst)); in emit_trunc_r64()
646 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_trunc_r64()
647 emit(ctx, andi, lo(dst), lo(dst), 0xffff); in emit_trunc_r64()
648 clobber_reg64(ctx, dst); in emit_trunc_r64()
655 const u8 dst[], u8 src, s16 off, u8 size) in emit_ldx() argument
660 emit(ctx, lbu, lo(dst), off, src); in emit_ldx()
661 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
665 emit(ctx, lhu, lo(dst), off, src); in emit_ldx()
666 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
670 emit(ctx, lw, lo(dst), off, src); in emit_ldx()
671 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
675 if (dst[1] == src) { in emit_ldx()
676 emit(ctx, lw, dst[0], off + 4, src); in emit_ldx()
677 emit(ctx, lw, dst[1], off, src); in emit_ldx()
679 emit(ctx, lw, dst[1], off, src); in emit_ldx()
680 emit(ctx, lw, dst[0], off + 4, src); in emit_ldx()
685 clobber_reg64(ctx, dst); in emit_ldx()
690 const u8 dst, const u8 src[], s16 off, u8 size) in emit_stx() argument
695 emit(ctx, sb, lo(src), off, dst); in emit_stx()
699 emit(ctx, sh, lo(src), off, dst); in emit_stx()
703 emit(ctx, sw, lo(src), off, dst); in emit_stx()
707 emit(ctx, sw, src[1], off, dst); in emit_stx()
708 emit(ctx, sw, src[0], off + 4, dst); in emit_stx()
715 u8 dst, u8 src, s16 off, u8 code) in emit_atomic_r32() argument
727 emit(ctx, move, MIPS_R_T9, dst); in emit_atomic_r32()
798 u8 dst, const u8 src[], s16 off, u8 code) in emit_atomic_r64() argument
812 emit(ctx, move, MIPS_R_T9, dst); in emit_atomic_r64()
873 static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) in emit_cmpxchg_r32() argument
885 emit(ctx, addiu, MIPS_R_T9, dst, off); in emit_cmpxchg_r32()
910 u8 dst, const u8 src[], s16 off) in emit_cmpxchg_r64() argument
924 emit(ctx, addiu, MIPS_R_T9, dst, off); in emit_cmpxchg_r64()
1093 const u8 dst[], s32 imm, s32 off, u8 op) in emit_jmp_i64() argument
1106 emit(ctx, addiu, tmp, lo(dst), -imm); in emit_jmp_i64()
1108 emit(ctx, xori, tmp, lo(dst), imm); in emit_jmp_i64()
1111 emit(ctx, xor, tmp, lo(dst), tmp); in emit_jmp_i64()
1114 emit(ctx, addu, MIPS_R_T9, hi(dst), 1); in emit_jmp_i64()
1117 emit(ctx, or, tmp, tmp, hi(dst)); in emit_jmp_i64()
1129 emit(ctx, andi, tmp, lo(dst), imm); in emit_jmp_i64()
1132 emit(ctx, and, tmp, lo(dst), tmp); in emit_jmp_i64()
1135 emit(ctx, or, tmp, tmp, hi(dst)); in emit_jmp_i64()
1143 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1148 emit_sltiu_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1153 emit_sltiu_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1158 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1163 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1168 emit_slti_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1173 emit_slti_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1178 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1186 const u8 dst[], const u8 src[], s32 off, u8 op) in emit_jmp_r64() argument
1199 emit(ctx, subu, t1, lo(dst), lo(src)); in emit_jmp_r64()
1200 emit(ctx, subu, t2, hi(dst), hi(src)); in emit_jmp_r64()
1211 emit(ctx, and, t1, lo(dst), lo(src)); in emit_jmp_r64()
1212 emit(ctx, and, t2, hi(dst), hi(src)); in emit_jmp_r64()
1221 emit_sltu_r64(ctx, t1, src, dst); in emit_jmp_r64()
1226 emit_sltu_r64(ctx, t1, dst, src); in emit_jmp_r64()
1231 emit_sltu_r64(ctx, t1, dst, src); in emit_jmp_r64()
1236 emit_sltu_r64(ctx, t1, src, dst); in emit_jmp_r64()
1241 emit_slt_r64(ctx, t1, src, dst); in emit_jmp_r64()
1246 emit_slt_r64(ctx, t1, dst, src); in emit_jmp_r64()
1251 emit_slt_r64(ctx, t1, dst, src); in emit_jmp_r64()
1256 emit_slt_r64(ctx, t1, src, dst); in emit_jmp_r64()
1465 const u8 *dst = bpf2mips32[insn->dst_reg]; in build_insn() local
1479 emit_mov_i(ctx, lo(dst), imm); in build_insn()
1480 emit_zext_ver(ctx, dst); in build_insn()
1486 emit_mov_i(ctx, hi(dst), 0); in build_insn()
1488 emit_mov_r(ctx, lo(dst), lo(src)); in build_insn()
1489 emit_zext_ver(ctx, dst); in build_insn()
1494 emit_alu_i(ctx, lo(dst), 0, BPF_NEG); in build_insn()
1495 emit_zext_ver(ctx, dst); in build_insn()
1521 emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); in build_insn()
1523 emit_alu_i(ctx, lo(dst), val, alu); in build_insn()
1525 emit_zext_ver(ctx, dst); in build_insn()
1549 emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); in build_insn()
1550 emit_zext_ver(ctx, dst); in build_insn()
1554 emit_mov_se_i64(ctx, dst, imm); in build_insn()
1558 emit_mov_r(ctx, lo(dst), lo(src)); in build_insn()
1559 emit_mov_r(ctx, hi(dst), hi(src)); in build_insn()
1563 emit_neg_i64(ctx, dst); in build_insn()
1567 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1578 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1587 emit_shift_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1591 emit_mul_i64(ctx, dst, imm); in build_insn()
1602 emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); in build_insn()
1614 emit_alu_r64(ctx, dst, src, BPF_OP(code)); in build_insn()
1622 emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); in build_insn()
1626 emit_mul_r64(ctx, dst, src); in build_insn()
1632 emit_divmod_r64(ctx, dst, src, BPF_OP(code)); in build_insn()
1645 emit_bswap_r64(ctx, dst, imm); in build_insn()
1647 emit_trunc_r64(ctx, dst, imm); in build_insn()
1651 emit_mov_i(ctx, lo(dst), imm); in build_insn()
1652 emit_mov_i(ctx, hi(dst), insn[1].imm); in build_insn()
1659 emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); in build_insn()
1677 emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); in build_insn()
1684 emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); in build_insn()
1702 emit_atomic_r(ctx, lo(dst), lo(src), off, imm); in build_insn()
1704 emit_atomic_r32(ctx, lo(dst), lo(src), in build_insn()
1711 emit_cmpxchg_r(ctx, lo(dst), lo(src), in build_insn()
1714 emit_cmpxchg_r32(ctx, lo(dst), lo(src), off); in build_insn()
1733 emit_atomic_r64(ctx, lo(dst), src, off, imm); in build_insn()
1736 emit_cmpxchg_r64(ctx, lo(dst), src, off); in build_insn()
1766 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
1767 emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); in build_insn()
1797 emit_jmp_i(ctx, lo(dst), imm, rel, jmp); in build_insn()
1801 emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); in build_insn()
1830 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
1831 emit_jmp_r64(ctx, dst, src, rel, jmp); in build_insn()
1860 emit_jmp_i64(ctx, dst, imm, rel, jmp); in build_insn()