summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJerin Jacob <jerinj@marvell.com>2019-09-03 16:29:37 +0530
committerThomas Monjalon <thomas@monjalon.net>2019-10-12 14:20:29 +0200
commit2acfae37f6b32b6ca245e9a571b9aa84ffd67dd2 (patch)
tree142e4f0b4c9c0f5548f60daf846f48fb0a13af7b
parente00906bdc700aa0397114b2773333d19aa612bae (diff)
downloaddpdk-next-eventdev-2acfae37f6b32b6ca245e9a571b9aa84ffd67dd2.zip
dpdk-next-eventdev-2acfae37f6b32b6ca245e9a571b9aa84ffd67dd2.tar.gz
dpdk-next-eventdev-2acfae37f6b32b6ca245e9a571b9aa84ffd67dd2.tar.xz
bpf/arm: add atomic-exchange-and-add operation
Implement XADD eBPF instruction using STADD arm64 instruction. If the given platform does not have atomics support, use LDXR and STXR pair for critical section instead of STADD. Signed-off-by: Jerin Jacob <jerinj@marvell.com> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
-rw-r--r--lib/librte_bpf/bpf_jit_arm64.c85
1 files changed, 84 insertions, 1 deletions
diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c
index c797c9c..62fa6a5 100644
--- a/lib/librte_bpf/bpf_jit_arm64.c
+++ b/lib/librte_bpf/bpf_jit_arm64.c
@@ -838,6 +838,83 @@ emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
}
static void
+emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0xb820001f;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
+}
+
+static void
+emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x885f7c00;
+ insn |= (!!is64) << 30;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
+}
+
+static void
+emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
+ uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x88007c00;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
+}
+
+static int
+has_atomics(void)
+{
+ int rc = 0;
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+ rc = 1;
+#endif
+ return rc;
+}
+
+static void
+emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
+ uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
+{
+ bool is64 = (BPF_SIZE(op) == EBPF_DW);
+ uint8_t rn;
+
+ if (off) {
+ emit_mov_imm(ctx, 1, tmp1, off);
+ emit_add(ctx, 1, tmp1, dst);
+ rn = tmp1;
+ } else {
+ rn = dst;
+ }
+
+ if (has_atomics()) {
+ emit_stadd(ctx, is64, src, rn);
+ } else {
+ emit_ldxr(ctx, is64, tmp2, rn);
+ emit_add(ctx, is64, tmp2, src);
+ emit_stxr(ctx, is64, tmp3, tmp2, rn);
+ emit_cbnz(ctx, is64, tmp3, -3);
+ }
+}
+
+static void
check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
const struct ebpf_insn *ins;
@@ -863,7 +940,7 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
static int
emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
- uint8_t op, dst, src, tmp1, tmp2;
+ uint8_t op, dst, src, tmp1, tmp2, tmp3;
const struct ebpf_insn *ins;
uint64_t u64;
int16_t off;
@@ -878,6 +955,7 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
+ tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
emit_prologue(ctx);
@@ -1067,6 +1145,11 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
emit_mov_imm(ctx, 1, tmp2, off);
emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
break;
+ /* STX XADD: lock *(size *)(dst + off) += src */
+ case (BPF_STX | EBPF_XADD | BPF_W):
+ case (BPF_STX | EBPF_XADD | EBPF_DW):
+ emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
+ break;
/* Return r0 */
case (BPF_JMP | EBPF_EXIT):
emit_epilogue(ctx);