leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bpf, arm64: take advantage of stack_depth tracking

Browse Source 
Make use of recently implemented stack_depth tracking for arm64 JIT,
so that stack usage can be reduced heavily for programs not using
tail calls at least.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Daniel Borkmann 2017-06-11 03:55:27 +02:00 committed by David S. Miller
parent 66e037ca57
commit f1c9eed7f4
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
arch/arm64/net/bpf_jit_comp.c
View File

@ -69,6 +69,7 @@ struct jit_ctx {
int epilogue_offset; int epilogue_offset;
int *offset; int *offset;
u32 *image; u32 *image;
u32 stack_size;
}; };
static inline void emit(const u32 insn, struct jit_ctx *ctx) static inline void emit(const u32 insn, struct jit_ctx *ctx)
@ -145,16 +146,11 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
/* Stack must be multiples of 16B */ /* Stack must be multiples of 16B */
#define STACK_ALIGN(sz) (((sz) + 15) & ~15) #define STACK_ALIGN(sz) (((sz) + 15) & ~15)
#define _STACK_SIZE \
(MAX_BPF_STACK \
+ 4 /* extra for skb_copy_bits buffer */)
#define STACK_SIZE STACK_ALIGN(_STACK_SIZE)
#define PROLOGUE_OFFSET 8 #define PROLOGUE_OFFSET 8
static int build_prologue(struct jit_ctx *ctx) static int build_prologue(struct jit_ctx *ctx)
{ {
const struct bpf_prog *prog = ctx->prog;
const u8 r6 = bpf2a64[BPF_REG_6]; const u8 r6 = bpf2a64[BPF_REG_6];
const u8 r7 = bpf2a64[BPF_REG_7]; const u8 r7 = bpf2a64[BPF_REG_7];
const u8 r8 = bpf2a64[BPF_REG_8]; const u8 r8 = bpf2a64[BPF_REG_8];
@ -176,9 +172,9 @@ static int build_prologue(struct jit_ctx *ctx)
* | | * | |
* | ... | BPF prog stack * | ... | BPF prog stack
* | | * | |
* +-----+ <= (BPF_FP - MAX_BPF_STACK) * +-----+ <= (BPF_FP - prog->aux->stack_depth)
* |RSVD | JIT scratchpad * |RSVD | JIT scratchpad
* current A64_SP => +-----+ <= (BPF_FP - STACK_SIZE) * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size)
* | | * | |
* | ... | Function call stack * | ... | Function call stack
* | | * | |
@ -202,8 +198,12 @@ static int build_prologue(struct jit_ctx *ctx)
/* Initialize tail_call_cnt */ /* Initialize tail_call_cnt */
emit(A64_MOVZ(1, tcc, 0, 0), ctx); emit(A64_MOVZ(1, tcc, 0, 0), ctx);
/* 4 byte extra for skb_copy_bits buffer */
ctx->stack_size = prog->aux->stack_depth + 4;
ctx->stack_size = STACK_ALIGN(ctx->stack_size);
/* Set up function call stack */ /* Set up function call stack */
emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
cur_offset = ctx->idx - idx0; cur_offset = ctx->idx - idx0;
if (cur_offset != PROLOGUE_OFFSET) { if (cur_offset != PROLOGUE_OFFSET) {
@ -288,7 +288,7 @@ static void build_epilogue(struct jit_ctx *ctx)
const u8 fp = bpf2a64[BPF_REG_FP]; const u8 fp = bpf2a64[BPF_REG_FP];
/* We're done with BPF stack */ /* We're done with BPF stack */
emit(A64_ADD_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
/* Restore fs (x25) and x26 */ /* Restore fs (x25) and x26 */
emit(A64_POP(fp, A64_R(26), A64_SP), ctx); emit(A64_POP(fp, A64_R(26), A64_SP), ctx);
@ -732,7 +732,7 @@ emit_cond_jmp:
return -EINVAL; return -EINVAL;
} }
emit_a64_mov_i64(r3, size, ctx); emit_a64_mov_i64(r3, size, ctx);
emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx); emit(A64_SUB_I(1, r4, fp, ctx->stack_size), ctx);
emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);
emit(A64_BLR(r5), ctx); emit(A64_BLR(r5), ctx);
emit(A64_MOV(1, r0, A64_R(0)), ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx);