mirror of
https://github.com/torvalds/linux.git
synced 2024-10-27 23:31:45 +00:00
Merge branch 'move-ld_abs-to-native-BPF'
Daniel Borkmann says: ==================== This set simplifies BPF JITs significantly by moving ld_abs/ld_ind to native BPF, for details see individual patches. Main rationale is in patch 'implement ld_abs/ld_ind in native bpf'. Thanks! v1 -> v2: - Added missing seen_lds_abs in LDX_MSH and use X = A initially due to being preserved on func call. - Added a large batch of cBPF tests into test_bpf. - Added x32 removal of LD_ABS/LD_IND, so all JITs are covered. ==================== Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
commit
5234ccf2be
|
@ -1452,83 +1452,6 @@ exit:
|
|||
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
|
||||
emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));
|
||||
break;
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
{
|
||||
const u8 r4 = bpf2a32[BPF_REG_6][1]; /* r4 = ptr to sk_buff */
|
||||
const u8 r0 = bpf2a32[BPF_REG_0][1]; /*r0: struct sk_buff *skb*/
|
||||
/* rtn value */
|
||||
const u8 r1 = bpf2a32[BPF_REG_0][0]; /* r1: int k */
|
||||
const u8 r2 = bpf2a32[BPF_REG_1][1]; /* r2: unsigned int size */
|
||||
const u8 r3 = bpf2a32[BPF_REG_1][0]; /* r3: void *buffer */
|
||||
const u8 r6 = bpf2a32[TMP_REG_1][1]; /* r6: void *(*func)(..) */
|
||||
int size;
|
||||
|
||||
/* Setting up first argument */
|
||||
emit(ARM_MOV_R(r0, r4), ctx);
|
||||
|
||||
/* Setting up second argument */
|
||||
emit_a32_mov_i(r1, imm, false, ctx);
|
||||
if (BPF_MODE(code) == BPF_IND)
|
||||
emit_a32_alu_r(r1, src_lo, false, sstk, ctx,
|
||||
false, false, BPF_ADD);
|
||||
|
||||
/* Setting up third argument */
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
size = 4;
|
||||
break;
|
||||
case BPF_H:
|
||||
size = 2;
|
||||
break;
|
||||
case BPF_B:
|
||||
size = 1;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
emit_a32_mov_i(r2, size, false, ctx);
|
||||
|
||||
/* Setting up fourth argument */
|
||||
emit(ARM_ADD_I(r3, ARM_SP, imm8m(SKB_BUFFER)), ctx);
|
||||
|
||||
/* Setting up function pointer to call */
|
||||
emit_a32_mov_i(r6, (unsigned int)bpf_load_pointer, false, ctx);
|
||||
emit_blx_r(r6, ctx);
|
||||
|
||||
emit(ARM_EOR_R(r1, r1, r1), ctx);
|
||||
/* Check if return address is NULL or not.
|
||||
* if NULL then jump to epilogue
|
||||
* else continue to load the value from retn address
|
||||
*/
|
||||
emit(ARM_CMP_I(r0, 0), ctx);
|
||||
jmp_offset = epilogue_offset(ctx);
|
||||
check_imm24(jmp_offset);
|
||||
_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
|
||||
|
||||
/* Load value from the address */
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
emit(ARM_LDR_I(r0, r0, 0), ctx);
|
||||
emit_rev32(r0, r0, ctx);
|
||||
break;
|
||||
case BPF_H:
|
||||
emit(ARM_LDRH_I(r0, r0, 0), ctx);
|
||||
emit_rev16(r0, r0, ctx);
|
||||
break;
|
||||
case BPF_B:
|
||||
emit(ARM_LDRB_I(r0, r0, 0), ctx);
|
||||
/* No need to reverse */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
/* ST: *(size *)(dst + off) = imm */
|
||||
case BPF_ST | BPF_MEM | BPF_W:
|
||||
case BPF_ST | BPF_MEM | BPF_H:
|
||||
|
|
|
@ -723,71 +723,6 @@ emit_cond_jmp:
|
|||
emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
|
||||
break;
|
||||
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
{
|
||||
const u8 r0 = bpf2a64[BPF_REG_0]; /* r0 = return value */
|
||||
const u8 r6 = bpf2a64[BPF_REG_6]; /* r6 = pointer to sk_buff */
|
||||
const u8 fp = bpf2a64[BPF_REG_FP];
|
||||
const u8 r1 = bpf2a64[BPF_REG_1]; /* r1: struct sk_buff *skb */
|
||||
const u8 r2 = bpf2a64[BPF_REG_2]; /* r2: int k */
|
||||
const u8 r3 = bpf2a64[BPF_REG_3]; /* r3: unsigned int size */
|
||||
const u8 r4 = bpf2a64[BPF_REG_4]; /* r4: void *buffer */
|
||||
const u8 r5 = bpf2a64[BPF_REG_5]; /* r5: void *(*func)(...) */
|
||||
int size;
|
||||
|
||||
emit(A64_MOV(1, r1, r6), ctx);
|
||||
emit_a64_mov_i(0, r2, imm, ctx);
|
||||
if (BPF_MODE(code) == BPF_IND)
|
||||
emit(A64_ADD(0, r2, r2, src), ctx);
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
size = 4;
|
||||
break;
|
||||
case BPF_H:
|
||||
size = 2;
|
||||
break;
|
||||
case BPF_B:
|
||||
size = 1;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
emit_a64_mov_i64(r3, size, ctx);
|
||||
emit(A64_SUB_I(1, r4, fp, ctx->stack_size), ctx);
|
||||
emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);
|
||||
emit(A64_BLR(r5), ctx);
|
||||
emit(A64_MOV(1, r0, A64_R(0)), ctx);
|
||||
|
||||
jmp_offset = epilogue_offset(ctx);
|
||||
check_imm19(jmp_offset);
|
||||
emit(A64_CBZ(1, r0, jmp_offset), ctx);
|
||||
emit(A64_MOV(1, r5, r0), ctx);
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
emit(A64_LDR32(r0, r5, A64_ZR), ctx);
|
||||
#ifndef CONFIG_CPU_BIG_ENDIAN
|
||||
emit(A64_REV32(0, r0, r0), ctx);
|
||||
#endif
|
||||
break;
|
||||
case BPF_H:
|
||||
emit(A64_LDRH(r0, r5, A64_ZR), ctx);
|
||||
#ifndef CONFIG_CPU_BIG_ENDIAN
|
||||
emit(A64_REV16(0, r0, r0), ctx);
|
||||
#endif
|
||||
break;
|
||||
case BPF_B:
|
||||
emit(A64_LDRB(r0, r5, A64_ZR), ctx);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
pr_err_once("unknown opcode %02x\n", code);
|
||||
return -EINVAL;
|
||||
|
|
|
@ -1267,110 +1267,6 @@ jeq_common:
|
|||
return -EINVAL;
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_B | BPF_ABS:
|
||||
case BPF_LD | BPF_H | BPF_ABS:
|
||||
case BPF_LD | BPF_W | BPF_ABS:
|
||||
case BPF_LD | BPF_DW | BPF_ABS:
|
||||
ctx->flags |= EBPF_SAVE_RA;
|
||||
|
||||
gen_imm_to_reg(insn, MIPS_R_A1, ctx);
|
||||
emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));
|
||||
|
||||
if (insn->imm < 0) {
|
||||
emit_const_to_reg(ctx, MIPS_R_T9, (u64)bpf_internal_load_pointer_neg_helper);
|
||||
} else {
|
||||
emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);
|
||||
emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);
|
||||
}
|
||||
goto ld_skb_common;
|
||||
|
||||
case BPF_LD | BPF_B | BPF_IND:
|
||||
case BPF_LD | BPF_H | BPF_IND:
|
||||
case BPF_LD | BPF_W | BPF_IND:
|
||||
case BPF_LD | BPF_DW | BPF_IND:
|
||||
ctx->flags |= EBPF_SAVE_RA;
|
||||
src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
|
||||
if (src < 0)
|
||||
return src;
|
||||
ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
|
||||
if (ts == REG_32BIT_ZERO_EX) {
|
||||
/* sign extend */
|
||||
emit_instr(ctx, sll, MIPS_R_A1, src, 0);
|
||||
src = MIPS_R_A1;
|
||||
}
|
||||
if (insn->imm >= S16_MIN && insn->imm <= S16_MAX) {
|
||||
emit_instr(ctx, daddiu, MIPS_R_A1, src, insn->imm);
|
||||
} else {
|
||||
gen_imm_to_reg(insn, MIPS_R_AT, ctx);
|
||||
emit_instr(ctx, daddu, MIPS_R_A1, MIPS_R_AT, src);
|
||||
}
|
||||
/* truncate to 32-bit int */
|
||||
emit_instr(ctx, sll, MIPS_R_A1, MIPS_R_A1, 0);
|
||||
emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);
|
||||
emit_instr(ctx, slt, MIPS_R_AT, MIPS_R_A1, MIPS_R_ZERO);
|
||||
|
||||
emit_const_to_reg(ctx, MIPS_R_T8, (u64)bpf_internal_load_pointer_neg_helper);
|
||||
emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);
|
||||
emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));
|
||||
emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_T8, MIPS_R_AT);
|
||||
|
||||
ld_skb_common:
|
||||
emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
|
||||
/* delay slot move */
|
||||
emit_instr(ctx, daddu, MIPS_R_A0, MIPS_R_S0, MIPS_R_ZERO);
|
||||
|
||||
/* Check the error value */
|
||||
b_off = b_imm(exit_idx, ctx);
|
||||
if (is_bad_offset(b_off)) {
|
||||
target = j_target(ctx, exit_idx);
|
||||
if (target == (unsigned int)-1)
|
||||
return -E2BIG;
|
||||
|
||||
if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
|
||||
ctx->offsets[this_idx] |= OFFSETS_B_CONV;
|
||||
ctx->long_b_conversion = 1;
|
||||
}
|
||||
emit_instr(ctx, bne, MIPS_R_V0, MIPS_R_ZERO, 4 * 3);
|
||||
emit_instr(ctx, nop);
|
||||
emit_instr(ctx, j, target);
|
||||
emit_instr(ctx, nop);
|
||||
} else {
|
||||
emit_instr(ctx, beq, MIPS_R_V0, MIPS_R_ZERO, b_off);
|
||||
emit_instr(ctx, nop);
|
||||
}
|
||||
|
||||
#ifdef __BIG_ENDIAN
|
||||
need_swap = false;
|
||||
#else
|
||||
need_swap = true;
|
||||
#endif
|
||||
dst = MIPS_R_V0;
|
||||
switch (BPF_SIZE(insn->code)) {
|
||||
case BPF_B:
|
||||
emit_instr(ctx, lbu, dst, 0, MIPS_R_V0);
|
||||
break;
|
||||
case BPF_H:
|
||||
emit_instr(ctx, lhu, dst, 0, MIPS_R_V0);
|
||||
if (need_swap)
|
||||
emit_instr(ctx, wsbh, dst, dst);
|
||||
break;
|
||||
case BPF_W:
|
||||
emit_instr(ctx, lw, dst, 0, MIPS_R_V0);
|
||||
if (need_swap) {
|
||||
emit_instr(ctx, wsbh, dst, dst);
|
||||
emit_instr(ctx, rotr, dst, dst, 16);
|
||||
}
|
||||
break;
|
||||
case BPF_DW:
|
||||
emit_instr(ctx, ld, dst, 0, MIPS_R_V0);
|
||||
if (need_swap) {
|
||||
emit_instr(ctx, dsbh, dst, dst);
|
||||
emit_instr(ctx, dshd, dst, dst);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
case BPF_ALU | BPF_END | BPF_FROM_BE:
|
||||
case BPF_ALU | BPF_END | BPF_FROM_LE:
|
||||
dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
# Arch-specific network modules
|
||||
#
|
||||
ifeq ($(CONFIG_PPC64),y)
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm64.o bpf_jit_comp64.o
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o
|
||||
else
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o
|
||||
endif
|
||||
|
|
|
@ -20,7 +20,7 @@
|
|||
* with our redzone usage.
|
||||
*
|
||||
* [ prev sp ] <-------------
|
||||
* [ nv gpr save area ] 8*8 |
|
||||
* [ nv gpr save area ] 6*8 |
|
||||
* [ tail_call_cnt ] 8 |
|
||||
* [ local_tmp_var ] 8 |
|
||||
* fp (r31) --> [ ebpf stack space ] upto 512 |
|
||||
|
@ -28,8 +28,8 @@
|
|||
* sp (r1) ---> [ stack pointer ] --------------
|
||||
*/
|
||||
|
||||
/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */
|
||||
#define BPF_PPC_STACK_SAVE (8*8)
|
||||
/* for gpr non volatile registers BPG_REG_6 to 10 */
|
||||
#define BPF_PPC_STACK_SAVE (6*8)
|
||||
/* for bpf JIT code internal usage */
|
||||
#define BPF_PPC_STACK_LOCALS 16
|
||||
/* stack frame excluding BPF stack, ensure this is quadword aligned */
|
||||
|
@ -39,10 +39,8 @@
|
|||
#ifndef __ASSEMBLY__
|
||||
|
||||
/* BPF register usage */
|
||||
#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 0)
|
||||
#define SKB_DATA_REG (MAX_BPF_JIT_REG + 1)
|
||||
#define TMP_REG_1 (MAX_BPF_JIT_REG + 2)
|
||||
#define TMP_REG_2 (MAX_BPF_JIT_REG + 3)
|
||||
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
|
||||
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
|
||||
|
||||
/* BPF to ppc register mappings */
|
||||
static const int b2p[] = {
|
||||
|
@ -63,40 +61,23 @@ static const int b2p[] = {
|
|||
[BPF_REG_FP] = 31,
|
||||
/* eBPF jit internal registers */
|
||||
[BPF_REG_AX] = 2,
|
||||
[SKB_HLEN_REG] = 25,
|
||||
[SKB_DATA_REG] = 26,
|
||||
[TMP_REG_1] = 9,
|
||||
[TMP_REG_2] = 10
|
||||
};
|
||||
|
||||
/* PPC NVR range -- update this if we ever use NVRs below r24 */
|
||||
#define BPF_PPC_NVR_MIN 24
|
||||
|
||||
/* Assembly helpers */
|
||||
#define DECLARE_LOAD_FUNC(func) u64 func(u64 r3, u64 r4); \
|
||||
u64 func##_negative_offset(u64 r3, u64 r4); \
|
||||
u64 func##_positive_offset(u64 r3, u64 r4);
|
||||
|
||||
DECLARE_LOAD_FUNC(sk_load_word);
|
||||
DECLARE_LOAD_FUNC(sk_load_half);
|
||||
DECLARE_LOAD_FUNC(sk_load_byte);
|
||||
|
||||
#define CHOOSE_LOAD_FUNC(imm, func) \
|
||||
(imm < 0 ? \
|
||||
(imm >= SKF_LL_OFF ? func##_negative_offset : func) : \
|
||||
func##_positive_offset)
|
||||
/* PPC NVR range -- update this if we ever use NVRs below r27 */
|
||||
#define BPF_PPC_NVR_MIN 27
|
||||
|
||||
#define SEEN_FUNC 0x1000 /* might call external helpers */
|
||||
#define SEEN_STACK 0x2000 /* uses BPF stack */
|
||||
#define SEEN_SKB 0x4000 /* uses sk_buff */
|
||||
#define SEEN_TAILCALL 0x8000 /* uses tail calls */
|
||||
#define SEEN_TAILCALL 0x4000 /* uses tail calls */
|
||||
|
||||
struct codegen_context {
|
||||
/*
|
||||
* This is used to track register usage as well
|
||||
* as calls to external helpers.
|
||||
* - register usage is tracked with corresponding
|
||||
* bits (r3-r10 and r25-r31)
|
||||
* bits (r3-r10 and r27-r31)
|
||||
* - rest of the bits can be used to track other
|
||||
* things -- for now, we use bits 16 to 23
|
||||
* encoded in SEEN_* macros above
|
||||
|
|
|
@ -1,180 +0,0 @@
|
|||
/*
|
||||
* bpf_jit_asm64.S: Packet/header access helper functions
|
||||
* for PPC64 BPF compiler.
|
||||
*
|
||||
* Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
|
||||
* IBM Corporation
|
||||
*
|
||||
* Based on bpf_jit_asm.S by Matt Evans
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; version 2
|
||||
* of the License.
|
||||
*/
|
||||
|
||||
#include <asm/ppc_asm.h>
|
||||
#include <asm/ptrace.h>
|
||||
#include "bpf_jit64.h"
|
||||
|
||||
/*
|
||||
* All of these routines are called directly from generated code,
|
||||
* with the below register usage:
|
||||
* r27 skb pointer (ctx)
|
||||
* r25 skb header length
|
||||
* r26 skb->data pointer
|
||||
* r4 offset
|
||||
*
|
||||
* Result is passed back in:
|
||||
* r8 data read in host endian format (accumulator)
|
||||
*
|
||||
* r9 is used as a temporary register
|
||||
*/
|
||||
|
||||
#define r_skb r27
|
||||
#define r_hlen r25
|
||||
#define r_data r26
|
||||
#define r_off r4
|
||||
#define r_val r8
|
||||
#define r_tmp r9
|
||||
|
||||
_GLOBAL_TOC(sk_load_word)
|
||||
cmpdi r_off, 0
|
||||
blt bpf_slow_path_word_neg
|
||||
b sk_load_word_positive_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_word_positive_offset)
|
||||
/* Are we accessing past headlen? */
|
||||
subi r_tmp, r_hlen, 4
|
||||
cmpd r_tmp, r_off
|
||||
blt bpf_slow_path_word
|
||||
/* Nope, just hitting the header. cr0 here is eq or gt! */
|
||||
LWZX_BE r_val, r_data, r_off
|
||||
blr /* Return success, cr0 != LT */
|
||||
|
||||
_GLOBAL_TOC(sk_load_half)
|
||||
cmpdi r_off, 0
|
||||
blt bpf_slow_path_half_neg
|
||||
b sk_load_half_positive_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_half_positive_offset)
|
||||
subi r_tmp, r_hlen, 2
|
||||
cmpd r_tmp, r_off
|
||||
blt bpf_slow_path_half
|
||||
LHZX_BE r_val, r_data, r_off
|
||||
blr
|
||||
|
||||
_GLOBAL_TOC(sk_load_byte)
|
||||
cmpdi r_off, 0
|
||||
blt bpf_slow_path_byte_neg
|
||||
b sk_load_byte_positive_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_byte_positive_offset)
|
||||
cmpd r_hlen, r_off
|
||||
ble bpf_slow_path_byte
|
||||
lbzx r_val, r_data, r_off
|
||||
blr
|
||||
|
||||
/*
|
||||
* Call out to skb_copy_bits:
|
||||
* Allocate a new stack frame here to remain ABI-compliant in
|
||||
* stashing LR.
|
||||
*/
|
||||
#define bpf_slow_path_common(SIZE) \
|
||||
mflr r0; \
|
||||
std r0, PPC_LR_STKOFF(r1); \
|
||||
stdu r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1); \
|
||||
mr r3, r_skb; \
|
||||
/* r4 = r_off as passed */ \
|
||||
addi r5, r1, STACK_FRAME_MIN_SIZE; \
|
||||
li r6, SIZE; \
|
||||
bl skb_copy_bits; \
|
||||
nop; \
|
||||
/* save r5 */ \
|
||||
addi r5, r1, STACK_FRAME_MIN_SIZE; \
|
||||
/* r3 = 0 on success */ \
|
||||
addi r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS; \
|
||||
ld r0, PPC_LR_STKOFF(r1); \
|
||||
mtlr r0; \
|
||||
cmpdi r3, 0; \
|
||||
blt bpf_error; /* cr0 = LT */
|
||||
|
||||
bpf_slow_path_word:
|
||||
bpf_slow_path_common(4)
|
||||
/* Data value is on stack, and cr0 != LT */
|
||||
LWZX_BE r_val, 0, r5
|
||||
blr
|
||||
|
||||
bpf_slow_path_half:
|
||||
bpf_slow_path_common(2)
|
||||
LHZX_BE r_val, 0, r5
|
||||
blr
|
||||
|
||||
bpf_slow_path_byte:
|
||||
bpf_slow_path_common(1)
|
||||
lbzx r_val, 0, r5
|
||||
blr
|
||||
|
||||
/*
|
||||
* Call out to bpf_internal_load_pointer_neg_helper
|
||||
*/
|
||||
#define sk_negative_common(SIZE) \
|
||||
mflr r0; \
|
||||
std r0, PPC_LR_STKOFF(r1); \
|
||||
stdu r1, -STACK_FRAME_MIN_SIZE(r1); \
|
||||
mr r3, r_skb; \
|
||||
/* r4 = r_off, as passed */ \
|
||||
li r5, SIZE; \
|
||||
bl bpf_internal_load_pointer_neg_helper; \
|
||||
nop; \
|
||||
addi r1, r1, STACK_FRAME_MIN_SIZE; \
|
||||
ld r0, PPC_LR_STKOFF(r1); \
|
||||
mtlr r0; \
|
||||
/* R3 != 0 on success */ \
|
||||
cmpldi r3, 0; \
|
||||
beq bpf_error_slow; /* cr0 = EQ */
|
||||
|
||||
bpf_slow_path_word_neg:
|
||||
lis r_tmp, -32 /* SKF_LL_OFF */
|
||||
cmpd r_off, r_tmp /* addr < SKF_* */
|
||||
blt bpf_error /* cr0 = LT */
|
||||
b sk_load_word_negative_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_word_negative_offset)
|
||||
sk_negative_common(4)
|
||||
LWZX_BE r_val, 0, r3
|
||||
blr
|
||||
|
||||
bpf_slow_path_half_neg:
|
||||
lis r_tmp, -32 /* SKF_LL_OFF */
|
||||
cmpd r_off, r_tmp /* addr < SKF_* */
|
||||
blt bpf_error /* cr0 = LT */
|
||||
b sk_load_half_negative_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_half_negative_offset)
|
||||
sk_negative_common(2)
|
||||
LHZX_BE r_val, 0, r3
|
||||
blr
|
||||
|
||||
bpf_slow_path_byte_neg:
|
||||
lis r_tmp, -32 /* SKF_LL_OFF */
|
||||
cmpd r_off, r_tmp /* addr < SKF_* */
|
||||
blt bpf_error /* cr0 = LT */
|
||||
b sk_load_byte_negative_offset
|
||||
|
||||
_GLOBAL_TOC(sk_load_byte_negative_offset)
|
||||
sk_negative_common(1)
|
||||
lbzx r_val, 0, r3
|
||||
blr
|
||||
|
||||
bpf_error_slow:
|
||||
/* fabricate a cr0 = lt */
|
||||
li r_tmp, -1
|
||||
cmpdi r_tmp, 0
|
||||
bpf_error:
|
||||
/*
|
||||
* Entered with cr0 = lt
|
||||
* Generated code will 'blt epilogue', returning 0.
|
||||
*/
|
||||
li r_val, 0
|
||||
blr
|
|
@ -59,7 +59,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
|
|||
* [ prev sp ] <-------------
|
||||
* [ ... ] |
|
||||
* sp (r1) ---> [ stack pointer ] --------------
|
||||
* [ nv gpr save area ] 8*8
|
||||
* [ nv gpr save area ] 6*8
|
||||
* [ tail_call_cnt ] 8
|
||||
* [ local_tmp_var ] 8
|
||||
* [ unused red zone ] 208 bytes protected
|
||||
|
@ -88,21 +88,6 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
|
|||
BUG();
|
||||
}
|
||||
|
||||
static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx)
|
||||
{
|
||||
/*
|
||||
* Load skb->len and skb->data_len
|
||||
* r3 points to skb
|
||||
*/
|
||||
PPC_LWZ(b2p[SKB_HLEN_REG], 3, offsetof(struct sk_buff, len));
|
||||
PPC_LWZ(b2p[TMP_REG_1], 3, offsetof(struct sk_buff, data_len));
|
||||
/* header_len = len - data_len */
|
||||
PPC_SUB(b2p[SKB_HLEN_REG], b2p[SKB_HLEN_REG], b2p[TMP_REG_1]);
|
||||
|
||||
/* skb->data pointer */
|
||||
PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data));
|
||||
}
|
||||
|
||||
static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
|
||||
{
|
||||
int i;
|
||||
|
@ -145,18 +130,6 @@ static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
|
|||
if (bpf_is_seen_register(ctx, i))
|
||||
PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
|
||||
|
||||
/*
|
||||
* Save additional non-volatile regs if we cache skb
|
||||
* Also, setup skb data
|
||||
*/
|
||||
if (ctx->seen & SEEN_SKB) {
|
||||
PPC_BPF_STL(b2p[SKB_HLEN_REG], 1,
|
||||
bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
|
||||
PPC_BPF_STL(b2p[SKB_DATA_REG], 1,
|
||||
bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
|
||||
bpf_jit_emit_skb_loads(image, ctx);
|
||||
}
|
||||
|
||||
/* Setup frame pointer to point to the bpf stack area */
|
||||
if (bpf_is_seen_register(ctx, BPF_REG_FP))
|
||||
PPC_ADDI(b2p[BPF_REG_FP], 1,
|
||||
|
@ -172,14 +145,6 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx
|
|||
if (bpf_is_seen_register(ctx, i))
|
||||
PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
|
||||
|
||||
/* Restore non-volatile registers used for skb cache */
|
||||
if (ctx->seen & SEEN_SKB) {
|
||||
PPC_BPF_LL(b2p[SKB_HLEN_REG], 1,
|
||||
bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
|
||||
PPC_BPF_LL(b2p[SKB_DATA_REG], 1,
|
||||
bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
|
||||
}
|
||||
|
||||
/* Tear down our stack frame */
|
||||
if (bpf_has_stack_frame(ctx)) {
|
||||
PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
|
||||
|
@ -753,23 +718,10 @@ emit_clear:
|
|||
ctx->seen |= SEEN_FUNC;
|
||||
func = (u8 *) __bpf_call_base + imm;
|
||||
|
||||
/* Save skb pointer if we need to re-cache skb data */
|
||||
if ((ctx->seen & SEEN_SKB) &&
|
||||
bpf_helper_changes_pkt_data(func))
|
||||
PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));
|
||||
|
||||
bpf_jit_emit_func_call(image, ctx, (u64)func);
|
||||
|
||||
/* move return value from r3 to BPF_REG_0 */
|
||||
PPC_MR(b2p[BPF_REG_0], 3);
|
||||
|
||||
/* refresh skb cache */
|
||||
if ((ctx->seen & SEEN_SKB) &&
|
||||
bpf_helper_changes_pkt_data(func)) {
|
||||
/* reload skb pointer to r3 */
|
||||
PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));
|
||||
bpf_jit_emit_skb_loads(image, ctx);
|
||||
}
|
||||
break;
|
||||
|
||||
/*
|
||||
|
@ -886,65 +838,6 @@ cond_branch:
|
|||
PPC_BCC(true_cond, addrs[i + 1 + off]);
|
||||
break;
|
||||
|
||||
/*
|
||||
* Loads from packet header/data
|
||||
* Assume 32-bit input value in imm and X (src_reg)
|
||||
*/
|
||||
|
||||
/* Absolute loads */
|
||||
case BPF_LD | BPF_W | BPF_ABS:
|
||||
func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_word);
|
||||
goto common_load_abs;
|
||||
case BPF_LD | BPF_H | BPF_ABS:
|
||||
func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_half);
|
||||
goto common_load_abs;
|
||||
case BPF_LD | BPF_B | BPF_ABS:
|
||||
func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_byte);
|
||||
common_load_abs:
|
||||
/*
|
||||
* Load from [imm]
|
||||
* Load into r4, which can just be passed onto
|
||||
* skb load helpers as the second parameter
|
||||
*/
|
||||
PPC_LI32(4, imm);
|
||||
goto common_load;
|
||||
|
||||
/* Indirect loads */
|
||||
case BPF_LD | BPF_W | BPF_IND:
|
||||
func = (u8 *)sk_load_word;
|
||||
goto common_load_ind;
|
||||
case BPF_LD | BPF_H | BPF_IND:
|
||||
func = (u8 *)sk_load_half;
|
||||
goto common_load_ind;
|
||||
case BPF_LD | BPF_B | BPF_IND:
|
||||
func = (u8 *)sk_load_byte;
|
||||
common_load_ind:
|
||||
/*
|
||||
* Load from [src_reg + imm]
|
||||
* Treat src_reg as a 32-bit value
|
||||
*/
|
||||
PPC_EXTSW(4, src_reg);
|
||||
if (imm) {
|
||||
if (imm >= -32768 && imm < 32768)
|
||||
PPC_ADDI(4, 4, IMM_L(imm));
|
||||
else {
|
||||
PPC_LI32(b2p[TMP_REG_1], imm);
|
||||
PPC_ADD(4, 4, b2p[TMP_REG_1]);
|
||||
}
|
||||
}
|
||||
|
||||
common_load:
|
||||
ctx->seen |= SEEN_SKB;
|
||||
ctx->seen |= SEEN_FUNC;
|
||||
bpf_jit_emit_func_call(image, ctx, (u64)func);
|
||||
|
||||
/*
|
||||
* Helper returns 'lt' condition on error, and an
|
||||
* appropriate return value in BPF_REG_0
|
||||
*/
|
||||
PPC_BCC(COND_LT, exit_addr);
|
||||
break;
|
||||
|
||||
/*
|
||||
* Tail call
|
||||
*/
|
||||
|
|
|
@ -2,4 +2,4 @@
|
|||
#
|
||||
# Arch-specific network modules
|
||||
#
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
|
||||
|
|
|
@ -1,116 +0,0 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* BPF Jit compiler for s390, help functions.
|
||||
*
|
||||
* Copyright IBM Corp. 2012,2015
|
||||
*
|
||||
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
|
||||
* Michael Holzheu <holzheu@linux.vnet.ibm.com>
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include "bpf_jit.h"
|
||||
|
||||
/*
|
||||
* Calling convention:
|
||||
* registers %r7-%r10, %r11,%r13, and %r15 are call saved
|
||||
*
|
||||
* Input (64 bit):
|
||||
* %r3 (%b2) = offset into skb data
|
||||
* %r6 (%b5) = return address
|
||||
* %r7 (%b6) = skb pointer
|
||||
* %r12 = skb data pointer
|
||||
*
|
||||
* Output:
|
||||
* %r14= %b0 = return value (read skb value)
|
||||
*
|
||||
* Work registers: %r2,%r4,%r5,%r14
|
||||
*
|
||||
* skb_copy_bits takes 4 parameters:
|
||||
* %r2 = skb pointer
|
||||
* %r3 = offset into skb data
|
||||
* %r4 = pointer to temp buffer
|
||||
* %r5 = length to copy
|
||||
* Return value in %r2: 0 = ok
|
||||
*
|
||||
* bpf_internal_load_pointer_neg_helper takes 3 parameters:
|
||||
* %r2 = skb pointer
|
||||
* %r3 = offset into data
|
||||
* %r4 = length to copy
|
||||
* Return value in %r2: Pointer to data
|
||||
*/
|
||||
|
||||
#define SKF_MAX_NEG_OFF -0x200000 /* SKF_LL_OFF from filter.h */
|
||||
|
||||
/*
|
||||
* Load SIZE bytes from SKB
|
||||
*/
|
||||
#define sk_load_common(NAME, SIZE, LOAD) \
|
||||
ENTRY(sk_load_##NAME); \
|
||||
ltgr %r3,%r3; /* Is offset negative? */ \
|
||||
jl sk_load_##NAME##_slow_neg; \
|
||||
ENTRY(sk_load_##NAME##_pos); \
|
||||
aghi %r3,SIZE; /* Offset + SIZE */ \
|
||||
clg %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */ \
|
||||
jh sk_load_##NAME##_slow; \
|
||||
LOAD %r14,-SIZE(%r3,%r12); /* Get data from skb */ \
|
||||
b OFF_OK(%r6); /* Return */ \
|
||||
\
|
||||
sk_load_##NAME##_slow:; \
|
||||
lgr %r2,%r7; /* Arg1 = skb pointer */ \
|
||||
aghi %r3,-SIZE; /* Arg2 = offset */ \
|
||||
la %r4,STK_OFF_TMP(%r15); /* Arg3 = temp bufffer */ \
|
||||
lghi %r5,SIZE; /* Arg4 = size */ \
|
||||
brasl %r14,skb_copy_bits; /* Get data from skb */ \
|
||||
LOAD %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */ \
|
||||
ltgr %r2,%r2; /* Set cc to (%r2 != 0) */ \
|
||||
br %r6; /* Return */
|
||||
|
||||
sk_load_common(word, 4, llgf) /* r14 = *(u32 *) (skb->data+offset) */
|
||||
sk_load_common(half, 2, llgh) /* r14 = *(u16 *) (skb->data+offset) */
|
||||
|
||||
/*
|
||||
* Load 1 byte from SKB (optimized version)
|
||||
*/
|
||||
/* r14 = *(u8 *) (skb->data+offset) */
|
||||
ENTRY(sk_load_byte)
|
||||
ltgr %r3,%r3 # Is offset negative?
|
||||
jl sk_load_byte_slow_neg
|
||||
ENTRY(sk_load_byte_pos)
|
||||
clg %r3,STK_OFF_HLEN(%r15) # Offset >= hlen?
|
||||
jnl sk_load_byte_slow
|
||||
llgc %r14,0(%r3,%r12) # Get byte from skb
|
||||
b OFF_OK(%r6) # Return OK
|
||||
|
||||
sk_load_byte_slow:
|
||||
lgr %r2,%r7 # Arg1 = skb pointer
|
||||
# Arg2 = offset
|
||||
la %r4,STK_OFF_TMP(%r15) # Arg3 = pointer to temp buffer
|
||||
lghi %r5,1 # Arg4 = size (1 byte)
|
||||
brasl %r14,skb_copy_bits # Get data from skb
|
||||
llgc %r14,STK_OFF_TMP(%r15) # Load result from temp buffer
|
||||
ltgr %r2,%r2 # Set cc to (%r2 != 0)
|
||||
br %r6 # Return cc
|
||||
|
||||
#define sk_negative_common(NAME, SIZE, LOAD) \
|
||||
sk_load_##NAME##_slow_neg:; \
|
||||
cgfi %r3,SKF_MAX_NEG_OFF; \
|
||||
jl bpf_error; \
|
||||
lgr %r2,%r7; /* Arg1 = skb pointer */ \
|
||||
/* Arg2 = offset */ \
|
||||
lghi %r4,SIZE; /* Arg3 = size */ \
|
||||
brasl %r14,bpf_internal_load_pointer_neg_helper; \
|
||||
ltgr %r2,%r2; \
|
||||
jz bpf_error; \
|
||||
LOAD %r14,0(%r2); /* Get data from pointer */ \
|
||||
xr %r3,%r3; /* Set cc to zero */ \
|
||||
br %r6; /* Return cc */
|
||||
|
||||
sk_negative_common(word, 4, llgf)
|
||||
sk_negative_common(half, 2, llgh)
|
||||
sk_negative_common(byte, 1, llgc)
|
||||
|
||||
bpf_error:
|
||||
# force a return 0 from jit handler
|
||||
ltgr %r15,%r15 # Set condition code
|
||||
br %r6
|
|
@ -16,9 +16,6 @@
|
|||
#include <linux/filter.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[];
|
||||
extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
/*
|
||||
|
@ -36,15 +33,6 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
|
|||
* | | |
|
||||
* | BPF stack | |
|
||||
* | | |
|
||||
* +---------------+ |
|
||||
* | 8 byte skbp | |
|
||||
* R15+176 -> +---------------+ |
|
||||
* | 8 byte hlen | |
|
||||
* R15+168 -> +---------------+ |
|
||||
* | 4 byte align | |
|
||||
* +---------------+ |
|
||||
* | 4 byte temp | |
|
||||
* | for bpf_jit.S | |
|
||||
* R15+160 -> +---------------+ |
|
||||
* | new backchain | |
|
||||
* R15+152 -> +---------------+ |
|
||||
|
@ -57,17 +45,11 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
|
|||
* The stack size used by the BPF program ("BPF stack" above) is passed
|
||||
* via "aux->stack_depth".
|
||||
*/
|
||||
#define STK_SPACE_ADD (8 + 8 + 4 + 4 + 160)
|
||||
#define STK_SPACE_ADD (160)
|
||||
#define STK_160_UNUSED (160 - 12 * 8)
|
||||
#define STK_OFF (STK_SPACE_ADD - STK_160_UNUSED)
|
||||
#define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */
|
||||
#define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */
|
||||
#define STK_OFF_SKBP 176 /* Offset of SKB pointer on stack */
|
||||
|
||||
#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */
|
||||
#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */
|
||||
|
||||
/* Offset to skip condition code check */
|
||||
#define OFF_OK 4
|
||||
|
||||
#endif /* __ARCH_S390_NET_BPF_JIT_H */
|
||||
|
|
|
@ -47,23 +47,21 @@ struct bpf_jit {
|
|||
|
||||
#define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */
|
||||
|
||||
#define SEEN_SKB 1 /* skb access */
|
||||
#define SEEN_MEM 2 /* use mem[] for temporary storage */
|
||||
#define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */
|
||||
#define SEEN_LITERAL 8 /* code uses literals */
|
||||
#define SEEN_FUNC 16 /* calls C functions */
|
||||
#define SEEN_TAIL_CALL 32 /* code uses tail calls */
|
||||
#define SEEN_REG_AX 64 /* code uses constant blinding */
|
||||
#define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB)
|
||||
#define SEEN_MEM (1 << 0) /* use mem[] for temporary storage */
|
||||
#define SEEN_RET0 (1 << 1) /* ret0_ip points to a valid return 0 */
|
||||
#define SEEN_LITERAL (1 << 2) /* code uses literals */
|
||||
#define SEEN_FUNC (1 << 3) /* calls C functions */
|
||||
#define SEEN_TAIL_CALL (1 << 4) /* code uses tail calls */
|
||||
#define SEEN_REG_AX (1 << 5) /* code uses constant blinding */
|
||||
#define SEEN_STACK (SEEN_FUNC | SEEN_MEM)
|
||||
|
||||
/*
|
||||
* s390 registers
|
||||
*/
|
||||
#define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */
|
||||
#define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */
|
||||
#define REG_SKB_DATA (MAX_BPF_JIT_REG + 2) /* SKB data register */
|
||||
#define REG_L (MAX_BPF_JIT_REG + 3) /* Literal pool register */
|
||||
#define REG_15 (MAX_BPF_JIT_REG + 4) /* Register 15 */
|
||||
#define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */
|
||||
#define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */
|
||||
#define REG_0 REG_W0 /* Register 0 */
|
||||
#define REG_1 REG_W1 /* Register 1 */
|
||||
#define REG_2 BPF_REG_1 /* Register 2 */
|
||||
|
@ -88,10 +86,8 @@ static const int reg2hex[] = {
|
|||
[BPF_REG_9] = 10,
|
||||
/* BPF stack pointer */
|
||||
[BPF_REG_FP] = 13,
|
||||
/* Register for blinding (shared with REG_SKB_DATA) */
|
||||
/* Register for blinding */
|
||||
[BPF_REG_AX] = 12,
|
||||
/* SKB data pointer */
|
||||
[REG_SKB_DATA] = 12,
|
||||
/* Work registers for s390x backend */
|
||||
[REG_W0] = 0,
|
||||
[REG_W1] = 1,
|
||||
|
@ -384,27 +380,6 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
|
|||
} while (re <= 15);
|
||||
}
|
||||
|
||||
/*
|
||||
* For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
|
||||
* we store the SKB header length on the stack and the SKB data
|
||||
* pointer in REG_SKB_DATA if BPF_REG_AX is not used.
|
||||
*/
|
||||
static void emit_load_skb_data_hlen(struct bpf_jit *jit)
|
||||
{
|
||||
/* Header length: llgf %w1,<len>(%b1) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
|
||||
offsetof(struct sk_buff, len));
|
||||
/* s %w1,<data_len>(%b1) */
|
||||
EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
|
||||
offsetof(struct sk_buff, data_len));
|
||||
/* stg %w1,ST_OFF_HLEN(%r0,%r15) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
|
||||
if (!(jit->seen & SEEN_REG_AX))
|
||||
/* lg %skb_data,data_off(%b1) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
|
||||
BPF_REG_1, offsetof(struct sk_buff, data));
|
||||
}
|
||||
|
||||
/*
|
||||
* Emit function prologue
|
||||
*
|
||||
|
@ -445,12 +420,6 @@ static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)
|
|||
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
|
||||
REG_15, 152);
|
||||
}
|
||||
if (jit->seen & SEEN_SKB) {
|
||||
emit_load_skb_data_hlen(jit);
|
||||
/* stg %b1,ST_OFF_SKBP(%r0,%r15) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_1, REG_0, REG_15,
|
||||
STK_OFF_SKBP);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -483,12 +452,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
|
|||
{
|
||||
struct bpf_insn *insn = &fp->insnsi[i];
|
||||
int jmp_off, last, insn_count = 1;
|
||||
unsigned int func_addr, mask;
|
||||
u32 dst_reg = insn->dst_reg;
|
||||
u32 src_reg = insn->src_reg;
|
||||
u32 *addrs = jit->addrs;
|
||||
s32 imm = insn->imm;
|
||||
s16 off = insn->off;
|
||||
unsigned int mask;
|
||||
|
||||
if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX)
|
||||
jit->seen |= SEEN_REG_AX;
|
||||
|
@ -970,13 +939,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
|
|||
EMIT2(0x0d00, REG_14, REG_W1);
|
||||
/* lgr %b0,%r2: load return value into %b0 */
|
||||
EMIT4(0xb9040000, BPF_REG_0, REG_2);
|
||||
if ((jit->seen & SEEN_SKB) &&
|
||||
bpf_helper_changes_pkt_data((void *)func)) {
|
||||
/* lg %b1,ST_OFF_SKBP(%r15) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,
|
||||
REG_15, STK_OFF_SKBP);
|
||||
emit_load_skb_data_hlen(jit);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case BPF_JMP | BPF_TAIL_CALL:
|
||||
|
@ -1176,73 +1138,6 @@ branch_oc:
|
|||
jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
|
||||
EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);
|
||||
break;
|
||||
/*
|
||||
* BPF_LD
|
||||
*/
|
||||
case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */
|
||||
case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */
|
||||
if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
|
||||
func_addr = __pa(sk_load_byte_pos);
|
||||
else
|
||||
func_addr = __pa(sk_load_byte);
|
||||
goto call_fn;
|
||||
case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */
|
||||
case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */
|
||||
if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
|
||||
func_addr = __pa(sk_load_half_pos);
|
||||
else
|
||||
func_addr = __pa(sk_load_half);
|
||||
goto call_fn;
|
||||
case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */
|
||||
case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */
|
||||
if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
|
||||
func_addr = __pa(sk_load_word_pos);
|
||||
else
|
||||
func_addr = __pa(sk_load_word);
|
||||
goto call_fn;
|
||||
call_fn:
|
||||
jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;
|
||||
REG_SET_SEEN(REG_14); /* Return address of possible func call */
|
||||
|
||||
/*
|
||||
* Implicit input:
|
||||
* BPF_REG_6 (R7) : skb pointer
|
||||
* REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX)
|
||||
*
|
||||
* Calculated input:
|
||||
* BPF_REG_2 (R3) : offset of byte(s) to fetch in skb
|
||||
* BPF_REG_5 (R6) : return address
|
||||
*
|
||||
* Output:
|
||||
* BPF_REG_0 (R14): data read from skb
|
||||
*
|
||||
* Scratch registers (BPF_REG_1-5)
|
||||
*/
|
||||
|
||||
/* Call function: llilf %w1,func_addr */
|
||||
EMIT6_IMM(0xc00f0000, REG_W1, func_addr);
|
||||
|
||||
/* Offset: lgfi %b2,imm */
|
||||
EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
|
||||
if (BPF_MODE(insn->code) == BPF_IND)
|
||||
/* agfr %b2,%src (%src is s32 here) */
|
||||
EMIT4(0xb9180000, BPF_REG_2, src_reg);
|
||||
|
||||
/* Reload REG_SKB_DATA if BPF_REG_AX is used */
|
||||
if (jit->seen & SEEN_REG_AX)
|
||||
/* lg %skb_data,data_off(%b6) */
|
||||
EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
|
||||
BPF_REG_6, offsetof(struct sk_buff, data));
|
||||
/* basr %b5,%w1 (%b5 is call saved) */
|
||||
EMIT2(0x0d00, BPF_REG_5, REG_W1);
|
||||
|
||||
/*
|
||||
* Note: For fast access we jump directly after the
|
||||
* jnz instruction from bpf_jit.S
|
||||
*/
|
||||
/* jnz <ret0> */
|
||||
EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
|
||||
break;
|
||||
default: /* too complex, give up */
|
||||
pr_err("Unknown opcode %02x\n", insn->code);
|
||||
return -1;
|
||||
|
|
|
@ -1,4 +1,7 @@
|
|||
#
|
||||
# Arch-specific network modules
|
||||
#
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_$(BITS).o bpf_jit_comp_$(BITS).o
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp_$(BITS).o
|
||||
ifeq ($(BITS),32)
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_32.o
|
||||
endif
|
||||
|
|
|
@ -33,35 +33,6 @@
|
|||
#define I5 0x1d
|
||||
#define FP 0x1e
|
||||
#define I7 0x1f
|
||||
|
||||
#define r_SKB L0
|
||||
#define r_HEADLEN L4
|
||||
#define r_SKB_DATA L5
|
||||
#define r_TMP G1
|
||||
#define r_TMP2 G3
|
||||
|
||||
/* assembly code in arch/sparc/net/bpf_jit_asm_64.S */
|
||||
extern u32 bpf_jit_load_word[];
|
||||
extern u32 bpf_jit_load_half[];
|
||||
extern u32 bpf_jit_load_byte[];
|
||||
extern u32 bpf_jit_load_byte_msh[];
|
||||
extern u32 bpf_jit_load_word_positive_offset[];
|
||||
extern u32 bpf_jit_load_half_positive_offset[];
|
||||
extern u32 bpf_jit_load_byte_positive_offset[];
|
||||
extern u32 bpf_jit_load_byte_msh_positive_offset[];
|
||||
extern u32 bpf_jit_load_word_negative_offset[];
|
||||
extern u32 bpf_jit_load_half_negative_offset[];
|
||||
extern u32 bpf_jit_load_byte_negative_offset[];
|
||||
extern u32 bpf_jit_load_byte_msh_negative_offset[];
|
||||
|
||||
#else
|
||||
#define r_RESULT %o0
|
||||
#define r_SKB %o0
|
||||
#define r_OFF %o1
|
||||
#define r_HEADLEN %l4
|
||||
#define r_SKB_DATA %l5
|
||||
#define r_TMP %g1
|
||||
#define r_TMP2 %g3
|
||||
#endif
|
||||
|
||||
#endif /* _BPF_JIT_H */
|
||||
|
|
|
@ -1,162 +0,0 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
#include "bpf_jit_64.h"
|
||||
|
||||
#define SAVE_SZ 176
|
||||
#define SCRATCH_OFF STACK_BIAS + 128
|
||||
#define BE_PTR(label) be,pn %xcc, label
|
||||
#define SIGN_EXTEND(reg) sra reg, 0, reg
|
||||
|
||||
#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
|
||||
|
||||
.text
|
||||
.globl bpf_jit_load_word
|
||||
bpf_jit_load_word:
|
||||
cmp r_OFF, 0
|
||||
bl bpf_slow_path_word_neg
|
||||
nop
|
||||
.globl bpf_jit_load_word_positive_offset
|
||||
bpf_jit_load_word_positive_offset:
|
||||
sub r_HEADLEN, r_OFF, r_TMP
|
||||
cmp r_TMP, 3
|
||||
ble bpf_slow_path_word
|
||||
add r_SKB_DATA, r_OFF, r_TMP
|
||||
andcc r_TMP, 3, %g0
|
||||
bne load_word_unaligned
|
||||
nop
|
||||
retl
|
||||
ld [r_TMP], r_RESULT
|
||||
load_word_unaligned:
|
||||
ldub [r_TMP + 0x0], r_OFF
|
||||
ldub [r_TMP + 0x1], r_TMP2
|
||||
sll r_OFF, 8, r_OFF
|
||||
or r_OFF, r_TMP2, r_OFF
|
||||
ldub [r_TMP + 0x2], r_TMP2
|
||||
sll r_OFF, 8, r_OFF
|
||||
or r_OFF, r_TMP2, r_OFF
|
||||
ldub [r_TMP + 0x3], r_TMP2
|
||||
sll r_OFF, 8, r_OFF
|
||||
retl
|
||||
or r_OFF, r_TMP2, r_RESULT
|
||||
|
||||
.globl bpf_jit_load_half
|
||||
bpf_jit_load_half:
|
||||
cmp r_OFF, 0
|
||||
bl bpf_slow_path_half_neg
|
||||
nop
|
||||
.globl bpf_jit_load_half_positive_offset
|
||||
bpf_jit_load_half_positive_offset:
|
||||
sub r_HEADLEN, r_OFF, r_TMP
|
||||
cmp r_TMP, 1
|
||||
ble bpf_slow_path_half
|
||||
add r_SKB_DATA, r_OFF, r_TMP
|
||||
andcc r_TMP, 1, %g0
|
||||
bne load_half_unaligned
|
||||
nop
|
||||
retl
|
||||
lduh [r_TMP], r_RESULT
|
||||
load_half_unaligned:
|
||||
ldub [r_TMP + 0x0], r_OFF
|
||||
ldub [r_TMP + 0x1], r_TMP2
|
||||
sll r_OFF, 8, r_OFF
|
||||
retl
|
||||
or r_OFF, r_TMP2, r_RESULT
|
||||
|
||||
.globl bpf_jit_load_byte
|
||||
bpf_jit_load_byte:
|
||||
cmp r_OFF, 0
|
||||
bl bpf_slow_path_byte_neg
|
||||
nop
|
||||
.globl bpf_jit_load_byte_positive_offset
|
||||
bpf_jit_load_byte_positive_offset:
|
||||
cmp r_OFF, r_HEADLEN
|
||||
bge bpf_slow_path_byte
|
||||
nop
|
||||
retl
|
||||
ldub [r_SKB_DATA + r_OFF], r_RESULT
|
||||
|
||||
#define bpf_slow_path_common(LEN) \
|
||||
save %sp, -SAVE_SZ, %sp; \
|
||||
mov %i0, %o0; \
|
||||
mov %i1, %o1; \
|
||||
add %fp, SCRATCH_OFF, %o2; \
|
||||
call skb_copy_bits; \
|
||||
mov (LEN), %o3; \
|
||||
cmp %o0, 0; \
|
||||
restore;
|
||||
|
||||
bpf_slow_path_word:
|
||||
bpf_slow_path_common(4)
|
||||
bl bpf_error
|
||||
ld [%sp + SCRATCH_OFF], r_RESULT
|
||||
retl
|
||||
nop
|
||||
bpf_slow_path_half:
|
||||
bpf_slow_path_common(2)
|
||||
bl bpf_error
|
||||
lduh [%sp + SCRATCH_OFF], r_RESULT
|
||||
retl
|
||||
nop
|
||||
bpf_slow_path_byte:
|
||||
bpf_slow_path_common(1)
|
||||
bl bpf_error
|
||||
ldub [%sp + SCRATCH_OFF], r_RESULT
|
||||
retl
|
||||
nop
|
||||
|
||||
#define bpf_negative_common(LEN) \
|
||||
save %sp, -SAVE_SZ, %sp; \
|
||||
mov %i0, %o0; \
|
||||
mov %i1, %o1; \
|
||||
SIGN_EXTEND(%o1); \
|
||||
call bpf_internal_load_pointer_neg_helper; \
|
||||
mov (LEN), %o2; \
|
||||
mov %o0, r_TMP; \
|
||||
cmp %o0, 0; \
|
||||
BE_PTR(bpf_error); \
|
||||
restore;
|
||||
|
||||
bpf_slow_path_word_neg:
|
||||
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||||
cmp r_OFF, r_TMP
|
||||
bl bpf_error
|
||||
nop
|
||||
.globl bpf_jit_load_word_negative_offset
|
||||
bpf_jit_load_word_negative_offset:
|
||||
bpf_negative_common(4)
|
||||
andcc r_TMP, 3, %g0
|
||||
bne load_word_unaligned
|
||||
nop
|
||||
retl
|
||||
ld [r_TMP], r_RESULT
|
||||
|
||||
bpf_slow_path_half_neg:
|
||||
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||||
cmp r_OFF, r_TMP
|
||||
bl bpf_error
|
||||
nop
|
||||
.globl bpf_jit_load_half_negative_offset
|
||||
bpf_jit_load_half_negative_offset:
|
||||
bpf_negative_common(2)
|
||||
andcc r_TMP, 1, %g0
|
||||
bne load_half_unaligned
|
||||
nop
|
||||
retl
|
||||
lduh [r_TMP], r_RESULT
|
||||
|
||||
bpf_slow_path_byte_neg:
|
||||
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||||
cmp r_OFF, r_TMP
|
||||
bl bpf_error
|
||||
nop
|
||||
.globl bpf_jit_load_byte_negative_offset
|
||||
bpf_jit_load_byte_negative_offset:
|
||||
bpf_negative_common(1)
|
||||
retl
|
||||
ldub [r_TMP], r_RESULT
|
||||
|
||||
bpf_error:
|
||||
/* Make the JIT program itself return zero. */
|
||||
ret
|
||||
restore %g0, %g0, %o0
|
|
@ -48,10 +48,6 @@ static void bpf_flush_icache(void *start_, void *end_)
|
|||
}
|
||||
}
|
||||
|
||||
#define SEEN_DATAREF 1 /* might call external helpers */
|
||||
#define SEEN_XREG 2 /* ebx is used */
|
||||
#define SEEN_MEM 4 /* use mem[] for temporary storage */
|
||||
|
||||
#define S13(X) ((X) & 0x1fff)
|
||||
#define S5(X) ((X) & 0x1f)
|
||||
#define IMMED 0x00002000
|
||||
|
@ -198,7 +194,6 @@ struct jit_ctx {
|
|||
bool tmp_1_used;
|
||||
bool tmp_2_used;
|
||||
bool tmp_3_used;
|
||||
bool saw_ld_abs_ind;
|
||||
bool saw_frame_pointer;
|
||||
bool saw_call;
|
||||
bool saw_tail_call;
|
||||
|
@ -207,9 +202,7 @@ struct jit_ctx {
|
|||
|
||||
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
|
||||
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
|
||||
#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2)
|
||||
#define SKB_DATA_REG (MAX_BPF_JIT_REG + 3)
|
||||
#define TMP_REG_3 (MAX_BPF_JIT_REG + 4)
|
||||
#define TMP_REG_3 (MAX_BPF_JIT_REG + 2)
|
||||
|
||||
/* Map BPF registers to SPARC registers */
|
||||
static const int bpf2sparc[] = {
|
||||
|
@ -238,9 +231,6 @@ static const int bpf2sparc[] = {
|
|||
[TMP_REG_1] = G1,
|
||||
[TMP_REG_2] = G2,
|
||||
[TMP_REG_3] = G3,
|
||||
|
||||
[SKB_HLEN_REG] = L4,
|
||||
[SKB_DATA_REG] = L5,
|
||||
};
|
||||
|
||||
static void emit(const u32 insn, struct jit_ctx *ctx)
|
||||
|
@ -800,25 +790,6 @@ static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
|
||||
{
|
||||
const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
|
||||
const u8 r_data = bpf2sparc[SKB_DATA_REG];
|
||||
const u8 r_tmp = bpf2sparc[TMP_REG_1];
|
||||
unsigned int off;
|
||||
|
||||
off = offsetof(struct sk_buff, len);
|
||||
emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
|
||||
|
||||
off = offsetof(struct sk_buff, data_len);
|
||||
emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
|
||||
|
||||
emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
|
||||
|
||||
off = offsetof(struct sk_buff, data);
|
||||
emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
|
||||
}
|
||||
|
||||
/* Just skip the save instruction and the ctx register move. */
|
||||
#define BPF_TAILCALL_PROLOGUE_SKIP 16
|
||||
#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
|
||||
|
@ -857,9 +828,6 @@ static void build_prologue(struct jit_ctx *ctx)
|
|||
|
||||
emit_reg_move(I0, O0, ctx);
|
||||
/* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
|
||||
|
||||
if (ctx->saw_ld_abs_ind)
|
||||
load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
|
||||
}
|
||||
|
||||
static void build_epilogue(struct jit_ctx *ctx)
|
||||
|
@ -1225,16 +1193,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
|
|||
u8 *func = ((u8 *)__bpf_call_base) + imm;
|
||||
|
||||
ctx->saw_call = true;
|
||||
if (ctx->saw_ld_abs_ind && bpf_helper_changes_pkt_data(func))
|
||||
emit_reg_move(bpf2sparc[BPF_REG_1], L7, ctx);
|
||||
|
||||
emit_call((u32 *)func, ctx);
|
||||
emit_nop(ctx);
|
||||
|
||||
emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
|
||||
|
||||
if (ctx->saw_ld_abs_ind && bpf_helper_changes_pkt_data(func))
|
||||
load_skb_regs(ctx, L7);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -1412,43 +1375,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
|
|||
emit_nop(ctx);
|
||||
break;
|
||||
}
|
||||
#define CHOOSE_LOAD_FUNC(K, func) \
|
||||
((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
|
||||
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
|
||||
goto common_load;
|
||||
/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
func = bpf_jit_load_word;
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
func = bpf_jit_load_half;
|
||||
goto common_load;
|
||||
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
func = bpf_jit_load_byte;
|
||||
common_load:
|
||||
ctx->saw_ld_abs_ind = true;
|
||||
|
||||
emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
|
||||
emit_loadimm(imm, O1, ctx);
|
||||
|
||||
if (BPF_MODE(code) == BPF_IND)
|
||||
emit_alu(ADD, src, O1, ctx);
|
||||
|
||||
emit_call(func, ctx);
|
||||
emit_alu_K(SRA, O1, 0, ctx);
|
||||
|
||||
emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
|
||||
break;
|
||||
|
||||
default:
|
||||
pr_err_once("unknown opcode %02x\n", code);
|
||||
|
@ -1583,12 +1509,11 @@ skip_init_ctx:
|
|||
build_epilogue(&ctx);
|
||||
|
||||
if (bpf_jit_enable > 1)
|
||||
pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
|
||||
pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c]\n", pass,
|
||||
image_size - (ctx.idx * 4),
|
||||
ctx.tmp_1_used ? '1' : ' ',
|
||||
ctx.tmp_2_used ? '2' : ' ',
|
||||
ctx.tmp_3_used ? '3' : ' ',
|
||||
ctx.saw_ld_abs_ind ? 'L' : ' ',
|
||||
ctx.saw_frame_pointer ? 'F' : ' ',
|
||||
ctx.saw_call ? 'C' : ' ',
|
||||
ctx.saw_tail_call ? 'T' : ' ');
|
||||
|
|
|
@ -5,6 +5,5 @@
|
|||
ifeq ($(CONFIG_X86_32),y)
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
|
||||
else
|
||||
OBJECT_FILES_NON_STANDARD_bpf_jit.o += y
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
|
||||
endif
|
||||
|
|
|
@ -1,154 +0,0 @@
|
|||
/* bpf_jit.S : BPF JIT helper functions
|
||||
*
|
||||
* Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; version 2
|
||||
* of the License.
|
||||
*/
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
/*
|
||||
* Calling convention :
|
||||
* rbx : skb pointer (callee saved)
|
||||
* esi : offset of byte(s) to fetch in skb (can be scratched)
|
||||
* r10 : copy of skb->data
|
||||
* r9d : hlen = skb->len - skb->data_len
|
||||
*/
|
||||
#define SKBDATA %r10
|
||||
#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
|
||||
|
||||
#define FUNC(name) \
|
||||
.globl name; \
|
||||
.type name, @function; \
|
||||
name:
|
||||
|
||||
FUNC(sk_load_word)
|
||||
test %esi,%esi
|
||||
js bpf_slow_path_word_neg
|
||||
|
||||
FUNC(sk_load_word_positive_offset)
|
||||
mov %r9d,%eax # hlen
|
||||
sub %esi,%eax # hlen - offset
|
||||
cmp $3,%eax
|
||||
jle bpf_slow_path_word
|
||||
mov (SKBDATA,%rsi),%eax
|
||||
bswap %eax /* ntohl() */
|
||||
ret
|
||||
|
||||
FUNC(sk_load_half)
|
||||
test %esi,%esi
|
||||
js bpf_slow_path_half_neg
|
||||
|
||||
FUNC(sk_load_half_positive_offset)
|
||||
mov %r9d,%eax
|
||||
sub %esi,%eax # hlen - offset
|
||||
cmp $1,%eax
|
||||
jle bpf_slow_path_half
|
||||
movzwl (SKBDATA,%rsi),%eax
|
||||
rol $8,%ax # ntohs()
|
||||
ret
|
||||
|
||||
FUNC(sk_load_byte)
|
||||
test %esi,%esi
|
||||
js bpf_slow_path_byte_neg
|
||||
|
||||
FUNC(sk_load_byte_positive_offset)
|
||||
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
|
||||
jle bpf_slow_path_byte
|
||||
movzbl (SKBDATA,%rsi),%eax
|
||||
ret
|
||||
|
||||
/* rsi contains offset and can be scratched */
|
||||
#define bpf_slow_path_common(LEN) \
|
||||
lea 32(%rbp), %rdx;\
|
||||
FRAME_BEGIN; \
|
||||
mov %rbx, %rdi; /* arg1 == skb */ \
|
||||
push %r9; \
|
||||
push SKBDATA; \
|
||||
/* rsi already has offset */ \
|
||||
mov $LEN,%ecx; /* len */ \
|
||||
call skb_copy_bits; \
|
||||
test %eax,%eax; \
|
||||
pop SKBDATA; \
|
||||
pop %r9; \
|
||||
FRAME_END
|
||||
|
||||
|
||||
bpf_slow_path_word:
|
||||
bpf_slow_path_common(4)
|
||||
js bpf_error
|
||||
mov 32(%rbp),%eax
|
||||
bswap %eax
|
||||
ret
|
||||
|
||||
bpf_slow_path_half:
|
||||
bpf_slow_path_common(2)
|
||||
js bpf_error
|
||||
mov 32(%rbp),%ax
|
||||
rol $8,%ax
|
||||
movzwl %ax,%eax
|
||||
ret
|
||||
|
||||
bpf_slow_path_byte:
|
||||
bpf_slow_path_common(1)
|
||||
js bpf_error
|
||||
movzbl 32(%rbp),%eax
|
||||
ret
|
||||
|
||||
#define sk_negative_common(SIZE) \
|
||||
FRAME_BEGIN; \
|
||||
mov %rbx, %rdi; /* arg1 == skb */ \
|
||||
push %r9; \
|
||||
push SKBDATA; \
|
||||
/* rsi already has offset */ \
|
||||
mov $SIZE,%edx; /* size */ \
|
||||
call bpf_internal_load_pointer_neg_helper; \
|
||||
test %rax,%rax; \
|
||||
pop SKBDATA; \
|
||||
pop %r9; \
|
||||
FRAME_END; \
|
||||
jz bpf_error
|
||||
|
||||
bpf_slow_path_word_neg:
|
||||
cmp SKF_MAX_NEG_OFF, %esi /* test range */
|
||||
jl bpf_error /* offset lower -> error */
|
||||
|
||||
FUNC(sk_load_word_negative_offset)
|
||||
sk_negative_common(4)
|
||||
mov (%rax), %eax
|
||||
bswap %eax
|
||||
ret
|
||||
|
||||
bpf_slow_path_half_neg:
|
||||
cmp SKF_MAX_NEG_OFF, %esi
|
||||
jl bpf_error
|
||||
|
||||
FUNC(sk_load_half_negative_offset)
|
||||
sk_negative_common(2)
|
||||
mov (%rax),%ax
|
||||
rol $8,%ax
|
||||
movzwl %ax,%eax
|
||||
ret
|
||||
|
||||
bpf_slow_path_byte_neg:
|
||||
cmp SKF_MAX_NEG_OFF, %esi
|
||||
jl bpf_error
|
||||
|
||||
FUNC(sk_load_byte_negative_offset)
|
||||
sk_negative_common(1)
|
||||
movzbl (%rax), %eax
|
||||
ret
|
||||
|
||||
bpf_error:
|
||||
# force a return 0 from jit handler
|
||||
xor %eax,%eax
|
||||
mov (%rbp),%rbx
|
||||
mov 8(%rbp),%r13
|
||||
mov 16(%rbp),%r14
|
||||
mov 24(%rbp),%r15
|
||||
add $40, %rbp
|
||||
leaveq
|
||||
ret
|
|
@ -17,15 +17,6 @@
|
|||
#include <asm/set_memory.h>
|
||||
#include <asm/nospec-branch.h>
|
||||
|
||||
/*
|
||||
* Assembly code in arch/x86/net/bpf_jit.S
|
||||
*/
|
||||
extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
|
||||
extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[];
|
||||
extern u8 sk_load_byte_positive_offset[];
|
||||
extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[];
|
||||
extern u8 sk_load_byte_negative_offset[];
|
||||
|
||||
static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
|
||||
{
|
||||
if (len == 1)
|
||||
|
@ -107,9 +98,6 @@ static int bpf_size_to_x86_bytes(int bpf_size)
|
|||
#define X86_JLE 0x7E
|
||||
#define X86_JG 0x7F
|
||||
|
||||
#define CHOOSE_LOAD_FUNC(K, func) \
|
||||
((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
|
||||
|
||||
/* Pick a register outside of BPF range for JIT internal work */
|
||||
#define AUX_REG (MAX_BPF_JIT_REG + 1)
|
||||
|
||||
|
@ -120,8 +108,8 @@ static int bpf_size_to_x86_bytes(int bpf_size)
|
|||
* register in load/store instructions, it always needs an
|
||||
* extra byte of encoding and is callee saved.
|
||||
*
|
||||
* R9 caches skb->len - skb->data_len
|
||||
* R10 caches skb->data, and used for blinding (if enabled)
|
||||
* Also x86-64 register R9 is unused. x86-64 register R10 is
|
||||
* used for blinding (if enabled).
|
||||
*/
|
||||
static const int reg2hex[] = {
|
||||
[BPF_REG_0] = 0, /* RAX */
|
||||
|
@ -196,19 +184,15 @@ static void jit_fill_hole(void *area, unsigned int size)
|
|||
|
||||
struct jit_context {
|
||||
int cleanup_addr; /* Epilogue code offset */
|
||||
bool seen_ld_abs;
|
||||
bool seen_ax_reg;
|
||||
};
|
||||
|
||||
/* Maximum number of bytes emitted while JITing one eBPF insn */
|
||||
#define BPF_MAX_INSN_SIZE 128
|
||||
#define BPF_INSN_SAFETY 64
|
||||
|
||||
#define AUX_STACK_SPACE \
|
||||
(32 /* Space for RBX, R13, R14, R15 */ + \
|
||||
8 /* Space for skb_copy_bits() buffer */)
|
||||
#define AUX_STACK_SPACE 40 /* Space for RBX, R13, R14, R15, tailcnt */
|
||||
|
||||
#define PROLOGUE_SIZE 37
|
||||
#define PROLOGUE_SIZE 37
|
||||
|
||||
/*
|
||||
* Emit x86-64 prologue code for BPF program and check its size.
|
||||
|
@ -232,20 +216,8 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
|
|||
/* sub rbp, AUX_STACK_SPACE */
|
||||
EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
|
||||
|
||||
/* All classic BPF filters use R6(rbx) save it */
|
||||
|
||||
/* mov qword ptr [rbp+0],rbx */
|
||||
EMIT4(0x48, 0x89, 0x5D, 0);
|
||||
|
||||
/*
|
||||
* bpf_convert_filter() maps classic BPF register X to R7 and uses R8
|
||||
* as temporary, so all tcpdump filters need to spill/fill R7(R13) and
|
||||
* R8(R14). R9(R15) spill could be made conditional, but there is only
|
||||
* one 'bpf_error' return path out of helper functions inside bpf_jit.S
|
||||
* The overhead of extra spill is negligible for any filter other
|
||||
* than synthetic ones. Therefore not worth adding complexity.
|
||||
*/
|
||||
|
||||
/* mov qword ptr [rbp+8],r13 */
|
||||
EMIT4(0x4C, 0x89, 0x6D, 8);
|
||||
/* mov qword ptr [rbp+16],r14 */
|
||||
|
@ -353,27 +325,6 @@ static void emit_bpf_tail_call(u8 **pprog)
|
|||
*pprog = prog;
|
||||
}
|
||||
|
||||
|
||||
static void emit_load_skb_data_hlen(u8 **pprog)
|
||||
{
|
||||
u8 *prog = *pprog;
|
||||
int cnt = 0;
|
||||
|
||||
/*
|
||||
* r9d = skb->len - skb->data_len (headlen)
|
||||
* r10 = skb->data
|
||||
*/
|
||||
/* mov %r9d, off32(%rdi) */
|
||||
EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len));
|
||||
|
||||
/* sub %r9d, off32(%rdi) */
|
||||
EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len));
|
||||
|
||||
/* mov %r10, off32(%rdi) */
|
||||
EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data));
|
||||
*pprog = prog;
|
||||
}
|
||||
|
||||
static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
|
||||
u32 dst_reg, const u32 imm32)
|
||||
{
|
||||
|
@ -462,8 +413,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
|
|||
{
|
||||
struct bpf_insn *insn = bpf_prog->insnsi;
|
||||
int insn_cnt = bpf_prog->len;
|
||||
bool seen_ld_abs = ctx->seen_ld_abs | (oldproglen == 0);
|
||||
bool seen_ax_reg = ctx->seen_ax_reg | (oldproglen == 0);
|
||||
bool seen_exit = false;
|
||||
u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
|
||||
int i, cnt = 0;
|
||||
|
@ -473,9 +422,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
|
|||
emit_prologue(&prog, bpf_prog->aux->stack_depth,
|
||||
bpf_prog_was_classic(bpf_prog));
|
||||
|
||||
if (seen_ld_abs)
|
||||
emit_load_skb_data_hlen(&prog);
|
||||
|
||||
for (i = 0; i < insn_cnt; i++, insn++) {
|
||||
const s32 imm32 = insn->imm;
|
||||
u32 dst_reg = insn->dst_reg;
|
||||
|
@ -483,13 +429,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
|
|||
u8 b2 = 0, b3 = 0;
|
||||
s64 jmp_offset;
|
||||
u8 jmp_cond;
|
||||
bool reload_skb_data;
|
||||
int ilen;
|
||||
u8 *func;
|
||||
|
||||
if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX)
|
||||
ctx->seen_ax_reg = seen_ax_reg = true;
|
||||
|
||||
switch (insn->code) {
|
||||
/* ALU */
|
||||
case BPF_ALU | BPF_ADD | BPF_X:
|
||||
|
@ -916,36 +858,12 @@ xadd: if (is_imm8(insn->off))
|
|||
case BPF_JMP | BPF_CALL:
|
||||
func = (u8 *) __bpf_call_base + imm32;
|
||||
jmp_offset = func - (image + addrs[i]);
|
||||
if (seen_ld_abs) {
|
||||
reload_skb_data = bpf_helper_changes_pkt_data(func);
|
||||
if (reload_skb_data) {
|
||||
EMIT1(0x57); /* push %rdi */
|
||||
jmp_offset += 22; /* pop, mov, sub, mov */
|
||||
} else {
|
||||
EMIT2(0x41, 0x52); /* push %r10 */
|
||||
EMIT2(0x41, 0x51); /* push %r9 */
|
||||
/*
|
||||
* We need to adjust jmp offset, since
|
||||
* pop %r9, pop %r10 take 4 bytes after call insn
|
||||
*/
|
||||
jmp_offset += 4;
|
||||
}
|
||||
}
|
||||
if (!imm32 || !is_simm32(jmp_offset)) {
|
||||
pr_err("unsupported BPF func %d addr %p image %p\n",
|
||||
imm32, func, image);
|
||||
return -EINVAL;
|
||||
}
|
||||
EMIT1_off32(0xE8, jmp_offset);
|
||||
if (seen_ld_abs) {
|
||||
if (reload_skb_data) {
|
||||
EMIT1(0x5F); /* pop %rdi */
|
||||
emit_load_skb_data_hlen(&prog);
|
||||
} else {
|
||||
EMIT2(0x41, 0x59); /* pop %r9 */
|
||||
EMIT2(0x41, 0x5A); /* pop %r10 */
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case BPF_JMP | BPF_TAIL_CALL:
|
||||
|
@ -1080,60 +998,6 @@ emit_jmp:
|
|||
}
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
func = sk_load_word;
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
func = CHOOSE_LOAD_FUNC(imm32, sk_load_word);
|
||||
common_load:
|
||||
ctx->seen_ld_abs = seen_ld_abs = true;
|
||||
jmp_offset = func - (image + addrs[i]);
|
||||
if (!func || !is_simm32(jmp_offset)) {
|
||||
pr_err("unsupported BPF func %d addr %p image %p\n",
|
||||
imm32, func, image);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (BPF_MODE(insn->code) == BPF_ABS) {
|
||||
/* mov %esi, imm32 */
|
||||
EMIT1_off32(0xBE, imm32);
|
||||
} else {
|
||||
/* mov %rsi, src_reg */
|
||||
EMIT_mov(BPF_REG_2, src_reg);
|
||||
if (imm32) {
|
||||
if (is_imm8(imm32))
|
||||
/* add %esi, imm8 */
|
||||
EMIT3(0x83, 0xC6, imm32);
|
||||
else
|
||||
/* add %esi, imm32 */
|
||||
EMIT2_off32(0x81, 0xC6, imm32);
|
||||
}
|
||||
}
|
||||
/*
|
||||
* skb pointer is in R6 (%rbx), it will be copied into
|
||||
* %rdi if skb_copy_bits() call is necessary.
|
||||
* sk_load_* helpers also use %r10 and %r9d.
|
||||
* See bpf_jit.S
|
||||
*/
|
||||
if (seen_ax_reg)
|
||||
/* r10 = skb->data, mov %r10, off32(%rbx) */
|
||||
EMIT3_off32(0x4c, 0x8b, 0x93,
|
||||
offsetof(struct sk_buff, data));
|
||||
EMIT1_off32(0xE8, jmp_offset); /* call */
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
func = sk_load_half;
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
func = CHOOSE_LOAD_FUNC(imm32, sk_load_half);
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
func = sk_load_byte;
|
||||
goto common_load;
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
func = CHOOSE_LOAD_FUNC(imm32, sk_load_byte);
|
||||
goto common_load;
|
||||
|
||||
case BPF_JMP | BPF_EXIT:
|
||||
if (seen_exit) {
|
||||
jmp_offset = ctx->cleanup_addr - addrs[i];
|
||||
|
|
|
@ -175,19 +175,13 @@ static const u8 bpf2ia32[][2] = {
|
|||
#define SCRATCH_SIZE 96
|
||||
|
||||
/* Total stack size used in JITed code */
|
||||
#define _STACK_SIZE \
|
||||
(stack_depth + \
|
||||
+ SCRATCH_SIZE + \
|
||||
+ 4 /* Extra space for skb_copy_bits buffer */)
|
||||
#define _STACK_SIZE (stack_depth + SCRATCH_SIZE)
|
||||
|
||||
#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
|
||||
|
||||
/* Get the offset of eBPF REGISTERs stored on scratch space. */
|
||||
#define STACK_VAR(off) (off)
|
||||
|
||||
/* Offset of skb_copy_bits buffer */
|
||||
#define SKB_BUFFER STACK_VAR(SCRATCH_SIZE)
|
||||
|
||||
/* Encode 'dst_reg' register into IA32 opcode 'byte' */
|
||||
static u8 add_1reg(u8 byte, u32 dst_reg)
|
||||
{
|
||||
|
@ -2276,134 +2270,6 @@ emit_jmp:
|
|||
return -EFAULT;
|
||||
}
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
{
|
||||
int size;
|
||||
const u8 *r6 = bpf2ia32[BPF_REG_6];
|
||||
|
||||
/* Setting up first argument */
|
||||
/* mov eax,dword ptr [ebp+off] */
|
||||
EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
|
||||
STACK_VAR(r6[0]));
|
||||
|
||||
/* Setting up second argument */
|
||||
if (BPF_MODE(code) == BPF_ABS) {
|
||||
/* mov %edx, imm32 */
|
||||
EMIT1_off32(0xBA, imm32);
|
||||
} else {
|
||||
if (sstk)
|
||||
/* mov edx,dword ptr [ebp+off] */
|
||||
EMIT3(0x8B, add_2reg(0x40, IA32_EBP,
|
||||
IA32_EDX),
|
||||
STACK_VAR(src_lo));
|
||||
else
|
||||
/* mov edx,src_lo */
|
||||
EMIT2(0x8B, add_2reg(0xC0, src_lo,
|
||||
IA32_EDX));
|
||||
if (imm32) {
|
||||
if (is_imm8(imm32))
|
||||
/* add %edx,imm8 */
|
||||
EMIT3(0x83, 0xC2, imm32);
|
||||
else
|
||||
/* add %edx,imm32 */
|
||||
EMIT2_off32(0x81, 0xC2, imm32);
|
||||
}
|
||||
}
|
||||
|
||||
/* Setting up third argument */
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
size = 4;
|
||||
break;
|
||||
case BPF_H:
|
||||
size = 2;
|
||||
break;
|
||||
case BPF_B:
|
||||
size = 1;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
/* mov ecx,val */
|
||||
EMIT2(0xB1, size);
|
||||
/* movzx ecx,ecx */
|
||||
EMIT3(0x0F, 0xB6, add_2reg(0xC0, IA32_ECX, IA32_ECX));
|
||||
|
||||
/* mov ebx,ebp */
|
||||
EMIT2(0x8B, add_2reg(0xC0, IA32_EBP, IA32_EBX));
|
||||
/* add %ebx,imm8 */
|
||||
EMIT3(0x83, add_1reg(0xC0, IA32_EBX), SKB_BUFFER);
|
||||
/* push ebx */
|
||||
EMIT1(0x53);
|
||||
|
||||
/* Setting up function pointer to call */
|
||||
/* mov ebx,imm32*/
|
||||
EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX),
|
||||
(unsigned int)bpf_load_pointer);
|
||||
|
||||
EMIT2(0xFF, add_1reg(0xD0, IA32_EBX));
|
||||
/* add %esp,4 */
|
||||
EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 4);
|
||||
/* xor edx,edx */
|
||||
EMIT2(0x33, add_2reg(0xC0, IA32_EDX, IA32_EDX));
|
||||
|
||||
/* mov dword ptr [ebp+off],eax */
|
||||
EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
|
||||
STACK_VAR(r0[0]));
|
||||
/* mov dword ptr [ebp+off],edx */
|
||||
EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
|
||||
STACK_VAR(r0[1]));
|
||||
|
||||
/*
|
||||
* Check if return address is NULL or not.
|
||||
* If NULL then jump to epilogue else continue
|
||||
* to load the value from retn address
|
||||
*/
|
||||
EMIT3(0x83, add_1reg(0xF8, IA32_EAX), 0);
|
||||
jmp_offset = ctx->cleanup_addr - addrs[i];
|
||||
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
jmp_offset += 7;
|
||||
break;
|
||||
case BPF_H:
|
||||
jmp_offset += 10;
|
||||
break;
|
||||
case BPF_B:
|
||||
jmp_offset += 6;
|
||||
break;
|
||||
}
|
||||
|
||||
EMIT2_off32(0x0F, IA32_JE + 0x10, jmp_offset);
|
||||
/* Load value from the address */
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_W:
|
||||
/* mov eax,[eax] */
|
||||
EMIT2(0x8B, 0x0);
|
||||
/* Emit 'bswap eax' */
|
||||
EMIT2(0x0F, add_1reg(0xC8, IA32_EAX));
|
||||
break;
|
||||
case BPF_H:
|
||||
EMIT3(0x0F, 0xB7, 0x0);
|
||||
EMIT1(0x66);
|
||||
EMIT3(0xC1, add_1reg(0xC8, IA32_EAX), 8);
|
||||
break;
|
||||
case BPF_B:
|
||||
EMIT3(0x0F, 0xB6, 0x0);
|
||||
break;
|
||||
}
|
||||
|
||||
/* mov dword ptr [ebp+off],eax */
|
||||
EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
|
||||
STACK_VAR(r0[0]));
|
||||
break;
|
||||
}
|
||||
/* STX XADD: lock *(u32 *)(dst + off) += src */
|
||||
case BPF_STX | BPF_XADD | BPF_W:
|
||||
/* STX XADD: lock *(u64 *)(dst + off) += src */
|
||||
|
|
|
@ -235,6 +235,8 @@ struct bpf_verifier_ops {
|
|||
struct bpf_insn_access_aux *info);
|
||||
int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
|
||||
const struct bpf_prog *prog);
|
||||
int (*gen_ld_abs)(const struct bpf_insn *orig,
|
||||
struct bpf_insn *insn_buf);
|
||||
u32 (*convert_ctx_access)(enum bpf_access_type type,
|
||||
const struct bpf_insn *src,
|
||||
struct bpf_insn *dst,
|
||||
|
@ -714,8 +716,6 @@ extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
|
|||
extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
|
||||
extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
|
||||
extern const struct bpf_func_proto bpf_get_current_comm_proto;
|
||||
extern const struct bpf_func_proto bpf_skb_vlan_push_proto;
|
||||
extern const struct bpf_func_proto bpf_skb_vlan_pop_proto;
|
||||
extern const struct bpf_func_proto bpf_get_stackid_proto;
|
||||
extern const struct bpf_func_proto bpf_get_stack_proto;
|
||||
extern const struct bpf_func_proto bpf_sock_map_update_proto;
|
||||
|
|
|
@ -47,7 +47,9 @@ struct xdp_buff;
|
|||
/* Additional register mappings for converted user programs. */
|
||||
#define BPF_REG_A BPF_REG_0
|
||||
#define BPF_REG_X BPF_REG_7
|
||||
#define BPF_REG_TMP BPF_REG_8
|
||||
#define BPF_REG_TMP BPF_REG_2 /* scratch reg */
|
||||
#define BPF_REG_D BPF_REG_8 /* data, callee-saved */
|
||||
#define BPF_REG_H BPF_REG_9 /* hlen, callee-saved */
|
||||
|
||||
/* Kernel hidden auxiliary/helper register for hardening step.
|
||||
* Only used by eBPF JITs. It's nothing more than a temporary
|
||||
|
|
|
@ -1802,6 +1802,30 @@ union bpf_attr {
|
|||
* Return
|
||||
* a non-negative value equal to or less than size on success, or
|
||||
* a negative error in case of failure.
|
||||
*
|
||||
* int skb_load_bytes_relative(const struct sk_buff *skb, u32 offset, void *to, u32 len, u32 start_header)
|
||||
* Description
|
||||
* This helper is similar to **bpf_skb_load_bytes**\ () in that
|
||||
* it provides an easy way to load *len* bytes from *offset*
|
||||
* from the packet associated to *skb*, into the buffer pointed
|
||||
* by *to*. The difference to **bpf_skb_load_bytes**\ () is that
|
||||
* a fifth argument *start_header* exists in order to select a
|
||||
* base offset to start from. *start_header* can be one of:
|
||||
*
|
||||
* **BPF_HDR_START_MAC**
|
||||
* Base offset to load data from is *skb*'s mac header.
|
||||
* **BPF_HDR_START_NET**
|
||||
* Base offset to load data from is *skb*'s network header.
|
||||
*
|
||||
* In general, "direct packet access" is the preferred method to
|
||||
* access packet data, however, this helper is in particular useful
|
||||
* in socket filters where *skb*\ **->data** does not always point
|
||||
* to the start of the mac header and where "direct packet access"
|
||||
* is not available.
|
||||
*
|
||||
* Return
|
||||
* 0 on success, or a negative error in case of failure.
|
||||
*
|
||||
*/
|
||||
#define __BPF_FUNC_MAPPER(FN) \
|
||||
FN(unspec), \
|
||||
|
@ -1871,7 +1895,8 @@ union bpf_attr {
|
|||
FN(bind), \
|
||||
FN(xdp_adjust_tail), \
|
||||
FN(skb_get_xfrm_state), \
|
||||
FN(get_stack),
|
||||
FN(get_stack), \
|
||||
FN(skb_load_bytes_relative),
|
||||
|
||||
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
|
||||
* function eBPF program intends to call
|
||||
|
@ -1932,6 +1957,12 @@ enum bpf_adj_room_mode {
|
|||
BPF_ADJ_ROOM_NET,
|
||||
};
|
||||
|
||||
/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */
|
||||
enum bpf_hdr_start_off {
|
||||
BPF_HDR_START_MAC,
|
||||
BPF_HDR_START_NET,
|
||||
};
|
||||
|
||||
/* user accessible mirror of in-kernel sk_buff.
|
||||
* new fields can only be added to the end of this structure
|
||||
*/
|
||||
|
|
|
@ -634,23 +634,6 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
|
|||
*to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_ABS | BPF_W:
|
||||
case BPF_LD | BPF_ABS | BPF_H:
|
||||
case BPF_LD | BPF_ABS | BPF_B:
|
||||
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
|
||||
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
|
||||
*to++ = BPF_LD_IND(from->code, BPF_REG_AX, 0);
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_IND | BPF_W:
|
||||
case BPF_LD | BPF_IND | BPF_H:
|
||||
case BPF_LD | BPF_IND | BPF_B:
|
||||
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
|
||||
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
|
||||
*to++ = BPF_ALU32_REG(BPF_ADD, BPF_REG_AX, from->src_reg);
|
||||
*to++ = BPF_LD_IND(from->code, BPF_REG_AX, 0);
|
||||
break;
|
||||
|
||||
case BPF_LD | BPF_IMM | BPF_DW:
|
||||
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);
|
||||
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
|
||||
|
@ -891,14 +874,7 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
|
|||
INSN_3(LDX, MEM, W), \
|
||||
INSN_3(LDX, MEM, DW), \
|
||||
/* Immediate based. */ \
|
||||
INSN_3(LD, IMM, DW), \
|
||||
/* Misc (old cBPF carry-over). */ \
|
||||
INSN_3(LD, ABS, B), \
|
||||
INSN_3(LD, ABS, H), \
|
||||
INSN_3(LD, ABS, W), \
|
||||
INSN_3(LD, IND, B), \
|
||||
INSN_3(LD, IND, H), \
|
||||
INSN_3(LD, IND, W)
|
||||
INSN_3(LD, IMM, DW)
|
||||
|
||||
bool bpf_opcode_in_insntable(u8 code)
|
||||
{
|
||||
|
@ -908,6 +884,13 @@ bool bpf_opcode_in_insntable(u8 code)
|
|||
[0 ... 255] = false,
|
||||
/* Now overwrite non-defaults ... */
|
||||
BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL),
|
||||
/* UAPI exposed, but rewritten opcodes. cBPF carry-over. */
|
||||
[BPF_LD | BPF_ABS | BPF_B] = true,
|
||||
[BPF_LD | BPF_ABS | BPF_H] = true,
|
||||
[BPF_LD | BPF_ABS | BPF_W] = true,
|
||||
[BPF_LD | BPF_IND | BPF_B] = true,
|
||||
[BPF_LD | BPF_IND | BPF_H] = true,
|
||||
[BPF_LD | BPF_IND | BPF_W] = true,
|
||||
};
|
||||
#undef BPF_INSN_3_TBL
|
||||
#undef BPF_INSN_2_TBL
|
||||
|
@ -938,8 +921,6 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
|
|||
#undef BPF_INSN_3_LBL
|
||||
#undef BPF_INSN_2_LBL
|
||||
u32 tail_call_cnt = 0;
|
||||
void *ptr;
|
||||
int off;
|
||||
|
||||
#define CONT ({ insn++; goto select_insn; })
|
||||
#define CONT_JMP ({ insn++; goto select_insn; })
|
||||
|
@ -1266,67 +1247,6 @@ out:
|
|||
atomic64_add((u64) SRC, (atomic64_t *)(unsigned long)
|
||||
(DST + insn->off));
|
||||
CONT;
|
||||
LD_ABS_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */
|
||||
off = IMM;
|
||||
load_word:
|
||||
/* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are only
|
||||
* appearing in the programs where ctx == skb
|
||||
* (see may_access_skb() in the verifier). All programs
|
||||
* keep 'ctx' in regs[BPF_REG_CTX] == BPF_R6,
|
||||
* bpf_convert_filter() saves it in BPF_R6, internal BPF
|
||||
* verifier will check that BPF_R6 == ctx.
|
||||
*
|
||||
* BPF_ABS and BPF_IND are wrappers of function calls,
|
||||
* so they scratch BPF_R1-BPF_R5 registers, preserve
|
||||
* BPF_R6-BPF_R9, and store return value into BPF_R0.
|
||||
*
|
||||
* Implicit input:
|
||||
* ctx == skb == BPF_R6 == CTX
|
||||
*
|
||||
* Explicit input:
|
||||
* SRC == any register
|
||||
* IMM == 32-bit immediate
|
||||
*
|
||||
* Output:
|
||||
* BPF_R0 - 8/16/32-bit skb data converted to cpu endianness
|
||||
*/
|
||||
|
||||
ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 4, &tmp);
|
||||
if (likely(ptr != NULL)) {
|
||||
BPF_R0 = get_unaligned_be32(ptr);
|
||||
CONT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
LD_ABS_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + imm32)) */
|
||||
off = IMM;
|
||||
load_half:
|
||||
ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 2, &tmp);
|
||||
if (likely(ptr != NULL)) {
|
||||
BPF_R0 = get_unaligned_be16(ptr);
|
||||
CONT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
LD_ABS_B: /* BPF_R0 = *(u8 *) (skb->data + imm32) */
|
||||
off = IMM;
|
||||
load_byte:
|
||||
ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 1, &tmp);
|
||||
if (likely(ptr != NULL)) {
|
||||
BPF_R0 = *(u8 *)ptr;
|
||||
CONT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
LD_IND_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + src_reg + imm32)) */
|
||||
off = IMM + SRC;
|
||||
goto load_word;
|
||||
LD_IND_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + src_reg + imm32)) */
|
||||
off = IMM + SRC;
|
||||
goto load_half;
|
||||
LD_IND_B: /* BPF_R0 = *(u8 *) (skb->data + src_reg + imm32) */
|
||||
off = IMM + SRC;
|
||||
goto load_byte;
|
||||
|
||||
default_label:
|
||||
/* If we ever reach this, we have a bug somewhere. Die hard here
|
||||
|
|
|
@ -3884,6 +3884,11 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!env->ops->gen_ld_abs) {
|
||||
verbose(env, "bpf verifier is misconfigured\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (env->subprog_cnt) {
|
||||
/* when program has LD_ABS insn JITs and interpreter assume
|
||||
* that r1 == ctx == skb which is not the case for callees
|
||||
|
@ -5519,6 +5524,25 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (BPF_CLASS(insn->code) == BPF_LD &&
|
||||
(BPF_MODE(insn->code) == BPF_ABS ||
|
||||
BPF_MODE(insn->code) == BPF_IND)) {
|
||||
cnt = env->ops->gen_ld_abs(insn, insn_buf);
|
||||
if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
|
||||
verbose(env, "bpf verifier is misconfigured\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
|
||||
if (!new_prog)
|
||||
return -ENOMEM;
|
||||
|
||||
delta += cnt - 1;
|
||||
env->prog = prog = new_prog;
|
||||
insn = new_prog->insnsi + i + delta;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (insn->code != (BPF_JMP | BPF_CALL))
|
||||
continue;
|
||||
if (insn->src_reg == BPF_PSEUDO_CALL)
|
||||
|
|
570
lib/test_bpf.c
570
lib/test_bpf.c
|
@ -386,116 +386,6 @@ static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
|
|||
return 0;
|
||||
}
|
||||
|
||||
#define PUSH_CNT 68
|
||||
/* test: {skb->data[0], vlan_push} x 68 + {skb->data[0], vlan_pop} x 68 */
|
||||
static int bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
|
||||
{
|
||||
unsigned int len = BPF_MAXINSNS;
|
||||
struct bpf_insn *insn;
|
||||
int i = 0, j, k = 0;
|
||||
|
||||
insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
|
||||
if (!insn)
|
||||
return -ENOMEM;
|
||||
|
||||
insn[i++] = BPF_MOV64_REG(R6, R1);
|
||||
loop:
|
||||
for (j = 0; j < PUSH_CNT; j++) {
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2);
|
||||
i++;
|
||||
insn[i++] = BPF_MOV64_REG(R1, R6);
|
||||
insn[i++] = BPF_MOV64_IMM(R2, 1);
|
||||
insn[i++] = BPF_MOV64_IMM(R3, 2);
|
||||
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
bpf_skb_vlan_push_proto.func - __bpf_call_base);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
|
||||
i++;
|
||||
}
|
||||
|
||||
for (j = 0; j < PUSH_CNT; j++) {
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2);
|
||||
i++;
|
||||
insn[i++] = BPF_MOV64_REG(R1, R6);
|
||||
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
bpf_skb_vlan_pop_proto.func - __bpf_call_base);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
|
||||
i++;
|
||||
}
|
||||
if (++k < 5)
|
||||
goto loop;
|
||||
|
||||
for (; i < len - 1; i++)
|
||||
insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xbef);
|
||||
|
||||
insn[len - 1] = BPF_EXIT_INSN();
|
||||
|
||||
self->u.ptr.insns = insn;
|
||||
self->u.ptr.len = len;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int bpf_fill_ld_abs_vlan_push_pop2(struct bpf_test *self)
|
||||
{
|
||||
struct bpf_insn *insn;
|
||||
|
||||
insn = kmalloc_array(16, sizeof(*insn), GFP_KERNEL);
|
||||
if (!insn)
|
||||
return -ENOMEM;
|
||||
|
||||
/* Due to func address being non-const, we need to
|
||||
* assemble this here.
|
||||
*/
|
||||
insn[0] = BPF_MOV64_REG(R6, R1);
|
||||
insn[1] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[2] = BPF_LD_ABS(BPF_H, 0);
|
||||
insn[3] = BPF_LD_ABS(BPF_W, 0);
|
||||
insn[4] = BPF_MOV64_REG(R7, R6);
|
||||
insn[5] = BPF_MOV64_IMM(R6, 0);
|
||||
insn[6] = BPF_MOV64_REG(R1, R7);
|
||||
insn[7] = BPF_MOV64_IMM(R2, 1);
|
||||
insn[8] = BPF_MOV64_IMM(R3, 2);
|
||||
insn[9] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
bpf_skb_vlan_push_proto.func - __bpf_call_base);
|
||||
insn[10] = BPF_MOV64_REG(R6, R7);
|
||||
insn[11] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[12] = BPF_LD_ABS(BPF_H, 0);
|
||||
insn[13] = BPF_LD_ABS(BPF_W, 0);
|
||||
insn[14] = BPF_MOV64_IMM(R0, 42);
|
||||
insn[15] = BPF_EXIT_INSN();
|
||||
|
||||
self->u.ptr.insns = insn;
|
||||
self->u.ptr.len = 16;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int bpf_fill_jump_around_ld_abs(struct bpf_test *self)
|
||||
{
|
||||
unsigned int len = BPF_MAXINSNS;
|
||||
struct bpf_insn *insn;
|
||||
int i = 0;
|
||||
|
||||
insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
|
||||
if (!insn)
|
||||
return -ENOMEM;
|
||||
|
||||
insn[i++] = BPF_MOV64_REG(R6, R1);
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JEQ, R0, 10, len - i - 2);
|
||||
i++;
|
||||
while (i < len - 1)
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 1);
|
||||
insn[i] = BPF_EXIT_INSN();
|
||||
|
||||
self->u.ptr.insns = insn;
|
||||
self->u.ptr.len = len;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __bpf_fill_stxdw(struct bpf_test *self, int size)
|
||||
{
|
||||
unsigned int len = BPF_MAXINSNS;
|
||||
|
@ -1987,40 +1877,6 @@ static struct bpf_test tests[] = {
|
|||
{ },
|
||||
{ { 0, -1 } }
|
||||
},
|
||||
{
|
||||
"INT: DIV + ABS",
|
||||
.u.insns_int = {
|
||||
BPF_ALU64_REG(BPF_MOV, R6, R1),
|
||||
BPF_LD_ABS(BPF_B, 3),
|
||||
BPF_ALU64_IMM(BPF_MOV, R2, 2),
|
||||
BPF_ALU32_REG(BPF_DIV, R0, R2),
|
||||
BPF_ALU64_REG(BPF_MOV, R8, R0),
|
||||
BPF_LD_ABS(BPF_B, 4),
|
||||
BPF_ALU64_REG(BPF_ADD, R8, R0),
|
||||
BPF_LD_IND(BPF_B, R8, -70),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
INTERNAL,
|
||||
{ 10, 20, 30, 40, 50 },
|
||||
{ { 4, 0 }, { 5, 10 } }
|
||||
},
|
||||
{
|
||||
/* This one doesn't go through verifier, but is just raw insn
|
||||
* as opposed to cBPF tests from here. Thus div by 0 tests are
|
||||
* done in test_verifier in BPF kselftests.
|
||||
*/
|
||||
"INT: DIV by -1",
|
||||
.u.insns_int = {
|
||||
BPF_ALU64_REG(BPF_MOV, R6, R1),
|
||||
BPF_ALU64_IMM(BPF_MOV, R7, -1),
|
||||
BPF_LD_ABS(BPF_B, 3),
|
||||
BPF_ALU32_REG(BPF_DIV, R0, R7),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
INTERNAL,
|
||||
{ 10, 20, 30, 40, 50 },
|
||||
{ { 3, 0 }, { 4, 0 } }
|
||||
},
|
||||
{
|
||||
"check: missing ret",
|
||||
.u.insns = {
|
||||
|
@ -2383,50 +2239,6 @@ static struct bpf_test tests[] = {
|
|||
{ },
|
||||
{ { 0, 1 } }
|
||||
},
|
||||
{
|
||||
"nmap reduced",
|
||||
.u.insns_int = {
|
||||
BPF_MOV64_REG(R6, R1),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, R0, 0x806, 28),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, R0, 0x806, 26),
|
||||
BPF_MOV32_IMM(R0, 18),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -64),
|
||||
BPF_LDX_MEM(BPF_W, R7, R10, -64),
|
||||
BPF_LD_IND(BPF_W, R7, 14),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -60),
|
||||
BPF_MOV32_IMM(R0, 280971478),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -56),
|
||||
BPF_LDX_MEM(BPF_W, R7, R10, -56),
|
||||
BPF_LDX_MEM(BPF_W, R0, R10, -60),
|
||||
BPF_ALU32_REG(BPF_SUB, R0, R7),
|
||||
BPF_JMP_IMM(BPF_JNE, R0, 0, 15),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, R0, 0x806, 13),
|
||||
BPF_MOV32_IMM(R0, 22),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -56),
|
||||
BPF_LDX_MEM(BPF_W, R7, R10, -56),
|
||||
BPF_LD_IND(BPF_H, R7, 14),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -52),
|
||||
BPF_MOV32_IMM(R0, 17366),
|
||||
BPF_STX_MEM(BPF_W, R10, R0, -48),
|
||||
BPF_LDX_MEM(BPF_W, R7, R10, -48),
|
||||
BPF_LDX_MEM(BPF_W, R0, R10, -52),
|
||||
BPF_ALU32_REG(BPF_SUB, R0, R7),
|
||||
BPF_JMP_IMM(BPF_JNE, R0, 0, 2),
|
||||
BPF_MOV32_IMM(R0, 256),
|
||||
BPF_EXIT_INSN(),
|
||||
BPF_MOV32_IMM(R0, 0),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
INTERNAL,
|
||||
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x06, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6},
|
||||
{ { 38, 256 } },
|
||||
.stack_depth = 64,
|
||||
},
|
||||
/* BPF_ALU | BPF_MOV | BPF_X */
|
||||
{
|
||||
"ALU_MOV_X: dst = 2",
|
||||
|
@ -5485,22 +5297,6 @@ static struct bpf_test tests[] = {
|
|||
{ { 1, 0xbee } },
|
||||
.fill_helper = bpf_fill_ld_abs_get_processor_id,
|
||||
},
|
||||
{
|
||||
"BPF_MAXINSNS: ld_abs+vlan_push/pop",
|
||||
{ },
|
||||
INTERNAL,
|
||||
{ 0x34 },
|
||||
{ { ETH_HLEN, 0xbef } },
|
||||
.fill_helper = bpf_fill_ld_abs_vlan_push_pop,
|
||||
},
|
||||
{
|
||||
"BPF_MAXINSNS: jump around ld_abs",
|
||||
{ },
|
||||
INTERNAL,
|
||||
{ 10, 11 },
|
||||
{ { 2, 10 } },
|
||||
.fill_helper = bpf_fill_jump_around_ld_abs,
|
||||
},
|
||||
/*
|
||||
* LD_IND / LD_ABS on fragmented SKBs
|
||||
*/
|
||||
|
@ -5682,6 +5478,53 @@ static struct bpf_test tests[] = {
|
|||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x05 } },
|
||||
},
|
||||
{
|
||||
"LD_IND byte positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xff } },
|
||||
},
|
||||
{
|
||||
"LD_IND byte positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_IND byte negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 } },
|
||||
},
|
||||
{
|
||||
"LD_IND byte negative offset, multiple calls",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3b),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 1),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 2),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 3),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 4),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x82 }, },
|
||||
},
|
||||
{
|
||||
"LD_IND halfword positive offset",
|
||||
.u.insns = {
|
||||
|
@ -5730,6 +5573,39 @@ static struct bpf_test tests[] = {
|
|||
},
|
||||
{ {0x40, 0x66cc } },
|
||||
},
|
||||
{
|
||||
"LD_IND halfword positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3d),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xffff } },
|
||||
},
|
||||
{
|
||||
"LD_IND halfword positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_IND halfword negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 } },
|
||||
},
|
||||
{
|
||||
"LD_IND word positive offset",
|
||||
.u.insns = {
|
||||
|
@ -5820,6 +5696,39 @@ static struct bpf_test tests[] = {
|
|||
},
|
||||
{ {0x40, 0x66cc77dd } },
|
||||
},
|
||||
{
|
||||
"LD_IND word positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3b),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xffffffff } },
|
||||
},
|
||||
{
|
||||
"LD_IND word positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_IND word negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 } },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte",
|
||||
.u.insns = {
|
||||
|
@ -5837,6 +5746,68 @@ static struct bpf_test tests[] = {
|
|||
},
|
||||
{ {0x40, 0xcc } },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xff } },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte negative offset, out of bounds load",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, -1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC | FLAG_EXPECTED_FAIL,
|
||||
.expected_errcode = -EINVAL,
|
||||
},
|
||||
{
|
||||
"LD_ABS byte negative offset, in bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x82 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS byte negative offset, multiple calls",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3c),
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3d),
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3e),
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x82 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword",
|
||||
.u.insns = {
|
||||
|
@ -5871,6 +5842,55 @@ static struct bpf_test tests[] = {
|
|||
},
|
||||
{ {0x40, 0x99ff } },
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3e),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xffff } },
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword negative offset, out of bounds load",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_H, -1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC | FLAG_EXPECTED_FAIL,
|
||||
.expected_errcode = -EINVAL,
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword negative offset, in bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x1982 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS halfword negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS word",
|
||||
.u.insns = {
|
||||
|
@ -5939,6 +5959,140 @@ static struct bpf_test tests[] = {
|
|||
},
|
||||
{ {0x40, 0x88ee99ff } },
|
||||
},
|
||||
{
|
||||
"LD_ABS word positive offset, all ff",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3c),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
|
||||
{ {0x40, 0xffffffff } },
|
||||
},
|
||||
{
|
||||
"LD_ABS word positive offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS word negative offset, out of bounds load",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_W, -1),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC | FLAG_EXPECTED_FAIL,
|
||||
.expected_errcode = -EINVAL,
|
||||
},
|
||||
{
|
||||
"LD_ABS word negative offset, in bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x25051982 }, },
|
||||
},
|
||||
{
|
||||
"LD_ABS word negative offset, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x3f, 0 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, preserved A",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0xffeebbaa }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, preserved A 2",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0x175e9d63),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3d),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3f),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x175e9d63 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, test result 1",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
|
||||
BPF_STMT(BPF_MISC | BPF_TXA, 0),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x14 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, test result 2",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e),
|
||||
BPF_STMT(BPF_MISC | BPF_TXA, 0),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x24 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, negative offset",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, -1),
|
||||
BPF_STMT(BPF_MISC | BPF_TXA, 0),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, negative offset 2",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, SKF_LL_OFF + 0x3e),
|
||||
BPF_STMT(BPF_MISC | BPF_TXA, 0),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0x24 }, },
|
||||
},
|
||||
{
|
||||
"LDX_MSH standalone, out of bounds",
|
||||
.u.insns = {
|
||||
BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
|
||||
BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x40),
|
||||
BPF_STMT(BPF_MISC | BPF_TXA, 0),
|
||||
BPF_STMT(BPF_RET | BPF_A, 0x0),
|
||||
},
|
||||
CLASSIC,
|
||||
{ [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
|
||||
{ {0x40, 0 }, },
|
||||
},
|
||||
/*
|
||||
* verify that the interpreter or JIT correctly sets A and X
|
||||
* to 0.
|
||||
|
@ -6127,14 +6281,6 @@ static struct bpf_test tests[] = {
|
|||
{},
|
||||
{ {0x1, 0x42 } },
|
||||
},
|
||||
{
|
||||
"LD_ABS with helper changing skb data",
|
||||
{ },
|
||||
INTERNAL,
|
||||
{ 0x34 },
|
||||
{ { ETH_HLEN, 42 } },
|
||||
.fill_helper = bpf_fill_ld_abs_vlan_push_pop2,
|
||||
},
|
||||
/* Checking interpreter vs JIT wrt signed extended imms. */
|
||||
{
|
||||
"JNE signed compare, test 1",
|
||||
|
|
|
@ -113,12 +113,12 @@ int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
|
|||
}
|
||||
EXPORT_SYMBOL(sk_filter_trim_cap);
|
||||
|
||||
BPF_CALL_1(__skb_get_pay_offset, struct sk_buff *, skb)
|
||||
BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb)
|
||||
{
|
||||
return skb_get_poff(skb);
|
||||
}
|
||||
|
||||
BPF_CALL_3(__skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
|
||||
BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
|
||||
{
|
||||
struct nlattr *nla;
|
||||
|
||||
|
@ -138,7 +138,7 @@ BPF_CALL_3(__skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
|
|||
return 0;
|
||||
}
|
||||
|
||||
BPF_CALL_3(__skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
|
||||
BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
|
||||
{
|
||||
struct nlattr *nla;
|
||||
|
||||
|
@ -162,13 +162,94 @@ BPF_CALL_3(__skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
|
|||
return 0;
|
||||
}
|
||||
|
||||
BPF_CALL_0(__get_raw_cpu_id)
|
||||
BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *,
|
||||
data, int, headlen, int, offset)
|
||||
{
|
||||
u8 tmp, *ptr;
|
||||
const int len = sizeof(tmp);
|
||||
|
||||
if (offset >= 0) {
|
||||
if (headlen - offset >= len)
|
||||
return *(u8 *)(data + offset);
|
||||
if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
|
||||
return tmp;
|
||||
} else {
|
||||
ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
|
||||
if (likely(ptr))
|
||||
return *(u8 *)ptr;
|
||||
}
|
||||
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb,
|
||||
int, offset)
|
||||
{
|
||||
return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len,
|
||||
offset);
|
||||
}
|
||||
|
||||
BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *,
|
||||
data, int, headlen, int, offset)
|
||||
{
|
||||
u16 tmp, *ptr;
|
||||
const int len = sizeof(tmp);
|
||||
|
||||
if (offset >= 0) {
|
||||
if (headlen - offset >= len)
|
||||
return get_unaligned_be16(data + offset);
|
||||
if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
|
||||
return be16_to_cpu(tmp);
|
||||
} else {
|
||||
ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
|
||||
if (likely(ptr))
|
||||
return get_unaligned_be16(ptr);
|
||||
}
|
||||
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb,
|
||||
int, offset)
|
||||
{
|
||||
return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len,
|
||||
offset);
|
||||
}
|
||||
|
||||
BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *,
|
||||
data, int, headlen, int, offset)
|
||||
{
|
||||
u32 tmp, *ptr;
|
||||
const int len = sizeof(tmp);
|
||||
|
||||
if (likely(offset >= 0)) {
|
||||
if (headlen - offset >= len)
|
||||
return get_unaligned_be32(data + offset);
|
||||
if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
|
||||
return be32_to_cpu(tmp);
|
||||
} else {
|
||||
ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
|
||||
if (likely(ptr))
|
||||
return get_unaligned_be32(ptr);
|
||||
}
|
||||
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb,
|
||||
int, offset)
|
||||
{
|
||||
return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len,
|
||||
offset);
|
||||
}
|
||||
|
||||
BPF_CALL_0(bpf_get_raw_cpu_id)
|
||||
{
|
||||
return raw_smp_processor_id();
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = {
|
||||
.func = __get_raw_cpu_id,
|
||||
.func = bpf_get_raw_cpu_id,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
};
|
||||
|
@ -318,16 +399,16 @@ static bool convert_bpf_extensions(struct sock_filter *fp,
|
|||
/* Emit call(arg1=CTX, arg2=A, arg3=X) */
|
||||
switch (fp->k) {
|
||||
case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
|
||||
*insn = BPF_EMIT_CALL(__skb_get_pay_offset);
|
||||
*insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset);
|
||||
break;
|
||||
case SKF_AD_OFF + SKF_AD_NLATTR:
|
||||
*insn = BPF_EMIT_CALL(__skb_get_nlattr);
|
||||
*insn = BPF_EMIT_CALL(bpf_skb_get_nlattr);
|
||||
break;
|
||||
case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
|
||||
*insn = BPF_EMIT_CALL(__skb_get_nlattr_nest);
|
||||
*insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest);
|
||||
break;
|
||||
case SKF_AD_OFF + SKF_AD_CPU:
|
||||
*insn = BPF_EMIT_CALL(__get_raw_cpu_id);
|
||||
*insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id);
|
||||
break;
|
||||
case SKF_AD_OFF + SKF_AD_RANDOM:
|
||||
*insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
|
||||
|
@ -354,26 +435,87 @@ static bool convert_bpf_extensions(struct sock_filter *fp,
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
|
||||
{
|
||||
const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS);
|
||||
int size = bpf_size_to_bytes(BPF_SIZE(fp->code));
|
||||
bool endian = BPF_SIZE(fp->code) == BPF_H ||
|
||||
BPF_SIZE(fp->code) == BPF_W;
|
||||
bool indirect = BPF_MODE(fp->code) == BPF_IND;
|
||||
const int ip_align = NET_IP_ALIGN;
|
||||
struct bpf_insn *insn = *insnp;
|
||||
int offset = fp->k;
|
||||
|
||||
if (!indirect &&
|
||||
((unaligned_ok && offset >= 0) ||
|
||||
(!unaligned_ok && offset >= 0 &&
|
||||
offset + ip_align >= 0 &&
|
||||
offset + ip_align % size == 0))) {
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
|
||||
*insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
|
||||
*insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP, size, 2 + endian);
|
||||
*insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, BPF_REG_D,
|
||||
offset);
|
||||
if (endian)
|
||||
*insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
|
||||
*insn++ = BPF_JMP_A(8);
|
||||
}
|
||||
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D);
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H);
|
||||
if (!indirect) {
|
||||
*insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset);
|
||||
} else {
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X);
|
||||
if (fp->k)
|
||||
*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset);
|
||||
}
|
||||
|
||||
switch (BPF_SIZE(fp->code)) {
|
||||
case BPF_B:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8);
|
||||
break;
|
||||
case BPF_H:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16);
|
||||
break;
|
||||
case BPF_W:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32);
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
*insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2);
|
||||
*insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
|
||||
*insn = BPF_EXIT_INSN();
|
||||
|
||||
*insnp = insn;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* bpf_convert_filter - convert filter program
|
||||
* @prog: the user passed filter program
|
||||
* @len: the length of the user passed filter program
|
||||
* @new_prog: allocated 'struct bpf_prog' or NULL
|
||||
* @new_len: pointer to store length of converted program
|
||||
* @seen_ld_abs: bool whether we've seen ld_abs/ind
|
||||
*
|
||||
* Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn'
|
||||
* style extended BPF (eBPF).
|
||||
* Conversion workflow:
|
||||
*
|
||||
* 1) First pass for calculating the new program length:
|
||||
* bpf_convert_filter(old_prog, old_len, NULL, &new_len)
|
||||
* bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs)
|
||||
*
|
||||
* 2) 2nd pass to remap in two passes: 1st pass finds new
|
||||
* jump offsets, 2nd pass remapping:
|
||||
* bpf_convert_filter(old_prog, old_len, new_prog, &new_len);
|
||||
* bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs)
|
||||
*/
|
||||
static int bpf_convert_filter(struct sock_filter *prog, int len,
|
||||
struct bpf_prog *new_prog, int *new_len)
|
||||
struct bpf_prog *new_prog, int *new_len,
|
||||
bool *seen_ld_abs)
|
||||
{
|
||||
int new_flen = 0, pass = 0, target, i, stack_off;
|
||||
struct bpf_insn *new_insn, *first_insn = NULL;
|
||||
|
@ -412,12 +554,27 @@ do_pass:
|
|||
* do this ourself. Initial CTX is present in BPF_REG_ARG1.
|
||||
*/
|
||||
*new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
|
||||
if (*seen_ld_abs) {
|
||||
/* For packet access in classic BPF, cache skb->data
|
||||
* in callee-saved BPF R8 and skb->len - skb->data_len
|
||||
* (headlen) in BPF R9. Since classic BPF is read-only
|
||||
* on CTX, we only need to cache it once.
|
||||
*/
|
||||
*new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
|
||||
BPF_REG_D, BPF_REG_CTX,
|
||||
offsetof(struct sk_buff, data));
|
||||
*new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX,
|
||||
offsetof(struct sk_buff, len));
|
||||
*new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX,
|
||||
offsetof(struct sk_buff, data_len));
|
||||
*new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP);
|
||||
}
|
||||
} else {
|
||||
new_insn += 3;
|
||||
}
|
||||
|
||||
for (i = 0; i < len; fp++, i++) {
|
||||
struct bpf_insn tmp_insns[6] = { };
|
||||
struct bpf_insn tmp_insns[32] = { };
|
||||
struct bpf_insn *insn = tmp_insns;
|
||||
|
||||
if (addrs)
|
||||
|
@ -460,6 +617,11 @@ do_pass:
|
|||
BPF_MODE(fp->code) == BPF_ABS &&
|
||||
convert_bpf_extensions(fp, &insn))
|
||||
break;
|
||||
if (BPF_CLASS(fp->code) == BPF_LD &&
|
||||
convert_bpf_ld_abs(fp, &insn)) {
|
||||
*seen_ld_abs = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) ||
|
||||
fp->code == (BPF_ALU | BPF_MOD | BPF_X)) {
|
||||
|
@ -562,21 +724,31 @@ jmp_rest:
|
|||
break;
|
||||
|
||||
/* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */
|
||||
case BPF_LDX | BPF_MSH | BPF_B:
|
||||
/* tmp = A */
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_A);
|
||||
case BPF_LDX | BPF_MSH | BPF_B: {
|
||||
struct sock_filter tmp = {
|
||||
.code = BPF_LD | BPF_ABS | BPF_B,
|
||||
.k = fp->k,
|
||||
};
|
||||
|
||||
*seen_ld_abs = true;
|
||||
|
||||
/* X = A */
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
|
||||
/* A = BPF_R0 = *(u8 *) (skb->data + K) */
|
||||
*insn++ = BPF_LD_ABS(BPF_B, fp->k);
|
||||
convert_bpf_ld_abs(&tmp, &insn);
|
||||
insn++;
|
||||
/* A &= 0xf */
|
||||
*insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
|
||||
/* A <<= 2 */
|
||||
*insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
|
||||
/* tmp = X */
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X);
|
||||
/* X = A */
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
|
||||
/* A = tmp */
|
||||
*insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
|
||||
break;
|
||||
|
||||
}
|
||||
/* RET_K is remaped into 2 insns. RET_A case doesn't need an
|
||||
* extra mov as BPF_REG_0 is already mapped into BPF_REG_A.
|
||||
*/
|
||||
|
@ -658,6 +830,8 @@ jmp_rest:
|
|||
if (!new_prog) {
|
||||
/* Only calculating new length. */
|
||||
*new_len = new_insn - first_insn;
|
||||
if (*seen_ld_abs)
|
||||
*new_len += 4; /* Prologue bits. */
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -1019,6 +1193,7 @@ static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
|
|||
struct sock_filter *old_prog;
|
||||
struct bpf_prog *old_fp;
|
||||
int err, new_len, old_len = fp->len;
|
||||
bool seen_ld_abs = false;
|
||||
|
||||
/* We are free to overwrite insns et al right here as it
|
||||
* won't be used at this point in time anymore internally
|
||||
|
@ -1040,7 +1215,8 @@ static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
|
|||
}
|
||||
|
||||
/* 1st pass: calculate the new program length. */
|
||||
err = bpf_convert_filter(old_prog, old_len, NULL, &new_len);
|
||||
err = bpf_convert_filter(old_prog, old_len, NULL, &new_len,
|
||||
&seen_ld_abs);
|
||||
if (err)
|
||||
goto out_err_free;
|
||||
|
||||
|
@ -1059,7 +1235,8 @@ static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
|
|||
fp->len = new_len;
|
||||
|
||||
/* 2nd pass: remap sock_filter insns into bpf_insn insns. */
|
||||
err = bpf_convert_filter(old_prog, old_len, fp, &new_len);
|
||||
err = bpf_convert_filter(old_prog, old_len, fp, &new_len,
|
||||
&seen_ld_abs);
|
||||
if (err)
|
||||
/* 2nd bpf_convert_filter() can fail only if it fails
|
||||
* to allocate memory, remapping must succeed. Note,
|
||||
|
@ -1507,6 +1684,47 @@ static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
|
|||
.arg4_type = ARG_CONST_SIZE,
|
||||
};
|
||||
|
||||
BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb,
|
||||
u32, offset, void *, to, u32, len, u32, start_header)
|
||||
{
|
||||
u8 *ptr;
|
||||
|
||||
if (unlikely(offset > 0xffff || len > skb_headlen(skb)))
|
||||
goto err_clear;
|
||||
|
||||
switch (start_header) {
|
||||
case BPF_HDR_START_MAC:
|
||||
ptr = skb_mac_header(skb) + offset;
|
||||
break;
|
||||
case BPF_HDR_START_NET:
|
||||
ptr = skb_network_header(skb) + offset;
|
||||
break;
|
||||
default:
|
||||
goto err_clear;
|
||||
}
|
||||
|
||||
if (likely(ptr >= skb_mac_header(skb) &&
|
||||
ptr + len <= skb_tail_pointer(skb))) {
|
||||
memcpy(to, ptr, len);
|
||||
return 0;
|
||||
}
|
||||
|
||||
err_clear:
|
||||
memset(to, 0, len);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = {
|
||||
.func = bpf_skb_load_bytes_relative,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
.arg2_type = ARG_ANYTHING,
|
||||
.arg3_type = ARG_PTR_TO_UNINIT_MEM,
|
||||
.arg4_type = ARG_CONST_SIZE,
|
||||
.arg5_type = ARG_ANYTHING,
|
||||
};
|
||||
|
||||
BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
|
||||
{
|
||||
/* Idea is the following: should the needed direct read/write
|
||||
|
@ -2181,7 +2399,7 @@ BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
|
|||
return ret;
|
||||
}
|
||||
|
||||
const struct bpf_func_proto bpf_skb_vlan_push_proto = {
|
||||
static const struct bpf_func_proto bpf_skb_vlan_push_proto = {
|
||||
.func = bpf_skb_vlan_push,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
|
@ -2189,7 +2407,6 @@ const struct bpf_func_proto bpf_skb_vlan_push_proto = {
|
|||
.arg2_type = ARG_ANYTHING,
|
||||
.arg3_type = ARG_ANYTHING,
|
||||
};
|
||||
EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto);
|
||||
|
||||
BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
|
||||
{
|
||||
|
@ -2203,13 +2420,12 @@ BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
|
|||
return ret;
|
||||
}
|
||||
|
||||
const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
|
||||
static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
|
||||
.func = bpf_skb_vlan_pop,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
};
|
||||
EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto);
|
||||
|
||||
static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
|
||||
{
|
||||
|
@ -3886,6 +4102,8 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
switch (func_id) {
|
||||
case BPF_FUNC_skb_load_bytes:
|
||||
return &bpf_skb_load_bytes_proto;
|
||||
case BPF_FUNC_skb_load_bytes_relative:
|
||||
return &bpf_skb_load_bytes_relative_proto;
|
||||
case BPF_FUNC_get_socket_cookie:
|
||||
return &bpf_get_socket_cookie_proto;
|
||||
case BPF_FUNC_get_socket_uid:
|
||||
|
@ -3903,6 +4121,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_skb_store_bytes_proto;
|
||||
case BPF_FUNC_skb_load_bytes:
|
||||
return &bpf_skb_load_bytes_proto;
|
||||
case BPF_FUNC_skb_load_bytes_relative:
|
||||
return &bpf_skb_load_bytes_relative_proto;
|
||||
case BPF_FUNC_skb_pull_data:
|
||||
return &bpf_skb_pull_data_proto;
|
||||
case BPF_FUNC_csum_diff:
|
||||
|
@ -4332,6 +4552,41 @@ static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
|
|||
return insn - insn_buf;
|
||||
}
|
||||
|
||||
static int bpf_gen_ld_abs(const struct bpf_insn *orig,
|
||||
struct bpf_insn *insn_buf)
|
||||
{
|
||||
bool indirect = BPF_MODE(orig->code) == BPF_IND;
|
||||
struct bpf_insn *insn = insn_buf;
|
||||
|
||||
/* We're guaranteed here that CTX is in R6. */
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
|
||||
if (!indirect) {
|
||||
*insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
|
||||
} else {
|
||||
*insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg);
|
||||
if (orig->imm)
|
||||
*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
|
||||
}
|
||||
|
||||
switch (BPF_SIZE(orig->code)) {
|
||||
case BPF_B:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache);
|
||||
break;
|
||||
case BPF_H:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache);
|
||||
break;
|
||||
case BPF_W:
|
||||
*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache);
|
||||
break;
|
||||
}
|
||||
|
||||
*insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2);
|
||||
*insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
|
||||
*insn++ = BPF_EXIT_INSN();
|
||||
|
||||
return insn - insn_buf;
|
||||
}
|
||||
|
||||
static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
|
||||
const struct bpf_prog *prog)
|
||||
{
|
||||
|
@ -5601,6 +5856,7 @@ const struct bpf_verifier_ops sk_filter_verifier_ops = {
|
|||
.get_func_proto = sk_filter_func_proto,
|
||||
.is_valid_access = sk_filter_is_valid_access,
|
||||
.convert_ctx_access = bpf_convert_ctx_access,
|
||||
.gen_ld_abs = bpf_gen_ld_abs,
|
||||
};
|
||||
|
||||
const struct bpf_prog_ops sk_filter_prog_ops = {
|
||||
|
@ -5612,6 +5868,7 @@ const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
|
|||
.is_valid_access = tc_cls_act_is_valid_access,
|
||||
.convert_ctx_access = tc_cls_act_convert_ctx_access,
|
||||
.gen_prologue = tc_cls_act_prologue,
|
||||
.gen_ld_abs = bpf_gen_ld_abs,
|
||||
};
|
||||
|
||||
const struct bpf_prog_ops tc_cls_act_prog_ops = {
|
||||
|
|
|
@ -1801,6 +1801,30 @@ union bpf_attr {
|
|||
* Return
|
||||
* a non-negative value equal to or less than size on success, or
|
||||
* a negative error in case of failure.
|
||||
*
|
||||
* int skb_load_bytes_relative(const struct sk_buff *skb, u32 offset, void *to, u32 len, u32 start_header)
|
||||
* Description
|
||||
* This helper is similar to **bpf_skb_load_bytes**\ () in that
|
||||
* it provides an easy way to load *len* bytes from *offset*
|
||||
* from the packet associated to *skb*, into the buffer pointed
|
||||
* by *to*. The difference to **bpf_skb_load_bytes**\ () is that
|
||||
* a fifth argument *start_header* exists in order to select a
|
||||
* base offset to start from. *start_header* can be one of:
|
||||
*
|
||||
* **BPF_HDR_START_MAC**
|
||||
* Base offset to load data from is *skb*'s mac header.
|
||||
* **BPF_HDR_START_NET**
|
||||
* Base offset to load data from is *skb*'s network header.
|
||||
*
|
||||
* In general, "direct packet access" is the preferred method to
|
||||
* access packet data, however, this helper is in particular useful
|
||||
* in socket filters where *skb*\ **->data** does not always point
|
||||
* to the start of the mac header and where "direct packet access"
|
||||
* is not available.
|
||||
*
|
||||
* Return
|
||||
* 0 on success, or a negative error in case of failure.
|
||||
*
|
||||
*/
|
||||
#define __BPF_FUNC_MAPPER(FN) \
|
||||
FN(unspec), \
|
||||
|
@ -1870,7 +1894,8 @@ union bpf_attr {
|
|||
FN(bind), \
|
||||
FN(xdp_adjust_tail), \
|
||||
FN(skb_get_xfrm_state), \
|
||||
FN(get_stack),
|
||||
FN(get_stack), \
|
||||
FN(skb_load_bytes_relative),
|
||||
|
||||
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
|
||||
* function eBPF program intends to call
|
||||
|
@ -1931,6 +1956,12 @@ enum bpf_adj_room_mode {
|
|||
BPF_ADJ_ROOM_NET,
|
||||
};
|
||||
|
||||
/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */
|
||||
enum bpf_hdr_start_off {
|
||||
BPF_HDR_START_MAC,
|
||||
BPF_HDR_START_NET,
|
||||
};
|
||||
|
||||
/* user accessible mirror of in-kernel sk_buff.
|
||||
* new fields can only be added to the end of this structure
|
||||
*/
|
||||
|
|
|
@ -47,7 +47,7 @@
|
|||
# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
|
||||
#endif
|
||||
|
||||
#define MAX_INSNS 512
|
||||
#define MAX_INSNS BPF_MAXINSNS
|
||||
#define MAX_FIXUPS 8
|
||||
#define MAX_NR_MAPS 4
|
||||
#define POINTER_VALUE 0xcafe4all
|
||||
|
@ -77,6 +77,8 @@ struct bpf_test {
|
|||
} result, result_unpriv;
|
||||
enum bpf_prog_type prog_type;
|
||||
uint8_t flags;
|
||||
__u8 data[TEST_DATA_LEN];
|
||||
void (*fill_helper)(struct bpf_test *self);
|
||||
};
|
||||
|
||||
/* Note we want this to be 64 bit aligned so that the end of our array is
|
||||
|
@ -94,6 +96,62 @@ struct other_val {
|
|||
long long bar;
|
||||
};
|
||||
|
||||
static void bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
|
||||
{
|
||||
/* test: {skb->data[0], vlan_push} x 68 + {skb->data[0], vlan_pop} x 68 */
|
||||
#define PUSH_CNT 51
|
||||
unsigned int len = BPF_MAXINSNS;
|
||||
struct bpf_insn *insn = self->insns;
|
||||
int i = 0, j, k = 0;
|
||||
|
||||
insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
|
||||
loop:
|
||||
for (j = 0; j < PUSH_CNT; j++) {
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
|
||||
i++;
|
||||
insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
|
||||
insn[i++] = BPF_MOV64_IMM(BPF_REG_2, 1);
|
||||
insn[i++] = BPF_MOV64_IMM(BPF_REG_3, 2);
|
||||
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
BPF_FUNC_skb_vlan_push),
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
|
||||
i++;
|
||||
}
|
||||
|
||||
for (j = 0; j < PUSH_CNT; j++) {
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
|
||||
i++;
|
||||
insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
|
||||
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
BPF_FUNC_skb_vlan_pop),
|
||||
insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
|
||||
i++;
|
||||
}
|
||||
if (++k < 5)
|
||||
goto loop;
|
||||
|
||||
for (; i < len - 1; i++)
|
||||
insn[i] = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, 0xbef);
|
||||
insn[len - 1] = BPF_EXIT_INSN();
|
||||
}
|
||||
|
||||
static void bpf_fill_jump_around_ld_abs(struct bpf_test *self)
|
||||
{
|
||||
struct bpf_insn *insn = self->insns;
|
||||
unsigned int len = BPF_MAXINSNS;
|
||||
int i = 0;
|
||||
|
||||
insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 0);
|
||||
insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 10, len - i - 2);
|
||||
i++;
|
||||
while (i < len - 1)
|
||||
insn[i++] = BPF_LD_ABS(BPF_B, 1);
|
||||
insn[i] = BPF_EXIT_INSN();
|
||||
}
|
||||
|
||||
static struct bpf_test tests[] = {
|
||||
{
|
||||
"add+sub+mul",
|
||||
|
@ -11725,6 +11783,197 @@ static struct bpf_test tests[] = {
|
|||
.result = ACCEPT,
|
||||
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
|
||||
},
|
||||
{
|
||||
"ld_abs: invalid op 1",
|
||||
.insns = {
|
||||
BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_ABS(BPF_DW, 0),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = REJECT,
|
||||
.errstr = "unknown opcode",
|
||||
},
|
||||
{
|
||||
"ld_abs: invalid op 2",
|
||||
.insns = {
|
||||
BPF_MOV32_IMM(BPF_REG_0, 256),
|
||||
BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_IND(BPF_DW, BPF_REG_0, 0),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = REJECT,
|
||||
.errstr = "unknown opcode",
|
||||
},
|
||||
{
|
||||
"ld_abs: nmap reduced",
|
||||
.insns = {
|
||||
BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 28),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 26),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 18),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -64),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -64),
|
||||
BPF_LD_IND(BPF_W, BPF_REG_7, 14),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -60),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 280971478),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -56),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -56),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -60),
|
||||
BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_7),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 15),
|
||||
BPF_LD_ABS(BPF_H, 12),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 13),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 22),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -56),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -56),
|
||||
BPF_LD_IND(BPF_H, BPF_REG_7, 14),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -52),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 17366),
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -48),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -48),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -52),
|
||||
BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_7),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 256),
|
||||
BPF_EXIT_INSN(),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 0),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x06, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 256,
|
||||
},
|
||||
{
|
||||
"ld_abs: div + abs, test 1",
|
||||
.insns = {
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_ABS(BPF_B, 3),
|
||||
BPF_ALU64_IMM(BPF_MOV, BPF_REG_2, 2),
|
||||
BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_2),
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_8, BPF_REG_0),
|
||||
BPF_LD_ABS(BPF_B, 4),
|
||||
BPF_ALU64_REG(BPF_ADD, BPF_REG_8, BPF_REG_0),
|
||||
BPF_LD_IND(BPF_B, BPF_REG_8, -70),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
10, 20, 30, 40, 50,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 10,
|
||||
},
|
||||
{
|
||||
"ld_abs: div + abs, test 2",
|
||||
.insns = {
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_ABS(BPF_B, 3),
|
||||
BPF_ALU64_IMM(BPF_MOV, BPF_REG_2, 2),
|
||||
BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_2),
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_8, BPF_REG_0),
|
||||
BPF_LD_ABS(BPF_B, 128),
|
||||
BPF_ALU64_REG(BPF_ADD, BPF_REG_8, BPF_REG_0),
|
||||
BPF_LD_IND(BPF_B, BPF_REG_8, -70),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
10, 20, 30, 40, 50,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 0,
|
||||
},
|
||||
{
|
||||
"ld_abs: div + abs, test 3",
|
||||
.insns = {
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
|
||||
BPF_ALU64_IMM(BPF_MOV, BPF_REG_7, 0),
|
||||
BPF_LD_ABS(BPF_B, 3),
|
||||
BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_7),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
10, 20, 30, 40, 50,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 0,
|
||||
},
|
||||
{
|
||||
"ld_abs: div + abs, test 4",
|
||||
.insns = {
|
||||
BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
|
||||
BPF_ALU64_IMM(BPF_MOV, BPF_REG_7, 0),
|
||||
BPF_LD_ABS(BPF_B, 256),
|
||||
BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_7),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
10, 20, 30, 40, 50,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 0,
|
||||
},
|
||||
{
|
||||
"ld_abs: vlan + abs, test 1",
|
||||
.insns = { },
|
||||
.data = {
|
||||
0x34,
|
||||
},
|
||||
.fill_helper = bpf_fill_ld_abs_vlan_push_pop,
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 0xbef,
|
||||
},
|
||||
{
|
||||
"ld_abs: vlan + abs, test 2",
|
||||
.insns = {
|
||||
BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
|
||||
BPF_LD_ABS(BPF_B, 0),
|
||||
BPF_LD_ABS(BPF_H, 0),
|
||||
BPF_LD_ABS(BPF_W, 0),
|
||||
BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
|
||||
BPF_MOV64_IMM(BPF_REG_6, 0),
|
||||
BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
|
||||
BPF_MOV64_IMM(BPF_REG_2, 1),
|
||||
BPF_MOV64_IMM(BPF_REG_3, 2),
|
||||
BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
|
||||
BPF_FUNC_skb_vlan_push),
|
||||
BPF_MOV64_REG(BPF_REG_6, BPF_REG_7),
|
||||
BPF_LD_ABS(BPF_B, 0),
|
||||
BPF_LD_ABS(BPF_H, 0),
|
||||
BPF_LD_ABS(BPF_W, 0),
|
||||
BPF_MOV64_IMM(BPF_REG_0, 42),
|
||||
BPF_EXIT_INSN(),
|
||||
},
|
||||
.data = {
|
||||
0x34,
|
||||
},
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 42,
|
||||
},
|
||||
{
|
||||
"ld_abs: jump around ld_abs",
|
||||
.insns = { },
|
||||
.data = {
|
||||
10, 11,
|
||||
},
|
||||
.fill_helper = bpf_fill_jump_around_ld_abs,
|
||||
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
|
||||
.result = ACCEPT,
|
||||
.retval = 10,
|
||||
},
|
||||
};
|
||||
|
||||
static int probe_filter_length(const struct bpf_insn *fp)
|
||||
|
@ -11828,7 +12077,7 @@ static int create_map_in_map(void)
|
|||
return outer_map_fd;
|
||||
}
|
||||
|
||||
static char bpf_vlog[32768];
|
||||
static char bpf_vlog[UINT_MAX >> 8];
|
||||
|
||||
static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog,
|
||||
int *map_fds)
|
||||
|
@ -11839,6 +12088,9 @@ static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog,
|
|||
int *fixup_prog = test->fixup_prog;
|
||||
int *fixup_map_in_map = test->fixup_map_in_map;
|
||||
|
||||
if (test->fill_helper)
|
||||
test->fill_helper(test);
|
||||
|
||||
/* Allocating HTs with 1 elem is fine here, since we only test
|
||||
* for verifier and not do a runtime lookup, so the only thing
|
||||
* that really matters is value size in this case.
|
||||
|
@ -11888,10 +12140,8 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
|
|||
int *passes, int *errors)
|
||||
{
|
||||
int fd_prog, expected_ret, reject_from_alignment;
|
||||
int prog_len, prog_type = test->prog_type;
|
||||
struct bpf_insn *prog = test->insns;
|
||||
int prog_len = probe_filter_length(prog);
|
||||
char data_in[TEST_DATA_LEN] = {};
|
||||
int prog_type = test->prog_type;
|
||||
int map_fds[MAX_NR_MAPS];
|
||||
const char *expected_err;
|
||||
uint32_t retval;
|
||||
|
@ -11901,6 +12151,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
|
|||
map_fds[i] = -1;
|
||||
|
||||
do_test_fixup(test, prog, map_fds);
|
||||
prog_len = probe_filter_length(prog);
|
||||
|
||||
fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER,
|
||||
prog, prog_len, test->flags & F_LOAD_WITH_STRICT_ALIGNMENT,
|
||||
|
@ -11940,8 +12191,9 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
|
|||
}
|
||||
|
||||
if (fd_prog >= 0) {
|
||||
err = bpf_prog_test_run(fd_prog, 1, data_in, sizeof(data_in),
|
||||
NULL, NULL, &retval, NULL);
|
||||
err = bpf_prog_test_run(fd_prog, 1, test->data,
|
||||
sizeof(test->data), NULL, NULL,
|
||||
&retval, NULL);
|
||||
if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) {
|
||||
printf("Unexpected bpf_prog_test_run error\n");
|
||||
goto fail_log;
|
||||
|
|
Loading…
Reference in New Issue
Block a user