forked from Minki/linux
409017847d
Existing BPF_CORE_READ() macro generates slightly suboptimal code. If
there are intermediate pointers to be read, initial source pointer is
going to be assigned into a temporary variable and then temporary
variable is going to be uniformly used as a "source" pointer for all
intermediate pointer reads. Schematically (ignoring all the type casts),
BPF_CORE_READ(s, a, b, c) is expanded into:
({
const void *__t = src;
bpf_probe_read(&__t, sizeof(*__t), &__t->a);
bpf_probe_read(&__t, sizeof(*__t), &__t->b);
typeof(s->a->b->c) __r;
bpf_probe_read(&__r, sizeof(*__r), &__t->c);
})
This initial `__t = src` makes calls more uniform, but causes slightly
less optimal register usage sometimes when compiled with Clang. This can
cascase into, e.g., more register spills.
This patch fixes this issue by generating more optimal sequence:
({
const void *__t;
bpf_probe_read(&__t, sizeof(*__t), &src->a); /* <-- src here */
bpf_probe_read(&__t, sizeof(*__t), &__t->b);
typeof(s->a->b->c) __r;
bpf_probe_read(&__r, sizeof(*__r), &__t->c);
})
Fixes: 7db3822ab9 ("libbpf: Add BPF_CORE_READ/BPF_CORE_READ_INTO helpers")
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20191011023847.275936-1-andriin@fb.com
168 lines
7.0 KiB
C
168 lines
7.0 KiB
C
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
|
#ifndef __BPF_CORE_READ_H__
|
|
#define __BPF_CORE_READ_H__
|
|
|
|
/*
|
|
* bpf_core_read() abstracts away bpf_probe_read() call and captures offset
|
|
* relocation for source address using __builtin_preserve_access_index()
|
|
* built-in, provided by Clang.
|
|
*
|
|
* __builtin_preserve_access_index() takes as an argument an expression of
|
|
* taking an address of a field within struct/union. It makes compiler emit
|
|
* a relocation, which records BTF type ID describing root struct/union and an
|
|
* accessor string which describes exact embedded field that was used to take
|
|
* an address. See detailed description of this relocation format and
|
|
* semantics in comments to struct bpf_offset_reloc in libbpf_internal.h.
|
|
*
|
|
* This relocation allows libbpf to adjust BPF instruction to use correct
|
|
* actual field offset, based on target kernel BTF type that matches original
|
|
* (local) BTF, used to record relocation.
|
|
*/
|
|
#define bpf_core_read(dst, sz, src) \
|
|
bpf_probe_read(dst, sz, \
|
|
(const void *)__builtin_preserve_access_index(src))
|
|
|
|
/*
|
|
* bpf_core_read_str() is a thin wrapper around bpf_probe_read_str()
|
|
* additionally emitting BPF CO-RE field relocation for specified source
|
|
* argument.
|
|
*/
|
|
#define bpf_core_read_str(dst, sz, src) \
|
|
bpf_probe_read_str(dst, sz, \
|
|
(const void *)__builtin_preserve_access_index(src))
|
|
|
|
#define ___concat(a, b) a ## b
|
|
#define ___apply(fn, n) ___concat(fn, n)
|
|
#define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N
|
|
|
|
/*
|
|
* return number of provided arguments; used for switch-based variadic macro
|
|
* definitions (see ___last, ___arrow, etc below)
|
|
*/
|
|
#define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
|
|
/*
|
|
* return 0 if no arguments are passed, N - otherwise; used for
|
|
* recursively-defined macros to specify termination (0) case, and generic
|
|
* (N) case (e.g., ___read_ptrs, ___core_read)
|
|
*/
|
|
#define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0)
|
|
|
|
#define ___last1(x) x
|
|
#define ___last2(a, x) x
|
|
#define ___last3(a, b, x) x
|
|
#define ___last4(a, b, c, x) x
|
|
#define ___last5(a, b, c, d, x) x
|
|
#define ___last6(a, b, c, d, e, x) x
|
|
#define ___last7(a, b, c, d, e, f, x) x
|
|
#define ___last8(a, b, c, d, e, f, g, x) x
|
|
#define ___last9(a, b, c, d, e, f, g, h, x) x
|
|
#define ___last10(a, b, c, d, e, f, g, h, i, x) x
|
|
#define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
|
|
|
#define ___nolast2(a, _) a
|
|
#define ___nolast3(a, b, _) a, b
|
|
#define ___nolast4(a, b, c, _) a, b, c
|
|
#define ___nolast5(a, b, c, d, _) a, b, c, d
|
|
#define ___nolast6(a, b, c, d, e, _) a, b, c, d, e
|
|
#define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f
|
|
#define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g
|
|
#define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h
|
|
#define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i
|
|
#define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
|
|
|
#define ___arrow1(a) a
|
|
#define ___arrow2(a, b) a->b
|
|
#define ___arrow3(a, b, c) a->b->c
|
|
#define ___arrow4(a, b, c, d) a->b->c->d
|
|
#define ___arrow5(a, b, c, d, e) a->b->c->d->e
|
|
#define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f
|
|
#define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g
|
|
#define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h
|
|
#define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i
|
|
#define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j
|
|
#define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
|
|
|
#define ___type(...) typeof(___arrow(__VA_ARGS__))
|
|
|
|
#define ___read(read_fn, dst, src_type, src, accessor) \
|
|
read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor)
|
|
|
|
/* "recursively" read a sequence of inner pointers using local __t var */
|
|
#define ___rd_first(src, a) ___read(bpf_core_read, &__t, ___type(src), src, a);
|
|
#define ___rd_last(...) \
|
|
___read(bpf_core_read, &__t, \
|
|
___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__));
|
|
#define ___rd_p1(...) const void *__t; ___rd_first(__VA_ARGS__)
|
|
#define ___rd_p2(...) ___rd_p1(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p3(...) ___rd_p2(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p4(...) ___rd_p3(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p5(...) ___rd_p4(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p6(...) ___rd_p5(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p7(...) ___rd_p6(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p8(...) ___rd_p7(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___rd_p9(...) ___rd_p8(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
|
#define ___read_ptrs(src, ...) \
|
|
___apply(___rd_p, ___narg(__VA_ARGS__))(src, __VA_ARGS__)
|
|
|
|
#define ___core_read0(fn, dst, src, a) \
|
|
___read(fn, dst, ___type(src), src, a);
|
|
#define ___core_readN(fn, dst, src, ...) \
|
|
___read_ptrs(src, ___nolast(__VA_ARGS__)) \
|
|
___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \
|
|
___last(__VA_ARGS__));
|
|
#define ___core_read(fn, dst, src, a, ...) \
|
|
___apply(___core_read, ___empty(__VA_ARGS__))(fn, dst, \
|
|
src, a, ##__VA_ARGS__)
|
|
|
|
/*
|
|
* BPF_CORE_READ_INTO() is a more performance-conscious variant of
|
|
* BPF_CORE_READ(), in which final field is read into user-provided storage.
|
|
* See BPF_CORE_READ() below for more details on general usage.
|
|
*/
|
|
#define BPF_CORE_READ_INTO(dst, src, a, ...) \
|
|
({ \
|
|
___core_read(bpf_core_read, dst, src, a, ##__VA_ARGS__) \
|
|
})
|
|
|
|
/*
|
|
* BPF_CORE_READ_STR_INTO() does same "pointer chasing" as
|
|
* BPF_CORE_READ() for intermediate pointers, but then executes (and returns
|
|
* corresponding error code) bpf_core_read_str() for final string read.
|
|
*/
|
|
#define BPF_CORE_READ_STR_INTO(dst, src, a, ...) \
|
|
({ \
|
|
___core_read(bpf_core_read_str, dst, src, a, ##__VA_ARGS__) \
|
|
})
|
|
|
|
/*
|
|
* BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially
|
|
* when there are few pointer chasing steps.
|
|
* E.g., what in non-BPF world (or in BPF w/ BCC) would be something like:
|
|
* int x = s->a.b.c->d.e->f->g;
|
|
* can be succinctly achieved using BPF_CORE_READ as:
|
|
* int x = BPF_CORE_READ(s, a.b.c, d.e, f, g);
|
|
*
|
|
* BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF
|
|
* CO-RE relocatable bpf_probe_read() wrapper) calls, logically equivalent to:
|
|
* 1. const void *__t = s->a.b.c;
|
|
* 2. __t = __t->d.e;
|
|
* 3. __t = __t->f;
|
|
* 4. return __t->g;
|
|
*
|
|
* Equivalence is logical, because there is a heavy type casting/preservation
|
|
* involved, as well as all the reads are happening through bpf_probe_read()
|
|
* calls using __builtin_preserve_access_index() to emit CO-RE relocations.
|
|
*
|
|
* N.B. Only up to 9 "field accessors" are supported, which should be more
|
|
* than enough for any practical purpose.
|
|
*/
|
|
#define BPF_CORE_READ(src, a, ...) \
|
|
({ \
|
|
___type(src, a, ##__VA_ARGS__) __r; \
|
|
BPF_CORE_READ_INTO(&__r, src, a, ##__VA_ARGS__); \
|
|
__r; \
|
|
})
|
|
|
|
#endif
|
|
|