mirror of
https://github.com/torvalds/linux.git
synced 2024-12-06 19:11:31 +00:00
de8f3a83b0
This work enables generic transfer of metadata from XDP into skb. The basic idea is that we can make use of the fact that the resulting skb must be linear and already comes with a larger headroom for supporting bpf_xdp_adjust_head(), which mangles xdp->data. Here, we base our work on a similar principle and introduce a small helper bpf_xdp_adjust_meta() for adjusting a new pointer called xdp->data_meta. Thus, the packet has a flexible and programmable room for meta data, followed by the actual packet data. struct xdp_buff is therefore laid out that we first point to data_hard_start, then data_meta directly prepended to data followed by data_end marking the end of packet. bpf_xdp_adjust_head() takes into account whether we have meta data already prepended and if so, memmove()s this along with the given offset provided there's enough room. xdp->data_meta is optional and programs are not required to use it. The rationale is that when we process the packet in XDP (e.g. as DoS filter), we can push further meta data along with it for the XDP_PASS case, and give the guarantee that a clsact ingress BPF program on the same device can pick this up for further post-processing. Since we work with skb there, we can also set skb->mark, skb->priority or other skb meta data out of BPF, thus having this scratch space generic and programmable allows for more flexibility than defining a direct 1:1 transfer of potentially new XDP members into skb (it's also more efficient as we don't need to initialize/handle each of such new members). The facility also works together with GRO aggregation. The scratch space at the head of the packet can be multiple of 4 byte up to 32 byte large. Drivers not yet supporting xdp->data_meta can simply be set up with xdp->data_meta as xdp->data + 1 as bpf_xdp_adjust_meta() will detect this and bail out, such that the subsequent match against xdp->data for later access is guaranteed to fail. The verifier treats xdp->data_meta/xdp->data the same way as we treat xdp->data/xdp->data_end pointer comparisons. The requirement for doing the compare against xdp->data is that it hasn't been modified from it's original address we got from ctx access. It may have a range marking already from prior successful xdp->data/xdp->data_end pointer comparisons though. Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Alexei Starovoitov <ast@kernel.org> Acked-by: John Fastabend <john.fastabend@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
175 lines
4.4 KiB
C
175 lines
4.4 KiB
C
/* Copyright (c) 2017 Facebook
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of version 2 of the GNU General Public
|
|
* License as published by the Free Software Foundation.
|
|
*/
|
|
#include <linux/bpf.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/etherdevice.h>
|
|
#include <linux/filter.h>
|
|
#include <linux/sched/signal.h>
|
|
|
|
static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx)
|
|
{
|
|
u32 ret;
|
|
|
|
preempt_disable();
|
|
rcu_read_lock();
|
|
ret = BPF_PROG_RUN(prog, ctx);
|
|
rcu_read_unlock();
|
|
preempt_enable();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static u32 bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *time)
|
|
{
|
|
u64 time_start, time_spent = 0;
|
|
u32 ret = 0, i;
|
|
|
|
if (!repeat)
|
|
repeat = 1;
|
|
time_start = ktime_get_ns();
|
|
for (i = 0; i < repeat; i++) {
|
|
ret = bpf_test_run_one(prog, ctx);
|
|
if (need_resched()) {
|
|
if (signal_pending(current))
|
|
break;
|
|
time_spent += ktime_get_ns() - time_start;
|
|
cond_resched();
|
|
time_start = ktime_get_ns();
|
|
}
|
|
}
|
|
time_spent += ktime_get_ns() - time_start;
|
|
do_div(time_spent, repeat);
|
|
*time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int bpf_test_finish(const union bpf_attr *kattr,
|
|
union bpf_attr __user *uattr, const void *data,
|
|
u32 size, u32 retval, u32 duration)
|
|
{
|
|
void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
|
|
int err = -EFAULT;
|
|
|
|
if (data_out && copy_to_user(data_out, data, size))
|
|
goto out;
|
|
if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
|
|
goto out;
|
|
if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
|
|
goto out;
|
|
if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
|
|
goto out;
|
|
err = 0;
|
|
out:
|
|
return err;
|
|
}
|
|
|
|
static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
|
|
u32 headroom, u32 tailroom)
|
|
{
|
|
void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
|
|
void *data;
|
|
|
|
if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
data = kzalloc(size + headroom + tailroom, GFP_USER);
|
|
if (!data)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
if (copy_from_user(data + headroom, data_in, size)) {
|
|
kfree(data);
|
|
return ERR_PTR(-EFAULT);
|
|
}
|
|
return data;
|
|
}
|
|
|
|
int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
|
|
union bpf_attr __user *uattr)
|
|
{
|
|
bool is_l2 = false, is_direct_pkt_access = false;
|
|
u32 size = kattr->test.data_size_in;
|
|
u32 repeat = kattr->test.repeat;
|
|
u32 retval, duration;
|
|
struct sk_buff *skb;
|
|
void *data;
|
|
int ret;
|
|
|
|
data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
|
|
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
|
|
if (IS_ERR(data))
|
|
return PTR_ERR(data);
|
|
|
|
switch (prog->type) {
|
|
case BPF_PROG_TYPE_SCHED_CLS:
|
|
case BPF_PROG_TYPE_SCHED_ACT:
|
|
is_l2 = true;
|
|
/* fall through */
|
|
case BPF_PROG_TYPE_LWT_IN:
|
|
case BPF_PROG_TYPE_LWT_OUT:
|
|
case BPF_PROG_TYPE_LWT_XMIT:
|
|
is_direct_pkt_access = true;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
skb = build_skb(data, 0);
|
|
if (!skb) {
|
|
kfree(data);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
|
|
__skb_put(skb, size);
|
|
skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
|
|
skb_reset_network_header(skb);
|
|
|
|
if (is_l2)
|
|
__skb_push(skb, ETH_HLEN);
|
|
if (is_direct_pkt_access)
|
|
bpf_compute_data_pointers(skb);
|
|
retval = bpf_test_run(prog, skb, repeat, &duration);
|
|
if (!is_l2)
|
|
__skb_push(skb, ETH_HLEN);
|
|
size = skb->len;
|
|
/* bpf program can never convert linear skb to non-linear */
|
|
if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
|
|
size = skb_headlen(skb);
|
|
ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
|
|
kfree_skb(skb);
|
|
return ret;
|
|
}
|
|
|
|
int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
|
|
union bpf_attr __user *uattr)
|
|
{
|
|
u32 size = kattr->test.data_size_in;
|
|
u32 repeat = kattr->test.repeat;
|
|
struct xdp_buff xdp = {};
|
|
u32 retval, duration;
|
|
void *data;
|
|
int ret;
|
|
|
|
data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
|
|
if (IS_ERR(data))
|
|
return PTR_ERR(data);
|
|
|
|
xdp.data_hard_start = data;
|
|
xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
|
|
xdp.data_meta = xdp.data;
|
|
xdp.data_end = xdp.data + size;
|
|
|
|
retval = bpf_test_run(prog, &xdp, repeat, &duration);
|
|
if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN)
|
|
size = xdp.data_end - xdp.data;
|
|
ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
|
|
kfree(data);
|
|
return ret;
|
|
}
|