forked from Minki/linux
Merge branch 'virtio-net-xdp-fixes'
Jason Wang says: ==================== several fixups for virtio-net XDP Merry Xmas and a Happy New year to all: This series tries to fixes several issues for virtio-net XDP which could be categorized into several parts: - fix several issues during XDP linearizing - allow csumed packet to work for XDP_PASS - make EWMA rxbuf size estimation works for XDP - forbid XDP when GUEST_UFO is support - remove big packet XDP support - add XDP support or small buffer Please see individual patches for details. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
commit
e57cbe48a6
@ -333,9 +333,9 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
|
||||
static void virtnet_xdp_xmit(struct virtnet_info *vi,
|
||||
struct receive_queue *rq,
|
||||
struct send_queue *sq,
|
||||
struct xdp_buff *xdp)
|
||||
struct xdp_buff *xdp,
|
||||
void *data)
|
||||
{
|
||||
struct page *page = virt_to_head_page(xdp->data);
|
||||
struct virtio_net_hdr_mrg_rxbuf *hdr;
|
||||
unsigned int num_sg, len;
|
||||
void *xdp_sent;
|
||||
@ -343,32 +343,46 @@ static void virtnet_xdp_xmit(struct virtnet_info *vi,
|
||||
|
||||
/* Free up any pending old buffers before queueing new ones. */
|
||||
while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
|
||||
struct page *sent_page = virt_to_head_page(xdp_sent);
|
||||
if (vi->mergeable_rx_bufs) {
|
||||
struct page *sent_page = virt_to_head_page(xdp_sent);
|
||||
|
||||
if (vi->mergeable_rx_bufs)
|
||||
put_page(sent_page);
|
||||
else
|
||||
give_pages(rq, sent_page);
|
||||
} else { /* small buffer */
|
||||
struct sk_buff *skb = xdp_sent;
|
||||
|
||||
kfree_skb(skb);
|
||||
}
|
||||
}
|
||||
|
||||
/* Zero header and leave csum up to XDP layers */
|
||||
hdr = xdp->data;
|
||||
memset(hdr, 0, vi->hdr_len);
|
||||
if (vi->mergeable_rx_bufs) {
|
||||
/* Zero header and leave csum up to XDP layers */
|
||||
hdr = xdp->data;
|
||||
memset(hdr, 0, vi->hdr_len);
|
||||
|
||||
num_sg = 1;
|
||||
sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
|
||||
num_sg = 1;
|
||||
sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
|
||||
} else { /* small buffer */
|
||||
struct sk_buff *skb = data;
|
||||
|
||||
/* Zero header and leave csum up to XDP layers */
|
||||
hdr = skb_vnet_hdr(skb);
|
||||
memset(hdr, 0, vi->hdr_len);
|
||||
|
||||
num_sg = 2;
|
||||
sg_init_table(sq->sg, 2);
|
||||
sg_set_buf(sq->sg, hdr, vi->hdr_len);
|
||||
skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
|
||||
}
|
||||
err = virtqueue_add_outbuf(sq->vq, sq->sg, num_sg,
|
||||
xdp->data, GFP_ATOMIC);
|
||||
data, GFP_ATOMIC);
|
||||
if (unlikely(err)) {
|
||||
if (vi->mergeable_rx_bufs)
|
||||
if (vi->mergeable_rx_bufs) {
|
||||
struct page *page = virt_to_head_page(xdp->data);
|
||||
|
||||
put_page(page);
|
||||
else
|
||||
give_pages(rq, page);
|
||||
} else /* small buffer */
|
||||
kfree_skb(data);
|
||||
return; // On error abort to avoid unnecessary kick
|
||||
} else if (!vi->mergeable_rx_bufs) {
|
||||
/* If not mergeable bufs must be big packets so cleanup pages */
|
||||
give_pages(rq, (struct page *)page->private);
|
||||
page->private = 0;
|
||||
}
|
||||
|
||||
virtqueue_kick(sq->vq);
|
||||
@ -377,23 +391,26 @@ static void virtnet_xdp_xmit(struct virtnet_info *vi,
|
||||
static u32 do_xdp_prog(struct virtnet_info *vi,
|
||||
struct receive_queue *rq,
|
||||
struct bpf_prog *xdp_prog,
|
||||
struct page *page, int offset, int len)
|
||||
void *data, int len)
|
||||
{
|
||||
int hdr_padded_len;
|
||||
struct xdp_buff xdp;
|
||||
void *buf;
|
||||
unsigned int qp;
|
||||
u32 act;
|
||||
u8 *buf;
|
||||
|
||||
buf = page_address(page) + offset;
|
||||
|
||||
if (vi->mergeable_rx_bufs)
|
||||
if (vi->mergeable_rx_bufs) {
|
||||
hdr_padded_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
|
||||
else
|
||||
hdr_padded_len = sizeof(struct padded_vnet_hdr);
|
||||
xdp.data = data + hdr_padded_len;
|
||||
xdp.data_end = xdp.data + (len - vi->hdr_len);
|
||||
buf = data;
|
||||
} else { /* small buffers */
|
||||
struct sk_buff *skb = data;
|
||||
|
||||
xdp.data = buf + hdr_padded_len;
|
||||
xdp.data_end = xdp.data + (len - vi->hdr_len);
|
||||
xdp.data = skb->data;
|
||||
xdp.data_end = xdp.data + len;
|
||||
buf = skb->data;
|
||||
}
|
||||
|
||||
act = bpf_prog_run_xdp(xdp_prog, &xdp);
|
||||
switch (act) {
|
||||
@ -403,8 +420,8 @@ static u32 do_xdp_prog(struct virtnet_info *vi,
|
||||
qp = vi->curr_queue_pairs -
|
||||
vi->xdp_queue_pairs +
|
||||
smp_processor_id();
|
||||
xdp.data = buf + (vi->mergeable_rx_bufs ? 0 : 4);
|
||||
virtnet_xdp_xmit(vi, rq, &vi->sq[qp], &xdp);
|
||||
xdp.data = buf;
|
||||
virtnet_xdp_xmit(vi, rq, &vi->sq[qp], &xdp, data);
|
||||
return XDP_TX;
|
||||
default:
|
||||
bpf_warn_invalid_xdp_action(act);
|
||||
@ -414,26 +431,17 @@ static u32 do_xdp_prog(struct virtnet_info *vi,
|
||||
}
|
||||
}
|
||||
|
||||
static struct sk_buff *receive_small(struct virtnet_info *vi, void *buf, unsigned int len)
|
||||
static struct sk_buff *receive_small(struct net_device *dev,
|
||||
struct virtnet_info *vi,
|
||||
struct receive_queue *rq,
|
||||
void *buf, unsigned int len)
|
||||
{
|
||||
struct sk_buff * skb = buf;
|
||||
struct bpf_prog *xdp_prog;
|
||||
|
||||
len -= vi->hdr_len;
|
||||
skb_trim(skb, len);
|
||||
|
||||
return skb;
|
||||
}
|
||||
|
||||
static struct sk_buff *receive_big(struct net_device *dev,
|
||||
struct virtnet_info *vi,
|
||||
struct receive_queue *rq,
|
||||
void *buf,
|
||||
unsigned int len)
|
||||
{
|
||||
struct bpf_prog *xdp_prog;
|
||||
struct page *page = buf;
|
||||
struct sk_buff *skb;
|
||||
|
||||
rcu_read_lock();
|
||||
xdp_prog = rcu_dereference(rq->xdp_prog);
|
||||
if (xdp_prog) {
|
||||
@ -442,7 +450,7 @@ static struct sk_buff *receive_big(struct net_device *dev,
|
||||
|
||||
if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
|
||||
goto err_xdp;
|
||||
act = do_xdp_prog(vi, rq, xdp_prog, page, 0, len);
|
||||
act = do_xdp_prog(vi, rq, xdp_prog, skb, len);
|
||||
switch (act) {
|
||||
case XDP_PASS:
|
||||
break;
|
||||
@ -456,18 +464,33 @@ static struct sk_buff *receive_big(struct net_device *dev,
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
|
||||
return skb;
|
||||
|
||||
err_xdp:
|
||||
rcu_read_unlock();
|
||||
dev->stats.rx_dropped++;
|
||||
kfree_skb(skb);
|
||||
xdp_xmit:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct sk_buff *receive_big(struct net_device *dev,
|
||||
struct virtnet_info *vi,
|
||||
struct receive_queue *rq,
|
||||
void *buf,
|
||||
unsigned int len)
|
||||
{
|
||||
struct page *page = buf;
|
||||
struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
|
||||
|
||||
if (unlikely(!skb))
|
||||
goto err;
|
||||
|
||||
return skb;
|
||||
|
||||
err_xdp:
|
||||
rcu_read_unlock();
|
||||
err:
|
||||
dev->stats.rx_dropped++;
|
||||
give_pages(rq, page);
|
||||
xdp_xmit:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -483,7 +506,7 @@ xdp_xmit:
|
||||
* anymore.
|
||||
*/
|
||||
static struct page *xdp_linearize_page(struct receive_queue *rq,
|
||||
u16 num_buf,
|
||||
u16 *num_buf,
|
||||
struct page *p,
|
||||
int offset,
|
||||
unsigned int *len)
|
||||
@ -497,7 +520,7 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
|
||||
memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
|
||||
page_off += *len;
|
||||
|
||||
while (--num_buf) {
|
||||
while (--*num_buf) {
|
||||
unsigned int buflen;
|
||||
unsigned long ctx;
|
||||
void *buf;
|
||||
@ -507,19 +530,22 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
|
||||
if (unlikely(!ctx))
|
||||
goto err_buf;
|
||||
|
||||
/* guard against a misconfigured or uncooperative backend that
|
||||
* is sending packet larger than the MTU.
|
||||
*/
|
||||
if ((page_off + buflen) > PAGE_SIZE)
|
||||
goto err_buf;
|
||||
|
||||
buf = mergeable_ctx_to_buf_address(ctx);
|
||||
p = virt_to_head_page(buf);
|
||||
off = buf - page_address(p);
|
||||
|
||||
/* guard against a misconfigured or uncooperative backend that
|
||||
* is sending packet larger than the MTU.
|
||||
*/
|
||||
if ((page_off + buflen) > PAGE_SIZE) {
|
||||
put_page(p);
|
||||
goto err_buf;
|
||||
}
|
||||
|
||||
memcpy(page_address(page) + page_off,
|
||||
page_address(p) + off, buflen);
|
||||
page_off += buflen;
|
||||
put_page(p);
|
||||
}
|
||||
|
||||
*len = page_off;
|
||||
@ -552,16 +578,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
||||
struct page *xdp_page;
|
||||
u32 act;
|
||||
|
||||
/* No known backend devices should send packets with
|
||||
* more than a single buffer when XDP conditions are
|
||||
* met. However it is not strictly illegal so the case
|
||||
* is handled as an exception and a warning is thrown.
|
||||
*/
|
||||
/* This happens when rx buffer size is underestimated */
|
||||
if (unlikely(num_buf > 1)) {
|
||||
bpf_warn_invalid_xdp_buffer();
|
||||
|
||||
/* linearize data for XDP */
|
||||
xdp_page = xdp_linearize_page(rq, num_buf,
|
||||
xdp_page = xdp_linearize_page(rq, &num_buf,
|
||||
page, offset, &len);
|
||||
if (!xdp_page)
|
||||
goto err_xdp;
|
||||
@ -575,16 +595,25 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
||||
* the receive path after XDP is loaded. In practice I
|
||||
* was not able to create this condition.
|
||||
*/
|
||||
if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
|
||||
if (unlikely(hdr->hdr.gso_type))
|
||||
goto err_xdp;
|
||||
|
||||
act = do_xdp_prog(vi, rq, xdp_prog, page, offset, len);
|
||||
act = do_xdp_prog(vi, rq, xdp_prog,
|
||||
page_address(xdp_page) + offset, len);
|
||||
switch (act) {
|
||||
case XDP_PASS:
|
||||
if (unlikely(xdp_page != page))
|
||||
__free_pages(xdp_page, 0);
|
||||
/* We can only create skb based on xdp_page. */
|
||||
if (unlikely(xdp_page != page)) {
|
||||
rcu_read_unlock();
|
||||
put_page(page);
|
||||
head_skb = page_to_skb(vi, rq, xdp_page,
|
||||
0, len, PAGE_SIZE);
|
||||
ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
|
||||
return head_skb;
|
||||
}
|
||||
break;
|
||||
case XDP_TX:
|
||||
ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
|
||||
if (unlikely(xdp_page != page))
|
||||
goto err_xdp;
|
||||
rcu_read_unlock();
|
||||
@ -593,6 +622,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
||||
default:
|
||||
if (unlikely(xdp_page != page))
|
||||
__free_pages(xdp_page, 0);
|
||||
ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
|
||||
goto err_xdp;
|
||||
}
|
||||
}
|
||||
@ -704,7 +734,7 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
|
||||
else if (vi->big_packets)
|
||||
skb = receive_big(dev, vi, rq, buf, len);
|
||||
else
|
||||
skb = receive_small(vi, buf, len);
|
||||
skb = receive_small(dev, vi, rq, buf, len);
|
||||
|
||||
if (unlikely(!skb))
|
||||
return;
|
||||
@ -1678,7 +1708,9 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog)
|
||||
int i, err;
|
||||
|
||||
if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
|
||||
virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6)) {
|
||||
virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
|
||||
virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
|
||||
virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO)) {
|
||||
netdev_warn(dev, "can't set XDP while host is implementing LRO, disable LRO first\n");
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user