mirror of
https://github.com/torvalds/linux.git
synced 2024-10-30 00:32:38 +00:00
virtio_net: rework mergeable buffer handling
Use the new _ctx virtio API to maintain true length for each buffer. Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
This commit is contained in:
parent
d45b897b11
commit
680557cf79
@ -250,24 +250,6 @@ static void skb_xmit_done(struct virtqueue *vq)
|
|||||||
netif_wake_subqueue(vi->dev, vq2txq(vq));
|
netif_wake_subqueue(vi->dev, vq2txq(vq));
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx)
|
|
||||||
{
|
|
||||||
unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1);
|
|
||||||
return (truesize + 1) * MERGEABLE_BUFFER_ALIGN;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx)
|
|
||||||
{
|
|
||||||
return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize)
|
|
||||||
{
|
|
||||||
unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN;
|
|
||||||
return (unsigned long)buf | (size - 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Called from bottom half context */
|
/* Called from bottom half context */
|
||||||
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
|
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
|
||||||
struct receive_queue *rq,
|
struct receive_queue *rq,
|
||||||
@ -511,15 +493,13 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
|
|||||||
|
|
||||||
while (--*num_buf) {
|
while (--*num_buf) {
|
||||||
unsigned int buflen;
|
unsigned int buflen;
|
||||||
unsigned long ctx;
|
|
||||||
void *buf;
|
void *buf;
|
||||||
int off;
|
int off;
|
||||||
|
|
||||||
ctx = (unsigned long)virtqueue_get_buf(rq->vq, &buflen);
|
buf = virtqueue_get_buf(rq->vq, &buflen);
|
||||||
if (unlikely(!ctx))
|
if (unlikely(!buf))
|
||||||
goto err_buf;
|
goto err_buf;
|
||||||
|
|
||||||
buf = mergeable_ctx_to_buf_address(ctx);
|
|
||||||
p = virt_to_head_page(buf);
|
p = virt_to_head_page(buf);
|
||||||
off = buf - page_address(p);
|
off = buf - page_address(p);
|
||||||
|
|
||||||
@ -548,10 +528,10 @@ err_buf:
|
|||||||
static struct sk_buff *receive_mergeable(struct net_device *dev,
|
static struct sk_buff *receive_mergeable(struct net_device *dev,
|
||||||
struct virtnet_info *vi,
|
struct virtnet_info *vi,
|
||||||
struct receive_queue *rq,
|
struct receive_queue *rq,
|
||||||
unsigned long ctx,
|
void *buf,
|
||||||
|
void *ctx,
|
||||||
unsigned int len)
|
unsigned int len)
|
||||||
{
|
{
|
||||||
void *buf = mergeable_ctx_to_buf_address(ctx);
|
|
||||||
struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
|
struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
|
||||||
u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
|
u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
|
||||||
struct page *page = virt_to_head_page(buf);
|
struct page *page = virt_to_head_page(buf);
|
||||||
@ -639,7 +619,13 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
|||||||
}
|
}
|
||||||
rcu_read_unlock();
|
rcu_read_unlock();
|
||||||
|
|
||||||
truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
|
if (unlikely(len > (unsigned long)ctx)) {
|
||||||
|
pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n",
|
||||||
|
dev->name, len, (unsigned long)ctx);
|
||||||
|
dev->stats.rx_length_errors++;
|
||||||
|
goto err_skb;
|
||||||
|
}
|
||||||
|
truesize = (unsigned long)ctx;
|
||||||
head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
|
head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
|
||||||
curr_skb = head_skb;
|
curr_skb = head_skb;
|
||||||
|
|
||||||
@ -648,7 +634,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
|||||||
while (--num_buf) {
|
while (--num_buf) {
|
||||||
int num_skb_frags;
|
int num_skb_frags;
|
||||||
|
|
||||||
ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
|
buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
|
||||||
if (unlikely(!ctx)) {
|
if (unlikely(!ctx)) {
|
||||||
pr_debug("%s: rx error: %d buffers out of %d missing\n",
|
pr_debug("%s: rx error: %d buffers out of %d missing\n",
|
||||||
dev->name, num_buf,
|
dev->name, num_buf,
|
||||||
@ -658,8 +644,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
|||||||
goto err_buf;
|
goto err_buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
buf = mergeable_ctx_to_buf_address(ctx);
|
|
||||||
page = virt_to_head_page(buf);
|
page = virt_to_head_page(buf);
|
||||||
|
if (unlikely(len > (unsigned long)ctx)) {
|
||||||
|
pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n",
|
||||||
|
dev->name, len, (unsigned long)ctx);
|
||||||
|
dev->stats.rx_length_errors++;
|
||||||
|
goto err_skb;
|
||||||
|
}
|
||||||
|
truesize = (unsigned long)ctx;
|
||||||
|
|
||||||
num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
|
num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
|
||||||
if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
|
if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
|
||||||
@ -675,7 +667,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
|
|||||||
head_skb->truesize += nskb->truesize;
|
head_skb->truesize += nskb->truesize;
|
||||||
num_skb_frags = 0;
|
num_skb_frags = 0;
|
||||||
}
|
}
|
||||||
truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
|
|
||||||
if (curr_skb != head_skb) {
|
if (curr_skb != head_skb) {
|
||||||
head_skb->data_len += len;
|
head_skb->data_len += len;
|
||||||
head_skb->len += len;
|
head_skb->len += len;
|
||||||
@ -700,14 +691,14 @@ err_xdp:
|
|||||||
err_skb:
|
err_skb:
|
||||||
put_page(page);
|
put_page(page);
|
||||||
while (--num_buf) {
|
while (--num_buf) {
|
||||||
ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
|
buf = virtqueue_get_buf(rq->vq, &len);
|
||||||
if (unlikely(!ctx)) {
|
if (unlikely(!buf)) {
|
||||||
pr_debug("%s: rx error: %d buffers missing\n",
|
pr_debug("%s: rx error: %d buffers missing\n",
|
||||||
dev->name, num_buf);
|
dev->name, num_buf);
|
||||||
dev->stats.rx_length_errors++;
|
dev->stats.rx_length_errors++;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx));
|
page = virt_to_head_page(buf);
|
||||||
put_page(page);
|
put_page(page);
|
||||||
}
|
}
|
||||||
err_buf:
|
err_buf:
|
||||||
@ -718,7 +709,7 @@ xdp_xmit:
|
|||||||
}
|
}
|
||||||
|
|
||||||
static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
|
static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
|
||||||
void *buf, unsigned int len)
|
void *buf, unsigned int len, void **ctx)
|
||||||
{
|
{
|
||||||
struct net_device *dev = vi->dev;
|
struct net_device *dev = vi->dev;
|
||||||
struct sk_buff *skb;
|
struct sk_buff *skb;
|
||||||
@ -729,9 +720,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
|
|||||||
pr_debug("%s: short packet %i\n", dev->name, len);
|
pr_debug("%s: short packet %i\n", dev->name, len);
|
||||||
dev->stats.rx_length_errors++;
|
dev->stats.rx_length_errors++;
|
||||||
if (vi->mergeable_rx_bufs) {
|
if (vi->mergeable_rx_bufs) {
|
||||||
unsigned long ctx = (unsigned long)buf;
|
put_page(virt_to_head_page(buf));
|
||||||
void *base = mergeable_ctx_to_buf_address(ctx);
|
|
||||||
put_page(virt_to_head_page(base));
|
|
||||||
} else if (vi->big_packets) {
|
} else if (vi->big_packets) {
|
||||||
give_pages(rq, buf);
|
give_pages(rq, buf);
|
||||||
} else {
|
} else {
|
||||||
@ -741,7 +730,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (vi->mergeable_rx_bufs)
|
if (vi->mergeable_rx_bufs)
|
||||||
skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len);
|
skb = receive_mergeable(dev, vi, rq, buf, ctx, len);
|
||||||
else if (vi->big_packets)
|
else if (vi->big_packets)
|
||||||
skb = receive_big(dev, vi, rq, buf, len);
|
skb = receive_big(dev, vi, rq, buf, len);
|
||||||
else
|
else
|
||||||
@ -869,7 +858,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
|
|||||||
struct page_frag *alloc_frag = &rq->alloc_frag;
|
struct page_frag *alloc_frag = &rq->alloc_frag;
|
||||||
unsigned int headroom = virtnet_get_headroom(vi);
|
unsigned int headroom = virtnet_get_headroom(vi);
|
||||||
char *buf;
|
char *buf;
|
||||||
unsigned long ctx;
|
void *ctx;
|
||||||
int err;
|
int err;
|
||||||
unsigned int len, hole;
|
unsigned int len, hole;
|
||||||
|
|
||||||
@ -879,7 +868,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
|
|||||||
|
|
||||||
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
|
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
|
||||||
buf += headroom; /* advance address leaving hole at front of pkt */
|
buf += headroom; /* advance address leaving hole at front of pkt */
|
||||||
ctx = mergeable_buf_to_ctx(buf, len);
|
ctx = (void *)(unsigned long)len;
|
||||||
get_page(alloc_frag->page);
|
get_page(alloc_frag->page);
|
||||||
alloc_frag->offset += len + headroom;
|
alloc_frag->offset += len + headroom;
|
||||||
hole = alloc_frag->size - alloc_frag->offset;
|
hole = alloc_frag->size - alloc_frag->offset;
|
||||||
@ -894,7 +883,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
|
|||||||
}
|
}
|
||||||
|
|
||||||
sg_init_one(rq->sg, buf, len);
|
sg_init_one(rq->sg, buf, len);
|
||||||
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp);
|
err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
|
||||||
if (err < 0)
|
if (err < 0)
|
||||||
put_page(virt_to_head_page(buf));
|
put_page(virt_to_head_page(buf));
|
||||||
|
|
||||||
@ -988,10 +977,20 @@ static int virtnet_receive(struct receive_queue *rq, int budget)
|
|||||||
void *buf;
|
void *buf;
|
||||||
struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
|
struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
|
||||||
|
|
||||||
while (received < budget &&
|
if (vi->mergeable_rx_bufs) {
|
||||||
(buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
|
void *ctx;
|
||||||
bytes += receive_buf(vi, rq, buf, len);
|
|
||||||
received++;
|
while (received < budget &&
|
||||||
|
(buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
|
||||||
|
bytes += receive_buf(vi, rq, buf, len, ctx);
|
||||||
|
received++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
while (received < budget &&
|
||||||
|
(buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
|
||||||
|
bytes += receive_buf(vi, rq, buf, len, NULL);
|
||||||
|
received++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
|
if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
|
||||||
@ -2014,9 +2013,7 @@ static void free_unused_bufs(struct virtnet_info *vi)
|
|||||||
|
|
||||||
while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
|
while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
|
||||||
if (vi->mergeable_rx_bufs) {
|
if (vi->mergeable_rx_bufs) {
|
||||||
unsigned long ctx = (unsigned long)buf;
|
put_page(virt_to_head_page(buf));
|
||||||
void *base = mergeable_ctx_to_buf_address(ctx);
|
|
||||||
put_page(virt_to_head_page(base));
|
|
||||||
} else if (vi->big_packets) {
|
} else if (vi->big_packets) {
|
||||||
give_pages(&vi->rq[i], buf);
|
give_pages(&vi->rq[i], buf);
|
||||||
} else {
|
} else {
|
||||||
|
Loading…
Reference in New Issue
Block a user