2018-05-02 11:01:23 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
/* XDP sockets
|
|
|
|
|
*
|
|
|
|
|
* AF_XDP sockets allows a channel between XDP programs and userspace
|
|
|
|
|
* applications.
|
|
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
|
*
|
|
|
|
|
* Author(s): Björn Töpel <bjorn.topel@intel.com>
|
|
|
|
|
* Magnus Karlsson <magnus.karlsson@intel.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
|
|
|
|
|
|
|
|
|
|
#include <linux/if_xdp.h>
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
|
#include <linux/sched/mm.h>
|
|
|
|
|
#include <linux/sched/signal.h>
|
|
|
|
|
#include <linux/sched/task.h>
|
|
|
|
|
#include <linux/socket.h>
|
|
|
|
|
#include <linux/file.h>
|
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
|
#include <linux/net.h>
|
|
|
|
|
#include <linux/netdevice.h>
|
2018-06-04 12:05:57 +00:00
|
|
|
#include <linux/rculist.h>
|
2018-05-02 11:01:23 +00:00
|
|
|
#include <net/xdp_sock.h>
|
2018-05-02 11:01:25 +00:00
|
|
|
#include <net/xdp.h>
|
2018-05-02 11:01:23 +00:00
|
|
|
|
2018-05-02 11:01:24 +00:00
|
|
|
#include "xsk_queue.h"
|
2018-05-02 11:01:23 +00:00
|
|
|
#include "xdp_umem.h"
|
|
|
|
|
|
2018-05-02 11:01:34 +00:00
|
|
|
#define TX_BATCH_SIZE 16
|
|
|
|
|
|
2018-05-02 11:01:23 +00:00
|
|
|
static struct xdp_sock *xdp_sk(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return (struct xdp_sock *)sk;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:28 +00:00
|
|
|
bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
|
|
|
|
|
{
|
2018-06-04 12:05:55 +00:00
|
|
|
return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&
|
|
|
|
|
READ_ONCE(xs->umem->fq);
|
2018-05-02 11:01:28 +00:00
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
|
|
|
|
|
{
|
|
|
|
|
return xskq_peek_addr(umem->fq, addr);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_peek_addr);
|
|
|
|
|
|
|
|
|
|
void xsk_umem_discard_addr(struct xdp_umem *umem)
|
|
|
|
|
{
|
|
|
|
|
xskq_discard_addr(umem->fq);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_discard_addr);
|
|
|
|
|
|
|
|
|
|
static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
|
2018-05-02 11:01:27 +00:00
|
|
|
{
|
|
|
|
|
void *buffer;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
u64 addr;
|
2018-06-04 11:57:11 +00:00
|
|
|
int err;
|
2018-05-02 11:01:27 +00:00
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
if (!xskq_peek_addr(xs->umem->fq, &addr) ||
|
|
|
|
|
len > xs->umem->chunk_size_nohr) {
|
2018-06-04 11:57:12 +00:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 11:01:27 +00:00
|
|
|
return -ENOSPC;
|
2018-06-04 11:57:12 +00:00
|
|
|
}
|
2018-05-02 11:01:27 +00:00
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
addr += xs->umem->headroom;
|
|
|
|
|
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
2018-05-02 11:01:27 +00:00
|
|
|
memcpy(buffer, xdp->data, len);
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
err = xskq_produce_batch_desc(xs->rx, addr, len);
|
2018-06-04 12:05:55 +00:00
|
|
|
if (!err) {
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
xskq_discard_addr(xs->umem->fq);
|
2018-06-04 12:05:55 +00:00
|
|
|
xdp_return_buff(xdp);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 11:01:27 +00:00
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 11:01:27 +00:00
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
|
2018-05-02 11:01:27 +00:00
|
|
|
{
|
2018-06-04 12:05:55 +00:00
|
|
|
int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);
|
2018-05-02 11:01:27 +00:00
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
if (err) {
|
2018-05-02 11:01:27 +00:00
|
|
|
xdp_return_buff(xdp);
|
2018-06-04 12:05:55 +00:00
|
|
|
xs->rx_dropped++;
|
|
|
|
|
}
|
2018-05-02 11:01:27 +00:00
|
|
|
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
|
{
|
|
|
|
|
u32 len;
|
|
|
|
|
|
|
|
|
|
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
len = xdp->data_end - xdp->data;
|
|
|
|
|
|
|
|
|
|
return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
|
|
|
|
|
__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:27 +00:00
|
|
|
void xsk_flush(struct xdp_sock *xs)
|
|
|
|
|
{
|
|
|
|
|
xskq_produce_flush_desc(xs->rx);
|
|
|
|
|
xs->sk.sk_data_ready(&xs->sk);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
|
{
|
2018-06-04 12:05:55 +00:00
|
|
|
u32 len = xdp->data_end - xdp->data;
|
|
|
|
|
void *buffer;
|
|
|
|
|
u64 addr;
|
2018-05-02 11:01:27 +00:00
|
|
|
int err;
|
|
|
|
|
|
2018-06-12 10:02:56 +00:00
|
|
|
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
if (!xskq_peek_addr(xs->umem->fq, &addr) ||
|
|
|
|
|
len > xs->umem->chunk_size_nohr) {
|
|
|
|
|
xs->rx_dropped++;
|
|
|
|
|
return -ENOSPC;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
addr += xs->umem->headroom;
|
|
|
|
|
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
|
|
|
|
memcpy(buffer, xdp->data, len);
|
|
|
|
|
err = xskq_produce_batch_desc(xs->rx, addr, len);
|
|
|
|
|
if (!err) {
|
|
|
|
|
xskq_discard_addr(xs->umem->fq);
|
2018-05-02 11:01:27 +00:00
|
|
|
xsk_flush(xs);
|
2018-06-04 12:05:55 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 11:01:27 +00:00
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 11:01:27 +00:00
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:57 +00:00
|
|
|
void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
|
|
|
|
|
{
|
|
|
|
|
xskq_produce_flush_addr_n(umem->cq, nb_entries);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_complete_tx);
|
|
|
|
|
|
|
|
|
|
void xsk_umem_consume_tx_done(struct xdp_umem *umem)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
|
|
|
|
|
xs->sk.sk_write_space(&xs->sk);
|
|
|
|
|
}
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx_done);
|
|
|
|
|
|
|
|
|
|
bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_desc desc;
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
|
|
|
|
|
if (!xskq_peek_desc(xs->tx, &desc))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if (xskq_produce_addr_lazy(umem->cq, desc.addr))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
*dma = xdp_umem_get_dma(umem, desc.addr);
|
|
|
|
|
*len = desc.len;
|
|
|
|
|
|
|
|
|
|
xskq_discard_desc(xs->tx);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx);
|
|
|
|
|
|
|
|
|
|
static int xsk_zc_xmit(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
struct net_device *dev = xs->dev;
|
|
|
|
|
|
|
|
|
|
return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:34 +00:00
|
|
|
static void xsk_destruct_skb(struct sk_buff *skb)
|
|
|
|
|
{
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
|
2018-05-02 11:01:34 +00:00
|
|
|
struct xdp_sock *xs = xdp_sk(skb->sk);
|
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
|
2018-05-02 11:01:34 +00:00
|
|
|
|
|
|
|
|
sock_wfree(skb);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_generic_xmit(struct sock *sk, struct msghdr *m,
|
|
|
|
|
size_t total_len)
|
|
|
|
|
{
|
|
|
|
|
u32 max_batch = TX_BATCH_SIZE;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
bool sent_frame = false;
|
|
|
|
|
struct xdp_desc desc;
|
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
|
|
if (unlikely(!xs->tx))
|
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
while (xskq_peek_desc(xs->tx, &desc)) {
|
|
|
|
|
char *buffer;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
u64 addr;
|
|
|
|
|
u32 len;
|
2018-05-02 11:01:34 +00:00
|
|
|
|
|
|
|
|
if (max_batch-- == 0) {
|
|
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
if (xskq_reserve_addr(xs->umem->cq)) {
|
2018-05-02 11:01:34 +00:00
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
len = desc.len;
|
|
|
|
|
if (unlikely(len > xs->dev->mtu)) {
|
|
|
|
|
err = -EMSGSIZE;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-22 07:34:58 +00:00
|
|
|
if (xs->queue_id >= xs->dev->real_num_tx_queues) {
|
|
|
|
|
err = -ENXIO;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:57 +00:00
|
|
|
skb = sock_alloc_send_skb(sk, len, 1, &err);
|
2018-05-02 11:01:34 +00:00
|
|
|
if (unlikely(!skb)) {
|
|
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb_put(skb, len);
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
addr = desc.addr;
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
2018-05-02 11:01:34 +00:00
|
|
|
err = skb_store_bits(skb, 0, buffer, len);
|
|
|
|
|
if (unlikely(err)) {
|
|
|
|
|
kfree_skb(skb);
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb->dev = xs->dev;
|
|
|
|
|
skb->priority = sk->sk_priority;
|
|
|
|
|
skb->mark = sk->sk_mark;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 11:57:13 +00:00
|
|
|
skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
|
2018-05-02 11:01:34 +00:00
|
|
|
skb->destructor = xsk_destruct_skb;
|
|
|
|
|
|
|
|
|
|
err = dev_direct_xmit(skb, xs->queue_id);
|
|
|
|
|
/* Ignore NET_XMIT_CN as packet might have been sent */
|
|
|
|
|
if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
|
|
|
|
|
err = -EAGAIN;
|
|
|
|
|
/* SKB consumed by dev_direct_xmit() */
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sent_frame = true;
|
|
|
|
|
xskq_discard_desc(xs->tx);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
if (sent_frame)
|
|
|
|
|
sk->sk_write_space(sk);
|
|
|
|
|
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
|
|
|
|
|
{
|
2018-06-04 12:05:57 +00:00
|
|
|
bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
|
2018-05-02 11:01:34 +00:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (unlikely(!xs->dev))
|
|
|
|
|
return -ENXIO;
|
|
|
|
|
if (unlikely(!(xs->dev->flags & IFF_UP)))
|
|
|
|
|
return -ENETDOWN;
|
2018-06-04 12:05:57 +00:00
|
|
|
if (need_wait)
|
|
|
|
|
return -EOPNOTSUPP;
|
2018-05-02 11:01:34 +00:00
|
|
|
|
2018-06-04 12:05:57 +00:00
|
|
|
return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
|
2018-05-02 11:01:34 +00:00
|
|
|
}
|
|
|
|
|
|
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller:
1) Add Maglev hashing scheduler to IPVS, from Inju Song.
2) Lots of new TC subsystem tests from Roman Mashak.
3) Add TCP zero copy receive and fix delayed acks and autotuning with
SO_RCVLOWAT, from Eric Dumazet.
4) Add XDP_REDIRECT support to mlx5 driver, from Jesper Dangaard
Brouer.
5) Add ttl inherit support to vxlan, from Hangbin Liu.
6) Properly separate ipv6 routes into their logically independant
components. fib6_info for the routing table, and fib6_nh for sets of
nexthops, which thus can be shared. From David Ahern.
7) Add bpf_xdp_adjust_tail helper, which can be used to generate ICMP
messages from XDP programs. From Nikita V. Shirokov.
8) Lots of long overdue cleanups to the r8169 driver, from Heiner
Kallweit.
9) Add BTF ("BPF Type Format"), from Martin KaFai Lau.
10) Add traffic condition monitoring to iwlwifi, from Luca Coelho.
11) Plumb extack down into fib_rules, from Roopa Prabhu.
12) Add Flower classifier offload support to igb, from Vinicius Costa
Gomes.
13) Add UDP GSO support, from Willem de Bruijn.
14) Add documentation for eBPF helpers, from Quentin Monnet.
15) Add TLS tx offload to mlx5, from Ilya Lesokhin.
16) Allow applications to be given the number of bytes available to read
on a socket via a control message returned from recvmsg(), from
Soheil Hassas Yeganeh.
17) Add x86_32 eBPF JIT compiler, from Wang YanQing.
18) Add AF_XDP sockets, with zerocopy support infrastructure as well.
From Björn Töpel.
19) Remove indirect load support from all of the BPF JITs and handle
these operations in the verifier by translating them into native BPF
instead. From Daniel Borkmann.
20) Add GRO support to ipv6 gre tunnels, from Eran Ben Elisha.
21) Allow XDP programs to do lookups in the main kernel routing tables
for forwarding. From David Ahern.
22) Allow drivers to store hardware state into an ELF section of kernel
dump vmcore files, and use it in cxgb4. From Rahul Lakkireddy.
23) Various RACK and loss detection improvements in TCP, from Yuchung
Cheng.
24) Add TCP SACK compression, from Eric Dumazet.
25) Add User Mode Helper support and basic bpfilter infrastructure, from
Alexei Starovoitov.
26) Support ports and protocol values in RTM_GETROUTE, from Roopa
Prabhu.
27) Support bulking in ->ndo_xdp_xmit() API, from Jesper Dangaard
Brouer.
28) Add lots of forwarding selftests, from Petr Machata.
29) Add generic network device failover driver, from Sridhar Samudrala.
* ra.kernel.org:/pub/scm/linux/kernel/git/davem/net-next: (1959 commits)
strparser: Add __strp_unpause and use it in ktls.
rxrpc: Fix terminal retransmission connection ID to include the channel
net: hns3: Optimize PF CMDQ interrupt switching process
net: hns3: Fix for VF mailbox receiving unknown message
net: hns3: Fix for VF mailbox cannot receiving PF response
bnx2x: use the right constant
Revert "net: sched: cls: Fix offloading when ingress dev is vxlan"
net: dsa: b53: Fix for brcm tag issue in Cygnus SoC
enic: fix UDP rss bits
netdev-FAQ: clarify DaveM's position for stable backports
rtnetlink: validate attributes in do_setlink()
mlxsw: Add extack messages for port_{un, }split failures
netdevsim: Add extack error message for devlink reload
devlink: Add extack to reload and port_{un, }split operations
net: metrics: add proper netlink validation
ipmr: fix error path when ipmr_new_table fails
ip6mr: only set ip6mr_table from setsockopt when ip6mr_new_table succeeds
net: hns3: remove unused hclgevf_cfg_func_mta_filter
netfilter: provide udp*_lib_lookup for nf_tproxy
qed*: Utilize FW 8.37.2.0
...
2018-06-07 01:39:49 +00:00
|
|
|
static __poll_t xsk_poll_mask(struct socket *sock, __poll_t events)
|
2018-05-02 11:01:27 +00:00
|
|
|
{
|
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller:
1) Add Maglev hashing scheduler to IPVS, from Inju Song.
2) Lots of new TC subsystem tests from Roman Mashak.
3) Add TCP zero copy receive and fix delayed acks and autotuning with
SO_RCVLOWAT, from Eric Dumazet.
4) Add XDP_REDIRECT support to mlx5 driver, from Jesper Dangaard
Brouer.
5) Add ttl inherit support to vxlan, from Hangbin Liu.
6) Properly separate ipv6 routes into their logically independant
components. fib6_info for the routing table, and fib6_nh for sets of
nexthops, which thus can be shared. From David Ahern.
7) Add bpf_xdp_adjust_tail helper, which can be used to generate ICMP
messages from XDP programs. From Nikita V. Shirokov.
8) Lots of long overdue cleanups to the r8169 driver, from Heiner
Kallweit.
9) Add BTF ("BPF Type Format"), from Martin KaFai Lau.
10) Add traffic condition monitoring to iwlwifi, from Luca Coelho.
11) Plumb extack down into fib_rules, from Roopa Prabhu.
12) Add Flower classifier offload support to igb, from Vinicius Costa
Gomes.
13) Add UDP GSO support, from Willem de Bruijn.
14) Add documentation for eBPF helpers, from Quentin Monnet.
15) Add TLS tx offload to mlx5, from Ilya Lesokhin.
16) Allow applications to be given the number of bytes available to read
on a socket via a control message returned from recvmsg(), from
Soheil Hassas Yeganeh.
17) Add x86_32 eBPF JIT compiler, from Wang YanQing.
18) Add AF_XDP sockets, with zerocopy support infrastructure as well.
From Björn Töpel.
19) Remove indirect load support from all of the BPF JITs and handle
these operations in the verifier by translating them into native BPF
instead. From Daniel Borkmann.
20) Add GRO support to ipv6 gre tunnels, from Eran Ben Elisha.
21) Allow XDP programs to do lookups in the main kernel routing tables
for forwarding. From David Ahern.
22) Allow drivers to store hardware state into an ELF section of kernel
dump vmcore files, and use it in cxgb4. From Rahul Lakkireddy.
23) Various RACK and loss detection improvements in TCP, from Yuchung
Cheng.
24) Add TCP SACK compression, from Eric Dumazet.
25) Add User Mode Helper support and basic bpfilter infrastructure, from
Alexei Starovoitov.
26) Support ports and protocol values in RTM_GETROUTE, from Roopa
Prabhu.
27) Support bulking in ->ndo_xdp_xmit() API, from Jesper Dangaard
Brouer.
28) Add lots of forwarding selftests, from Petr Machata.
29) Add generic network device failover driver, from Sridhar Samudrala.
* ra.kernel.org:/pub/scm/linux/kernel/git/davem/net-next: (1959 commits)
strparser: Add __strp_unpause and use it in ktls.
rxrpc: Fix terminal retransmission connection ID to include the channel
net: hns3: Optimize PF CMDQ interrupt switching process
net: hns3: Fix for VF mailbox receiving unknown message
net: hns3: Fix for VF mailbox cannot receiving PF response
bnx2x: use the right constant
Revert "net: sched: cls: Fix offloading when ingress dev is vxlan"
net: dsa: b53: Fix for brcm tag issue in Cygnus SoC
enic: fix UDP rss bits
netdev-FAQ: clarify DaveM's position for stable backports
rtnetlink: validate attributes in do_setlink()
mlxsw: Add extack messages for port_{un, }split failures
netdevsim: Add extack error message for devlink reload
devlink: Add extack to reload and port_{un, }split operations
net: metrics: add proper netlink validation
ipmr: fix error path when ipmr_new_table fails
ip6mr: only set ip6mr_table from setsockopt when ip6mr_new_table succeeds
net: hns3: remove unused hclgevf_cfg_func_mta_filter
netfilter: provide udp*_lib_lookup for nf_tproxy
qed*: Utilize FW 8.37.2.0
...
2018-06-07 01:39:49 +00:00
|
|
|
__poll_t mask = datagram_poll_mask(sock, events);
|
2018-05-02 11:01:27 +00:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (xs->rx && !xskq_empty_desc(xs->rx))
|
|
|
|
|
mask |= POLLIN | POLLRDNORM;
|
2018-05-02 11:01:34 +00:00
|
|
|
if (xs->tx && !xskq_full_desc(xs->tx))
|
|
|
|
|
mask |= POLLOUT | POLLWRNORM;
|
2018-05-02 11:01:27 +00:00
|
|
|
|
|
|
|
|
return mask;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:25 +00:00
|
|
|
static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
|
|
|
|
|
bool umem_queue)
|
2018-05-02 11:01:24 +00:00
|
|
|
{
|
|
|
|
|
struct xsk_queue *q;
|
|
|
|
|
|
|
|
|
|
if (entries == 0 || *queue || !is_power_of_2(entries))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2018-05-02 11:01:25 +00:00
|
|
|
q = xskq_create(entries, umem_queue);
|
2018-05-02 11:01:24 +00:00
|
|
|
if (!q)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2018-05-22 07:35:01 +00:00
|
|
|
/* Make sure queue is ready before it can be seen by others */
|
|
|
|
|
smp_wmb();
|
2018-05-02 11:01:24 +00:00
|
|
|
*queue = q;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:23 +00:00
|
|
|
static int xsk_release(struct socket *sock)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
2018-05-02 11:01:26 +00:00
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-02 11:01:23 +00:00
|
|
|
struct net *net;
|
|
|
|
|
|
|
|
|
|
if (!sk)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
net = sock_net(sk);
|
|
|
|
|
|
|
|
|
|
local_bh_disable();
|
|
|
|
|
sock_prot_inuse_add(net, sk->sk_prot, -1);
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
2018-05-02 11:01:26 +00:00
|
|
|
if (xs->dev) {
|
2018-05-22 07:34:56 +00:00
|
|
|
/* Wait for driver to stop using the xdp socket. */
|
|
|
|
|
synchronize_net();
|
|
|
|
|
dev_put(xs->dev);
|
2018-05-02 11:01:26 +00:00
|
|
|
xs->dev = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:23 +00:00
|
|
|
sock_orphan(sk);
|
|
|
|
|
sock->sk = NULL;
|
|
|
|
|
|
|
|
|
|
sk_refcnt_debug_release(sk);
|
|
|
|
|
sock_put(sk);
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:26 +00:00
|
|
|
static struct socket *xsk_lookup_xsk_from_fd(int fd)
|
|
|
|
|
{
|
|
|
|
|
struct socket *sock;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
sock = sockfd_lookup(fd, &err);
|
|
|
|
|
if (!sock)
|
|
|
|
|
return ERR_PTR(-ENOTSOCK);
|
|
|
|
|
|
|
|
|
|
if (sock->sk->sk_family != PF_XDP) {
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
return ERR_PTR(-ENOPROTOOPT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return sock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
|
|
|
|
|
{
|
|
|
|
|
struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-22 07:34:56 +00:00
|
|
|
struct net_device *dev;
|
2018-06-04 12:05:55 +00:00
|
|
|
u32 flags, qid;
|
2018-05-02 11:01:26 +00:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
|
|
if (addr_len < sizeof(struct sockaddr_xdp))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
if (sxdp->sxdp_family != AF_XDP)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 07:34:56 +00:00
|
|
|
if (xs->dev) {
|
|
|
|
|
err = -EBUSY;
|
|
|
|
|
goto out_release;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:26 +00:00
|
|
|
dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
|
|
|
|
|
if (!dev) {
|
|
|
|
|
err = -ENODEV;
|
|
|
|
|
goto out_release;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:32 +00:00
|
|
|
if (!xs->rx && !xs->tx) {
|
2018-05-02 11:01:26 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
qid = sxdp->sxdp_queue_id;
|
|
|
|
|
|
|
|
|
|
if ((xs->rx && qid >= dev->real_num_rx_queues) ||
|
|
|
|
|
(xs->tx && qid >= dev->real_num_tx_queues)) {
|
2018-05-02 11:01:26 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
flags = sxdp->sxdp_flags;
|
|
|
|
|
|
|
|
|
|
if (flags & XDP_SHARED_UMEM) {
|
2018-05-02 11:01:26 +00:00
|
|
|
struct xdp_sock *umem_xs;
|
|
|
|
|
struct socket *sock;
|
|
|
|
|
|
2018-06-04 12:05:55 +00:00
|
|
|
if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) {
|
|
|
|
|
/* Cannot specify flags for shared sockets. */
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:26 +00:00
|
|
|
if (xs->umem) {
|
|
|
|
|
/* We have already our own. */
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
|
|
|
|
|
if (IS_ERR(sock)) {
|
|
|
|
|
err = PTR_ERR(sock);
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
umem_xs = xdp_sk(sock->sk);
|
|
|
|
|
if (!umem_xs->umem) {
|
|
|
|
|
/* No umem to inherit. */
|
|
|
|
|
err = -EBADF;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
goto out_unlock;
|
2018-06-04 12:05:55 +00:00
|
|
|
} else if (umem_xs->dev != dev || umem_xs->queue_id != qid) {
|
2018-05-02 11:01:26 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
xdp_get_umem(umem_xs->umem);
|
|
|
|
|
xs->umem = umem_xs->umem;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
2018-05-02 11:01:27 +00:00
|
|
|
} else {
|
|
|
|
|
/* This xsk has its own umem. */
|
|
|
|
|
xskq_set_umem(xs->umem->fq, &xs->umem->props);
|
2018-05-02 11:01:31 +00:00
|
|
|
xskq_set_umem(xs->umem->cq, &xs->umem->props);
|
2018-06-04 12:05:55 +00:00
|
|
|
|
|
|
|
|
err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out_unlock;
|
2018-05-02 11:01:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
xs->dev = dev;
|
2018-06-04 12:05:57 +00:00
|
|
|
xs->zc = xs->umem->zc;
|
|
|
|
|
xs->queue_id = qid;
|
2018-05-02 11:01:26 +00:00
|
|
|
xskq_set_umem(xs->rx, &xs->umem->props);
|
2018-05-02 11:01:34 +00:00
|
|
|
xskq_set_umem(xs->tx, &xs->umem->props);
|
2018-06-04 12:05:57 +00:00
|
|
|
xdp_add_sk_umem(xs->umem, xs);
|
2018-05-02 11:01:26 +00:00
|
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
|
if (err)
|
|
|
|
|
dev_put(dev);
|
|
|
|
|
out_release:
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:23 +00:00
|
|
|
static int xsk_setsockopt(struct socket *sock, int level, int optname,
|
|
|
|
|
char __user *optval, unsigned int optlen)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
|
|
switch (optname) {
|
2018-05-02 11:01:25 +00:00
|
|
|
case XDP_RX_RING:
|
2018-05-02 11:01:32 +00:00
|
|
|
case XDP_TX_RING:
|
2018-05-02 11:01:25 +00:00
|
|
|
{
|
|
|
|
|
struct xsk_queue **q;
|
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
|
|
if (optlen < sizeof(entries))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-02 11:01:32 +00:00
|
|
|
q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
|
2018-05-02 11:01:25 +00:00
|
|
|
err = xsk_init_queue(entries, q, false);
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
2018-05-02 11:01:23 +00:00
|
|
|
case XDP_UMEM_REG:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_umem_reg mr;
|
|
|
|
|
struct xdp_umem *umem;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(&mr, optval, sizeof(mr)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 07:35:02 +00:00
|
|
|
if (xs->umem) {
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return -EBUSY;
|
|
|
|
|
}
|
2018-05-02 11:01:23 +00:00
|
|
|
|
2018-05-22 07:35:02 +00:00
|
|
|
umem = xdp_umem_create(&mr);
|
|
|
|
|
if (IS_ERR(umem)) {
|
2018-05-02 11:01:23 +00:00
|
|
|
mutex_unlock(&xs->mutex);
|
2018-05-22 07:35:02 +00:00
|
|
|
return PTR_ERR(umem);
|
2018-05-02 11:01:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Make sure umem is ready before it can be seen by others */
|
|
|
|
|
smp_wmb();
|
|
|
|
|
xs->umem = umem;
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 11:01:24 +00:00
|
|
|
case XDP_UMEM_FILL_RING:
|
2018-05-02 11:01:31 +00:00
|
|
|
case XDP_UMEM_COMPLETION_RING:
|
2018-05-02 11:01:24 +00:00
|
|
|
{
|
|
|
|
|
struct xsk_queue **q;
|
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 07:35:02 +00:00
|
|
|
if (!xs->umem) {
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:31 +00:00
|
|
|
q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
|
|
|
|
|
&xs->umem->cq;
|
2018-05-02 11:01:25 +00:00
|
|
|
err = xsk_init_queue(entries, q, true);
|
2018-05-02 11:01:24 +00:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
2018-05-02 11:01:23 +00:00
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:35 +00:00
|
|
|
static int xsk_getsockopt(struct socket *sock, int level, int optname,
|
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
int len;
|
|
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
|
|
if (get_user(len, optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (len < 0)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
switch (optname) {
|
|
|
|
|
case XDP_STATISTICS:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_statistics stats;
|
|
|
|
|
|
|
|
|
|
if (len < sizeof(stats))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
|
stats.rx_dropped = xs->rx_dropped;
|
|
|
|
|
stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
|
|
|
|
|
stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
if (copy_to_user(optval, &stats, sizeof(stats)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (put_user(sizeof(stats), optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 07:34:59 +00:00
|
|
|
case XDP_MMAP_OFFSETS:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_mmap_offsets off;
|
|
|
|
|
|
|
|
|
|
if (len < sizeof(off))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
|
|
|
|
|
off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
|
|
|
|
|
off.rx.desc = offsetof(struct xdp_rxtx_ring, desc);
|
|
|
|
|
off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
|
|
|
|
|
off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
|
|
|
|
|
off.tx.desc = offsetof(struct xdp_rxtx_ring, desc);
|
|
|
|
|
|
|
|
|
|
off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
|
|
|
|
|
off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
|
|
|
|
|
off.fr.desc = offsetof(struct xdp_umem_ring, desc);
|
|
|
|
|
off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
|
|
|
|
|
off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
|
|
|
|
|
off.cr.desc = offsetof(struct xdp_umem_ring, desc);
|
|
|
|
|
|
|
|
|
|
len = sizeof(off);
|
|
|
|
|
if (copy_to_user(optval, &off, len))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (put_user(len, optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 11:01:35 +00:00
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:24 +00:00
|
|
|
static int xsk_mmap(struct file *file, struct socket *sock,
|
|
|
|
|
struct vm_area_struct *vma)
|
|
|
|
|
{
|
2018-06-07 13:37:34 +00:00
|
|
|
loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
|
2018-05-02 11:01:24 +00:00
|
|
|
unsigned long size = vma->vm_end - vma->vm_start;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sock->sk);
|
|
|
|
|
struct xsk_queue *q = NULL;
|
2018-05-22 07:35:01 +00:00
|
|
|
struct xdp_umem *umem;
|
2018-05-02 11:01:24 +00:00
|
|
|
unsigned long pfn;
|
|
|
|
|
struct page *qpg;
|
|
|
|
|
|
2018-05-02 11:01:25 +00:00
|
|
|
if (offset == XDP_PGOFF_RX_RING) {
|
2018-05-22 07:35:01 +00:00
|
|
|
q = READ_ONCE(xs->rx);
|
2018-05-02 11:01:32 +00:00
|
|
|
} else if (offset == XDP_PGOFF_TX_RING) {
|
2018-05-22 07:35:01 +00:00
|
|
|
q = READ_ONCE(xs->tx);
|
2018-05-02 11:01:25 +00:00
|
|
|
} else {
|
2018-05-22 07:35:01 +00:00
|
|
|
umem = READ_ONCE(xs->umem);
|
|
|
|
|
if (!umem)
|
2018-05-02 11:01:25 +00:00
|
|
|
return -EINVAL;
|
2018-05-02 11:01:24 +00:00
|
|
|
|
2018-05-02 11:01:25 +00:00
|
|
|
if (offset == XDP_UMEM_PGOFF_FILL_RING)
|
2018-05-22 07:35:01 +00:00
|
|
|
q = READ_ONCE(umem->fq);
|
2018-05-02 11:01:31 +00:00
|
|
|
else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
|
2018-05-22 07:35:01 +00:00
|
|
|
q = READ_ONCE(umem->cq);
|
2018-05-02 11:01:25 +00:00
|
|
|
}
|
2018-05-02 11:01:24 +00:00
|
|
|
|
|
|
|
|
if (!q)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
qpg = virt_to_head_page(q->ring);
|
|
|
|
|
if (size > (PAGE_SIZE << compound_order(qpg)))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
|
|
|
|
|
return remap_pfn_range(vma, vma->vm_start, pfn,
|
|
|
|
|
size, vma->vm_page_prot);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 11:01:23 +00:00
|
|
|
static struct proto xsk_proto = {
|
|
|
|
|
.name = "XDP",
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
.obj_size = sizeof(struct xdp_sock),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct proto_ops xsk_proto_ops = {
|
2018-05-18 12:00:24 +00:00
|
|
|
.family = PF_XDP,
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
.release = xsk_release,
|
|
|
|
|
.bind = xsk_bind,
|
|
|
|
|
.connect = sock_no_connect,
|
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
|
.accept = sock_no_accept,
|
|
|
|
|
.getname = sock_no_getname,
|
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller:
1) Add Maglev hashing scheduler to IPVS, from Inju Song.
2) Lots of new TC subsystem tests from Roman Mashak.
3) Add TCP zero copy receive and fix delayed acks and autotuning with
SO_RCVLOWAT, from Eric Dumazet.
4) Add XDP_REDIRECT support to mlx5 driver, from Jesper Dangaard
Brouer.
5) Add ttl inherit support to vxlan, from Hangbin Liu.
6) Properly separate ipv6 routes into their logically independant
components. fib6_info for the routing table, and fib6_nh for sets of
nexthops, which thus can be shared. From David Ahern.
7) Add bpf_xdp_adjust_tail helper, which can be used to generate ICMP
messages from XDP programs. From Nikita V. Shirokov.
8) Lots of long overdue cleanups to the r8169 driver, from Heiner
Kallweit.
9) Add BTF ("BPF Type Format"), from Martin KaFai Lau.
10) Add traffic condition monitoring to iwlwifi, from Luca Coelho.
11) Plumb extack down into fib_rules, from Roopa Prabhu.
12) Add Flower classifier offload support to igb, from Vinicius Costa
Gomes.
13) Add UDP GSO support, from Willem de Bruijn.
14) Add documentation for eBPF helpers, from Quentin Monnet.
15) Add TLS tx offload to mlx5, from Ilya Lesokhin.
16) Allow applications to be given the number of bytes available to read
on a socket via a control message returned from recvmsg(), from
Soheil Hassas Yeganeh.
17) Add x86_32 eBPF JIT compiler, from Wang YanQing.
18) Add AF_XDP sockets, with zerocopy support infrastructure as well.
From Björn Töpel.
19) Remove indirect load support from all of the BPF JITs and handle
these operations in the verifier by translating them into native BPF
instead. From Daniel Borkmann.
20) Add GRO support to ipv6 gre tunnels, from Eran Ben Elisha.
21) Allow XDP programs to do lookups in the main kernel routing tables
for forwarding. From David Ahern.
22) Allow drivers to store hardware state into an ELF section of kernel
dump vmcore files, and use it in cxgb4. From Rahul Lakkireddy.
23) Various RACK and loss detection improvements in TCP, from Yuchung
Cheng.
24) Add TCP SACK compression, from Eric Dumazet.
25) Add User Mode Helper support and basic bpfilter infrastructure, from
Alexei Starovoitov.
26) Support ports and protocol values in RTM_GETROUTE, from Roopa
Prabhu.
27) Support bulking in ->ndo_xdp_xmit() API, from Jesper Dangaard
Brouer.
28) Add lots of forwarding selftests, from Petr Machata.
29) Add generic network device failover driver, from Sridhar Samudrala.
* ra.kernel.org:/pub/scm/linux/kernel/git/davem/net-next: (1959 commits)
strparser: Add __strp_unpause and use it in ktls.
rxrpc: Fix terminal retransmission connection ID to include the channel
net: hns3: Optimize PF CMDQ interrupt switching process
net: hns3: Fix for VF mailbox receiving unknown message
net: hns3: Fix for VF mailbox cannot receiving PF response
bnx2x: use the right constant
Revert "net: sched: cls: Fix offloading when ingress dev is vxlan"
net: dsa: b53: Fix for brcm tag issue in Cygnus SoC
enic: fix UDP rss bits
netdev-FAQ: clarify DaveM's position for stable backports
rtnetlink: validate attributes in do_setlink()
mlxsw: Add extack messages for port_{un, }split failures
netdevsim: Add extack error message for devlink reload
devlink: Add extack to reload and port_{un, }split operations
net: metrics: add proper netlink validation
ipmr: fix error path when ipmr_new_table fails
ip6mr: only set ip6mr_table from setsockopt when ip6mr_new_table succeeds
net: hns3: remove unused hclgevf_cfg_func_mta_filter
netfilter: provide udp*_lib_lookup for nf_tproxy
qed*: Utilize FW 8.37.2.0
...
2018-06-07 01:39:49 +00:00
|
|
|
.poll_mask = xsk_poll_mask,
|
2018-05-18 12:00:24 +00:00
|
|
|
.ioctl = sock_no_ioctl,
|
|
|
|
|
.listen = sock_no_listen,
|
|
|
|
|
.shutdown = sock_no_shutdown,
|
|
|
|
|
.setsockopt = xsk_setsockopt,
|
|
|
|
|
.getsockopt = xsk_getsockopt,
|
|
|
|
|
.sendmsg = xsk_sendmsg,
|
|
|
|
|
.recvmsg = sock_no_recvmsg,
|
|
|
|
|
.mmap = xsk_mmap,
|
|
|
|
|
.sendpage = sock_no_sendpage,
|
2018-05-02 11:01:23 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void xsk_destruct(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
|
return;
|
|
|
|
|
|
2018-05-02 11:01:25 +00:00
|
|
|
xskq_destroy(xs->rx);
|
2018-05-02 11:01:32 +00:00
|
|
|
xskq_destroy(xs->tx);
|
2018-06-04 12:05:57 +00:00
|
|
|
xdp_del_sk_umem(xs->umem, xs);
|
2018-05-02 11:01:23 +00:00
|
|
|
xdp_put_umem(xs->umem);
|
|
|
|
|
|
|
|
|
|
sk_refcnt_debug_dec(sk);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_create(struct net *net, struct socket *sock, int protocol,
|
|
|
|
|
int kern)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk;
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
if (!ns_capable(net->user_ns, CAP_NET_RAW))
|
|
|
|
|
return -EPERM;
|
|
|
|
|
if (sock->type != SOCK_RAW)
|
|
|
|
|
return -ESOCKTNOSUPPORT;
|
|
|
|
|
|
|
|
|
|
if (protocol)
|
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
|
|
|
|
|
|
sock->state = SS_UNCONNECTED;
|
|
|
|
|
|
|
|
|
|
sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
|
|
|
|
|
if (!sk)
|
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
|
|
sock->ops = &xsk_proto_ops;
|
|
|
|
|
|
|
|
|
|
sock_init_data(sock, sk);
|
|
|
|
|
|
|
|
|
|
sk->sk_family = PF_XDP;
|
|
|
|
|
|
|
|
|
|
sk->sk_destruct = xsk_destruct;
|
|
|
|
|
sk_refcnt_debug_inc(sk);
|
|
|
|
|
|
|
|
|
|
xs = xdp_sk(sk);
|
|
|
|
|
mutex_init(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
local_bh_disable();
|
|
|
|
|
sock_prot_inuse_add(net, &xsk_proto, 1);
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct net_proto_family xsk_family_ops = {
|
|
|
|
|
.family = PF_XDP,
|
|
|
|
|
.create = xsk_create,
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int __init xsk_init(void)
|
|
|
|
|
{
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
err = proto_register(&xsk_proto, 0 /* no slab */);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
err = sock_register(&xsk_family_ops);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out_proto;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
out_proto:
|
|
|
|
|
proto_unregister(&xsk_proto);
|
|
|
|
|
out:
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fs_initcall(xsk_init);
|
