2018-05-18 12:00:21 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* XDP user-space ring structure
|
2018-05-02 11:01:24 +00:00
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_XSK_QUEUE_H
|
|
|
|
#define _LINUX_XSK_QUEUE_H
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/if_xdp.h>
|
|
|
|
|
|
|
|
#include "xdp_umem_props.h"
|
|
|
|
|
2018-05-02 11:01:27 +00:00
|
|
|
#define RX_BATCH_SIZE 16
|
|
|
|
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 07:34:59 +00:00
|
|
|
struct xdp_ring {
|
|
|
|
u32 producer ____cacheline_aligned_in_smp;
|
|
|
|
u32 consumer ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the RX and TX queues for packets */
|
|
|
|
struct xdp_rxtx_ring {
|
|
|
|
struct xdp_ring ptrs;
|
|
|
|
struct xdp_desc desc[0] ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the fill and completion queues for buffers */
|
|
|
|
struct xdp_umem_ring {
|
|
|
|
struct xdp_ring ptrs;
|
|
|
|
u32 desc[0] ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
2018-05-02 11:01:24 +00:00
|
|
|
struct xsk_queue {
|
|
|
|
struct xdp_umem_props umem_props;
|
|
|
|
u32 ring_mask;
|
|
|
|
u32 nentries;
|
|
|
|
u32 prod_head;
|
|
|
|
u32 prod_tail;
|
|
|
|
u32 cons_head;
|
|
|
|
u32 cons_tail;
|
|
|
|
struct xdp_ring *ring;
|
|
|
|
u64 invalid_descs;
|
|
|
|
};
|
|
|
|
|
2018-05-02 11:01:27 +00:00
|
|
|
/* Common functions operating for both RXTX and umem queues */
|
|
|
|
|
2018-05-02 11:01:35 +00:00
|
|
|
static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return q ? q->invalid_descs : 0;
|
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:27 +00:00
|
|
|
static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)
|
|
|
|
{
|
|
|
|
u32 entries = q->prod_tail - q->cons_tail;
|
|
|
|
|
|
|
|
if (entries == 0) {
|
|
|
|
/* Refresh the local pointer */
|
|
|
|
q->prod_tail = READ_ONCE(q->ring->producer);
|
|
|
|
entries = q->prod_tail - q->cons_tail;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (entries > dcnt) ? dcnt : entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt)
|
|
|
|
{
|
|
|
|
u32 free_entries = q->nentries - (producer - q->cons_tail);
|
|
|
|
|
|
|
|
if (free_entries >= dcnt)
|
|
|
|
return free_entries;
|
|
|
|
|
|
|
|
/* Refresh the local tail pointer */
|
|
|
|
q->cons_tail = READ_ONCE(q->ring->consumer);
|
|
|
|
return q->nentries - (producer - q->cons_tail);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* UMEM queue */
|
|
|
|
|
|
|
|
static inline bool xskq_is_valid_id(struct xsk_queue *q, u32 idx)
|
|
|
|
{
|
|
|
|
if (unlikely(idx >= q->umem_props.nframes)) {
|
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32 *xskq_validate_id(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
while (q->cons_tail != q->cons_head) {
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
unsigned int idx = q->cons_tail & q->ring_mask;
|
|
|
|
|
|
|
|
if (xskq_is_valid_id(q, ring->desc[idx]))
|
|
|
|
return &ring->desc[idx];
|
|
|
|
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32 *xskq_peek_id(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring;
|
|
|
|
|
|
|
|
if (q->cons_tail == q->cons_head) {
|
|
|
|
WRITE_ONCE(q->ring->consumer, q->cons_tail);
|
|
|
|
q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
|
|
|
|
|
|
|
|
/* Order consumer and data */
|
|
|
|
smp_rmb();
|
|
|
|
|
|
|
|
return xskq_validate_id(q);
|
|
|
|
}
|
|
|
|
|
|
|
|
ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
return &ring->desc[q->cons_tail & q->ring_mask];
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_discard_id(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
q->cons_tail++;
|
|
|
|
(void)xskq_validate_id(q);
|
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:34 +00:00
|
|
|
static inline int xskq_produce_id(struct xsk_queue *q, u32 id)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
|
|
|
ring->desc[q->prod_tail++ & q->ring_mask] = id;
|
|
|
|
|
|
|
|
/* Order producer and data */
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
WRITE_ONCE(q->ring->producer, q->prod_tail);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int xskq_reserve_id(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
if (xskq_nb_free(q, q->prod_head, 1) == 0)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
q->prod_head++;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Rx/Tx queue */
|
|
|
|
|
|
|
|
static inline bool xskq_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d)
|
|
|
|
{
|
|
|
|
u32 buff_len;
|
|
|
|
|
|
|
|
if (unlikely(d->idx >= q->umem_props.nframes)) {
|
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
buff_len = q->umem_props.frame_size;
|
|
|
|
if (unlikely(d->len > buff_len || d->len == 0 ||
|
|
|
|
d->offset > buff_len || d->offset + d->len > buff_len)) {
|
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct xdp_desc *xskq_validate_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
while (q->cons_tail != q->cons_head) {
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
unsigned int idx = q->cons_tail & q->ring_mask;
|
|
|
|
|
|
|
|
if (xskq_is_valid_desc(q, &ring->desc[idx])) {
|
|
|
|
if (desc)
|
|
|
|
*desc = ring->desc[idx];
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct xdp_desc *xskq_peek_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
struct xdp_rxtx_ring *ring;
|
|
|
|
|
|
|
|
if (q->cons_tail == q->cons_head) {
|
|
|
|
WRITE_ONCE(q->ring->consumer, q->cons_tail);
|
|
|
|
q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
|
|
|
|
|
|
|
|
/* Order consumer and data */
|
|
|
|
smp_rmb();
|
|
|
|
|
|
|
|
return xskq_validate_desc(q, desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
*desc = ring->desc[q->cons_tail & q->ring_mask];
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_discard_desc(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
q->cons_tail++;
|
|
|
|
(void)xskq_validate_desc(q, NULL);
|
|
|
|
}
|
2018-05-02 11:01:27 +00:00
|
|
|
|
|
|
|
static inline int xskq_produce_batch_desc(struct xsk_queue *q,
|
|
|
|
u32 id, u32 len, u16 offset)
|
|
|
|
{
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
unsigned int idx;
|
|
|
|
|
|
|
|
if (xskq_nb_free(q, q->prod_head, 1) == 0)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
idx = (q->prod_head++) & q->ring_mask;
|
|
|
|
ring->desc[idx].idx = id;
|
|
|
|
ring->desc[idx].len = len;
|
|
|
|
ring->desc[idx].offset = offset;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_produce_flush_desc(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* Order producer and data */
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
q->prod_tail = q->prod_head,
|
|
|
|
WRITE_ONCE(q->ring->producer, q->prod_tail);
|
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:34 +00:00
|
|
|
static inline bool xskq_full_desc(struct xsk_queue *q)
|
|
|
|
{
|
2018-05-18 12:00:23 +00:00
|
|
|
return xskq_nb_avail(q, q->nentries) == q->nentries;
|
2018-05-02 11:01:34 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:27 +00:00
|
|
|
static inline bool xskq_empty_desc(struct xsk_queue *q)
|
|
|
|
{
|
2018-05-18 12:00:23 +00:00
|
|
|
return xskq_nb_free(q, q->prod_tail, 1) == q->nentries;
|
2018-05-02 11:01:27 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:26 +00:00
|
|
|
void xskq_set_umem(struct xsk_queue *q, struct xdp_umem_props *umem_props);
|
2018-05-02 11:01:25 +00:00
|
|
|
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
|
2018-05-02 11:01:27 +00:00
|
|
|
void xskq_destroy(struct xsk_queue *q_ops);
|
2018-05-02 11:01:24 +00:00
|
|
|
|
|
|
|
#endif /* _LINUX_XSK_QUEUE_H */
|