Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-24 21:21:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

xsk: use generic DMA sync shortcut instead of a custom one

Browse Source 
XSk infra's been using its own DMA sync shortcut to try avoiding
redundant function calls. Now that there is a generic one, remove
the custom implementation and rely on the generic helpers.
xsk_buff_dma_sync_for_cpu() doesn't need the second argument anymore,
remove it.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
This commit is contained in:
Alexander Lobakin 2024-05-07 13:20:26 +02:00 committed by Christoph Hellwig
parent 4321de4497
commit 163943ac00
13 changed files with 21 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/net/ethernet/engleder/tsnep_main.c
View File

@ -1587,7 +1587,7 @@ static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
length = __le32_to_cpu(entry->desc_wb->properties) & length = __le32_to_cpu(entry->desc_wb->properties) &
TSNEP_DESC_LENGTH_MASK; TSNEP_DESC_LENGTH_MASK;
xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);
xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); xsk_buff_dma_sync_for_cpu(entry->xdp);
/* RX metadata with timestamps is in front of actual data, /* RX metadata with timestamps is in front of actual data,
* subtract metadata size to get length of actual data and * subtract metadata size to get length of actual data and

2
drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c
View File

@ -55,7 +55,7 @@ static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv,
xdp_set_data_meta_invalid(xdp_buff); xdp_set_data_meta_invalid(xdp_buff);
xdp_buff->rxq = &ch->xdp_rxq; xdp_buff->rxq = &ch->xdp_rxq;
xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool); xsk_buff_dma_sync_for_cpu(xdp_buff);
xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff); xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff);
/* xdp.data pointer may have changed */ /* xdp.data pointer may have changed */

2
drivers/net/ethernet/intel/i40e/i40e_xsk.c
View File

@ -483,7 +483,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
bi = *i40e_rx_bi(rx_ring, next_to_process); bi = *i40e_rx_bi(rx_ring, next_to_process);
xsk_buff_set_size(bi, size); xsk_buff_set_size(bi, size);
xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); xsk_buff_dma_sync_for_cpu(bi);
if (!first) if (!first)
first = bi; first = bi;

2
drivers/net/ethernet/intel/ice/ice_xsk.c
View File

@ -879,7 +879,7 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget)
ICE_RX_FLX_DESC_PKT_LEN_M; ICE_RX_FLX_DESC_PKT_LEN_M;
xsk_buff_set_size(xdp, size); xsk_buff_set_size(xdp, size);
xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); xsk_buff_dma_sync_for_cpu(xdp);
if (!first) { if (!first) {
first = xdp; first = xdp;

2
drivers/net/ethernet/intel/igc/igc_main.c
View File

@ -2813,7 +2813,7 @@ static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
} }
bi->xdp->data_end = bi->xdp->data + size; bi->xdp->data_end = bi->xdp->data + size;
xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool); xsk_buff_dma_sync_for_cpu(bi->xdp);
res = __igc_xdp_run_prog(adapter, prog, bi->xdp); res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
switch (res) { switch (res) {

2
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
View File

@ -304,7 +304,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
} }
bi->xdp->data_end = bi->xdp->data + size; bi->xdp->data_end = bi->xdp->data + size;
xsk_buff_dma_sync_for_cpu(bi->xdp, rx_ring->xsk_pool); xsk_buff_dma_sync_for_cpu(bi->xdp);
xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp);
if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) {

4
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
View File

@ -270,7 +270,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
mxbuf->cqe = cqe; mxbuf->cqe = cqe;
xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); xsk_buff_dma_sync_for_cpu(&mxbuf->xdp);
net_prefetch(mxbuf->xdp.data); net_prefetch(mxbuf->xdp.data);
/* Possible flows: /* Possible flows:
@ -319,7 +319,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
mxbuf->cqe = cqe; mxbuf->cqe = cqe;
xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); xsk_buff_dma_sync_for_cpu(&mxbuf->xdp);
net_prefetch(mxbuf->xdp.data); net_prefetch(mxbuf->xdp.data);
prog = rcu_dereference(rq->xdp_prog); prog = rcu_dereference(rq->xdp_prog);

2
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
View File

@ -917,7 +917,7 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
if (!rq->xsk_pool) { if (!rq->xsk_pool) {
count = mlx5e_refill_rx_wqes(rq, head, wqe_bulk); count = mlx5e_refill_rx_wqes(rq, head, wqe_bulk);
} else if (likely(!rq->xsk_pool->dma_need_sync)) { } else if (likely(!dma_dev_need_sync(rq->pdev))) {
mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk); mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk);
count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk); count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk);
} else { } else {

2
drivers/net/ethernet/netronome/nfp/nfd3/xsk.c
View File

@ -184,7 +184,7 @@ nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget,
xrxbuf->xdp->data += meta_len; xrxbuf->xdp->data += meta_len;
xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len;
xdp_set_data_meta_invalid(xrxbuf->xdp); xdp_set_data_meta_invalid(xrxbuf->xdp);
xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool); xsk_buff_dma_sync_for_cpu(xrxbuf->xdp);
net_prefetch(xrxbuf->xdp->data); net_prefetch(xrxbuf->xdp->data);
if (meta_len) { if (meta_len) {

2
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
View File

@ -5352,7 +5352,7 @@ read_again:
/* RX buffer is good and fit into a XSK pool buffer */ /* RX buffer is good and fit into a XSK pool buffer */
buf->xdp->data_end = buf->xdp->data + buf1_len; buf->xdp->data_end = buf->xdp->data + buf1_len;
xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); xsk_buff_dma_sync_for_cpu(buf->xdp);
prog = READ_ONCE(priv->xdp_prog); prog = READ_ONCE(priv->xdp_prog);
res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);

7
include/net/xdp_sock_drv.h
View File

@ -219,13 +219,10 @@ static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool
return meta; return meta;
} }
static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
{ {
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
if (!pool->dma_need_sync)
return;
xp_dma_sync_for_cpu(xskb); xp_dma_sync_for_cpu(xskb);
} }
@ -402,7 +399,7 @@ static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool
return NULL; return NULL;
} }
static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
{ {
} }

14
include/net/xsk_buff_pool.h
View File

@ -43,7 +43,6 @@ struct xsk_dma_map {
refcount_t users; refcount_t users;
struct list_head list; /* Protected by the RTNL_LOCK */ struct list_head list; /* Protected by the RTNL_LOCK */
u32 dma_pages_cnt; u32 dma_pages_cnt;
bool dma_need_sync;
}; };
struct xsk_buff_pool { struct xsk_buff_pool {
@ -82,7 +81,6 @@ struct xsk_buff_pool {
u8 tx_metadata_len; /* inherited from umem */ u8 tx_metadata_len; /* inherited from umem */
u8 cached_need_wakeup; u8 cached_need_wakeup;
bool uses_need_wakeup; bool uses_need_wakeup;
bool dma_need_sync;
bool unaligned; bool unaligned;
bool tx_sw_csum; bool tx_sw_csum;
void *addrs; void *addrs;
@ -155,21 +153,17 @@ static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb)
return xskb->frame_dma; return xskb->frame_dma;
} }
void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb);
static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
{ {
xp_dma_sync_for_cpu_slow(xskb); dma_sync_single_for_cpu(xskb->pool->dev, xskb->dma,
xskb->pool->frame_len,
DMA_BIDIRECTIONAL);
} }
void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
size_t size);
static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool, static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,
dma_addr_t dma, size_t size) dma_addr_t dma, size_t size)
{ {
if (!pool->dma_need_sync) dma_sync_single_for_device(pool->dev, dma, size, DMA_BIDIRECTIONAL);
return;
xp_dma_sync_for_device_slow(pool, dma, size);
} }
/* Masks for xdp_umem_page flags. /* Masks for xdp_umem_page flags.

29
net/xdp/xsk_buff_pool.c
View File

@ -338,7 +338,6 @@ static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_devi
dma_map->netdev = netdev; dma_map->netdev = netdev;
dma_map->dev = dev; dma_map->dev = dev;
dma_map->dma_need_sync = false;
dma_map->dma_pages_cnt = nr_pages; dma_map->dma_pages_cnt = nr_pages;
refcount_set(&dma_map->users, 1); refcount_set(&dma_map->users, 1);
list_add(&dma_map->list, &umem->xsk_dma_list); list_add(&dma_map->list, &umem->xsk_dma_list);
@ -424,7 +423,6 @@ static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_
pool->dev = dma_map->dev; pool->dev = dma_map->dev;
pool->dma_pages_cnt = dma_map->dma_pages_cnt; pool->dma_pages_cnt = dma_map->dma_pages_cnt;
pool->dma_need_sync = dma_map->dma_need_sync;
memcpy(pool->dma_pages, dma_map->dma_pages, memcpy(pool->dma_pages, dma_map->dma_pages,
pool->dma_pages_cnt * sizeof(*pool->dma_pages)); pool->dma_pages_cnt * sizeof(*pool->dma_pages));
@ -460,8 +458,6 @@ int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
__xp_dma_unmap(dma_map, attrs); __xp_dma_unmap(dma_map, attrs);
return -ENOMEM; return -ENOMEM;
} }
if (dma_need_sync(dev, dma))
dma_map->dma_need_sync = true;
dma_map->dma_pages[i] = dma; dma_map->dma_pages[i] = dma;
} }
@ -557,11 +553,9 @@ struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)
xskb->xdp.data_meta = xskb->xdp.data; xskb->xdp.data_meta = xskb->xdp.data;
xskb->xdp.flags = 0; xskb->xdp.flags = 0;
if (pool->dma_need_sync) { if (pool->dev)
dma_sync_single_range_for_device(pool->dev, xskb->dma, 0, xp_dma_sync_for_device(pool, xskb->dma, pool->frame_len);
pool->frame_len,
DMA_BIDIRECTIONAL);
}
return &xskb->xdp; return &xskb->xdp;
} }
EXPORT_SYMBOL(xp_alloc); EXPORT_SYMBOL(xp_alloc);
@ -633,7 +627,7 @@ u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
{ {
u32 nb_entries1 = 0, nb_entries2; u32 nb_entries1 = 0, nb_entries2;
if (unlikely(pool->dma_need_sync)) { if (unlikely(pool->dev && dma_dev_need_sync(pool->dev))) {
struct xdp_buff *buff; struct xdp_buff *buff;
/* Slow path */ /* Slow path */
@ -693,18 +687,3 @@ dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)
(addr & ~PAGE_MASK); (addr & ~PAGE_MASK);
} }
EXPORT_SYMBOL(xp_raw_get_dma); EXPORT_SYMBOL(xp_raw_get_dma);
void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb)
{
dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,
xskb->pool->frame_len, DMA_BIDIRECTIONAL);
}
EXPORT_SYMBOL(xp_dma_sync_for_cpu_slow);
void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
size_t size)
{
dma_sync_single_range_for_device(pool->dev, dma, 0,
size, DMA_BIDIRECTIONAL);
}
EXPORT_SYMBOL(xp_dma_sync_for_device_slow);