Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-22 12:11:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

net/mlx5e: xsk: Improve need_wakeup logic

Browse Source 
XSK need_wakeup mechanism allows the driver to stop busy waiting for
buffers when the fill ring is empty, yield to the application and signal
it that the driver needs to be waken up after the application refills
the fill ring.

Add protection against the race condition on the RX (refill) side: if
the application refills buffers after xskrq->post_wqes is called, but
before mlx5e_xsk_update_rx_wakeup, NAPI will exit, skipping taking these
buffers to the hardware WQ, and the application won't wake it up again.

Optimize the whole need_wakeup logic, removing unneeded flows, to
compensate for this new check.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Maxim Mikityanskiy 2022-10-01 21:56:21 -07:00 committed by Jakub Kicinski
parent 1ca6492ec9
commit cfb4d09c30
4 changed files with 23 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
View File

@ -5,7 +5,6 @@
#define __MLX5_EN_XSK_RX_H__
#include "en.h"
#include <net/xdp_sock_drv.h>
/* RX data path */
@ -21,17 +20,4 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *wi,
u32 cqe_bcnt);
static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err)
{
if (!xsk_uses_need_wakeup(rq->xsk_pool))
return alloc_err;
if (unlikely(alloc_err))
xsk_set_rx_need_wakeup(rq->xsk_pool);
else
xsk_clear_rx_need_wakeup(rq->xsk_pool);
return false;
}
#endif /* __MLX5_EN_XSK_RX_H__ */

12
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
View File

@ -5,7 +5,6 @@
#define __MLX5_EN_XSK_TX_H__
#include "en.h"
#include <net/xdp_sock_drv.h>
/* TX data path */
@ -13,15 +12,4 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget);
static inline void mlx5e_xsk_update_tx_wakeup(struct mlx5e_xdpsq *sq)
{
if (!xsk_uses_need_wakeup(sq->xsk_pool))
return;
if (sq->pc != sq->cc)
xsk_clear_tx_need_wakeup(sq->xsk_pool);
else
xsk_set_tx_need_wakeup(sq->xsk_pool);
}
#endif /* __MLX5_EN_XSK_TX_H__ */

1
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
View File

@ -41,6 +41,7 @@
#include <net/gro.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <net/xdp_sock_drv.h>
#include "en.h"
#include "en/txrx.h"
#include "en_tc.h"

33
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
View File

@ -31,6 +31,7 @@
*/
#include <linux/irq.h>
#include <net/xdp_sock_drv.h>
#include "en.h"
#include "en/txrx.h"
#include "en/xdp.h"
@ -86,26 +87,36 @@ void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
static bool mlx5e_napi_xsk_post(struct mlx5e_xdpsq *xsksq, struct mlx5e_rq *xskrq)
{
bool need_wakeup = xsk_uses_need_wakeup(xskrq->xsk_pool);
bool busy_xsk = false, xsk_rx_alloc_err;
/* Handle the race between the application querying need_wakeup and the
* driver setting it:
* 1. Update need_wakeup both before and after the TX. If it goes to
* "yes", it can only happen with the first update.
* 2. If the application queried need_wakeup before we set it, the
* packets will be transmitted anyway, even w/o a wakeup.
* 3. Give a chance to clear need_wakeup after new packets were queued
* for TX.
/* If SQ is empty, there are no TX completions to trigger NAPI, so set
* need_wakeup. Do it before queuing packets for TX to avoid race
* condition with userspace.
*/
mlx5e_xsk_update_tx_wakeup(xsksq);
if (need_wakeup && xsksq->pc == xsksq->cc)
xsk_set_tx_need_wakeup(xsksq->xsk_pool);
busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET);
mlx5e_xsk_update_tx_wakeup(xsksq);
/* If we queued some packets for TX, no need for wakeup anymore. */
if (need_wakeup && xsksq->pc != xsksq->cc)
xsk_clear_tx_need_wakeup(xsksq->xsk_pool);
/* If WQ is empty, RX won't trigger NAPI, so set need_wakeup. Do it
* before refilling to avoid race condition with userspace.
*/
if (need_wakeup && !mlx5e_rqwq_get_cur_sz(xskrq))
xsk_set_rx_need_wakeup(xskrq->xsk_pool);
xsk_rx_alloc_err = INDIRECT_CALL_2(xskrq->post_wqes,
mlx5e_post_rx_mpwqes,
mlx5e_post_rx_wqes,
xskrq);
busy_xsk |= mlx5e_xsk_update_rx_wakeup(xskrq, xsk_rx_alloc_err);
/* Ask for wakeup if WQ is not full after refill. */
if (!need_wakeup)
busy_xsk |= xsk_rx_alloc_err;
else if (xsk_rx_alloc_err)
xsk_set_rx_need_wakeup(xskrq->xsk_pool);
else
xsk_clear_rx_need_wakeup(xskrq->xsk_pool);
return busy_xsk;
}