sfc: Add support for busy polling

This patch adds the sfc driver code for implementing busy polling.
It adds ndo_busy_poll method and locking between it and napi poll.
It also adds each napi to the napi_hash right after netif_napi_add().

Uses efx_start_eventq and efx_stop_eventq in the self tests.

Signed-off-by: Shradha Shah <sshah@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Alexandre Rames 2014-07-22 14:03:25 +01:00 committed by David S. Miller
parent 2dbe82d0d1
commit 36763266bb
5 changed files with 208 additions and 7 deletions

View File

@ -272,6 +272,9 @@ static int efx_poll(struct napi_struct *napi, int budget)
struct efx_nic *efx = channel->efx;
int spent;
if (!efx_channel_lock_napi(channel))
return budget;
netif_vdbg(efx, intr, efx->net_dev,
"channel %d NAPI poll executing on CPU %d\n",
channel->channel, raw_smp_processor_id());
@ -311,6 +314,7 @@ static int efx_poll(struct napi_struct *napi, int budget)
efx_nic_eventq_read_ack(channel);
}
efx_channel_unlock_napi(channel);
return spent;
}
@ -357,7 +361,7 @@ static int efx_init_eventq(struct efx_channel *channel)
}
/* Enable event queue processing and NAPI */
static void efx_start_eventq(struct efx_channel *channel)
void efx_start_eventq(struct efx_channel *channel)
{
netif_dbg(channel->efx, ifup, channel->efx->net_dev,
"chan %d start event queue\n", channel->channel);
@ -366,17 +370,20 @@ static void efx_start_eventq(struct efx_channel *channel)
channel->enabled = true;
smp_wmb();
efx_channel_enable(channel);
napi_enable(&channel->napi_str);
efx_nic_eventq_read_ack(channel);
}
/* Disable event queue processing and NAPI */
static void efx_stop_eventq(struct efx_channel *channel)
void efx_stop_eventq(struct efx_channel *channel)
{
if (!channel->enabled)
return;
napi_disable(&channel->napi_str);
while (!efx_channel_disable(channel))
usleep_range(1000, 20000);
channel->enabled = false;
}
@ -1960,6 +1967,8 @@ static void efx_init_napi_channel(struct efx_channel *channel)
channel->napi_dev = efx->net_dev;
netif_napi_add(channel->napi_dev, &channel->napi_str,
efx_poll, napi_weight);
napi_hash_add(&channel->napi_str);
efx_channel_init_lock(channel);
}
static void efx_init_napi(struct efx_nic *efx)
@ -1972,8 +1981,10 @@ static void efx_init_napi(struct efx_nic *efx)
static void efx_fini_napi_channel(struct efx_channel *channel)
{
if (channel->napi_dev)
if (channel->napi_dev) {
netif_napi_del(&channel->napi_str);
napi_hash_del(&channel->napi_str);
}
channel->napi_dev = NULL;
}
@ -2008,6 +2019,37 @@ static void efx_netpoll(struct net_device *net_dev)
#endif
#ifdef CONFIG_NET_RX_BUSY_POLL
static int efx_busy_poll(struct napi_struct *napi)
{
struct efx_channel *channel =
container_of(napi, struct efx_channel, napi_str);
struct efx_nic *efx = channel->efx;
int budget = 4;
int old_rx_packets, rx_packets;
if (!netif_running(efx->net_dev))
return LL_FLUSH_FAILED;
if (!efx_channel_lock_poll(channel))
return LL_FLUSH_BUSY;
old_rx_packets = channel->rx_queue.rx_packets;
efx_process_channel(channel, budget);
rx_packets = channel->rx_queue.rx_packets - old_rx_packets;
/* There is no race condition with NAPI here.
* NAPI will automatically be rescheduled if it yielded during busy
* polling, because it was not able to take the lock and thus returned
* the full budget.
*/
efx_channel_unlock_poll(channel);
return rx_packets;
}
#endif
/**************************************************************************
*
* Kernel net device interface
@ -2177,6 +2219,9 @@ static const struct net_device_ops efx_farch_netdev_ops = {
.ndo_poll_controller = efx_netpoll,
#endif
.ndo_setup_tc = efx_setup_tc,
#ifdef CONFIG_NET_RX_BUSY_POLL
.ndo_busy_poll = efx_busy_poll,
#endif
#ifdef CONFIG_RFS_ACCEL
.ndo_rx_flow_steer = efx_filter_rfs,
#endif
@ -2197,6 +2242,9 @@ static const struct net_device_ops efx_ef10_netdev_ops = {
#ifdef CONFIG_NET_POLL_CONTROLLER
.ndo_poll_controller = efx_netpoll,
#endif
#ifdef CONFIG_NET_RX_BUSY_POLL
.ndo_busy_poll = efx_busy_poll,
#endif
#ifdef CONFIG_RFS_ACCEL
.ndo_rx_flow_steer = efx_filter_rfs,
#endif

View File

@ -194,6 +194,8 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
bool rx_may_override_tx);
void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
unsigned int *rx_usecs, bool *rx_adaptive);
void efx_stop_eventq(struct efx_channel *channel);
void efx_start_eventq(struct efx_channel *channel);
/* Dummy PHY ops for PHY drivers */
int efx_port_dummy_op_int(struct efx_nic *efx);

View File

@ -28,6 +28,7 @@
#include <linux/vmalloc.h>
#include <linux/i2c.h>
#include <linux/mtd/mtd.h>
#include <net/busy_poll.h>
#include "enum.h"
#include "bitfield.h"
@ -387,6 +388,8 @@ enum efx_sync_events_state {
* @irq_moderation: IRQ moderation value (in hardware ticks)
* @napi_dev: Net device used with NAPI
* @napi_str: NAPI control structure
* @state: state for NAPI vs busy polling
* @state_lock: lock protecting @state
* @eventq: Event queue buffer
* @eventq_mask: Event queue pointer mask
* @eventq_read_ptr: Event queue read pointer
@ -424,6 +427,22 @@ struct efx_channel {
unsigned int irq_moderation;
struct net_device *napi_dev;
struct napi_struct napi_str;
#ifdef CONFIG_NET_RX_BUSY_POLL
unsigned int state;
spinlock_t state_lock;
#define EFX_CHANNEL_STATE_IDLE 0
#define EFX_CHANNEL_STATE_NAPI (1 << 0) /* NAPI owns this channel */
#define EFX_CHANNEL_STATE_POLL (1 << 1) /* poll owns this channel */
#define EFX_CHANNEL_STATE_DISABLED (1 << 2) /* channel is disabled */
#define EFX_CHANNEL_STATE_NAPI_YIELD (1 << 3) /* NAPI yielded this channel */
#define EFX_CHANNEL_STATE_POLL_YIELD (1 << 4) /* poll yielded this channel */
#define EFX_CHANNEL_OWNED \
(EFX_CHANNEL_STATE_NAPI | EFX_CHANNEL_STATE_POLL)
#define EFX_CHANNEL_LOCKED \
(EFX_CHANNEL_OWNED | EFX_CHANNEL_STATE_DISABLED)
#define EFX_CHANNEL_USER_PEND \
(EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_POLL_YIELD)
#endif /* CONFIG_NET_RX_BUSY_POLL */
struct efx_special_buffer eventq;
unsigned int eventq_mask;
unsigned int eventq_read_ptr;
@ -457,6 +476,135 @@ struct efx_channel {
u32 sync_timestamp_minor;
};
#ifdef CONFIG_NET_RX_BUSY_POLL
static inline void efx_channel_init_lock(struct efx_channel *channel)
{
spin_lock_init(&channel->state_lock);
}
/* Called from the device poll routine to get ownership of a channel. */
static inline bool efx_channel_lock_napi(struct efx_channel *channel)
{
bool rc = true;
spin_lock_bh(&channel->state_lock);
if (channel->state & EFX_CHANNEL_LOCKED) {
WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI);
channel->state |= EFX_CHANNEL_STATE_NAPI_YIELD;
rc = false;
} else {
/* we don't care if someone yielded */
channel->state = EFX_CHANNEL_STATE_NAPI;
}
spin_unlock_bh(&channel->state_lock);
return rc;
}
static inline void efx_channel_unlock_napi(struct efx_channel *channel)
{
spin_lock_bh(&channel->state_lock);
WARN_ON(channel->state &
(EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_NAPI_YIELD));
channel->state &= EFX_CHANNEL_STATE_DISABLED;
spin_unlock_bh(&channel->state_lock);
}
/* Called from efx_busy_poll(). */
static inline bool efx_channel_lock_poll(struct efx_channel *channel)
{
bool rc = true;
spin_lock_bh(&channel->state_lock);
if ((channel->state & EFX_CHANNEL_LOCKED)) {
channel->state |= EFX_CHANNEL_STATE_POLL_YIELD;
rc = false;
} else {
/* preserve yield marks */
channel->state |= EFX_CHANNEL_STATE_POLL;
}
spin_unlock_bh(&channel->state_lock);
return rc;
}
/* Returns true if NAPI tried to get the channel while it was locked. */
static inline void efx_channel_unlock_poll(struct efx_channel *channel)
{
spin_lock_bh(&channel->state_lock);
WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI);
/* will reset state to idle, unless channel is disabled */
channel->state &= EFX_CHANNEL_STATE_DISABLED;
spin_unlock_bh(&channel->state_lock);
}
/* True if a socket is polling, even if it did not get the lock. */
static inline bool efx_channel_busy_polling(struct efx_channel *channel)
{
WARN_ON(!(channel->state & EFX_CHANNEL_OWNED));
return channel->state & EFX_CHANNEL_USER_PEND;
}
static inline void efx_channel_enable(struct efx_channel *channel)
{
spin_lock_bh(&channel->state_lock);
channel->state = EFX_CHANNEL_STATE_IDLE;
spin_unlock_bh(&channel->state_lock);
}
/* False if the channel is currently owned. */
static inline bool efx_channel_disable(struct efx_channel *channel)
{
bool rc = true;
spin_lock_bh(&channel->state_lock);
if (channel->state & EFX_CHANNEL_OWNED)
rc = false;
channel->state |= EFX_CHANNEL_STATE_DISABLED;
spin_unlock_bh(&channel->state_lock);
return rc;
}
#else /* CONFIG_NET_RX_BUSY_POLL */
static inline void efx_channel_init_lock(struct efx_channel *channel)
{
}
static inline bool efx_channel_lock_napi(struct efx_channel *channel)
{
return true;
}
static inline void efx_channel_unlock_napi(struct efx_channel *channel)
{
}
static inline bool efx_channel_lock_poll(struct efx_channel *channel)
{
return false;
}
static inline void efx_channel_unlock_poll(struct efx_channel *channel)
{
}
static inline bool efx_channel_busy_polling(struct efx_channel *channel)
{
return false;
}
static inline void efx_channel_enable(struct efx_channel *channel)
{
}
static inline bool efx_channel_disable(struct efx_channel *channel)
{
return true;
}
#endif /* CONFIG_NET_RX_BUSY_POLL */
/**
* struct efx_msi_context - Context for each MSI
* @efx: The associated NIC

View File

@ -462,6 +462,7 @@ efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
skb_record_rx_queue(skb, channel->rx_queue.core_index);
skb_mark_napi_id(skb, &channel->napi_str);
gro_result = napi_gro_frags(napi);
if (gro_result != GRO_DROP)
channel->irq_mod_score += 2;
@ -520,6 +521,8 @@ static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
/* Move past the ethernet header */
skb->protocol = eth_type_trans(skb, efx->net_dev);
skb_mark_napi_id(skb, &channel->napi_str);
return skb;
}
@ -666,7 +669,8 @@ void __efx_rx_packet(struct efx_channel *channel)
if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;
if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb)
if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb &&
!efx_channel_busy_polling(channel))
efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh);
else
efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);

View File

@ -188,7 +188,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx,
schedule_timeout_uninterruptible(wait);
efx_for_each_channel(channel, efx) {
napi_disable(&channel->napi_str);
efx_stop_eventq(channel);
if (channel->eventq_read_ptr !=
read_ptr[channel->channel]) {
set_bit(channel->channel, &napi_ran);
@ -200,8 +200,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx,
if (efx_nic_event_test_irq_cpu(channel) >= 0)
clear_bit(channel->channel, &int_pend);
}
napi_enable(&channel->napi_str);
efx_nic_eventq_read_ack(channel);
efx_start_eventq(channel);
}
wait *= 2;