Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-27 13:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'net-ena-implement-adaptive-interrupt-moderation-using-dim'

Browse Source 
Arthur Kiyanovski says:

====================
net: ena: implement adaptive interrupt moderation using dim

In this patchset we replace our adaptive interrupt moderation
implementation with the dim library implementation.
The dim library showed great improvement in throughput, latency
and CPU usage in different scenarios on ARM CPUs.
This patchset also includes a few bug fixes to the parts of the
old implementation of adaptive interrupt moderation that were left.

Changes from V1 patchset:
Removed stray empty lines from patches 01/11, 09/11.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2019-09-16 22:06:03 +02:00
commit b63e1a02d7
6 changed files with 127 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
drivers/net/ethernet/amazon/ena/ena_admin_defs.h
View File

@ -808,6 +808,12 @@ struct ena_admin_host_info {
u16 num_cpus;
u16 reserved;
/* 1 :0 : reserved
* 2 : interrupt_moderation
* 31:3 : reserved
*/
u32 driver_supported_features;
};
struct ena_admin_rss_ind_table_entry {
@ -1110,6 +1116,8 @@ struct ena_admin_ena_mmio_req_read_less_resp {
#define ENA_ADMIN_HOST_INFO_DEVICE_MASK GENMASK(7, 3)
#define ENA_ADMIN_HOST_INFO_BUS_SHIFT 8
#define ENA_ADMIN_HOST_INFO_BUS_MASK GENMASK(15, 8)
#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT 2
#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK BIT(2)
/* aenq_common_desc */
#define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0)

177
drivers/net/ethernet/amazon/ena/ena_com.c
View File

@ -1278,40 +1278,33 @@ static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
return 0;
}
static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
{
size_t size;
size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
ena_dev->intr_moder_tbl =
devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
if (!ena_dev->intr_moder_tbl)
return -ENOMEM;
ena_com_config_default_interrupt_moderation_table(ena_dev);
return 0;
}
static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
u16 intr_delay_resolution)
{
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
unsigned int i;
/* Initial value of intr_delay_resolution might be 0 */
u16 prev_intr_delay_resolution =
ena_dev->intr_delay_resolution ?
ena_dev->intr_delay_resolution :
ENA_DEFAULT_INTR_DELAY_RESOLUTION;
if (!intr_delay_resolution) {
pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
intr_delay_resolution = 1;
intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
}
ena_dev->intr_delay_resolution = intr_delay_resolution;
/* update Rx */
for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
ena_dev->intr_moder_rx_interval =
ena_dev->intr_moder_rx_interval *
prev_intr_delay_resolution /
intr_delay_resolution;
/* update Tx */
ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
ena_dev->intr_moder_tx_interval =
ena_dev->intr_moder_tx_interval *
prev_intr_delay_resolution /
intr_delay_resolution;
ena_dev->intr_delay_resolution = intr_delay_resolution;
}
/*****************************************************************************/
@ -2776,42 +2769,34 @@ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
ENA_ADMIN_INTERRUPT_MODERATION);
}
int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
u32 tx_coalesce_usecs)
static int ena_com_update_nonadaptive_moderation_interval(u32 coalesce_usecs,
u32 intr_delay_resolution,
u32 *intr_moder_interval)
{
if (!ena_dev->intr_delay_resolution) {
if (!intr_delay_resolution) {
pr_err("Illegal interrupt delay granularity value\n");
return -EFAULT;
}
ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
ena_dev->intr_delay_resolution;
*intr_moder_interval = coalesce_usecs / intr_delay_resolution;
return 0;
}
int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
u32 tx_coalesce_usecs)
{
return ena_com_update_nonadaptive_moderation_interval(tx_coalesce_usecs,
ena_dev->intr_delay_resolution,
&ena_dev->intr_moder_tx_interval);
}
int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
u32 rx_coalesce_usecs)
{
if (!ena_dev->intr_delay_resolution) {
pr_err("Illegal interrupt delay granularity value\n");
return -EFAULT;
}
/* We use LOWEST entry of moderation table for storing
* nonadaptive interrupt coalescing values
*/
ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
rx_coalesce_usecs / ena_dev->intr_delay_resolution;
return 0;
}
void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
{
if (ena_dev->intr_moder_tbl)
devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
ena_dev->intr_moder_tbl = NULL;
return ena_com_update_nonadaptive_moderation_interval(rx_coalesce_usecs,
ena_dev->intr_delay_resolution,
&ena_dev->intr_moder_rx_interval);
}
int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
@ -2838,66 +2823,14 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
return rc;
}
rc = ena_com_init_interrupt_moderation_table(ena_dev);
if (rc)
goto err;
/* if moderation is supported by device we set adaptive moderation */
delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
/* Disable adaptive moderation by default - can be enabled from
* ethtool
*/
/* Disable adaptive moderation by default - can be enabled later */
ena_com_disable_adaptive_moderation(ena_dev);
return 0;
err:
ena_com_destroy_interrupt_moderation(ena_dev);
return rc;
}
void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
{
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
if (!intr_moder_tbl)
return;
intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
ENA_INTR_LOWEST_USECS;
intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
ENA_INTR_LOWEST_PKTS;
intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
ENA_INTR_LOWEST_BYTES;
intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
ENA_INTR_LOW_USECS;
intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
ENA_INTR_LOW_PKTS;
intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
ENA_INTR_LOW_BYTES;
intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
ENA_INTR_MID_USECS;
intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
ENA_INTR_MID_PKTS;
intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
ENA_INTR_MID_BYTES;
intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
ENA_INTR_HIGH_USECS;
intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
ENA_INTR_HIGH_PKTS;
intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
ENA_INTR_HIGH_BYTES;
intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
ENA_INTR_HIGHEST_USECS;
intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
ENA_INTR_HIGHEST_PKTS;
intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
ENA_INTR_HIGHEST_BYTES;
}
unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
@ -2907,49 +2840,7 @@ unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *
unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
{
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
if (intr_moder_tbl)
return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
return 0;
}
void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry)
{
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
return;
intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
if (ena_dev->intr_delay_resolution)
intr_moder_tbl[level].intr_moder_interval /=
ena_dev->intr_delay_resolution;
intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
/* use hardcoded value until ethtool supports bytecount parameter */
if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
}
void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry)
{
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
return;
entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
if (ena_dev->intr_delay_resolution)
entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
entry->pkts_per_interval =
intr_moder_tbl[level].pkts_per_interval;
entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
return ena_dev->intr_moder_rx_interval;
}
int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,

151
drivers/net/ethernet/amazon/ena/ena_com.h
View File

@ -72,46 +72,14 @@
/*****************************************************************************/
/* ENA adaptive interrupt moderation settings */
#define ENA_INTR_LOWEST_USECS (0)
#define ENA_INTR_LOWEST_PKTS (3)
#define ENA_INTR_LOWEST_BYTES (2 * 1524)
#define ENA_INTR_LOW_USECS (32)
#define ENA_INTR_LOW_PKTS (12)
#define ENA_INTR_LOW_BYTES (16 * 1024)
#define ENA_INTR_MID_USECS (80)
#define ENA_INTR_MID_PKTS (48)
#define ENA_INTR_MID_BYTES (64 * 1024)
#define ENA_INTR_HIGH_USECS (128)
#define ENA_INTR_HIGH_PKTS (96)
#define ENA_INTR_HIGH_BYTES (128 * 1024)
#define ENA_INTR_HIGHEST_USECS (192)
#define ENA_INTR_HIGHEST_PKTS (128)
#define ENA_INTR_HIGHEST_BYTES (192 * 1024)
#define ENA_INTR_INITIAL_TX_INTERVAL_USECS 196
#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 4
#define ENA_INTR_DELAY_OLD_VALUE_WEIGHT 6
#define ENA_INTR_DELAY_NEW_VALUE_WEIGHT 4
#define ENA_INTR_MODER_LEVEL_STRIDE 2
#define ENA_INTR_BYTE_COUNT_NOT_SUPPORTED 0xFFFFFF
#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0
#define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1
#define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF
#define ENA_FEATURE_MAX_QUEUE_EXT_VER 1
enum ena_intr_moder_level {
ENA_INTR_MODER_LOWEST = 0,
ENA_INTR_MODER_LOW,
ENA_INTR_MODER_MID,
ENA_INTR_MODER_HIGH,
ENA_INTR_MODER_HIGHEST,
ENA_INTR_MAX_NUM_OF_LEVELS,
};
struct ena_llq_configurations {
enum ena_admin_llq_header_location llq_header_location;
enum ena_admin_llq_ring_entry_size llq_ring_entry_size;
@ -120,12 +88,6 @@ struct ena_llq_configurations {
u16 llq_ring_entry_size_value;
};
struct ena_intr_moder_entry {
unsigned int intr_moder_interval;
unsigned int pkts_per_interval;
unsigned int bytes_per_interval;
};
enum queue_direction {
ENA_COM_IO_QUEUE_DIRECTION_TX,
ENA_COM_IO_QUEUE_DIRECTION_RX
@ -376,7 +338,13 @@ struct ena_com_dev {
struct ena_host_attribute host_attr;
bool adaptive_coalescing;
u16 intr_delay_resolution;
/* interrupt moderation intervals are in usec divided by
* intr_delay_resolution, which is supplied by the device.
*/
u32 intr_moder_tx_interval;
u32 intr_moder_rx_interval;
struct ena_intr_moder_entry *intr_moder_tbl;
struct ena_com_llq_info llq_info;
@ -914,11 +882,6 @@ int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
*/
int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev);
/* ena_com_destroy_interrupt_moderation - Destroy interrupt moderation resources
* @ena_dev: ENA communication layer struct
*/
void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev);
/* ena_com_interrupt_moderation_supported - Return if interrupt moderation
* capability is supported by the device.
*
@ -926,12 +889,6 @@ void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev);
*/
bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev);
/* ena_com_config_default_interrupt_moderation_table - Restore the interrupt
* moderation table back to the default parameters.
* @ena_dev: ENA communication layer struct
*/
void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev);
/* ena_com_update_nonadaptive_moderation_interval_tx - Update the
* non-adaptive interval in Tx direction.
* @ena_dev: ENA communication layer struct
@ -968,29 +925,6 @@ unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *
*/
unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev);
/* ena_com_init_intr_moderation_entry - Update a single entry in the interrupt
* moderation table.
* @ena_dev: ENA communication layer struct
* @level: Interrupt moderation table level
* @entry: Entry value
*
* Update a single entry in the interrupt moderation table.
*/
void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry);
/* ena_com_get_intr_moderation_entry - Init ena_intr_moder_entry.
* @ena_dev: ENA communication layer struct
* @level: Interrupt moderation table level
* @entry: Entry to fill.
*
* Initialize the entry according to the adaptive interrupt moderation table.
*/
void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry);
/* ena_com_config_dev_mode - Configure the placement policy of the device.
* @ena_dev: ENA communication layer struct
* @llq_features: LLQ feature descriptor, retrieve via
@ -1016,75 +950,6 @@ static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_d
ena_dev->adaptive_coalescing = false;
}
/* ena_com_calculate_interrupt_delay - Calculate new interrupt delay
* @ena_dev: ENA communication layer struct
* @pkts: Number of packets since the last update
* @bytes: Number of bytes received since the last update.
* @smoothed_interval: Returned interval
* @moder_tbl_idx: Current table level as input update new level as return
* value.
*/
static inline void ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev,
unsigned int pkts,
unsigned int bytes,
unsigned int *smoothed_interval,
unsigned int *moder_tbl_idx)
{
enum ena_intr_moder_level curr_moder_idx, new_moder_idx;
struct ena_intr_moder_entry *curr_moder_entry;
struct ena_intr_moder_entry *pred_moder_entry;
struct ena_intr_moder_entry *new_moder_entry;
struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
unsigned int interval;
/* We apply adaptive moderation on Rx path only.
* Tx uses static interrupt moderation.
*/
if (!pkts || !bytes)
/* Tx interrupt, or spurious interrupt,
* in both cases we just use same delay values
*/
return;
curr_moder_idx = (enum ena_intr_moder_level)(*moder_tbl_idx);
if (unlikely(curr_moder_idx >= ENA_INTR_MAX_NUM_OF_LEVELS)) {
pr_err("Wrong moderation index %u\n", curr_moder_idx);
return;
}
curr_moder_entry = &intr_moder_tbl[curr_moder_idx];
new_moder_idx = curr_moder_idx;
if (curr_moder_idx == ENA_INTR_MODER_LOWEST) {
if ((pkts > curr_moder_entry->pkts_per_interval) ||
(bytes > curr_moder_entry->bytes_per_interval))
new_moder_idx =
(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
} else {
pred_moder_entry = &intr_moder_tbl[curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE];
if ((pkts <= pred_moder_entry->pkts_per_interval) ||
(bytes <= pred_moder_entry->bytes_per_interval))
new_moder_idx =
(enum ena_intr_moder_level)(curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE);
else if ((pkts > curr_moder_entry->pkts_per_interval) ||
(bytes > curr_moder_entry->bytes_per_interval)) {
if (curr_moder_idx != ENA_INTR_MODER_HIGHEST)
new_moder_idx =
(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
}
}
new_moder_entry = &intr_moder_tbl[new_moder_idx];
interval = new_moder_entry->intr_moder_interval;
*smoothed_interval = (
(interval * ENA_INTR_DELAY_NEW_VALUE_WEIGHT +
ENA_INTR_DELAY_OLD_VALUE_WEIGHT * (*smoothed_interval)) + 5) /
10;
*moder_tbl_idx = new_moder_idx;
}
/* ena_com_update_intr_reg - Prepare interrupt register
* @intr_reg: interrupt register to update.
* @rx_delay_interval: Rx interval in usecs

87
drivers/net/ethernet/amazon/ena/ena_ethtool.c
View File

@ -305,32 +305,21 @@ static int ena_get_coalesce(struct net_device *net_dev,
{
struct ena_adapter *adapter = netdev_priv(net_dev);
struct ena_com_dev *ena_dev = adapter->ena_dev;
struct ena_intr_moder_entry intr_moder_entry;
if (!ena_com_interrupt_moderation_supported(ena_dev)) {
/* the devie doesn't support interrupt moderation */
return -EOPNOTSUPP;
}
coalesce->tx_coalesce_usecs =
ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) /
ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) *
ena_dev->intr_delay_resolution;
if (!ena_com_get_adaptive_moderation_enabled(ena_dev)) {
if (!ena_com_get_adaptive_moderation_enabled(ena_dev))
coalesce->rx_coalesce_usecs =
ena_com_get_nonadaptive_moderation_interval_rx(ena_dev)
/ ena_dev->intr_delay_resolution;
} else {
ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry);
coalesce->rx_coalesce_usecs_low = intr_moder_entry.intr_moder_interval;
coalesce->rx_max_coalesced_frames_low = intr_moder_entry.pkts_per_interval;
* ena_dev->intr_delay_resolution;
ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry);
coalesce->rx_coalesce_usecs = intr_moder_entry.intr_moder_interval;
coalesce->rx_max_coalesced_frames = intr_moder_entry.pkts_per_interval;
ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry);
coalesce->rx_coalesce_usecs_high = intr_moder_entry.intr_moder_interval;
coalesce->rx_max_coalesced_frames_high = intr_moder_entry.pkts_per_interval;
}
coalesce->use_adaptive_rx_coalesce =
ena_com_get_adaptive_moderation_enabled(ena_dev);
@ -348,12 +337,22 @@ static void ena_update_tx_rings_intr_moderation(struct ena_adapter *adapter)
adapter->tx_ring[i].smoothed_interval = val;
}
static void ena_update_rx_rings_intr_moderation(struct ena_adapter *adapter)
{
unsigned int val;
int i;
val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev);
for (i = 0; i < adapter->num_queues; i++)
adapter->rx_ring[i].smoothed_interval = val;
}
static int ena_set_coalesce(struct net_device *net_dev,
struct ethtool_coalesce *coalesce)
{
struct ena_adapter *adapter = netdev_priv(net_dev);
struct ena_com_dev *ena_dev = adapter->ena_dev;
struct ena_intr_moder_entry intr_moder_entry;
int rc;
if (!ena_com_interrupt_moderation_supported(ena_dev)) {
@ -361,22 +360,6 @@ static int ena_set_coalesce(struct net_device *net_dev,
return -EOPNOTSUPP;
}
if (coalesce->rx_coalesce_usecs_irq ||
coalesce->rx_max_coalesced_frames_irq ||
coalesce->tx_coalesce_usecs_irq ||
coalesce->tx_max_coalesced_frames ||
coalesce->tx_max_coalesced_frames_irq ||
coalesce->stats_block_coalesce_usecs ||
coalesce->use_adaptive_tx_coalesce ||
coalesce->pkt_rate_low ||
coalesce->tx_coalesce_usecs_low ||
coalesce->tx_max_coalesced_frames_low ||
coalesce->pkt_rate_high ||
coalesce->tx_coalesce_usecs_high ||
coalesce->tx_max_coalesced_frames_high ||
coalesce->rate_sample_interval)
return -EINVAL;
rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev,
coalesce->tx_coalesce_usecs);
if (rc)
@ -384,37 +367,23 @@ static int ena_set_coalesce(struct net_device *net_dev,
ena_update_tx_rings_intr_moderation(adapter);
if (ena_com_get_adaptive_moderation_enabled(ena_dev)) {
if (!coalesce->use_adaptive_rx_coalesce) {
ena_com_disable_adaptive_moderation(ena_dev);
rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
coalesce->rx_coalesce_usecs);
return rc;
}
} else { /* was in non-adaptive mode */
if (coalesce->use_adaptive_rx_coalesce) {
if (coalesce->use_adaptive_rx_coalesce) {
if (!ena_com_get_adaptive_moderation_enabled(ena_dev))
ena_com_enable_adaptive_moderation(ena_dev);
} else {
rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
coalesce->rx_coalesce_usecs);
return rc;
}
return 0;
}
intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_low;
intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_low;
intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry);
rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
coalesce->rx_coalesce_usecs);
if (rc)
return rc;
intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs;
intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames;
intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry);
ena_update_rx_rings_intr_moderation(adapter);
intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_high;
intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_high;
intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry);
if (!coalesce->use_adaptive_rx_coalesce) {
if (ena_com_get_adaptive_moderation_enabled(ena_dev))
ena_com_disable_adaptive_moderation(ena_dev);
}
return 0;
}

83
drivers/net/ethernet/amazon/ena/ena_netdev.c
View File

@ -158,7 +158,6 @@ static void ena_init_io_rings_common(struct ena_adapter *adapter,
ring->adapter = adapter;
ring->ena_dev = adapter->ena_dev;
ring->per_napi_packets = 0;
ring->per_napi_bytes = 0;
ring->cpu = 0;
ring->first_interrupt = false;
ring->no_interrupt_event_cnt = 0;
@ -196,6 +195,7 @@ static void ena_init_io_rings(struct ena_adapter *adapter)
rxr->smoothed_interval =
ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
rxr->empty_rx_queue = 0;
adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
}
}
@ -712,6 +712,7 @@ static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
for (i = 0; i < adapter->num_queues; i++) {
ena_qid = ENA_IO_RXQ_IDX(i);
cancel_work_sync(&adapter->ena_napi[i].dim.work);
ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
}
}
@ -832,9 +833,6 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
__netif_tx_unlock(txq);
}
tx_ring->per_napi_bytes += tx_bytes;
tx_ring->per_napi_packets += tx_pkts;
return tx_pkts;
}
@ -1118,7 +1116,6 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
} while (likely(res_budget));
work_done = budget - res_budget;
rx_ring->per_napi_bytes += total_len;
rx_ring->per_napi_packets += work_done;
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->rx_stats.bytes += total_len;
@ -1155,35 +1152,50 @@ error:
return 0;
}
void ena_adjust_intr_moderation(struct ena_ring *rx_ring,
struct ena_ring *tx_ring)
static void ena_dim_work(struct work_struct *w)
{
/* We apply adaptive moderation on Rx path only.
* Tx uses static interrupt moderation.
*/
ena_com_calculate_interrupt_delay(rx_ring->ena_dev,
rx_ring->per_napi_packets,
rx_ring->per_napi_bytes,
&rx_ring->smoothed_interval,
&rx_ring->moder_tbl_idx);
struct dim *dim = container_of(w, struct dim, work);
struct dim_cq_moder cur_moder =
net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
dim->state = DIM_START_MEASURE;
}
static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
{
struct dim_sample dim_sample;
struct ena_ring *rx_ring = ena_napi->rx_ring;
if (!rx_ring->per_napi_packets)
return;
rx_ring->non_empty_napi_events++;
dim_update_sample(rx_ring->non_empty_napi_events,
rx_ring->rx_stats.cnt,
rx_ring->rx_stats.bytes,
&dim_sample);
net_dim(&ena_napi->dim, dim_sample);
/* Reset per napi packets/bytes */
tx_ring->per_napi_packets = 0;
tx_ring->per_napi_bytes = 0;
rx_ring->per_napi_packets = 0;
rx_ring->per_napi_bytes = 0;
}
static void ena_unmask_interrupt(struct ena_ring *tx_ring,
struct ena_ring *rx_ring)
{
struct ena_eth_io_intr_reg intr_reg;
u32 rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
rx_ring->smoothed_interval :
ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
/* Update intr register: rx intr delay,
* tx intr delay and interrupt unmask
*/
ena_com_update_intr_reg(&intr_reg,
rx_ring->smoothed_interval,
rx_interval,
tx_ring->smoothed_interval,
true);
@ -1260,9 +1272,11 @@ static int ena_io_poll(struct napi_struct *napi, int budget)
* from the interrupt context (vs from sk_busy_loop)
*/
if (napi_complete_done(napi, rx_work_done)) {
/* Tx and Rx share the same interrupt vector */
/* We apply adaptive moderation on Rx path only.
* Tx uses static interrupt moderation.
*/
if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
ena_adjust_intr_moderation(rx_ring, tx_ring);
ena_adjust_adaptive_rx_intr_moderation(ena_napi);
ena_unmask_interrupt(tx_ring, rx_ring);
}
@ -1552,14 +1566,6 @@ static void ena_napi_enable_all(struct ena_adapter *adapter)
napi_enable(&adapter->ena_napi[i].napi);
}
static void ena_restore_ethtool_params(struct ena_adapter *adapter)
{
adapter->tx_usecs = 0;
adapter->rx_usecs = 0;
adapter->tx_frames = 1;
adapter->rx_frames = 1;
}
/* Configure the Rx forwarding */
static int ena_rss_configure(struct ena_adapter *adapter)
{
@ -1609,8 +1615,6 @@ static int ena_up_complete(struct ena_adapter *adapter)
/* enable transmits */
netif_tx_start_all_queues(adapter->netdev);
ena_restore_ethtool_params(adapter);
ena_napi_enable_all(adapter);
return 0;
@ -1740,13 +1744,16 @@ static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
rc = ena_create_io_rx_queue(adapter, i);
if (rc)
goto create_err;
INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
}
return 0;
create_err:
while (i--)
while (i--) {
cancel_work_sync(&adapter->ena_napi[i].dim.work);
ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
}
return rc;
}
@ -2419,6 +2426,9 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev,
("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
host_info->num_cpus = num_online_cpus();
host_info->driver_supported_features =
ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK;
rc = ena_com_set_host_attributes(ena_dev);
if (rc) {
if (rc == -EOPNOTSUPP)
@ -3485,10 +3495,12 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
calc_queue_ctx.pdev = pdev;
/* initial Tx interrupt delay, Assumes 1 usec granularity.
/* Initial Tx and RX interrupt delay. Assumes 1 usec granularity.
* Updated during device initialization with the real granularity
*/
ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
io_queue_num = ena_calc_io_queue_num(pdev, ena_dev, &get_feat_ctx);
rc = ena_calc_queue_size(&calc_queue_ctx);
if (rc || io_queue_num <= 0) {
@ -3618,7 +3630,6 @@ err_free_msix:
ena_free_mgmnt_irq(adapter);
ena_disable_msix(adapter);
err_worker_destroy:
ena_com_destroy_interrupt_moderation(ena_dev);
del_timer(&adapter->timer_service);
err_netdev_destroy:
free_netdev(netdev);
@ -3679,8 +3690,6 @@ static void ena_remove(struct pci_dev *pdev)
pci_disable_device(pdev);
ena_com_destroy_interrupt_moderation(ena_dev);
vfree(ena_dev);
}

8
drivers/net/ethernet/amazon/ena/ena_netdev.h
View File

@ -34,6 +34,7 @@
#define ENA_H
#include <linux/bitops.h>
#include <linux/dim.h>
#include <linux/etherdevice.h>
#include <linux/inetdevice.h>
#include <linux/interrupt.h>
@ -153,6 +154,7 @@ struct ena_napi {
struct ena_ring *tx_ring;
struct ena_ring *rx_ring;
u32 qid;
struct dim dim;
};
struct ena_calc_queue_size_ctx {
@ -278,8 +280,7 @@ struct ena_ring {
struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];
u32 smoothed_interval;
u32 per_napi_packets;
u32 per_napi_bytes;
enum ena_intr_moder_level moder_tbl_idx;
u16 non_empty_napi_events;
struct u64_stats_sync syncp;
union {
struct ena_stats_tx tx_stats;
@ -329,9 +330,6 @@ struct ena_adapter {
u32 missing_tx_completion_threshold;
u32 tx_usecs, rx_usecs; /* interrupt moderation */
u32 tx_frames, rx_frames; /* interrupt moderation */
u32 requested_tx_ring_size;
u32 requested_rx_ring_size;