linux/include/net/dst.h
Zhang Yanmin f1dd9c379c [NET]: Fix tbench regression in 2.6.25-rc1
Comparing with kernel 2.6.24, tbench result has regression with
2.6.25-rc1.

1) On 2 quad-core processor stoakley: 4%.
2) On 4 quad-core processor tigerton: more than 30%.

bisect located below patch.

b4ce92775c is first bad commit
commit b4ce92775c
Author: Herbert Xu <herbert@gondor.apana.org.au>
Date:   Tue Nov 13 21:33:32 2007 -0800

    [IPV6]: Move nfheader_len into rt6_info

    The dst member nfheader_len is only used by IPv6.  It's also currently
    creating a rather ugly alignment hole in struct dst.  Therefore this patch
    moves it from there into struct rt6_info.

Above patch changes the cache line alignment, especially member
__refcnt. I did a testing by adding 2 unsigned long pading before
lastuse, so the 3 members, lastuse/__refcnt/__use, are moved to next
cache line. The performance is recovered.

I created a patch to rearrange the members in struct dst_entry.

With Eric and Valdis Kletnieks's suggestion, I made finer arrangement.

1) Move tclassid under ops in case CONFIG_NET_CLS_ROUTE=y. So
   sizeof(dst_entry)=200 no matter if CONFIG_NET_CLS_ROUTE=y/n. I
   tested many patches on my 16-core tigerton by moving tclassid to
   different place. It looks like tclassid could also have impact on
   performance.  If moving tclassid before metrics, or just don't move
   tclassid, the performance isn't good. So I move it behind metrics.

2) Add comments before __refcnt.

On 16-core tigerton:

If CONFIG_NET_CLS_ROUTE=y, the result with below patch is about 18%
better than the one without the patch;

If CONFIG_NET_CLS_ROUTE=n, the result with below patch is about 30%
better than the one without the patch.

With 32bit 2.6.25-rc1 on 8-core stoakley, the new patch doesn't
introduce regression.

Thank Eric, Valdis, and David!

Signed-off-by: Zhang Yanmin <yanmin.zhang@intel.com>
Acked-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-03-12 22:52:37 -07:00

300 lines
6.4 KiB
C

/*
* net/dst.h Protocol independent destination cache definitions.
*
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
*
*/
#ifndef _NET_DST_H
#define _NET_DST_H
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/rcupdate.h>
#include <linux/jiffies.h>
#include <net/neighbour.h>
#include <asm/processor.h>
/*
* 0 - no debugging messages
* 1 - rare events and bugs (default)
* 2 - trace mode.
*/
#define RT_CACHE_DEBUG 0
#define DST_GC_MIN (HZ/10)
#define DST_GC_INC (HZ/2)
#define DST_GC_MAX (120*HZ)
/* Each dst_entry has reference count and sits in some parent list(s).
* When it is removed from parent list, it is "freed" (dst_free).
* After this it enters dead state (dst->obsolete > 0) and if its refcnt
* is zero, it can be destroyed immediately, otherwise it is added
* to gc list and garbage collector periodically checks the refcnt.
*/
struct sk_buff;
struct dst_entry
{
struct rcu_head rcu_head;
struct dst_entry *child;
struct net_device *dev;
short error;
short obsolete;
int flags;
#define DST_HOST 1
#define DST_NOXFRM 2
#define DST_NOPOLICY 4
#define DST_NOHASH 8
unsigned long expires;
unsigned short header_len; /* more space at head required */
unsigned short trailer_len; /* space to reserve at tail */
unsigned int rate_tokens;
unsigned long rate_last; /* rate limiting for ICMP */
struct dst_entry *path;
struct neighbour *neighbour;
struct hh_cache *hh;
struct xfrm_state *xfrm;
int (*input)(struct sk_buff*);
int (*output)(struct sk_buff*);
struct dst_ops *ops;
u32 metrics[RTAX_MAX];
#ifdef CONFIG_NET_CLS_ROUTE
__u32 tclassid;
#endif
/*
* __refcnt wants to be on a different cache line from
* input/output/ops or performance tanks badly
*/
atomic_t __refcnt; /* client references */
int __use;
unsigned long lastuse;
union {
struct dst_entry *next;
struct rtable *rt_next;
struct rt6_info *rt6_next;
struct dn_route *dn_next;
};
};
struct dst_ops
{
unsigned short family;
__be16 protocol;
unsigned gc_thresh;
int (*gc)(struct dst_ops *ops);
struct dst_entry * (*check)(struct dst_entry *, __u32 cookie);
void (*destroy)(struct dst_entry *);
void (*ifdown)(struct dst_entry *,
struct net_device *dev, int how);
struct dst_entry * (*negative_advice)(struct dst_entry *);
void (*link_failure)(struct sk_buff *);
void (*update_pmtu)(struct dst_entry *dst, u32 mtu);
int (*local_out)(struct sk_buff *skb);
int entry_size;
atomic_t entries;
struct kmem_cache *kmem_cachep;
struct net *dst_net;
};
#ifdef __KERNEL__
static inline u32
dst_metric(const struct dst_entry *dst, int metric)
{
return dst->metrics[metric-1];
}
static inline u32 dst_mtu(const struct dst_entry *dst)
{
u32 mtu = dst_metric(dst, RTAX_MTU);
/*
* Alexey put it here, so ask him about it :)
*/
barrier();
return mtu;
}
static inline u32
dst_allfrag(const struct dst_entry *dst)
{
int ret = dst_metric(dst, RTAX_FEATURES) & RTAX_FEATURE_ALLFRAG;
/* Yes, _exactly_. This is paranoia. */
barrier();
return ret;
}
static inline int
dst_metric_locked(struct dst_entry *dst, int metric)
{
return dst_metric(dst, RTAX_LOCK) & (1<<metric);
}
static inline void dst_hold(struct dst_entry * dst)
{
atomic_inc(&dst->__refcnt);
}
static inline void dst_use(struct dst_entry *dst, unsigned long time)
{
dst_hold(dst);
dst->__use++;
dst->lastuse = time;
}
static inline
struct dst_entry * dst_clone(struct dst_entry * dst)
{
if (dst)
atomic_inc(&dst->__refcnt);
return dst;
}
static inline
void dst_release(struct dst_entry * dst)
{
if (dst) {
WARN_ON(atomic_read(&dst->__refcnt) < 1);
smp_mb__before_atomic_dec();
atomic_dec(&dst->__refcnt);
}
}
/* Children define the path of the packet through the
* Linux networking. Thus, destinations are stackable.
*/
static inline struct dst_entry *dst_pop(struct dst_entry *dst)
{
struct dst_entry *child = dst_clone(dst->child);
dst_release(dst);
return child;
}
extern int dst_discard(struct sk_buff *skb);
extern void * dst_alloc(struct dst_ops * ops);
extern void __dst_free(struct dst_entry * dst);
extern struct dst_entry *dst_destroy(struct dst_entry * dst);
static inline void dst_free(struct dst_entry * dst)
{
if (dst->obsolete > 1)
return;
if (!atomic_read(&dst->__refcnt)) {
dst = dst_destroy(dst);
if (!dst)
return;
}
__dst_free(dst);
}
static inline void dst_rcu_free(struct rcu_head *head)
{
struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head);
dst_free(dst);
}
static inline void dst_confirm(struct dst_entry *dst)
{
if (dst)
neigh_confirm(dst->neighbour);
}
static inline void dst_negative_advice(struct dst_entry **dst_p)
{
struct dst_entry * dst = *dst_p;
if (dst && dst->ops->negative_advice)
*dst_p = dst->ops->negative_advice(dst);
}
static inline void dst_link_failure(struct sk_buff *skb)
{
struct dst_entry * dst = skb->dst;
if (dst && dst->ops && dst->ops->link_failure)
dst->ops->link_failure(skb);
}
static inline void dst_set_expires(struct dst_entry *dst, int timeout)
{
unsigned long expires = jiffies + timeout;
if (expires == 0)
expires = 1;
if (dst->expires == 0 || time_before(expires, dst->expires))
dst->expires = expires;
}
/* Output packet to network from transport. */
static inline int dst_output(struct sk_buff *skb)
{
return skb->dst->output(skb);
}
/* Input packet from network to transport. */
static inline int dst_input(struct sk_buff *skb)
{
int err;
for (;;) {
err = skb->dst->input(skb);
if (likely(err == 0))
return err;
/* Oh, Jamal... Seems, I will not forgive you this mess. :-) */
if (unlikely(err != NET_XMIT_BYPASS))
return err;
}
}
static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
{
if (dst->obsolete)
dst = dst->ops->check(dst, cookie);
return dst;
}
extern void dst_init(void);
/* Flags for xfrm_lookup flags argument. */
enum {
XFRM_LOOKUP_WAIT = 1 << 0,
XFRM_LOOKUP_ICMP = 1 << 1,
};
struct flowi;
#ifndef CONFIG_XFRM
static inline int xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
struct sock *sk, int flags)
{
return 0;
}
static inline int __xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
struct sock *sk, int flags)
{
return 0;
}
#else
extern int xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
struct sock *sk, int flags);
extern int __xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
struct sock *sk, int flags);
#endif
#endif
#endif /* _NET_DST_H */