From 960fdfdeb9e85a67bed136bc945c541ba61c2bdd Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 14:52:30 +0100
Subject: [PATCH 01/13] xfrm: input: constify xfrm_input_afinfo
Nothing writes to these structures (the module owner was not used).
While at it, size xfrm_input_afinfo[] by the highest existing xfrm family
(INET6), not AF_MAX.
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 5 ++---
net/ipv4/xfrm4_protocol.c | 3 +--
net/ipv6/xfrm6_protocol.c | 3 +--
net/xfrm/xfrm_input.c | 31 +++++++++++--------------------
4 files changed, 15 insertions(+), 27 deletions(-)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index d9a81dcef53e..da0e4dd653e2 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -349,13 +349,12 @@ struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family);
struct xfrm_input_afinfo {
unsigned int family;
- struct module *owner;
int (*callback)(struct sk_buff *skb, u8 protocol,
int err);
};
-int xfrm_input_register_afinfo(struct xfrm_input_afinfo *afinfo);
-int xfrm_input_unregister_afinfo(struct xfrm_input_afinfo *afinfo);
+int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo);
+int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo);
void xfrm_state_delete_tunnel(struct xfrm_state *x);
diff --git a/net/ipv4/xfrm4_protocol.c b/net/ipv4/xfrm4_protocol.c
index dccefa9d84cf..8dd0e6ab8606 100644
--- a/net/ipv4/xfrm4_protocol.c
+++ b/net/ipv4/xfrm4_protocol.c
@@ -188,9 +188,8 @@ static const struct net_protocol ipcomp4_protocol = {
.netns_ok = 1,
};
-static struct xfrm_input_afinfo xfrm4_input_afinfo = {
+static const struct xfrm_input_afinfo xfrm4_input_afinfo = {
.family = AF_INET,
- .owner = THIS_MODULE,
.callback = xfrm4_rcv_cb,
};
diff --git a/net/ipv6/xfrm6_protocol.c b/net/ipv6/xfrm6_protocol.c
index 54d13f8dbbae..b2dc8ce49378 100644
--- a/net/ipv6/xfrm6_protocol.c
+++ b/net/ipv6/xfrm6_protocol.c
@@ -162,9 +162,8 @@ static const struct inet6_protocol ipcomp6_protocol = {
.flags = INET6_PROTO_NOPOLICY,
};
-static struct xfrm_input_afinfo xfrm6_input_afinfo = {
+static const struct xfrm_input_afinfo xfrm6_input_afinfo = {
.family = AF_INET6,
- .owner = THIS_MODULE,
.callback = xfrm6_rcv_cb,
};
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index 3213fe8027be..8722294c6e59 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -19,19 +19,18 @@
static struct kmem_cache *secpath_cachep __read_mostly;
static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
-static struct xfrm_input_afinfo __rcu *xfrm_input_afinfo[NPROTO];
+static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];
static struct gro_cells gro_cells;
static struct net_device xfrm_napi_dev;
-int xfrm_input_register_afinfo(struct xfrm_input_afinfo *afinfo)
+int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
{
int err = 0;
- if (unlikely(afinfo == NULL))
- return -EINVAL;
- if (unlikely(afinfo->family >= NPROTO))
+ if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
return -EAFNOSUPPORT;
+
spin_lock_bh(&xfrm_input_afinfo_lock);
if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
err = -EEXIST;
@@ -42,14 +41,10 @@ int xfrm_input_register_afinfo(struct xfrm_input_afinfo *afinfo)
}
EXPORT_SYMBOL(xfrm_input_register_afinfo);
-int xfrm_input_unregister_afinfo(struct xfrm_input_afinfo *afinfo)
+int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
{
int err = 0;
- if (unlikely(afinfo == NULL))
- return -EINVAL;
- if (unlikely(afinfo->family >= NPROTO))
- return -EAFNOSUPPORT;
spin_lock_bh(&xfrm_input_afinfo_lock);
if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
@@ -63,12 +58,13 @@ int xfrm_input_unregister_afinfo(struct xfrm_input_afinfo *afinfo)
}
EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
-static struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
+static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
{
- struct xfrm_input_afinfo *afinfo;
+ const struct xfrm_input_afinfo *afinfo;
- if (unlikely(family >= NPROTO))
+ if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
return NULL;
+
rcu_read_lock();
afinfo = rcu_dereference(xfrm_input_afinfo[family]);
if (unlikely(!afinfo))
@@ -76,22 +72,17 @@ static struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
return afinfo;
}
-static void xfrm_input_put_afinfo(struct xfrm_input_afinfo *afinfo)
-{
- rcu_read_unlock();
-}
-
static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
int err)
{
int ret;
- struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
+ const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
if (!afinfo)
return -EAFNOSUPPORT;
ret = afinfo->callback(skb, protocol, err);
- xfrm_input_put_afinfo(afinfo);
+ rcu_read_unlock();
return ret;
}
From f5e2bb4f5b2252e8f170f59bece2cf9a2efc8e6a Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:14 +0100
Subject: [PATCH 02/13] xfrm: policy: xfrm_get_tos cannot fail
The comment makes it look like get_tos() is used to validate something,
but it turns out the comment was about xfrm_find_bundle() which got removed
years ago.
xfrm_get_tos will return either the tos (ipv4) or 0 (ipv6).
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
net/xfrm/xfrm_policy.c | 19 +++++--------------
1 file changed, 5 insertions(+), 14 deletions(-)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 99ad1af2927f..6fffa2fac607 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -1538,19 +1538,13 @@ xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
}
-/* Check that the bundle accepts the flow and its components are
- * still valid.
- */
-
-static inline int xfrm_get_tos(const struct flowi *fl, int family)
+static int xfrm_get_tos(const struct flowi *fl, int family)
{
- struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
- int tos;
+ struct xfrm_policy_afinfo *afinfo;
+ int tos = 0;
- if (!afinfo)
- return -EINVAL;
-
- tos = afinfo->get_tos(fl);
+ afinfo = xfrm_policy_get_afinfo(family);
+ tos = afinfo ? afinfo->get_tos(fl) : 0;
xfrm_policy_put_afinfo(afinfo);
@@ -1705,9 +1699,6 @@ static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
tos = xfrm_get_tos(fl, family);
- err = tos;
- if (tos < 0)
- goto put_states;
dst_hold(dst);
From 2b61997aa0c68ae033d066ac2d9905ada81b761a Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:15 +0100
Subject: [PATCH 03/13] xfrm: policy: xfrm_policy_unregister_afinfo can return
void
Nothing checks the return value. Also, the errors returned on unregister
are impossible (we only support INET and INET6, so no way
xfrm_policy_afinfo[afinfo->family] can be anything other than 'afinfo'
itself).
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 2 +-
net/xfrm/xfrm_policy.c | 35 +++++++++++++----------------------
2 files changed, 14 insertions(+), 23 deletions(-)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index da0e4dd653e2..39037b1cce7a 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -304,7 +304,7 @@ struct xfrm_policy_afinfo {
};
int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo);
-int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo);
+void xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo);
void km_policy_notify(struct xfrm_policy *xp, int dir,
const struct km_event *c);
void km_state_notify(struct xfrm_state *x, const struct km_event *c);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 6fffa2fac607..794148f76ae2 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -2883,34 +2883,25 @@ int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
}
EXPORT_SYMBOL(xfrm_policy_register_afinfo);
-int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
+void xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
{
- int err = 0;
- if (unlikely(afinfo == NULL))
- return -EINVAL;
+ struct dst_ops *dst_ops = afinfo->dst_ops;
+
if (unlikely(afinfo->family >= NPROTO))
- return -EAFNOSUPPORT;
- spin_lock(&xfrm_policy_afinfo_lock);
- if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
- if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
- err = -EINVAL;
- else
- RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
+ return;
+
+ if (likely(xfrm_policy_afinfo[afinfo->family] != afinfo)) {
+ RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
NULL);
}
- spin_unlock(&xfrm_policy_afinfo_lock);
- if (!err) {
- struct dst_ops *dst_ops = afinfo->dst_ops;
- synchronize_rcu();
+ synchronize_rcu();
- dst_ops->kmem_cachep = NULL;
- dst_ops->check = NULL;
- dst_ops->negative_advice = NULL;
- dst_ops->link_failure = NULL;
- afinfo->garbage_collect = NULL;
- }
- return err;
+ dst_ops->kmem_cachep = NULL;
+ dst_ops->check = NULL;
+ dst_ops->negative_advice = NULL;
+ dst_ops->link_failure = NULL;
+ afinfo->garbage_collect = NULL;
}
EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
From 3d7d25a68ea5153d9d0d01c8c83acf644eab9704 Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:16 +0100
Subject: [PATCH 04/13] xfrm: policy: remove garbage_collect callback
Just call xfrm_garbage_collect_deferred() directly.
This gets rid of a write to afinfo in register/unregister and allows to
constify afinfo later on.
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 2 +-
net/ipv4/xfrm4_policy.c | 2 +-
net/ipv6/xfrm6_policy.c | 2 +-
net/xfrm/xfrm_policy.c | 6 ++----
4 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 39037b1cce7a..a22368edd105 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -282,7 +282,6 @@ struct xfrm_dst;
struct xfrm_policy_afinfo {
unsigned short family;
struct dst_ops *dst_ops;
- void (*garbage_collect)(struct net *net);
struct dst_entry *(*dst_lookup)(struct net *net,
int tos, int oif,
const xfrm_address_t *saddr,
@@ -1169,6 +1168,7 @@ static inline void xfrm_sk_free_policy(struct sock *sk)
}
void xfrm_garbage_collect(struct net *net);
+void xfrm_garbage_collect_deferred(struct net *net);
#else
diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c
index 6a7ff6957535..77ca91d7b79c 100644
--- a/net/ipv4/xfrm4_policy.c
+++ b/net/ipv4/xfrm4_policy.c
@@ -219,7 +219,7 @@ static inline int xfrm4_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm4_dst_ops);
- xfrm4_policy_afinfo.garbage_collect(net);
+ xfrm_garbage_collect_deferred(net);
return (dst_entries_get_slow(ops) > ops->gc_thresh * 2);
}
diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c
index e0f71c01d728..c273f22b12af 100644
--- a/net/ipv6/xfrm6_policy.c
+++ b/net/ipv6/xfrm6_policy.c
@@ -220,7 +220,7 @@ static inline int xfrm6_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm6_dst_ops);
- xfrm6_policy_afinfo.garbage_collect(net);
+ xfrm_garbage_collect_deferred(net);
return dst_entries_get_fast(ops) > ops->gc_thresh * 2;
}
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 794148f76ae2..80695a157b6e 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -2733,10 +2733,11 @@ void xfrm_garbage_collect(struct net *net)
}
EXPORT_SYMBOL(xfrm_garbage_collect);
-static void xfrm_garbage_collect_deferred(struct net *net)
+void xfrm_garbage_collect_deferred(struct net *net)
{
flow_cache_flush_deferred(net);
}
+EXPORT_SYMBOL(xfrm_garbage_collect_deferred);
static void xfrm_init_pmtu(struct dst_entry *dst)
{
@@ -2873,8 +2874,6 @@ int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
dst_ops->link_failure = xfrm_link_failure;
if (likely(dst_ops->neigh_lookup == NULL))
dst_ops->neigh_lookup = xfrm_neigh_lookup;
- if (likely(afinfo->garbage_collect == NULL))
- afinfo->garbage_collect = xfrm_garbage_collect_deferred;
rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
}
spin_unlock(&xfrm_policy_afinfo_lock);
@@ -2901,7 +2900,6 @@ void xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
dst_ops->check = NULL;
dst_ops->negative_advice = NULL;
dst_ops->link_failure = NULL;
- afinfo->garbage_collect = NULL;
}
EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
From a2817d8b279bc8fe62da76e6019eb9ff9d4e319c Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:17 +0100
Subject: [PATCH 05/13] xfrm: policy: remove family field
Only needed it to register the policy backend at init time.
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 5 ++---
net/ipv4/xfrm4_policy.c | 5 +----
net/ipv6/xfrm6_policy.c | 5 +----
net/xfrm/xfrm_policy.c | 34 +++++++++++++++++-----------------
4 files changed, 21 insertions(+), 28 deletions(-)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index a22368edd105..6e061309adca 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -280,7 +280,6 @@ struct net_device;
struct xfrm_type;
struct xfrm_dst;
struct xfrm_policy_afinfo {
- unsigned short family;
struct dst_ops *dst_ops;
struct dst_entry *(*dst_lookup)(struct net *net,
int tos, int oif,
@@ -302,8 +301,8 @@ struct xfrm_policy_afinfo {
struct dst_entry *(*blackhole_route)(struct net *net, struct dst_entry *orig);
};
-int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo);
-void xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo);
+int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family);
+void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo);
void km_policy_notify(struct xfrm_policy *xp, int dir,
const struct km_event *c);
void km_state_notify(struct xfrm_state *x, const struct km_event *c);
diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c
index 77ca91d7b79c..25613539766f 100644
--- a/net/ipv4/xfrm4_policy.c
+++ b/net/ipv4/xfrm4_policy.c
@@ -17,8 +17,6 @@
#include <net/ip.h>
#include <net/l3mdev.h>
-static struct xfrm_policy_afinfo xfrm4_policy_afinfo;
-
static struct dst_entry *__xfrm4_dst_lookup(struct net *net, struct flowi4 *fl4,
int tos, int oif,
const xfrm_address_t *saddr,
@@ -272,7 +270,6 @@ static struct dst_ops xfrm4_dst_ops_template = {
};
static struct xfrm_policy_afinfo xfrm4_policy_afinfo = {
- .family = AF_INET,
.dst_ops = &xfrm4_dst_ops_template,
.dst_lookup = xfrm4_dst_lookup,
.get_saddr = xfrm4_get_saddr,
@@ -376,7 +373,7 @@ static struct pernet_operations __net_initdata xfrm4_net_ops = {
static void __init xfrm4_policy_init(void)
{
- xfrm_policy_register_afinfo(&xfrm4_policy_afinfo);
+ xfrm_policy_register_afinfo(&xfrm4_policy_afinfo, AF_INET);
}
void __init xfrm4_init(void)
diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c
index c273f22b12af..3217e85334e3 100644
--- a/net/ipv6/xfrm6_policy.c
+++ b/net/ipv6/xfrm6_policy.c
@@ -25,8 +25,6 @@
#include <net/mip6.h>
#endif
-static struct xfrm_policy_afinfo xfrm6_policy_afinfo;
-
static struct dst_entry *xfrm6_dst_lookup(struct net *net, int tos, int oif,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr)
@@ -292,7 +290,6 @@ static struct dst_ops xfrm6_dst_ops_template = {
};
static struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
- .family = AF_INET6,
.dst_ops = &xfrm6_dst_ops_template,
.dst_lookup = xfrm6_dst_lookup,
.get_saddr = xfrm6_get_saddr,
@@ -305,7 +302,7 @@ static struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
static int __init xfrm6_policy_init(void)
{
- return xfrm_policy_register_afinfo(&xfrm6_policy_afinfo);
+ return xfrm_policy_register_afinfo(&xfrm6_policy_afinfo, AF_INET6);
}
static void xfrm6_policy_fini(void)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 80695a157b6e..7698785876d7 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -45,7 +45,7 @@ struct xfrm_flo {
};
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
-static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO]
+static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
__read_mostly;
static struct kmem_cache *xfrm_dst_cache __read_mostly;
@@ -103,11 +103,11 @@ bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl
return false;
}
-static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
+static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
{
- struct xfrm_policy_afinfo *afinfo;
+ const struct xfrm_policy_afinfo *afinfo;
- if (unlikely(family >= NPROTO))
+ if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
return NULL;
rcu_read_lock();
afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
@@ -2848,15 +2848,15 @@ static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
return dst->path->ops->neigh_lookup(dst, skb, daddr);
}
-int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
+int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
{
int err = 0;
- if (unlikely(afinfo == NULL))
- return -EINVAL;
- if (unlikely(afinfo->family >= NPROTO))
+
+ if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
return -EAFNOSUPPORT;
+
spin_lock(&xfrm_policy_afinfo_lock);
- if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
+ if (unlikely(xfrm_policy_afinfo[family] != NULL))
err = -EEXIST;
else {
struct dst_ops *dst_ops = afinfo->dst_ops;
@@ -2874,7 +2874,7 @@ int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
dst_ops->link_failure = xfrm_link_failure;
if (likely(dst_ops->neigh_lookup == NULL))
dst_ops->neigh_lookup = xfrm_neigh_lookup;
- rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
+ rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
}
spin_unlock(&xfrm_policy_afinfo_lock);
@@ -2882,16 +2882,16 @@ int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
}
EXPORT_SYMBOL(xfrm_policy_register_afinfo);
-void xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
+void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
{
struct dst_ops *dst_ops = afinfo->dst_ops;
+ int i;
- if (unlikely(afinfo->family >= NPROTO))
- return;
-
- if (likely(xfrm_policy_afinfo[afinfo->family] != afinfo)) {
- RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
- NULL);
+ for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
+ if (xfrm_policy_afinfo[i] != afinfo)
+ continue;
+ RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
+ break;
}
synchronize_rcu();
From bdba9fe01e1bcb942c95a1a332b27ef829c87df4 Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:18 +0100
Subject: [PATCH 06/13] xfrm: policy: remove xfrm_policy_put_afinfo
Alternative is to keep it an make the (unused) afinfo arg const to avoid
the compiler warnings once the afinfo structs get constified.
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
net/xfrm/xfrm_policy.c | 21 ++++++++-------------
1 file changed, 8 insertions(+), 13 deletions(-)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 7698785876d7..80ef0dcf0e29 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -116,11 +116,6 @@ static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short fa
return afinfo;
}
-static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
-{
- rcu_read_unlock();
-}
-
static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
int tos, int oif,
const xfrm_address_t *saddr,
@@ -136,7 +131,7 @@ static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return dst;
}
@@ -1436,7 +1431,7 @@ xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
if (unlikely(afinfo == NULL))
return -EINVAL;
err = afinfo->get_saddr(net, oif, local, remote);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return err;
}
@@ -1546,7 +1541,7 @@ static int xfrm_get_tos(const struct flowi *fl, int family)
afinfo = xfrm_policy_get_afinfo(family);
tos = afinfo ? afinfo->get_tos(fl) : 0;
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return tos;
}
@@ -1632,7 +1627,7 @@ static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
} else
xdst = ERR_PTR(-ENOBUFS);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return xdst;
}
@@ -1649,7 +1644,7 @@ static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
err = afinfo->init_path(path, dst, nfheader_len);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return err;
}
@@ -1666,7 +1661,7 @@ static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
err = afinfo->fill_dst(xdst, dev, fl);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return err;
}
@@ -2215,7 +2210,7 @@ static struct dst_entry *make_blackhole(struct net *net, u16 family,
} else {
ret = afinfo->blackhole_route(net, dst_orig);
}
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return ret;
}
@@ -2465,7 +2460,7 @@ int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
afinfo->decode_session(skb, fl, reverse);
err = security_xfrm_decode_session(skb, &fl->flowi_secid);
- xfrm_policy_put_afinfo(afinfo);
+ rcu_read_unlock();
return err;
}
EXPORT_SYMBOL(__xfrm_decode_session);
From 37b103830ec3e52a761bb647eb78da22a1fe4e09 Mon Sep 17 00:00:00 2001
From: Florian Westphal <fw@strlen.de>
Date: Tue, 7 Feb 2017 15:00:19 +0100
Subject: [PATCH 07/13] xfrm: policy: make policy backend const
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
net/ipv4/xfrm4_policy.c | 2 +-
net/ipv6/xfrm6_policy.c | 2 +-
net/xfrm/xfrm_policy.c | 18 +++++++++---------
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c
index 25613539766f..71b4ecc195c7 100644
--- a/net/ipv4/xfrm4_policy.c
+++ b/net/ipv4/xfrm4_policy.c
@@ -269,7 +269,7 @@ static struct dst_ops xfrm4_dst_ops_template = {
.gc_thresh = INT_MAX,
};
-static struct xfrm_policy_afinfo xfrm4_policy_afinfo = {
+static const struct xfrm_policy_afinfo xfrm4_policy_afinfo = {
.dst_ops = &xfrm4_dst_ops_template,
.dst_lookup = xfrm4_dst_lookup,
.get_saddr = xfrm4_get_saddr,
diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c
index 3217e85334e3..79651bc71bf0 100644
--- a/net/ipv6/xfrm6_policy.c
+++ b/net/ipv6/xfrm6_policy.c
@@ -289,7 +289,7 @@ static struct dst_ops xfrm6_dst_ops_template = {
.gc_thresh = INT_MAX,
};
-static struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
+static const struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
.dst_ops = &xfrm6_dst_ops_template,
.dst_lookup = xfrm6_dst_lookup,
.get_saddr = xfrm6_get_saddr,
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 80ef0dcf0e29..04ed1a1ae019 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -45,7 +45,7 @@ struct xfrm_flo {
};
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
-static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
+static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
__read_mostly;
static struct kmem_cache *xfrm_dst_cache __read_mostly;
@@ -122,7 +122,7 @@ static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
const xfrm_address_t *daddr,
int family)
{
- struct xfrm_policy_afinfo *afinfo;
+ const struct xfrm_policy_afinfo *afinfo;
struct dst_entry *dst;
afinfo = xfrm_policy_get_afinfo(family);
@@ -1426,7 +1426,7 @@ xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
xfrm_address_t *remote, unsigned short family)
{
int err;
- struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
if (unlikely(afinfo == NULL))
return -EINVAL;
@@ -1535,7 +1535,7 @@ xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
static int xfrm_get_tos(const struct flowi *fl, int family)
{
- struct xfrm_policy_afinfo *afinfo;
+ const struct xfrm_policy_afinfo *afinfo;
int tos = 0;
afinfo = xfrm_policy_get_afinfo(family);
@@ -1598,7 +1598,7 @@ static const struct flow_cache_ops xfrm_bundle_fc_ops = {
static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
{
- struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
struct dst_ops *dst_ops;
struct xfrm_dst *xdst;
@@ -1635,7 +1635,7 @@ static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
int nfheader_len)
{
- struct xfrm_policy_afinfo *afinfo =
+ const struct xfrm_policy_afinfo *afinfo =
xfrm_policy_get_afinfo(dst->ops->family);
int err;
@@ -1652,7 +1652,7 @@ static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
const struct flowi *fl)
{
- struct xfrm_policy_afinfo *afinfo =
+ const struct xfrm_policy_afinfo *afinfo =
xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
int err;
@@ -2201,7 +2201,7 @@ error:
static struct dst_entry *make_blackhole(struct net *net, u16 family,
struct dst_entry *dst_orig)
{
- struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
struct dst_entry *ret;
if (!afinfo) {
@@ -2452,7 +2452,7 @@ xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int star
int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
unsigned int family, int reverse)
{
- struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
int err;
if (unlikely(afinfo == NULL))
From b0fcee825c0ad05057a97d1f4685e1b9e9d00c53 Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:39:24 +0100
Subject: [PATCH 08/13] xfrm: Add a secpath_set helper.
Add a new helper to set the secpath to the skb.
This avoids code duplication, as this is used
in multiple places.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 1 +
net/ipv6/xfrm6_input.c | 15 +++------------
net/xfrm/xfrm_input.c | 34 ++++++++++++++++++++++------------
3 files changed, 26 insertions(+), 24 deletions(-)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 6e061309adca..287635df4eef 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -1006,6 +1006,7 @@ secpath_put(struct sec_path *sp)
}
struct sec_path *secpath_dup(struct sec_path *src);
+int secpath_set(struct sk_buff *skb);
static inline void
secpath_reset(struct sk_buff *skb)
diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c
index b5789562aded..662fb2c3e765 100644
--- a/net/ipv6/xfrm6_input.c
+++ b/net/ipv6/xfrm6_input.c
@@ -69,18 +69,9 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr,
struct xfrm_state *x = NULL;
int i = 0;
- /* Allocate new secpath or COW existing one. */
- if (!skb->sp || atomic_read(&skb->sp->refcnt) != 1) {
- struct sec_path *sp;
-
- sp = secpath_dup(skb->sp);
- if (!sp) {
- XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
- goto drop;
- }
- if (skb->sp)
- secpath_put(skb->sp);
- skb->sp = sp;
+ if (secpath_set(skb)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
+ goto drop;
}
if (1 + skb->sp->len == XFRM_MAX_DEPTH) {
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index 8722294c6e59..d8f913bb6919 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -117,6 +117,24 @@ struct sec_path *secpath_dup(struct sec_path *src)
}
EXPORT_SYMBOL(secpath_dup);
+int secpath_set(struct sk_buff *skb)
+{
+ struct sec_path *sp;
+
+ /* Allocate new secpath or COW existing one. */
+ if (!skb->sp || atomic_read(&skb->sp->refcnt) != 1) {
+ sp = secpath_dup(skb->sp);
+ if (!sp)
+ return -ENOMEM;
+
+ if (skb->sp)
+ secpath_put(skb->sp);
+ skb->sp = sp;
+ }
+ return 0;
+}
+EXPORT_SYMBOL(secpath_set);
+
/* Fetch spi and seq from ipsec header */
int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
@@ -212,18 +230,10 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
break;
}
- /* Allocate new secpath or COW existing one. */
- if (!skb->sp || atomic_read(&skb->sp->refcnt) != 1) {
- struct sec_path *sp;
-
- sp = secpath_dup(skb->sp);
- if (!sp) {
- XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
- goto drop;
- }
- if (skb->sp)
- secpath_put(skb->sp);
- skb->sp = sp;
+ err = secpath_set(skb);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
+ goto drop;
}
seq = 0;
From 5f114163f2f5eb2edbb49c4d3e0b405c7a8a7e2a Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:39:39 +0100
Subject: [PATCH 09/13] net: Add a skb_gro_flush_final helper.
Add a skb_gro_flush_final helper to prepare for consuming
skbs in call_gro_receive. We will extend this helper to not
touch the skb if the skb is consumed by a gro callback with
a followup patch. We need this to handle the upcomming IPsec
ESP callbacks as they reinject the skb to the napi_gro_receive
asynchronous. The handler is used in all gro_receive functions
that can call the ESP gro handlers.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/linux/netdevice.h | 5 +++++
net/ethernet/eth.c | 2 +-
net/ipv4/af_inet.c | 2 +-
net/ipv6/ip6_offload.c | 2 +-
4 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 58afbd1cc659..f9da3acfee9a 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2661,6 +2661,11 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
remcsum_unadjust((__sum16 *)ptr, grc->delta);
}
+static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
+{
+ NAPI_GRO_CB(skb)->flush |= flush;
+}
+
static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
unsigned short type,
const void *daddr, const void *saddr,
diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c
index efdaaab735fc..c666ff0dd88b 100644
--- a/net/ethernet/eth.c
+++ b/net/ethernet/eth.c
@@ -474,7 +474,7 @@ struct sk_buff **eth_gro_receive(struct sk_buff **head,
out_unlock:
rcu_read_unlock();
out:
- NAPI_GRO_CB(skb)->flush |= flush;
+ skb_gro_flush_final(skb, pp, flush);
return pp;
}
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 685ba53df2d1..602d40f43687 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1423,7 +1423,7 @@ out_unlock:
rcu_read_unlock();
out:
- NAPI_GRO_CB(skb)->flush |= flush;
+ skb_gro_flush_final(skb, pp, flush);
return pp;
}
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index fc7b4017ba24..0838e6d01d2e 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -253,7 +253,7 @@ out_unlock:
rcu_read_unlock();
out:
- NAPI_GRO_CB(skb)->flush |= flush;
+ skb_gro_flush_final(skb, pp, flush);
return pp;
}
From 25393d3fc055b76587fcc91627aee8c345400c3a Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:39:44 +0100
Subject: [PATCH 10/13] net: Prepare gro for packet consuming gro callbacks
The upcomming IPsec ESP gro callbacks will consume the skb,
so prepare for that.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/linux/netdevice.h | 9 +++++++++
net/core/dev.c | 7 +++++++
net/xfrm/Kconfig | 4 ++++
3 files changed, 20 insertions(+)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index f9da3acfee9a..9e5d1cd9d975 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -352,6 +352,7 @@ enum gro_result {
GRO_HELD,
GRO_NORMAL,
GRO_DROP,
+ GRO_CONSUMED,
};
typedef enum gro_result gro_result_t;
@@ -2661,10 +2662,18 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
remcsum_unadjust((__sum16 *)ptr, grc->delta);
}
+#ifdef CONFIG_XFRM_OFFLOAD
+static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
+{
+ if (PTR_ERR(pp) != -EINPROGRESS)
+ NAPI_GRO_CB(skb)->flush |= flush;
+}
+#else
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
{
NAPI_GRO_CB(skb)->flush |= flush;
}
+#endif
static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
unsigned short type,
diff --git a/net/core/dev.c b/net/core/dev.c
index 3e1a60102e64..64efbb9e4436 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4505,6 +4505,11 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
if (&ptype->list == head)
goto normal;
+ if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
+ ret = GRO_CONSUMED;
+ goto ok;
+ }
+
same_flow = NAPI_GRO_CB(skb)->same_flow;
ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
@@ -4609,6 +4614,7 @@ static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
case GRO_HELD:
case GRO_MERGED:
+ case GRO_CONSUMED:
break;
}
@@ -4680,6 +4686,7 @@ static gro_result_t napi_frags_finish(struct napi_struct *napi,
break;
case GRO_MERGED:
+ case GRO_CONSUMED:
break;
}
diff --git a/net/xfrm/Kconfig b/net/xfrm/Kconfig
index bda1a13628a8..a484451c72ec 100644
--- a/net/xfrm/Kconfig
+++ b/net/xfrm/Kconfig
@@ -5,6 +5,10 @@ config XFRM
bool
depends on NET
+config XFRM_OFFLOAD
+ bool
+ depends on XFRM
+
config XFRM_ALGO
tristate
select XFRM
From 1e29537034e388c7e72eac43cfcda1d23131623b Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:39:49 +0100
Subject: [PATCH 11/13] xfrm: Export xfrm_parse_spi.
We need it in the ESP offload handlers, so export it.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 1 +
net/xfrm/xfrm_input.c | 1 +
2 files changed, 2 insertions(+)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 287635df4eef..fe8db3d87e4f 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -1520,6 +1520,7 @@ int xfrm4_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
int encap_type);
int xfrm4_transport_finish(struct sk_buff *skb, int async);
int xfrm4_rcv(struct sk_buff *skb);
+int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq);
static inline int xfrm4_rcv_spi(struct sk_buff *skb, int nexthdr, __be32 spi)
{
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index d8f913bb6919..86f8a8de5252 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -170,6 +170,7 @@ int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
*seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
return 0;
}
+EXPORT_SYMBOL(xfrm_parse_spi);
int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
{
From 54ef207ac8f7a17d677082157a29f4df8499dc81 Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:39:54 +0100
Subject: [PATCH 12/13] xfrm: Extend the sec_path for IPsec offloading
We need to keep per packet offloading informations across
the layers. So we extend the sec_path to carry these for
the input and output offload codepath.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 41 +++++++++++++++++++++++++++++++++++++++++
net/xfrm/xfrm_input.c | 2 ++
2 files changed, 43 insertions(+)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index fe8db3d87e4f..10086a0986f8 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -498,6 +498,7 @@ struct xfrm_tmpl {
};
#define XFRM_MAX_DEPTH 6
+#define XFRM_MAX_OFFLOAD_DEPTH 1
struct xfrm_policy_walk_entry {
struct list_head all;
@@ -973,10 +974,41 @@ static inline void xfrm_dst_destroy(struct xfrm_dst *xdst)
void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev);
+struct xfrm_offload {
+ /* Output sequence number for replay protection on offloading. */
+ struct {
+ __u32 low;
+ __u32 hi;
+ } seq;
+
+ __u32 flags;
+#define SA_DELETE_REQ 1
+#define CRYPTO_DONE 2
+#define CRYPTO_NEXT_DONE 4
+#define CRYPTO_FALLBACK 8
+#define XFRM_GSO_SEGMENT 16
+#define XFRM_GRO 32
+
+ __u32 status;
+#define CRYPTO_SUCCESS 1
+#define CRYPTO_GENERIC_ERROR 2
+#define CRYPTO_TRANSPORT_AH_AUTH_FAILED 4
+#define CRYPTO_TRANSPORT_ESP_AUTH_FAILED 8
+#define CRYPTO_TUNNEL_AH_AUTH_FAILED 16
+#define CRYPTO_TUNNEL_ESP_AUTH_FAILED 32
+#define CRYPTO_INVALID_PACKET_SYNTAX 64
+#define CRYPTO_INVALID_PROTOCOL 128
+
+ __u8 proto;
+};
+
struct sec_path {
atomic_t refcnt;
int len;
+ int olen;
+
struct xfrm_state *xvec[XFRM_MAX_DEPTH];
+ struct xfrm_offload ovec[XFRM_MAX_OFFLOAD_DEPTH];
};
static inline int secpath_exists(struct sk_buff *skb)
@@ -1776,6 +1808,15 @@ static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb)
{
return skb->sp->xvec[skb->sp->len - 1];
}
+static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb)
+{
+ struct sec_path *sp = skb->sp;
+
+ if (!sp || !sp->olen || sp->len != sp->olen)
+ return NULL;
+
+ return &sp->ovec[sp->olen - 1];
+}
#endif
static inline int xfrm_mark_get(struct nlattr **attrs, struct xfrm_mark *m)
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index 86f8a8de5252..d2ff71230864 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -105,6 +105,8 @@ struct sec_path *secpath_dup(struct sec_path *src)
return NULL;
sp->len = 0;
+ sp->olen = 0;
+
if (src) {
int i;
From 7785bba299a8dc8fe8390a0183dad3cafb3f1d80 Mon Sep 17 00:00:00 2001
From: Steffen Klassert <steffen.klassert@secunet.com>
Date: Wed, 15 Feb 2017 09:40:00 +0100
Subject: [PATCH 13/13] esp: Add a software GRO codepath
This patch adds GRO ifrastructure and callbacks for ESP on
ipv4 and ipv6.
In case the GRO layer detects an ESP packet, the
esp{4,6}_gro_receive() function does a xfrm state lookup
and calls the xfrm input layer if it finds a matching state.
The packet will be decapsulated and reinjected it into layer 2.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/net/xfrm.h | 1 +
net/ipv4/Kconfig | 13 ++++
net/ipv4/Makefile | 1 +
net/ipv4/esp4_offload.c | 106 +++++++++++++++++++++++++++++++
net/ipv4/xfrm4_input.c | 6 ++
net/ipv4/xfrm4_mode_transport.c | 4 +-
net/ipv6/Kconfig | 13 ++++
net/ipv6/Makefile | 1 +
net/ipv6/esp6_offload.c | 108 ++++++++++++++++++++++++++++++++
net/ipv6/xfrm6_input.c | 7 +++
net/ipv6/xfrm6_mode_transport.c | 4 +-
net/xfrm/xfrm_input.c | 31 +++++++--
12 files changed, 288 insertions(+), 7 deletions(-)
create mode 100644 net/ipv4/esp4_offload.c
create mode 100644 net/ipv6/esp6_offload.c
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 10086a0986f8..14d82bf16692 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -682,6 +682,7 @@ struct xfrm_spi_skb_cb {
unsigned int daddroff;
unsigned int family;
+ __be32 seq;
};
#define XFRM_SPI_SKB_CB(__skb) ((struct xfrm_spi_skb_cb *)&((__skb)->cb[0]))
diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig
index 6e7baaf814c6..e0878c3f66a8 100644
--- a/net/ipv4/Kconfig
+++ b/net/ipv4/Kconfig
@@ -360,6 +360,19 @@ config INET_ESP
If unsure, say Y.
+config INET_ESP_OFFLOAD
+ tristate "IP: ESP transformation offload"
+ depends on INET_ESP
+ select XFRM_OFFLOAD
+ default n
+ ---help---
+ Support for ESP transformation offload. This makes sense
+ only if this system really does IPsec and want to do it
+ with high throughput. A typical desktop system does not
+ need it, even if it does IPsec.
+
+ If unsure, say N.
+
config INET_IPCOMP
tristate "IP: IPComp transformation"
select INET_XFRM_TUNNEL
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 48af58a5686e..c6d4238ff94a 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_NET_IPVTI) += ip_vti.o
obj-$(CONFIG_SYN_COOKIES) += syncookies.o
obj-$(CONFIG_INET_AH) += ah4.o
obj-$(CONFIG_INET_ESP) += esp4.o
+obj-$(CONFIG_INET_ESP_OFFLOAD) += esp4_offload.o
obj-$(CONFIG_INET_IPCOMP) += ipcomp.o
obj-$(CONFIG_INET_XFRM_TUNNEL) += xfrm4_tunnel.o
obj-$(CONFIG_INET_XFRM_MODE_BEET) += xfrm4_mode_beet.o
diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c
new file mode 100644
index 000000000000..1de442632406
--- /dev/null
+++ b/net/ipv4/esp4_offload.c
@@ -0,0 +1,106 @@
+/*
+ * IPV4 GSO/GRO offload support
+ * Linux INET implementation
+ *
+ * Copyright (C) 2016 secunet Security Networks AG
+ * Author: Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * ESP GRO support
+ */
+
+#include <linux/skbuff.h>
+#include <linux/init.h>
+#include <net/protocol.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <net/ip.h>
+#include <net/xfrm.h>
+#include <net/esp.h>
+#include <linux/scatterlist.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <net/udp.h>
+
+static struct sk_buff **esp4_gro_receive(struct sk_buff **head,
+ struct sk_buff *skb)
+{
+ int offset = skb_gro_offset(skb);
+ struct xfrm_offload *xo;
+ struct xfrm_state *x;
+ __be32 seq;
+ __be32 spi;
+ int err;
+
+ skb_pull(skb, offset);
+
+ if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
+ goto out;
+
+ err = secpath_set(skb);
+ if (err)
+ goto out;
+
+ if (skb->sp->len == XFRM_MAX_DEPTH)
+ goto out;
+
+ x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
+ (xfrm_address_t *)&ip_hdr(skb)->daddr,
+ spi, IPPROTO_ESP, AF_INET);
+ if (!x)
+ goto out;
+
+ skb->sp->xvec[skb->sp->len++] = x;
+ skb->sp->olen++;
+
+ xo = xfrm_offload(skb);
+ if (!xo) {
+ xfrm_state_put(x);
+ goto out;
+ }
+ xo->flags |= XFRM_GRO;
+
+ XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
+ XFRM_SPI_SKB_CB(skb)->family = AF_INET;
+ XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
+ XFRM_SPI_SKB_CB(skb)->seq = seq;
+
+ /* We don't need to handle errors from xfrm_input, it does all
+ * the error handling and frees the resources on error. */
+ xfrm_input(skb, IPPROTO_ESP, spi, -2);
+
+ return ERR_PTR(-EINPROGRESS);
+out:
+ skb_push(skb, offset);
+ NAPI_GRO_CB(skb)->same_flow = 0;
+ NAPI_GRO_CB(skb)->flush = 1;
+
+ return NULL;
+}
+
+static const struct net_offload esp4_offload = {
+ .callbacks = {
+ .gro_receive = esp4_gro_receive,
+ },
+};
+
+static int __init esp4_offload_init(void)
+{
+ return inet_add_offload(&esp4_offload, IPPROTO_ESP);
+}
+
+static void __exit esp4_offload_exit(void)
+{
+ inet_del_offload(&esp4_offload, IPPROTO_ESP);
+}
+
+module_init(esp4_offload_init);
+module_exit(esp4_offload_exit);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
diff --git a/net/ipv4/xfrm4_input.c b/net/ipv4/xfrm4_input.c
index 62e1e72db461..1fc684111ce6 100644
--- a/net/ipv4/xfrm4_input.c
+++ b/net/ipv4/xfrm4_input.c
@@ -40,6 +40,7 @@ drop:
int xfrm4_transport_finish(struct sk_buff *skb, int async)
{
+ struct xfrm_offload *xo = xfrm_offload(skb);
struct iphdr *iph = ip_hdr(skb);
iph->protocol = XFRM_MODE_SKB_CB(skb)->protocol;
@@ -53,6 +54,11 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async)
iph->tot_len = htons(skb->len);
ip_send_check(iph);
+ if (xo && (xo->flags & XFRM_GRO)) {
+ skb_mac_header_rebuild(skb);
+ return 0;
+ }
+
NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
dev_net(skb->dev), NULL, skb, skb->dev, NULL,
xfrm4_rcv_encap_finish);
diff --git a/net/ipv4/xfrm4_mode_transport.c b/net/ipv4/xfrm4_mode_transport.c
index fd840c7d75ea..4acc0508c5eb 100644
--- a/net/ipv4/xfrm4_mode_transport.c
+++ b/net/ipv4/xfrm4_mode_transport.c
@@ -43,6 +43,7 @@ static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
{
int ihl = skb->data - skb_transport_header(skb);
+ struct xfrm_offload *xo = xfrm_offload(skb);
if (skb->transport_header != skb->network_header) {
memmove(skb_transport_header(skb),
@@ -50,7 +51,8 @@ static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
skb->network_header = skb->transport_header;
}
ip_hdr(skb)->tot_len = htons(skb->len + ihl);
- skb_reset_transport_header(skb);
+ if (!xo || !(xo->flags & XFRM_GRO))
+ skb_reset_transport_header(skb);
return 0;
}
diff --git a/net/ipv6/Kconfig b/net/ipv6/Kconfig
index ec1267e2bd1f..b2a85cca91f7 100644
--- a/net/ipv6/Kconfig
+++ b/net/ipv6/Kconfig
@@ -75,6 +75,19 @@ config INET6_ESP
If unsure, say Y.
+config INET6_ESP_OFFLOAD
+ tristate "IPv6: ESP transformation offload"
+ depends on INET6_ESP
+ select XFRM_OFFLOAD
+ default n
+ ---help---
+ Support for ESP transformation offload. This makes sense
+ only if this system really does IPsec and want to do it
+ with high throughput. A typical desktop system does not
+ need it, even if it does IPsec.
+
+ If unsure, say N.
+
config INET6_IPCOMP
tristate "IPv6: IPComp transformation"
select INET6_XFRM_TUNNEL
diff --git a/net/ipv6/Makefile b/net/ipv6/Makefile
index a9e9fec387ce..217e9ff0e24b 100644
--- a/net/ipv6/Makefile
+++ b/net/ipv6/Makefile
@@ -30,6 +30,7 @@ ipv6-objs += $(ipv6-y)
obj-$(CONFIG_INET6_AH) += ah6.o
obj-$(CONFIG_INET6_ESP) += esp6.o
+obj-$(CONFIG_INET6_ESP_OFFLOAD) += esp6_offload.o
obj-$(CONFIG_INET6_IPCOMP) += ipcomp6.o
obj-$(CONFIG_INET6_XFRM_TUNNEL) += xfrm6_tunnel.o
obj-$(CONFIG_INET6_TUNNEL) += tunnel6.o
diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c
new file mode 100644
index 000000000000..d914eb93204a
--- /dev/null
+++ b/net/ipv6/esp6_offload.c
@@ -0,0 +1,108 @@
+/*
+ * IPV6 GSO/GRO offload support
+ * Linux INET implementation
+ *
+ * Copyright (C) 2016 secunet Security Networks AG
+ * Author: Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * ESP GRO support
+ */
+
+#include <linux/skbuff.h>
+#include <linux/init.h>
+#include <net/protocol.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <net/ip.h>
+#include <net/xfrm.h>
+#include <net/esp.h>
+#include <linux/scatterlist.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <net/ip6_route.h>
+#include <net/ipv6.h>
+#include <linux/icmpv6.h>
+
+static struct sk_buff **esp6_gro_receive(struct sk_buff **head,
+ struct sk_buff *skb)
+{
+ int offset = skb_gro_offset(skb);
+ struct xfrm_offload *xo;
+ struct xfrm_state *x;
+ __be32 seq;
+ __be32 spi;
+ int err;
+
+ skb_pull(skb, offset);
+
+ if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
+ goto out;
+
+ err = secpath_set(skb);
+ if (err)
+ goto out;
+
+ if (skb->sp->len == XFRM_MAX_DEPTH)
+ goto out;
+
+ x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
+ (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
+ spi, IPPROTO_ESP, AF_INET6);
+ if (!x)
+ goto out;
+
+ skb->sp->xvec[skb->sp->len++] = x;
+ skb->sp->olen++;
+
+ xo = xfrm_offload(skb);
+ if (!xo) {
+ xfrm_state_put(x);
+ goto out;
+ }
+ xo->flags |= XFRM_GRO;
+
+ XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
+ XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
+ XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
+ XFRM_SPI_SKB_CB(skb)->seq = seq;
+
+ /* We don't need to handle errors from xfrm_input, it does all
+ * the error handling and frees the resources on error. */
+ xfrm_input(skb, IPPROTO_ESP, spi, -2);
+
+ return ERR_PTR(-EINPROGRESS);
+out:
+ skb_push(skb, offset);
+ NAPI_GRO_CB(skb)->same_flow = 0;
+ NAPI_GRO_CB(skb)->flush = 1;
+
+ return NULL;
+}
+
+static const struct net_offload esp6_offload = {
+ .callbacks = {
+ .gro_receive = esp6_gro_receive,
+ },
+};
+
+static int __init esp6_offload_init(void)
+{
+ return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
+}
+
+static void __exit esp6_offload_exit(void)
+{
+ inet6_del_offload(&esp6_offload, IPPROTO_ESP);
+}
+
+module_init(esp6_offload_init);
+module_exit(esp6_offload_exit);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c
index 662fb2c3e765..08a807b29298 100644
--- a/net/ipv6/xfrm6_input.c
+++ b/net/ipv6/xfrm6_input.c
@@ -33,6 +33,8 @@ EXPORT_SYMBOL(xfrm6_rcv_spi);
int xfrm6_transport_finish(struct sk_buff *skb, int async)
{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+
skb_network_header(skb)[IP6CB(skb)->nhoff] =
XFRM_MODE_SKB_CB(skb)->protocol;
@@ -44,6 +46,11 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async)
ipv6_hdr(skb)->payload_len = htons(skb->len);
__skb_push(skb, skb->data - skb_network_header(skb));
+ if (xo && (xo->flags & XFRM_GRO)) {
+ skb_mac_header_rebuild(skb);
+ return -1;
+ }
+
NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING,
dev_net(skb->dev), NULL, skb, skb->dev, NULL,
ip6_rcv_finish);
diff --git a/net/ipv6/xfrm6_mode_transport.c b/net/ipv6/xfrm6_mode_transport.c
index 4e344105b3fd..4439ee44c8b0 100644
--- a/net/ipv6/xfrm6_mode_transport.c
+++ b/net/ipv6/xfrm6_mode_transport.c
@@ -47,6 +47,7 @@ static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
{
int ihl = skb->data - skb_transport_header(skb);
+ struct xfrm_offload *xo = xfrm_offload(skb);
if (skb->transport_header != skb->network_header) {
memmove(skb_transport_header(skb),
@@ -55,7 +56,8 @@ static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
}
ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
sizeof(struct ipv6hdr));
- skb_reset_transport_header(skb);
+ if (!xo || !(xo->flags & XFRM_GRO))
+ skb_reset_transport_header(skb);
return 0;
}
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index d2ff71230864..46bdb4fbed0b 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -207,14 +207,23 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
unsigned int family;
int decaps = 0;
int async = 0;
+ struct xfrm_offload *xo;
+ bool xfrm_gro = false;
- /* A negative encap_type indicates async resumption. */
if (encap_type < 0) {
- async = 1;
x = xfrm_input_state(skb);
- seq = XFRM_SKB_CB(skb)->seq.input.low;
family = x->outer_mode->afinfo->family;
- goto resume;
+
+ /* An encap_type of -1 indicates async resumption. */
+ if (encap_type == -1) {
+ async = 1;
+ seq = XFRM_SKB_CB(skb)->seq.input.low;
+ goto resume;
+ }
+ /* encap_type < -1 indicates a GRO call. */
+ encap_type = 0;
+ seq = XFRM_SPI_SKB_CB(skb)->seq;
+ goto lock;
}
daddr = (xfrm_address_t *)(skb_network_header(skb) +
@@ -260,6 +269,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
skb->sp->xvec[skb->sp->len++] = x;
+lock:
spin_lock(&x->lock);
if (unlikely(x->km.state != XFRM_STATE_VALID)) {
@@ -381,7 +391,18 @@ resume:
gro_cells_receive(&gro_cells, skb);
return 0;
} else {
- return x->inner_mode->afinfo->transport_finish(skb, async);
+ xo = xfrm_offload(skb);
+ if (xo)
+ xfrm_gro = xo->flags & XFRM_GRO;
+
+ err = x->inner_mode->afinfo->transport_finish(skb, async);
+ if (xfrm_gro) {
+ skb_dst_drop(skb);
+ gro_cells_receive(&gro_cells, skb);
+ return err;
+ }
+
+ return err;
}
drop_unlock: