mirror of
https://github.com/torvalds/linux.git
synced 2024-12-12 22:23:55 +00:00
b73c3d0e4f
For a connected socket we can precompute the flow hash for setting in skb->hash on output. This is a performance advantage over calculating the skb->hash for every packet on the connection. The computation is done using the common hash algorithm to be consistent with computations done for packets of the connection in other states where thers is no socket (e.g. time-wait, syn-recv, syn-cookies). This patch adds sk_txhash to the sock structure. inet_set_txhash and ip6_set_txhash functions are added which are called from points in TCP and UDP where socket moves to established state. skb_set_hash_from_sk is a function which sets skb->hash from the sock txhash value. This is called in UDP and TCP transmit path when transmitting within the context of a socket. Tested: ran super_netperf with 200 TCP_RR streams over a vxlan interface (in this case skb_get_hash called on every TX packet to create a UDP source port). Before fix: 95.02% CPU utilization 154/256/505 90/95/99% latencies 1.13042e+06 tps Time in functions: 0.28% skb_flow_dissect 0.21% __skb_get_hash After fix: 94.95% CPU utilization 156/254/485 90/95/99% latencies 1.15447e+06 Neither __skb_get_hash nor skb_flow_dissect appear in perf Signed-off-by: Tom Herbert <therbert@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
124 lines
3.0 KiB
C
124 lines
3.0 KiB
C
/*
|
|
* common UDP/RAW code
|
|
* Linux INET implementation
|
|
*
|
|
* Authors:
|
|
* Hideaki YOSHIFUJI <yoshfuji@linux-ipv6.org>
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/module.h>
|
|
#include <linux/ip.h>
|
|
#include <linux/in.h>
|
|
#include <net/ip.h>
|
|
#include <net/sock.h>
|
|
#include <net/route.h>
|
|
#include <net/tcp_states.h>
|
|
|
|
int ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
|
|
{
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
struct sockaddr_in *usin = (struct sockaddr_in *) uaddr;
|
|
struct flowi4 *fl4;
|
|
struct rtable *rt;
|
|
__be32 saddr;
|
|
int oif;
|
|
int err;
|
|
|
|
|
|
if (addr_len < sizeof(*usin))
|
|
return -EINVAL;
|
|
|
|
if (usin->sin_family != AF_INET)
|
|
return -EAFNOSUPPORT;
|
|
|
|
sk_dst_reset(sk);
|
|
|
|
lock_sock(sk);
|
|
|
|
oif = sk->sk_bound_dev_if;
|
|
saddr = inet->inet_saddr;
|
|
if (ipv4_is_multicast(usin->sin_addr.s_addr)) {
|
|
if (!oif)
|
|
oif = inet->mc_index;
|
|
if (!saddr)
|
|
saddr = inet->mc_addr;
|
|
}
|
|
fl4 = &inet->cork.fl.u.ip4;
|
|
rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr,
|
|
RT_CONN_FLAGS(sk), oif,
|
|
sk->sk_protocol,
|
|
inet->inet_sport, usin->sin_port, sk);
|
|
if (IS_ERR(rt)) {
|
|
err = PTR_ERR(rt);
|
|
if (err == -ENETUNREACH)
|
|
IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
|
|
goto out;
|
|
}
|
|
|
|
if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) {
|
|
ip_rt_put(rt);
|
|
err = -EACCES;
|
|
goto out;
|
|
}
|
|
if (!inet->inet_saddr)
|
|
inet->inet_saddr = fl4->saddr; /* Update source address */
|
|
if (!inet->inet_rcv_saddr) {
|
|
inet->inet_rcv_saddr = fl4->saddr;
|
|
if (sk->sk_prot->rehash)
|
|
sk->sk_prot->rehash(sk);
|
|
}
|
|
inet->inet_daddr = fl4->daddr;
|
|
inet->inet_dport = usin->sin_port;
|
|
sk->sk_state = TCP_ESTABLISHED;
|
|
inet_set_txhash(sk);
|
|
inet->inet_id = jiffies;
|
|
|
|
sk_dst_set(sk, &rt->dst);
|
|
err = 0;
|
|
out:
|
|
release_sock(sk);
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(ip4_datagram_connect);
|
|
|
|
/* Because UDP xmit path can manipulate sk_dst_cache without holding
|
|
* socket lock, we need to use sk_dst_set() here,
|
|
* even if we own the socket lock.
|
|
*/
|
|
void ip4_datagram_release_cb(struct sock *sk)
|
|
{
|
|
const struct inet_sock *inet = inet_sk(sk);
|
|
const struct ip_options_rcu *inet_opt;
|
|
__be32 daddr = inet->inet_daddr;
|
|
struct dst_entry *dst;
|
|
struct flowi4 fl4;
|
|
struct rtable *rt;
|
|
|
|
rcu_read_lock();
|
|
|
|
dst = __sk_dst_get(sk);
|
|
if (!dst || !dst->obsolete || dst->ops->check(dst, 0)) {
|
|
rcu_read_unlock();
|
|
return;
|
|
}
|
|
inet_opt = rcu_dereference(inet->inet_opt);
|
|
if (inet_opt && inet_opt->opt.srr)
|
|
daddr = inet_opt->opt.faddr;
|
|
rt = ip_route_output_ports(sock_net(sk), &fl4, sk, daddr,
|
|
inet->inet_saddr, inet->inet_dport,
|
|
inet->inet_sport, sk->sk_protocol,
|
|
RT_CONN_FLAGS(sk), sk->sk_bound_dev_if);
|
|
|
|
dst = !IS_ERR(rt) ? &rt->dst : NULL;
|
|
sk_dst_set(sk, dst);
|
|
|
|
rcu_read_unlock();
|
|
}
|
|
EXPORT_SYMBOL_GPL(ip4_datagram_release_cb);
|