diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 77eb60d2b496..d8050382b189 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -558,6 +558,15 @@ extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, unsigned int to, struct ts_config *config, struct ts_state *state); +extern __u32 __skb_get_rxhash(struct sk_buff *skb); +static inline __u32 skb_get_rxhash(struct sk_buff *skb) +{ + if (!skb->rxhash) + skb->rxhash = __skb_get_rxhash(skb); + + return skb->rxhash; +} + #ifdef NET_SKBUFF_DATA_USES_OFFSET static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) { diff --git a/net/core/dev.c b/net/core/dev.c index 1ae654391442..586a11cb4398 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2259,6 +2259,83 @@ static inline void ____napi_schedule(struct softnet_data *sd, __raise_softirq_irqoff(NET_RX_SOFTIRQ); } +/* + * __skb_get_rxhash: calculate a flow hash based on src/dst addresses + * and src/dst port numbers. Returns a non-zero hash number on success + * and 0 on failure. + */ +__u32 __skb_get_rxhash(struct sk_buff *skb) +{ + int nhoff, hash = 0; + struct ipv6hdr *ip6; + struct iphdr *ip; + u8 ip_proto; + u32 addr1, addr2, ihl; + union { + u32 v32; + u16 v16[2]; + } ports; + + nhoff = skb_network_offset(skb); + + switch (skb->protocol) { + case __constant_htons(ETH_P_IP): + if (!pskb_may_pull(skb, sizeof(*ip) + nhoff)) + goto done; + + ip = (struct iphdr *) skb->data + nhoff; + ip_proto = ip->protocol; + addr1 = (__force u32) ip->saddr; + addr2 = (__force u32) ip->daddr; + ihl = ip->ihl; + break; + case __constant_htons(ETH_P_IPV6): + if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff)) + goto done; + + ip6 = (struct ipv6hdr *) skb->data + nhoff; + ip_proto = ip6->nexthdr; + addr1 = (__force u32) ip6->saddr.s6_addr32[3]; + addr2 = (__force u32) ip6->daddr.s6_addr32[3]; + ihl = (40 >> 2); + break; + default: + goto done; + } + + switch (ip_proto) { + case IPPROTO_TCP: + case IPPROTO_UDP: + case IPPROTO_DCCP: + case IPPROTO_ESP: + case IPPROTO_AH: + case IPPROTO_SCTP: + case IPPROTO_UDPLITE: + if (pskb_may_pull(skb, (ihl * 4) + 4 + nhoff)) { + ports.v32 = * (__force u32 *) (skb->data + nhoff + + (ihl * 4)); + if (ports.v16[1] < ports.v16[0]) + swap(ports.v16[0], ports.v16[1]); + break; + } + default: + ports.v32 = 0; + break; + } + + /* get a consistent hash (same value on both flow directions) */ + if (addr2 < addr1) + swap(addr1, addr2); + + hash = jhash_3words(addr1, addr2, ports.v32, hashrnd); + if (!hash) + hash = 1; + +done: + return hash; +} +EXPORT_SYMBOL(__skb_get_rxhash); + #ifdef CONFIG_RPS /* One global table that all flow-based protocols share. */ @@ -2273,20 +2350,12 @@ EXPORT_SYMBOL(rps_sock_flow_table); static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow **rflowp) { - struct ipv6hdr *ip6; - struct iphdr *ip; struct netdev_rx_queue *rxqueue; struct rps_map *map; struct rps_dev_flow_table *flow_table; struct rps_sock_flow_table *sock_flow_table; int cpu = -1; - u8 ip_proto; u16 tcpu; - u32 addr1, addr2, ihl; - union { - u32 v32; - u16 v16[2]; - } ports; if (skb_rx_queue_recorded(skb)) { u16 index = skb_get_rx_queue(skb); @@ -2303,60 +2372,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, if (!rxqueue->rps_map && !rxqueue->rps_flow_table) goto done; - if (skb->rxhash) - goto got_hash; /* Skip hash computation on packet header */ - - switch (skb->protocol) { - case __constant_htons(ETH_P_IP): - if (!pskb_may_pull(skb, sizeof(*ip))) - goto done; - - ip = (struct iphdr *) skb->data; - ip_proto = ip->protocol; - addr1 = (__force u32) ip->saddr; - addr2 = (__force u32) ip->daddr; - ihl = ip->ihl; - break; - case __constant_htons(ETH_P_IPV6): - if (!pskb_may_pull(skb, sizeof(*ip6))) - goto done; - - ip6 = (struct ipv6hdr *) skb->data; - ip_proto = ip6->nexthdr; - addr1 = (__force u32) ip6->saddr.s6_addr32[3]; - addr2 = (__force u32) ip6->daddr.s6_addr32[3]; - ihl = (40 >> 2); - break; - default: + if (!skb_get_rxhash(skb)) goto done; - } - switch (ip_proto) { - case IPPROTO_TCP: - case IPPROTO_UDP: - case IPPROTO_DCCP: - case IPPROTO_ESP: - case IPPROTO_AH: - case IPPROTO_SCTP: - case IPPROTO_UDPLITE: - if (pskb_may_pull(skb, (ihl * 4) + 4)) { - ports.v32 = * (__force u32 *) (skb->data + (ihl * 4)); - if (ports.v16[1] < ports.v16[0]) - swap(ports.v16[0], ports.v16[1]); - break; - } - default: - ports.v32 = 0; - break; - } - /* get a consistent hash (same value on both flow directions) */ - if (addr2 < addr1) - swap(addr1, addr2); - skb->rxhash = jhash_3words(addr1, addr2, ports.v32, hashrnd); - if (!skb->rxhash) - skb->rxhash = 1; - -got_hash: flow_table = rcu_dereference(rxqueue->rps_flow_table); sock_flow_table = rcu_dereference(rps_sock_flow_table); if (flow_table && sock_flow_table) {