2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
*
|
|
|
|
* PACKET - implements raw packet sockets.
|
|
|
|
*
|
2005-05-05 23:16:16 +00:00
|
|
|
* Authors: Ross Biro
|
2005-04-16 22:20:36 +00:00
|
|
|
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
|
|
|
|
* Alan Cox, <gw4pts@gw4pts.ampr.org>
|
|
|
|
*
|
2007-02-09 14:25:10 +00:00
|
|
|
* Fixes:
|
2005-04-16 22:20:36 +00:00
|
|
|
* Alan Cox : verify_area() now used correctly
|
|
|
|
* Alan Cox : new skbuff lists, look ma no backlogs!
|
|
|
|
* Alan Cox : tidied skbuff lists.
|
|
|
|
* Alan Cox : Now uses generic datagram routines I
|
|
|
|
* added. Also fixed the peek/read crash
|
|
|
|
* from all old Linux datagram code.
|
|
|
|
* Alan Cox : Uses the improved datagram code.
|
|
|
|
* Alan Cox : Added NULL's for socket options.
|
|
|
|
* Alan Cox : Re-commented the code.
|
|
|
|
* Alan Cox : Use new kernel side addressing
|
|
|
|
* Rob Janssen : Correct MTU usage.
|
|
|
|
* Dave Platt : Counter leaks caused by incorrect
|
|
|
|
* interrupt locking and some slightly
|
|
|
|
* dubious gcc output. Can you read
|
|
|
|
* compiler: it said _VOLATILE_
|
|
|
|
* Richard Kooijman : Timestamp fixes.
|
|
|
|
* Alan Cox : New buffers. Use sk->mac.raw.
|
|
|
|
* Alan Cox : sendmsg/recvmsg support.
|
|
|
|
* Alan Cox : Protocol setting support
|
|
|
|
* Alexey Kuznetsov : Untied from IPv4 stack.
|
|
|
|
* Cyrus Durgin : Fixed kerneld for kmod.
|
|
|
|
* Michal Ostrowski : Module initialization cleanup.
|
2007-02-09 14:25:10 +00:00
|
|
|
* Ulises Alonso : Frame number limit removal and
|
2005-04-16 22:20:36 +00:00
|
|
|
* packet_set_ring memory leak.
|
2005-09-21 07:11:37 +00:00
|
|
|
* Eric Biederman : Allow for > 8 byte hardware addresses.
|
|
|
|
* The convention is that longer addresses
|
|
|
|
* will simply extend the hardware address
|
2007-02-09 14:25:10 +00:00
|
|
|
* byte arrays at the end of sockaddr_ll
|
2005-09-21 07:11:37 +00:00
|
|
|
* and packet_mreq.
|
2009-05-19 05:11:22 +00:00
|
|
|
* Johann Baudy : Added TX RING.
|
2011-08-19 10:18:16 +00:00
|
|
|
* Chetan Loke : Implemented TPACKET_V3 block abstraction
|
|
|
|
* layer.
|
|
|
|
* Copyright (C) 2011, <lokec@ccs.neu.edu>
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/mm.h>
|
2006-01-11 20:17:47 +00:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/fcntl.h>
|
|
|
|
#include <linux/socket.h>
|
|
|
|
#include <linux/in.h>
|
|
|
|
#include <linux/inet.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/if_packet.h>
|
|
|
|
#include <linux/wireless.h>
|
2007-02-05 07:33:10 +00:00
|
|
|
#include <linux/kernel.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/kmod.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
#include <linux/vmalloc.h>
|
2007-09-12 10:01:34 +00:00
|
|
|
#include <net/net_namespace.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/ip.h>
|
|
|
|
#include <net/protocol.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <net/sock.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/timer.h>
|
2016-12-24 19:46:01 +00:00
|
|
|
#include <linux/uaccess.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/ioctls.h>
|
|
|
|
#include <asm/page.h>
|
2006-10-19 20:08:53 +00:00
|
|
|
#include <asm/cacheflush.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/io.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/poll.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/init.h>
|
2009-01-30 22:12:06 +00:00
|
|
|
#include <linux/mutex.h>
|
2009-10-27 01:40:35 +00:00
|
|
|
#include <linux/if_vlan.h>
|
2010-02-05 04:24:10 +00:00
|
|
|
#include <linux/virtio_net.h>
|
2010-04-07 22:41:28 +00:00
|
|
|
#include <linux/errqueue.h>
|
2010-06-02 12:53:56 +00:00
|
|
|
#include <linux/net_tstamp.h>
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
#include <linux/percpu.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_INET
|
|
|
|
#include <net/inet_common.h>
|
|
|
|
#endif
|
2015-08-15 02:31:34 +00:00
|
|
|
#include <linux/bpf.h>
|
2016-06-07 16:06:34 +00:00
|
|
|
#include <net/compat.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-08-13 05:49:39 +00:00
|
|
|
#include "internal.h"
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
Assumptions:
|
|
|
|
- if device has no dev->hard_header routine, it adds and removes ll header
|
|
|
|
inside itself. In this case ll header is invisible outside of device,
|
|
|
|
but higher levels still should reserve dev->hard_header_len.
|
|
|
|
Some devices are enough clever to reallocate skb, when header
|
|
|
|
will not fit to reserved space (tunnel), another ones are silly
|
|
|
|
(PPP).
|
|
|
|
- packet socket receives packets with pulled ll header,
|
|
|
|
so that SOCK_RAW should push it back.
|
|
|
|
|
|
|
|
On receive:
|
|
|
|
-----------
|
|
|
|
|
|
|
|
Incoming, dev->hard_header!=NULL
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> ll header
|
|
|
|
data -> data
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
Outgoing, dev->hard_header!=NULL
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> ll header
|
|
|
|
data -> ll header
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
Incoming, dev->hard_header==NULL
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> UNKNOWN position. It is very likely, that it points to ll
|
|
|
|
header. PPP makes it, that is wrong, because introduce
|
2007-07-19 01:44:35 +00:00
|
|
|
assymetry between rx and tx paths.
|
2007-04-11 04:21:55 +00:00
|
|
|
data -> data
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
Outgoing, dev->hard_header==NULL
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> data. ll header is still not built!
|
|
|
|
data -> data
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
Resume
|
|
|
|
If dev->hard_header==NULL we are unlikely to restore sensible ll header.
|
|
|
|
|
|
|
|
|
|
|
|
On transmit:
|
|
|
|
------------
|
|
|
|
|
|
|
|
dev->hard_header != NULL
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> ll header
|
|
|
|
data -> ll header
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
dev->hard_header == NULL (ll header is added by device, we cannot control it)
|
2007-04-11 04:21:55 +00:00
|
|
|
mac_header -> data
|
|
|
|
data -> data
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
We should set nh.raw on output to correct posistion,
|
|
|
|
packet classifier depends on it.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Private packet socket structures. */
|
|
|
|
|
2005-09-21 07:11:37 +00:00
|
|
|
/* identical to struct packet_mreq except it has
|
|
|
|
* a longer address field.
|
|
|
|
*/
|
2009-07-21 21:57:59 +00:00
|
|
|
struct packet_mreq_max {
|
2005-09-21 07:11:37 +00:00
|
|
|
int mr_ifindex;
|
|
|
|
unsigned short mr_type;
|
|
|
|
unsigned short mr_alen;
|
|
|
|
unsigned char mr_address[MAX_ADDR_LEN];
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
2007-05-29 20:12:50 +00:00
|
|
|
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr {
|
|
|
|
struct tpacket_hdr *h1;
|
|
|
|
struct tpacket2_hdr *h2;
|
|
|
|
struct tpacket3_hdr *h3;
|
|
|
|
void *raw;
|
|
|
|
};
|
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
|
2009-05-19 05:11:22 +00:00
|
|
|
int closing, int tx_ring);
|
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
#define V3_ALIGNMENT (8)
|
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
#define BLK_PLUS_PRIV(sz_of_priv) \
|
|
|
|
(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
|
|
|
|
|
|
|
|
#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
|
|
|
|
#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
|
|
|
|
#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
|
|
|
|
#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
|
|
|
|
#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
|
|
|
|
#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
|
|
|
|
#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
struct packet_sock;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct packet_type *pt, struct net_device *orig_dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
static void *packet_previous_frame(struct packet_sock *po,
|
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
int status);
|
|
|
|
static void packet_increment_head(struct packet_ring_buffer *buff);
|
2017-05-24 15:34:11 +00:00
|
|
|
static int prb_curr_blk_in_use(struct tpacket_block_desc *);
|
2011-08-25 10:43:30 +00:00
|
|
|
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *);
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_retire_current_block(struct tpacket_kbdq_core *,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *, unsigned int status);
|
2011-08-25 10:43:30 +00:00
|
|
|
static int prb_queue_frozen(struct tpacket_kbdq_core *);
|
|
|
|
static void prb_open_block(struct tpacket_kbdq_core *,
|
|
|
|
struct tpacket_block_desc *);
|
2017-10-24 08:46:26 +00:00
|
|
|
static void prb_retire_rx_blk_timer_expired(struct timer_list *);
|
2011-08-25 10:43:30 +00:00
|
|
|
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
|
|
|
|
static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
|
|
|
|
static void prb_clear_rxhash(struct tpacket_kbdq_core *,
|
|
|
|
struct tpacket3_hdr *);
|
|
|
|
static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
|
|
|
|
struct tpacket3_hdr *);
|
2005-04-16 22:20:36 +00:00
|
|
|
static void packet_flush_mclist(struct sock *sk);
|
2018-05-02 11:01:33 +00:00
|
|
|
static u16 packet_pick_tx_queue(struct sk_buff *skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-02-05 07:33:10 +00:00
|
|
|
struct packet_skb_cb {
|
|
|
|
union {
|
|
|
|
struct sockaddr_pkt pkt;
|
2015-03-01 12:58:28 +00:00
|
|
|
union {
|
|
|
|
/* Trick: alias skb original length with
|
|
|
|
* ll.sll_family and ll.protocol in order
|
|
|
|
* to save room.
|
|
|
|
*/
|
|
|
|
unsigned int origlen;
|
|
|
|
struct sockaddr_ll ll;
|
|
|
|
};
|
2007-02-05 07:33:10 +00:00
|
|
|
} sa;
|
|
|
|
};
|
|
|
|
|
2015-09-23 18:45:08 +00:00
|
|
|
#define vio_le() virtio_legacy_is_little_endian()
|
|
|
|
|
2007-02-05 07:33:10 +00:00
|
|
|
#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
|
2007-02-05 07:31:32 +00:00
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
|
2011-08-19 10:18:16 +00:00
|
|
|
#define GET_PBLOCK_DESC(x, bid) \
|
2011-08-25 10:43:30 +00:00
|
|
|
((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
|
2011-08-19 10:18:16 +00:00
|
|
|
#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
|
2011-08-25 10:43:30 +00:00
|
|
|
((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
|
2011-08-19 10:18:16 +00:00
|
|
|
#define GET_NEXT_PRB_BLK_NUM(x) \
|
|
|
|
(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
|
|
|
|
((x)->kactive_blk_num+1) : 0)
|
|
|
|
|
2011-07-05 08:45:05 +00:00
|
|
|
static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
|
|
|
|
static void __fanout_link(struct sock *sk, struct packet_sock *po);
|
|
|
|
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
static int packet_direct_xmit(struct sk_buff *skb)
|
|
|
|
{
|
2018-05-02 11:01:33 +00:00
|
|
|
return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
}
|
|
|
|
|
2013-12-06 10:36:15 +00:00
|
|
|
static struct net_device *packet_cached_dev_get(struct packet_sock *po)
|
|
|
|
{
|
|
|
|
struct net_device *dev;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
dev = rcu_dereference(po->cached_dev);
|
|
|
|
if (likely(dev))
|
|
|
|
dev_hold(dev);
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return dev;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_cached_dev_assign(struct packet_sock *po,
|
|
|
|
struct net_device *dev)
|
|
|
|
{
|
|
|
|
rcu_assign_pointer(po->cached_dev, dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_cached_dev_reset(struct packet_sock *po)
|
|
|
|
{
|
|
|
|
RCU_INIT_POINTER(po->cached_dev, NULL);
|
|
|
|
}
|
|
|
|
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
static bool packet_use_direct_xmit(const struct packet_sock *po)
|
|
|
|
{
|
|
|
|
return po->xmit == packet_direct_xmit;
|
|
|
|
}
|
|
|
|
|
packet: check for ndo_select_queue during queue selection
Mathias reported that on an AMD Geode LX embedded board (ALiX)
with ath9k driver PACKET_QDISC_BYPASS, introduced in commit
d346a3fae3ff ("packet: introduce PACKET_QDISC_BYPASS socket
option"), triggers a WARN_ON() coming from the driver itself
via 066dae93bdf ("ath9k: rework tx queue selection and fix
queue stopping/waking").
The reason why this happened is that ndo_select_queue() call
is not invoked from direct xmit path i.e. for ieee80211 subsystem
that sets queue and TID (similar to 802.1d tag) which is being
put into the frame through 802.11e (WMM, QoS). If that is not
set, pending frame counter for e.g. ath9k can get messed up.
So the WARN_ON() in ath9k is absolutely legitimate. Generally,
the hw queue selection in ieee80211 depends on the type of
traffic, and priorities are set according to ieee80211_ac_numbers
mapping; working in a similar way as DiffServ only on a lower
layer, so that the AP can favour frames that have "real-time"
requirements like voice or video data frames.
Therefore, check for presence of ndo_select_queue() in netdev
ops and, if available, invoke it with a fallback handler to
__packet_pick_tx_queue(), so that driver such as bnx2x, ixgbe,
or mlx4 can still select a hw queue for transmission in
relation to the current CPU while e.g. ieee80211 subsystem
can make their own choices.
Reported-by: Mathias Kretschmer <mathias.kretschmer@fokus.fraunhofer.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-02-16 14:55:22 +00:00
|
|
|
static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
{
|
2013-12-12 21:39:55 +00:00
|
|
|
return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:33 +00:00
|
|
|
static u16 packet_pick_tx_queue(struct sk_buff *skb)
|
packet: check for ndo_select_queue during queue selection
Mathias reported that on an AMD Geode LX embedded board (ALiX)
with ath9k driver PACKET_QDISC_BYPASS, introduced in commit
d346a3fae3ff ("packet: introduce PACKET_QDISC_BYPASS socket
option"), triggers a WARN_ON() coming from the driver itself
via 066dae93bdf ("ath9k: rework tx queue selection and fix
queue stopping/waking").
The reason why this happened is that ndo_select_queue() call
is not invoked from direct xmit path i.e. for ieee80211 subsystem
that sets queue and TID (similar to 802.1d tag) which is being
put into the frame through 802.11e (WMM, QoS). If that is not
set, pending frame counter for e.g. ath9k can get messed up.
So the WARN_ON() in ath9k is absolutely legitimate. Generally,
the hw queue selection in ieee80211 depends on the type of
traffic, and priorities are set according to ieee80211_ac_numbers
mapping; working in a similar way as DiffServ only on a lower
layer, so that the AP can favour frames that have "real-time"
requirements like voice or video data frames.
Therefore, check for presence of ndo_select_queue() in netdev
ops and, if available, invoke it with a fallback handler to
__packet_pick_tx_queue(), so that driver such as bnx2x, ixgbe,
or mlx4 can still select a hw queue for transmission in
relation to the current CPU while e.g. ieee80211 subsystem
can make their own choices.
Reported-by: Mathias Kretschmer <mathias.kretschmer@fokus.fraunhofer.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-02-16 14:55:22 +00:00
|
|
|
{
|
2018-05-02 11:01:33 +00:00
|
|
|
struct net_device *dev = skb->dev;
|
packet: check for ndo_select_queue during queue selection
Mathias reported that on an AMD Geode LX embedded board (ALiX)
with ath9k driver PACKET_QDISC_BYPASS, introduced in commit
d346a3fae3ff ("packet: introduce PACKET_QDISC_BYPASS socket
option"), triggers a WARN_ON() coming from the driver itself
via 066dae93bdf ("ath9k: rework tx queue selection and fix
queue stopping/waking").
The reason why this happened is that ndo_select_queue() call
is not invoked from direct xmit path i.e. for ieee80211 subsystem
that sets queue and TID (similar to 802.1d tag) which is being
put into the frame through 802.11e (WMM, QoS). If that is not
set, pending frame counter for e.g. ath9k can get messed up.
So the WARN_ON() in ath9k is absolutely legitimate. Generally,
the hw queue selection in ieee80211 depends on the type of
traffic, and priorities are set according to ieee80211_ac_numbers
mapping; working in a similar way as DiffServ only on a lower
layer, so that the AP can favour frames that have "real-time"
requirements like voice or video data frames.
Therefore, check for presence of ndo_select_queue() in netdev
ops and, if available, invoke it with a fallback handler to
__packet_pick_tx_queue(), so that driver such as bnx2x, ixgbe,
or mlx4 can still select a hw queue for transmission in
relation to the current CPU while e.g. ieee80211 subsystem
can make their own choices.
Reported-by: Mathias Kretschmer <mathias.kretschmer@fokus.fraunhofer.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-02-16 14:55:22 +00:00
|
|
|
const struct net_device_ops *ops = dev->netdev_ops;
|
|
|
|
u16 queue_index;
|
|
|
|
|
|
|
|
if (ops->ndo_select_queue) {
|
|
|
|
queue_index = ops->ndo_select_queue(dev, skb, NULL,
|
|
|
|
__packet_pick_tx_queue);
|
|
|
|
queue_index = netdev_cap_txqueue(dev, queue_index);
|
|
|
|
} else {
|
|
|
|
queue_index = __packet_pick_tx_queue(dev, skb);
|
|
|
|
}
|
|
|
|
|
2018-05-02 11:01:33 +00:00
|
|
|
return queue_index;
|
packet: check for ndo_select_queue during queue selection
Mathias reported that on an AMD Geode LX embedded board (ALiX)
with ath9k driver PACKET_QDISC_BYPASS, introduced in commit
d346a3fae3ff ("packet: introduce PACKET_QDISC_BYPASS socket
option"), triggers a WARN_ON() coming from the driver itself
via 066dae93bdf ("ath9k: rework tx queue selection and fix
queue stopping/waking").
The reason why this happened is that ndo_select_queue() call
is not invoked from direct xmit path i.e. for ieee80211 subsystem
that sets queue and TID (similar to 802.1d tag) which is being
put into the frame through 802.11e (WMM, QoS). If that is not
set, pending frame counter for e.g. ath9k can get messed up.
So the WARN_ON() in ath9k is absolutely legitimate. Generally,
the hw queue selection in ieee80211 depends on the type of
traffic, and priorities are set according to ieee80211_ac_numbers
mapping; working in a similar way as DiffServ only on a lower
layer, so that the AP can favour frames that have "real-time"
requirements like voice or video data frames.
Therefore, check for presence of ndo_select_queue() in netdev
ops and, if available, invoke it with a fallback handler to
__packet_pick_tx_queue(), so that driver such as bnx2x, ixgbe,
or mlx4 can still select a hw queue for transmission in
relation to the current CPU while e.g. ieee80211 subsystem
can make their own choices.
Reported-by: Mathias Kretschmer <mathias.kretschmer@fokus.fraunhofer.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-02-16 14:55:22 +00:00
|
|
|
}
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
/* __register_prot_hook must be invoked through register_prot_hook
|
2011-07-04 08:44:29 +00:00
|
|
|
* or from a context in which asynchronous accesses to the packet
|
|
|
|
* socket is not possible (packet_create()).
|
|
|
|
*/
|
2018-04-23 21:37:03 +00:00
|
|
|
static void __register_prot_hook(struct sock *sk)
|
2011-07-04 08:44:29 +00:00
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2013-11-21 15:50:58 +00:00
|
|
|
|
2011-07-04 08:44:29 +00:00
|
|
|
if (!po->running) {
|
2013-12-06 10:36:15 +00:00
|
|
|
if (po->fanout)
|
2011-07-05 08:45:05 +00:00
|
|
|
__fanout_link(sk, po);
|
2013-12-06 10:36:15 +00:00
|
|
|
else
|
2011-07-05 08:45:05 +00:00
|
|
|
dev_add_pack(&po->prot_hook);
|
2013-11-21 15:50:58 +00:00
|
|
|
|
2011-07-04 08:44:29 +00:00
|
|
|
sock_hold(sk);
|
|
|
|
po->running = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
static void register_prot_hook(struct sock *sk)
|
|
|
|
{
|
|
|
|
lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
|
|
|
|
__register_prot_hook(sk);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If the sync parameter is true, we will temporarily drop
|
2011-07-04 08:44:29 +00:00
|
|
|
* the po->bind_lock and do a synchronize_net to make sure no
|
|
|
|
* asynchronous packet processing paths still refer to the elements
|
|
|
|
* of po->prot_hook. If the sync parameter is false, it is the
|
|
|
|
* callers responsibility to take care of this.
|
|
|
|
*/
|
|
|
|
static void __unregister_prot_hook(struct sock *sk, bool sync)
|
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
lockdep_assert_held_once(&po->bind_lock);
|
|
|
|
|
2011-07-04 08:44:29 +00:00
|
|
|
po->running = 0;
|
2013-12-06 10:36:15 +00:00
|
|
|
|
|
|
|
if (po->fanout)
|
2011-07-05 08:45:05 +00:00
|
|
|
__fanout_unlink(sk, po);
|
2013-12-06 10:36:15 +00:00
|
|
|
else
|
2011-07-05 08:45:05 +00:00
|
|
|
__dev_remove_pack(&po->prot_hook);
|
2013-11-21 15:50:58 +00:00
|
|
|
|
2011-07-04 08:44:29 +00:00
|
|
|
__sock_put(sk);
|
|
|
|
|
|
|
|
if (sync) {
|
|
|
|
spin_unlock(&po->bind_lock);
|
|
|
|
synchronize_net();
|
|
|
|
spin_lock(&po->bind_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unregister_prot_hook(struct sock *sk, bool sync)
|
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
|
|
|
|
if (po->running)
|
|
|
|
__unregister_prot_hook(sk, sync);
|
|
|
|
}
|
|
|
|
|
2014-11-24 11:32:16 +00:00
|
|
|
static inline struct page * __pure pgv_to_page(void *addr)
|
2010-12-01 02:52:20 +00:00
|
|
|
{
|
|
|
|
if (is_vmalloc_addr(addr))
|
|
|
|
return vmalloc_to_page(addr);
|
|
|
|
return virt_to_page(addr);
|
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static void __packet_set_status(struct packet_sock *po, void *frame, int status)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr h;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
h.raw = frame;
|
2008-07-15 05:50:15 +00:00
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
2009-05-19 05:11:22 +00:00
|
|
|
h.h1->tp_status = status;
|
2010-12-01 02:52:20 +00:00
|
|
|
flush_dcache_page(pgv_to_page(&h.h1->tp_status));
|
2008-07-15 05:50:15 +00:00
|
|
|
break;
|
|
|
|
case TPACKET_V2:
|
2009-05-19 05:11:22 +00:00
|
|
|
h.h2->tp_status = status;
|
2010-12-01 02:52:20 +00:00
|
|
|
flush_dcache_page(pgv_to_page(&h.h2->tp_status));
|
2008-07-15 05:50:15 +00:00
|
|
|
break;
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
2017-01-03 14:31:47 +00:00
|
|
|
h.h3->tp_status = status;
|
|
|
|
flush_dcache_page(pgv_to_page(&h.h3->tp_status));
|
|
|
|
break;
|
2009-05-19 05:11:22 +00:00
|
|
|
default:
|
2011-08-19 10:18:16 +00:00
|
|
|
WARN(1, "TPACKET version not supported.\n");
|
2009-05-19 05:11:22 +00:00
|
|
|
BUG();
|
2008-07-15 05:50:15 +00:00
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
smp_wmb();
|
2008-07-15 05:50:15 +00:00
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static int __packet_get_status(struct packet_sock *po, void *frame)
|
2008-07-15 05:50:15 +00:00
|
|
|
{
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr h;
|
2008-07-15 05:50:15 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
smp_rmb();
|
|
|
|
|
2008-07-15 05:50:15 +00:00
|
|
|
h.raw = frame;
|
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
2010-12-01 02:52:20 +00:00
|
|
|
flush_dcache_page(pgv_to_page(&h.h1->tp_status));
|
2009-05-19 05:11:22 +00:00
|
|
|
return h.h1->tp_status;
|
2008-07-15 05:50:15 +00:00
|
|
|
case TPACKET_V2:
|
2010-12-01 02:52:20 +00:00
|
|
|
flush_dcache_page(pgv_to_page(&h.h2->tp_status));
|
2009-05-19 05:11:22 +00:00
|
|
|
return h.h2->tp_status;
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
2017-01-03 14:31:47 +00:00
|
|
|
flush_dcache_page(pgv_to_page(&h.h3->tp_status));
|
|
|
|
return h.h3->tp_status;
|
2009-05-19 05:11:22 +00:00
|
|
|
default:
|
2011-08-19 10:18:16 +00:00
|
|
|
WARN(1, "TPACKET version not supported.\n");
|
2009-05-19 05:11:22 +00:00
|
|
|
BUG();
|
|
|
|
return 0;
|
2008-07-15 05:50:15 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
|
|
|
|
unsigned int flags)
|
2013-04-23 00:39:29 +00:00
|
|
|
{
|
|
|
|
struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
|
|
|
|
|
2014-07-25 22:01:31 +00:00
|
|
|
if (shhwtstamps &&
|
|
|
|
(flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
|
|
|
|
ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
|
|
|
|
return TP_STATUS_TS_RAW_HARDWARE;
|
2013-04-23 00:39:29 +00:00
|
|
|
|
|
|
|
if (ktime_to_timespec_cond(skb->tstamp, ts))
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
return TP_STATUS_TS_SOFTWARE;
|
2013-04-23 00:39:29 +00:00
|
|
|
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
return 0;
|
2013-04-23 00:39:29 +00:00
|
|
|
}
|
|
|
|
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
|
|
|
|
struct sk_buff *skb)
|
2013-04-23 00:39:28 +00:00
|
|
|
{
|
|
|
|
union tpacket_uhdr h;
|
|
|
|
struct timespec ts;
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
__u32 ts_status;
|
2013-04-23 00:39:28 +00:00
|
|
|
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
|
|
|
|
return 0;
|
2013-04-23 00:39:28 +00:00
|
|
|
|
|
|
|
h.raw = frame;
|
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
h.h1->tp_sec = ts.tv_sec;
|
|
|
|
h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
|
|
|
|
break;
|
|
|
|
case TPACKET_V2:
|
|
|
|
h.h2->tp_sec = ts.tv_sec;
|
|
|
|
h.h2->tp_nsec = ts.tv_nsec;
|
|
|
|
break;
|
|
|
|
case TPACKET_V3:
|
2017-01-05 01:34:28 +00:00
|
|
|
h.h3->tp_sec = ts.tv_sec;
|
|
|
|
h.h3->tp_nsec = ts.tv_nsec;
|
|
|
|
break;
|
2013-04-23 00:39:28 +00:00
|
|
|
default:
|
|
|
|
WARN(1, "TPACKET version not supported.\n");
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* one flush is safe, as both fields always lie on the same cacheline */
|
|
|
|
flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
|
|
|
|
smp_wmb();
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
|
|
|
|
return ts_status;
|
2013-04-23 00:39:28 +00:00
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static void *packet_lookup_frame(struct packet_sock *po,
|
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
unsigned int position,
|
|
|
|
int status)
|
|
|
|
{
|
|
|
|
unsigned int pg_vec_pos, frame_offset;
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr h;
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
pg_vec_pos = position / rb->frames_per_block;
|
|
|
|
frame_offset = position % rb->frames_per_block;
|
|
|
|
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
h.raw = rb->pg_vec[pg_vec_pos].buffer +
|
|
|
|
(frame_offset * rb->frame_size);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
if (status != __packet_get_status(po, h.raw))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return h.raw;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *packet_current_frame(struct packet_sock *po,
|
2009-05-19 05:11:22 +00:00
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
int status)
|
|
|
|
{
|
|
|
|
return packet_lookup_frame(po, rb, rb->head, status);
|
|
|
|
}
|
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
del_timer_sync(&pkc->retire_blk_timer);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
|
|
|
|
struct sk_buff_head *rb_queue)
|
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2015-07-28 12:21:26 +00:00
|
|
|
pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2013-11-29 08:53:23 +00:00
|
|
|
spin_lock_bh(&rb_queue->lock);
|
2011-08-19 10:18:16 +00:00
|
|
|
pkc->delete_blk_timer = 1;
|
2013-11-29 08:53:23 +00:00
|
|
|
spin_unlock_bh(&rb_queue->lock);
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
prb_del_retire_blk_timer(pkc);
|
|
|
|
}
|
|
|
|
|
2015-06-22 07:09:16 +00:00
|
|
|
static void prb_setup_retire_blk_timer(struct packet_sock *po)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2015-06-22 07:09:16 +00:00
|
|
|
pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
2017-10-24 08:46:26 +00:00
|
|
|
timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
|
|
|
|
0);
|
|
|
|
pkc->retire_blk_timer.expires = jiffies;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int prb_calc_retire_blk_tmo(struct packet_sock *po,
|
|
|
|
int blk_size_in_bytes)
|
|
|
|
{
|
|
|
|
struct net_device *dev;
|
|
|
|
unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
|
2016-02-24 18:58:10 +00:00
|
|
|
struct ethtool_link_ksettings ecmd;
|
2011-09-03 03:34:30 +00:00
|
|
|
int err;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2011-09-03 03:34:30 +00:00
|
|
|
rtnl_lock();
|
|
|
|
dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
|
|
|
|
if (unlikely(!dev)) {
|
|
|
|
rtnl_unlock();
|
2011-08-19 10:18:16 +00:00
|
|
|
return DEFAULT_PRB_RETIRE_TOV;
|
2011-09-03 03:34:30 +00:00
|
|
|
}
|
2016-02-24 18:58:10 +00:00
|
|
|
err = __ethtool_get_link_ksettings(dev, &ecmd);
|
2011-09-03 03:34:30 +00:00
|
|
|
rtnl_unlock();
|
|
|
|
if (!err) {
|
|
|
|
/*
|
|
|
|
* If the link speed is so slow you don't really
|
|
|
|
* need to worry about perf anyways
|
|
|
|
*/
|
2016-02-24 18:58:10 +00:00
|
|
|
if (ecmd.base.speed < SPEED_1000 ||
|
|
|
|
ecmd.base.speed == SPEED_UNKNOWN) {
|
2011-09-03 03:34:30 +00:00
|
|
|
return DEFAULT_PRB_RETIRE_TOV;
|
2012-06-27 03:56:12 +00:00
|
|
|
} else {
|
|
|
|
msec = 1;
|
2016-02-24 18:58:10 +00:00
|
|
|
div = ecmd.base.speed / 1000;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
|
|
|
|
|
|
|
|
if (div)
|
|
|
|
mbits /= div;
|
|
|
|
|
|
|
|
tmo = mbits * msec;
|
|
|
|
|
|
|
|
if (div)
|
|
|
|
return tmo+1;
|
|
|
|
return tmo;
|
|
|
|
}
|
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
|
2011-08-19 10:18:16 +00:00
|
|
|
union tpacket_req_u *req_u)
|
|
|
|
{
|
|
|
|
p1->feature_req_word = req_u->req3.tp_feature_req_word;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_prb_bdqc(struct packet_sock *po,
|
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
struct pgv *pg_vec,
|
2015-06-22 07:09:16 +00:00
|
|
|
union tpacket_req_u *req_u)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
2013-12-06 05:29:36 +00:00
|
|
|
struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_block_desc *pbd;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
memset(p1, 0x0, sizeof(*p1));
|
|
|
|
|
|
|
|
p1->knxt_seq_num = 1;
|
|
|
|
p1->pkbdq = pg_vec;
|
2011-08-25 10:43:30 +00:00
|
|
|
pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
|
2012-06-03 17:41:40 +00:00
|
|
|
p1->pkblk_start = pg_vec[0].buffer;
|
2011-08-19 10:18:16 +00:00
|
|
|
p1->kblk_size = req_u->req3.tp_block_size;
|
|
|
|
p1->knum_blocks = req_u->req3.tp_block_nr;
|
|
|
|
p1->hdrlen = po->tp_hdrlen;
|
|
|
|
p1->version = po->tp_version;
|
|
|
|
p1->last_kactive_blk_num = 0;
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats3.tp_freeze_q_cnt = 0;
|
2011-08-19 10:18:16 +00:00
|
|
|
if (req_u->req3.tp_retire_blk_tov)
|
|
|
|
p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
|
|
|
|
else
|
|
|
|
p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
|
|
|
|
req_u->req3.tp_block_size);
|
|
|
|
p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
|
|
|
|
p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
|
|
|
|
|
2014-08-15 16:16:04 +00:00
|
|
|
p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
|
2011-08-19 10:18:16 +00:00
|
|
|
prb_init_ft_ops(p1, req_u);
|
2015-06-22 07:09:16 +00:00
|
|
|
prb_setup_retire_blk_timer(po);
|
2011-08-19 10:18:16 +00:00
|
|
|
prb_open_block(p1, pbd);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Do NOT update the last_blk_num first.
|
|
|
|
* Assumes sk_buff_head lock is held.
|
|
|
|
*/
|
2011-08-25 10:43:30 +00:00
|
|
|
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
mod_timer(&pkc->retire_blk_timer,
|
|
|
|
jiffies + pkc->tov_in_jiffies);
|
|
|
|
pkc->last_kactive_blk_num = pkc->kactive_blk_num;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Timer logic:
|
|
|
|
* 1) We refresh the timer only when we open a block.
|
|
|
|
* By doing this we don't waste cycles refreshing the timer
|
|
|
|
* on packet-by-packet basis.
|
|
|
|
*
|
|
|
|
* With a 1MB block-size, on a 1Gbps line, it will take
|
|
|
|
* i) ~8 ms to fill a block + ii) memcpy etc.
|
|
|
|
* In this cut we are not accounting for the memcpy time.
|
|
|
|
*
|
|
|
|
* So, if the user sets the 'tmo' to 10ms then the timer
|
|
|
|
* will never fire while the block is still getting filled
|
|
|
|
* (which is what we want). However, the user could choose
|
|
|
|
* to close a block early and that's fine.
|
|
|
|
*
|
|
|
|
* But when the timer does fire, we check whether or not to refresh it.
|
|
|
|
* Since the tmo granularity is in msecs, it is not too expensive
|
|
|
|
* to refresh the timer, lets say every '8' msecs.
|
|
|
|
* Either the user can set the 'tmo' or we can derive it based on
|
|
|
|
* a) line-speed and b) block-size.
|
|
|
|
* prb_calc_retire_blk_tmo() calculates the tmo.
|
|
|
|
*
|
|
|
|
*/
|
2017-10-24 08:46:26 +00:00
|
|
|
static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
2017-10-24 08:46:26 +00:00
|
|
|
struct packet_sock *po =
|
|
|
|
from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
|
2013-12-06 05:29:36 +00:00
|
|
|
struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
2011-08-19 10:18:16 +00:00
|
|
|
unsigned int frozen;
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_block_desc *pbd;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
spin_lock(&po->sk.sk_receive_queue.lock);
|
|
|
|
|
|
|
|
frozen = prb_queue_frozen(pkc);
|
|
|
|
pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
|
|
|
|
|
|
|
if (unlikely(pkc->delete_blk_timer))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* We only need to plug the race when the block is partially filled.
|
|
|
|
* tpacket_rcv:
|
|
|
|
* lock(); increment BLOCK_NUM_PKTS; unlock()
|
|
|
|
* copy_bits() is in progress ...
|
|
|
|
* timer fires on other cpu:
|
|
|
|
* we can't retire the current block because copy_bits
|
|
|
|
* is in progress.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
if (BLOCK_NUM_PKTS(pbd)) {
|
|
|
|
while (atomic_read(&pkc->blk_fill_in_prog)) {
|
|
|
|
/* Waiting for skb_copy_bits to finish... */
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
|
|
|
|
if (!frozen) {
|
2015-02-24 05:18:28 +00:00
|
|
|
if (!BLOCK_NUM_PKTS(pbd)) {
|
|
|
|
/* An empty block. Just refresh the timer. */
|
|
|
|
goto refresh_timer;
|
|
|
|
}
|
2011-08-19 10:18:16 +00:00
|
|
|
prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
|
|
|
|
if (!prb_dispatch_next_block(pkc, po))
|
|
|
|
goto refresh_timer;
|
|
|
|
else
|
|
|
|
goto out;
|
|
|
|
} else {
|
|
|
|
/* Case 1. Queue was frozen because user-space was
|
|
|
|
* lagging behind.
|
|
|
|
*/
|
2017-05-24 15:34:11 +00:00
|
|
|
if (prb_curr_blk_in_use(pbd)) {
|
2011-08-19 10:18:16 +00:00
|
|
|
/*
|
|
|
|
* Ok, user-space is still behind.
|
|
|
|
* So just refresh the timer.
|
|
|
|
*/
|
|
|
|
goto refresh_timer;
|
|
|
|
} else {
|
|
|
|
/* Case 2. queue was frozen,user-space caught up,
|
|
|
|
* now the link went idle && the timer fired.
|
|
|
|
* We don't have a block to close.So we open this
|
|
|
|
* block and restart the timer.
|
|
|
|
* opening a block thaws the queue,restarts timer
|
|
|
|
* Thawing/timer-refresh is a side effect.
|
|
|
|
*/
|
|
|
|
prb_open_block(pkc, pbd);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
refresh_timer:
|
|
|
|
_prb_refresh_rx_retire_blk_timer(pkc);
|
|
|
|
|
|
|
|
out:
|
|
|
|
spin_unlock(&po->sk.sk_receive_queue.lock);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_block_desc *pbd1, __u32 status)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
/* Flush everything minus the block header */
|
|
|
|
|
|
|
|
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
|
|
|
u8 *start, *end;
|
|
|
|
|
|
|
|
start = (u8 *)pbd1;
|
|
|
|
|
|
|
|
/* Skip the block header(we know header WILL fit in 4K) */
|
|
|
|
start += PAGE_SIZE;
|
|
|
|
|
|
|
|
end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
|
|
|
|
for (; start < end; start += PAGE_SIZE)
|
|
|
|
flush_dcache_page(pgv_to_page(start));
|
|
|
|
|
|
|
|
smp_wmb();
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Now update the block status. */
|
|
|
|
|
|
|
|
BLOCK_STATUS(pbd1) = status;
|
|
|
|
|
|
|
|
/* Flush the block header */
|
|
|
|
|
|
|
|
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
|
|
|
start = (u8 *)pbd1;
|
|
|
|
flush_dcache_page(pgv_to_page(start));
|
|
|
|
|
|
|
|
smp_wmb();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Side effect:
|
|
|
|
*
|
|
|
|
* 1) flush the block
|
|
|
|
* 2) Increment active_blk_num
|
|
|
|
*
|
|
|
|
* Note:We DONT refresh the timer on purpose.
|
|
|
|
* Because almost always the next block will be opened.
|
|
|
|
*/
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_close_block(struct tpacket_kbdq_core *pkc1,
|
|
|
|
struct tpacket_block_desc *pbd1,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *po, unsigned int stat)
|
|
|
|
{
|
|
|
|
__u32 status = TP_STATUS_USER | stat;
|
|
|
|
|
|
|
|
struct tpacket3_hdr *last_pkt;
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
|
2014-12-19 03:49:25 +00:00
|
|
|
struct sock *sk = &po->sk;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2013-04-19 06:12:29 +00:00
|
|
|
if (po->stats.stats3.tp_drops)
|
2011-08-19 10:18:16 +00:00
|
|
|
status |= TP_STATUS_LOSING;
|
|
|
|
|
|
|
|
last_pkt = (struct tpacket3_hdr *)pkc1->prev;
|
|
|
|
last_pkt->tp_next_offset = 0;
|
|
|
|
|
|
|
|
/* Get the ts of the last pkt */
|
|
|
|
if (BLOCK_NUM_PKTS(pbd1)) {
|
|
|
|
h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
|
|
|
|
h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
|
|
|
|
} else {
|
2015-02-24 05:18:28 +00:00
|
|
|
/* Ok, we tmo'd - so get the current time.
|
|
|
|
*
|
|
|
|
* It shouldn't really happen as we don't close empty
|
|
|
|
* blocks. See prb_retire_rx_blk_timer_expired().
|
|
|
|
*/
|
2011-08-19 10:18:16 +00:00
|
|
|
struct timespec ts;
|
|
|
|
getnstimeofday(&ts);
|
|
|
|
h1->ts_last_pkt.ts_sec = ts.tv_sec;
|
|
|
|
h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
|
|
|
|
}
|
|
|
|
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
/* Flush the block */
|
|
|
|
prb_flush_block(pkc1, pbd1, status);
|
|
|
|
|
2014-12-19 03:49:25 +00:00
|
|
|
sk->sk_data_ready(sk);
|
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
pkc->reset_pending_on_curr_blk = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Side effect of opening a block:
|
|
|
|
*
|
|
|
|
* 1) prb_queue is thawed.
|
|
|
|
* 2) retire_blk_timer is refreshed.
|
|
|
|
*
|
|
|
|
*/
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_open_block(struct tpacket_kbdq_core *pkc1,
|
|
|
|
struct tpacket_block_desc *pbd1)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
struct timespec ts;
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
smp_rmb();
|
|
|
|
|
2013-05-03 02:57:00 +00:00
|
|
|
/* We could have just memset this but we will lose the
|
|
|
|
* flexibility of making the priv area sticky
|
|
|
|
*/
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2013-05-03 02:57:00 +00:00
|
|
|
BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
|
|
|
|
BLOCK_NUM_PKTS(pbd1) = 0;
|
|
|
|
BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2013-05-03 02:57:00 +00:00
|
|
|
getnstimeofday(&ts);
|
|
|
|
|
|
|
|
h1->ts_first_pkt.ts_sec = ts.tv_sec;
|
|
|
|
h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
2013-05-03 02:57:00 +00:00
|
|
|
pkc1->pkblk_start = (char *)pbd1;
|
|
|
|
pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
|
|
|
|
|
|
|
BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
|
|
|
BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
|
|
|
|
|
|
|
|
pbd1->version = pkc1->version;
|
|
|
|
pkc1->prev = pkc1->nxt_offset;
|
|
|
|
pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
|
|
|
|
|
|
|
|
prb_thaw_queue(pkc1);
|
|
|
|
_prb_refresh_rx_retire_blk_timer(pkc1);
|
|
|
|
|
|
|
|
smp_wmb();
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Queue freeze logic:
|
|
|
|
* 1) Assume tp_block_nr = 8 blocks.
|
|
|
|
* 2) At time 't0', user opens Rx ring.
|
|
|
|
* 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
|
|
|
|
* 4) user-space is either sleeping or processing block '0'.
|
|
|
|
* 5) tpacket_rcv is currently filling block '7', since there is no space left,
|
|
|
|
* it will close block-7,loop around and try to fill block '0'.
|
|
|
|
* call-flow:
|
|
|
|
* __packet_lookup_frame_in_block
|
|
|
|
* prb_retire_current_block()
|
|
|
|
* prb_dispatch_next_block()
|
|
|
|
* |->(BLOCK_STATUS == USER) evaluates to true
|
|
|
|
* 5.1) Since block-0 is currently in-use, we just freeze the queue.
|
|
|
|
* 6) Now there are two cases:
|
|
|
|
* 6.1) Link goes idle right after the queue is frozen.
|
|
|
|
* But remember, the last open_block() refreshed the timer.
|
|
|
|
* When this timer expires,it will refresh itself so that we can
|
|
|
|
* re-open block-0 in near future.
|
|
|
|
* 6.2) Link is busy and keeps on receiving packets. This is a simple
|
|
|
|
* case and __packet_lookup_frame_in_block will check if block-0
|
|
|
|
* is free and can now be re-used.
|
|
|
|
*/
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *po)
|
|
|
|
{
|
|
|
|
pkc->reset_pending_on_curr_blk = 1;
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats3.tp_freeze_q_cnt++;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the next block is free then we will dispatch it
|
|
|
|
* and return a good offset.
|
|
|
|
* Else, we will freeze the queue.
|
|
|
|
* So, caller must check the return value.
|
|
|
|
*/
|
2011-08-25 10:43:30 +00:00
|
|
|
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *po)
|
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_block_desc *pbd;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
smp_rmb();
|
|
|
|
|
|
|
|
/* 1. Get current block num */
|
|
|
|
pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
|
|
|
|
|
|
|
/* 2. If this block is currently in_use then freeze the queue */
|
|
|
|
if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
|
|
|
|
prb_freeze_queue(pkc, po);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* 3.
|
|
|
|
* open this block and return the offset where the first packet
|
|
|
|
* needs to get stored.
|
|
|
|
*/
|
|
|
|
prb_open_block(pkc, pbd);
|
|
|
|
return (void *)pkc->nxt_offset;
|
|
|
|
}
|
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_sock *po, unsigned int status)
|
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
/* retire/close the current block */
|
|
|
|
if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
|
|
|
|
/*
|
|
|
|
* Plug the case where copy_bits() is in progress on
|
|
|
|
* cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
|
|
|
|
* have space to copy the pkt in the current block and
|
|
|
|
* called prb_retire_current_block()
|
|
|
|
*
|
|
|
|
* We don't need to worry about the TMO case because
|
|
|
|
* the timer-handler already handled this case.
|
|
|
|
*/
|
|
|
|
if (!(status & TP_STATUS_BLK_TMO)) {
|
|
|
|
while (atomic_read(&pkc->blk_fill_in_prog)) {
|
|
|
|
/* Waiting for skb_copy_bits to finish... */
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
prb_close_block(pkc, pbd, po, status);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-05-24 15:34:11 +00:00
|
|
|
static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
return TP_STATUS_USER & BLOCK_STATUS(pbd);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
return pkc->reset_pending_on_curr_blk;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
|
2011-08-19 10:18:16 +00:00
|
|
|
atomic_dec(&pkc->blk_fill_in_prog);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct tpacket3_hdr *ppd)
|
|
|
|
{
|
2013-12-16 06:12:06 +00:00
|
|
|
ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct tpacket3_hdr *ppd)
|
|
|
|
{
|
|
|
|
ppd->hv1.tp_rxhash = 0;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct tpacket3_hdr *ppd)
|
|
|
|
{
|
2015-01-13 16:13:44 +00:00
|
|
|
if (skb_vlan_tag_present(pkc->skb)) {
|
|
|
|
ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
|
2013-12-17 13:53:40 +00:00
|
|
|
ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
|
|
|
|
ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
2011-08-19 10:18:16 +00:00
|
|
|
} else {
|
2012-08-20 03:34:03 +00:00
|
|
|
ppd->hv1.tp_vlan_tci = 0;
|
2013-12-17 13:53:40 +00:00
|
|
|
ppd->hv1.tp_vlan_tpid = 0;
|
2012-08-20 03:34:03 +00:00
|
|
|
ppd->tp_status = TP_STATUS_AVAILABLE;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-08-25 10:43:30 +00:00
|
|
|
static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct tpacket3_hdr *ppd)
|
|
|
|
{
|
2013-12-17 13:53:40 +00:00
|
|
|
ppd->hv1.tp_padding = 0;
|
2011-08-19 10:18:16 +00:00
|
|
|
prb_fill_vlan_info(pkc, ppd);
|
|
|
|
|
|
|
|
if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
|
|
|
|
prb_fill_rxhash(pkc, ppd);
|
|
|
|
else
|
|
|
|
prb_clear_rxhash(pkc, ppd);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void prb_fill_curr_block(char *curr,
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc,
|
|
|
|
struct tpacket_block_desc *pbd,
|
2011-08-19 10:18:16 +00:00
|
|
|
unsigned int len)
|
|
|
|
{
|
|
|
|
struct tpacket3_hdr *ppd;
|
|
|
|
|
|
|
|
ppd = (struct tpacket3_hdr *)curr;
|
|
|
|
ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
|
|
|
|
pkc->prev = curr;
|
|
|
|
pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
|
|
|
|
BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
|
|
|
|
BLOCK_NUM_PKTS(pbd) += 1;
|
|
|
|
atomic_inc(&pkc->blk_fill_in_prog);
|
|
|
|
prb_run_all_ft_ops(pkc, ppd);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Assumes caller has the sk->rx_queue.lock */
|
|
|
|
static void *__packet_lookup_frame_in_block(struct packet_sock *po,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
int status,
|
|
|
|
unsigned int len
|
|
|
|
)
|
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc;
|
|
|
|
struct tpacket_block_desc *pbd;
|
2011-08-19 10:18:16 +00:00
|
|
|
char *curr, *end;
|
|
|
|
|
2012-06-03 17:41:40 +00:00
|
|
|
pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
2011-08-19 10:18:16 +00:00
|
|
|
pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
|
|
|
|
|
|
|
/* Queue is frozen when user space is lagging behind */
|
|
|
|
if (prb_queue_frozen(pkc)) {
|
|
|
|
/*
|
|
|
|
* Check if that last block which caused the queue to freeze,
|
|
|
|
* is still in_use by user-space.
|
|
|
|
*/
|
2017-05-24 15:34:11 +00:00
|
|
|
if (prb_curr_blk_in_use(pbd)) {
|
2011-08-19 10:18:16 +00:00
|
|
|
/* Can't record this packet */
|
|
|
|
return NULL;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Ok, the block was released by user-space.
|
|
|
|
* Now let's open that block.
|
|
|
|
* opening a block also thaws the queue.
|
|
|
|
* Thawing is a side effect.
|
|
|
|
*/
|
|
|
|
prb_open_block(pkc, pbd);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
smp_mb();
|
|
|
|
curr = pkc->nxt_offset;
|
|
|
|
pkc->skb = skb;
|
2012-06-03 17:41:40 +00:00
|
|
|
end = (char *)pbd + pkc->kblk_size;
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
/* first try the current block */
|
|
|
|
if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
|
|
|
|
prb_fill_curr_block(curr, pkc, pbd, len);
|
|
|
|
return (void *)curr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Ok, close the current block */
|
|
|
|
prb_retire_current_block(pkc, po, 0);
|
|
|
|
|
|
|
|
/* Now, try to dispatch the next block */
|
|
|
|
curr = (char *)prb_dispatch_next_block(pkc, po);
|
|
|
|
if (curr) {
|
|
|
|
pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
|
|
|
prb_fill_curr_block(curr, pkc, pbd, len);
|
|
|
|
return (void *)curr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No free blocks are available.user_space hasn't caught up yet.
|
|
|
|
* Queue was just frozen and now this packet will get dropped.
|
|
|
|
*/
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *packet_current_rx_frame(struct packet_sock *po,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct sk_buff *skb,
|
|
|
|
int status, unsigned int len)
|
|
|
|
{
|
|
|
|
char *curr = NULL;
|
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
case TPACKET_V2:
|
|
|
|
curr = packet_lookup_frame(po, &po->rx_ring,
|
|
|
|
po->rx_ring.head, status);
|
|
|
|
return curr;
|
|
|
|
case TPACKET_V3:
|
|
|
|
return __packet_lookup_frame_in_block(po, skb, status, len);
|
|
|
|
default:
|
|
|
|
WARN(1, "TPACKET version not supported\n");
|
|
|
|
BUG();
|
2012-08-06 16:27:10 +00:00
|
|
|
return NULL;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *prb_lookup_block(struct packet_sock *po,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_ring_buffer *rb,
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
unsigned int idx,
|
2011-08-19 10:18:16 +00:00
|
|
|
int status)
|
|
|
|
{
|
2011-08-25 10:43:30 +00:00
|
|
|
struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
|
2011-08-19 10:18:16 +00:00
|
|
|
|
|
|
|
if (status != BLOCK_STATUS(pbd))
|
|
|
|
return NULL;
|
|
|
|
return pbd;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static int prb_previous_blk_num(struct packet_ring_buffer *rb)
|
2011-08-19 10:18:16 +00:00
|
|
|
{
|
|
|
|
unsigned int prev;
|
|
|
|
if (rb->prb_bdqc.kactive_blk_num)
|
|
|
|
prev = rb->prb_bdqc.kactive_blk_num-1;
|
|
|
|
else
|
|
|
|
prev = rb->prb_bdqc.knum_blocks-1;
|
|
|
|
return prev;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Assumes caller has held the rx_queue.lock */
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *__prb_previous_block(struct packet_sock *po,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
int status)
|
|
|
|
{
|
|
|
|
unsigned int previous = prb_previous_blk_num(rb);
|
|
|
|
return prb_lookup_block(po, rb, previous, status);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *packet_previous_rx_frame(struct packet_sock *po,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
int status)
|
|
|
|
{
|
|
|
|
if (po->tp_version <= TPACKET_V2)
|
|
|
|
return packet_previous_frame(po, rb, status);
|
|
|
|
|
|
|
|
return __prb_previous_block(po, rb, status);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void packet_increment_rx_head(struct packet_sock *po,
|
2011-08-19 10:18:16 +00:00
|
|
|
struct packet_ring_buffer *rb)
|
|
|
|
{
|
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
case TPACKET_V2:
|
|
|
|
return packet_increment_head(rb);
|
|
|
|
case TPACKET_V3:
|
|
|
|
default:
|
|
|
|
WARN(1, "TPACKET version not supported.\n");
|
|
|
|
BUG();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void *packet_previous_frame(struct packet_sock *po,
|
2009-05-19 05:11:22 +00:00
|
|
|
struct packet_ring_buffer *rb,
|
|
|
|
int status)
|
|
|
|
{
|
|
|
|
unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
|
|
|
|
return packet_lookup_frame(po, rb, previous, status);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static void packet_increment_head(struct packet_ring_buffer *buff)
|
2009-05-19 05:11:22 +00:00
|
|
|
{
|
|
|
|
buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
|
|
|
|
}
|
|
|
|
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
static void packet_inc_pending(struct packet_ring_buffer *rb)
|
|
|
|
{
|
|
|
|
this_cpu_inc(*rb->pending_refcnt);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_dec_pending(struct packet_ring_buffer *rb)
|
|
|
|
{
|
|
|
|
this_cpu_dec(*rb->pending_refcnt);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
|
|
|
|
{
|
|
|
|
unsigned int refcnt = 0;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
/* We don't use pending refcount in rx_ring. */
|
|
|
|
if (rb->pending_refcnt == NULL)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
|
|
|
|
|
|
|
|
return refcnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_alloc_pending(struct packet_sock *po)
|
|
|
|
{
|
|
|
|
po->rx_ring.pending_refcnt = NULL;
|
|
|
|
|
|
|
|
po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
|
|
|
|
if (unlikely(po->tx_ring.pending_refcnt == NULL))
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_free_pending(struct packet_sock *po)
|
|
|
|
{
|
|
|
|
free_percpu(po->tx_ring.pending_refcnt);
|
|
|
|
}
|
|
|
|
|
2015-05-12 15:56:47 +00:00
|
|
|
#define ROOM_POW_OFF 2
|
|
|
|
#define ROOM_NONE 0x0
|
|
|
|
#define ROOM_LOW 0x1
|
|
|
|
#define ROOM_NORMAL 0x2
|
|
|
|
|
|
|
|
static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
{
|
2015-05-12 15:56:47 +00:00
|
|
|
int idx, len;
|
|
|
|
|
|
|
|
len = po->rx_ring.frame_max + 1;
|
|
|
|
idx = po->rx_ring.head;
|
|
|
|
if (pow_off)
|
|
|
|
idx += len >> pow_off;
|
|
|
|
if (idx >= len)
|
|
|
|
idx -= len;
|
|
|
|
return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
|
|
|
|
{
|
|
|
|
int idx, len;
|
|
|
|
|
|
|
|
len = po->rx_ring.prb_bdqc.knum_blocks;
|
|
|
|
idx = po->rx_ring.prb_bdqc.kactive_blk_num;
|
|
|
|
if (pow_off)
|
|
|
|
idx += len >> pow_off;
|
|
|
|
if (idx >= len)
|
|
|
|
idx -= len;
|
|
|
|
return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
|
|
|
|
}
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
|
2015-05-12 15:56:47 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = &po->sk;
|
|
|
|
int ret = ROOM_NONE;
|
|
|
|
|
|
|
|
if (po->prot_hook.func != tpacket_rcv) {
|
|
|
|
int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
- (skb ? skb->truesize : 0);
|
2015-05-12 15:56:47 +00:00
|
|
|
if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
|
|
|
|
return ROOM_NORMAL;
|
|
|
|
else if (avail > 0)
|
|
|
|
return ROOM_LOW;
|
|
|
|
else
|
|
|
|
return ROOM_NONE;
|
|
|
|
}
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
|
2015-05-12 15:56:47 +00:00
|
|
|
if (po->tp_version == TPACKET_V3) {
|
|
|
|
if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
|
|
|
|
ret = ROOM_NORMAL;
|
|
|
|
else if (__tpacket_v3_has_room(po, 0))
|
|
|
|
ret = ROOM_LOW;
|
|
|
|
} else {
|
|
|
|
if (__tpacket_has_room(po, ROOM_POW_OFF))
|
|
|
|
ret = ROOM_NORMAL;
|
|
|
|
else if (__tpacket_has_room(po, 0))
|
|
|
|
ret = ROOM_LOW;
|
|
|
|
}
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
bool has_room;
|
|
|
|
|
2015-05-14 19:25:02 +00:00
|
|
|
spin_lock_bh(&po->sk.sk_receive_queue.lock);
|
|
|
|
ret = __packet_rcv_has_room(po, skb);
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
has_room = ret == ROOM_NORMAL;
|
|
|
|
if (po->pressure == has_room)
|
2015-05-14 19:25:02 +00:00
|
|
|
po->pressure = !has_room;
|
|
|
|
spin_unlock_bh(&po->sk.sk_receive_queue.lock);
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
|
2015-05-12 15:56:47 +00:00
|
|
|
return ret;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static void packet_sock_destruct(struct sock *sk)
|
|
|
|
{
|
2010-04-07 22:41:28 +00:00
|
|
|
skb_queue_purge(&sk->sk_error_queue);
|
|
|
|
|
2008-07-26 04:43:18 +00:00
|
|
|
WARN_ON(atomic_read(&sk->sk_rmem_alloc));
|
2017-06-30 10:08:00 +00:00
|
|
|
WARN_ON(refcount_read(&sk->sk_wmem_alloc));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD)) {
|
2009-07-21 21:57:59 +00:00
|
|
|
pr_err("Attempt to release alive packet socket: %p\n", sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2007-11-11 05:38:48 +00:00
|
|
|
sk_refcnt_debug_dec(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2015-05-12 15:56:49 +00:00
|
|
|
static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
u32 rxhash;
|
|
|
|
int i, count = 0;
|
|
|
|
|
|
|
|
rxhash = skb_get_hash(skb);
|
|
|
|
for (i = 0; i < ROLLOVER_HLEN; i++)
|
|
|
|
if (po->rollover->history[i] == rxhash)
|
|
|
|
count++;
|
|
|
|
|
|
|
|
po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
|
|
|
|
return count > (ROLLOVER_HLEN >> 1);
|
|
|
|
}
|
|
|
|
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
static unsigned int fanout_demux_hash(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
2011-07-05 08:45:05 +00:00
|
|
|
{
|
2016-07-01 20:07:50 +00:00
|
|
|
return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
|
|
|
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
static unsigned int fanout_demux_lb(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
2011-07-05 08:45:05 +00:00
|
|
|
{
|
2015-06-17 19:59:34 +00:00
|
|
|
unsigned int val = atomic_inc_return(&f->rr_cur);
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2015-06-17 19:59:34 +00:00
|
|
|
return val % num;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int fanout_demux_cpu(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
|
|
|
{
|
|
|
|
return smp_processor_id() % num;
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
|
|
|
|
2013-08-28 20:13:09 +00:00
|
|
|
static unsigned int fanout_demux_rnd(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
|
|
|
{
|
random32: add prandom_u32_max and convert open coded users
Many functions have open coded a function that returns a random
number in range [0,N-1]. Under the assumption that we have a PRNG
such as taus113 with being well distributed in [0, ~0U] space,
we can implement such a function as uword t = (n*m')>>32, where
m' is a random number obtained from PRNG, n the right open interval
border and t our resulting random number, with n,m',t in u32 universe.
Lets go with Joe and simply call it prandom_u32_max(), although
technically we have an right open interval endpoint, but that we
have documented. Other users can further be migrated to the new
prandom_u32_max() function later on; for now, we need to make sure
to migrate reciprocal_divide() users for the reciprocal_divide()
follow-up fixup since their function signatures are going to change.
Joint work with Hannes Frederic Sowa.
Cc: Jakub Zawadzki <darkjames-ws@darkjames.pl>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-22 01:29:39 +00:00
|
|
|
return prandom_u32_max(num);
|
2013-08-28 20:13:09 +00:00
|
|
|
}
|
|
|
|
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
static unsigned int fanout_demux_rollover(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
2015-05-12 15:56:45 +00:00
|
|
|
unsigned int idx, bool try_self,
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
unsigned int num)
|
2011-07-06 08:56:38 +00:00
|
|
|
{
|
2015-05-17 23:44:02 +00:00
|
|
|
struct packet_sock *po, *po_next, *po_skip = NULL;
|
2015-05-12 15:56:50 +00:00
|
|
|
unsigned int i, j, room = ROOM_NONE;
|
2011-07-06 08:56:38 +00:00
|
|
|
|
2015-05-12 15:56:46 +00:00
|
|
|
po = pkt_sk(f->arr[idx]);
|
2015-05-12 15:56:49 +00:00
|
|
|
|
|
|
|
if (try_self) {
|
|
|
|
room = packet_rcv_has_room(po, skb);
|
|
|
|
if (room == ROOM_NORMAL ||
|
|
|
|
(room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
|
|
|
|
return idx;
|
2015-05-17 23:44:02 +00:00
|
|
|
po_skip = po;
|
2015-05-12 15:56:49 +00:00
|
|
|
}
|
2015-05-12 15:56:45 +00:00
|
|
|
|
2015-05-12 15:56:46 +00:00
|
|
|
i = j = min_t(int, po->rollover->sock, num - 1);
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
do {
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
po_next = pkt_sk(f->arr[i]);
|
2015-05-17 23:44:02 +00:00
|
|
|
if (po_next != po_skip && !po_next->pressure &&
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
if (i != j)
|
2015-05-12 15:56:46 +00:00
|
|
|
po->rollover->sock = i;
|
2015-05-12 15:56:50 +00:00
|
|
|
atomic_long_inc(&po->rollover->num);
|
|
|
|
if (room == ROOM_LOW)
|
|
|
|
atomic_long_inc(&po->rollover->num_huge);
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
return i;
|
|
|
|
}
|
2015-05-12 15:56:45 +00:00
|
|
|
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
if (++i == num)
|
|
|
|
i = 0;
|
|
|
|
} while (i != j);
|
|
|
|
|
2015-05-12 15:56:50 +00:00
|
|
|
atomic_long_inc(&po->rollover->num_failed);
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
return idx;
|
|
|
|
}
|
|
|
|
|
2014-01-22 21:01:44 +00:00
|
|
|
static unsigned int fanout_demux_qm(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
|
|
|
{
|
|
|
|
return skb_get_queue_mapping(skb) % num;
|
|
|
|
}
|
|
|
|
|
2015-08-15 02:31:34 +00:00
|
|
|
static unsigned int fanout_demux_bpf(struct packet_fanout *f,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
unsigned int num)
|
|
|
|
{
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
unsigned int ret = 0;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
prog = rcu_dereference(f->bpf_prog);
|
|
|
|
if (prog)
|
2015-10-07 17:55:41 +00:00
|
|
|
ret = bpf_prog_run_clear_cb(prog, skb) % num;
|
2015-08-15 02:31:34 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
|
|
|
|
{
|
|
|
|
return f->flags & (flag >> 8);
|
2011-07-06 08:56:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct packet_type *pt, struct net_device *orig_dev)
|
2011-07-05 08:45:05 +00:00
|
|
|
{
|
|
|
|
struct packet_fanout *f = pt->af_packet_priv;
|
2015-06-16 14:59:11 +00:00
|
|
|
unsigned int num = READ_ONCE(f->num_members);
|
2015-10-09 18:44:54 +00:00
|
|
|
struct net *net = read_pnet(&f->net);
|
2011-07-05 08:45:05 +00:00
|
|
|
struct packet_sock *po;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
unsigned int idx;
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2015-10-09 18:44:54 +00:00
|
|
|
if (!net_eq(dev_net(dev), net) || !num) {
|
2011-07-05 08:45:05 +00:00
|
|
|
kfree_skb(skb);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-02-20 05:24:27 +00:00
|
|
|
if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
|
2015-10-09 18:44:54 +00:00
|
|
|
skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
|
2015-02-20 05:24:27 +00:00
|
|
|
if (!skb)
|
|
|
|
return 0;
|
|
|
|
}
|
2011-07-06 08:56:38 +00:00
|
|
|
switch (f->type) {
|
|
|
|
case PACKET_FANOUT_HASH:
|
|
|
|
default:
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
idx = fanout_demux_hash(f, skb, num);
|
2011-07-06 08:56:38 +00:00
|
|
|
break;
|
|
|
|
case PACKET_FANOUT_LB:
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
idx = fanout_demux_lb(f, skb, num);
|
2011-07-06 08:56:38 +00:00
|
|
|
break;
|
|
|
|
case PACKET_FANOUT_CPU:
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
idx = fanout_demux_cpu(f, skb, num);
|
|
|
|
break;
|
2013-08-28 20:13:09 +00:00
|
|
|
case PACKET_FANOUT_RND:
|
|
|
|
idx = fanout_demux_rnd(f, skb, num);
|
|
|
|
break;
|
2014-01-22 21:01:44 +00:00
|
|
|
case PACKET_FANOUT_QM:
|
|
|
|
idx = fanout_demux_qm(f, skb, num);
|
|
|
|
break;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
case PACKET_FANOUT_ROLLOVER:
|
2015-05-12 15:56:45 +00:00
|
|
|
idx = fanout_demux_rollover(f, skb, 0, false, num);
|
2011-07-06 08:56:38 +00:00
|
|
|
break;
|
2015-08-15 02:31:34 +00:00
|
|
|
case PACKET_FANOUT_CBPF:
|
2015-08-15 02:31:35 +00:00
|
|
|
case PACKET_FANOUT_EBPF:
|
2015-08-15 02:31:34 +00:00
|
|
|
idx = fanout_demux_bpf(f, skb, num);
|
|
|
|
break;
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
|
|
|
|
2015-05-12 15:56:45 +00:00
|
|
|
if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
|
|
|
|
idx = fanout_demux_rollover(f, skb, idx, true, num);
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2015-05-12 15:56:45 +00:00
|
|
|
po = pkt_sk(f->arr[idx]);
|
2011-07-05 08:45:05 +00:00
|
|
|
return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
|
|
|
|
}
|
|
|
|
|
2012-08-16 05:36:48 +00:00
|
|
|
DEFINE_MUTEX(fanout_mutex);
|
|
|
|
EXPORT_SYMBOL_GPL(fanout_mutex);
|
2011-07-05 08:45:05 +00:00
|
|
|
static LIST_HEAD(fanout_list);
|
2017-04-21 14:56:11 +00:00
|
|
|
static u16 fanout_next_id;
|
2011-07-05 08:45:05 +00:00
|
|
|
|
|
|
|
static void __fanout_link(struct sock *sk, struct packet_sock *po)
|
|
|
|
{
|
|
|
|
struct packet_fanout *f = po->fanout;
|
|
|
|
|
|
|
|
spin_lock(&f->lock);
|
|
|
|
f->arr[f->num_members] = sk;
|
|
|
|
smp_wmb();
|
|
|
|
f->num_members++;
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
if (f->num_members == 1)
|
|
|
|
dev_add_pack(&f->prot_hook);
|
2011-07-05 08:45:05 +00:00
|
|
|
spin_unlock(&f->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
|
|
|
|
{
|
|
|
|
struct packet_fanout *f = po->fanout;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
spin_lock(&f->lock);
|
|
|
|
for (i = 0; i < f->num_members; i++) {
|
|
|
|
if (f->arr[i] == sk)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
BUG_ON(i >= f->num_members);
|
|
|
|
f->arr[i] = f->arr[f->num_members - 1];
|
|
|
|
f->num_members--;
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
if (f->num_members == 0)
|
|
|
|
__dev_remove_pack(&f->prot_hook);
|
2011-07-05 08:45:05 +00:00
|
|
|
spin_unlock(&f->lock);
|
|
|
|
}
|
|
|
|
|
2013-12-23 03:31:38 +00:00
|
|
|
static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
|
2012-08-16 22:02:58 +00:00
|
|
|
{
|
2015-10-09 18:29:32 +00:00
|
|
|
if (sk->sk_family != PF_PACKET)
|
|
|
|
return false;
|
2012-08-16 22:02:58 +00:00
|
|
|
|
2015-10-09 18:29:32 +00:00
|
|
|
return ptype->af_packet_priv == pkt_sk(sk)->fanout;
|
2012-08-16 22:02:58 +00:00
|
|
|
}
|
|
|
|
|
2015-08-15 02:31:34 +00:00
|
|
|
static void fanout_init_data(struct packet_fanout *f)
|
|
|
|
{
|
|
|
|
switch (f->type) {
|
|
|
|
case PACKET_FANOUT_LB:
|
|
|
|
atomic_set(&f->rr_cur, 0);
|
|
|
|
break;
|
|
|
|
case PACKET_FANOUT_CBPF:
|
2015-08-15 02:31:35 +00:00
|
|
|
case PACKET_FANOUT_EBPF:
|
2015-08-15 02:31:34 +00:00
|
|
|
RCU_INIT_POINTER(f->bpf_prog, NULL);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
|
|
|
|
{
|
|
|
|
struct bpf_prog *old;
|
|
|
|
|
|
|
|
spin_lock(&f->lock);
|
|
|
|
old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
|
|
|
|
rcu_assign_pointer(f->bpf_prog, new);
|
|
|
|
spin_unlock(&f->lock);
|
|
|
|
|
|
|
|
if (old) {
|
|
|
|
synchronize_net();
|
|
|
|
bpf_prog_destroy(old);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
|
|
|
|
unsigned int len)
|
|
|
|
{
|
|
|
|
struct bpf_prog *new;
|
|
|
|
struct sock_fprog fprog;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
|
|
|
|
return -EPERM;
|
|
|
|
if (len != sizeof(fprog))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&fprog, data, len))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2015-10-02 13:17:33 +00:00
|
|
|
ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
|
2015-08-15 02:31:34 +00:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
__fanout_set_data_bpf(po->fanout, new);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-08-15 02:31:35 +00:00
|
|
|
static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
|
|
|
|
unsigned int len)
|
|
|
|
{
|
|
|
|
struct bpf_prog *new;
|
|
|
|
u32 fd;
|
|
|
|
|
|
|
|
if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
|
|
|
|
return -EPERM;
|
|
|
|
if (len != sizeof(fd))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&fd, data, len))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2016-06-30 15:24:44 +00:00
|
|
|
new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
|
2015-08-15 02:31:35 +00:00
|
|
|
if (IS_ERR(new))
|
|
|
|
return PTR_ERR(new);
|
|
|
|
|
|
|
|
__fanout_set_data_bpf(po->fanout, new);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-08-15 02:31:34 +00:00
|
|
|
static int fanout_set_data(struct packet_sock *po, char __user *data,
|
|
|
|
unsigned int len)
|
|
|
|
{
|
|
|
|
switch (po->fanout->type) {
|
|
|
|
case PACKET_FANOUT_CBPF:
|
|
|
|
return fanout_set_data_cbpf(po, data, len);
|
2015-08-15 02:31:35 +00:00
|
|
|
case PACKET_FANOUT_EBPF:
|
|
|
|
return fanout_set_data_ebpf(po, data, len);
|
2015-08-15 02:31:34 +00:00
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
static void fanout_release_data(struct packet_fanout *f)
|
|
|
|
{
|
|
|
|
switch (f->type) {
|
|
|
|
case PACKET_FANOUT_CBPF:
|
2015-08-15 02:31:35 +00:00
|
|
|
case PACKET_FANOUT_EBPF:
|
2015-08-15 02:31:34 +00:00
|
|
|
__fanout_set_data_bpf(f, NULL);
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2017-04-21 14:56:11 +00:00
|
|
|
static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
|
|
|
|
{
|
|
|
|
struct packet_fanout *f;
|
|
|
|
|
|
|
|
list_for_each_entry(f, &fanout_list, list) {
|
|
|
|
if (f->id == candidate_id &&
|
|
|
|
read_pnet(&f->net) == sock_net(sk)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
|
|
|
|
{
|
|
|
|
u16 id = fanout_next_id;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (__fanout_id_is_free(sk, id)) {
|
|
|
|
*new_id = id;
|
|
|
|
fanout_next_id = id + 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
id++;
|
|
|
|
} while (id != fanout_next_id);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2011-07-05 08:43:20 +00:00
|
|
|
static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
|
2011-07-05 08:45:05 +00:00
|
|
|
{
|
2017-02-14 17:03:51 +00:00
|
|
|
struct packet_rollover *rollover = NULL;
|
2011-07-05 08:45:05 +00:00
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
struct packet_fanout *f, *match;
|
2011-07-05 08:43:20 +00:00
|
|
|
u8 type = type_flags & 0xff;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
u8 flags = type_flags >> 8;
|
2011-07-05 08:45:05 +00:00
|
|
|
int err;
|
|
|
|
|
|
|
|
switch (type) {
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
case PACKET_FANOUT_ROLLOVER:
|
|
|
|
if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
|
|
|
|
return -EINVAL;
|
2011-07-05 08:45:05 +00:00
|
|
|
case PACKET_FANOUT_HASH:
|
|
|
|
case PACKET_FANOUT_LB:
|
2011-07-06 08:56:38 +00:00
|
|
|
case PACKET_FANOUT_CPU:
|
2013-08-28 20:13:09 +00:00
|
|
|
case PACKET_FANOUT_RND:
|
2014-01-22 21:01:44 +00:00
|
|
|
case PACKET_FANOUT_QM:
|
2015-08-15 02:31:34 +00:00
|
|
|
case PACKET_FANOUT_CBPF:
|
2015-08-15 02:31:35 +00:00
|
|
|
case PACKET_FANOUT_EBPF:
|
2011-07-05 08:45:05 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2017-02-14 17:03:51 +00:00
|
|
|
mutex_lock(&fanout_mutex);
|
|
|
|
|
|
|
|
err = -EALREADY;
|
2011-07-05 08:45:05 +00:00
|
|
|
if (po->fanout)
|
2017-02-14 17:03:51 +00:00
|
|
|
goto out;
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2015-05-17 23:44:02 +00:00
|
|
|
if (type == PACKET_FANOUT_ROLLOVER ||
|
|
|
|
(type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
|
2017-02-14 17:03:51 +00:00
|
|
|
err = -ENOMEM;
|
|
|
|
rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
|
|
|
|
if (!rollover)
|
|
|
|
goto out;
|
|
|
|
atomic_long_set(&rollover->num, 0);
|
|
|
|
atomic_long_set(&rollover->num_huge, 0);
|
|
|
|
atomic_long_set(&rollover->num_failed, 0);
|
2015-05-12 15:56:46 +00:00
|
|
|
}
|
|
|
|
|
2017-04-21 14:56:11 +00:00
|
|
|
if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
|
|
|
|
if (id != 0) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (!fanout_find_new_id(sk, &id)) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/* ephemeral flag for the first socket in the group: drop it */
|
|
|
|
flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
|
|
|
|
}
|
|
|
|
|
2011-07-05 08:45:05 +00:00
|
|
|
match = NULL;
|
|
|
|
list_for_each_entry(f, &fanout_list, list) {
|
|
|
|
if (f->id == id &&
|
|
|
|
read_pnet(&f->net) == sock_net(sk)) {
|
|
|
|
match = f;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2011-07-07 13:41:29 +00:00
|
|
|
err = -EINVAL;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
if (match && match->flags != flags)
|
2011-07-07 13:41:29 +00:00
|
|
|
goto out;
|
2011-07-05 08:45:05 +00:00
|
|
|
if (!match) {
|
2011-07-07 13:41:29 +00:00
|
|
|
err = -ENOMEM;
|
2011-07-05 08:45:05 +00:00
|
|
|
match = kzalloc(sizeof(*match), GFP_KERNEL);
|
2011-07-07 13:41:29 +00:00
|
|
|
if (!match)
|
|
|
|
goto out;
|
|
|
|
write_pnet(&match->net, sock_net(sk));
|
|
|
|
match->id = id;
|
|
|
|
match->type = type;
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
match->flags = flags;
|
2011-07-07 13:41:29 +00:00
|
|
|
INIT_LIST_HEAD(&match->list);
|
|
|
|
spin_lock_init(&match->lock);
|
2017-06-30 10:08:10 +00:00
|
|
|
refcount_set(&match->sk_ref, 0);
|
2015-08-15 02:31:34 +00:00
|
|
|
fanout_init_data(match);
|
2011-07-07 13:41:29 +00:00
|
|
|
match->prot_hook.type = po->prot_hook.type;
|
|
|
|
match->prot_hook.dev = po->prot_hook.dev;
|
|
|
|
match->prot_hook.func = packet_rcv_fanout;
|
|
|
|
match->prot_hook.af_packet_priv = match;
|
2012-08-16 22:02:58 +00:00
|
|
|
match->prot_hook.id_match = match_fanout_group;
|
2011-07-07 13:41:29 +00:00
|
|
|
list_add(&match->list, &fanout_list);
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
2011-07-07 13:41:29 +00:00
|
|
|
err = -EINVAL;
|
2017-09-14 21:14:41 +00:00
|
|
|
|
|
|
|
spin_lock(&po->bind_lock);
|
|
|
|
if (po->running &&
|
|
|
|
match->type == type &&
|
2011-07-07 13:41:29 +00:00
|
|
|
match->prot_hook.type == po->prot_hook.type &&
|
|
|
|
match->prot_hook.dev == po->prot_hook.dev) {
|
|
|
|
err = -ENOSPC;
|
2017-06-30 10:08:10 +00:00
|
|
|
if (refcount_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
|
2011-07-07 13:41:29 +00:00
|
|
|
__dev_remove_pack(&po->prot_hook);
|
|
|
|
po->fanout = match;
|
2017-11-28 15:44:29 +00:00
|
|
|
po->rollover = rollover;
|
|
|
|
rollover = NULL;
|
2017-06-30 10:08:10 +00:00
|
|
|
refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
|
2011-07-07 13:41:29 +00:00
|
|
|
__fanout_link(sk, po);
|
|
|
|
err = 0;
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
|
|
|
}
|
2017-09-14 21:14:41 +00:00
|
|
|
spin_unlock(&po->bind_lock);
|
|
|
|
|
|
|
|
if (err && !refcount_read(&match->sk_ref)) {
|
|
|
|
list_del(&match->list);
|
|
|
|
kfree(match);
|
|
|
|
}
|
|
|
|
|
2011-07-07 13:41:29 +00:00
|
|
|
out:
|
2017-11-28 15:44:29 +00:00
|
|
|
kfree(rollover);
|
2017-02-14 17:03:51 +00:00
|
|
|
mutex_unlock(&fanout_mutex);
|
2011-07-05 08:45:05 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
|
|
|
|
* pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
|
|
|
|
* It is the responsibility of the caller to call fanout_release_data() and
|
|
|
|
* free the returned packet_fanout (after synchronize_net())
|
|
|
|
*/
|
|
|
|
static struct packet_fanout *fanout_release(struct sock *sk)
|
2011-07-05 08:45:05 +00:00
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
struct packet_fanout *f;
|
|
|
|
|
2012-08-16 05:36:48 +00:00
|
|
|
mutex_lock(&fanout_mutex);
|
2017-02-14 17:03:51 +00:00
|
|
|
f = po->fanout;
|
|
|
|
if (f) {
|
|
|
|
po->fanout = NULL;
|
|
|
|
|
2017-06-30 10:08:10 +00:00
|
|
|
if (refcount_dec_and_test(&f->sk_ref))
|
2017-02-14 17:03:51 +00:00
|
|
|
list_del(&f->list);
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
else
|
|
|
|
f = NULL;
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
|
|
|
mutex_unlock(&fanout_mutex);
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
|
|
|
|
return f;
|
2011-07-05 08:45:05 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-11-11 22:25:42 +00:00
|
|
|
static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
|
|
|
|
struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
/* Earlier code assumed this would be a VLAN pkt, double-check
|
|
|
|
* this now that we have the actual packet in hand. We can only
|
|
|
|
* do this check on Ethernet devices.
|
|
|
|
*/
|
|
|
|
if (unlikely(dev->type != ARPHRD_ETHER))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
|
|
|
|
}
|
|
|
|
|
2005-12-22 20:49:22 +00:00
|
|
|
static const struct proto_ops packet_ops;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-12-22 20:49:22 +00:00
|
|
|
static const struct proto_ops packet_ops_spkt;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct packet_type *pt, struct net_device *orig_dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
struct sockaddr_pkt *spkt;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When we registered the protocol we saved the socket in the data
|
|
|
|
* field for just this event.
|
|
|
|
*/
|
|
|
|
|
|
|
|
sk = pt->af_packet_priv;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Yank back the headers [hope the device set this
|
|
|
|
* right or kerboom...]
|
|
|
|
*
|
|
|
|
* Incoming packets have ll header pulled,
|
|
|
|
* push it back.
|
|
|
|
*
|
2007-03-19 22:33:04 +00:00
|
|
|
* For outgoing ones skb->data == skb_mac_header(skb)
|
2005-04-16 22:20:36 +00:00
|
|
|
* so that this procedure is noop.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (skb->pkt_type == PACKET_LOOPBACK)
|
|
|
|
goto out;
|
|
|
|
|
2009-11-25 23:14:13 +00:00
|
|
|
if (!net_eq(dev_net(dev), sock_net(sk)))
|
2007-11-20 06:28:35 +00:00
|
|
|
goto out;
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
skb = skb_share_check(skb, GFP_ATOMIC);
|
|
|
|
if (skb == NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto oom;
|
|
|
|
|
|
|
|
/* drop any routing info */
|
2009-06-02 05:19:30 +00:00
|
|
|
skb_dst_drop(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-07-12 18:57:52 +00:00
|
|
|
/* drop conntrack reference */
|
|
|
|
nf_reset(skb);
|
|
|
|
|
2007-02-05 07:33:10 +00:00
|
|
|
spkt = &PACKET_SKB_CB(skb)->sa.pkt;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-19 22:33:04 +00:00
|
|
|
skb_push(skb, skb->data - skb_mac_header(skb));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The SOCK_PACKET socket receives _all_ frames.
|
|
|
|
*/
|
|
|
|
|
|
|
|
spkt->spkt_family = dev->type;
|
|
|
|
strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
|
|
|
|
spkt->spkt_protocol = skb->protocol;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Charge the memory to the socket. This is done specifically
|
|
|
|
* to prevent sockets using all the memory up.
|
|
|
|
*/
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
if (sock_queue_rcv_skb(sk, skb) == 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
out:
|
|
|
|
kfree_skb(skb);
|
|
|
|
oom:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Output a raw packet to a device layer. This bypasses all the other
|
|
|
|
* protocol layers and you must therefore supply it with a complete frame
|
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2015-03-02 07:37:48 +00:00
|
|
|
static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
|
|
|
|
size_t len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
2014-01-17 21:53:15 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
|
2009-12-15 05:47:03 +00:00
|
|
|
struct sk_buff *skb = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct net_device *dev;
|
2016-04-03 03:08:12 +00:00
|
|
|
struct sockcm_cookie sockc;
|
2009-07-21 21:57:59 +00:00
|
|
|
__be16 proto = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
int err;
|
2012-02-11 15:39:30 +00:00
|
|
|
int extra_len = 0;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2007-02-09 14:25:10 +00:00
|
|
|
* Get and verify the address.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
if (saddr) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (msg->msg_namelen < sizeof(struct sockaddr))
|
2009-07-21 21:57:59 +00:00
|
|
|
return -EINVAL;
|
|
|
|
if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
|
|
|
|
proto = saddr->spkt_protocol;
|
|
|
|
} else
|
|
|
|
return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2007-02-09 14:25:10 +00:00
|
|
|
* Find the device first to size check it
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2012-06-10 08:59:28 +00:00
|
|
|
saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
|
2009-12-15 05:47:03 +00:00
|
|
|
retry:
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
|
2005-04-16 22:20:36 +00:00
|
|
|
err = -ENODEV;
|
|
|
|
if (dev == NULL)
|
|
|
|
goto out_unlock;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2007-01-26 03:30:36 +00:00
|
|
|
err = -ENETDOWN;
|
|
|
|
if (!(dev->flags & IFF_UP))
|
|
|
|
goto out_unlock;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2009-07-21 21:57:59 +00:00
|
|
|
* You may not queue a frame bigger than the mtu. This is the lowest level
|
|
|
|
* raw protocol and you must do your own fragmentation at this level.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2012-02-11 15:39:30 +00:00
|
|
|
if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
|
|
|
|
if (!netif_supports_nofcs(dev)) {
|
|
|
|
err = -EPROTONOSUPPORT;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
extra_len = 4; /* We're doing our own CRC */
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
err = -EMSGSIZE;
|
2012-02-11 15:39:30 +00:00
|
|
|
if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_unlock;
|
|
|
|
|
2009-12-15 05:47:03 +00:00
|
|
|
if (!skb) {
|
|
|
|
size_t reserved = LL_RESERVED_SPACE(dev);
|
2011-11-18 02:20:05 +00:00
|
|
|
int tlen = dev->needed_tailroom;
|
2009-12-15 05:47:03 +00:00
|
|
|
unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
|
|
|
|
|
|
|
|
rcu_read_unlock();
|
2011-11-18 02:20:05 +00:00
|
|
|
skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
|
2009-12-15 05:47:03 +00:00
|
|
|
if (skb == NULL)
|
|
|
|
return -ENOBUFS;
|
|
|
|
/* FIXME: Save some space for broken drivers that write a hard
|
|
|
|
* header at transmission time by themselves. PPP is the notable
|
|
|
|
* one here. This should really be fixed at the driver level.
|
|
|
|
*/
|
|
|
|
skb_reserve(skb, reserved);
|
|
|
|
skb_reset_network_header(skb);
|
|
|
|
|
|
|
|
/* Try to align data part correctly */
|
|
|
|
if (hhlen) {
|
|
|
|
skb->data -= hhlen;
|
|
|
|
skb->tail -= hhlen;
|
|
|
|
if (len < hhlen)
|
|
|
|
skb_reset_network_header(skb);
|
|
|
|
}
|
2014-04-07 01:25:44 +00:00
|
|
|
err = memcpy_from_msg(skb_put(skb, len), msg, len);
|
2009-12-15 05:47:03 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free;
|
|
|
|
goto retry;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2016-03-10 02:58:34 +00:00
|
|
|
if (!dev_validate_header(dev, skb->data, len)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
2015-11-11 22:25:42 +00:00
|
|
|
if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
|
|
|
|
!packet_extra_vlan_len_allowed(dev, skb)) {
|
|
|
|
err = -EMSGSIZE;
|
|
|
|
goto out_unlock;
|
2011-02-11 09:35:18 +00:00
|
|
|
}
|
2009-12-15 05:47:03 +00:00
|
|
|
|
2016-07-19 05:40:51 +00:00
|
|
|
sockc.tsflags = sk->sk_tsflags;
|
2016-04-03 03:08:12 +00:00
|
|
|
if (msg->msg_controllen) {
|
|
|
|
err = sock_cmsg_send(sk, msg, &sockc);
|
2016-07-20 22:01:18 +00:00
|
|
|
if (unlikely(err))
|
2016-04-03 03:08:12 +00:00
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
skb->protocol = proto;
|
|
|
|
skb->dev = dev;
|
|
|
|
skb->priority = sk->sk_priority;
|
2009-10-01 19:14:46 +00:00
|
|
|
skb->mark = sk->sk_mark;
|
2013-04-14 08:08:13 +00:00
|
|
|
|
2016-04-03 03:08:12 +00:00
|
|
|
sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-02-11 15:39:30 +00:00
|
|
|
if (unlikely(extra_len == 4))
|
|
|
|
skb->no_fcs = 1;
|
|
|
|
|
2013-03-26 23:11:22 +00:00
|
|
|
skb_probe_transport_header(skb, 0);
|
2013-03-25 20:19:57 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
dev_queue_xmit(skb);
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_unlock();
|
2009-07-21 21:57:59 +00:00
|
|
|
return len;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
out_unlock:
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_unlock();
|
2009-12-15 05:47:03 +00:00
|
|
|
out_free:
|
|
|
|
kfree_skb(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2015-10-07 17:55:41 +00:00
|
|
|
static unsigned int run_filter(struct sk_buff *skb,
|
|
|
|
const struct sock *sk,
|
|
|
|
unsigned int res)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sk_filter *filter;
|
2006-08-31 22:28:39 +00:00
|
|
|
|
2011-01-18 07:46:52 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
filter = rcu_dereference(sk->sk_filter);
|
2007-01-24 23:21:02 +00:00
|
|
|
if (filter != NULL)
|
2015-10-07 17:55:41 +00:00
|
|
|
res = bpf_prog_run_clear_cb(filter->prog, skb);
|
2011-01-18 07:46:52 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-01-24 23:21:02 +00:00
|
|
|
return res;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2016-02-03 23:02:14 +00:00
|
|
|
static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
|
|
|
|
size_t *len)
|
|
|
|
{
|
|
|
|
struct virtio_net_hdr vnet_hdr;
|
|
|
|
|
|
|
|
if (*len < sizeof(vnet_hdr))
|
|
|
|
return -EINVAL;
|
|
|
|
*len -= sizeof(vnet_hdr);
|
|
|
|
|
2018-06-06 15:23:01 +00:00
|
|
|
if (virtio_net_hdr_from_skb(skb, &vnet_hdr, vio_le(), true, 0))
|
2016-02-03 23:02:14 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2010-12-06 20:50:09 +00:00
|
|
|
* This function makes lazy skb cloning in hope that most of packets
|
|
|
|
* are discarded by BPF.
|
|
|
|
*
|
|
|
|
* Note tricky part: we DO mangle shared skb! skb->data, skb->len
|
|
|
|
* and skb->cb are mangled. It works because (and until) packets
|
|
|
|
* falling here are owned by current CPU. Output packets are cloned
|
|
|
|
* by dev_queue_xmit_nit(), input packets are processed by net_bh
|
|
|
|
* sequencially, so that if we return skb to original state on exit,
|
|
|
|
* we will not harm anyone.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct packet_type *pt, struct net_device *orig_dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
struct sockaddr_ll *sll;
|
|
|
|
struct packet_sock *po;
|
2009-07-21 21:57:59 +00:00
|
|
|
u8 *skb_head = skb->data;
|
2005-04-16 22:20:36 +00:00
|
|
|
int skb_len = skb->len;
|
2007-01-24 23:21:02 +00:00
|
|
|
unsigned int snaplen, res;
|
2016-04-14 21:10:04 +00:00
|
|
|
bool is_drop_n_account = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (skb->pkt_type == PACKET_LOOPBACK)
|
|
|
|
goto drop;
|
|
|
|
|
|
|
|
sk = pt->af_packet_priv;
|
|
|
|
po = pkt_sk(sk);
|
|
|
|
|
2009-11-25 23:14:13 +00:00
|
|
|
if (!net_eq(dev_net(dev), sock_net(sk)))
|
2007-11-20 06:28:35 +00:00
|
|
|
goto drop;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
skb->dev = dev;
|
|
|
|
|
2007-10-09 08:40:57 +00:00
|
|
|
if (dev->header_ops) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* The device has an explicit notion of ll header,
|
2010-12-06 20:50:09 +00:00
|
|
|
* exported to higher levels.
|
|
|
|
*
|
|
|
|
* Otherwise, the device hides details of its frame
|
|
|
|
* structure, so that corresponding packet head is
|
|
|
|
* never delivered to user.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
if (sk->sk_type != SOCK_DGRAM)
|
2007-03-19 22:33:04 +00:00
|
|
|
skb_push(skb, skb->data - skb_mac_header(skb));
|
2005-04-16 22:20:36 +00:00
|
|
|
else if (skb->pkt_type == PACKET_OUTGOING) {
|
|
|
|
/* Special case: outgoing packets have ll header at head */
|
2007-03-11 01:16:10 +00:00
|
|
|
skb_pull(skb, skb_network_offset(skb));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
snaplen = skb->len;
|
|
|
|
|
2007-01-24 23:21:02 +00:00
|
|
|
res = run_filter(skb, sk, snaplen);
|
|
|
|
if (!res)
|
2006-08-31 22:28:39 +00:00
|
|
|
goto drop_n_restore;
|
2007-01-24 23:21:02 +00:00
|
|
|
if (snaplen > res)
|
|
|
|
snaplen = res;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-12-21 07:11:44 +00:00
|
|
|
if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto drop_n_acct;
|
|
|
|
|
|
|
|
if (skb_shared(skb)) {
|
|
|
|
struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
|
|
|
|
if (nskb == NULL)
|
|
|
|
goto drop_n_acct;
|
|
|
|
|
|
|
|
if (skb_head != skb->data) {
|
|
|
|
skb->data = skb_head;
|
|
|
|
skb->len = skb_len;
|
|
|
|
}
|
2012-04-19 02:24:42 +00:00
|
|
|
consume_skb(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
skb = nskb;
|
|
|
|
}
|
|
|
|
|
2015-03-01 12:58:29 +00:00
|
|
|
sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
|
2007-02-05 07:33:10 +00:00
|
|
|
|
|
|
|
sll = &PACKET_SKB_CB(skb)->sa.ll;
|
2005-04-16 22:20:36 +00:00
|
|
|
sll->sll_hatype = dev->type;
|
|
|
|
sll->sll_pkttype = skb->pkt_type;
|
2007-11-11 06:03:25 +00:00
|
|
|
if (unlikely(po->origdev))
|
2007-04-20 23:05:39 +00:00
|
|
|
sll->sll_ifindex = orig_dev->ifindex;
|
|
|
|
else
|
|
|
|
sll->sll_ifindex = dev->ifindex;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-09-27 05:13:38 +00:00
|
|
|
sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-03-01 12:58:28 +00:00
|
|
|
/* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
|
|
|
|
* Use their space for storing the original skb length.
|
|
|
|
*/
|
|
|
|
PACKET_SKB_CB(skb)->sa.origlen = skb->len;
|
2007-02-05 07:31:32 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (pskb_trim(skb, snaplen))
|
|
|
|
goto drop_n_acct;
|
|
|
|
|
|
|
|
skb_set_owner_r(skb, sk);
|
|
|
|
skb->dev = NULL;
|
2009-06-02 05:19:30 +00:00
|
|
|
skb_dst_drop(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-07-12 18:57:52 +00:00
|
|
|
/* drop conntrack reference */
|
|
|
|
nf_reset(skb);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_lock(&sk->sk_receive_queue.lock);
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats1.tp_packets++;
|
2015-03-01 12:58:30 +00:00
|
|
|
sock_skb_set_dropcount(sk, skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
__skb_queue_tail(&sk->sk_receive_queue, skb);
|
|
|
|
spin_unlock(&sk->sk_receive_queue.lock);
|
2014-04-11 20:15:36 +00:00
|
|
|
sk->sk_data_ready(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
drop_n_acct:
|
2016-04-14 21:10:04 +00:00
|
|
|
is_drop_n_account = true;
|
make PACKET_STATISTICS getsockopt report consistently between ring and non-ring
This is a minor change.
Up until kernel 2.6.32, getsockopt(fd, SOL_PACKET, PACKET_STATISTICS,
...) would return total and dropped packets since its last invocation. The
introduction of socket queue overflow reporting [1] changed drop
rate calculation in the normal packet socket path, but not when using a
packet ring. As a result, the getsockopt now returns different statistics
depending on the reception method used. With a ring, it still returns the
count since the last call, as counts are incremented in tpacket_rcv and
reset in getsockopt. Without a ring, it returns 0 if no drops occurred
since the last getsockopt and the total drops over the lifespan of
the socket otherwise. The culprit is this line in packet_rcv, executed
on a drop:
drop_n_acct:
po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);
As it shows, the new drop number it taken from the socket drop counter,
which is not reset at getsockopt. I put together a small example
that demonstrates the issue [2]. It runs for 10 seconds and overflows
the queue/ring on every odd second. The reported drop rates are:
ring: 16, 0, 16, 0, 16, ...
non-ring: 0, 15, 0, 30, 0, 46, 0, 60, 0 , 74.
Note how the even ring counts monotonically increase. Because the
getsockopt adds tp_drops to tp_packets, total counts are similarly
reported cumulatively. Long story short, reinstating the original code, as
the below patch does, fixes the issue at the cost of additional per-packet
cycles. Another solution that does not introduce per-packet overhead
is be to keep the current data path, record the value of sk_drops at
getsockopt() at call N in a new field in struct packetsock and subtract
that when reporting at call N+1. I'll be happy to code that, instead,
it's just more messy.
[1] http://patchwork.ozlabs.org/patch/35665/
[2] http://kernel.googlecode.com/files/test-packetsock-getstatistics.c
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-09-30 10:38:28 +00:00
|
|
|
spin_lock(&sk->sk_receive_queue.lock);
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats1.tp_drops++;
|
make PACKET_STATISTICS getsockopt report consistently between ring and non-ring
This is a minor change.
Up until kernel 2.6.32, getsockopt(fd, SOL_PACKET, PACKET_STATISTICS,
...) would return total and dropped packets since its last invocation. The
introduction of socket queue overflow reporting [1] changed drop
rate calculation in the normal packet socket path, but not when using a
packet ring. As a result, the getsockopt now returns different statistics
depending on the reception method used. With a ring, it still returns the
count since the last call, as counts are incremented in tpacket_rcv and
reset in getsockopt. Without a ring, it returns 0 if no drops occurred
since the last getsockopt and the total drops over the lifespan of
the socket otherwise. The culprit is this line in packet_rcv, executed
on a drop:
drop_n_acct:
po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);
As it shows, the new drop number it taken from the socket drop counter,
which is not reset at getsockopt. I put together a small example
that demonstrates the issue [2]. It runs for 10 seconds and overflows
the queue/ring on every odd second. The reported drop rates are:
ring: 16, 0, 16, 0, 16, ...
non-ring: 0, 15, 0, 30, 0, 46, 0, 60, 0 , 74.
Note how the even ring counts monotonically increase. Because the
getsockopt adds tp_drops to tp_packets, total counts are similarly
reported cumulatively. Long story short, reinstating the original code, as
the below patch does, fixes the issue at the cost of additional per-packet
cycles. Another solution that does not introduce per-packet overhead
is be to keep the current data path, record the value of sk_drops at
getsockopt() at call N in a new field in struct packetsock and subtract
that when reporting at call N+1. I'll be happy to code that, instead,
it's just more messy.
[1] http://patchwork.ozlabs.org/patch/35665/
[2] http://kernel.googlecode.com/files/test-packetsock-getstatistics.c
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-09-30 10:38:28 +00:00
|
|
|
atomic_inc(&sk->sk_drops);
|
|
|
|
spin_unlock(&sk->sk_receive_queue.lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
drop_n_restore:
|
|
|
|
if (skb_head != skb->data && skb_shared(skb)) {
|
|
|
|
skb->data = skb_head;
|
|
|
|
skb->len = skb_len;
|
|
|
|
}
|
|
|
|
drop:
|
2016-04-14 21:10:04 +00:00
|
|
|
if (!is_drop_n_account)
|
|
|
|
consume_skb(skb);
|
|
|
|
else
|
|
|
|
kfree_skb(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct packet_type *pt, struct net_device *orig_dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
struct packet_sock *po;
|
|
|
|
struct sockaddr_ll *sll;
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr h;
|
2009-07-21 21:57:59 +00:00
|
|
|
u8 *skb_head = skb->data;
|
2005-04-16 22:20:36 +00:00
|
|
|
int skb_len = skb->len;
|
2007-01-24 23:21:02 +00:00
|
|
|
unsigned int snaplen, res;
|
2011-08-19 10:18:16 +00:00
|
|
|
unsigned long status = TP_STATUS_USER;
|
2008-07-15 05:50:15 +00:00
|
|
|
unsigned short macoff, netoff, hdrlen;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sk_buff *copy_skb = NULL;
|
2008-07-15 05:50:15 +00:00
|
|
|
struct timespec ts;
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
__u32 ts_status;
|
2016-04-14 21:10:04 +00:00
|
|
|
bool is_drop_n_account = false;
|
2017-08-28 18:29:41 +00:00
|
|
|
bool do_vnet = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-12-17 13:53:32 +00:00
|
|
|
/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
|
|
|
|
* We may add members to them until current aligned size without forcing
|
|
|
|
* userspace to call getsockopt(..., PACKET_HDRLEN, ...).
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
|
|
|
|
BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (skb->pkt_type == PACKET_LOOPBACK)
|
|
|
|
goto drop;
|
|
|
|
|
|
|
|
sk = pt->af_packet_priv;
|
|
|
|
po = pkt_sk(sk);
|
|
|
|
|
2009-11-25 23:14:13 +00:00
|
|
|
if (!net_eq(dev_net(dev), sock_net(sk)))
|
2007-11-20 06:28:35 +00:00
|
|
|
goto drop;
|
|
|
|
|
2007-10-09 08:40:57 +00:00
|
|
|
if (dev->header_ops) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sk->sk_type != SOCK_DGRAM)
|
2007-03-19 22:33:04 +00:00
|
|
|
skb_push(skb, skb->data - skb_mac_header(skb));
|
2005-04-16 22:20:36 +00:00
|
|
|
else if (skb->pkt_type == PACKET_OUTGOING) {
|
|
|
|
/* Special case: outgoing packets have ll header at head */
|
2007-03-11 01:16:10 +00:00
|
|
|
skb_pull(skb, skb_network_offset(skb));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
snaplen = skb->len;
|
|
|
|
|
2007-01-24 23:21:02 +00:00
|
|
|
res = run_filter(skb, sk, snaplen);
|
|
|
|
if (!res)
|
2006-08-31 22:28:39 +00:00
|
|
|
goto drop_n_restore;
|
2015-03-23 06:11:12 +00:00
|
|
|
|
|
|
|
if (skb->ip_summed == CHECKSUM_PARTIAL)
|
|
|
|
status |= TP_STATUS_CSUMNOTREADY;
|
2015-03-23 06:11:13 +00:00
|
|
|
else if (skb->pkt_type != PACKET_OUTGOING &&
|
|
|
|
(skb->ip_summed == CHECKSUM_COMPLETE ||
|
|
|
|
skb_csum_unnecessary(skb)))
|
|
|
|
status |= TP_STATUS_CSUM_VALID;
|
2015-03-23 06:11:12 +00:00
|
|
|
|
2007-01-24 23:21:02 +00:00
|
|
|
if (snaplen > res)
|
|
|
|
snaplen = res;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (sk->sk_type == SOCK_DGRAM) {
|
2008-07-19 01:05:19 +00:00
|
|
|
macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
|
|
|
|
po->tp_reserve;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2012-04-15 05:58:06 +00:00
|
|
|
unsigned int maclen = skb_network_offset(skb);
|
2008-07-15 05:50:15 +00:00
|
|
|
netoff = TPACKET_ALIGN(po->tp_hdrlen +
|
2008-07-19 01:05:19 +00:00
|
|
|
(maclen < 16 ? 16 : maclen)) +
|
2016-02-03 23:02:15 +00:00
|
|
|
po->tp_reserve;
|
2017-08-28 18:29:41 +00:00
|
|
|
if (po->has_vnet_hdr) {
|
2016-02-03 23:02:15 +00:00
|
|
|
netoff += sizeof(struct virtio_net_hdr);
|
2017-08-28 18:29:41 +00:00
|
|
|
do_vnet = true;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
macoff = netoff - maclen;
|
|
|
|
}
|
2011-08-19 10:18:16 +00:00
|
|
|
if (po->tp_version <= TPACKET_V2) {
|
|
|
|
if (macoff + snaplen > po->rx_ring.frame_size) {
|
|
|
|
if (po->copy_thresh &&
|
2011-12-21 07:11:44 +00:00
|
|
|
atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
|
2011-08-19 10:18:16 +00:00
|
|
|
if (skb_shared(skb)) {
|
|
|
|
copy_skb = skb_clone(skb, GFP_ATOMIC);
|
|
|
|
} else {
|
|
|
|
copy_skb = skb_get(skb);
|
|
|
|
skb_head = skb->data;
|
|
|
|
}
|
|
|
|
if (copy_skb)
|
|
|
|
skb_set_owner_r(copy_skb, sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2011-08-19 10:18:16 +00:00
|
|
|
snaplen = po->rx_ring.frame_size - macoff;
|
2017-08-28 18:29:41 +00:00
|
|
|
if ((int)snaplen < 0) {
|
2011-08-19 10:18:16 +00:00
|
|
|
snaplen = 0;
|
2017-08-28 18:29:41 +00:00
|
|
|
do_vnet = false;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2014-08-15 16:16:04 +00:00
|
|
|
} else if (unlikely(macoff + snaplen >
|
|
|
|
GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
|
|
|
|
u32 nval;
|
|
|
|
|
|
|
|
nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
|
|
|
|
pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
|
|
|
|
snaplen, nval, macoff);
|
|
|
|
snaplen = nval;
|
|
|
|
if (unlikely((int)snaplen < 0)) {
|
|
|
|
snaplen = 0;
|
|
|
|
macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
|
2017-08-28 18:29:41 +00:00
|
|
|
do_vnet = false;
|
2014-08-15 16:16:04 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
spin_lock(&sk->sk_receive_queue.lock);
|
2011-08-19 10:18:16 +00:00
|
|
|
h.raw = packet_current_rx_frame(po, skb,
|
|
|
|
TP_STATUS_KERNEL, (macoff+snaplen));
|
2008-07-15 05:50:15 +00:00
|
|
|
if (!h.raw)
|
2016-02-03 23:02:15 +00:00
|
|
|
goto drop_n_account;
|
2011-08-19 10:18:16 +00:00
|
|
|
if (po->tp_version <= TPACKET_V2) {
|
|
|
|
packet_increment_rx_head(po, &po->rx_ring);
|
|
|
|
/*
|
|
|
|
* LOSING will be reported till you read the stats,
|
|
|
|
* because it's COR - Clear On Read.
|
|
|
|
* Anyways, moving it for V1/V2 only as V3 doesn't need this
|
|
|
|
* at packet level.
|
|
|
|
*/
|
2013-04-19 06:12:29 +00:00
|
|
|
if (po->stats.stats1.tp_drops)
|
2011-08-19 10:18:16 +00:00
|
|
|
status |= TP_STATUS_LOSING;
|
|
|
|
}
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats1.tp_packets++;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (copy_skb) {
|
|
|
|
status |= TP_STATUS_COPY;
|
|
|
|
__skb_queue_tail(&sk->sk_receive_queue, copy_skb);
|
|
|
|
}
|
|
|
|
spin_unlock(&sk->sk_receive_queue.lock);
|
|
|
|
|
2017-08-28 18:29:41 +00:00
|
|
|
if (do_vnet) {
|
2016-11-18 23:40:42 +00:00
|
|
|
if (virtio_net_hdr_from_skb(skb, h.raw + macoff -
|
|
|
|
sizeof(struct virtio_net_hdr),
|
2018-06-06 15:23:01 +00:00
|
|
|
vio_le(), true, 0)) {
|
2016-02-03 23:02:15 +00:00
|
|
|
spin_lock(&sk->sk_receive_queue.lock);
|
|
|
|
goto drop_n_account;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-07-15 05:50:15 +00:00
|
|
|
skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
|
|
|
|
if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
|
2013-04-23 00:39:29 +00:00
|
|
|
getnstimeofday(&ts);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
status |= ts_status;
|
|
|
|
|
2008-07-15 05:50:15 +00:00
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
h.h1->tp_len = skb->len;
|
|
|
|
h.h1->tp_snaplen = snaplen;
|
|
|
|
h.h1->tp_mac = macoff;
|
|
|
|
h.h1->tp_net = netoff;
|
2013-04-16 01:29:11 +00:00
|
|
|
h.h1->tp_sec = ts.tv_sec;
|
|
|
|
h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
|
2008-07-15 05:50:15 +00:00
|
|
|
hdrlen = sizeof(*h.h1);
|
|
|
|
break;
|
|
|
|
case TPACKET_V2:
|
|
|
|
h.h2->tp_len = skb->len;
|
|
|
|
h.h2->tp_snaplen = snaplen;
|
|
|
|
h.h2->tp_mac = macoff;
|
|
|
|
h.h2->tp_net = netoff;
|
|
|
|
h.h2->tp_sec = ts.tv_sec;
|
|
|
|
h.h2->tp_nsec = ts.tv_nsec;
|
2015-01-13 16:13:44 +00:00
|
|
|
if (skb_vlan_tag_present(skb)) {
|
|
|
|
h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
|
2013-12-17 13:53:40 +00:00
|
|
|
h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
|
|
|
|
status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
2011-06-01 06:49:10 +00:00
|
|
|
} else {
|
|
|
|
h.h2->tp_vlan_tci = 0;
|
2013-12-17 13:53:40 +00:00
|
|
|
h.h2->tp_vlan_tpid = 0;
|
2011-06-01 06:49:10 +00:00
|
|
|
}
|
2013-12-17 13:53:36 +00:00
|
|
|
memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
|
2008-07-15 05:50:15 +00:00
|
|
|
hdrlen = sizeof(*h.h2);
|
|
|
|
break;
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
/* tp_nxt_offset,vlan are already populated above.
|
|
|
|
* So DONT clear those fields here
|
|
|
|
*/
|
|
|
|
h.h3->tp_status |= status;
|
|
|
|
h.h3->tp_len = skb->len;
|
|
|
|
h.h3->tp_snaplen = snaplen;
|
|
|
|
h.h3->tp_mac = macoff;
|
|
|
|
h.h3->tp_net = netoff;
|
|
|
|
h.h3->tp_sec = ts.tv_sec;
|
|
|
|
h.h3->tp_nsec = ts.tv_nsec;
|
2013-12-17 13:53:36 +00:00
|
|
|
memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
|
2011-08-19 10:18:16 +00:00
|
|
|
hdrlen = sizeof(*h.h3);
|
|
|
|
break;
|
2008-07-15 05:50:15 +00:00
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-15 05:50:15 +00:00
|
|
|
sll = h.raw + TPACKET_ALIGN(hdrlen);
|
2007-09-27 05:13:38 +00:00
|
|
|
sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
sll->sll_family = AF_PACKET;
|
|
|
|
sll->sll_hatype = dev->type;
|
|
|
|
sll->sll_protocol = skb->protocol;
|
|
|
|
sll->sll_pkttype = skb->pkt_type;
|
2007-11-11 06:03:25 +00:00
|
|
|
if (unlikely(po->origdev))
|
2007-04-20 23:05:39 +00:00
|
|
|
sll->sll_ifindex = orig_dev->ifindex;
|
|
|
|
else
|
|
|
|
sll->sll_ifindex = dev->ifindex;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-12-07 08:11:33 +00:00
|
|
|
smp_mb();
|
2014-01-19 10:46:53 +00:00
|
|
|
|
2010-12-07 04:26:16 +00:00
|
|
|
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
2014-01-19 10:46:53 +00:00
|
|
|
if (po->tp_version <= TPACKET_V2) {
|
2010-12-01 02:52:20 +00:00
|
|
|
u8 *start, *end;
|
|
|
|
|
2014-01-19 10:46:53 +00:00
|
|
|
end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
|
|
|
|
macoff + snaplen);
|
|
|
|
|
|
|
|
for (start = h.raw; start < end; start += PAGE_SIZE)
|
|
|
|
flush_dcache_page(pgv_to_page(start));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2014-01-19 10:46:53 +00:00
|
|
|
smp_wmb();
|
2010-12-07 04:26:16 +00:00
|
|
|
#endif
|
2014-01-19 10:46:53 +00:00
|
|
|
|
2014-12-19 03:49:25 +00:00
|
|
|
if (po->tp_version <= TPACKET_V2) {
|
2011-08-19 10:18:16 +00:00
|
|
|
__packet_set_status(po, h.raw, status);
|
2014-12-19 03:49:25 +00:00
|
|
|
sk->sk_data_ready(sk);
|
|
|
|
} else {
|
2011-08-19 10:18:16 +00:00
|
|
|
prb_clear_blk_fill_status(&po->rx_ring);
|
2014-12-19 03:49:25 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
drop_n_restore:
|
|
|
|
if (skb_head != skb->data && skb_shared(skb)) {
|
|
|
|
skb->data = skb_head;
|
|
|
|
skb->len = skb_len;
|
|
|
|
}
|
|
|
|
drop:
|
2016-04-14 21:10:04 +00:00
|
|
|
if (!is_drop_n_account)
|
|
|
|
consume_skb(skb);
|
|
|
|
else
|
|
|
|
kfree_skb(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
2016-02-03 23:02:15 +00:00
|
|
|
drop_n_account:
|
2016-04-14 21:10:04 +00:00
|
|
|
is_drop_n_account = true;
|
2013-04-19 06:12:29 +00:00
|
|
|
po->stats.stats1.tp_drops++;
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&sk->sk_receive_queue.lock);
|
|
|
|
|
2014-04-11 20:15:36 +00:00
|
|
|
sk->sk_data_ready(sk);
|
2009-02-25 00:36:42 +00:00
|
|
|
kfree_skb(copy_skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto drop_n_restore;
|
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static void tpacket_destruct_skb(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(skb->sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
if (likely(po->tx_ring.pg_vec)) {
|
2014-01-19 10:46:53 +00:00
|
|
|
void *ph;
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
__u32 ts;
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
ph = skb_shinfo(skb)->destructor_arg;
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
packet_dec_pending(&po->tx_ring);
|
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
Currently, there is no way to find out which timestamp is reported in
tpacket{,2,3}_hdr's tp_sec, tp_{n,u}sec members. It can be one of
SOF_TIMESTAMPING_SYS_HARDWARE, SOF_TIMESTAMPING_RAW_HARDWARE,
SOF_TIMESTAMPING_SOFTWARE, or a fallback variant late call from the
PF_PACKET code in software.
Therefore, report in the tp_status member of the ring buffer which
timestamp has been reported for RX and TX path. This should not break
anything for the following reasons: i) in RX ring path, the user needs
to test for tp_status & TP_STATUS_USER, and later for other flags as
well such as TP_STATUS_VLAN_VALID et al, so adding other flags will
do no harm; ii) in TX ring path, time stamps with PACKET_TIMESTAMP
socketoption are not available resp. had no effect except that the
application setting this is buggy. Next to TP_STATUS_AVAILABLE, the
user also should check for other flags such as TP_STATUS_WRONG_FORMAT
to reclaim frames to the application. Thus, in case TX ts are turned
off (default case), nothing happens to the application logic, and in
case we want to use this new feature, we now can also check which of
the ts source is reported in the status field as provided in the docs.
Reported-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-04-23 00:39:31 +00:00
|
|
|
|
|
|
|
ts = __packet_set_timestamp(po, ph, skb);
|
|
|
|
__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
sock_wfree(skb);
|
|
|
|
}
|
|
|
|
|
packet: infer protocol from ethernet header if unset
In case no struct sockaddr_ll has been passed to packet
socket's sendmsg() when doing a TX_RING flush run, then
skb->protocol is set to po->num instead, which is the protocol
passed via socket(2)/bind(2).
Applications only xmitting can go the path of allocating the
socket as socket(PF_PACKET, <mode>, 0) and do a bind(2) on the
TX_RING with sll_protocol of 0. That way, register_prot_hook()
is neither called on creation nor on bind time, which saves
cycles when there's no interest in capturing anyway.
That leaves us however with po->num 0 instead and therefore
the TX_RING flush run sets skb->protocol to 0 as well. Eric
reported that this leads to problems when using tools like
trafgen over bonding device. I.e. the bonding's hash function
could invoke the kernel's flow dissector, which depends on
skb->protocol being properly set. In the current situation, all
the traffic is then directed to a single slave.
Fix it up by inferring skb->protocol from the Ethernet header
when not set and we have ARPHRD_ETHER device type. This is only
done in case of SOCK_RAW and where we have a dev->hard_header_len
length. In case of ARPHRD_ETHER devices, this is guaranteed to
cover ETH_HLEN, and therefore being accessed on the skb after
the skb_store_bits().
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-11 22:25:43 +00:00
|
|
|
static void tpacket_set_protocol(const struct net_device *dev,
|
|
|
|
struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
if (dev->type == ARPHRD_ETHER) {
|
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
skb->protocol = eth_hdr(skb)->h_proto;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-03 23:02:14 +00:00
|
|
|
static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
|
|
|
|
{
|
|
|
|
if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
|
|
|
|
(__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
|
|
|
|
__virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
|
|
|
|
__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
|
|
|
|
vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
|
|
|
|
__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
|
|
|
|
__virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
|
|
|
|
|
|
|
|
if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
|
|
|
|
struct virtio_net_hdr *vnet_hdr)
|
|
|
|
{
|
|
|
|
if (*len < sizeof(*vnet_hdr))
|
|
|
|
return -EINVAL;
|
|
|
|
*len -= sizeof(*vnet_hdr);
|
|
|
|
|
2016-11-02 02:09:04 +00:00
|
|
|
if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
|
2016-02-03 23:02:14 +00:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return __packet_snd_vnet_parse(vnet_hdr, *len);
|
|
|
|
}
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
|
2016-02-03 23:02:16 +00:00
|
|
|
void *frame, struct net_device *dev, void *data, int tp_len,
|
2016-04-03 03:08:12 +00:00
|
|
|
__be16 proto, unsigned char *addr, int hlen, int copylen,
|
|
|
|
const struct sockcm_cookie *sockc)
|
2009-05-19 05:11:22 +00:00
|
|
|
{
|
2013-04-16 01:57:46 +00:00
|
|
|
union tpacket_uhdr ph;
|
2016-02-03 23:02:16 +00:00
|
|
|
int to_write, offset, len, nr_frags, len_max;
|
2009-05-19 05:11:22 +00:00
|
|
|
struct socket *sock = po->sk.sk_socket;
|
|
|
|
struct page *page;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
ph.raw = frame;
|
|
|
|
|
|
|
|
skb->protocol = proto;
|
|
|
|
skb->dev = dev;
|
|
|
|
skb->priority = po->sk.sk_priority;
|
2009-10-01 19:14:46 +00:00
|
|
|
skb->mark = po->sk.sk_mark;
|
2016-04-03 03:08:12 +00:00
|
|
|
sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
|
2009-05-19 05:11:22 +00:00
|
|
|
skb_shinfo(skb)->destructor_arg = ph.raw;
|
|
|
|
|
2011-11-18 02:20:04 +00:00
|
|
|
skb_reserve(skb, hlen);
|
2009-05-19 05:11:22 +00:00
|
|
|
skb_reset_network_header(skb);
|
2013-03-25 20:19:57 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
to_write = tp_len;
|
|
|
|
|
|
|
|
if (sock->type == SOCK_DGRAM) {
|
|
|
|
err = dev_hard_header(skb, dev, ntohs(proto), addr,
|
|
|
|
NULL, tp_len);
|
|
|
|
if (unlikely(err < 0))
|
|
|
|
return -EINVAL;
|
2016-02-03 23:02:17 +00:00
|
|
|
} else if (copylen) {
|
2016-03-10 02:58:34 +00:00
|
|
|
int hdrlen = min_t(int, copylen, tp_len);
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
skb_push(skb, dev->hard_header_len);
|
2016-02-03 23:02:17 +00:00
|
|
|
skb_put(skb, copylen - dev->hard_header_len);
|
2016-03-10 02:58:34 +00:00
|
|
|
err = skb_store_bits(skb, 0, data, hdrlen);
|
2009-05-19 05:11:22 +00:00
|
|
|
if (unlikely(err))
|
|
|
|
return err;
|
2016-03-10 02:58:34 +00:00
|
|
|
if (!dev_validate_header(dev, skb->data, hdrlen))
|
|
|
|
return -EINVAL;
|
packet: infer protocol from ethernet header if unset
In case no struct sockaddr_ll has been passed to packet
socket's sendmsg() when doing a TX_RING flush run, then
skb->protocol is set to po->num instead, which is the protocol
passed via socket(2)/bind(2).
Applications only xmitting can go the path of allocating the
socket as socket(PF_PACKET, <mode>, 0) and do a bind(2) on the
TX_RING with sll_protocol of 0. That way, register_prot_hook()
is neither called on creation nor on bind time, which saves
cycles when there's no interest in capturing anyway.
That leaves us however with po->num 0 instead and therefore
the TX_RING flush run sets skb->protocol to 0 as well. Eric
reported that this leads to problems when using tools like
trafgen over bonding device. I.e. the bonding's hash function
could invoke the kernel's flow dissector, which depends on
skb->protocol being properly set. In the current situation, all
the traffic is then directed to a single slave.
Fix it up by inferring skb->protocol from the Ethernet header
when not set and we have ARPHRD_ETHER device type. This is only
done in case of SOCK_RAW and where we have a dev->hard_header_len
length. In case of ARPHRD_ETHER devices, this is guaranteed to
cover ETH_HLEN, and therefore being accessed on the skb after
the skb_store_bits().
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-11 22:25:43 +00:00
|
|
|
if (!skb->protocol)
|
|
|
|
tpacket_set_protocol(dev, skb);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
2016-03-10 02:58:34 +00:00
|
|
|
data += hdrlen;
|
|
|
|
to_write -= hdrlen;
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
offset = offset_in_page(data);
|
|
|
|
len_max = PAGE_SIZE - offset;
|
|
|
|
len = ((to_write > len_max) ? len_max : to_write);
|
|
|
|
|
|
|
|
skb->data_len = to_write;
|
|
|
|
skb->len += to_write;
|
|
|
|
skb->truesize += to_write;
|
2017-06-30 10:08:00 +00:00
|
|
|
refcount_add(to_write, &po->sk.sk_wmem_alloc);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
while (likely(to_write)) {
|
|
|
|
nr_frags = skb_shinfo(skb)->nr_frags;
|
|
|
|
|
|
|
|
if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
|
2009-07-21 21:57:59 +00:00
|
|
|
pr_err("Packet exceed the number of skb frags(%lu)\n",
|
|
|
|
MAX_SKB_FRAGS);
|
2009-05-19 05:11:22 +00:00
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
2010-12-01 02:52:20 +00:00
|
|
|
page = pgv_to_page(data);
|
|
|
|
data += len;
|
2009-05-19 05:11:22 +00:00
|
|
|
flush_dcache_page(page);
|
|
|
|
get_page(page);
|
2010-12-01 02:52:20 +00:00
|
|
|
skb_fill_page_desc(skb, nr_frags, page, offset, len);
|
2009-05-19 05:11:22 +00:00
|
|
|
to_write -= len;
|
|
|
|
offset = 0;
|
|
|
|
len_max = PAGE_SIZE;
|
|
|
|
len = ((to_write > len_max) ? len_max : to_write);
|
|
|
|
}
|
|
|
|
|
2015-11-11 22:25:41 +00:00
|
|
|
skb_probe_transport_header(skb, 0);
|
2015-11-11 22:25:40 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
return tp_len;
|
|
|
|
}
|
|
|
|
|
2016-02-03 23:02:16 +00:00
|
|
|
static int tpacket_parse_header(struct packet_sock *po, void *frame,
|
|
|
|
int size_max, void **data)
|
|
|
|
{
|
|
|
|
union tpacket_uhdr ph;
|
|
|
|
int tp_len, off;
|
|
|
|
|
|
|
|
ph.raw = frame;
|
|
|
|
|
|
|
|
switch (po->tp_version) {
|
2017-01-03 14:31:47 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
if (ph.h3->tp_next_offset != 0) {
|
|
|
|
pr_warn_once("variable sized slot not supported");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
tp_len = ph.h3->tp_len;
|
|
|
|
break;
|
2016-02-03 23:02:16 +00:00
|
|
|
case TPACKET_V2:
|
|
|
|
tp_len = ph.h2->tp_len;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
tp_len = ph.h1->tp_len;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (unlikely(tp_len > size_max)) {
|
|
|
|
pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
|
|
|
|
return -EMSGSIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlikely(po->tp_tx_has_off)) {
|
|
|
|
int off_min, off_max;
|
|
|
|
|
|
|
|
off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
|
|
|
|
off_max = po->tx_ring.frame_size - tp_len;
|
|
|
|
if (po->sk.sk_type == SOCK_DGRAM) {
|
|
|
|
switch (po->tp_version) {
|
2017-01-03 14:31:47 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
off = ph.h3->tp_net;
|
|
|
|
break;
|
2016-02-03 23:02:16 +00:00
|
|
|
case TPACKET_V2:
|
|
|
|
off = ph.h2->tp_net;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
off = ph.h1->tp_net;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
switch (po->tp_version) {
|
2017-01-03 14:31:47 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
off = ph.h3->tp_mac;
|
|
|
|
break;
|
2016-02-03 23:02:16 +00:00
|
|
|
case TPACKET_V2:
|
|
|
|
off = ph.h2->tp_mac;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
off = ph.h1->tp_mac;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (unlikely((off < off_min) || (off_max < off)))
|
|
|
|
return -EINVAL;
|
|
|
|
} else {
|
|
|
|
off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
|
|
|
|
}
|
|
|
|
|
|
|
|
*data = frame + off;
|
|
|
|
return tp_len;
|
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
struct net_device *dev;
|
2016-02-03 23:02:17 +00:00
|
|
|
struct virtio_net_hdr *vnet_hdr = NULL;
|
2016-04-03 03:08:12 +00:00
|
|
|
struct sockcm_cookie sockc;
|
2009-05-19 05:11:22 +00:00
|
|
|
__be16 proto;
|
2013-08-08 00:11:00 +00:00
|
|
|
int err, reserve = 0;
|
2009-07-21 21:57:59 +00:00
|
|
|
void *ph;
|
2014-01-17 21:53:15 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
|
2014-01-15 15:25:35 +00:00
|
|
|
bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
|
2009-05-19 05:11:22 +00:00
|
|
|
int tp_len, size_max;
|
|
|
|
unsigned char *addr;
|
2016-02-03 23:02:16 +00:00
|
|
|
void *data;
|
2009-05-19 05:11:22 +00:00
|
|
|
int len_sum = 0;
|
2012-08-20 03:34:03 +00:00
|
|
|
int status = TP_STATUS_AVAILABLE;
|
2016-02-03 23:02:17 +00:00
|
|
|
int hlen, tlen, copylen = 0;
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
mutex_lock(&po->pg_vec_lock);
|
|
|
|
|
2013-12-06 10:36:15 +00:00
|
|
|
if (likely(saddr == NULL)) {
|
2013-11-21 15:50:58 +00:00
|
|
|
dev = packet_cached_dev_get(po);
|
2009-05-19 05:11:22 +00:00
|
|
|
proto = po->num;
|
|
|
|
addr = NULL;
|
|
|
|
} else {
|
|
|
|
err = -EINVAL;
|
|
|
|
if (msg->msg_namelen < sizeof(struct sockaddr_ll))
|
|
|
|
goto out;
|
|
|
|
if (msg->msg_namelen < (saddr->sll_halen
|
|
|
|
+ offsetof(struct sockaddr_ll,
|
|
|
|
sll_addr)))
|
|
|
|
goto out;
|
|
|
|
proto = saddr->sll_protocol;
|
|
|
|
addr = saddr->sll_addr;
|
2011-06-01 07:18:53 +00:00
|
|
|
dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
err = -ENXIO;
|
|
|
|
if (unlikely(dev == NULL))
|
|
|
|
goto out;
|
|
|
|
err = -ENETDOWN;
|
|
|
|
if (unlikely(!(dev->flags & IFF_UP)))
|
|
|
|
goto out_put;
|
|
|
|
|
2017-05-12 18:19:15 +00:00
|
|
|
sockc.tsflags = po->sk.sk_tsflags;
|
|
|
|
if (msg->msg_controllen) {
|
|
|
|
err = sock_cmsg_send(&po->sk, msg, &sockc);
|
|
|
|
if (unlikely(err))
|
|
|
|
goto out_put;
|
|
|
|
}
|
|
|
|
|
2015-11-11 22:25:44 +00:00
|
|
|
if (po->sk.sk_socket->type == SOCK_RAW)
|
|
|
|
reserve = dev->hard_header_len;
|
2009-05-19 05:11:22 +00:00
|
|
|
size_max = po->tx_ring.frame_size
|
2009-10-29 10:19:11 +00:00
|
|
|
- (po->tp_hdrlen - sizeof(struct sockaddr_ll));
|
2009-05-19 05:11:22 +00:00
|
|
|
|
2016-02-03 23:02:17 +00:00
|
|
|
if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
|
2015-11-11 22:25:44 +00:00
|
|
|
size_max = dev->mtu + reserve + VLAN_HLEN;
|
2013-08-08 00:11:00 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
do {
|
|
|
|
ph = packet_current_frame(po, &po->tx_ring,
|
2014-01-15 15:25:35 +00:00
|
|
|
TP_STATUS_SEND_REQUEST);
|
2009-05-19 05:11:22 +00:00
|
|
|
if (unlikely(ph == NULL)) {
|
2014-01-15 15:25:35 +00:00
|
|
|
if (need_wait && need_resched())
|
|
|
|
schedule();
|
2009-05-19 05:11:22 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2016-02-03 23:02:16 +00:00
|
|
|
skb = NULL;
|
|
|
|
tp_len = tpacket_parse_header(po, ph, size_max, &data);
|
|
|
|
if (tp_len < 0)
|
|
|
|
goto tpacket_error;
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
status = TP_STATUS_SEND_REQUEST;
|
2011-11-18 02:20:04 +00:00
|
|
|
hlen = LL_RESERVED_SPACE(dev);
|
|
|
|
tlen = dev->needed_tailroom;
|
2016-02-03 23:02:17 +00:00
|
|
|
if (po->has_vnet_hdr) {
|
|
|
|
vnet_hdr = data;
|
|
|
|
data += sizeof(*vnet_hdr);
|
|
|
|
tp_len -= sizeof(*vnet_hdr);
|
|
|
|
if (tp_len < 0 ||
|
|
|
|
__packet_snd_vnet_parse(vnet_hdr, tp_len)) {
|
|
|
|
tp_len = -EINVAL;
|
|
|
|
goto tpacket_error;
|
|
|
|
}
|
|
|
|
copylen = __virtio16_to_cpu(vio_le(),
|
|
|
|
vnet_hdr->hdr_len);
|
|
|
|
}
|
2016-03-10 02:58:34 +00:00
|
|
|
copylen = max_t(int, copylen, dev->hard_header_len);
|
2009-05-19 05:11:22 +00:00
|
|
|
skb = sock_alloc_send_skb(&po->sk,
|
2016-02-03 23:02:17 +00:00
|
|
|
hlen + tlen + sizeof(struct sockaddr_ll) +
|
|
|
|
(copylen - dev->hard_header_len),
|
2015-05-08 13:44:37 +00:00
|
|
|
!need_wait, &err);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
2015-05-08 13:44:37 +00:00
|
|
|
if (unlikely(skb == NULL)) {
|
|
|
|
/* we assume the socket was initially writeable ... */
|
|
|
|
if (likely(len_sum > 0))
|
|
|
|
err = len_sum;
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out_status;
|
2015-05-08 13:44:37 +00:00
|
|
|
}
|
2016-02-03 23:02:16 +00:00
|
|
|
tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
|
2016-04-03 03:08:12 +00:00
|
|
|
addr, hlen, copylen, &sockc);
|
2015-07-28 10:57:01 +00:00
|
|
|
if (likely(tp_len >= 0) &&
|
2015-11-11 22:25:44 +00:00
|
|
|
tp_len > dev->mtu + reserve &&
|
2016-02-03 23:02:17 +00:00
|
|
|
!po->has_vnet_hdr &&
|
2015-11-11 22:25:42 +00:00
|
|
|
!packet_extra_vlan_len_allowed(dev, skb))
|
|
|
|
tp_len = -EMSGSIZE;
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
if (unlikely(tp_len < 0)) {
|
2016-02-03 23:02:16 +00:00
|
|
|
tpacket_error:
|
2009-05-19 05:11:22 +00:00
|
|
|
if (po->tp_loss) {
|
|
|
|
__packet_set_status(po, ph,
|
|
|
|
TP_STATUS_AVAILABLE);
|
|
|
|
packet_increment_head(&po->tx_ring);
|
|
|
|
kfree_skb(skb);
|
|
|
|
continue;
|
|
|
|
} else {
|
|
|
|
status = TP_STATUS_WRONG_FORMAT;
|
|
|
|
err = tp_len;
|
|
|
|
goto out_status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-18 23:40:41 +00:00
|
|
|
if (po->has_vnet_hdr && virtio_net_hdr_to_skb(skb, vnet_hdr,
|
|
|
|
vio_le())) {
|
2016-02-03 23:02:17 +00:00
|
|
|
tp_len = -EINVAL;
|
|
|
|
goto tpacket_error;
|
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
skb->destructor = tpacket_destruct_skb;
|
|
|
|
__packet_set_status(po, ph, TP_STATUS_SENDING);
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
packet_inc_pending(&po->tx_ring);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
status = TP_STATUS_SEND_REQUEST;
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
err = po->xmit(skb);
|
2010-01-10 22:04:19 +00:00
|
|
|
if (unlikely(err > 0)) {
|
|
|
|
err = net_xmit_errno(err);
|
|
|
|
if (err && __packet_get_status(po, ph) ==
|
|
|
|
TP_STATUS_AVAILABLE) {
|
|
|
|
/* skb was destructed already */
|
|
|
|
skb = NULL;
|
|
|
|
goto out_status;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* skb was dropped but not destructed yet;
|
|
|
|
* let's treat it like congestion or err < 0
|
|
|
|
*/
|
|
|
|
err = 0;
|
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
packet_increment_head(&po->tx_ring);
|
|
|
|
len_sum += tp_len;
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
} while (likely((ph != NULL) ||
|
|
|
|
/* Note: packet_read_pending() might be slow if we have
|
|
|
|
* to call it as it's per_cpu variable, but in fast-path
|
|
|
|
* we already short-circuit the loop with the first
|
|
|
|
* condition, and luckily don't have to go that path
|
|
|
|
* anyway.
|
|
|
|
*/
|
|
|
|
(need_wait && packet_read_pending(&po->tx_ring))));
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
err = len_sum;
|
|
|
|
goto out_put;
|
|
|
|
|
|
|
|
out_status:
|
|
|
|
__packet_set_status(po, ph, status);
|
|
|
|
kfree_skb(skb);
|
|
|
|
out_put:
|
2013-11-21 15:50:58 +00:00
|
|
|
dev_put(dev);
|
2009-05-19 05:11:22 +00:00
|
|
|
out:
|
|
|
|
mutex_unlock(&po->pg_vec_lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
|
|
|
|
size_t reserve, size_t len,
|
|
|
|
size_t linear, int noblock,
|
|
|
|
int *err)
|
2010-02-05 04:24:10 +00:00
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
/* Under a page? Don't bother with paged skb. */
|
|
|
|
if (prepad + len < PAGE_SIZE || !linear)
|
|
|
|
linear = len;
|
|
|
|
|
|
|
|
skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
|
2013-08-08 21:38:47 +00:00
|
|
|
err, 0);
|
2010-02-05 04:24:10 +00:00
|
|
|
if (!skb)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
skb_reserve(skb, reserve);
|
|
|
|
skb_put(skb, linear);
|
|
|
|
skb->data_len = len - linear;
|
|
|
|
skb->len += len - linear;
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
2014-01-17 21:53:15 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sk_buff *skb;
|
|
|
|
struct net_device *dev;
|
2006-11-08 08:26:29 +00:00
|
|
|
__be16 proto;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned char *addr;
|
2011-06-01 07:18:53 +00:00
|
|
|
int err, reserve = 0;
|
2015-10-08 21:56:49 +00:00
|
|
|
struct sockcm_cookie sockc;
|
2010-02-05 04:24:10 +00:00
|
|
|
struct virtio_net_hdr vnet_hdr = { 0 };
|
|
|
|
int offset = 0;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2017-09-26 16:20:17 +00:00
|
|
|
bool has_vnet_hdr = false;
|
2017-02-07 20:57:21 +00:00
|
|
|
int hlen, tlen, linear;
|
2012-02-11 15:39:30 +00:00
|
|
|
int extra_len = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2007-02-09 14:25:10 +00:00
|
|
|
* Get and verify the address.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2013-12-06 10:36:15 +00:00
|
|
|
if (likely(saddr == NULL)) {
|
2013-11-21 15:50:58 +00:00
|
|
|
dev = packet_cached_dev_get(po);
|
2005-04-16 22:20:36 +00:00
|
|
|
proto = po->num;
|
|
|
|
addr = NULL;
|
|
|
|
} else {
|
|
|
|
err = -EINVAL;
|
|
|
|
if (msg->msg_namelen < sizeof(struct sockaddr_ll))
|
|
|
|
goto out;
|
2005-09-21 07:11:37 +00:00
|
|
|
if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
|
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
proto = saddr->sll_protocol;
|
|
|
|
addr = saddr->sll_addr;
|
2011-06-01 07:18:53 +00:00
|
|
|
dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
err = -ENXIO;
|
2013-11-21 15:50:58 +00:00
|
|
|
if (unlikely(dev == NULL))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_unlock;
|
2007-01-26 03:30:36 +00:00
|
|
|
err = -ENETDOWN;
|
2013-11-21 15:50:58 +00:00
|
|
|
if (unlikely(!(dev->flags & IFF_UP)))
|
2007-01-26 03:30:36 +00:00
|
|
|
goto out_unlock;
|
|
|
|
|
2016-07-19 05:40:51 +00:00
|
|
|
sockc.tsflags = sk->sk_tsflags;
|
2015-10-08 21:56:49 +00:00
|
|
|
sockc.mark = sk->sk_mark;
|
|
|
|
if (msg->msg_controllen) {
|
|
|
|
err = sock_cmsg_send(sk, msg, &sockc);
|
|
|
|
if (unlikely(err))
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2013-11-21 15:50:58 +00:00
|
|
|
if (sock->type == SOCK_RAW)
|
|
|
|
reserve = dev->hard_header_len;
|
2010-02-05 04:24:10 +00:00
|
|
|
if (po->has_vnet_hdr) {
|
2016-02-03 23:02:14 +00:00
|
|
|
err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
|
|
|
|
if (err)
|
2010-02-05 04:24:10 +00:00
|
|
|
goto out_unlock;
|
2017-09-26 16:20:17 +00:00
|
|
|
has_vnet_hdr = true;
|
2010-02-05 04:24:10 +00:00
|
|
|
}
|
|
|
|
|
2012-02-11 15:39:30 +00:00
|
|
|
if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
|
|
|
|
if (!netif_supports_nofcs(dev)) {
|
|
|
|
err = -EPROTONOSUPPORT;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
extra_len = 4; /* We're doing our own CRC */
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
err = -EMSGSIZE;
|
2016-02-03 23:02:14 +00:00
|
|
|
if (!vnet_hdr.gso_type &&
|
|
|
|
(len > dev->mtu + reserve + VLAN_HLEN + extra_len))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_unlock;
|
|
|
|
|
2010-02-05 04:24:10 +00:00
|
|
|
err = -ENOBUFS;
|
2011-11-18 02:20:04 +00:00
|
|
|
hlen = LL_RESERVED_SPACE(dev);
|
|
|
|
tlen = dev->needed_tailroom;
|
2017-02-07 20:57:21 +00:00
|
|
|
linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
|
|
|
|
linear = max(linear, min_t(int, len, dev->hard_header_len));
|
|
|
|
skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
|
2010-02-05 04:24:10 +00:00
|
|
|
msg->msg_flags & MSG_DONTWAIT, &err);
|
2009-07-21 21:57:59 +00:00
|
|
|
if (skb == NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_unlock;
|
|
|
|
|
2018-05-11 17:24:25 +00:00
|
|
|
skb_reset_network_header(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-09 08:36:32 +00:00
|
|
|
err = -EINVAL;
|
2014-11-19 18:10:16 +00:00
|
|
|
if (sock->type == SOCK_DGRAM) {
|
|
|
|
offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
|
2015-01-11 18:01:16 +00:00
|
|
|
if (unlikely(offset < 0))
|
2014-11-19 18:10:16 +00:00
|
|
|
goto out_free;
|
2018-05-11 17:24:25 +00:00
|
|
|
} else if (reserve) {
|
2018-05-24 22:10:30 +00:00
|
|
|
skb_reserve(skb, -reserve);
|
2014-11-19 18:10:16 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Returns -EFAULT on error */
|
2014-11-24 15:42:55 +00:00
|
|
|
err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free;
|
2013-04-14 08:08:13 +00:00
|
|
|
|
2016-03-10 02:58:34 +00:00
|
|
|
if (sock->type == SOCK_RAW &&
|
|
|
|
!dev_validate_header(dev, skb->data, len)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_free;
|
|
|
|
}
|
|
|
|
|
2016-04-03 03:08:12 +00:00
|
|
|
sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-02-03 23:02:14 +00:00
|
|
|
if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
|
2015-11-11 22:25:42 +00:00
|
|
|
!packet_extra_vlan_len_allowed(dev, skb)) {
|
|
|
|
err = -EMSGSIZE;
|
|
|
|
goto out_free;
|
2011-02-11 09:35:18 +00:00
|
|
|
}
|
|
|
|
|
2013-08-08 00:11:00 +00:00
|
|
|
skb->protocol = proto;
|
|
|
|
skb->dev = dev;
|
2005-04-16 22:20:36 +00:00
|
|
|
skb->priority = sk->sk_priority;
|
2015-10-08 21:56:49 +00:00
|
|
|
skb->mark = sockc.mark;
|
packet: check for ndo_select_queue during queue selection
Mathias reported that on an AMD Geode LX embedded board (ALiX)
with ath9k driver PACKET_QDISC_BYPASS, introduced in commit
d346a3fae3ff ("packet: introduce PACKET_QDISC_BYPASS socket
option"), triggers a WARN_ON() coming from the driver itself
via 066dae93bdf ("ath9k: rework tx queue selection and fix
queue stopping/waking").
The reason why this happened is that ndo_select_queue() call
is not invoked from direct xmit path i.e. for ieee80211 subsystem
that sets queue and TID (similar to 802.1d tag) which is being
put into the frame through 802.11e (WMM, QoS). If that is not
set, pending frame counter for e.g. ath9k can get messed up.
So the WARN_ON() in ath9k is absolutely legitimate. Generally,
the hw queue selection in ieee80211 depends on the type of
traffic, and priorities are set according to ieee80211_ac_numbers
mapping; working in a similar way as DiffServ only on a lower
layer, so that the AP can favour frames that have "real-time"
requirements like voice or video data frames.
Therefore, check for presence of ndo_select_queue() in netdev
ops and, if available, invoke it with a fallback handler to
__packet_pick_tx_queue(), so that driver such as bnx2x, ixgbe,
or mlx4 can still select a hw queue for transmission in
relation to the current CPU while e.g. ieee80211 subsystem
can make their own choices.
Reported-by: Mathias Kretschmer <mathias.kretschmer@fokus.fraunhofer.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-02-16 14:55:22 +00:00
|
|
|
|
2017-09-26 16:20:17 +00:00
|
|
|
if (has_vnet_hdr) {
|
2016-11-18 23:40:41 +00:00
|
|
|
err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
|
2016-02-03 23:02:14 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free;
|
|
|
|
len += sizeof(vnet_hdr);
|
2010-02-05 04:24:10 +00:00
|
|
|
}
|
|
|
|
|
2015-11-11 22:25:41 +00:00
|
|
|
skb_probe_transport_header(skb, reserve);
|
|
|
|
|
2012-02-11 15:39:30 +00:00
|
|
|
if (unlikely(extra_len == 4))
|
|
|
|
skb->no_fcs = 1;
|
|
|
|
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
err = po->xmit(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err > 0 && (err = net_xmit_errno(err)) != 0)
|
|
|
|
goto out_unlock;
|
|
|
|
|
2013-11-21 15:50:58 +00:00
|
|
|
dev_put(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
return len;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
out_free:
|
|
|
|
kfree_skb(skb);
|
|
|
|
out_unlock:
|
2013-11-21 15:50:58 +00:00
|
|
|
if (dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
dev_put(dev);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2015-03-02 07:37:48 +00:00
|
|
|
static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
|
2009-05-19 05:11:22 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
if (po->tx_ring.pg_vec)
|
|
|
|
return tpacket_snd(po, msg);
|
|
|
|
else
|
|
|
|
return packet_snd(sock, msg, len);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Close a PACKET socket. This is fairly simple. We immediately go
|
|
|
|
* to 'closed' state and remove our protocol entry in the device list.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int packet_release(struct socket *sock)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po;
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
struct packet_fanout *f;
|
2007-11-20 06:28:35 +00:00
|
|
|
struct net *net;
|
2011-08-19 10:18:16 +00:00
|
|
|
union tpacket_req_u req_u;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (!sk)
|
|
|
|
return 0;
|
|
|
|
|
2008-03-25 17:26:21 +00:00
|
|
|
net = sock_net(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
po = pkt_sk(sk);
|
|
|
|
|
2012-08-21 01:06:47 +00:00
|
|
|
mutex_lock(&net->packet.sklist_lock);
|
2010-02-22 07:57:18 +00:00
|
|
|
sk_del_node_init_rcu(sk);
|
2012-08-21 01:06:47 +00:00
|
|
|
mutex_unlock(&net->packet.sklist_lock);
|
|
|
|
|
|
|
|
preempt_disable();
|
2008-11-24 08:09:29 +00:00
|
|
|
sock_prot_inuse_add(net, sk->sk_prot, -1);
|
2012-08-21 01:06:47 +00:00
|
|
|
preempt_enable();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-02-22 07:57:18 +00:00
|
|
|
spin_lock(&po->bind_lock);
|
2011-07-04 08:44:29 +00:00
|
|
|
unregister_prot_hook(sk, false);
|
2013-12-06 10:36:15 +00:00
|
|
|
packet_cached_dev_reset(po);
|
|
|
|
|
2011-06-01 07:18:52 +00:00
|
|
|
if (po->prot_hook.dev) {
|
|
|
|
dev_put(po->prot_hook.dev);
|
|
|
|
po->prot_hook.dev = NULL;
|
|
|
|
}
|
2010-02-22 07:57:18 +00:00
|
|
|
spin_unlock(&po->bind_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
packet_flush_mclist(sk);
|
|
|
|
|
2018-04-16 00:52:04 +00:00
|
|
|
lock_sock(sk);
|
2013-02-01 07:21:41 +00:00
|
|
|
if (po->rx_ring.pg_vec) {
|
|
|
|
memset(&req_u, 0, sizeof(req_u));
|
2011-08-19 10:18:16 +00:00
|
|
|
packet_set_ring(sk, &req_u, 1, 0);
|
2013-02-01 07:21:41 +00:00
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
|
2013-02-01 07:21:41 +00:00
|
|
|
if (po->tx_ring.pg_vec) {
|
|
|
|
memset(&req_u, 0, sizeof(req_u));
|
2011-08-19 10:18:16 +00:00
|
|
|
packet_set_ring(sk, &req_u, 1, 1);
|
2013-02-01 07:21:41 +00:00
|
|
|
}
|
2018-04-16 00:52:04 +00:00
|
|
|
release_sock(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
f = fanout_release(sk);
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2010-02-22 07:57:18 +00:00
|
|
|
synchronize_net();
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
|
|
|
|
if (f) {
|
2017-11-28 15:44:29 +00:00
|
|
|
kfree(po->rollover);
|
packet: Do not call fanout_release from atomic contexts
Commit 6664498280cf ("packet: call fanout_release, while UNREGISTERING a
netdev"), unfortunately, introduced the following issues.
1. calling mutex_lock(&fanout_mutex) (fanout_release()) from inside
rcu_read-side critical section. rcu_read_lock disables preemption, most often,
which prohibits calling sleeping functions.
[ ] include/linux/rcupdate.h:560 Illegal context switch in RCU read-side critical section!
[ ]
[ ] rcu_scheduler_active = 1, debug_locks = 0
[ ] 4 locks held by ovs-vswitchd/1969:
[ ] #0: (cb_lock){++++++}, at: [<ffffffff8158a6c9>] genl_rcv+0x19/0x40
[ ] #1: (ovs_mutex){+.+.+.}, at: [<ffffffffa04878ca>] ovs_vport_cmd_del+0x4a/0x100 [openvswitch]
[ ] #2: (rtnl_mutex){+.+.+.}, at: [<ffffffff81564157>] rtnl_lock+0x17/0x20
[ ] #3: (rcu_read_lock){......}, at: [<ffffffff81614165>] packet_notifier+0x5/0x3f0
[ ]
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810c9077>] lockdep_rcu_suspicious+0x107/0x110
[ ] [<ffffffff810a2da7>] ___might_sleep+0x57/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff810de93f>] ? vprintk_default+0x1f/0x30
[ ] [<ffffffff81186e88>] ? printk+0x4d/0x4f
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
2. calling mutex_lock(&fanout_mutex) inside spin_lock(&po->bind_lock).
"sleeping function called from invalid context"
[ ] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:620
[ ] in_atomic(): 1, irqs_disabled(): 0, pid: 1969, name: ovs-vswitchd
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff810a2f52>] ___might_sleep+0x202/0x210
[ ] [<ffffffff810a2fd0>] __might_sleep+0x70/0x90
[ ] [<ffffffff8162e80c>] mutex_lock_nested+0x3c/0x3a0
[ ] [<ffffffff816106dd>] fanout_release+0x1d/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
3. calling dev_remove_pack(&fanout->prot_hook), from inside
spin_lock(&po->bind_lock) or rcu_read-side critical-section. dev_remove_pack()
-> synchronize_net(), which might sleep.
[ ] BUG: scheduling while atomic: ovs-vswitchd/1969/0x00000002
[ ] INFO: lockdep is turned off.
[ ] Call Trace:
[ ] [<ffffffff813770c1>] dump_stack+0x85/0xc4
[ ] [<ffffffff81186274>] __schedule_bug+0x64/0x73
[ ] [<ffffffff8162b8cb>] __schedule+0x6b/0xd10
[ ] [<ffffffff8162c5db>] schedule+0x6b/0x80
[ ] [<ffffffff81630b1d>] schedule_timeout+0x38d/0x410
[ ] [<ffffffff810ea3fd>] synchronize_sched_expedited+0x53d/0x810
[ ] [<ffffffff810ea6de>] synchronize_rcu_expedited+0xe/0x10
[ ] [<ffffffff8154eab5>] synchronize_net+0x35/0x50
[ ] [<ffffffff8154eae3>] dev_remove_pack+0x13/0x20
[ ] [<ffffffff8161077e>] fanout_release+0xbe/0xe0
[ ] [<ffffffff81614459>] packet_notifier+0x2f9/0x3f0
4. fanout_release() races with calls from different CPU.
To fix the above problems, remove the call to fanout_release() under
rcu_read_lock(). Instead, call __dev_remove_pack(&fanout->prot_hook) and
netdev_run_todo will be happy that &dev->ptype_specific list is empty. In order
to achieve this, I moved dev_{add,remove}_pack() out of fanout_{add,release} to
__fanout_{link,unlink}. So, call to {,__}unregister_prot_hook() will make sure
fanout->prot_hook is removed as well.
Fixes: 6664498280cf ("packet: call fanout_release, while UNREGISTERING a netdev")
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Anoob Soman <anoob.soman@citrix.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-02-15 20:25:39 +00:00
|
|
|
fanout_release_data(f);
|
|
|
|
kfree(f);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Now the socket is dead. No more input will appear.
|
|
|
|
*/
|
|
|
|
sock_orphan(sk);
|
|
|
|
sock->sk = NULL;
|
|
|
|
|
|
|
|
/* Purge queues */
|
|
|
|
|
|
|
|
skb_queue_purge(&sk->sk_receive_queue);
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
packet_free_pending(po);
|
2007-11-11 05:38:48 +00:00
|
|
|
sk_refcnt_debug_release(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
sock_put(sk);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Attach a packet hook.
|
|
|
|
*/
|
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
|
|
|
|
__be16 proto)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2015-07-27 22:32:21 +00:00
|
|
|
struct net_device *dev_curr;
|
packet: improve socket create/bind latency in some cases
Most people acquire PF_PACKET sockets with a protocol argument in
the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for
all its sockets. Most likely, at some point in time a subsequent
bind() call will follow, e.g. in libpcap with ...
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_ifindex = ifindex;
sll.sll_protocol = htons(ETH_P_ALL);
... as arguments. What happens in the kernel is that already
in socket() syscall, we install a proto hook via register_prot_hook()
if our protocol argument is != 0. Yet, in bind() we're almost
doing the same work by doing a unregister_prot_hook() with an
expensive synchronize_net() call in case during socket() the proto
was != 0, plus follow-up register_prot_hook() with a bound device
to it this time, in order to limit traffic we get.
In the case when the protocol and user supplied device index (== 0)
does not change from socket() to bind(), we can spare us doing
the same work twice. Similarly for re-binding to the same device
and protocol. For these scenarios, we can decrease create/bind
latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case)
with this patch.
Alternatively, for the first case, if people care, they should
simply create their sockets with proto == 0 argument and define
the protocol during bind() as this saves a call to synchronize_net()
as well (sock-bind-3 case).
In all other cases, we're tied to user space behaviour we must not
change, also since a bind() is not strictly required. Thus, we need
the synchronize_net() to make sure no asynchronous packet processing
paths still refer to the previous elements of po->prot_hook.
In case of mmap()ed sockets, the workflow that includes bind() is
socket() -> setsockopt(<ring>) -> bind(). In that case, a pair of
{__unregister, register}_prot_hook is being called from setsockopt()
in order to install the new protocol receive handler. Thus, when
we call bind and can skip a re-hook, we have already previously
installed the new handler. For fanout, this is handled different
entirely, so we should be good.
Timings on an i7-3520M machine:
* sock-bind-1: 89 us
* sock-bind-2: 7447 us
* sock-bind-3: 75 us
sock-bind-1:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-2:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-3:
socket(PF_PACKET, SOCK_RAW, 0) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:34 +00:00
|
|
|
__be16 proto_curr;
|
|
|
|
bool need_rehook;
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
struct net_device *dev = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
bool unlisted = false;
|
2011-07-05 08:45:05 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
spin_lock(&po->bind_lock);
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
|
2017-09-26 16:19:37 +00:00
|
|
|
if (po->fanout) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
if (name) {
|
|
|
|
dev = dev_get_by_name_rcu(sock_net(sk), name);
|
|
|
|
if (!dev) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
} else if (ifindex) {
|
|
|
|
dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
|
|
|
|
if (!dev) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dev)
|
|
|
|
dev_hold(dev);
|
2013-12-06 10:36:15 +00:00
|
|
|
|
packet: improve socket create/bind latency in some cases
Most people acquire PF_PACKET sockets with a protocol argument in
the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for
all its sockets. Most likely, at some point in time a subsequent
bind() call will follow, e.g. in libpcap with ...
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_ifindex = ifindex;
sll.sll_protocol = htons(ETH_P_ALL);
... as arguments. What happens in the kernel is that already
in socket() syscall, we install a proto hook via register_prot_hook()
if our protocol argument is != 0. Yet, in bind() we're almost
doing the same work by doing a unregister_prot_hook() with an
expensive synchronize_net() call in case during socket() the proto
was != 0, plus follow-up register_prot_hook() with a bound device
to it this time, in order to limit traffic we get.
In the case when the protocol and user supplied device index (== 0)
does not change from socket() to bind(), we can spare us doing
the same work twice. Similarly for re-binding to the same device
and protocol. For these scenarios, we can decrease create/bind
latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case)
with this patch.
Alternatively, for the first case, if people care, they should
simply create their sockets with proto == 0 argument and define
the protocol during bind() as this saves a call to synchronize_net()
as well (sock-bind-3 case).
In all other cases, we're tied to user space behaviour we must not
change, also since a bind() is not strictly required. Thus, we need
the synchronize_net() to make sure no asynchronous packet processing
paths still refer to the previous elements of po->prot_hook.
In case of mmap()ed sockets, the workflow that includes bind() is
socket() -> setsockopt(<ring>) -> bind(). In that case, a pair of
{__unregister, register}_prot_hook is being called from setsockopt()
in order to install the new protocol receive handler. Thus, when
we call bind and can skip a re-hook, we have already previously
installed the new handler. For fanout, this is handled different
entirely, so we should be good.
Timings on an i7-3520M machine:
* sock-bind-1: 89 us
* sock-bind-2: 7447 us
* sock-bind-3: 75 us
sock-bind-1:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-2:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-3:
socket(PF_PACKET, SOCK_RAW, 0) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:34 +00:00
|
|
|
proto_curr = po->prot_hook.type;
|
|
|
|
dev_curr = po->prot_hook.dev;
|
|
|
|
|
|
|
|
need_rehook = proto_curr != proto || dev_curr != dev;
|
|
|
|
|
|
|
|
if (need_rehook) {
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
if (po->running) {
|
|
|
|
rcu_read_unlock();
|
2017-11-28 16:03:30 +00:00
|
|
|
/* prevents packet_notifier() from calling
|
|
|
|
* register_prot_hook()
|
|
|
|
*/
|
|
|
|
po->num = 0;
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
__unregister_prot_hook(sk, true);
|
|
|
|
rcu_read_lock();
|
|
|
|
dev_curr = po->prot_hook.dev;
|
|
|
|
if (dev)
|
|
|
|
unlisted = !dev_get_by_index_rcu(sock_net(sk),
|
|
|
|
dev->ifindex);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-11-28 16:03:30 +00:00
|
|
|
BUG_ON(po->running);
|
packet: improve socket create/bind latency in some cases
Most people acquire PF_PACKET sockets with a protocol argument in
the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for
all its sockets. Most likely, at some point in time a subsequent
bind() call will follow, e.g. in libpcap with ...
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_ifindex = ifindex;
sll.sll_protocol = htons(ETH_P_ALL);
... as arguments. What happens in the kernel is that already
in socket() syscall, we install a proto hook via register_prot_hook()
if our protocol argument is != 0. Yet, in bind() we're almost
doing the same work by doing a unregister_prot_hook() with an
expensive synchronize_net() call in case during socket() the proto
was != 0, plus follow-up register_prot_hook() with a bound device
to it this time, in order to limit traffic we get.
In the case when the protocol and user supplied device index (== 0)
does not change from socket() to bind(), we can spare us doing
the same work twice. Similarly for re-binding to the same device
and protocol. For these scenarios, we can decrease create/bind
latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case)
with this patch.
Alternatively, for the first case, if people care, they should
simply create their sockets with proto == 0 argument and define
the protocol during bind() as this saves a call to synchronize_net()
as well (sock-bind-3 case).
In all other cases, we're tied to user space behaviour we must not
change, also since a bind() is not strictly required. Thus, we need
the synchronize_net() to make sure no asynchronous packet processing
paths still refer to the previous elements of po->prot_hook.
In case of mmap()ed sockets, the workflow that includes bind() is
socket() -> setsockopt(<ring>) -> bind(). In that case, a pair of
{__unregister, register}_prot_hook is being called from setsockopt()
in order to install the new protocol receive handler. Thus, when
we call bind and can skip a re-hook, we have already previously
installed the new handler. For fanout, this is handled different
entirely, so we should be good.
Timings on an i7-3520M machine:
* sock-bind-1: 89 us
* sock-bind-2: 7447 us
* sock-bind-3: 75 us
sock-bind-1:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-2:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-3:
socket(PF_PACKET, SOCK_RAW, 0) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:34 +00:00
|
|
|
po->num = proto;
|
|
|
|
po->prot_hook.type = proto;
|
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
if (unlikely(unlisted)) {
|
|
|
|
dev_put(dev);
|
|
|
|
po->prot_hook.dev = NULL;
|
|
|
|
po->ifindex = -1;
|
|
|
|
packet_cached_dev_reset(po);
|
|
|
|
} else {
|
|
|
|
po->prot_hook.dev = dev;
|
|
|
|
po->ifindex = dev ? dev->ifindex : 0;
|
|
|
|
packet_cached_dev_assign(po, dev);
|
|
|
|
}
|
packet: improve socket create/bind latency in some cases
Most people acquire PF_PACKET sockets with a protocol argument in
the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for
all its sockets. Most likely, at some point in time a subsequent
bind() call will follow, e.g. in libpcap with ...
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_ifindex = ifindex;
sll.sll_protocol = htons(ETH_P_ALL);
... as arguments. What happens in the kernel is that already
in socket() syscall, we install a proto hook via register_prot_hook()
if our protocol argument is != 0. Yet, in bind() we're almost
doing the same work by doing a unregister_prot_hook() with an
expensive synchronize_net() call in case during socket() the proto
was != 0, plus follow-up register_prot_hook() with a bound device
to it this time, in order to limit traffic we get.
In the case when the protocol and user supplied device index (== 0)
does not change from socket() to bind(), we can spare us doing
the same work twice. Similarly for re-binding to the same device
and protocol. For these scenarios, we can decrease create/bind
latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case)
with this patch.
Alternatively, for the first case, if people care, they should
simply create their sockets with proto == 0 argument and define
the protocol during bind() as this saves a call to synchronize_net()
as well (sock-bind-3 case).
In all other cases, we're tied to user space behaviour we must not
change, also since a bind() is not strictly required. Thus, we need
the synchronize_net() to make sure no asynchronous packet processing
paths still refer to the previous elements of po->prot_hook.
In case of mmap()ed sockets, the workflow that includes bind() is
socket() -> setsockopt(<ring>) -> bind(). In that case, a pair of
{__unregister, register}_prot_hook is being called from setsockopt()
in order to install the new protocol receive handler. Thus, when
we call bind and can skip a re-hook, we have already previously
installed the new handler. For fanout, this is handled different
entirely, so we should be good.
Timings on an i7-3520M machine:
* sock-bind-1: 89 us
* sock-bind-2: 7447 us
* sock-bind-3: 75 us
sock-bind-1:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-2:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-3:
socket(PF_PACKET, SOCK_RAW, 0) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:34 +00:00
|
|
|
}
|
2015-07-27 22:32:21 +00:00
|
|
|
if (dev_curr)
|
|
|
|
dev_put(dev_curr);
|
2013-12-06 10:36:15 +00:00
|
|
|
|
packet: improve socket create/bind latency in some cases
Most people acquire PF_PACKET sockets with a protocol argument in
the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for
all its sockets. Most likely, at some point in time a subsequent
bind() call will follow, e.g. in libpcap with ...
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_ifindex = ifindex;
sll.sll_protocol = htons(ETH_P_ALL);
... as arguments. What happens in the kernel is that already
in socket() syscall, we install a proto hook via register_prot_hook()
if our protocol argument is != 0. Yet, in bind() we're almost
doing the same work by doing a unregister_prot_hook() with an
expensive synchronize_net() call in case during socket() the proto
was != 0, plus follow-up register_prot_hook() with a bound device
to it this time, in order to limit traffic we get.
In the case when the protocol and user supplied device index (== 0)
does not change from socket() to bind(), we can spare us doing
the same work twice. Similarly for re-binding to the same device
and protocol. For these scenarios, we can decrease create/bind
latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case)
with this patch.
Alternatively, for the first case, if people care, they should
simply create their sockets with proto == 0 argument and define
the protocol during bind() as this saves a call to synchronize_net()
as well (sock-bind-3 case).
In all other cases, we're tied to user space behaviour we must not
change, also since a bind() is not strictly required. Thus, we need
the synchronize_net() to make sure no asynchronous packet processing
paths still refer to the previous elements of po->prot_hook.
In case of mmap()ed sockets, the workflow that includes bind() is
socket() -> setsockopt(<ring>) -> bind(). In that case, a pair of
{__unregister, register}_prot_hook is being called from setsockopt()
in order to install the new protocol receive handler. Thus, when
we call bind and can skip a re-hook, we have already previously
installed the new handler. For fanout, this is handled different
entirely, so we should be good.
Timings on an i7-3520M machine:
* sock-bind-1: 89 us
* sock-bind-2: 7447 us
* sock-bind-3: 75 us
sock-bind-1:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-2:
socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
sock-bind-3:
socket(PF_PACKET, SOCK_RAW, 0) = 3
bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1),
pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:34 +00:00
|
|
|
if (proto == 0 || !need_rehook)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_unlock;
|
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
|
2011-07-04 08:44:29 +00:00
|
|
|
register_prot_hook(sk);
|
2007-11-13 05:05:20 +00:00
|
|
|
} else {
|
|
|
|
sk->sk_err = ENETDOWN;
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
sk->sk_error_report(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
out_unlock:
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&po->bind_lock);
|
|
|
|
release_sock(sk);
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Bind a packet socket to a device
|
|
|
|
*/
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
|
|
|
|
int addr_len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-07-21 21:57:59 +00:00
|
|
|
struct sock *sk = sock->sk;
|
net: don't call strlen() on the user buffer in packet_bind_spkt()
KMSAN (KernelMemorySanitizer, a new error detection tool) reports use of
uninitialized memory in packet_bind_spkt():
Acked-by: Eric Dumazet <edumazet@google.com>
==================================================================
BUG: KMSAN: use of unitialized memory
CPU: 0 PID: 1074 Comm: packet Not tainted 4.8.0-rc6+ #1891
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs
01/01/2011
0000000000000000 ffff88006b6dfc08 ffffffff82559ae8 ffff88006b6dfb48
ffffffff818a7c91 ffffffff85b9c870 0000000000000092 ffffffff85b9c550
0000000000000000 0000000000000092 00000000ec400911 0000000000000002
Call Trace:
[< inline >] __dump_stack lib/dump_stack.c:15
[<ffffffff82559ae8>] dump_stack+0x238/0x290 lib/dump_stack.c:51
[<ffffffff818a6626>] kmsan_report+0x276/0x2e0 mm/kmsan/kmsan.c:1003
[<ffffffff818a783b>] __msan_warning+0x5b/0xb0
mm/kmsan/kmsan_instr.c:424
[< inline >] strlen lib/string.c:484
[<ffffffff8259b58d>] strlcpy+0x9d/0x200 lib/string.c:144
[<ffffffff84b2eca4>] packet_bind_spkt+0x144/0x230
net/packet/af_packet.c:3132
[<ffffffff84242e4d>] SYSC_bind+0x40d/0x5f0 net/socket.c:1370
[<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
[<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f
arch/x86/entry/entry_64.o:?
chained origin: 00000000eba00911
[<ffffffff810bb787>] save_stack_trace+0x27/0x50
arch/x86/kernel/stacktrace.c:67
[< inline >] kmsan_save_stack_with_flags mm/kmsan/kmsan.c:322
[< inline >] kmsan_save_stack mm/kmsan/kmsan.c:334
[<ffffffff818a59f8>] kmsan_internal_chain_origin+0x118/0x1e0
mm/kmsan/kmsan.c:527
[<ffffffff818a7773>] __msan_set_alloca_origin4+0xc3/0x130
mm/kmsan/kmsan_instr.c:380
[<ffffffff84242b69>] SYSC_bind+0x129/0x5f0 net/socket.c:1356
[<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
[<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f
arch/x86/entry/entry_64.o:?
origin description: ----address@SYSC_bind (origin=00000000eb400911)
==================================================================
(the line numbers are relative to 4.8-rc6, but the bug persists
upstream)
, when I run the following program as root:
=====================================
#include <string.h>
#include <sys/socket.h>
#include <netpacket/packet.h>
#include <net/ethernet.h>
int main() {
struct sockaddr addr;
memset(&addr, 0xff, sizeof(addr));
addr.sa_family = AF_PACKET;
int fd = socket(PF_PACKET, SOCK_PACKET, htons(ETH_P_ALL));
bind(fd, &addr, sizeof(addr));
return 0;
}
=====================================
This happens because addr.sa_data copied from the userspace is not
zero-terminated, and copying it with strlcpy() in packet_bind_spkt()
results in calling strlen() on the kernel copy of that non-terminated
buffer.
Signed-off-by: Alexander Potapenko <glider@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-03-01 11:57:20 +00:00
|
|
|
char name[sizeof(uaddr->sa_data) + 1];
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Check legality
|
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2006-01-24 00:28:02 +00:00
|
|
|
if (addr_len != sizeof(struct sockaddr))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
net: don't call strlen() on the user buffer in packet_bind_spkt()
KMSAN (KernelMemorySanitizer, a new error detection tool) reports use of
uninitialized memory in packet_bind_spkt():
Acked-by: Eric Dumazet <edumazet@google.com>
==================================================================
BUG: KMSAN: use of unitialized memory
CPU: 0 PID: 1074 Comm: packet Not tainted 4.8.0-rc6+ #1891
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs
01/01/2011
0000000000000000 ffff88006b6dfc08 ffffffff82559ae8 ffff88006b6dfb48
ffffffff818a7c91 ffffffff85b9c870 0000000000000092 ffffffff85b9c550
0000000000000000 0000000000000092 00000000ec400911 0000000000000002
Call Trace:
[< inline >] __dump_stack lib/dump_stack.c:15
[<ffffffff82559ae8>] dump_stack+0x238/0x290 lib/dump_stack.c:51
[<ffffffff818a6626>] kmsan_report+0x276/0x2e0 mm/kmsan/kmsan.c:1003
[<ffffffff818a783b>] __msan_warning+0x5b/0xb0
mm/kmsan/kmsan_instr.c:424
[< inline >] strlen lib/string.c:484
[<ffffffff8259b58d>] strlcpy+0x9d/0x200 lib/string.c:144
[<ffffffff84b2eca4>] packet_bind_spkt+0x144/0x230
net/packet/af_packet.c:3132
[<ffffffff84242e4d>] SYSC_bind+0x40d/0x5f0 net/socket.c:1370
[<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
[<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f
arch/x86/entry/entry_64.o:?
chained origin: 00000000eba00911
[<ffffffff810bb787>] save_stack_trace+0x27/0x50
arch/x86/kernel/stacktrace.c:67
[< inline >] kmsan_save_stack_with_flags mm/kmsan/kmsan.c:322
[< inline >] kmsan_save_stack mm/kmsan/kmsan.c:334
[<ffffffff818a59f8>] kmsan_internal_chain_origin+0x118/0x1e0
mm/kmsan/kmsan.c:527
[<ffffffff818a7773>] __msan_set_alloca_origin4+0xc3/0x130
mm/kmsan/kmsan_instr.c:380
[<ffffffff84242b69>] SYSC_bind+0x129/0x5f0 net/socket.c:1356
[<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
[<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f
arch/x86/entry/entry_64.o:?
origin description: ----address@SYSC_bind (origin=00000000eb400911)
==================================================================
(the line numbers are relative to 4.8-rc6, but the bug persists
upstream)
, when I run the following program as root:
=====================================
#include <string.h>
#include <sys/socket.h>
#include <netpacket/packet.h>
#include <net/ethernet.h>
int main() {
struct sockaddr addr;
memset(&addr, 0xff, sizeof(addr));
addr.sa_family = AF_PACKET;
int fd = socket(PF_PACKET, SOCK_PACKET, htons(ETH_P_ALL));
bind(fd, &addr, sizeof(addr));
return 0;
}
=====================================
This happens because addr.sa_data copied from the userspace is not
zero-terminated, and copying it with strlcpy() in packet_bind_spkt()
results in calling strlen() on the kernel copy of that non-terminated
buffer.
Signed-off-by: Alexander Potapenko <glider@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-03-01 11:57:20 +00:00
|
|
|
/* uaddr->sa_data comes from the userspace, it's not guaranteed to be
|
|
|
|
* zero-terminated.
|
|
|
|
*/
|
|
|
|
memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
|
|
|
|
name[sizeof(uaddr->sa_data)] = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
|
|
|
|
{
|
2009-07-21 21:57:59 +00:00
|
|
|
struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
|
|
|
|
struct sock *sk = sock->sk;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check legality
|
|
|
|
*/
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (addr_len < sizeof(struct sockaddr_ll))
|
|
|
|
return -EINVAL;
|
|
|
|
if (sll->sll_family != AF_PACKET)
|
|
|
|
return -EINVAL;
|
|
|
|
|
packet: race condition in packet_bind
There is a race conditions between packet_notifier and packet_bind{_spkt}.
It happens if packet_notifier(NETDEV_UNREGISTER) executes between the
time packet_bind{_spkt} takes a reference on the new netdevice and the
time packet_do_bind sets po->ifindex.
In this case the notification can be missed.
If this happens during a dev_change_net_namespace this can result in the
netdevice to be moved to the new namespace while the packet_sock in the
old namespace still holds a reference on it. When the netdevice is later
deleted in the new namespace the deletion hangs since the packet_sock
is not found in the new namespace' &net->packet.sklist.
It can be reproduced with the script below.
This patch makes packet_do_bind check again for the presence of the
netdevice in the packet_sock's namespace after the synchronize_net
in unregister_prot_hook.
More in general it also uses the rcu lock for the duration of the bind
to stop dev_change_net_namespace/rollback_registered_many from
going past the synchronize_net following unlist_netdevice, so that
no NETDEV_UNREGISTER notifications can happen on the new netdevice
while the bind is executing. In order to do this some code from
packet_bind{_spkt} is consolidated into packet_do_dev.
import socket, os, time, sys
proto=7
realDev='em1'
vlanId=400
if len(sys.argv) > 1:
vlanId=int(sys.argv[1])
dev='vlan%d' % vlanId
os.system('taskset -p 0x10 %d' % os.getpid())
s = socket.socket(socket.PF_PACKET, socket.SOCK_RAW, proto)
os.system('ip link add link %s name %s type vlan id %d' %
(realDev, dev, vlanId))
os.system('ip netns add dummy')
pid=os.fork()
if pid == 0:
# dev should be moved while packet_do_bind is in synchronize net
os.system('taskset -p 0x20000 %d' % os.getpid())
os.system('ip link set %s netns dummy' % dev)
os.system('ip netns exec dummy ip link del %s' % dev)
s.close()
sys.exit(0)
time.sleep(.004)
try:
s.bind(('%s' % dev, proto+1))
except:
print 'Could not bind socket'
s.close()
os.system('ip netns del dummy')
sys.exit(0)
os.waitpid(pid, 0)
s.close()
os.system('ip netns del dummy')
sys.exit(0)
Signed-off-by: Francesco Ruggeri <fruggeri@arista.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-11-05 16:16:14 +00:00
|
|
|
return packet_do_bind(sk, NULL, sll->sll_ifindex,
|
|
|
|
sll->sll_protocol ? : pkt_sk(sk)->num);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct proto packet_proto = {
|
|
|
|
.name = "PACKET",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.obj_size = sizeof(struct packet_sock),
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
2007-02-09 14:25:10 +00:00
|
|
|
* Create a packet of type SOCK_PACKET.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2009-11-06 06:18:14 +00:00
|
|
|
static int packet_create(struct net *net, struct socket *sock, int protocol,
|
|
|
|
int kern)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
struct packet_sock *po;
|
2006-11-08 08:26:29 +00:00
|
|
|
__be16 proto = (__force __be16)protocol; /* weird, but documented */
|
2005-04-16 22:20:36 +00:00
|
|
|
int err;
|
|
|
|
|
2012-11-16 03:03:07 +00:00
|
|
|
if (!ns_capable(net->user_ns, CAP_NET_RAW))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EPERM;
|
2007-05-29 20:16:31 +00:00
|
|
|
if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
|
|
|
|
sock->type != SOCK_PACKET)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ESOCKTNOSUPPORT;
|
|
|
|
|
|
|
|
sock->state = SS_UNCONNECTED;
|
|
|
|
|
|
|
|
err = -ENOBUFS;
|
2015-05-09 02:09:13 +00:00
|
|
|
sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sk == NULL)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
sock->ops = &packet_ops;
|
|
|
|
if (sock->type == SOCK_PACKET)
|
|
|
|
sock->ops = &packet_ops_spkt;
|
2007-05-29 20:16:31 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
sock_init_data(sock, sk);
|
|
|
|
|
|
|
|
po = pkt_sk(sk);
|
|
|
|
sk->sk_family = PF_PACKET;
|
2006-11-08 08:26:29 +00:00
|
|
|
po->num = proto;
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
po->xmit = dev_queue_xmit;
|
2013-12-06 10:36:15 +00:00
|
|
|
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
err = packet_alloc_pending(po);
|
|
|
|
if (err)
|
|
|
|
goto out2;
|
|
|
|
|
2013-12-06 10:36:15 +00:00
|
|
|
packet_cached_dev_reset(po);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
sk->sk_destruct = packet_sock_destruct;
|
2007-11-11 05:38:48 +00:00
|
|
|
sk_refcnt_debug_inc(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Attach a protocol block
|
|
|
|
*/
|
|
|
|
|
|
|
|
spin_lock_init(&po->bind_lock);
|
2009-01-30 22:12:06 +00:00
|
|
|
mutex_init(&po->pg_vec_lock);
|
2015-05-12 15:56:46 +00:00
|
|
|
po->rollover = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
po->prot_hook.func = packet_rcv;
|
2007-05-29 20:16:31 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sock->type == SOCK_PACKET)
|
|
|
|
po->prot_hook.func = packet_rcv_spkt;
|
2007-05-29 20:16:31 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
po->prot_hook.af_packet_priv = sk;
|
|
|
|
|
2006-11-08 08:26:29 +00:00
|
|
|
if (proto) {
|
|
|
|
po->prot_hook.type = proto;
|
2018-04-23 21:37:03 +00:00
|
|
|
__register_prot_hook(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-08-21 01:06:47 +00:00
|
|
|
mutex_lock(&net->packet.sklist_lock);
|
2010-02-22 07:57:18 +00:00
|
|
|
sk_add_node_rcu(sk, &net->packet.sklist);
|
2012-08-21 01:06:47 +00:00
|
|
|
mutex_unlock(&net->packet.sklist_lock);
|
|
|
|
|
|
|
|
preempt_disable();
|
2008-11-19 22:25:35 +00:00
|
|
|
sock_prot_inuse_add(net, &packet_proto, 1);
|
2012-08-21 01:06:47 +00:00
|
|
|
preempt_enable();
|
2010-02-22 07:57:18 +00:00
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
return 0;
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
out2:
|
|
|
|
sk_free(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Pull a packet from our receive queue and hand it to the user.
|
|
|
|
* If necessary we block.
|
|
|
|
*/
|
|
|
|
|
2015-03-02 07:37:48 +00:00
|
|
|
static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
|
|
|
|
int flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
int copied, err;
|
2010-02-05 04:24:10 +00:00
|
|
|
int vnet_hdr_len = 0;
|
2015-03-01 12:58:28 +00:00
|
|
|
unsigned int origlen = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
err = -EINVAL;
|
2010-04-07 22:41:28 +00:00
|
|
|
if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
/* What error should we return now? EUNATTACH? */
|
|
|
|
if (pkt_sk(sk)->ifindex < 0)
|
|
|
|
return -ENODEV;
|
|
|
|
#endif
|
|
|
|
|
2010-04-07 22:41:28 +00:00
|
|
|
if (flags & MSG_ERRQUEUE) {
|
2013-07-19 17:40:09 +00:00
|
|
|
err = sock_recv_errqueue(sk, msg, len,
|
|
|
|
SOL_PACKET, PACKET_TX_TIMESTAMP);
|
2010-04-07 22:41:28 +00:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Call the generic datagram receiver. This handles all sorts
|
|
|
|
* of horrible races and re-entrancy so we can forget about it
|
|
|
|
* in the protocol layers.
|
|
|
|
*
|
|
|
|
* Now it will return ENETDOWN, if device have just gone down,
|
|
|
|
* but then it will block.
|
|
|
|
*/
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2007-02-09 14:25:10 +00:00
|
|
|
* An error occurred so return it. Because skb_recv_datagram()
|
2005-04-16 22:20:36 +00:00
|
|
|
* handles the blocking we don't see and worry about blocking
|
|
|
|
* retries.
|
|
|
|
*/
|
|
|
|
|
2006-01-24 00:28:02 +00:00
|
|
|
if (skb == NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
if (pkt_sk(sk)->pressure)
|
|
|
|
packet_rcv_has_room(pkt_sk(sk), NULL);
|
|
|
|
|
2010-02-05 04:24:10 +00:00
|
|
|
if (pkt_sk(sk)->has_vnet_hdr) {
|
2016-02-03 23:02:14 +00:00
|
|
|
err = packet_rcv_vnet(msg, skb, &len);
|
|
|
|
if (err)
|
2010-02-05 04:24:10 +00:00
|
|
|
goto out_free;
|
2016-02-03 23:02:14 +00:00
|
|
|
vnet_hdr_len = sizeof(struct virtio_net_hdr);
|
2010-02-05 04:24:10 +00:00
|
|
|
}
|
|
|
|
|
2013-11-21 02:14:22 +00:00
|
|
|
/* You lose any data beyond the buffer you gave. If it worries
|
|
|
|
* a user program they can ask the device for its MTU
|
|
|
|
* anyway.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
copied = skb->len;
|
2009-07-21 21:57:59 +00:00
|
|
|
if (copied > len) {
|
|
|
|
copied = len;
|
|
|
|
msg->msg_flags |= MSG_TRUNC;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2014-11-05 21:46:40 +00:00
|
|
|
err = skb_copy_datagram_msg(skb, 0, msg, copied);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free;
|
|
|
|
|
2015-03-01 12:58:28 +00:00
|
|
|
if (sock->type != SOCK_PACKET) {
|
|
|
|
struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
|
|
|
|
|
|
|
|
/* Original length was stored in sockaddr_ll fields */
|
|
|
|
origlen = PACKET_SKB_CB(skb)->sa.origlen;
|
|
|
|
sll->sll_family = AF_PACKET;
|
|
|
|
sll->sll_protocol = skb->protocol;
|
|
|
|
}
|
|
|
|
|
net: Generalize socket rx gap / receive queue overflow cmsg
Create a new socket level option to report number of queue overflows
Recently I augmented the AF_PACKET protocol to report the number of frames lost
on the socket receive queue between any two enqueued frames. This value was
exported via a SOL_PACKET level cmsg. AFter I completed that work it was
requested that this feature be generalized so that any datagram oriented socket
could make use of this option. As such I've created this patch, It creates a
new SOL_SOCKET level option called SO_RXQ_OVFL, which when enabled exports a
SOL_SOCKET level cmsg that reports the nubmer of times the sk_receive_queue
overflowed between any two given frames. It also augments the AF_PACKET
protocol to take advantage of this new feature (as it previously did not touch
sk->sk_drops, which this patch uses to record the overflow count). Tested
successfully by me.
Notes:
1) Unlike my previous patch, this patch simply records the sk_drops value, which
is not a number of drops between packets, but rather a total number of drops.
Deltas must be computed in user space.
2) While this patch currently works with datagram oriented protocols, it will
also be accepted by non-datagram oriented protocols. I'm not sure if thats
agreeable to everyone, but my argument in favor of doing so is that, for those
protocols which aren't applicable to this option, sk_drops will always be zero,
and reporting no drops on a receive queue that isn't used for those
non-participating protocols seems reasonable to me. This also saves us having
to code in a per-protocol opt in mechanism.
3) This applies cleanly to net-next assuming that commit
977750076d98c7ff6cbda51858bb5a5894a9d9ab (my af packet cmsg patch) is reverted
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-10-12 20:26:31 +00:00
|
|
|
sock_recv_ts_and_drops(msg, sk, skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-11-21 02:14:22 +00:00
|
|
|
if (msg->msg_name) {
|
|
|
|
/* If the address length field is there to be filled
|
|
|
|
* in, we fill it in now.
|
|
|
|
*/
|
|
|
|
if (sock->type == SOCK_PACKET) {
|
2014-01-17 21:53:15 +00:00
|
|
|
__sockaddr_check_size(sizeof(struct sockaddr_pkt));
|
2013-11-21 02:14:22 +00:00
|
|
|
msg->msg_namelen = sizeof(struct sockaddr_pkt);
|
|
|
|
} else {
|
|
|
|
struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
|
2015-03-01 12:58:28 +00:00
|
|
|
|
2013-11-21 02:14:22 +00:00
|
|
|
msg->msg_namelen = sll->sll_halen +
|
|
|
|
offsetof(struct sockaddr_ll, sll_addr);
|
|
|
|
}
|
2007-02-05 07:33:10 +00:00
|
|
|
memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
|
|
|
|
msg->msg_namelen);
|
2013-11-21 02:14:22 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-02-05 07:31:32 +00:00
|
|
|
if (pkt_sk(sk)->auxdata) {
|
2007-02-05 07:33:10 +00:00
|
|
|
struct tpacket_auxdata aux;
|
|
|
|
|
|
|
|
aux.tp_status = TP_STATUS_USER;
|
|
|
|
if (skb->ip_summed == CHECKSUM_PARTIAL)
|
|
|
|
aux.tp_status |= TP_STATUS_CSUMNOTREADY;
|
2015-03-23 06:11:13 +00:00
|
|
|
else if (skb->pkt_type != PACKET_OUTGOING &&
|
|
|
|
(skb->ip_summed == CHECKSUM_COMPLETE ||
|
|
|
|
skb_csum_unnecessary(skb)))
|
|
|
|
aux.tp_status |= TP_STATUS_CSUM_VALID;
|
|
|
|
|
2015-03-01 12:58:28 +00:00
|
|
|
aux.tp_len = origlen;
|
2007-02-05 07:33:10 +00:00
|
|
|
aux.tp_snaplen = skb->len;
|
|
|
|
aux.tp_mac = 0;
|
2007-03-11 01:16:10 +00:00
|
|
|
aux.tp_net = skb_network_offset(skb);
|
2015-01-13 16:13:44 +00:00
|
|
|
if (skb_vlan_tag_present(skb)) {
|
|
|
|
aux.tp_vlan_tci = skb_vlan_tag_get(skb);
|
2013-12-17 13:53:40 +00:00
|
|
|
aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
|
|
|
|
aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
2011-06-01 06:49:10 +00:00
|
|
|
} else {
|
|
|
|
aux.tp_vlan_tci = 0;
|
2013-12-17 13:53:40 +00:00
|
|
|
aux.tp_vlan_tpid = 0;
|
2011-06-01 06:49:10 +00:00
|
|
|
}
|
2007-02-05 07:33:10 +00:00
|
|
|
put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
|
2007-02-05 07:31:32 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Free or return the buffer as appropriate. Again this
|
|
|
|
* hides all the races and re-entrancy issues from us.
|
|
|
|
*/
|
2010-02-05 04:24:10 +00:00
|
|
|
err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
out_free:
|
|
|
|
skb_free_datagram(sk, skb);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
|
2018-02-12 19:00:20 +00:00
|
|
|
int peer)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct net_device *dev;
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
|
|
|
if (peer)
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
|
|
|
uaddr->sa_family = AF_PACKET;
|
2013-06-12 14:02:27 +00:00
|
|
|
memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
|
|
|
|
if (dev)
|
2013-06-12 14:02:27 +00:00
|
|
|
strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-02-12 19:00:20 +00:00
|
|
|
return sizeof(*uaddr);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
|
2018-02-12 19:00:20 +00:00
|
|
|
int peer)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct net_device *dev;
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2009-11-08 05:51:19 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (peer)
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
|
|
|
sll->sll_family = AF_PACKET;
|
|
|
|
sll->sll_ifindex = po->ifindex;
|
|
|
|
sll->sll_protocol = po->num;
|
2010-11-10 20:09:10 +00:00
|
|
|
sll->sll_pkttype = 0;
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (dev) {
|
|
|
|
sll->sll_hatype = dev->type;
|
|
|
|
sll->sll_halen = dev->addr_len;
|
|
|
|
memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
|
|
|
|
} else {
|
|
|
|
sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
|
|
|
|
sll->sll_halen = 0;
|
|
|
|
}
|
2009-11-02 09:43:32 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-02-12 19:00:20 +00:00
|
|
|
return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-07-15 03:49:46 +00:00
|
|
|
static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
|
|
|
|
int what)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
switch (i->type) {
|
|
|
|
case PACKET_MR_MULTICAST:
|
2010-03-02 20:40:01 +00:00
|
|
|
if (i->alen != dev->addr_len)
|
|
|
|
return -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (what > 0)
|
2010-04-01 21:22:57 +00:00
|
|
|
return dev_mc_add(dev, i->addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
else
|
2010-04-01 21:22:57 +00:00
|
|
|
return dev_mc_del(dev, i->addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
case PACKET_MR_PROMISC:
|
2008-07-15 03:49:46 +00:00
|
|
|
return dev_set_promiscuity(dev, what);
|
2005-04-16 22:20:36 +00:00
|
|
|
case PACKET_MR_ALLMULTI:
|
2008-07-15 03:49:46 +00:00
|
|
|
return dev_set_allmulti(dev, what);
|
2009-05-19 18:27:17 +00:00
|
|
|
case PACKET_MR_UNICAST:
|
2010-03-02 20:40:01 +00:00
|
|
|
if (i->alen != dev->addr_len)
|
|
|
|
return -EINVAL;
|
2009-05-19 18:27:17 +00:00
|
|
|
if (what > 0)
|
2010-04-01 21:22:09 +00:00
|
|
|
return dev_uc_add(dev, i->addr);
|
2009-05-19 18:27:17 +00:00
|
|
|
else
|
2010-04-01 21:22:09 +00:00
|
|
|
return dev_uc_del(dev, i->addr);
|
2009-05-19 18:27:17 +00:00
|
|
|
break;
|
2009-07-21 21:57:59 +00:00
|
|
|
default:
|
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2008-07-15 03:49:46 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2015-03-09 18:51:04 +00:00
|
|
|
static void packet_dev_mclist_delete(struct net_device *dev,
|
|
|
|
struct packet_mclist **mlp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-03-09 18:51:04 +00:00
|
|
|
struct packet_mclist *ml;
|
|
|
|
|
|
|
|
while ((ml = *mlp) != NULL) {
|
|
|
|
if (ml->ifindex == dev->ifindex) {
|
|
|
|
packet_dev_mc(dev, ml, -1);
|
|
|
|
*mlp = ml->next;
|
|
|
|
kfree(ml);
|
|
|
|
} else
|
|
|
|
mlp = &ml->next;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-09-21 07:11:37 +00:00
|
|
|
static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
struct packet_mclist *ml, *i;
|
|
|
|
struct net_device *dev;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
rtnl_lock();
|
|
|
|
|
|
|
|
err = -ENODEV;
|
2008-03-25 17:26:21 +00:00
|
|
|
dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!dev)
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
err = -EINVAL;
|
2010-03-02 20:40:01 +00:00
|
|
|
if (mreq->mr_alen > dev->addr_len)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto done;
|
|
|
|
|
|
|
|
err = -ENOBUFS;
|
2006-01-11 23:56:43 +00:00
|
|
|
i = kmalloc(sizeof(*i), GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (i == NULL)
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
for (ml = po->mclist; ml; ml = ml->next) {
|
|
|
|
if (ml->ifindex == mreq->mr_ifindex &&
|
|
|
|
ml->type == mreq->mr_type &&
|
|
|
|
ml->alen == mreq->mr_alen &&
|
|
|
|
memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
|
|
|
|
ml->count++;
|
|
|
|
/* Free the new element ... */
|
|
|
|
kfree(i);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
i->type = mreq->mr_type;
|
|
|
|
i->ifindex = mreq->mr_ifindex;
|
|
|
|
i->alen = mreq->mr_alen;
|
|
|
|
memcpy(i->addr, mreq->mr_address, i->alen);
|
2016-04-10 10:52:28 +00:00
|
|
|
memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
|
2005-04-16 22:20:36 +00:00
|
|
|
i->count = 1;
|
|
|
|
i->next = po->mclist;
|
|
|
|
po->mclist = i;
|
2008-07-15 03:49:46 +00:00
|
|
|
err = packet_dev_mc(dev, i, 1);
|
|
|
|
if (err) {
|
|
|
|
po->mclist = i->next;
|
|
|
|
kfree(i);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
done:
|
|
|
|
rtnl_unlock();
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2005-09-21 07:11:37 +00:00
|
|
|
static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct packet_mclist *ml, **mlp;
|
|
|
|
|
|
|
|
rtnl_lock();
|
|
|
|
|
|
|
|
for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
|
|
|
|
if (ml->ifindex == mreq->mr_ifindex &&
|
|
|
|
ml->type == mreq->mr_type &&
|
|
|
|
ml->alen == mreq->mr_alen &&
|
|
|
|
memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
|
|
|
|
if (--ml->count == 0) {
|
|
|
|
struct net_device *dev;
|
|
|
|
*mlp = ml->next;
|
2009-10-16 06:38:46 +00:00
|
|
|
dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
|
|
|
|
if (dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
packet_dev_mc(dev, ml, -1);
|
|
|
|
kfree(ml);
|
|
|
|
}
|
2015-03-09 18:51:04 +00:00
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
rtnl_unlock();
|
2015-03-09 18:51:04 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_flush_mclist(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
struct packet_mclist *ml;
|
|
|
|
|
|
|
|
if (!po->mclist)
|
|
|
|
return;
|
|
|
|
|
|
|
|
rtnl_lock();
|
|
|
|
while ((ml = po->mclist) != NULL) {
|
|
|
|
struct net_device *dev;
|
|
|
|
|
|
|
|
po->mclist = ml->next;
|
2009-10-16 06:38:46 +00:00
|
|
|
dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
|
|
|
|
if (dev != NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
packet_dev_mc(dev, ml, -1);
|
|
|
|
kfree(ml);
|
|
|
|
}
|
|
|
|
rtnl_unlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2009-09-30 23:12:20 +00:00
|
|
|
packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
2007-02-05 07:31:32 +00:00
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (level != SOL_PACKET)
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
switch (optname) {
|
2007-02-09 14:25:10 +00:00
|
|
|
case PACKET_ADD_MEMBERSHIP:
|
2005-04-16 22:20:36 +00:00
|
|
|
case PACKET_DROP_MEMBERSHIP:
|
|
|
|
{
|
2005-09-21 07:11:37 +00:00
|
|
|
struct packet_mreq_max mreq;
|
|
|
|
int len = optlen;
|
|
|
|
memset(&mreq, 0, sizeof(mreq));
|
|
|
|
if (len < sizeof(struct packet_mreq))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
2005-09-21 07:11:37 +00:00
|
|
|
if (len > sizeof(mreq))
|
|
|
|
len = sizeof(mreq);
|
2009-07-21 21:57:59 +00:00
|
|
|
if (copy_from_user(&mreq, optval, len))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EFAULT;
|
2005-09-21 07:11:37 +00:00
|
|
|
if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
|
|
|
|
return -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (optname == PACKET_ADD_MEMBERSHIP)
|
|
|
|
ret = packet_mc_add(sk, &mreq);
|
|
|
|
else
|
|
|
|
ret = packet_mc_drop(sk, &mreq);
|
|
|
|
return ret;
|
|
|
|
}
|
2007-05-29 20:12:50 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
case PACKET_RX_RING:
|
2009-05-19 05:11:22 +00:00
|
|
|
case PACKET_TX_RING:
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-08-19 10:18:16 +00:00
|
|
|
union tpacket_req_u req_u;
|
|
|
|
int len;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-04-16 00:52:04 +00:00
|
|
|
lock_sock(sk);
|
2011-08-19 10:18:16 +00:00
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
case TPACKET_V2:
|
|
|
|
len = sizeof(req_u.req);
|
|
|
|
break;
|
|
|
|
case TPACKET_V3:
|
|
|
|
default:
|
|
|
|
len = sizeof(req_u.req3);
|
|
|
|
break;
|
|
|
|
}
|
2018-04-16 00:52:04 +00:00
|
|
|
if (optlen < len) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
} else {
|
|
|
|
if (copy_from_user(&req_u.req, optval, len))
|
|
|
|
ret = -EFAULT;
|
|
|
|
else
|
|
|
|
ret = packet_set_ring(sk, &req_u, 0,
|
|
|
|
optname == PACKET_TX_RING);
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
case PACKET_COPY_THRESH:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
if (optlen != sizeof(val))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
2009-07-21 21:57:59 +00:00
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
pkt_sk(sk)->copy_thresh = val;
|
|
|
|
return 0;
|
|
|
|
}
|
2008-07-15 05:50:15 +00:00
|
|
|
case PACKET_VERSION:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
switch (val) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
case TPACKET_V2:
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
2016-11-30 22:55:36 +00:00
|
|
|
break;
|
2008-07-15 05:50:15 +00:00
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2016-11-30 22:55:36 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
} else {
|
|
|
|
po->tp_version = val;
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
2008-07-15 05:50:15 +00:00
|
|
|
}
|
2008-07-19 01:05:19 +00:00
|
|
|
case PACKET_RESERVE:
|
|
|
|
{
|
|
|
|
unsigned int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
2017-03-29 14:11:22 +00:00
|
|
|
if (val > INT_MAX)
|
|
|
|
return -EINVAL;
|
2017-08-10 16:41:58 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
} else {
|
|
|
|
po->tp_reserve = val;
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
2008-07-19 01:05:19 +00:00
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
case PACKET_LOSS:
|
|
|
|
{
|
|
|
|
unsigned int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
2018-04-23 21:37:03 +00:00
|
|
|
|
|
|
|
lock_sock(sk);
|
|
|
|
if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
} else {
|
|
|
|
po->tp_loss = !!val;
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
2007-02-05 07:31:32 +00:00
|
|
|
case PACKET_AUXDATA:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen < sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
lock_sock(sk);
|
2007-02-05 07:31:32 +00:00
|
|
|
po->auxdata = !!val;
|
2018-04-23 21:37:03 +00:00
|
|
|
release_sock(sk);
|
2007-02-05 07:31:32 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2007-04-20 23:05:39 +00:00
|
|
|
case PACKET_ORIGDEV:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen < sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
lock_sock(sk);
|
2007-04-20 23:05:39 +00:00
|
|
|
po->origdev = !!val;
|
2018-04-23 21:37:03 +00:00
|
|
|
release_sock(sk);
|
2007-04-20 23:05:39 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2010-02-05 04:24:10 +00:00
|
|
|
case PACKET_VNET_HDR:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (sock->type != SOCK_RAW)
|
|
|
|
return -EINVAL;
|
|
|
|
if (optlen < sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2018-04-23 21:37:03 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
} else {
|
|
|
|
po->has_vnet_hdr = !!val;
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
2010-02-05 04:24:10 +00:00
|
|
|
}
|
2010-06-02 12:53:56 +00:00
|
|
|
case PACKET_TIMESTAMP:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
po->tp_tstamp = val;
|
|
|
|
return 0;
|
|
|
|
}
|
2011-07-05 08:45:05 +00:00
|
|
|
case PACKET_FANOUT:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return fanout_add(sk, val & 0xffff, val >> 16);
|
|
|
|
}
|
2015-08-15 02:31:34 +00:00
|
|
|
case PACKET_FANOUT_DATA:
|
|
|
|
{
|
|
|
|
if (!po->fanout)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return fanout_set_data(po, optval, optlen);
|
|
|
|
}
|
2012-11-06 23:10:47 +00:00
|
|
|
case PACKET_TX_HAS_OFF:
|
|
|
|
{
|
|
|
|
unsigned int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
2018-04-23 21:37:03 +00:00
|
|
|
|
|
|
|
lock_sock(sk);
|
|
|
|
if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
} else {
|
|
|
|
po->tp_tx_has_off = !!val;
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
2012-11-06 23:10:47 +00:00
|
|
|
return 0;
|
|
|
|
}
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
case PACKET_QDISC_BYPASS:
|
|
|
|
{
|
|
|
|
int val;
|
|
|
|
|
|
|
|
if (optlen != sizeof(val))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&val, optval, sizeof(val)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
default:
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int packet_getsockopt(struct socket *sock, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
{
|
|
|
|
int len;
|
2012-04-19 21:56:11 +00:00
|
|
|
int val, lv = sizeof(val);
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2012-04-19 21:56:11 +00:00
|
|
|
void *data = &val;
|
2013-04-19 06:12:29 +00:00
|
|
|
union tpacket_stats_u st;
|
2015-05-12 15:56:50 +00:00
|
|
|
struct tpacket_rollover_stats rstats;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (level != SOL_PACKET)
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
2006-01-24 00:28:02 +00:00
|
|
|
if (get_user(len, optlen))
|
|
|
|
return -EFAULT;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (len < 0)
|
|
|
|
return -EINVAL;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
switch (optname) {
|
2005-04-16 22:20:36 +00:00
|
|
|
case PACKET_STATISTICS:
|
|
|
|
spin_lock_bh(&sk->sk_receive_queue.lock);
|
2013-04-19 06:12:29 +00:00
|
|
|
memcpy(&st, &po->stats, sizeof(st));
|
|
|
|
memset(&po->stats, 0, sizeof(po->stats));
|
|
|
|
spin_unlock_bh(&sk->sk_receive_queue.lock);
|
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
if (po->tp_version == TPACKET_V3) {
|
2012-04-19 21:56:11 +00:00
|
|
|
lv = sizeof(struct tpacket_stats_v3);
|
2013-08-19 20:40:22 +00:00
|
|
|
st.stats3.tp_packets += st.stats3.tp_drops;
|
2013-04-19 06:12:29 +00:00
|
|
|
data = &st.stats3;
|
2011-08-19 10:18:16 +00:00
|
|
|
} else {
|
2012-04-19 21:56:11 +00:00
|
|
|
lv = sizeof(struct tpacket_stats);
|
2013-08-19 20:40:22 +00:00
|
|
|
st.stats1.tp_packets += st.stats1.tp_drops;
|
2013-04-19 06:12:29 +00:00
|
|
|
data = &st.stats1;
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
2013-04-19 06:12:29 +00:00
|
|
|
|
2007-02-05 07:31:32 +00:00
|
|
|
break;
|
|
|
|
case PACKET_AUXDATA:
|
|
|
|
val = po->auxdata;
|
2007-04-20 23:05:39 +00:00
|
|
|
break;
|
|
|
|
case PACKET_ORIGDEV:
|
|
|
|
val = po->origdev;
|
2010-02-05 04:24:10 +00:00
|
|
|
break;
|
|
|
|
case PACKET_VNET_HDR:
|
|
|
|
val = po->has_vnet_hdr;
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
2008-07-15 05:50:15 +00:00
|
|
|
case PACKET_VERSION:
|
|
|
|
val = po->tp_version;
|
|
|
|
break;
|
|
|
|
case PACKET_HDRLEN:
|
|
|
|
if (len > sizeof(int))
|
|
|
|
len = sizeof(int);
|
2017-04-25 16:51:46 +00:00
|
|
|
if (len < sizeof(int))
|
|
|
|
return -EINVAL;
|
2008-07-15 05:50:15 +00:00
|
|
|
if (copy_from_user(&val, optval, len))
|
|
|
|
return -EFAULT;
|
|
|
|
switch (val) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
val = sizeof(struct tpacket_hdr);
|
|
|
|
break;
|
|
|
|
case TPACKET_V2:
|
|
|
|
val = sizeof(struct tpacket2_hdr);
|
|
|
|
break;
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
val = sizeof(struct tpacket3_hdr);
|
|
|
|
break;
|
2008-07-15 05:50:15 +00:00
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
break;
|
2008-07-19 01:05:19 +00:00
|
|
|
case PACKET_RESERVE:
|
|
|
|
val = po->tp_reserve;
|
|
|
|
break;
|
2009-05-19 05:11:22 +00:00
|
|
|
case PACKET_LOSS:
|
|
|
|
val = po->tp_loss;
|
|
|
|
break;
|
2010-06-02 12:53:56 +00:00
|
|
|
case PACKET_TIMESTAMP:
|
|
|
|
val = po->tp_tstamp;
|
|
|
|
break;
|
2011-07-05 08:45:05 +00:00
|
|
|
case PACKET_FANOUT:
|
|
|
|
val = (po->fanout ?
|
|
|
|
((u32)po->fanout->id |
|
packet: packet fanout rollover during socket overload
Changes:
v3->v2: rebase (no other changes)
passes selftest
v2->v1: read f->num_members only once
fix bug: test rollover mode + flag
Minimize packet drop in a fanout group. If one socket is full,
roll over packets to another from the group. Maintain flow
affinity during normal load using an rxhash fanout policy, while
dispersing unexpected traffic storms that hit a single cpu, such
as spoofed-source DoS flows. Rollover breaks affinity for flows
arriving at saturated sockets during those conditions.
The patch adds a fanout policy ROLLOVER that rotates between sockets,
filling each socket before moving to the next. It also adds a fanout
flag ROLLOVER. If passed along with any other fanout policy, the
primary policy is applied until the chosen socket is full. Then,
rollover selects another socket, to delay packet drop until the
entire system is saturated.
Probing sockets is not free. Selecting the last used socket, as
rollover does, is a greedy approach that maximizes chance of
success, at the cost of extreme load imbalance. In practice, with
sufficiently long queues to absorb bursts, sockets are drained in
parallel and load balance looks uniform in `top`.
To avoid contention, scales counters with number of sockets and
accesses them lockfree. Values are bounds checked to ensure
correctness.
Tested using an application with 9 threads pinned to CPUs, one socket
per thread and sufficient busywork per packet operation to limits each
thread to handling 32 Kpps. When sent 500 Kpps single UDP stream
packets, a FANOUT_CPU setup processes 32 Kpps in total without this
patch, 270 Kpps with the patch. Tested with read() and with a packet
ring (V1).
Also, passes psock_fanout.c unit test added to selftests.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-19 10:18:11 +00:00
|
|
|
((u32)po->fanout->type << 16) |
|
|
|
|
((u32)po->fanout->flags << 24)) :
|
2011-07-05 08:45:05 +00:00
|
|
|
0);
|
|
|
|
break;
|
2015-05-12 15:56:50 +00:00
|
|
|
case PACKET_ROLLOVER_STATS:
|
2017-11-28 15:44:29 +00:00
|
|
|
if (!po->rollover)
|
2015-05-12 15:56:50 +00:00
|
|
|
return -EINVAL;
|
2017-11-28 15:44:29 +00:00
|
|
|
rstats.tp_all = atomic_long_read(&po->rollover->num);
|
|
|
|
rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
|
|
|
|
rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
|
|
|
|
data = &rstats;
|
|
|
|
lv = sizeof(rstats);
|
2015-05-12 15:56:50 +00:00
|
|
|
break;
|
2012-11-06 23:10:47 +00:00
|
|
|
case PACKET_TX_HAS_OFF:
|
|
|
|
val = po->tp_tx_has_off;
|
|
|
|
break;
|
packet: introduce PACKET_QDISC_BYPASS socket option
This patch introduces a PACKET_QDISC_BYPASS socket option, that
allows for using a similar xmit() function as in pktgen instead
of taking the dev_queue_xmit() path. This can be very useful when
PF_PACKET applications are required to be used in a similar
scenario as pktgen, but with full, flexible packet payload that
needs to be provided, for example.
On default, nothing changes in behaviour for normal PF_PACKET
TX users, so everything stays as is for applications. New users,
however, can now set PACKET_QDISC_BYPASS if needed to prevent
own packets from i) reentering packet_rcv() and ii) to directly
push the frame to the driver.
In doing so we can increase pps (here 64 byte packets) for
PF_PACKET a bit:
# CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**]
1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436
2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779
3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610
4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114
5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422
6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261
7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619
8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879
9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447
10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689
11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412
[...]
20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081
[**]: qdisc path with packet_rcv(), how probably most people
seem to use it (hopefully not anymore if not needed)
The test was done using a modified trafgen, sending a simple
static 64 bytes packet, on all CPUs. The trick in the fast
"qdisc path" case, is to avoid reentering packet_rcv() by
setting the RAW socket protocol to zero, like:
socket(PF_PACKET, SOCK_RAW, 0);
Tradeoffs are documented as well in this patch, clearly, if
queues are busy, we will drop more packets, tc disciplines are
ignored, and these packets are not visible to taps anymore. For
a pktgen like scenario, we argue that this is acceptable.
The pointer to the xmit function has been placed in packet
socket structure hole between cached_dev and prot_hook that
is hot anyway as we're working on cached_dev in each send path.
Done in joint work together with Jesper Dangaard Brouer.
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-12-06 10:36:17 +00:00
|
|
|
case PACKET_QDISC_BYPASS:
|
|
|
|
val = packet_use_direct_xmit(po);
|
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
default:
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
}
|
|
|
|
|
2012-04-19 21:56:11 +00:00
|
|
|
if (len > lv)
|
|
|
|
len = lv;
|
2006-01-24 00:28:02 +00:00
|
|
|
if (put_user(len, optlen))
|
|
|
|
return -EFAULT;
|
2007-02-05 07:31:32 +00:00
|
|
|
if (copy_to_user(optval, data, len))
|
|
|
|
return -EFAULT;
|
2006-01-24 00:28:02 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-06-07 16:06:34 +00:00
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
|
|
|
|
char __user *optval, unsigned int optlen)
|
|
|
|
{
|
|
|
|
struct packet_sock *po = pkt_sk(sock->sk);
|
|
|
|
|
|
|
|
if (level != SOL_PACKET)
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
if (optname == PACKET_FANOUT_DATA &&
|
|
|
|
po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
|
|
|
|
optval = (char __user *)get_compat_bpf_fprog(optval);
|
|
|
|
if (!optval)
|
|
|
|
return -EFAULT;
|
|
|
|
optlen = sizeof(struct sock_fprog);
|
|
|
|
}
|
|
|
|
|
|
|
|
return packet_setsockopt(sock, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2013-05-28 01:30:21 +00:00
|
|
|
static int packet_notifier(struct notifier_block *this,
|
|
|
|
unsigned long msg, void *ptr)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
2013-05-28 01:30:21 +00:00
|
|
|
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
|
2008-03-25 12:47:49 +00:00
|
|
|
struct net *net = dev_net(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-02-22 07:57:18 +00:00
|
|
|
rcu_read_lock();
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 01:06:00 +00:00
|
|
|
sk_for_each_rcu(sk, &net->packet.sklist) {
|
2005-04-16 22:20:36 +00:00
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
|
|
|
|
|
|
|
switch (msg) {
|
|
|
|
case NETDEV_UNREGISTER:
|
|
|
|
if (po->mclist)
|
2015-03-09 18:51:04 +00:00
|
|
|
packet_dev_mclist_delete(dev, &po->mclist);
|
2007-05-29 20:12:50 +00:00
|
|
|
/* fallthrough */
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
case NETDEV_DOWN:
|
|
|
|
if (dev->ifindex == po->ifindex) {
|
|
|
|
spin_lock(&po->bind_lock);
|
|
|
|
if (po->running) {
|
2011-07-04 08:44:29 +00:00
|
|
|
__unregister_prot_hook(sk, false);
|
2005-04-16 22:20:36 +00:00
|
|
|
sk->sk_err = ENETDOWN;
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
sk->sk_error_report(sk);
|
|
|
|
}
|
|
|
|
if (msg == NETDEV_UNREGISTER) {
|
2013-12-06 10:36:15 +00:00
|
|
|
packet_cached_dev_reset(po);
|
2005-04-16 22:20:36 +00:00
|
|
|
po->ifindex = -1;
|
2011-06-01 07:18:52 +00:00
|
|
|
if (po->prot_hook.dev)
|
|
|
|
dev_put(po->prot_hook.dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
po->prot_hook.dev = NULL;
|
|
|
|
}
|
|
|
|
spin_unlock(&po->bind_lock);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case NETDEV_UP:
|
2010-02-22 07:57:18 +00:00
|
|
|
if (dev->ifindex == po->ifindex) {
|
|
|
|
spin_lock(&po->bind_lock);
|
2011-07-04 08:44:29 +00:00
|
|
|
if (po->num)
|
|
|
|
register_prot_hook(sk);
|
2010-02-22 07:57:18 +00:00
|
|
|
spin_unlock(&po->bind_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2010-02-22 07:57:18 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static int packet_ioctl(struct socket *sock, unsigned int cmd,
|
|
|
|
unsigned long arg)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
switch (cmd) {
|
2009-07-21 21:57:59 +00:00
|
|
|
case SIOCOUTQ:
|
|
|
|
{
|
|
|
|
int amount = sk_wmem_alloc_get(sk);
|
2009-06-18 02:05:41 +00:00
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
return put_user(amount, (int __user *)arg);
|
|
|
|
}
|
|
|
|
case SIOCINQ:
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
int amount = 0;
|
|
|
|
|
|
|
|
spin_lock_bh(&sk->sk_receive_queue.lock);
|
|
|
|
skb = skb_peek(&sk->sk_receive_queue);
|
|
|
|
if (skb)
|
|
|
|
amount = skb->len;
|
|
|
|
spin_unlock_bh(&sk->sk_receive_queue.lock);
|
|
|
|
return put_user(amount, (int __user *)arg);
|
|
|
|
}
|
|
|
|
case SIOCGSTAMP:
|
|
|
|
return sock_get_timestamp(sk, (struct timeval __user *)arg);
|
|
|
|
case SIOCGSTAMPNS:
|
|
|
|
return sock_get_timestampns(sk, (struct timespec __user *)arg);
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_INET
|
2009-07-21 21:57:59 +00:00
|
|
|
case SIOCADDRT:
|
|
|
|
case SIOCDELRT:
|
|
|
|
case SIOCDARP:
|
|
|
|
case SIOCGARP:
|
|
|
|
case SIOCSARP:
|
|
|
|
case SIOCGIFADDR:
|
|
|
|
case SIOCSIFADDR:
|
|
|
|
case SIOCGIFBRDADDR:
|
|
|
|
case SIOCSIFBRDADDR:
|
|
|
|
case SIOCGIFNETMASK:
|
|
|
|
case SIOCSIFNETMASK:
|
|
|
|
case SIOCGIFDSTADDR:
|
|
|
|
case SIOCSIFDSTADDR:
|
|
|
|
case SIOCSIFFLAGS:
|
|
|
|
return inet_dgram_ops.ioctl(sock, cmd, arg);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2009-07-21 21:57:59 +00:00
|
|
|
default:
|
|
|
|
return -ENOIOCTLCMD;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-04-09 13:27:37 +00:00
|
|
|
static __poll_t packet_poll_mask(struct socket *sock, __poll_t events)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2018-04-09 13:27:37 +00:00
|
|
|
__poll_t mask = datagram_poll_mask(sock, events);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock_bh(&sk->sk_receive_queue.lock);
|
2009-05-19 05:11:22 +00:00
|
|
|
if (po->rx_ring.pg_vec) {
|
2011-08-19 10:18:16 +00:00
|
|
|
if (!packet_previous_rx_frame(po, &po->rx_ring,
|
|
|
|
TP_STATUS_KERNEL))
|
2018-02-11 22:34:03 +00:00
|
|
|
mask |= EPOLLIN | EPOLLRDNORM;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
packet: rollover lock contention avoidance
Rollover has to call packet_rcv_has_room on sockets in the fanout
group to find a socket to migrate to. This operation is expensive
especially if the packet sockets use rings, when a lock has to be
acquired.
Avoid pounding on the lock by all sockets by temporarily marking a
socket as "under memory pressure" when such pressure is detected.
While set, only the socket owner may call packet_rcv_has_room on the
socket. Once it detects normal conditions, it clears the flag. The
socket is not used as a victim by any other socket in the meantime.
Under reasonably balanced load, each socket writer frequently calls
packet_rcv_has_room and clears its own pressure field. As a backup
for when the socket is rarely written to, also clear the flag on
reading (packet_recvmsg, packet_poll) if this can be done cheaply
(i.e., without calling packet_rcv_has_room). This is only for
edge cases.
Tested:
Ran bench_rollover: a process with 8 sockets in a single fanout
group, each pinned to a single cpu that receives one nic recv
interrupt. RPS and RFS are disabled. The benchmark uses packet
rx_ring, which has to take a lock when determining whether a
socket has room.
Sent 3.5 Mpps of UDP traffic with sufficient entropy to spread
uniformly across the packet sockets (and inserted an iptables
rule to drop in PREROUTING to avoid protocol stack processing).
Without this patch, all sockets try to migrate traffic to
neighbors, causing lock contention when searching for a non-
empty neighbor. The lock is the top 9 entries.
perf record -a -g sleep 5
- 17.82% bench_rollover [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
- 99.00% spin_lock
+ 81.77% packet_rcv_has_room.isra.41
+ 18.23% tpacket_rcv
+ 0.84% packet_rcv_has_room.isra.41
+ 5.20% ksoftirqd/6 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.15% ksoftirqd/1 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.14% ksoftirqd/2 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/7 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.12% ksoftirqd/5 [kernel.kallsyms] [k] _raw_spin_lock
+ 5.10% ksoftirqd/4 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.66% ksoftirqd/0 [kernel.kallsyms] [k] _raw_spin_lock
+ 4.45% ksoftirqd/3 [kernel.kallsyms] [k] _raw_spin_lock
+ 1.55% bench_rollover [kernel.kallsyms] [k] packet_rcv_has_room.isra.41
On net-next with this patch, this lock contention is no longer a
top entry. Most time is spent in the actual read function. Next up
are other locks:
+ 15.52% bench_rollover bench_rollover [.] reader
+ 4.68% swapper [kernel.kallsyms] [k] memcpy_erms
+ 2.77% swapper [kernel.kallsyms] [k] packet_lookup_frame.isra.51
+ 2.56% ksoftirqd/1 [kernel.kallsyms] [k] memcpy_erms
+ 2.16% swapper [kernel.kallsyms] [k] tpacket_rcv
+ 1.93% swapper [kernel.kallsyms] [k] mlx4_en_process_rx_cq
Looking closer at the remaining _raw_spin_lock, the cost of probing
in rollover is now comparable to the cost of taking the lock later
in tpacket_rcv.
- 1.51% swapper [kernel.kallsyms] [k] _raw_spin_lock
- _raw_spin_lock
+ 33.41% packet_rcv_has_room
+ 28.15% tpacket_rcv
+ 19.54% enqueue_to_backlog
+ 6.45% __free_pages_ok
+ 2.78% packet_rcv_fanout
+ 2.13% fanout_demux_rollover
+ 2.01% netif_receive_skb_internal
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-12 15:56:48 +00:00
|
|
|
if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
|
2015-05-14 19:25:02 +00:00
|
|
|
po->pressure = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock_bh(&sk->sk_receive_queue.lock);
|
2009-05-19 05:11:22 +00:00
|
|
|
spin_lock_bh(&sk->sk_write_queue.lock);
|
|
|
|
if (po->tx_ring.pg_vec) {
|
|
|
|
if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
|
2018-02-11 22:34:03 +00:00
|
|
|
mask |= EPOLLOUT | EPOLLWRNORM;
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
|
|
|
spin_unlock_bh(&sk->sk_write_queue.lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
return mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* Dirty? Well, I still did not learn better way to account
|
|
|
|
* for user mmaps.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static void packet_mm_open(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct file *file = vma->vm_file;
|
2009-07-21 21:57:59 +00:00
|
|
|
struct socket *sock = file->private_data;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sock *sk = sock->sk;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sk)
|
|
|
|
atomic_inc(&pkt_sk(sk)->mapped);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_mm_close(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct file *file = vma->vm_file;
|
2009-07-21 21:57:59 +00:00
|
|
|
struct socket *sock = file->private_data;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sock *sk = sock->sk;
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sk)
|
|
|
|
atomic_dec(&pkt_sk(sk)->mapped);
|
|
|
|
}
|
|
|
|
|
2009-09-27 18:29:37 +00:00
|
|
|
static const struct vm_operations_struct packet_mmap_ops = {
|
2009-07-21 21:57:59 +00:00
|
|
|
.open = packet_mm_open,
|
|
|
|
.close = packet_mm_close,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
|
|
|
|
unsigned int len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2005-12-07 00:38:35 +00:00
|
|
|
for (i = 0; i < len; i++) {
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
if (likely(pg_vec[i].buffer)) {
|
2010-12-01 02:52:57 +00:00
|
|
|
if (is_vmalloc_addr(pg_vec[i].buffer))
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
vfree(pg_vec[i].buffer);
|
|
|
|
else
|
|
|
|
free_pages((unsigned long)pg_vec[i].buffer,
|
|
|
|
order);
|
|
|
|
pg_vec[i].buffer = NULL;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
kfree(pg_vec);
|
|
|
|
}
|
|
|
|
|
2011-11-02 11:00:49 +00:00
|
|
|
static char *alloc_one_pg_vec_page(unsigned long order)
|
2005-12-07 00:38:35 +00:00
|
|
|
{
|
2014-01-19 10:46:53 +00:00
|
|
|
char *buffer;
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
|
|
|
|
__GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
|
|
|
|
|
|
|
|
buffer = (char *) __get_free_pages(gfp_flags, order);
|
|
|
|
if (buffer)
|
|
|
|
return buffer;
|
|
|
|
|
2014-01-19 10:46:53 +00:00
|
|
|
/* __get_free_pages failed, fall back to vmalloc */
|
2010-11-20 07:31:54 +00:00
|
|
|
buffer = vzalloc((1 << order) * PAGE_SIZE);
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
if (buffer)
|
|
|
|
return buffer;
|
|
|
|
|
2014-01-19 10:46:53 +00:00
|
|
|
/* vmalloc failed, lets dig into swap here */
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
gfp_flags &= ~__GFP_NORETRY;
|
2014-01-19 10:46:53 +00:00
|
|
|
buffer = (char *) __get_free_pages(gfp_flags, order);
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
if (buffer)
|
|
|
|
return buffer;
|
|
|
|
|
2014-01-19 10:46:53 +00:00
|
|
|
/* complete and utter failure */
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
return NULL;
|
2005-12-07 00:38:35 +00:00
|
|
|
}
|
|
|
|
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
|
2005-12-07 00:38:35 +00:00
|
|
|
{
|
|
|
|
unsigned int block_nr = req->tp_block_nr;
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
struct pgv *pg_vec;
|
2005-12-07 00:38:35 +00:00
|
|
|
int i;
|
|
|
|
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
|
2005-12-07 00:38:35 +00:00
|
|
|
if (unlikely(!pg_vec))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
for (i = 0; i < block_nr; i++) {
|
2010-12-01 02:52:57 +00:00
|
|
|
pg_vec[i].buffer = alloc_one_pg_vec_page(order);
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
if (unlikely(!pg_vec[i].buffer))
|
2005-12-07 00:38:35 +00:00
|
|
|
goto out_free_pgvec;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
return pg_vec;
|
|
|
|
|
|
|
|
out_free_pgvec:
|
|
|
|
free_pg_vec(pg_vec, order, block_nr);
|
|
|
|
pg_vec = NULL;
|
|
|
|
goto out;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-08-19 10:18:16 +00:00
|
|
|
static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
|
2009-05-19 05:11:22 +00:00
|
|
|
int closing, int tx_ring)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
struct pgv *pg_vec = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2006-11-08 08:26:29 +00:00
|
|
|
int was_running, order = 0;
|
2009-05-19 05:11:22 +00:00
|
|
|
struct packet_ring_buffer *rb;
|
|
|
|
struct sk_buff_head *rb_queue;
|
2006-11-08 08:26:29 +00:00
|
|
|
__be16 num;
|
2011-08-19 10:18:16 +00:00
|
|
|
int err = -EINVAL;
|
|
|
|
/* Added to avoid minimal code churn */
|
|
|
|
struct tpacket_req *req = &req_u->req;
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
rb = tx_ring ? &po->tx_ring : &po->rx_ring;
|
|
|
|
rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
err = -EBUSY;
|
|
|
|
if (!closing) {
|
|
|
|
if (atomic_read(&po->mapped))
|
|
|
|
goto out;
|
packet: use percpu mmap tx frame pending refcount
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc()
and one atomic_dec() call in skb destructor and use a percpu
reference count instead in order to determine if packets are
still pending to be sent out. Micro-benchmark with [1] that has
been slightly modified (that is, protcol = 0 in socket(2) and
bind(2)), example on a rather crappy testing machine; I expect
it to scale and have even better results on bigger machines:
./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs:
With patch: 4,022,015 cyc
Without patch: 4,812,994 cyc
time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable:
With patch:
real 1m32.241s
user 0m0.287s
sys 1m29.316s
Without patch:
real 1m38.386s
user 0m0.265s
sys 1m35.572s
In function tpacket_snd(), it is okay to use packet_read_pending()
since in fast-path we short-circuit the condition already with
ph != NULL, since we have next frames to process. In case we have
MSG_DONTWAIT, we also do not execute this path as need_wait is
false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is
okay to call a packet_read_pending(), because when we ever reach
that path, we're done processing outgoing frames anyway and only
look if there are skbs still outstanding to be orphaned. We can
stay lockless in this percpu counter since it's acceptable when we
reach this path for the sum to be imprecise first, but we'll level
out at 0 after all pending frames have reached the skb destructor
eventually through tx reclaim. When people pin a tx process to
particular CPUs, we expect overflows to happen in the reference
counter as on one CPU we expect heavy increase; and distributed
through ksoftirqd on all CPUs a decrease, for example. As
David Laight points out, since the C language doesn't define the
result of signed int overflow (i.e. rather than wrap, it is
allowed to saturate as a possible outcome), we have to use
unsigned int as reference count. The sum over all CPUs when tx
is complete will result in 0 again.
The BUG_ON() in tpacket_destruct_skb() we can remove as well. It
can _only_ be set from inside tpacket_snd() path and we made sure
to increase tx_ring.pending in any case before we called po->xmit(skb).
So testing for tx_ring.pending == 0 is not too useful. Instead, it
would rather have been useful to test if lower layers didn't orphan
the skb so that we're missing ring slots being put back to
TP_STATUS_AVAILABLE. But such a bug will be caught in user space
already as we end up realizing that we do not have any
TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set.
Btw, in case of RX_RING path, we do not make use of the pending
member, therefore we also don't need to use up any percpu memory
here. Also note that __alloc_percpu() already returns a zero-filled
percpu area, so initialization is done already.
[1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-01-15 15:25:36 +00:00
|
|
|
if (packet_read_pending(rb))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
if (req->tp_block_nr) {
|
|
|
|
/* Sanity tests and some calculations */
|
|
|
|
err = -EBUSY;
|
|
|
|
if (unlikely(rb->pg_vec))
|
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-15 05:50:15 +00:00
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V1:
|
|
|
|
po->tp_hdrlen = TPACKET_HDRLEN;
|
|
|
|
break;
|
|
|
|
case TPACKET_V2:
|
|
|
|
po->tp_hdrlen = TPACKET2_HDRLEN;
|
|
|
|
break;
|
2011-08-19 10:18:16 +00:00
|
|
|
case TPACKET_V3:
|
|
|
|
po->tp_hdrlen = TPACKET3_HDRLEN;
|
|
|
|
break;
|
2008-07-15 05:50:15 +00:00
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
err = -EINVAL;
|
2005-12-07 00:38:35 +00:00
|
|
|
if (unlikely((int)req->tp_block_size <= 0))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
2015-11-17 09:40:21 +00:00
|
|
|
if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
2014-08-15 16:16:04 +00:00
|
|
|
if (po->tp_version >= TPACKET_V3 &&
|
2017-03-29 14:11:20 +00:00
|
|
|
req->tp_block_size <=
|
2018-06-01 16:23:02 +00:00
|
|
|
BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + sizeof(struct tpacket3_hdr))
|
2014-08-15 16:16:04 +00:00
|
|
|
goto out;
|
2008-07-19 01:05:19 +00:00
|
|
|
if (unlikely(req->tp_frame_size < po->tp_hdrlen +
|
2009-05-19 05:11:22 +00:00
|
|
|
po->tp_reserve))
|
|
|
|
goto out;
|
2005-12-07 00:38:35 +00:00
|
|
|
if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-11-17 09:38:36 +00:00
|
|
|
rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
|
|
|
|
if (unlikely(rb->frames_per_block == 0))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
2017-03-29 14:11:21 +00:00
|
|
|
if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
|
|
|
|
goto out;
|
2009-05-19 05:11:22 +00:00
|
|
|
if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
|
|
|
|
req->tp_frame_nr))
|
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
err = -ENOMEM;
|
2005-12-07 00:38:35 +00:00
|
|
|
order = get_order(req->tp_block_size);
|
|
|
|
pg_vec = alloc_pg_vec(req, order);
|
|
|
|
if (unlikely(!pg_vec))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
2011-08-19 10:18:16 +00:00
|
|
|
switch (po->tp_version) {
|
|
|
|
case TPACKET_V3:
|
2017-01-03 14:31:47 +00:00
|
|
|
/* Block transmit is not supported yet */
|
|
|
|
if (!tx_ring) {
|
2015-06-22 07:09:16 +00:00
|
|
|
init_prb_bdqc(po, rb, pg_vec, req_u);
|
2017-01-03 14:31:47 +00:00
|
|
|
} else {
|
|
|
|
struct tpacket_req3 *req3 = &req_u->req3;
|
|
|
|
|
|
|
|
if (req3->tp_retire_blk_tov ||
|
|
|
|
req3->tp_sizeof_priv ||
|
|
|
|
req3->tp_feature_req_word) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
2014-02-18 12:20:51 +00:00
|
|
|
break;
|
2011-08-19 10:18:16 +00:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
|
|
|
/* Done */
|
|
|
|
else {
|
|
|
|
err = -EINVAL;
|
2005-12-07 00:38:35 +00:00
|
|
|
if (unlikely(req->tp_frame_nr))
|
2009-05-19 05:11:22 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* Detach socket from network */
|
|
|
|
spin_lock(&po->bind_lock);
|
|
|
|
was_running = po->running;
|
|
|
|
num = po->num;
|
|
|
|
if (was_running) {
|
|
|
|
po->num = 0;
|
2011-07-04 08:44:29 +00:00
|
|
|
__unregister_prot_hook(sk, false);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
spin_unlock(&po->bind_lock);
|
2007-02-09 14:25:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
synchronize_net();
|
|
|
|
|
|
|
|
err = -EBUSY;
|
2009-01-30 22:12:06 +00:00
|
|
|
mutex_lock(&po->pg_vec_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (closing || atomic_read(&po->mapped) == 0) {
|
|
|
|
err = 0;
|
2009-05-19 05:11:22 +00:00
|
|
|
spin_lock_bh(&rb_queue->lock);
|
2010-12-11 00:02:20 +00:00
|
|
|
swap(rb->pg_vec, pg_vec);
|
2009-05-19 05:11:22 +00:00
|
|
|
rb->frame_max = (req->tp_frame_nr - 1);
|
|
|
|
rb->head = 0;
|
|
|
|
rb->frame_size = req->tp_frame_size;
|
|
|
|
spin_unlock_bh(&rb_queue->lock);
|
|
|
|
|
2010-12-11 00:02:20 +00:00
|
|
|
swap(rb->pg_vec_order, order);
|
|
|
|
swap(rb->pg_vec_len, req->tp_block_nr);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
|
|
|
|
po->prot_hook.func = (po->rx_ring.pg_vec) ?
|
|
|
|
tpacket_rcv : packet_rcv;
|
|
|
|
skb_queue_purge(rb_queue);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (atomic_read(&po->mapped))
|
2009-07-21 21:57:59 +00:00
|
|
|
pr_err("packet_mmap: vma is busy: %d\n",
|
|
|
|
atomic_read(&po->mapped));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-01-30 22:12:06 +00:00
|
|
|
mutex_unlock(&po->pg_vec_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock(&po->bind_lock);
|
2011-07-04 08:44:29 +00:00
|
|
|
if (was_running) {
|
2005-04-16 22:20:36 +00:00
|
|
|
po->num = num;
|
2011-07-04 08:44:29 +00:00
|
|
|
register_prot_hook(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
spin_unlock(&po->bind_lock);
|
packet: fix use-after-free in prb_retire_rx_blk_timer_expired()
There are multiple reports showing we have a use-after-free in
the timer prb_retire_rx_blk_timer_expired(), where we use struct
tpacket_kbdq_core::pkbdq, a pg_vec, after it gets freed by
free_pg_vec().
The interesting part is it is not freed via packet_release() but
via packet_setsockopt(), which means we are not closing the socket.
Looking into the big and fat function packet_set_ring(), this could
happen if we satisfy the following conditions:
1. closing == 0, not on packet_release() path
2. req->tp_block_nr == 0, we don't allocate a new pg_vec
3. rx_ring->pg_vec is already set as V3, which means we already called
packet_set_ring() wtih req->tp_block_nr > 0 previously
4. req->tp_frame_nr == 0, pass sanity check
5. po->mapped == 0, never called mmap()
In this scenario we are clearing the old rx_ring->pg_vec, so we need
to free this pg_vec, but we don't stop the timer on this path because
of closing==0.
The timer has to be stopped as long as we need to free pg_vec, therefore
the check on closing!=0 is wrong, we should check pg_vec!=NULL instead.
Thanks to liujian for testing different fixes.
Reported-by: alexander.levin@verizon.com
Reported-by: Dave Jones <davej@codemonkey.org.uk>
Reported-by: liujian (CE) <liujian56@huawei.com>
Tested-by: liujian (CE) <liujian56@huawei.com>
Cc: Ding Tianhong <dingtianhong@huawei.com>
Cc: Willem de Bruijn <willemdebruijn.kernel@gmail.com>
Signed-off-by: Cong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-24 17:07:32 +00:00
|
|
|
if (pg_vec && (po->tp_version > TPACKET_V2)) {
|
2011-08-19 10:18:16 +00:00
|
|
|
/* Because we don't support block-based V3 on tx-ring */
|
|
|
|
if (!tx_ring)
|
2015-07-28 12:21:26 +00:00
|
|
|
prb_shutdown_retire_blk_timer(po, rb_queue);
|
2011-08-19 10:18:16 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (pg_vec)
|
|
|
|
free_pg_vec(pg_vec, order, req->tp_block_nr);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2009-05-19 05:11:22 +00:00
|
|
|
static int packet_mmap(struct file *file, struct socket *sock,
|
|
|
|
struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct packet_sock *po = pkt_sk(sk);
|
2009-05-19 05:11:22 +00:00
|
|
|
unsigned long size, expected_size;
|
|
|
|
struct packet_ring_buffer *rb;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long start;
|
|
|
|
int err = -EINVAL;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (vma->vm_pgoff)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2009-01-30 22:12:06 +00:00
|
|
|
mutex_lock(&po->pg_vec_lock);
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
expected_size = 0;
|
|
|
|
for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
|
|
|
|
if (rb->pg_vec) {
|
|
|
|
expected_size += rb->pg_vec_len
|
|
|
|
* rb->pg_vec_pages
|
|
|
|
* PAGE_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (expected_size == 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
2009-05-19 05:11:22 +00:00
|
|
|
|
|
|
|
size = vma->vm_end - vma->vm_start;
|
|
|
|
if (size != expected_size)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
start = vma->vm_start;
|
2009-05-19 05:11:22 +00:00
|
|
|
for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
|
|
|
|
if (rb->pg_vec == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
for (i = 0; i < rb->pg_vec_len; i++) {
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
struct page *page;
|
|
|
|
void *kaddr = rb->pg_vec[i].buffer;
|
2009-05-19 05:11:22 +00:00
|
|
|
int pg_num;
|
|
|
|
|
2010-12-01 02:52:57 +00:00
|
|
|
for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
|
|
|
|
page = pgv_to_page(kaddr);
|
2009-05-19 05:11:22 +00:00
|
|
|
err = vm_insert_page(vma, start, page);
|
|
|
|
if (unlikely(err))
|
|
|
|
goto out;
|
|
|
|
start += PAGE_SIZE;
|
packet: Enhance AF_PACKET implementation to not require high order contiguous memory allocation (v4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Version 4 of this patch.
Change notes:
1) Removed extra memset. Didn't think kcalloc added a GFP_ZERO the way kzalloc did :)
Summary:
It was shown to me recently that systems under high load were driven very deep
into swap when tcpdump was run. The reason this happened was because the
AF_PACKET protocol has a SET_RINGBUFFER socket option that allows the user space
application to specify how many entries an AF_PACKET socket will have and how
large each entry will be. It seems the default setting for tcpdump is to set
the ring buffer to 32 entries of 64 Kb each, which implies 32 order 5
allocation. Thats difficult under good circumstances, and horrid under memory
pressure.
I thought it would be good to make that a bit more usable. I was going to do a
simple conversion of the ring buffer from contigous pages to iovecs, but
unfortunately, the metadata which AF_PACKET places in these buffers can easily
span a page boundary, and given that these buffers get mapped into user space,
and the data layout doesn't easily allow for a change to padding between frames
to avoid that, a simple iovec change is just going to break user space ABI
consistency.
So I've done this, I've added a three tiered mechanism to the af_packet set_ring
socket option. It attempts to allocate memory in the following order:
1) Using __get_free_pages with GFP_NORETRY set, so as to fail quickly without
digging into swap
2) Using vmalloc
3) Using __get_free_pages with GFP_NORETRY clear, causing us to try as hard as
needed to get the memory
The effect is that we don't disturb the system as much when we're under load,
while still being able to conduct tcpdumps effectively.
Tested successfully by me.
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Reported-by: Maciej Żenczykowski <zenczykowski@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-16 18:26:47 +00:00
|
|
|
kaddr += PAGE_SIZE;
|
2009-05-19 05:11:22 +00:00
|
|
|
}
|
2005-12-07 00:38:35 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-05-19 05:11:22 +00:00
|
|
|
|
2005-12-07 00:38:35 +00:00
|
|
|
atomic_inc(&po->mapped);
|
2005-04-16 22:20:36 +00:00
|
|
|
vma->vm_ops = &packet_mmap_ops;
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
out:
|
2009-01-30 22:12:06 +00:00
|
|
|
mutex_unlock(&po->pg_vec_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2005-12-22 20:49:22 +00:00
|
|
|
static const struct proto_ops packet_ops_spkt = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.family = PF_PACKET,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = packet_release,
|
|
|
|
.bind = packet_bind_spkt,
|
|
|
|
.connect = sock_no_connect,
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
.accept = sock_no_accept,
|
|
|
|
.getname = packet_getname_spkt,
|
2018-04-09 13:27:37 +00:00
|
|
|
.poll_mask = datagram_poll_mask,
|
2005-04-16 22:20:36 +00:00
|
|
|
.ioctl = packet_ioctl,
|
|
|
|
.listen = sock_no_listen,
|
|
|
|
.shutdown = sock_no_shutdown,
|
|
|
|
.setsockopt = sock_no_setsockopt,
|
|
|
|
.getsockopt = sock_no_getsockopt,
|
|
|
|
.sendmsg = packet_sendmsg_spkt,
|
|
|
|
.recvmsg = packet_recvmsg,
|
|
|
|
.mmap = sock_no_mmap,
|
|
|
|
.sendpage = sock_no_sendpage,
|
|
|
|
};
|
|
|
|
|
2005-12-22 20:49:22 +00:00
|
|
|
static const struct proto_ops packet_ops = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.family = PF_PACKET,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = packet_release,
|
|
|
|
.bind = packet_bind,
|
|
|
|
.connect = sock_no_connect,
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
.accept = sock_no_accept,
|
2007-02-09 14:25:10 +00:00
|
|
|
.getname = packet_getname,
|
2018-04-09 13:27:37 +00:00
|
|
|
.poll_mask = packet_poll_mask,
|
2005-04-16 22:20:36 +00:00
|
|
|
.ioctl = packet_ioctl,
|
|
|
|
.listen = sock_no_listen,
|
|
|
|
.shutdown = sock_no_shutdown,
|
|
|
|
.setsockopt = packet_setsockopt,
|
|
|
|
.getsockopt = packet_getsockopt,
|
2016-06-07 16:06:34 +00:00
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
.compat_setsockopt = compat_packet_setsockopt,
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
.sendmsg = packet_sendmsg,
|
|
|
|
.recvmsg = packet_recvmsg,
|
|
|
|
.mmap = packet_mmap,
|
|
|
|
.sendpage = sock_no_sendpage,
|
|
|
|
};
|
|
|
|
|
2009-10-05 05:58:39 +00:00
|
|
|
static const struct net_proto_family packet_family_ops = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.family = PF_PACKET,
|
|
|
|
.create = packet_create,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct notifier_block packet_netdev_notifier = {
|
2009-07-21 21:57:59 +00:00
|
|
|
.notifier_call = packet_notifier,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
|
|
|
|
|
|
static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
|
2010-02-22 07:57:18 +00:00
|
|
|
__acquires(RCU)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2007-11-20 06:31:54 +00:00
|
|
|
struct net *net = seq_file_net(seq);
|
2010-02-22 07:57:18 +00:00
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
|
|
{
|
2007-12-16 22:04:02 +00:00
|
|
|
struct net *net = seq_file_net(seq);
|
2010-02-22 07:57:18 +00:00
|
|
|
return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void packet_seq_stop(struct seq_file *seq, void *v)
|
2010-02-22 07:57:18 +00:00
|
|
|
__releases(RCU)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-02-22 07:57:18 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2007-02-09 14:25:10 +00:00
|
|
|
static int packet_seq_show(struct seq_file *seq, void *v)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
if (v == SEQ_START_TOKEN)
|
|
|
|
seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
|
|
|
|
else {
|
2010-02-08 23:19:29 +00:00
|
|
|
struct sock *s = sk_entry(v);
|
2005-04-16 22:20:36 +00:00
|
|
|
const struct packet_sock *po = pkt_sk(s);
|
|
|
|
|
|
|
|
seq_printf(seq,
|
net: convert %p usage to %pK
The %pK format specifier is designed to hide exposed kernel pointers,
specifically via /proc interfaces. Exposing these pointers provides an
easy target for kernel write vulnerabilities, since they reveal the
locations of writable structures containing easily triggerable function
pointers. The behavior of %pK depends on the kptr_restrict sysctl.
If kptr_restrict is set to 0, no deviation from the standard %p behavior
occurs. If kptr_restrict is set to 1, the default, if the current user
(intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG
(currently in the LSM tree), kernel pointers using %pK are printed as 0's.
If kptr_restrict is set to 2, kernel pointers using %pK are printed as
0's regardless of privileges. Replacing with 0's was chosen over the
default "(null)", which cannot be parsed by userland %p, which expects
"(nil)".
The supporting code for kptr_restrict and %pK are currently in the -mm
tree. This patch converts users of %p in net/ to %pK. Cases of printing
pointers to the syslog are not covered, since this would eliminate useful
information for postmortem debugging and the reading of the syslog is
already optionally protected by the dmesg_restrict sysctl.
Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com>
Cc: James Morris <jmorris@namei.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Thomas Graf <tgraf@infradead.org>
Cc: Eugene Teo <eugeneteo@kernel.org>
Cc: Kees Cook <kees.cook@canonical.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: David S. Miller <davem@davemloft.net>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Eric Paris <eparis@parisplace.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-05-23 12:17:35 +00:00
|
|
|
"%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
|
2005-04-16 22:20:36 +00:00
|
|
|
s,
|
2017-06-30 10:08:01 +00:00
|
|
|
refcount_read(&s->sk_refcnt),
|
2005-04-16 22:20:36 +00:00
|
|
|
s->sk_type,
|
|
|
|
ntohs(po->num),
|
|
|
|
po->ifindex,
|
|
|
|
po->running,
|
|
|
|
atomic_read(&s->sk_rmem_alloc),
|
2012-05-24 07:10:10 +00:00
|
|
|
from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
|
2009-07-21 21:57:59 +00:00
|
|
|
sock_i_ino(s));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-07-11 06:07:31 +00:00
|
|
|
static const struct seq_operations packet_seq_ops = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.start = packet_seq_start,
|
|
|
|
.next = packet_seq_next,
|
|
|
|
.stop = packet_seq_stop,
|
|
|
|
.show = packet_seq_show,
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2010-01-17 03:35:32 +00:00
|
|
|
static int __net_init packet_net_init(struct net *net)
|
2007-11-20 06:28:35 +00:00
|
|
|
{
|
2012-08-21 01:06:47 +00:00
|
|
|
mutex_init(&net->packet.sklist_lock);
|
2007-12-11 12:19:54 +00:00
|
|
|
INIT_HLIST_HEAD(&net->packet.sklist);
|
2007-11-20 06:28:35 +00:00
|
|
|
|
2018-04-10 17:42:55 +00:00
|
|
|
if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
|
|
|
|
sizeof(struct seq_net_private)))
|
2007-11-20 06:28:35 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-01-17 03:35:32 +00:00
|
|
|
static void __net_exit packet_net_exit(struct net *net)
|
2007-11-20 06:28:35 +00:00
|
|
|
{
|
2013-02-18 01:34:56 +00:00
|
|
|
remove_proc_entry("packet", net->proc_net);
|
2017-11-12 19:27:49 +00:00
|
|
|
WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
|
2007-11-20 06:28:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations packet_net_ops = {
|
|
|
|
.init = packet_net_init,
|
|
|
|
.exit = packet_net_exit,
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static void __exit packet_exit(void)
|
|
|
|
{
|
|
|
|
unregister_netdevice_notifier(&packet_netdev_notifier);
|
2007-11-20 06:28:35 +00:00
|
|
|
unregister_pernet_subsys(&packet_net_ops);
|
2005-04-16 22:20:36 +00:00
|
|
|
sock_unregister(PF_PACKET);
|
|
|
|
proto_unregister(&packet_proto);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init packet_init(void)
|
|
|
|
{
|
|
|
|
int rc = proto_register(&packet_proto, 0);
|
|
|
|
|
|
|
|
if (rc != 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
sock_register(&packet_family_ops);
|
2007-11-20 06:28:35 +00:00
|
|
|
register_pernet_subsys(&packet_net_ops);
|
2005-04-16 22:20:36 +00:00
|
|
|
register_netdevice_notifier(&packet_netdev_notifier);
|
|
|
|
out:
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
module_init(packet_init);
|
|
|
|
module_exit(packet_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
|
|
MODULE_ALIAS_NETPROTO(PF_PACKET);
|