5b74283ab2
When unmapping pages it is necessary to flush the TLB. If that page was
accessed by another CPU then an IPI is used to flush the remote CPU. That
is a lot of IPIs if kswapd is scanning and unmapping >100K pages per
second.
There already is a window between when a page is unmapped and when it is
TLB flushed. This series increases the window so multiple pages can be
flushed using a single IPI. This should be safe or the kernel is hosed
already.
Patch 1 simply made the rest of the series easier to write as ftrace
could identify all the senders of TLB flush IPIS.
Patch 2 tracks what CPUs potentially map a PFN and then sends an IPI
to flush the entire TLB.
Patch 3 tracks when there potentially are writable TLB entries that
need to be batched differently
Patch 4 increases SWAP_CLUSTER_MAX to further batch flushes
The performance impact is documented in the changelogs but in the optimistic
case on a 4-socket machine the full series reduces interrupts from 900K
interrupts/second to 60K interrupts/second.
This patch (of 4):
It is easy to trace when an IPI is received to flush a TLB but harder to
detect what event sent it. This patch makes it easy to identify the
source of IPIs being transmitted for TLB flushes on x86.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
64 lines
1.4 KiB
C
64 lines
1.4 KiB
C
#undef TRACE_SYSTEM
|
|
#define TRACE_SYSTEM tlb
|
|
|
|
#if !defined(_TRACE_TLB_H) || defined(TRACE_HEADER_MULTI_READ)
|
|
#define _TRACE_TLB_H
|
|
|
|
#include <linux/mm_types.h>
|
|
#include <linux/tracepoint.h>
|
|
|
|
#define TLB_FLUSH_REASON \
|
|
EM( TLB_FLUSH_ON_TASK_SWITCH, "flush on task switch" ) \
|
|
EM( TLB_REMOTE_SHOOTDOWN, "remote shootdown" ) \
|
|
EM( TLB_LOCAL_SHOOTDOWN, "local shootdown" ) \
|
|
EM( TLB_LOCAL_MM_SHOOTDOWN, "local mm shootdown" ) \
|
|
EMe( TLB_REMOTE_SEND_IPI, "remote ipi send" )
|
|
|
|
/*
|
|
* First define the enums in TLB_FLUSH_REASON to be exported to userspace
|
|
* via TRACE_DEFINE_ENUM().
|
|
*/
|
|
#undef EM
|
|
#undef EMe
|
|
#define EM(a,b) TRACE_DEFINE_ENUM(a);
|
|
#define EMe(a,b) TRACE_DEFINE_ENUM(a);
|
|
|
|
TLB_FLUSH_REASON
|
|
|
|
/*
|
|
* Now redefine the EM() and EMe() macros to map the enums to the strings
|
|
* that will be printed in the output.
|
|
*/
|
|
#undef EM
|
|
#undef EMe
|
|
#define EM(a,b) { a, b },
|
|
#define EMe(a,b) { a, b }
|
|
|
|
TRACE_EVENT_CONDITION(tlb_flush,
|
|
|
|
TP_PROTO(int reason, unsigned long pages),
|
|
TP_ARGS(reason, pages),
|
|
|
|
TP_CONDITION(cpu_online(smp_processor_id())),
|
|
|
|
TP_STRUCT__entry(
|
|
__field( int, reason)
|
|
__field(unsigned long, pages)
|
|
),
|
|
|
|
TP_fast_assign(
|
|
__entry->reason = reason;
|
|
__entry->pages = pages;
|
|
),
|
|
|
|
TP_printk("pages:%ld reason:%s (%d)",
|
|
__entry->pages,
|
|
__print_symbolic(__entry->reason, TLB_FLUSH_REASON),
|
|
__entry->reason)
|
|
);
|
|
|
|
#endif /* _TRACE_TLB_H */
|
|
|
|
/* This part must be outside protection */
|
|
#include <trace/define_trace.h>
|