2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* linux/fs/buffer.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992, 2002 Linus Torvalds
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
|
|
|
|
*
|
|
|
|
* Removed a lot of unnecessary code and simplified things now that
|
|
|
|
* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
|
|
|
|
*
|
|
|
|
* Speed up hash, lru, and free list operations. Use gfp() for allocating
|
|
|
|
* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
|
|
|
|
*
|
|
|
|
* Added 32k buffer block sizes - these are required older ARM systems. - RMK
|
|
|
|
*
|
|
|
|
* async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/slab.h>
|
2006-01-11 20:17:46 +00:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/quotaops.h>
|
|
|
|
#include <linux/highmem.h>
|
2011-11-17 04:57:37 +00:00
|
|
|
#include <linux/export.h>
|
2015-06-02 14:37:23 +00:00
|
|
|
#include <linux/backing-dev.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/hash.h>
|
|
|
|
#include <linux/suspend.h>
|
|
|
|
#include <linux/buffer_head.h>
|
2006-12-10 10:19:27 +00:00
|
|
|
#include <linux/task_io_accounting_ops.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/bio.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/mpage.h>
|
[PATCH] spinlock consolidation
This patch (written by me and also containing many suggestions of Arjan van
de Ven) does a major cleanup of the spinlock code. It does the following
things:
- consolidates and enhances the spinlock/rwlock debugging code
- simplifies the asm/spinlock.h files
- encapsulates the raw spinlock type and moves generic spinlock
features (such as ->break_lock) into the generic code.
- cleans up the spinlock code hierarchy to get rid of the spaghetti.
Most notably there's now only a single variant of the debugging code,
located in lib/spinlock_debug.c. (previously we had one SMP debugging
variant per architecture, plus a separate generic one for UP builds)
Also, i've enhanced the rwlock debugging facility, it will now track
write-owners. There is new spinlock-owner/CPU-tracking on SMP builds too.
All locks have lockup detection now, which will work for both soft and hard
spin/rwlock lockups.
The arch-level include files now only contain the minimally necessary
subset of the spinlock code - all the rest that can be generalized now
lives in the generic headers:
include/asm-i386/spinlock_types.h | 16
include/asm-x86_64/spinlock_types.h | 16
I have also split up the various spinlock variants into separate files,
making it easier to see which does what. The new layout is:
SMP | UP
----------------------------|-----------------------------------
asm/spinlock_types_smp.h | linux/spinlock_types_up.h
linux/spinlock_types.h | linux/spinlock_types.h
asm/spinlock_smp.h | linux/spinlock_up.h
linux/spinlock_api_smp.h | linux/spinlock_api_up.h
linux/spinlock.h | linux/spinlock.h
/*
* here's the role of the various spinlock/rwlock related include files:
*
* on SMP builds:
*
* asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the
* initializers
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* asm/spinlock.h: contains the __raw_spin_*()/etc. lowlevel
* implementations, mostly inline assembly code
*
* (also included on UP-debug builds:)
*
* linux/spinlock_api_smp.h:
* contains the prototypes for the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*
* on UP builds:
*
* linux/spinlock_type_up.h:
* contains the generic, simplified UP spinlock type.
* (which is an empty structure on non-debug builds)
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* linux/spinlock_up.h:
* contains the __raw_spin_*()/etc. version of UP
* builds. (which are NOPs on non-debug, non-preempt
* builds)
*
* (included on UP-non-debug builds:)
*
* linux/spinlock_api_up.h:
* builds the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*/
All SMP and UP architectures are converted by this patch.
arm, i386, ia64, ppc, ppc64, s390/s390x, x64 was build-tested via
crosscompilers. m32r, mips, sh, sparc, have not been tested yet, but should
be mostly fine.
From: Grant Grundler <grundler@parisc-linux.org>
Booted and lightly tested on a500-44 (64-bit, SMP kernel, dual CPU).
Builds 32-bit SMP kernel (not booted or tested). I did not try to build
non-SMP kernels. That should be trivial to fix up later if necessary.
I converted bit ops atomic_hash lock to raw_spinlock_t. Doing so avoids
some ugly nesting of linux/*.h and asm/*.h files. Those particular locks
are well tested and contained entirely inside arch specific code. I do NOT
expect any new issues to arise with them.
If someone does ever need to use debug/metrics with them, then they will
need to unravel this hairball between spinlocks, atomic ops, and bit ops
that exist only because parisc has exactly one atomic instruction: LDCW
(load and clear word).
From: "Luck, Tony" <tony.luck@intel.com>
ia64 fix
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjanv@infradead.org>
Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
Cc: Matthew Wilcox <willy@debian.org>
Signed-off-by: Hirokazu Takata <takata@linux-m32r.org>
Signed-off-by: Mikael Pettersson <mikpe@csd.uu.se>
Signed-off-by: Benoit Boissinot <benoit.boissinot@ens-lyon.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-10 07:25:56 +00:00
|
|
|
#include <linux/bit_spinlock.h>
|
2013-01-11 21:06:36 +00:00
|
|
|
#include <trace/events/block.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
|
2015-06-02 14:39:48 +00:00
|
|
|
static int submit_bh_wbc(int rw, struct buffer_head *bh,
|
|
|
|
unsigned long bio_flags,
|
|
|
|
struct writeback_control *wbc);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
|
|
|
|
|
2012-12-12 21:52:15 +00:00
|
|
|
void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
bh->b_end_io = handler;
|
|
|
|
bh->b_private = private;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(init_buffer);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-11 21:06:35 +00:00
|
|
|
inline void touch_buffer(struct buffer_head *bh)
|
|
|
|
{
|
2013-01-11 21:06:36 +00:00
|
|
|
trace_block_touch_buffer(bh);
|
2013-01-11 21:06:35 +00:00
|
|
|
mark_page_accessed(bh->b_page);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(touch_buffer);
|
|
|
|
|
2008-02-08 12:19:52 +00:00
|
|
|
void __lock_buffer(struct buffer_head *bh)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
sched: Remove proliferation of wait_on_bit() action functions
The current "wait_on_bit" interface requires an 'action'
function to be provided which does the actual waiting.
There are over 20 such functions, many of them identical.
Most cases can be satisfied by one of just two functions, one
which uses io_schedule() and one which just uses schedule().
So:
Rename wait_on_bit and wait_on_bit_lock to
wait_on_bit_action and wait_on_bit_lock_action
to make it explicit that they need an action function.
Introduce new wait_on_bit{,_lock} and wait_on_bit{,_lock}_io
which are *not* given an action function but implicitly use
a standard one.
The decision to error-out if a signal is pending is now made
based on the 'mode' argument rather than being encoded in the action
function.
All instances of the old wait_on_bit and wait_on_bit_lock which
can use the new version have been changed accordingly and their
action functions have been discarded.
wait_on_bit{_lock} does not return any specific error code in the
event of a signal so the caller must check for non-zero and
interpolate their own error code as appropriate.
The wait_on_bit() call in __fscache_wait_on_invalidate() was
ambiguous as it specified TASK_UNINTERRUPTIBLE but used
fscache_wait_bit_interruptible as an action function.
David Howells confirms this should be uniformly
"uninterruptible"
The main remaining user of wait_on_bit{,_lock}_action is NFS
which needs to use a freezer-aware schedule() call.
A comment in fs/gfs2/glock.c notes that having multiple 'action'
functions is useful as they display differently in the 'wchan'
field of 'ps'. (and /proc/$PID/wchan).
As the new bit_wait{,_io} functions are tagged "__sched", they
will not show up at all, but something higher in the stack. So
the distinction will still be visible, only with different
function names (gds2_glock_wait versus gfs2_glock_dq_wait in the
gfs2/glock.c case).
Since first version of this patch (against 3.15) two new action
functions appeared, on in NFS and one in CIFS. CIFS also now
uses an action function that makes the same freezer aware
schedule call as NFS.
Signed-off-by: NeilBrown <neilb@suse.de>
Acked-by: David Howells <dhowells@redhat.com> (fscache, keys)
Acked-by: Steven Whitehouse <swhiteho@redhat.com> (gfs2)
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steve French <sfrench@samba.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140707051603.28027.72349.stgit@notabene.brown
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-07-07 05:16:04 +00:00
|
|
|
wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__lock_buffer);
|
|
|
|
|
2008-02-08 12:19:52 +00:00
|
|
|
void unlock_buffer(struct buffer_head *bh)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2008-10-19 03:27:00 +00:00
|
|
|
clear_bit_unlock(BH_Lock, &bh->b_state);
|
2014-03-17 17:06:10 +00:00
|
|
|
smp_mb__after_atomic();
|
2005-04-16 22:20:36 +00:00
|
|
|
wake_up_bit(&bh->b_state, BH_Lock);
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(unlock_buffer);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-07-03 22:02:05 +00:00
|
|
|
/*
|
|
|
|
* Returns if the page has dirty or writeback buffers. If all the buffers
|
|
|
|
* are unlocked and clean then the PageDirty information is stale. If
|
|
|
|
* any of the pages are locked, it is assumed they are locked for IO.
|
|
|
|
*/
|
|
|
|
void buffer_check_dirty_writeback(struct page *page,
|
|
|
|
bool *dirty, bool *writeback)
|
|
|
|
{
|
|
|
|
struct buffer_head *head, *bh;
|
|
|
|
*dirty = false;
|
|
|
|
*writeback = false;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (PageWriteback(page))
|
|
|
|
*writeback = true;
|
|
|
|
|
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (buffer_locked(bh))
|
|
|
|
*writeback = true;
|
|
|
|
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
*dirty = true;
|
|
|
|
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(buffer_check_dirty_writeback);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Block until a buffer comes unlocked. This doesn't stop it
|
|
|
|
* from becoming locked again - you have to lock it yourself
|
|
|
|
* if you want to preserve its state.
|
|
|
|
*/
|
|
|
|
void __wait_on_buffer(struct buffer_head * bh)
|
|
|
|
{
|
sched: Remove proliferation of wait_on_bit() action functions
The current "wait_on_bit" interface requires an 'action'
function to be provided which does the actual waiting.
There are over 20 such functions, many of them identical.
Most cases can be satisfied by one of just two functions, one
which uses io_schedule() and one which just uses schedule().
So:
Rename wait_on_bit and wait_on_bit_lock to
wait_on_bit_action and wait_on_bit_lock_action
to make it explicit that they need an action function.
Introduce new wait_on_bit{,_lock} and wait_on_bit{,_lock}_io
which are *not* given an action function but implicitly use
a standard one.
The decision to error-out if a signal is pending is now made
based on the 'mode' argument rather than being encoded in the action
function.
All instances of the old wait_on_bit and wait_on_bit_lock which
can use the new version have been changed accordingly and their
action functions have been discarded.
wait_on_bit{_lock} does not return any specific error code in the
event of a signal so the caller must check for non-zero and
interpolate their own error code as appropriate.
The wait_on_bit() call in __fscache_wait_on_invalidate() was
ambiguous as it specified TASK_UNINTERRUPTIBLE but used
fscache_wait_bit_interruptible as an action function.
David Howells confirms this should be uniformly
"uninterruptible"
The main remaining user of wait_on_bit{,_lock}_action is NFS
which needs to use a freezer-aware schedule() call.
A comment in fs/gfs2/glock.c notes that having multiple 'action'
functions is useful as they display differently in the 'wchan'
field of 'ps'. (and /proc/$PID/wchan).
As the new bit_wait{,_io} functions are tagged "__sched", they
will not show up at all, but something higher in the stack. So
the distinction will still be visible, only with different
function names (gds2_glock_wait versus gfs2_glock_dq_wait in the
gfs2/glock.c case).
Since first version of this patch (against 3.15) two new action
functions appeared, on in NFS and one in CIFS. CIFS also now
uses an action function that makes the same freezer aware
schedule call as NFS.
Signed-off-by: NeilBrown <neilb@suse.de>
Acked-by: David Howells <dhowells@redhat.com> (fscache, keys)
Acked-by: Steven Whitehouse <swhiteho@redhat.com> (gfs2)
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steve French <sfrench@samba.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140707051603.28027.72349.stgit@notabene.brown
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-07-07 05:16:04 +00:00
|
|
|
wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(__wait_on_buffer);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
static void
|
|
|
|
__clear_page_buffers(struct page *page)
|
|
|
|
{
|
|
|
|
ClearPagePrivate(page);
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:40 +00:00
|
|
|
set_page_private(page, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
|
fs: merge I/O error prints into one line
buffer.c uses two printk calls to print these messages:
[67353.422338] Buffer I/O error on device sdr, logical block 212868488
[67353.422338] lost page write due to I/O error on sdr
In a busy system, they may be interleaved with other prints,
losing the context for the second message. Merge them into
one line with one printk call so the prints are atomic.
Also, differentiate between async page writes, sync page writes, and
async page reads.
Also, shorten "device" to "dev" to match the block layer prints:
[67353.467906] blk_update_request: critical target error, dev sdr, sector
1707107328
Also, use %llu rather than %Lu.
Resulting prints look like:
[ 1356.437006] blk_update_request: critical target error, dev sdr, sector 1719693992
[ 1361.383522] quiet_error: 659876 callbacks suppressed
[ 1361.385816] Buffer I/O error on dev sdr, logical block 256902912, lost async page write
[ 1361.385819] Buffer I/O error on dev sdr, logical block 256903644, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-21 19:55:09 +00:00
|
|
|
static void buffer_io_error(struct buffer_head *bh, char *msg)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-21 19:55:11 +00:00
|
|
|
if (!test_bit(BH_Quiet, &bh->b_state))
|
|
|
|
printk_ratelimited(KERN_ERR
|
2015-04-13 12:31:37 +00:00
|
|
|
"Buffer I/O error on dev %pg, logical block %llu%s\n",
|
|
|
|
bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-10-16 08:24:47 +00:00
|
|
|
* End-of-IO handler helper function which does not touch the bh after
|
|
|
|
* unlocking it.
|
|
|
|
* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
|
|
|
|
* a race there is benign: unlock_buffer() only use the bh's address for
|
|
|
|
* hashing after unlocking the buffer, so it doesn't actually touch the bh
|
|
|
|
* itself.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-10-16 08:24:47 +00:00
|
|
|
static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
|
|
|
/* This happens, due to failed READA attempts. */
|
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
2007-10-16 08:24:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Default synchronous end-of-IO handler.. Just mark it up-to-date and
|
|
|
|
* unlock the buffer. This is what ll_rw_block uses too.
|
|
|
|
*/
|
|
|
|
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
__end_buffer_read_notouch(bh, uptodate);
|
2005-04-16 22:20:36 +00:00
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(end_buffer_read_sync);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-21 19:55:11 +00:00
|
|
|
buffer_io_error(bh, ", lost sync page write");
|
2005-04-16 22:20:36 +00:00
|
|
|
set_buffer_write_io_error(bh);
|
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(end_buffer_write_sync);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Various filesystems appear to want __find_get_block to be non-blocking.
|
|
|
|
* But it's the page lock which protects the buffers. To get around this,
|
|
|
|
* we get exclusion from try_to_free_buffers with the blockdev mapping's
|
|
|
|
* private_lock.
|
|
|
|
*
|
|
|
|
* Hack idea: for the blockdev mapping, i_bufferlist_lock contention
|
|
|
|
* may be quite high. This code could TryLock the page, and if that
|
|
|
|
* succeeds, there is no need to take private_lock. (But if
|
|
|
|
* private_lock is contended then so is mapping->tree_lock).
|
|
|
|
*/
|
|
|
|
static struct buffer_head *
|
2005-11-07 08:59:39 +00:00
|
|
|
__find_get_block_slow(struct block_device *bdev, sector_t block)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inode *bd_inode = bdev->bd_inode;
|
|
|
|
struct address_space *bd_mapping = bd_inode->i_mapping;
|
|
|
|
struct buffer_head *ret = NULL;
|
|
|
|
pgoff_t index;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct buffer_head *head;
|
|
|
|
struct page *page;
|
|
|
|
int all_mapped = 1;
|
|
|
|
|
|
|
|
index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
|
2014-06-04 23:10:31 +00:00
|
|
|
page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
spin_lock(&bd_mapping->private_lock);
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
goto out_unlock;
|
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
do {
|
2009-04-02 23:56:46 +00:00
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
all_mapped = 0;
|
|
|
|
else if (bh->b_blocknr == block) {
|
2005-04-16 22:20:36 +00:00
|
|
|
ret = bh;
|
|
|
|
get_bh(bh);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/* we might be here because some of the buffers on this page are
|
|
|
|
* not mapped. This is due to various races between
|
|
|
|
* file io on the block device and getblk. It gets dealt with
|
|
|
|
* elsewhere, don't buffer_error if we had some unmapped buffers
|
|
|
|
*/
|
|
|
|
if (all_mapped) {
|
|
|
|
printk("__find_get_block_slow() failed. "
|
|
|
|
"block=%llu, b_blocknr=%llu\n",
|
2006-03-26 09:38:00 +00:00
|
|
|
(unsigned long long)block,
|
|
|
|
(unsigned long long)bh->b_blocknr);
|
|
|
|
printk("b_state=0x%08lx, b_size=%zu\n",
|
|
|
|
bh->b_state, bh->b_size);
|
2015-04-13 12:31:37 +00:00
|
|
|
printk("device %pg blocksize: %d\n", bdev,
|
2011-11-01 00:09:00 +00:00
|
|
|
1 << bd_inode->i_blkbits);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
out_unlock:
|
|
|
|
spin_unlock(&bd_mapping->private_lock);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-09-23 17:37:09 +00:00
|
|
|
* Kick the writeback threads then try to free up some ZONE_NORMAL memory.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
static void free_more_memory(void)
|
|
|
|
{
|
2008-04-28 09:12:18 +00:00
|
|
|
struct zone *zone;
|
2008-04-28 09:12:14 +00:00
|
|
|
int nid;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-10-08 03:54:10 +00:00
|
|
|
wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
|
2005-04-16 22:20:36 +00:00
|
|
|
yield();
|
|
|
|
|
2008-04-28 09:12:14 +00:00
|
|
|
for_each_online_node(nid) {
|
2008-04-28 09:12:18 +00:00
|
|
|
(void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
|
|
|
|
gfp_zone(GFP_NOFS), NULL,
|
|
|
|
&zone);
|
|
|
|
if (zone)
|
2008-04-28 09:12:16 +00:00
|
|
|
try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
|
2009-03-31 22:23:31 +00:00
|
|
|
GFP_NOFS, NULL);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* I/O completion handler for block_read_full_page() - pages
|
|
|
|
* which come unlocked at the end of I/O.
|
|
|
|
*/
|
|
|
|
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-07-08 00:56:56 +00:00
|
|
|
struct buffer_head *first;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct buffer_head *tmp;
|
|
|
|
struct page *page;
|
|
|
|
int page_uptodate = 1;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_async_read(bh));
|
|
|
|
|
|
|
|
page = bh->b_page;
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
|
|
|
clear_buffer_uptodate(bh);
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-21 19:55:11 +00:00
|
|
|
buffer_io_error(bh, ", async page read");
|
2005-04-16 22:20:36 +00:00
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Be _very_ careful from here on. Bad things can happen if
|
|
|
|
* two buffer heads end IO at almost the same time and both
|
|
|
|
* decide that the page is now completely done.
|
|
|
|
*/
|
2005-07-08 00:56:56 +00:00
|
|
|
first = page_buffers(page);
|
|
|
|
local_irq_save(flags);
|
|
|
|
bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
|
2005-04-16 22:20:36 +00:00
|
|
|
clear_buffer_async_read(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh;
|
|
|
|
do {
|
|
|
|
if (!buffer_uptodate(tmp))
|
|
|
|
page_uptodate = 0;
|
|
|
|
if (buffer_async_read(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
} while (tmp != bh);
|
2005-07-08 00:56:56 +00:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If none of the buffers had errors and they are all
|
|
|
|
* uptodate then we can set the page uptodate.
|
|
|
|
*/
|
|
|
|
if (page_uptodate && !PageError(page))
|
|
|
|
SetPageUptodate(page);
|
|
|
|
unlock_page(page);
|
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2005-07-08 00:56:56 +00:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Completion handler for block_write_full_page() - pages which are unlocked
|
|
|
|
* during I/O, and which have PageWriteback cleared upon I/O completion.
|
|
|
|
*/
|
2009-04-15 17:22:38 +00:00
|
|
|
void end_buffer_async_write(struct buffer_head *bh, int uptodate)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-07-08 00:56:56 +00:00
|
|
|
struct buffer_head *first;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct buffer_head *tmp;
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_async_write(bh));
|
|
|
|
|
|
|
|
page = bh->b_page;
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-21 19:55:11 +00:00
|
|
|
buffer_io_error(bh, ", lost async page write");
|
2005-04-16 22:20:36 +00:00
|
|
|
set_bit(AS_EIO, &page->mapping->flags);
|
2006-10-17 07:10:19 +00:00
|
|
|
set_buffer_write_io_error(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
|
2005-07-08 00:56:56 +00:00
|
|
|
first = page_buffers(page);
|
|
|
|
local_irq_save(flags);
|
|
|
|
bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
clear_buffer_async_write(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh->b_this_page;
|
|
|
|
while (tmp != bh) {
|
|
|
|
if (buffer_async_write(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
}
|
2005-07-08 00:56:56 +00:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
end_page_writeback(page);
|
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2005-07-08 00:56:56 +00:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(end_buffer_async_write);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If a page's buffers are under async readin (end_buffer_async_read
|
|
|
|
* completion) then there is a possibility that another thread of
|
|
|
|
* control could lock one of the buffers after it has completed
|
|
|
|
* but while some of the other buffers have not completed. This
|
|
|
|
* locked buffer would confuse end_buffer_async_read() into not unlocking
|
|
|
|
* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
|
|
|
|
* that this buffer is not under async I/O.
|
|
|
|
*
|
|
|
|
* The page comes unlocked when it has no locked buffer_async buffers
|
|
|
|
* left.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone starting new async I/O reads any of
|
|
|
|
* the buffers.
|
|
|
|
*
|
|
|
|
* PageWriteback is used to prevent simultaneous writeout of the same
|
|
|
|
* page.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone from starting writeback of a page which is
|
|
|
|
* under read I/O (PageWriteback is only ever set against a locked page).
|
|
|
|
*/
|
|
|
|
static void mark_buffer_async_read(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
bh->b_end_io = end_buffer_async_read;
|
|
|
|
set_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
2009-09-22 23:43:51 +00:00
|
|
|
static void mark_buffer_async_write_endio(struct buffer_head *bh,
|
|
|
|
bh_end_io_t *handler)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-04-15 17:22:38 +00:00
|
|
|
bh->b_end_io = handler;
|
2005-04-16 22:20:36 +00:00
|
|
|
set_buffer_async_write(bh);
|
|
|
|
}
|
2009-04-15 17:22:38 +00:00
|
|
|
|
|
|
|
void mark_buffer_async_write(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
mark_buffer_async_write_endio(bh, end_buffer_async_write);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
EXPORT_SYMBOL(mark_buffer_async_write);
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fs/buffer.c contains helper functions for buffer-backed address space's
|
|
|
|
* fsync functions. A common requirement for buffer-based filesystems is
|
|
|
|
* that certain data from the backing blockdev needs to be written out for
|
|
|
|
* a successful fsync(). For example, ext2 indirect blocks need to be
|
|
|
|
* written back and waited upon before fsync() returns.
|
|
|
|
*
|
|
|
|
* The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
|
|
|
|
* inode_has_buffers() and invalidate_inode_buffers() are provided for the
|
|
|
|
* management of a list of dependent buffers at ->i_mapping->private_list.
|
|
|
|
*
|
|
|
|
* Locking is a little subtle: try_to_free_buffers() will remove buffers
|
|
|
|
* from their controlling inode's queue when they are being freed. But
|
|
|
|
* try_to_free_buffers() will be operating against the *blockdev* mapping
|
|
|
|
* at the time, not against the S_ISREG file which depends on those buffers.
|
|
|
|
* So the locking for private_list is via the private_lock in the address_space
|
|
|
|
* which backs the buffers. Which is different from the address_space
|
|
|
|
* against which the buffers are listed. So for a particular address_space,
|
|
|
|
* mapping->private_lock does *not* protect mapping->private_list! In fact,
|
|
|
|
* mapping->private_list will always be protected by the backing blockdev's
|
|
|
|
* ->private_lock.
|
|
|
|
*
|
|
|
|
* Which introduces a requirement: all buffers on an address_space's
|
|
|
|
* ->private_list must be from the same address_space: the blockdev's.
|
|
|
|
*
|
|
|
|
* address_spaces which do not place buffers at ->private_list via these
|
|
|
|
* utility functions are free to use private_lock and private_list for
|
|
|
|
* whatever they want. The only requirement is that list_empty(private_list)
|
|
|
|
* be true at clear_inode() time.
|
|
|
|
*
|
|
|
|
* FIXME: clear_inode should not call invalidate_inode_buffers(). The
|
|
|
|
* filesystems should do that. invalidate_inode_buffers() should just go
|
|
|
|
* BUG_ON(!list_empty).
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
|
|
|
|
* take an address_space, not an inode. And it should be called
|
|
|
|
* mark_buffer_dirty_fsync() to clearly define why those buffers are being
|
|
|
|
* queued up.
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
|
|
|
|
* list if it is already on a list. Because if the buffer is on a list,
|
|
|
|
* it *must* already be on the right one. If not, the filesystem is being
|
|
|
|
* silly. This will save a ton of locking. But first we have to ensure
|
|
|
|
* that buffers are taken *off* the old inode's list when they are freed
|
|
|
|
* (presumably in truncate). That requires careful auditing of all
|
|
|
|
* filesystems (do it inside bforget()). It could also be done by bringing
|
|
|
|
* b_inode back.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The buffer's backing address_space's private_lock must be held
|
|
|
|
*/
|
2008-07-30 05:33:47 +00:00
|
|
|
static void __remove_assoc_queue(struct buffer_head *bh)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 07:10:19 +00:00
|
|
|
WARN_ON(!bh->b_assoc_map);
|
|
|
|
if (buffer_write_io_error(bh))
|
|
|
|
set_bit(AS_EIO, &bh->b_assoc_map->flags);
|
|
|
|
bh->b_assoc_map = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int inode_has_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
return !list_empty(&inode->i_data.private_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* osync is designed to support O_SYNC io. It waits synchronously for
|
|
|
|
* all already-submitted IO to complete, but does not queue any new
|
|
|
|
* writes to the disk.
|
|
|
|
*
|
|
|
|
* To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
|
|
|
|
* you dirty the buffers, and then use osync_inode_buffers to wait for
|
|
|
|
* completion. Any other dirty buffers which are not yet queued for
|
|
|
|
* write will not be flushed to disk by the osync.
|
|
|
|
*/
|
|
|
|
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head *p;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
repeat:
|
|
|
|
list_for_each_prev(p, list) {
|
|
|
|
bh = BH_ENTRY(p);
|
|
|
|
if (buffer_locked(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock(lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2010-03-23 10:06:58 +00:00
|
|
|
static void do_thaw_one(struct super_block *sb, void *unused)
|
2009-03-31 22:23:46 +00:00
|
|
|
{
|
2010-03-23 10:06:58 +00:00
|
|
|
while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
|
2015-04-13 12:31:37 +00:00
|
|
|
printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
|
2010-03-23 10:06:58 +00:00
|
|
|
}
|
2009-03-31 22:23:46 +00:00
|
|
|
|
2010-03-23 10:06:58 +00:00
|
|
|
static void do_thaw_all(struct work_struct *work)
|
|
|
|
{
|
|
|
|
iterate_supers(do_thaw_one, NULL);
|
2009-04-08 11:44:08 +00:00
|
|
|
kfree(work);
|
2009-03-31 22:23:46 +00:00
|
|
|
printk(KERN_WARNING "Emergency Thaw complete\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* emergency_thaw_all -- forcibly thaw every frozen filesystem
|
|
|
|
*
|
|
|
|
* Used for emergency unfreeze of all filesystems via SysRq
|
|
|
|
*/
|
|
|
|
void emergency_thaw_all(void)
|
|
|
|
{
|
2009-04-08 11:44:08 +00:00
|
|
|
struct work_struct *work;
|
|
|
|
|
|
|
|
work = kmalloc(sizeof(*work), GFP_ATOMIC);
|
|
|
|
if (work) {
|
|
|
|
INIT_WORK(work, do_thaw_all);
|
|
|
|
schedule_work(work);
|
|
|
|
}
|
2009-03-31 22:23:46 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
2008-03-01 06:02:31 +00:00
|
|
|
* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
|
2005-05-01 15:59:26 +00:00
|
|
|
* @mapping: the mapping which wants those buffers written
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* Starts I/O against the buffers at mapping->private_list, and waits upon
|
|
|
|
* that I/O.
|
|
|
|
*
|
2005-05-01 15:59:26 +00:00
|
|
|
* Basically, this is a convenience function for fsync().
|
|
|
|
* @mapping is a file or directory which needs those buffers to be written for
|
|
|
|
* a successful fsync().
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
int sync_mapping_buffers(struct address_space *mapping)
|
|
|
|
{
|
2012-12-12 00:02:35 +00:00
|
|
|
struct address_space *buffer_mapping = mapping->private_data;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (buffer_mapping == NULL || list_empty(&mapping->private_list))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return fsync_buffers_list(&buffer_mapping->private_lock,
|
|
|
|
&mapping->private_list);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(sync_mapping_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Called when we've recently written block `bblock', and it is known that
|
|
|
|
* `bblock' was for a buffer_boundary() buffer. This means that the block at
|
|
|
|
* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
|
|
|
|
* dirty, schedule it for IO. So that indirects merge nicely with their data.
|
|
|
|
*/
|
|
|
|
void write_boundary_block(struct block_device *bdev,
|
|
|
|
sector_t bblock, unsigned blocksize)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
|
|
|
|
if (bh) {
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
ll_rw_block(WRITE, 1, &bh);
|
|
|
|
put_bh(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
|
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
struct address_space *buffer_mapping = bh->b_page->mapping;
|
|
|
|
|
|
|
|
mark_buffer_dirty(bh);
|
2012-12-12 00:02:35 +00:00
|
|
|
if (!mapping->private_data) {
|
|
|
|
mapping->private_data = buffer_mapping;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2012-12-12 00:02:35 +00:00
|
|
|
BUG_ON(mapping->private_data != buffer_mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2008-02-08 12:21:59 +00:00
|
|
|
if (!bh->b_assoc_map) {
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
list_move_tail(&bh->b_assoc_buffers,
|
|
|
|
&mapping->private_list);
|
2006-10-17 07:10:19 +00:00
|
|
|
bh->b_assoc_map = mapping;
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty_inode);
|
|
|
|
|
2007-07-17 11:03:34 +00:00
|
|
|
/*
|
|
|
|
* Mark the page dirty, and set it dirty in the radix tree, and mark the inode
|
|
|
|
* dirty.
|
|
|
|
*
|
|
|
|
* If warn is true, then emit a warning if the page is not uptodate and has
|
|
|
|
* not been truncated.
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
*
|
|
|
|
* The caller must hold mem_cgroup_begin_page_stat() lock.
|
2007-07-17 11:03:34 +00:00
|
|
|
*/
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
static void __set_page_dirty(struct page *page, struct address_space *mapping,
|
|
|
|
struct mem_cgroup *memcg, int warn)
|
2007-07-17 11:03:34 +00:00
|
|
|
{
|
2014-02-06 20:04:28 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&mapping->tree_lock, flags);
|
2007-07-17 11:03:34 +00:00
|
|
|
if (page->mapping) { /* Race with truncate? */
|
|
|
|
WARN_ON_ONCE(warn && !PageUptodate(page));
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
account_page_dirtied(page, mapping, memcg);
|
2007-07-17 11:03:34 +00:00
|
|
|
radix_tree_tag_set(&mapping->page_tree,
|
|
|
|
page_index(page), PAGECACHE_TAG_DIRTY);
|
|
|
|
}
|
2014-02-06 20:04:28 +00:00
|
|
|
spin_unlock_irqrestore(&mapping->tree_lock, flags);
|
2007-07-17 11:03:34 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Add a page to the dirty page list.
|
|
|
|
*
|
|
|
|
* It is a sad fact of life that this function is called from several places
|
|
|
|
* deeply under spinlocking. It may not sleep.
|
|
|
|
*
|
|
|
|
* If the page has buffers, the uptodate buffers are set dirty, to preserve
|
|
|
|
* dirty-state coherency between the page and the buffers. It the page does
|
|
|
|
* not have buffers then when they are later attached they will all be set
|
|
|
|
* dirty.
|
|
|
|
*
|
|
|
|
* The buffers are dirtied before the page is dirtied. There's a small race
|
|
|
|
* window in which a writepage caller may see the page cleanness but not the
|
|
|
|
* buffer dirtiness. That's fine. If this code were to set the page dirty
|
|
|
|
* before the buffers, a concurrent writepage caller could clear the page dirty
|
|
|
|
* bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
|
|
|
|
* page on the dirty page list.
|
|
|
|
*
|
|
|
|
* We use private_lock to lock against try_to_free_buffers while using the
|
|
|
|
* page's buffer list. Also use this to protect against clean buffers being
|
|
|
|
* added to the page after it was set dirty.
|
|
|
|
*
|
|
|
|
* FIXME: may need to call ->reservepage here as well. That's rather up to the
|
|
|
|
* address_space though.
|
|
|
|
*/
|
|
|
|
int __set_page_dirty_buffers(struct page *page)
|
|
|
|
{
|
2009-03-19 18:32:05 +00:00
|
|
|
int newly_dirty;
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
struct mem_cgroup *memcg;
|
2007-07-17 11:03:34 +00:00
|
|
|
struct address_space *mapping = page_mapping(page);
|
2006-10-10 02:36:54 +00:00
|
|
|
|
|
|
|
if (unlikely(!mapping))
|
|
|
|
return !TestSetPageDirty(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock(&mapping->private_lock);
|
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh = head;
|
|
|
|
|
|
|
|
do {
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
/*
|
|
|
|
* Use mem_group_begin_page_stat() to keep PageDirty synchronized with
|
|
|
|
* per-memcg dirty page counters.
|
|
|
|
*/
|
|
|
|
memcg = mem_cgroup_begin_page_stat(page);
|
2009-03-19 18:32:05 +00:00
|
|
|
newly_dirty = !TestSetPageDirty(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&mapping->private_lock);
|
|
|
|
|
2009-03-19 18:32:05 +00:00
|
|
|
if (newly_dirty)
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
__set_page_dirty(page, mapping, memcg, 1);
|
|
|
|
|
|
|
|
mem_cgroup_end_page_stat(memcg);
|
|
|
|
|
|
|
|
if (newly_dirty)
|
|
|
|
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
|
|
|
|
|
2009-03-19 18:32:05 +00:00
|
|
|
return newly_dirty;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__set_page_dirty_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write out and wait upon a list of buffers.
|
|
|
|
*
|
|
|
|
* We have conflicting pressures: we want to make sure that all
|
|
|
|
* initially dirty buffers get waited on, but that any subsequently
|
|
|
|
* dirtied buffers don't. After all, we don't want fsync to last
|
|
|
|
* forever if somebody is actively writing to the file.
|
|
|
|
*
|
|
|
|
* Do this in two main stages: first we copy dirty buffers to a
|
|
|
|
* temporary inode list, queueing the writes as we go. Then we clean
|
|
|
|
* up, waiting for those writes to complete.
|
|
|
|
*
|
|
|
|
* During this second stage, any subsequent updates to the file may end
|
|
|
|
* up refiling the buffer on the original inode's dirty list again, so
|
|
|
|
* there is a chance we will end up with a buffer queued for write but
|
|
|
|
* not yet completed on that list. So, as a final cleanup we go through
|
|
|
|
* the osync code to catch these locked, dirty buffers without requeuing
|
|
|
|
* any newly dirty buffers for write.
|
|
|
|
*/
|
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head tmp;
|
2011-03-10 07:52:07 +00:00
|
|
|
struct address_space *mapping;
|
2005-04-16 22:20:36 +00:00
|
|
|
int err = 0, err2;
|
2011-03-17 09:51:40 +00:00
|
|
|
struct blk_plug plug;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
INIT_LIST_HEAD(&tmp);
|
2011-03-17 09:51:40 +00:00
|
|
|
blk_start_plug(&plug);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
while (!list_empty(list)) {
|
|
|
|
bh = BH_ENTRY(list->next);
|
2008-02-08 12:21:59 +00:00
|
|
|
mapping = bh->b_assoc_map;
|
2006-10-17 07:10:19 +00:00
|
|
|
__remove_assoc_queue(bh);
|
2008-02-08 12:21:59 +00:00
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
2005-04-16 22:20:36 +00:00
|
|
|
if (buffer_dirty(bh) || buffer_locked(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers, &tmp);
|
2008-02-08 12:21:59 +00:00
|
|
|
bh->b_assoc_map = mapping;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
/*
|
|
|
|
* Ensure any pending I/O completes so that
|
2010-08-11 15:06:24 +00:00
|
|
|
* write_dirty_buffer() actually writes the
|
|
|
|
* current contents - it is a noop if I/O is
|
|
|
|
* still in flight on potentially older
|
|
|
|
* contents.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2011-03-09 10:56:30 +00:00
|
|
|
write_dirty_buffer(bh, WRITE_SYNC);
|
2009-04-06 12:48:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Kick off IO for the previous mapping. Note
|
|
|
|
* that we will not run the very last mapping,
|
|
|
|
* wait_on_buffer() will do that for us
|
|
|
|
* through sync_buffer().
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-17 09:51:40 +00:00
|
|
|
spin_unlock(lock);
|
|
|
|
blk_finish_plug(&plug);
|
|
|
|
spin_lock(lock);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
while (!list_empty(&tmp)) {
|
|
|
|
bh = BH_ENTRY(tmp.prev);
|
|
|
|
get_bh(bh);
|
2008-02-08 12:21:59 +00:00
|
|
|
mapping = bh->b_assoc_map;
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers,
|
2008-03-04 22:28:33 +00:00
|
|
|
&mapping->private_list);
|
2008-02-08 12:21:59 +00:00
|
|
|
bh->b_assoc_map = mapping;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
err2 = osync_buffers_list(lock, list);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
else
|
|
|
|
return err2;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Invalidate any and all dirty buffers on a given inode. We are
|
|
|
|
* probably unmounting the fs, but that doesn't mean we have already
|
|
|
|
* done a sync(). Just drop the buffers from the inode list.
|
|
|
|
*
|
|
|
|
* NOTE: we take the inode's blockdev's mapping's private_lock. Which
|
|
|
|
* assumes that all the buffers are against the blockdev. Not true
|
|
|
|
* for reiserfs.
|
|
|
|
*/
|
|
|
|
void invalidate_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
|
|
|
struct list_head *list = &mapping->private_list;
|
2012-12-12 00:02:35 +00:00
|
|
|
struct address_space *buffer_mapping = mapping->private_data;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
while (!list_empty(list))
|
|
|
|
__remove_assoc_queue(BH_ENTRY(list->next));
|
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
}
|
2008-09-23 16:24:08 +00:00
|
|
|
EXPORT_SYMBOL(invalidate_inode_buffers);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove any clean buffers from the inode's buffer list. This is called
|
|
|
|
* when we're trying to free the inode itself. Those buffers can pin it.
|
|
|
|
*
|
|
|
|
* Returns true if all buffers were removed.
|
|
|
|
*/
|
|
|
|
int remove_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
|
|
|
struct list_head *list = &mapping->private_list;
|
2012-12-12 00:02:35 +00:00
|
|
|
struct address_space *buffer_mapping = mapping->private_data;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
while (!list_empty(list)) {
|
|
|
|
struct buffer_head *bh = BH_ENTRY(list->next);
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
}
|
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create the appropriate buffers when given a page for data area and
|
|
|
|
* the size of each buffer.. Use the bh->b_this_page linked list to
|
|
|
|
* follow the buffers created. Return NULL if unable to create more
|
|
|
|
* buffers.
|
|
|
|
*
|
|
|
|
* The retry flag is used to differentiate async IO (paging, swapping)
|
|
|
|
* which may not fail from ordinary buffer allocations.
|
|
|
|
*/
|
|
|
|
struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
|
|
|
|
int retry)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
long offset;
|
|
|
|
|
|
|
|
try_again:
|
|
|
|
head = NULL;
|
|
|
|
offset = PAGE_SIZE;
|
|
|
|
while ((offset -= size) >= 0) {
|
|
|
|
bh = alloc_buffer_head(GFP_NOFS);
|
|
|
|
if (!bh)
|
|
|
|
goto no_grow;
|
|
|
|
|
|
|
|
bh->b_this_page = head;
|
|
|
|
bh->b_blocknr = -1;
|
|
|
|
head = bh;
|
|
|
|
|
|
|
|
bh->b_size = size;
|
|
|
|
|
|
|
|
/* Link the buffer to its page */
|
|
|
|
set_bh_page(bh, page, offset);
|
|
|
|
}
|
|
|
|
return head;
|
|
|
|
/*
|
|
|
|
* In case anything failed, we just free everything we got.
|
|
|
|
*/
|
|
|
|
no_grow:
|
|
|
|
if (head) {
|
|
|
|
do {
|
|
|
|
bh = head;
|
|
|
|
head = head->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
} while (head);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return failure for non-async IO requests. Async IO requests
|
|
|
|
* are not allowed to fail, so we have to wait until buffer heads
|
|
|
|
* become available. But we don't want tasks sleeping with
|
|
|
|
* partially complete buffers, so all were released above.
|
|
|
|
*/
|
|
|
|
if (!retry)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* We're _really_ low on memory. Now we just
|
|
|
|
* wait for old buffer heads to become free due to
|
|
|
|
* finishing IO. Since this is an async request and
|
|
|
|
* the reserve list is empty, we're sure there are
|
|
|
|
* async buffer heads in use.
|
|
|
|
*/
|
|
|
|
free_more_memory();
|
|
|
|
goto try_again;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(alloc_page_buffers);
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
link_dev_buffers(struct page *page, struct buffer_head *head)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *tail;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
}
|
|
|
|
|
2012-11-29 20:31:52 +00:00
|
|
|
static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
|
|
|
|
{
|
|
|
|
sector_t retval = ~((sector_t)0);
|
|
|
|
loff_t sz = i_size_read(bdev->bd_inode);
|
|
|
|
|
|
|
|
if (sz) {
|
|
|
|
unsigned int sizebits = blksize_bits(size);
|
|
|
|
retval = (sz >> sizebits);
|
|
|
|
}
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Initialise the state of a blockdev page's buffers.
|
|
|
|
*/
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
static sector_t
|
2005-04-16 22:20:36 +00:00
|
|
|
init_page_buffers(struct page *page, struct block_device *bdev,
|
|
|
|
sector_t block, int size)
|
|
|
|
{
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh = head;
|
|
|
|
int uptodate = PageUptodate(page);
|
2012-11-29 20:31:52 +00:00
|
|
|
sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh)) {
|
|
|
|
init_buffer(bh, NULL, NULL);
|
|
|
|
bh->b_bdev = bdev;
|
|
|
|
bh->b_blocknr = block;
|
|
|
|
if (uptodate)
|
|
|
|
set_buffer_uptodate(bh);
|
block: don't mark buffers beyond end of disk as mapped
Hi,
We have a bug report open where a squashfs image mounted on ppc64 would
exhibit errors due to trying to read beyond the end of the disk. It can
easily be reproduced by doing the following:
[root@ibm-p750e-02-lp3 ~]# ls -l install.img
-rw-r--r-- 1 root root 142032896 Apr 30 16:46 install.img
[root@ibm-p750e-02-lp3 ~]# mount -o loop ./install.img /mnt/test
[root@ibm-p750e-02-lp3 ~]# dd if=/dev/loop0 of=/dev/null
dd: reading `/dev/loop0': Input/output error
277376+0 records in
277376+0 records out
142016512 bytes (142 MB) copied, 0.9465 s, 150 MB/s
In dmesg, you'll find the following:
squashfs: version 4.0 (2009/01/31) Phillip Lougher
[ 43.106012] attempt to access beyond end of device
[ 43.106029] loop0: rw=0, want=277410, limit=277408
[ 43.106039] Buffer I/O error on device loop0, logical block 138704
[ 43.106053] attempt to access beyond end of device
[ 43.106057] loop0: rw=0, want=277412, limit=277408
[ 43.106061] Buffer I/O error on device loop0, logical block 138705
[ 43.106066] attempt to access beyond end of device
[ 43.106070] loop0: rw=0, want=277414, limit=277408
[ 43.106073] Buffer I/O error on device loop0, logical block 138706
[ 43.106078] attempt to access beyond end of device
[ 43.106081] loop0: rw=0, want=277416, limit=277408
[ 43.106085] Buffer I/O error on device loop0, logical block 138707
[ 43.106089] attempt to access beyond end of device
[ 43.106093] loop0: rw=0, want=277418, limit=277408
[ 43.106096] Buffer I/O error on device loop0, logical block 138708
[ 43.106101] attempt to access beyond end of device
[ 43.106104] loop0: rw=0, want=277420, limit=277408
[ 43.106108] Buffer I/O error on device loop0, logical block 138709
[ 43.106112] attempt to access beyond end of device
[ 43.106116] loop0: rw=0, want=277422, limit=277408
[ 43.106120] Buffer I/O error on device loop0, logical block 138710
[ 43.106124] attempt to access beyond end of device
[ 43.106128] loop0: rw=0, want=277424, limit=277408
[ 43.106131] Buffer I/O error on device loop0, logical block 138711
[ 43.106135] attempt to access beyond end of device
[ 43.106139] loop0: rw=0, want=277426, limit=277408
[ 43.106143] Buffer I/O error on device loop0, logical block 138712
[ 43.106147] attempt to access beyond end of device
[ 43.106151] loop0: rw=0, want=277428, limit=277408
[ 43.106154] Buffer I/O error on device loop0, logical block 138713
[ 43.106158] attempt to access beyond end of device
[ 43.106162] loop0: rw=0, want=277430, limit=277408
[ 43.106166] attempt to access beyond end of device
[ 43.106169] loop0: rw=0, want=277432, limit=277408
...
[ 43.106307] attempt to access beyond end of device
[ 43.106311] loop0: rw=0, want=277470, limit=2774
Squashfs manages to read in the end block(s) of the disk during the
mount operation. Then, when dd reads the block device, it leads to
block_read_full_page being called with buffers that are beyond end of
disk, but are marked as mapped. Thus, it would end up submitting read
I/O against them, resulting in the errors mentioned above. I fixed the
problem by modifying init_page_buffers to only set the buffer mapped if
it fell inside of i_size.
Cheers,
Jeff
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
Acked-by: Nick Piggin <npiggin@kernel.dk>
--
Changes from v1->v2: re-used max_block, as suggested by Nick Piggin.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-05-11 14:34:10 +00:00
|
|
|
if (block < end_block)
|
|
|
|
set_buffer_mapped(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
block++;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Caller needs to validate requested block against end of device.
|
|
|
|
*/
|
|
|
|
return end_block;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create the page-cache page that contains the requested block.
|
|
|
|
*
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
* This is used purely for blockdev mappings.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
static int
|
2005-04-16 22:20:36 +00:00
|
|
|
grow_dev_page(struct block_device *bdev, sector_t block,
|
2014-09-05 02:04:42 +00:00
|
|
|
pgoff_t index, int size, int sizebits, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inode *inode = bdev->bd_inode;
|
|
|
|
struct page *page;
|
|
|
|
struct buffer_head *bh;
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
sector_t end_block;
|
|
|
|
int ret = 0; /* Will call free_more_memory() */
|
2013-10-16 20:47:00 +00:00
|
|
|
gfp_t gfp_mask;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-11-07 00:28:49 +00:00
|
|
|
gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
|
2014-09-05 02:04:42 +00:00
|
|
|
|
2013-10-16 20:47:00 +00:00
|
|
|
/*
|
|
|
|
* XXX: __getblk_slow() can not really deal with failure and
|
|
|
|
* will endlessly loop on improvised global reclaim. Prefer
|
|
|
|
* looping in the allocator rather than here, at least that
|
|
|
|
* code knows what it's doing.
|
|
|
|
*/
|
|
|
|
gfp_mask |= __GFP_NOFAIL;
|
|
|
|
|
|
|
|
page = find_or_create_page(inode->i_mapping, index, gfp_mask);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!page)
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-26 16:24:46 +00:00
|
|
|
BUG_ON(!PageLocked(page));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
bh = page_buffers(page);
|
|
|
|
if (bh->b_size == size) {
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
end_block = init_page_buffers(page, bdev,
|
2014-09-22 00:53:03 +00:00
|
|
|
(sector_t)index << sizebits,
|
|
|
|
size);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
goto done;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
if (!try_to_free_buffers(page))
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate some buffers for this page
|
|
|
|
*/
|
|
|
|
bh = alloc_page_buffers(page, size, 0);
|
|
|
|
if (!bh)
|
|
|
|
goto failed;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Link the page to the buffers and initialise them. Take the
|
|
|
|
* lock to be atomic wrt __find_get_block(), which does not
|
|
|
|
* run under the page lock.
|
|
|
|
*/
|
|
|
|
spin_lock(&inode->i_mapping->private_lock);
|
|
|
|
link_dev_buffers(page, bh);
|
2014-09-22 00:53:03 +00:00
|
|
|
end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
|
|
|
|
size);
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&inode->i_mapping->private_lock);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
done:
|
|
|
|
ret = (block < end_block) ? 1 : -ENXIO;
|
2005-04-16 22:20:36 +00:00
|
|
|
failed:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create buffers for the specified block device block's page. If
|
|
|
|
* that page was dirty, the buffers are set dirty also.
|
|
|
|
*/
|
2006-01-14 21:20:43 +00:00
|
|
|
static int
|
2014-09-05 02:04:42 +00:00
|
|
|
grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
pgoff_t index;
|
|
|
|
int sizebits;
|
|
|
|
|
|
|
|
sizebits = -1;
|
|
|
|
do {
|
|
|
|
sizebits++;
|
|
|
|
} while ((size << sizebits) < PAGE_SIZE);
|
|
|
|
|
|
|
|
index = block >> sizebits;
|
|
|
|
|
2006-10-11 08:21:46 +00:00
|
|
|
/*
|
|
|
|
* Check for a block which wants to lie outside our maximum possible
|
|
|
|
* pagecache index. (this comparison is done using sector_t types).
|
|
|
|
*/
|
|
|
|
if (unlikely(index != block >> sizebits)) {
|
|
|
|
printk(KERN_ERR "%s: requested out-of-range block %llu for "
|
2015-04-13 12:31:37 +00:00
|
|
|
"device %pg\n",
|
2008-04-30 07:55:09 +00:00
|
|
|
__func__, (unsigned long long)block,
|
2015-04-13 12:31:37 +00:00
|
|
|
bdev);
|
2006-10-11 08:21:46 +00:00
|
|
|
return -EIO;
|
|
|
|
}
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Create a page with the proper size buffers.. */
|
2014-09-05 02:04:42 +00:00
|
|
|
return grow_dev_page(bdev, block, index, size, sizebits, gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2014-09-05 02:04:42 +00:00
|
|
|
struct buffer_head *
|
|
|
|
__getblk_slow(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
/* Size must be multiple of hard sectorsize */
|
2009-05-22 21:17:49 +00:00
|
|
|
if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
|
2005-04-16 22:20:36 +00:00
|
|
|
(size < 512 || size > PAGE_SIZE))) {
|
|
|
|
printk(KERN_ERR "getblk(): invalid block size %d requested\n",
|
|
|
|
size);
|
2009-05-22 21:17:49 +00:00
|
|
|
printk(KERN_ERR "logical block size: %d\n",
|
|
|
|
bdev_logical_block_size(bdev));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
dump_stack();
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
for (;;) {
|
|
|
|
struct buffer_head *bh;
|
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
bh = __find_get_block(bdev, block, size);
|
|
|
|
if (bh)
|
|
|
|
return bh;
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
|
2014-09-05 02:04:42 +00:00
|
|
|
ret = grow_buffers(bdev, block, size, gfp);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 10:17:36 +00:00
|
|
|
if (ret < 0)
|
|
|
|
return NULL;
|
|
|
|
if (ret == 0)
|
|
|
|
free_more_memory();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
2014-09-05 02:04:42 +00:00
|
|
|
EXPORT_SYMBOL(__getblk_slow);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The relationship between dirty buffers and dirty pages:
|
|
|
|
*
|
|
|
|
* Whenever a page has any dirty buffers, the page's dirty bit is set, and
|
|
|
|
* the page is tagged dirty in its radix tree.
|
|
|
|
*
|
|
|
|
* At all times, the dirtiness of the buffers represents the dirtiness of
|
|
|
|
* subsections of the page. If the page has buffers, the page dirty bit is
|
|
|
|
* merely a hint about the true dirty state.
|
|
|
|
*
|
|
|
|
* When a page is set dirty in its entirety, all its buffers are marked dirty
|
|
|
|
* (if the page has buffers).
|
|
|
|
*
|
|
|
|
* When a buffer is marked dirty, its page is dirtied, but the page's other
|
|
|
|
* buffers are not.
|
|
|
|
*
|
|
|
|
* Also. When blockdev buffers are explicitly read with bread(), they
|
|
|
|
* individually become uptodate. But their backing page remains not
|
|
|
|
* uptodate - even if all of its buffers are uptodate. A subsequent
|
|
|
|
* block_read_full_page() against that page will discover all the uptodate
|
|
|
|
* buffers, will set the page uptodate and will perform no I/O.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mark_buffer_dirty - mark a buffer_head as needing writeout
|
2005-05-01 15:59:26 +00:00
|
|
|
* @bh: the buffer_head to mark dirty
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* mark_buffer_dirty() will set the dirty bit against the buffer, then set its
|
|
|
|
* backing page dirty, then tag the page as dirty in its address_space's radix
|
|
|
|
* tree and then attach the address_space's inode to its superblock's dirty
|
|
|
|
* inode list.
|
|
|
|
*
|
|
|
|
* mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
|
2011-03-22 11:23:36 +00:00
|
|
|
* mapping->tree_lock and mapping->host->i_lock.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2008-02-08 12:19:52 +00:00
|
|
|
void mark_buffer_dirty(struct buffer_head *bh)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2007-07-17 11:03:34 +00:00
|
|
|
WARN_ON_ONCE(!buffer_uptodate(bh));
|
2008-04-04 21:38:17 +00:00
|
|
|
|
2013-01-11 21:06:36 +00:00
|
|
|
trace_block_dirty_buffer(bh);
|
|
|
|
|
2008-04-04 21:38:17 +00:00
|
|
|
/*
|
|
|
|
* Very *carefully* optimize the it-is-already-dirty case.
|
|
|
|
*
|
|
|
|
* Don't let the final "is it dirty" escape to before we
|
|
|
|
* perhaps modified the buffer.
|
|
|
|
*/
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-03-19 18:32:05 +00:00
|
|
|
if (!test_set_buffer_dirty(bh)) {
|
|
|
|
struct page *page = bh->b_page;
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
struct address_space *mapping = NULL;
|
|
|
|
struct mem_cgroup *memcg;
|
|
|
|
|
|
|
|
memcg = mem_cgroup_begin_page_stat(page);
|
2009-08-22 00:40:08 +00:00
|
|
|
if (!TestSetPageDirty(page)) {
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
mapping = page_mapping(page);
|
2009-08-22 00:40:08 +00:00
|
|
|
if (mapping)
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
__set_page_dirty(page, mapping, memcg, 0);
|
2009-08-22 00:40:08 +00:00
|
|
|
}
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-22 21:13:16 +00:00
|
|
|
mem_cgroup_end_page_stat(memcg);
|
|
|
|
if (mapping)
|
|
|
|
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
|
2009-03-19 18:32:05 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Decrement a buffer_head's reference count. If all buffers against a page
|
|
|
|
* have zero reference count, are clean and unlocked, and if the page is clean
|
|
|
|
* and unlocked then try_to_free_buffers() may strip the buffers from the page
|
|
|
|
* in preparation for freeing it (sometimes, rarely, buffers are removed from
|
|
|
|
* a page but it ends up not being freed, and buffers may later be reattached).
|
|
|
|
*/
|
|
|
|
void __brelse(struct buffer_head * buf)
|
|
|
|
{
|
|
|
|
if (atomic_read(&buf->b_count)) {
|
|
|
|
put_bh(buf);
|
|
|
|
return;
|
|
|
|
}
|
2008-07-26 02:45:40 +00:00
|
|
|
WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(__brelse);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* bforget() is like brelse(), except it discards any
|
|
|
|
* potentially dirty data.
|
|
|
|
*/
|
|
|
|
void __bforget(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
clear_buffer_dirty(bh);
|
2008-02-08 12:21:59 +00:00
|
|
|
if (bh->b_assoc_map) {
|
2005-04-16 22:20:36 +00:00
|
|
|
struct address_space *buffer_mapping = bh->b_page->mapping;
|
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 07:10:19 +00:00
|
|
|
bh->b_assoc_map = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
__brelse(bh);
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(__bforget);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
static struct buffer_head *__bread_slow(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return bh;
|
|
|
|
} else {
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
brelse(bh);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
|
|
|
|
* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
|
|
|
|
* refcount elevated by one when they're in an LRU. A buffer can only appear
|
|
|
|
* once in a particular CPU's LRU. A single buffer can be present in multiple
|
|
|
|
* CPU's LRUs at the same time.
|
|
|
|
*
|
|
|
|
* This is a transparent caching front-end to sb_bread(), sb_getblk() and
|
|
|
|
* sb_find_get_block().
|
|
|
|
*
|
|
|
|
* The LRUs themselves only need locking against invalidate_bh_lrus. We use
|
|
|
|
* a local interrupt disable for that.
|
|
|
|
*/
|
|
|
|
|
2014-10-09 22:29:38 +00:00
|
|
|
#define BH_LRU_SIZE 16
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
struct bh_lru {
|
|
|
|
struct buffer_head *bhs[BH_LRU_SIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define bh_lru_lock() local_irq_disable()
|
|
|
|
#define bh_lru_unlock() local_irq_enable()
|
|
|
|
#else
|
|
|
|
#define bh_lru_lock() preempt_disable()
|
|
|
|
#define bh_lru_unlock() preempt_enable()
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline void check_irqs_on(void)
|
|
|
|
{
|
|
|
|
#ifdef irqs_disabled
|
|
|
|
BUG_ON(irqs_disabled());
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The LRU management algorithm is dopey-but-simple. Sorry.
|
|
|
|
*/
|
|
|
|
static void bh_lru_install(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
struct buffer_head *evictee = NULL;
|
|
|
|
|
|
|
|
check_irqs_on();
|
|
|
|
bh_lru_lock();
|
2010-12-06 17:16:28 +00:00
|
|
|
if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
|
2005-04-16 22:20:36 +00:00
|
|
|
struct buffer_head *bhs[BH_LRU_SIZE];
|
|
|
|
int in;
|
|
|
|
int out = 0;
|
|
|
|
|
|
|
|
get_bh(bh);
|
|
|
|
bhs[out++] = bh;
|
|
|
|
for (in = 0; in < BH_LRU_SIZE; in++) {
|
2010-12-06 17:16:28 +00:00
|
|
|
struct buffer_head *bh2 =
|
|
|
|
__this_cpu_read(bh_lrus.bhs[in]);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (bh2 == bh) {
|
|
|
|
__brelse(bh2);
|
|
|
|
} else {
|
|
|
|
if (out >= BH_LRU_SIZE) {
|
|
|
|
BUG_ON(evictee != NULL);
|
|
|
|
evictee = bh2;
|
|
|
|
} else {
|
|
|
|
bhs[out++] = bh2;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
while (out < BH_LRU_SIZE)
|
|
|
|
bhs[out++] = NULL;
|
2013-12-03 23:32:53 +00:00
|
|
|
memcpy(this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
bh_lru_unlock();
|
|
|
|
|
|
|
|
if (evictee)
|
|
|
|
__brelse(evictee);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Look up the bh in this cpu's LRU. If it's there, move it to the head.
|
|
|
|
*/
|
2006-01-14 21:20:43 +00:00
|
|
|
static struct buffer_head *
|
2007-02-12 08:52:14 +00:00
|
|
|
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *ret = NULL;
|
2007-02-12 08:52:14 +00:00
|
|
|
unsigned int i;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
check_irqs_on();
|
|
|
|
bh_lru_lock();
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
2010-12-06 17:16:28 +00:00
|
|
|
struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
fs: check bh blocknr earlier when searching lru
It's very common for the buffer heads in the lru to have different block
numbers. By comparing the blocknr before the bdev and size we can
reduce the cost of searching in the very common case where all the
entries have the same bdev and size.
In quick hot cache cycle counting tests on a single fs workstation this
cut the cost of a miss by about 20%.
A diff of the disassembly shows the reordering of the bdev and blocknr
comparisons. This is in such a tiny loop that skipping one comparison
is a meaningful portion of the total work being done:
1628: 83 c1 01 add $0x1,%ecx
162b: 83 f9 08 cmp $0x8,%ecx
162e: 74 60 je 1690 <__find_get_block+0xa0>
1630: 89 c8 mov %ecx,%eax
1632: 65 4c 8b 04 c5 00 00 mov %gs:0x0(,%rax,8),%r8
1639: 00 00
163b: 4d 85 c0 test %r8,%r8
163e: 4c 89 c3 mov %r8,%rbx
1641: 74 e5 je 1628 <__find_get_block+0x38>
- 1643: 4d 3b 68 30 cmp 0x30(%r8),%r13
+ 1643: 4d 3b 68 18 cmp 0x18(%r8),%r13
1647: 75 df jne 1628 <__find_get_block+0x38>
- 1649: 4d 3b 60 18 cmp 0x18(%r8),%r12
+ 1649: 4d 3b 60 30 cmp 0x30(%r8),%r12
164d: 75 d9 jne 1628 <__find_get_block+0x38>
164f: 49 39 50 20 cmp %rdx,0x20(%r8)
1653: 75 d3 jne 1628 <__find_get_block+0x38>
Signed-off-by: Zach Brown <zab@zabbo.net>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:05 +00:00
|
|
|
if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
|
|
|
|
bh->b_size == size) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (i) {
|
|
|
|
while (i) {
|
2010-12-06 17:16:28 +00:00
|
|
|
__this_cpu_write(bh_lrus.bhs[i],
|
|
|
|
__this_cpu_read(bh_lrus.bhs[i - 1]));
|
2005-04-16 22:20:36 +00:00
|
|
|
i--;
|
|
|
|
}
|
2010-12-06 17:16:28 +00:00
|
|
|
__this_cpu_write(bh_lrus.bhs[0], bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
get_bh(bh);
|
|
|
|
ret = bh;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
bh_lru_unlock();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform a pagecache lookup for the matching buffer. If it's there, refresh
|
|
|
|
* it in the LRU and mark it as accessed. If it is not present then return
|
|
|
|
* NULL
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2007-02-12 08:52:14 +00:00
|
|
|
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
|
|
|
|
|
|
|
|
if (bh == NULL) {
|
2014-06-04 23:10:31 +00:00
|
|
|
/* __find_get_block_slow will mark the page accessed */
|
2005-11-07 08:59:39 +00:00
|
|
|
bh = __find_get_block_slow(bdev, block);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (bh)
|
|
|
|
bh_lru_install(bh);
|
2014-06-04 23:10:31 +00:00
|
|
|
} else
|
2005-04-16 22:20:36 +00:00
|
|
|
touch_buffer(bh);
|
2014-06-04 23:10:31 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__find_get_block);
|
|
|
|
|
|
|
|
/*
|
2014-09-05 02:04:42 +00:00
|
|
|
* __getblk_gfp() will locate (and, if necessary, create) the buffer_head
|
2005-04-16 22:20:36 +00:00
|
|
|
* which corresponds to the passed block_device, block and size. The
|
|
|
|
* returned buffer has its reference count incremented.
|
|
|
|
*
|
2014-09-05 02:04:42 +00:00
|
|
|
* __getblk_gfp() will lock up the machine if grow_dev_page's
|
|
|
|
* try_to_free_buffers() attempt is failing. FIXME, perhaps?
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2014-09-05 02:04:42 +00:00
|
|
|
__getblk_gfp(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, block, size);
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
if (bh == NULL)
|
2014-09-05 02:04:42 +00:00
|
|
|
bh = __getblk_slow(bdev, block, size, gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
return bh;
|
|
|
|
}
|
2014-09-05 02:04:42 +00:00
|
|
|
EXPORT_SYMBOL(__getblk_gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Do async read-ahead on a buffer..
|
|
|
|
*/
|
2007-02-12 08:52:14 +00:00
|
|
|
void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = __getblk(bdev, block, size);
|
2005-10-30 23:03:15 +00:00
|
|
|
if (likely(bh)) {
|
|
|
|
ll_rw_block(READA, 1, &bh);
|
|
|
|
brelse(bh);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__breadahead);
|
|
|
|
|
|
|
|
/**
|
2014-09-05 02:04:42 +00:00
|
|
|
* __bread_gfp() - reads a specified block and returns the bh
|
2005-05-01 15:59:26 +00:00
|
|
|
* @bdev: the block_device to read from
|
2005-04-16 22:20:36 +00:00
|
|
|
* @block: number of block
|
|
|
|
* @size: size (in bytes) to read
|
2014-09-05 02:04:42 +00:00
|
|
|
* @gfp: page allocation flag
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Reads a specified block, and returns buffer head that contains it.
|
2014-09-05 02:04:42 +00:00
|
|
|
* The page cache can be allocated from non-movable area
|
|
|
|
* not to prevent page migration if you set gfp to zero.
|
2005-04-16 22:20:36 +00:00
|
|
|
* It returns NULL if the block was unreadable.
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2014-09-05 02:04:42 +00:00
|
|
|
__bread_gfp(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-09-05 02:04:42 +00:00
|
|
|
struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-10-30 23:03:15 +00:00
|
|
|
if (likely(bh) && !buffer_uptodate(bh))
|
2005-04-16 22:20:36 +00:00
|
|
|
bh = __bread_slow(bh);
|
|
|
|
return bh;
|
|
|
|
}
|
2014-09-05 02:04:42 +00:00
|
|
|
EXPORT_SYMBOL(__bread_gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* invalidate_bh_lrus() is called rarely - but not only at unmount.
|
|
|
|
* This doesn't race because it runs in each cpu either in irq
|
|
|
|
* or with preempt disabled.
|
|
|
|
*/
|
|
|
|
static void invalidate_bh_lru(void *arg)
|
|
|
|
{
|
|
|
|
struct bh_lru *b = &get_cpu_var(bh_lrus);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
|
|
|
put_cpu_var(bh_lrus);
|
|
|
|
}
|
2012-03-28 21:42:45 +00:00
|
|
|
|
|
|
|
static bool has_bh_in_lru(int cpu, void *dummy)
|
|
|
|
{
|
|
|
|
struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
|
|
|
|
int i;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-03-28 21:42:45 +00:00
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
if (b->bhs[i])
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-05-06 21:49:55 +00:00
|
|
|
void invalidate_bh_lrus(void)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-03-28 21:42:45 +00:00
|
|
|
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
rewrite rd
This is a rewrite of the ramdisk block device driver.
The old one is really difficult because it effectively implements a block
device which serves data out of its own buffer cache. It relies on the dirty
bit being set, to pin its backing store in cache, however there are non
trivial paths which can clear the dirty bit (eg. try_to_free_buffers()),
which had recently lead to data corruption. And in general it is completely
wrong for a block device driver to do this.
The new one is more like a regular block device driver. It has no idea about
vm/vfs stuff. It's backing store is similar to the buffer cache (a simple
radix-tree of pages), but it doesn't know anything about page cache (the pages
in the radix tree are not pagecache pages).
There is one slight downside -- direct block device access and filesystem
metadata access goes through an extra copy and gets stored in RAM twice.
However, this downside is only slight, because the real buffercache of the
device is now reclaimable (because we're not playing crazy games with it), so
under memory intensive situations, footprint should effectively be the same --
maybe even a slight advantage to the new driver because it can also reclaim
buffer heads.
The fact that it now goes through all the regular vm/fs paths makes it
much more useful for testing, too.
text data bss dec hex filename
2837 849 384 4070 fe6 drivers/block/rd.o
3528 371 12 3911 f47 drivers/block/brd.o
Text is larger, but data and bss are smaller, making total size smaller.
A few other nice things about it:
- Similar structure and layout to the new loop device handlinag.
- Dynamic ramdisk creation.
- Runtime flexible buffer head size (because it is no longer part of the
ramdisk code).
- Boot / load time flexible ramdisk size, which could easily be extended
to a per-ramdisk runtime changeable size (eg. with an ioctl).
- Can use highmem for the backing store.
[akpm@linux-foundation.org: fix build]
[byron.bbradley@gmail.com: make rd_size non-static]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Byron Bradley <byron.bbradley@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-08 12:19:49 +00:00
|
|
|
EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
void set_bh_page(struct buffer_head *bh,
|
|
|
|
struct page *page, unsigned long offset)
|
|
|
|
{
|
|
|
|
bh->b_page = page;
|
2006-03-26 16:24:46 +00:00
|
|
|
BUG_ON(offset >= PAGE_SIZE);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (PageHighMem(page))
|
|
|
|
/*
|
|
|
|
* This catches illegal uses and preserves the offset:
|
|
|
|
*/
|
|
|
|
bh->b_data = (char *)(0 + offset);
|
|
|
|
else
|
|
|
|
bh->b_data = page_address(page) + offset;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(set_bh_page);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Called when truncating a buffer on a page completely.
|
|
|
|
*/
|
2014-06-04 23:10:29 +00:00
|
|
|
|
|
|
|
/* Bits that are cleared during an invalidate */
|
|
|
|
#define BUFFER_FLAGS_DISCARD \
|
|
|
|
(1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
|
|
|
|
1 << BH_Delay | 1 << BH_Unwritten)
|
|
|
|
|
2006-01-14 21:20:43 +00:00
|
|
|
static void discard_buffer(struct buffer_head * bh)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-06-04 23:10:29 +00:00
|
|
|
unsigned long b_state, b_state_old;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
lock_buffer(bh);
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
bh->b_bdev = NULL;
|
2014-06-04 23:10:29 +00:00
|
|
|
b_state = bh->b_state;
|
|
|
|
for (;;) {
|
|
|
|
b_state_old = cmpxchg(&bh->b_state, b_state,
|
|
|
|
(b_state & ~BUFFER_FLAGS_DISCARD));
|
|
|
|
if (b_state_old == b_state)
|
|
|
|
break;
|
|
|
|
b_state = b_state_old;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2011-09-01 00:22:57 +00:00
|
|
|
* block_invalidatepage - invalidate part or all of a buffer-backed page
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* @page: the page which is affected
|
2013-05-22 03:17:23 +00:00
|
|
|
* @offset: start of the range to invalidate
|
|
|
|
* @length: length of the range to invalidate
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* block_invalidatepage() is called when all or part of the page has become
|
2011-09-01 00:22:57 +00:00
|
|
|
* invalidated by a truncate operation.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* block_invalidatepage() does not have to release all buffers, but it must
|
|
|
|
* ensure that no dirty buffer is left outside @offset and that no I/O
|
|
|
|
* is underway against any of the blocks which are outside the truncation
|
|
|
|
* point. Because the caller is about to free (and possibly reuse) those
|
|
|
|
* blocks on-disk.
|
|
|
|
*/
|
2013-05-22 03:17:23 +00:00
|
|
|
void block_invalidatepage(struct page *page, unsigned int offset,
|
|
|
|
unsigned int length)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *head, *bh, *next;
|
|
|
|
unsigned int curr_off = 0;
|
2013-05-22 03:17:23 +00:00
|
|
|
unsigned int stop = length + offset;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
goto out;
|
|
|
|
|
2013-05-22 03:17:23 +00:00
|
|
|
/*
|
|
|
|
* Check for overflow
|
|
|
|
*/
|
|
|
|
BUG_ON(stop > PAGE_CACHE_SIZE || stop < length);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
unsigned int next_off = curr_off + bh->b_size;
|
|
|
|
next = bh->b_this_page;
|
|
|
|
|
2013-05-22 03:17:23 +00:00
|
|
|
/*
|
|
|
|
* Are we still fully in range ?
|
|
|
|
*/
|
|
|
|
if (next_off > stop)
|
|
|
|
goto out;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* is this block fully invalidated?
|
|
|
|
*/
|
|
|
|
if (offset <= curr_off)
|
|
|
|
discard_buffer(bh);
|
|
|
|
curr_off = next_off;
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We release buffers only if the entire page is being invalidated.
|
|
|
|
* The get_block cached value has been unconditionally invalidated,
|
|
|
|
* so real IO is not possible anymore.
|
|
|
|
*/
|
|
|
|
if (offset == 0)
|
2006-03-26 09:37:18 +00:00
|
|
|
try_to_release_page(page, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
2006-03-26 09:37:18 +00:00
|
|
|
return;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_invalidatepage);
|
|
|
|
|
2013-05-22 03:17:23 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* We attach and possibly dirty the buffers atomically wrt
|
|
|
|
* __set_page_dirty_buffers() via private_lock. try_to_free_buffers
|
|
|
|
* is already excluded via the page lock.
|
|
|
|
*/
|
|
|
|
void create_empty_buffers(struct page *page,
|
|
|
|
unsigned long blocksize, unsigned long b_state)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head, *tail;
|
|
|
|
|
|
|
|
head = alloc_page_buffers(page, blocksize, 1);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
bh->b_state |= b_state;
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
|
|
|
|
|
|
|
spin_lock(&page->mapping->private_lock);
|
|
|
|
if (PageUptodate(page) || PageDirty(page)) {
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (PageDirty(page))
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
if (PageUptodate(page))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
spin_unlock(&page->mapping->private_lock);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(create_empty_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are taking a block for data and we don't want any output from any
|
|
|
|
* buffer-cache aliases starting from return from that function and
|
|
|
|
* until the moment when something will explicitly mark the buffer
|
|
|
|
* dirty (hopefully that will not happen until we will free that block ;-)
|
|
|
|
* We don't even need to mark it not-uptodate - nobody can expect
|
|
|
|
* anything from a newly allocated buffer anyway. We used to used
|
|
|
|
* unmap_buffer() for such invalidation, but that was wrong. We definitely
|
|
|
|
* don't want to mark the alias unmapped, for example - it would confuse
|
|
|
|
* anyone who might pick it with bread() afterwards...
|
|
|
|
*
|
|
|
|
* Also.. Note that bforget() doesn't lock the buffer. So there can
|
|
|
|
* be writeout I/O going on against recently-freed buffers. We don't
|
|
|
|
* wait on that I/O in bforget() - it's more efficient to wait on the I/O
|
|
|
|
* only if we really need to. That happens here.
|
|
|
|
*/
|
|
|
|
void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
|
|
|
|
{
|
|
|
|
struct buffer_head *old_bh;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
|
2005-11-07 08:59:39 +00:00
|
|
|
old_bh = __find_get_block_slow(bdev, block);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (old_bh) {
|
|
|
|
clear_buffer_dirty(old_bh);
|
|
|
|
wait_on_buffer(old_bh);
|
|
|
|
clear_buffer_req(old_bh);
|
|
|
|
__brelse(old_bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(unmap_underlying_metadata);
|
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
/*
|
|
|
|
* Size is a power-of-two in the range 512..PAGE_SIZE,
|
|
|
|
* and the case we care about most is PAGE_SIZE.
|
|
|
|
*
|
|
|
|
* So this *could* possibly be written with those
|
|
|
|
* constraints in mind (relevant mostly if some
|
|
|
|
* architecture has a slow bit-scan instruction)
|
|
|
|
*/
|
|
|
|
static inline int block_size_bits(unsigned int blocksize)
|
|
|
|
{
|
|
|
|
return ilog2(blocksize);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
|
|
|
|
{
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
|
|
|
|
return page_buffers(page);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* NOTE! All mapped/uptodate combinations are valid:
|
|
|
|
*
|
|
|
|
* Mapped Uptodate Meaning
|
|
|
|
*
|
|
|
|
* No No "unknown" - must do get_block()
|
|
|
|
* No Yes "hole" - zero-filled
|
|
|
|
* Yes No "allocated" - allocated on disk, not read in
|
|
|
|
* Yes Yes "valid" - allocated and up-to-date in memory.
|
|
|
|
*
|
|
|
|
* "Dirty" is valid only with the last case (mapped+uptodate).
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* While block_write_full_page is writing back the dirty buffers under
|
|
|
|
* the page lock, whoever dirtied the buffers may decide to clean them
|
|
|
|
* again at any time. We handle that by only looking at the buffer
|
|
|
|
* state inside lock_buffer().
|
|
|
|
*
|
|
|
|
* If block_write_full_page() is called for regular writeback
|
|
|
|
* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
|
|
|
|
* locked buffer. This only can happen if someone has written the buffer
|
|
|
|
* directly, with submit_bh(). At the address_space level PageWriteback
|
|
|
|
* prevents this contention from occurring.
|
2009-04-07 22:12:43 +00:00
|
|
|
*
|
|
|
|
* If block_write_full_page() is called with wbc->sync_mode ==
|
2011-03-09 10:56:30 +00:00
|
|
|
* WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
|
|
|
|
* causes the writes to be flagged as synchronous writes.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
static int __block_write_full_page(struct inode *inode, struct page *page,
|
2009-04-15 17:22:38 +00:00
|
|
|
get_block_t *get_block, struct writeback_control *wbc,
|
|
|
|
bh_end_io_t *handler)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
sector_t block;
|
|
|
|
sector_t last_block;
|
2005-05-05 23:15:48 +00:00
|
|
|
struct buffer_head *bh, *head;
|
2012-11-29 18:21:43 +00:00
|
|
|
unsigned int blocksize, bbits;
|
2005-04-16 22:20:36 +00:00
|
|
|
int nr_underway = 0;
|
2015-06-02 14:37:23 +00:00
|
|
|
int write_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
head = create_page_buffers(page, inode,
|
2005-04-16 22:20:36 +00:00
|
|
|
(1 << BH_Dirty)|(1 << BH_Uptodate));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Be very careful. We have no exclusion from __set_page_dirty_buffers
|
|
|
|
* here, and the (potentially unmapped) buffers may become dirty at
|
|
|
|
* any time. If a buffer becomes dirty here after we've inspected it
|
|
|
|
* then we just miss that fact, and the page stays dirty.
|
|
|
|
*
|
|
|
|
* Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
|
|
|
|
* handle that here by just cleaning them.
|
|
|
|
*/
|
|
|
|
|
|
|
|
bh = head;
|
2012-11-29 18:21:43 +00:00
|
|
|
blocksize = bh->b_size;
|
|
|
|
bbits = block_size_bits(blocksize);
|
|
|
|
|
|
|
|
block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
|
|
|
|
last_block = (i_size_read(inode) - 1) >> bbits;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get all the dirty buffers mapped to disk addresses and
|
|
|
|
* handle any aliases from the underlying blockdev's mapping.
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
if (block > last_block) {
|
|
|
|
/*
|
|
|
|
* mapped buffers outside i_size will occur, because
|
|
|
|
* this page can be outside i_size when there is a
|
|
|
|
* truncate in progress.
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* The buffer was zeroed by block_write_full_page()
|
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
set_buffer_uptodate(bh);
|
2008-07-11 23:27:31 +00:00
|
|
|
} else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
|
|
|
|
buffer_dirty(bh)) {
|
2006-03-26 09:38:00 +00:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 22:20:36 +00:00
|
|
|
err = get_block(inode, block, bh, 1);
|
|
|
|
if (err)
|
|
|
|
goto recover;
|
2008-07-11 23:27:31 +00:00
|
|
|
clear_buffer_delay(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (buffer_new(bh)) {
|
|
|
|
/* blockdev mappings never come here */
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
unmap_underlying_metadata(bh->b_bdev,
|
|
|
|
bh->b_blocknr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
block++;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
continue;
|
|
|
|
/*
|
|
|
|
* If it's a fully non-blocking write attempt and we cannot
|
|
|
|
* lock the buffer then redirty the page. Note that this can
|
2009-09-23 17:37:09 +00:00
|
|
|
* potentially cause a busy-wait loop from writeback threads
|
|
|
|
* and kswapd activity, but those code paths have their own
|
|
|
|
* higher-level throttling.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2010-10-26 21:21:26 +00:00
|
|
|
if (wbc->sync_mode != WB_SYNC_NONE) {
|
2005-04-16 22:20:36 +00:00
|
|
|
lock_buffer(bh);
|
2008-08-02 10:02:13 +00:00
|
|
|
} else if (!trylock_buffer(bh)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
redirty_page_for_writepage(wbc, page);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
2009-04-15 17:22:38 +00:00
|
|
|
mark_buffer_async_write_endio(bh, handler);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page and its buffers are protected by PageWriteback(), so we can
|
|
|
|
* drop the bh refcounts early.
|
|
|
|
*/
|
|
|
|
BUG_ON(PageWriteback(page));
|
|
|
|
set_page_writeback(page);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
2015-06-02 14:39:48 +00:00
|
|
|
submit_bh_wbc(write_op, bh, 0, wbc);
|
2005-04-16 22:20:36 +00:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2005-05-05 23:15:47 +00:00
|
|
|
unlock_page(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
err = 0;
|
|
|
|
done:
|
|
|
|
if (nr_underway == 0) {
|
|
|
|
/*
|
|
|
|
* The page was marked dirty, but the buffers were
|
|
|
|
* clean. Someone wrote them back by hand with
|
|
|
|
* ll_rw_block/submit_bh. A rare case.
|
|
|
|
*/
|
|
|
|
end_page_writeback(page);
|
2007-05-06 21:49:05 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* The page and buffer_heads can be released at any time from
|
|
|
|
* here on.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
recover:
|
|
|
|
/*
|
|
|
|
* ENOSPC, or some other error. We may already have added some
|
|
|
|
* blocks to the file, so we need to write these out to avoid
|
|
|
|
* exposing stale data.
|
|
|
|
* The page is currently locked and not marked for writeback
|
|
|
|
*/
|
|
|
|
bh = head;
|
|
|
|
/* Recovery: lock and submit the mapped buffers */
|
|
|
|
do {
|
2008-07-11 23:27:31 +00:00
|
|
|
if (buffer_mapped(bh) && buffer_dirty(bh) &&
|
|
|
|
!buffer_delay(bh)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
lock_buffer(bh);
|
2009-04-15 17:22:38 +00:00
|
|
|
mark_buffer_async_write_endio(bh, handler);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The buffer may have been set dirty during
|
|
|
|
* attachment to a dirty page.
|
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
|
|
|
SetPageError(page);
|
|
|
|
BUG_ON(PageWriteback(page));
|
2007-05-08 07:23:27 +00:00
|
|
|
mapping_set_error(page->mapping, err);
|
2005-04-16 22:20:36 +00:00
|
|
|
set_page_writeback(page);
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
|
|
|
clear_buffer_dirty(bh);
|
2015-06-02 14:39:48 +00:00
|
|
|
submit_bh_wbc(write_op, bh, 0, wbc);
|
2005-04-16 22:20:36 +00:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2007-02-20 21:57:54 +00:00
|
|
|
unlock_page(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2007-10-16 08:25:01 +00:00
|
|
|
/*
|
|
|
|
* If a page has any new buffers, zero them out here, and mark them uptodate
|
|
|
|
* and dirty so they'll be written out (in order to prevent uninitialised
|
|
|
|
* block data from leaking). And clear the new bit.
|
|
|
|
*/
|
|
|
|
void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
unsigned int block_start, block_end;
|
|
|
|
struct buffer_head *head, *bh;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
return;
|
|
|
|
|
|
|
|
bh = head = page_buffers(page);
|
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + bh->b_size;
|
|
|
|
|
|
|
|
if (buffer_new(bh)) {
|
|
|
|
if (block_end > from && block_start < to) {
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
unsigned start, size;
|
|
|
|
|
|
|
|
start = max(from, block_start);
|
|
|
|
size = min(to, block_end) - start;
|
|
|
|
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, start, size);
|
2007-10-16 08:25:01 +00:00
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(page_zero_new_buffers);
|
|
|
|
|
2010-10-06 08:47:23 +00:00
|
|
|
int __block_write_begin(struct page *page, loff_t pos, unsigned len,
|
2010-06-04 09:29:57 +00:00
|
|
|
get_block_t *get_block)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-10-06 08:47:23 +00:00
|
|
|
unsigned from = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
unsigned to = from + len;
|
2010-06-04 09:29:57 +00:00
|
|
|
struct inode *inode = page->mapping->host;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned block_start, block_end;
|
|
|
|
sector_t block;
|
|
|
|
int err = 0;
|
|
|
|
unsigned blocksize, bbits;
|
|
|
|
struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
BUG_ON(from > PAGE_CACHE_SIZE);
|
|
|
|
BUG_ON(to > PAGE_CACHE_SIZE);
|
|
|
|
BUG_ON(from > to);
|
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
head = create_page_buffers(page, inode, 0);
|
|
|
|
blocksize = head->b_size;
|
|
|
|
bbits = block_size_bits(blocksize);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
|
|
|
|
|
|
|
|
for(bh = head, block_start = 0; bh != head || !block_start;
|
|
|
|
block++, block_start=block_end, bh = bh->b_this_page) {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (buffer_new(bh))
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 09:38:00 +00:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 22:20:36 +00:00
|
|
|
err = get_block(inode, block, bh, 1);
|
|
|
|
if (err)
|
2005-05-05 23:15:45 +00:00
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (buffer_new(bh)) {
|
|
|
|
unmap_underlying_metadata(bh->b_bdev,
|
|
|
|
bh->b_blocknr);
|
|
|
|
if (PageUptodate(page)) {
|
2007-10-16 08:25:00 +00:00
|
|
|
clear_buffer_new(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
set_buffer_uptodate(bh);
|
2007-10-16 08:25:00 +00:00
|
|
|
mark_buffer_dirty(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
}
|
2008-02-05 06:28:29 +00:00
|
|
|
if (block_end > to || block_start < from)
|
|
|
|
zero_user_segments(page,
|
|
|
|
to, block_end,
|
|
|
|
block_start, from);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
|
2007-02-12 08:51:41 +00:00
|
|
|
!buffer_unwritten(bh) &&
|
2005-04-16 22:20:36 +00:00
|
|
|
(block_start < from || block_end > to)) {
|
|
|
|
ll_rw_block(READ, 1, &bh);
|
|
|
|
*wait_bh++=bh;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If we issued read requests - let them complete.
|
|
|
|
*/
|
|
|
|
while(wait_bh > wait) {
|
|
|
|
wait_on_buffer(*--wait_bh);
|
|
|
|
if (!buffer_uptodate(*wait_bh))
|
2005-05-05 23:15:45 +00:00
|
|
|
err = -EIO;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2011-06-13 22:58:27 +00:00
|
|
|
if (unlikely(err))
|
2007-10-16 08:25:01 +00:00
|
|
|
page_zero_new_buffers(page, from, to);
|
2005-04-16 22:20:36 +00:00
|
|
|
return err;
|
|
|
|
}
|
2010-10-06 08:47:23 +00:00
|
|
|
EXPORT_SYMBOL(__block_write_begin);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
static int __block_commit_write(struct inode *inode, struct page *page,
|
|
|
|
unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
unsigned block_start, block_end;
|
|
|
|
int partial = 0;
|
|
|
|
unsigned blocksize;
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
bh = head = page_buffers(page);
|
|
|
|
blocksize = bh->b_size;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
block_start = 0;
|
|
|
|
do {
|
2005-04-16 22:20:36 +00:00
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
partial = 1;
|
|
|
|
} else {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
2007-10-16 08:25:01 +00:00
|
|
|
clear_buffer_new(bh);
|
2012-11-29 18:21:43 +00:00
|
|
|
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is a partial write which happened to make all buffers
|
|
|
|
* uptodate then we can optimize away a bogus readpage() for
|
|
|
|
* the next read(). Here we 'discover' whether the page went
|
|
|
|
* uptodate as a result of this (potentially partial) write.
|
|
|
|
*/
|
|
|
|
if (!partial)
|
|
|
|
SetPageUptodate(page);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-10-16 08:25:01 +00:00
|
|
|
/*
|
2010-06-04 09:29:58 +00:00
|
|
|
* block_write_begin takes care of the basic task of block allocation and
|
|
|
|
* bringing partial write blocks uptodate first.
|
|
|
|
*
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 15:05:33 +00:00
|
|
|
* The filesystem needs to handle block truncation upon failure.
|
2007-10-16 08:25:01 +00:00
|
|
|
*/
|
2010-06-04 09:29:58 +00:00
|
|
|
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
|
|
|
|
unsigned flags, struct page **pagep, get_block_t *get_block)
|
2007-10-16 08:25:01 +00:00
|
|
|
{
|
2010-06-04 09:29:57 +00:00
|
|
|
pgoff_t index = pos >> PAGE_CACHE_SHIFT;
|
2007-10-16 08:25:01 +00:00
|
|
|
struct page *page;
|
2010-06-04 09:29:57 +00:00
|
|
|
int status;
|
2007-10-16 08:25:01 +00:00
|
|
|
|
2010-06-04 09:29:57 +00:00
|
|
|
page = grab_cache_page_write_begin(mapping, index, flags);
|
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
2007-10-16 08:25:01 +00:00
|
|
|
|
2010-06-04 09:29:57 +00:00
|
|
|
status = __block_write_begin(page, pos, len, get_block);
|
2007-10-16 08:25:01 +00:00
|
|
|
if (unlikely(status)) {
|
2010-06-04 09:29:57 +00:00
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
page = NULL;
|
2007-10-16 08:25:01 +00:00
|
|
|
}
|
|
|
|
|
2010-06-04 09:29:57 +00:00
|
|
|
*pagep = page;
|
2007-10-16 08:25:01 +00:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_begin);
|
|
|
|
|
|
|
|
int block_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned start;
|
|
|
|
|
|
|
|
start = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
|
|
|
|
if (unlikely(copied < len)) {
|
|
|
|
/*
|
|
|
|
* The buffers that were written will now be uptodate, so we
|
|
|
|
* don't have to worry about a readpage reading them and
|
|
|
|
* overwriting a partial write. However if we have encountered
|
|
|
|
* a short write and only partially written into a buffer, it
|
|
|
|
* will not be marked uptodate, so a readpage might come in and
|
|
|
|
* destroy our partial write.
|
|
|
|
*
|
|
|
|
* Do the simplest thing, and just treat any short write to a
|
|
|
|
* non uptodate page as a zero-length write, and force the
|
|
|
|
* caller to redo the whole thing.
|
|
|
|
*/
|
|
|
|
if (!PageUptodate(page))
|
|
|
|
copied = 0;
|
|
|
|
|
|
|
|
page_zero_new_buffers(page, start+copied, start+len);
|
|
|
|
}
|
|
|
|
flush_dcache_page(page);
|
|
|
|
|
|
|
|
/* This could be a short (even 0-length) commit */
|
|
|
|
__block_commit_write(inode, page, start, start+copied);
|
|
|
|
|
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_end);
|
|
|
|
|
|
|
|
int generic_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
vfs: fix data corruption when blocksize < pagesize for mmaped data
->page_mkwrite() is used by filesystems to allocate blocks under a page
which is becoming writeably mmapped in some process' address space. This
allows a filesystem to return a page fault if there is not enough space
available, user exceeds quota or similar problem happens, rather than
silently discarding data later when writepage is called.
However VFS fails to call ->page_mkwrite() in all the cases where
filesystems need it when blocksize < pagesize. For example when
blocksize = 1024, pagesize = 4096 the following is problematic:
ftruncate(fd, 0);
pwrite(fd, buf, 1024, 0);
map = mmap(NULL, 1024, PROT_WRITE, MAP_SHARED, fd, 0);
map[0] = 'a'; ----> page_mkwrite() for index 0 is called
ftruncate(fd, 10000); /* or even pwrite(fd, buf, 1, 10000) */
mremap(map, 1024, 10000, 0);
map[4095] = 'a'; ----> no page_mkwrite() called
At the moment ->page_mkwrite() is called, filesystem can allocate only
one block for the page because i_size == 1024. Otherwise it would create
blocks beyond i_size which is generally undesirable. But later at
->writepage() time, we also need to store data at offset 4095 but we
don't have block allocated for it.
This patch introduces a helper function filesystems can use to have
->page_mkwrite() called at all the necessary moments.
Signed-off-by: Jan Kara <jack@suse.cz>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Cc: stable@vger.kernel.org
2014-10-02 01:49:18 +00:00
|
|
|
loff_t old_size = inode->i_size;
|
2008-07-11 23:27:31 +00:00
|
|
|
int i_size_changed = 0;
|
2007-10-16 08:25:01 +00:00
|
|
|
|
|
|
|
copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No need to use i_size_read() here, the i_size
|
|
|
|
* cannot change under us because we hold i_mutex.
|
|
|
|
*
|
|
|
|
* But it's important to update i_size while still holding page lock:
|
|
|
|
* page writeout could otherwise come in and zero beyond i_size.
|
|
|
|
*/
|
|
|
|
if (pos+copied > inode->i_size) {
|
|
|
|
i_size_write(inode, pos+copied);
|
2008-07-11 23:27:31 +00:00
|
|
|
i_size_changed = 1;
|
2007-10-16 08:25:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
vfs: fix data corruption when blocksize < pagesize for mmaped data
->page_mkwrite() is used by filesystems to allocate blocks under a page
which is becoming writeably mmapped in some process' address space. This
allows a filesystem to return a page fault if there is not enough space
available, user exceeds quota or similar problem happens, rather than
silently discarding data later when writepage is called.
However VFS fails to call ->page_mkwrite() in all the cases where
filesystems need it when blocksize < pagesize. For example when
blocksize = 1024, pagesize = 4096 the following is problematic:
ftruncate(fd, 0);
pwrite(fd, buf, 1024, 0);
map = mmap(NULL, 1024, PROT_WRITE, MAP_SHARED, fd, 0);
map[0] = 'a'; ----> page_mkwrite() for index 0 is called
ftruncate(fd, 10000); /* or even pwrite(fd, buf, 1, 10000) */
mremap(map, 1024, 10000, 0);
map[4095] = 'a'; ----> no page_mkwrite() called
At the moment ->page_mkwrite() is called, filesystem can allocate only
one block for the page because i_size == 1024. Otherwise it would create
blocks beyond i_size which is generally undesirable. But later at
->writepage() time, we also need to store data at offset 4095 but we
don't have block allocated for it.
This patch introduces a helper function filesystems can use to have
->page_mkwrite() called at all the necessary moments.
Signed-off-by: Jan Kara <jack@suse.cz>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Cc: stable@vger.kernel.org
2014-10-02 01:49:18 +00:00
|
|
|
if (old_size < pos)
|
|
|
|
pagecache_isize_extended(inode, old_size, pos);
|
2008-07-11 23:27:31 +00:00
|
|
|
/*
|
|
|
|
* Don't mark the inode dirty under page lock. First, it unnecessarily
|
|
|
|
* makes the holding time of page lock longer. Second, it forces lock
|
|
|
|
* ordering of page lock and transaction start for journaling
|
|
|
|
* filesystems.
|
|
|
|
*/
|
|
|
|
if (i_size_changed)
|
|
|
|
mark_inode_dirty(inode);
|
|
|
|
|
2007-10-16 08:25:01 +00:00
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(generic_write_end);
|
|
|
|
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 22:46:36 +00:00
|
|
|
/*
|
|
|
|
* block_is_partially_uptodate checks whether buffers within a page are
|
|
|
|
* uptodate or not.
|
|
|
|
*
|
|
|
|
* Returns true if all buffers which correspond to a file portion
|
|
|
|
* we want to read are uptodate.
|
|
|
|
*/
|
2014-02-03 02:16:54 +00:00
|
|
|
int block_is_partially_uptodate(struct page *page, unsigned long from,
|
|
|
|
unsigned long count)
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 22:46:36 +00:00
|
|
|
{
|
|
|
|
unsigned block_start, block_end, blocksize;
|
|
|
|
unsigned to;
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
return 0;
|
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
head = page_buffers(page);
|
|
|
|
blocksize = head->b_size;
|
2014-02-03 02:16:54 +00:00
|
|
|
to = min_t(unsigned, PAGE_CACHE_SIZE - from, count);
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 22:46:36 +00:00
|
|
|
to = from + to;
|
|
|
|
if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end > from && block_start < to) {
|
|
|
|
if (!buffer_uptodate(bh)) {
|
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (block_end >= to)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_is_partially_uptodate);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Generic "read page" function for block devices that have the normal
|
|
|
|
* get_block functionality. This is most of the block device filesystems.
|
|
|
|
* Reads the page asynchronously --- the unlock_buffer() and
|
|
|
|
* set/clear_buffer_uptodate() functions propagate buffer state into the
|
|
|
|
* page struct once IO has completed.
|
|
|
|
*/
|
|
|
|
int block_read_full_page(struct page *page, get_block_t *get_block)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
sector_t iblock, lblock;
|
|
|
|
struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
|
2012-11-29 18:21:43 +00:00
|
|
|
unsigned int blocksize, bbits;
|
2005-04-16 22:20:36 +00:00
|
|
|
int nr, i;
|
|
|
|
int fully_mapped = 1;
|
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
head = create_page_buffers(page, inode, 0);
|
|
|
|
blocksize = head->b_size;
|
|
|
|
bbits = block_size_bits(blocksize);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-11-29 18:21:43 +00:00
|
|
|
iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
|
|
|
|
lblock = (i_size_read(inode)+blocksize-1) >> bbits;
|
2005-04-16 22:20:36 +00:00
|
|
|
bh = head;
|
|
|
|
nr = 0;
|
|
|
|
i = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!buffer_mapped(bh)) {
|
2005-05-17 04:53:49 +00:00
|
|
|
int err = 0;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
fully_mapped = 0;
|
|
|
|
if (iblock < lblock) {
|
2006-03-26 09:38:00 +00:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-05-17 04:53:49 +00:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
|
|
|
if (err)
|
2005-04-16 22:20:36 +00:00
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
if (!buffer_mapped(bh)) {
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, i * blocksize, blocksize);
|
2005-05-17 04:53:49 +00:00
|
|
|
if (!err)
|
|
|
|
set_buffer_uptodate(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* get_block() might have updated the buffer
|
|
|
|
* synchronously
|
|
|
|
*/
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
arr[nr++] = bh;
|
|
|
|
} while (i++, iblock++, (bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
if (fully_mapped)
|
|
|
|
SetPageMappedToDisk(page);
|
|
|
|
|
|
|
|
if (!nr) {
|
|
|
|
/*
|
|
|
|
* All buffers are uptodate - we can set the page uptodate
|
|
|
|
* as well. But not if get_block() returned an error.
|
|
|
|
*/
|
|
|
|
if (!PageError(page))
|
|
|
|
SetPageUptodate(page);
|
|
|
|
unlock_page(page);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Stage two: lock the buffers */
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
lock_buffer(bh);
|
|
|
|
mark_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stage 3: start the IO. Check for uptodateness
|
|
|
|
* inside the buffer lock in case another process reading
|
|
|
|
* the underlying blockdev brought it uptodate (the sct fix).
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
end_buffer_async_read(bh, 1);
|
|
|
|
else
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(block_read_full_page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* utility function for filesystems that need to do work on expanding
|
2007-10-16 08:25:07 +00:00
|
|
|
* truncates. Uses filesystem pagecache writes to allow the filesystem to
|
2005-04-16 22:20:36 +00:00
|
|
|
* deal with the hole.
|
|
|
|
*/
|
2007-10-16 08:25:07 +00:00
|
|
|
int generic_cont_expand_simple(struct inode *inode, loff_t size)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
struct page *page;
|
2007-10-16 08:25:07 +00:00
|
|
|
void *fsdata;
|
2005-04-16 22:20:36 +00:00
|
|
|
int err;
|
|
|
|
|
2009-08-20 16:35:06 +00:00
|
|
|
err = inode_newsize_ok(inode, size);
|
|
|
|
if (err)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_begin(NULL, mapping, size, 0,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
2006-01-08 09:02:13 +00:00
|
|
|
goto out;
|
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
|
|
|
|
BUG_ON(err > 0);
|
2006-01-08 09:02:13 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(generic_cont_expand_simple);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-04-29 07:59:01 +00:00
|
|
|
static int cont_expand_zero(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, loff_t *bytes)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned blocksize = 1 << inode->i_blkbits;
|
2007-10-16 08:25:07 +00:00
|
|
|
struct page *page;
|
|
|
|
void *fsdata;
|
|
|
|
pgoff_t index, curidx;
|
|
|
|
loff_t curpos;
|
|
|
|
unsigned zerofrom, offset, len;
|
|
|
|
int err = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
index = pos >> PAGE_CACHE_SHIFT;
|
|
|
|
offset = pos & ~PAGE_CACHE_MASK;
|
|
|
|
|
|
|
|
while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
|
|
|
|
zerofrom = curpos & ~PAGE_CACHE_MASK;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
2007-10-16 08:25:07 +00:00
|
|
|
len = PAGE_CACHE_SIZE - zerofrom;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_begin(file, mapping, curpos, len,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, zerofrom, len);
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_end(file, mapping, curpos, len, len,
|
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2008-04-28 09:16:28 +00:00
|
|
|
|
|
|
|
balance_dirty_pages_ratelimited(mapping);
|
2014-07-27 17:00:41 +00:00
|
|
|
|
|
|
|
if (unlikely(fatal_signal_pending(current))) {
|
|
|
|
err = -EINTR;
|
|
|
|
goto out;
|
|
|
|
}
|
2007-10-16 08:25:07 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
/* page covers the boundary, find the boundary offset */
|
|
|
|
if (index == curidx) {
|
|
|
|
zerofrom = curpos & ~PAGE_CACHE_MASK;
|
2005-04-16 22:20:36 +00:00
|
|
|
/* if we will expand the thing last block will be filled */
|
2007-10-16 08:25:07 +00:00
|
|
|
if (offset <= zerofrom) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (zerofrom & (blocksize-1)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
2007-10-16 08:25:07 +00:00
|
|
|
len = offset - zerofrom;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_begin(file, mapping, curpos, len,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, zerofrom, len);
|
2007-10-16 08:25:07 +00:00
|
|
|
err = pagecache_write_end(file, mapping, curpos, len, len,
|
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2007-10-16 08:25:07 +00:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For moronic filesystems that do not allow holes in file.
|
|
|
|
* We may have to extend the file.
|
|
|
|
*/
|
2010-06-04 09:29:55 +00:00
|
|
|
int cont_write_begin(struct file *file, struct address_space *mapping,
|
2007-10-16 08:25:07 +00:00
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata,
|
|
|
|
get_block_t *get_block, loff_t *bytes)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned blocksize = 1 << inode->i_blkbits;
|
|
|
|
unsigned zerofrom;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = cont_expand_zero(file, mapping, pos, bytes);
|
|
|
|
if (err)
|
2010-06-04 09:29:58 +00:00
|
|
|
return err;
|
2007-10-16 08:25:07 +00:00
|
|
|
|
|
|
|
zerofrom = *bytes & ~PAGE_CACHE_MASK;
|
|
|
|
if (pos+len > *bytes && zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2010-06-04 09:29:58 +00:00
|
|
|
return block_write_begin(mapping, pos, len, flags, pagep, get_block);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(cont_write_begin);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
int block_commit_write(struct page *page, unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
__block_commit_write(inode,page,from,to);
|
|
|
|
return 0;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(block_commit_write);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-07-19 07:39:55 +00:00
|
|
|
/*
|
|
|
|
* block_page_mkwrite() is not allowed to change the file size as it gets
|
|
|
|
* called from a page fault handler when a page is first dirtied. Hence we must
|
|
|
|
* be careful to check for EOF conditions here. We set the page up correctly
|
|
|
|
* for a written page which means we get ENOSPC checking when writing into
|
|
|
|
* holes and correct delalloc and unwritten extent mapping on filesystems that
|
|
|
|
* support these features.
|
|
|
|
*
|
|
|
|
* We are not allowed to take the i_mutex here so we have to play games to
|
|
|
|
* protect against truncate races as the page could now be beyond EOF. Because
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 15:05:33 +00:00
|
|
|
* truncate writes the inode size before removing pages, once we have the
|
2007-07-19 07:39:55 +00:00
|
|
|
* page lock we can determine safely if the page is beyond EOF. If it is not
|
|
|
|
* beyond EOF, then the page is guaranteed safe against truncation until we
|
|
|
|
* unlock the page.
|
2011-05-23 22:23:35 +00:00
|
|
|
*
|
2012-06-12 14:20:37 +00:00
|
|
|
* Direct callers of this function should protect against filesystem freezing
|
2015-10-13 22:51:02 +00:00
|
|
|
* using sb_start_pagefault() - sb_end_pagefault() functions.
|
2007-07-19 07:39:55 +00:00
|
|
|
*/
|
2015-10-13 22:51:02 +00:00
|
|
|
int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
|
2011-05-23 22:23:34 +00:00
|
|
|
get_block_t get_block)
|
2007-07-19 07:39:55 +00:00
|
|
|
{
|
2009-03-31 22:23:21 +00:00
|
|
|
struct page *page = vmf->page;
|
2013-01-23 22:07:38 +00:00
|
|
|
struct inode *inode = file_inode(vma->vm_file);
|
2007-07-19 07:39:55 +00:00
|
|
|
unsigned long end;
|
|
|
|
loff_t size;
|
2011-05-23 22:23:34 +00:00
|
|
|
int ret;
|
2007-07-19 07:39:55 +00:00
|
|
|
|
|
|
|
lock_page(page);
|
|
|
|
size = i_size_read(inode);
|
|
|
|
if ((page->mapping != inode->i_mapping) ||
|
2007-07-20 07:31:45 +00:00
|
|
|
(page_offset(page) > size)) {
|
2011-05-23 22:23:34 +00:00
|
|
|
/* We overload EFAULT to mean page got truncated */
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto out_unlock;
|
2007-07-19 07:39:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* page is wholly or partially inside EOF */
|
|
|
|
if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
|
|
|
|
end = size & ~PAGE_CACHE_MASK;
|
|
|
|
else
|
|
|
|
end = PAGE_CACHE_SIZE;
|
|
|
|
|
2010-10-06 08:47:23 +00:00
|
|
|
ret = __block_write_begin(page, 0, end, get_block);
|
2007-07-19 07:39:55 +00:00
|
|
|
if (!ret)
|
|
|
|
ret = block_commit_write(page, 0, end);
|
|
|
|
|
2011-05-23 22:23:34 +00:00
|
|
|
if (unlikely(ret < 0))
|
|
|
|
goto out_unlock;
|
2011-05-23 22:23:35 +00:00
|
|
|
set_page_dirty(page);
|
mm: only enforce stable page writes if the backing device requires it
Create a helper function to check if a backing device requires stable
page writes and, if so, performs the necessary wait. Then, make it so
that all points in the memory manager that handle making pages writable
use the helper function. This should provide stable page write support
to most filesystems, while eliminating unnecessary waiting for devices
that don't require the feature.
Before this patchset, all filesystems would block, regardless of whether
or not it was necessary. ext3 would wait, but still generate occasional
checksum errors. The network filesystems were left to do their own
thing, so they'd wait too.
After this patchset, all the disk filesystems except ext3 and btrfs will
wait only if the hardware requires it. ext3 (if necessary) snapshots
pages instead of blocking, and btrfs provides its own bdi so the mm will
never wait. Network filesystems haven't been touched, so either they
provide their own stable page guarantees or they don't block at all.
The blocking behavior is back to what it was before 3.0 if you don't
have a disk requiring stable page writes.
Here's the result of using dbench to test latency on ext2:
3.8.0-rc3:
Operation Count AvgLat MaxLat
----------------------------------------
WriteX 109347 0.028 59.817
ReadX 347180 0.004 3.391
Flush 15514 29.828 287.283
Throughput 57.429 MB/sec 4 clients 4 procs max_latency=287.290 ms
3.8.0-rc3 + patches:
WriteX 105556 0.029 4.273
ReadX 335004 0.005 4.112
Flush 14982 30.540 298.634
Throughput 55.4496 MB/sec 4 clients 4 procs max_latency=298.650 ms
As you can see, the maximum write latency drops considerably with this
patch enabled. The other filesystems (ext3/ext4/xfs/btrfs) behave
similarly, but see the cover letter for those results.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Acked-by: Steven Whitehouse <swhiteho@redhat.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Artem Bityutskiy <dedekind1@gmail.com>
Cc: Joel Becker <jlbec@evilplan.org>
Cc: Mark Fasheh <mfasheh@suse.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Eric Van Hensbergen <ericvh@gmail.com>
Cc: Ron Minnich <rminnich@sandia.gov>
Cc: Latchesar Ionkov <lucho@ionkov.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-22 00:42:51 +00:00
|
|
|
wait_for_stable_page(page);
|
2011-05-23 22:23:34 +00:00
|
|
|
return 0;
|
|
|
|
out_unlock:
|
|
|
|
unlock_page(page);
|
2007-07-19 07:39:55 +00:00
|
|
|
return ret;
|
2011-05-23 22:23:34 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(block_page_mkwrite);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2007-10-16 08:25:25 +00:00
|
|
|
* nobh_write_begin()'s prereads are special: the buffer_heads are freed
|
2005-04-16 22:20:36 +00:00
|
|
|
* immediately, while under the page lock. So it needs a special end_io
|
|
|
|
* handler which does not touch the bh after unlocking it.
|
|
|
|
*/
|
|
|
|
static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
2007-10-16 08:24:47 +00:00
|
|
|
__end_buffer_read_notouch(bh, uptodate);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
/*
|
|
|
|
* Attach the singly-linked list of buffers created by nobh_write_begin, to
|
|
|
|
* the page (converting it to circular linked list and taking care of page
|
|
|
|
* dirty races).
|
|
|
|
*/
|
|
|
|
static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
|
|
|
|
spin_lock(&page->mapping->private_lock);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (PageDirty(page))
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
if (!bh->b_this_page)
|
|
|
|
bh->b_this_page = head;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
spin_unlock(&page->mapping->private_lock);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2010-06-04 09:29:54 +00:00
|
|
|
* On entry, the page is fully not uptodate.
|
|
|
|
* On exit the page is fully uptodate in the areas outside (from,to)
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 15:05:33 +00:00
|
|
|
* The filesystem needs to handle block truncation upon failure.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2010-06-04 09:29:54 +00:00
|
|
|
int nobh_write_begin(struct address_space *mapping,
|
2007-10-16 08:25:25 +00:00
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata,
|
2005-04-16 22:20:36 +00:00
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
2007-10-16 08:25:25 +00:00
|
|
|
struct inode *inode = mapping->host;
|
2005-04-16 22:20:36 +00:00
|
|
|
const unsigned blkbits = inode->i_blkbits;
|
|
|
|
const unsigned blocksize = 1 << blkbits;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
struct buffer_head *head, *bh;
|
2007-10-16 08:25:25 +00:00
|
|
|
struct page *page;
|
|
|
|
pgoff_t index;
|
|
|
|
unsigned from, to;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned block_in_page;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
unsigned block_start, block_end;
|
2005-04-16 22:20:36 +00:00
|
|
|
sector_t block_in_file;
|
|
|
|
int nr_reads = 0;
|
|
|
|
int ret = 0;
|
|
|
|
int is_mapped_to_disk = 1;
|
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
index = pos >> PAGE_CACHE_SHIFT;
|
|
|
|
from = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
to = from + len;
|
|
|
|
|
fs: symlink write_begin allocation context fix
With the write_begin/write_end aops, page_symlink was broken because it
could no longer pass a GFP_NOFS type mask into the point where the
allocations happened. They are done in write_begin, which would always
assume that the filesystem can be entered from reclaim. This bug could
cause filesystem deadlocks.
The funny thing with having a gfp_t mask there is that it doesn't really
allow the caller to arbitrarily tinker with the context in which it can be
called. It couldn't ever be GFP_ATOMIC, for example, because it needs to
take the page lock. The only thing any callers care about is __GFP_FS
anyway, so turn that into a single flag.
Add a new flag for write_begin, AOP_FLAG_NOFS. Filesystems can now act on
this flag in their write_begin function. Change __grab_cache_page to
accept a nofs argument as well, to honour that flag (while we're there,
change the name to grab_cache_page_write_begin which is more instructive
and does away with random leading underscores).
This is really a more flexible way to go in the end anyway -- if a
filesystem happens to want any extra allocations aside from the pagecache
ones in ints write_begin function, it may now use GFP_KERNEL (rather than
GFP_NOFS) for common case allocations (eg. ocfs2_alloc_write_ctxt, for a
random example).
[kosaki.motohiro@jp.fujitsu.com: fix ubifs]
[kosaki.motohiro@jp.fujitsu.com: fix fuse]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: <stable@kernel.org> [2.6.28.x]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
[ Cleaned up the calling convention: just pass in the AOP flags
untouched to the grab_cache_page_write_begin() function. That
just simplifies everybody, and may even allow future expansion of the
logic. - Linus ]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-01-04 20:00:53 +00:00
|
|
|
page = grab_cache_page_write_begin(mapping, index, flags);
|
2007-10-16 08:25:25 +00:00
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
|
|
|
*pagep = page;
|
|
|
|
*fsdata = NULL;
|
|
|
|
|
|
|
|
if (page_has_buffers(page)) {
|
2010-10-25 06:01:12 +00:00
|
|
|
ret = __block_write_begin(page, pos, len, get_block);
|
|
|
|
if (unlikely(ret))
|
|
|
|
goto out_release;
|
|
|
|
return ret;
|
2007-10-16 08:25:25 +00:00
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (PageMappedToDisk(page))
|
|
|
|
return 0;
|
|
|
|
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
/*
|
|
|
|
* Allocate buffers so that we can keep track of state, and potentially
|
|
|
|
* attach them to the page if an error occurs. In the common case of
|
|
|
|
* no error, they will just be freed again without ever being attached
|
|
|
|
* to the page (which is all OK, because we're under the page lock).
|
|
|
|
*
|
|
|
|
* Be careful: the buffer linked list is a NULL terminated one, rather
|
|
|
|
* than the circular one we're used to.
|
|
|
|
*/
|
|
|
|
head = alloc_page_buffers(page, blocksize, 0);
|
2007-10-16 08:25:25 +00:00
|
|
|
if (!head) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out_release;
|
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We loop across all blocks in the page, whether or not they are
|
|
|
|
* part of the affected region. This is so we can discover if the
|
|
|
|
* page is fully mapped-to-disk.
|
|
|
|
*/
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
for (block_start = 0, block_in_page = 0, bh = head;
|
2005-04-16 22:20:36 +00:00
|
|
|
block_start < PAGE_CACHE_SIZE;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
|
2005-04-16 22:20:36 +00:00
|
|
|
int create;
|
|
|
|
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
block_end = block_start + blocksize;
|
|
|
|
bh->b_state = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
create = 1;
|
|
|
|
if (block_start >= to)
|
|
|
|
create = 0;
|
|
|
|
ret = get_block(inode, block_in_file + block_in_page,
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
bh, create);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (ret)
|
|
|
|
goto failed;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
if (!buffer_mapped(bh))
|
2005-04-16 22:20:36 +00:00
|
|
|
is_mapped_to_disk = 0;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
if (buffer_new(bh))
|
|
|
|
unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
set_buffer_uptodate(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
}
|
|
|
|
if (buffer_new(bh) || !buffer_mapped(bh)) {
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user_segments(page, block_start, from,
|
|
|
|
to, block_end);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
if (buffer_uptodate(bh))
|
2005-04-16 22:20:36 +00:00
|
|
|
continue; /* reiserfs does this */
|
|
|
|
if (block_start < from || block_end > to) {
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
lock_buffer(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_nobh;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
nr_reads++;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (nr_reads) {
|
|
|
|
/*
|
|
|
|
* The page is locked, so these buffers are protected from
|
|
|
|
* any VM or truncate activity. Hence we don't need to care
|
|
|
|
* for the buffer_head refcounts.
|
|
|
|
*/
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
for (bh = head; bh; bh = bh->b_this_page) {
|
2005-04-16 22:20:36 +00:00
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
ret = -EIO;
|
|
|
|
}
|
|
|
|
if (ret)
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (is_mapped_to_disk)
|
|
|
|
SetPageMappedToDisk(page);
|
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
*fsdata = head; /* to be released by nobh_write_end */
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
failed:
|
2007-10-16 08:25:25 +00:00
|
|
|
BUG_ON(!ret);
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
* Error recovery is a bit difficult. We need to zero out blocks that
|
|
|
|
* were newly allocated, and dirty them to ensure they get written out.
|
|
|
|
* Buffers need to be attached to the page at this point, otherwise
|
|
|
|
* the handling of potential IO errors during writeout would be hard
|
|
|
|
* (could try doing synchronous writeout, but what if that fails too?)
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-10-16 08:25:25 +00:00
|
|
|
attach_nobh_buffers(page, head);
|
|
|
|
page_zero_new_buffers(page, from, to);
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
out_release:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
*pagep = NULL;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 15:05:33 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2007-10-16 08:25:25 +00:00
|
|
|
EXPORT_SYMBOL(nobh_write_begin);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
int nobh_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
2007-10-21 04:57:41 +00:00
|
|
|
struct buffer_head *head = fsdata;
|
2007-10-16 08:25:25 +00:00
|
|
|
struct buffer_head *bh;
|
vfs: fix data leak in nobh_write_end()
Current nobh_write_end() implementation ignore partial writes(copied < len)
case if page was fully mapped and simply mark page as Uptodate, which is
totally wrong because area [pos+copied, pos+len) wasn't updated explicitly in
previous write_begin call. It simply contains garbage from pagecache and
result in data leakage.
#TEST_CASE_BEGIN:
~~~~~~~~~~~~~~~~
In fact issue triggered by classical testcase
open("/mnt/test", O_RDWR|O_CREAT|O_TRUNC, 0666) = 3
ftruncate(3, 409600) = 0
writev(3, [{"a", 1}, {NULL, 4095}], 2) = 1
##TESTCASE_SOURCE:
~~~~~~~~~~~~~~~~~
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <errno.h>
int main(int argc, char **argv)
{
int fd, ret;
void* p;
struct iovec iov[2];
fd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
ftruncate(fd, 409600);
iov[0].iov_base="a";
iov[0].iov_len=1;
iov[1].iov_base=NULL;
iov[1].iov_len=4096;
ret = writev(fd, iov, sizeof(iov)/sizeof(struct iovec));
printf("writev = %d, err = %d\n", ret, errno);
return 0;
}
##TESTCASE RESULT:
~~~~~~~~~~~~~~~~~~
[root@ts63 ~]# mount | grep mnt2
/dev/mapper/test on /mnt2 type ext2 (rw,nobh)
[root@ts63 ~]# /tmp/writev /mnt2/test
writev = 1, err = 0
[root@ts63 ~]# hexdump -C /mnt2/test
00000000 61 65 62 6f 6f 74 00 00 f0 b9 b4 59 3a 00 00 00 |aeboot.....Y:...|
00000010 20 00 00 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000020 df df df df df df df df df df df df df df df df |................|
00000030 3a 00 00 00 2a 00 00 00 21 00 00 00 00 00 00 00 |:...*...!.......|
00000040 60 c0 8c 00 00 00 00 00 40 4a 8d 00 00 00 00 00 |`.......@J......|
00000050 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000060 74 69 6d 65 20 64 64 20 69 66 3d 2f 64 65 76 2f |time dd if=/dev/|
00000070 6c 6f 6f 70 30 20 20 6f 66 3d 2f 64 65 76 2f 6e |loop0 of=/dev/n|
skip..
00000f50 00 00 00 00 00 00 00 00 31 00 00 00 00 00 00 00 |........1.......|
00000f60 6d 6b 66 73 2e 65 78 74 33 20 2f 64 65 76 2f 76 |mkfs.ext3 /dev/v|
00000f70 7a 76 67 2f 74 65 73 74 20 2d 62 34 30 39 36 00 |zvg/test -b4096.|
00000f80 a0 fe 8c 00 00 00 00 00 21 00 00 00 00 00 00 00 |........!.......|
00000f90 23 31 32 30 35 39 35 30 34 30 34 00 3a 00 00 00 |#1205950404.:...|
00000fa0 20 00 8d 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000fb0 d0 cf 8c 00 00 00 00 00 10 d0 8c 00 00 00 00 00 |................|
00000fc0 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000fd0 6d 6f 75 6e 74 20 2f 64 65 76 2f 76 7a 76 67 2f |mount /dev/vzvg/|
00000fe0 74 65 73 74 20 20 2f 76 7a 20 2d 6f 20 64 61 74 |test /vz -o dat|
00000ff0 61 3d 77 72 69 74 65 62 61 63 6b 00 00 00 00 00 |a=writeback.....|
00001000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
As you can see file's page contains garbage from pagecache instead of zeros.
#TEST_CASE_END
Attached patch:
- Add sanity check BUG_ON in order to prevent incorrect usage by caller,
This is function invariant because page can has buffers and in no zero
*fadata pointer at the same time.
- Always attach buffers to page is it is partial write case.
- Always switch back to generic_write_end if page has buffers.
This is reasonable because if page already has buffer then generic_write_begin
was called previously.
Signed-off-by: Dmitri Monakhov <dmonakhov@openvz.org>
Reviewed-by: Nick Piggin <npiggin@suse.de>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-03-28 21:15:52 +00:00
|
|
|
BUG_ON(fsdata != NULL && page_has_buffers(page));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-02-06 20:59:26 +00:00
|
|
|
if (unlikely(copied < len) && head)
|
vfs: fix data leak in nobh_write_end()
Current nobh_write_end() implementation ignore partial writes(copied < len)
case if page was fully mapped and simply mark page as Uptodate, which is
totally wrong because area [pos+copied, pos+len) wasn't updated explicitly in
previous write_begin call. It simply contains garbage from pagecache and
result in data leakage.
#TEST_CASE_BEGIN:
~~~~~~~~~~~~~~~~
In fact issue triggered by classical testcase
open("/mnt/test", O_RDWR|O_CREAT|O_TRUNC, 0666) = 3
ftruncate(3, 409600) = 0
writev(3, [{"a", 1}, {NULL, 4095}], 2) = 1
##TESTCASE_SOURCE:
~~~~~~~~~~~~~~~~~
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <errno.h>
int main(int argc, char **argv)
{
int fd, ret;
void* p;
struct iovec iov[2];
fd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
ftruncate(fd, 409600);
iov[0].iov_base="a";
iov[0].iov_len=1;
iov[1].iov_base=NULL;
iov[1].iov_len=4096;
ret = writev(fd, iov, sizeof(iov)/sizeof(struct iovec));
printf("writev = %d, err = %d\n", ret, errno);
return 0;
}
##TESTCASE RESULT:
~~~~~~~~~~~~~~~~~~
[root@ts63 ~]# mount | grep mnt2
/dev/mapper/test on /mnt2 type ext2 (rw,nobh)
[root@ts63 ~]# /tmp/writev /mnt2/test
writev = 1, err = 0
[root@ts63 ~]# hexdump -C /mnt2/test
00000000 61 65 62 6f 6f 74 00 00 f0 b9 b4 59 3a 00 00 00 |aeboot.....Y:...|
00000010 20 00 00 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000020 df df df df df df df df df df df df df df df df |................|
00000030 3a 00 00 00 2a 00 00 00 21 00 00 00 00 00 00 00 |:...*...!.......|
00000040 60 c0 8c 00 00 00 00 00 40 4a 8d 00 00 00 00 00 |`.......@J......|
00000050 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000060 74 69 6d 65 20 64 64 20 69 66 3d 2f 64 65 76 2f |time dd if=/dev/|
00000070 6c 6f 6f 70 30 20 20 6f 66 3d 2f 64 65 76 2f 6e |loop0 of=/dev/n|
skip..
00000f50 00 00 00 00 00 00 00 00 31 00 00 00 00 00 00 00 |........1.......|
00000f60 6d 6b 66 73 2e 65 78 74 33 20 2f 64 65 76 2f 76 |mkfs.ext3 /dev/v|
00000f70 7a 76 67 2f 74 65 73 74 20 2d 62 34 30 39 36 00 |zvg/test -b4096.|
00000f80 a0 fe 8c 00 00 00 00 00 21 00 00 00 00 00 00 00 |........!.......|
00000f90 23 31 32 30 35 39 35 30 34 30 34 00 3a 00 00 00 |#1205950404.:...|
00000fa0 20 00 8d 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000fb0 d0 cf 8c 00 00 00 00 00 10 d0 8c 00 00 00 00 00 |................|
00000fc0 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000fd0 6d 6f 75 6e 74 20 2f 64 65 76 2f 76 7a 76 67 2f |mount /dev/vzvg/|
00000fe0 74 65 73 74 20 20 2f 76 7a 20 2d 6f 20 64 61 74 |test /vz -o dat|
00000ff0 61 3d 77 72 69 74 65 62 61 63 6b 00 00 00 00 00 |a=writeback.....|
00001000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
As you can see file's page contains garbage from pagecache instead of zeros.
#TEST_CASE_END
Attached patch:
- Add sanity check BUG_ON in order to prevent incorrect usage by caller,
This is function invariant because page can has buffers and in no zero
*fadata pointer at the same time.
- Always attach buffers to page is it is partial write case.
- Always switch back to generic_write_end if page has buffers.
This is reasonable because if page already has buffer then generic_write_begin
was called previously.
Signed-off-by: Dmitri Monakhov <dmonakhov@openvz.org>
Reviewed-by: Nick Piggin <npiggin@suse.de>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-03-28 21:15:52 +00:00
|
|
|
attach_nobh_buffers(page, head);
|
|
|
|
if (page_has_buffers(page))
|
|
|
|
return generic_write_end(file, mapping, pos, len,
|
|
|
|
copied, page, fsdata);
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 08:24:48 +00:00
|
|
|
|
2007-02-20 21:58:09 +00:00
|
|
|
SetPageUptodate(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
set_page_dirty(page);
|
2007-10-16 08:25:25 +00:00
|
|
|
if (pos+copied > inode->i_size) {
|
|
|
|
i_size_write(inode, pos+copied);
|
2005-04-16 22:20:36 +00:00
|
|
|
mark_inode_dirty(inode);
|
|
|
|
}
|
2007-10-16 08:25:25 +00:00
|
|
|
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
|
|
|
while (head) {
|
|
|
|
bh = head;
|
|
|
|
head = head->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
return copied;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2007-10-16 08:25:25 +00:00
|
|
|
EXPORT_SYMBOL(nobh_write_end);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* nobh_writepage() - based on block_full_write_page() except
|
|
|
|
* that it tries to operate without attaching bufferheads to
|
|
|
|
* the page.
|
|
|
|
*/
|
|
|
|
int nobh_writepage(struct page *page, get_block_t *get_block,
|
|
|
|
struct writeback_control *wbc)
|
|
|
|
{
|
|
|
|
struct inode * const inode = page->mapping->host;
|
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Is the page fully inside i_size? */
|
|
|
|
if (page->index < end_index)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* Is the page fully outside i_size? (truncate in progress) */
|
|
|
|
offset = i_size & (PAGE_CACHE_SIZE-1);
|
|
|
|
if (page->index >= end_index+1 || !offset) {
|
|
|
|
/*
|
|
|
|
* The page may have dirty, unmapped buffers. For example,
|
|
|
|
* they may have been added in ext3_writepage(). Make them
|
|
|
|
* freeable here, so the page does not leak.
|
|
|
|
*/
|
|
|
|
#if 0
|
|
|
|
/* Not really sure about this - do we need this ? */
|
|
|
|
if (page->mapping->a_ops->invalidatepage)
|
|
|
|
page->mapping->a_ops->invalidatepage(page, offset);
|
|
|
|
#endif
|
|
|
|
unlock_page(page);
|
|
|
|
return 0; /* don't care */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page straddles i_size. It must be zeroed out on each and every
|
|
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
|
|
|
* in multiples of the page size. For a file that is not a multiple of
|
|
|
|
* the page size, the remaining memory is zeroed when mapped, and
|
|
|
|
* writes to that region are not written out to the file."
|
|
|
|
*/
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user_segment(page, offset, PAGE_CACHE_SIZE);
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
ret = mpage_writepage(page, get_block, wbc);
|
|
|
|
if (ret == -EAGAIN)
|
2009-04-15 17:22:38 +00:00
|
|
|
ret = __block_write_full_page(inode, page, get_block, wbc,
|
|
|
|
end_buffer_async_write);
|
2005-04-16 22:20:36 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(nobh_writepage);
|
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
int nobh_truncate_page(struct address_space *mapping,
|
|
|
|
loff_t from, get_block_t *get_block)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
pgoff_t index = from >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset = from & (PAGE_CACHE_SIZE-1);
|
2007-10-16 08:25:25 +00:00
|
|
|
unsigned blocksize;
|
|
|
|
sector_t iblock;
|
|
|
|
unsigned length, pos;
|
|
|
|
struct inode *inode = mapping->host;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct page *page;
|
2007-10-16 08:25:25 +00:00
|
|
|
struct buffer_head map_bh;
|
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
length = offset & (blocksize - 1);
|
|
|
|
|
|
|
|
/* Block boundary? Nothing to do */
|
|
|
|
if (!length)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
length = blocksize - length;
|
|
|
|
iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
page = grab_cache_page(mapping, index);
|
2007-10-16 08:25:25 +00:00
|
|
|
err = -ENOMEM;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
2007-10-16 08:25:25 +00:00
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
has_buffers:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
return block_truncate_page(mapping, from, get_block);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Find the buffer that contains "offset" */
|
|
|
|
pos = blocksize;
|
|
|
|
while (offset >= pos) {
|
|
|
|
iblock++;
|
|
|
|
pos += blocksize;
|
|
|
|
}
|
|
|
|
|
Fix nobh_truncate_page() to not pass stack garbage to get_block()
The nobh_truncate_page() function is used by ext2, exofs, and jfs. Of
these three, only ext2 and jfs's get_block() function pays attention
to bh->b_size --- which is normally always the filesystem blocksize
except when the get_block() function is called by either
mpage_readpage(), mpage_readpages(), or the direct I/O routines in
fs/direct_io.c.
Unfortunately, nobh_truncate_page() does not initialize map_bh before
calling the filesystem-supplied get_block() function. So ext2 and jfs
will try to calculate the number of blocks to map by taking stack
garbage and shifting it left by inode->i_blkbits. This should be
*mostly* harmless (except the filesystem will do some unnneeded work)
unless the stack garbage is less than filesystem's blocksize, in which
case maxblocks will be zero, and the attempt to find out whether or
not the filesystem has a hole at a given logical block will fail, and
the page cache entry might not get zero'ed out.
Also if the stack garbage in in map_bh->state happens to have the
BH_Mapped bit set, there could be an attempt to call readpage() on a
non-existent page, which could cause nobh_truncate_page() to return an
error when it should not.
Fix this by initializing map_bh->state and map_bh->size.
Fortunately, it's probably fairly unlikely that ext2 and jfs users
mount with nobh these days.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
Cc: Dave Kleikamp <shaggy@linux.vnet.ibm.com>
Cc: linux-fsdevel@vger.kernel.org
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2009-05-12 11:37:56 +00:00
|
|
|
map_bh.b_size = blocksize;
|
|
|
|
map_bh.b_state = 0;
|
2007-10-16 08:25:25 +00:00
|
|
|
err = get_block(inode, iblock, &map_bh, 0);
|
|
|
|
if (err)
|
|
|
|
goto unlock;
|
|
|
|
/* unmapped? It's a hole - nothing to do */
|
|
|
|
if (!buffer_mapped(&map_bh))
|
|
|
|
goto unlock;
|
|
|
|
|
|
|
|
/* Ok, it's mapped. Make sure it's up-to-date */
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
err = mapping->a_ops->readpage(NULL, page);
|
|
|
|
if (err) {
|
|
|
|
page_cache_release(page);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
lock_page(page);
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
err = -EIO;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
if (page_has_buffers(page))
|
|
|
|
goto has_buffers;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, offset, length);
|
2007-10-16 08:25:25 +00:00
|
|
|
set_page_dirty(page);
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
unlock:
|
2005-04-16 22:20:36 +00:00
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
2007-10-16 08:25:25 +00:00
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(nobh_truncate_page);
|
|
|
|
|
|
|
|
int block_truncate_page(struct address_space *mapping,
|
|
|
|
loff_t from, get_block_t *get_block)
|
|
|
|
{
|
|
|
|
pgoff_t index = from >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset = from & (PAGE_CACHE_SIZE-1);
|
|
|
|
unsigned blocksize;
|
2006-01-08 09:03:05 +00:00
|
|
|
sector_t iblock;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned length, pos;
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
struct page *page;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
length = offset & (blocksize - 1);
|
|
|
|
|
|
|
|
/* Block boundary? Nothing to do */
|
|
|
|
if (!length)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
length = blocksize - length;
|
2006-01-08 09:03:05 +00:00
|
|
|
iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
page = grab_cache_page(mapping, index);
|
|
|
|
err = -ENOMEM;
|
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
create_empty_buffers(page, blocksize, 0);
|
|
|
|
|
|
|
|
/* Find the buffer that contains "offset" */
|
|
|
|
bh = page_buffers(page);
|
|
|
|
pos = blocksize;
|
|
|
|
while (offset >= pos) {
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
iblock++;
|
|
|
|
pos += blocksize;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 09:38:00 +00:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 22:20:36 +00:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
|
|
|
if (err)
|
|
|
|
goto unlock;
|
|
|
|
/* unmapped? It's a hole - nothing to do */
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Ok, it's mapped. Make sure it's up-to-date */
|
|
|
|
if (PageUptodate(page))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
|
2007-02-12 08:51:41 +00:00
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
err = -EIO;
|
|
|
|
ll_rw_block(READ, 1, &bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
/* Uhhuh. Read error. Complain and punt. */
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user(page, offset, length);
|
2005-04-16 22:20:36 +00:00
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(block_truncate_page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The generic ->writepage function for buffer-backed address_spaces
|
|
|
|
*/
|
2014-06-04 23:07:43 +00:00
|
|
|
int block_write_full_page(struct page *page, get_block_t *get_block,
|
|
|
|
struct writeback_control *wbc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inode * const inode = page->mapping->host;
|
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset;
|
|
|
|
|
|
|
|
/* Is the page fully inside i_size? */
|
|
|
|
if (page->index < end_index)
|
2009-04-15 17:22:38 +00:00
|
|
|
return __block_write_full_page(inode, page, get_block, wbc,
|
2014-06-04 23:07:43 +00:00
|
|
|
end_buffer_async_write);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Is the page fully outside i_size? (truncate in progress) */
|
|
|
|
offset = i_size & (PAGE_CACHE_SIZE-1);
|
|
|
|
if (page->index >= end_index+1 || !offset) {
|
|
|
|
/*
|
|
|
|
* The page may have dirty, unmapped buffers. For example,
|
|
|
|
* they may have been added in ext3_writepage(). Make them
|
|
|
|
* freeable here, so the page does not leak.
|
|
|
|
*/
|
2013-05-22 03:17:23 +00:00
|
|
|
do_invalidatepage(page, 0, PAGE_CACHE_SIZE);
|
2005-04-16 22:20:36 +00:00
|
|
|
unlock_page(page);
|
|
|
|
return 0; /* don't care */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page straddles i_size. It must be zeroed out on each and every
|
2009-12-11 21:35:40 +00:00
|
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
2005-04-16 22:20:36 +00:00
|
|
|
* in multiples of the page size. For a file that is not a multiple of
|
|
|
|
* the page size, the remaining memory is zeroed when mapped, and
|
|
|
|
* writes to that region are not written out to the file."
|
|
|
|
*/
|
2008-02-05 06:28:29 +00:00
|
|
|
zero_user_segment(page, offset, PAGE_CACHE_SIZE);
|
2014-06-04 23:07:43 +00:00
|
|
|
return __block_write_full_page(inode, page, get_block, wbc,
|
|
|
|
end_buffer_async_write);
|
2009-04-15 17:22:38 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(block_write_full_page);
|
2009-04-15 17:22:38 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
|
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
|
|
|
struct buffer_head tmp;
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
tmp.b_state = 0;
|
|
|
|
tmp.b_blocknr = 0;
|
2006-03-26 09:38:00 +00:00
|
|
|
tmp.b_size = 1 << inode->i_blkbits;
|
2005-04-16 22:20:36 +00:00
|
|
|
get_block(inode, block, &tmp, 0);
|
|
|
|
return tmp.b_blocknr;
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(generic_block_bmap);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-07-20 13:29:37 +00:00
|
|
|
static void end_bio_bh_io_sync(struct bio *bio)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = bio->bi_private;
|
|
|
|
|
2015-07-24 18:37:59 +00:00
|
|
|
if (unlikely(bio_flagged(bio, BIO_QUIET)))
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 09:24:35 +00:00
|
|
|
set_bit(BH_Quiet, &bh->b_state);
|
|
|
|
|
2015-07-20 13:29:37 +00:00
|
|
|
bh->b_end_io(bh, !bio->bi_error);
|
2005-04-16 22:20:36 +00:00
|
|
|
bio_put(bio);
|
|
|
|
}
|
|
|
|
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
/*
|
|
|
|
* This allows us to do IO even on the odd last sectors
|
2014-10-09 22:26:53 +00:00
|
|
|
* of a device, even if the block size is some multiple
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
* of the physical sector size.
|
|
|
|
*
|
|
|
|
* We'll just truncate the bio to the size of the device,
|
|
|
|
* and clear the end of the buffer head manually.
|
|
|
|
*
|
|
|
|
* Truly out-of-range accesses will turn into actual IO
|
|
|
|
* errors, this only handles the "we need to be able to
|
|
|
|
* do IO at the final sector" case.
|
|
|
|
*/
|
2014-10-09 22:26:55 +00:00
|
|
|
void guard_bio_eod(int rw, struct bio *bio)
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
{
|
|
|
|
sector_t maxsector;
|
2014-10-09 22:26:53 +00:00
|
|
|
struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
|
|
|
|
unsigned truncated_bytes;
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
|
|
|
|
maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
|
|
|
|
if (!maxsector)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the *whole* IO is past the end of the device,
|
|
|
|
* let it through, and the IO layer will turn it into
|
|
|
|
* an EIO.
|
|
|
|
*/
|
2013-10-11 22:44:27 +00:00
|
|
|
if (unlikely(bio->bi_iter.bi_sector >= maxsector))
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
return;
|
|
|
|
|
2013-10-11 22:44:27 +00:00
|
|
|
maxsector -= bio->bi_iter.bi_sector;
|
2014-10-09 22:26:53 +00:00
|
|
|
if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
return;
|
|
|
|
|
2014-10-09 22:26:53 +00:00
|
|
|
/* Uhhuh. We've got a bio that straddles the device size! */
|
|
|
|
truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
|
|
|
|
/* Truncate the bio.. */
|
2014-10-09 22:26:53 +00:00
|
|
|
bio->bi_iter.bi_size -= truncated_bytes;
|
|
|
|
bvec->bv_len -= truncated_bytes;
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
|
|
|
|
/* ..and clear the end of the buffer for reads */
|
2012-12-05 17:01:24 +00:00
|
|
|
if ((rw & RW_MASK) == READ) {
|
2014-10-09 22:26:53 +00:00
|
|
|
zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
|
|
|
|
truncated_bytes);
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-02 14:39:48 +00:00
|
|
|
static int submit_bh_wbc(int rw, struct buffer_head *bh,
|
|
|
|
unsigned long bio_flags, struct writeback_control *wbc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct bio *bio;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_locked(bh));
|
|
|
|
BUG_ON(!buffer_mapped(bh));
|
|
|
|
BUG_ON(!bh->b_end_io);
|
2009-05-12 20:22:37 +00:00
|
|
|
BUG_ON(buffer_delay(bh));
|
|
|
|
BUG_ON(buffer_unwritten(bh));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2008-08-22 08:00:36 +00:00
|
|
|
* Only clear out a write error when rewriting
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2008-08-22 08:00:36 +00:00
|
|
|
if (test_set_buffer_req(bh) && (rw & WRITE))
|
2005-04-16 22:20:36 +00:00
|
|
|
clear_buffer_write_io_error(bh);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* from here on down, it's all bio -- do the initial mapping,
|
|
|
|
* submit_bio -> generic_make_request may further map this bio around
|
|
|
|
*/
|
|
|
|
bio = bio_alloc(GFP_NOIO, 1);
|
|
|
|
|
2015-05-28 18:50:51 +00:00
|
|
|
if (wbc) {
|
2015-06-02 14:39:48 +00:00
|
|
|
wbc_init_bio(wbc, bio);
|
2015-05-28 18:50:51 +00:00
|
|
|
wbc_account_io(wbc, bh->b_page, bh->b_size);
|
|
|
|
}
|
2015-06-02 14:37:23 +00:00
|
|
|
|
2013-10-11 22:44:27 +00:00
|
|
|
bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
|
2005-04-16 22:20:36 +00:00
|
|
|
bio->bi_bdev = bh->b_bdev;
|
|
|
|
|
2014-12-22 11:48:42 +00:00
|
|
|
bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
|
|
|
|
BUG_ON(bio->bi_iter.bi_size != bh->b_size);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
bio->bi_end_io = end_bio_bh_io_sync;
|
|
|
|
bio->bi_private = bh;
|
2013-04-29 22:07:25 +00:00
|
|
|
bio->bi_flags |= bio_flags;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
/* Take care of bh's that straddle the end of the device */
|
2014-10-09 22:26:53 +00:00
|
|
|
guard_bio_eod(rw, bio);
|
vfs: avoid "attempt to access beyond end of device" warnings
The block device access simplification that avoided accessing the (racy)
block size information (commit bbec0270bdd8: "blkdev_max_block: make
private to fs/buffer.c") no longer checks the maximum block size in the
block mapping path.
That was _almost_ as simple as just removing the code entirely, because
the readers and writers all check the size of the device anyway, so
under normal circumstances it "just worked".
However, the block size may be such that the end of the device may
straddle one single buffer_head. At which point we may still want to
access the end of the device, but the buffer we use to access it
partially extends past the end.
The 'bd_set_size()' function intentionally sets the block size to avoid
this, but mounting the device - or setting the block size by hand to
some other value - can modify that block size.
So instead, teach 'submit_bh()' about the special case of the buffer
head straddling the end of the device, and turning such an access into a
smaller IO access, avoiding the problem.
This, btw, also means that unlike before, we can now access the whole
device regardless of device block size setting. So now, even if the
device size is only 512-byte aligned, we can read and write even the
last sector even when having a much bigger block size for accessing the
rest of the device.
So with this, we could now get rid of the 'bd_set_size()' block size
code entirely - resulting in faster IO for the common case - but that
would be a separate patch.
Reported-and-tested-by: Romain Francoise <romain@orebokech.com>
Reporeted-and-tested-by: Meelis Roos <mroos@linux.ee>
Reported-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-12-04 16:25:11 +00:00
|
|
|
|
2013-04-20 23:58:37 +00:00
|
|
|
if (buffer_meta(bh))
|
|
|
|
rw |= REQ_META;
|
|
|
|
if (buffer_prio(bh))
|
|
|
|
rw |= REQ_PRIO;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
submit_bio(rw, bio);
|
2015-05-26 19:59:53 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2015-06-02 14:37:23 +00:00
|
|
|
|
|
|
|
int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)
|
|
|
|
{
|
2015-06-02 14:39:48 +00:00
|
|
|
return submit_bh_wbc(rw, bh, bio_flags, NULL);
|
2015-06-02 14:37:23 +00:00
|
|
|
}
|
2013-04-29 22:07:25 +00:00
|
|
|
EXPORT_SYMBOL_GPL(_submit_bh);
|
|
|
|
|
|
|
|
int submit_bh(int rw, struct buffer_head *bh)
|
|
|
|
{
|
2015-06-02 14:39:48 +00:00
|
|
|
return submit_bh_wbc(rw, bh, 0, NULL);
|
2013-04-29 22:07:25 +00:00
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(submit_bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* ll_rw_block: low-level access to block devices (DEPRECATED)
|
2010-08-11 15:06:24 +00:00
|
|
|
* @rw: whether to %READ or %WRITE or maybe %READA (readahead)
|
2005-04-16 22:20:36 +00:00
|
|
|
* @nr: number of &struct buffer_heads in the array
|
|
|
|
* @bhs: array of pointers to &struct buffer_head
|
|
|
|
*
|
2005-09-06 22:19:10 +00:00
|
|
|
* ll_rw_block() takes an array of pointers to &struct buffer_heads, and
|
|
|
|
* requests an I/O operation on them, either a %READ or a %WRITE. The third
|
2010-08-11 15:06:24 +00:00
|
|
|
* %READA option is described in the documentation for generic_make_request()
|
|
|
|
* which ll_rw_block() calls.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* This function drops any buffer that it cannot get a lock on (with the
|
2010-08-11 15:06:24 +00:00
|
|
|
* BH_Lock state bit), any buffer that appears to be clean when doing a write
|
|
|
|
* request, and any buffer that appears to be up-to-date when doing read
|
|
|
|
* request. Further it marks as clean buffers that are processed for
|
|
|
|
* writing (the buffer cache won't assume that they are actually clean
|
|
|
|
* until the buffer gets unlocked).
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* ll_rw_block sets b_end_io to simple completion handler that marks
|
2014-02-18 13:54:36 +00:00
|
|
|
* the buffer up-to-date (if appropriate), unlocks the buffer and wakes
|
2005-04-16 22:20:36 +00:00
|
|
|
* any waiters.
|
|
|
|
*
|
|
|
|
* All of the buffers must be for the same device, and must also be a
|
|
|
|
* multiple of the current approved size for the device.
|
|
|
|
*/
|
|
|
|
void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
struct buffer_head *bh = bhs[i];
|
|
|
|
|
2010-08-11 15:06:24 +00:00
|
|
|
if (!trylock_buffer(bh))
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
2010-08-11 15:06:24 +00:00
|
|
|
if (rw == WRITE) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
2005-04-16 22:24:07 +00:00
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
2006-02-03 11:04:43 +00:00
|
|
|
get_bh(bh);
|
2010-08-11 15:06:24 +00:00
|
|
|
submit_bh(WRITE, bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (!buffer_uptodate(bh)) {
|
2005-04-16 22:24:07 +00:00
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
2006-02-03 11:04:43 +00:00
|
|
|
get_bh(bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
submit_bh(rw, bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(ll_rw_block);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-08-11 15:06:24 +00:00
|
|
|
void write_dirty_buffer(struct buffer_head *bh, int rw)
|
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!test_clear_buffer_dirty(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
|
|
|
get_bh(bh);
|
|
|
|
submit_bh(rw, bh);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(write_dirty_buffer);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* For a data-integrity writeout, we need to wait upon any in-progress I/O
|
|
|
|
* and then start new I/O and then wait upon it. The caller must have a ref on
|
|
|
|
* the buffer_head.
|
|
|
|
*/
|
2010-08-11 15:05:45 +00:00
|
|
|
int __sync_dirty_buffer(struct buffer_head *bh, int rw)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
WARN_ON(atomic_read(&bh->b_count) < 1);
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
2010-08-11 15:05:45 +00:00
|
|
|
ret = submit_bh(rw, bh);
|
2005-04-16 22:20:36 +00:00
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!ret && !buffer_uptodate(bh))
|
|
|
|
ret = -EIO;
|
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
2010-08-11 15:05:45 +00:00
|
|
|
EXPORT_SYMBOL(__sync_dirty_buffer);
|
|
|
|
|
|
|
|
int sync_dirty_buffer(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
return __sync_dirty_buffer(bh, WRITE_SYNC);
|
|
|
|
}
|
2009-09-22 23:43:51 +00:00
|
|
|
EXPORT_SYMBOL(sync_dirty_buffer);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* try_to_free_buffers() checks if all the buffers on this particular page
|
|
|
|
* are unused, and releases them if so.
|
|
|
|
*
|
|
|
|
* Exclusion against try_to_free_buffers may be obtained by either
|
|
|
|
* locking the page or by holding its mapping's private_lock.
|
|
|
|
*
|
|
|
|
* If the page is dirty but all the buffers are clean then we need to
|
|
|
|
* be sure to mark the page clean as well. This is because the page
|
|
|
|
* may be against a block device, and a later reattachment of buffers
|
|
|
|
* to a dirty page will set *all* buffers dirty. Which would corrupt
|
|
|
|
* filesystem data on the same device.
|
|
|
|
*
|
|
|
|
* The same applies to regular filesystem pages: if all the buffers are
|
|
|
|
* clean then we set the page clean and proceed. To do that, we require
|
|
|
|
* total exclusion from __set_page_dirty_buffers(). That is obtained with
|
|
|
|
* private_lock.
|
|
|
|
*
|
|
|
|
* try_to_free_buffers() is non-blocking.
|
|
|
|
*/
|
|
|
|
static inline int buffer_busy(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
return atomic_read(&bh->b_count) |
|
|
|
|
(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
|
|
|
|
{
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
2005-05-01 15:58:39 +00:00
|
|
|
if (buffer_write_io_error(bh) && page->mapping)
|
2005-04-16 22:20:36 +00:00
|
|
|
set_bit(AS_EIO, &page->mapping->flags);
|
|
|
|
if (buffer_busy(bh))
|
|
|
|
goto failed;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
|
2008-02-08 12:21:59 +00:00
|
|
|
if (bh->b_assoc_map)
|
2005-04-16 22:20:36 +00:00
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
*buffers_to_free = head;
|
|
|
|
__clear_page_buffers(page);
|
|
|
|
return 1;
|
|
|
|
failed:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int try_to_free_buffers(struct page *page)
|
|
|
|
{
|
|
|
|
struct address_space * const mapping = page->mapping;
|
|
|
|
struct buffer_head *buffers_to_free = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 20:47:06 +00:00
|
|
|
if (PageWriteback(page))
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (mapping == NULL) { /* can this still happen? */
|
|
|
|
ret = drop_buffers(page, &buffers_to_free);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock(&mapping->private_lock);
|
|
|
|
ret = drop_buffers(page, &buffers_to_free);
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 20:47:06 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the filesystem writes its buffers by hand (eg ext3)
|
|
|
|
* then we can have clean buffers against a dirty page. We
|
|
|
|
* clean the page here; otherwise the VM will never notice
|
|
|
|
* that the filesystem did any IO at all.
|
|
|
|
*
|
|
|
|
* Also, during truncate, discard_buffer will have marked all
|
|
|
|
* the page's buffers clean. We discover that here and clean
|
|
|
|
* the page also.
|
2007-01-30 03:36:27 +00:00
|
|
|
*
|
|
|
|
* private_lock must be held over this entire operation in order
|
|
|
|
* to synchronise against __set_page_dirty_buffers and prevent the
|
|
|
|
* dirty bit from being lost.
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 20:47:06 +00:00
|
|
|
*/
|
2015-05-22 21:13:15 +00:00
|
|
|
if (ret)
|
|
|
|
cancel_dirty_page(page);
|
2007-01-30 03:36:27 +00:00
|
|
|
spin_unlock(&mapping->private_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
if (buffers_to_free) {
|
|
|
|
struct buffer_head *bh = buffers_to_free;
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != buffers_to_free);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(try_to_free_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* There are no bdflush tunables left. But distributions are
|
|
|
|
* still running obsolete flush daemons, so we terminate them here.
|
|
|
|
*
|
|
|
|
* Use of bdflush() is deprecated and will be removed in a future kernel.
|
2009-09-23 17:37:09 +00:00
|
|
|
* The `flush-X' kernel threads fully replace bdflush daemons and this call.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-01-14 13:14:12 +00:00
|
|
|
SYSCALL_DEFINE2(bdflush, int, func, long, data)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
static int msg_count;
|
|
|
|
|
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
|
|
return -EPERM;
|
|
|
|
|
|
|
|
if (msg_count < 5) {
|
|
|
|
msg_count++;
|
|
|
|
printk(KERN_INFO
|
|
|
|
"warning: process `%s' used the obsolete bdflush"
|
|
|
|
" system call\n", current->comm);
|
|
|
|
printk(KERN_INFO "Fix your initscripts?\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (func == 1)
|
|
|
|
do_exit(0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer-head allocation
|
|
|
|
*/
|
2012-05-15 09:29:52 +00:00
|
|
|
static struct kmem_cache *bh_cachep __read_mostly;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Once the number of bh's in the machine exceeds this level, we start
|
|
|
|
* stripping them in writeback.
|
|
|
|
*/
|
2013-02-23 00:35:46 +00:00
|
|
|
static unsigned long max_buffer_heads;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
int buffer_heads_over_limit;
|
|
|
|
|
|
|
|
struct bh_accounting {
|
|
|
|
int nr; /* Number of live bh's */
|
|
|
|
int ratelimit; /* Limit cacheline bouncing */
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
|
|
|
|
|
|
|
|
static void recalc_bh_state(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int tot = 0;
|
|
|
|
|
2010-12-06 17:40:05 +00:00
|
|
|
if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
2010-12-06 17:16:28 +00:00
|
|
|
__this_cpu_write(bh_accounting.ratelimit, 0);
|
2006-03-24 11:18:10 +00:00
|
|
|
for_each_online_cpu(i)
|
2005-04-16 22:20:36 +00:00
|
|
|
tot += per_cpu(bh_accounting, i).nr;
|
|
|
|
buffer_heads_over_limit = (tot > max_buffer_heads);
|
|
|
|
}
|
2010-12-06 17:16:28 +00:00
|
|
|
|
2005-10-07 06:46:04 +00:00
|
|
|
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-03-10 23:20:33 +00:00
|
|
|
struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (ret) {
|
2007-05-17 05:10:57 +00:00
|
|
|
INIT_LIST_HEAD(&ret->b_assoc_buffers);
|
2010-12-06 17:16:28 +00:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_inc(bh_accounting.nr);
|
2005-04-16 22:20:36 +00:00
|
|
|
recalc_bh_state();
|
2010-12-06 17:16:28 +00:00
|
|
|
preempt_enable();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(alloc_buffer_head);
|
|
|
|
|
|
|
|
void free_buffer_head(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
BUG_ON(!list_empty(&bh->b_assoc_buffers));
|
|
|
|
kmem_cache_free(bh_cachep, bh);
|
2010-12-06 17:16:28 +00:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_dec(bh_accounting.nr);
|
2005-04-16 22:20:36 +00:00
|
|
|
recalc_bh_state();
|
2010-12-06 17:16:28 +00:00
|
|
|
preempt_enable();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(free_buffer_head);
|
|
|
|
|
|
|
|
static void buffer_exit_cpu(int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct bh_lru *b = &per_cpu(bh_lrus, cpu);
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
2010-12-06 17:16:28 +00:00
|
|
|
this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
|
2006-03-24 11:18:10 +00:00
|
|
|
per_cpu(bh_accounting, cpu).nr = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int buffer_cpu_notify(struct notifier_block *self,
|
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
2007-05-09 09:35:10 +00:00
|
|
|
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
|
2005-04-16 22:20:36 +00:00
|
|
|
buffer_exit_cpu((unsigned long)hcpu);
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
2008-01-29 04:58:26 +00:00
|
|
|
/**
|
2008-03-20 00:01:00 +00:00
|
|
|
* bh_uptodate_or_lock - Test whether the buffer is uptodate
|
2008-01-29 04:58:26 +00:00
|
|
|
* @bh: struct buffer_head
|
|
|
|
*
|
|
|
|
* Return true if the buffer is up-to-date and false,
|
|
|
|
* with the buffer locked, if not.
|
|
|
|
*/
|
|
|
|
int bh_uptodate_or_lock(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
if (!buffer_uptodate(bh)) {
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
return 0;
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bh_uptodate_or_lock);
|
|
|
|
|
|
|
|
/**
|
2008-03-20 00:01:00 +00:00
|
|
|
* bh_submit_read - Submit a locked buffer for reading
|
2008-01-29 04:58:26 +00:00
|
|
|
* @bh: struct buffer_head
|
|
|
|
*
|
|
|
|
* Returns zero on success and -EIO on error.
|
|
|
|
*/
|
|
|
|
int bh_submit_read(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
BUG_ON(!buffer_locked(bh));
|
|
|
|
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
return 0;
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bh_submit_read);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void __init buffer_init(void)
|
|
|
|
{
|
2013-02-23 00:35:46 +00:00
|
|
|
unsigned long nrpages;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-02-05 06:28:36 +00:00
|
|
|
bh_cachep = kmem_cache_create("buffer_head",
|
|
|
|
sizeof(struct buffer_head), 0,
|
|
|
|
(SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
|
|
|
|
SLAB_MEM_SPREAD),
|
2010-03-10 23:20:33 +00:00
|
|
|
NULL);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Limit the bh occupancy to 10% of ZONE_NORMAL
|
|
|
|
*/
|
|
|
|
nrpages = (nr_free_buffer_pages() * 10) / 100;
|
|
|
|
max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
|
|
|
|
hotcpu_notifier(buffer_cpu_notify, 0);
|
|
|
|
}
|