Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-26 06:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

statx: optimize copy of struct statx to userspace

Browse Source 
I found that statx() was significantly slower than stat().  As a
microbenchmark, I compared 10,000,000 invocations of fstat() on a tmpfs
file to the same with statx() passed a NULL path:

	$ time ./stat_benchmark

	real	0m1.464s
	user	0m0.275s
	sys	0m1.187s

	$ time ./statx_benchmark

	real	0m5.530s
	user	0m0.281s
	sys	0m5.247s

statx is expected to be a little slower than stat because struct statx
is larger than struct stat, but not by *that* much.  It turns out that
most of the overhead was in copying struct statx to userspace, mostly in
all the stac/clac instructions that got generated for each __put_user()
call.  (This was on x86_64, but some other architectures, e.g. arm64,
have something similar now too.)

stat() instead initializes its struct on the stack and copies it to
userspace with a single call to copy_to_user().  This turns out to be
much faster, and changing statx to do this makes it almost as fast as
stat:

	$ time ./statx_benchmark

	real	0m1.624s
	user	0m0.270s
	sys	0m1.354s

For zeroing the reserved fields, start by zeroing the full struct with
memset.  This makes it clear that every byte copied to userspace is
initialized, even implicit padding bytes (though there are none
currently).  In the scenarios I tested, it also performed the same as a
designated initializer.  Manually initializing each field was still
slightly faster, but would have been more error-prone and less
verifiable.

Also rename statx_set_result() to cp_statx() for consistency with
cp_old_stat() et al., and make it noinline so that struct statx doesn't
add to the stack usage during the main portion of the syscall execution.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
This commit is contained in:
Eric Biggers 2017-03-31 18:31:48 +01:00 committed by Al Viro
parent b15fb70b82
commit 64bd72048a
1 changed files with 29 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
fs/stat.c
View File

@ -510,46 +510,37 @@ SYSCALL_DEFINE4(fstatat64, int, dfd, const char __user *, filename,
} }
#endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */ #endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */
static inline int __put_timestamp(struct timespec *kts, static noinline_for_stack int
struct statx_timestamp __user *uts) cp_statx(const struct kstat *stat, struct statx __user *buffer)
{ {
return (__put_user(kts->tv_sec, &uts->tv_sec ) || struct statx tmp;
__put_user(kts->tv_nsec, &uts->tv_nsec ) ||
__put_user(0, &uts->__reserved ));
}
/* memset(&tmp, 0, sizeof(tmp));
* Set the statx results.
*/
static long statx_set_result(struct kstat *stat, struct statx __user *buffer)
{
uid_t uid = from_kuid_munged(current_user_ns(), stat->uid);
gid_t gid = from_kgid_munged(current_user_ns(), stat->gid);
if (__put_user(stat->result_mask, &buffer->stx_mask ) || tmp.stx_mask = stat->result_mask;
__put_user(stat->mode, &buffer->stx_mode ) || tmp.stx_blksize = stat->blksize;
__clear_user(&buffer->__spare0, sizeof(buffer->__spare0)) || tmp.stx_attributes = stat->attributes;
__put_user(stat->nlink, &buffer->stx_nlink ) || tmp.stx_nlink = stat->nlink;
__put_user(uid, &buffer->stx_uid ) || tmp.stx_uid = from_kuid_munged(current_user_ns(), stat->uid);
__put_user(gid, &buffer->stx_gid ) || tmp.stx_gid = from_kgid_munged(current_user_ns(), stat->gid);
__put_user(stat->attributes, &buffer->stx_attributes ) || tmp.stx_mode = stat->mode;
__put_user(stat->blksize, &buffer->stx_blksize ) || tmp.stx_ino = stat->ino;
__put_user(MAJOR(stat->rdev), &buffer->stx_rdev_major ) || tmp.stx_size = stat->size;
__put_user(MINOR(stat->rdev), &buffer->stx_rdev_minor ) || tmp.stx_blocks = stat->blocks;
__put_user(MAJOR(stat->dev), &buffer->stx_dev_major ) || tmp.stx_atime.tv_sec = stat->atime.tv_sec;
__put_user(MINOR(stat->dev), &buffer->stx_dev_minor ) || tmp.stx_atime.tv_nsec = stat->atime.tv_nsec;
__put_timestamp(&stat->atime, &buffer->stx_atime ) || tmp.stx_btime.tv_sec = stat->btime.tv_sec;
__put_timestamp(&stat->btime, &buffer->stx_btime ) || tmp.stx_btime.tv_nsec = stat->btime.tv_nsec;
__put_timestamp(&stat->ctime, &buffer->stx_ctime ) || tmp.stx_ctime.tv_sec = stat->ctime.tv_sec;
__put_timestamp(&stat->mtime, &buffer->stx_mtime ) || tmp.stx_ctime.tv_nsec = stat->ctime.tv_nsec;
__put_user(stat->ino, &buffer->stx_ino ) || tmp.stx_mtime.tv_sec = stat->mtime.tv_sec;
__put_user(stat->size, &buffer->stx_size ) || tmp.stx_mtime.tv_nsec = stat->mtime.tv_nsec;
__put_user(stat->blocks, &buffer->stx_blocks ) || tmp.stx_rdev_major = MAJOR(stat->rdev);
__clear_user(&buffer->__spare1, sizeof(buffer->__spare1)) || tmp.stx_rdev_minor = MINOR(stat->rdev);
__clear_user(&buffer->__spare2, sizeof(buffer->__spare2))) tmp.stx_dev_major = MAJOR(stat->dev);
return -EFAULT; tmp.stx_dev_minor = MINOR(stat->dev);
return 0; return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0;
} }
/** /**
@ -573,8 +564,6 @@ SYSCALL_DEFINE5(statx,
if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE) if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE)
return -EINVAL; return -EINVAL;
if (!access_ok(VERIFY_WRITE, buffer, sizeof(*buffer)))
return -EFAULT;
if (filename) if (filename)
error = vfs_statx(dfd, filename, flags, &stat, mask); error = vfs_statx(dfd, filename, flags, &stat, mask);
@ -582,7 +571,8 @@ SYSCALL_DEFINE5(statx,
error = vfs_statx_fd(dfd, &stat, mask, flags); error = vfs_statx_fd(dfd, &stat, mask, flags);
if (error) if (error)
return error; return error;
return statx_set_result(&stat, buffer);
return cp_statx(&stat, buffer);
} }
/* Caller is here responsible for sufficient locking (ie. inode->i_lock) */ /* Caller is here responsible for sufficient locking (ie. inode->i_lock) */