leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib/vdso: Provide generic VDSO implementation

Browse Source 
In the last few years the kernel gained quite some architecture specific
vdso implementations which contain very similar code.

Introduce a generic VDSO implementation of gettimeofday() which will be
shareable between architectures once they are converted over.

The implementation is based on the current x86 VDSO code.

[ tglx: Massaged changelog and made the kernel doc tabular ]

Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Shijith Thotton <sthotton@marvell.com>
Tested-by: Andre Przywara <andre.przywara@arm.com>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-mips@vger.kernel.org
Cc: linux-kselftest@vger.kernel.org
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: Mark Salyzyn <salyzyn@android.com>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Dmitry Safonov <0x7f454c46@gmail.com>
Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk>
Cc: Huw Davies <huw@codeweavers.com>
Link: https://lkml.kernel.org/r/20190621095252.32307-3-vincenzo.frascino@arm.com
This commit is contained in:
Vincenzo Frascino 2019-06-21 10:52:29 +01:00 committed by Thomas Gleixner
parent 361f8aee9b
commit 00b26474c2
5 changed files with 343 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
include/vdso/helpers.h Normal file
View File

@ -0,0 +1,56 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __VDSO_HELPERS_H
#define __VDSO_HELPERS_H
#ifndef __ASSEMBLY__
#include <vdso/datapage.h>
static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
{
u32 seq;
while ((seq = READ_ONCE(vd->seq)) & 1)
cpu_relax();
smp_rmb();
return seq;
}
static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
u32 start)
{
u32 seq;
smp_rmb();
seq = READ_ONCE(vd->seq);
return seq != start;
}
static __always_inline void vdso_write_begin(struct vdso_data *vd)
{
/*
* WRITE_ONCE it is required otherwise the compiler can validly tear
* updates to vd[x].seq and it is possible that the value seen by the
* reader it is inconsistent.
*/
WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
smp_wmb();
}
static __always_inline void vdso_write_end(struct vdso_data *vd)
{
smp_wmb();
/*
* WRITE_ONCE it is required otherwise the compiler can validly tear
* updates to vd[x].seq and it is possible that the value seen by the
* reader it is inconsistent.
*/
WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
}
#endif /* !__ASSEMBLY__ */
#endif /* __VDSO_HELPERS_H */

5
lib/Kconfig
View File

@ -576,6 +576,11 @@ config OID_REGISTRY
config UCS2_STRING config UCS2_STRING
tristate tristate
#
# generic vdso
#
source "lib/vdso/Kconfig"
source "lib/fonts/Kconfig" source "lib/fonts/Kconfig"
config SG_SPLIT config SG_SPLIT

36
lib/vdso/Kconfig Normal file
View File

@ -0,0 +1,36 @@
# SPDX-License-Identifier: GPL-2.0
config HAVE_GENERIC_VDSO
bool
if HAVE_GENERIC_VDSO
config GENERIC_GETTIMEOFDAY
bool
help
This is a generic implementation of gettimeofday vdso.
Each architecture that enables this feature has to
provide the fallback implementation.
config GENERIC_VDSO_32
bool
depends on GENERIC_GETTIMEOFDAY && !64BIT
help
This config option helps to avoid possible performance issues
in 32 bit only architectures.
config GENERIC_COMPAT_VDSO
bool
help
This config option enables the compat VDSO layer.
config CROSS_COMPILE_COMPAT_VDSO
string "32 bit Toolchain prefix for compat vDSO"
default ""
depends on GENERIC_COMPAT_VDSO
help
Defines the cross-compiler prefix for compiling compat vDSO.
If a 64 bit compiler (i.e. x86_64) can compile the VDSO for
32 bit, it does not need to define this parameter.
endif

22
lib/vdso/Makefile Normal file
View File

@ -0,0 +1,22 @@
# SPDX-License-Identifier: GPL-2.0
GENERIC_VDSO_MK_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH))
c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c)
# This cmd checks that the vdso library does not contain absolute relocation
# It has to be called after the linking of the vdso library and requires it
# as a parameter.
#
# $(ARCH_REL_TYPE_ABS) is defined in the arch specific makefile and corresponds
# to the absolute relocation types printed by "objdump -R" and accepted by the
# dynamic linker.
ifndef ARCH_REL_TYPE_ABS
$(error ARCH_REL_TYPE_ABS is not set)
endif
quiet_cmd_vdso_check = VDSOCHK $@
cmd_vdso_check = if $(OBJDUMP) -R $@ | egrep -h "$(ARCH_REL_TYPE_ABS)"; \
then (echo >&2 "$@: dynamic relocations are not supported"; \
rm -f $@; /bin/false); fi

224
lib/vdso/gettimeofday.c Normal file
View File

@ -0,0 +1,224 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Generic userspace implementations of gettimeofday() and similar.
*/
#include <linux/compiler.h>
#include <linux/math64.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/hrtimer_defs.h>
#include <vdso/datapage.h>
#include <vdso/helpers.h>
/*
* The generic vDSO implementation requires that gettimeofday.h
* provides:
* - __arch_get_vdso_data(): to get the vdso datapage.
* - __arch_get_hw_counter(): to get the hw counter based on the
* clock_mode.
* - gettimeofday_fallback(): fallback for gettimeofday.
* - clock_gettime_fallback(): fallback for clock_gettime.
* - clock_getres_fallback(): fallback for clock_getres.
*/
#include <asm/vdso/gettimeofday.h>
static int do_hres(const struct vdso_data *vd, clockid_t clk,
struct __kernel_timespec *ts)
{
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
u64 cycles, last, sec, ns;
u32 seq;
do {
seq = vdso_read_begin(vd);
cycles = __arch_get_hw_counter(vd->clock_mode) &
vd->mask;
ns = vdso_ts->nsec;
last = vd->cycle_last;
if (unlikely((s64)cycles < 0))
return clock_gettime_fallback(clk, ts);
if (cycles > last)
ns += (cycles - last) * vd->mult;
ns >>= vd->shift;
sec = vdso_ts->sec;
} while (unlikely(vdso_read_retry(vd, seq)));
/*
* Do this outside the loop: a race inside the loop could result
* in __iter_div_u64_rem() being extremely slow.
*/
ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return 0;
}
static void do_coarse(const struct vdso_data *vd, clockid_t clk,
struct __kernel_timespec *ts)
{
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
u32 seq;
do {
seq = vdso_read_begin(vd);
ts->tv_sec = vdso_ts->sec;
ts->tv_nsec = vdso_ts->nsec;
} while (unlikely(vdso_read_retry(vd, seq)));
}
static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
const struct vdso_data *vd = __arch_get_vdso_data();
u32 msk;
/* Check for negative values or invalid clocks */
if (unlikely((u32) clock >= MAX_CLOCKS))
goto fallback;
/*
* Convert the clockid to a bitmask and use it to check which
* clocks are handled in the VDSO directly.
*/
msk = 1U << clock;
if (likely(msk & VDSO_HRES)) {
return do_hres(&vd[CS_HRES_COARSE], clock, ts);
} else if (msk & VDSO_COARSE) {
do_coarse(&vd[CS_HRES_COARSE], clock, ts);
return 0;
} else if (msk & VDSO_RAW) {
return do_hres(&vd[CS_RAW], clock, ts);
}
fallback:
return clock_gettime_fallback(clock, ts);
}
static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
struct __kernel_timespec ts;
int ret;
if (res == NULL)
goto fallback;
ret = __cvdso_clock_gettime(clock, &ts);
if (ret == 0) {
res->tv_sec = ts.tv_sec;
res->tv_nsec = ts.tv_nsec;
}
return ret;
fallback:
return clock_gettime_fallback(clock, (struct __kernel_timespec *)res);
}
static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
const struct vdso_data *vd = __arch_get_vdso_data();
if (likely(tv != NULL)) {
struct __kernel_timespec ts;
if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
return gettimeofday_fallback(tv, tz);
tv->tv_sec = ts.tv_sec;
tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
}
if (unlikely(tz != NULL)) {
tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
}
return 0;
}
#ifdef VDSO_HAS_TIME
static __maybe_unused time_t __cvdso_time(time_t *time)
{
const struct vdso_data *vd = __arch_get_vdso_data();
time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
if (time)
*time = t;
return t;
}
#endif /* VDSO_HAS_TIME */
#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
const struct vdso_data *vd = __arch_get_vdso_data();
u64 ns;
u32 msk;
u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
/* Check for negative values or invalid clocks */
if (unlikely((u32) clock >= MAX_CLOCKS))
goto fallback;
/*
* Convert the clockid to a bitmask and use it to check which
* clocks are handled in the VDSO directly.
*/
msk = 1U << clock;
if (msk & VDSO_HRES) {
/*
* Preserves the behaviour of posix_get_hrtimer_res().
*/
ns = hrtimer_res;
} else if (msk & VDSO_COARSE) {
/*
* Preserves the behaviour of posix_get_coarse_res().
*/
ns = LOW_RES_NSEC;
} else if (msk & VDSO_RAW) {
/*
* Preserves the behaviour of posix_get_hrtimer_res().
*/
ns = hrtimer_res;
} else {
goto fallback;
}
if (res) {
res->tv_sec = 0;
res->tv_nsec = ns;
}
return 0;
fallback:
return clock_getres_fallback(clock, res);
}
static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
struct __kernel_timespec ts;
int ret;
if (res == NULL)
goto fallback;
ret = __cvdso_clock_getres(clock, &ts);
if (ret == 0) {
res->tv_sec = ts.tv_sec;
res->tv_nsec = ts.tv_nsec;
}
return ret;
fallback:
return clock_getres_fallback(clock, (struct __kernel_timespec *)res);
}
#endif /* VDSO_HAS_CLOCK_GETRES */