Andy Lutomirski
04d98e96a0
commit 1ddf0b1b11aa8a90cef6706e935fc31c75c406ba upstream.
In Linux 3.18 and below, GCC hoists the lsl instructions in the
pvclock code all the way to the beginning of __vdso_clock_gettime,
slowing the non-paravirt case significantly. For unknown reasons,
presumably related to the removal of a branch, the performance issue
is gone as of
e76b027e6408 x86,vdso: Use LSL unconditionally for vgetcpu
but I don't trust GCC enough to expect the problem to stay fixed.
There should be no correctness issue, because the __getcpu calls in
__vdso_vlock_gettime were never necessary in the first place.
Note to stable maintainers: In 3.18 and below, depending on
configuration, gcc 4.9.2 generates code like this:
9c3: 44 0f 03 e8 lsl %ax,%r13d
9c7: 45 89 eb mov %r13d,%r11d
9ca: 0f 03 d8 lsl %ax,%ebx
This patch won't apply as is to any released kernel, but I'll send a
trivial backported version if needed.
[
Backported by Andy Lutomirski. Should apply to all affected
versions. This fixes a functionality bug as well as a performance
bug: buggy kernels can infinite loop in __vdso_clock_gettime on
affected compilers. See, for exammple:
https://bugzilla.redhat.com/show_bug.cgi?id=1178975
]
Fixes: 51c19b4f59 x86: vdso: pvclock gettime support
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
45 lines
883 B
C
45 lines
883 B
C
#ifndef _ASM_X86_VSYSCALL_H
|
|
#define _ASM_X86_VSYSCALL_H
|
|
|
|
#include <linux/seqlock.h>
|
|
#include <uapi/asm/vsyscall.h>
|
|
|
|
#define VGETCPU_RDTSCP 1
|
|
#define VGETCPU_LSL 2
|
|
|
|
/* kernel space (writeable) */
|
|
extern int vgetcpu_mode;
|
|
extern struct timezone sys_tz;
|
|
|
|
#include <asm/vvar.h>
|
|
|
|
extern void map_vsyscall(void);
|
|
|
|
/*
|
|
* Called on instruction fetch fault in vsyscall page.
|
|
* Returns true if handled.
|
|
*/
|
|
extern bool emulate_vsyscall(struct pt_regs *regs, unsigned long address);
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
#define VGETCPU_CPU_MASK 0xfff
|
|
|
|
static inline unsigned int __getcpu(void)
|
|
{
|
|
unsigned int p;
|
|
|
|
if (VVAR(vgetcpu_mode) == VGETCPU_RDTSCP) {
|
|
/* Load per CPU data from RDTSCP */
|
|
native_read_tscp(&p);
|
|
} else {
|
|
/* Load per CPU data from GDT */
|
|
asm volatile ("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
|
|
}
|
|
|
|
return p;
|
|
}
|
|
#endif /* CONFIG_X86_64 */
|
|
|
|
#endif /* _ASM_X86_VSYSCALL_H */
|