mirror of
https://github.com/Fishwaldo/Star64_linux.git
synced 2025-03-17 20:54:10 +00:00
x86: Add magic AMD return-thunk
Note: needs to be in a section distinct from Retpolines such that the Retpoline RET substitution cannot possibly use immediate jumps. ORC unwinding for zen_untrain_ret() and __x86_return_thunk() is a little tricky but works due to the fact that zen_untrain_ret() doesn't have any stack ops and as such will emit a single ORC entry at the start (+0x3f). Meanwhile, unwinding an IP, including the __x86_return_thunk() one (+0x40) will search for the largest ORC entry smaller or equal to the IP, these will find the one ORC entry (+0x3f) and all works. [ Alexandre: SVM part. ] [ bp: Build fix, massages. ] Suggested-by: Andrew Cooper <Andrew.Cooper3@citrix.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Borislav Petkov <bp@suse.de> Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org> Signed-off-by: Borislav Petkov <bp@suse.de>
This commit is contained in:
parent
951ddecf43
commit
a149180fbc
9 changed files with 128 additions and 9 deletions
|
@ -96,6 +96,7 @@ SYM_CODE_START(entry_SYSCALL_64)
|
||||||
|
|
||||||
SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
|
SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
|
||||||
ANNOTATE_NOENDBR
|
ANNOTATE_NOENDBR
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/* Construct struct pt_regs on stack */
|
/* Construct struct pt_regs on stack */
|
||||||
pushq $__USER_DS /* pt_regs->ss */
|
pushq $__USER_DS /* pt_regs->ss */
|
||||||
|
@ -717,6 +718,7 @@ native_irq_return_ldt:
|
||||||
pushq %rdi /* Stash user RDI */
|
pushq %rdi /* Stash user RDI */
|
||||||
swapgs /* to kernel GS */
|
swapgs /* to kernel GS */
|
||||||
SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
|
SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
movq PER_CPU_VAR(espfix_waddr), %rdi
|
movq PER_CPU_VAR(espfix_waddr), %rdi
|
||||||
movq %rax, (0*8)(%rdi) /* user RAX */
|
movq %rax, (0*8)(%rdi) /* user RAX */
|
||||||
|
@ -911,6 +913,7 @@ SYM_CODE_START_LOCAL(paranoid_entry)
|
||||||
* be retrieved from a kernel internal table.
|
* be retrieved from a kernel internal table.
|
||||||
*/
|
*/
|
||||||
SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
|
SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Handling GSBASE depends on the availability of FSGSBASE.
|
* Handling GSBASE depends on the availability of FSGSBASE.
|
||||||
|
@ -1020,6 +1023,7 @@ SYM_CODE_START_LOCAL(error_entry)
|
||||||
FENCE_SWAPGS_USER_ENTRY
|
FENCE_SWAPGS_USER_ENTRY
|
||||||
/* We have user CR3. Change to kernel CR3. */
|
/* We have user CR3. Change to kernel CR3. */
|
||||||
SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
|
SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
|
leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
|
||||||
.Lerror_entry_from_usermode_after_swapgs:
|
.Lerror_entry_from_usermode_after_swapgs:
|
||||||
|
@ -1072,6 +1076,7 @@ SYM_CODE_START_LOCAL(error_entry)
|
||||||
swapgs
|
swapgs
|
||||||
FENCE_SWAPGS_USER_ENTRY
|
FENCE_SWAPGS_USER_ENTRY
|
||||||
SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
|
SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Pretend that the exception came from user mode: set up pt_regs
|
* Pretend that the exception came from user mode: set up pt_regs
|
||||||
|
@ -1167,6 +1172,7 @@ SYM_CODE_START(asm_exc_nmi)
|
||||||
movq %rsp, %rdx
|
movq %rsp, %rdx
|
||||||
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
||||||
UNWIND_HINT_IRET_REGS base=%rdx offset=8
|
UNWIND_HINT_IRET_REGS base=%rdx offset=8
|
||||||
|
UNTRAIN_RET
|
||||||
pushq 5*8(%rdx) /* pt_regs->ss */
|
pushq 5*8(%rdx) /* pt_regs->ss */
|
||||||
pushq 4*8(%rdx) /* pt_regs->rsp */
|
pushq 4*8(%rdx) /* pt_regs->rsp */
|
||||||
pushq 3*8(%rdx) /* pt_regs->flags */
|
pushq 3*8(%rdx) /* pt_regs->flags */
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
#include <asm/irqflags.h>
|
#include <asm/irqflags.h>
|
||||||
#include <asm/asm.h>
|
#include <asm/asm.h>
|
||||||
#include <asm/smap.h>
|
#include <asm/smap.h>
|
||||||
|
#include <asm/nospec-branch.h>
|
||||||
#include <linux/linkage.h>
|
#include <linux/linkage.h>
|
||||||
#include <linux/err.h>
|
#include <linux/err.h>
|
||||||
|
|
||||||
|
@ -72,6 +73,7 @@ SYM_CODE_START(entry_SYSENTER_compat)
|
||||||
pushq $__USER32_CS /* pt_regs->cs */
|
pushq $__USER32_CS /* pt_regs->cs */
|
||||||
pushq $0 /* pt_regs->ip = 0 (placeholder) */
|
pushq $0 /* pt_regs->ip = 0 (placeholder) */
|
||||||
SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)
|
SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* User tracing code (ptrace or signal handlers) might assume that
|
* User tracing code (ptrace or signal handlers) might assume that
|
||||||
|
@ -190,6 +192,7 @@ SYM_CODE_START(entry_SYSCALL_compat)
|
||||||
|
|
||||||
SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
|
SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
|
||||||
ANNOTATE_NOENDBR
|
ANNOTATE_NOENDBR
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/* Construct struct pt_regs on stack */
|
/* Construct struct pt_regs on stack */
|
||||||
pushq $__USER32_DS /* pt_regs->ss */
|
pushq $__USER32_DS /* pt_regs->ss */
|
||||||
|
@ -332,6 +335,7 @@ SYM_CODE_START(entry_INT80_compat)
|
||||||
pushq 0*8(%rax) /* regs->orig_ax */
|
pushq 0*8(%rax) /* regs->orig_ax */
|
||||||
.Lint80_keep_stack:
|
.Lint80_keep_stack:
|
||||||
|
|
||||||
|
UNTRAIN_RET
|
||||||
PUSH_AND_CLEAR_REGS rax=$-ENOSYS
|
PUSH_AND_CLEAR_REGS rax=$-ENOSYS
|
||||||
UNWIND_HINT_REGS
|
UNWIND_HINT_REGS
|
||||||
|
|
||||||
|
|
|
@ -301,6 +301,7 @@
|
||||||
#define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
|
#define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
|
||||||
#define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */
|
#define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */
|
||||||
#define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */
|
#define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */
|
||||||
|
#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */
|
||||||
|
|
||||||
/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
|
/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
|
||||||
#define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */
|
#define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */
|
||||||
|
|
|
@ -55,7 +55,8 @@
|
||||||
#else
|
#else
|
||||||
# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \
|
# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \
|
||||||
(1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \
|
(1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \
|
||||||
(1 << (X86_FEATURE_RETHUNK & 31)))
|
(1 << (X86_FEATURE_RETHUNK & 31)) | \
|
||||||
|
(1 << (X86_FEATURE_UNRET & 31)))
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_INTEL_IOMMU_SVM
|
#ifdef CONFIG_INTEL_IOMMU_SVM
|
||||||
|
|
|
@ -112,6 +112,22 @@
|
||||||
#endif
|
#endif
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
|
||||||
|
* return thunk isn't mapped into the userspace tables (then again, AMD
|
||||||
|
* typically has NO_MELTDOWN).
|
||||||
|
*
|
||||||
|
* Doesn't clobber any registers but does require a stable stack.
|
||||||
|
*
|
||||||
|
* As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
|
||||||
|
* where we have a stack but before any RET instruction.
|
||||||
|
*/
|
||||||
|
.macro UNTRAIN_RET
|
||||||
|
#ifdef CONFIG_RETPOLINE
|
||||||
|
ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET
|
||||||
|
#endif
|
||||||
|
.endm
|
||||||
|
|
||||||
#else /* __ASSEMBLY__ */
|
#else /* __ASSEMBLY__ */
|
||||||
|
|
||||||
#define ANNOTATE_RETPOLINE_SAFE \
|
#define ANNOTATE_RETPOLINE_SAFE \
|
||||||
|
@ -124,6 +140,7 @@ typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
|
||||||
extern retpoline_thunk_t __x86_indirect_thunk_array[];
|
extern retpoline_thunk_t __x86_indirect_thunk_array[];
|
||||||
|
|
||||||
extern void __x86_return_thunk(void);
|
extern void __x86_return_thunk(void);
|
||||||
|
extern void zen_untrain_ret(void);
|
||||||
|
|
||||||
#ifdef CONFIG_RETPOLINE
|
#ifdef CONFIG_RETPOLINE
|
||||||
|
|
||||||
|
|
|
@ -141,7 +141,7 @@ SECTIONS
|
||||||
|
|
||||||
#ifdef CONFIG_RETPOLINE
|
#ifdef CONFIG_RETPOLINE
|
||||||
__indirect_thunk_start = .;
|
__indirect_thunk_start = .;
|
||||||
*(.text.__x86.indirect_thunk)
|
*(.text.__x86.*)
|
||||||
__indirect_thunk_end = .;
|
__indirect_thunk_end = .;
|
||||||
#endif
|
#endif
|
||||||
} :text =0xcccc
|
} :text =0xcccc
|
||||||
|
|
|
@ -110,6 +110,15 @@ SYM_FUNC_START(__svm_vcpu_run)
|
||||||
mov %r15, VCPU_R15(%_ASM_AX)
|
mov %r15, VCPU_R15(%_ASM_AX)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
|
||||||
|
* untrained as soon as we exit the VM and are back to the
|
||||||
|
* kernel. This should be done before re-enabling interrupts
|
||||||
|
* because interrupt handlers won't sanitize 'ret' if the return is
|
||||||
|
* from the kernel.
|
||||||
|
*/
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Clear all general purpose registers except RSP and RAX to prevent
|
* Clear all general purpose registers except RSP and RAX to prevent
|
||||||
* speculative use of the guest's values, even those that are reloaded
|
* speculative use of the guest's values, even those that are reloaded
|
||||||
|
@ -190,6 +199,15 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
|
||||||
FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
|
FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
|
||||||
|
* untrained as soon as we exit the VM and are back to the
|
||||||
|
* kernel. This should be done before re-enabling interrupts
|
||||||
|
* because interrupt handlers won't sanitize RET if the return is
|
||||||
|
* from the kernel.
|
||||||
|
*/
|
||||||
|
UNTRAIN_RET
|
||||||
|
|
||||||
pop %_ASM_BX
|
pop %_ASM_BX
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_64
|
||||||
|
|
|
@ -72,11 +72,67 @@ SYM_CODE_END(__x86_indirect_thunk_array)
|
||||||
* This function name is magical and is used by -mfunction-return=thunk-extern
|
* This function name is magical and is used by -mfunction-return=thunk-extern
|
||||||
* for the compiler to generate JMPs to it.
|
* for the compiler to generate JMPs to it.
|
||||||
*/
|
*/
|
||||||
SYM_CODE_START(__x86_return_thunk)
|
.section .text.__x86.return_thunk
|
||||||
UNWIND_HINT_EMPTY
|
|
||||||
ANNOTATE_NOENDBR
|
/*
|
||||||
|
* Safety details here pertain to the AMD Zen{1,2} microarchitecture:
|
||||||
|
* 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for
|
||||||
|
* alignment within the BTB.
|
||||||
|
* 2) The instruction at zen_untrain_ret must contain, and not
|
||||||
|
* end with, the 0xc3 byte of the RET.
|
||||||
|
* 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread
|
||||||
|
* from re-poisioning the BTB prediction.
|
||||||
|
*/
|
||||||
|
.align 64
|
||||||
|
.skip 63, 0xcc
|
||||||
|
SYM_FUNC_START_NOALIGN(zen_untrain_ret);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* As executed from zen_untrain_ret, this is:
|
||||||
|
*
|
||||||
|
* TEST $0xcc, %bl
|
||||||
|
* LFENCE
|
||||||
|
* JMP __x86_return_thunk
|
||||||
|
*
|
||||||
|
* Executing the TEST instruction has a side effect of evicting any BTB
|
||||||
|
* prediction (potentially attacker controlled) attached to the RET, as
|
||||||
|
* __x86_return_thunk + 1 isn't an instruction boundary at the moment.
|
||||||
|
*/
|
||||||
|
.byte 0xf6
|
||||||
|
|
||||||
|
/*
|
||||||
|
* As executed from __x86_return_thunk, this is a plain RET.
|
||||||
|
*
|
||||||
|
* As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
|
||||||
|
*
|
||||||
|
* We subsequently jump backwards and architecturally execute the RET.
|
||||||
|
* This creates a correct BTB prediction (type=ret), but in the
|
||||||
|
* meantime we suffer Straight Line Speculation (because the type was
|
||||||
|
* no branch) which is halted by the INT3.
|
||||||
|
*
|
||||||
|
* With SMT enabled and STIBP active, a sibling thread cannot poison
|
||||||
|
* RET's prediction to a type of its choice, but can evict the
|
||||||
|
* prediction due to competitive sharing. If the prediction is
|
||||||
|
* evicted, __x86_return_thunk will suffer Straight Line Speculation
|
||||||
|
* which will be contained safely by the INT3.
|
||||||
|
*/
|
||||||
|
SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL)
|
||||||
ret
|
ret
|
||||||
int3
|
int3
|
||||||
SYM_CODE_END(__x86_return_thunk)
|
SYM_CODE_END(__x86_return_thunk)
|
||||||
|
|
||||||
__EXPORT_THUNK(__x86_return_thunk)
|
/*
|
||||||
|
* Ensure the TEST decoding / BTB invalidation is complete.
|
||||||
|
*/
|
||||||
|
lfence
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Jump back and execute the RET in the middle of the TEST instruction.
|
||||||
|
* INT3 is for SLS protection.
|
||||||
|
*/
|
||||||
|
jmp __x86_return_thunk
|
||||||
|
int3
|
||||||
|
SYM_FUNC_END(zen_untrain_ret)
|
||||||
|
__EXPORT_THUNK(zen_untrain_ret)
|
||||||
|
|
||||||
|
EXPORT_SYMBOL(__x86_return_thunk)
|
||||||
|
|
|
@ -1302,7 +1302,7 @@ static void add_retpoline_call(struct objtool_file *file, struct instruction *in
|
||||||
annotate_call_site(file, insn, false);
|
annotate_call_site(file, insn, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void add_return_call(struct objtool_file *file, struct instruction *insn)
|
static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
* Return thunk tail calls are really just returns in disguise,
|
* Return thunk tail calls are really just returns in disguise,
|
||||||
|
@ -1311,7 +1311,8 @@ static void add_return_call(struct objtool_file *file, struct instruction *insn)
|
||||||
insn->type = INSN_RETURN;
|
insn->type = INSN_RETURN;
|
||||||
insn->retpoline_safe = true;
|
insn->retpoline_safe = true;
|
||||||
|
|
||||||
list_add_tail(&insn->call_node, &file->return_thunk_list);
|
if (add)
|
||||||
|
list_add_tail(&insn->call_node, &file->return_thunk_list);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool same_function(struct instruction *insn1, struct instruction *insn2)
|
static bool same_function(struct instruction *insn1, struct instruction *insn2)
|
||||||
|
@ -1367,7 +1368,7 @@ static int add_jump_destinations(struct objtool_file *file)
|
||||||
add_retpoline_call(file, insn);
|
add_retpoline_call(file, insn);
|
||||||
continue;
|
continue;
|
||||||
} else if (reloc->sym->return_thunk) {
|
} else if (reloc->sym->return_thunk) {
|
||||||
add_return_call(file, insn);
|
add_return_call(file, insn, true);
|
||||||
continue;
|
continue;
|
||||||
} else if (insn->func) {
|
} else if (insn->func) {
|
||||||
/*
|
/*
|
||||||
|
@ -1387,6 +1388,21 @@ static int add_jump_destinations(struct objtool_file *file)
|
||||||
|
|
||||||
jump_dest = find_insn(file, dest_sec, dest_off);
|
jump_dest = find_insn(file, dest_sec, dest_off);
|
||||||
if (!jump_dest) {
|
if (!jump_dest) {
|
||||||
|
struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This is a special case for zen_untrain_ret().
|
||||||
|
* It jumps to __x86_return_thunk(), but objtool
|
||||||
|
* can't find the thunk's starting RET
|
||||||
|
* instruction, because the RET is also in the
|
||||||
|
* middle of another instruction. Objtool only
|
||||||
|
* knows about the outer instruction.
|
||||||
|
*/
|
||||||
|
if (sym && sym->return_thunk) {
|
||||||
|
add_return_call(file, insn, false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
|
WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
|
||||||
insn->sec, insn->offset, dest_sec->name,
|
insn->sec, insn->offset, dest_sec->name,
|
||||||
dest_off);
|
dest_off);
|
||||||
|
|
Loading…
Add table
Reference in a new issue