Skip to content

Commit dd84441

Browse files
dwmw2ingomolnar
dwmw2
authored and
ingomolnar
committed
x86/speculation: Use IBRS if available before calling into firmware
Retpoline means the kernel is safe because it has no indirect branches. But firmware isn't, so use IBRS for firmware calls if it's available. Block preemption while IBRS is set, although in practice the call sites already had to be doing that. Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware code, from an NMI handler. I don't want to touch that with a bargepole. Signed-off-by: David Woodhouse <[email protected]> Reviewed-by: Thomas Gleixner <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: [email protected] Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Ingo Molnar <[email protected]>
1 parent d1c9910 commit dd84441

5 files changed

Lines changed: 63 additions & 12 deletions

File tree

‎arch/x86/include/asm/apm.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
#ifndef _ASM_X86_MACH_DEFAULT_APM_H
88
#define _ASM_X86_MACH_DEFAULT_APM_H
99

10+
#include <asm/nospec-branch.h>
11+
1012
#ifdef APM_ZERO_SEGS
1113
# define APM_DO_ZERO_SEGS \
1214
"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
3234
* N.B. We do NOT need a cld after the BIOS call
3335
* because we always save and restore the flags.
3436
*/
37+
firmware_restrict_branch_speculation_start();
3538
__asm__ __volatile__(APM_DO_ZERO_SEGS
3639
"pushl %%edi\n\t"
3740
"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
4447
"=S" (*esi)
4548
: "a" (func), "b" (ebx_in), "c" (ecx_in)
4649
: "memory", "cc");
50+
firmware_restrict_branch_speculation_end();
4751
}
4852

4953
static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
5660
* N.B. We do NOT need a cld after the BIOS call
5761
* because we always save and restore the flags.
5862
*/
63+
firmware_restrict_branch_speculation_start();
5964
__asm__ __volatile__(APM_DO_ZERO_SEGS
6065
"pushl %%edi\n\t"
6166
"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
6873
"=S" (si)
6974
: "a" (func), "b" (ebx_in), "c" (ecx_in)
7075
: "memory", "cc");
76+
firmware_restrict_branch_speculation_end();
7177
return error;
7278
}
7379

‎arch/x86/include/asm/cpufeatures.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@
213213
#define X86_FEATURE_SEV ( 7*32+20) /* AMD Secure Encrypted Virtualization */
214214

215215
#define X86_FEATURE_USE_IBPB ( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */
216+
#define X86_FEATURE_USE_IBRS_FW ( 7*32+22) /* "" Use IBRS during runtime firmware calls */
216217

217218
/* Virtualization flags: Linux defined, word 8 */
218219
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */

‎arch/x86/include/asm/efi.h‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <asm/pgtable.h>
77
#include <asm/processor-flags.h>
88
#include <asm/tlb.h>
9+
#include <asm/nospec-branch.h>
910

1011
/*
1112
* We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
3637

3738
extern asmlinkage unsigned long efi_call_phys(void *, ...);
3839

39-
#define arch_efi_call_virt_setup() kernel_fpu_begin()
40-
#define arch_efi_call_virt_teardown() kernel_fpu_end()
40+
#define arch_efi_call_virt_setup() \
41+
({ \
42+
kernel_fpu_begin(); \
43+
firmware_restrict_branch_speculation_start(); \
44+
})
45+
46+
#define arch_efi_call_virt_teardown() \
47+
({ \
48+
firmware_restrict_branch_speculation_end(); \
49+
kernel_fpu_end(); \
50+
})
51+
4152

4253
/*
4354
* Wrap all the virtual calls in a way that forces the parameters on the stack.
@@ -73,6 +84,7 @@ struct efi_scratch {
7384
efi_sync_low_kernel_mappings(); \
7485
preempt_disable(); \
7586
__kernel_fpu_begin(); \
87+
firmware_restrict_branch_speculation_start(); \
7688
\
7789
if (efi_scratch.use_pgd) { \
7890
efi_scratch.prev_cr3 = __read_cr3(); \
@@ -91,6 +103,7 @@ struct efi_scratch {
91103
__flush_tlb_all(); \
92104
} \
93105
\
106+
firmware_restrict_branch_speculation_end(); \
94107
__kernel_fpu_end(); \
95108
preempt_enable(); \
96109
})

‎arch/x86/include/asm/nospec-branch.h‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -219,17 +219,38 @@ static inline void vmexit_fill_RSB(void)
219219
#endif
220220
}
221221

222+
#define alternative_msr_write(_msr, _val, _feature) \
223+
asm volatile(ALTERNATIVE("", \
224+
"movl %[msr], %%ecx\n\t" \
225+
"movl %[val], %%eax\n\t" \
226+
"movl $0, %%edx\n\t" \
227+
"wrmsr", \
228+
_feature) \
229+
: : [msr] "i" (_msr), [val] "i" (_val) \
230+
: "eax", "ecx", "edx", "memory")
231+
222232
static inline void indirect_branch_prediction_barrier(void)
223233
{
224-
asm volatile(ALTERNATIVE("",
225-
"movl %[msr], %%ecx\n\t"
226-
"movl %[val], %%eax\n\t"
227-
"movl $0, %%edx\n\t"
228-
"wrmsr",
229-
X86_FEATURE_USE_IBPB)
230-
: : [msr] "i" (MSR_IA32_PRED_CMD),
231-
[val] "i" (PRED_CMD_IBPB)
232-
: "eax", "ecx", "edx", "memory");
234+
alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
235+
X86_FEATURE_USE_IBPB);
236+
}
237+
238+
/*
239+
* With retpoline, we must use IBRS to restrict branch prediction
240+
* before calling into firmware.
241+
*/
242+
static inline void firmware_restrict_branch_speculation_start(void)
243+
{
244+
preempt_disable();
245+
alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,
246+
X86_FEATURE_USE_IBRS_FW);
247+
}
248+
249+
static inline void firmware_restrict_branch_speculation_end(void)
250+
{
251+
alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,
252+
X86_FEATURE_USE_IBRS_FW);
253+
preempt_enable();
233254
}
234255

235256
#endif /* __ASSEMBLY__ */

‎arch/x86/kernel/cpu/bugs.c‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,15 @@ static void __init spectre_v2_select_mitigation(void)
300300
setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
301301
pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
302302
}
303+
304+
/*
305+
* Retpoline means the kernel is safe because it has no indirect
306+
* branches. But firmware isn't, so use IBRS to protect that.
307+
*/
308+
if (boot_cpu_has(X86_FEATURE_IBRS)) {
309+
setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
310+
pr_info("Enabling Restricted Speculation for firmware calls\n");
311+
}
303312
}
304313

305314
#undef pr_fmt
@@ -326,8 +335,9 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c
326335
if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
327336
return sprintf(buf, "Not affected\n");
328337

329-
return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
338+
return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
330339
boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
340+
boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
331341
spectre_v2_module_string());
332342
}
333343
#endif

0 commit comments

Comments
 (0)