From df0b501074796bd81af8c2c7204878ec17f8e326 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 26 Apr 2019 19:39:05 +0100 Subject: [PATCH] x86/boot: Use trampoline_protmode_entry in place Instead of relocating this to low memory, just use it directly from its physical address within the Xen image. Use relocations because it runs without a stack and can't easily be made location-agnostic. This is step 1 in making the boot process avoid scribbling on low memory, so that Xen can boot in a reserved "crash kernel" region with kexec/kdump. Working for boot, but faults when a secondary CPU enables paging because the full physical Xen image is no longer in idle_pg_table by that point. Will fix by putting it *back* into the block which is copied to low memory (which will be made dependent on no-real-mode in a subsequent patch), and by re-relocating it on demand for the APs. So the BSP with no-real-mode will use it in place and relocated to xen_phys_start as it currently is in this patch, then in __start_xen() some low memory will be correctly allocated and it'll be copied (and relocated again) to there. --- xen/arch/x86/boot/head.S | 13 +++- xen/arch/x86/boot/trampoline.S | 102 ++++----------------------- xen/arch/x86/boot/trampoline32.S | 116 +++++++++++++++++++++++++++++++ xen/arch/x86/boot/wakeup.S | 2 +- xen/arch/x86/cpu/common.c | 2 +- xen/arch/x86/cpu/intel.c | 2 +- xen/arch/x86/setup.c | 17 ++++- xen/arch/x86/xen.lds.S | 3 + xen/include/asm-x86/config.h | 2 + 9 files changed, 164 insertions(+), 95 deletions(-) create mode 100644 xen/arch/x86/boot/trampoline32.S diff --git a/xen/arch/x86/boot/head.S b/xen/arch/x86/boot/head.S index db19ac6fd8..da314df42a 100644 --- a/xen/arch/x86/boot/head.S +++ b/xen/arch/x86/boot/head.S @@ -696,7 +696,16 @@ trampoline_setup: lea __PAGE_HYPERVISOR+sym_esi(l1_identmap),%edi mov %edi,sym_fs(l2_bootmap) - /* Apply relocations to bootstrap trampoline. */ + /* Apply (plain physical) relocations to protected mode trampoline. */ + mov $sym_offs(__trampoline32_rel_start),%edi +1: + mov %fs:(%edi),%eax + add %esi,%fs:(%edi,%eax) + add $4,%edi + cmp $sym_offs(__trampoline32_rel_stop),%edi + jb 1b + + /* Apply relocations to 16-bit bootstrap trampoline. */ mov sym_fs(trampoline_phys),%edx mov $sym_offs(__trampoline_rel_start),%edi 1: @@ -756,4 +765,6 @@ ENTRY(trampoline_start) #include "trampoline.S" ENTRY(trampoline_end) +#include "trampoline32.S" + #include "x86_64.S" diff --git a/xen/arch/x86/boot/trampoline.S b/xen/arch/x86/boot/trampoline.S index 5588c7986a..69641edca1 100644 --- a/xen/arch/x86/boot/trampoline.S +++ b/xen/arch/x86/boot/trampoline.S @@ -34,6 +34,15 @@ .long 111b - (off) - .; \ .popsection +#define protsym(s) ((s)-__XEN_VIRT_START) +#define protsym_rel(sym, off, opnd...) \ + protsym(sym),##opnd; \ +111:; \ + .pushsection .trampoline32_rel, "a"; \ + .long 111b - (off) - .; \ + .popsection + + /* Start of the permanent trampoline code. */ .code16 @@ -50,7 +59,7 @@ GLOBAL(trampoline_realmode_entry) xor %ax, %ax inc %ax lmsw %ax # CR0.PE = 1 (enter protected mode) - ljmpl $BOOT_CS32,$bootsym_rel(trampoline_protmode_entry,6) + ljmpl $BOOT_CS32,$protsym_rel(trampoline_protmode_entry,6) .balign 8 .word 0 @@ -85,97 +94,9 @@ trampoline_gdt: .long trampoline_gdt + BOOT_PSEUDORM_DS + 2 - . .popsection -GLOBAL(trampoline_misc_enable_off) - .quad 0 - -GLOBAL(cpuid_ext_features) - .long 0 - -GLOBAL(trampoline_xen_phys_start) - .long 0 - GLOBAL(trampoline_cpu_started) .byte 0 - .code32 -trampoline_protmode_entry: - /* Set up a few descriptors: on entry only CS is guaranteed good. */ - mov $BOOT_DS,%eax - mov %eax,%ds - mov %eax,%es - - /* Set up FPU. */ - fninit - - /* Initialise CR4. */ - mov $X86_CR4_PAE,%ecx - mov %ecx,%cr4 - - /* Load pagetable base register. */ - mov $sym_offs(idle_pg_table),%eax - add bootsym_rel(trampoline_xen_phys_start,4,%eax) - mov %eax,%cr3 - - /* Adjust IA32_MISC_ENABLE if needed (for NX enabling below). */ - mov bootsym_rel(trampoline_misc_enable_off,4,%esi) - mov bootsym_rel(trampoline_misc_enable_off+4,4,%edi) - mov %esi,%eax - or %edi,%eax - jz 1f - mov $MSR_IA32_MISC_ENABLE,%ecx - rdmsr - not %esi - not %edi - and %esi,%eax - and %edi,%edx - wrmsr -1: - - /* Set up EFER (Extended Feature Enable Register). */ - mov bootsym_rel(cpuid_ext_features,4,%edi) - movl $MSR_EFER,%ecx - rdmsr - or $EFER_LME|EFER_SCE,%eax /* Long Mode + SYSCALL/SYSRET */ - bt $cpufeat_bit(X86_FEATURE_NX),%edi /* No Execute? */ - jnc 1f - btsl $_EFER_NX,%eax /* No Execute */ -1: wrmsr - - mov $(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_NE |\ - X86_CR0_ET | X86_CR0_MP | X86_CR0_PE), %eax - mov %eax,%cr0 - jmp 1f -1: - - /* Now in compatibility mode. Long-jump into 64-bit mode. */ - ljmp $BOOT_CS64,$bootsym_rel(start64,6) - - .code64 -start64: - /* Jump to high mappings. */ - movabs $__high_start, %rdi - -#ifdef CONFIG_INDIRECT_THUNK - /* - * If booting virtualised, or hot-onlining a CPU, sibling threads can - * attempt Branch Target Injection against this jmp. - * - * We've got no usable stack so can't use a RETPOLINE thunk, and are - * further than disp32 from the high mappings so couldn't use - * JUMP_THUNK even if it was a non-RETPOLINE thunk. Furthermore, an - * LFENCE isn't necessarily safe to use at this point. - * - * As this isn't a hotpath, use a fully serialising event to reduce - * the speculation window as much as possible. %ebx needs preserving - * for __high_start. - */ - mov %ebx, %esi - cpuid - mov %esi, %ebx -#endif - - jmpq *%rdi - #include "wakeup.S" /* The first page of trampoline is permanent, the rest boot-time only. */ @@ -274,7 +195,8 @@ trampoline_boot_cpu_entry: xor %ebx,%ebx /* Jump to the common bootstrap entry point. */ - jmp trampoline_protmode_entry + mov $protsym_rel(trampoline_protmode_entry,4,%eax) + jmp *%eax #include "video.h" diff --git a/xen/arch/x86/boot/trampoline32.S b/xen/arch/x86/boot/trampoline32.S new file mode 100644 index 0000000000..19d60d6ac1 --- /dev/null +++ b/xen/arch/x86/boot/trampoline32.S @@ -0,0 +1,116 @@ +/* + * Relocatable 32-bit trampoline code + * + * Care must taken when referencing symbols: they live in the relocated + * trampoline and in the hypervisor binary. The hypervisor symbols can either + * be accessed by their virtual address or by the physical address. When + * using the physical address eventually the physical start address of the + * hypervisor must be taken into account: after early boot the hypervisor + * will copy itself to high memory and writes its physical start address to + * trampoline_xen_phys_start in the low memory trampoline copy. + * + * Parts of the trampoline are needed for early boot only, while some other + * parts are needed as long as the hypervisor is active (e.g. wakeup code + * after suspend, bringup code for secondary cpus). The permanent parts should + * not reference any temporary low memory trampoline parts as those parts are + * not guaranteed to persist. + */ + +#undef protsym +#define protsym(s) ((s)-__XEN_VIRT_START) +#define protsym_rel(sym, off, opnd...) \ + protsym(sym),##opnd; \ +111:; \ + .pushsection .trampoline32_rel, "a"; \ + .long 111b - (off) - .; \ + .popsection + + +GLOBAL(trampoline_misc_enable_off) + .quad 0 + +GLOBAL(cpuid_ext_features) + .long 0 + +GLOBAL(trampoline_xen_phys_start) + .long 0 + + .code32 +trampoline_protmode_entry: + /* Set up a few descriptors: on entry only CS is guaranteed good. */ + mov $BOOT_DS,%eax + mov %eax,%ds + mov %eax,%es + + /* Set up FPU. */ + fninit + + /* Initialise CR4. */ + mov $X86_CR4_PAE,%ecx + mov %ecx,%cr4 + + /* Load pagetable base register. */ + mov $sym_offs(idle_pg_table),%eax + add protsym_rel(trampoline_xen_phys_start,4,%eax) + mov %eax,%cr3 + + /* Adjust IA32_MISC_ENABLE if needed (for NX enabling below). */ + mov protsym_rel(trampoline_misc_enable_off,4,%esi) + mov protsym_rel(trampoline_misc_enable_off+4,4,%edi) + mov %esi,%eax + or %edi,%eax + jz 1f + mov $MSR_IA32_MISC_ENABLE,%ecx + rdmsr + not %esi + not %edi + and %esi,%eax + and %edi,%edx + wrmsr +1: + + /* Set up EFER (Extended Feature Enable Register). */ + mov protsym_rel(cpuid_ext_features,4,%edi) + movl $MSR_EFER,%ecx + rdmsr + or $EFER_LME|EFER_SCE,%eax /* Long Mode + SYSCALL/SYSRET */ + bt $cpufeat_bit(X86_FEATURE_NX),%edi /* No Execute? */ + jnc 1f + btsl $_EFER_NX,%eax /* No Execute */ +1: wrmsr + + mov $(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_NE |\ + X86_CR0_ET | X86_CR0_MP | X86_CR0_PE), %eax + mov %eax,%cr0 + jmp 1f +1: + + /* Now in compatibility mode. Long-jump into 64-bit mode. */ + ljmp $BOOT_CS64,$protsym_rel(start64,6) + + .code64 +start64: + /* Jump to high mappings. */ + movabs $__high_start, %rdi + +#ifdef CONFIG_INDIRECT_THUNK + /* + * If booting virtualised, or hot-onlining a CPU, sibling threads can + * attempt Branch Target Injection against this jmp. + * + * We've got no usable stack so can't use a RETPOLINE thunk, and are + * further than disp32 from the high mappings so couldn't use + * JUMP_THUNK even if it was a non-RETPOLINE thunk. Furthermore, an + * LFENCE isn't necessarily safe to use at this point. + * + * As this isn't a hotpath, use a fully serialising event to reduce + * the speculation window as much as possible. %ebx needs preserving + * for __high_start. + */ + mov %ebx, %esi + cpuid + mov %esi, %ebx +#endif + + jmpq *%rdi + diff --git a/xen/arch/x86/boot/wakeup.S b/xen/arch/x86/boot/wakeup.S index f9632eef95..87cf755f29 100644 --- a/xen/arch/x86/boot/wakeup.S +++ b/xen/arch/x86/boot/wakeup.S @@ -139,7 +139,7 @@ wakeup_32: /* Will cpuid feature change after resume? */ /* Set up EFER (Extended Feature Enable Register). */ - mov bootsym_rel(cpuid_ext_features,4,%edi) + mov protsym_rel(cpuid_ext_features,4,%edi) test $0x20100800,%edi /* SYSCALL/SYSRET, No Execute, Long Mode? */ jz .Lskip_eferw movl $MSR_EFER,%ecx diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c index 1d21ff9467..5f6d522f30 100644 --- a/xen/arch/x86/cpu/common.c +++ b/xen/arch/x86/cpu/common.c @@ -408,7 +408,7 @@ static void generic_identify(struct cpuinfo_x86 *c) &c->x86_capability[cpufeat_word(X86_FEATURE_LAHF_LM)], &c->x86_capability[cpufeat_word(X86_FEATURE_SYSCALL)]); if (c == &boot_cpu_data) - bootsym(cpuid_ext_features) = + protsym(cpuid_ext_features) = c->x86_capability[cpufeat_word(X86_FEATURE_NX)]; if (c->extended_cpuid_level >= 0x80000004) diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c index 8e23ed6379..d6ac9d326d 100644 --- a/xen/arch/x86/cpu/intel.c +++ b/xen/arch/x86/cpu/intel.c @@ -240,7 +240,7 @@ static void early_init_intel(struct cpuinfo_x86 *c) MSR_IA32_MISC_ENABLE_XD_DISABLE); if (disable) { wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable); - bootsym(trampoline_misc_enable_off) |= disable; + protsym(trampoline_misc_enable_off) |= disable; } if (disable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c index a353d76f9a..49a7ea4750 100644 --- a/xen/arch/x86/setup.c +++ b/xen/arch/x86/setup.c @@ -664,6 +664,20 @@ static char * __init cmdline_cook(char *p, const char *loader_name) return p; } +extern const s32 __trampoline32_rel_start[], __trampoline32_rel_stop[]; + +static void __init relocate_trampoline32(unsigned long delta) +{ + const s32 *trampoline_ptr; + + /* Apply relocations to trampoline. */ + for ( trampoline_ptr = __trampoline32_rel_start; + trampoline_ptr < __trampoline32_rel_stop; + ++trampoline_ptr ) + *(u32 *)(*trampoline_ptr + (long)trampoline_ptr) += delta; +} + + void __init noreturn __start_xen(unsigned long mbi_p) { char *memmap_type = NULL; @@ -1035,8 +1049,9 @@ void __init noreturn __start_xen(unsigned long mbi_p) /* Select relocation address. */ e = end - reloc_size; + relocate_trampoline32(e - xen_phys_start); xen_phys_start = e; - bootsym(trampoline_xen_phys_start) = e; + protsym(trampoline_xen_phys_start) = e; /* * Perform relocation to new physical address. diff --git a/xen/arch/x86/xen.lds.S b/xen/arch/x86/xen.lds.S index a76e2ec8ef..59cf242f5e 100644 --- a/xen/arch/x86/xen.lds.S +++ b/xen/arch/x86/xen.lds.S @@ -220,6 +220,9 @@ SECTIONS *(.init.data.rel) *(.init.data.rel.*) . = ALIGN(4); + __trampoline32_rel_start = .; + *(.trampoline32_rel) + __trampoline32_rel_stop = .; __trampoline_rel_start = .; *(.trampoline_rel) __trampoline_rel_stop = .; diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h index 9ef9d03ca7..10f54bdf7e 100644 --- a/xen/include/asm-x86/config.h +++ b/xen/include/asm-x86/config.h @@ -95,6 +95,8 @@ extern unsigned long trampoline_phys; #define bootsym(sym) \ (*RELOC_HIDE((typeof(&(sym)))__va(__pa(&(sym))), \ trampoline_phys-__pa(trampoline_start))) +/* 32-bit trampoline is used in-place for now */ +#define protsym(sym) (sym) extern char trampoline_start[], trampoline_end[]; extern char trampoline_realmode_entry[]; extern unsigned int trampoline_xen_phys_start; -- 2.17.2