x86/decompressor: Call trampoline as a normal function

Move the long return to switch to 32-bit mode into the trampoline code
so it can be called as an ordinary function. This will allow it to be
called directly from C code in a subsequent patch.

While at it, reorganize the code somewhat to keep the prologue and
epilogue of the function together, making the code a bit easier to
follow. Also, given that the trampoline is now entered in 64-bit mode, a
simple RIP-relative reference can be used to take the address of the
exit point.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Link: https://lore.kernel.org/r/20230807162720.545787-10-ardb@kernel.org

authored by

Ard Biesheuvel and committed by

Borislav Petkov (AMD) 2 years ago e8972a76 00c6b097

+36 -45

2 changed files

expand all

arch

x86

boot

compressed

head_64.S

pgtable.h

+35 -44

arch/x86/boot/compressed/head_64.S

··· 447 447 /* Save the trampoline address in RCX */ 448 448 movq %rax, %rcx 449 449 450 - /* Set up 32-bit addressable stack */ 451 - leaq TRAMPOLINE_32BIT_STACK_END(%rcx), %rsp 452 - 453 - /* 454 - * Preserve live 64-bit registers on the stack: this is necessary 455 - * because the architecture does not guarantee that GPRs will retain 456 - * their full 64-bit values across a 32-bit mode switch. 457 - */ 458 - pushq %r15 459 - pushq %rbp 460 - pushq %rbx 461 - 462 - /* 463 - * Push the 64-bit address of trampoline_return() onto the new stack. 464 - * It will be used by the trampoline to return to the main code. Due to 465 - * the 32-bit mode switch, it cannot be kept it in a register either. 466 - */ 467 - leaq trampoline_return(%rip), %rdi 468 - pushq %rdi 469 - 470 - /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */ 471 - pushq $__KERNEL32_CS 472 450 leaq TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax 473 - pushq %rax 474 - lretq 475 - trampoline_return: 476 - /* Restore live 64-bit registers */ 477 - popq %rbx 478 - popq %rbp 479 - popq %r15 480 - 481 - /* Restore the stack, the 32-bit trampoline uses its own stack */ 482 - leaq rva(boot_stack_end)(%rbx), %rsp 451 + call *%rax 483 452 484 453 /* 485 454 * cleanup_trampoline() would restore trampoline memory. ··· 530 561 jmp *%rax 531 562 SYM_FUNC_END(.Lrelocated) 532 563 533 - .code32 534 564 /* 535 565 * This is the 32-bit trampoline that will be copied over to low memory. 536 566 * ··· 538 570 * Non zero RDX means trampoline needs to enable 5-level paging. 539 571 */ 540 572 SYM_CODE_START(trampoline_32bit_src) 573 + /* 574 + * Preserve live 64-bit registers on the stack: this is necessary 575 + * because the architecture does not guarantee that GPRs will retain 576 + * their full 64-bit values across a 32-bit mode switch. 577 + */ 578 + pushq %r15 579 + pushq %rbp 580 + pushq %rbx 581 + 582 + /* Set up 32-bit addressable stack and push the old RSP value */ 583 + leaq (TRAMPOLINE_32BIT_STACK_END - 8)(%rcx), %rbx 584 + movq %rsp, (%rbx) 585 + movq %rbx, %rsp 586 + 587 + /* Take the address of the trampoline exit code */ 588 + leaq .Lret(%rip), %rbx 589 + 590 + /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */ 591 + pushq $__KERNEL32_CS 592 + leaq 0f(%rip), %rax 593 + pushq %rax 594 + lretq 595 + 596 + .Lret: 597 + /* Restore the preserved 64-bit registers */ 598 + movq (%rsp), %rsp 599 + popq %rbx 600 + popq %rbp 601 + popq %r15 602 + retq 603 + 604 + .code32 605 + 0: 541 606 /* Set up data and stack segments */ 542 607 movl $__KERNEL_DS, %eax 543 608 movl %eax, %ds ··· 634 633 1: 635 634 movl %eax, %cr4 636 635 637 - /* Calculate address of paging_enabled() once we are executing in the trampoline */ 638 - leal .Lpaging_enabled - trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_OFFSET(%ecx), %eax 639 - 640 636 /* Prepare the stack for far return to Long Mode */ 641 637 pushl $__KERNEL_CS 642 - pushl %eax 638 + pushl %ebx 643 639 644 640 /* Enable paging again. */ 645 641 movl %cr0, %eax ··· 646 648 lret 647 649 SYM_CODE_END(trampoline_32bit_src) 648 650 649 - .code64 650 - SYM_FUNC_START_LOCAL_NOALIGN(.Lpaging_enabled) 651 - /* Return from the trampoline */ 652 - retq 653 - SYM_FUNC_END(.Lpaging_enabled) 654 - 655 651 /* 656 652 * The trampoline code has a size limit. 657 653 * Make sure we fail to compile if the trampoline code grows ··· 653 661 */ 654 662 .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE 655 663 656 - .code32 657 664 SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmode) 658 665 /* This isn't an x86-64 CPU, so hang intentionally, we cannot continue */ 659 666 1:

+1 -1

arch/x86/boot/compressed/pgtable.h

··· 6 6 #define TRAMPOLINE_32BIT_PGTABLE_OFFSET 0 7 7 8 8 #define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE 9 - #define TRAMPOLINE_32BIT_CODE_SIZE 0x80 9 + #define TRAMPOLINE_32BIT_CODE_SIZE 0xA0 10 10 11 11 #define TRAMPOLINE_32BIT_STACK_END TRAMPOLINE_32BIT_SIZE 12 12

Configure Feed

Configure Feed