kern/arch/x86/boot/setup64.S

/* Copyright (C) 2021,2022 fef <owo@fef.moe>.  All rights reserved. */

#include <asm/common.h>

#include <arch/multiboot.h>
#include <arch/page.h>
#include <arch/sched.h>
#include <arch/segment.h>

/*
 * Early boot sequence on amd64.
 *
 * This is loosely based on the 32-bit example code from the OSDev.org wiki:
 * <https://wiki.osdev.org/Higher_Half_x86_Bare_Bones>
 * mixed with some of the Long Mode tutorial code:
 * <https://wiki.osdev.org/Setting_Up_Long_Mode>
 *
 * When entering from the bootloader, we are still in 32-bit protected mode,
 * meaning we have to enable long mode ourselves.
 *
 * Our basic setup procedure is:
 * - set up the stack (unlike on i386, where we do it only as the last step)
 * - perform some sanity checks, like whether the CPU actually supports 64 bits
 * - load the GDT at its *physical* address and use its 32-bit segments
 * - populate the page maps and %cr3 with the 4-level 64-bit structures:
 *     + the lowest 1GB (including the entire kernel image) is identity mapped
 *     + the same mapping is mirrored to KERNBASE (0xffffffff80000000)
 *     + the page tables are mapped recursively at X86_PMAP_OFFSET
 * - enable IA-32e mode by setting IA32_EFER.LME = 1 (see Intel SDM vol 4, tbl 2-2)
 * - enable paging and jump to the higher half mapping
 * - update rsp and rbp to also point to the high address
 * - reload the GDT at its higher half address
 * - discard the identity mapping of low memory
 * - call _boot()
 */

	.code32 /* we enter in 32-bit protected mode */

	/* from kernel64.ld */
	.extern _image_start_phys
	.extern _image_end_phys

	.extern	_boot /* main boot routine -- see ./boot.c */

	/* initial page maps -- see ../mm/amd64/page.c */
	.extern _pml4t
	.extern _pdpt0

	/* GDT stuff -- see ../mm/segment.S */
	.extern _x86_gdt_desc
	.extern _x86_gdt_desc_phys

	.extern _x86_write_tss_base
	.extern _x86_check_multiboot
	.extern _x86_check_cpuid
	.extern _x86_check_ext_cpuid
	.extern _x86_check_ia32e

	.section .multiboot.text, "ax", @progbits

/* C code is linked against high memory, this gets the physical address */
#define PADDR(c_symbol) (c_symbol - KERNBASE)

/*
 * actual setup routine
 */

ENTRY(_setup)
	/*
	 * set up the stack
	 */
	movl	$PADDR(stack_top), %esp
	/* "previous" %rip (for ktrace) */
	pushl	$0
	pushl	$0
	/* "previous" %rbp (for ktrace) */
	pushl	$0
	pushl	$0
	movl	%esp, %ebp

	push	%ebx /* temporarily stash multiboot tag address away */

	/*
	 * the bootloader should have loaded the multiboot magic
	 * value into eax, check if that is really the case
	 */
	push	%eax
	call	_x86_check_multiboot
	addl	$4, %esp

	/*
	 * check if the CPU supports the CPUID instruction
	 * (prints an error and halt if it doesn't)
	 */
	call	_x86_check_cpuid

	/*
	 * check if the CPU supports extended CPUID addresses
	 * (prints an error and halts if it doesn't)
	 */
	call	_x86_check_ext_cpuid

	/*
	 * check if the CPU supports IA-32e mode
	 * (prints an error and halts if it doesn't)
	 */
	call	_x86_check_ia32e

	/*
	 * load the base values for kernel and user TSS into the corresponding GDT entries
	 */
	pushl	$PADDR(_x86_kern_tss)
	pushl	$PADDR(_x86_gdt + X86_KERN_TSS)
	/* _x86_write_tss_base(&_x86_gdt[X86_KERN_TSS / sizeof(_x86_gdt[0])], &_x86_kern_tss); */
	call	_x86_write_tss_base

	pushl	$PADDR(_x86_user_tss)
	pushl	$PADDR(_x86_gdt + X86_USER_TSS)
	/* _x86_write_tss_base(&_x86_gdt[X86_USER_TSS / sizeof(_x86_gdt[0])], &_x86_user_tss); */
	call	_x86_write_tss_base
	addl	$16, %esp

	/*
	 * load our own GDT (at its physical address) and its 32-bit segments
	 */
	lgdt	_x86_gdt_desc_phys
	ljmp	$(X86_32_KERN_CS), $1f
1:	movl	$(X86_KERN_DS), %eax
	movw	%ax, %ds
	movw	%ax, %es
	movw	%ax, %fs
	movw	%ax, %gs
	movw	%ax, %ss

	/* XXX do we really need to load a TSS? */
	movl	$X86_KERN_TSS, %eax
	ltr	%ax

#if (KERNBASE % (1 << X86_PDPT_SHIFT)) != 0
#error "KERNBASE must be aligned to at least a PDP entry (1 GB)"
#endif
#if (X86_PMAP_OFFSET % (1 << X86_PML4T_SHIFT)) != 0
#error "X86_PMAP_OFFSET must be aligned to at least a PML4 entry (512 GB)"
#endif

#define V48 0xffff000000000000
#define PDPT_OFFSET(ptr) (( (((ptr) - V48) >> X86_PDPT_SHIFT) % 512 ) * 8)
#define PML4T_OFFSET(ptr) ( ((ptr) - V48) >> (X86_PML4T_SHIFT) * 8 )

	/*
	 * statically map the low 2 GB to itself and to the high kernel half
	 */
	/* for the identity mapping */
	movl	$0x00000083, PADDR(_pdpt0) /* present (0), write (1), huge (7) */
	movl	$0x40000083, PADDR(_pdpt0 + 8)
	/* For the -2GB at the end of virtual memory.  We use the same PDP for
	 * both low and high memory, so techincally this creates a total of four
	 * mappings (+0 GB, +510 GB, -512 GB, -2 GB), but we remove all except
	 * the -2GB one once we have transitioned to high memory. */
	movl	$0x00000083, PADDR(_pdpt0 + PDPT_OFFSET(KERNBASE))
	movl	$0x40000083, PADDR(_pdpt0 + PDPT_OFFSET(KERNBASE + 0x40000000))

	movl	$PADDR(_pdpt0 + 0x003), PADDR(_pml4t) /* present (0), write (1), huge (7) */
	movl	$PADDR(_pdpt0 + 0x003), PADDR(_pml4t + PML4T_OFFSET(KERNBASE))

	/* map the PML4 to itself */
	movl	$PADDR(_pml4t + 0x003), PADDR(_pml4t + PML4T_OFFSET(X86_PMAP_OFFSET))
	movb	$0x80, PADDR(_pml4t + PML4T_OFFSET(X86_PMAP_OFFSET) + 7) /* NX bit */

	/*
	 * ensure paging is disabled by clearing CR0.PG (bit 31)
	 */
	movl	%cr0, %eax
	andl	$0x7fffffff, %eax
	movl	%eax, %cr0

	/*
	 * enable:
	 *   CR4.PSE (Page Size Extensions, bit 4)
	 *   CR4.PAE (Physical Address Extension, bit 5)
	 *   CR4.PGE (Page Global Enable, bit 7)
	 */
	movl	%cr4, %eax
	orl	$0x000000b0, %eax
	movl	%eax, %cr4

	/* load cr3 with the PML4 */
	movl	$PADDR(_pml4t), %eax
	movl	%eax, %cr3

	/*
	 * enable IA-32e by setting IA32_EFER.LME (bit 8)
	 * (and also set No Execute Enable (bit 11) while we're at it)
	 */
	movl	$0xc0000080, %ecx
	rdmsr
	orl	$0x00000900, %eax
	wrmsr

	/*
	 * enable:
	 *   CR0.PG (Paging, bit 31)
	 *   CR0.WP (Write Protect, bit 16)
	 */
	movl	%cr0, %eax
	orl	$0x80010000, %eax
	movl	%eax, %cr0

	/* remember when we pushed the multiboot tag address stored in ebx
	 * like, 100 lines ago?  Yeah we restore that now, and put it into
	 * the right register to pass it as the first parameter to _boot(). */
	pop	%edi

	/* we can't jump to the high half of 64-bit memory directly since this is
	 * still a 32-bit instruction, so we need to add an intermediate step to
	 * a trampoline which can make the actual 64-bit far jump to high memory */
	ljmpl	$X86_64_KERN_CS, $trampoline
END(_setup)

/*
 * native 64-bit code starts here
 */

	.code64

L_ENTRY(trampoline)
	movabsq	$_setup_highmem, %rcx
	jmpq	*%rcx
L_END(trampoline)

	.text

L_ENTRY(_setup_highmem)
	/*
	 * update all pointers to virtual address space
	 */
	movabsq	$KERNBASE, %rax
	addq	%rax, %rsp
	addq	%rax, %rbp
	addq	%rax, %rdi /* multiboot tag */

	/*
	 * reload the GDT, this time with the virtual mapping
	 */
	lgdt	_x86_gdt_desc
	/* data segments are ignored in 64-bit mode, load the null descriptor */
1:	xor	%eax, %eax
	movw	%ax, %ds
	movw	%ax, %es
	movw	%ax, %fs
	movw	%ax, %gs
	movw	%ax, %ss

	/* reset RFLAGS */
	pushq	$0
	popfq

	/* remove the low memory identity mapping and bonk the TLB */
	movl	$0, _pdpt0
	movl	$0, _pdpt0 + 8
	movl	$0, _pml4t
	movq	%cr3, %rax
	movq	%rax, %cr3

	callq	_boot

	/* this should Never Be Reached(TM) */
	cli
2:	hlt
	jmp 2b
L_END(_setup_highmem)

	.section .multiboot.data, "a", @progbits

	.section .bootstrap_stack, "aw", @nobits
	.align KERN_STACK_SIZE
stack_bottom:
	.skip KERN_STACK_SIZE
stack_top: