d31f62a955
This will be needed when we support memory un-mapping, or the same user mode page tables on multiple CPUs. Neither are implemented yet. Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
127 lines
3.8 KiB
C
127 lines
3.8 KiB
C
/*
|
|
* Copyright (c) 2017 Intel Corporation
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
#include <kernel.h>
|
|
#include <sys/speculation.h>
|
|
#include <syscall_handler.h>
|
|
#include <kernel_arch_func.h>
|
|
#include <ksched.h>
|
|
#include <x86_mmu.h>
|
|
|
|
#ifndef CONFIG_X86_KPTI
|
|
/* Update the to the incoming thread's page table, and update the location of
|
|
* the privilege elevation stack.
|
|
*
|
|
* May be called ONLY during context switch and when supervisor threads drop
|
|
* synchronously to user mode. Hot code path!
|
|
*
|
|
* Nothing to do here if KPTI is enabled. We are in supervisor mode, so the
|
|
* active page tables are the kernel's page tables. If the incoming thread is
|
|
* in user mode we are going to switch CR3 to the thread-specific tables when
|
|
* we go through z_x86_trampoline_to_user.
|
|
*
|
|
* We don't need to update the privilege mode initial stack pointer either,
|
|
* privilege elevation always lands on the trampoline stack and the irq/sycall
|
|
* code has to manually transition off of it to the appropriate stack after
|
|
* switching page tables.
|
|
*/
|
|
void z_x86_swap_update_page_tables(struct k_thread *incoming)
|
|
{
|
|
uintptr_t ptables_phys;
|
|
|
|
#ifndef CONFIG_X86_64
|
|
/* 64-bit uses syscall/sysret which switches stacks manually,
|
|
* tss64.psp is updated unconditionally in __resume
|
|
*/
|
|
if ((incoming->base.user_options & K_USER) != 0) {
|
|
_main_tss.esp0 = (uintptr_t)incoming->arch.psp;
|
|
}
|
|
#endif
|
|
|
|
/* Check first that we actually need to do this, since setting
|
|
* CR3 involves an expensive full TLB flush.
|
|
*/
|
|
ptables_phys = incoming->arch.ptables;
|
|
|
|
if (ptables_phys != z_x86_cr3_get()) {
|
|
z_x86_cr3_set(ptables_phys);
|
|
}
|
|
}
|
|
#endif /* CONFIG_X86_KPTI */
|
|
|
|
FUNC_NORETURN static void drop_to_user(k_thread_entry_t user_entry,
|
|
void *p1, void *p2, void *p3)
|
|
{
|
|
uint32_t stack_end;
|
|
|
|
/* Transition will reset stack pointer to initial, discarding
|
|
* any old context since this is a one-way operation
|
|
*/
|
|
stack_end = Z_STACK_PTR_ALIGN(_current->stack_info.start +
|
|
_current->stack_info.size -
|
|
_current->stack_info.delta);
|
|
|
|
z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end,
|
|
_current->stack_info.start);
|
|
CODE_UNREACHABLE;
|
|
}
|
|
|
|
/* Preparation steps needed for all threads if user mode is turned on.
|
|
*
|
|
* Returns the initial entry point to swap into.
|
|
*/
|
|
void *z_x86_userspace_prepare_thread(struct k_thread *thread)
|
|
{
|
|
void *initial_entry;
|
|
struct z_x86_thread_stack_header *header =
|
|
(struct z_x86_thread_stack_header *)thread->stack_obj;
|
|
|
|
thread->arch.psp =
|
|
header->privilege_stack + sizeof(header->privilege_stack);
|
|
|
|
if ((thread->base.user_options & K_USER) != 0U) {
|
|
z_x86_thread_pt_init(thread);
|
|
initial_entry = drop_to_user;
|
|
} else {
|
|
thread->arch.ptables = (uintptr_t)&z_x86_kernel_ptables;
|
|
initial_entry = z_thread_entry;
|
|
}
|
|
|
|
return initial_entry;
|
|
}
|
|
|
|
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
|
void *p1, void *p2, void *p3)
|
|
{
|
|
k_spinlock_key_t key;
|
|
|
|
z_x86_thread_pt_init(_current);
|
|
|
|
key = k_spin_lock(&z_mem_domain_lock);
|
|
/* Apply memory domain configuration, if assigned. Threads that
|
|
* started in user mode already had this done via z_setup_new_thread()
|
|
*/
|
|
z_x86_apply_mem_domain(_current, _current->mem_domain_info.mem_domain);
|
|
k_spin_unlock(&z_mem_domain_lock, key);
|
|
|
|
#ifndef CONFIG_X86_KPTI
|
|
/* We're synchronously dropping into user mode from a thread that
|
|
* used to be in supervisor mode. K_USER flag has now been set, but
|
|
* Need to swap from the kernel's page tables to the per-thread page
|
|
* tables.
|
|
*
|
|
* Safe to update page tables from here, all tables are identity-
|
|
* mapped and memory areas used before the ring 3 transition all
|
|
* have the same attributes wrt supervisor mode access.
|
|
*
|
|
* Threads that started in user mode already had this applied on
|
|
* initial context switch.
|
|
*/
|
|
z_x86_swap_update_page_tables(_current);
|
|
#endif
|
|
|
|
drop_to_user(user_entry, p1, p2, p3);
|
|
}
|