diff --git a/arch/Kconfig b/arch/Kconfig index 774840dda0f..27dbbc0b10c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1148,3 +1148,9 @@ config ARCH_HAS_CUSTOM_BUSY_WAIT It's possible that an architecture port cannot or does not want to use the provided k_busy_wait(), but instead must do something custom. It must enable this option in that case. + +config ARCH_HAS_CUSTOM_CURRENT_IMPL + bool + help + Select when architecture implements arch_current_thread() & + arch_current_thread_set(). diff --git a/doc/releases/release-notes-4.1.rst b/doc/releases/release-notes-4.1.rst index b19ad21b0e7..bb52564d168 100644 --- a/doc/releases/release-notes-4.1.rst +++ b/doc/releases/release-notes-4.1.rst @@ -36,6 +36,14 @@ Deprecated in this release Architectures ************* +* Common + + * Introduced :kconfig:option:`CONFIG_ARCH_HAS_CUSTOM_CURRENT_IMPL`, which can be selected when + an architecture implemented and enabled its own :c:func:`arch_current_thread` and + :c:func:`arch_current_thread_set` functions for faster retrieval of the current CPU's thread + pointer. When enabled, ``_current`` variable will be routed to the + :c:func:`arch_current_thread` (:github:`80716`). + * ARC * ARM diff --git a/include/zephyr/arch/arch_inlines.h b/include/zephyr/arch/arch_inlines.h index 0f32159e2f1..04c4a649f1e 100644 --- a/include/zephyr/arch/arch_inlines.h +++ b/include/zephyr/arch/arch_inlines.h @@ -34,4 +34,6 @@ #include #endif +#include + #endif /* ZEPHYR_INCLUDE_ARCH_INLINES_H_ */ diff --git a/include/zephyr/arch/common/arch_inlines.h b/include/zephyr/arch/common/arch_inlines.h new file mode 100644 index 00000000000..0490dba71aa --- /dev/null +++ b/include/zephyr/arch/common/arch_inlines.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 Meta Platforms. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_ZEPHYR_ARCH_COMMON_ARCH_INLINES_H_ +#define ZEPHYR_INCLUDE_ZEPHYR_ARCH_COMMON_ARCH_INLINES_H_ + +#ifndef ZEPHYR_INCLUDE_ARCH_INLINES_H_ +#error "This header shouldn't be included directly" +#endif /* ZEPHYR_INCLUDE_ARCH_INLINES_H_ */ + +#ifndef _ASMLANGUAGE + +#include + +#ifndef CONFIG_ARCH_HAS_CUSTOM_CURRENT_IMPL +static ALWAYS_INLINE struct k_thread *arch_current_thread(void) +{ +#ifdef CONFIG_SMP + /* In SMP, _current is a field read from _current_cpu, which + * can race with preemption before it is read. We must lock + * local interrupts when reading it. + */ + unsigned int k = arch_irq_lock(); + + struct k_thread *ret = _current_cpu->current; + + arch_irq_unlock(k); +#else + struct k_thread *ret = _kernel.cpus[0].current; +#endif /* CONFIG_SMP */ + return ret; +} + +static ALWAYS_INLINE void arch_current_thread_set(struct k_thread *thread) +{ + _current_cpu->current = thread; +} +#endif /* CONFIG_ARCH_HAS_CUSTOM_CURRENT_IMPL */ + +#endif /* _ASMLANGUAGE */ + +#endif /* ZEPHYR_INCLUDE_ZEPHYR_ARCH_COMMON_ARCH_INLINES_H_ */ diff --git a/include/zephyr/kernel_structs.h b/include/zephyr/kernel_structs.h index cf7daff9a6c..2467598175f 100644 --- a/include/zephyr/kernel_structs.h +++ b/include/zephyr/kernel_structs.h @@ -260,7 +260,7 @@ bool z_smp_cpu_mobile(void); #define _current_cpu ({ __ASSERT_NO_MSG(!z_smp_cpu_mobile()); \ arch_curr_cpu(); }) -#define _current k_sched_current_thread_query() +#define _current arch_current_thread() #else #define _current_cpu (&_kernel.cpus[0]) diff --git a/kernel/include/kswap.h b/kernel/include/kswap.h index d3638b6179a..03c9967b2b6 100644 --- a/kernel/include/kswap.h +++ b/kernel/include/kswap.h @@ -134,7 +134,7 @@ static ALWAYS_INLINE unsigned int do_swap(unsigned int key, #endif /* CONFIG_SMP */ z_thread_mark_switched_out(); z_sched_switch_spin(new_thread); - _current_cpu->current = new_thread; + arch_current_thread_set(new_thread); #ifdef CONFIG_TIMESLICING z_reset_time_slice(new_thread); @@ -260,6 +260,6 @@ static inline void z_dummy_thread_init(struct k_thread *dummy_thread) dummy_thread->base.slice_ticks = 0; #endif /* CONFIG_TIMESLICE_PER_THREAD */ - _current_cpu->current = dummy_thread; + arch_current_thread_set(dummy_thread); } #endif /* ZEPHYR_KERNEL_INCLUDE_KSWAP_H_ */ diff --git a/kernel/mmu.c b/kernel/mmu.c index 788f30ff730..617b02997dd 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -1674,7 +1674,7 @@ static bool do_page_fault(void *addr, bool pin) #endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ key = k_spin_lock(&z_mm_lock); - faulting_thread = _current_cpu->current; + faulting_thread = _current; status = arch_page_location_get(addr, &page_in_location); if (status == ARCH_PAGE_LOCATION_BAD) { diff --git a/kernel/sched.c b/kernel/sched.c index eda1a3e0908..8faeea27cec 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -838,11 +838,11 @@ struct k_thread *z_swap_next_thread(void) } #ifdef CONFIG_USE_SWITCH -/* Just a wrapper around _current = xxx with tracing */ +/* Just a wrapper around arch_current_thread_set(xxx) with tracing */ static inline void set_current(struct k_thread *new_thread) { z_thread_mark_switched_out(); - _current_cpu->current = new_thread; + arch_current_thread_set(new_thread); } /** @@ -1230,20 +1230,7 @@ static inline void z_vrfy_k_wakeup(k_tid_t thread) k_tid_t z_impl_k_sched_current_thread_query(void) { -#ifdef CONFIG_SMP - /* In SMP, _current is a field read from _current_cpu, which - * can race with preemption before it is read. We must lock - * local interrupts when reading it. - */ - unsigned int k = arch_irq_lock(); -#endif /* CONFIG_SMP */ - - k_tid_t ret = _current_cpu->current; - -#ifdef CONFIG_SMP - arch_irq_unlock(k); -#endif /* CONFIG_SMP */ - return ret; + return arch_current_thread(); } #ifdef CONFIG_USERSPACE diff --git a/kernel/thread.c b/kernel/thread.c index 69728a403d9..8dabaa8db2c 100644 --- a/kernel/thread.c +++ b/kernel/thread.c @@ -946,8 +946,8 @@ void z_thread_mark_switched_out(void) #ifdef CONFIG_TRACING #ifdef CONFIG_THREAD_LOCAL_STORAGE /* Dummy thread won't have TLS set up to run arbitrary code */ - if (!_current_cpu->current || - (_current_cpu->current->base.thread_state & _THREAD_DUMMY) != 0) + if (!_current || + (_current->base.thread_state & _THREAD_DUMMY) != 0) return; #endif /* CONFIG_THREAD_LOCAL_STORAGE */ SYS_PORT_TRACING_FUNC(k_thread, switched_out);