diff --git a/Cadence/Xtensa/Makefile b/Cadence/Xtensa/Makefile index a446b831f..460bde70e 100644 --- a/Cadence/Xtensa/Makefile +++ b/Cadence/Xtensa/Makefile @@ -30,6 +30,11 @@ ifeq ($(OVERLAY),1) OVLYFLAGS = -DXT_USE_THREAD_SAFE_CLIB=1 -DXT_USE_OVLY=1 endif +# Build without SMP support +ifeq ($(SMP),0) +SMPFLAGS = -DconfigNUMBER_OF_CORES=1 +endif + SRCROOT = $(subst /,$(S),$(CURDIR)) TSTROOT = $(subst /,$(S),$(abspath $(SRCROOT)$(S)..$(S)..$(S)..$(S)..$(S)..$(S)..$(S)Demo$(S)ThirdParty$(S)Partner-Supported-Demos$(S)Cadence_Xtensa_ISS_xt-clang$(SMALL))) BLDROOT = $(TSTROOT)$(S)build @@ -79,7 +84,7 @@ endif IFLAGS = \ -I$(FR_SRCDIR)$(S)include -I$(XT_SRCDIR) -I$(TSTROOT)$(S)common$(S)config_files -I$(BLDDIR) -CSTD = -std=c99 +CSTD = -std=c11 ifeq ($(OVERLAY),1) CFLAGS = -Os -g else @@ -87,8 +92,11 @@ CFLAGS = -O2 -g endif WFLAGS = -Werror -Wall -Wextra CFGFLAGS ?= -CCFLAGS = $(CSTD) $(CFGFLAGS) $(CFLAGS) $(WFLAGS) -mno-coproc -mlongcalls -ffunction-sections -mno-l32r-flix $(DFLAGS) $(MPUFLAGS) $(OVLYFLAGS) -ASFLAGS = $(CCFLAGS) +CCFLAGS = $(CSTD) $(CFGFLAGS) $(CFLAGS) $(WFLAGS) -mno-coproc -mlongcalls -ffunction-sections -mno-l32r-flix $(DFLAGS) $(MPUFLAGS) $(OVLYFLAGS) $(SMPFLAGS) + +# Avoid LTO due to build issues related to inline assembly +CCFLAGS_F = $(filter-out -flto,$(CCFLAGS)) +ASFLAGS = $(CCFLAGS_F) # File-specific flags @@ -120,12 +128,12 @@ $(OSLIB) : $(LIB_O_LIST) $(AR) -rs $@ $^ $(BLDDIR)/asm-offsets.h : asm-offsets.c $(BLDDIR)/.mkdir - $(CC) $(CCFLAGS) $(IFLAGS) -MD -MF $(subst .h,.d,$@) -MT $@ -ffunction-sections -fdata-sections -Wl,--gc-sections -o $@.exe $< + $(CC) $(CCFLAGS_F) $(IFLAGS) -MD -MF $(subst .h,.d,$@) -MT $@ -ffunction-sections -fdata-sections -Wl,--gc-sections -o $@.exe $< -lxtutil $(ISS) $@.exe > $@ $(RM) $(subst /,$(S),$@.exe) $(BLDDIR)/%.o : %.c $(BLDDIR)/.mkdir - $(CC) $(CCFLAGS) $(IFLAGS) $(FLAGS_$*) -mtext-section-literals -MD -MF $(subst .o,.d,$@) -c -o $@ $< + $(CC) $(CCFLAGS_F) $(IFLAGS) $(FLAGS_$*) -mtext-section-literals -MD -MF $(subst .o,.d,$@) -c -o $@ $< $(BLDDIR)/%.o : %.S $(BLDDIR)/.mkdir $(CC) $(ASFLAGS) $(IFLAGS) $(FLAGS_$*) -MD -MF $(subst .o,.d,$@) -c -o $@ $< diff --git a/Cadence/Xtensa/asm-offsets.c b/Cadence/Xtensa/asm-offsets.c index bdeb16bc0..fde46118a 100644 --- a/Cadence/Xtensa/asm-offsets.c +++ b/Cadence/Xtensa/asm-offsets.c @@ -1,5 +1,6 @@ #include #include + #include "../../../../../tasks.c" #define DEFINE(sym,val) \ @@ -18,7 +19,14 @@ int main(void) #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) DEFINE(TCB_END_OF_STACK_OFF, offsetof(TCB_t, pxEndOfStack)); #endif +#if ( ( configUSE_CORE_AFFINITY == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + DEFINE(TCB_CORE_AFFINITY_MASK_OFF, offsetof(TCB_t, uxCoreAffinityMask)); +#endif #if ( configUSE_C_RUNTIME_TLS_SUPPORT == 1 ) DEFINE(TCB_IMPURE_PTR_OFF, offsetof(TCB_t, xTLSBlock)); #endif + DEFINE(PORTINT_NEST_OFF, offsetof(xt_internal_data_t, port_interruptNesting)); + DEFINE(PORTINT_SWITCH_OFF, offsetof(xt_internal_data_t, port_switch_flag)); + DEFINE(PORTINT_INTENABLE_OFF, offsetof(xt_internal_data_t, xt_intenable)); + DEFINE(PORTINT_VPRI_MASK_OFF, offsetof(xt_internal_data_t, xt_vpri_mask)); } diff --git a/Cadence/Xtensa/mpu.S b/Cadence/Xtensa/mpu.S index 5dfd3104e..9aa194a0c 100644 --- a/Cadence/Xtensa/mpu.S +++ b/Cadence/Xtensa/mpu.S @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -30,6 +30,7 @@ #include "xtensa_rtos.h" #include "portmacro.h" #include "asm-offsets.h" +#include "xtensa_asm.h" #include @@ -82,7 +83,7 @@ Entry Conditions: .align 4 _xt_mpu_restore: - movi A2, pxCurrentTCB + pxctcb A2, A3 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i A2, A2, 0 beqz A2, .Lset diff --git a/Cadence/Xtensa/port.c b/Cadence/Xtensa/port.c index c46764ee1..4907ce5c7 100644 --- a/Cadence/Xtensa/port.c +++ b/Cadence/Xtensa/port.c @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -42,19 +42,19 @@ #include #endif +#include "FreeRTOS.h" +#include "xtensa_config.h" + #include "xtensa_api.h" #include "xtensa_rtos.h" -#include "FreeRTOS.h" #include "task.h" /* Heap area (see heap_4.c). When MPU in use, align it to the MPU region boundary to avoid overlapping with non-heap data. */ -#if portUSING_MPU_WRAPPERS +#if portUSING_MPU_WRAPPERS && configAPPLICATION_ALLOCATED_HEAP #define HEAP_SIZE ((configTOTAL_HEAP_SIZE + XCHAL_MPU_ALIGN - 1) & -XCHAL_MPU_ALIGN) PRIVILEGED_DATA uint8_t ucHeap[ HEAP_SIZE ] __attribute__((aligned(XCHAL_MPU_ALIGN))); -#else -uint8_t ucHeap[ configTOTAL_HEAP_SIZE ]; #endif #if portUSING_MPU_WRAPPERS @@ -81,6 +81,12 @@ extern void _xt_task_start( void ); extern void _xt_task_start_user( void ); #endif +#if ( configNUMBER_OF_CORES > 1 ) +extern uint32_t _bss_table_start; +extern uint32_t _bss_table_end; +extern void __bss_init(uint32_t * table_start, uint32_t * table_end); +#endif + // Timer tick interval in cycles. static uint32_t xt_tick_cycles; TickType_t xMaxSuppressedTicks; @@ -99,8 +105,181 @@ int32_t xt_sw_intnum = -1; // Duplicate of inaccessible xSchedulerRunning. uint32_t port_xSchedulerRunning = 0U; -// Interrupt nesting level. -uint32_t port_interruptNesting = 0U; +#if (defined __DYNAMIC_REENT__) + #if ( configNUMBER_OF_CORES > 1 ) + #if XSHAL_CLIB == XTHAL_CLIB_XCLIB + #define _XT_INTDATA_REENT_INIT(x) NULL, NULL, { 0 }, + #elif XSHAL_CLIB == XTHAL_CLIB_NEWLIB + #if ( XT_USE_DATARAM ) + #define _XT_INTDATA_REENT_INIT(x) NULL, NULL, _REENT_INIT(_xt_intdata.xt_reent), + #else + #define _XT_INTDATA_REENT_INIT(x) NULL, NULL, _REENT_INIT(_xt_intdata[(x)].xt_reent), + #endif + #else + #error Specified CLIB not reentrant + #endif + #else + #define _XT_INTDATA_REENT_INIT NULL, + #endif // configNUMBER_OF_CORES > 1 +#else + #if ( configNUMBER_OF_CORES > 1 ) + #define _XT_INTDATA_REENT_INIT(x) + #else + #define _XT_INTDATA_REENT_INIT + #endif +#endif // __DYNAMIC_REENT__ + +#if ( configNUMBER_OF_CORES == 1 ) + +// Interrupt nesting level and task switch flag maintained together. +xt_internal_data_t _xt_intdata = { + 0, 0, 0, 0xffffffff, _XT_INTDATA_REENT_INIT +}; + +#else + +#if ( XT_USE_DATARAM ) + +// Per-core struct contains interrupt variables and uxCriticalNestings +// When in dataram, structure is in per-core memory and not padded. +xt_internal_data_t XT_DATARAM_ATTR +_xt_intdata = { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(0) }; + +#else // XT_USE_DATARAM + +// Per-core struct contains interrupt variables and uxCriticalNestings +// When in shared sysram, structure is padded to cache line and indexed per-core +xt_internal_data_t __attribute__((aligned (XCHAL_DCACHE_LINESIZE))) +_xt_intdata[ configNUMBER_OF_CORES ] = { + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(0) { 0 } }, +#if ( configNUMBER_OF_CORES >= 2 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(1) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES >= 3 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(2) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES >= 4 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(3) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES >= 5 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(4) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES >= 6 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(5) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES >= 7 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(6) { 0 } }, +#endif +#if ( configNUMBER_OF_CORES == 8 ) + { 0, 0, 0, 0, 0xffffffff, 0, _XT_INTDATA_REENT_INIT(7) { 0 } }, +#endif +}; + +#endif // XT_USE_DATARAM + +PRIVILEGED_DATA xt_mutex __attribute__((aligned (XCHAL_DCACHE_LINESIZE))) _xt_mutex_ISR; +PRIVILEGED_DATA xt_mutex __attribute__((aligned (XCHAL_DCACHE_LINESIZE))) _xt_mutex_task; + +#if (XT_USE_L2RAM) +XTHAL_L2_SETUP(XCHAL_L2RAM_RESET_PADDR, XT_L2RAM_SIXTEENTHS, XT_L2CACHE_SIXTEENTHS); +#endif + +/* + * Initialize the mutex. + */ +void +xt_mutex_init(xt_mutex_p pmtx) +{ + if (pmtx != NULL) { + pmtx->owner = 0U; + pmtx->count = 0U; + } +} + +/* + * Lock the mutex, busy wait until lock acquired. Can be called repeatedly + * to lock an already owned mutex. Note the locking is not protected against + * interrupts. This is a potentially blocking function and should not be + * called from an interrupt handler anyway. + */ +int32_t +xt_mutex_lock(xt_mutex_p pmtx) +{ + uint32_t id = (portGET_CORE_ID()) + 1U; + + if (pmtx != NULL) { + if (pmtx->owner == id) { + pmtx->count++; + } + else { + int32_t ret; + + do { +#if XCHAL_HAVE_EXCLUSIVE + /* Streamline implementation for SMP case. + * %0 : ret + * %1 : address &(pmtx->owner) + * : test value (0) optimized w/ bnez + * %2 : set value (id) preserved for loop + * %3 : temp reg 1 + * %4 : temp reg 2 + */ + uint32_t t1 = 0, t2 = 1; /* Different values trick optimizer */ + __asm__ volatile ("mov %3, %2 /* %3 = copy of set_value */ \n\t" + "1: \n\t" + "l32ex %0, %1 /* %0 = *address, set monitor */ \n\t" + "bnez %0, 2f /* skip write if *address != 0 */ \n\t" + "mov %4, %3 /* %4 = set_value */ \n\t" + "s32ex %4, %1 /* *address = set_value */ \n\t" + "getex %4 /* get result of store */ \n\t" + "beqz %4, 1b \n\t" + "2: \n\t" + "clrex /* in case we skipped write */ \n\t" + : "=&r"(ret) + : "r"(&(pmtx->owner)), "r"(id), "r"(t1), "r"(t2)); +#else + ret = xthal_compare_and_set((int32_t *) &(pmtx->owner), 0, (int32_t) id); +#endif + } while (ret != 0); + pmtx->count = 1U; + } + return 0; + } + + return -1; +} + +/* + * Unlock the mutex. Can be called repeatedly to unlock the same mutex. + * The lock is only released when the lock count goes to zero. + */ +int32_t +xt_mutex_unlock(xt_mutex_p pmtx) +{ + uint32_t id = (portGET_CORE_ID()) + 1U; + + if ((pmtx != NULL) && (pmtx->owner == id)) { + pmtx->count--; + if (pmtx->count == 0U) { + pmtx->owner = 0U; + } + return 0; + } + + return -1; +} + + +// Ensure SMP initialization flag values are non-zero so it gets linked +// into .data and not .bss. +typedef enum { + XT_SMP_SYNC_INITVAL = 1, + XT_SMP_SYNC_DONE = 2, +} xt_smp_sync_t; + +volatile xt_smp_sync_t xt_smp_sync = XT_SMP_SYNC_INITVAL; + +#endif // ( configNUMBER_OF_CORES == 1 ) #undef MPU_WRAPPERS_INCLUDED_FROM_API_FILE @@ -134,12 +313,12 @@ static void xt_tick_handler( void ) // Interrupts upto configMAX_SYSCALL_INTERRUPT_PRIORITY must be // disabled before calling xTaskIncrementTick as it accesses the // kernel lists. - interruptMask = portSET_INTERRUPT_MASK_FROM_ISR(); + interruptMask = taskENTER_CRITICAL_FROM_ISR(); { ret = xTaskIncrementTick(); ++xt_tick_count; } - portCLEAR_INTERRUPT_MASK_FROM_ISR( interruptMask ); + taskEXIT_CRITICAL_FROM_ISR( interruptMask ); portYIELD_FROM_ISR( ret ); @@ -187,6 +366,25 @@ static void xt_tick_timer_stop( void ) xt_set_ccompare( XT_TIMER_INDEX, 0 ); } +#if ( configNUMBER_OF_CORES > 1 ) +//----------------------------------------------------------------------------- +// IPI interrupts used for multicore scheduler +//----------------------------------------------------------------------------- +const uint32_t xt_ipi_intnum[configNUMBER_OF_CORES] = XCHAL_SUBSYS_IPI_S0_INTLIST; + +//----------------------------------------------------------------------------- +// portYIELD_CORE IPI handler wrapper +//----------------------------------------------------------------------------- +static void xt_ipi_yield_wrapper( void * arg ) +{ + // Flag a context switch and exit; _Interrupt() will do the rest. + // Do NOT call vPortYieldFromInt() directly, which would result in twice + // saving and clearing CPENABLE, corrupting the coprocessor state. + UNUSED(arg); + portYIELD_FROM_ISR(1); // Flag a context switch and exit +} +#endif + //----------------------------------------------------------------------------- // Start the scheduler. //----------------------------------------------------------------------------- @@ -194,8 +392,13 @@ BaseType_t xPortStartScheduler( void ) { #if XCHAL_HAVE_XEA3 extern void xt_sched_handler(void * arg); + extern void xt_unhandled_interrupt(void * arg); int32_t i; #endif + #if (configNUMBER_OF_CORES > 1 ) + uint32_t c; + uint32_t my_core = portGET_CORE_ID(); + #endif // Interrupts are disabled at this point and stack contains PS with // enabled interrupts when task context is restored. @@ -214,8 +417,11 @@ BaseType_t xPortStartScheduler( void ) // Select a software interrupt to use for scheduling. for (i = 0; i < XCHAL_NUM_INTERRUPTS; i++) { if ((Xthal_inttype[i] == XTHAL_INTTYPE_SOFTWARE) && (Xthal_intlevel[i] == 1)) { - xt_sw_intnum = i; - break; + if (xt_get_interrupt_handler(i) == &xt_unhandled_interrupt) { + // Finalize the interrupt if not already in use + xt_sw_intnum = i; + break; + } } } @@ -233,15 +439,54 @@ BaseType_t xPortStartScheduler( void ) #if XCHAL_HAVE_ISL XT_WSR_ISL(0); #endif - #endif + #endif // XCHAL_HAVE_XEA3 + + #if ( configNUMBER_OF_CORES > 1 ) + // Initialize SMP mutexes + if (my_core == 0) { + xt_mutex_init(&_xt_mutex_ISR); + xt_mutex_init(&_xt_mutex_task); + } else { + // Ensure core 0 started first. + // NOTE: if this assert triggers, ensure all nonzero cores start in RunStall. + // On XTSC, that may entail setting the RunOnReset InitValue in subsys.yml. + configASSERT( port_xSchedulerRunning ); + } + + // Limit the number of cacheops to prevent hangs in case the test uses large + // number of CacheOPs at the same time on multiple cores + xthal_L2_prefetch_set_limit(XCHAL_L2CC_MAX_REQ/2); + + // Configure inter-processor interrupts that can be triggered by other cores; + // used for portYIELD_CORE(). + for (c = 0; c < configNUMBER_OF_CORES; c++) { + if (c != my_core) { + if (!xt_set_interrupt_handler(xt_ipi_intnum[c], xt_ipi_yield_wrapper, NULL)) { + return pdFALSE; + } + xt_interrupt_enable(xt_ipi_intnum[c]); + } + } + if (my_core == configTICK_CORE) { + // Set up and enable timer tick. + xt_tick_timer_init(); + } + #else // configNUMBER_OF_CORES // Set up and enable timer tick. xt_tick_timer_init(); + #endif // configNUMBER_OF_CORES #if XT_USE_THREAD_SAFE_CLIB // Init C library - vPortClibInit(); - #endif + #if ( configNUMBER_OF_CORES > 1 ) + if (my_core == 0) + #endif // ( configNUMBER_OF_CORES > 1 ) + { + // Init C library + vPortClibInit(); + } + #endif // XT_USE_THREAD_SAFE_CLIB #if portUSING_MPU_WRAPPERS // Setup MPU @@ -251,6 +496,25 @@ BaseType_t xPortStartScheduler( void ) port_xSchedulerRunning = 1U; + #if ( configNUMBER_OF_CORES > 1 ) + if (my_core == 0) { + // Cache-coherence means writeback operations are unnecessary. + xt_smp_sync = XT_SMP_SYNC_DONE; + + // Release other cores last + if (xthal_run_cores(XTSUB_RUN_ALL_CORES)) { + return pdFALSE; + } + } else { + // Used by xt-gdb thread-aware debug support + _XT_INTDATA(my_core).xt_core_init_done = 1; + } + #endif + + // Spill and invalidate prior register windows so that solicited + // restores do not inadvertently pick up starting register window + xthal_window_spill(); + // Cannot be directly called from C; never returns __asm__ volatile ("call0 _frxt_dispatch\n"); @@ -260,7 +524,7 @@ BaseType_t xPortStartScheduler( void ) BaseType_t xPortIsInsideInterrupt( void ) { - return port_interruptNesting > 0 ? pdTRUE : pdFALSE; + return (_XT_INTDATA(portGET_CORE_ID()).port_interruptNesting) > 0 ? pdTRUE : pdFALSE; } //----------------------------------------------------------------------------- @@ -272,6 +536,46 @@ void vPortEndScheduler( void ) port_xSchedulerRunning = 0U; } + +#if ( configNUMBER_OF_CORES > 1 ) +//----------------------------------------------------------------------------- +// Starting with port v3.11, the SMP init process is slightly different: only +// core 0 calls main(); other cores enter the scheduler directly via _start(). +// +// Since SMP requires coherent shared memory, core 0 must do the following: +// 1) initialize BSS in shared memories, and +// 2) call __clibrary_init() +// +// Nonzero cores will skip these steps and wait here until core 0 calls +// xPortStartScheduler(), ensuring all cores are synchronized, regardless of +// the order reset was released. +// +// This process is implemented by overriding the __memmap_init() hook in the +// XTOS CRT init sequence. Note that BSS is not initialized at this point so +// we must only reference initialized global data. Any systems that require +// additional MMU/TLB setup will need to hook in here as well. +//----------------------------------------------------------------------------- +void __memmap_init(void) +{ + if (portGET_CORE_ID() > 0) { + portDISABLE_INTERRUPTS(); + while (xt_smp_sync != XT_SMP_SYNC_DONE) { + // Busy-wait + } + + // By this point core 0 will have initialized BSS in shared memory + // but we still need to initialize per-core BSS segments + __bss_init(&_bss_table_start, &_bss_table_end); + + (void) xPortStartScheduler(); + + // Does not return here + configASSERT( 0 ); + } +} +#endif // ( configNUMBER_OF_CORES > 1 ) + + //----------------------------------------------------------------------------- // Stack initialization. // Reserve coprocessor save area if needed, construct a dummy stack frame and @@ -518,6 +822,7 @@ void vPortSuppressTicksAndSleep( TickType_t xExpectedIdleTime ) #if portUSING_MPU_WRAPPERS extern void vPortResetPrivilege(BaseType_t previous); + void vPortEnterCritical( void ) { // TODO: handle configALLOW_UNPRIVILEGED_CRITICAL_SECTIONS @@ -530,7 +835,6 @@ void vPortEnterCritical( void ) } else { - // TODO: handle port_interruptNesting vTaskEnterCritical(); } } @@ -547,7 +851,6 @@ void vPortExitCritical( void ) } else { - // TODO: handle port_interruptNesting vTaskExitCritical(); } } diff --git a/Cadence/Xtensa/portasm.S b/Cadence/Xtensa/portasm.S index 9acb089a2..e311d1ede 100644 --- a/Cadence/Xtensa/portasm.S +++ b/Cadence/Xtensa/portasm.S @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -30,25 +30,34 @@ #include "xtensa_rtos.h" #include "asm-offsets.h" #include "portmacro.h" +#include "xtensa_asm.h" - .extern pxCurrentTCB - -/* -******************************************************************************* -* Interrupt stack. The size of the interrupt stack is determined by the config -* parameter "configISR_STACK_SIZE" in FreeRTOSConfig.h -******************************************************************************* -*/ +#if XCHAL_HAVE_XEA3 .data .align 16 - .global port_switch_flag -port_switch_flag: - .word 0 - .global port_yield_flag port_yield_flag: .word 0 +#endif + +#if (XCHAL_CP_NUM > 0) && (configNUMBER_OF_CORES > 1) && (configUSE_CORE_AFFINITY == 1) + /* Keep some affinity bitmasks to quickly decide whether to save CP state */ + .data +port_affinity_masks: + .word ~0x1 + .word ~0x2 + #if ( configNUMBER_OF_CORES > 2 ) + .word ~0x4 + .word ~0x8 + #if ( configNUMBER_OF_CORES > 4 ) + .word ~0x10 + .word ~0x20 + .word ~0x40 + .word ~0x80 + #endif + #endif +#endif #if portUSING_MPU_WRAPPERS @@ -57,6 +66,7 @@ port_yield_flag: .text #endif + /* ******************************************************************************* * _frxt_setup_switch @@ -74,9 +84,9 @@ _frxt_setup_switch: ENTRY(16) - movi a2, port_switch_flag + pintdata a2, a3 movi a3, 1 - s32i a3, a2, 0 + s32i a3, a2, PORTINT_SWITCH_OFF /* port_switch_flag = 1 */ RET(16) @@ -117,20 +127,28 @@ _frxt_int_enter: Manage nesting directly rather than call the generic IntEnter() (in windowed ABI we can't call a C function here anyway because PS.EXCM is still set). */ + pintdata a3, a2 /* a3 <- &port_interruptNesting */ movi a2, port_xSchedulerRunning - movi a3, port_interruptNesting l32i a2, a2, 0 /* a2 = port_xSchedulerRunning */ beqz a2, 1f /* scheduler not running, no tasks */ - l32i a2, a3, 0 /* a2 = port_interruptNesting */ + l32i a2, a3, PORTINT_NEST_OFF /* a2 = port_interruptNesting */ addi a2, a2, 1 /* increment nesting count */ - s32i a2, a3, 0 /* save nesting count */ + s32i a2, a3, PORTINT_NEST_OFF /* save nesting count */ bnei a2, 1, .Lnested /* !=0 before incr, so nested */ - movi a2, pxCurrentTCB + pxctcb a2, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a2, a2, 0 /* a2 = current TCB */ beqz a2, 1f s32i a1, a2, TCB_TOP_OF_STACK_OFF /* pxCurrentTCB->pxTopOfStack = SP */ - movi a1, xt_interrupt_stack_top /* a1 = top of intr stack */ + movi a1, xt_interrupt_stack_top /* a1 = top of core 0 intr stack */ + #if ( configNUMBER_OF_CORES > 1 ) + beqz a3, 1f /* a3 contains coreid from pxctcb */ + movi a2, configISR_STACK_SIZE +.Lset_int_stack: + add a1, a1, a2 + addi a3, a3, -1 + bnez a3, .Lset_int_stack + #endif .Lnested: 1: @@ -161,26 +179,25 @@ _frxt_int_enter: .align 4 _frxt_int_exit: + pintdata a3, a2 /* a3 <- &port_interruptNesting */ movi a2, port_xSchedulerRunning - movi a3, port_interruptNesting rsil a0, XT_IRQ_LOCK_LEVEL /* lock out interrupts */ l32i a2, a2, 0 /* a2 = port_xSchedulerRunning */ beqz a2, .Lnoswitch /* scheduler not running, no tasks */ - l32i a2, a3, 0 /* a2 = port_interruptNesting */ + l32i a2, a3, PORTINT_NEST_OFF /* a2 = port_interruptNesting */ addi a2, a2, -1 /* decrement nesting count */ - s32i a2, a3, 0 /* save nesting count */ + s32i a2, a3, PORTINT_NEST_OFF /* save nesting count */ bnez a2, .Lnesting /* !=0 after decr so still nested */ - movi a2, pxCurrentTCB + pxctcb a2, a4 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a2, a2, 0 /* a2 = current TCB */ beqz a2, 1f /* no task ? go to dispatcher */ l32i a1, a2, TCB_TOP_OF_STACK_OFF /* SP = pxCurrentTCB->pxTopOfStack */ - movi a2, port_switch_flag /* address of switch flag */ - l32i a3, a2, 0 /* a3 = port_switch_flag */ - beqz a3, .Lnoswitch /* flag = 0 means no switch reqd */ - movi a3, 0 - s32i a3, a2, 0 /* zero out the flag for next time */ + l32i a2, a3, PORTINT_SWITCH_OFF /* a2 = port_switch_flag */ + beqz a2, .Lnoswitch /* flag = 0 means no switch reqd */ + movi a2, 0 + s32i a2, a3, PORTINT_SWITCH_OFF /* zero out the flag for next time */ 1: /* @@ -272,12 +289,17 @@ _frxt_dispatch: s32i a3, a2, 0 /* Indicate thread yield. */ #ifdef __XTENSA_CALL0_ABI__ + #if ( configNUMBER_OF_CORES > 1 ) + coreid a2 + #endif call0 vTaskSwitchContext // Get next TCB to resume - movi a2, pxCurrentTCB + pxctcb a2, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ #else - movi a2, pxCurrentTCB + // When (configNUMBER_OF_CORES > 1) we call vTaskSwitchContext(coreId) + // ... and pxctcb places coreid into second register parameter (for SMP) + pxctcb a2, a10 /* pxCurrentTCB or pxCurrentTCBs[] */ call8 vTaskSwitchContext // Get next TCB to resume -#endif +#endif // __XTENSA_CALL0_ABI__ l32i a3, a2, 0 /* a3 = pxCurrentTCB */ l32i a1, a3, TCB_TOP_OF_STACK_OFF /* SP = next_TCB->pxTopOfStack */ @@ -296,10 +318,15 @@ _frxt_dispatch: #if XCHAL_HAVE_XEA2 #ifdef __XTENSA_CALL0_ABI__ + #if ( configNUMBER_OF_CORES > 1 ) + coreid a2 + #endif call0 vTaskSwitchContext // Get next TCB to resume - movi a2, pxCurrentTCB + pxctcb a2, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ #else - movi a2, pxCurrentTCB + // When (configNUMBER_OF_CORES > 1) we call vTaskSwitchContext(coreId) + // ... and pxctcb places coreid into second register parameter (for SMP) + pxctcb a2, a6 /* pxCurrentTCB or pxCurrentTCBs[] */ call4 vTaskSwitchContext // Get next TCB to resume #endif l32i a3, a2, 0 @@ -326,6 +353,12 @@ _frxt_dispatch: skip_lowering_priv: #endif + #if XCHAL_HAVE_LOOPS + /* LCOUNT was not saved upon solicited yield; clear it to a known state */ + movi a3, 0 + wsr a3, LCOUNT + #endif + l32i a3, sp, XT_STK_PS #ifdef __XTENSA_CALL0_ABI__ l32i a12, sp, XT_STK_A12 @@ -351,7 +384,7 @@ skip_lowering_priv: #if XCHAL_CP_NUM > 0 /* Restore CPENABLE from task's co-processor save area. */ - movi a3, pxCurrentTCB + pxctcb a3, a2 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a3, a3, 0 l32i a2, a3, TCB_END_OF_STACK_OFF /* a2 = base of coproc save area */ l16ui a3, a2, XT_CPENABLE /* a3 = saved CPENABLE value */ @@ -439,7 +472,7 @@ vPortYield: movi a8, PS_STACK_MASK | PS_DI_MASK /* disable interrupts. */ xps a2, a8 - movi a3, pxCurrentTCB + pxctcb a3, a4 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a4, a3, 0 /* a4 = pxCurrentTCB */ #if XCHAL_CP_NUM > 0 @@ -525,7 +558,7 @@ vPortYield: call0 _xt_coproc_savecs #endif - movi a2, pxCurrentTCB + pxctcb a2, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ movi a3, 0 l32i a2, a2, 0 /* a2 = pxCurrentTCB */ s32i a3, sp, XT_STK_EXIT /* 0 to flag as solicited frame */ @@ -539,7 +572,7 @@ vPortYield: beqz a2, 1f s16i a3, a2, XT_CPENABLE /* clear saved cpenable */ 1: - #endif + #endif /* XCHAL_CP_NUM > 0 */ /* Tail-call dispatcher. */ call0 _frxt_dispatch @@ -574,15 +607,42 @@ vPortYieldFromInt: #if XCHAL_CP_NUM > 0 /* Save CPENABLE in task's co-processor save area, and clear CPENABLE. */ - movi a3, pxCurrentTCB /* cp_state = */ + pxctcb a3, a6 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a3, a3, 0 l32i a2, a3, TCB_END_OF_STACK_OFF - rsr a3, CPENABLE - s16i a3, a2, XT_CPENABLE /* cp_state->cpenable = CPENABLE; */ - movi a3, 0 - wsr a3, CPENABLE /* disable all co-processors */ -#endif + rsr a4, CPENABLE + + #if ( configNUMBER_OF_CORES > 1 ) + beqz a4, .Lnocp_save + + #if ( configUSE_CORE_AFFINITY == 1 ) + /* Task can run on other cores if affinity mask has bit(s) set for other cores */ + movi a5, port_affinity_masks + addx4 a5, a6, a5 + l32i a5, a5, 0 + l32i a3, a3, TCB_CORE_AFFINITY_MASK_OFF + bnone a3, a5, .Lnocp_save /* skip save if task only runnable on current core */ +.Lcp_save: + #endif /* ( configUSE_CORE_AFFINITY == 1 ) */ + + #ifdef __XTENSA_CALL0_ABI__ + /* No need to save a0 since we will call _frxt_dispatch later, which doesn't return */ + call0 _xt_coproc_saveall + #else + mov a6, a2 + call4 _xt_coproc_saveall + #endif /* __XTENSA_CALL0_ABI__ */ + j .Lcp_done +.Lnocp_save: + #endif /* ( configNUMBER_OF_CORES > 1 ) */ + + s16i a4, a2, XT_CPENABLE /* cp_state->cpenable = CPENABLE; */ + + movi a4, 0 + wsr a4, CPENABLE /* disable all co-processors */ +.Lcp_done: +#endif /* XCHAL_CP_NUM > 0 */ #ifdef __XTENSA_CALL0_ABI__ /* Tail-call dispatcher. */ @@ -607,10 +667,16 @@ _xt_task_start: movi a0, 0 wsr a0, CPENABLE /* disable all coprocessors */ #endif +#if XCHAL_HAVE_LOOPS + #if XCHAL_CP_NUM == 0 + movi a0, 0 /* skip if (a0 <- 0) by above loop logic */ + #endif + wsr a0, LCOUNT /* ensure loops are disabled */ +#endif #if XCHAL_HAVE_XEA3 /* On entry, a8 = task entry point, a9 = top of stack / coproc save area */ - movi a0, pxCurrentTCB + pxctcb a0, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a0, a0, 0 s32i a9, a0, TCB_END_OF_STACK_OFF /* TCB->pxEndOfStack = CP save area */ movi a0, 0 /* terminate call stack for gdb */ @@ -622,7 +688,7 @@ _xt_task_start: wsr a0, PS l32i a0, sp, XT_STK_PC /* retrieve interruptee's PC */ wsr a0, EPC_1 - movi a3, pxCurrentTCB + pxctcb a3, a0 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a3, a3, 0 l32i a0, sp, XT_STK_A0 /* retrieve interruptee's A0 */ l32i sp, sp, XT_STK_A1 /* remove interrupt stack frame */ @@ -650,7 +716,7 @@ _xt_task_start_user: #endif /* On entry, a8 = task entry point, a9 = top of stack / coproc save area */ - movi a0, pxCurrentTCB + pxctcb a0, a3 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a0, a0, 0 s32i a9, a0, TCB_END_OF_STACK_OFF /* TCB->pxEndOfStack = CP save area */ rsr a0, PS @@ -671,6 +737,7 @@ _xt_task_start_user: * May only be called when a task is running, not within an interrupt handler (returns 0 in that case). * May only be called from assembly code by the 'call0' instruction. Does NOT obey ABI conventions. * Returns in a15 a pointer to the base of the co-processor state save area for the current task. +* May trash a8, which is already trashed by all callers in xtensa_coproc_handler.S. * See the detailed description of the XT_RTOS_ENTER macro in xtensa_rtos.h. * ********************************************************************************************************** @@ -686,10 +753,10 @@ _xt_task_coproc_state: movi a15, port_xSchedulerRunning /* if (port_xSchedulerRunning */ l32i a15, a15, 0 beqz a15, 2f - movi a15, port_interruptNesting /* && port_interruptNesting == 0 */ - l32i a15, a15, 0 + pintdata a15, a8 /* a15 <- &port_interruptNesting */ + l32i a15, a15, PORTINT_NEST_OFF bnez a15, 1f - movi a15, pxCurrentTCB + pxctcb a15, a8 /* pxCurrentTCB or pxCurrentTCBs[] */ l32i a15, a15, 0 /* && pxCurrentTCB != 0) { */ beqz a15, 2f l32i a15, a15, TCB_END_OF_STACK_OFF /* a15 = pxCurrentTCB->pxEndOfStack */ @@ -700,3 +767,130 @@ _xt_task_coproc_state: ret #endif /* XCHAL_CP_NUM > 0 */ + + +#if (defined configNUMBER_OF_CORES) && ( configNUMBER_OF_CORES > 1 ) + +/* +------------------------------------------------------------------------------- + Initial (startup) stack for SMP systems + + For SMP configurations, (configNUMBER_OF_CORES * XT_SYSTEM_STACK_SIZE) bytes + are allocated. Avoid placing this in per-core memory (usually dataram) as + that is a valuable resource. +------------------------------------------------------------------------------- +*/ + + .data + + .global xt_startup_stacks + .type xt_startup_stacks,@object + .align 16 + +xt_startup_stacks: + .space XT_SYSTEM_STACK_SIZE // Working space, argv[0], overflow + .space XT_USER_SIZE // Extra for core 0: main()/printf()/... + + .global xt_startup_stack_top + .type xt_startup_stack_top,@object +xt_startup_stack_top: + .rept configNUMBER_OF_CORES - 1 + .space XT_SYSTEM_STACK_SIZE + .endr + + + .text + .global __stack_init + .type __stack_init,@function + .align 4 + +__stack_init: + + // NOTE: called via call0/callx0, return address is in a0. + // NOTE: must not rely on valid stack pointer / BSS values. + + movi a2, XT_SYSTEM_STACK_SIZE // must be multiple of 16 bytes + coreid a3 + + movi a4, xt_startup_stack_top // default SP (top of core 0 stack) +2: + beqz a3, 3f + add a4, a4, a2 + addi a3, a3, -1 + j 2b +3: + mov a1, a4 // set stack pointer + ret + + .size __stack_init, . - __stack_init + + + .text + .global __bss_init + .type __bss_init,@function + .align 4 + +__bss_init: + // void __bss_init(uint32_t * table_start, uint32_t * table_end) + // + // table_start -- points to first entry in BSS table (2 words) + // table_end -- points to end of BSS table + // + // This function clears BSS sections in dataram for all cores. + // BSS sections not in dataram are cleared only by core 0. + + abi_entry 16, 8 + coreid a4 + +.L1: + bgeu a2, a3, .Lret // at table end, finish + l32i a5, a2, 0 // entry start address + l32i a6, a2, 4 // entry end address + addi a2, a2, 8 // advance to next entry + beq a5, a6, .L1 // empty section, skip + +#if defined (XCHAL_DATARAM0_VADDR) + movi a7, XCHAL_DATARAM0_VADDR + movi a8, XCHAL_DATARAM0_SIZE + sub a7, a5, a7 // a7 = start_addr - dataram0_start_addr + bgeu a7, a8, .L2 // not within dataram0 range + j .Lclear +#endif +.L2: +#if defined (XCHAL_DATARAM1_VADDR) + movi a7, XCHAL_DATARAM1_VADDR + movi a8, XCHAL_DATARAM1_SIZE + sub a7, a5, a7 // a7 = start_addr - dataram1_start_addr + bgeu a7, a8, .L3 // not within dataram1 range + j .Lclear +#endif +.L3: + bnez a4, .L1 // not core 0, skip +.Lclear: + movi a7, 0 // value to store + sub a10, a6, a5 // a10 = length, assumed a multiple of 4 + bbci.l a10, 2, .L1z + s32i a7, a5, 0 // clear 4 bytes to make length multiple of 8 + addi a5, a5, 4 +.L1z: + bbci.l a10, 3, .L2z + s32i a7, a5, 0 // clear 8 bytes to make length multiple of 16 + s32i a7, a5, 4 + addi a5, a5, 8 +.L2z: + srli a10, a10, 4 // length is now multiple of 16, divide by 16 + floopnez a10, clearzte + s32i a7, a5, 0 // clear 16 bytes at a time... + s32i a7, a5, 4 + s32i a7, a5, 8 + s32i a7, a5, 12 + addi a5, a5, 16 + floopend a10, clearzte + bltu a2, a3, .L1 + +.Lret: + abi_return + + .size __bss_init, . - __bss_init + +#endif /* configNUMBER_OF_CORES > 1 */ diff --git a/Cadence/Xtensa/portclib.c b/Cadence/Xtensa/portclib.c index 04b6ecc5b..6db8be594 100644 --- a/Cadence/Xtensa/portclib.c +++ b/Cadence/Xtensa/portclib.c @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -31,6 +31,8 @@ #if XT_USE_THREAD_SAFE_CLIB +#define MTX_LOCK_ATTEMPTS_BEFORE_YIELD 5 + #if XSHAL_CLIB == XTHAL_CLIB_XCLIB #include @@ -87,8 +89,22 @@ _Mtxdst(_Rmtx * mtx) void _Mtxlock(_Rmtx * mtx) { + int retries = 0; + TickType_t ticks_to_wait = portMAX_DELAY; if ((mtx != NULL) && (*mtx != NULL)) { - xSemaphoreTakeRecursive(*mtx, portMAX_DELAY); +#if ( ( INCLUDE_xTaskGetSchedulerState == 1 ) || ( configUSE_TIMERS == 1 ) ) + if (xTaskGetSchedulerState() == taskSCHEDULER_SUSPENDED) { + // Making this a non-blocking call enables the heap_3 memory manager, + // which calls malloc() with the scheduler suspended (and would trigger + // an assertion at queue.c:1675). + ticks_to_wait = 0; + } +#endif + while (xSemaphoreTakeRecursive(*mtx, ticks_to_wait) != pdPASS) { + if (++retries >= MTX_LOCK_ATTEMPTS_BEFORE_YIELD) { + taskYIELD(); + } + } } } @@ -103,16 +119,17 @@ _Mtxunlock(_Rmtx * mtx) } } +extern char _end[]; +extern char _heap_sentry; +char * _heap_sentry_ptr = &_heap_sentry; +char * heap_ptr; + //----------------------------------------------------------------------------- // Called by malloc() to allocate blocks of memory from the heap. //----------------------------------------------------------------------------- void * _sbrk_r (struct _reent * reent, int32_t incr) { - extern char _end[]; - extern char _heap_sentry; - static char * _heap_sentry_ptr = &_heap_sentry; - static char * heap_ptr; char * base; if (!heap_ptr) @@ -166,13 +183,28 @@ static uint32_t ulClibInitDone = 0; void __malloc_lock(struct _reent * ptr) { + int retries = 0; + TickType_t ticks_to_wait = portMAX_DELAY; + // Suppress compiler warning. (void) ptr; if (!ulClibInitDone) return; - xSemaphoreTakeRecursive(xClibMutex, portMAX_DELAY); +#if ( ( INCLUDE_xTaskGetSchedulerState == 1 ) || ( configUSE_TIMERS == 1 ) ) + if (xTaskGetSchedulerState() == taskSCHEDULER_SUSPENDED) { + // Making this a non-blocking call enables the heap_3 memory manager, + // which calls malloc() with the scheduler suspended (and would trigger + // an assertion at queue.c:1675). + ticks_to_wait = 0; + } +#endif + while (xSemaphoreTakeRecursive(xClibMutex, ticks_to_wait) != pdPASS) { + if (++retries >= MTX_LOCK_ATTEMPTS_BEFORE_YIELD) { + taskYIELD(); + } + } } //----------------------------------------------------------------------------- @@ -210,16 +242,17 @@ __env_unlock(struct _reent * ptr) __malloc_unlock(ptr); } +extern char _end[]; +extern char _heap_sentry; +char * _heap_sentry_ptr = &_heap_sentry; +char * heap_ptr; + //----------------------------------------------------------------------------- // Called by malloc() to allocate blocks of memory from the heap. //----------------------------------------------------------------------------- void * _sbrk_r (struct _reent * reent, int32_t incr) { - extern char _end[]; - extern char _heap_sentry; - static char * _heap_sentry_ptr = &_heap_sentry; - static char * heap_ptr; char * base; if (!heap_ptr) @@ -258,4 +291,56 @@ _reclaim_reent(struct _reent * ptr) #endif /* XSHAL_CLIB == XTHAL_CLIB_NEWLIB */ +//----------------------------------------------------------------------------- +// If xclib/newlib support overriding reent_ptr_ as a function, use it instead +// of FreeRTOS' configSET_TLS_BLOCK() hook. Required for coherent libc support +// on SMP; single-core is overridden too since _reent_ptr is not an lvalue. +//----------------------------------------------------------------------------- +#if (defined __DYNAMIC_REENT__) + + #if (configNUMBER_OF_CORES > 1) + +struct _reent * +__getreent(void) +{ + xt_internal_data_t *xt_intdata_p = &(_XT_INTDATA(portGET_CORE_ID())); + #if ( configUSE_C_RUNTIME_TLS_SUPPORT == 1 ) + if (xt_intdata_p->xt_reent_p) { + return xt_intdata_p->xt_reent_p; + } + #endif /* configUSE_C_RUNTIME_TLS_SUPPORT */ + + // If TLS not configured or libc is used prior to starting the scheduler + return &(xt_intdata_p->xt_reent); +} + + #else /* (configNUMBER_OF_CORES == 1) */ + +#if XSHAL_CLIB == XTHAL_CLIB_XCLIB +static struct _reent xt_reent __attribute__ ((section (".clib.percpu.bss"))); +#endif + +struct _reent * +__getreent(void) +{ + #if ( configUSE_C_RUNTIME_TLS_SUPPORT == 1 ) + if (_xt_intdata.xt_reent_p) { + return _xt_intdata.xt_reent_p; + } + #endif /* configUSE_C_RUNTIME_TLS_SUPPORT */ + + // If TLS not configured or libc is used prior to starting the scheduler + #if XSHAL_CLIB == XTHAL_CLIB_XCLIB + return &xt_reent; + #elif XSHAL_CLIB == XTHAL_CLIB_NEWLIB + return _impure_ptr; + #else + #error The selected C runtime library is not thread safe. + #endif +} + + #endif /* configNUMBER_OF_CORES */ + +#endif /* __DYNAMIC_REENT__ */ + #endif /* XT_USE_THREAD_SAFE_CLIB */ diff --git a/Cadence/Xtensa/portmacro.h b/Cadence/Xtensa/portmacro.h index 40df6a265..ff27a9e7e 100644 --- a/Cadence/Xtensa/portmacro.h +++ b/Cadence/Xtensa/portmacro.h @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -41,10 +41,19 @@ #ifndef __ASSEMBLER__ #include +#include #include #include -#include /* required for XSHAL_CLIB */ +#include /* required for XSHAL_CLIB */ + +/* required for SMP support */ +#if (XCHAL_SW_VERSION >= 1506000) +#include +#elif (XCHAL_SW_VERSION >= 1505000) +#include "xtsubsystem_patch.h" +#endif +#include /*----------------------------------------------------------- * Port specific definitions. @@ -68,7 +77,7 @@ typedef portSTACK_TYPE StackType_t; typedef portBASE_TYPE BaseType_t; -typedef unsigned portBASE_TYPE UBaseType_t; +typedef unsigned portBASE_TYPE UBaseType_t; #if( configTICK_TYPE_WIDTH_IN_BITS == TICK_TYPE_WIDTH_16_BITS ) typedef uint16_t TickType_t; @@ -112,12 +121,6 @@ portENABLE_INTERRUPTS(void) #endif } -// Nested critical sections. Nesting managed by FreeRTOS. -// This is fine for a single core. TODO: revisit for SMP support. -// NOTE: consider Espressif solution: GCC/Xtensa_ESP32/include/portmacro.h - -#define portCRITICAL_NESTING_IN_TCB 1 - extern void vTaskEnterCritical(void); extern void vTaskExitCritical(void); @@ -158,7 +161,7 @@ extern void vPortExitCritical(void); #endif // These allow nested interrupt disabling and restoring via local registers or stack. -// They can be called from interrupts context. +// They can be called from interrupt context. static inline uint32_t portENTER_CRITICAL_NESTED(void) { @@ -191,24 +194,225 @@ portEXIT_CRITICAL_NESTED(uint32_t state) } // These FreeRTOS versions are similar to the nested versions above +#define portSET_INTERRUPT_MASK() portENTER_CRITICAL_NESTED() +#define portCLEAR_INTERRUPT_MASK(state) portEXIT_CRITICAL_NESTED(state) #define portSET_INTERRUPT_MASK_FROM_ISR() portENTER_CRITICAL_NESTED() #define portCLEAR_INTERRUPT_MASK_FROM_ISR(state) portEXIT_CRITICAL_NESTED(state) + BaseType_t xPortRaisePrivilege( void ); /*-----------------------------------------------------------*/ /* Architecture specifics. */ -#define portSTACK_GROWTH ( -1 ) -#define portTICK_PERIOD_MS ( ( TickType_t ) 1000 / configTICK_RATE_HZ ) +#define portSTACK_GROWTH ( -1 ) +#define portTICK_PERIOD_MS ( ( TickType_t ) 1000 / configTICK_RATE_HZ ) #ifdef configBYTE_ALIGNMENT -#define portBYTE_ALIGNMENT configBYTE_ALIGNMENT +#define portBYTE_ALIGNMENT configBYTE_ALIGNMENT #elif XCHAL_DATA_WIDTH < 16 -#define portBYTE_ALIGNMENT XCHAL_DATA_WIDTH +#define portBYTE_ALIGNMENT XCHAL_DATA_WIDTH +#else +#define portBYTE_ALIGNMENT 16 +#endif +#define portNOP() XT_NOP() +/* XT_MEMW() not required for generic C code memory barriers */ +#define portMEMORY_BARRIER() __asm__ volatile ("" ::: "memory") +/*-----------------------------------------------------------*/ + +/* Multicore specifics. */ +#define portMAX_CORE_COUNT 8 + +#ifndef configNUMBER_OF_CORES + #define configNUMBER_OF_CORES 1 +#elif ( configNUMBER_OF_CORES < 1 || configNUMBER_OF_CORES > portMAX_CORE_COUNT ) + #error "Invalid number of cores specified in config!" +#endif + + +/* FreeRTOS core id is always zero based; set to 0 for single-core case */ +#if ( configNUMBER_OF_CORES > 1 ) + +/* Various checks to confirm config is compatible with FreeRTOS SMP */ +#if ( !XCHAL_DCACHE_IS_COHERENT || ( XCHAL_SUBSYS_NUM_CORES == 1 )) + #error "SMP support requires Coherent Multicore Subsystem" +#endif + +#if !XCHAL_HAVE_PRID + #error "SMP support requires PRID" +#endif + +#if !XCHAL_HAVE_EXCLUSIVE + #error "SMP support requires Exclusive Store to implement locks" +#endif + +#if portUSING_MPU_WRAPPERS + #error "SMP support requires FreeRTOS MPU wrappers to be off" +#endif + +#if (XCHAL_SW_VERSION < 1505000) + #error "SMP requires Xtensa toolchain RJ.5 or later" +#endif + +#if !XCHAL_SUBSYS_IPI_NUM_SETS + #error "SMP support requires at least one set of inter-processor interrupts (IPIs)" +#endif + +#if (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C0_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C0 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C1_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C1 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 2) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C2_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C2 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 3) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C3_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C3 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 4) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C4_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C4 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 5) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C5_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C5 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 6) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C6_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C6 core interrupt is > XCHAL_EXCM_LEVEL +#endif +#if (configNUMBER_OF_CORES >= 7) && (XCHAL_INT_LEVEL(XCHAL_SUBSYS_IPI_S0C7_INTNUM) > XCHAL_EXCM_LEVEL) +#error IPI S0C7 core interrupt is > XCHAL_EXCM_LEVEL +#endif + +#ifndef configTICK_CORE + #define configTICK_CORE 0 +#elif ( configTICK_CORE < 0 || configTICK_CORE >= configNUMBER_OF_CORES ) + #error "Invalid tick core specified in config!" +#endif +#if ( configUSE_CORE_AFFINITY == 1 ) + #define configTIMER_SERVICE_TASK_CORE_AFFINITY ( 1 << configTICK_CORE ) +#endif + + /* The Xtensa SMP port maintains an array of xt_internal_data_t structures, + * which are padded to a cache line boundary. This prevents cache thrashing + * since various members are read or written by their own core. If new fields + * are added, XT_PERCORE_DATA_SIZE must be adjusted accordingly. + */ +#if (defined __DYNAMIC_REENT__) + #define XT_PERCORE_REENT_DATA_SIZE (8 + sizeof(struct _reent)) #else -#define portBYTE_ALIGNMENT 16 + #define XT_PERCORE_REENT_DATA_SIZE 0 +#endif + #define XT_PERCORE_DATA_SIZE (sizeof(UBaseType_t) + 20 + XT_PERCORE_REENT_DATA_SIZE) + + typedef struct xt_internal_data { + uint32_t port_interruptNesting; // First field for asm efficiency + uint32_t port_switch_flag; + UBaseType_t uxCriticalNestings; + uint32_t xt_intenable; + uint32_t xt_vpri_mask; + uint32_t xt_core_init_done; +#if (defined __DYNAMIC_REENT__) + struct _reent *xt_reent_p; // When xclib defines _reent_ptr() + void * xt_reent_pad_align; // Ensure xt_reent is 16-byte aligned + struct _reent xt_reent; +#endif +#if ( XT_USE_DATARAM == 0 ) + uint8_t pad[XCHAL_DCACHE_LINESIZE - + (XT_PERCORE_DATA_SIZE & (XCHAL_DCACHE_LINESIZE - 1))]; +#endif + } xt_internal_data_t; + + static_assert( offsetof(xt_internal_data_t, port_interruptNesting) == 0, + "Bad xt_internal_data field order" ); +#if ( XT_USE_DATARAM == 0 ) + static_assert( ((sizeof(xt_internal_data_t) & (XCHAL_DCACHE_LINESIZE - 1)) == 0), + "Incorrect xt_internal_data padding" ); +#endif + + #define portGET_CORE_ID() xthal_get_coreid() + #define portYIELD_CORE(xCoreID) xthal_ipi_trigger(xCoreID) + #define portCRITICAL_NESTING_IN_TCB 0 // Nesting managed by port for SMP + + /* Mutex APIs for SMP locks -- based on XTOS implementation. + * Requires Xtensa Exclusive Store and PRID options to be present. + * Must reside in shared memory and declared statically (not on the stack). + * Align and pad to cache line size for best performance. + */ + typedef struct xt_mutex { + uint32_t owner; + uint32_t count; + uint8_t pad[XCHAL_DCACHE_LINESIZE - 2 * sizeof(uint32_t)]; + } xt_mutex; + + typedef xt_mutex * xt_mutex_p; + + extern xt_mutex _xt_mutex_ISR; + extern xt_mutex _xt_mutex_task; + + extern void xt_mutex_init(xt_mutex_p pmtx); + extern int32_t xt_mutex_lock(xt_mutex_p pmtx); + extern int32_t xt_mutex_unlock(xt_mutex_p pmtx); + + #define portGET_ISR_LOCK( xCoreID ) xt_mutex_lock(&_xt_mutex_ISR) + #define portRELEASE_ISR_LOCK( xCoreID ) xt_mutex_unlock(&_xt_mutex_ISR) + #define portGET_TASK_LOCK( xCoreID ) xt_mutex_lock(&_xt_mutex_task) + #define portRELEASE_TASK_LOCK( xCoreID ) xt_mutex_unlock(&_xt_mutex_task) + + // Per-core data struct can be kept in dataram or indexed in shared sysram + #if ( XT_USE_DATARAM ) + extern xt_internal_data_t _xt_intdata; + #define _XT_INTDATA(...) (_xt_intdata) + #else + extern xt_internal_data_t _xt_intdata[ configNUMBER_OF_CORES ]; + #define _XT_INTDATA(c) (_xt_intdata[(c)]) + #endif + + // uxCriticalNestings maintained within per-core data + #define portGET_CRITICAL_NESTING_COUNT( xCoreID ) ( _XT_INTDATA( xCoreID ).uxCriticalNestings ) + #define portSET_CRITICAL_NESTING_COUNT( xCoreID, x ) ( (_XT_INTDATA( xCoreID ).uxCriticalNestings) = ( x ) ) + #define portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ) ( (_XT_INTDATA( xCoreID ).uxCriticalNestings) ++ ) + #define portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ) ( (_XT_INTDATA( xCoreID ).uxCriticalNestings) -- ) + + // port_interruptNesting maintained within per-core data + #define portINCREMENT_INTERRUPT_NESTING_COUNT() ( (_XT_INTDATA( portGET_CORE_ID() ).port_interruptNesting) ++ ) + #define portDECREMENT_INTERRUPT_NESTING_COUNT() ( (_XT_INTDATA( portGET_CORE_ID() ).port_interruptNesting) -- ) + + extern UBaseType_t vTaskEnterCriticalFromISR(void); + extern void vTaskExitCriticalFromISR(UBaseType_t uxSavedInterruptStatus); + #define portENTER_CRITICAL_FROM_ISR() vTaskEnterCriticalFromISR() + #define portEXIT_CRITICAL_FROM_ISR(x) vTaskExitCriticalFromISR(x) + +#else // configNUMBER_OF_CORES + + /* The single-core Xtensa port maintains a single structure with interrupt-related + * data structures, which for efficiency are accessed as offsets from a base + * structure pointer. + */ + typedef struct xt_internal_data { + uint32_t port_interruptNesting; // First field for asm efficiency + uint32_t port_switch_flag; + uint32_t xt_intenable; + uint32_t xt_vpri_mask; +#if (defined __DYNAMIC_REENT__) + struct _reent *xt_reent_p; // When xclib defines _reent_ptr() #endif -#define portNOP() XT_NOP() -#define portMEMORY_BARRIER() XT_MEMW() + } xt_internal_data_t; + + static_assert( offsetof(xt_internal_data_t, port_interruptNesting) == 0, "Bad xt_internal_data field order" ); + + #define portGET_CORE_ID() 0 + #define portYIELD_CORE( xCoreID ) UNUSED( xCoreID ) + #define portCRITICAL_NESTING_IN_TCB 1 // Nesting managed by FreeRTOS fine for 1 core + + #define portGET_ISR_LOCK( xCoreID ) + #define portRELEASE_ISR_LOCK( xCoreID ) + #define portGET_TASK_LOCK( xCoreID ) + #define portRELEASE_TASK_LOCK( xCoreID ) + + extern xt_internal_data_t _xt_intdata; + #define _XT_INTDATA(...) ( _xt_intdata ) + #define portINCREMENT_INTERRUPT_NESTING_COUNT() ( _xt_intdata.port_interruptNesting++ ) + #define portDECREMENT_INTERRUPT_NESTING_COUNT() ( _xt_intdata.port_interruptNesting-- ) + +#endif // configNUMBER_OF_CORES /*-----------------------------------------------------------*/ /* Fine resolution time */ @@ -220,6 +424,7 @@ BaseType_t xPortRaisePrivilege( void ); /* Kernel utilities. */ BaseType_t xPortIsInsideInterrupt( void ); void vPortYield( void ); +void vPortYieldFromInt( void ); void _frxt_setup_switch( void ); #define portYIELD() vPortYield() #define portYIELD_FROM_ISR( xHigherPriorityTaskWoken ) \ diff --git a/Cadence/Xtensa/portmpu.c b/Cadence/Xtensa/portmpu.c index ea7ba59be..514e53e9e 100644 --- a/Cadence/Xtensa/portmpu.c +++ b/Cadence/Xtensa/portmpu.c @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -408,7 +408,6 @@ prvSetupMPU(void) } typedef void TCB_t; -extern volatile TCB_t * volatile pxCurrentTCB; static void PRIVILEGED_FUNCTION init_private_mpu_regions(const struct xMEMORY_REGION * const xRegions, diff --git a/Cadence/Xtensa/readme_xtensa.txt b/Cadence/Xtensa/readme_xtensa.txt index 2239f31b5..b55443ff4 100644 --- a/Cadence/Xtensa/readme_xtensa.txt +++ b/Cadence/Xtensa/readme_xtensa.txt @@ -1,7 +1,7 @@ FreeRTOS Port for Xtensa Configurable Processors ================================================ - FreeRTOS Kernel Version 11.0.0 + FreeRTOS Kernel Version 11.3.0 Introduction @@ -11,7 +11,8 @@ This document describes the Xtensa port for FreeRTOS multitasking RTOS. For an introduction to FreeRTOS itself, please refer to FreeRTOS documentation. -This port currently works with FreeRTOS kernel version 11.0.0. +This port was developed and tested with FreeRTOS kernel version 11.2.0, +and also currently works with FreeRTOS kernel version 11.3.0. Xtensa Configuration Requirements and Restrictions @@ -76,8 +77,8 @@ kernel from: https://github.com/FreeRTOS/FreeRTOS-Kernel https://www.freertos.org/ -The Xtensa port files are included in the official package under the -"Partner-Supported-Ports" and "Partner-Supported-Demos" submodules, but +The Xtensa port files are included in the official package under the +"Partner-Supported-Ports" and "Partner-Supported-Demos" submodules, but may not be the latest versions available. All source is provided along with a Makefile that works for any host @@ -110,9 +111,9 @@ Building FreeRTOS for Xtensa ---------------------------- To build the FreeRTOS library and the example programs, go into the -directory 'Demo/.../Cadence_Xtensa_ISS_xt-clang' and use the makefile -in that directory. "make all" will build all the examples. There is -another makefile in the 'portable/.../Cadence/Xtensa' directory that +directory 'Demo/.../Cadence_Xtensa_ISS_xt-clang' and use the makefile +in that directory. "make all" will build all the examples. There is +another makefile in the 'portable/.../Cadence/Xtensa' directory that builds just the FreeRTOS library. By default, you will build for the Xtensa instruction set simulator. If @@ -234,7 +235,7 @@ define this to 1 if either newlib or xclib is detected. The space for the per-thread C library context data is allocated within the FreeRTOS TCB structure. -The MPU example must be built separately since it requires the FreeRTOS +The MPU example must be built separately since it requires the FreeRTOS library to be rebuilt with -DportUSING_MPU_WRAPPERS=1 -DportALIGN_SECTIONS which is handled by the makefile if you do the following: @@ -766,4 +767,83 @@ Overlay Support for more details. +SMP Support For Xtensa LX +------------------------- + +FreeRTOS v11 SMP configuration is supported beginning with Xtensa port +version 3.10. General details regarding SMP on FreeRTOS can be found here: + +https://www.freertos.org/Documentation/02-Kernel/02-Kernel-features/13-Symmetric-multiprocessing-introduction + +Important information regarding Xtensa SMP support: + +- Current core configuration requirements for FreeRTOS SMP: + + 1. MPU hardware (for coherence) + 2. 1 set of inter-processor interrupts (IPIs) <= EXCM_LEVEL + 3. Only coherent LX8 multicore configurations are supported at this time; + as such, the exclusive access option is subsequently required + 4. Unrelated to SMP, 1 timer per core <= EXCM_LEVEL and + 1 software interrupt per core <= EXCM_LEVEL are required + +- SMP support requires Xtensa toolchain version RJ-2025.5-p1 or newer. + +- Starting with Xtensa port v3.13, SMP mode is enabled by default for LX/SMP + Xtensa configurations (where XCHAL_SUBSYS_NUM_CORES > 1), and is disabled + for all other configs. SMP mode is enabled within the kernel by defining + configNUMBER_OF_CORES > 1. For the Xtensa Demo suite, this setting is + found in common/config_files/FreeRTOSConfig.h, and can be enabled by + running "make SMP=1" in Cadence_Xtensa_ISS_xt-clang/. + +- SMP support relies on coherent shared memory being enabled prior to FreeRTOS + initialization. It is enabled by default in the LX8 multicore boot code. + +- SMP core power-shutoff (PSO) support is not standardized in FreeRTOS-Kernel. + Xtensa PSO support is described in Cadence_Xtensa_ISS_xt-clang/README.md. + +- MPU hardware support is required for the Xtensa coherence protocol to + function, as all coherent memory regions must be configured as inner- + shareable or outer-shareable. This is usually specified in the MPU table + that is linked into the executable and used to program the MPU at boot-up. + +- MPU software support in FreeRTOS is currently not compatible with SMP and + must be disabled. This allows FreeRTOS to maintain a fully-coherent memory + map such that system state is always available and shared across cores. + +- Overlay software support in FreeRTOS is currently not compatible with SMP + and must be disabled. + +- FreeRTOS manages coprocessor state through "lazy"/on-demand context switches + for efficiency, as described in xtensa_context.h. On SMP systems, any tasks + that reference coprocessor state should be pinned to a specific core to + minimize unsolicited context switch overhead; otherwise, full coprocessor + state will be saved and restored to ensure CP state is coherent across cores. + +- SMP examples are provided in common/application_code/cadence_code/xt_smp.c + and common/application_code/cadence_code/xt_mc_demo.c and can be built + by running "make SMP=1" in Cadence_Xtensa_ISS_xt-clang/. + +- The Xtensa system interrupt stack (mentioned above) is replicated per-core + in order to properly handle interrupts on a shared-memory system. These + stacks are typically too large to be an efficient use of dataram, so they + are replicated and placed alongside other default .data objects. + +- A single interrupt dispatch table is shared for all cores in the SMP system. + Therefore, registering an ISR on one core will result in the same handler + being registered for that interrupt on all cores. However, interrupts are + still enabled and disabled on a per-core basis. + +- Xtensa-specific config option "XT_USE_L2RAM" moves data structures for timer, + scheduler, etc. to L2RAM instead of the default L2-cached system memory. + This can improve context switching performance. L1 data cache ensures + coherence in both cases. See xtensa_config.h for more details. This option + is disabled by default. + +- Xtensa-specific config option "XT_USE_DATARAM" moves data structures for + interrupt dispatch and reentrancy to dataram instead of the default L2-cached + system memory. This removes the need to index these structures per-core and + can improve performance. See xtensa_config.h for more details. This option + is disabled by default. + + -End- diff --git a/Cadence/Xtensa/relnotes.txt b/Cadence/Xtensa/relnotes.txt index 78334fbcb..a01c91dc1 100644 --- a/Cadence/Xtensa/relnotes.txt +++ b/Cadence/Xtensa/relnotes.txt @@ -1,3 +1,43 @@ +Notes for Version 3.13 +---------------------- +- FreeRTOS SMP support for newlib (experimental). +- FreeRTOS SMP context switching performance improvements. +- FreeRTOS SMP config option "XT_USE_L2RAM" moves data into L2RAM + for context switching performance improvements. Disabled by default. +- FreeRTOS SMP config option "XT_USE_DATARAM" moves per-core data into + dataram for performance improvements. Disabled by default. +- Based on FreeRTOS-Kernel V11.2.0 release in 3/2025. +- Also tested on FreeRTOS-Kernel V11.3.0 release in 3/2026. + + +Notes for Version 3.12 +---------------------- + +- FreeRTOS SMP fully implements thread-safe and multi-core reentrancy + using xclib built with __DYNAMIC_REENT__ enabled. +- FreeRTOS SMP fixes for some coprocessor save/restore problems. + + +Notes for Version 3.11 +---------------------- + +- FreeRTOS SMP only calls main() on core 0; non-zero cores bypass main() + and wait for core 0 to start the scheduler before running tasks. +- FreeRTOS SMP no longer requires dataram. +- FreeRTOS SMP no longer requires non-zero cores to be held in RunStall + during initialization. +- See readme_xtensa.txt for updated details and requirements. + + +Notes for Version 3.10 +---------------------- + +- This version supports FreeRTOS SMP. + See readme_xtensa.txt for details and requirements. + +- Bugfix for imprecise exception handling (NX). + + Notes for Version 3.00 ---------------------- diff --git a/Cadence/Xtensa/xtensa_api.h b/Cadence/Xtensa/xtensa_api.h index 4fd8ade11..02996e9ec 100644 --- a/Cadence/Xtensa/xtensa_api.h +++ b/Cadence/Xtensa/xtensa_api.h @@ -79,6 +79,16 @@ extern xt_exc_handler xt_set_exception_handler( uint32_t n, xt_exc_handler f ) P extern xt_handler xt_set_interrupt_handler( uint32_t n, xt_handler f, void * arg ) PRIVILEGED_FUNCTION; +/* +------------------------------------------------------------------------------- + Call this function to get a handler for the specified interrupt. + + n - Interrupt number. +------------------------------------------------------------------------------- +*/ +extern xt_handler xt_get_interrupt_handler( uint32_t n ) PRIVILEGED_FUNCTION; + + /* ------------------------------------------------------------------------------- Call this function to enable the specified interrupt. diff --git a/Cadence/Xtensa/xtensa_asm.h b/Cadence/Xtensa/xtensa_asm.h new file mode 100644 index 000000000..638b4f719 --- /dev/null +++ b/Cadence/Xtensa/xtensa_asm.h @@ -0,0 +1,111 @@ +/* + * FreeRTOS Kernel + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. + * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * https://www.FreeRTOS.org + * https://github.com/FreeRTOS + * + */ + +#ifndef XTENSA_ASM_H +#define XTENSA_ASM_H + +#include + +#if (defined configNUMBER_OF_CORES) && ( configNUMBER_OF_CORES > 1 ) +#define XT_SMP_MACROS 1 +#else +#define XT_SMP_MACROS 0 +#endif + +#if XT_SMP_MACROS + +/* +******************************************************************************* +* Macro to return core ID into regieter r. +* Useful for assembly implementations of xthal_get_coreid(). +******************************************************************************* +*/ + .macro coreid r + rsr.prid \r +#if XCHAL_SUBSYS_CORE_ID_BITS + extui \r, \r, 0, XCHAL_SUBSYS_CORE_ID_BITS +#endif + .endm + +#endif /* XT_SMP_MACROS */ + +/* +******************************************************************************* +* Macro to load a pointer to the current core's current TCB into register r. +* NOTE: Trashes register t and places coreid in it on SMP configurations. +******************************************************************************* +*/ +#if XT_SMP_MACROS + .extern pxCurrentTCBs +#else + .extern pxCurrentTCB +#endif + + .macro pxctcb r, t +#if XT_SMP_MACROS + coreid \t + movi \r, pxCurrentTCBs + addx4 \r, \t, \r +#else + movi \r, pxCurrentTCB +#endif + .endm + +/* +******************************************************************************* +* Macro to load a pointer to the current core's xt_internal_data_t block, +* which contains the port_interruptNesting (first field) and port_switch_flag +* variables, along with other data on SMP configurations. +* NOTE: Trashes register t on SMP configurations. +******************************************************************************* +*/ +#if XT_SMP_MACROS + // ( XT_USE_DATARAM == 0 ) : + // Defined in C as: xt_internal_data_t _xt_intdata[ configNUMBER_OF_CORES ]; + // where sizeof(xt_internal_data_t ) == XCHAL_DCACHE_LINESIZE + // ( XT_USE_DATARAM == 1 ) : + // Defined in C as: xt_internal_data_t _xt_intdata __attribute__ ((section(".dram0.data"))); + .extern _xt_intdata +#else + // Defined in C as: xt_internal_data_t _xt_intdata; + .extern _xt_intdata +#endif + + .macro pintdata r, t +#if XT_SMP_MACROS && ( XT_USE_DATARAM == 0 ) + coreid \t + movi \r, _xt_intdata + slli \t, \t, XCHAL_DCACHE_LINEWIDTH + add \r, \r, \t +#else + movi \r, _xt_intdata +#endif + .endm + +#endif /* XTENSA_ASM_H */ diff --git a/Cadence/Xtensa/xtensa_config.h b/Cadence/Xtensa/xtensa_config.h index c5de48269..2fb42bb2c 100644 --- a/Cadence/Xtensa/xtensa_config.h +++ b/Cadence/Xtensa/xtensa_config.h @@ -130,7 +130,7 @@ #define _impure_ptr _reent_ptr void _reclaim_reent(struct _reent * ptr); - #endif + #endif // !__ASSEMBLER__ #elif XSHAL_CLIB == XTHAL_CLIB_NEWLIB #define XT_HAVE_THREAD_SAFE_CLIB 1 #if !defined __ASSEMBLER__ @@ -139,14 +139,24 @@ #define XT_CLIB_GLOBAL_PTR _impure_ptr void _reclaim_reent(struct _reent * ptr); - #endif - #else + #endif // !__ASSEMBLER__ + #else // XTHAL_CLIB_XCLIB || XTHAL_CLIB_NEWLIB #define XT_HAVE_THREAD_SAFE_CLIB 0 #error The selected C runtime library is not thread safe. - #endif + #endif // XTHAL_CLIB_XCLIB || XTHAL_CLIB_NEWLIB + #if (defined __DYNAMIC_REENT__) + // For xclib/newlib with support for custom reent_ptr_() we keep + // XT_CLIB_GLOBAL_PTR within interrupt data struct + #if (configNUMBER_OF_CORES > 1) + #define configSET_TLS_BLOCK(xTLSBlock) ( _XT_INTDATA(portGET_CORE_ID()).xt_reent_p = \ + &( xTLSBlock ) ) + #else + #define configSET_TLS_BLOCK(xTLSBlock) ( _xt_intdata.xt_reent_p = &( xTLSBlock ) ) + #endif + #endif // __DYNAMIC_REENT__ #else #define XT_CLIB_CONTEXT_AREA_SIZE 0 -#endif +#endif // XT_USE_THREAD_SAFE_CLIB /*------------------------------------------------------------------------------ Extra size -- interrupt frame plus coprocessor save area plus hook space. @@ -181,6 +191,64 @@ /* Default system (interrupt) stack size */ #define XT_SYSTEM_STACK_SIZE 0x400 +/** + * XT_USE_L2RAM is defined in xtensa_config.h and can be enabled to improve + * performance for SMP configurations. When set, all "PRIVILEGED_DATA" + * structures are moved to L2RAM instead of L2-cached sysram. Both locations + * are cached per coherence protocol in each core's L1 data cache. + * + * It's worth noting that the default FreeRTOS heap is privileged and is moved + * along with these structures, so sufficient L2RAM space must be provisioned. + */ +#if (configNUMBER_OF_CORES > 1) + /* Not usable if L2 is configured as all-cache */ + #if XCHAL_L2CACHE_ONLY + #undef XT_USE_L2RAM + #define XT_USE_L2RAM 0 + #endif + /* Default is to not use L2RAM for shared data structures; + * enabling this can improve context switching performance. + */ + #if !(defined XT_USE_L2RAM) + #define XT_USE_L2RAM 0 + #endif +#else + #undef XT_USE_L2RAM + #define XT_USE_L2RAM 0 +#endif + +#if XT_USE_L2RAM + /** + * The 50/50 L2CACHE/L2RAM split defined here may not work for all systems. + * These defines must be absolute numerical values, not expressions. + */ + #define XT_L2RAM_SIXTEENTHS 8 + #define XT_L2CACHE_SIXTEENTHS 8 + + #define portMOVE_PRIVILEGED_DATA __attribute__( ( section( ".l2ram.bss" ) ) ) +#endif + +/** + * XT_USE_DATARAM is defined in xtensa_config.h and can be enabled to improve + * performance for SMP configurations. When set, the _xt_intdata per-core + * structures are moved to dataram on each core. This reduces overhead of + * indexing the shared structure and provides fast access to RTOS data. + * + * TODO: may also improve performance for single-core configs. + */ +#if ((configNUMBER_OF_CORES > 1) && (XCHAL_NUM_DATARAM > 0)) + #if !(defined XT_USE_DATARAM) + #define XT_USE_DATARAM 0 + #endif +#else + #undef XT_USE_DATARAM + #define XT_USE_DATARAM 0 +#endif + +#if XT_USE_DATARAM + #define XT_DATARAM_ATTR __attribute__ ((section(".dram0.data"))) +#endif + /* *INDENT-OFF* */ #ifdef __cplusplus } diff --git a/Cadence/Xtensa/xtensa_context.S b/Cadence/Xtensa/xtensa_context.S index 69a18f9ae..8f6b436ee 100644 --- a/Cadence/Xtensa/xtensa_context.S +++ b/Cadence/Xtensa/xtensa_context.S @@ -1,6 +1,6 @@ - /* +/* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -143,9 +143,14 @@ _xt_context_save: s32i a3, sp, XT_STK_LBEG rsr a3, LEND s32i a3, sp, XT_STK_LEND - rsr a3, LCOUNT + movi a3, 0 + #if (defined XT_USE_OVLY) && (defined XT_OV_SCRATCH) && (XT_OV_SCRATCH == LCOUNT) + rsr a3, LCOUNT /* preserve LCOUNT for overlay store logic */ + #else + xsr a3, LCOUNT /* clear LCOUNT to prevent looping in ISR */ + #endif // XT_USE_OVLY s32i a3, sp, XT_STK_LCOUNT - #endif + #endif // XCHAL_HAVE_LOOPS #if XCHAL_HAVE_EXCLUSIVE /* Save and clear state of ATOMCTL */ @@ -186,10 +191,15 @@ _xt_context_save: _xt_overlay_get_state a9, a12, a13 s32i a9, sp, XT_STK_OVLY /* save overlay state */ #if (defined XT_OV_SCRATCH) /* In newer liboverlay versions */ + #if (XT_OV_SCRATCH == LCOUNT) + movi a9, 0 /* clear LCOUNT to prevent looping in ISR */ + xsr a9, XT_OV_SCRATCH /* save XT_OV_SCRATCH/LCOUNT register */ + #else rsr a9, XT_OV_SCRATCH /* save XT_OV_SCRATCH register */ + #endif // XT_OV_SCRATCH == LCOUNT s32i a9, sp, XT_STK_OVLY_SCRATCH - #endif - #endif + #endif // XT_OV_SCRATCH + #endif // XT_USE_OVLY l32i a12, sp, XT_STK_A12 /* recover original a9,12,13 */ l32i a13, sp, XT_STK_A13 @@ -314,13 +324,13 @@ _xt_context_restore: #ifdef XT_USE_SWPRI /* Restore virtual interrupt priority and interrupt enable */ - movi a3, xt_intdata - l32i a4, a3, 0 /* a4 = xt_intenable */ - l32i a5, sp, XT_STK_VPRI /* a5 = saved xt_vpri_mask */ + pintdata a3, a4 + l32i a4, a3, PORTINT_INTENABLE_OFF /* a4 = xt_intenable */ + l32i a5, sp, XT_STK_VPRI /* a5 = saved xt_vpri_mask */ and a4, a4, a5 - wsr a4, INTENABLE /* update INTENABLE */ - s32i a5, a3, 4 /* restore xt_vpri_mask */ - #endif + wsr a4, INTENABLE /* update INTENABLE */ + s32i a5, a3, PORTINT_VPRI_MASK_OFF /* restore xt_vpri_mask */ + #endif // XT_USE_SWPRI l32i a3, sp, XT_STK_SAR l32i a2, sp, XT_STK_A2 @@ -388,7 +398,11 @@ _xt_coproc_init: /* Initialize thread co-processor ownerships to 0 (unowned). */ movi a2, _xt_coproc_owner_sa /* a2 = base of owner array */ +#if (defined configNUMBER_OF_CORES) + addi a3, a2, (XCHAL_CP_MAX * configNUMBER_OF_CORES) << 2 /* a3 = top+1 of owner array */ +#else addi a3, a2, XCHAL_CP_MAX << 2 /* a3 = top+1 of owner array */ +#endif movi a4, 0 /* a4 = 0 (unowned) */ 1: s32i a4, a2, 0 addi a2, a2, 4 @@ -432,7 +446,11 @@ _xt_coproc_release: ENTRY0 /* a2 = base of save area */ movi a3, _xt_coproc_owner_sa /* a3 = base of owner array */ +#if (defined configNUMBER_OF_CORES) + addi a4, a3, (XCHAL_CP_MAX * configNUMBER_OF_CORES) << 2 /* a4 = top+1 of owner array */ +#else addi a4, a3, XCHAL_CP_MAX << 2 /* a4 = top+1 of owner array */ +#endif movi a5, 0 /* a5 = 0 (unowned) */ #if XCHAL_HAVE_XEA3 diff --git a/Cadence/Xtensa/xtensa_context.h b/Cadence/Xtensa/xtensa_context.h index 084f89b05..b261d482c 100644 --- a/Cadence/Xtensa/xtensa_context.h +++ b/Cadence/Xtensa/xtensa_context.h @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -49,6 +49,9 @@ #include #include +/* For configNUMBER_OF_CORES */ +#include + /* Align a value up to nearest n-byte boundary, where n is a power of 2. */ #ifdef __ASSEMBLER__ @@ -272,6 +275,19 @@ XSTRUCT_END(XtExcFrame) co-processor state (which can be quite large) and in particular remove that overhead from interrupt handlers. + An exception to this rule applies to SMP configurations. If a thread using + a coprocessor is pinned to a specific core (using the task "Affinity" APIs) + then the usual "lazy" co-processor context switching is used. However, if a + task can migrate to another core, its co-processor state needs to be saved + before it is swapped out since it may be required on a different core, + increasing context switching latency. For this reason, it is recommended + (but not required) that all tasks using a co-processor be pinned to a core. + + NOTE: SMP applications should only change affinity masks on the currently- + running task or on a task that has fully saved/released its coprocessors; + changing the affinity on an inactive preempted task could result in + coprocessor data loss and must be avoided. + The co-processor state save area may be in any convenient per-thread location such as in the thread control block or above the thread stack area. It need not be in the interrupt stack frame since interrupts don't use co-processors. @@ -287,7 +303,7 @@ XSTRUCT_END(XtExcFrame) When a thread solicits a context-swtich, its CPENABLE is cleared - the compiler has saved the (caller-saved) co-proc state if it needs to. When a non-running thread loses ownership of a CP, its bit is cleared. - When a thread runs, it's XT_CPENABLE is loaded into the CPENABLE reg. + When a thread runs, its XT_CPENABLE is loaded into the CPENABLE reg. Avoids co-processor exceptions when no change of ownership is needed. XT_CPSTORED @@ -413,6 +429,7 @@ XSTRUCT_END(XtExcFrame) #ifdef __ASSEMBLER__ #include "asm-offsets.h" +#include "xtensa_asm.h" // RTOS-specific entry macro. Use only a8, a12-a14. @@ -427,11 +444,11 @@ XSTRUCT_END(XtExcFrame) // Check scheduler state and interrupt nest state. + pintdata a10, a8 // a10 <- &port_interruptNesting movi a8, port_xSchedulerRunning - movi a9, port_interruptNesting l32i a8, a8, 0 // a8 <- port_xSchedulerRunning beqz a8, .Lnested // scheduler not running, no tasks - l32i a8, a9, 0 // a8 <- port_interruptNesting + l32i a8, a10, PORTINT_NEST_OFF // a8 <- port_interruptNesting bnez a8, .Lnested // != 0 means nested, skip ahead movi a8, port_yield_flag l32i a9, a8, 0 // a9 <- port_yield_flag @@ -440,14 +457,13 @@ XSTRUCT_END(XtExcFrame) s32i a9, a8, 0 // zero out for next time j .Lyield // no context save needed 2: - movi a8, pxCurrentTCB + pxctcb a8, a9 // pxCurrentTCB or pxCurrentTCBs[] l32i a9, a8, 0 // a9 <- pxCurrentTCB beqz a9, .Lsched // no current, go to scheduler - movi a10, port_switch_flag - l32i a11, a10, 0 // a11 <- port_switch_flag + l32i a11, a10, PORTINT_SWITCH_OFF // a11 <- port_switch_flag beqz a11, .Lnested // = 0 means no switch movi a11, 0 - s32i a11, a10, 0 // zero out for next time + s32i a11, a10, PORTINT_SWITCH_OFF // zero out for next time // Preemption, save remaining state of current (outgoing) thread @@ -496,18 +512,24 @@ XSTRUCT_END(XtExcFrame) addi a1, a1, -XT_STK_FRMSZ -32 #ifdef __XTENSA_WINDOWED_ABI__ - movi a10, vTaskSwitchContext - callx8 a10 +#if ( configNUMBER_OF_CORES > 1 ) + coreid a10, a9 +#endif + movi a9, vTaskSwitchContext + callx8 a9 #else - movi a10, vTaskSwitchContext - callx0 a10 +#if ( configNUMBER_OF_CORES > 1 ) + coreid a2, a9 #endif + movi a9, vTaskSwitchContext + callx0 a9 +#endif // __XTENSA_WINDOWED_ABI__ .Lyield: // Come here directly if the outgoing task yielded. pxCurrentTCB // has already been updated. - movi a9, pxCurrentTCB + pxctcb a9, a10 // pxCurrentTCB or pxCurrentTCBs[] l32i a9, a9, 0 // a9 <- pxCurrentTCB #if XCHAL_CP_NUM > 0 diff --git a/Cadence/Xtensa/xtensa_coproc_handler.S b/Cadence/Xtensa/xtensa_coproc_handler.S index e59a00223..ce868225f 100644 --- a/Cadence/Xtensa/xtensa_coproc_handler.S +++ b/Cadence/Xtensa/xtensa_coproc_handler.S @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -34,6 +34,7 @@ #include "xtensa_context.h" #include "xtensa_rtos.h" +#include "xtensa_asm.h" #if XCHAL_CP_NUM > 0 @@ -49,7 +50,11 @@ .global _xt_coproc_owner_sa .align 16,,XCHAL_CP_MAX << 2 // minimize crossing cache boundaries _xt_coproc_owner_sa: +#if (defined configNUMBER_OF_CORES) + .rept (XCHAL_CP_MAX * configNUMBER_OF_CORES) +#else .rept XCHAL_CP_MAX +#endif .word 0 .endr @@ -107,6 +112,22 @@ _xt_coproc_handler: addx4 a0, a5, a0 // a0 = &_xt_coproc_mask[n] l32i a0, a0, 0 // a0 = (n << 16) | (1 << n) movi a3, _xt_coproc_owner_sa +#if ( configNUMBER_OF_CORES > 1 ) + // Increment _xt_coproc_owner_sa (a3) to per-core area + coreid a2 +#ifdef XCHAL_HAVE_MUL32 + movi a6, (XCHAL_CP_MAX * 4) + mull a2, a2, a6 + add a3, a3, a2 +#else +1: + beqz a2, 2f + addi a3, a3, (XCHAL_CP_MAX * 4) + addi a2, a2, -1 + j 1b +2: +#endif +#endif extui a2, a0, 0, 16 // coprocessor bitmask portion or a4, a4, a2 // a4 = CPENABLE | (1 << n) wsr a4, CPENABLE @@ -202,6 +223,107 @@ _xt_coproc_handler: j .L_xt_coproc_err +#if ( configNUMBER_OF_CORES > 1 ) + +//----------------------------------------------------------------------------- +// _xt_coproc_saveall +// +// Saves all active coprocessor state to current thread's save area and disables +// all CPENABLE bits. Obeys ABI conventions on entry. +// Register use: +// a0 - on entry, return address +// a1 - ... +// a2 - pointer to coprocessor state save area +// a3-a15 - may all be used and trashed by this routine. +//----------------------------------------------------------------------------- + + .text + .align 4 + .global _xt_coproc_saveall + .type _xt_coproc_saveall,@function +_xt_coproc_saveall: + + ENTRY(16) + + // Get coprocessor state save area of new owner thread + mov a11, a2 // a11 = owner's save area + beqz a11, .L_xt_coproc_saveall_inv // not in a thread (invalid) + l32i a4, a11, XT_CP_ASA // actual save area address + beqz a4, .L_xt_coproc_saveall_inv // thread has no save area + + // Find first CP to save + movi a9, 0 // a9 = CP index + movi a8, 1 // a8 = CP mask +.L_check_done: + movi a3, XCHAL_CP_MASK // a3 = Valid CP mask + rsr a10, CPENABLE // a10 = CPENABLE + and a10, a10, a3 // Ignore invalid CPENABLE bits + beqz a10, .L_xt_coproc_saveall_done + bany a10, a8, .L_save_cp +.L_check_next: + addi a9, a9, 1 // a9 = next CP index + slli a8, a8, 1 // a8 = next CP mask + bany a10, a8, .L_save_cp + j .L_check_next + +.L_save_cp: + // Clear coprocessor owner thread (save area ptr) + movi a3, _xt_coproc_owner_sa + + // Increment _xt_coproc_owner_sa (a3) to per-core area + coreid a5 +1: + beqz a5, 2f + addi a3, a3, (XCHAL_CP_MAX * 4) + addi a5, a5, -1 + j 1b +2: // NOTE: a5 is 0 by the time we get here + + addx4 a3, a9, a3 // a3 = &_xt_coproc_owner_sa[n] + s32i a5, a3, 0 // _xt_coproc_owner_sa[n] = NULL + + // We need to save owner's coprocessor state + movi a5, _xt_coproc_sa_offset + + // Mark owner state as no longer active (CPENABLE bit n clear) + xor a4, a10, a8 // clear CP in owner's CPENABLE + s16i a4, a11, XT_CPENABLE // update owner's CPENABLE + + addx4 a5, a9, a5 // a5 = &_xt_coproc_sa_offset[n] + + // Mark owner state as saved (CPSTORED bit n set) + l16ui a4, a11, XT_CPSTORED // a4 = old owner's CPSTORED + l32i a5, a5, 0 // a5 = XT_CP[n]_SA offset + or a4, a4, a8 // set CP in old owner's CPSTORED + s16i a4, a11, XT_CPSTORED // update old owner's CPSTORED + l32i a2, a11, XT_CP_ASA // ptr to actual (aligned) save area + mov a3, a9 + add a2, a2, a5 // a2 = old owner's area for CP n + + // The config-specific HAL macro invoked below destroys a2-a6. + // It is theoretically possible for Xtensa processor designers to write TIE + // that causes more address registers to be affected, but it is generally + // unlikely. If that ever happens, more registers needs to be saved/restored + // around this macro invocation, and the value in a11 needs to be recomputed. + + xchal_cpi_store_funcbody + xor a10, a10, a8 // clear CP in CPENABLE + wsr a10, CPENABLE + bnez a10, .L_check_next + +.L_xt_coproc_saveall_done: + movi a2, 0 // a2 <- 0 == OK +.L_xt_coproc_saveall_err: + RET(16) + +.L_xt_coproc_saveall_inv: + // Coprocessor exception occurred outside a thread or the thread + // did not allocate space to save coprocessor state. Return error. + movi a2, 1 + j .L_xt_coproc_saveall_err + +#endif // ( configNUMBER_OF_CORES > 1 ) + //----------------------------------------------------------------------------- // _xt_coproc_savecs // @@ -240,68 +362,116 @@ _xt_coproc_savecs: s16i a11, a15, XT_CP_CS_ST // save mask of CPs being stored movi a13, _xt_coproc_sa_offset // array of CP save offsets l32i a15, a15, XT_CP_ASA // a15 = base of aligned save area +#if ( configNUMBER_OF_CORES > 1 ) + movi a12, _xt_coproc_owner_sa + coreid a14 +#ifdef XCHAL_HAVE_MUL32 + movi a10, (XCHAL_CP_MAX * 4) + mull a14, a14, a10 + add a12, a12, a14 +#else +1: + beqz a14, 2f + addi a12, a12, (XCHAL_CP_MAX * 4) // a12 = &_xt_coproc_owner_sa[core] + addi a14, a14, -1 + j 1b +2: +#endif +#endif /* configNUMBER_OF_CORES > 1 */ #if XCHAL_CP0_SA_SIZE bbci.l a11, 0, 2f // CP 0 not enabled l32i a14, a13, 0 // a14 = _xt_coproc_sa_offset[0] - add a12, a14, a15 // a12 = save area for CP 0 - xchal_cp0_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 // a14 = save area for CP 0 + xchal_cp0_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 0 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP1_SA_SIZE bbci.l a11, 1, 2f // CP 1 not enabled l32i a14, a13, 4 // a14 = _xt_coproc_sa_offset[1] - add a12, a14, a15 // a12 = save area for CP 1 - xchal_cp1_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 // a14 = save area for CP 1 + xchal_cp1_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 4 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP2_SA_SIZE bbci.l a11, 2, 2f l32i a14, a13, 8 - add a12, a14, a15 - xchal_cp2_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp2_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 8 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP3_SA_SIZE bbci.l a11, 3, 2f l32i a14, a13, 12 - add a12, a14, a15 - xchal_cp3_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp3_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 12 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP4_SA_SIZE bbci.l a11, 4, 2f l32i a14, a13, 16 - add a12, a14, a15 - xchal_cp4_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp4_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 16 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP5_SA_SIZE bbci.l a11, 5, 2f l32i a14, a13, 20 - add a12, a14, a15 - xchal_cp5_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp5_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 20 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP6_SA_SIZE bbci.l a11, 6, 2f l32i a14, a13, 24 - add a12, a14, a15 - xchal_cp6_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp6_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 24 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif #if XCHAL_CP7_SA_SIZE bbci.l a11, 7, 2f l32i a14, a13, 28 - add a12, a14, a15 - xchal_cp7_store a12, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL + add a14, a14, a15 + xchal_cp7_store a14, a7, a8, a9, a10 continue=0 ofs=-1 select=XTHAL_SAS_TIE|XTHAL_SAS_NOCC|XTHAL_SAS_CALE alloc=XTHAL_SAS_ALL +#if ( configNUMBER_OF_CORES > 1 ) + movi a7, 0 + s32i a7, a12, 28 // _xt_coproc_owner_sa[core][n] = NULL +#endif 2: #endif diff --git a/Cadence/Xtensa/xtensa_idma.c b/Cadence/Xtensa/xtensa_idma.c new file mode 100644 index 000000000..f9bd1c7bc --- /dev/null +++ b/Cadence/Xtensa/xtensa_idma.c @@ -0,0 +1,351 @@ + /* + * FreeRTOS Kernel + * Copyright (C) 2019-2026 Cadence Design Systems, Inc. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * https://www.FreeRTOS.org + * https://github.com/FreeRTOS + * + */ + +/* + * Xtensa-specific FreeRTOS interface to the libidma OS functions. This allows + * FreeRTOS threads to use iDMA channels safely. If you wish to provide your + * own implementation then this one is easily overridden from application code. + * However make sure to provide a full replacement for the public API. + */ + +#include +#include + +#if XCHAL_HAVE_IDMA + +#include + +#include "FreeRTOS.h" +#include "xtensa_api.h" + +#include "task.h" +#include "semphr.h" + +#if (defined INCLUDE_xTaskGetCurrentTaskHandle) && (INCLUDE_xTaskGetCurrentTaskHandle) + +//----------------------------------------------------------------------------- +// Allow one buffer per channel per thread. Normally this is in excess of what +// is actually needed, so adjust as necessary. Most threads will end up using +// only one channel at a time, and not all threads in an application even use +// iDMA. +// +// Thread-local storage is currently not used here because the assumption is +// that most threads will not use iDMA, so it is not useful to tie up one or +// more TLS slots in every thread. +// +// Since FreeRTOS task suspend/resume APIs are not interrupt-safe, a +// counting semaphore implements the block/unblock APIs. Recommend enabling +// configSUPPORT_STATIC_ALLOCATION to speed up semaphore creation. +//----------------------------------------------------------------------------- + +typedef struct { + idma_buf_t * buf_list[XCHAL_IDMA_NUM_CHANNELS]; + TaskHandle_t thread; +#if ( configNUMBER_OF_CORES > 1 ) + uint32_t core; +#endif +#if ( configSUPPORT_STATIC_ALLOCATION ) + StaticSemaphore_t sem_buf; +#endif + SemaphoreHandle_t sem_handle; +} xt_idma_buf_info_t; + +//----------------------------------------------------------------------------- +// Statically allocate some number of thread buffer info structs. Adjust this +// as needed. For large values, the linear searches will not be efficient but +// we don't expect MAX_THREADS to be large. Besides, channel buffer changes +// should be relatively infrequent. +// +// On multicore systems, this structure is also protected by a mutex. +//----------------------------------------------------------------------------- + +#define MAX_THREADS 8 + +static ALIGNDCACHE xt_idma_buf_info_t xt_idma_buf_info[MAX_THREADS]; + +#if ( configNUMBER_OF_CORES > 1 ) +#if ( configSUPPORT_STATIC_ALLOCATION ) +static StaticSemaphore_t xt_idma_mtx_buf; +#endif +static SemaphoreHandle_t xt_idma_mtx_handle = NULL; +#endif + + +//----------------------------------------------------------------------------- +// idma_register_interrupts +//----------------------------------------------------------------------------- +int32_t +idma_register_interrupts(int32_t ch, os_handler done_handler, os_handler err_handler) +{ +#if XCHAL_HAVE_INTERRUPTS + if (ch < XCHAL_IDMA_NUM_CHANNELS) { + uint32_t doneint = (uint32_t) XCHAL_IDMA_CH0_DONE_INTERRUPT + (uint32_t) ch; + uint32_t errint = (uint32_t) XCHAL_IDMA_CH0_ERR_INTERRUPT + (uint32_t) ch; + int32_t ret = 0; + xt_handler oldh; + + oldh = xt_set_interrupt_handler(doneint, done_handler, cvt_int32_to_voidp(ch)); + ret += (oldh == NULL) ? 1 : 0; + xt_interrupt_enable(doneint); + + oldh = xt_set_interrupt_handler(errint, err_handler, cvt_int32_to_voidp(ch)); + ret += (oldh == NULL) ? 1 : 0; + xt_interrupt_enable(errint); + + return (ret == 0) ? 0 : -1; + } +#endif + return -1; +} + + +//----------------------------------------------------------------------------- +// idma_disable_interrupts +//----------------------------------------------------------------------------- +uint32_t +idma_disable_interrupts(void) +{ + return xthal_disable_interrupts(); +} + + +//----------------------------------------------------------------------------- +// idma_enable_interrupts +//----------------------------------------------------------------------------- +void +idma_enable_interrupts(uint32_t level) +{ + xthal_restore_interrupts(level); +} + + +//----------------------------------------------------------------------------- +// idma_thread_id +//----------------------------------------------------------------------------- +void * +idma_thread_id(void) +{ + return xTaskGetCurrentTaskHandle(); +} + + +//----------------------------------------------------------------------------- +// idma_thread_block +//----------------------------------------------------------------------------- +void +idma_thread_block(void * thread) +{ + if (thread != NULL) { + int32_t i; + for (i = 0; i < MAX_THREADS; i++) { + if (xt_idma_buf_info[i].thread == thread) { + break; + } + } + configASSERT (i < MAX_THREADS); + xSemaphoreTake(xt_idma_buf_info[i].sem_handle, portMAX_DELAY); + } +} + + +//----------------------------------------------------------------------------- +// idma_thread_unblock +//----------------------------------------------------------------------------- +void +idma_thread_unblock(void * thread) +{ + if (thread != NULL) { + int32_t i; + for (i = 0; i < MAX_THREADS; i++) { + if (xt_idma_buf_info[i].thread == thread) { + break; + } + } + configASSERT (i < MAX_THREADS); + xSemaphoreGive(xt_idma_buf_info[i].sem_handle); + } +} + + +//----------------------------------------------------------------------------- +// idma_chan_buf_set +//----------------------------------------------------------------------------- +void +idma_chan_buf_set(int32_t ch, idma_buf_t * buf) +{ + if (ch < XCHAL_IDMA_NUM_CHANNELS) { + TaskHandle_t thread = xTaskGetCurrentTaskHandle(); + int32_t j = -1; + int32_t i; + uint32_t ps; + +#if ( configNUMBER_OF_CORES > 1 ) + if (!xt_idma_mtx_handle) { +#if ( configSUPPORT_STATIC_ALLOCATION ) + xt_idma_mtx_handle = xSemaphoreCreateMutexStatic(&xt_idma_mtx_buf); +#else + xt_idma_mtx_handle = xSemaphoreCreateMutex(); +#endif + } + xSemaphoreTake(xt_idma_mtx_handle, portMAX_DELAY); +#endif + // Find a free slot and allocate it. This requires interrupts + // disabled because we're manipulating global shared data. + ps = xthal_disable_interrupts(); + + for (i = 0; i < MAX_THREADS; i++) { + if (xt_idma_buf_info[i].thread == thread) { + break; + } + if ((xt_idma_buf_info[i].thread == NULL) && (j < 0)) { + j = i; + } + } + if (i < MAX_THREADS) { + xt_idma_buf_info[i].buf_list[ch] = buf; + } + else if (j >= 0) { + xt_idma_buf_info[j].thread = thread; + xt_idma_buf_info[j].buf_list[ch] = buf; +#if ( configNUMBER_OF_CORES > 1 ) + xt_idma_buf_info[j].core = portGET_CORE_ID(); +#endif +#if ( configSUPPORT_STATIC_ALLOCATION ) + xt_idma_buf_info[j].sem_handle = + xSemaphoreCreateCountingStatic(1, 0, &xt_idma_buf_info[j].sem_buf); +#else + xt_idma_buf_info[j].sem_handle = xSemaphoreCreateCounting(1, 0); +#endif + } + else { + configASSERT(0); + } + + xthal_restore_interrupts(ps); +#if ( configNUMBER_OF_CORES > 1 ) + xSemaphoreGive(xt_idma_mtx_handle); +#endif + } +} + + +//----------------------------------------------------------------------------- +// idma_chan_buf_get +//----------------------------------------------------------------------------- +idma_buf_t * +idma_chan_buf_get(int32_t ch) +{ + if (ch < XCHAL_IDMA_NUM_CHANNELS) { + TaskHandle_t thread = xTaskGetCurrentTaskHandle(); + int32_t i; + + for (i = 0; i < MAX_THREADS; i++) { + if (xt_idma_buf_info[i].thread == thread) { +#if ( configNUMBER_OF_CORES > 1 ) + // Confirm thread has not changed cores + configASSERT(xt_idma_buf_info[i].core == portGET_CORE_ID()); +#endif + return xt_idma_buf_info[i].buf_list[ch]; + } + } + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// idma_chan_buf_clear +//----------------------------------------------------------------------------- +void +idma_chan_buf_clear(int32_t ch) +{ + if (ch < XCHAL_IDMA_NUM_CHANNELS) { + TaskHandle_t thread = xTaskGetCurrentTaskHandle(); + int32_t i; + int32_t j; + uint32_t ps; +#if !( configSUPPORT_STATIC_ALLOCATION ) + SemaphoreHandle_t sem_to_delete = NULL; +#endif + +#if ( configNUMBER_OF_CORES > 1 ) + xSemaphoreTake(xt_idma_mtx_handle, portMAX_DELAY); +#endif + // The following requires interrupts disabled because we're + // manipulating global shared data. + ps = xthal_disable_interrupts(); + + for (i = 0; i < MAX_THREADS; i++) { + if (xt_idma_buf_info[i].thread == thread) { + // Clear the slot buffer for the channel. + xt_idma_buf_info[i].buf_list[ch] = NULL; + + // If this slot is now unused, free it up. + for (j = 0; j < XCHAL_IDMA_NUM_CHANNELS; j++) { + if (xt_idma_buf_info[i].buf_list[j] != NULL) { + break; + } + } + if (j == XCHAL_IDMA_NUM_CHANNELS) { +#if ( configNUMBER_OF_CORES > 1 ) + // Confirm thread has not changed cores + configASSERT(xt_idma_buf_info[i].core == portGET_CORE_ID()); +#endif +#if !( configSUPPORT_STATIC_ALLOCATION ) + // Delete sem after releasing lock and enabling interrupts + sem_to_delete = xt_idma_buf_info[i].sem_handle; +#endif + xt_idma_buf_info[i].sem_handle = NULL; + xt_idma_buf_info[i].thread = NULL; + } + + break; + } + } + + xthal_restore_interrupts(ps); +#if ( configNUMBER_OF_CORES > 1 ) + xSemaphoreGive(xt_idma_mtx_handle); +#endif +#if !( configSUPPORT_STATIC_ALLOCATION ) + if (sem_to_delete != NULL) { + vSemaphoreDelete(sem_to_delete); + } +#endif + } +} + +#else + +#warn INCLUDE_xTaskGetCurrentTaskHandle required for threaded iDMA support + +#endif // (defined INCLUDE_xTaskGetCurrentTaskHandle) && (INCLUDE_xTaskGetCurrentTaskHandle) + +#endif // XCHAL_HAVE_IDMA + diff --git a/Cadence/Xtensa/xtensa_intr.c b/Cadence/Xtensa/xtensa_intr.c index 3ca5cd51b..66c8faaba 100644 --- a/Cadence/Xtensa/xtensa_intr.c +++ b/Cadence/Xtensa/xtensa_intr.c @@ -33,6 +33,7 @@ */ #include +#include #include #include @@ -42,6 +43,8 @@ #include "xtensa_api.h" +#include "FreeRTOS.h" + #if XCHAL_HAVE_EXCEPTIONS @@ -56,6 +59,18 @@ void xt_unhandled_exception( XtExcFrame * frame ) { (void) frame; +#if ( configNUMBER_OF_CORES > 1 ) + static int uexc_recursion_depth = 0; + if (++uexc_recursion_depth >= configNUMBER_OF_CORES) { + /* In the event that calling _exit() results in an unhandled + * exception, stop the subsequent infinite loop. + */ + while ( 1 ); + } + if (portGET_CORE_ID() > 0) { + _exit( -1 ); + } +#endif exit( -1 ); } @@ -94,12 +109,6 @@ xt_set_exception_handler( uint32_t n, xt_exc_handler f ) #if XCHAL_HAVE_INTERRUPTS -#if XCHAL_HAVE_XEA2 && (XCHAL_NUM_INTERRUPTS <= 32) -/* Defined in xtensa_intr_asm.S */ -extern uint32_t xt_intenable; -extern uint32_t xt_vpri_mask; -#endif - /* Handler table is in xtensa_intr_asm.S */ typedef struct xt_handler_table_entry { xt_handler handler; @@ -120,6 +129,11 @@ void xt_unhandled_interrupt( void * arg ) { (void) arg; +#if ( configNUMBER_OF_CORES > 1 ) + if (portGET_CORE_ID() > 0) { + _exit( -1 ); + } +#endif exit( -1 ); } @@ -128,6 +142,7 @@ xt_unhandled_interrupt( void * arg ) This function registers a handler for the specified interrupt. The "arg" parameter specifies the argument to be passed to the handler when it is invoked. The function returns the address of the previous handler. + Note that the previous handler's argument will be lost. On error, it returns NULL. */ xt_handler @@ -172,6 +187,40 @@ xt_set_interrupt_handler( uint32_t n, xt_handler f, void * arg ) } +/* + This function returns a handler for the specified interrupt. + If no handler has been registered, it returns a pointer to + xt_unhandled_interrupt(). + If the interrupt number or level are invalid, it returns NULL. +*/ +xt_handler +xt_get_interrupt_handler( uint32_t n ) +{ + xt_handler_table_entry * entry; + + if ( n >= (uint32_t) XCHAL_NUM_INTERRUPTS ) + { + // Invalid interrupt number. + return NULL; + } + +#if XCHAL_HAVE_XEA2 + if ( Xthal_intlevel[n] > XCHAL_EXCM_LEVEL ) + { + // Priority level too high to safely handle in C. + return NULL; + } +#endif + +#if (XT_USE_INT_WRAPPER || XCHAL_HAVE_XEA3) + entry = _xt_interrupt_table + n + 1; +#else + entry = _xt_interrupt_table + n; +#endif + return entry->handler; +} + + /* This function enables the interrupt whose number is specified as the argument. @@ -182,11 +231,16 @@ xt_interrupt_enable( uint32_t intnum ) #if XCHAL_HAVE_XEA2 && (XCHAL_NUM_INTERRUPTS <= 32) if ( intnum < (uint32_t) XCHAL_NUM_INTERRUPTS ) { + #if ( configNUMBER_OF_CORES > 1 ) + xt_internal_data_t *intdatap = &(_XT_INTDATA(portGET_CORE_ID())); + #else + xt_internal_data_t *intdatap = &_xt_intdata; + #endif int ps = XT_RSIL( 15 ); // New INTENABLE = (xt_intenable | mask) & xt_vpri_mask. - xt_intenable |= ( 1U << intnum ); - XT_WSR_INTENABLE( xt_intenable & xt_vpri_mask ); + intdatap->xt_intenable |= ( 1U << intnum ); + XT_WSR_INTENABLE( intdatap->xt_intenable & intdatap->xt_vpri_mask ); XT_WSR_PS( ps ); XT_RSYNC(); } @@ -206,11 +260,16 @@ xt_interrupt_disable( uint32_t intnum ) #if XCHAL_HAVE_XEA2 && (XCHAL_NUM_INTERRUPTS <= 32) if ( intnum < (uint32_t) XCHAL_NUM_INTERRUPTS ) { + #if ( configNUMBER_OF_CORES > 1 ) + xt_internal_data_t *intdatap = &(_XT_INTDATA(portGET_CORE_ID())); + #else + xt_internal_data_t *intdatap = &_xt_intdata; + #endif int ps = XT_RSIL( 15 ); // New INTENABLE = (xt_intenable & ~mask) & xt_vpri_mask. - xt_intenable &= ~( 1U << intnum ); - XT_WSR_INTENABLE( xt_intenable & xt_vpri_mask ); + intdatap->xt_intenable &= ~( 1U << intnum ); + XT_WSR_INTENABLE( intdatap->xt_intenable & intdatap->xt_vpri_mask ); XT_WSR_PS( ps ); XT_RSYNC(); } @@ -230,7 +289,13 @@ xt_interrupt_enabled( uint32_t intnum ) #if XCHAL_HAVE_XEA2 && (XCHAL_NUM_INTERRUPTS <= 32) if ( intnum < (uint32_t) XCHAL_NUM_INTERRUPTS ) { - return ( (xt_intenable & (1U << intnum)) != 0 ) ? 1U : 0; + #if ( configNUMBER_OF_CORES > 1 ) + xt_internal_data_t *intdatap = &(_XT_INTDATA(portGET_CORE_ID())); + #else + xt_internal_data_t *intdatap = &_xt_intdata; + #endif + + return ( (intdatap->xt_intenable & (1U << intnum)) != 0 ) ? 1U : 0; } return 0; #else diff --git a/Cadence/Xtensa/xtensa_intr_asm.S b/Cadence/Xtensa/xtensa_intr_asm.S index c495e2c03..48be589d0 100644 --- a/Cadence/Xtensa/xtensa_intr_asm.S +++ b/Cadence/Xtensa/xtensa_intr_asm.S @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -45,30 +45,18 @@ /* ------------------------------------------------------------------------------- INTENABLE virtualization information. + + Moved to _xt_intdata structure for better cache performance ------------------------------------------------------------------------------- */ -#if XCHAL_HAVE_XEA2 && (XCHAL_NUM_INTERRUPTS <= 32) - - .data - .global xt_intdata - .align 8 -xt_intdata: - .global xt_intenable - .type xt_intenable,@object - .size xt_intenable,4 - .global xt_vpri_mask - .type xt_vpri_mask,@object - .size xt_vpri_mask,4 - -xt_intenable: .word 0 /* Virtual INTENABLE */ -xt_vpri_mask: .word 0xFFFFFFFF /* Virtual priority mask */ - -#endif - /* ------------------------------------------------------------------------------- System interrupt stack. + + For SMP configurations, (configNUMBER_OF_CORES * configISR_STACK_SIZE) bytes + are allocated. Avoid placing this in per-core memory (usually dataram) as + that is a valuable resource. ------------------------------------------------------------------------------- */ @@ -85,6 +73,11 @@ xt_vpri_mask: .word 0xFFFFFFFF /* Virtual priority mask */ xt_interrupt_stack: .space configISR_STACK_SIZE xt_interrupt_stack_top: +#if ( configNUMBER_OF_CORES > 1 ) + .rept configNUMBER_OF_CORES - 1 + .space configISR_STACK_SIZE + .endr +#endif /* @@ -92,6 +85,10 @@ xt_interrupt_stack_top: Table of C-callable interrupt handlers for each interrupt. For XEA2 configs, not all slots can be filled, because interrupts at level > EXCM_LEVEL will not be dispatched to a C handler by default. + + NOTE: This is not replicated for SMP systems, so registering an interrupt + handler on one core will register the same handler for that interrupt on all + cores. However, Interrupts are enabled and disabled on a per-core basis. ------------------------------------------------------------------------------- */ diff --git a/Cadence/Xtensa/xtensa_intr_wrapper.c b/Cadence/Xtensa/xtensa_intr_wrapper.c index 5c3d19d7e..4419f3b81 100644 --- a/Cadence/Xtensa/xtensa_intr_wrapper.c +++ b/Cadence/Xtensa/xtensa_intr_wrapper.c @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -47,8 +47,6 @@ typedef struct xt_handler_table_entry { extern xt_handler_table_entry _xt_interrupt_table[XCHAL_NUM_INTERRUPTS + 1]; extern int32_t xt_sw_intnum; -extern int32_t port_switch_flag; -extern uint32_t port_interruptNesting; static int32_t xt_wflag; @@ -67,7 +65,7 @@ xt_interrupt_wrapper(void * arg) xt_handler handler; state = portENTER_CRITICAL_NESTED(); - port_interruptNesting++; + portINCREMENT_INTERRUPT_NESTING_COUNT(); portEXIT_CRITICAL_NESTED(state); /* Load handler address and argument from table. Note that the @@ -86,13 +84,13 @@ xt_interrupt_wrapper(void * arg) if (xt_wflag != 0) { xt_wflag = 0; } - else if (port_switch_flag) { + else if (_xt_intdata.port_switch_flag) { xt_wflag = 1; xt_interrupt_trigger(xt_sw_intnum); } state = portENTER_CRITICAL_NESTED(); - port_interruptNesting--; + portDECREMENT_INTERRUPT_NESTING_COUNT(); portEXIT_CRITICAL_NESTED(state); } diff --git a/Cadence/Xtensa/xtensa_rtos.h b/Cadence/Xtensa/xtensa_rtos.h index c1c287f15..88a2ffd8a 100644 --- a/Cadence/Xtensa/xtensa_rtos.h +++ b/Cadence/Xtensa/xtensa_rtos.h @@ -1,6 +1,6 @@ /* * FreeRTOS Kernel - * Copyright (C) 2015-2024 Cadence Design Systems, Inc. + * Copyright (C) 2015-2025 Cadence Design Systems, Inc. * Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: MIT @@ -250,8 +250,8 @@ Xtensa Port Version. *******************************************************************************/ -#define XTENSA_PORT_VERSION 3.00 -#define XTENSA_PORT_VERSION_STRING "3.00" +#define XTENSA_PORT_VERSION 3.13 +#define XTENSA_PORT_VERSION_STRING "3.13" #define XT_IRQ_LOCK_LEVEL XCHAL_EXCM_LEVEL diff --git a/Cadence/Xtensa/xtensa_vectors.S b/Cadence/Xtensa/xtensa_vectors.S index 04fa6c2a0..6f2a47464 100644 --- a/Cadence/Xtensa/xtensa_vectors.S +++ b/Cadence/Xtensa/xtensa_vectors.S @@ -324,12 +324,12 @@ add a2, a4, a4 /* a2 = a4 << 1 */ addi a2, a2, -1 /* a2 = mask of 1's <= a4 bit */ and a2, a2, a3 /* a2 = mask of all bits <= a4 at this level */ - movi a3, xt_intdata - l32i a6, a3, 4 /* a6 = xt_vpri_mask */ + pintdata a3, a6 + l32i a6, a3, PORTINT_VPRI_MASK_OFF /* a6 = xt_vpri_mask */ neg a2, a2 addi a2, a2, -1 /* a2 = mask to apply */ and a5, a6, a2 /* mask off all bits <= a4 bit */ - s32i a5, a3, 4 /* update xt_vpri_mask */ + s32i a5, a3, PORTINT_VPRI_MASK_OFF /* update xt_vpri_mask */ rsr a3, INTENABLE and a3, a3, a2 /* mask off all bits <= a4 bit */ wsr a3, INTENABLE @@ -375,9 +375,9 @@ */ rsil a3, \level - movi a3, xt_intdata - l32i a4, a3, 0 /* a4 = xt_intenable */ - s32i a2, a3, 4 /* update xt_vpri_mask */ + pintdata a3, a4 + l32i a4, a3, PORTINT_INTENABLE_OFF /* a4 = xt_intenable */ + s32i a2, a3, PORTINT_VPRI_MASK_OFF /* update xt_vpri_mask */ and a4, a4, a2 /* a4 = masked intenable */ wsr a4, INTENABLE /* update INTENABLE */ @@ -427,8 +427,8 @@ _xt_panic: simcall #else rsil a2, XT_IRQ_LOCK_LEVEL /* disable all low & med ints */ -1: j 1b /* loop infinitely */ #endif +1: j 1b /* loop infinitely */ .section .rodata, "a" .align 4 diff --git a/Cadence/Xtensa/xtensa_vectors_xea3.S b/Cadence/Xtensa/xtensa_vectors_xea3.S index 54ba28197..ef75e91ce 100644 --- a/Cadence/Xtensa/xtensa_vectors_xea3.S +++ b/Cadence/Xtensa/xtensa_vectors_xea3.S @@ -276,11 +276,11 @@ _xt_entry: // a0 holds return address (Tailchain+3) // For call0: // a11 holds ExcCause, also saved in [oldsp - 72] - // a15 holds exception SP, a1 points to exception frame + // a15 holds exception SP, a1 points to exception frame (for precise exc) // For windowed: // a3 holds ExcCause, also saved in [oldsp - 72] - // a1 holds new SP - // New SP + XT_STK_XTRA_SZ points to exception frame + // a7 holds exception SP, a1 points to exception frame (for precise exc) + // For imprecise exceptions a1 is now pointing into the interrupt stack. .global _xt_exception @@ -307,8 +307,26 @@ _xt_exception: #else mov a2, a1 // Argument = Exception frame ptr #endif + extui a5, a3, EXCCAUSE_IMPR_SHIFT, EXCCAUSE_IMPR_BITS // Extract imprecise bits + bnez a5, .Limprecise jx a4 // Return directly from handler +.Limprecise: + // For imprecise exceptions the stack has been switched to the + // interrupt stack, but the exception frame is on the application + // stack. Adjust the argument (a2) accordingly. +#ifdef __XTENSA_CALL0_ABI__ + mov a2, a15 // a15 holds exception SP +#else + mov a2, a7 // a7 holds exception SP +#endif +#if portUSING_MPU_WRAPPERS && !(defined __XTENSA_CALL0_ABI__) + addi a2, a2, -96 - 4 // Adjust for XT_STK_A2 +#else + addi a2, a2, -96 +#endif + jx a4 // Jump to handler + #if portUSING_MPU_WRAPPERS .align 4 do_is_priv: diff --git a/Cadence/Xtensa/xtsubsystem_patch.h b/Cadence/Xtensa/xtsubsystem_patch.h new file mode 100644 index 000000000..64cb85444 --- /dev/null +++ b/Cadence/Xtensa/xtsubsystem_patch.h @@ -0,0 +1,289 @@ +/* + * xtsubsystem.h -- definitions for the Xtensa multicore subsystem + */ + +/* + * Copyright (c) 2024-2025 Cadence Design Systems, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef XTENSA_XTSUBSYSTEM_H +#define XTENSA_XTSUBSYSTEM_H + +#include +#include +#include +#include + +/* + * XTSubsystem Registers + * + * The register window is broken down into 2 portions: + * 1. Inter-processor interrupt (IPI) register window + * 2. XTSubsystem control register window + * + * Register naming suffix description: + * + * - _BASE : base address of a block relative to APB + * - _B : base address of a block indexed by core/set ID + * - _I : increment amount per core/set index + * - _O : offset to register block + */ + +/* Xtsubsystem block offset relative to APB base */ +#define XTSUB_IPI_O UINT32_C(0xfc000) +#define XTSUB_IPI_BASE (XTSUB_IPI_O) +#define XTSUB_IPI_SIZE UINT32_C(0x3000) + +/* Xtsubsystem IPI register definitions */ +#define XTSUB_IPI_S0C_B UINT32_C(0x0000) /* Inter-processor interrupt set 0 base */ +#define XTSUB_IPI_S0C_I UINT32_C(4) /* IPI per-core increment */ +#define XTSUB_IPI_S_I UINT32_C(0x0100) /* IPI per-set increment */ + +/* Xtsubsystem control register (post-IPI) block definitions */ +#define XTSUB_CTLREG_O UINT32_C(0x3000) +#define XTSUB_CTLREG_BASE (XTSUB_IPI_BASE + XTSUB_CTLREG_O) +#define XTSUB_CTLREG_SIZE UINT32_C(0x1000) + +/* Xtsubsystem full register block definitions */ +#define XTSUB_TOTAL_BASE (XTSUB_IPI_BASE) +#define XTSUB_TOTAL_SIZE (XTSUB_IPI_SIZE + XTSUB_CTLREG_SIZE) + +/* Register offsets must be <= 1020 (decimal) to be used as imm8 values. + * Must add in XTSUB_CTLREG_BASE to be used as absolute addresses. + */ +#define XTSUB_RUN_ON_RESET UINT32_C(0x0020) +#define XTSUB_START_VEC_SEL UINT32_C(0x0024) +#define XTSUB_ALT_RESET_VEC_B UINT32_C(0x0030) +#define XTSUB_ALT_RESET_VEC_I UINT32_C(4) +#define XTSUB_CC_CTRL UINT32_C(0x0100) +#define XTSUB_CC_STAT UINT32_C(0x0104) +#define XTSUB_PWAIT_STAT UINT32_C(0x0108) +#define XTSUB_NUM_CORES UINT32_C(0x010c) +#define XTSUB_CC_PWR_CTRL_B UINT32_C(0x0200) +#define XTSUB_CC_PWR_CTRL_I UINT32_C(4) +#define XTSUB_CC_TIME_LO UINT32_C(0x02f8) +#define XTSUB_CC_TIME_HI UINT32_C(0x02fc) +#define XTSUB_CC_TIMECMP_LO_B UINT32_C(0x0300) +#define XTSUB_CC_TIMECMP_HI_B UINT32_C(0x0304) +#define XTSUB_CC_TIMECMP_I UINT32_C(8) + +/* Definitions for XTSUB_RUN_ON_RESET */ +#define XTSUB_RUN_ALL_CORES ((1 << XCHAL_SUBSYS_NUM_CORES) - UINT32_C(2)) + +#if !defined(_ASMLANGUAGE) && !defined(_NOCLANGUAGE) && !defined(__ASSEMBLER__) + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This function puts the Xtsubsystem base address into the location + * pointed to by xtsub_addr. + * + * Parameters: + * xtsub_addr Pointer to memory to be written on success + * + * It returns: + * XTHAL_INVALID If xtsub_addr is NULL, + * XTHAL_UNSUPPORTED APB is not configured, or + * XTHAL_SUCCESS on success + */ +XT_INLINE int32_t xthal_get_xtsub_address(uint32_t *xtsub_addr) +{ + int32_t ret = xthal_get_apb_address(xtsub_addr); + if (ret == XTHAL_SUCCESS) { + *xtsub_addr += XTSUB_TOTAL_BASE; + } + return ret; +} + +/* + * This function triggers an inter-processor interrupt (IPI) on the specified + * core using IPI set 0. The remote core must not also be the current core. + * The interrupt type is not checked but must be configured as "external edge." + * + * By convention, this function triggers processor interrupt , i.e. + * XCHAL_SUBSYS_IPI_S0C_INTNUM, on "core_id" where corresponds to the + * ID of the core calling this function. Processor interrupt can be + * mapped to an external BInterrupt pin using XCHAL_EXTINT_NUM. + * + * Parameters: + * core_id Remote core on which to trigger an IPI + * + * It returns: + * XTHAL_INVALID If the specified core is invalid, + * XTHAL_UNSUPPORTED APB is not configured, or + * XTHAL_SUCCESS on success + */ +XT_INLINE int32_t xthal_ipi_trigger(uint32_t core_id) +{ + uint32_t curr_core_id = xthal_get_coreid(); + uint32_t ipi_reg_addr; + int32_t ret; + + if ((core_id == curr_core_id) || (core_id >= XCHAL_SUBSYS_NUM_CORES)) { + return XTHAL_INVALID; + } + ret = xthal_get_xtsub_address(&ipi_reg_addr); + if (ret != XTHAL_SUCCESS) { + return ret; + } + + ipi_reg_addr += XTSUB_IPI_S0C_B + (XTSUB_IPI_S0C_I * curr_core_id); + + // Writing 1 to bit b sends an interrupt pulse to core b. + // XTSC models this as an MMIO register, so writing a 0 is required + // to complete the pulse. RTL does nothing when 0 is written. + *(volatile uint32_t *)ipi_reg_addr = (1 << core_id); + *(volatile uint32_t *)ipi_reg_addr = 0; + return XTHAL_SUCCESS; +} + + +/* + * Multicore Timer APIs. + * + * Always 64 bits wide, unlike other HAL timer APIs. + * Minimal error checking is performed in the interest of speed. + * + * NOTE: Some routines are written in inline assembly to preserve + * instruction order without the performance impact of memw operations + * (which can be added to volatile pointers due to serialization). + * Using assembly for these operations prevents bundling of loads and + * stores but we still benefit from bundling other ops. + */ + +#define STR(x) #x +#define XSTR(x) STR(x) +#define APB_BASE_ADDR_MASK UINT32_C(0xfffff000) + +/* Internal helper function optimized for cct APIs */ +XT_INLINE uint32_t *xthal_get_xtsub_cctbaseaddr(void) +{ +#if XCHAL_SUBSYS_HAVE_CCTIMER + + // Optimization: do not call xthal_get_apb_address(), which would + // involve an extra function call in addition to stack operations + // for parameter/return value management +#if (XCHAL_HAVE_XEA2 && XCHAL_HAVE_APB) || XCHAL_HAVE_PROGRAMMABLE_APB + uint32_t apb_addr = XT_RSR_APB0CFG() & APB_BASE_ADDR_MASK; +#elif (XCHAL_HAVE_APB) + uint32_t apb_addr = XCHAL_APB_BASEADDR; +#endif + return (uint32_t *)(apb_addr + XTSUB_CTLREG_BASE); + +#else + + return NULL; + +#endif /* XCHAL_SUBSYS_HAVE_CCTIMER */ +} + +XT_INLINE uint64_t xthal_get_cct_cycle_count(void) +{ +#if XCHAL_SUBSYS_HAVE_CCTIMER + + uint32_t *regbase = xthal_get_xtsub_cctbaseaddr(); + uint32_t lo, hi1, hi2; + __asm__ volatile ("l32i %0, %3, " XSTR(XTSUB_CC_TIME_HI) "\n\t" + "l32i %1, %3, " XSTR(XTSUB_CC_TIME_LO) "\n\t" + "l32i %2, %3, " XSTR(XTSUB_CC_TIME_HI) "\n\t" + : "=&r"(hi1), "=&r"(lo), "=r"(hi2) : "a"(regbase) : "memory" ); + if (hi1 != hi2) { + // Reread to get consistent cycle count + lo = XT_L32I((const int *)regbase, XTSUB_CC_TIME_LO); + } + return ((uint64_t)hi2 << UINT32_C(32)) | (uint64_t)lo; + +#else + + return 0; + +#endif /* XCHAL_SUBSYS_HAVE_CCTIMER */ +} + +XT_INLINE uint64_t xthal_get_cct_cycle_compare(void) +{ +#if XCHAL_SUBSYS_HAVE_CCTIMER + + uint32_t id = xthal_get_coreid(); + uint32_t *regbase = xthal_get_xtsub_cctbaseaddr() + + (XTSUB_CC_TIMECMP_I * id) / sizeof(uint32_t); + // Reads with intrinsics here is fine since TIMECMP isn't changing + uint32_t hi = XT_L32I((const int *)regbase, XTSUB_CC_TIMECMP_HI_B); + uint32_t lo = XT_L32I((const int *)regbase, XTSUB_CC_TIMECMP_LO_B); + return ((uint64_t)hi << UINT32_C(32)) | (uint64_t)lo; + +#else + + return 0; + +#endif /* XCHAL_SUBSYS_HAVE_CCTIMER */ +} + +XT_INLINE void xthal_set_cct_cycle_compare(uint64_t val) +{ +#if XCHAL_SUBSYS_HAVE_CCTIMER + + uint32_t id = xthal_get_coreid(); + uint32_t *regbase = xthal_get_xtsub_cctbaseaddr() + + (XTSUB_CC_TIMECMP_I * id) / sizeof(uint32_t); + uint32_t lo = (uint32_t)(val & UINT32_MAX); + uint32_t hi = (uint32_t)(val >> UINT32_C(32)); + // Value is latched upon writing low word; write high word first. + // For performance reasons, no precautions are taken to avoid an + // interrupt triggering between the two writes; the caller should + // protect against that if needed. + __asm__ volatile ("s32i %0, %2, " XSTR(XTSUB_CC_TIMECMP_HI_B) "\n\t" + "s32i %1, %2, " XSTR(XTSUB_CC_TIMECMP_LO_B) "\n\t" + :: "r"(hi), "r"(lo), "a"(regbase) : "memory" ); + +#else + + UNUSED(val); + +#endif /* XCHAL_SUBSYS_HAVE_CCTIMER */ +} + +XT_INLINE void xthal_cct_interrupt_clear(void) +{ +#if XCHAL_SUBSYS_HAVE_CCTIMER + + uint64_t cctimecmp_disable = (uint64_t)-1; + uint64_t reread_val; + xthal_set_cct_cycle_compare(cctimecmp_disable); + + // Reread compare register to ensure that APB writes have completed + reread_val = xthal_get_cct_cycle_compare(); + UNUSED(reread_val); + +#endif /* XCHAL_SUBSYS_HAVE_CCTIMER */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* !__ASSEMBLER__ */ + +#endif /* XTENSA_XTSUBSYSTEM_H */ +