rp2,esp32,extmod: Implement UPDATE_SUBMODULES in CMake.
Rather than having Make calling CMake to generate a list of submodules and then run a Make target (which is complex and prone to masking other errors), implement the submodule update logic in CMake itself. Internal CMake-side changes are that GIT_SUBMODULES is now a CMake list, and the trigger variable name is changed from ECHO_SUBMODULES to UPDATE_SUBMODULES. The run is otherwise 100% a normal CMake run now, so most of the other special casing can be removed. Signed-off-by: Angus Gratton <angus@redyak.com.au>
This commit is contained in:
Angus Gratton
Damien George
parent
2db0c0225f
commit
cccac2cc01
7 changed files with 111 additions and 104 deletions
|
|
@ -107,62 +107,57 @@ set(MICROPY_SOURCE_LIB_LIBM_SQRT_HW ${MICROPY_DIR}/lib/libm/thumb_vfp_sqrtf.c)
|
|||
|
||||
if(MICROPY_PY_BTREE)
|
||||
set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/berkeley-db-1.xx)
|
||||
list(APPEND GIT_SUBMODULES lib/berkeley-db-1.xx)
|
||||
|
||||
if(ECHO_SUBMODULES)
|
||||
# No-op, we're just doing submodule/variant discovery.
|
||||
# Cannot run the add_library/target_include_directories rules (even though
|
||||
# the build won't run) because IDF will attempt verify the files exist.
|
||||
elseif(NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
|
||||
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
|
||||
# Regular build, submodule not initialised -- fail with a clear error.
|
||||
message(FATAL_ERROR " MICROPY_PY_BTREE is enabled but the berkeley-db submodule is not initialised.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
|
||||
else()
|
||||
# Regular build, we have the submodule.
|
||||
add_library(micropy_extmod_btree OBJECT
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
|
||||
)
|
||||
|
||||
target_include_directories(micropy_extmod_btree PRIVATE
|
||||
${MICROPY_LIB_BERKELEY_DIR}/include
|
||||
)
|
||||
|
||||
if(NOT BERKELEY_DB_CONFIG_FILE)
|
||||
set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
|
||||
endif()
|
||||
|
||||
target_compile_definitions(micropy_extmod_btree PRIVATE
|
||||
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
|
||||
)
|
||||
|
||||
# The include directories and compile definitions below are needed to build
|
||||
# modbtree.c and should be added to the main MicroPython target.
|
||||
|
||||
list(APPEND MICROPY_INC_CORE
|
||||
"${MICROPY_LIB_BERKELEY_DIR}/include"
|
||||
)
|
||||
|
||||
list(APPEND MICROPY_DEF_CORE
|
||||
MICROPY_PY_BTREE=1
|
||||
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
|
||||
)
|
||||
|
||||
list(APPEND MICROPY_SOURCE_EXTMOD
|
||||
${MICROPY_EXTMOD_DIR}/modbtree.c
|
||||
)
|
||||
endif()
|
||||
|
||||
add_library(micropy_extmod_btree OBJECT
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
|
||||
${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
|
||||
)
|
||||
|
||||
target_include_directories(micropy_extmod_btree PRIVATE
|
||||
${MICROPY_LIB_BERKELEY_DIR}/include
|
||||
)
|
||||
|
||||
if(NOT BERKELEY_DB_CONFIG_FILE)
|
||||
set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
|
||||
endif()
|
||||
|
||||
target_compile_definitions(micropy_extmod_btree PRIVATE
|
||||
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
|
||||
)
|
||||
|
||||
# The include directories and compile definitions below are needed to build
|
||||
# modbtree.c and should be added to the main MicroPython target.
|
||||
|
||||
list(APPEND MICROPY_INC_CORE
|
||||
"${MICROPY_LIB_BERKELEY_DIR}/include"
|
||||
)
|
||||
|
||||
list(APPEND MICROPY_DEF_CORE
|
||||
MICROPY_PY_BTREE=1
|
||||
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
|
||||
)
|
||||
|
||||
list(APPEND MICROPY_SOURCE_EXTMOD
|
||||
${MICROPY_EXTMOD_DIR}/modbtree.c
|
||||
)
|
||||
endif()
|
||||
|
||||
# Library for mbedtls
|
||||
|
|
@ -350,5 +345,5 @@ if(MICROPY_PY_LWIP)
|
|||
${MICROPY_LIB_LWIP_DIR}/include
|
||||
)
|
||||
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/lwip)
|
||||
list(APPEND GIT_SUBMODULES lib/lwip)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -106,12 +106,10 @@ size-components:
|
|||
size-files:
|
||||
$(call RUN_IDF_PY,size-files)
|
||||
|
||||
# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
|
||||
# print out the value of the GIT_SUBMODULES variable, prefixed with
|
||||
# "GIT_SUBMODULES", and then abort. This extracts out that line from the idf.py
|
||||
# output and passes the list of submodules to py/mkrules.mk which does the
|
||||
# `git submodule init` on each.
|
||||
# Run idf.py with the UPDATE_SUBMODULES flag to update
|
||||
# necessary submodules for this board.
|
||||
#
|
||||
# This is done in a dedicated build directory as some CMake cache values are not
|
||||
# set correctly if not all submodules are loaded yet.
|
||||
submodules:
|
||||
@GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \
|
||||
grep '^GIT_SUBMODULES=' | cut -d= -f2); \
|
||||
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
|
||||
IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D UPDATE_SUBMODULES=1 reconfigure
|
||||
|
|
|
|||
|
|
@ -71,8 +71,8 @@ list(APPEND MICROPY_SOURCE_DRIVERS
|
|||
${MICROPY_DIR}/drivers/dht/dht.c
|
||||
)
|
||||
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
|
||||
if(MICROPY_PY_TINYUSB AND NOT ECHO_SUBMODULES)
|
||||
list(APPEND GIT_SUBMODULES lib/tinyusb)
|
||||
if(MICROPY_PY_TINYUSB)
|
||||
set(TINYUSB_SRC "${MICROPY_DIR}/lib/tinyusb/src")
|
||||
string(TOUPPER OPT_MCU_${IDF_TARGET} tusb_mcu)
|
||||
|
||||
|
|
@ -195,6 +195,17 @@ if (MICROPY_USER_LDFRAGMENTS)
|
|||
set(MICROPY_LDFRAGMENTS ${MICROPY_USER_LDFRAGMENTS})
|
||||
endif()
|
||||
|
||||
if (UPDATE_SUBMODULES)
|
||||
# ESP-IDF checks if some paths exist before CMake does. Some paths don't
|
||||
# yet exist if this is an UPDATE_SUBMODULES pass on a brand new checkout, so remove
|
||||
# any path which might not exist yet. A "real" build will not set UPDATE_SUBMODULES.
|
||||
unset(MICROPY_SOURCE_TINYUSB)
|
||||
unset(MICROPY_SOURCE_EXTMOD)
|
||||
unset(MICROPY_SOURCE_LIB)
|
||||
unset(MICROPY_INC_TINYUSB)
|
||||
unset(MICROPY_INC_CORE)
|
||||
endif()
|
||||
|
||||
# Register the main IDF component.
|
||||
idf_component_register(
|
||||
SRCS
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ if (MICROPY_PY_NETWORK_CYW43)
|
|||
endif()
|
||||
|
||||
# Necessary submodules for all boards.
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mbedtls)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
|
||||
list(APPEND GIT_SUBMODULES lib/mbedtls)
|
||||
list(APPEND GIT_SUBMODULES lib/tinyusb)
|
||||
|
||||
# Include component cmake fragments
|
||||
include(${MICROPY_DIR}/py/py.cmake)
|
||||
|
|
@ -340,7 +340,7 @@ if (MICROPY_PY_BLUETOOTH_CYW43)
|
|||
endif()
|
||||
|
||||
if (MICROPY_BLUETOOTH_BTSTACK)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/btstack)
|
||||
list(APPEND GIT_SUBMODULES lib/btstack)
|
||||
|
||||
list(APPEND MICROPY_SOURCE_PORT mpbtstackport.c)
|
||||
|
||||
|
|
@ -358,8 +358,8 @@ if (MICROPY_BLUETOOTH_BTSTACK)
|
|||
endif()
|
||||
|
||||
if(MICROPY_BLUETOOTH_NIMBLE)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mynewt-nimble)
|
||||
if(NOT (${ECHO_SUBMODULES}) AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
|
||||
list(APPEND GIT_SUBMODULES lib/mynewt-nimble)
|
||||
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
|
||||
message(FATAL_ERROR " mynewt-nimble not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
|
||||
endif()
|
||||
|
||||
|
|
@ -386,8 +386,8 @@ target_include_directories(${MICROPY_TARGET} PRIVATE
|
|||
)
|
||||
|
||||
if (MICROPY_PY_NETWORK_CYW43)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/cyw43-driver)
|
||||
if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
|
||||
list(APPEND GIT_SUBMODULES lib/cyw43-driver)
|
||||
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
|
||||
message(FATAL_ERROR " cyw43-driver not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
|
||||
endif()
|
||||
|
||||
|
|
@ -433,8 +433,8 @@ if (MICROPY_PY_NETWORK_NINAW10)
|
|||
endif()
|
||||
|
||||
if (MICROPY_PY_NETWORK_WIZNET5K)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/wiznet5k)
|
||||
if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
|
||||
list(APPEND GIT_SUBMODULES lib/wiznet5k)
|
||||
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
|
||||
message(FATAL_ERROR " wiznet5k not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
|
||||
endif()
|
||||
|
||||
|
|
|
|||
|
|
@ -65,15 +65,11 @@ all:
|
|||
clean:
|
||||
$(RM) -rf $(BUILD)
|
||||
|
||||
# First ensure that pico-sdk is initialised, then use cmake to pick everything
|
||||
# else (including board-specific dependencies).
|
||||
# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
|
||||
# print out the value of the GIT_SUBMODULES variable, prefixed with
|
||||
# "GIT_SUBMODULES", and then abort. This extracts out that line from the cmake
|
||||
# output and passes the list of submodules to py/mkrules.mk which does the
|
||||
# `git submodule init` on each.
|
||||
# First ensure that pico-sdk is initialised, then run CMake with the
|
||||
# UPDATE_SUBMODULES flag to update necessary submodules for this board.
|
||||
#
|
||||
# This is done in a dedicated build directory as some CMake cache values are not
|
||||
# set correctly if not all submodules are loaded yet.
|
||||
submodules:
|
||||
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="lib/pico-sdk" submodules
|
||||
@GIT_SUBMODULES=$$(cmake -B $(BUILD)/submodules -DECHO_SUBMODULES=1 ${CMAKE_ARGS} -S . 2>&1 | \
|
||||
grep '^GIT_SUBMODULES=' | cut -d= -f2); \
|
||||
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
|
||||
cmake -S . -B $(BUILD)/submodules -DUPDATE_SUBMODULES=1 ${CMAKE_ARGS}
|
||||
|
|
|
|||
|
|
@ -209,16 +209,11 @@ if(MICROPY_FROZEN_MANIFEST)
|
|||
# Note: target_compile_definitions already added earlier.
|
||||
|
||||
if(NOT MICROPY_LIB_DIR)
|
||||
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/micropython-lib)
|
||||
list(APPEND GIT_SUBMODULES lib/micropython-lib)
|
||||
set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib)
|
||||
endif()
|
||||
|
||||
if(ECHO_SUBMODULES)
|
||||
# No-op, we're just doing submodule/variant discovery.
|
||||
# Note: All the following rules are safe to run in discovery mode even
|
||||
# though the submodule might not be available as they do not directly depend
|
||||
# on anything from the submodule.
|
||||
elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
|
||||
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
|
||||
message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
|
||||
endif()
|
||||
|
||||
|
|
@ -272,12 +267,29 @@ if(MICROPY_FROZEN_MANIFEST)
|
|||
)
|
||||
endif()
|
||||
|
||||
# Update submodules
|
||||
if(ECHO_SUBMODULES)
|
||||
# If cmake is run with GIT_SUBMODULES defined on command line, process the port / board
|
||||
# settings then print the final GIT_SUBMODULES variable and exit.
|
||||
# Note: the GIT_SUBMODULES is done via echo rather than message, as message splits
|
||||
# the output onto multiple lines
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}")
|
||||
message(FATAL_ERROR "Done")
|
||||
# Update submodules, this is invoked on some ports via 'make submodules'.
|
||||
#
|
||||
# Note: This logic has a Makefile equivalent in py/mkrules.mk
|
||||
if(UPDATE_SUBMODULES AND GIT_SUBMODULES)
|
||||
macro(run_git)
|
||||
execute_process(COMMAND git ${ARGV} WORKING_DIRECTORY ${MICROPY_DIR}
|
||||
RESULT_VARIABLE RES)
|
||||
endmacro()
|
||||
|
||||
list(JOIN GIT_SUBMODULES " " GIT_SUBMODULES_MSG)
|
||||
message("Updating submodules: ${GIT_SUBMODULES_MSG}")
|
||||
run_git(submodule sync ${GIT_SUBMODULES})
|
||||
if(RES EQUAL 0)
|
||||
# If available, do blobless partial clones of submodules to save time and space.
|
||||
# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
|
||||
run_git(submodule update --init --filter=blob:none ${GIT_SUBMODULES})
|
||||
# Fallback to standard submodule update if blobless isn't available (earlier than git 2.36.0)
|
||||
if (NOT RES EQUAL 0)
|
||||
run_git(submodule update --init ${GIT_SUBMODULES})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (NOT RES EQUAL 0)
|
||||
message(FATAL_ERROR "Submodule update failed")
|
||||
endif()
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -262,19 +262,14 @@ endif
|
|||
# If available, do blobless partial clones of submodules to save time and space.
|
||||
# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
|
||||
# Fallback to standard submodule update if blobless isn't available (earlier than 2.36.0)
|
||||
#
|
||||
# Note: This target has a CMake equivalent in py/mkrules.cmake
|
||||
submodules:
|
||||
$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
|
||||
ifneq ($(GIT_SUBMODULES),)
|
||||
$(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES)
|
||||
$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \
|
||||
git submodule update --init $(GIT_SUBMODULES)
|
||||
else
|
||||
ifeq ($(GIT_SUBMODULES_FAIL_IF_EMPTY),1)
|
||||
# If you see this error, it may mean the internal step run by the port's build
|
||||
# system to find git submodules has failed. Double-check dependencies are set correctly.
|
||||
$(ECHO) "Internal build error: The submodule list should not be empty."
|
||||
exit 1
|
||||
endif
|
||||
endif
|
||||
.PHONY: submodules
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue