From d00d0dc6998e41fc118e42b711c90bee214696ed Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Sat, 19 Sep 2026 18:50:17 -0700 Subject: [PATCH] build: retry git clones a few times when they fail I'm pretty fed up with GitHub's reliability. Sometimes a clone just fails, their uptime has been spotty lately. Using our dependency auto-build feature for more packages, more frequently in CI, the opportunities for spurious failures have grown enough that it seems like every CI workflow (with 29 job variants these days) has a good chance of at least one job variant failing. That's a lot of noise in what's supposed to be a reliable signal about whether the PR passes or fails the testsuite on its own merits. So here is me being a little desperte -- I'm adding a CMake function execute_process_with_retry, that we use with the git clone of dependencies to try up to 5 times before a full failure. After the first failure, it waits 15 seconds. The delay doubles with each failure. If all attempts fail, it's a full job failure. Hopefully, any spurious GitHub clone unavailability that lasts less than a few minutes will just be glossed over and let us complete CI, at the cost of perhaps a minute or two delay, which seems like a small price to pay. Assisted by: Claude Code / claude-opus-5 Signed-off-by: Larry Gritz --- src/cmake/dependency_utils.cmake | 105 ++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/src/cmake/dependency_utils.cmake b/src/cmake/dependency_utils.cmake index 85d8ef70b5..7a5a1ead18 100644 --- a/src/cmake/dependency_utils.cmake +++ b/src/cmake/dependency_utils.cmake @@ -47,6 +47,14 @@ endif () set_option (${PROJECT_NAME}_DEPENDENCY_BUILD_ALLOW_UNVERIFIED_TAGS "Allow dependency auto-build to use unverified tags -- Dangerous" OFF) +# Downloads of dependency source sometimes fail for transient reasons (the +# hosting site being briefly unreachable, especially from CI runners), so +# retry a failed download rather than failing the whole build. +set_cache (${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES 5 + "Number of times to retry a failed dependency download" ADVANCED) +set_cache (${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY 15 + "Seconds to wait before the first dependency download retry (doubles each retry)" ADVANCED) + # Track all build deps we find with checked_find_package set (CFP_ALL_BUILD_DEPS_FOUND "") @@ -675,6 +683,80 @@ function(remove_prefixes_from_variable VAR_TYPE VAR_NAME PREFIXES) endif() endfunction() +# execute_process() workalike for a command that can fail for transient +# reasons, such as anything needing network access: connection failures to a +# hosting site are common enough on CI runners that a single attempt is +# unreliable. Wait ${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY seconds +# after a failure, doubling the wait each time, for up to +# ${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES retries before giving up. +# +# Usage: +# execute_process_with_retry (COMMAND [args...] +# [WORKING_DIRECTORY ] +# [CLEANUP ] +# [RESULT_VARIABLE ] +# [ERROR_VARIABLE ] +# [QUIET]) +# +# CLEANUP names a directory holding partial results (such as a half-finished +# clone) to remove before each retry. +# +function (execute_process_with_retry) + cmake_parse_arguments(_epr # prefix + # noValueKeywords: + "QUIET" + # singleValueKeywords: + "WORKING_DIRECTORY;CLEANUP;RESULT_VARIABLE;ERROR_VARIABLE" + # multiValueKeywords: + "COMMAND" + # argsToParse: + ${ARGN}) + + unset (_epr_workdir) + if (_epr_WORKING_DIRECTORY) + set (_epr_workdir WORKING_DIRECTORY ${_epr_WORKING_DIRECTORY}) + endif () + unset (_epr_quiet) + if (_epr_QUIET) + set (_epr_quiet OUTPUT_QUIET) + endif () + + set (_epr_delay ${${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY}) + set (_epr_retries ${${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES}) + set (_epr_tries 0) + while (TRUE) + execute_process (COMMAND ${_epr_COMMAND} + ${_epr_workdir} + RESULT_VARIABLE _epr_result + ERROR_VARIABLE _epr_errors + ERROR_STRIP_TRAILING_WHITESPACE + ${_epr_quiet}) + if (_epr_result EQUAL 0) + break () + endif () + if (_epr_tries GREATER_EQUAL _epr_retries) + break () + endif () + math (EXPR _epr_tries "${_epr_tries} + 1") + message (STATUS "${ColorYellow}Failed: ${_epr_errors}${ColorReset}") + message (STATUS "${ColorYellow}Retrying in ${_epr_delay} seconds " + "(retry ${_epr_tries} of ${_epr_retries})${ColorReset}") + if (_epr_CLEANUP AND EXISTS ${_epr_CLEANUP}) + file (REMOVE_RECURSE ${_epr_CLEANUP}) + endif () + execute_process (COMMAND ${CMAKE_COMMAND} -E sleep ${_epr_delay}) + math (EXPR _epr_delay "${_epr_delay} * 2") + endwhile () + + if (_epr_RESULT_VARIABLE) + set (${_epr_RESULT_VARIABLE} ${_epr_result} PARENT_SCOPE) + endif () + if (_epr_ERROR_VARIABLE) + set (${_epr_ERROR_VARIABLE} "${_epr_errors}" PARENT_SCOPE) + endif () +endfunction () + + # Helper to build a dependency with CMake. Given a package name, git repo and # tag, and optional cmake args, it will clone the repo into the surrounding # project's build area, configures, and build sit, and installs it into a @@ -721,6 +803,7 @@ macro (build_dependency_with_cmake pkgname) unset (${pkgname}_GIT_CLONE_ARGS) unset (_pkg_exec_quiet) + unset (_pkg_retry_quiet) if (NOT "${_pkg_GIT_TAG}" STREQUAL "") # If a tag or branch is specified, do a shallow clone for efficiency. list (APPEND ${pkgname}_GIT_CLONE_ARGS -b ${_pkg_GIT_TAG} --depth 1) @@ -728,6 +811,7 @@ macro (build_dependency_with_cmake pkgname) if (_pkg_QUIET OR "${_pkg_QUIET}" STREQUAL "") list (APPEND ${pkgname}_GIT_CLONE_ARGS -q) set (_pkg_exec_quiet OUTPUT_QUIET) + set (_pkg_retry_quiet QUIET) endif () # Clone the repo if we don't already have it @@ -736,12 +820,16 @@ macro (build_dependency_with_cmake pkgname) message (STATUS "COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} " "${${pkgname}_LOCAL_SOURCE_DIR} " "${${pkgname}_GIT_CLONE_ARGS}") - execute_process(COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} - ${${pkgname}_LOCAL_SOURCE_DIR} - ${${pkgname}_GIT_CLONE_ARGS} - ERROR_VARIABLE ${pkgname}_clone_errors - ${_pkg_exec_quiet}) - if (NOT IS_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR}) + execute_process_with_retry ( + COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} + ${${pkgname}_LOCAL_SOURCE_DIR} + ${${pkgname}_GIT_CLONE_ARGS} + CLEANUP ${${pkgname}_LOCAL_SOURCE_DIR} + RESULT_VARIABLE ${pkgname}_clone_result + ERROR_VARIABLE ${pkgname}_clone_errors + ${_pkg_retry_quiet}) + if (NOT ${pkgname}_clone_result EQUAL 0 + OR NOT IS_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR}) message (FATAL_ERROR "Could not download ${_pkg_GIT_REPOSITORY}: ${${pkgname}_clone_errors}") endif () endif () @@ -794,13 +882,12 @@ macro (build_dependency_with_cmake pkgname) # commits are the ones pinned by the verified superproject commit and # inherit its supply-chain guarantee. if (NOT "${_pkg_GIT_SUBMODULES}" STREQUAL "") - execute_process( + execute_process_with_retry ( COMMAND ${GIT_EXECUTABLE} submodule update --init --depth 1 -- ${_pkg_GIT_SUBMODULES} WORKING_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR} RESULT_VARIABLE _pkg_submodule_result ERROR_VARIABLE _pkg_submodule_errors - ERROR_STRIP_TRAILING_WHITESPACE - ${_pkg_exec_quiet}) + ${_pkg_retry_quiet}) if (NOT _pkg_submodule_result EQUAL 0) message (FATAL_ERROR "${pkgname}: git submodule update failed: ${_pkg_submodule_errors}") endif ()