diff --git a/src/cmake/dependency_utils.cmake b/src/cmake/dependency_utils.cmake index 85d8ef70b5..7a5a1ead18 100644 --- a/src/cmake/dependency_utils.cmake +++ b/src/cmake/dependency_utils.cmake @@ -47,6 +47,14 @@ endif () set_option (${PROJECT_NAME}_DEPENDENCY_BUILD_ALLOW_UNVERIFIED_TAGS "Allow dependency auto-build to use unverified tags -- Dangerous" OFF) +# Downloads of dependency source sometimes fail for transient reasons (the +# hosting site being briefly unreachable, especially from CI runners), so +# retry a failed download rather than failing the whole build. +set_cache (${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES 5 + "Number of times to retry a failed dependency download" ADVANCED) +set_cache (${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY 15 + "Seconds to wait before the first dependency download retry (doubles each retry)" ADVANCED) + # Track all build deps we find with checked_find_package set (CFP_ALL_BUILD_DEPS_FOUND "") @@ -675,6 +683,80 @@ function(remove_prefixes_from_variable VAR_TYPE VAR_NAME PREFIXES) endif() endfunction() +# execute_process() workalike for a command that can fail for transient +# reasons, such as anything needing network access: connection failures to a +# hosting site are common enough on CI runners that a single attempt is +# unreliable. Wait ${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY seconds +# after a failure, doubling the wait each time, for up to +# ${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES retries before giving up. +# +# Usage: +# execute_process_with_retry (COMMAND [args...] +# [WORKING_DIRECTORY ] +# [CLEANUP ] +# [RESULT_VARIABLE ] +# [ERROR_VARIABLE ] +# [QUIET]) +# +# CLEANUP names a directory holding partial results (such as a half-finished +# clone) to remove before each retry. +# +function (execute_process_with_retry) + cmake_parse_arguments(_epr # prefix + # noValueKeywords: + "QUIET" + # singleValueKeywords: + "WORKING_DIRECTORY;CLEANUP;RESULT_VARIABLE;ERROR_VARIABLE" + # multiValueKeywords: + "COMMAND" + # argsToParse: + ${ARGN}) + + unset (_epr_workdir) + if (_epr_WORKING_DIRECTORY) + set (_epr_workdir WORKING_DIRECTORY ${_epr_WORKING_DIRECTORY}) + endif () + unset (_epr_quiet) + if (_epr_QUIET) + set (_epr_quiet OUTPUT_QUIET) + endif () + + set (_epr_delay ${${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRY_DELAY}) + set (_epr_retries ${${PROJECT_NAME}_DEPENDENCY_DOWNLOAD_RETRIES}) + set (_epr_tries 0) + while (TRUE) + execute_process (COMMAND ${_epr_COMMAND} + ${_epr_workdir} + RESULT_VARIABLE _epr_result + ERROR_VARIABLE _epr_errors + ERROR_STRIP_TRAILING_WHITESPACE + ${_epr_quiet}) + if (_epr_result EQUAL 0) + break () + endif () + if (_epr_tries GREATER_EQUAL _epr_retries) + break () + endif () + math (EXPR _epr_tries "${_epr_tries} + 1") + message (STATUS "${ColorYellow}Failed: ${_epr_errors}${ColorReset}") + message (STATUS "${ColorYellow}Retrying in ${_epr_delay} seconds " + "(retry ${_epr_tries} of ${_epr_retries})${ColorReset}") + if (_epr_CLEANUP AND EXISTS ${_epr_CLEANUP}) + file (REMOVE_RECURSE ${_epr_CLEANUP}) + endif () + execute_process (COMMAND ${CMAKE_COMMAND} -E sleep ${_epr_delay}) + math (EXPR _epr_delay "${_epr_delay} * 2") + endwhile () + + if (_epr_RESULT_VARIABLE) + set (${_epr_RESULT_VARIABLE} ${_epr_result} PARENT_SCOPE) + endif () + if (_epr_ERROR_VARIABLE) + set (${_epr_ERROR_VARIABLE} "${_epr_errors}" PARENT_SCOPE) + endif () +endfunction () + + # Helper to build a dependency with CMake. Given a package name, git repo and # tag, and optional cmake args, it will clone the repo into the surrounding # project's build area, configures, and build sit, and installs it into a @@ -721,6 +803,7 @@ macro (build_dependency_with_cmake pkgname) unset (${pkgname}_GIT_CLONE_ARGS) unset (_pkg_exec_quiet) + unset (_pkg_retry_quiet) if (NOT "${_pkg_GIT_TAG}" STREQUAL "") # If a tag or branch is specified, do a shallow clone for efficiency. list (APPEND ${pkgname}_GIT_CLONE_ARGS -b ${_pkg_GIT_TAG} --depth 1) @@ -728,6 +811,7 @@ macro (build_dependency_with_cmake pkgname) if (_pkg_QUIET OR "${_pkg_QUIET}" STREQUAL "") list (APPEND ${pkgname}_GIT_CLONE_ARGS -q) set (_pkg_exec_quiet OUTPUT_QUIET) + set (_pkg_retry_quiet QUIET) endif () # Clone the repo if we don't already have it @@ -736,12 +820,16 @@ macro (build_dependency_with_cmake pkgname) message (STATUS "COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} " "${${pkgname}_LOCAL_SOURCE_DIR} " "${${pkgname}_GIT_CLONE_ARGS}") - execute_process(COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} - ${${pkgname}_LOCAL_SOURCE_DIR} - ${${pkgname}_GIT_CLONE_ARGS} - ERROR_VARIABLE ${pkgname}_clone_errors - ${_pkg_exec_quiet}) - if (NOT IS_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR}) + execute_process_with_retry ( + COMMAND ${GIT_EXECUTABLE} clone ${_pkg_GIT_REPOSITORY} + ${${pkgname}_LOCAL_SOURCE_DIR} + ${${pkgname}_GIT_CLONE_ARGS} + CLEANUP ${${pkgname}_LOCAL_SOURCE_DIR} + RESULT_VARIABLE ${pkgname}_clone_result + ERROR_VARIABLE ${pkgname}_clone_errors + ${_pkg_retry_quiet}) + if (NOT ${pkgname}_clone_result EQUAL 0 + OR NOT IS_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR}) message (FATAL_ERROR "Could not download ${_pkg_GIT_REPOSITORY}: ${${pkgname}_clone_errors}") endif () endif () @@ -794,13 +882,12 @@ macro (build_dependency_with_cmake pkgname) # commits are the ones pinned by the verified superproject commit and # inherit its supply-chain guarantee. if (NOT "${_pkg_GIT_SUBMODULES}" STREQUAL "") - execute_process( + execute_process_with_retry ( COMMAND ${GIT_EXECUTABLE} submodule update --init --depth 1 -- ${_pkg_GIT_SUBMODULES} WORKING_DIRECTORY ${${pkgname}_LOCAL_SOURCE_DIR} RESULT_VARIABLE _pkg_submodule_result ERROR_VARIABLE _pkg_submodule_errors - ERROR_STRIP_TRAILING_WHITESPACE - ${_pkg_exec_quiet}) + ${_pkg_retry_quiet}) if (NOT _pkg_submodule_result EQUAL 0) message (FATAL_ERROR "${pkgname}: git submodule update failed: ${_pkg_submodule_errors}") endif ()