From 09698aa2810bf036802eb664af685fea83bc934f Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 14:13:15 -0500 Subject: [PATCH 01/28] Added simple README-build-in-container.md explaining how to work inside the container Ticket: ENT-12599 Changelog: none --- README-build-in-container.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 README-build-in-container.md diff --git a/README-build-in-container.md b/README-build-in-container.md new file mode 100644 index 000000000..d8e0b3fdd --- /dev/null +++ b/README-build-in-container.md @@ -0,0 +1,8 @@ +Build with ./build-in-container.py + +Using ./build-in-container.py --shell you can run the build interactively and debug issues. +Start in the container by running /srv/source/buildscripts/build-in-container-inner.sh + +This will copy repository sources that are needed from the read-only /srv location to read-write work area in /home/builder/build. + +Continue debugging by running steps in /home/builder/buildscripts/build-scripts/0*.sh From a2521d6fdf03be4e1cf784c315eb9f92552c1f91 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 14:22:13 -0500 Subject: [PATCH 02/28] Added 0000-system-dependencies step to calls setup-cfengine-build-host.sh This also required adding /etc/cfengine-in-container.flag for the setup script to operate properly. Ticket: ENT-12599 Changelog: none --- build-scripts/0000-system-dependencies | 5 +++++ container/Dockerfile.debian | 3 +++ container/Dockerfile.mingw | 3 +++ container/Dockerfile.rhel | 3 +++ container/Dockerfile.rhel7 | 3 +++ container/Dockerfile.tarballs | 4 ++++ 6 files changed, 21 insertions(+) create mode 100755 build-scripts/0000-system-dependencies diff --git a/build-scripts/0000-system-dependencies b/build-scripts/0000-system-dependencies new file mode 100755 index 000000000..8a6db48de --- /dev/null +++ b/build-scripts/0000-system-dependencies @@ -0,0 +1,5 @@ +#!/bin/sh + +set -ex +thisdir="$(dirname "$0")" +sudo bash "$thisdir"/../ci/setup-cfengine-build-host.sh diff --git a/container/Dockerfile.debian b/container/Dockerfile.debian index bc139e0f9..ee9be9321 100644 --- a/container/Dockerfile.debian +++ b/container/Dockerfile.debian @@ -44,6 +44,9 @@ RUN /tmp/linux-install-rust.sh RUN useradd -m -s /bin/bash builder \ && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder +# let setup-cfengine-build-host.sh know that we are in a container +RUN touch /etc/cfengine-in-container.flag + USER builder WORKDIR /home/builder diff --git a/container/Dockerfile.mingw b/container/Dockerfile.mingw index a8b9faec9..7982b5fd7 100644 --- a/container/Dockerfile.mingw +++ b/container/Dockerfile.mingw @@ -38,6 +38,9 @@ RUN /tmp/linux-install-rust.sh x86_64-pc-windows-gnu RUN useradd -m -s /bin/bash builder \ && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder +# Let setup-cfengine-build-host.sh know we are in a container +RUN touch /etc/cfengine-in-container.flag + USER builder WORKDIR /home/builder diff --git a/container/Dockerfile.rhel b/container/Dockerfile.rhel index 12d5a2179..4d82a419d 100644 --- a/container/Dockerfile.rhel +++ b/container/Dockerfile.rhel @@ -59,6 +59,9 @@ RUN if [ -n "${EXTRA_PKGS}" ]; then dnf install -y ${EXTRA_PKGS} && dnf clean al RUN useradd -m -s /bin/bash builder \ && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder +# let setup-cfengine-build-host.sh know we are in a container +RUN touch /etc/cfengine-in-container.flag + USER builder WORKDIR /home/builder diff --git a/container/Dockerfile.rhel7 b/container/Dockerfile.rhel7 index eb61542af..cadf71d8d 100644 --- a/container/Dockerfile.rhel7 +++ b/container/Dockerfile.rhel7 @@ -51,6 +51,9 @@ RUN /tmp/linux-install-rust.sh RUN useradd -m -s /bin/bash builder \ && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder +# let setup-cfengine-build-host.sh know we are in a container +RUN touch /etc/cfengine-in-container.flag + USER builder WORKDIR /home/builder diff --git a/container/Dockerfile.tarballs b/container/Dockerfile.tarballs index 0a06d2121..c040039e2 100644 --- a/container/Dockerfile.tarballs +++ b/container/Dockerfile.tarballs @@ -28,6 +28,10 @@ RUN apt-get -qy update && apt-get -y install \ RUN useradd -m -s /bin/bash builder \ && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder +# let setup-cfengine-build-host.sh know we are in a container and bootstrapping +RUN touch /etc/cfengine-in-container.flag +RUN touch /etc/cfengine-bootstrap-pr-host.flag + USER builder WORKDIR /home/builder From 5b64fc6600bfbe12f4c03a91118590151ede79ef Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 14:23:31 -0500 Subject: [PATCH 03/28] sq --- build-in-container-inner.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index a77d0d2af..155bb3792 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -4,6 +4,9 @@ set -e # Configuration via environment variables: # PROJECT, BUILD_TYPE, EXPLICIT_ROLE, BUILD_NUMBER, EXPLICIT_VERSION +# let setup-cfengine-build-host.sh know we are in a container +sudo touch /etc/cfengine-in-container.flag + BASEDIR=/home/builder/build export BASEDIR export AUTOBUILD_PATH="$BASEDIR/buildscripts" From 0f0c05f37932c7a39d8a328026e5359759587bee Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 14:25:49 -0500 Subject: [PATCH 04/28] Adjusted build scripts to support an OUTPUT environment variable Ticket: ENT-12599 Changelog: none --- build-in-container-inner.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index 155bb3792..6b1607cf4 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -10,12 +10,14 @@ sudo touch /etc/cfengine-in-container.flag BASEDIR=/home/builder/build export BASEDIR export AUTOBUILD_PATH="$BASEDIR/buildscripts" +OUTPUT=/output +export OUTPUT mkdir -p "$BASEDIR" # Bind-mounted directories may be owned by the host user's UID. # Fix ownership so builder can write to them. -sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" /output +sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" "$OUTPUT" # And hand ownership back to the host user on the way out. if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then @@ -130,7 +132,7 @@ install_mission_portal_deps() ( # Lets whoever consumes the output check that it arrived intact. Sorted in the C # locale so that the list itself comes out the same every time. write_sha256sums() ( - cd /output + cd "$OUTPUT" # shellcheck disable=SC2094 # > Make sure not to read and write the same file in the same pipeline. # find leaves it out by name, so the list never covers itself. @@ -228,4 +230,4 @@ write_sha256sums echo "" echo "=== Build complete ===" -ls -lh /output/ +ls -lh "$OUTPUT"/ From ee937b7d65da158719835be3257bdd7016ee8ff5 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 14:35:18 -0500 Subject: [PATCH 05/28] Added numbered symlinks for build steps and adjustments to enable running them in sequence Ticket: ENT-12599 Changelog: none --- build-in-container-inner.sh | 115 +----------- build-scripts/0005-repositories | 1 + build-scripts/0010-autogen | 1 + build-scripts/0020-clean-buildmachine | 1 + build-scripts/0030-bootstrap-mission-portal | 1 + build-scripts/0040-build-tarballs | 1 + build-scripts/0050-unpack-tarballs | 1 + build-scripts/0060-install-dependencies | 1 + build-scripts/0070-configure | 1 + build-scripts/0080-generate-source-tarballs | 1 + build-scripts/0090-compile | 1 + build-scripts/0100-produce-debug-symbols | 1 + build-scripts/0110-package | 1 + build-scripts/0120-prepare-results | 1 + build-scripts/0130-test | 1 + build-scripts/0140-prepare-results | 1 + build-scripts/autogen | 14 +- build-scripts/bootstrap-mission-portal | 55 ++++++ build-scripts/bootstrap-tarballs | 189 +------------------- build-scripts/build-tarballs | 70 ++++++++ build-scripts/compare-versions | 4 + build-scripts/create-empty-test | 1 + build-scripts/functions | 4 +- build-scripts/generate-source-tarballs | 4 +- build-scripts/repositories | 19 ++ build-scripts/revision-file | 4 +- build-scripts/test | 8 +- build-scripts/unpack-tarballs | 10 ++ ci/setup-cfengine-build-host.sh | 7 +- ci/setup-ci-host.sh | 13 +- 30 files changed, 226 insertions(+), 306 deletions(-) create mode 120000 build-scripts/0005-repositories create mode 120000 build-scripts/0010-autogen create mode 120000 build-scripts/0020-clean-buildmachine create mode 120000 build-scripts/0030-bootstrap-mission-portal create mode 120000 build-scripts/0040-build-tarballs create mode 120000 build-scripts/0050-unpack-tarballs create mode 120000 build-scripts/0060-install-dependencies create mode 120000 build-scripts/0070-configure create mode 120000 build-scripts/0080-generate-source-tarballs create mode 120000 build-scripts/0090-compile create mode 120000 build-scripts/0100-produce-debug-symbols create mode 120000 build-scripts/0110-package create mode 120000 build-scripts/0120-prepare-results create mode 120000 build-scripts/0130-test create mode 120000 build-scripts/0140-prepare-results create mode 100755 build-scripts/bootstrap-mission-portal mode change 100755 => 100644 build-scripts/bootstrap-tarballs create mode 100755 build-scripts/build-tarballs create mode 100755 build-scripts/repositories diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index 6b1607cf4..538d8da7a 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -90,45 +90,6 @@ fi export SOURCE_DATE_EPOCH echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" -install_mission_portal_deps() ( - set -e - - if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then - echo "Installing npm dependencies..." - npm ci --prefix "$BASEDIR/mission-portal/public/scripts/" - echo "Building react components..." - npm run build --prefix "$BASEDIR/mission-portal/public/scripts/" - rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules" - fi - - if [ -f "$BASEDIR/mission-portal/composer.json" ]; then - echo "Installing Mission Portal PHP dependencies..." - (cd "$BASEDIR/mission-portal" && composer install --no-dev --ignore-platform-reqs --prefer-dist) - fi - - if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then - echo "Installing Nova API PHP dependencies..." - (cd "$BASEDIR/nova/api/http" && composer install --no-dev --ignore-platform-reqs --prefer-dist) - fi - - if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then - echo "Compiling Mission Portal styles..." - mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css" - (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" && - lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css) - fi - - if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then - echo "Installing LDAP API PHP dependencies..." - (cd "$BASEDIR/mission-portal/ldap" && composer install --no-dev --ignore-platform-reqs --prefer-dist) - fi - - # Composer falls back to git clone when GitHub's anonymous zipball - # rate limit is hit, leaving non-reproducible .git directories in the - # vendor tree. Strip them. - find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} + -) - # Lets whoever consumes the output check that it arrived intact. Sorted in the C # locale so that the list itself comes out the same every time. write_sha256sums() ( @@ -140,50 +101,6 @@ write_sha256sums() ( | LC_ALL=C sort | xargs -r sha256sum > sha256sums.txt ) -# Build the source tarballs. They are the same whichever platform builds them, -# so only this image builds them, and nothing else here does. /output is -# /tarballs on the host, as the packages' /output is per label. -# -# Each tarball's timestamps follow its own repository: Makefile.am in core and in -# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking -# it from the last commit keeps a tarball identical until its own sources change. -build_tarballs() ( - set -e - - ( - cd "$BASEDIR/core" - SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) - export SOURCE_DATE_EPOCH - echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" - - rm -f cfengine-3.*.tar.gz - # Configure so the dist target exists, undone again below. - ./configure -C - make dist - mv cfengine-3.*.tar.gz /output/ - make distclean - ) - - ( - cd "$BASEDIR/masterfiles" - SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) - export SOURCE_DATE_EPOCH - echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" - - rm -f cfengine-masterfiles*.tar.gz - ./configure - make dist # source tarball: cfengine-masterfiles-.tar.gz - make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz - mv cfengine-masterfiles*.tar.gz /output/ - make distclean - ) - - write_sha256sums -) - -# === Step runner with failure reporting === -# Disable set -e so we can capture exit codes and report which step failed. -set +e run_step() { local name="$1" shift @@ -198,33 +115,13 @@ run_step() { } # === Build steps === -run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen" - -if [ "$TARBALLS" = yes ]; then - run_step "02-tarballs" build_tarballs - echo "" - echo "=== Build complete ===" - ls -lh /output/ - exit 0 -fi +NO_TESTS=true +export NO_TESTS -run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies" -# Mission Portal is an Enterprise/nova-only component; its sources are only -# synced when PROJECT=nova. Skip this step for community hubs. -if [ "$PROJECT" = "nova" ] && [ "$EXPLICIT_ROLE" = "hub" ]; then - run_step "03-mission-portal-deps" install_mission_portal_deps -fi -run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure" -run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile" -run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package" - -# === Copy output packages === -# Packages are created under $BASEDIR// by dpkg-buildpackage / rpmbuild. -# Exclude deps-packaging to avoid copying dependency packages. -find "$BASEDIR" -maxdepth 4 \ - -path "$BASEDIR/buildscripts/deps-packaging" -prune -o \ - \( -name '*.deb' -o -name '*.rpm' -o -name '*.msi' -o -name '*.pkg.tar.gz' \) -print \ - -exec cp {} /output/ \; +for script in "$BASEDIR/buildscripts/build-scripts"/0*; do + name="$(basename "$script")" + run_step "$name" "$script" +done write_sha256sums diff --git a/build-scripts/0005-repositories b/build-scripts/0005-repositories new file mode 120000 index 000000000..e69f3dc94 --- /dev/null +++ b/build-scripts/0005-repositories @@ -0,0 +1 @@ +repositories \ No newline at end of file diff --git a/build-scripts/0010-autogen b/build-scripts/0010-autogen new file mode 120000 index 000000000..563ea1500 --- /dev/null +++ b/build-scripts/0010-autogen @@ -0,0 +1 @@ +autogen \ No newline at end of file diff --git a/build-scripts/0020-clean-buildmachine b/build-scripts/0020-clean-buildmachine new file mode 120000 index 000000000..3e44a57b8 --- /dev/null +++ b/build-scripts/0020-clean-buildmachine @@ -0,0 +1 @@ +clean-buildmachine \ No newline at end of file diff --git a/build-scripts/0030-bootstrap-mission-portal b/build-scripts/0030-bootstrap-mission-portal new file mode 120000 index 000000000..da139a1d1 --- /dev/null +++ b/build-scripts/0030-bootstrap-mission-portal @@ -0,0 +1 @@ +bootstrap-mission-portal \ No newline at end of file diff --git a/build-scripts/0040-build-tarballs b/build-scripts/0040-build-tarballs new file mode 120000 index 000000000..0d1b19ec2 --- /dev/null +++ b/build-scripts/0040-build-tarballs @@ -0,0 +1 @@ +build-tarballs \ No newline at end of file diff --git a/build-scripts/0050-unpack-tarballs b/build-scripts/0050-unpack-tarballs new file mode 120000 index 000000000..f4e734e68 --- /dev/null +++ b/build-scripts/0050-unpack-tarballs @@ -0,0 +1 @@ +unpack-tarballs \ No newline at end of file diff --git a/build-scripts/0060-install-dependencies b/build-scripts/0060-install-dependencies new file mode 120000 index 000000000..365df31b2 --- /dev/null +++ b/build-scripts/0060-install-dependencies @@ -0,0 +1 @@ +install-dependencies \ No newline at end of file diff --git a/build-scripts/0070-configure b/build-scripts/0070-configure new file mode 120000 index 000000000..cabc7046d --- /dev/null +++ b/build-scripts/0070-configure @@ -0,0 +1 @@ +configure \ No newline at end of file diff --git a/build-scripts/0080-generate-source-tarballs b/build-scripts/0080-generate-source-tarballs new file mode 120000 index 000000000..ae7ba0935 --- /dev/null +++ b/build-scripts/0080-generate-source-tarballs @@ -0,0 +1 @@ +generate-source-tarballs \ No newline at end of file diff --git a/build-scripts/0090-compile b/build-scripts/0090-compile new file mode 120000 index 000000000..f8808babb --- /dev/null +++ b/build-scripts/0090-compile @@ -0,0 +1 @@ +compile \ No newline at end of file diff --git a/build-scripts/0100-produce-debug-symbols b/build-scripts/0100-produce-debug-symbols new file mode 120000 index 000000000..017abf496 --- /dev/null +++ b/build-scripts/0100-produce-debug-symbols @@ -0,0 +1 @@ +produce-debug-symbols \ No newline at end of file diff --git a/build-scripts/0110-package b/build-scripts/0110-package new file mode 120000 index 000000000..597acc768 --- /dev/null +++ b/build-scripts/0110-package @@ -0,0 +1 @@ +package \ No newline at end of file diff --git a/build-scripts/0120-prepare-results b/build-scripts/0120-prepare-results new file mode 120000 index 000000000..b1b48550e --- /dev/null +++ b/build-scripts/0120-prepare-results @@ -0,0 +1 @@ +prepare-results \ No newline at end of file diff --git a/build-scripts/0130-test b/build-scripts/0130-test new file mode 120000 index 000000000..30d74d258 --- /dev/null +++ b/build-scripts/0130-test @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/build-scripts/0140-prepare-results b/build-scripts/0140-prepare-results new file mode 120000 index 000000000..b1b48550e --- /dev/null +++ b/build-scripts/0140-prepare-results @@ -0,0 +1 @@ +prepare-results \ No newline at end of file diff --git a/build-scripts/autogen b/build-scripts/autogen index 236f30b25..7ba0dbef3 100755 --- a/build-scripts/autogen +++ b/build-scripts/autogen @@ -18,6 +18,10 @@ # ``` # +# note that this script should be a no-op if an artifacts.tgz is present +thisdir="$(dirname "$0")" +bash "$thisdir"/../ci/setup-ci-host.sh --bootstrap + # Get the BASEDIR variable holding the path to where our repos are checked out . "$(dirname "$0")/functions" @@ -61,11 +65,15 @@ done # Run autogen.sh on each repository for proj in $projects; do # autogen.sh is quite verbose, so only print the output in case of failure - log_debug "Running autogen.sh for project $proj..." ( cd "$BASEDIR/$proj" - export NO_CONFIGURE=1 - run_and_print_on_failure ./autogen.sh + if [ ! -f configure ] || [ ! -f CFVERSION ]; then + log_debug "Running autogen.sh for project $proj..." + export NO_CONFIGURE=1 + run_and_print_on_failure ./autogen.sh + else + log_debug "Skipping autogen.sh in $proj as configure and/or CFVERSION files already exist" + fi ) done diff --git a/build-scripts/bootstrap-mission-portal b/build-scripts/bootstrap-mission-portal new file mode 100755 index 000000000..c59b7063d --- /dev/null +++ b/build-scripts/bootstrap-mission-portal @@ -0,0 +1,55 @@ +. "$(dirname "$0")"/functions +. detect-environment +. compile-options +. version + set -e + +if [ "$ROLE" != "hub" ]; then + echo "ROLE is not hub so skipping $0" + exit 0 +fi + +if command -v composer >/dev/null; then + COMPOSER=$(command -v composer) +elif [ -f /usr/local/bin/composer.phar ]; then + COMPOSER=/usr/local/bin/composer.phar +else + echo "Error: could not find composer command in PATH or at /usr/local/bin/composer.phar" + exit 1 +fi + + +if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then + echo "Installing npm dependencies..." + npm ci --prefix "$BASEDIR/mission-portal/public/scripts/" + echo "Building react components..." + npm run build --prefix "$BASEDIR/mission-portal/public/scripts/" + rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules" +fi + +if [ -f "$BASEDIR/mission-portal/composer.json" ]; then + echo "Installing Mission Portal PHP dependencies..." + (cd "$BASEDIR/mission-portal" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist) +fi + +if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then + echo "Installing Nova API PHP dependencies..." + (cd "$BASEDIR/nova/api/http" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist) +fi + +if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then + echo "Compiling Mission Portal styles..." + mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css" + (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" && + lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css) +fi + +if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then + echo "Installing LDAP API PHP dependencies..." + (cd "$BASEDIR/mission-portal/ldap" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist) +fi + +# Composer falls back to git clone when GitHub's anonymous zipball +# rate limit is hit, leaving non-reproducible .git directories in the +# vendor tree. Strip them. +find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} + diff --git a/build-scripts/bootstrap-tarballs b/build-scripts/bootstrap-tarballs old mode 100755 new mode 100644 index 4a27d3009..aac9fbe3f --- a/build-scripts/bootstrap-tarballs +++ b/build-scripts/bootstrap-tarballs @@ -1,184 +1,7 @@ -#!/bin/bash - -# This script is supposed generate tarballs from the core & masterfiles -# repositories. We also use these tarballs later in the build process to make -# sure that they actually work. -# -# Currently this script does a lot more than bootstrap tarballs. -# It also fetches pull request info and installs PHP and javascript dependencies. -# The history behind this is that this script is run early in the bootstrap-pr job, -# and these extra steps effectively saves compute time for subsequent hub builds. -# Maybe we should rename it to only bootstrap (see ENT-13064). -# -# You will first need to run the autogen script. E.g.: -# PROJECT=community ./buildscripts/build-scripts/autogen -# Then you can run it like this: -# BUILD_TYPE=DEBUG ./buildscripts/build-scripts/bootstrap-tarballs -# -# The script expects the following repositories to be side by side: -# . -# ├── buildscripts -# ├── core -# ├── enterprise -# ├── nova -# ├── mission-portal -# ├── libntech -# └── masterfiles -# -# ^ When building community you won't need enterprise, nova, mission-portal -# - -_dir=$(readlink -e "$(dirname "$0")") -# refactored a few functions into single file scripts for easier development/debugging, see ENT-12741 and ENT-12595 -# Easier to add a path to a script than source a file of functions. -export PATH="$_dir"/bin:$PATH -. "$(dirname "$0")"/functions -. detect-environment -. compile-options -. version - -mkdir -p "$BASEDIR"/output/tarballs - -# the first part of the script is not really critical -set +e - -# Get information about PRs among the used revisions. -# These PRs will have to be notified of build progress. -# -# Variables such as MISSION_PORTAL_REV may be set by the CI (Jenkins) to build and test multiple PR's together. -# The variables typically hold a branch name or a pull request ID. -# PR IDs can also be preceded by 'pull' or 'origin/pull'. -# E.g.: -# - MISSION_PORTAL_REV=pull/1755 -# - MISSION_PORTAL_REV=origin/pull/1755 -# where the trailing number is the pull request ID. -# -# Furthermore, they can be suffixed by anything after a subsequent slash (/). -# We usually use: -# - ID alone -# - pull/ID/head -# - pull/ID/merge -# -# This loop fetches information about the PRs if the respective variable is set. -# -for repo_spec in cfengine/buildscripts cfengine/core cfengine/masterfiles cfengine/enterprise cfengine/nova cfengine/mission-portal NorthernTechHQ/libntech; do - # remove organization/ from start of repo_spec - # E.g. 'cfengine/mission-portal' -> 'mission-portal' - repo="${repo_spec#*/}" - - # Convert to uppercase, swap hyphens with underscore and append '_REV' - # E.g. 'mission-portal' -> 'MISSION_PORTAL_REV' - rev_param_name="$(echo "$repo" | tr '[:lower:]-' '[:upper:]_')_REV" - - # Try to dereference the result from above and skip the rest of the loop - # unless the variable is defined. - revision="${!rev_param_name}" || continue - - # remove "origin/" (if any) - revision="${revision##origin/}" - - # Check to see if the resolved variable starts with 'pull/' - if expr "$revision" : "pull/" >/dev/null; then - # Extract the revision number. E.g. 'pull/1755' -> '1755' - pr_nr="$(echo "$revision" | cut -d/ -f2)" - - get-github-pull-request-info "$repo_spec" "$pr_nr" >>"$BASEDIR"/output/PRs - fi -done - -# now script failures should fail the script +#!/usr/bin/env bash +# legacy bootstrap-tarballs was split into bootstrap and bootstrap-mission-portal +# todo: remove this script when it is no longer called from anywhere set -e - -# Build tarball from core repository -cd "$BASEDIR"/core -rm -f cfengine-3.*.tar.gz -git rev-parse HEAD >"$BASEDIR"/output/core-commitID -# Configure in order to run "make dist", deleted later. -log_debug "Running configure on core repository..." -run_and_print_on_failure ./configure -C -# Normalize source timestamps to avoid errors like: -# configure: error: newly created file is older than distributed files! -find . -exec touch -t 202501010000.00 {} + -log_debug "Running make dist on core repository..." -run_and_print_on_failure make dist -mv cfengine-3.*.tar.gz "$BASEDIR"/output/tarballs/ -log_debug "Running make distclean on core repository..." -run_and_print_on_failure make distclean - -# Build tarballs from masterfiles repository -cd "$BASEDIR"/masterfiles -rm -f cfengine-masterfiles*.tar.gz -git rev-parse HEAD >"$BASEDIR"/output/masterfiles-commitID -# Configure in order to run "make dist", deleted later. -log_debug "Running configure on masterfiles repository..." -run_and_print_on_failure ./configure -# Normalize source timestamps to avoid errors like: -# configure: error: newly created file is older than distributed files! -find . -exec touch -t 202501010000.00 {} + -log_debug "Running make dist on masterfiles repository..." -run_and_print_on_failure make dist # source tarball -log_debug "Running make tar-package on masterfiles repository..." -run_and_print_on_failure make tar-package # package tarball (containing all files as if they were installed under "prefix".) -mv cfengine-masterfiles*.tar.gz "$BASEDIR"/output/tarballs/ -log_debug "Running make distclean on masterfiles repository..." -run_and_print_on_failure make distclean - -# Compute a checksum list that can be used to verify the integrity of the -# tarballs -cd "$BASEDIR"/output/tarballs -sha256sum -- *.tar.gz >sha256sums.txt -# Add the BSD (16-bit) checksum of the checksum list to it's filename. This way -# you can verify the integrity of the checksum list itself. -CKSUM=$(sum sha256sums.txt | cut -d ' ' -f 1) -mv sha256sums.txt sha256sums."$CKSUM".txt - -log_debug "Installing javascript npm dependencies..." -( - if test -f "$BASEDIR"/mission-portal/public/scripts/package.json; then - cd "$BASEDIR"/mission-portal/public/scripts - # display node & npm versions - npm --version - node --version - # install dependencies from npmjs - run_and_print_on_failure npm ci --prefix "$BASEDIR"/mission-portal/public/scripts/ - # build react components - run_and_print_on_failure npm run build --prefix "$BASEDIR"/mission-portal/public/scripts/ - # remove node_modules since the bundles are already built - run_and_print_on_failure rm -rf "$BASEDIR"/mission-portal/public/scripts/node_modules - fi -) - -log_debug "Installing PHP composer dependencies from mission-portal repository..." -( - if test -f "$BASEDIR"/mission-portal/composer.json; then - cd "$BASEDIR"/mission-portal - # install PHP dependencies from composer - run_and_print_on_failure php /usr/bin/composer install --no-dev - fi -) - -log_debug "Installing PHP composer dependencies from nova repository..." -( - if test -f "$BASEDIR"/nova/api/http/composer.json; then - cd "$BASEDIR"/nova/api/http - # install PHP dependencies from composer - run_and_print_on_failure php /usr/bin/composer install --no-dev --ignore-platform-reqs - fi -) - -log_debug "Compiling Mission Portal styles..." -( - if test -f "$BASEDIR"/mission-portal/public/themes/default/bootstrap/cfengine_theme.less; then - cd "$BASEDIR"/mission-portal/public/themes/default/bootstrap - run_and_print_on_failure npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css - fi -) - -log_debug "Installing LDAP API PHP composer dependencies..." -( - if test -f "$BASEDIR"/mission-portal/ldap/composer.json; then - cd "$BASEDIR"/mission-portal/ldap - # install PHP dependencies from composer - run_and_print_on_failure php /usr/bin/composer install --no-dev - fi -) +thisdir="$(dirname "$0")" +bash "$thisdir"/bootstrap +bash "$thisdir"/bootstrap-mission-portal diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs new file mode 100755 index 000000000..c7f3406bf --- /dev/null +++ b/build-scripts/build-tarballs @@ -0,0 +1,70 @@ +#!/bin/sh + +# +# This script builds tarballs for core and masterfiles for later use. +# +# Build the source tarballs. They are the same whichever platform builds them, +# so only this image builds them, and nothing else here does. +# +# This script looks for two optional env vars: +# - TARBALLS - if yes, then run this script, otherwise do not. The default is to not run this script. + this variable is provided by build-in-container.py +# - OUTPUT - optional, for container builds this should be /output. The default is /output +# +# Each tarball's timestamps follow its own repository: Makefile.am in core and in +# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking +# it from the last commit keeps a tarball identical until its own sources change. + +# The script expects the following repositories to exist side by side: +# . +# ├── buildscripts +# ├── core +# └── masterfiles +# The script can be run as described in the usage message below: + +usage() { + echo "Usage: [TARBALLS=yes] $0" +} + +. "$(dirname "$0")"/functions +. detect-environment +. compile-options +. version +set -e + +if [ -z "$TARBALLS" ] || [ "$TARBALLS" != "yes" ]; then + echo "TARBALLS env variable did not have a value of yes, so skipping $0" + exit 0 +fi + +if [ -x "$OUTPUT" ]; then + OUTPUT=output +fi + +( + cd "$BASEDIR/core" + SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) + export SOURCE_DATE_EPOCH + echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + + rm -f cfengine-3.*.tar.gz + # Configure so the dist target exists, undone again below. + ./configure -C + make dist + mv cfengine-3.*.tar.gz "$OUTPUT" + make distclean +) + +( + cd "$BASEDIR/masterfiles" + SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) + export SOURCE_DATE_EPOCH + echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + + rm -f cfengine-masterfiles*.tar.gz + ./configure + make dist # source tarball: cfengine-masterfiles-.tar.gz + make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz + mv cfengine-masterfiles*.tar.gz "$OUTPUT" + make distclean +) diff --git a/build-scripts/compare-versions b/build-scripts/compare-versions index c17ba4e6c..a0c0d5d75 100755 --- a/build-scripts/compare-versions +++ b/build-scripts/compare-versions @@ -54,6 +54,10 @@ fi # themselves, however, in my opinion it reads better this way) for proj_i in $projects; do for proj_j in $projects; do + if [ ! -f "$BASEDIR/$proj_i/CFVERSION" ]; then + echo "Missing CFVERSION file in $BASEDIR/$proj_i" + exit 1 + fi # The CFVERSION file is read into the tr command which removes # whitespace and the sed command extracts the major, minor & patch # version number from the string. diff --git a/build-scripts/create-empty-test b/build-scripts/create-empty-test index e7905f59c..8e5f52dbf 100755 --- a/build-scripts/create-empty-test +++ b/build-scripts/create-empty-test @@ -5,6 +5,7 @@ # test report saying that there were no tests to report. for repo in core nova; do + mkdir -p "$BASEDIR/$repo/tests/unit" cat <"$BASEDIR/$repo/tests/unit/no_tests.xml" diff --git a/build-scripts/functions b/build-scripts/functions index 769b8f55c..95dbd98a3 100644 --- a/build-scripts/functions +++ b/build-scripts/functions @@ -75,8 +75,10 @@ export_variables() { esac AUTOBUILD_PATH=$(dirname "$SCRIPTDIR") - + else + SCRIPTDIR="$AUTOBUILD_PATH"/build-scripts fi + export SCRIPTDIR BASEDIR=$(dirname "$AUTOBUILD_PATH") export BASEDIR diff --git a/build-scripts/generate-source-tarballs b/build-scripts/generate-source-tarballs index e99f6cc67..4c3271a86 100755 --- a/build-scripts/generate-source-tarballs +++ b/build-scripts/generate-source-tarballs @@ -26,9 +26,11 @@ . detect-environment . compile-options +cd "$BASEDIR" + for repo in core masterfiles; do log_debug "Running target make dist in $repo repository..." - (cd $repo && run_and_print_on_failure "$MAKE" V=1 dist) + (cd "$repo" && run_and_print_on_failure "$MAKE" V=1 dist) done # Copy to the directory that's being uploaded to buildcache diff --git a/build-scripts/repositories b/build-scripts/repositories new file mode 100755 index 000000000..2ee171071 --- /dev/null +++ b/build-scripts/repositories @@ -0,0 +1,19 @@ +set -ex +. "$(dirname "$0")"/functions + +REPOS="core masterfiles" +if test "$PROJECT" = "nova"; then + REPOS="$REPOS enterprise" +fi +if [ "$PROJECT" = "nova" ] && [ "$EXPLICIT_ROLE" = "hub" ]; then + REPOS="$REPOS nova mission-portal" +fi + +cd "$BASEDIR" + +for repo in $REPOS; do + if [ ! -d "$BASEDIR/$repo" ]; then + echo "cloning $repo at pwd $(pwd)" + git clone git@github.com:cfengine/$repo --recursive + fi +done diff --git a/build-scripts/revision-file b/build-scripts/revision-file index c3445f0af..6d35edec5 100755 --- a/build-scripts/revision-file +++ b/build-scripts/revision-file @@ -79,8 +79,8 @@ for _dir in $_dirs; do echo "$(basename "$0"): Debug: Revision file already exists in $_dir" fi else - echo "$(basename "$0"): Error: Expected to find the '$_dir' directory in '$BASEDIR', but it's not there" - exit 1 + # just a warning below, to enable just building with buildscripts for dependencies only without any other repositories like core, etc. + echo "$(basename "$0"): Warning: Expected to find the '$_dir' directory in '$BASEDIR', but it's not there" fi done diff --git a/build-scripts/test b/build-scripts/test index 524b67074..114e4c1f1 100755 --- a/build-scripts/test +++ b/build-scripts/test @@ -4,14 +4,18 @@ . detect-environment . compile-options -# $NO_TEST and $NO_ACCEPTANCE_TESTS are usually set by Jenkins +# NO_TESTS and NO_ACCEPTANCE_TESTS are usually set by Jenkins if [ "true" = "$NO_TESTS" ] || [ "true" = "$NO_ACCEPTANCE_TESTS" ]; then # In order to keep Jenkins happy, we create an empty test report saying that there were no tests to report. - log_debug "Creating empty test (\$NO_TESTS=$NO_TEST, \$NO_ACCEPTANCE_TESTS=$NO_ACCEPTANCE_TESTS)" + log_debug "Creating empty test (\$NO_TESTS=$NO_TESTS, \$NO_ACCEPTANCE_TESTS=$NO_ACCEPTANCE_TESTS)" create-empty-test exit 0 fi +# pre-reqs during a build, clean all packages and re-install from packages built in $HOME/.cache probably +"$BASEDIR"/buildscripts/build-scripts/clean-buildmachine +"$BASEDIR"/buildscripts/build-scripts/install-dependencies + if [ -n "$TEST_MACHINE" ]; then # Prepares the test machine by creating a chroot environment. log_debug "Preparing test machine" diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index bb67aa9af..2ba83143e 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -28,9 +28,19 @@ . detect-environment . compile-options +if [ ! -d "$BASEDIR"/output/tarballs ]; then + echo "No $BASEDIR/output/tarballs so skipping $0" + exit 0 +fi + SOURCE_TARBALL="$BASEDIR/output/tarballs/cfengine-3.*.tar.gz" MASTERFILES_TARBALL=$(find "$BASEDIR"/output/tarballs/ -name 'cfengine-masterfiles*.tar.gz' -a ! \( -name '*pkg.tar.gz' \)) +if [ ! -f "$SOURCE_TARBALL" ] || [ ! -f "$MASTERFILES_TARBALL" ]; then + echo "Did not find SOURCE_TARBALL: $SOURCE_TARBALL or MASTERFILES_TARBALL: $MASTERFILES_TARBALL so skipping $0 script" + exit 0 +fi + # DELETE the git-checked-out directories, they are tainted with # ./configure artifacts anyway. The tarballs are unpacked and symlinked # into place. That way unpacking of tarballs on all platforms is diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index 9bf06059c..bd259e1d4 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -8,6 +8,11 @@ if [ -n "$cfengine_role" ]; then touch /etc/cfengine-"$cfengine_role".flag fi +if uname | grep -i darwin >/dev/null; then + echo "Error: MacOS(Darwin) not supported yet." + exit 1 +fi + # install needed packages and software for a build host set -e if [ "$(id -u)" != "0" ]; then @@ -62,7 +67,7 @@ function cleanup() { set -e set -x [ -f /var/log/messages ] && tail /var/log/messages - command -v journalctl >/dev/null && journalctl | grep -P '(error|fail)' + command -v journalctl >/dev/null && journalctl | grep -P '(error|fail)' || true if command -v apt >/dev/null 2>&1; then # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false # Replace systemctl with a no-op stub that always succeeds. We can't diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh index b646fb9d5..1bb53afdf 100755 --- a/ci/setup-ci-host.sh +++ b/ci/setup-ci-host.sh @@ -27,17 +27,22 @@ function file-line() function github-known-hosts() { - echo "ensuring github hostkeys are added to /home/jenkins/.ssh/known_hosts" + mkdir ~/.ssh + echo "ensuring github hostkeys are added to ~/.ssh/known_hosts" grep '^github.com' "$thisdir"/known_hosts | while read -r key; do - file-line /home/jenkins/.ssh/known_hosts "$key" + file-line ~/.ssh/known_hosts "$key" done - chown jenkins /home/jenkins/.ssh/known_hosts - chmod 0600 /home/jenkins/.ssh/known_hosts + chown $(id -un) ~/.ssh/known_hosts + chmod 0600 ~/.ssh/known_hosts } echo "ensuring that github.com hostkeys are in ~/.ssh/known_hosts" github-known-hosts +if [ "$1" = "--bootstrap" ]; then + exit 0 +fi + # we setup some vars for platform versions to make it easier to make choice later # default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such redhat=0 From 416cffdf6d940d9c039dd91c50e497cb91b5c0e9 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 15:08:36 -0500 Subject: [PATCH 06/28] allow for build-tarballs only run if TARBALLS is yes --- build-in-container-inner.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index 538d8da7a..af58f91ef 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -115,6 +115,15 @@ run_step() { } # === Build steps === + +if [ "$TARBALLS" = yes ]; then + run_step "build-tarballs" build-tarballs + echo "" + echo "=== Build complete ===" + ls -lh "$OUTPUT" + exit 0 +fi + NO_TESTS=true export NO_TESTS From 1f3278132e5f678ecc00c3a081ed4f34b8b7b14e Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 15:29:12 -0500 Subject: [PATCH 07/28] refinement --- build-in-container-inner.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index af58f91ef..0c8bffd31 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -105,7 +105,7 @@ run_step() { local name="$1" shift echo "=== Running $name ===" - "$@" + "$name $@" local rc=$? if [ $rc -ne 0 ]; then echo "" @@ -117,7 +117,7 @@ run_step() { # === Build steps === if [ "$TARBALLS" = yes ]; then - run_step "build-tarballs" build-tarballs + run_step build-tarballs echo "" echo "=== Build complete ===" ls -lh "$OUTPUT" @@ -129,7 +129,7 @@ export NO_TESTS for script in "$BASEDIR/buildscripts/build-scripts"/0*; do name="$(basename "$script")" - run_step "$name" "$script" + run_step "$name" done write_sha256sums From 0d3d5be9f6db6c9461511f67473009ea66706cd4 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Thu, 27 Aug 2026 15:48:07 -0500 Subject: [PATCH 08/28] tweaks --- build-in-container-inner.sh | 3 ++- build-scripts/build-tarballs | 2 +- ci/setup-ci-host.sh | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh index 0c8bffd31..98a1bd0d9 100755 --- a/build-in-container-inner.sh +++ b/build-in-container-inner.sh @@ -105,7 +105,7 @@ run_step() { local name="$1" shift echo "=== Running $name ===" - "$name $@" + "$BASEDIR/buildscripts/build-scripts/$name" "$@" local rc=$? if [ $rc -ne 0 ]; then echo "" @@ -117,6 +117,7 @@ run_step() { # === Build steps === if [ "$TARBALLS" = yes ]; then + run_step autogen run_step build-tarballs echo "" echo "=== Build complete ===" diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index c7f3406bf..e833b4e7e 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -8,7 +8,7 @@ # # This script looks for two optional env vars: # - TARBALLS - if yes, then run this script, otherwise do not. The default is to not run this script. - this variable is provided by build-in-container.py +# this variable is provided by build-in-container.py # - OUTPUT - optional, for container builds this should be /output. The default is /output # # Each tarball's timestamps follow its own repository: Makefile.am in core and in diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh index 1bb53afdf..d0a71aa89 100755 --- a/ci/setup-ci-host.sh +++ b/ci/setup-ci-host.sh @@ -36,13 +36,14 @@ function github-known-hosts() chmod 0600 ~/.ssh/known_hosts } -echo "ensuring that github.com hostkeys are in ~/.ssh/known_hosts" -github-known-hosts - if [ "$1" = "--bootstrap" ]; then + # todo, setup needed dependencies for a bootstrap host exit 0 fi +echo "ensuring that github.com hostkeys are in ~/.ssh/known_hosts" +github-known-hosts + # we setup some vars for platform versions to make it easier to make choice later # default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such redhat=0 From 4babaf87f76e006fc5398022450fac565178a78b Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 08:07:17 -0500 Subject: [PATCH 09/28] fix and tidy build-tarballs --- build-scripts/build-tarballs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index e833b4e7e..0d15b40a0 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -33,24 +33,31 @@ usage() { set -e if [ -z "$TARBALLS" ] || [ "$TARBALLS" != "yes" ]; then - echo "TARBALLS env variable did not have a value of yes, so skipping $0" + log_debug "TARBALLS env variable did not have a value of yes, so skipping $0" exit 0 fi -if [ -x "$OUTPUT" ]; then +if [ -z "$OUTPUT" ]; then OUTPUT=output fi +if [ ! -d "$OUTPUT" ]; then + log_debug "notice: OUTPUT dir: $OUTPUT does not exist. Creating it now." + mkdir -p "$OUTPUT" +fi + ( cd "$BASEDIR/core" SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) export SOURCE_DATE_EPOCH - echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + log_debug "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" rm -f cfengine-3.*.tar.gz # Configure so the dist target exists, undone again below. - ./configure -C - make dist + log_debug "Running ./configure -C in core..." + run_and_print_on_failure ./configure -C + log_debug "Running make dist in core..." + run_and_print_on_failure make dist mv cfengine-3.*.tar.gz "$OUTPUT" make distclean ) @@ -59,12 +66,15 @@ fi cd "$BASEDIR/masterfiles" SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) export SOURCE_DATE_EPOCH - echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + log_debug "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" rm -f cfengine-masterfiles*.tar.gz - ./configure - make dist # source tarball: cfengine-masterfiles-.tar.gz - make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz + log_debug "Running ./configure in masterfiles..." + run_and_print_on_failure ./configure + log_debug "Running make dist in masterfiles..." + run_and_print_on_failure make dist # source tarball: cfengine-masterfiles-.tar.gz + log_debug "Running make tar-package in masterfiles..." + run_and_print_on_failure make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz mv cfengine-masterfiles*.tar.gz "$OUTPUT" make distclean ) From 81b8b674155ca7f46502de0b845947930327e33b Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 08:12:34 -0500 Subject: [PATCH 10/28] more tidy of logging --- build-scripts/build-tarballs | 14 ++++++++++++-- build-scripts/version | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index 0d15b40a0..fc33c62df 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -53,13 +53,18 @@ fi log_debug "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" rm -f cfengine-3.*.tar.gz + # Configure so the dist target exists, undone again below. log_debug "Running ./configure -C in core..." run_and_print_on_failure ./configure -C + log_debug "Running make dist in core..." run_and_print_on_failure make dist + mv cfengine-3.*.tar.gz "$OUTPUT" - make distclean + + log_debug "Running make distclean in core..." + run_and_print_on_failure make distclean ) ( @@ -69,12 +74,17 @@ fi log_debug "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" rm -f cfengine-masterfiles*.tar.gz + log_debug "Running ./configure in masterfiles..." run_and_print_on_failure ./configure + log_debug "Running make dist in masterfiles..." run_and_print_on_failure make dist # source tarball: cfengine-masterfiles-.tar.gz + log_debug "Running make tar-package in masterfiles..." run_and_print_on_failure make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz + mv cfengine-masterfiles*.tar.gz "$OUTPUT" - make distclean + log_debug "Running make distclean in masterfiles..." + run_and_print_on_failure make distclean ) diff --git a/build-scripts/version b/build-scripts/version index d1f392e1f..9026569bf 100644 --- a/build-scripts/version +++ b/build-scripts/version @@ -19,11 +19,11 @@ # repository. The script is executed when you run the autogen.sh. CORE_VERSION=$(cat "$BASEDIR"/core/CFVERSION) VERSION="${EXPLICIT_VERSION:-$CORE_VERSION}" -echo "$(basename $0): Debug: Detected CFEngine version number $VERSION" +log_debug "Detected CFEngine version number $VERSION" # The RELEASE number is the actual release of the package. It starts as 1 and # would incremented each time you create a new release of the CFEngine package # of the same version. RELEASE="${EXPLICIT_RELEASE:-1}" export RELEASE -echo "$(basename $0): Debug: Detected CFEngine package release number $RELEASE" +log_debug "Detected CFEngine package release number $RELEASE" From f5e9fb493b1f20107ab0a597e37deb407791c126 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 08:16:43 -0500 Subject: [PATCH 11/28] chmod +x bootstrap-tarballs --- build-scripts/bootstrap-tarballs | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-scripts/bootstrap-tarballs diff --git a/build-scripts/bootstrap-tarballs b/build-scripts/bootstrap-tarballs old mode 100644 new mode 100755 From fbaebf3d3f25a37ffdf0adb56e556af7ad073887 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 08:32:11 -0500 Subject: [PATCH 12/28] fix refactored bootstrap-tarballs --- build-scripts/bootstrap-tarballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-scripts/bootstrap-tarballs b/build-scripts/bootstrap-tarballs index aac9fbe3f..72b2a28a1 100755 --- a/build-scripts/bootstrap-tarballs +++ b/build-scripts/bootstrap-tarballs @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# legacy bootstrap-tarballs was split into bootstrap and bootstrap-mission-portal +# legacy bootstrap-tarballs was split into build-tarballs and bootstrap-mission-portal # todo: remove this script when it is no longer called from anywhere set -e thisdir="$(dirname "$0")" -bash "$thisdir"/bootstrap +bash "$thisdir"/build-tarballs bash "$thisdir"/bootstrap-mission-portal From da47d1a8fb8d21c4c1a4ebc84a83e88235ce608c Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 09:36:49 -0500 Subject: [PATCH 13/28] fix build-tarballs for jenkins and build-in-container --- build-scripts/build-tarballs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index fc33c62df..ca5027ed4 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -11,6 +11,8 @@ # this variable is provided by build-in-container.py # - OUTPUT - optional, for container builds this should be /output. The default is /output # +# Additionally, if we are running in a Jenkins job, JOB_NAME will be set and if bootstrap-pr then the script will run fully. +# # Each tarball's timestamps follow its own repository: Makefile.am in core and in # masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking # it from the last commit keeps a tarball identical until its own sources change. @@ -33,8 +35,10 @@ usage() { set -e if [ -z "$TARBALLS" ] || [ "$TARBALLS" != "yes" ]; then - log_debug "TARBALLS env variable did not have a value of yes, so skipping $0" - exit 0 + if [ -z "$JOB_NAME" ] || [ "$JOB_NAME" != "bootstrap-pr" ]; then + log_debug "Neither TARBALLS=yes or JOB_NAME=bootstrap-pr was in environment variables so skipping $0" + exit 0 + fi fi if [ -z "$OUTPUT" ]; then From 4cfbfefd5933d492742b6ded92f91c13e7ba42fc Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 09:45:38 -0500 Subject: [PATCH 14/28] fix output dir for pr-pipeline --- build-scripts/build-tarballs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index ca5027ed4..9c5a3d5e9 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -46,10 +46,12 @@ if [ -z "$OUTPUT" ]; then fi if [ ! -d "$OUTPUT" ]; then - log_debug "notice: OUTPUT dir: $OUTPUT does not exist. Creating it now." + log_debug "OUTPUT dir: $OUTPUT does not exist. Creating it now." mkdir -p "$OUTPUT" fi +log_debug "OUTPUT dir is $OUTPUT" + ( cd "$BASEDIR/core" SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) From 84c079b2bd25f5676669bdedc94d0c3c5be9c048 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 10:16:14 -0500 Subject: [PATCH 15/28] fix build-tarballs output directory --- build-scripts/build-tarballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index 9c5a3d5e9..78eddcb55 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -67,7 +67,7 @@ log_debug "OUTPUT dir is $OUTPUT" log_debug "Running make dist in core..." run_and_print_on_failure make dist - mv cfengine-3.*.tar.gz "$OUTPUT" + mv cfengine-3.*.tar.gz "$OUTPUT"/ log_debug "Running make distclean in core..." run_and_print_on_failure make distclean @@ -90,7 +90,7 @@ log_debug "OUTPUT dir is $OUTPUT" log_debug "Running make tar-package in masterfiles..." run_and_print_on_failure make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz - mv cfengine-masterfiles*.tar.gz "$OUTPUT" + mv cfengine-masterfiles*.tar.gz "$OUTPUT"/ log_debug "Running make distclean in masterfiles..." run_and_print_on_failure make distclean ) From 56ddb72f5449501ae631a8ba306e4f5590634da6 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 10:29:56 -0500 Subject: [PATCH 16/28] fix output dir in jenkins --- build-scripts/build-tarballs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index 78eddcb55..ab2f1109a 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -42,7 +42,7 @@ if [ -z "$TARBALLS" ] || [ "$TARBALLS" != "yes" ]; then fi if [ -z "$OUTPUT" ]; then - OUTPUT=output + OUTPUT="$BASEDIR"/output/tarballs/ fi if [ ! -d "$OUTPUT" ]; then @@ -67,7 +67,7 @@ log_debug "OUTPUT dir is $OUTPUT" log_debug "Running make dist in core..." run_and_print_on_failure make dist - mv cfengine-3.*.tar.gz "$OUTPUT"/ + mv cfengine-3.*.tar.gz "$OUTPUT" log_debug "Running make distclean in core..." run_and_print_on_failure make distclean @@ -90,7 +90,7 @@ log_debug "OUTPUT dir is $OUTPUT" log_debug "Running make tar-package in masterfiles..." run_and_print_on_failure make tar-package # package tarball: cfengine-masterfiles-.pkg.tar.gz - mv cfengine-masterfiles*.tar.gz "$OUTPUT"/ + mv cfengine-masterfiles*.tar.gz "$OUTPUT" log_debug "Running make distclean in masterfiles..." run_and_print_on_failure make distclean ) From 79cdfe67f1de797467e73d941977d468f5ad5b6f Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 10:30:13 -0500 Subject: [PATCH 17/28] remove fail2ban in containers, no need since no sshd --- ci/cfengine-build-host-setup.cf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index ceb93eb10..b21a36a40 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -36,7 +36,8 @@ bundle agent cfengine_build_host_setup "ntp"; debian|ubuntu:: - "fail2ban" comment => "Ban IPs with repeated failed SSH auth attempts"; + "fail2ban" comment => "Ban IPs with repeated failed SSH auth attempts", + if => "not_in_container"; "libltdl7" package_policy => "delete"; "libltdl-dev" package_policy => "delete"; "binutils"; @@ -176,10 +177,13 @@ bundle agent cfengine_build_host_setup !(redhat_7|centos_7).(redhat|centos).(yum_dnf_conf_ok).epel_release_ok:: "fail2ban-server" - comment => "Ban IPs with repeated failed SSH auth attempts. On centos/rhel 8+ we must specify individual packages instead of just fail2ban as package method will append -*.* which would include conflicting shorewall and shorewall-lite packages."; + comment => "Ban IPs with repeated failed SSH auth attempts. On centos/rhel 8+ we must specify individual packages instead of just fail2ban as package method will append -*.* which would include conflicting shorewall and shorewall-lite packages.", + if => "not_in_container"; - "fail2ban-sendmail"; - "fail2ban-firewalld"; + "fail2ban-sendmail" + if => "not_in_container"; + "fail2ban-firewalld" + if => "not_in_container"; "ccache"; "fakeroot"; "perl-JSON-PP"; @@ -513,6 +517,7 @@ root - core unlimited ); "/etc/fail2ban/jail.local" + if => "not_in_container", create => "true", content => "[sshd] enabled = true @@ -661,10 +666,11 @@ jenkins ALL=NOPASSWD: /usr/bin/podman services: any:: "fail2ban" + if => "not_in_container", service_policy => "start", comment => "Ensure fail2ban is running"; - fail2ban_config_changed:: + fail2ban_config_changed.not_in_container:: "fail2ban" service_policy => "restart", comment => "Restart fail2ban to apply jail configuration"; From d9fedc6b8e0975edacd9c8d7ce311e696f2ff69c Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 10:48:09 -0500 Subject: [PATCH 18/28] debug unpack-tarballs in jenkins --- build-scripts/unpack-tarballs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 2ba83143e..d5b4fa2d2 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -33,6 +33,11 @@ if [ ! -d "$BASEDIR"/output/tarballs ]; then exit 0 fi +log_debug "CRAIG DEBUG location of tarballs..." +set -x +find "$BASEDIR" -name '*.tar.gz' +set +x + SOURCE_TARBALL="$BASEDIR/output/tarballs/cfengine-3.*.tar.gz" MASTERFILES_TARBALL=$(find "$BASEDIR"/output/tarballs/ -name 'cfengine-masterfiles*.tar.gz' -a ! \( -name '*pkg.tar.gz' \)) From ec82042d541eeee1c5a1e8cdf590eb4af5e3abb5 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 11:53:32 -0500 Subject: [PATCH 19/28] no sshd and no fail2ban in containers --- ci/cfengine-build-host-setup.cf | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index b21a36a40..a54f1c0af 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -36,8 +36,10 @@ bundle agent cfengine_build_host_setup "ntp"; debian|ubuntu:: - "fail2ban" comment => "Ban IPs with repeated failed SSH auth attempts", + "fail2ban" + comment => "Ban IPs with repeated failed SSH auth attempts", if => "not_in_container"; + "libltdl7" package_policy => "delete"; "libltdl-dev" package_policy => "delete"; "binutils"; @@ -180,10 +182,8 @@ bundle agent cfengine_build_host_setup comment => "Ban IPs with repeated failed SSH auth attempts. On centos/rhel 8+ we must specify individual packages instead of just fail2ban as package method will append -*.* which would include conflicting shorewall and shorewall-lite packages.", if => "not_in_container"; - "fail2ban-sendmail" - if => "not_in_container"; - "fail2ban-firewalld" - if => "not_in_container"; + "fail2ban-sendmail" if => "not_in_container"; + "fail2ban-firewalld" if => "not_in_container"; "ccache"; "fakeroot"; "perl-JSON-PP"; @@ -281,7 +281,6 @@ bundle agent cfengine_build_host_setup "not_in_container" expression => not(fileexists("/etc/cfengine-in-container.flag")), comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not."; - @if minimum_version(3.23) # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=7) ubuntu:: @@ -296,7 +295,6 @@ bundle agent cfengine_build_host_setup "leech2_build_toolchain_host" expression => version_compare("$(sys.os_version_major)", ">=", "7"); @endif - any:: "have_rust" expression => fileexists("/opt/rust/bin/rustc"); "have_protoc" expression => fileexists("/usr/local/bin/protoc"); @@ -385,11 +383,12 @@ bundle agent cfengine_build_host_setup comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option"; commands: - sshd_hardened:: + sshd_hardened.not_in_container:: 'kill -1 $(pgrep -f "sshd -D")' handle => "sshd_restarted", contain => in_shell, comment => "Reload sshd config with SIGHUP(1) to apply hardened configuration"; + have_tmp_mount:: "mount -o remount,size=5G /tmp" comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.", @@ -529,6 +528,7 @@ findtime = 600", comment => "Configure fail2ban to ban IPs after 5 failed SSH attempts within 10 minutes"; "$(sshd_config_files)" + if => "not_in_container", edit_line => comment_lines_matching( "^$(sshd_hardening_directives)\s+(?!no\s*$).*", "#" ), @@ -537,7 +537,7 @@ findtime = 600", "/etc/ssh/sshd_config" edit_line => prepend_if_no_line("$(sshd_hardening_directives) no"), - if => fileexists("/etc/ssh/sshd_config"), + if => and("not_in_container", fileexists("/etc/ssh/sshd_config")), classes => if_repaired("sshd_hardened"), comment => "Ensure SSH hardening directives are at the top of sshd_config, before any Include"; @@ -647,7 +647,7 @@ jenkins ALL=NOPASSWD: /usr/bin/podman !have_sys_user.(suse|sles|opensuse):: "useradd -u 3 sys -g sys" contain => in_shell; - linux:: + linux.not_in_container:: "sshd -T 2>/dev/null | grep -qiE '^PermitRootLogin no'" depends_on => { "sshd_restarted" }, contain => in_shell, @@ -664,9 +664,8 @@ jenkins ALL=NOPASSWD: /usr/bin/podman comment => "Verify KbdInteractiveAuthentication (OpenSSH 8.7+) or ChallengeResponseAuthentication (older) is disabled"; services: - any:: + not_in_container:: "fail2ban" - if => "not_in_container", service_policy => "start", comment => "Ensure fail2ban is running"; From 5670b9e30718bdac3c6ae2761a3ae19b23ca3d6b Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 11:57:40 -0500 Subject: [PATCH 20/28] debug outputs --- build-scripts/configure | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build-scripts/configure b/build-scripts/configure index ba88698e3..8cc50f60b 100755 --- a/build-scripts/configure +++ b/build-scripts/configure @@ -25,6 +25,12 @@ . compile-options . version +echo "CRAIG debug location of bootstrap-pr output files" +set -x +find "$BASEDIR" -name '*.tar.gz' +set +x + + # Make sure the PROJECT variable is set correctly case "$PROJECT" in community) From c561211fbbb9e0b951ff7a9ad3871e50d537f8dc Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 13:14:45 -0500 Subject: [PATCH 21/28] debug unpack-tarballs in jenkins --- build-scripts/unpack-tarballs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index d5b4fa2d2..10410d6e2 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -39,7 +39,10 @@ find "$BASEDIR" -name '*.tar.gz' set +x SOURCE_TARBALL="$BASEDIR/output/tarballs/cfengine-3.*.tar.gz" +echo "CRAIG found SOURCE_TARBALL=$SOURCE_TARBALL" + MASTERFILES_TARBALL=$(find "$BASEDIR"/output/tarballs/ -name 'cfengine-masterfiles*.tar.gz' -a ! \( -name '*pkg.tar.gz' \)) +echo "CRAIG found MASTERFILES_TARBALL=$MASTERFILES_TARBALL" if [ ! -f "$SOURCE_TARBALL" ] || [ ! -f "$MASTERFILES_TARBALL" ]; then echo "Did not find SOURCE_TARBALL: $SOURCE_TARBALL or MASTERFILES_TARBALL: $MASTERFILES_TARBALL so skipping $0 script" From cef0ba8e3a7252a2a8c6a9fd7e0e00dee47c21c8 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 13:24:04 -0500 Subject: [PATCH 22/28] fixup unpack-tarballs for jenkins --- build-scripts/unpack-tarballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 10410d6e2..f43606dd2 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -38,7 +38,7 @@ set -x find "$BASEDIR" -name '*.tar.gz' set +x -SOURCE_TARBALL="$BASEDIR/output/tarballs/cfengine-3.*.tar.gz" +SOURCE_TARBALL="$(ls $BASEDIR/output/tarballs/cfengine-3.*.tar.gz)" echo "CRAIG found SOURCE_TARBALL=$SOURCE_TARBALL" MASTERFILES_TARBALL=$(find "$BASEDIR"/output/tarballs/ -name 'cfengine-masterfiles*.tar.gz' -a ! \( -name '*pkg.tar.gz' \)) From 9759df600e0abddf06b3a2eae77d9b7b7bac3e1c Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 13:49:46 -0500 Subject: [PATCH 23/28] debug package script output location --- build-scripts/package | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build-scripts/package b/build-scripts/package index f98378a49..54b46b96e 100755 --- a/build-scripts/package +++ b/build-scripts/package @@ -344,12 +344,18 @@ deb) # Build the DEB package log_debug "Building DEB package with dpkg-buildpackage" + log_debug "CRAIG debug DEB package location, pwd: $(pwd)" log_debug "DEB_BUILD_OPTIONS: $DEB_BUILD_OPTIONS" ( cd "$BASEDIR/$PKG/pkg" export DEB_BUILD_OPTIONS="$DEB_BUILD_OPTIONS" dpkg-buildpackage -b -us -uc -rfakeroot # -b: binary only, -us/-uc: don't sign ) || false + log_debug "CRAIG find .deb files in BASEDIR: $BASEDIR" +set -x +find "$BASEDIR" -name '*.deb' +set +x + # Also create generic TAR package for manual installation log_debug "Creating generic TAR package for DEB build" From 41f818cca6f38342c68a18694950e5cafd7c13a8 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 14:11:30 -0500 Subject: [PATCH 24/28] more output debugs --- build-scripts/package | 2 +- build-scripts/prepare-results | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build-scripts/package b/build-scripts/package index 54b46b96e..4d94954da 100755 --- a/build-scripts/package +++ b/build-scripts/package @@ -353,7 +353,7 @@ deb) ) || false log_debug "CRAIG find .deb files in BASEDIR: $BASEDIR" set -x -find "$BASEDIR" -name '*.deb' +find "$BASEDIR" -name 'cfengine*.deb' set +x diff --git a/build-scripts/prepare-results b/build-scripts/prepare-results index 3903a6fe9..9ed3d4276 100755 --- a/build-scripts/prepare-results +++ b/build-scripts/prepare-results @@ -1,4 +1,6 @@ #!/bin/sh +set -x +echo CRAIG DEBUG $0 # Prepare build artifacts and test results for collection/archival. # This script collects built packages and test results from various locations From f9ded414470d045580210c2e018aebf686536710 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 14:14:53 -0500 Subject: [PATCH 25/28] reduce platforms for build while testing --- Jenkinsfile | 2 +- build-scripts/canaries.txt | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 build-scripts/canaries.txt diff --git a/Jenkinsfile b/Jenkinsfile index 181827d8e..6394b7ad9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ // builds. readTrusted reads it from the revision this job is for, without // cloning the repo. def selectedLabels() { - def sel = readTrusted('build-scripts/labels.txt').readLines() + def sel = readTrusted('build-scripts/canaries.txt').readLines() .collect { it.trim() } .findAll { it.startsWith('PACKAGES_') && it ==~ params.LABEL_FILTER } if (!sel) { diff --git a/build-scripts/canaries.txt b/build-scripts/canaries.txt new file mode 100644 index 000000000..94d367bb2 --- /dev/null +++ b/build-scripts/canaries.txt @@ -0,0 +1,9 @@ +# canary labels + +PACKAGES_HUB_x86_64_linux_redhat_8 + +PACKAGES_HUB_x86_64_linux_ubuntu_22 + +PACKAGES_x86_64_linux_redhat_7 + +PACKAGES_x86_64_mingw From 8311a4ed344d443a905bf4c1b6e9b0648c89c4bc Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 14:52:22 -0500 Subject: [PATCH 26/28] add back commitID flag files needed by vagrant-pr and testing-pr --- build-scripts/build-tarballs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index ab2f1109a..9506ec8bc 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -60,6 +60,9 @@ log_debug "OUTPUT dir is $OUTPUT" rm -f cfengine-3.*.tar.gz + log_debug "Writing flag file used by vagrant-pr and testing-pr: $BASEDIR/output/core-commitID" + git rev-parse HEAD >"$BASEDIR"/output/core-commitID + # Configure so the dist target exists, undone again below. log_debug "Running ./configure -C in core..." run_and_print_on_failure ./configure -C @@ -81,6 +84,9 @@ log_debug "OUTPUT dir is $OUTPUT" rm -f cfengine-masterfiles*.tar.gz + log_debug "Writing flag file used by vagrant-pr and testing-pr: $BASEDIR/output/masterfiles-commitID" + git rev-parse HEAD >"$BASEDIR"/output/masterfiles-commitID + log_debug "Running ./configure in masterfiles..." run_and_print_on_failure ./configure From ad594456d7d9e2bbaf45b630869023d9becf449c Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 14:54:11 -0500 Subject: [PATCH 27/28] add back output/PRs file needed by jenkins --- build-scripts/0035-generate-pull-request-file | 1 + build-scripts/generate-pull-request-file | 70 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 120000 build-scripts/0035-generate-pull-request-file create mode 100755 build-scripts/generate-pull-request-file diff --git a/build-scripts/0035-generate-pull-request-file b/build-scripts/0035-generate-pull-request-file new file mode 120000 index 000000000..019f7567e --- /dev/null +++ b/build-scripts/0035-generate-pull-request-file @@ -0,0 +1 @@ +generate-pull-request-file \ No newline at end of file diff --git a/build-scripts/generate-pull-request-file b/build-scripts/generate-pull-request-file new file mode 100755 index 000000000..23680a9fa --- /dev/null +++ b/build-scripts/generate-pull-request-file @@ -0,0 +1,70 @@ +#!/bin/bash + +# Get information about PRs among the used revisions. +# These PRs will have to be notified of build progress. +# +# Variables such as MISSION_PORTAL_REV may be set by the CI (Jenkins) to build and test multiple PR's together. +# The variables typically hold a branch name or a pull request ID. +# PR IDs can also be preceded by 'pull' or 'origin/pull'. +# E.g.: +# - MISSION_PORTAL_REV=pull/1755 +# - MISSION_PORTAL_REV=origin/pull/1755 +# where the trailing number is the pull request ID. +# +# Furthermore, they can be suffixed by anything after a subsequent slash (/). +# We usually use: +# - ID alone +# - pull/ID/head +# - pull/ID/merge +# +# This loop fetches information about the PRs if the respective variable is set. +# +# The script expects the following repositories to be side by side: +# . +# ├── buildscripts +# ├── core +# ├── enterprise +# ├── nova +# ├── mission-portal +# ├── libntech +# └── masterfiles +# +# ^ When building community you won't need enterprise, nova, mission-portal +# + +_dir=$(readlink -e "$(dirname "$0")") +# refactored a few functions into single file scripts for easier development/debugging, see ENT-12741 and ENT-12595 +# Easier to add a path to a script than source a file of functions. +export PATH="$_dir"/bin:$PATH +. "$(dirname "$0")"/functions +. detect-environment +. compile-options +. version + +# errors are not critical +set +e + +for repo_spec in cfengine/buildscripts cfengine/core cfengine/masterfiles cfengine/enterprise cfengine/nova cfengine/mission-portal NorthernTechHQ/libntech; do + # remove organization/ from start of repo_spec + # E.g. 'cfengine/mission-portal' -> 'mission-portal' + repo="${repo_spec#*/}" + + # Convert to uppercase, swap hyphens with underscore and append '_REV' + # E.g. 'mission-portal' -> 'MISSION_PORTAL_REV' + rev_param_name="$(echo "$repo" | tr '[:lower:]-' '[:upper:]_')_REV" + + # Try to dereference the result from above and skip the rest of the loop + # unless the variable is defined. + revision="${!rev_param_name}" || continue + + # remove "origin/" (if any) + revision="${revision##origin/}" + + # Check to see if the resolved variable starts with 'pull/' + if expr "$revision" : "pull/" >/dev/null; then + # Extract the revision number. E.g. 'pull/1755' -> '1755' + pr_nr="$(echo "$revision" | cut -d/ -f2)" + + get-github-pull-request-info "$repo_spec" "$pr_nr" >>"$BASEDIR"/output/PRs + fi +done From a70c9e36c8a42450449da7f0b6cdefec89fdc18f Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 28 Aug 2026 15:09:53 -0500 Subject: [PATCH 28/28] fixup build-tarballs and location of commitID files --- build-scripts/build-tarballs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-scripts/build-tarballs b/build-scripts/build-tarballs index 9506ec8bc..ac4262f6b 100755 --- a/build-scripts/build-tarballs +++ b/build-scripts/build-tarballs @@ -60,8 +60,8 @@ log_debug "OUTPUT dir is $OUTPUT" rm -f cfengine-3.*.tar.gz - log_debug "Writing flag file used by vagrant-pr and testing-pr: $BASEDIR/output/core-commitID" - git rev-parse HEAD >"$BASEDIR"/output/core-commitID + log_debug "Writing flag file used by vagrant-pr and testing-pr: $OUTPUT/core-commitID" + git rev-parse HEAD > "$OUTPUT"/core-commitID # Configure so the dist target exists, undone again below. log_debug "Running ./configure -C in core..." @@ -84,8 +84,8 @@ log_debug "OUTPUT dir is $OUTPUT" rm -f cfengine-masterfiles*.tar.gz - log_debug "Writing flag file used by vagrant-pr and testing-pr: $BASEDIR/output/masterfiles-commitID" - git rev-parse HEAD >"$BASEDIR"/output/masterfiles-commitID + log_debug "Writing flag file used by vagrant-pr and testing-pr: $OUTPUT/masterfiles-commitID" + git rev-parse HEAD >"$OUTPUT"/masterfiles-commitID log_debug "Running ./configure in masterfiles..." run_and_print_on_failure ./configure