Added numbered symlinks to build steps for ease of use

Jenkinsfile

@@ -10,7 +10,7 @@
 // builds. readTrusted reads it from the revision this job is for, without
 // cloning the repo.
 def selectedLabels() {
-  def sel = readTrusted('build-scripts/labels.txt').readLines()
+  def sel = readTrusted('build-scripts/canaries.txt').readLines()
       .collect { it.trim() }
       .findAll { it.startsWith('PACKAGES_') && it ==~ params.LABEL_FILTER }
   if (!sel) {

README-build-in-container.md

@@ -0,0 +1,8 @@
+Build with ./build-in-container.py
+
+Using ./build-in-container.py --shell you can run the build interactively and debug issues.
+Start in the container by running /srv/source/buildscripts/build-in-container-inner.sh
+
+This will copy repository sources that are needed from the read-only /srv location to read-write work area in /home/builder/build.
+
+Continue debugging by running steps in /home/builder/buildscripts/build-scripts/0*.sh

build-in-container-inner.sh

@@ -4,15 +4,20 @@ set -e
 # Configuration via environment variables:
 #   PROJECT, BUILD_TYPE, EXPLICIT_ROLE, BUILD_NUMBER, EXPLICIT_VERSION
 
+# let setup-cfengine-build-host.sh know we are in a container
+sudo touch /etc/cfengine-in-container.flag
+
 BASEDIR=/home/builder/build
 export BASEDIR
 export AUTOBUILD_PATH="$BASEDIR/buildscripts"
+OUTPUT=/output
+export OUTPUT
 
 mkdir -p "$BASEDIR"
 
 # Bind-mounted directories may be owned by the host user's UID.
 # Fix ownership so builder can write to them.
-sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" /output
+sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" "$OUTPUT"
 
 # And hand ownership back to the host user on the way out.
 if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then
@@ -85,105 +90,22 @@ fi
 export SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 
-install_mission_portal_deps() (
-    set -e
-
-    if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then
-        echo "Installing npm dependencies..."
-        npm ci --prefix "$BASEDIR/mission-portal/public/scripts/"
-        echo "Building react components..."
-        npm run build --prefix "$BASEDIR/mission-portal/public/scripts/"
-        rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules"
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/composer.json" ]; then
-        echo "Installing Mission Portal PHP dependencies..."
-        (cd "$BASEDIR/mission-portal" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then
-        echo "Installing Nova API PHP dependencies..."
-        (cd "$BASEDIR/nova/api/http" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then
-        echo "Compiling Mission Portal styles..."
-        mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css"
-        (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" &&
-            lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then
-        echo "Installing LDAP API PHP dependencies..."
-        (cd "$BASEDIR/mission-portal/ldap" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    # Composer falls back to git clone when GitHub's anonymous zipball
-    # rate limit is hit, leaving non-reproducible .git directories in the
-    # vendor tree. Strip them.
-    find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +
-)
-
 # Lets whoever consumes the output check that it arrived intact. Sorted in the C
 # locale so that the list itself comes out the same every time.
 write_sha256sums() (
-    cd /output
+    cd "$OUTPUT"
     # shellcheck disable=SC2094
     # > Make sure not to read and write the same file in the same pipeline.
     # find leaves it out by name, so the list never covers itself.
     find . -maxdepth 1 -type f ! -name sha256sums.txt -printf '%P\n' \
         | LC_ALL=C sort | xargs -r sha256sum > sha256sums.txt
 )
 
-# Build the source tarballs. They are the same whichever platform builds them,
-# so only this image builds them, and nothing else here does. /output is
-# <output-dir>/tarballs on the host, as the packages' /output is per label.
-#
-# Each tarball's timestamps follow its own repository: Makefile.am in core and in
-# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking
-# it from the last commit keeps a tarball identical until its own sources change.
-build_tarballs() (
-    set -e
-
-    (
-        cd "$BASEDIR/core"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-3.*.tar.gz
-        # Configure so the dist target exists, undone again below.
-        ./configure -C
-        make dist
-        mv cfengine-3.*.tar.gz /output/
-        make distclean
-    )
-
-    (
-        cd "$BASEDIR/masterfiles"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-masterfiles*.tar.gz
-        ./configure
-        make dist        # source tarball:  cfengine-masterfiles-<version>.tar.gz
-        make tar-package # package tarball: cfengine-masterfiles-<version>.pkg.tar.gz
-        mv cfengine-masterfiles*.tar.gz /output/
-        make distclean
-    )
-
-    write_sha256sums
-)
-
-# === Step runner with failure reporting ===
-# Disable set -e so we can capture exit codes and report which step failed.
-set +e
 run_step() {
     local name="$1"
     shift
     echo "=== Running $name ==="
-    "$@"
+    "$BASEDIR/buildscripts/build-scripts/$name" "$@"
     local rc=$?
     if [ $rc -ne 0 ]; then
         echo ""
@@ -193,36 +115,26 @@ run_step() {
 }
 
 # === Build steps ===
-run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen"
 
 if [ "$TARBALLS" = yes ]; then
-    run_step "02-tarballs" build_tarballs
+    run_step autogen
+    run_step build-tarballs
     echo ""
     echo "=== Build complete ==="
-    ls -lh /output/
+    ls -lh "$OUTPUT"
     exit 0
 fi
 
-run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies"
-# Mission Portal is an Enterprise/nova-only component; its sources are only
-# synced when PROJECT=nova. Skip this step for community hubs.
-if [ "$PROJECT" = "nova" ] && [ "$EXPLICIT_ROLE" = "hub" ]; then
-    run_step "03-mission-portal-deps" install_mission_portal_deps
-fi
-run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure"
-run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile"
-run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package"
-
-# === Copy output packages ===
-# Packages are created under $BASEDIR/<project>/ by dpkg-buildpackage / rpmbuild.
-# Exclude deps-packaging to avoid copying dependency packages.
-find "$BASEDIR" -maxdepth 4 \
-    -path "$BASEDIR/buildscripts/deps-packaging" -prune -o \
-    \( -name '*.deb' -o -name '*.rpm' -o -name '*.msi' -o -name '*.pkg.tar.gz' \) -print \
-    -exec cp {} /output/ \;
+NO_TESTS=true
+export NO_TESTS
+
+for script in "$BASEDIR/buildscripts/build-scripts"/0*; do
+  name="$(basename "$script")"
+  run_step "$name"
+done
 
 write_sha256sums
 
 echo ""
 echo "=== Build complete ==="
-ls -lh /output/
+ls -lh "$OUTPUT"/

build-scripts/0000-system-dependencies

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -ex
+thisdir="$(dirname "$0")"
+sudo bash "$thisdir"/../ci/setup-cfengine-build-host.sh

build-scripts/0005-repositories

@@ -0,0 +1 @@
+repositories

build-scripts/0010-autogen

@@ -0,0 +1 @@
+autogen

build-scripts/0020-clean-buildmachine

@@ -0,0 +1 @@
+clean-buildmachine

build-scripts/0030-bootstrap-mission-portal

@@ -0,0 +1 @@
+bootstrap-mission-portal

build-scripts/0035-generate-pull-request-file

@@ -0,0 +1 @@
+generate-pull-request-file

build-scripts/0040-build-tarballs

@@ -0,0 +1 @@
+build-tarballs

build-scripts/0050-unpack-tarballs

@@ -0,0 +1 @@
+unpack-tarballs

build-scripts/0060-install-dependencies

@@ -0,0 +1 @@
+install-dependencies

build-scripts/0070-configure

@@ -0,0 +1 @@
+configure

build-scripts/0080-generate-source-tarballs

@@ -0,0 +1 @@
+generate-source-tarballs

build-scripts/0090-compile

@@ -0,0 +1 @@
+compile

build-scripts/0100-produce-debug-symbols

@@ -0,0 +1 @@
+produce-debug-symbols

build-scripts/0110-package

@@ -0,0 +1 @@
+package

build-scripts/0120-prepare-results

@@ -0,0 +1 @@
+prepare-results

build-scripts/0130-test

@@ -0,0 +1 @@
+test

build-scripts/0140-prepare-results

@@ -0,0 +1 @@
+prepare-results

build-scripts/autogen

@@ -18,6 +18,10 @@
 # ```
 #
 
+# note that this script should be a no-op if an artifacts.tgz is present
+thisdir="$(dirname "$0")"
+bash "$thisdir"/../ci/setup-ci-host.sh --bootstrap
+
 # Get the BASEDIR variable holding the path to where our repos are checked out
 . "$(dirname "$0")/functions"
 
@@ -61,11 +65,15 @@ done
 # Run autogen.sh on each repository
 for proj in $projects; do
     # autogen.sh is quite verbose, so only print the output in case of failure
-    log_debug "Running autogen.sh for project $proj..."
     (
         cd "$BASEDIR/$proj"
-        export NO_CONFIGURE=1
-        run_and_print_on_failure ./autogen.sh
+        if [ ! -f configure ] || [ ! -f CFVERSION ]; then
+            log_debug "Running autogen.sh for project $proj..."
+            export NO_CONFIGURE=1
+            run_and_print_on_failure ./autogen.sh
+        else
+            log_debug "Skipping autogen.sh in $proj as configure and/or CFVERSION files already exist"
+        fi
     )
 done
 

build-scripts/bootstrap-mission-portal

@@ -0,0 +1,55 @@
+. "$(dirname "$0")"/functions
+. detect-environment
+. compile-options
+. version
+    set -e
+
+if [ "$ROLE" != "hub" ]; then
+  echo "ROLE is not hub so skipping $0"
+  exit 0
+fi
+
+if command -v composer >/dev/null; then
+  COMPOSER=$(command -v composer)
+elif [ -f /usr/local/bin/composer.phar ]; then
+  COMPOSER=/usr/local/bin/composer.phar
+else
+  echo "Error: could not find composer command in PATH or at /usr/local/bin/composer.phar"
+  exit 1
+fi
+
+
+if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then
+    echo "Installing npm dependencies..."
+    npm ci --prefix "$BASEDIR/mission-portal/public/scripts/"
+    echo "Building react components..."
+    npm run build --prefix "$BASEDIR/mission-portal/public/scripts/"
+    rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules"
+fi
+
+if [ -f "$BASEDIR/mission-portal/composer.json" ]; then
+    echo "Installing Mission Portal PHP dependencies..."
+    (cd "$BASEDIR/mission-portal" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist)
+fi
+
+if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then
+    echo "Installing Nova API PHP dependencies..."
+    (cd "$BASEDIR/nova/api/http" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist)
+fi
+
+if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then
+    echo "Compiling Mission Portal styles..."
+    mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css"
+    (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" &&
+        lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css)
+fi
+
+if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then
+    echo "Installing LDAP API PHP dependencies..."
+    (cd "$BASEDIR/mission-portal/ldap" && php "$COMPOSER" install --no-dev --ignore-platform-reqs --prefer-dist)
+fi
+
+# Composer falls back to git clone when GitHub's anonymous zipball
+# rate limit is hit, leaving non-reproducible .git directories in the
+# vendor tree. Strip them.
+find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +