diff --git a/.github/workflows/almalinux-compose-test-ppc64le.yml b/.github/workflows/almalinux-compose-test-ppc64le.yml index 58c8da9..21161f9 100644 --- a/.github/workflows/almalinux-compose-test-ppc64le.yml +++ b/.github/workflows/almalinux-compose-test-ppc64le.yml @@ -2,9 +2,15 @@ name: almalinux-compose-test-ppc64le run-name: AlmaLinux ${{ inputs.version_major }} compose test (ppc64le)${{ inputs.pungi_repository && ' +pungi' || '' }}${{ inputs.pulp_repository && ' +pulp' || '' }} -# Placeholder: GitHub dispatches a workflow only if a file with its path is on -# the default branch. The implementation (QEMU TCG emulated ppc64le guests) is -# on the ci/tcg-ppc64le-s390x branch: run the workflow from there. +# The tests run in AlmaLinux ppc64le containers (podman, systemd as PID 1, the +# almalinux/-init images, ci/container-guests.sh), split between the +# containers like on x86_64 and aarch64: in the AlmaLinux organization on a +# ppc64le runner, natively; elsewhere, e.g. in forks, on an x86_64 runner, their +# binaries run by QEMU user-mode emulation. The containers share the runner's +# kernel, so the tests run with CONTAINERTEST=1: those exercising the kernel, +# the boot or the hardware skip themselves. Emulated, QEMU needs to be 10.1 or +# newer, so Ubuntu 26.04: the 8.2.2 of Ubuntu 24.04 miscompiles POWER9 vector +# loads/stores (QEMU issue 1769). on: workflow_dispatch: @@ -50,13 +56,718 @@ on: type: boolean default: true + # Allow this workflow to be called from the 'almalinux-compose-test' orchestrator. + # 'workflow_call' has no 'choice' input type, so 'version_major' is a string here. + workflow_call: + inputs: + version_major: + description: 'AlmaLinux major version' + required: true + type: string + pungi_repository: + description: 'Add pungi repositories' + type: boolean + default: false + pulp_repository: + description: 'Add pulp repositories' + type: boolean + default: true + testing_repository: + description: 'Add testing repository' + type: boolean + default: false + disable_default_repos: + description: 'Disable system default repositories' + type: boolean + default: false + rerun_failed: + description: 'Re-run failed tests' + type: boolean + default: true + notify_mattermost: + description: 'Send results notification to Mattermost' + type: boolean + default: true + jobs: compose-test: name: AlmaLinux ${{ inputs.version_major }} - runs-on: ubuntu-26.04-ppc64le-p10 + # In the AlmaLinux organization the containers run natively on a POWER + # runner, a POWER10 one for AlmaLinux 10 (built for POWER10); elsewhere, + # e.g. in forks, on an x86_64 runner, emulated. KVM is of no use (no + # nested-virt). GitHub-hosted runners stop jobs after 6 hours. + runs-on: >- + ${{ + github.repository_owner == 'AlmaLinux' && + (startsWith(inputs.version_major, '10') && 'ubuntu-26.04-ppc64le-p10' || 'ubuntu-26.04-ppc64le') + || + 'ubuntu-26.04' + }} + timeout-minutes: 360 + + env: + # Number of guests the tests are split between, run in parallel + shards: 4 + # Whether the containers run natively (see runs-on) or emulated + native: ${{ github.repository_owner == 'AlmaLinux' }} + # The factor the tests' time limits ('duration') are scaled by: emulation + # is several times slower than the hardware they are meant for, and + # a ppc64le runner may be slower than an x86_64 one + duration_scale: ${{ github.repository_owner == 'AlmaLinux' && 2 || 8 }} + steps: - - name: Not implemented on this branch + - uses: actions/checkout@v7 + + - name: Prepare stuff + run: | + # Environment variables + pungi_latest_result=latest_result_almalinux + pulp_root=${{ inputs.version_major }} + tmt_options= + rpm_gpg_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-AlmaLinux + release_version=${{ inputs.version_major }} + dnf_crb_repo='CRB' + dnf_extras_repo='extras' + # systemd-enabled image of the version + image=docker.io/almalinux/${{ inputs.version_major }}-init:latest + + case ${{ inputs.version_major }} in + 8) + dnf_crb_repo='PowerTools' + ;; + 9) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-9" + ;; + 10) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + ;; + 10-kitten) + release_version=10 + pulp_root=kitten-10 + pungi_latest_result="${pungi_latest_result}-kitten" + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + dnf_extras_repo='extras-common' + ;; + esac + # Release major version + echo "release_version=${release_version}" >> $GITHUB_ENV + + # Pulp repository root of the release + echo "pulp_root=${pulp_root}" >> $GITHUB_ENV + + # Pungi repository latest results directory + echo "pungi_latest_result=${pungi_latest_result}" >> $GITHUB_ENV + + # Name of repository to enable (PowerTools/CRB) + if [ "x${{ inputs.pulp_repository }}" = "xtrue" ]; then + # Lowercase the name for path in pulp's URL + pulp_crb_repo="${dnf_crb_repo,,}" + fi + echo "dnf_crb_repo=${dnf_crb_repo}" >> $GITHUB_ENV + echo "pulp_crb_repo=${pulp_crb_repo}" >> $GITHUB_ENV + echo "dnf_extras_repo=${dnf_extras_repo}" >> $GITHUB_ENV + + # TMT extra options; these go right after 'tmt', before the + # subcommand -- tmt 1.78 dropped the step-level '--feeling-safe'. + echo "tmt_options=${tmt_options}" >> $GITHUB_ENV + + # RPM_GPG_KEY file + echo "rpm_gpg_key=${rpm_gpg_key}" >> $GITHUB_ENV + + # Image the containers run + echo "image=${image}" >> $GITHUB_ENV + + # TMT tests run directory (in the containers) + echo "tmt_run_dir=/var/tmp/tmt/run-001" >> $GITHUB_ENV + + # Names of the containers, one per shard of the tests + echo "guests=$(seq -f 'almalinux-%g' -s ' ' 0 $(( shards - 1 )))" >> $GITHUB_ENV + + # Local directory on the runner to gather artifacts pulled from the containers + artifacts_dir="${{ github.workspace }}/artifacts" + mkdir -p "${artifacts_dir}" + echo "artifacts_dir=${artifacts_dir}" >> $GITHUB_ENV + + # For nvidia drivers tests + case ${{ inputs.version_major }} in + 8) + nvidia=false + ;; + *) + nvidia=true + ;; + esac + echo "nvidia=${nvidia}" >> $GITHUB_ENV + + - name: Configure apt + run: | + # Prefer the regional EC2 mirror over archive.ubuntu.com (throttled/unreachable + # from the runners at times), and make apt fail fast with retries. + imds_token=$(curl -sf --max-time 5 -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' || true) + region=$(curl -sf --max-time 5 -H "X-aws-ec2-metadata-token: $imds_token" http://169.254.169.254/latest/meta-data/placement/region || true) + if [ -n "$region" ]; then + sudo sed -i "s|https\?://archive.ubuntu.com/ubuntu|http://${region}.ec2.archive.ubuntu.com/ubuntu|" /etc/apt/sources.list.d/ubuntu.sources + fi + printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' | sudo tee /etc/apt/apt.conf.d/99-ci-retries + + - name: Install podman + run: | + command -v podman > /dev/null || { sudo apt-get -y update; sudo apt-get -y install podman; } + echo "[Debug] $(uname -m) runner, $(podman --version)" + + - name: Install QEMU user-mode emulation + if: env.native != 'true' + run: | + # Its binfmt_misc handlers let the kernel run ppc64le binaries + sudo apt-get -y update + sudo apt-get -y install qemu-user-binfmt + # Registered without the C (credentials) flag, setuid programs (passwd, + # newgrp, su, ...) would run without their privileges: register again with it + handler=$(grep -h "^:qemu-ppc64le:" /usr/lib/binfmt.d/qemu-ppc64le.conf) + echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-ppc64le > /dev/null + echo "${handler%:*}:OCPF" | sudo tee /proc/sys/fs/binfmt_misc/register > /dev/null + cat /proc/sys/fs/binfmt_misc/qemu-ppc64le + qemu_version=$(qemu-ppc64le --version | sed -nE 's/^qemu-ppc64le version ([0-9]+\.[0-9]+).*/\1/p') + echo "[Debug] qemu-ppc64le ${qemu_version}, $(podman --version)" + # Refuse an emulator with QEMU issue 1769 rather than let the run fail hours in + if [ "$(printf '%s\n' "${qemu_version}" 10.1 | sort -V | head -1)" != "10.1" ]; then + echo "[Error] QEMU ${qemu_version} is too old for ppc64le (needs 10.1 or newer, see QEMU issue 1769); run this job on Ubuntu 26.04 or newer" + exit 1 + fi + + + - name: Start the containers + run: | + # The containers' root is the runner's, so their systemds and services + # share its inotify instances: with too few D-Bus failed to start in some + # ("dirwatch_new: Too many open files"). Raise the limit where the runner + # allows it (a container runner does not), and show how much is used. + echo 8192 | sudo tee /proc/sys/fs/inotify/max_user_instances > /dev/null 2>&1 || + echo "[Debug] inotify instances limit not raised: not allowed on the runner" + sudo guests="${guests}" ci/container-guests.sh ppc64le "${image}" + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + + - name: Create AlmaLinux pungi repository + if: inputs.pungi_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pungi.repo" + [almalinux-${{ env.release_version }}-appstream-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/AppStream/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-appstream-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-baseos-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/BaseOS/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-baseos-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_crb_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_extras_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux pulp repository + if: inputs.pulp_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pulp.repo" + [almalinux-${{ env.pulp_root }}-appstream-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-appstream-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-appstream-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-baseos-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-baseos-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-baseos-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux testing pulp repository + # Kitten has no testing repository, so skip it there + if: inputs.pulp_repository && inputs.testing_repository && inputs.version_major != '10-kitten' + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-testing-pulp.repo" + [almalinux-${{ env.pulp_root }}-testing-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-testing-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-testing-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux nvidia pulp repository + if: env.nvidia == 'true' && inputs.pulp_repository + run: | + # Repositories of the 'almalinux-nvidia' product + # https://build.almalinux.org/product/777 + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-nvidia-pulp.repo" + [almalinux-${{ env.pulp_root }}-nvidia-pulp] + baseurl = https://build.almalinux.org/pulp/content/copr/anfimovdm-almalinux-nvidia-almalinux-${{ env.pulp_root }}-$arch-dr/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-nvidia-pulp + gpgkey=file://${{ env.rpm_gpg_key }},file:///etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA-CUDA-${{ env.release_version }} + EOF + + # No 'Publish tested repository state': ci/compose_state.py gates the + # x86_64 and aarch64 publishing only + + - name: Push repositories to the containers + run: | + ci/each-guest.sh ' + set -e + for repo in almalinux-pungi almalinux-pulp almalinux-testing-pulp almalinux-nvidia-pulp; do + f="${{ env.artifacts_dir }}/${repo}.repo" + if [ -e "$f" ]; then + echo "[Debug] Installing ${repo}.repo" + sudo podman cp "$f" $guest:/etc/yum.repos.d/${repo}.repo + fi + done + ' + + - name: Update the system + run: | + # No reboot: the containers run the runner's kernel + ci/each-guest.sh ' + set -e + sudo podman exec $guest dnf -y clean all + sudo podman exec $guest dnf -y upgrade + # The image has packages an installed system does not (curl-minimal, + # which conflicts with the curl the tests install) and lacks some it + # has (hostname) + sudo podman exec $guest sh -c "! rpm -q curl-minimal >/dev/null || dnf -y swap curl-minimal curl" + sudo podman exec $guest dnf -y install hostname + ' + + - name: Get compose-tests onto the containers + run: | + # Scale the tests' time limits in the copy the containers get, as plain + # values: the tmt 1.26 of AlmaLinux 8 has neither policies nor + # duration multiplication. Tests without one get tmt's default, 5m. + grep -rlE '^duration:' --include=main.fmf tests | xargs perl -pi -e ' + my %unit = (s => 1, m => 60, h => 3600, d => 86400); + s/^duration:\s*(\d+)([smhd])\s*$/"duration: " . $1 * $unit{$2} * $ENV{duration_scale} . "s\n"/e' + echo "duration: $(( 5 * 60 * duration_scale ))s" > tests/main.fmf + echo "[Debug] Test time limits:"; grep -rE '^duration:' --include=main.fmf tests + + tar --exclude=./artifacts -C "${{ github.workspace }}" -czf /tmp/compose-tests.tgz . + ci/each-guest.sh ' + set -e + sudo podman cp /tmp/compose-tests.tgz $guest:/tmp/compose-tests.tgz + sudo podman exec $guest sh -c "rm -rf /compose-tests && mkdir -p /compose-tests && tar -C /compose-tests -xzf /tmp/compose-tests.tgz" + ' + + - name: Get system release + run: | + echo "system_release=$(sudo podman exec almalinux-0 cat /etc/almalinux-release)" >> $GITHUB_ENV + + - name: Add testing repository + # With pulp the testing repo comes from pulp (see the pulp step above). + # Kitten has no testing repository, so skip it there. + if: inputs.testing_repository && !inputs.pulp_repository && inputs.version_major != '10-kitten' + run: | + ci/each-guest.sh 'sudo podman exec $guest dnf install -y almalinux-release-testing' + + - name: Prepare test infrastructure + run: | + cat > /tmp/ct-prepare.sh <<'REMOTE' + enable_repo=${{ env.dnf_crb_repo }} + dnf_opts="--enablerepo=${enable_repo,,}" + if [ "${{ inputs.disable_default_repos }}" = "true" ]; then + # Disable all enabled system repositories + for enabled_repo in $(dnf repolist --enabled | awk '/repo id/ {A=1; next} A {print $1}' | grep -v -E 'pungi|pulp'); do + echo "[Debug] Disable '${enabled_repo}' repository" + dnf config-manager --set-disabled ${enabled_repo} + done + dnf_opts= + fi + case ${{ inputs.version_major }} in + 10-kitten) + dnf install -y ${dnf_opts} epel-release almalinux-kitten-release-devel + [ "${{ inputs.disable_default_repos }}" = "false" ] && \ + dnf_opts="${dnf_opts} --enablerepo=devel" + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 10) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 9) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt initscripts-service + ;; + *) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt tmt-report-html initscripts + ;; + esac + export PATH=$PATH:/usr/local/bin + echo "[Debug] tmt version: $(tmt --version)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-prepare.sh' + + - name: Split tests between the containers + run: | + # Enabled tests for this distro, split by duration (ci/test-durations/) + # into one tmt 'test --name' regex per container, pushed to /tmp/ct-shard.regex + sudo podman exec almalinux-0 sh -c 'export PATH=$PATH:/usr/local/bin; cd /compose-tests; tmt -c distro=centos-stream-${{ env.release_version }} -c container=yes tests ls --enabled' \ + | grep '^/' > ./tests_enabled.txt + python3 ci/shard.py split --shards ${shards} --variant '${{ inputs.version_major }}-ppc64le' < ./tests_enabled.txt > ./shards.txt + for guest in ${guests}; do + read -r regex <&3 + echo "${regex}" > ./ct-shard.regex + sudo podman cp ./ct-shard.regex ${guest}:/tmp/ct-shard.regex + done 3< ./shards.txt + + - name: Run tests + id: run-tests + continue-on-error: true + run: | + # continue-on-error lets us always collect logs and notify; capture the + # real exit code so a later step can tell a clean run (tmt 0=all pass, + # 1=some failed) from an aborted one (e.g. 2 = tmt error, or the + # container died mid-run). + set +e + cat > /tmp/ct-run.sh <<'REMOTE' + export PATH=$PATH:/usr/local/bin + cd /compose-tests + tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" + REMOTE + # Count root's inotify instances while the tests run: their peak, and who + # held them, shows how near the runner's limit the tests get (see "Start + # the containers") + sudo ci/inotify-peak.sh "${{ runner.temp }}/inotify-peak" & + inotify_peak=$! + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + sudo pkill -f ci/inotify-peak.sh; wait ${inotify_peak} + echo "[Debug] inotify instances in use at the peak while testing: $(head -1 "${{ runner.temp }}/inotify-peak") (of root, limit $(cat /proc/sys/fs/inotify/max_user_instances))" + tail -n +2 "${{ runner.temp }}/inotify-peak" | expand -t 6,20 | sed 's/^/[Debug] /' + # The worst exit code of the containers: one outside 0/1 (2 = tmt error, + # others = the container died) wins over 1 (some failed) over 0 (all pass). + rc=0 + for guest in ${guests}; do + guest_rc=$(cat ./tmt_rc.${guest} 2>/dev/null || echo 255) + echo "[Debug] ${guest} tmt exit code: ${guest_rc}" + case "${guest_rc}" in + 0) ;; + 1) [ "${rc}" = 0 ] && rc=1 ;; + *) rc=${guest_rc} ;; + esac + done + echo "[Debug] tmt exit code: ${rc}" + echo "tmt_rc=${rc}" >> "$GITHUB_OUTPUT" + exit ${rc} + + - name: ${{ inputs.version_major }} tests results + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + #[Debug] + find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml" -exec cat {} \; + + out=/tmp/ct-collect + rm -rf "$out"; mkdir -p "$out" + for f in $(find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml"); do + cp -av "$f" "$out/$(basename "$(dirname "$(dirname "$f")")").results.yaml" + done + chmod -R a+r "$out" + REMOTE + # Prefix the results with the container: almalinux-0.legacy.results.yaml + rm -rf /tmp/ct-collect + sudo podman cp ${guest}:/tmp/ct-collect /tmp/ct-collect 2>/dev/null || mkdir -p /tmp/ct-collect + sudo chown -R "$(id -u):$(id -g)" /tmp/ct-collect + for f in /tmp/ct-collect/*.results.yaml; do + [ -e "$f" ] && mv "$f" "${{ env.artifacts_dir }}/${guest}.$(basename "$f")" + done + done + + - name: ${{ inputs.version_major }} tests log + run: | + for guest in ${guests}; do + #[Debug] + sudo podman exec ${guest} cat ${{ env.tmt_run_dir }}/log.txt + + sudo podman cp ${guest}:${{ env.tmt_run_dir }}/log.txt "${{ env.artifacts_dir }}/${guest}.log.txt" || true + done + + - name: ${{ inputs.version_major }} tests output + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + cd ${{ env.tmt_run_dir }}/ + tar cf /tmp/output.tar $( find . -name output.txt ) || true + REMOTE + sudo podman cp ${guest}:/tmp/output.tar "${{ env.artifacts_dir }}/${guest}.output.tar" || true + done + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests log + path: ${{ env.artifacts_dir }}/*.log.txt + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests results + path: ${{ env.artifacts_dir }}/*.results.yaml + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests output + path: ${{ env.artifacts_dir }}/*.output.tar + + - name: Collect the containers' journals + if: always() + run: | + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + mkdir -p "${{ runner.temp }}/journals" + for guest in ${guests}; do + sudo podman exec ${guest} journalctl --no-pager -o short-precise \ + > "${{ runner.temp }}/journals/${guest}.journal.txt" 2>&1 || true + done + + - uses: actions/upload-artifact@v7 + # The containers' systemd journals: services that do not start in them + if: always() + with: + name: ${{ inputs.version_major }}-ppc64le containers journal + path: ${{ runner.temp }}/journals/*.journal.txt + if-no-files-found: ignore + + - name: Evaluate test results + id: eval + if: always() + run: | + # Tally the collected results.yaml, build the failed-tests list, and + # decide whether the run actually completed. + if ! command -v yq >/dev/null 2>&1; then + case $(uname -m) in x86_64) yq_arch=amd64 ;; aarch64) yq_arch=arm64 ;; *) yq_arch=$(uname -m) ;; esac + sudo curl -fsSL -o /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${yq_arch} + sudo chmod +x /usr/local/bin/yq + fi + + # The failed-tests list gets ' ' lines so that a failed + # test is re-run in the container it failed in. A container with no + # results at all (every shard runs at least 0_common) means its run + # was aborted. + shopt -s nullglob + total=0; passed=0; failed=0; errored=0; empty_guest=false + : > "${artifacts_dir}/tests_failed.txt" + for guest in ${guests}; do + guest_total=0 + for f in "${artifacts_dir}/${guest}".*.results.yaml; do + guest_total=$(( guest_total + $(yq '. | length' "$f" 2>/dev/null || echo 0) )) + passed=$(( passed + $(yq '[.[] | select(.result == "pass")] | length' "$f" 2>/dev/null || echo 0) )) + failed=$(( failed + $(yq '[.[] | select(.result == "fail")] | length' "$f" 2>/dev/null || echo 0) )) + errored=$((errored + $(yq '[.[] | select(.result == "error")] | length' "$f" 2>/dev/null || echo 0) )) + yq '.[] | select(.result == "fail" or .result == "error") | .name' "$f" 2>/dev/null \ + | sed "s|^|${guest} |" >> "${artifacts_dir}/tests_failed.txt" + done + total=$(( total + guest_total )) + [ "${guest_total}" -eq 0 ] && empty_guest=true + done + failed_total=$(( failed + errored )) + + echo "total=${total}" >> "$GITHUB_OUTPUT" + echo "passed=${passed}" >> "$GITHUB_OUTPUT" + echo "failed=${failed_total}" >> "$GITHUB_OUTPUT" + + # Completeness: a clean tmt run exits 0 (all pass) or 1 (some failed). + # Anything else (2 = tmt error, the container died, ...) or + # zero recorded results means the run was aborted before finishing. + rc='${{ steps.run-tests.outputs.tmt_rc }}' + incomplete=false + case "${rc}" in 0|1) ;; *) incomplete=true ;; esac + [ "${total}" -eq 0 ] && incomplete=true + [ "${empty_guest}" = true ] && incomplete=true + echo "incomplete=${incomplete}" >> "$GITHUB_OUTPUT" + + echo "[Debug] total=${total} passed=${passed} failed=${failed_total} tmt_rc=${rc} incomplete=${incomplete}" + + - name: Re-run failed tests, prepare tests summary + # Only when the run completed and recorded failures; an aborted run is + # handled by the completeness gate at the end of the job. + if: ${{ always() && steps.eval.outputs.incomplete == 'false' && steps.eval.outputs.failed != '0' }} + run: | + # Tests results and the step exit code + rerun_results= + exit_code=0 + + # Read failed tests list: ' ' lines + while read -r guest test_failed; do + [ -n "${test_failed}" ] || continue + echo "[Debug] Re-run '${test_failed}' on ${guest}" + # Exact name as a tmt regex, '+' and '.' made literal (p_gcc-c++) + test_regex="^$(echo "${test_failed}" | sed 's/[+.]/[&]/g')\$" + + # Include failed test name into results summary + test_result="${test_failed}" + + # Re-run specific failed test, in the container it failed in + if [ "${{ inputs.rerun_failed }}" = "true" ]; then + if sudo podman exec ${guest} sh -c "export PATH=\$PATH:/usr/local/bin; cd /compose-tests; tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name \"${test_regex}\"" < /dev/null; then + test_result="${test_failed} [re-run ✅]" + else + test_result="${test_failed} [re-run ❌]" + exit_code=1 + fi + else + echo "[Debug]" + echo "If without re-run, fail this step as the '${test_failed}' test previously failed." + exit_code=1 + fi + + # Format test results in list format: 'item1', 'item2', ... + [ "x${rerun_results}" = "x" ] && rerun_results="'${test_result}'" || rerun_results="${rerun_results}, '${test_result}'" + + done < "${{ env.artifacts_dir }}/tests_failed.txt" + + # Export header and test results + [ "x${rerun_results}" = "x" ] && summary_header="✅ All tests pass." || summary_header="❌ Failed tests:" + echo "rerun_results=${rerun_results}" >> $GITHUB_ENV + echo "summary_header=${summary_header}" >> $GITHUB_ENV + + exit $exit_code + + - name: Print tests summary + id: summary + # Always run so the run-page summary and the Mattermost text are produced + # for both passing and failing runs. + if: always() + uses: actions/github-script@v9 + with: + result-encoding: string + script: | + const repos = [ + 'pulp - ${{ inputs.pulp_repository && '➕' || '➖' }}', + 'pungi - ${{ inputs.pungi_repository && '➕' || '➖' }}', + 'testing - ${{ inputs.testing_repository && '➕' || '➖' }}', + 'default repos - ${{ inputs.disable_default_repos && '➖' || '➕' }}', + ]; + // Re-run annotated failures (set only when the re-run step ran). + const rerun = [${{ env.rerun_results }}]; + + // Tests not run in the containers, hardcoded: the fmf adjust rules + // disable them for the tmt context container=yes, or they skip + // themselves when CONTAINERTEST=1 + const skipped = [ + '/tests/attr — a container may not mount a loop device', + '/tests/libbpf-tools — BPF programs on the runner\'s kernel, from a container', + '/tests/libvirt/net — a container may not create libvirt networks', + '/tests/perf — perf events of the runner\'s kernel, from a container', + '/tests/podman — no nested containers', + '/tests/selinux — no SELinux in a container', + '/tests/sysstat/iostat/cpu, /tests/sysstat/iostat/disk — a container may not drop the page cache', + '/tests/legacy/p_kvdo — kvdo is a kernel module', + '/tests/legacy/p_osbuild — an image build needs loop devices and mounts', + 'parts of other /tests/legacy tests — checks needing a whole system skip themselves', + ]; + + const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); + const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); + const total = parseInt('${{ steps.eval.outputs.total }}' || '0', 10); + const incomplete = '${{ steps.eval.outputs.incomplete }}' === 'true'; + const rc = '${{ steps.run-tests.outputs.tmt_rc }}'; + + // The counts above are the FIRST run. Reconcile them with the re-run + // outcomes: entries are annotated '… [re-run ✅]' (recovered) / + // '… [re-run ❌]' (still failing); unmarked means re-run was disabled. + const reran = rerun.some(r => r.includes('[re-run')); + const recovered = rerun.filter(r => r.includes('✅')).length; + const stillFailing = rerun.length - recovered; + + let header; + if (incomplete) header = `⚠️ Test run did NOT complete — only ${total} results recorded (tmt exit ${rc}).`; + else if (stillFailing > 0) header = `❌ ${stillFailing} of ${total} tests failed.`; + else if (recovered > 0) header = `✅ All ${total} tests passed (${recovered} recovered on re-run).`; + else header = `✅ All ${total} tests passed.`; + + const counts = `Passed: ${passed} · Failed: ${failed} · Total: ${total}`; + const rerunLine = reran ? `Re-run — recovered: ${recovered} · still failing: ${stillFailing}` : ''; + + // Run-page summary + let summary = core.summary + .addHeading('${{ env.system_release }}', '4') + .addHeading('Used repositories:', '4') + .addList(repos, true) + .addHeading(header, '4') + .addRaw(counts).addBreak(); + if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); + summary.addList(rerun, true) + .addHeading('Skipped in the containers:', '4') + .addList(skipped) + .write(); + + // Failure reason as the 'compose-fail' annotation of this job + if (incomplete || stillFailing > 0) { + core.error(`${header} ${rerun.join(', ')}`.trim().slice(0, 3800), { title: 'compose-fail' }); + } + + // Markdown text for the Mattermost notification + const runUrl = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const text = [ + '#### AlmaLinux ${{ inputs.version_major }} ppc64le — compose tests', + '**${{ env.system_release }}**', + '', + header, + counts, + ...(rerunLine ? [rerunLine] : []), + '', + '**Used repositories:**', + ...repos.map(r => '- ' + r), + ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), + '', + '**Skipped in the containers:**', + ...skipped.map(r => '- ' + r), + '', + '[Workflow run](' + runUrl + ')', + ].join('\n'); + core.setOutput('text', text); + + - name: Send notification to Mattermost + uses: mattermost/action-mattermost-notify@v2.1.0 + # Notify on both pass and fail; never fail the job on a notification error + if: ${{ always() && inputs.notify_mattermost }} + continue-on-error: true + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }} + MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }} + MATTERMOST_USERNAME: ${{ github.triggering_actor }} + TEXT: | + ${{ steps.summary.outputs.text }} + + - name: Fail if the test run did not complete + # Runs last (after notification) so an aborted/incomplete run — e.g. a + # test that kills the container, leaving an empty results.yaml — is + # reported as a failure instead of a false green. + if: ${{ always() && steps.eval.outputs.incomplete == 'true' }} run: | - echo "::error::No ppc64le compose tests on '${{ github.ref_name }}': run this workflow from the ci/tcg-ppc64le-s390x branch." - exit 0 + echo "::error::Test run did not complete: tmt exit '${{ steps.run-tests.outputs.tmt_rc }}', ${{ steps.eval.outputs.total }} results recorded." + exit 1 diff --git a/.github/workflows/almalinux-compose-test-s390x.yml b/.github/workflows/almalinux-compose-test-s390x.yml index 830a63f..17a3842 100644 --- a/.github/workflows/almalinux-compose-test-s390x.yml +++ b/.github/workflows/almalinux-compose-test-s390x.yml @@ -2,9 +2,13 @@ name: almalinux-compose-test-s390x run-name: AlmaLinux ${{ inputs.version_major }} compose test (s390x)${{ inputs.pungi_repository && ' +pungi' || '' }}${{ inputs.pulp_repository && ' +pulp' || '' }} -# Placeholder: GitHub dispatches a workflow only if a file with its path is on -# the default branch. The implementation (QEMU TCG emulated s390x guests) is -# on the ci/tcg-ppc64le-s390x branch: run the workflow from there. +# The tests run in AlmaLinux s390x containers (podman, systemd as PID 1, the +# almalinux/-init images, ci/container-guests.sh), split between the +# containers like on x86_64 and aarch64: in the AlmaLinux organization on a +# s390x runner, natively; elsewhere, e.g. in forks, on an x86_64 runner, their +# binaries run by QEMU user-mode emulation. The containers share the runner's +# kernel, so the tests run with CONTAINERTEST=1: those exercising the kernel, +# the boot or the hardware skip themselves. on: workflow_dispatch: @@ -50,13 +54,736 @@ on: type: boolean default: true + # Allow this workflow to be called from the 'almalinux-compose-test' orchestrator. + # 'workflow_call' has no 'choice' input type, so 'version_major' is a string here. + workflow_call: + inputs: + version_major: + description: 'AlmaLinux major version' + required: true + type: string + pungi_repository: + description: 'Add pungi repositories' + type: boolean + default: false + pulp_repository: + description: 'Add pulp repositories' + type: boolean + default: true + testing_repository: + description: 'Add testing repository' + type: boolean + default: false + disable_default_repos: + description: 'Disable system default repositories' + type: boolean + default: false + rerun_failed: + description: 'Re-run failed tests' + type: boolean + default: true + notify_mattermost: + description: 'Send results notification to Mattermost' + type: boolean + default: true + jobs: compose-test: name: AlmaLinux ${{ inputs.version_major }} - runs-on: ubuntu-26.04-s390x + # In the AlmaLinux organization the containers run natively on an s390x + # runner; elsewhere, e.g. in forks, on an x86_64 runner, emulated. KVM is + # of no use (no nested-virt). GitHub-hosted runners stop jobs after 6 hours. + runs-on: >- + ${{ + github.repository_owner == 'AlmaLinux' && + 'ubuntu-26.04-s390x' + || + 'ubuntu-26.04' + }} + timeout-minutes: 360 + + env: + # Number of guests the tests are split between, run in parallel + shards: 4 + # Whether the containers run natively (see runs-on) or emulated + native: ${{ github.repository_owner == 'AlmaLinux' }} + # The factor the tests' time limits ('duration') are scaled by: emulation + # is several times slower than the hardware they are meant for, and + # an s390x runner may be slower than an x86_64 one + duration_scale: ${{ github.repository_owner == 'AlmaLinux' && 2 || 8 }} + steps: - - name: Not implemented on this branch + - uses: actions/checkout@v7 + + - name: Prepare stuff + run: | + # Environment variables + pungi_latest_result=latest_result_almalinux + pulp_root=${{ inputs.version_major }} + tmt_options= + rpm_gpg_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-AlmaLinux + release_version=${{ inputs.version_major }} + dnf_crb_repo='CRB' + dnf_extras_repo='extras' + # systemd-enabled image of the version + image=docker.io/almalinux/${{ inputs.version_major }}-init:latest + + case ${{ inputs.version_major }} in + 8) + dnf_crb_repo='PowerTools' + ;; + 9) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-9" + ;; + 10) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + ;; + 10-kitten) + release_version=10 + pulp_root=kitten-10 + pungi_latest_result="${pungi_latest_result}-kitten" + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + dnf_extras_repo='extras-common' + ;; + esac + # Release major version + echo "release_version=${release_version}" >> $GITHUB_ENV + + # Pulp repository root of the release + echo "pulp_root=${pulp_root}" >> $GITHUB_ENV + + # Pungi repository latest results directory + echo "pungi_latest_result=${pungi_latest_result}" >> $GITHUB_ENV + + # Name of repository to enable (PowerTools/CRB) + if [ "x${{ inputs.pulp_repository }}" = "xtrue" ]; then + # Lowercase the name for path in pulp's URL + pulp_crb_repo="${dnf_crb_repo,,}" + fi + echo "dnf_crb_repo=${dnf_crb_repo}" >> $GITHUB_ENV + echo "pulp_crb_repo=${pulp_crb_repo}" >> $GITHUB_ENV + echo "dnf_extras_repo=${dnf_extras_repo}" >> $GITHUB_ENV + + # TMT extra options; these go right after 'tmt', before the + # subcommand -- tmt 1.78 dropped the step-level '--feeling-safe'. + echo "tmt_options=${tmt_options}" >> $GITHUB_ENV + + # RPM_GPG_KEY file + echo "rpm_gpg_key=${rpm_gpg_key}" >> $GITHUB_ENV + + # Image the containers run + echo "image=${image}" >> $GITHUB_ENV + + # TMT tests run directory (in the containers) + echo "tmt_run_dir=/var/tmp/tmt/run-001" >> $GITHUB_ENV + + # Names of the containers, one per shard of the tests + echo "guests=$(seq -f 'almalinux-%g' -s ' ' 0 $(( shards - 1 )))" >> $GITHUB_ENV + + # Local directory on the runner to gather artifacts pulled from the containers + artifacts_dir="${{ github.workspace }}/artifacts" + mkdir -p "${artifacts_dir}" + echo "artifacts_dir=${artifacts_dir}" >> $GITHUB_ENV + + # For nvidia drivers tests + case ${{ inputs.version_major }} in + 8) + nvidia=false + ;; + *) + nvidia=true + ;; + esac + echo "nvidia=${nvidia}" >> $GITHUB_ENV + + - name: Configure apt + run: | + # Prefer the regional EC2 mirror over archive.ubuntu.com (throttled/unreachable + # from the runners at times), and make apt fail fast with retries. + imds_token=$(curl -sf --max-time 5 -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' || true) + region=$(curl -sf --max-time 5 -H "X-aws-ec2-metadata-token: $imds_token" http://169.254.169.254/latest/meta-data/placement/region || true) + if [ -n "$region" ]; then + sudo sed -i "s|https\?://archive.ubuntu.com/ubuntu|http://${region}.ec2.archive.ubuntu.com/ubuntu|" /etc/apt/sources.list.d/ubuntu.sources + fi + printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' | sudo tee /etc/apt/apt.conf.d/99-ci-retries + + - name: Install podman + run: | + command -v podman > /dev/null || { sudo apt-get -y update; sudo apt-get -y install podman; } + echo "[Debug] $(uname -m) runner, $(podman --version)" + + - name: Install QEMU user-mode emulation + if: env.native != 'true' + run: | + # Its binfmt_misc handlers let the kernel run s390x binaries + sudo apt-get -y update + sudo apt-get -y install qemu-user-binfmt + # Registered without the C (credentials) flag, setuid programs (passwd, + # newgrp, su, ...) would run without their privileges: register again with it + handler=$(grep -h "^:qemu-s390x:" /usr/lib/binfmt.d/qemu-s390x.conf) + echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-s390x > /dev/null + echo "${handler%:*}:OCPF" | sudo tee /proc/sys/fs/binfmt_misc/register > /dev/null + cat /proc/sys/fs/binfmt_misc/qemu-s390x + echo "[Debug] $(qemu-s390x --version | head -1), $(podman --version)" + + - name: Unload the runner's confining AppArmor profiles + if: env.native == 'true' + run: | + # The runner (an Incus container on the s390x one) confines programs + # by path with its AppArmor profiles, the containers' ones too: started + # as /usr/bin/hostname, /usr/bin/gs, ... a program gets the runner's + # profile, which fails it on the containers' files, even in complain + # mode: hostname fails to load libc ("cannot apply additional memory + # protection after relocation"). Unload these profiles: the runner is + # thrown away after the job. Not those flagged unconfined: they confine + # nothing, they grant (user namespaces to podman, crun, ...), and the + # containers run under one of them. + if [ ! -d /sys/kernel/security/apparmor ]; then + echo "[Debug] No AppArmor on the runner" + exit 0 + fi + failed= + for profile in /etc/apparmor.d/*; do + [ -f "${profile}" ] || continue + grep -q 'flags=([^)]*unconfined' "${profile}" && continue + sudo apparmor_parser -R "${profile}" 2> /dev/null || failed="${failed} ${profile##*/}" + done + [ -z "${failed}" ] || echo "[Debug] Not unloaded (not loaded?):${failed}" + sudo aa-status 2>&1 | head -5 || true + + - name: Start the containers + run: | + # The containers' root is the runner's, so their systemds and services + # share its inotify instances: with too few D-Bus failed to start in some + # ("dirwatch_new: Too many open files"). Raise the limit where the runner + # allows it (a container runner does not), and show how much is used. + echo 8192 | sudo tee /proc/sys/fs/inotify/max_user_instances > /dev/null 2>&1 || + echo "[Debug] inotify instances limit not raised: not allowed on the runner" + sudo guests="${guests}" ci/container-guests.sh s390x "${image}" + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + + - name: Create AlmaLinux pungi repository + if: inputs.pungi_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pungi.repo" + [almalinux-${{ env.release_version }}-appstream-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/AppStream/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-appstream-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-baseos-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/BaseOS/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-baseos-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_crb_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_extras_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux pulp repository + if: inputs.pulp_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pulp.repo" + [almalinux-${{ env.pulp_root }}-appstream-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-appstream-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-appstream-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-baseos-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-baseos-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-baseos-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux testing pulp repository + # Kitten has no testing repository, so skip it there + if: inputs.pulp_repository && inputs.testing_repository && inputs.version_major != '10-kitten' + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-testing-pulp.repo" + [almalinux-${{ env.pulp_root }}-testing-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-testing-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-testing-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux nvidia pulp repository + if: env.nvidia == 'true' && inputs.pulp_repository + run: | + # Repositories of the 'almalinux-nvidia' product + # https://build.almalinux.org/product/777 + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-nvidia-pulp.repo" + [almalinux-${{ env.pulp_root }}-nvidia-pulp] + baseurl = https://build.almalinux.org/pulp/content/copr/anfimovdm-almalinux-nvidia-almalinux-${{ env.pulp_root }}-$arch-dr/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-nvidia-pulp + gpgkey=file://${{ env.rpm_gpg_key }},file:///etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA-CUDA-${{ env.release_version }} + EOF + + # No 'Publish tested repository state': ci/compose_state.py gates the + # x86_64 and aarch64 publishing only + + - name: Push repositories to the containers + run: | + ci/each-guest.sh ' + set -e + for repo in almalinux-pungi almalinux-pulp almalinux-testing-pulp almalinux-nvidia-pulp; do + f="${{ env.artifacts_dir }}/${repo}.repo" + if [ -e "$f" ]; then + echo "[Debug] Installing ${repo}.repo" + sudo podman cp "$f" $guest:/etc/yum.repos.d/${repo}.repo + fi + done + ' + + - name: Update the system + run: | + # No reboot: the containers run the runner's kernel + ci/each-guest.sh ' + set -e + sudo podman exec $guest dnf -y clean all + sudo podman exec $guest dnf -y upgrade + # The image has packages an installed system does not (curl-minimal, + # which conflicts with the curl the tests install) and lacks some it + # has (hostname) + sudo podman exec $guest sh -c "! rpm -q curl-minimal >/dev/null || dnf -y swap curl-minimal curl" + sudo podman exec $guest dnf -y install hostname + ' + + - name: Get compose-tests onto the containers + run: | + # Scale the tests' time limits in the copy the containers get, as plain + # values: the tmt 1.26 of AlmaLinux 8 has neither policies nor + # duration multiplication. Tests without one get tmt's default, 5m. + grep -rlE '^duration:' --include=main.fmf tests | xargs perl -pi -e ' + my %unit = (s => 1, m => 60, h => 3600, d => 86400); + s/^duration:\s*(\d+)([smhd])\s*$/"duration: " . $1 * $unit{$2} * $ENV{duration_scale} . "s\n"/e' + echo "duration: $(( 5 * 60 * duration_scale ))s" > tests/main.fmf + echo "[Debug] Test time limits:"; grep -rE '^duration:' --include=main.fmf tests + + tar --exclude=./artifacts -C "${{ github.workspace }}" -czf /tmp/compose-tests.tgz . + ci/each-guest.sh ' + set -e + sudo podman cp /tmp/compose-tests.tgz $guest:/tmp/compose-tests.tgz + sudo podman exec $guest sh -c "rm -rf /compose-tests && mkdir -p /compose-tests && tar -C /compose-tests -xzf /tmp/compose-tests.tgz" + ' + + - name: Get system release + run: | + echo "system_release=$(sudo podman exec almalinux-0 cat /etc/almalinux-release)" >> $GITHUB_ENV + + - name: Add testing repository + # With pulp the testing repo comes from pulp (see the pulp step above). + # Kitten has no testing repository, so skip it there. + if: inputs.testing_repository && !inputs.pulp_repository && inputs.version_major != '10-kitten' + run: | + ci/each-guest.sh 'sudo podman exec $guest dnf install -y almalinux-release-testing' + + - name: Prepare test infrastructure + run: | + cat > /tmp/ct-prepare.sh <<'REMOTE' + enable_repo=${{ env.dnf_crb_repo }} + dnf_opts="--enablerepo=${enable_repo,,}" + if [ "${{ inputs.disable_default_repos }}" = "true" ]; then + # Disable all enabled system repositories + for enabled_repo in $(dnf repolist --enabled | awk '/repo id/ {A=1; next} A {print $1}' | grep -v -E 'pungi|pulp'); do + echo "[Debug] Disable '${enabled_repo}' repository" + dnf config-manager --set-disabled ${enabled_repo} + done + dnf_opts= + fi + case ${{ inputs.version_major }} in + 10-kitten) + dnf install -y ${dnf_opts} epel-release almalinux-kitten-release-devel + [ "${{ inputs.disable_default_repos }}" = "false" ] && \ + dnf_opts="${dnf_opts} --enablerepo=devel" + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 10) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 9) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt initscripts-service + ;; + *) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt tmt-report-html initscripts + ;; + esac + export PATH=$PATH:/usr/local/bin + echo "[Debug] tmt version: $(tmt --version)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-prepare.sh' + + - name: Split tests between the containers + run: | + # Enabled tests for this distro, split by duration (ci/test-durations/) + # into one tmt 'test --name' regex per container, pushed to /tmp/ct-shard.regex + sudo podman exec almalinux-0 sh -c 'export PATH=$PATH:/usr/local/bin; cd /compose-tests; tmt -c distro=centos-stream-${{ env.release_version }} -c container=yes tests ls --enabled' \ + | grep '^/' > ./tests_enabled.txt + python3 ci/shard.py split --shards ${shards} --variant '${{ inputs.version_major }}-s390x' < ./tests_enabled.txt > ./shards.txt + for guest in ${guests}; do + read -r regex <&3 + echo "${regex}" > ./ct-shard.regex + sudo podman cp ./ct-shard.regex ${guest}:/tmp/ct-shard.regex + done 3< ./shards.txt + + - name: Run tests + id: run-tests + continue-on-error: true + run: | + # continue-on-error lets us always collect logs and notify; capture the + # real exit code so a later step can tell a clean run (tmt 0=all pass, + # 1=some failed) from an aborted one (e.g. 2 = tmt error, or the + # container died mid-run). + set +e + cat > /tmp/ct-run.sh <<'REMOTE' + export PATH=$PATH:/usr/local/bin + cd /compose-tests + tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" + REMOTE + # Count root's inotify instances while the tests run: their peak, and who + # held them, shows how near the runner's limit the tests get (see "Start + # the containers") + sudo ci/inotify-peak.sh "${{ runner.temp }}/inotify-peak" & + inotify_peak=$! + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + sudo pkill -f ci/inotify-peak.sh; wait ${inotify_peak} + echo "[Debug] inotify instances in use at the peak while testing: $(head -1 "${{ runner.temp }}/inotify-peak") (of root, limit $(cat /proc/sys/fs/inotify/max_user_instances))" + tail -n +2 "${{ runner.temp }}/inotify-peak" | expand -t 6,20 | sed 's/^/[Debug] /' + # The worst exit code of the containers: one outside 0/1 (2 = tmt error, + # others = the container died) wins over 1 (some failed) over 0 (all pass). + rc=0 + for guest in ${guests}; do + guest_rc=$(cat ./tmt_rc.${guest} 2>/dev/null || echo 255) + echo "[Debug] ${guest} tmt exit code: ${guest_rc}" + case "${guest_rc}" in + 0) ;; + 1) [ "${rc}" = 0 ] && rc=1 ;; + *) rc=${guest_rc} ;; + esac + done + echo "[Debug] tmt exit code: ${rc}" + echo "tmt_rc=${rc}" >> "$GITHUB_OUTPUT" + exit ${rc} + + - name: ${{ inputs.version_major }} tests results + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + #[Debug] + find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml" -exec cat {} \; + + out=/tmp/ct-collect + rm -rf "$out"; mkdir -p "$out" + for f in $(find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml"); do + cp -av "$f" "$out/$(basename "$(dirname "$(dirname "$f")")").results.yaml" + done + chmod -R a+r "$out" + REMOTE + # Prefix the results with the container: almalinux-0.legacy.results.yaml + rm -rf /tmp/ct-collect + sudo podman cp ${guest}:/tmp/ct-collect /tmp/ct-collect 2>/dev/null || mkdir -p /tmp/ct-collect + sudo chown -R "$(id -u):$(id -g)" /tmp/ct-collect + for f in /tmp/ct-collect/*.results.yaml; do + [ -e "$f" ] && mv "$f" "${{ env.artifacts_dir }}/${guest}.$(basename "$f")" + done + done + + - name: ${{ inputs.version_major }} tests log + run: | + for guest in ${guests}; do + #[Debug] + sudo podman exec ${guest} cat ${{ env.tmt_run_dir }}/log.txt + + sudo podman cp ${guest}:${{ env.tmt_run_dir }}/log.txt "${{ env.artifacts_dir }}/${guest}.log.txt" || true + done + + - name: ${{ inputs.version_major }} tests output + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + cd ${{ env.tmt_run_dir }}/ + tar cf /tmp/output.tar $( find . -name output.txt ) || true + REMOTE + sudo podman cp ${guest}:/tmp/output.tar "${{ env.artifacts_dir }}/${guest}.output.tar" || true + done + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests log + path: ${{ env.artifacts_dir }}/*.log.txt + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests results + path: ${{ env.artifacts_dir }}/*.results.yaml + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests output + path: ${{ env.artifacts_dir }}/*.output.tar + + - name: Collect the containers' journals + if: always() + run: | + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + mkdir -p "${{ runner.temp }}/journals" + for guest in ${guests}; do + sudo podman exec ${guest} journalctl --no-pager -o short-precise \ + > "${{ runner.temp }}/journals/${guest}.journal.txt" 2>&1 || true + done + + - uses: actions/upload-artifact@v7 + # The containers' systemd journals: services that do not start in them + if: always() + with: + name: ${{ inputs.version_major }}-s390x containers journal + path: ${{ runner.temp }}/journals/*.journal.txt + if-no-files-found: ignore + + - name: Evaluate test results + id: eval + if: always() + run: | + # Tally the collected results.yaml, build the failed-tests list, and + # decide whether the run actually completed. + if ! command -v yq >/dev/null 2>&1; then + case $(uname -m) in x86_64) yq_arch=amd64 ;; aarch64) yq_arch=arm64 ;; *) yq_arch=$(uname -m) ;; esac + sudo curl -fsSL -o /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${yq_arch} + sudo chmod +x /usr/local/bin/yq + fi + + # The failed-tests list gets ' ' lines so that a failed + # test is re-run in the container it failed in. A container with no + # results at all (every shard runs at least 0_common) means its run + # was aborted. + shopt -s nullglob + total=0; passed=0; failed=0; errored=0; empty_guest=false + : > "${artifacts_dir}/tests_failed.txt" + for guest in ${guests}; do + guest_total=0 + for f in "${artifacts_dir}/${guest}".*.results.yaml; do + guest_total=$(( guest_total + $(yq '. | length' "$f" 2>/dev/null || echo 0) )) + passed=$(( passed + $(yq '[.[] | select(.result == "pass")] | length' "$f" 2>/dev/null || echo 0) )) + failed=$(( failed + $(yq '[.[] | select(.result == "fail")] | length' "$f" 2>/dev/null || echo 0) )) + errored=$((errored + $(yq '[.[] | select(.result == "error")] | length' "$f" 2>/dev/null || echo 0) )) + yq '.[] | select(.result == "fail" or .result == "error") | .name' "$f" 2>/dev/null \ + | sed "s|^|${guest} |" >> "${artifacts_dir}/tests_failed.txt" + done + total=$(( total + guest_total )) + [ "${guest_total}" -eq 0 ] && empty_guest=true + done + failed_total=$(( failed + errored )) + + echo "total=${total}" >> "$GITHUB_OUTPUT" + echo "passed=${passed}" >> "$GITHUB_OUTPUT" + echo "failed=${failed_total}" >> "$GITHUB_OUTPUT" + + # Completeness: a clean tmt run exits 0 (all pass) or 1 (some failed). + # Anything else (2 = tmt error, the container died, ...) or + # zero recorded results means the run was aborted before finishing. + rc='${{ steps.run-tests.outputs.tmt_rc }}' + incomplete=false + case "${rc}" in 0|1) ;; *) incomplete=true ;; esac + [ "${total}" -eq 0 ] && incomplete=true + [ "${empty_guest}" = true ] && incomplete=true + echo "incomplete=${incomplete}" >> "$GITHUB_OUTPUT" + + echo "[Debug] total=${total} passed=${passed} failed=${failed_total} tmt_rc=${rc} incomplete=${incomplete}" + + - name: Re-run failed tests, prepare tests summary + # Only when the run completed and recorded failures; an aborted run is + # handled by the completeness gate at the end of the job. + if: ${{ always() && steps.eval.outputs.incomplete == 'false' && steps.eval.outputs.failed != '0' }} + run: | + # Tests results and the step exit code + rerun_results= + exit_code=0 + + # Read failed tests list: ' ' lines + while read -r guest test_failed; do + [ -n "${test_failed}" ] || continue + echo "[Debug] Re-run '${test_failed}' on ${guest}" + # Exact name as a tmt regex, '+' and '.' made literal (p_gcc-c++) + test_regex="^$(echo "${test_failed}" | sed 's/[+.]/[&]/g')\$" + + # Include failed test name into results summary + test_result="${test_failed}" + + # Re-run specific failed test, in the container it failed in + if [ "${{ inputs.rerun_failed }}" = "true" ]; then + if sudo podman exec ${guest} sh -c "export PATH=\$PATH:/usr/local/bin; cd /compose-tests; tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name \"${test_regex}\"" < /dev/null; then + test_result="${test_failed} [re-run ✅]" + else + test_result="${test_failed} [re-run ❌]" + exit_code=1 + fi + else + echo "[Debug]" + echo "If without re-run, fail this step as the '${test_failed}' test previously failed." + exit_code=1 + fi + + # Format test results in list format: 'item1', 'item2', ... + [ "x${rerun_results}" = "x" ] && rerun_results="'${test_result}'" || rerun_results="${rerun_results}, '${test_result}'" + + done < "${{ env.artifacts_dir }}/tests_failed.txt" + + # Export header and test results + [ "x${rerun_results}" = "x" ] && summary_header="✅ All tests pass." || summary_header="❌ Failed tests:" + echo "rerun_results=${rerun_results}" >> $GITHUB_ENV + echo "summary_header=${summary_header}" >> $GITHUB_ENV + + exit $exit_code + + - name: Print tests summary + id: summary + # Always run so the run-page summary and the Mattermost text are produced + # for both passing and failing runs. + if: always() + uses: actions/github-script@v9 + with: + result-encoding: string + script: | + const repos = [ + 'pulp - ${{ inputs.pulp_repository && '➕' || '➖' }}', + 'pungi - ${{ inputs.pungi_repository && '➕' || '➖' }}', + 'testing - ${{ inputs.testing_repository && '➕' || '➖' }}', + 'default repos - ${{ inputs.disable_default_repos && '➖' || '➕' }}', + ]; + // Re-run annotated failures (set only when the re-run step ran). + const rerun = [${{ env.rerun_results }}]; + + // Tests not run in the containers, hardcoded: the fmf adjust rules + // disable them for the tmt context container=yes, or they skip + // themselves when CONTAINERTEST=1 + const skipped = [ + '/tests/attr — a container may not mount a loop device', + '/tests/libbpf-tools — BPF programs on the runner\'s kernel, from a container', + '/tests/libvirt/net — a container may not create libvirt networks', + '/tests/perf — perf events of the runner\'s kernel, from a container', + '/tests/podman — no nested containers', + '/tests/selinux — no SELinux in a container', + '/tests/sysstat/iostat/cpu, /tests/sysstat/iostat/disk — a container may not drop the page cache', + '/tests/legacy/p_kvdo — kvdo is a kernel module', + '/tests/legacy/p_osbuild — an image build needs loop devices and mounts', + ...('${{ env.native }}' === 'true' ? [] : ['/tests/libvirt/socket: sysinfo check — no /proc/sysinfo on the runner\'s kernel']), + 'parts of other /tests/legacy tests — checks needing a whole system skip themselves', + ]; + + const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); + const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); + const total = parseInt('${{ steps.eval.outputs.total }}' || '0', 10); + const incomplete = '${{ steps.eval.outputs.incomplete }}' === 'true'; + const rc = '${{ steps.run-tests.outputs.tmt_rc }}'; + + // The counts above are the FIRST run. Reconcile them with the re-run + // outcomes: entries are annotated '… [re-run ✅]' (recovered) / + // '… [re-run ❌]' (still failing); unmarked means re-run was disabled. + const reran = rerun.some(r => r.includes('[re-run')); + const recovered = rerun.filter(r => r.includes('✅')).length; + const stillFailing = rerun.length - recovered; + + let header; + if (incomplete) header = `⚠️ Test run did NOT complete — only ${total} results recorded (tmt exit ${rc}).`; + else if (stillFailing > 0) header = `❌ ${stillFailing} of ${total} tests failed.`; + else if (recovered > 0) header = `✅ All ${total} tests passed (${recovered} recovered on re-run).`; + else header = `✅ All ${total} tests passed.`; + + const counts = `Passed: ${passed} · Failed: ${failed} · Total: ${total}`; + const rerunLine = reran ? `Re-run — recovered: ${recovered} · still failing: ${stillFailing}` : ''; + + // Run-page summary + let summary = core.summary + .addHeading('${{ env.system_release }}', '4') + .addHeading('Used repositories:', '4') + .addList(repos, true) + .addHeading(header, '4') + .addRaw(counts).addBreak(); + if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); + summary.addList(rerun, true) + .addHeading('Skipped in the containers:', '4') + .addList(skipped) + .write(); + + // Failure reason as the 'compose-fail' annotation of this job + if (incomplete || stillFailing > 0) { + core.error(`${header} ${rerun.join(', ')}`.trim().slice(0, 3800), { title: 'compose-fail' }); + } + + // Markdown text for the Mattermost notification + const runUrl = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const text = [ + '#### AlmaLinux ${{ inputs.version_major }} s390x — compose tests', + '**${{ env.system_release }}**', + '', + header, + counts, + ...(rerunLine ? [rerunLine] : []), + '', + '**Used repositories:**', + ...repos.map(r => '- ' + r), + ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), + '', + '**Skipped in the containers:**', + ...skipped.map(r => '- ' + r), + '', + '[Workflow run](' + runUrl + ')', + ].join('\n'); + core.setOutput('text', text); + + - name: Send notification to Mattermost + uses: mattermost/action-mattermost-notify@v2.1.0 + # Notify on both pass and fail; never fail the job on a notification error + if: ${{ always() && inputs.notify_mattermost }} + continue-on-error: true + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }} + MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }} + MATTERMOST_USERNAME: ${{ github.triggering_actor }} + TEXT: | + ${{ steps.summary.outputs.text }} + + - name: Fail if the test run did not complete + # Runs last (after notification) so an aborted/incomplete run — e.g. a + # test that kills the container, leaving an empty results.yaml — is + # reported as a failure instead of a false green. + if: ${{ always() && steps.eval.outputs.incomplete == 'true' }} run: | - echo "::error::No s390x compose tests on '${{ github.ref_name }}': run this workflow from the ci/tcg-ppc64le-s390x branch." - exit 0 + echo "::error::Test run did not complete: tmt exit '${{ steps.run-tests.outputs.tmt_rc }}', ${{ steps.eval.outputs.total }} results recorded." + exit 1 diff --git a/ci/container-guests.sh b/ci/container-guests.sh new file mode 100755 index 0000000..29c91b8 --- /dev/null +++ b/ci/container-guests.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Start AlmaLinux containers of an architecture with podman, one per name in +# $guests, systemd as PID 1: natively on a runner of that architecture, or +# run by QEMU user-mode emulation (the qemu- binfmt_misc handler must be +# registered) on another one. Waits until systemd has booted every container +# and checks that the containers resolve names. +# Run commands in one with 'podman exec ...'; remove them with +# 'podman rm -f -t 0 ': they do not stop on systemd's stop signal under +# emulation. +# +# guests='almalinux-0 almalinux-1' ci/container-guests.sh ppc64le docker.io/almalinux/9-init:latest +# +# Extra 'podman run' options can be given in $podman_run_options. + +usage="usage: $0 ARCH IMAGE" +arch=${1:?${usage}} +image=${2:?${usage}} +set -e -o pipefail + +if [ "$(uname -m)" != "${arch}" ] && [ ! -e "/proc/sys/fs/binfmt_misc/qemu-${arch}" ]; then + echo "[Error] No qemu-${arch} binfmt_misc handler: ${arch} binaries cannot run here" >&2 + exit 1 +fi + +podman pull -q --platform "linux/${arch}" "${image}" > /dev/null + +# The containers reach the runner's network through podman's default one, +# 10.88.0.0/16. Some runners are in it themselves (an s390x runner's DNS server +# was 10.88.92.1): the containers then send to the bridge what is for them, and +# cannot resolve names. Use the first subnet that is none of the runner's +# routes, nameservers included, on a network of our own if it is not podman's. +subnet=$(python3 -I - <<'EOF' +import ipaddress, json, re, subprocess +routes = json.loads(subprocess.run(["ip", "-j", "-4", "route", "show", "table", "all"], + capture_output=True, text=True, check=True).stdout) +used = [ipaddress.ip_network(r["dst"], strict=False) for r in routes + if r.get("dst", "default") != "default" and not r.get("dev", "").startswith("podman")] +for resolv in ("/etc/resolv.conf", "/run/systemd/resolve/resolv.conf"): + try: + used += [ipaddress.ip_network(ns) for ns in + re.findall(r"^nameserver\s+([0-9.]+)\s*$", open(resolv).read(), re.M)] + except OSError: + pass +candidates = [f"10.{n}.0.0/16" for n in range(88, 100)] + ["172.30.0.0/16", "192.168.200.0/24"] +print(next(c for c in map(ipaddress.ip_network, candidates) + if not any(c.overlaps(u) for u in used if not u.is_loopback))) +EOF +) +network=podman +if [ "${subnet}" != 10.88.0.0/16 ]; then + network=compose-tests + echo "[Debug] podman's network 10.88.0.0/16 overlaps the runner's: the containers get ${subnet}" + podman network exists "${network}" || + podman network create --disable-dns --subnet "${subnet}" "${network}" > /dev/null +fi + +for guest in ${guests:?}; do + podman run -d --name "${guest}" --hostname "${guest}" --platform "linux/${arch}" \ + --network "${network}" --systemd=always ${podman_run_options:-} "${image}" > /dev/null +done + +for guest in ${guests}; do + for i in $(seq 1 60); do + state=$(podman exec "${guest}" systemctl is-system-running 2> /dev/null) || true + case "${state}" in running|degraded) break ;; esac + if [ "${i}" -eq 60 ]; then + echo "[Error] ${guest}: systemd not up after 10 minutes ('${state}')" >&2 + exit 1 + fi + sleep 10 + done + + # Under emulation the units with credentials (ImportCredential=) fail at + # boot with 243/CREDENTIALS: QEMU does not translate the mount calls systemd + # sets them up with. Run systemd-sysusers and systemd-tmpfiles ourselves, + # give journald (it imports credentials since AlmaLinux 10) a copy of its + # unit without them, and start again what failed at boot, D-Bus with it. + echo "[Debug] ${guest}: failed at boot: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)" + # The services installed later fail the same way: libvirtd and virtsecretd + # load an encrypted credential since libvirt 12.8 (AlmaLinux Kitten). Clear + # their credentials with a drop-in. Not with a top-level one for every + # service: under s390x emulation systemd reports a unit with a drop-in as + # changed on disk, and postgresql-setup then refuses to initialize the + # database. + if [ "$(uname -m)" != "${arch}" ]; then + podman exec "${guest}" sh -c ' + for unit in libvirtd virtsecretd; do + mkdir -p /etc/systemd/system/${unit}.service.d + printf "%s\n" "[Service]" LoadCredential= LoadCredentialEncrypted= \ + > /etc/systemd/system/${unit}.service.d/99-emulated-no-credentials.conf + done + ' + fi + podman exec "${guest}" sh -c ' + systemd-sysusers + systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev + unit=/usr/lib/systemd/system/systemd-journald.service + if grep -q "^ImportCredential=" "${unit}"; then + sed "/^ImportCredential=/d" "${unit}" > /etc/systemd/system/systemd-journald.service + fi + systemctl daemon-reload + failed=$(systemctl --failed --no-legend --plain | awk "{print \$1}" | + grep -vE "^systemd-(sysusers|tmpfiles-setup|tmpfiles-setup-dev|tmpfiles-setup-dev-early|firstboot)\.service$") + systemctl reset-failed + [ -z "${failed}" ] || systemctl restart ${failed} + ' || true + + echo "[Debug] ${guest}: $(podman exec "${guest}" sh -c 'echo "$(uname -m), $(cat /etc/almalinux-release), systemd $(systemctl is-system-running)"')" + echo "[Debug] ${guest}: still failed: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)" +done + +# The containers resolve names with the runner's upstream DNS servers, through +# podman's network. On the native s390x runner that failed now and then for a +# whole job: dnf then waited 2.5 minutes for the first name. Check it, show +# why it fails, and reload podman's network (its firewall rules) once. +dns_name=mirrors.almalinux.org + +dns_works() { + for guest in ${guests}; do + podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null || return 1 + done +} + +dns_debug() { + echo "[Debug] runner: ${dns_name} $(getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve'), nameservers: $(awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)" + if [ -r /run/systemd/resolve/resolv.conf ]; then + echo "[Debug] runner: systemd-resolved upstream nameservers: $(awk '/^nameserver/ {print $2}' /run/systemd/resolve/resolv.conf | xargs)" + fi + echo "[Debug] podman network: $(podman network inspect "${network}" --format '{{.Driver}} {{.NetworkInterface}} {{range .Subnets}}{{.Subnet}} {{end}}dns_enabled={{.DNSEnabled}}'), backend $(podman info --format '{{.Host.NetworkBackend}}'), ip_forward=$(cat /proc/sys/net/ipv4/ip_forward)" + for guest in ${guests}; do + echo "[Debug] ${guest}: nameservers: $(podman exec "${guest}" awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)," \ + "${dns_name} $(podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve')," \ + "HTTPS to 1.1.1.1: $(podman exec "${guest}" curl -s -o /dev/null --max-time 15 -w '%{http_code}' https://1.1.1.1/ || echo failed)" + done +} + +for i in $(seq 1 4); do + dns_works && break + if [ "${i}" -eq 4 ]; then + echo "[Warning] The containers cannot resolve ${dns_name}: reloading podman's network" + dns_debug + podman network reload --all > /dev/null + if dns_works; then + echo "[Debug] The containers resolve ${dns_name} after reloading podman's network" + break + else + echo "[Error] The containers still cannot resolve ${dns_name}" >&2 + dns_debug + exit 1 + fi + fi + sleep 15 +done diff --git a/ci/inotify-peak.sh b/ci/inotify-peak.sh new file mode 100755 index 0000000..c1bbc6f --- /dev/null +++ b/ci/inotify-peak.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Count the inotify instances root's processes hold every 5 seconds, until +# killed, and keep the largest count in FILE: on its first line the count and +# when, then how many each process held and where it ran (a container or the +# runner). Run as root, so that every process can be seen. +# +# sudo ci/inotify-peak.sh /tmp/inotify-peak & ...; sudo kill $!; cat /tmp/inotify-peak + +file=${1:?usage: $0 FILE} +# Containers' IDs and names: a process' cgroup tells the container it runs in +containers=$(podman ps --no-trunc --format '{{.ID}} {{.Names}}' 2> /dev/null) +peak=0 +while :; do + fds=$(find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null) + count=$(grep -c . <<< "${fds}") + if [ "${count}" -gt "${peak}" ]; then + peak=${count} + { + echo "${peak} at $(date -u +%H:%M:%S)" + cut -d/ -f3 <<< "${fds}" | sort | uniq -c | while read -r n pid; do + id=$(grep -o 'libpod-[0-9a-f]\{64\}' "/proc/${pid}/cgroup" 2> /dev/null | head -1) + where=$(awk -v id="${id#libpod-}" '$1 == id {print $2}' <<< "${containers}") + printf '%s\t%s\t%s\n' "${n}" "$(cat "/proc/${pid}/comm" 2> /dev/null || echo '(gone)')" "${where:-runner}" + done | awk -F '\t' '{n[$3 "\t" $2] += $1} END {for (k in n) print n[k] "\t" k}' | sort -k2,2 -k1,1rn + } > "${file}.new" && mv "${file}.new" "${file}" + fi + sleep 5 +done diff --git a/tests/attr/main.fmf b/tests/attr/main.fmf index 473d5de..7cf770c 100644 --- a/tests/attr/main.fmf +++ b/tests/attr/main.fmf @@ -2,3 +2,7 @@ summary: attr smoke tests description: Verify that attr works as expected test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: a container may not mount a loop device diff --git a/tests/legacy/functions.sh b/tests/legacy/functions.sh index 1c8bcb7..eedc9d4 100755 --- a/tests/legacy/functions.sh +++ b/tests/legacy/functions.sh @@ -210,7 +210,12 @@ function t_GetPkgVer # Description: get the arch function t_GetArch { - rpm -q kernel --queryformat '%{arch}\n' | head -n 1 + if rpm -q kernel > /dev/null 2>&1; then + rpm -q kernel --queryformat '%{arch}\n' | head -n 1 + else + # A container has no kernel package + uname -m + fi } # Set the arch diff --git a/tests/legacy/p_bash/bash_version_info_test.sh b/tests/legacy/p_bash/bash_version_info_test.sh index 988910e..b9da76a 100755 --- a/tests/legacy/p_bash/bash_version_info_test.sh +++ b/tests/legacy/p_bash/bash_version_info_test.sh @@ -5,6 +5,7 @@ t_Log "Running $0 - check that bash version info is the same with upstream." -bash --version | grep -qE "(i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64)-redhat-linux-gnu" +# s390x was built for the IBM vendor up to AlmaLinux 9 +bash --version | grep -qE "((i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64|s390x)-redhat|s390x-ibm)-linux-gnu" t_CheckExitStatus $? diff --git a/tests/legacy/p_exim/_helper_exim_ehlo.expect b/tests/legacy/p_exim/_helper_exim_ehlo.expect index 9250885..82ffb25 100755 --- a/tests/legacy/p_exim/_helper_exim_ehlo.expect +++ b/tests/legacy/p_exim/_helper_exim_ehlo.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Christoph Galuschka -set timeout 1 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 1 +} spawn telnet localhost 25 expect "220 *" send -- "ehlo localhost\n" diff --git a/tests/legacy/p_exim/_helper_exim_helo.expect b/tests/legacy/p_exim/_helper_exim_helo.expect index 941c55c..5743ebd 100755 --- a/tests/legacy/p_exim/_helper_exim_helo.expect +++ b/tests/legacy/p_exim/_helper_exim_helo.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Christoph Galuschka -set timeout 1 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 1 +} spawn telnet localhost 25 expect "220 *" send -- "helo localhost\n" diff --git a/tests/legacy/p_gzip/20-gzip-test b/tests/legacy/p_gzip/20-gzip-test index b27852e..62f610a 100755 --- a/tests/legacy/p_gzip/20-gzip-test +++ b/tests/legacy/p_gzip/20-gzip-test @@ -78,21 +78,13 @@ gzip -c $FILE | gunzip | grep -q 'gzip-test of single file' || ExitFail # Check the correct info is listed for -l the output is # compressed uncompressed ratio uncompressed_name -# 59 25 24.0% /var/tmp/gzip-test.txt -# -# The md5sum is 4cbaba004a815915904bd524ede5edec -# -# The 32-bit gzip behaves differently: -# -# compressed uncompressed ratio uncompressed_name # 59 25 -8.0% /var/tmp/gzip-test.txt -# -# The md5sum is caf6dee7a56022a840316520134967a4 t_Log "Check the output of -l" gzip $FILE -mdhash=$(gzip -l $FILE.gz | md5sum | cut -f1 -d' ') -[ "$mdhash" == "4cbaba004a815915904bd524ede5edec" -o "$mdhash" == "caf6dee7a56022a840316520134967a4" ] || ExitFail +# Only the uncompressed size and name: the compressed size depends on the +# compressor, e.g. 63 bytes on an s390x machine compressing in hardware (DFLTCC) +gzip -l $FILE.gz | awk 'NR == 2 {print $2, $4}' | grep -qx "25 $FILE" || ExitFail # Check that -q works which removes the header information from -l t_Log "Check -q reduced the output verbosity" diff --git a/tests/legacy/p_kvdo/test.sh b/tests/legacy/p_kvdo/test.sh index a9e7df2..76ae6e1 100755 --- a/tests/legacy/p_kvdo/test.sh +++ b/tests/legacy/p_kvdo/test.sh @@ -1,6 +1,11 @@ #!/bin/bash -e source ../functions.sh + +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running $0 - kvdo is a kernel module: running in container -> SKIP" + exit 0 +fi ./0-install_kvdo.sh ./kvdo_runtest.sh ./check_sb.sh \ No newline at end of file diff --git a/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect b/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect index 7e01d18..16f71e7 100755 --- a/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect +++ b/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Athmane Madjoudj -set timeout 2 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 2 +} spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/cat ssh_test_file send -- "\r" expect eof diff --git a/tests/legacy/p_openssh/_helper_sshd_user_login.expect b/tests/legacy/p_openssh/_helper_sshd_user_login.expect index 802f7b7..c85078e 100755 --- a/tests/legacy/p_openssh/_helper_sshd_user_login.expect +++ b/tests/legacy/p_openssh/_helper_sshd_user_login.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Athmane Madjoudj -set timeout 2 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 2 +} spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/ls match_max 6000 expect "*?assword:*" diff --git a/tests/legacy/p_osbuild/test.sh b/tests/legacy/p_osbuild/test.sh index 24d4a0f..7d16233 100755 --- a/tests/legacy/p_osbuild/test.sh +++ b/tests/legacy/p_osbuild/test.sh @@ -2,6 +2,19 @@ source ../functions.sh +# In an emulated guest (the ppc64le and s390x CI on forks) building an image +# takes more than an hour. +if [ "x${emulated_guest}" = "xtrue" ]; then + t_Log "Running $0 - osbuild: emulated guest -> SKIP" + exit 0 +fi + +# Building an image needs loop devices and mounts a container does not get +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running $0 - osbuild: running in container -> SKIP" + exit 0 +fi + # osbuild-composer ships x86_64 (v3-baseline) repo definitions only — there is # no x86_64_v2 arch block in the repositories JSON, and the build chroot pulls # v3 glibc which aborts on a v2 host. Detect the v2 variant once via the diff --git a/tests/legacy/p_tcpdump/01_tcpdump_nic.sh b/tests/legacy/p_tcpdump/01_tcpdump_nic.sh index 1cf4a8b..2e14421 100755 --- a/tests/legacy/p_tcpdump/01_tcpdump_nic.sh +++ b/tests/legacy/p_tcpdump/01_tcpdump_nic.sh @@ -4,6 +4,11 @@ t_Log "Running $0 - TCPdump test to Default-GW with IPv4" +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running in container -> SKIP" + exit 0 +fi + # Grabing Default-Router if NIC def_gw_out=$(ip route list default|grep "default via"|head -n 1) def_gw=$(echo $def_gw_out | awk '{print $3}') diff --git a/tests/libbpf-tools/main.fmf b/tests/libbpf-tools/main.fmf index 29ed631..a83da47 100644 --- a/tests/libbpf-tools/main.fmf +++ b/tests/libbpf-tools/main.fmf @@ -4,5 +4,8 @@ test: ./test.sh framework: beakerlib enabled: true adjust: - when: distro == centos-7 or distro == centos-stream-8 - enabled: false + - when: distro == centos-7 or distro == centos-stream-8 + enabled: false + - when: container == yes + enabled: false + because: BPF programs on the runner's kernel, from a container diff --git a/tests/libbpf-tools/test.sh b/tests/libbpf-tools/test.sh index 4910c69..12943b6 100755 --- a/tests/libbpf-tools/test.sh +++ b/tests/libbpf-tools/test.sh @@ -66,6 +66,8 @@ rlJournalStart rlPhaseEnd for tool in "${tools_versions_test[@]}"; do + # Not shipped for s390x + [ "${tool}" = bpf-numamove ] && [ "$(uname -m)" = s390x ] && continue rlPhaseStartTest "${tool} get version" rlRun "${tool} -V" rlPhaseEnd diff --git a/tests/libvirt/main.fmf b/tests/libvirt/main.fmf index 20ecbea..64cc5f5 100644 --- a/tests/libvirt/main.fmf +++ b/tests/libvirt/main.fmf @@ -5,5 +5,5 @@ framework: beakerlib require: [libvirt,python3-libvirt] enabled: true adjust: - when: arch == aarch64 - enabled: false + - when: arch == aarch64 + enabled: false diff --git a/tests/libvirt/net/main.fmf b/tests/libvirt/net/main.fmf index 56a373f..f3d75f0 100644 --- a/tests/libvirt/net/main.fmf +++ b/tests/libvirt/net/main.fmf @@ -1,3 +1,7 @@ summary: libvirt networking smoke tests description: libvirt networking smoke tests enabled: true +adjust+: + - when: container == yes + enabled: false + because: a container may not create libvirt networks diff --git a/tests/libvirt/socket/test.sh b/tests/libvirt/socket/test.sh index 919248d..9b275f5 100755 --- a/tests/libvirt/socket/test.sh +++ b/tests/libvirt/socket/test.sh @@ -23,9 +23,15 @@ rlJournalStart rlRun "virsh -c qemu:///system nodeinfo" rlPhaseEnd - rlPhaseStartTest "should get sysinfo on qemu" - rlRun "virsh -c qemu:///system sysinfo" - rlPhaseEnd + # On s390x libvirt reads the host's system information from /proc/sysinfo, + # which only an s390x kernel provides: not a container on another one's + if [ "$(uname -m)" = s390x ] && [ ! -e /proc/sysinfo ]; then + rlLog "No /proc/sysinfo (s390x on another architecture's kernel): not checking sysinfo" + else + rlPhaseStartTest "should get sysinfo on qemu" + rlRun "virsh -c qemu:///system sysinfo" + rlPhaseEnd + fi rlPhaseStartTest "should access test socket with python" rlRun -s "python3 ./pytestaccess.py" diff --git a/tests/perf/main.fmf b/tests/perf/main.fmf index cbe598f..4a0a543 100644 --- a/tests/perf/main.fmf +++ b/tests/perf/main.fmf @@ -2,3 +2,7 @@ summary: perf tests description: basic perf functionality tests test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: perf events of the runner's kernel, from a container diff --git a/tests/podman/main.fmf b/tests/podman/main.fmf index b138cbf..d9228be 100644 --- a/tests/podman/main.fmf +++ b/tests/podman/main.fmf @@ -6,3 +6,6 @@ enabled: true adjust: - when: distro == centos-7 enabled: false + - when: container == yes + enabled: false + because: no nested containers diff --git a/tests/selinux/main.fmf b/tests/selinux/main.fmf index 36d39af..4db328d 100644 --- a/tests/selinux/main.fmf +++ b/tests/selinux/main.fmf @@ -2,3 +2,7 @@ summary: selinux smoke tests description: basic selinux smoke tests test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: no SELinux in a container diff --git a/tests/sysstat/iostat/main.fmf b/tests/sysstat/iostat/main.fmf new file mode 100644 index 0000000..55ebeb8 --- /dev/null +++ b/tests/sysstat/iostat/main.fmf @@ -0,0 +1,4 @@ +adjust: + - when: container == yes + enabled: false + because: a container may not drop the page cache