From 1259bb30e7e3d57277ae1f3a44ae4fc77bd4c454 Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:30:02 +0200 Subject: [PATCH 1/8] ci: start AlmaLinux containers of an architecture with podman ci/container-guests.sh starts a container per name in $guests from an almalinux/-init image, systemd as PID 1, of the given architecture: natively on a runner of that architecture, run by QEMU user-mode emulation on another one. It waits until systemd has booted each container and fixes what does not boot under emulation: units with credentials (ImportCredential=) fail there with 243/CREDENTIALS, so it runs systemd-sysusers and systemd-tmpfiles itself, gives journald (which imports credentials since AlmaLinux 10) a copy of its unit without them, and restarts what failed at boot, D-Bus with it. The services installed later fail the same way: libvirtd and virtsecretd load an encrypted credential since libvirt 12.8 (AlmaLinux Kitten), so under emulation a drop-in clears their credentials. Not a top-level drop-in for every service: under s390x emulation systemd reports a unit with a drop-in as changed on disk, and postgresql-setup then refuses to initialize the database. It lists the units that failed at boot and those still failed. --- ci/container-guests.sh | 79 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100755 ci/container-guests.sh diff --git a/ci/container-guests.sh b/ci/container-guests.sh new file mode 100755 index 0000000..cb50915 --- /dev/null +++ b/ci/container-guests.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Start AlmaLinux containers of an architecture with podman, one per name in +# $guests, systemd as PID 1: natively on a runner of that architecture, or +# run by QEMU user-mode emulation (the qemu- binfmt_misc handler must be +# registered) on another one. Waits until systemd has booted every container. +# Run commands in one with 'podman exec ...'; remove them with +# 'podman rm -f -t 0 ': they do not stop on systemd's stop signal under +# emulation. +# +# guests='almalinux-0 almalinux-1' ci/container-guests.sh ppc64le docker.io/almalinux/9-init:latest +# +# Extra 'podman run' options can be given in $podman_run_options. + +usage="usage: $0 ARCH IMAGE" +arch=${1:?${usage}} +image=${2:?${usage}} +set -e -o pipefail + +if [ "$(uname -m)" != "${arch}" ] && [ ! -e "/proc/sys/fs/binfmt_misc/qemu-${arch}" ]; then + echo "[Error] No qemu-${arch} binfmt_misc handler: ${arch} binaries cannot run here" >&2 + exit 1 +fi + +podman pull -q --platform "linux/${arch}" "${image}" > /dev/null + +for guest in ${guests:?}; do + podman run -d --name "${guest}" --hostname "${guest}" --platform "linux/${arch}" \ + --systemd=always ${podman_run_options:-} "${image}" > /dev/null +done + +for guest in ${guests}; do + for i in $(seq 1 60); do + state=$(podman exec "${guest}" systemctl is-system-running 2> /dev/null) || true + case "${state}" in running|degraded) break ;; esac + if [ "${i}" -eq 60 ]; then + echo "[Error] ${guest}: systemd not up after 10 minutes ('${state}')" >&2 + exit 1 + fi + sleep 10 + done + + # Under emulation the units with credentials (ImportCredential=) fail at + # boot with 243/CREDENTIALS: QEMU does not translate the mount calls systemd + # sets them up with. Run systemd-sysusers and systemd-tmpfiles ourselves, + # give journald (it imports credentials since AlmaLinux 10) a copy of its + # unit without them, and start again what failed at boot, D-Bus with it. + echo "[Debug] ${guest}: failed at boot: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)" + # The services installed later fail the same way: libvirtd and virtsecretd + # load an encrypted credential since libvirt 12.8 (AlmaLinux Kitten). Clear + # their credentials with a drop-in. Not with a top-level one for every + # service: under s390x emulation systemd reports a unit with a drop-in as + # changed on disk, and postgresql-setup then refuses to initialize the + # database. + if [ "$(uname -m)" != "${arch}" ]; then + podman exec "${guest}" sh -c ' + for unit in libvirtd virtsecretd; do + mkdir -p /etc/systemd/system/${unit}.service.d + printf "%s\n" "[Service]" LoadCredential= LoadCredentialEncrypted= \ + > /etc/systemd/system/${unit}.service.d/99-emulated-no-credentials.conf + done + ' + fi + podman exec "${guest}" sh -c ' + systemd-sysusers + systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev + unit=/usr/lib/systemd/system/systemd-journald.service + if grep -q "^ImportCredential=" "${unit}"; then + sed "/^ImportCredential=/d" "${unit}" > /etc/systemd/system/systemd-journald.service + fi + systemctl daemon-reload + failed=$(systemctl --failed --no-legend --plain | awk "{print \$1}" | + grep -vE "^systemd-(sysusers|tmpfiles-setup|tmpfiles-setup-dev|tmpfiles-setup-dev-early|firstboot)\.service$") + systemctl reset-failed + [ -z "${failed}" ] || systemctl restart ${failed} + ' || true + + echo "[Debug] ${guest}: $(podman exec "${guest}" sh -c 'echo "$(uname -m), $(cat /etc/almalinux-release), systemd $(systemctl is-system-running)"')" + echo "[Debug] ${guest}: still failed: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)" +done From 88d7cbc73e215fbba9499a33c8814ff016c277a3 Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:30:30 +0200 Subject: [PATCH 2/8] ci: run the ppc64le compose tests in containers Run the compose tests in 4 AlmaLinux ppc64le containers (podman, systemd as PID 1, ci/container-guests.sh), split between them like on x86_64 and aarch64. The containers share the runner's kernel, so the tests run with the tmt context container=yes and CONTAINERTEST=1. In the AlmaLinux organization the containers run natively on a POWER runner, a POWER10 one for AlmaLinux 10 (ubuntu-26.04-ppc64le(-p10)). Elsewhere, e.g. in forks, they run on an x86_64 runner (ubuntu-26.04) under QEMU user-mode emulation, which needs QEMU 10.1 or newer (QEMU issue 1769) and the binfmt_misc C flag for setuid programs; the tests' time limits are scaled x8 there instead of x2, and they get emulated_guest=true. The containers are made more like an installed system: curl-minimal is swapped for curl and hostname installed, rsync (tmt from pip) on AlmaLinux 10 and initscripts (/sbin/service) on AlmaLinux 8. The runner's inotify instances limit, which the rootful containers share, is raised where the runner allows it, and its use shown. --- .../almalinux-compose-test-ppc64le.yml | 696 +++++++++++++++++- 1 file changed, 689 insertions(+), 7 deletions(-) diff --git a/.github/workflows/almalinux-compose-test-ppc64le.yml b/.github/workflows/almalinux-compose-test-ppc64le.yml index 58c8da9..4b74b29 100644 --- a/.github/workflows/almalinux-compose-test-ppc64le.yml +++ b/.github/workflows/almalinux-compose-test-ppc64le.yml @@ -2,9 +2,15 @@ name: almalinux-compose-test-ppc64le run-name: AlmaLinux ${{ inputs.version_major }} compose test (ppc64le)${{ inputs.pungi_repository && ' +pungi' || '' }}${{ inputs.pulp_repository && ' +pulp' || '' }} -# Placeholder: GitHub dispatches a workflow only if a file with its path is on -# the default branch. The implementation (QEMU TCG emulated ppc64le guests) is -# on the ci/tcg-ppc64le-s390x branch: run the workflow from there. +# The tests run in AlmaLinux ppc64le containers (podman, systemd as PID 1, the +# almalinux/-init images, ci/container-guests.sh), split between the +# containers like on x86_64 and aarch64: in the AlmaLinux organization on a +# ppc64le runner, natively; elsewhere, e.g. in forks, on an x86_64 runner, their +# binaries run by QEMU user-mode emulation. The containers share the runner's +# kernel, so the tests run with CONTAINERTEST=1: those exercising the kernel, +# the boot or the hardware skip themselves. Emulated, QEMU needs to be 10.1 or +# newer, so Ubuntu 26.04: the 8.2.2 of Ubuntu 24.04 miscompiles POWER9 vector +# loads/stores (QEMU issue 1769). on: workflow_dispatch: @@ -50,13 +56,689 @@ on: type: boolean default: true + # Allow this workflow to be called from the 'almalinux-compose-test' orchestrator. + # 'workflow_call' has no 'choice' input type, so 'version_major' is a string here. + workflow_call: + inputs: + version_major: + description: 'AlmaLinux major version' + required: true + type: string + pungi_repository: + description: 'Add pungi repositories' + type: boolean + default: false + pulp_repository: + description: 'Add pulp repositories' + type: boolean + default: true + testing_repository: + description: 'Add testing repository' + type: boolean + default: false + disable_default_repos: + description: 'Disable system default repositories' + type: boolean + default: false + rerun_failed: + description: 'Re-run failed tests' + type: boolean + default: true + notify_mattermost: + description: 'Send results notification to Mattermost' + type: boolean + default: true + jobs: compose-test: name: AlmaLinux ${{ inputs.version_major }} - runs-on: ubuntu-26.04-ppc64le-p10 + # In the AlmaLinux organization the containers run natively on a POWER + # runner, a POWER10 one for AlmaLinux 10 (built for POWER10); elsewhere, + # e.g. in forks, on an x86_64 runner, emulated. KVM is of no use (no + # nested-virt). GitHub-hosted runners stop jobs after 6 hours. + runs-on: >- + ${{ + github.repository_owner == 'AlmaLinux' && + (startsWith(inputs.version_major, '10') && 'ubuntu-26.04-ppc64le-p10' || 'ubuntu-26.04-ppc64le') + || + 'ubuntu-26.04' + }} + timeout-minutes: 360 + + env: + # Number of guests the tests are split between, run in parallel + shards: 4 + # Whether the containers run natively (see runs-on) or emulated + native: ${{ github.repository_owner == 'AlmaLinux' }} + # The factor the tests' time limits ('duration') are scaled by: emulation + # is several times slower than the hardware they are meant for, and + # a ppc64le runner may be slower than an x86_64 one + duration_scale: ${{ github.repository_owner == 'AlmaLinux' && 2 || 8 }} + steps: - - name: Not implemented on this branch + - uses: actions/checkout@v7 + + - name: Prepare stuff + run: | + # Environment variables + pungi_latest_result=latest_result_almalinux + pulp_root=${{ inputs.version_major }} + tmt_options= + rpm_gpg_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-AlmaLinux + release_version=${{ inputs.version_major }} + dnf_crb_repo='CRB' + dnf_extras_repo='extras' + # systemd-enabled image of the version + image=docker.io/almalinux/${{ inputs.version_major }}-init:latest + + case ${{ inputs.version_major }} in + 8) + dnf_crb_repo='PowerTools' + ;; + 9) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-9" + ;; + 10) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + ;; + 10-kitten) + release_version=10 + pulp_root=kitten-10 + pungi_latest_result="${pungi_latest_result}-kitten" + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + dnf_extras_repo='extras-common' + ;; + esac + # Release major version + echo "release_version=${release_version}" >> $GITHUB_ENV + + # Pulp repository root of the release + echo "pulp_root=${pulp_root}" >> $GITHUB_ENV + + # Pungi repository latest results directory + echo "pungi_latest_result=${pungi_latest_result}" >> $GITHUB_ENV + + # Name of repository to enable (PowerTools/CRB) + if [ "x${{ inputs.pulp_repository }}" = "xtrue" ]; then + # Lowercase the name for path in pulp's URL + pulp_crb_repo="${dnf_crb_repo,,}" + fi + echo "dnf_crb_repo=${dnf_crb_repo}" >> $GITHUB_ENV + echo "pulp_crb_repo=${pulp_crb_repo}" >> $GITHUB_ENV + echo "dnf_extras_repo=${dnf_extras_repo}" >> $GITHUB_ENV + + # TMT extra options; these go right after 'tmt', before the + # subcommand -- tmt 1.78 dropped the step-level '--feeling-safe'. + echo "tmt_options=${tmt_options}" >> $GITHUB_ENV + + # RPM_GPG_KEY file + echo "rpm_gpg_key=${rpm_gpg_key}" >> $GITHUB_ENV + + # Image the containers run + echo "image=${image}" >> $GITHUB_ENV + + # TMT tests run directory (in the containers) + echo "tmt_run_dir=/var/tmp/tmt/run-001" >> $GITHUB_ENV + + # Names of the containers, one per shard of the tests + echo "guests=$(seq -f 'almalinux-%g' -s ' ' 0 $(( shards - 1 )))" >> $GITHUB_ENV + + # Local directory on the runner to gather artifacts pulled from the containers + artifacts_dir="${{ github.workspace }}/artifacts" + mkdir -p "${artifacts_dir}" + echo "artifacts_dir=${artifacts_dir}" >> $GITHUB_ENV + + # For nvidia drivers tests + case ${{ inputs.version_major }} in + 8) + nvidia=false + ;; + *) + nvidia=true + ;; + esac + echo "nvidia=${nvidia}" >> $GITHUB_ENV + + - name: Configure apt + run: | + # Prefer the regional EC2 mirror over archive.ubuntu.com (throttled/unreachable + # from the runners at times), and make apt fail fast with retries. + imds_token=$(curl -sf --max-time 5 -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' || true) + region=$(curl -sf --max-time 5 -H "X-aws-ec2-metadata-token: $imds_token" http://169.254.169.254/latest/meta-data/placement/region || true) + if [ -n "$region" ]; then + sudo sed -i "s|https\?://archive.ubuntu.com/ubuntu|http://${region}.ec2.archive.ubuntu.com/ubuntu|" /etc/apt/sources.list.d/ubuntu.sources + fi + printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' | sudo tee /etc/apt/apt.conf.d/99-ci-retries + + - name: Install podman + run: | + command -v podman > /dev/null || { sudo apt-get -y update; sudo apt-get -y install podman; } + echo "[Debug] $(uname -m) runner, $(podman --version)" + + - name: Install QEMU user-mode emulation + if: env.native != 'true' + run: | + # Its binfmt_misc handlers let the kernel run ppc64le binaries + sudo apt-get -y update + sudo apt-get -y install qemu-user-binfmt + # Registered without the C (credentials) flag, setuid programs (passwd, + # newgrp, su, ...) would run without their privileges: register again with it + handler=$(grep -h "^:qemu-ppc64le:" /usr/lib/binfmt.d/qemu-ppc64le.conf) + echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-ppc64le > /dev/null + echo "${handler%:*}:OCPF" | sudo tee /proc/sys/fs/binfmt_misc/register > /dev/null + cat /proc/sys/fs/binfmt_misc/qemu-ppc64le + qemu_version=$(qemu-ppc64le --version | sed -nE 's/^qemu-ppc64le version ([0-9]+\.[0-9]+).*/\1/p') + echo "[Debug] qemu-ppc64le ${qemu_version}, $(podman --version)" + # Refuse an emulator with QEMU issue 1769 rather than let the run fail hours in + if [ "$(printf '%s\n' "${qemu_version}" 10.1 | sort -V | head -1)" != "10.1" ]; then + echo "[Error] QEMU ${qemu_version} is too old for ppc64le (needs 10.1 or newer, see QEMU issue 1769); run this job on Ubuntu 26.04 or newer" + exit 1 + fi + + + - name: Start the containers + run: | + # The containers' root is the runner's, so their systemds and services + # share its inotify instances: with too few D-Bus failed to start in some + # ("dirwatch_new: Too many open files"). Raise the limit where the runner + # allows it (a container runner does not), and show how much is used. + echo 8192 | sudo tee /proc/sys/fs/inotify/max_user_instances > /dev/null 2>&1 || + echo "[Debug] inotify instances limit not raised: not allowed on the runner" + sudo guests="${guests}" ci/container-guests.sh ppc64le "${image}" + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + + - name: Create AlmaLinux pungi repository + if: inputs.pungi_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pungi.repo" + [almalinux-${{ env.release_version }}-appstream-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/AppStream/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-appstream-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-baseos-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/BaseOS/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-baseos-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_crb_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi] + baseurl = https://ppc64le-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_extras_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux pulp repository + if: inputs.pulp_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pulp.repo" + [almalinux-${{ env.pulp_root }}-appstream-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-appstream-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-appstream-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-baseos-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-baseos-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-baseos-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux testing pulp repository + # Kitten has no testing repository, so skip it there + if: inputs.pulp_repository && inputs.testing_repository && inputs.version_major != '10-kitten' + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-testing-pulp.repo" + [almalinux-${{ env.pulp_root }}-testing-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-testing-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-testing-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux nvidia pulp repository + if: env.nvidia == 'true' && inputs.pulp_repository + run: | + # Repositories of the 'almalinux-nvidia' product + # https://build.almalinux.org/product/777 + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-nvidia-pulp.repo" + [almalinux-${{ env.pulp_root }}-nvidia-pulp] + baseurl = https://build.almalinux.org/pulp/content/copr/anfimovdm-almalinux-nvidia-almalinux-${{ env.pulp_root }}-$arch-dr/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-nvidia-pulp + gpgkey=file://${{ env.rpm_gpg_key }},file:///etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA-CUDA-${{ env.release_version }} + EOF + + # No 'Publish tested repository state': ci/compose_state.py gates the + # x86_64 and aarch64 publishing only + + - name: Push repositories to the containers + run: | + ci/each-guest.sh ' + set -e + for repo in almalinux-pungi almalinux-pulp almalinux-testing-pulp almalinux-nvidia-pulp; do + f="${{ env.artifacts_dir }}/${repo}.repo" + if [ -e "$f" ]; then + echo "[Debug] Installing ${repo}.repo" + sudo podman cp "$f" $guest:/etc/yum.repos.d/${repo}.repo + fi + done + ' + + - name: Update the system + run: | + # No reboot: the containers run the runner's kernel + ci/each-guest.sh ' + set -e + sudo podman exec $guest dnf -y clean all + sudo podman exec $guest dnf -y upgrade + # The image has packages an installed system does not (curl-minimal, + # which conflicts with the curl the tests install) and lacks some it + # has (hostname) + sudo podman exec $guest sh -c "! rpm -q curl-minimal >/dev/null || dnf -y swap curl-minimal curl" + sudo podman exec $guest dnf -y install hostname + ' + + - name: Get compose-tests onto the containers + run: | + # Scale the tests' time limits in the copy the containers get, as plain + # values: the tmt 1.26 of AlmaLinux 8 has neither policies nor + # duration multiplication. Tests without one get tmt's default, 5m. + grep -rlE '^duration:' --include=main.fmf tests | xargs perl -pi -e ' + my %unit = (s => 1, m => 60, h => 3600, d => 86400); + s/^duration:\s*(\d+)([smhd])\s*$/"duration: " . $1 * $unit{$2} * $ENV{duration_scale} . "s\n"/e' + echo "duration: $(( 5 * 60 * duration_scale ))s" > tests/main.fmf + echo "[Debug] Test time limits:"; grep -rE '^duration:' --include=main.fmf tests + + tar --exclude=./artifacts -C "${{ github.workspace }}" -czf /tmp/compose-tests.tgz . + ci/each-guest.sh ' + set -e + sudo podman cp /tmp/compose-tests.tgz $guest:/tmp/compose-tests.tgz + sudo podman exec $guest sh -c "rm -rf /compose-tests && mkdir -p /compose-tests && tar -C /compose-tests -xzf /tmp/compose-tests.tgz" + ' + + - name: Get system release + run: | + echo "system_release=$(sudo podman exec almalinux-0 cat /etc/almalinux-release)" >> $GITHUB_ENV + + - name: Add testing repository + # With pulp the testing repo comes from pulp (see the pulp step above). + # Kitten has no testing repository, so skip it there. + if: inputs.testing_repository && !inputs.pulp_repository && inputs.version_major != '10-kitten' + run: | + ci/each-guest.sh 'sudo podman exec $guest dnf install -y almalinux-release-testing' + + - name: Prepare test infrastructure + run: | + cat > /tmp/ct-prepare.sh <<'REMOTE' + enable_repo=${{ env.dnf_crb_repo }} + dnf_opts="--enablerepo=${enable_repo,,}" + if [ "${{ inputs.disable_default_repos }}" = "true" ]; then + # Disable all enabled system repositories + for enabled_repo in $(dnf repolist --enabled | awk '/repo id/ {A=1; next} A {print $1}' | grep -v -E 'pungi|pulp'); do + echo "[Debug] Disable '${enabled_repo}' repository" + dnf config-manager --set-disabled ${enabled_repo} + done + dnf_opts= + fi + case ${{ inputs.version_major }} in + 10-kitten) + dnf install -y ${dnf_opts} epel-release almalinux-kitten-release-devel + [ "${{ inputs.disable_default_repos }}" = "false" ] && \ + dnf_opts="${dnf_opts} --enablerepo=devel" + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 10) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 9) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt initscripts-service + ;; + *) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt tmt-report-html initscripts + ;; + esac + export PATH=$PATH:/usr/local/bin + echo "[Debug] tmt version: $(tmt --version)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-prepare.sh' + + - name: Split tests between the containers + run: | + # Enabled tests for this distro, split by duration (ci/test-durations/) + # into one tmt 'test --name' regex per container, pushed to /tmp/ct-shard.regex + sudo podman exec almalinux-0 sh -c 'export PATH=$PATH:/usr/local/bin; cd /compose-tests; tmt -c distro=centos-stream-${{ env.release_version }} -c container=yes tests ls --enabled' \ + | grep '^/' > ./tests_enabled.txt + python3 ci/shard.py split --shards ${shards} --variant '${{ inputs.version_major }}-ppc64le' < ./tests_enabled.txt > ./shards.txt + for guest in ${guests}; do + read -r regex <&3 + echo "${regex}" > ./ct-shard.regex + sudo podman cp ./ct-shard.regex ${guest}:/tmp/ct-shard.regex + done 3< ./shards.txt + + - name: Run tests + id: run-tests + continue-on-error: true + run: | + # continue-on-error lets us always collect logs and notify; capture the + # real exit code so a later step can tell a clean run (tmt 0=all pass, + # 1=some failed) from an aborted one (e.g. 2 = tmt error, or the + # container died mid-run). + set +e + cat > /tmp/ct-run.sh <<'REMOTE' + export PATH=$PATH:/usr/local/bin + cd /compose-tests + tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + # The worst exit code of the containers: one outside 0/1 (2 = tmt error, + # others = the container died) wins over 1 (some failed) over 0 (all pass). + rc=0 + for guest in ${guests}; do + guest_rc=$(cat ./tmt_rc.${guest} 2>/dev/null || echo 255) + echo "[Debug] ${guest} tmt exit code: ${guest_rc}" + case "${guest_rc}" in + 0) ;; + 1) [ "${rc}" = 0 ] && rc=1 ;; + *) rc=${guest_rc} ;; + esac + done + echo "[Debug] tmt exit code: ${rc}" + echo "tmt_rc=${rc}" >> "$GITHUB_OUTPUT" + exit ${rc} + + - name: ${{ inputs.version_major }} tests results + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + #[Debug] + find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml" -exec cat {} \; + + out=/tmp/ct-collect + rm -rf "$out"; mkdir -p "$out" + for f in $(find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml"); do + cp -av "$f" "$out/$(basename "$(dirname "$(dirname "$f")")").results.yaml" + done + chmod -R a+r "$out" + REMOTE + # Prefix the results with the container: almalinux-0.legacy.results.yaml + rm -rf /tmp/ct-collect + sudo podman cp ${guest}:/tmp/ct-collect /tmp/ct-collect 2>/dev/null || mkdir -p /tmp/ct-collect + sudo chown -R "$(id -u):$(id -g)" /tmp/ct-collect + for f in /tmp/ct-collect/*.results.yaml; do + [ -e "$f" ] && mv "$f" "${{ env.artifacts_dir }}/${guest}.$(basename "$f")" + done + done + + - name: ${{ inputs.version_major }} tests log + run: | + for guest in ${guests}; do + #[Debug] + sudo podman exec ${guest} cat ${{ env.tmt_run_dir }}/log.txt + + sudo podman cp ${guest}:${{ env.tmt_run_dir }}/log.txt "${{ env.artifacts_dir }}/${guest}.log.txt" || true + done + + - name: ${{ inputs.version_major }} tests output + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + cd ${{ env.tmt_run_dir }}/ + tar cf /tmp/output.tar $( find . -name output.txt ) || true + REMOTE + sudo podman cp ${guest}:/tmp/output.tar "${{ env.artifacts_dir }}/${guest}.output.tar" || true + done + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests log + path: ${{ env.artifacts_dir }}/*.log.txt + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests results + path: ${{ env.artifacts_dir }}/*.results.yaml + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-ppc64le tests output + path: ${{ env.artifacts_dir }}/*.output.tar + + - name: Collect the containers' journals + if: always() + run: | + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + mkdir -p "${{ runner.temp }}/journals" + for guest in ${guests}; do + sudo podman exec ${guest} journalctl --no-pager -o short-precise \ + > "${{ runner.temp }}/journals/${guest}.journal.txt" 2>&1 || true + done + + - uses: actions/upload-artifact@v7 + # The containers' systemd journals: services that do not start in them + if: always() + with: + name: ${{ inputs.version_major }}-ppc64le containers journal + path: ${{ runner.temp }}/journals/*.journal.txt + if-no-files-found: ignore + + - name: Evaluate test results + id: eval + if: always() + run: | + # Tally the collected results.yaml, build the failed-tests list, and + # decide whether the run actually completed. + if ! command -v yq >/dev/null 2>&1; then + case $(uname -m) in x86_64) yq_arch=amd64 ;; aarch64) yq_arch=arm64 ;; *) yq_arch=$(uname -m) ;; esac + sudo curl -fsSL -o /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${yq_arch} + sudo chmod +x /usr/local/bin/yq + fi + + # The failed-tests list gets ' ' lines so that a failed + # test is re-run in the container it failed in. A container with no + # results at all (every shard runs at least 0_common) means its run + # was aborted. + shopt -s nullglob + total=0; passed=0; failed=0; errored=0; empty_guest=false + : > "${artifacts_dir}/tests_failed.txt" + for guest in ${guests}; do + guest_total=0 + for f in "${artifacts_dir}/${guest}".*.results.yaml; do + guest_total=$(( guest_total + $(yq '. | length' "$f" 2>/dev/null || echo 0) )) + passed=$(( passed + $(yq '[.[] | select(.result == "pass")] | length' "$f" 2>/dev/null || echo 0) )) + failed=$(( failed + $(yq '[.[] | select(.result == "fail")] | length' "$f" 2>/dev/null || echo 0) )) + errored=$((errored + $(yq '[.[] | select(.result == "error")] | length' "$f" 2>/dev/null || echo 0) )) + yq '.[] | select(.result == "fail" or .result == "error") | .name' "$f" 2>/dev/null \ + | sed "s|^|${guest} |" >> "${artifacts_dir}/tests_failed.txt" + done + total=$(( total + guest_total )) + [ "${guest_total}" -eq 0 ] && empty_guest=true + done + failed_total=$(( failed + errored )) + + echo "total=${total}" >> "$GITHUB_OUTPUT" + echo "passed=${passed}" >> "$GITHUB_OUTPUT" + echo "failed=${failed_total}" >> "$GITHUB_OUTPUT" + + # Completeness: a clean tmt run exits 0 (all pass) or 1 (some failed). + # Anything else (2 = tmt error, the container died, ...) or + # zero recorded results means the run was aborted before finishing. + rc='${{ steps.run-tests.outputs.tmt_rc }}' + incomplete=false + case "${rc}" in 0|1) ;; *) incomplete=true ;; esac + [ "${total}" -eq 0 ] && incomplete=true + [ "${empty_guest}" = true ] && incomplete=true + echo "incomplete=${incomplete}" >> "$GITHUB_OUTPUT" + + echo "[Debug] total=${total} passed=${passed} failed=${failed_total} tmt_rc=${rc} incomplete=${incomplete}" + + - name: Re-run failed tests, prepare tests summary + # Only when the run completed and recorded failures; an aborted run is + # handled by the completeness gate at the end of the job. + if: ${{ always() && steps.eval.outputs.incomplete == 'false' && steps.eval.outputs.failed != '0' }} + run: | + # Tests results and the step exit code + rerun_results= + exit_code=0 + + # Read failed tests list: ' ' lines + while read -r guest test_failed; do + [ -n "${test_failed}" ] || continue + echo "[Debug] Re-run '${test_failed}' on ${guest}" + # Exact name as a tmt regex, '+' and '.' made literal (p_gcc-c++) + test_regex="^$(echo "${test_failed}" | sed 's/[+.]/[&]/g')\$" + + # Include failed test name into results summary + test_result="${test_failed}" + + # Re-run specific failed test, in the container it failed in + if [ "${{ inputs.rerun_failed }}" = "true" ]; then + if sudo podman exec ${guest} sh -c "export PATH=\$PATH:/usr/local/bin; cd /compose-tests; tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name \"${test_regex}\"" < /dev/null; then + test_result="${test_failed} [re-run ✅]" + else + test_result="${test_failed} [re-run ❌]" + exit_code=1 + fi + else + echo "[Debug]" + echo "If without re-run, fail this step as the '${test_failed}' test previously failed." + exit_code=1 + fi + + # Format test results in list format: 'item1', 'item2', ... + [ "x${rerun_results}" = "x" ] && rerun_results="'${test_result}'" || rerun_results="${rerun_results}, '${test_result}'" + + done < "${{ env.artifacts_dir }}/tests_failed.txt" + + # Export header and test results + [ "x${rerun_results}" = "x" ] && summary_header="✅ All tests pass." || summary_header="❌ Failed tests:" + echo "rerun_results=${rerun_results}" >> $GITHUB_ENV + echo "summary_header=${summary_header}" >> $GITHUB_ENV + + exit $exit_code + + - name: Print tests summary + id: summary + # Always run so the run-page summary and the Mattermost text are produced + # for both passing and failing runs. + if: always() + uses: actions/github-script@v9 + with: + result-encoding: string + script: | + const repos = [ + 'pulp - ${{ inputs.pulp_repository && '➕' || '➖' }}', + 'pungi - ${{ inputs.pungi_repository && '➕' || '➖' }}', + 'testing - ${{ inputs.testing_repository && '➕' || '➖' }}', + 'default repos - ${{ inputs.disable_default_repos && '➖' || '➕' }}', + ]; + // Re-run annotated failures (set only when the re-run step ran). + const rerun = [${{ env.rerun_results }}]; + + + const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); + const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); + const total = parseInt('${{ steps.eval.outputs.total }}' || '0', 10); + const incomplete = '${{ steps.eval.outputs.incomplete }}' === 'true'; + const rc = '${{ steps.run-tests.outputs.tmt_rc }}'; + + // The counts above are the FIRST run. Reconcile them with the re-run + // outcomes: entries are annotated '… [re-run ✅]' (recovered) / + // '… [re-run ❌]' (still failing); unmarked means re-run was disabled. + const reran = rerun.some(r => r.includes('[re-run')); + const recovered = rerun.filter(r => r.includes('✅')).length; + const stillFailing = rerun.length - recovered; + + let header; + if (incomplete) header = `⚠️ Test run did NOT complete — only ${total} results recorded (tmt exit ${rc}).`; + else if (stillFailing > 0) header = `❌ ${stillFailing} of ${total} tests failed.`; + else if (recovered > 0) header = `✅ All ${total} tests passed (${recovered} recovered on re-run).`; + else header = `✅ All ${total} tests passed.`; + + const counts = `Passed: ${passed} · Failed: ${failed} · Total: ${total}`; + const rerunLine = reran ? `Re-run — recovered: ${recovered} · still failing: ${stillFailing}` : ''; + + // Run-page summary + let summary = core.summary + .addHeading('${{ env.system_release }}', '4') + .addHeading('Used repositories:', '4') + .addList(repos, true) + .addHeading(header, '4') + .addRaw(counts).addBreak(); + if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); + summary.addList(rerun, true).write(); + + // Failure reason as the 'compose-fail' annotation of this job + if (incomplete || stillFailing > 0) { + core.error(`${header} ${rerun.join(', ')}`.trim().slice(0, 3800), { title: 'compose-fail' }); + } + + // Markdown text for the Mattermost notification + const runUrl = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const text = [ + '#### AlmaLinux ${{ inputs.version_major }} ppc64le — compose tests', + '**${{ env.system_release }}**', + '', + header, + counts, + ...(rerunLine ? [rerunLine] : []), + '', + '**Used repositories:**', + ...repos.map(r => '- ' + r), + ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), + '', + '[Workflow run](' + runUrl + ')', + ].join('\n'); + core.setOutput('text', text); + + - name: Send notification to Mattermost + uses: mattermost/action-mattermost-notify@v2.1.0 + # Notify on both pass and fail; never fail the job on a notification error + if: ${{ always() && inputs.notify_mattermost }} + continue-on-error: true + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }} + MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }} + MATTERMOST_USERNAME: ${{ github.triggering_actor }} + TEXT: | + ${{ steps.summary.outputs.text }} + + - name: Fail if the test run did not complete + # Runs last (after notification) so an aborted/incomplete run — e.g. a + # test that kills the container, leaving an empty results.yaml — is + # reported as a failure instead of a false green. + if: ${{ always() && steps.eval.outputs.incomplete == 'true' }} run: | - echo "::error::No ppc64le compose tests on '${{ github.ref_name }}': run this workflow from the ci/tcg-ppc64le-s390x branch." - exit 0 + echo "::error::Test run did not complete: tmt exit '${{ steps.run-tests.outputs.tmt_rc }}', ${{ steps.eval.outputs.total }} results recorded." + exit 1 From 7c25f5539daaf8f58427b83e52a673b31a4ba19d Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:30:39 +0200 Subject: [PATCH 3/8] ci: run the s390x compose tests in containers Run the compose tests in 4 AlmaLinux s390x containers like the ppc64le ones: natively in the AlmaLinux organization (ubuntu-26.04-s390x), under QEMU user-mode emulation on an x86_64 runner elsewhere, e.g. in forks. The native s390x runner is an Incus container with AppArmor and Ubuntu's profiles, and podman runs the containers unconfined, so a program started from a path a runner profile attaches to gets that profile: anything started as /usr/bin/hostname or /usr/bin/gs failed to load libc ("cannot apply additional memory protection after relocation: Permission denied"), even with the profile in complain mode. Before starting the containers unload these profiles, but not those flagged unconfined: they confine nothing, they grant (user namespaces to podman, crun, ...), the containers run under one of them, and reloaded in complain mode they broke podman's overlay storage. --- .../almalinux-compose-test-s390x.yml | 711 +++++++++++++++++- 1 file changed, 704 insertions(+), 7 deletions(-) diff --git a/.github/workflows/almalinux-compose-test-s390x.yml b/.github/workflows/almalinux-compose-test-s390x.yml index 830a63f..54d5525 100644 --- a/.github/workflows/almalinux-compose-test-s390x.yml +++ b/.github/workflows/almalinux-compose-test-s390x.yml @@ -2,9 +2,13 @@ name: almalinux-compose-test-s390x run-name: AlmaLinux ${{ inputs.version_major }} compose test (s390x)${{ inputs.pungi_repository && ' +pungi' || '' }}${{ inputs.pulp_repository && ' +pulp' || '' }} -# Placeholder: GitHub dispatches a workflow only if a file with its path is on -# the default branch. The implementation (QEMU TCG emulated s390x guests) is -# on the ci/tcg-ppc64le-s390x branch: run the workflow from there. +# The tests run in AlmaLinux s390x containers (podman, systemd as PID 1, the +# almalinux/-init images, ci/container-guests.sh), split between the +# containers like on x86_64 and aarch64: in the AlmaLinux organization on a +# s390x runner, natively; elsewhere, e.g. in forks, on an x86_64 runner, their +# binaries run by QEMU user-mode emulation. The containers share the runner's +# kernel, so the tests run with CONTAINERTEST=1: those exercising the kernel, +# the boot or the hardware skip themselves. on: workflow_dispatch: @@ -50,13 +54,706 @@ on: type: boolean default: true + # Allow this workflow to be called from the 'almalinux-compose-test' orchestrator. + # 'workflow_call' has no 'choice' input type, so 'version_major' is a string here. + workflow_call: + inputs: + version_major: + description: 'AlmaLinux major version' + required: true + type: string + pungi_repository: + description: 'Add pungi repositories' + type: boolean + default: false + pulp_repository: + description: 'Add pulp repositories' + type: boolean + default: true + testing_repository: + description: 'Add testing repository' + type: boolean + default: false + disable_default_repos: + description: 'Disable system default repositories' + type: boolean + default: false + rerun_failed: + description: 'Re-run failed tests' + type: boolean + default: true + notify_mattermost: + description: 'Send results notification to Mattermost' + type: boolean + default: true + jobs: compose-test: name: AlmaLinux ${{ inputs.version_major }} - runs-on: ubuntu-26.04-s390x + # In the AlmaLinux organization the containers run natively on an s390x + # runner; elsewhere, e.g. in forks, on an x86_64 runner, emulated. KVM is + # of no use (no nested-virt). GitHub-hosted runners stop jobs after 6 hours. + runs-on: >- + ${{ + github.repository_owner == 'AlmaLinux' && + 'ubuntu-26.04-s390x' + || + 'ubuntu-26.04' + }} + timeout-minutes: 360 + + env: + # Number of guests the tests are split between, run in parallel + shards: 4 + # Whether the containers run natively (see runs-on) or emulated + native: ${{ github.repository_owner == 'AlmaLinux' }} + # The factor the tests' time limits ('duration') are scaled by: emulation + # is several times slower than the hardware they are meant for, and + # an s390x runner may be slower than an x86_64 one + duration_scale: ${{ github.repository_owner == 'AlmaLinux' && 2 || 8 }} + steps: - - name: Not implemented on this branch + - uses: actions/checkout@v7 + + - name: Prepare stuff + run: | + # Environment variables + pungi_latest_result=latest_result_almalinux + pulp_root=${{ inputs.version_major }} + tmt_options= + rpm_gpg_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-AlmaLinux + release_version=${{ inputs.version_major }} + dnf_crb_repo='CRB' + dnf_extras_repo='extras' + # systemd-enabled image of the version + image=docker.io/almalinux/${{ inputs.version_major }}-init:latest + + case ${{ inputs.version_major }} in + 8) + dnf_crb_repo='PowerTools' + ;; + 9) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-9" + ;; + 10) + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + ;; + 10-kitten) + release_version=10 + pulp_root=kitten-10 + pungi_latest_result="${pungi_latest_result}-kitten" + tmt_options='--feeling-safe' + rpm_gpg_key="${rpm_gpg_key}-10" + dnf_extras_repo='extras-common' + ;; + esac + # Release major version + echo "release_version=${release_version}" >> $GITHUB_ENV + + # Pulp repository root of the release + echo "pulp_root=${pulp_root}" >> $GITHUB_ENV + + # Pungi repository latest results directory + echo "pungi_latest_result=${pungi_latest_result}" >> $GITHUB_ENV + + # Name of repository to enable (PowerTools/CRB) + if [ "x${{ inputs.pulp_repository }}" = "xtrue" ]; then + # Lowercase the name for path in pulp's URL + pulp_crb_repo="${dnf_crb_repo,,}" + fi + echo "dnf_crb_repo=${dnf_crb_repo}" >> $GITHUB_ENV + echo "pulp_crb_repo=${pulp_crb_repo}" >> $GITHUB_ENV + echo "dnf_extras_repo=${dnf_extras_repo}" >> $GITHUB_ENV + + # TMT extra options; these go right after 'tmt', before the + # subcommand -- tmt 1.78 dropped the step-level '--feeling-safe'. + echo "tmt_options=${tmt_options}" >> $GITHUB_ENV + + # RPM_GPG_KEY file + echo "rpm_gpg_key=${rpm_gpg_key}" >> $GITHUB_ENV + + # Image the containers run + echo "image=${image}" >> $GITHUB_ENV + + # TMT tests run directory (in the containers) + echo "tmt_run_dir=/var/tmp/tmt/run-001" >> $GITHUB_ENV + + # Names of the containers, one per shard of the tests + echo "guests=$(seq -f 'almalinux-%g' -s ' ' 0 $(( shards - 1 )))" >> $GITHUB_ENV + + # Local directory on the runner to gather artifacts pulled from the containers + artifacts_dir="${{ github.workspace }}/artifacts" + mkdir -p "${artifacts_dir}" + echo "artifacts_dir=${artifacts_dir}" >> $GITHUB_ENV + + # For nvidia drivers tests + case ${{ inputs.version_major }} in + 8) + nvidia=false + ;; + *) + nvidia=true + ;; + esac + echo "nvidia=${nvidia}" >> $GITHUB_ENV + + - name: Configure apt + run: | + # Prefer the regional EC2 mirror over archive.ubuntu.com (throttled/unreachable + # from the runners at times), and make apt fail fast with retries. + imds_token=$(curl -sf --max-time 5 -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' || true) + region=$(curl -sf --max-time 5 -H "X-aws-ec2-metadata-token: $imds_token" http://169.254.169.254/latest/meta-data/placement/region || true) + if [ -n "$region" ]; then + sudo sed -i "s|https\?://archive.ubuntu.com/ubuntu|http://${region}.ec2.archive.ubuntu.com/ubuntu|" /etc/apt/sources.list.d/ubuntu.sources + fi + printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' | sudo tee /etc/apt/apt.conf.d/99-ci-retries + + - name: Install podman + run: | + command -v podman > /dev/null || { sudo apt-get -y update; sudo apt-get -y install podman; } + echo "[Debug] $(uname -m) runner, $(podman --version)" + + - name: Install QEMU user-mode emulation + if: env.native != 'true' + run: | + # Its binfmt_misc handlers let the kernel run s390x binaries + sudo apt-get -y update + sudo apt-get -y install qemu-user-binfmt + # Registered without the C (credentials) flag, setuid programs (passwd, + # newgrp, su, ...) would run without their privileges: register again with it + handler=$(grep -h "^:qemu-s390x:" /usr/lib/binfmt.d/qemu-s390x.conf) + echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-s390x > /dev/null + echo "${handler%:*}:OCPF" | sudo tee /proc/sys/fs/binfmt_misc/register > /dev/null + cat /proc/sys/fs/binfmt_misc/qemu-s390x + echo "[Debug] $(qemu-s390x --version | head -1), $(podman --version)" + + - name: Unload the runner's confining AppArmor profiles + if: env.native == 'true' + run: | + # The runner (an Incus container on the s390x one) confines programs + # by path with its AppArmor profiles, the containers' ones too: started + # as /usr/bin/hostname, /usr/bin/gs, ... a program gets the runner's + # profile, which fails it on the containers' files, even in complain + # mode: hostname fails to load libc ("cannot apply additional memory + # protection after relocation"). Unload these profiles: the runner is + # thrown away after the job. Not those flagged unconfined: they confine + # nothing, they grant (user namespaces to podman, crun, ...), and the + # containers run under one of them. + if [ ! -d /sys/kernel/security/apparmor ]; then + echo "[Debug] No AppArmor on the runner" + exit 0 + fi + failed= + for profile in /etc/apparmor.d/*; do + [ -f "${profile}" ] || continue + grep -q 'flags=([^)]*unconfined' "${profile}" && continue + sudo apparmor_parser -R "${profile}" 2> /dev/null || failed="${failed} ${profile##*/}" + done + [ -z "${failed}" ] || echo "[Debug] Not unloaded (not loaded?):${failed}" + sudo aa-status 2>&1 | head -5 || true + + - name: Start the containers + run: | + # The containers' root is the runner's, so their systemds and services + # share its inotify instances: with too few D-Bus failed to start in some + # ("dirwatch_new: Too many open files"). Raise the limit where the runner + # allows it (a container runner does not), and show how much is used. + echo 8192 | sudo tee /proc/sys/fs/inotify/max_user_instances > /dev/null 2>&1 || + echo "[Debug] inotify instances limit not raised: not allowed on the runner" + sudo guests="${guests}" ci/container-guests.sh s390x "${image}" + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + + - name: Create AlmaLinux pungi repository + if: inputs.pungi_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pungi.repo" + [almalinux-${{ env.release_version }}-appstream-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/AppStream/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-appstream-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-baseos-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/BaseOS/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-baseos-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_crb_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_crb_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi] + baseurl = https://s390x-pungi-${{ env.release_version }}.almalinux.dev/almalinux/${{ env.release_version }}/$arch/${{ env.pungi_latest_result }}/compose/${{ env.dnf_extras_repo }}/$arch/os/ + enabled = 1 + name = almalinux-${{ env.release_version }}-${{ env.dnf_extras_repo }}-pungi + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux pulp repository + if: inputs.pulp_repository + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-pulp.repo" + [almalinux-${{ env.pulp_root }}-appstream-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-appstream-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-appstream-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-baseos-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-baseos-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-baseos-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.pulp_crb_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + + [almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-${{ env.dnf_extras_repo }}-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux testing pulp repository + # Kitten has no testing repository, so skip it there + if: inputs.pulp_repository && inputs.testing_repository && inputs.version_major != '10-kitten' + run: | + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-testing-pulp.repo" + [almalinux-${{ env.pulp_root }}-testing-pulp] + baseurl = https://build.almalinux.org/pulp/content/prod/almalinux-${{ env.pulp_root }}-testing-$arch/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-testing-pulp + gpgkey=file://${{ env.rpm_gpg_key }} + EOF + + - name: Create AlmaLinux nvidia pulp repository + if: env.nvidia == 'true' && inputs.pulp_repository + run: | + # Repositories of the 'almalinux-nvidia' product + # https://build.almalinux.org/product/777 + cat <<'EOF'>"${{ env.artifacts_dir }}/almalinux-nvidia-pulp.repo" + [almalinux-${{ env.pulp_root }}-nvidia-pulp] + baseurl = https://build.almalinux.org/pulp/content/copr/anfimovdm-almalinux-nvidia-almalinux-${{ env.pulp_root }}-$arch-dr/ + enabled = 1 + name = almalinux-${{ env.pulp_root }}-nvidia-pulp + gpgkey=file://${{ env.rpm_gpg_key }},file:///etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA-CUDA-${{ env.release_version }} + EOF + + # No 'Publish tested repository state': ci/compose_state.py gates the + # x86_64 and aarch64 publishing only + + - name: Push repositories to the containers + run: | + ci/each-guest.sh ' + set -e + for repo in almalinux-pungi almalinux-pulp almalinux-testing-pulp almalinux-nvidia-pulp; do + f="${{ env.artifacts_dir }}/${repo}.repo" + if [ -e "$f" ]; then + echo "[Debug] Installing ${repo}.repo" + sudo podman cp "$f" $guest:/etc/yum.repos.d/${repo}.repo + fi + done + ' + + - name: Update the system + run: | + # No reboot: the containers run the runner's kernel + ci/each-guest.sh ' + set -e + sudo podman exec $guest dnf -y clean all + sudo podman exec $guest dnf -y upgrade + # The image has packages an installed system does not (curl-minimal, + # which conflicts with the curl the tests install) and lacks some it + # has (hostname) + sudo podman exec $guest sh -c "! rpm -q curl-minimal >/dev/null || dnf -y swap curl-minimal curl" + sudo podman exec $guest dnf -y install hostname + ' + + - name: Get compose-tests onto the containers + run: | + # Scale the tests' time limits in the copy the containers get, as plain + # values: the tmt 1.26 of AlmaLinux 8 has neither policies nor + # duration multiplication. Tests without one get tmt's default, 5m. + grep -rlE '^duration:' --include=main.fmf tests | xargs perl -pi -e ' + my %unit = (s => 1, m => 60, h => 3600, d => 86400); + s/^duration:\s*(\d+)([smhd])\s*$/"duration: " . $1 * $unit{$2} * $ENV{duration_scale} . "s\n"/e' + echo "duration: $(( 5 * 60 * duration_scale ))s" > tests/main.fmf + echo "[Debug] Test time limits:"; grep -rE '^duration:' --include=main.fmf tests + + tar --exclude=./artifacts -C "${{ github.workspace }}" -czf /tmp/compose-tests.tgz . + ci/each-guest.sh ' + set -e + sudo podman cp /tmp/compose-tests.tgz $guest:/tmp/compose-tests.tgz + sudo podman exec $guest sh -c "rm -rf /compose-tests && mkdir -p /compose-tests && tar -C /compose-tests -xzf /tmp/compose-tests.tgz" + ' + + - name: Get system release + run: | + echo "system_release=$(sudo podman exec almalinux-0 cat /etc/almalinux-release)" >> $GITHUB_ENV + + - name: Add testing repository + # With pulp the testing repo comes from pulp (see the pulp step above). + # Kitten has no testing repository, so skip it there. + if: inputs.testing_repository && !inputs.pulp_repository && inputs.version_major != '10-kitten' + run: | + ci/each-guest.sh 'sudo podman exec $guest dnf install -y almalinux-release-testing' + + - name: Prepare test infrastructure + run: | + cat > /tmp/ct-prepare.sh <<'REMOTE' + enable_repo=${{ env.dnf_crb_repo }} + dnf_opts="--enablerepo=${enable_repo,,}" + if [ "${{ inputs.disable_default_repos }}" = "true" ]; then + # Disable all enabled system repositories + for enabled_repo in $(dnf repolist --enabled | awk '/repo id/ {A=1; next} A {print $1}' | grep -v -E 'pungi|pulp'); do + echo "[Debug] Disable '${enabled_repo}' repository" + dnf config-manager --set-disabled ${enabled_repo} + done + dnf_opts= + fi + case ${{ inputs.version_major }} in + 10-kitten) + dnf install -y ${dnf_opts} epel-release almalinux-kitten-release-devel + [ "${{ inputs.disable_default_repos }}" = "false" ] && \ + dnf_opts="${dnf_opts} --enablerepo=devel" + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 10) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} python3-pip beakerlib initscripts-service rsync + export PATH=$PATH:/usr/local/bin; pip install flexparser==0.3.1 tmt + ;; + 9) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt initscripts-service + ;; + *) + dnf install -y ${dnf_opts} epel-release + dnf install -y ${dnf_opts} tmt tmt-report-html initscripts + ;; + esac + export PATH=$PATH:/usr/local/bin + echo "[Debug] tmt version: $(tmt --version)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-prepare.sh' + + - name: Split tests between the containers + run: | + # Enabled tests for this distro, split by duration (ci/test-durations/) + # into one tmt 'test --name' regex per container, pushed to /tmp/ct-shard.regex + sudo podman exec almalinux-0 sh -c 'export PATH=$PATH:/usr/local/bin; cd /compose-tests; tmt -c distro=centos-stream-${{ env.release_version }} -c container=yes tests ls --enabled' \ + | grep '^/' > ./tests_enabled.txt + python3 ci/shard.py split --shards ${shards} --variant '${{ inputs.version_major }}-s390x' < ./tests_enabled.txt > ./shards.txt + for guest in ${guests}; do + read -r regex <&3 + echo "${regex}" > ./ct-shard.regex + sudo podman cp ./ct-shard.regex ${guest}:/tmp/ct-shard.regex + done 3< ./shards.txt + + - name: Run tests + id: run-tests + continue-on-error: true + run: | + # continue-on-error lets us always collect logs and notify; capture the + # real exit code so a later step can tell a clean run (tmt 0=all pass, + # 1=some failed) from an aborted one (e.g. 2 = tmt error, or the + # container died mid-run). + set +e + cat > /tmp/ct-run.sh <<'REMOTE' + export PATH=$PATH:/usr/local/bin + cd /compose-tests + tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" + REMOTE + ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + # The worst exit code of the containers: one outside 0/1 (2 = tmt error, + # others = the container died) wins over 1 (some failed) over 0 (all pass). + rc=0 + for guest in ${guests}; do + guest_rc=$(cat ./tmt_rc.${guest} 2>/dev/null || echo 255) + echo "[Debug] ${guest} tmt exit code: ${guest_rc}" + case "${guest_rc}" in + 0) ;; + 1) [ "${rc}" = 0 ] && rc=1 ;; + *) rc=${guest_rc} ;; + esac + done + echo "[Debug] tmt exit code: ${rc}" + echo "tmt_rc=${rc}" >> "$GITHUB_OUTPUT" + exit ${rc} + + - name: ${{ inputs.version_major }} tests results + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + #[Debug] + find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml" -exec cat {} \; + + out=/tmp/ct-collect + rm -rf "$out"; mkdir -p "$out" + for f in $(find ${{ env.tmt_run_dir }}/plans/*/execute -name "results.yaml"); do + cp -av "$f" "$out/$(basename "$(dirname "$(dirname "$f")")").results.yaml" + done + chmod -R a+r "$out" + REMOTE + # Prefix the results with the container: almalinux-0.legacy.results.yaml + rm -rf /tmp/ct-collect + sudo podman cp ${guest}:/tmp/ct-collect /tmp/ct-collect 2>/dev/null || mkdir -p /tmp/ct-collect + sudo chown -R "$(id -u):$(id -g)" /tmp/ct-collect + for f in /tmp/ct-collect/*.results.yaml; do + [ -e "$f" ] && mv "$f" "${{ env.artifacts_dir }}/${guest}.$(basename "$f")" + done + done + + - name: ${{ inputs.version_major }} tests log + run: | + for guest in ${guests}; do + #[Debug] + sudo podman exec ${guest} cat ${{ env.tmt_run_dir }}/log.txt + + sudo podman cp ${guest}:${{ env.tmt_run_dir }}/log.txt "${{ env.artifacts_dir }}/${guest}.log.txt" || true + done + + - name: ${{ inputs.version_major }} tests output + run: | + for guest in ${guests}; do + sudo podman exec -i ${guest} bash -s <<'REMOTE' + cd ${{ env.tmt_run_dir }}/ + tar cf /tmp/output.tar $( find . -name output.txt ) || true + REMOTE + sudo podman cp ${guest}:/tmp/output.tar "${{ env.artifacts_dir }}/${guest}.output.tar" || true + done + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests log + path: ${{ env.artifacts_dir }}/*.log.txt + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests results + path: ${{ env.artifacts_dir }}/*.results.yaml + + - uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.version_major }}-s390x tests output + path: ${{ env.artifacts_dir }}/*.output.tar + + - name: Collect the containers' journals + if: always() + run: | + echo "[Debug] inotify instances in use: $(sudo find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null | wc -l) of root, limit $(cat /proc/sys/fs/inotify/max_user_instances)" + mkdir -p "${{ runner.temp }}/journals" + for guest in ${guests}; do + sudo podman exec ${guest} journalctl --no-pager -o short-precise \ + > "${{ runner.temp }}/journals/${guest}.journal.txt" 2>&1 || true + done + + - uses: actions/upload-artifact@v7 + # The containers' systemd journals: services that do not start in them + if: always() + with: + name: ${{ inputs.version_major }}-s390x containers journal + path: ${{ runner.temp }}/journals/*.journal.txt + if-no-files-found: ignore + + - name: Evaluate test results + id: eval + if: always() + run: | + # Tally the collected results.yaml, build the failed-tests list, and + # decide whether the run actually completed. + if ! command -v yq >/dev/null 2>&1; then + case $(uname -m) in x86_64) yq_arch=amd64 ;; aarch64) yq_arch=arm64 ;; *) yq_arch=$(uname -m) ;; esac + sudo curl -fsSL -o /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${yq_arch} + sudo chmod +x /usr/local/bin/yq + fi + + # The failed-tests list gets ' ' lines so that a failed + # test is re-run in the container it failed in. A container with no + # results at all (every shard runs at least 0_common) means its run + # was aborted. + shopt -s nullglob + total=0; passed=0; failed=0; errored=0; empty_guest=false + : > "${artifacts_dir}/tests_failed.txt" + for guest in ${guests}; do + guest_total=0 + for f in "${artifacts_dir}/${guest}".*.results.yaml; do + guest_total=$(( guest_total + $(yq '. | length' "$f" 2>/dev/null || echo 0) )) + passed=$(( passed + $(yq '[.[] | select(.result == "pass")] | length' "$f" 2>/dev/null || echo 0) )) + failed=$(( failed + $(yq '[.[] | select(.result == "fail")] | length' "$f" 2>/dev/null || echo 0) )) + errored=$((errored + $(yq '[.[] | select(.result == "error")] | length' "$f" 2>/dev/null || echo 0) )) + yq '.[] | select(.result == "fail" or .result == "error") | .name' "$f" 2>/dev/null \ + | sed "s|^|${guest} |" >> "${artifacts_dir}/tests_failed.txt" + done + total=$(( total + guest_total )) + [ "${guest_total}" -eq 0 ] && empty_guest=true + done + failed_total=$(( failed + errored )) + + echo "total=${total}" >> "$GITHUB_OUTPUT" + echo "passed=${passed}" >> "$GITHUB_OUTPUT" + echo "failed=${failed_total}" >> "$GITHUB_OUTPUT" + + # Completeness: a clean tmt run exits 0 (all pass) or 1 (some failed). + # Anything else (2 = tmt error, the container died, ...) or + # zero recorded results means the run was aborted before finishing. + rc='${{ steps.run-tests.outputs.tmt_rc }}' + incomplete=false + case "${rc}" in 0|1) ;; *) incomplete=true ;; esac + [ "${total}" -eq 0 ] && incomplete=true + [ "${empty_guest}" = true ] && incomplete=true + echo "incomplete=${incomplete}" >> "$GITHUB_OUTPUT" + + echo "[Debug] total=${total} passed=${passed} failed=${failed_total} tmt_rc=${rc} incomplete=${incomplete}" + + - name: Re-run failed tests, prepare tests summary + # Only when the run completed and recorded failures; an aborted run is + # handled by the completeness gate at the end of the job. + if: ${{ always() && steps.eval.outputs.incomplete == 'false' && steps.eval.outputs.failed != '0' }} + run: | + # Tests results and the step exit code + rerun_results= + exit_code=0 + + # Read failed tests list: ' ' lines + while read -r guest test_failed; do + [ -n "${test_failed}" ] || continue + echo "[Debug] Re-run '${test_failed}' on ${guest}" + # Exact name as a tmt regex, '+' and '.' made literal (p_gcc-c++) + test_regex="^$(echo "${test_failed}" | sed 's/[+.]/[&]/g')\$" + + # Include failed test name into results summary + test_result="${test_failed}" + + # Re-run specific failed test, in the container it failed in + if [ "${{ inputs.rerun_failed }}" = "true" ]; then + if sudo podman exec ${guest} sh -c "export PATH=\$PATH:/usr/local/bin; cd /compose-tests; tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name \"${test_regex}\"" < /dev/null; then + test_result="${test_failed} [re-run ✅]" + else + test_result="${test_failed} [re-run ❌]" + exit_code=1 + fi + else + echo "[Debug]" + echo "If without re-run, fail this step as the '${test_failed}' test previously failed." + exit_code=1 + fi + + # Format test results in list format: 'item1', 'item2', ... + [ "x${rerun_results}" = "x" ] && rerun_results="'${test_result}'" || rerun_results="${rerun_results}, '${test_result}'" + + done < "${{ env.artifacts_dir }}/tests_failed.txt" + + # Export header and test results + [ "x${rerun_results}" = "x" ] && summary_header="✅ All tests pass." || summary_header="❌ Failed tests:" + echo "rerun_results=${rerun_results}" >> $GITHUB_ENV + echo "summary_header=${summary_header}" >> $GITHUB_ENV + + exit $exit_code + + - name: Print tests summary + id: summary + # Always run so the run-page summary and the Mattermost text are produced + # for both passing and failing runs. + if: always() + uses: actions/github-script@v9 + with: + result-encoding: string + script: | + const repos = [ + 'pulp - ${{ inputs.pulp_repository && '➕' || '➖' }}', + 'pungi - ${{ inputs.pungi_repository && '➕' || '➖' }}', + 'testing - ${{ inputs.testing_repository && '➕' || '➖' }}', + 'default repos - ${{ inputs.disable_default_repos && '➖' || '➕' }}', + ]; + // Re-run annotated failures (set only when the re-run step ran). + const rerun = [${{ env.rerun_results }}]; + + + const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); + const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); + const total = parseInt('${{ steps.eval.outputs.total }}' || '0', 10); + const incomplete = '${{ steps.eval.outputs.incomplete }}' === 'true'; + const rc = '${{ steps.run-tests.outputs.tmt_rc }}'; + + // The counts above are the FIRST run. Reconcile them with the re-run + // outcomes: entries are annotated '… [re-run ✅]' (recovered) / + // '… [re-run ❌]' (still failing); unmarked means re-run was disabled. + const reran = rerun.some(r => r.includes('[re-run')); + const recovered = rerun.filter(r => r.includes('✅')).length; + const stillFailing = rerun.length - recovered; + + let header; + if (incomplete) header = `⚠️ Test run did NOT complete — only ${total} results recorded (tmt exit ${rc}).`; + else if (stillFailing > 0) header = `❌ ${stillFailing} of ${total} tests failed.`; + else if (recovered > 0) header = `✅ All ${total} tests passed (${recovered} recovered on re-run).`; + else header = `✅ All ${total} tests passed.`; + + const counts = `Passed: ${passed} · Failed: ${failed} · Total: ${total}`; + const rerunLine = reran ? `Re-run — recovered: ${recovered} · still failing: ${stillFailing}` : ''; + + // Run-page summary + let summary = core.summary + .addHeading('${{ env.system_release }}', '4') + .addHeading('Used repositories:', '4') + .addList(repos, true) + .addHeading(header, '4') + .addRaw(counts).addBreak(); + if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); + summary.addList(rerun, true).write(); + + // Failure reason as the 'compose-fail' annotation of this job + if (incomplete || stillFailing > 0) { + core.error(`${header} ${rerun.join(', ')}`.trim().slice(0, 3800), { title: 'compose-fail' }); + } + + // Markdown text for the Mattermost notification + const runUrl = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const text = [ + '#### AlmaLinux ${{ inputs.version_major }} s390x — compose tests', + '**${{ env.system_release }}**', + '', + header, + counts, + ...(rerunLine ? [rerunLine] : []), + '', + '**Used repositories:**', + ...repos.map(r => '- ' + r), + ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), + '', + '[Workflow run](' + runUrl + ')', + ].join('\n'); + core.setOutput('text', text); + + - name: Send notification to Mattermost + uses: mattermost/action-mattermost-notify@v2.1.0 + # Notify on both pass and fail; never fail the job on a notification error + if: ${{ always() && inputs.notify_mattermost }} + continue-on-error: true + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }} + MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }} + MATTERMOST_USERNAME: ${{ github.triggering_actor }} + TEXT: | + ${{ steps.summary.outputs.text }} + + - name: Fail if the test run did not complete + # Runs last (after notification) so an aborted/incomplete run — e.g. a + # test that kills the container, leaving an empty results.yaml — is + # reported as a failure instead of a false green. + if: ${{ always() && steps.eval.outputs.incomplete == 'true' }} run: | - echo "::error::No s390x compose tests on '${{ github.ref_name }}': run this workflow from the ci/tcg-ppc64le-s390x branch." - exit 0 + echo "::error::Test run did not complete: tmt exit '${{ steps.run-tests.outputs.tmt_rc }}', ${{ steps.eval.outputs.total }} results recorded." + exit 1 From 71a4f943aa4e35dcac90ff73be3d8bd80e09d9e4 Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:30:57 +0200 Subject: [PATCH 4/8] tests: run in containers, skipping what a container cannot The ppc64le and s390x tests run in containers (tmt context container=yes, CONTAINERTEST=1), which share the runner's kernel. Disable for container=yes the tests a container cannot run: attr (no loop device mounts), libbpf-tools and perf (BPF programs and perf events of the runner's kernel), libvirt/net (no libvirt networks), podman (no nested containers), selinux (no SELinux), sysstat/iostat (no dropping the page cache). Skip with CONTAINERTEST=1 p_kvdo (a kernel module), the NIC check of p_tcpdump and p_osbuild (an image build needs loop devices and mounts). t_GetArch takes the architecture from uname when there is no kernel package. List the skipped tests, with the reason, in the run-page summary and the Mattermost message of the ppc64le and s390x workflows: they only count the tests that ran. --- .../almalinux-compose-test-ppc64le.yml | 23 +++++++++++++++++- .../almalinux-compose-test-s390x.yml | 24 ++++++++++++++++++- tests/attr/main.fmf | 4 ++++ tests/legacy/functions.sh | 7 +++++- tests/legacy/p_kvdo/test.sh | 5 ++++ tests/legacy/p_osbuild/test.sh | 6 +++++ tests/legacy/p_tcpdump/01_tcpdump_nic.sh | 5 ++++ tests/libbpf-tools/main.fmf | 7 ++++-- tests/libvirt/main.fmf | 4 ++-- tests/libvirt/net/main.fmf | 4 ++++ tests/perf/main.fmf | 4 ++++ tests/podman/main.fmf | 3 +++ tests/selinux/main.fmf | 4 ++++ tests/sysstat/iostat/main.fmf | 4 ++++ 14 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 tests/sysstat/iostat/main.fmf diff --git a/.github/workflows/almalinux-compose-test-ppc64le.yml b/.github/workflows/almalinux-compose-test-ppc64le.yml index 4b74b29..4263008 100644 --- a/.github/workflows/almalinux-compose-test-ppc64le.yml +++ b/.github/workflows/almalinux-compose-test-ppc64le.yml @@ -666,6 +666,21 @@ jobs: // Re-run annotated failures (set only when the re-run step ran). const rerun = [${{ env.rerun_results }}]; + // Tests not run in the containers, hardcoded: the fmf adjust rules + // disable them for the tmt context container=yes, or they skip + // themselves when CONTAINERTEST=1 + const skipped = [ + '/tests/attr — a container may not mount a loop device', + '/tests/libbpf-tools — BPF programs on the runner\'s kernel, from a container', + '/tests/libvirt/net — a container may not create libvirt networks', + '/tests/perf — perf events of the runner\'s kernel, from a container', + '/tests/podman — no nested containers', + '/tests/selinux — no SELinux in a container', + '/tests/sysstat/iostat/cpu, /tests/sysstat/iostat/disk — a container may not drop the page cache', + '/tests/legacy/p_kvdo — kvdo is a kernel module', + '/tests/legacy/p_osbuild — an image build needs loop devices and mounts', + 'parts of other /tests/legacy tests — checks needing a whole system skip themselves', + ]; const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); @@ -697,7 +712,10 @@ jobs: .addHeading(header, '4') .addRaw(counts).addBreak(); if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); - summary.addList(rerun, true).write(); + summary.addList(rerun, true) + .addHeading('Skipped in the containers:', '4') + .addList(skipped) + .write(); // Failure reason as the 'compose-fail' annotation of this job if (incomplete || stillFailing > 0) { @@ -718,6 +736,9 @@ jobs: ...repos.map(r => '- ' + r), ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), '', + '**Skipped in the containers:**', + ...skipped.map(r => '- ' + r), + '', '[Workflow run](' + runUrl + ')', ].join('\n'); core.setOutput('text', text); diff --git a/.github/workflows/almalinux-compose-test-s390x.yml b/.github/workflows/almalinux-compose-test-s390x.yml index 54d5525..6b23ce5 100644 --- a/.github/workflows/almalinux-compose-test-s390x.yml +++ b/.github/workflows/almalinux-compose-test-s390x.yml @@ -681,6 +681,22 @@ jobs: // Re-run annotated failures (set only when the re-run step ran). const rerun = [${{ env.rerun_results }}]; + // Tests not run in the containers, hardcoded: the fmf adjust rules + // disable them for the tmt context container=yes, or they skip + // themselves when CONTAINERTEST=1 + const skipped = [ + '/tests/attr — a container may not mount a loop device', + '/tests/libbpf-tools — BPF programs on the runner\'s kernel, from a container', + '/tests/libvirt/net — a container may not create libvirt networks', + '/tests/perf — perf events of the runner\'s kernel, from a container', + '/tests/podman — no nested containers', + '/tests/selinux — no SELinux in a container', + '/tests/sysstat/iostat/cpu, /tests/sysstat/iostat/disk — a container may not drop the page cache', + '/tests/legacy/p_kvdo — kvdo is a kernel module', + '/tests/legacy/p_osbuild — an image build needs loop devices and mounts', + ...('${{ env.native }}' === 'true' ? [] : ['/tests/libvirt/socket: sysinfo check — no /proc/sysinfo on the runner\'s kernel']), + 'parts of other /tests/legacy tests — checks needing a whole system skip themselves', + ]; const passed = parseInt('${{ steps.eval.outputs.passed }}' || '0', 10); const failed = parseInt('${{ steps.eval.outputs.failed }}' || '0', 10); @@ -712,7 +728,10 @@ jobs: .addHeading(header, '4') .addRaw(counts).addBreak(); if (rerunLine) summary = summary.addRaw(rerunLine).addBreak(); - summary.addList(rerun, true).write(); + summary.addList(rerun, true) + .addHeading('Skipped in the containers:', '4') + .addList(skipped) + .write(); // Failure reason as the 'compose-fail' annotation of this job if (incomplete || stillFailing > 0) { @@ -733,6 +752,9 @@ jobs: ...repos.map(r => '- ' + r), ...(rerun.length ? ['', '**Re-run failed tests:**', ...rerun.map(r => '- ' + r)] : []), '', + '**Skipped in the containers:**', + ...skipped.map(r => '- ' + r), + '', '[Workflow run](' + runUrl + ')', ].join('\n'); core.setOutput('text', text); diff --git a/tests/attr/main.fmf b/tests/attr/main.fmf index 473d5de..7cf770c 100644 --- a/tests/attr/main.fmf +++ b/tests/attr/main.fmf @@ -2,3 +2,7 @@ summary: attr smoke tests description: Verify that attr works as expected test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: a container may not mount a loop device diff --git a/tests/legacy/functions.sh b/tests/legacy/functions.sh index 1c8bcb7..eedc9d4 100755 --- a/tests/legacy/functions.sh +++ b/tests/legacy/functions.sh @@ -210,7 +210,12 @@ function t_GetPkgVer # Description: get the arch function t_GetArch { - rpm -q kernel --queryformat '%{arch}\n' | head -n 1 + if rpm -q kernel > /dev/null 2>&1; then + rpm -q kernel --queryformat '%{arch}\n' | head -n 1 + else + # A container has no kernel package + uname -m + fi } # Set the arch diff --git a/tests/legacy/p_kvdo/test.sh b/tests/legacy/p_kvdo/test.sh index a9e7df2..76ae6e1 100755 --- a/tests/legacy/p_kvdo/test.sh +++ b/tests/legacy/p_kvdo/test.sh @@ -1,6 +1,11 @@ #!/bin/bash -e source ../functions.sh + +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running $0 - kvdo is a kernel module: running in container -> SKIP" + exit 0 +fi ./0-install_kvdo.sh ./kvdo_runtest.sh ./check_sb.sh \ No newline at end of file diff --git a/tests/legacy/p_osbuild/test.sh b/tests/legacy/p_osbuild/test.sh index 24d4a0f..4ca7d96 100755 --- a/tests/legacy/p_osbuild/test.sh +++ b/tests/legacy/p_osbuild/test.sh @@ -2,6 +2,12 @@ source ../functions.sh +# Building an image needs loop devices and mounts a container does not get +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running $0 - osbuild: running in container -> SKIP" + exit 0 +fi + # osbuild-composer ships x86_64 (v3-baseline) repo definitions only — there is # no x86_64_v2 arch block in the repositories JSON, and the build chroot pulls # v3 glibc which aborts on a v2 host. Detect the v2 variant once via the diff --git a/tests/legacy/p_tcpdump/01_tcpdump_nic.sh b/tests/legacy/p_tcpdump/01_tcpdump_nic.sh index 1cf4a8b..2e14421 100755 --- a/tests/legacy/p_tcpdump/01_tcpdump_nic.sh +++ b/tests/legacy/p_tcpdump/01_tcpdump_nic.sh @@ -4,6 +4,11 @@ t_Log "Running $0 - TCPdump test to Default-GW with IPv4" +if [ "$CONTAINERTEST" -eq "1" ]; then + t_Log "Running in container -> SKIP" + exit 0 +fi + # Grabing Default-Router if NIC def_gw_out=$(ip route list default|grep "default via"|head -n 1) def_gw=$(echo $def_gw_out | awk '{print $3}') diff --git a/tests/libbpf-tools/main.fmf b/tests/libbpf-tools/main.fmf index 29ed631..a83da47 100644 --- a/tests/libbpf-tools/main.fmf +++ b/tests/libbpf-tools/main.fmf @@ -4,5 +4,8 @@ test: ./test.sh framework: beakerlib enabled: true adjust: - when: distro == centos-7 or distro == centos-stream-8 - enabled: false + - when: distro == centos-7 or distro == centos-stream-8 + enabled: false + - when: container == yes + enabled: false + because: BPF programs on the runner's kernel, from a container diff --git a/tests/libvirt/main.fmf b/tests/libvirt/main.fmf index 20ecbea..64cc5f5 100644 --- a/tests/libvirt/main.fmf +++ b/tests/libvirt/main.fmf @@ -5,5 +5,5 @@ framework: beakerlib require: [libvirt,python3-libvirt] enabled: true adjust: - when: arch == aarch64 - enabled: false + - when: arch == aarch64 + enabled: false diff --git a/tests/libvirt/net/main.fmf b/tests/libvirt/net/main.fmf index 56a373f..f3d75f0 100644 --- a/tests/libvirt/net/main.fmf +++ b/tests/libvirt/net/main.fmf @@ -1,3 +1,7 @@ summary: libvirt networking smoke tests description: libvirt networking smoke tests enabled: true +adjust+: + - when: container == yes + enabled: false + because: a container may not create libvirt networks diff --git a/tests/perf/main.fmf b/tests/perf/main.fmf index cbe598f..4a0a543 100644 --- a/tests/perf/main.fmf +++ b/tests/perf/main.fmf @@ -2,3 +2,7 @@ summary: perf tests description: basic perf functionality tests test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: perf events of the runner's kernel, from a container diff --git a/tests/podman/main.fmf b/tests/podman/main.fmf index b138cbf..d9228be 100644 --- a/tests/podman/main.fmf +++ b/tests/podman/main.fmf @@ -6,3 +6,6 @@ enabled: true adjust: - when: distro == centos-7 enabled: false + - when: container == yes + enabled: false + because: no nested containers diff --git a/tests/selinux/main.fmf b/tests/selinux/main.fmf index 36d39af..4db328d 100644 --- a/tests/selinux/main.fmf +++ b/tests/selinux/main.fmf @@ -2,3 +2,7 @@ summary: selinux smoke tests description: basic selinux smoke tests test: ./test.sh framework: beakerlib +adjust: + - when: container == yes + enabled: false + because: no SELinux in a container diff --git a/tests/sysstat/iostat/main.fmf b/tests/sysstat/iostat/main.fmf new file mode 100644 index 0000000..55ebeb8 --- /dev/null +++ b/tests/sysstat/iostat/main.fmf @@ -0,0 +1,4 @@ +adjust: + - when: container == yes + enabled: false + because: a container may not drop the page cache From ac8224002974c0bb607ff3e70d41aed6be27ca32 Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:31:05 +0200 Subject: [PATCH 5/8] tests: fix the s390x failures that are not product bugs - p_bash: bash on s390x is built for the IBM vendor up to AlmaLinux 9 (s390x-ibm-linux-gnu), for redhat since 10 (s390x-redhat-linux-gnu). - libbpf-tools: bpf-numamove is not shipped for s390x. - libvirt/socket: libvirt reads the host's system information on s390x from /proc/sysinfo, which only an s390x kernel has: an emulated s390x container on another one has none, so skip the sysinfo check there. - p_gzip: s390x machines with DFLTCC (z15 and newer) compress in hardware, which gives the test file another compressed size: check the uncompressed size and name of 'gzip -l' only, not the md5sum of its whole output. --- tests/legacy/p_bash/bash_version_info_test.sh | 3 ++- tests/legacy/p_gzip/20-gzip-test | 14 +++----------- tests/libbpf-tools/test.sh | 2 ++ tests/libvirt/socket/test.sh | 12 +++++++++--- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/legacy/p_bash/bash_version_info_test.sh b/tests/legacy/p_bash/bash_version_info_test.sh index 988910e..b9da76a 100755 --- a/tests/legacy/p_bash/bash_version_info_test.sh +++ b/tests/legacy/p_bash/bash_version_info_test.sh @@ -5,6 +5,7 @@ t_Log "Running $0 - check that bash version info is the same with upstream." -bash --version | grep -qE "(i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64)-redhat-linux-gnu" +# s390x was built for the IBM vendor up to AlmaLinux 9 +bash --version | grep -qE "((i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64|s390x)-redhat|s390x-ibm)-linux-gnu" t_CheckExitStatus $? diff --git a/tests/legacy/p_gzip/20-gzip-test b/tests/legacy/p_gzip/20-gzip-test index b27852e..62f610a 100755 --- a/tests/legacy/p_gzip/20-gzip-test +++ b/tests/legacy/p_gzip/20-gzip-test @@ -78,21 +78,13 @@ gzip -c $FILE | gunzip | grep -q 'gzip-test of single file' || ExitFail # Check the correct info is listed for -l the output is # compressed uncompressed ratio uncompressed_name -# 59 25 24.0% /var/tmp/gzip-test.txt -# -# The md5sum is 4cbaba004a815915904bd524ede5edec -# -# The 32-bit gzip behaves differently: -# -# compressed uncompressed ratio uncompressed_name # 59 25 -8.0% /var/tmp/gzip-test.txt -# -# The md5sum is caf6dee7a56022a840316520134967a4 t_Log "Check the output of -l" gzip $FILE -mdhash=$(gzip -l $FILE.gz | md5sum | cut -f1 -d' ') -[ "$mdhash" == "4cbaba004a815915904bd524ede5edec" -o "$mdhash" == "caf6dee7a56022a840316520134967a4" ] || ExitFail +# Only the uncompressed size and name: the compressed size depends on the +# compressor, e.g. 63 bytes on an s390x machine compressing in hardware (DFLTCC) +gzip -l $FILE.gz | awk 'NR == 2 {print $2, $4}' | grep -qx "25 $FILE" || ExitFail # Check that -q works which removes the header information from -l t_Log "Check -q reduced the output verbosity" diff --git a/tests/libbpf-tools/test.sh b/tests/libbpf-tools/test.sh index 4910c69..12943b6 100755 --- a/tests/libbpf-tools/test.sh +++ b/tests/libbpf-tools/test.sh @@ -66,6 +66,8 @@ rlJournalStart rlPhaseEnd for tool in "${tools_versions_test[@]}"; do + # Not shipped for s390x + [ "${tool}" = bpf-numamove ] && [ "$(uname -m)" = s390x ] && continue rlPhaseStartTest "${tool} get version" rlRun "${tool} -V" rlPhaseEnd diff --git a/tests/libvirt/socket/test.sh b/tests/libvirt/socket/test.sh index 919248d..9b275f5 100755 --- a/tests/libvirt/socket/test.sh +++ b/tests/libvirt/socket/test.sh @@ -23,9 +23,15 @@ rlJournalStart rlRun "virsh -c qemu:///system nodeinfo" rlPhaseEnd - rlPhaseStartTest "should get sysinfo on qemu" - rlRun "virsh -c qemu:///system sysinfo" - rlPhaseEnd + # On s390x libvirt reads the host's system information from /proc/sysinfo, + # which only an s390x kernel provides: not a container on another one's + if [ "$(uname -m)" = s390x ] && [ ! -e /proc/sysinfo ]; then + rlLog "No /proc/sysinfo (s390x on another architecture's kernel): not checking sysinfo" + else + rlPhaseStartTest "should get sysinfo on qemu" + rlRun "virsh -c qemu:///system sysinfo" + rlPhaseEnd + fi rlPhaseStartTest "should access test socket with python" rlRun -s "python3 ./pytestaccess.py" From 728c5f211badbb51bcb148d42ab3b6a54e543d4a Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Fri, 30 Oct 2026 14:31:14 +0200 Subject: [PATCH 6/8] tests: take the emulated guests' slowness into account The ppc64le and s390x tests run emulated on forks (emulated_guest=true), several times slower than on the hardware the tests are meant for. - p_exim, p_openssh: the expect helpers wait 1 or 2 seconds for an answer, too short there: wait 30 seconds in an emulated guest. - p_osbuild: building an image takes more than an hour there: skip it in an emulated guest. --- tests/legacy/p_exim/_helper_exim_ehlo.expect | 7 ++++++- tests/legacy/p_exim/_helper_exim_helo.expect | 7 ++++++- .../p_openssh/_helper_sshd_user_login-with-key.expect | 7 ++++++- tests/legacy/p_openssh/_helper_sshd_user_login.expect | 7 ++++++- tests/legacy/p_osbuild/test.sh | 7 +++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/tests/legacy/p_exim/_helper_exim_ehlo.expect b/tests/legacy/p_exim/_helper_exim_ehlo.expect index 9250885..82ffb25 100755 --- a/tests/legacy/p_exim/_helper_exim_ehlo.expect +++ b/tests/legacy/p_exim/_helper_exim_ehlo.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Christoph Galuschka -set timeout 1 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 1 +} spawn telnet localhost 25 expect "220 *" send -- "ehlo localhost\n" diff --git a/tests/legacy/p_exim/_helper_exim_helo.expect b/tests/legacy/p_exim/_helper_exim_helo.expect index 941c55c..5743ebd 100755 --- a/tests/legacy/p_exim/_helper_exim_helo.expect +++ b/tests/legacy/p_exim/_helper_exim_helo.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Christoph Galuschka -set timeout 1 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 1 +} spawn telnet localhost 25 expect "220 *" send -- "helo localhost\n" diff --git a/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect b/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect index 7e01d18..16f71e7 100755 --- a/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect +++ b/tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Athmane Madjoudj -set timeout 2 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 2 +} spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/cat ssh_test_file send -- "\r" expect eof diff --git a/tests/legacy/p_openssh/_helper_sshd_user_login.expect b/tests/legacy/p_openssh/_helper_sshd_user_login.expect index 802f7b7..c85078e 100755 --- a/tests/legacy/p_openssh/_helper_sshd_user_login.expect +++ b/tests/legacy/p_openssh/_helper_sshd_user_login.expect @@ -1,7 +1,12 @@ #!/usr/bin/expect -f # Author: Athmane Madjoudj -set timeout 2 +# An emulated guest (the CI on forks) answers more slowly +if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} { + set timeout 30 +} else { + set timeout 2 +} spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/ls match_max 6000 expect "*?assword:*" diff --git a/tests/legacy/p_osbuild/test.sh b/tests/legacy/p_osbuild/test.sh index 4ca7d96..7d16233 100755 --- a/tests/legacy/p_osbuild/test.sh +++ b/tests/legacy/p_osbuild/test.sh @@ -2,6 +2,13 @@ source ../functions.sh +# In an emulated guest (the ppc64le and s390x CI on forks) building an image +# takes more than an hour. +if [ "x${emulated_guest}" = "xtrue" ]; then + t_Log "Running $0 - osbuild: emulated guest -> SKIP" + exit 0 +fi + # Building an image needs loop devices and mounts a container does not get if [ "$CONTAINERTEST" -eq "1" ]; then t_Log "Running $0 - osbuild: running in container -> SKIP" From 8232ddb99220264383d815ae566b8a0313d47cdc Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Mon, 2 Nov 2026 10:31:52 +0200 Subject: [PATCH 7/8] ci: give the containers a network they resolve names through On the native s390x runner the containers could not resolve any name for a whole job, in 3 of 21 runs, while the runner itself could: "Update the system" waited 2.5 minutes for its first name in every container, then failed with "Could not resolve host: mirrors.almalinux.org". The containers reach the runner's network through podman's default one, 10.88.0.0/16, and such a runner's DNS server was 10.88.92.1: the containers sent their queries to the bridge, so no name resolved (HTTPS to an address still worked). Before starting the containers, take the first of 10.88.0.0/16, 10.89.0.0/16, ... that is none of the runner's routes nor nameservers. When that is not 10.88.0.0/16, start the containers on a network of their own with it, without DNS like podman's default network, so that they still use the runner's nameservers. After starting them check that each one resolves a name, for about a minute. If they still cannot, print what DNS goes through (the runner's and the containers' nameservers, podman's network, IP forwarding, whether a container reaches the internet by address), reload podman's network, which re-applies its firewall rules, and check again. Fail the step at once if that does not help. --- ci/container-guests.sh | 78 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ci/container-guests.sh b/ci/container-guests.sh index cb50915..29c91b8 100755 --- a/ci/container-guests.sh +++ b/ci/container-guests.sh @@ -2,7 +2,8 @@ # Start AlmaLinux containers of an architecture with podman, one per name in # $guests, systemd as PID 1: natively on a runner of that architecture, or # run by QEMU user-mode emulation (the qemu- binfmt_misc handler must be -# registered) on another one. Waits until systemd has booted every container. +# registered) on another one. Waits until systemd has booted every container +# and checks that the containers resolve names. # Run commands in one with 'podman exec ...'; remove them with # 'podman rm -f -t 0 ': they do not stop on systemd's stop signal under # emulation. @@ -23,9 +24,39 @@ fi podman pull -q --platform "linux/${arch}" "${image}" > /dev/null +# The containers reach the runner's network through podman's default one, +# 10.88.0.0/16. Some runners are in it themselves (an s390x runner's DNS server +# was 10.88.92.1): the containers then send to the bridge what is for them, and +# cannot resolve names. Use the first subnet that is none of the runner's +# routes, nameservers included, on a network of our own if it is not podman's. +subnet=$(python3 -I - <<'EOF' +import ipaddress, json, re, subprocess +routes = json.loads(subprocess.run(["ip", "-j", "-4", "route", "show", "table", "all"], + capture_output=True, text=True, check=True).stdout) +used = [ipaddress.ip_network(r["dst"], strict=False) for r in routes + if r.get("dst", "default") != "default" and not r.get("dev", "").startswith("podman")] +for resolv in ("/etc/resolv.conf", "/run/systemd/resolve/resolv.conf"): + try: + used += [ipaddress.ip_network(ns) for ns in + re.findall(r"^nameserver\s+([0-9.]+)\s*$", open(resolv).read(), re.M)] + except OSError: + pass +candidates = [f"10.{n}.0.0/16" for n in range(88, 100)] + ["172.30.0.0/16", "192.168.200.0/24"] +print(next(c for c in map(ipaddress.ip_network, candidates) + if not any(c.overlaps(u) for u in used if not u.is_loopback))) +EOF +) +network=podman +if [ "${subnet}" != 10.88.0.0/16 ]; then + network=compose-tests + echo "[Debug] podman's network 10.88.0.0/16 overlaps the runner's: the containers get ${subnet}" + podman network exists "${network}" || + podman network create --disable-dns --subnet "${subnet}" "${network}" > /dev/null +fi + for guest in ${guests:?}; do podman run -d --name "${guest}" --hostname "${guest}" --platform "linux/${arch}" \ - --systemd=always ${podman_run_options:-} "${image}" > /dev/null + --network "${network}" --systemd=always ${podman_run_options:-} "${image}" > /dev/null done for guest in ${guests}; do @@ -77,3 +108,46 @@ for guest in ${guests}; do echo "[Debug] ${guest}: $(podman exec "${guest}" sh -c 'echo "$(uname -m), $(cat /etc/almalinux-release), systemd $(systemctl is-system-running)"')" echo "[Debug] ${guest}: still failed: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)" done + +# The containers resolve names with the runner's upstream DNS servers, through +# podman's network. On the native s390x runner that failed now and then for a +# whole job: dnf then waited 2.5 minutes for the first name. Check it, show +# why it fails, and reload podman's network (its firewall rules) once. +dns_name=mirrors.almalinux.org + +dns_works() { + for guest in ${guests}; do + podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null || return 1 + done +} + +dns_debug() { + echo "[Debug] runner: ${dns_name} $(getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve'), nameservers: $(awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)" + if [ -r /run/systemd/resolve/resolv.conf ]; then + echo "[Debug] runner: systemd-resolved upstream nameservers: $(awk '/^nameserver/ {print $2}' /run/systemd/resolve/resolv.conf | xargs)" + fi + echo "[Debug] podman network: $(podman network inspect "${network}" --format '{{.Driver}} {{.NetworkInterface}} {{range .Subnets}}{{.Subnet}} {{end}}dns_enabled={{.DNSEnabled}}'), backend $(podman info --format '{{.Host.NetworkBackend}}'), ip_forward=$(cat /proc/sys/net/ipv4/ip_forward)" + for guest in ${guests}; do + echo "[Debug] ${guest}: nameservers: $(podman exec "${guest}" awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)," \ + "${dns_name} $(podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve')," \ + "HTTPS to 1.1.1.1: $(podman exec "${guest}" curl -s -o /dev/null --max-time 15 -w '%{http_code}' https://1.1.1.1/ || echo failed)" + done +} + +for i in $(seq 1 4); do + dns_works && break + if [ "${i}" -eq 4 ]; then + echo "[Warning] The containers cannot resolve ${dns_name}: reloading podman's network" + dns_debug + podman network reload --all > /dev/null + if dns_works; then + echo "[Debug] The containers resolve ${dns_name} after reloading podman's network" + break + else + echo "[Error] The containers still cannot resolve ${dns_name}" >&2 + dns_debug + exit 1 + fi + fi + sleep 15 +done From 42c38d0e5ae45dec40d048c2ae3bd5ce78460d83 Mon Sep 17 00:00:00 2001 From: Yuriy Kohut Date: Mon, 2 Nov 2026 13:30:22 +0200 Subject: [PATCH 8/8] ci: log the peak of the inotify instances in use while testing The count of root's inotify instances in use logged after the containers start does not show how near the runner's limit a run gets: in the native ppc64le runs that failed, with the limit at 128, D-Bus and libvirtd ran out of them minutes into the tests and for the rest of them. Count root's inotify instances every 5 seconds while the tests run and log the largest count, with how many each process held and in which container, to tell what holds them when it happens again. --- .../almalinux-compose-test-ppc64le.yml | 8 ++++++ .../almalinux-compose-test-s390x.yml | 8 ++++++ ci/inotify-peak.sh | 28 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100755 ci/inotify-peak.sh diff --git a/.github/workflows/almalinux-compose-test-ppc64le.yml b/.github/workflows/almalinux-compose-test-ppc64le.yml index 4263008..21161f9 100644 --- a/.github/workflows/almalinux-compose-test-ppc64le.yml +++ b/.github/workflows/almalinux-compose-test-ppc64le.yml @@ -460,7 +460,15 @@ jobs: cd /compose-tests tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" REMOTE + # Count root's inotify instances while the tests run: their peak, and who + # held them, shows how near the runner's limit the tests get (see "Start + # the containers") + sudo ci/inotify-peak.sh "${{ runner.temp }}/inotify-peak" & + inotify_peak=$! ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + sudo pkill -f ci/inotify-peak.sh; wait ${inotify_peak} + echo "[Debug] inotify instances in use at the peak while testing: $(head -1 "${{ runner.temp }}/inotify-peak") (of root, limit $(cat /proc/sys/fs/inotify/max_user_instances))" + tail -n +2 "${{ runner.temp }}/inotify-peak" | expand -t 6,20 | sed 's/^/[Debug] /' # The worst exit code of the containers: one outside 0/1 (2 = tmt error, # others = the container died) wins over 1 (some failed) over 0 (all pass). rc=0 diff --git a/.github/workflows/almalinux-compose-test-s390x.yml b/.github/workflows/almalinux-compose-test-s390x.yml index 6b23ce5..17a3842 100644 --- a/.github/workflows/almalinux-compose-test-s390x.yml +++ b/.github/workflows/almalinux-compose-test-s390x.yml @@ -475,7 +475,15 @@ jobs: cd /compose-tests tmt ${{ env.tmt_options }} -vvv -c distro=centos-stream-${{ env.release_version }} -c container=yes run --all --environment pungi_repository=${{ inputs.pungi_repository }} --environment pulp_repository=${{ inputs.pulp_repository }} --environment CONTAINERTEST=1 --environment emulated_guest=${{ env.native != 'true' }} provision --how=local test --name "$(cat /tmp/ct-shard.regex)" REMOTE + # Count root's inotify instances while the tests run: their peak, and who + # held them, shows how near the runner's limit the tests get (see "Start + # the containers") + sudo ci/inotify-peak.sh "${{ runner.temp }}/inotify-peak" & + inotify_peak=$! ci/each-guest.sh 'sudo podman exec -i $guest bash -s < /tmp/ct-run.sh; echo $? > ./tmt_rc.$guest' + sudo pkill -f ci/inotify-peak.sh; wait ${inotify_peak} + echo "[Debug] inotify instances in use at the peak while testing: $(head -1 "${{ runner.temp }}/inotify-peak") (of root, limit $(cat /proc/sys/fs/inotify/max_user_instances))" + tail -n +2 "${{ runner.temp }}/inotify-peak" | expand -t 6,20 | sed 's/^/[Debug] /' # The worst exit code of the containers: one outside 0/1 (2 = tmt error, # others = the container died) wins over 1 (some failed) over 0 (all pass). rc=0 diff --git a/ci/inotify-peak.sh b/ci/inotify-peak.sh new file mode 100755 index 0000000..c1bbc6f --- /dev/null +++ b/ci/inotify-peak.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Count the inotify instances root's processes hold every 5 seconds, until +# killed, and keep the largest count in FILE: on its first line the count and +# when, then how many each process held and where it ran (a container or the +# runner). Run as root, so that every process can be seen. +# +# sudo ci/inotify-peak.sh /tmp/inotify-peak & ...; sudo kill $!; cat /tmp/inotify-peak + +file=${1:?usage: $0 FILE} +# Containers' IDs and names: a process' cgroup tells the container it runs in +containers=$(podman ps --no-trunc --format '{{.ID}} {{.Names}}' 2> /dev/null) +peak=0 +while :; do + fds=$(find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null) + count=$(grep -c . <<< "${fds}") + if [ "${count}" -gt "${peak}" ]; then + peak=${count} + { + echo "${peak} at $(date -u +%H:%M:%S)" + cut -d/ -f3 <<< "${fds}" | sort | uniq -c | while read -r n pid; do + id=$(grep -o 'libpod-[0-9a-f]\{64\}' "/proc/${pid}/cgroup" 2> /dev/null | head -1) + where=$(awk -v id="${id#libpod-}" '$1 == id {print $2}' <<< "${containers}") + printf '%s\t%s\t%s\n' "${n}" "$(cat "/proc/${pid}/comm" 2> /dev/null || echo '(gone)')" "${where:-runner}" + done | awk -F '\t' '{n[$3 "\t" $2] += $1} END {for (k in n) print n[k] "\t" k}' | sort -k2,2 -k1,1rn + } > "${file}.new" && mv "${file}.new" "${file}" + fi + sleep 5 +done