Run the ppc64le and s390x compose tests in containers

ci/container-guests.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+# Start AlmaLinux containers of an architecture with podman, one per name in
+# $guests, systemd as PID 1: natively on a runner of that architecture, or
+# run by QEMU user-mode emulation (the qemu-<arch> binfmt_misc handler must be
+# registered) on another one. Waits until systemd has booted every container
+# and checks that the containers resolve names.
+# Run commands in one with 'podman exec <guest> ...'; remove them with
+# 'podman rm -f -t 0 <guest>': they do not stop on systemd's stop signal under
+# emulation.
+#
+#   guests='almalinux-0 almalinux-1' ci/container-guests.sh ppc64le docker.io/almalinux/9-init:latest
+#
+# Extra 'podman run' options can be given in $podman_run_options.
+
+usage="usage: $0 ARCH IMAGE"
+arch=${1:?${usage}}
+image=${2:?${usage}}
+set -e -o pipefail
+
+if [ "$(uname -m)" != "${arch}" ] && [ ! -e "/proc/sys/fs/binfmt_misc/qemu-${arch}" ]; then
+  echo "[Error] No qemu-${arch} binfmt_misc handler: ${arch} binaries cannot run here" >&2
+  exit 1
+fi
+
+podman pull -q --platform "linux/${arch}" "${image}" > /dev/null
+
+# The containers reach the runner's network through podman's default one,
+# 10.88.0.0/16. Some runners are in it themselves (an s390x runner's DNS server
+# was 10.88.92.1): the containers then send to the bridge what is for them, and
+# cannot resolve names. Use the first subnet that is none of the runner's
+# routes, nameservers included, on a network of our own if it is not podman's.
+subnet=$(python3 -I - <<'EOF'
+import ipaddress, json, re, subprocess
+routes = json.loads(subprocess.run(["ip", "-j", "-4", "route", "show", "table", "all"],
+                                   capture_output=True, text=True, check=True).stdout)
+used = [ipaddress.ip_network(r["dst"], strict=False) for r in routes
+        if r.get("dst", "default") != "default" and not r.get("dev", "").startswith("podman")]
+for resolv in ("/etc/resolv.conf", "/run/systemd/resolve/resolv.conf"):
+    try:
+        used += [ipaddress.ip_network(ns) for ns in
+                 re.findall(r"^nameserver\s+([0-9.]+)\s*$", open(resolv).read(), re.M)]
+    except OSError:
+        pass
+candidates = [f"10.{n}.0.0/16" for n in range(88, 100)] + ["172.30.0.0/16", "192.168.200.0/24"]
+print(next(c for c in map(ipaddress.ip_network, candidates)
+           if not any(c.overlaps(u) for u in used if not u.is_loopback)))
+EOF
+)
+network=podman
+if [ "${subnet}" != 10.88.0.0/16 ]; then
+  network=compose-tests
+  echo "[Debug] podman's network 10.88.0.0/16 overlaps the runner's: the containers get ${subnet}"
+  podman network exists "${network}" ||
+    podman network create --disable-dns --subnet "${subnet}" "${network}" > /dev/null
+fi
+
+for guest in ${guests:?}; do
+  podman run -d --name "${guest}" --hostname "${guest}" --platform "linux/${arch}" \
+    --network "${network}" --systemd=always ${podman_run_options:-} "${image}" > /dev/null
+done
+
+for guest in ${guests}; do
+  for i in $(seq 1 60); do
+    state=$(podman exec "${guest}" systemctl is-system-running 2> /dev/null) || true
+    case "${state}" in running|degraded) break ;; esac
+    if [ "${i}" -eq 60 ]; then
+      echo "[Error] ${guest}: systemd not up after 10 minutes ('${state}')" >&2
+      exit 1
+    fi
+    sleep 10
+  done
+
+  # Under emulation the units with credentials (ImportCredential=) fail at
+  # boot with 243/CREDENTIALS: QEMU does not translate the mount calls systemd
+  # sets them up with. Run systemd-sysusers and systemd-tmpfiles ourselves,
+  # give journald (it imports credentials since AlmaLinux 10) a copy of its
+  # unit without them, and start again what failed at boot, D-Bus with it.
+  echo "[Debug] ${guest}: failed at boot: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)"
+  # The services installed later fail the same way: libvirtd and virtsecretd
+  # load an encrypted credential since libvirt 12.8 (AlmaLinux Kitten). Clear
+  # their credentials with a drop-in. Not with a top-level one for every
+  # service: under s390x emulation systemd reports a unit with a drop-in as
+  # changed on disk, and postgresql-setup then refuses to initialize the
+  # database.
+  if [ "$(uname -m)" != "${arch}" ]; then
+    podman exec "${guest}" sh -c '
+      for unit in libvirtd virtsecretd; do
+        mkdir -p /etc/systemd/system/${unit}.service.d
+        printf "%s\n" "[Service]" LoadCredential= LoadCredentialEncrypted= \
+          > /etc/systemd/system/${unit}.service.d/99-emulated-no-credentials.conf
+      done
+    '
+  fi
+  podman exec "${guest}" sh -c '
+    systemd-sysusers
+    systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev
+    unit=/usr/lib/systemd/system/systemd-journald.service
+    if grep -q "^ImportCredential=" "${unit}"; then
+      sed "/^ImportCredential=/d" "${unit}" > /etc/systemd/system/systemd-journald.service
+    fi
+    systemctl daemon-reload
+    failed=$(systemctl --failed --no-legend --plain | awk "{print \$1}" |
+      grep -vE "^systemd-(sysusers|tmpfiles-setup|tmpfiles-setup-dev|tmpfiles-setup-dev-early|firstboot)\.service$")
+    systemctl reset-failed
+    [ -z "${failed}" ] || systemctl restart ${failed}
+  ' || true
+
+  echo "[Debug] ${guest}: $(podman exec "${guest}" sh -c 'echo "$(uname -m), $(cat /etc/almalinux-release), systemd $(systemctl is-system-running)"')"
+  echo "[Debug] ${guest}: still failed: $(podman exec "${guest}" systemctl --failed --no-legend --plain | awk '{print $1}' | xargs)"
+done
+
+# The containers resolve names with the runner's upstream DNS servers, through
+# podman's network. On the native s390x runner that failed now and then for a
+# whole job: dnf then waited 2.5 minutes for the first name. Check it, show
+# why it fails, and reload podman's network (its firewall rules) once.
+dns_name=mirrors.almalinux.org
+
+dns_works() {
+  for guest in ${guests}; do
+    podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null || return 1
+  done
+}
+
+dns_debug() {
+  echo "[Debug] runner: ${dns_name} $(getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve'), nameservers: $(awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)"
+  if [ -r /run/systemd/resolve/resolv.conf ]; then
+    echo "[Debug] runner: systemd-resolved upstream nameservers: $(awk '/^nameserver/ {print $2}' /run/systemd/resolve/resolv.conf | xargs)"
+  fi
+  echo "[Debug] podman network: $(podman network inspect "${network}" --format '{{.Driver}} {{.NetworkInterface}} {{range .Subnets}}{{.Subnet}} {{end}}dns_enabled={{.DNSEnabled}}'), backend $(podman info --format '{{.Host.NetworkBackend}}'), ip_forward=$(cat /proc/sys/net/ipv4/ip_forward)"
+  for guest in ${guests}; do
+    echo "[Debug] ${guest}: nameservers: $(podman exec "${guest}" awk '/^nameserver/ {print $2}' /etc/resolv.conf | xargs)," \
+      "${dns_name} $(podman exec "${guest}" timeout 15 getent hosts "${dns_name}" > /dev/null && echo resolves || echo 'does not resolve')," \
+      "HTTPS to 1.1.1.1: $(podman exec "${guest}" curl -s -o /dev/null --max-time 15 -w '%{http_code}' https://1.1.1.1/ || echo failed)"
+  done
+}
+
+for i in $(seq 1 4); do
+  dns_works && break
+  if [ "${i}" -eq 4 ]; then
+    echo "[Warning] The containers cannot resolve ${dns_name}: reloading podman's network"
+    dns_debug
+    podman network reload --all > /dev/null
+    if dns_works; then
+      echo "[Debug] The containers resolve ${dns_name} after reloading podman's network"
+      break
+    else
+      echo "[Error] The containers still cannot resolve ${dns_name}" >&2
+      dns_debug
+      exit 1
+    fi
+  fi
+  sleep 15
+done

ci/inotify-peak.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Count the inotify instances root's processes hold every 5 seconds, until
+# killed, and keep the largest count in FILE: on its first line the count and
+# when, then how many each process held and where it ran (a container or the
+# runner). Run as root, so that every process can be seen.
+#
+#   sudo ci/inotify-peak.sh /tmp/inotify-peak & ...; sudo kill $!; cat /tmp/inotify-peak
+
+file=${1:?usage: $0 FILE}
+# Containers' IDs and names: a process' cgroup tells the container it runs in
+containers=$(podman ps --no-trunc --format '{{.ID}} {{.Names}}' 2> /dev/null)
+peak=0
+while :; do
+  fds=$(find /proc/[0-9]*/fd -user 0 -lname 'anon_inode:inotify' 2> /dev/null)
+  count=$(grep -c . <<< "${fds}")
+  if [ "${count}" -gt "${peak}" ]; then
+    peak=${count}
+    {
+      echo "${peak} at $(date -u +%H:%M:%S)"
+      cut -d/ -f3 <<< "${fds}" | sort | uniq -c | while read -r n pid; do
+        id=$(grep -o 'libpod-[0-9a-f]\{64\}' "/proc/${pid}/cgroup" 2> /dev/null | head -1)
+        where=$(awk -v id="${id#libpod-}" '$1 == id {print $2}' <<< "${containers}")
+        printf '%s\t%s\t%s\n' "${n}" "$(cat "/proc/${pid}/comm" 2> /dev/null || echo '(gone)')" "${where:-runner}"
+      done | awk -F '\t' '{n[$3 "\t" $2] += $1} END {for (k in n) print n[k] "\t" k}' | sort -k2,2 -k1,1rn
+    } > "${file}.new" && mv "${file}.new" "${file}"
+  fi
+  sleep 5
+done

tests/attr/main.fmf

@@ -2,3 +2,7 @@ summary: attr smoke tests
 description: Verify that attr works as expected
 test: ./test.sh
 framework: beakerlib
+adjust:
+  - when: container == yes
+    enabled: false
+    because: a container may not mount a loop device

tests/legacy/functions.sh

@@ -210,7 +210,12 @@ function t_GetPkgVer
 # Description: get the arch
 function t_GetArch
 {
-	rpm -q kernel --queryformat '%{arch}\n' | head -n 1
+	if rpm -q kernel > /dev/null 2>&1; then
+		rpm -q kernel --queryformat '%{arch}\n' | head -n 1
+	else
+		# A container has no kernel package
+		uname -m
+	fi
 }
 
 # Set the arch

tests/legacy/p_bash/bash_version_info_test.sh

@@ -5,6 +5,7 @@
 
 t_Log "Running $0 -  check that bash version info is the same with upstream."
 
-bash --version | grep -qE "(i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64)-redhat-linux-gnu"
+# s390x was built for the IBM vendor up to AlmaLinux 9
+bash --version | grep -qE "((i386|i686|x86_64|aarch64|armv7hl|powerpc64le|powerpc64|s390x)-redhat|s390x-ibm)-linux-gnu"
 
 t_CheckExitStatus $?

tests/legacy/p_exim/_helper_exim_ehlo.expect

@@ -1,7 +1,12 @@
 #!/usr/bin/expect -f
 # Author: Christoph Galuschka <tigalch@tigalch.org>
 
-set timeout 1
+# An emulated guest (the CI on forks) answers more slowly
+if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} {
+    set timeout 30
+} else {
+    set timeout 1
+}
 spawn telnet localhost 25
 expect "220 *"
 send -- "ehlo localhost\n"

tests/legacy/p_exim/_helper_exim_helo.expect

@@ -1,7 +1,12 @@
 #!/usr/bin/expect -f
 # Author: Christoph Galuschka <tigalch@tigalch.org>
 
-set timeout 1
+# An emulated guest (the CI on forks) answers more slowly
+if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} {
+    set timeout 30
+} else {
+    set timeout 1
+}
 spawn telnet localhost 25
 expect "220 *"
 send -- "helo localhost\n"

tests/legacy/p_gzip/20-gzip-test

@@ -78,21 +78,13 @@ gzip -c $FILE | gunzip | grep -q 'gzip-test of single file' || ExitFail
 
 # Check the correct info is listed for -l the output is
 #         compressed        uncompressed  ratio uncompressed_name
-#                 59                  25  24.0% /var/tmp/gzip-test.txt
-#
-# The md5sum is 4cbaba004a815915904bd524ede5edec
-#
-# The 32-bit gzip behaves differently:
-# 
-#         compressed        uncompressed  ratio uncompressed_name
 #                 59                  25  -8.0% /var/tmp/gzip-test.txt
-#
-# The md5sum is caf6dee7a56022a840316520134967a4
 
 t_Log "Check the output of -l"
 gzip $FILE
-mdhash=$(gzip -l $FILE.gz | md5sum | cut -f1 -d' ')
-[ "$mdhash" == "4cbaba004a815915904bd524ede5edec" -o "$mdhash" == "caf6dee7a56022a840316520134967a4" ] || ExitFail
+# Only the uncompressed size and name: the compressed size depends on the
+# compressor, e.g. 63 bytes on an s390x machine compressing in hardware (DFLTCC)
+gzip -l $FILE.gz | awk 'NR == 2 {print $2, $4}' | grep -qx "25 $FILE" || ExitFail
 
 # Check that -q works which removes the header information from -l
 t_Log "Check -q reduced the output verbosity"

tests/legacy/p_kvdo/test.sh

@@ -1,6 +1,11 @@
 #!/bin/bash -e
 
 source ../functions.sh
+
+if [ "$CONTAINERTEST" -eq "1" ]; then
+    t_Log "Running $0 - kvdo is a kernel module: running in container -> SKIP"
+    exit 0
+fi
 ./0-install_kvdo.sh
 ./kvdo_runtest.sh
 ./check_sb.sh

tests/legacy/p_openssh/_helper_sshd_user_login-with-key.expect

@@ -1,7 +1,12 @@
 #!/usr/bin/expect -f
 # Author: Athmane Madjoudj <athmanem@gmail.com>
 
-set timeout 2
+# An emulated guest (the CI on forks) answers more slowly
+if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} {
+    set timeout 30
+} else {
+    set timeout 2
+}
 spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/cat ssh_test_file
 send -- "\r"
 expect eof

tests/legacy/p_openssh/_helper_sshd_user_login.expect

@@ -1,7 +1,12 @@
 #!/usr/bin/expect -f
 # Author: Athmane Madjoudj <athmanem@gmail.com>
 
-set timeout 2
+# An emulated guest (the CI on forks) answers more slowly
+if {[info exists env(emulated_guest)] && $env(emulated_guest) eq "true"} {
+    set timeout 30
+} else {
+    set timeout 2
+}
 spawn ssh -o StrictHostKeyChecking=no sshtest@localhost /bin/ls
 match_max 6000
 expect "*?assword:*"

tests/legacy/p_osbuild/test.sh

@@ -2,6 +2,19 @@
 
 source ../functions.sh
 
+# In an emulated guest (the ppc64le and s390x CI on forks) building an image
+# takes more than an hour.
+if [ "x${emulated_guest}" = "xtrue" ]; then
+    t_Log "Running $0 - osbuild: emulated guest -> SKIP"
+    exit 0
+fi
+
+# Building an image needs loop devices and mounts a container does not get
+if [ "$CONTAINERTEST" -eq "1" ]; then
+    t_Log "Running $0 - osbuild: running in container -> SKIP"
+    exit 0
+fi
+
 # osbuild-composer ships x86_64 (v3-baseline) repo definitions only — there is
 # no x86_64_v2 arch block in the repositories JSON, and the build chroot pulls
 # v3 glibc which aborts on a v2 host. Detect the v2 variant once via the

tests/legacy/p_tcpdump/01_tcpdump_nic.sh

@@ -4,6 +4,11 @@
 
 t_Log "Running $0 - TCPdump test to Default-GW with IPv4"
 
+if [ "$CONTAINERTEST" -eq "1" ]; then
+    t_Log "Running in container -> SKIP"
+    exit 0
+fi
+
 # Grabing Default-Router if NIC
 def_gw_out=$(ip route list default|grep "default via"|head -n 1)
 def_gw=$(echo $def_gw_out | awk '{print $3}')

tests/libbpf-tools/main.fmf

@@ -4,5 +4,8 @@ test: ./test.sh
 framework: beakerlib
 enabled: true
 adjust:
-  when: distro == centos-7 or distro == centos-stream-8
-  enabled: false
+  - when: distro == centos-7 or distro == centos-stream-8
+    enabled: false
+  - when: container == yes
+    enabled: false
+    because: BPF programs on the runner's kernel, from a container

tests/libbpf-tools/test.sh

@@ -66,6 +66,8 @@ rlJournalStart
   rlPhaseEnd
 
 for tool in "${tools_versions_test[@]}"; do
+  # Not shipped for s390x
+  [ "${tool}" = bpf-numamove ] && [ "$(uname -m)" = s390x ] && continue
   rlPhaseStartTest "${tool} get version"
   rlRun "${tool} -V"
   rlPhaseEnd

tests/libvirt/main.fmf

@@ -5,5 +5,5 @@ framework: beakerlib
 require: [libvirt,python3-libvirt]
 enabled: true
 adjust:
-  when: arch == aarch64
-  enabled: false
+  - when: arch == aarch64
+    enabled: false

tests/libvirt/net/main.fmf

@@ -1,3 +1,7 @@
 summary: libvirt networking smoke tests
 description: libvirt networking smoke tests
 enabled: true
+adjust+:
+  - when: container == yes
+    enabled: false
+    because: a container may not create libvirt networks

tests/libvirt/socket/test.sh

@@ -23,9 +23,15 @@ rlJournalStart
   rlRun "virsh -c qemu:///system nodeinfo"
   rlPhaseEnd
 
-  rlPhaseStartTest "should get sysinfo on qemu"
-  rlRun "virsh -c qemu:///system sysinfo"
-  rlPhaseEnd
+  # On s390x libvirt reads the host's system information from /proc/sysinfo,
+  # which only an s390x kernel provides: not a container on another one's
+  if [ "$(uname -m)" = s390x ] && [ ! -e /proc/sysinfo ]; then
+    rlLog "No /proc/sysinfo (s390x on another architecture's kernel): not checking sysinfo"
+  else
+    rlPhaseStartTest "should get sysinfo on qemu"
+    rlRun "virsh -c qemu:///system sysinfo"
+    rlPhaseEnd
+  fi
 
   rlPhaseStartTest "should access test socket with python"
   rlRun -s "python3 ./pytestaccess.py"