diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ff41be5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + target-branch: dev diff --git a/.github/workflows/regression_test.yml b/.github/workflows/regression_test.yml index b4554ab..73de83c 100644 --- a/.github/workflows/regression_test.yml +++ b/.github/workflows/regression_test.yml @@ -1,25 +1,29 @@ -# This is a basic workflow that is manually triggered - name: regression_test -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: workflow_dispatch: push: - branches: [ master ] + branches: [dev, master] pull_request: - branches: [ master ] + branches: [dev, master] + +concurrency: + group: levelx-regression-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "linux_job" run_tests: permissions: contents: read issues: read checks: write pull-requests: write + # GitHub validates deployment permissions even when deployment is skipped. + # The reusable test job scopes its token separately. pages: write id-token: write - uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@master \ No newline at end of file + uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@b37cd4a81a1cb8c2ebefc438220ab7f009e13362 + with: + coverage_name: merged + coverage_thresholds: '68 100' + skip_deploy: ${{ github.ref != 'refs/heads/master' || github.event_name == 'pull_request' }} diff --git a/scripts/build.sh b/scripts/build.sh index 0ec3d05..5b78ef2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -10,4 +10,6 @@ # SPDX-License-Identifier: MIT ############################################################################## -$(dirname `realpath $0`)/../test/cmake/run.sh build all \ No newline at end of file +set -euo pipefail + +exec "$(dirname "$(realpath "$0")")/../test/cmake/run.sh" build all diff --git a/scripts/install.sh b/scripts/install.sh index bcf4a0c..17575bc 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,30 +10,49 @@ # SPDX-License-Identifier: MIT ############################################################################## -# +set -euo pipefail + +# Retry transient package and network failures a bounded number of times. +retry() { + local attempt + for attempt in 1 2 3; do + if "$@"; then + return 0 + fi + if [ "$attempt" -lt 3 ]; then + sleep $((attempt * 5)) + fi + done + return 1 +} -# Remove large folder -rm -rf /opt/hostedtoolcache +apt_options=(-o Acquire::Retries=3 -o DPkg::Lock::Timeout=60) +if ! retry sudo timeout 150 apt-get "${apt_options[@]}" update; then + echo "Package index update failed; package installation will verify availability." >&2 +fi +retry sudo timeout 150 apt-get "${apt_options[@]}" install -y \ + cmake gcc-14 gcc-14-multilib git ninja-build python3-venv \ + unifdef p7zip-full tofrodos gawk -# Install necessary softwares for Ubuntu. +venv_dir="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/levelx-ci-venv" +python3 -m venv "$venv_dir" +retry timeout 120 "$venv_dir/bin/python" -m pip install \ + --retries 3 --timeout 30 gcovr==8.6 -sudo apt-get update -sudo apt-get install -y \ - gcc-multilib \ - git \ - g++ \ - python3-pip \ - ninja-build \ - unifdef \ - p7zip-full \ - tofrodos \ - gawk \ - software-properties-common +cc="${CC:-gcc-14}" +gcov="${GCOV:-gcov-14}" +cc_version=$("$cc" -dumpfullversion) +gcov_version=$("$gcov" --version | sed -n '1{s/.* \([0-9][0-9]*\.[0-9][0-9]*\(\.[0-9][0-9]*\)\?\).*/\1/p;}') +if [ -z "$gcov_version" ] || [ "$cc_version" != "$gcov_version" ]; then + echo "Compiler $cc and coverage tool $gcov have different versions." >&2 + exit 1 +fi -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - -CODENAME=$(lsb_release -c | cut -f2 -d':' | sed 's/\t//') -apt-add-repository "deb https://apt.kitware.com/ubuntu/ $CODENAME main" +if [ -n "${GITHUB_ENV:-}" ]; then + printf 'CC=%s\nGCOV=%s\n' "$cc" "$gcov" >> "$GITHUB_ENV" + printf '%s\n' "$venv_dir/bin" >> "$GITHUB_PATH" +fi -python3 -m pip install --upgrade pip -pip3 install gcovr==4.1 -pip install --upgrade cmake \ No newline at end of file +"$venv_dir/bin/gcovr" --version | head -1 +"$cc" --version | head -1 +"$gcov" --version | head -1 diff --git a/scripts/test.sh b/scripts/test.sh index 1c4be22..763b793 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -10,4 +10,10 @@ # SPDX-License-Identifier: MIT ############################################################################## -$(dirname `realpath $0`)/../test/cmake/run.sh test all \ No newline at end of file +set -euo pipefail + +test_dir="$(dirname "$(realpath "$0")")/../test/cmake" +"$test_dir/run.sh" test all +if [ "${TX_COVERAGE:-OFF}" = ON ]; then + "$test_dir/check_coverage.sh" +fi diff --git a/test/cmake/CMakeLists.txt b/test/cmake/CMakeLists.txt index fbcd573..eb35819 100644 --- a/test/cmake/CMakeLists.txt +++ b/test/cmake/CMakeLists.txt @@ -78,14 +78,19 @@ add_subdirectory(regression) add_subdirectory(samples) # Coverage -if(CMAKE_BUILD_TYPE MATCHES ".*_coverage") +option(TX_COVERAGE "Instrument every regression configuration" OFF) +if(TX_COVERAGE OR CMAKE_BUILD_TYPE MATCHES ".*_coverage") target_compile_options(levelx PRIVATE -fprofile-arcs -ftest-coverage) target_link_options(levelx PRIVATE -fprofile-arcs -ftest-coverage) endif() # Build ThreadX library once -execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/run.sh build_libs) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/run.sh build_libs + RESULT_VARIABLE dependency_status) +if(NOT dependency_status EQUAL 0) + message(FATAL_ERROR "Dependency build failed: ${dependency_status}") +endif() add_custom_target(build_libs ALL COMMAND ${CMAKE_CURRENT_LIST_DIR}/run.sh build_libs) add_dependencies(levelx build_libs) diff --git a/test/cmake/README.md b/test/cmake/README.md new file mode 100644 index 0000000..3b204a8 --- /dev/null +++ b/test/cmake/README.md @@ -0,0 +1,62 @@ +# Linux regression tests + +Use GCC 14, matching gcov 14, CMake, Ninja and gcovr 8.6. The CI installer +supports Ubuntu 24.04. From the repository root: + +```sh +export CC=gcc-14 GCOV=gcov-14 TX_COVERAGE=ON +./scripts/build.sh +./scripts/test.sh +``` + +The runner verifies the dependency commits recorded in `threadx-revision.txt` +and `filex-revision.txt`. The FileX revision is the tested head of +[eclipse-threadx/filex#106](https://github.com/eclipse-threadx/filex/pull/106), +which is still open. Updating a dependency requires changing its pin and +rerunning all ten configurations. An existing checkout with another revision +or modified tracked files is rejected. + +All configurations need the FileX source tree. Non-standalone configurations +link the shared ThreadX and FileX libraries; standalone configurations build +FileX with standalone support. LevelX has no Windows regression port. + +`TX_COVERAGE=ON` instruments every LevelX configuration. A complete test run +clears old coverage first, runs every configuration, and collects coverage even +when a test fails. Reports under `coverage_report/per_configuration` contain +JSON, XML and HTML for each configuration. The `merged` reports combine all ten +JSON inputs and use repository-relative source names. Missing, empty or +unmeasured inputs fail collection. Both line and branch floors are enforced by +`coverage.sh`; the workflow also enforces its integer line floor. + +After a successful full run, `./test/cmake/check_coverage.sh` verifies rejection +of missing, empty and unmeasured coverage inputs, then restores and remerges the +valid reports. + +The reusable workflow pin supplies bounded install, build and test steps and +retains test and coverage artifacts on failure. Only master push or manual +runs can deploy coverage. Dependabot updates target dev; GitHub activates this +configuration once it is present on the default branch. + +## Coverage target + +The measured GCC 14 union is 1,632/2,400 lines (68.00%) and 1,596/2,439 +branches (65.44%). The enforced floors are 68.0% lines and 65.4% branches. +The 100% target still needs 768 lines and 843 branches covered. The largest +line gaps are the FileX simulator adapters (164), NOR block reclaim (69), NOR +extended open (65), and NAND block data movement (38). Other gaps include +media-error paths, metadata allocation, sector release and simulator failures. +These sources remain in the denominator. + +The ECC regression checks every single-bit position in a 512-byte page, +corrections in both halves, and uncorrectable errors. All four ECC helper files +have full line coverage. The full, driver-interface and combined NOR cache +configurations contribute 97, 51 and 65 source lines absent from the default +configuration respectively. Standalone configurations select a subset of the +same source lines; their reports still participate in the union. Function +merging uses the earliest declaration line because driver-interface macros +place otherwise identical function declarations on different lines. + +The ECC test views aligned `USHORT` storage through a character pointer because +the ECC implementation accesses words. This is a deviation from advisory +MISRA C:2004 Rule 11.4; character access preserves alignment and is explicitly +permitted by the character-pointer exception in MISRA C:2012/2023 Rule 11.3. diff --git a/test/cmake/check_coverage.sh b/test/cmake/check_coverage.sh new file mode 100755 index 0000000..4bdc07c --- /dev/null +++ b/test/cmake/check_coverage.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright (c) 2026 Eclipse ThreadX contributors +# SPDX-License-Identifier: MIT + +set -euo pipefail +cd "$(dirname "$0")" + +# Verify rejection of damaged inputs using a completed coverage run. +base=coverage_report/per_configuration/default_build_coverage +backup=$(mktemp -d) +cp "$base.json" "$backup/input.json" +cp "$base.xml" "$backup/input.xml" +# Restore the original inputs on success or failure. +restore() { + cp "$backup/input.json" "$base.json" + cp "$backup/input.xml" "$base.xml" + rm -r "$backup" +} +trap restore EXIT + +# Require the collector to reject each damaged input. +expect_failure() { + if ./coverage.sh --merge > "$backup/output.log" 2>&1; then + echo "Coverage accepted $1." >&2 + exit 1 + fi + echo "Rejected $1." +} + +rm "$base.json" +expect_failure 'a missing JSON input' +: > "$base.json" +expect_failure 'an empty JSON input' +printf '{"files":[]}\n' > "$base.json" +expect_failure 'an unmeasured JSON input' +cp "$backup/input.json" "$base.json" +printf '\n' > "$base.xml" +expect_failure 'an unmeasured XML input' +cp "$backup/input.xml" "$base.xml" +./coverage.sh --merge diff --git a/test/cmake/coverage.sh b/test/cmake/coverage.sh index 817b62f..cf9468a 100755 --- a/test/cmake/coverage.sh +++ b/test/cmake/coverage.sh @@ -10,10 +10,144 @@ # SPDX-License-Identifier: MIT ############################################################################## +set -euo pipefail -set -e +cd "$(dirname "$0")" +repo_root=$(cd ../.. && pwd) +report_dir=coverage_report +configurations=( + default_build_coverage free_sector_verify_build full_build standalone_build + standalone_free_sector_verify_build standalone_full_build + new_driver_interface_build nor_obsolete_cache_build nor_mapping_cache_build + nor_obsolete_mapping_cache_build +) -cd $(dirname $0) -mkdir -p coverage_report/$1 -gcovr --object-directory=build/$1/levelx/CMakeFiles/levelx.dir/common/src -r ../../common/src --xml-pretty --output coverage_report/$1.xml -gcovr --object-directory=build/$1/levelx/CMakeFiles/levelx.dir/common/src -r ../../common/src --html --html-details --output coverage_report/$1/index.html +# Validate measured sources and make the XML source root portable. +check_report() { + python3 - "$@" <<'PY' +import json +import pathlib +import sys +import xml.etree.ElementTree as ET + +report = pathlib.Path(sys.argv[1]) +tree = ET.parse(report) +root = tree.getroot() +for source in root.findall('./sources/source'): + source.text = '.' +tree.write(report, encoding='utf-8', xml_declaration=True) +classes = root.findall('.//class') +if int(root.get('lines-valid', '0')) == 0 or not classes: + raise SystemExit(f'{report}: report contains no measured LevelX files') +if any(not item.get('filename', '').startswith('common/src/') for item in classes): + raise SystemExit(f'{report}: report contains a source outside common/src') +if len(sys.argv) > 2: + with open(sys.argv[2], encoding='utf-8') as stream: + data = json.load(stream) + if not data.get('files') or not any(item.get('lines') for item in data['files']): + raise SystemExit(f'{sys.argv[2]}: tracefile contains no measured files') + if any(not item.get('file', '').startswith('common/src/') for item in data['files']): + raise SystemExit(f'{sys.argv[2]}: tracefile contains a source outside common/src') +print(f"{report}: lines {root.get('lines-covered')}/{root.get('lines-valid')} " + f"({float(root.get('line-rate', '0')) * 100:.2f}%), branches " + f"{root.get('branches-covered')}/{root.get('branches-valid')} " + f"({float(root.get('branch-rate', '0')) * 100:.2f}%)") +PY +} + +# Enforce separate line and branch coverage floors on the union. +check_merged_floor() { + python3 - "$1" <<'PY' +import sys +import xml.etree.ElementTree as ET + +root = ET.parse(sys.argv[1]).getroot() +for label, attribute, minimum in ( + ('line', 'lines', 680), + ('branch', 'branches', 654), +): + covered = int(root.get(f'{attribute}-covered', '0')) + valid = int(root.get(f'{attribute}-valid', '0')) + if valid == 0 or covered * 1000 < valid * minimum: + raise SystemExit( + f'{sys.argv[1]}: {label} coverage {covered}/{valid} ' + f'is below {minimum / 10:.1f}%' + ) +PY +} + +if [ "${1:-}" = --clean ]; then + if [ -d "$report_dir" ]; then + rm -r -- "$report_dir" + fi + if [ -d build ]; then + find build -type f -name '*.gcda' -delete + fi + exit 0 +fi + +if [ "${1:-}" = --merge ]; then + trace_args=() + for configuration in "${configurations[@]}"; do + base="$report_dir/per_configuration/$configuration" + for path in "$base.json" "$base.xml" "$base/index.html"; do + if [ ! -s "$path" ]; then + echo "Missing or empty coverage report: $path" >&2 + exit 1 + fi + done + check_report "$base.xml" "$base.json" + trace_args+=(--add-tracefile "$base.json") + done + + # Driver-interface macros place the same function on different source lines. + trace_args+=(--merge-mode-functions=merge-use-line-min) + mkdir -p "$report_dir/merged" + gcovr -r "$repo_root" "${trace_args[@]}" --json "$report_dir/merged.json" --xml-pretty \ + --output "$report_dir/merged.xml" + gcovr -r "$repo_root" "${trace_args[@]}" --html --html-details \ + --output "$report_dir/merged/index.html" + check_report "$report_dir/merged.xml" "$report_dir/merged.json" + check_merged_floor "$report_dir/merged.xml" + exit 0 +fi + +configuration="${1:-}" +valid=0 +for item in "${configurations[@]}"; do + if [ "$configuration" = "$item" ]; then + valid=1 + break + fi +done +if [ "$valid" -ne 1 ]; then + echo "Unknown coverage configuration: $configuration" >&2 + exit 1 +fi + +cc_name=$(basename "${CC:-gcc}") +if [ -n "${GCOV:-}" ]; then + gcov="$GCOV" +elif [[ "$cc_name" = gcc* ]]; then + gcov="gcov${cc_name#gcc}" +else + gcov=gcov +fi +if ! command -v "$gcov" >/dev/null 2>&1; then + echo "Coverage tool $gcov is unavailable." >&2 + exit 1 +fi + +objects="$PWD/build/$configuration/levelx/CMakeFiles/levelx.dir/common/src" +if [ ! -d "$objects" ] || [ -z "$(find "$objects" -name '*.gcda' -print -quit)" ]; then + echo "No LevelX coverage data for $configuration." >&2 + exit 1 +fi + +base="$report_dir/per_configuration/$configuration" +mkdir -p "$base" +gcovr --gcov-executable "$gcov" -r "$repo_root" -f "$repo_root/common/src" \ + "$objects" --json "$base.json" --xml-pretty --output "$base.xml" +gcovr --gcov-executable "$gcov" -r "$repo_root" -f "$repo_root/common/src" \ + "$objects" --html --html-details --output "$base/index.html" +check_report "$base.xml" "$base.json" diff --git a/test/cmake/filex-revision.txt b/test/cmake/filex-revision.txt new file mode 100644 index 0000000..6a821fd --- /dev/null +++ b/test/cmake/filex-revision.txt @@ -0,0 +1 @@ +79c703d917e648c615ae6e0c253e63a73bb134a5 diff --git a/test/cmake/regression/CMakeLists.txt b/test/cmake/regression/CMakeLists.txt index f92c71a..9ac1b61 100644 --- a/test/cmake/regression/CMakeLists.txt +++ b/test/cmake/regression/CMakeLists.txt @@ -6,6 +6,7 @@ project(regression_test LANGUAGES C) set(SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../../regression) set(regression_test_cases + ${SOURCE_DIR}/levelx_nand_ecc_test.c ${SOURCE_DIR}/levelx_nand_flash_test.c ${SOURCE_DIR}/levelx_nor_flash_test.c ${SOURCE_DIR}/levelx_nor_flash_test_cache.c) diff --git a/test/cmake/run.sh b/test/cmake/run.sh index 3f8159d..4c50615 100755 --- a/test/cmake/run.sh +++ b/test/cmake/run.sh @@ -10,11 +10,34 @@ # SPDX-License-Identifier: MIT ############################################################################## +set -euo pipefail -cd $(dirname $0) +cd "$(dirname "$0")" +for dependency in threadx filex; do + revision=$(cat "$dependency-revision.txt") + repository="https://github.com/eclipse-threadx/$dependency.git" + if [ ! -e "$dependency" ]; then + git init -q "$dependency" + git -C "$dependency" remote add origin "$repository" + timeout 180 git -C "$dependency" fetch --depth 1 origin "$revision" + git -C "$dependency" checkout -q --detach FETCH_HEAD + fi + if [ "$(git -C "$dependency" rev-parse HEAD)" != "$revision" ] || + [ -n "$(git -C "$dependency" status --porcelain --untracked-files=no)" ]; then + echo "$dependency checkout does not match its pinned revision." >&2 + exit 1 + fi +done -# if threadx repo does not exist, clone it -[ -d threadx ] || git clone https://github.com/eclipse-threadx/threadx.git --depth 1 -[ -d filex ] || git clone https://github.com/eclipse-threadx/filex.git --depth 1 -[ -f .run.sh ] || ln -sf threadx/scripts/cmake_bootstrap.sh .run.sh -./.run.sh $* \ No newline at end of file +bootstrap=threadx/scripts/cmake_bootstrap.sh +if [ ! -f "$bootstrap" ]; then + echo "ThreadX bootstrap script is missing." >&2 + exit 1 +fi + +if [ "${1:-}" = test ] && [ "${2:-}" = all ] && [ "${TX_COVERAGE:-OFF}" = ON ]; then + ./coverage.sh --clean +fi + +ln -sfn "$bootstrap" .run.sh +exec ./.run.sh "$@" diff --git a/test/cmake/threadx-revision.txt b/test/cmake/threadx-revision.txt new file mode 100644 index 0000000..b01234d --- /dev/null +++ b/test/cmake/threadx-revision.txt @@ -0,0 +1 @@ +b37cd4a81a1cb8c2ebefc438220ab7f009e13362 diff --git a/test/regression/levelx_nand_ecc_test.c b/test/regression/levelx_nand_ecc_test.c new file mode 100644 index 0000000..2831bcd --- /dev/null +++ b/test/regression/levelx_nand_ecc_test.c @@ -0,0 +1,100 @@ +/*************************************************************************** + * Copyright (c) 2026 Eclipse ThreadX contributors + * + * This program and the accompanying materials are made available under the + * terms of the MIT License which is available at + * https://opensource.org/licenses/MIT. + * + * AI Disclosure: This file was largely AI-generated by Codex (GPT-6). + * The AI-generated portions may be considered public domain (CC0-1.0) + * and not subject to the project's licence. The human contributor has + * reviewed and verified that the code is correct. + * + * SPDX-License-Identifier: MIT and CC0-1.0 + **************************************************************************/ + +#include +#include "lx_api.h" + +#ifndef LX_STANDALONE_ENABLE +/* Supply the application hook required by the shared ThreadX library. */ +VOID tx_application_define(VOID *first_unused_memory) +{ + (void) first_unused_memory; +} +#endif + +/* Check page ECC correction across both 256-byte portions of a page. */ +int main(void) +{ + +LX_NAND_FLASH flash = {0}; +USHORT storage[256]; +UCHAR *page = (UCHAR *) storage; +UCHAR expected[512]; +UCHAR ecc[6]; +UINT i; +UINT bit; +int failed = 0; + + flash.lx_nand_flash_bytes_per_page = 512; + for (i = 0; i < 512; i++) + { + expected[i] = (UCHAR) (i & 0xFFU); + } + memcpy(page, expected, sizeof(expected)); + if (lx_nand_flash_page_ecc_compute(&flash, page, ecc) != LX_SUCCESS) + { + failed = 1; + } + if (lx_nand_flash_page_ecc_check(&flash, page, ecc) != LX_SUCCESS) + { + failed = 1; + } + + /* Every single data bit must be corrected without changing other bytes. */ + for (i = 0; i < 512; i++) + { + for (bit = 0; bit < 8; bit++) + { + memcpy(page, expected, sizeof(expected)); + page[i] ^= (UCHAR) (1U << bit); + if (lx_nand_flash_page_ecc_check(&flash, page, ecc) != LX_NAND_ERROR_CORRECTED) + { + failed = 1; + } + if (memcmp(page, expected, sizeof(expected)) != 0) + { + failed = 1; + } + } + } + + /* Corrections in both portions must retain the corrected status. */ + page[0] ^= 1U; + page[256] ^= 1U; + if (lx_nand_flash_page_ecc_check(&flash, page, ecc) != LX_NAND_ERROR_CORRECTED) + { + failed = 1; + } + if (memcmp(page, expected, sizeof(expected)) != 0) + { + failed = 1; + } + + /* An uncorrectable second portion must override a first-portion correction. */ + page[0] ^= 1U; + page[256] ^= 3U; + if (lx_nand_flash_page_ecc_check(&flash, page, ecc) != LX_NAND_ERROR_NOT_CORRECTED) + { + failed = 1; + } + memcpy(page, expected, sizeof(expected)); + page[0] ^= 3U; + if (lx_nand_flash_page_ecc_check(&flash, page, ecc) != LX_NAND_ERROR_NOT_CORRECTED) + { + failed = 1; + } + + return(failed); +}