diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..29a1c5fbe --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + target-branch: dev + schedule: + interval: weekly + ignore: + # The cross-repository bootstrap and workflow are reviewed together. + - dependency-name: eclipse-threadx/threadx/.github/workflows/regression_template.yml diff --git a/.github/workflows/regression_test.yml b/.github/workflows/regression_test.yml index 5576b211c..e2d39acd2 100644 --- a/.github/workflows/regression_test.yml +++ b/.github/workflows/regression_test.yml @@ -1,29 +1,27 @@ name: USBX Regression Test -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: workflow_dispatch: inputs: tests_to_run: - description: 'all, single or multiple of default_build_coverage error_check_build_full_coverage tracex_enable_build device_buffer_owner_build device_zero_copy_build nofx_build_coverage optimized_build standalone_device_build_coverage standalone_device_buffer_owner_build standalone_device_zero_copy_build standalone_host_build_coverage standalone_build_coverage generic_build otg_support_build memory_management_build_coverage msrc_rtos_build msrc_standalone_build' - required: false + description: 'all, or space-separated profiles from test/cmake/usbx/CMakeLists.txt' + required: true default: 'all' + type: string skip_coverage: required: false type: boolean default: false - coverage_name: - required: false - default: 'default_build_coverage' push: - branches: [ master ] + branches: [dev, master] pull_request: - branches: [ master ] + branches: [dev, master] -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: +concurrency: + group: usbx-regression-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: manual_tests: if: github.event_name == 'workflow_dispatch' permissions: @@ -31,16 +29,19 @@ jobs: issues: read checks: write pull-requests: write + # GitHub validates deployment permissions even when that job is skipped. + # The called test job narrows its own token scope. pages: write id-token: write - - uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@master + uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@b37cd4a81a1cb8c2ebefc438220ab7f009e13362 with: cmake_path: ./test/cmake/usbx - build_script: ./scripts/build.sh ${{ inputs.tests_to_run }} - test_script: ./scripts/test.sh ${{ inputs.tests_to_run }} - coverage_name: ${{ inputs.coverage_name }} - skip_coverage: ${{ !!inputs.skip_coverage }} + build_script: ./scripts/build.sh --manual + test_script: ./scripts/test.sh --manual + coverage_name: ${{ inputs.tests_to_run == 'all' && 'merged' || 'subset' }} + coverage_thresholds: '0 100' + skip_coverage: ${{ inputs.skip_coverage }} + skip_deploy: ${{ github.ref != 'refs/heads/master' || inputs.tests_to_run != 'all' }} auto_tests: if: github.event_name != 'workflow_dispatch' @@ -49,13 +50,15 @@ jobs: issues: read checks: write pull-requests: write + # The reusable workflow declares a separate deployment job. pages: write id-token: write - - uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@master + uses: eclipse-threadx/threadx/.github/workflows/regression_template.yml@b37cd4a81a1cb8c2ebefc438220ab7f009e13362 with: cmake_path: ./test/cmake/usbx build_script: ./scripts/build.sh all test_script: ./scripts/test.sh all - coverage_name: default_build_coverage + coverage_name: merged + coverage_thresholds: '66 100' skip_coverage: false + skip_deploy: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.gitignore b/.gitignore index f187e2a32..c42604ed1 100755 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ compile_commands.json CTestTestfile.cmake coverage_report/ + +/test/externals/ +/test/cmake/usbx/.run.sh +/test/cmake/usbx/libs/ +__pycache__/ diff --git a/samples/demo_usbx.c b/samples/demo_usbx.c index 1b4ddaad7..ce7349253 100644 --- a/samples/demo_usbx.c +++ b/samples/demo_usbx.c @@ -8,10 +8,12 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This is a small demo of the USBX */ #include "ux_api.h" +#include "ux_device_stack.h" #include "ux_system.h" #include "ux_utility.h" #include "ux_host_class_dpump.h" diff --git a/scripts/build.sh b/scripts/build.sh index 3c73f76a3..242931de1 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -10,4 +10,10 @@ # SPDX-License-Identifier: MIT ############################################################################## -$(dirname `realpath $0`)/../test/cmake/usbx/run.sh build $@ +set -euo pipefail +export CC=${CC:-gcc-14} +export GCOV=${GCOV:-gcov-14} +export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-4} +"$(dirname "$(realpath "$0")")/../test/cmake/usbx/run.sh" build "$@" +echo "Checking runner and coverage failure handling in isolated fixtures." +python3 "$(dirname "$(realpath "$0")")/../test/cmake/usbx/test_runner.py" diff --git a/scripts/install.sh b/scripts/install.sh index 5ba1cde68..850cc3c9e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,27 +10,50 @@ # SPDX-License-Identifier: MIT ############################################################################## -# -# Install necessary softwares for Ubuntu. - -sudo apt-get update -sudo apt-get install -y \ - gcc-multilib \ - git \ - g++ \ - python3-pip \ - ninja-build \ - unifdef \ - p7zip-full \ - tofrodos \ - gawk \ - software-properties-common - -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - -CODENAME=$(lsb_release -c | cut -f2 -d':' | sed 's/\t//') -sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $CODENAME main" -sudo apt-get -y install cmake - -python3 -m pip install --upgrade pip -pip3 install gcovr==4.1 -pip3 install --upgrade cmake +set -euo pipefail + +retry() { + local attempt + for attempt in 1 2 3; do + if "$@"; then + return 0 + fi + if [ "$attempt" -lt 3 ]; then + sleep $((attempt * 5)) + fi + done + return 1 +} + +apt_options=(-o Acquire::Retries=3 -o DPkg::Lock::Timeout=60) +if ! retry sudo timeout 150 apt-get "${apt_options[@]}" update; then + echo "Package index update failed; package installation will verify availability." >&2 +fi +retry sudo timeout 150 apt-get "${apt_options[@]}" install -y \ + cmake gcc-14 gcc-14-multilib git ninja-build python3-venv \ + unifdef p7zip-full tofrodos gawk + +venv_dir="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/usbx-ci-venv" +python3 -m venv "$venv_dir" +retry timeout 120 "$venv_dir/bin/python" -m pip install \ + --retries 3 --timeout 30 gcovr==8.6 + +cc="${CC:-gcc-14}" +gcov="${GCOV:-gcov-14}" +cc_version=$("$cc" -dumpfullversion) +gcov_version=$("$gcov" --version | sed -n '1{s/.* \([0-9][0-9]*\.[0-9][0-9]*\(\.[0-9][0-9]*\)\?\).*/\1/p;}') +if [ -z "$gcov_version" ] || [ "$cc_version" != "$gcov_version" ]; then + echo "Compiler $cc and coverage tool $gcov have different versions." >&2 + exit 1 +fi + +if [ -n "${GITHUB_ENV:-}" ]; then + printf 'CC=%s\nGCOV=%s\n' "$cc" "$gcov" >> "$GITHUB_ENV" + printf '%s\n' "$venv_dir/bin" >> "$GITHUB_PATH" +fi + +"$venv_dir/bin/gcovr" --version | head -1 +"$cc" --version | head -1 +"$gcov" --version | head -1 + +printf 'Coverage environment: %s\n' "$venv_dir/bin/activate" diff --git a/scripts/test.sh b/scripts/test.sh index 7a3720cd6..6a591a44c 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -10,4 +10,10 @@ # SPDX-License-Identifier: MIT ############################################################################## -CTEST_PARALLEL_LEVEL=4 $(dirname `realpath $0`)/../test/cmake/usbx/run.sh test $@ +set -euo pipefail +export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-4} +export CTEST_REPEAT_FAIL=1 +export CC=${CC:-gcc-14} +export GCOV=${GCOV:-gcov-14} +export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-4} +exec "$(dirname "$(realpath "$0")")/../test/cmake/usbx/run.sh" test "$@" diff --git a/test/cmake/usbx/BASELINE.md b/test/cmake/usbx/BASELINE.md new file mode 100644 index 000000000..bb043ec02 --- /dev/null +++ b/test/cmake/usbx/BASELINE.md @@ -0,0 +1,40 @@ +# GCC 14 baseline + +Measured at USBX commit `ffc0f79d951c716a03addcf7255020328e0d5b47`, with the +dependency revisions in `dependencies.txt`, GCC/gcov 14.3.0 and gcovr 8.6. +All 17 configurations were configured and built from clean trees with CMake and +Ninja. CTest ran every configuration once, including configurations with build +failures: 3,019/3,043 cases passed in 666.17 seconds of summed CTest wall time. + +Twenty-three cases could not run because five regression sources had incompatible +pointer arguments or missing declarations. The sample also lacked device-stack +declarations. One standalone printer transfer assertion failed. These results +are retained as failures; a subsequent isolated printer pass does not erase them. + +Coverage below uses the existing profile-specific exclusions, so percentages are +not comparable across profiles and cannot be averaged or merged. A dash means +the profile was not instrumented. Standalone-device collection initially failed +gcovr's 2^32 suspicious-hit heuristic; the same raw counters were then collected +with the documented 2^40 threshold and unchanged exclusions. No tests were rerun +to produce that coverage. Build times are Ninja execution spans and exclude +configuration and dependency setup. + +| Profile | Passed / tests | Build s | Test s | Lines hit / valid | Branches hit / valid | +| --- | ---: | ---: | ---: | ---: | ---: | +| default_build_coverage | 427/430 | 10.838 | 13.89 | 7701/8014 | 3420/3738 | +| error_check_build_full_coverage | 425/430 | 11.190 | 13.53 | 14767/24084 | 6238/11250 | +| tracex_enable_build | 427/430 | 11.390 | 13.48 | — | — | +| device_buffer_owner_build | 427/430 | 11.371 | 11.18 | — | — | +| device_zero_copy_build | 427/430 | 11.271 | 16.53 | — | — | +| nofx_build_coverage | 46/46 | 6.778 | 5.57 | 2751/7955 | 993/3738 | +| optimized_build | 129/129 | 6.893 | 3.46 | — | — | +| standalone_device_build_coverage | 55/57 | 6.669 | 185.88 | 2625/3098 | 880/1279 | +| standalone_device_buffer_owner_build | 56/57 | 6.782 | 185.39 | — | — | +| standalone_device_zero_copy_build | 56/57 | 6.896 | 177.48 | — | — | +| standalone_host_build_coverage | 44/44 | 6.633 | 20.78 | 4005/5255 | 1464/2289 | +| standalone_build_coverage | 20/20 | 5.412 | 0.25 | 2011/8277 | 720/3567 | +| generic_build | 1/1 | 6.047 | 0.01 | — | — | +| otg_support_build | 427/430 | 10.614 | 11.31 | — | — | +| memory_management_build_coverage | 11/11 | 6.619 | 4.63 | 4748/8069 | 1715/3780 | +| msrc_rtos_build | 33/33 | 6.482 | 0.76 | — | — | +| msrc_standalone_build | 8/8 | 5.069 | 2.04 | — | — | diff --git a/test/cmake/usbx/CMakeLists.txt b/test/cmake/usbx/CMakeLists.txt index fb2f69f75..b070c3b9d 100644 --- a/test/cmake/usbx/CMakeLists.txt +++ b/test/cmake/usbx/CMakeLists.txt @@ -5,6 +5,8 @@ cmake_policy(SET CMP0077 NEW) project(usbx_test LANGUAGES C) +option(USBX_CI_COVERAGE "Instrument every selected configuration" "$ENV{TX_COVERAGE}") + # Use customized ux_user.h set(UX_USER_FILE ${CMAKE_CURRENT_SOURCE_DIR}/ux_user.h) @@ -215,7 +217,7 @@ else() message(STATUS "Building usbx BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") endif() if(NOT BUILD_SHARED_LIBS) - if(CMAKE_BUILD_TYPE MATCHES ".*_coverage") + if(USBX_CI_COVERAGE OR CMAKE_BUILD_TYPE MATCHES ".*_coverage") add_link_options(-fprofile-arcs) add_link_options(-lgcov) endif() @@ -255,13 +257,17 @@ if(NOT (CMAKE_BUILD_TYPE MATCHES "standalone.*")) endif() # Coverage -if(CMAKE_BUILD_TYPE MATCHES ".*_coverage") +if(USBX_CI_COVERAGE OR CMAKE_BUILD_TYPE MATCHES ".*_coverage") target_compile_options(usbx PRIVATE -fprofile-arcs -ftest-coverage) target_link_options(usbx PRIVATE -fprofile-arcs -ftest-coverage) endif() # Build ThreadX library once -execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/run.sh build_libs) +execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/run.sh build_libs + RESULT_VARIABLE dependency_status) +if(NOT dependency_status EQUAL 0) + message(FATAL_ERROR "Dependency build failed: ${dependency_status}") +endif() add_custom_target(build_libs ALL COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/run.sh build_libs) diff --git a/test/cmake/usbx/README.md b/test/cmake/usbx/README.md new file mode 100644 index 000000000..1011555e9 --- /dev/null +++ b/test/cmake/usbx/README.md @@ -0,0 +1,62 @@ +# Linux regression and coverage + +The regression workflow builds and tests all 17 profiles on every pull request to +`dev` or `master`. Its check name is `auto_tests / run_tests`. Workflow +success alone does not enforce merging: repository rules must separately require +that check. The staged Otterdog ruleset must remain disabled until this workflow +is available on `dev`. + +Run `scripts/install.sh` on Ubuntu 24.04, then use GCC 14 and its matching gcov with +CMake and Ninja: + +```sh +TX_COVERAGE=ON ./scripts/build.sh all +TX_COVERAGE=ON ./scripts/test.sh all +python3 test/cmake/usbx/test_runner.py +``` + +The installer installs gcovr 8.6 in a virtual environment and exports its location +through GitHub Actions environment files. Outside Actions, activate the reported +virtual environment before running coverage. `CC` and `GCOV` can select matching +compiler and coverage executables. The default build and test concurrency is four. +CTest runs each case once; failing cases are not retried into a passing result. + +`dependencies.txt` records immutable ThreadX, FileX and NetX Duo revisions. FileX +PR #106 is open at the pinned revision; this setup does not assume it has merged. +Existing dependency checkouts must be clean and match their pins. Remove an old +checkout explicitly when updating a pin. The ThreadX bootstrap supplies JUnit +output without runtime edits to dependency files. + +Pass profile names instead of `all` to either script, or select them in the manual +workflow. Build and test output names the exact selection. JUnit files, test logs +and `build/results.txt` report actual counts and elapsed test times. The generic +profile's single CTest case is a build-only placeholder, not a functional USB test. + +Every selected profile is instrumented when `TX_COVERAGE=ON`, including the generic +build. Raw JSON, Cobertura XML and HTML use one inclusion policy: all compiled C +sources matching `common/*/src/*.c`. Host controllers, simulators, optional classes +and untested source lines remain in the denominator. Dependency and test code are +outside that policy. No profile-specific diagnostic exclusion affects the union. + +Each input must have nonempty JSON, XML and HTML, measured source lines, and +repository-relative paths. The merger checks that JSON and XML agree and that its +measured and covered source-line sets equal the exact unions of its inputs. +Reports and execution counters are cleared before each test selection. A subset +cannot consume an earlier full run's reports or publish complete coverage. +Available diagnostics and coverage survive failing tests; missing coverage itself +fails the runner. Full-suite line and branch floors are in `coverage-floors.json`. +Standalone polling produces more than six billion positive hits in a single +profile. Direct GCC 14 output confirms balanced loop counts. Collection uses a +2^40 suspicious-hit threshold instead of gcovr's default 2^32 heuristic; negative +counts and parse errors remain fatal, and no lines are ignored to obtain a report. + +Only successful complete runs on `master` can deploy Pages. Callers grant the +permissions required for GitHub to validate the pinned reusable workflow's +separate deployment job; its test job retains narrower permissions. The template +bounds installation, build and test steps and uploads diagnostics on failure. +The manual selector is parsed from the event file, never interpolated as shell +code. All pull requests retain the full matrix; there is no conditional routing. + +Dependabot targets `dev` once its configuration is present on the default branch. +The cross-repository workflow/bootstrap revision requires a separate review and +is excluded from routine action updates. diff --git a/test/cmake/usbx/VALIDATION.md b/test/cmake/usbx/VALIDATION.md new file mode 100644 index 000000000..df5eb34d8 --- /dev/null +++ b/test/cmake/usbx/VALIDATION.md @@ -0,0 +1,75 @@ +# Full-matrix validation + +GCC/gcov 14.3.0, gcovr 8.6, CMake and Ninja: 3,043/3,043 tests passed across +all 17 configurations, with no skips or retries. Summed CTest wall time was +674.14 seconds. The generic profile contributes one build-only placeholder. +The compiler errors and original coverage are recorded in [BASELINE.md](BASELINE.md). + +| Profile | Passed / tests | Build s | Test s | Lines hit / valid | Branches hit / valid | +| --- | ---: | ---: | ---: | ---: | ---: | +| default_build_coverage | 430/430 | 11.879 | 13.99 | 14228/24018 | 5741/11162 | +| error_check_build_full_coverage | 430/430 | 12.779 | 15.38 | 14914/24084 | 6385/11250 | +| tracex_enable_build | 430/430 | 12.593 | 14.91 | 14698/24806 | 5768/11272 | +| device_buffer_owner_build | 430/430 | 12.885 | 14.69 | 15337/25269 | 5994/11474 | +| device_zero_copy_build | 430/430 | 12.952 | 14.87 | 15291/25242 | 5974/11474 | +| nofx_build_coverage | 46/46 | 7.399 | 3.96 | 3302/23982 | 1179/11208 | +| optimized_build | 129/129 | 8.226 | 3.71 | 3604/22895 | 1329/10645 | +| standalone_device_build_coverage | 57/57 | 7.577 | 190.04 | 8520/24155 | 3131/11106 | +| standalone_device_buffer_owner_build | 57/57 | 8.021 | 179.81 | 9044/25436 | 3256/11381 | +| standalone_device_zero_copy_build | 57/57 | 7.503 | 180.82 | 8904/25214 | 3225/11319 | +| standalone_host_build_coverage | 44/44 | 6.951 | 20.78 | 7659/24626 | 2725/11307 | +| standalone_build_coverage | 20/20 | 5.490 | 0.25 | 2965/24467 | 1037/11107 | +| generic_build | 1/1 | 6.589 | 0.00 | 0/24070 | 0/11310 | +| otg_support_build | 430/430 | 11.547 | 13.46 | 14289/24140 | 5789/11292 | +| memory_management_build_coverage | 11/11 | 7.130 | 4.53 | 5527/24073 | 1966/11204 | +| msrc_rtos_build | 33/33 | 7.019 | 0.86 | 7271/24595 | 2602/11492 | +| msrc_standalone_build | 8/8 | 5.786 | 2.08 | 3567/24991 | 1189/11363 | + +Build times are Ninja execution spans, excluding configuration and dependency +setup. Coverage includes all compiled common C sources under the same policy +in every profile. The merge contains exactly the raw source-line union: +**20,991/31,649 lines (66.3244%) and 12,118/22,004 branches (55.0718%)**. +Every expected JSON, XML and HTML input was nonempty. All source paths were +repository-relative. An independent raw-JSON set comparison also matched both +the measured and covered line unions. Standalone transfer/task sources absent +from the default build contribute to the merged denominator. + +Floors are 66% lines and 54.5% branches, with small margins below the measured +results for simulator scheduling variation. They guard against regression; they +do not meet the project target of 100%. The broader denominator must not be +compared to the baseline's filtered default report as if they measured the same +sources. + +Remaining uncovered source lines are included, not excluded: + +| Source area | Covered / measured lines | Uncovered lines | +| --- | ---: | ---: | +| Core | 4707/5425 | 718 | +| Device classes | 6568/8962 | 2394 | +| Host classes | 7995/12407 | 4412 | +| Hardware host controllers | 0/2317 | 2317 | +| Network integration | 164/195 | 31 | +| PictBridge | 1557/2343 | 786 | + +Closing these gaps requires controller simulation or hardware tests, additional +class/protocol scenarios, and error-path tests. Hardware tests were not run. +The descriptor-size regression covers zero, three-byte and eight-byte invalid +fields, empty layouts and alignment boundaries; the two previously unhit lines +are now covered and that source has 100% measured line coverage in the default +profile. + +Runner tests exercise failed fetch/checkout, a real CMake dependency error, a +real failing CTest case, missing/empty coverage inputs, unsafe manual selectors, +and a real gcovr merge with overlapping and disjoint lines. Eight tests pass. +The complete report also passes the configured line and branch floors. +An additional failing CTest case injected into the built generic profile returned +failure while retaining real gcovr JSON/XML/HTML and JUnit diagnostics. A following +manual generic-only run passed 1/1 and produced exactly one profile's reports, +labelled subset, with no stale merged report. + +The baseline standalone printer assertion failed once and passed in the complete +revised run. No printer behavior change or retry is claimed as its fix. + +Actionlint, ShellCheck and whitespace checks pass. The Ubuntu installer completes +with matching GCC/gcov and the pinned gcovr environment. The staged dev ruleset +remains disabled until workflow rollout; passing CI is not merge enforcement. diff --git a/test/cmake/usbx/coverage-floors.json b/test/cmake/usbx/coverage-floors.json new file mode 100644 index 000000000..dd7a99fca --- /dev/null +++ b/test/cmake/usbx/coverage-floors.json @@ -0,0 +1 @@ +{"lines": 66.0, "branches": 54.5} diff --git a/test/cmake/usbx/coverage.py b/test/cmake/usbx/coverage.py new file mode 100644 index 000000000..97ffcadc3 --- /dev/null +++ b/test/cmake/usbx/coverage.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Collect one consistent source universe and verify every merge input.""" +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parent +REPORT = HERE / 'coverage_report' +POLICY = r'common/[^/]+/src/[^/]+\.c$' + + +def profiles(): + """Read the same profile list used by CMake and the bootstrap.""" + text = (HERE / 'CMakeLists.txt').read_text() + block = text.split('set(BUILD_CONFIGURATIONS', 1)[1].split(')', 1)[0] + return re.findall(r'[a-z_]*build[a-z_]*', block) + + +def nonempty(path): + """Reject absent and zero-byte artifacts before publishing.""" + if not path.is_file() or not path.stat().st_size: + raise ValueError(f'Missing or empty coverage input: {path.name}') + + +def source_name(name): + """Require repository-relative USBX source paths without traversal.""" + if PurePosixPath(name).is_absolute() or '..' in PurePosixPath(name).parts or not re.fullmatch(POLICY, name): + raise ValueError(f'Invalid coverage source: {name}') + + +def trace_lines(path): + """Return measured and covered source-line sets from an unfiltered trace.""" + nonempty(path) + data = json.loads(path.read_text()) + measured, covered = set(), set() + for entry in data['files']: + name = entry['file'] + source_name(name) + for line in entry['lines']: + if line.get('gcovr/excluded', False) or line.get('gcovr/noncode', False): + continue + key = (name, line['line_number']) + measured.add(key) + if line['count'] > 0: + covered.add(key) + if not measured: + raise ValueError(f'No measured source lines: {path.name}') + return measured, covered + + +def xml_lines(path): + """Validate Cobertura denominators and repository-relative filenames.""" + nonempty(path) + root = ET.parse(path).getroot() + measured, covered = set(), set() + for entry in root.findall('.//class'): + name = entry.attrib['filename'] + source_name(name) + for line in entry.findall('./lines/line'): + key = (name, int(line.attrib['number'])) + measured.add(key) + if int(line.attrib['hits']) > 0: + covered.add(key) + if not measured or len(measured) != int(root.attrib['lines-valid']): + raise ValueError(f'Invalid source-line denominator: {path.name}') + if len(covered) != int(root.attrib['lines-covered']): + raise ValueError(f'Invalid covered-line count: {path.name}') + if any(source.text not in ('.', '') for source in root.findall('./sources/source')): + raise ValueError('Coverage source root must be repository-relative') + return measured, covered + + +def relative_xml(path): + """Normalize gcovr's source root without changing file or line records.""" + tree = ET.parse(path) + for source in tree.findall('./sources/source'): + source.text = '.' + tree.write(path, encoding='utf-8', xml_declaration=True) + + +def generate(base, inputs): + """Generate all formats from the same gcovr invocation and policy.""" + base.mkdir(parents=True, exist_ok=True) + args = ['gcovr', '--root', str(ROOT), '--filter', POLICY, + '--gcov-executable', os.environ.get('GCOV', 'gcov-14'), + '--merge-mode-functions', 'separate', + # Standalone polling legitimately exceeds gcovr's 32-bit heuristic. + '--gcov-suspicious-hits-threshold', str(2**40), + '--json', str(base.with_suffix('.json')), + '--xml', str(base.with_suffix('.xml')), '--xml-pretty', + '--html-details', str(base / 'index.html'), *inputs] + subprocess.run(args, cwd=ROOT, check=True) + relative_xml(base.with_suffix('.xml')) + verify(base) + + +def verify(base): + """Check all expected formats and their exact line-set agreement.""" + nonempty(base / 'index.html') + actual = trace_lines(base.with_suffix('.json')) + if actual != xml_lines(base.with_suffix('.xml')): + raise ValueError(f'JSON/XML line mismatch: {base.name}') + return actual + + +def merge(): + """Merge only the current selection, retaining every measured input line.""" + # Remove old aggregate reports before checking inputs, including failure paths. + for name in ('merged', 'subset'): + shutil.rmtree(REPORT / name, ignore_errors=True) + for suffix in ('.json', '.xml'): + (REPORT / (name + suffix)).unlink(missing_ok=True) + selection = (REPORT / 'profiles.txt').read_text().splitlines() + available = profiles() + if not selection or len(set(selection)) != len(selection) or not set(selection) <= set(available): + raise ValueError('Invalid coverage profile selection') + expected, hit = set(), set() + inputs = [] + for profile in selection: + base = REPORT / 'per_configuration' / profile + lines, covered = verify(base) + expected |= lines + hit |= covered + inputs += ['--add-tracefile', str(base.with_suffix('.json'))] + complete = set(selection) == set(available) and (REPORT / 'full-suite').is_file() + base = REPORT / ('merged' if complete else 'subset') + generate(base, inputs) + if verify(base) != (expected, hit): + raise ValueError('Merged report differs from the exact source-line union') + root = ET.parse(base.with_suffix('.xml')).getroot() + summary = f"{'Complete' if complete else 'Subset'} coverage: {len(selection)}/{len(available)} profiles; " + for label in ('lines', 'branches'): + covered, valid = int(root.attrib[f'{label}-covered']), int(root.attrib[f'{label}-valid']) + if valid == 0: + raise ValueError(f'Empty {label} denominator') + rate = covered * 100 / valid + summary += f'{label} {covered}/{valid} ({rate:.4f}%); ' + print(summary) + (REPORT / 'summary.txt').write_text(summary + '\n') + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as stream: + stream.write(summary + '\n') + if complete: + floors = json.loads((HERE / 'coverage-floors.json').read_text()) + for label, minimum in floors.items(): + if int(root.attrib[f'{label}-covered']) * 100 < int(root.attrib[f'{label}-valid']) * minimum: + raise ValueError(f'{label} coverage is below {minimum}%') + + +def main(): + """Dispatch collection or union validation with a failing exit on any error.""" + if sys.argv[1:] == ['--merge']: + merge() + elif len(sys.argv) == 2 and sys.argv[1] in profiles(): + profile = sys.argv[1] + if profile not in (REPORT / 'profiles.txt').read_text().splitlines(): + raise ValueError('Profile does not belong to this test run') + objects = HERE / 'build' / profile / 'usbx' / 'CMakeFiles' / 'usbx.dir' / 'common' + if not any(objects.rglob('*.gcno')): + raise ValueError(f'No instrumented objects: {profile}') + generate(REPORT / 'per_configuration' / profile, [str(objects)]) + else: + raise ValueError('Expected a known profile or --merge') + + +if __name__ == '__main__': + try: + main() + except (ValueError, OSError, KeyError, ET.ParseError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) diff --git a/test/cmake/usbx/coverage.sh b/test/cmake/usbx/coverage.sh index f5f30c714..9efe8a176 100755 --- a/test/cmake/usbx/coverage.sh +++ b/test/cmake/usbx/coverage.sh @@ -1,124 +1,4 @@ #!/bin/bash - -set -e - -cd $(dirname $0) -root_path=$(cd ../../../common; pwd) - -# Exclude strings -exclude_file_list=" $root_path/*/src/*_class_asix*" -exclude_file_list+=" $root_path/*/src/*_class_audio*" -exclude_file_list+=" $root_path/*/src/*_class_ccid*" -exclude_file_list+=" $root_path/*/src/*_class_dfu*" -exclude_file_list+=" $root_path/*/src/*_class_gser*" -exclude_file_list+=" $root_path/*/src/*_class_pima*" -exclude_file_list+=" $root_path/*/src/*_class_rndis*" -exclude_file_list+=" $root_path/*/src/*_class_printer*" -exclude_file_list+=" $root_path/*/src/*_class_prolific*" -exclude_file_list+=" $root_path/*/src/*_class_swar*" -exclude_file_list+=" $root_path/*/src/*_class_video*" -exclude_file_list+=" $root_path/*/src/*_hnp_*" -exclude_file_list+=" $root_path/*/src/*_role_*" -exclude_file_list+=" $root_path/*/src/ux_utility_set_interrupt_handler.c" -exclude_file_list+=" $root_path/*/src/*dcd_sim_slave*" -exclude_file_list+=" $root_path/*/src/*device_class_dpump*" -exclude_file_list+=" $root_path/*/src/*hcd_sim_host*" -exclude_file_list+=" $root_path/*/src/*host_class_dpump*" -exclude_file_list+=" $root_path/*/src/*ux_network_driver*" - -# Device HID interrupt OUT related -exclude_file_list+=" $root_path/*/src/ux_device_class_hid_read.c" -exclude_file_list+=" $root_path/*/src/ux_device_class_hid_receiver*" - -# CB/CBI related -exclude_file_list+=" $root_path/*/src/*_storage*_cb.c" -exclude_file_list+=" $root_path/*/src/*_storage*_cbi.c" - -# CD/DVD related things -exclude_file_list+=" $root_path/*/src/*_storage_get_status*" -exclude_file_list+=" $root_path/*/src/*_storage_get_configuration*" -exclude_file_list+=" $root_path/*/src/*_storage_get_performance*" -exclude_file_list+=" $root_path/*/src/*_storage_read_disk_information*" -exclude_file_list+=" $root_path/*/src/*_storage_report_key*" -exclude_file_list+=" $root_path/*/src/*_storage_read_dvd*" -exclude_file_list+=" $root_path/*/src/*_storage_read_toc*" - -# Obsolete -exclude_file_list+=" $root_path/*/src/ux_device_stack_interface_get.c" -exclude_file_list+=" $root_path/*/src/ux_host_stack_delay_ms.c" - -# Host controllers -exclude_file_list+=" $root_path/usbx_host_controllers/src/*" - -# Pictbridge related files -exclude_file_list+=" $root_path/usbx_pictbridge/src/*" - -# Host related files -exclude_host_list=" $root_path/*/src/*_host_*" - -# Device related files -exclude_device_list=" $root_path/*/src/*_device_*" - -# RTOS related files -exclude_rtos_list=" $root_path/*/src/*_thread*" -exclude_rtos_list+=" $root_path/*/src/*_event_*" -exclude_rtos_list+=" $root_path/*/src/*_mutex_*" -exclude_rtos_list+=" $root_path/*/src/*_semaphore_*" -exclude_rtos_list+=" $root_path/*/src/*_timer_*" - -exclude_rtos_list+=" $root_path/*/src/*_cdc_ecm_*" - -exclude_rtos_list+=" $root_path/*/src/*_hub_*" - -exclude_rtos_list+=" $root_path/*/src/*_cdc_acm_capabilities_get.c" -exclude_rtos_list+=" $root_path/*/src/*_cdc_acm_configure.c" - -exclude_rtos_list+=" $root_path/*/src/*_hid_configure.c" -exclude_rtos_list+=" $root_path/*/src/*_hid_descriptor_parse.c" -exclude_rtos_list+=" $root_path/*/src/*_hid_report_descriptor_get.c" - -exclude_rtos_list+=" $root_path/*/src/*_storage_configure.c" -exclude_rtos_list+=" $root_path/*/src/*_storage_media_mount.c" -exclude_rtos_list+=" $root_path/*/src/*_storage_media_open.c" -exclude_rtos_list+=" $root_path/*/src/*_storage_partition_read.c" -exclude_rtos_list+=" $root_path/*/src/*_storage_transport.c" - -# Standalone related files -exclude_standalone_list=" $root_path/*/src/*_run.c" - -if [[ $1 = *"_full_coverage" ]]; then - - exclude_options="" -else - - exclude_options="" - for f in $exclude_file_list;do - exclude_options+=" -e $f" - done - - if [[ $1 = *"_device_"* ]]; then - for f in $exclude_host_list;do - exclude_options+=" -e $f" - done - fi - - if [[ $1 = *"_host_"* ]]; then - for f in $exclude_device_list;do - exclude_options+=" -e $f" - done - fi - - if [[ $1 = "standalone_"* ]]; then - for f in $exclude_rtos_list;do - exclude_options+=" -e $f" - done - else - for f in $exclude_standalone_list;do - exclude_options+=" -e $f" - done - fi -fi - -mkdir -p coverage_report/$1 -gcovr --object-directory=build/$1/usbx/CMakeFiles/usbx.dir/common -r ../../../common $exclude_options --xml-pretty --output coverage_report/$1.xml -gcovr --object-directory=build/$1/usbx/CMakeFiles/usbx.dir/common -r ../../../common $exclude_options --html --html-details --html-high-threshold 100.0 --output coverage_report/$1/index.html +set -euo pipefail +cd "$(dirname "$0")" +exec python3 coverage.py "$@" diff --git a/test/cmake/usbx/dependencies.txt b/test/cmake/usbx/dependencies.txt new file mode 100644 index 000000000..5fd5cf43e --- /dev/null +++ b/test/cmake/usbx/dependencies.txt @@ -0,0 +1,3 @@ +threadx b37cd4a81a1cb8c2ebefc438220ab7f009e13362 +filex 79c703d917e648c615ae6e0c253e63a73bb134a5 +netxduo 400e647de10a71763e9facea2b98667bfa93b285 diff --git a/test/cmake/usbx/report.py b/test/cmake/usbx/report.py new file mode 100644 index 000000000..11a31ca46 --- /dev/null +++ b/test/cmake/usbx/report.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Summarize the actual JUnit results, including failed or missing runs.""" +import os +import re +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +rows = ['| Profile | Passed / tests | Seconds |', '| --- | ---: | ---: |'] +failed = False +for profile in sys.argv[1:]: + path = Path('build') / profile / f'{profile}.xml' + try: + root = ET.parse(path).getroot() + cases = root.findall('.//testcase') + passed = sum(case.get('status') == 'run' and + case.find('failure') is None and case.find('skipped') is None + for case in cases) + log = path.with_suffix('.txt').read_text() + duration = re.search(r'Total Test time \(real\) =\s*([0-9.]+) sec', log) + if duration is None: + raise ValueError(f'Missing duration: {profile}') + seconds = float(duration.group(1)) + rows.append(f'| {profile} | {passed} / {len(cases)} | {seconds:.2f} |') + failed |= not cases or passed != len(cases) + except (OSError, ET.ParseError, ValueError): + rows.append(f'| {profile} | missing results | — |') + failed = True +text = '\n'.join(rows) + '\n' +print(text) +Path('build/results.md').write_text(text, encoding='utf-8') +# The reusable workflow uploads build/*.txt on failure as well as success. +Path('build/results.txt').write_text(text, encoding='utf-8') +if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') as stream: + stream.write(text) +sys.exit(int(failed)) diff --git a/test/cmake/usbx/run.sh b/test/cmake/usbx/run.sh index 040343f3b..e5a9197a5 100755 --- a/test/cmake/usbx/run.sh +++ b/test/cmake/usbx/run.sh @@ -1,17 +1,106 @@ #!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" -cd $(dirname $0) +# Resolve only the reviewed revisions; never alter an existing checkout. +mkdir -p ../../externals +while read -r dependency revision; do + [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 + destination="../../externals/$dependency" + if [ ! -e "$destination" ]; then + temporary=$(mktemp -d "../../externals/.${dependency}.XXXXXX") + trap 'rm -rf -- "$temporary"' EXIT + git init -q "$temporary" + git -C "$temporary" remote add origin "https://github.com/eclipse-threadx/$dependency.git" + timeout 180 git -C "$temporary" fetch --depth 1 origin "$revision" + git -C "$temporary" checkout -q --detach "$revision" + mv -- "$temporary" "$destination" + trap - EXIT + fi + if [ "$(git -C "$destination" rev-parse HEAD)" != "$revision" ] || + [ -n "$(git -C "$destination" status --porcelain --untracked-files=no)" ]; then + echo "Dependency $dependency must be clean at $revision." >&2 + exit 1 + fi + printf 'Dependency: %s %s\n' "$dependency" "$revision" +done < dependencies.txt -# Checkout externals -[ -d externals ] || mkdir ../../externals -git clone https://github.com/eclipse-threadx/threadx.git ../../externals/threadx -git clone https://github.com/eclipse-threadx/netxduo.git ../../externals/netxduo -git clone https://github.com/eclipse-threadx/filex.git ../../externals/filex +bootstrap=../../externals/threadx/scripts/cmake_bootstrap.sh +[ -s "$bootstrap" ] +ln -sfn "$bootstrap" .run.sh +if [ "${1:-}" = build_libs ]; then + exec ./.run.sh build_libs +fi -# Add junit output for ctest generation -if ! grep -q "\-\-output\-junit \$1.xml" ../../externals/threadx/scripts/cmake_bootstrap.sh; then - sed -i 's/ctest $parallel --timeout 1000 -O $1.txt/& --output-junit $1.xml/g' ../../externals/threadx/scripts/cmake_bootstrap.sh +command=${1:-} +shift || true +manual_full=true +if [ "${1:-}" = --manual ]; then + manual_full=$(python3 -c 'import json, os; print(str(json.load(open(os.environ["GITHUB_EVENT_PATH"]))["inputs"].get("tests_to_run", "all") == "all").lower())') + mapfile -t manual < <(python3 -c 'import json, os; print("\n".join(json.load(open(os.environ["GITHUB_EVENT_PATH"]))["inputs"].get("tests_to_run", "all").split()))') + [ "${#manual[@]}" -gt 0 ] || exit 1 + set -- "${manual[@]}" +fi +mapfile -t available < <(sed -n '/^set(BUILD_CONFIGURATIONS/,/^ )/p' CMakeLists.txt | grep -oE '[a-z_]*build[a-z_]*') +if [ "$#" -eq 0 ]; then + selected=("${available[0]}") +elif [ "$*" = all ]; then + selected=("${available[@]}") +else + selected=("$@") +fi +for profile in "${selected[@]}"; do + if ! printf '%s\n' "${available[@]}" | grep -qxF "$profile"; then + echo "Unknown profile: $profile" >&2 + exit 1 + fi +done +if [ "$(printf '%s\n' "${selected[@]}" | sort -u | wc -l)" -ne "${#selected[@]}" ]; then + echo 'Duplicate profiles are not allowed.' >&2 + exit 1 fi +full_suite=false +if [ "$*" = all ] && "$manual_full"; then full_suite=true; fi +printf 'Selected profiles (%s): %s\n' "${#selected[@]}" "${selected[*]}" +export CTEST_REPEAT_FAIL=1 +export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-4} -[ -f .run.sh ] || ln -sf ../../externals/threadx/scripts/cmake_bootstrap.sh .run.sh -./.run.sh $* \ No newline at end of file +case "$command" in + build) + # Fresh build trees prevent compiler, instrumentation and gcda reuse. + rm -rf coverage_report + rm -f build/built-profiles.txt + for profile in "${selected[@]}"; do + rm -rf -- "build/$profile" + done + ./.run.sh build "${selected[@]}" + mkdir -p build + printf '%s\n' "${selected[@]}" > build/built-profiles.txt + printf 'Built profiles (%s): %s\n' "${#selected[@]}" "${selected[*]}" + ;; + test) + # Clear reports even for subsets, before validating their build inputs. + rm -rf coverage_report + mkdir -p coverage_report + printf '%s\n' "${selected[@]}" > coverage_report/profiles.txt + if "$full_suite"; then touch coverage_report/full-suite; fi + for profile in "${selected[@]}"; do + grep -qxF "$profile" build/built-profiles.txt + [ -s "build/$profile/CTestTestfile.cmake" ] + if [ "${TX_COVERAGE:-OFF}" = ON ]; then + grep -qxF 'USBX_CI_COVERAGE:BOOL=ON' "build/$profile/CMakeCache.txt" + fi + find "build/$profile" -name '*.gcda' -delete + rm -f -- "build/$profile/$profile.xml" + done + status=0 + ./.run.sh test "${selected[@]}" || status=$? + python3 report.py "${selected[@]}" || status=1 + printf 'Tested profiles (%s): %s\n' "${#selected[@]}" "${selected[*]}" + exit "$status" + ;; + *) + echo 'Usage: run.sh {build|test|build_libs} [all|profile ...]' >&2 + exit 1 + ;; +esac diff --git a/test/cmake/usbx/test_runner.py b/test/cmake/usbx/test_runner.py new file mode 100644 index 000000000..d3b7deb1a --- /dev/null +++ b/test/cmake/usbx/test_runner.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Exercise failure propagation in isolated runner trees using real CTest.""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location('coverage_checks', HERE / 'coverage.py') +coverage = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(coverage) + + +class RunnerFailures(unittest.TestCase): + """Fault injections leave the production build and dependency trees intact.""" + + def setUp(self): + """Create a disposable runner with the same pinned dependencies.""" + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.runner = self.root / 'test/cmake/usbx' + self.runner.mkdir(parents=True) + externals = self.root / 'test/externals' + externals.mkdir() + for name in ('threadx', 'filex', 'netxduo'): + (externals / name).symlink_to(HERE.parents[1] / 'externals' / name) + for name in ('run.sh', 'dependencies.txt', 'report.py'): + shutil.copy2(HERE / name, self.runner / name) + (self.runner / 'CMakeLists.txt').write_text('set(BUILD_CONFIGURATIONS\n generic_build\n )\n') + self.env = dict(os.environ, CC='gcc-14', GCOV='gcov-14', + CTEST_PARALLEL_LEVEL='1', TX_COVERAGE='OFF') + + def run_command(self, *arguments): + """Capture diagnostics and return the runner's actual exit status.""" + return subprocess.run(['bash', './run.sh', *arguments], cwd=self.runner, + env=self.env, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=60) + + def test_manual_input_is_data(self): + """Shell metacharacters in a manual selector cannot execute commands.""" + event = self.root / 'event.json' + event.write_text(json.dumps({'inputs': {'tests_to_run': 'generic_build; touch injected'}})) + self.env['GITHUB_EVENT_PATH'] = str(event) + result = self.run_command('test', '--manual') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('Unknown profile', result.stdout) + self.assertFalse((self.runner / 'injected').exists()) + + def test_checkout_failure(self): + """A failed fetch cannot continue into configuration or testing.""" + (self.root / 'test/externals/threadx').unlink() + bin_dir = self.root / 'bin' + bin_dir.mkdir() + git = shutil.which('git') + wrapper = bin_dir / 'git' + wrapper.write_text(f'#!/bin/bash\nfor arg in "$@"; do\n' + ' if [ "$arg" = fetch ]; then exit 71; fi\ndone\n' + f'exec {git} "$@"\n') + wrapper.chmod(0o755) + self.env['PATH'] = str(bin_dir) + os.pathsep + self.env['PATH'] + result = self.run_command('build', 'all') + self.assertEqual(result.returncode, 71, result.stdout) + self.assertFalse((self.runner / 'build').exists()) + + self.env['USBX_TEST_DEPENDENCY'] = str(HERE.parents[1] / 'externals/threadx') + wrapper.write_text( + '#!/bin/bash\n' + 'if [ "${3:-}" = checkout ]; then exit 72; fi\n' + f'if [ "${{3:-}}" = fetch ]; then exec {git} -C "$2" fetch ' + '--depth 1 "$USBX_TEST_DEPENDENCY" "$7"; fi\n' + f'exec {git} "$@"\n') + result = self.run_command('build', 'all') + self.assertEqual(result.returncode, 72, result.stdout) + self.assertFalse((self.runner / 'build').exists()) + + def test_dependency_build_failure(self): + """An actual CMake dependency error propagates through the bootstrap.""" + libs = self.runner / 'libs' + libs.mkdir() + (libs / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.13)\nproject(failure NONE)\n' + 'message(FATAL_ERROR "injected dependency failure")\n') + result = self.run_command('build_libs') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('injected dependency failure', result.stdout) + + (libs / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.13)\nproject(failure NONE)\n' + 'add_custom_target(failing_dependency ALL COMMAND ${CMAKE_COMMAND} -E false)\n') + result = self.run_command('build_libs') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('failing_dependency', result.stdout) + self.assertIn('FAILED', result.stdout) + + def test_ctest_failure_keeps_diagnostics_and_collects_coverage(self): + """A failing CTest case stays red while collection still executes.""" + source = self.root / 'fixture' + source.mkdir() + (source / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.13)\nproject(failure NONE)\n' + 'enable_testing()\nadd_test(NAME injected_failure COMMAND ' + '${CMAKE_COMMAND} -E false)\n') + build = self.runner / 'build/generic_build' + subprocess.run(['cmake', '-S', str(source), '-B', str(build), '-G', 'Ninja', + '-DUSBX_CI_COVERAGE:BOOL=ON'], check=True, capture_output=True) + subprocess.run(['cmake', '--build', str(build)], check=True, capture_output=True) + (build.parent / 'built-profiles.txt').write_text('generic_build\n') + report = self.runner / 'coverage_report' + report.mkdir() + (report / 'merged.xml').write_text('stale') + # Record both bootstrap coverage callbacks independently of gcovr. + collector = self.runner / 'coverage.sh' + collector.write_text('#!/bin/bash\nprintf "%s\\n" "$1" >> coverage_report/collected.txt\n') + collector.chmod(0o755) + self.env['TX_COVERAGE'] = 'ON' + result = self.run_command('test', 'all') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('injected_failure', (build / 'generic_build.xml').read_text()) + self.assertTrue(list((build / 'Testing/Temporary').glob('LastTest*.log'))) + self.assertEqual((report / 'collected.txt').read_text(), 'generic_build\n--merge\n') + self.assertFalse((report / 'merged.xml').exists()) + + def test_missing_coverage_fails_runner(self): + """Passing CTest cannot hide missing coverage objects or tracefiles.""" + source = self.root / 'fixture' + source.mkdir() + (source / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.13)\nproject(passing NONE)\n' + 'enable_testing()\nadd_test(NAME passing COMMAND ' + '${CMAKE_COMMAND} -E true)\n') + build = self.runner / 'build/generic_build' + subprocess.run(['cmake', '-S', str(source), '-B', str(build), '-G', 'Ninja', + '-DUSBX_CI_COVERAGE:BOOL=ON'], check=True, capture_output=True) + subprocess.run(['cmake', '--build', str(build)], check=True, capture_output=True) + (build.parent / 'built-profiles.txt').write_text('generic_build\n') + for name in ('coverage.sh', 'coverage.py', 'coverage-floors.json'): + shutil.copy2(HERE / name, self.runner / name) + self.env['TX_COVERAGE'] = 'ON' + result = self.run_command('test', 'all') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('100% tests passed', result.stdout) + self.assertIn('No instrumented objects', result.stdout) + self.assertIn('Missing or empty coverage input', result.stdout) + + collector = self.runner / 'coverage.sh' + collector.write_text( + '#!/bin/bash\nset -e\n' + 'if [ "$1" = --merge ]; then exec python3 coverage.py --merge; fi\n' + 'mkdir -p coverage_report/per_configuration/generic_build\n' + 'touch coverage_report/per_configuration/generic_build/index.html\n' + 'touch coverage_report/per_configuration/generic_build.json\n' + 'touch coverage_report/per_configuration/generic_build.xml\n') + result = self.run_command('test', 'all') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn('100% tests passed', result.stdout) + self.assertIn('Missing or empty coverage input', result.stdout) + + +class CoverageInputs(unittest.TestCase): + """Reject missing, empty, invalid and non-relative raw inputs.""" + + def setUp(self): + """Allocate a temporary tracefile independent of matrix outputs.""" + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.path = Path(self.temporary.name) / 'trace.json' + + def test_missing_and_empty(self): + """Both missing and zero-byte files fail before merging.""" + with self.assertRaises(ValueError): + coverage.trace_lines(self.path) + self.path.touch() + with self.assertRaises(ValueError): + coverage.trace_lines(self.path) + self.path.write_text('{"files": []}') + with self.assertRaises(ValueError): + coverage.trace_lines(self.path) + + def test_source_policy_and_lines(self): + """Keep host and device sources, including their uncovered lines.""" + files = [{'file': f'common/{area}/src/probe.c', 'lines': [ + {'line_number': 10, 'count': 1}, {'line_number': 20, 'count': 0}] + } for area in ('core', 'usbx_host_classes', 'usbx_device_classes')] + self.path.write_text(json.dumps({'files': files})) + measured, covered = coverage.trace_lines(self.path) + self.assertEqual(len(measured), 6) + self.assertEqual(len(covered), 3) + for name in ('/absolute/probe.c', '../probe.c', 'test/probe.c'): + files[0]['file'] = name + self.path.write_text(json.dumps({'files': files})) + with self.assertRaises(ValueError): + coverage.trace_lines(self.path) + + +class CoverageMerge(unittest.TestCase): + """Exercise gcovr's real merge with overlapping and disjoint source lines.""" + + def test_union_and_missing_input_invalidation(self): + """The union keeps unhit lines and invalidates stale aggregate reports.""" + with tempfile.TemporaryDirectory() as directory: + previous = coverage.REPORT + coverage.REPORT = Path(directory) + self.addCleanup(setattr, coverage, 'REPORT', previous) + selection = coverage.profiles()[:2] + (coverage.REPORT / 'profiles.txt').write_text('\n'.join(selection) + '\n') + source = 'common/core/src/ux_dcd_sim_slave_endpoint_create.c' + expected = set() + for profile, numbers in zip(selection, ((87, 90), (87, 93))): + lines = [] + for number in numbers: + expected.add((source, number)) + lines.append({'line_number': number, 'count': int(number == 87), + 'branches': [{'count': 1, 'fallthrough': True, + 'throw': False, 'source_block_id': 2, + 'destination_block_id': 3}]}) + raw = coverage.REPORT / f'{profile}-input.json' + raw.write_text(json.dumps({'gcovr/format_version': '0.14', 'files': [ + {'file': source, 'lines': lines, 'functions': []}]})) + coverage.generate(coverage.REPORT / 'per_configuration' / profile, + ['--add-tracefile', str(raw)]) + coverage.merge() + measured, covered = coverage.verify(coverage.REPORT / 'subset') + self.assertEqual(measured, expected) + self.assertEqual(covered, {(source, 87)}) + self.assertFalse((coverage.REPORT / 'merged.xml').exists()) + (coverage.REPORT / 'per_configuration' / (selection[1] + '.json')).write_text('') + with self.assertRaises(ValueError): + coverage.merge() + self.assertFalse((coverage.REPORT / 'subset.xml').exists()) + + +if __name__ == '__main__': + os.environ.pop('GITHUB_STEP_SUMMARY', None) + unittest.main() diff --git a/test/regression/usbx_ux_device_class_hid_descriptor_send_test.c b/test/regression/usbx_ux_device_class_hid_descriptor_send_test.c index b86bb29c9..cbf40b047 100644 --- a/test/regression/usbx_ux_device_class_hid_descriptor_send_test.c +++ b/test/regression/usbx_ux_device_class_hid_descriptor_send_test.c @@ -8,8 +8,10 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ #include "usbx_test_common_hid.h" +#include "ux_host_stack.h" #include "ux_host_class_hid_keyboard.h" diff --git a/test/regression/usbx_ux_host_stack_device_configuration_get_test.c b/test/regression/usbx_ux_host_stack_device_configuration_get_test.c index 1655076c7..97e71cf7e 100644 --- a/test/regression/usbx_ux_host_stack_device_configuration_get_test.c +++ b/test/regression/usbx_ux_host_stack_device_configuration_get_test.c @@ -8,6 +8,7 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This test is designed to test the ux_host_stack_device_configuration_get. */ @@ -15,6 +16,7 @@ #include "tx_api.h" #include "ux_api.h" #include "ux_system.h" +#include "ux_host_stack.h" #include "ux_utility.h" #include "ux_host_class_dpump.h" #include "ux_device_class_dpump.h" diff --git a/test/regression/usbx_ux_utility_descriptor_struct_test.c b/test/regression/usbx_ux_utility_descriptor_struct_test.c index 67a83c9a9..cfe8b2186 100644 --- a/test/regression/usbx_ux_utility_descriptor_struct_test.c +++ b/test/regression/usbx_ux_utility_descriptor_struct_test.c @@ -8,6 +8,7 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This test is designed to test the ux_utility_descriptor_parse. */ @@ -144,6 +145,19 @@ CHAR *memory_pointer; static void ux_test_thread_simulation_0_entry(ULONG arg) { +UCHAR invalid_zero[] = {0}; +UCHAR invalid_three[] = {1, 3}; +UCHAR invalid_eight[] = {8}; +UCHAR aligned_fields[] = {1, 2, 4}; + + /* Reject unsupported field sizes and align valid descriptor layouts. */ + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(invalid_zero, 1, 3u) == 0); + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(invalid_three, 2, 3u) == 0); + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(invalid_eight, 1, 3u) == 0); + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(aligned_fields, 0, 3u) == 0); + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(aligned_fields, 3, 3u) == 8); + UX_TEST_ASSERT(_ux_utility_descriptor_parse_size(aligned_fields, 3, 15u) == 16); + /* Print STD framework descriptor's sizes */ printf("\n"); printf("Descriptor %7s %7s %7s\n", "nFields", "Size4", "SizeNew"); diff --git a/test/regression/usbx_uxe_device_video_test.c b/test/regression/usbx_uxe_device_video_test.c index 9a4a3f00c..f87d1eca3 100644 --- a/test/regression/usbx_uxe_device_video_test.c +++ b/test/regression/usbx_uxe_device_video_test.c @@ -8,6 +8,7 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This test is designed to test the ux_utility_descriptor_pack. */ @@ -559,9 +560,10 @@ static void ux_test_thread_simulation_0_entry(ULONG arg) UINT status; UX_DEVICE_CLASS_VIDEO_STREAM dummy_video_stream; ULONG dummy_length; -UCHAR payload_data[64]; -ULONG payload_length; -UCHAR **dummy_payload = &payload_data; +UCHAR payload_data[64]; +ULONG payload_length; +UCHAR *payload = payload_data; +UCHAR **dummy_payload = &payload; dummy_length = ux_device_class_video_max_payload_length(UX_NULL); if (dummy_length != 0) diff --git a/test/regression/usbx_uxe_host_pima_test.c b/test/regression/usbx_uxe_host_pima_test.c index e22a69fb5..6fc500ce0 100644 --- a/test/regression/usbx_uxe_host_pima_test.c +++ b/test/regression/usbx_uxe_host_pima_test.c @@ -8,6 +8,7 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This test is designed to test the ux_utility_descriptor_pack. */ @@ -105,6 +106,7 @@ UINT status; UX_HOST_CLASS_PIMA dummy_pima_inst; UX_HOST_CLASS_PIMA *dummy_pima = &dummy_pima_inst; UX_HOST_CLASS_PIMA_SESSION pima_session; +UX_HOST_CLASS_PIMA_DEVICE pima_device; UCHAR object_buffer[64]; UX_HOST_CLASS_PIMA_OBJECT object; ULONG object_actual_length; @@ -113,7 +115,7 @@ UX_HOST_CLASS_PIMA_STORAGE storage; ULONG object_handles_array[32]; /* Unit test for function ux_host_class_pima_device_info_get() */ - status = ux_host_class_pima_device_info_get(NX_NULL, &pima_session); + status = ux_host_class_pima_device_info_get(NX_NULL, &pima_device); UX_TEST_CHECK_CODE(UX_INVALID_PARAMETER ,status); status = ux_host_class_pima_device_info_get(dummy_pima, NX_NULL); diff --git a/test/regression/usbx_uxe_host_video_test.c b/test/regression/usbx_uxe_host_video_test.c index 23d4ef114..309ad8cd2 100644 --- a/test/regression/usbx_uxe_host_video_test.c +++ b/test/regression/usbx_uxe_host_video_test.c @@ -8,6 +8,7 @@ /* */ /* SPDX-License-Identifier: MIT */ /***************************************************************************/ +/* Portions of this file were generated with AI assistance. */ /* This test is designed to test the ux_utility_descriptor_pack. */ @@ -105,6 +106,7 @@ UINT status; UX_HOST_CLASS_VIDEO dummy_video_inst; UX_HOST_CLASS_VIDEO *dummy_video = &dummy_video_inst; UCHAR dummy_buffer[64]; +UCHAR *buffer = dummy_buffer; UX_HOST_CLASS_VIDEO_CONTROL dummy_video_control; UX_HOST_CLASS_VIDEO_TRANSFER_REQUEST dummy_video_request; ULONG max_payload = 0xff; @@ -160,11 +162,11 @@ ULONG max_payload = 0xff; UX_TEST_CHECK_CODE(UX_INVALID_PARAMETER, status); /* ux_host_class_video_transfer_buffers_add() */ - status = ux_host_class_video_transfer_buffers_add(UX_NULL, &dummy_buffer, 1); + status = ux_host_class_video_transfer_buffers_add(UX_NULL, &buffer, 1); UX_TEST_CHECK_CODE(UX_INVALID_PARAMETER, status); status = ux_host_class_video_transfer_buffers_add(dummy_video, UX_NULL, 1); UX_TEST_CHECK_CODE(UX_INVALID_PARAMETER, status); - status = ux_host_class_video_transfer_buffers_add(dummy_video, &dummy_buffer, 0); + status = ux_host_class_video_transfer_buffers_add(dummy_video, &buffer, 0); UX_TEST_CHECK_CODE(UX_INVALID_PARAMETER, status); /* ux_host_class_video_transfer_callback_set() */