diff --git a/.github/workflows/check-core-staging.yml b/.github/workflows/check-core-staging.yml index 7a55867..0a4434e 100644 --- a/.github/workflows/check-core-staging.yml +++ b/.github/workflows/check-core-staging.yml @@ -18,18 +18,20 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: openDAQ/openDAQ-CI/.github/actions/resolve-channel@v3 + - uses: openDAQ/openDAQ-CI/.github/actions/resolve-channel@jira/TBBAS-3547-staging-pull-by-annotations id: channel - name: Read the pinned core ref shell: bash - run: echo "CORE_REF=$(cat opendaq_ref)" >> "$GITHUB_ENV" + run: echo "OPENDAQ_REF=$(cat opendaq_ref)" >> "$GITHUB_ENV" - - uses: openDAQ/openDAQ-CI/.github/actions/staging-check@v3 + - uses: openDAQ/openDAQ-CI/.github/actions/staging-check@jira/TBBAS-3547-staging-pull-by-annotations env: CHANNEL: ${{ steps.channel.outputs.channel }} with: packages: >- [ - { "path": "ghcr.io/openDAQ/$CHANNEL/staging/opendaq", "tag": "$CORE_REF" } + { "packages": [ + { "path": "ghcr.io/openDAQ/$CHANNEL/staging/opendaq", "annotations": { "org.opencontainers.image.revision": "$OPENDAQ_REF" } } + ] } ] diff --git a/.github/workflows/ci-downstream.yml b/.github/workflows/ci-downstream.yml index 6b2a436..d573698 100644 --- a/.github/workflows/ci-downstream.yml +++ b/.github/workflows/ci-downstream.yml @@ -12,7 +12,7 @@ on: jobs: call-opendaq-reusable: name: LT Streaming Modern - uses: openDAQ/openDAQ-CI/.github/workflows/reusable.yml@v3 + uses: openDAQ/openDAQ-CI/.github/workflows/reusable.yml@jira/TBBAS-3547-staging-pull-by-annotations with: opendaq-ref: ${{ github.event.inputs.opendaq-ref || '' }} include-jobs: > diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index f124f32..98f0a4f 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -15,7 +15,7 @@ jobs: name: Stagings permissions: packages: write - uses: openDAQ/openDAQ-CI/.github/workflows/staging-cleanup.yml@v3 + uses: openDAQ/openDAQ-CI/.github/workflows/staging-cleanup.yml@jira/TBBAS-3547-staging-pull-by-annotations with: dry-run: ${{ inputs.dry-run || false }} staging-retentions: >- diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 86775aa..be7dee6 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -4,6 +4,10 @@ on: schedule: - cron: '0 2 * * 1' workflow_dispatch: + inputs: + opendaq-ref: + description: "openDAQ commit, branch or tag. If empty, uses the opendaq_ref file from the repo." + default: "" jobs: @@ -11,18 +15,71 @@ jobs: name: Resolve the core commit runs-on: ubuntu-latest outputs: - core-ref: ${{ steps.core.outputs.ref }} + opendaq-ref: ${{ steps.core.outputs.ref }} channel: ${{ steps.channel.outputs.channel }} + jobs: ${{ steps.stagings.outputs.jobs }} + staging-pull: ${{ steps.stagings.outputs.staging-pull }} + opendaq-artifacts: ${{ steps.stagings.outputs.opendaq-artifacts }} steps: - uses: actions/checkout@v4 - - uses: openDAQ/openDAQ-CI/.github/actions/resolve-channel@v3 + - uses: openDAQ/openDAQ-CI/.github/actions/resolve-channel@jira/TBBAS-3547-staging-pull-by-annotations id: channel - - name: Read the pinned core ref + # the input, or else opendaq_ref; a branch or a tag is resolved to the commit, which is + # what the core staging is found by and what this staging records + - name: Resolve the core commit id: core shell: bash - run: echo "ref=$(cat opendaq_ref)" >> "$GITHUB_OUTPUT" + env: + REF: ${{ inputs.opendaq-ref }} + run: | + ref="${REF:-$(cat opendaq_ref)}" + sha="$ref" + if ! [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then + sha=$(git ls-remote https://github.com/openDAQ/openDAQ "$ref" "$ref^{}" | sort -k2 | tail -1 | cut -f1) + fi + [ -n "$sha" ] || { echo "::error::$ref names no openDAQ commit"; exit 1; } + echo "core: $ref -> $sha" + echo "ref=$sha" >> "$GITHUB_OUTPUT" + + # the jobs to build, and the openDAQ staging each of them builds against + - name: Request the openDAQ stagings + id: stagings + shell: python3 {0} + env: + CHANNEL: ${{ steps.channel.outputs.channel }} + OPENDAQ_REF: ${{ steps.core.outputs.ref }} + run: | + import json, os + + # job -> the openDAQ staging it builds against, by triplet + opendaq = { + "windows-2022-x86_64-msvs-v143-release": "x86_64-windows-msvc-143-release", + "windows-2022-x86-msvs-v143-release": "x86-windows-msvc-143-release", + "manylinux_2_28-x86_64-ninja-gcc-14-release": "x86_64-manylinux_2_28-gcc-14-release", + "manylinux_2_28-armv8-ninja-gcc-14-release": "armv8-manylinux_2_28-gcc-14-release", + "ubuntu-22.04-x86_64-ninja-gcc-release": "x86_64-linux-gcc-11-release", + "ubuntu-22.04-armv8-ninja-gcc-release": "armv8-linux-gcc-11-release", + "macos-26-x86_64-ninja-appleclang-21-release": "x86_64-macos-apple-clang-21-release", + "macos-26-armv8-ninja-appleclang-21-release": "armv8-macos-apple-clang-21-release", + } + + # each staging, of the resolved commit, as an artifact of its own ... + pull = [{"artifact": f"opendaq-{triplet}", "packages": [{ + "path": f"ghcr.io/openDAQ/{os.environ['CHANNEL']}/staging/opendaq", + "annotations": {"org.opencontainers.image.revision": os.environ["OPENDAQ_REF"], + "com.opendaq.package.triplet": triplet}}]} + for triplet in opendaq.values()] + + # ... which its job unpacks into staging-deps + artifacts = [{"match-jobs": [job], "artifacts": [{"name": f"opendaq-{triplet}", "dir": "staging-deps"}]} + for job, triplet in opendaq.items()] + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"jobs={json.dumps(list(opendaq))}\n") + f.write(f"staging-pull={json.dumps(pull)}\n") + f.write(f"opendaq-artifacts={json.dumps(artifacts)[1:-1]}\n") # no brackets: it goes inside packages stage: needs: resolve @@ -30,42 +87,14 @@ jobs: permissions: contents: read packages: write - uses: openDAQ/openDAQ-CI/.github/workflows/stage.yml@v3 + uses: openDAQ/openDAQ-CI/.github/workflows/stage.yml@jira/TBBAS-3547-staging-pull-by-annotations with: timeout: '[{"timeout-minutes": 180}]' - staging-pull: >- - [ - { "artifact": "core-windows-2022-x86_64-msvs-v143-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "core-windows-2022-x86-msvs-v143-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86" } }] }, - { "artifact": "core-manylinux_2_28-x86_64-ninja-gcc-14-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "manylinux_2_28", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "core-manylinux_2_28-armv8-ninja-gcc-14-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "manylinux_2_28", "com.opendaq.settings.arch": "armv8" } }] }, - { "artifact": "core-ubuntu-22.04-x86_64-ninja-gcc-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "core-ubuntu-22.04-armv8-ninja-gcc-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "armv8" } }] }, - { "artifact": "core-macos-26-x86_64-ninja-appleclang-21-release","packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "core-macos-26-armv8-ninja-appleclang-21-release", "packages": [{ "path": "ghcr.io/openDAQ/${{ needs.resolve.outputs.channel }}/staging/opendaq", "tag": "${{ needs.resolve.outputs.core-ref }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "armv8" } }] } - ] - include-jobs: >- - [ - "windows-2022-x86_64-msvs-v143-release", - "windows-2022-x86-msvs-v143-release", - "manylinux_2_28-x86_64-ninja-gcc-14-release", - "manylinux_2_28-armv8-ninja-gcc-14-release", - "ubuntu-22.04-x86_64-ninja-gcc-release", - "ubuntu-22.04-armv8-ninja-gcc-release", - "macos-26-x86_64-ninja-appleclang-21-release", - "macos-26-armv8-ninja-appleclang-21-release" - ] + include-jobs: ${{ needs.resolve.outputs.jobs }} + staging-pull: ${{ needs.resolve.outputs.staging-pull }} packages: >- [ - { "match-jobs": ["windows-2022-x86_64-msvs-v143-release"], "artifacts": [{ "name": "core-windows-2022-x86_64-msvs-v143-release", "dir": "staging-deps" }] }, - { "match-jobs": ["windows-2022-x86-msvs-v143-release"], "artifacts": [{ "name": "core-windows-2022-x86-msvs-v143-release", "dir": "staging-deps" }] }, - { "match-jobs": ["manylinux_2_28-x86_64-ninja-gcc-14-release"], "artifacts": [{ "name": "core-manylinux_2_28-x86_64-ninja-gcc-14-release", "dir": "staging-deps" }] }, - { "match-jobs": ["manylinux_2_28-armv8-ninja-gcc-14-release"], "artifacts": [{ "name": "core-manylinux_2_28-armv8-ninja-gcc-14-release", "dir": "staging-deps" }] }, - { "match-jobs": ["ubuntu-22.04-x86_64-ninja-gcc-release"], "artifacts": [{ "name": "core-ubuntu-22.04-x86_64-ninja-gcc-release", "dir": "staging-deps" }] }, - { "match-jobs": ["ubuntu-22.04-armv8-ninja-gcc-release"], "artifacts": [{ "name": "core-ubuntu-22.04-armv8-ninja-gcc-release", "dir": "staging-deps" }] }, - { "match-jobs": ["macos-26-x86_64-ninja-appleclang-21-release"],"artifacts": [{ "name": "core-macos-26-x86_64-ninja-appleclang-21-release", "dir": "staging-deps" }] }, - { "match-jobs": ["macos-26-armv8-ninja-appleclang-21-release"], "artifacts": [{ "name": "core-macos-26-armv8-ninja-appleclang-21-release", "dir": "staging-deps" }] }, - + ${{ needs.resolve.outputs.opendaq-artifacts }}, { "match-jobs": ["ubuntu-*"], "apt-install": ["libssl-dev"] }, { "match-jobs": ["manylinux*"], "yum-install": ["openssl-devel"] }, { "match-jobs": ["windows-*-x86-*"], "run": "vcpkg install openssl:x86-windows; Add-Content -Path $env:GITHUB_PATH -Value C:\\vcpkg\\installed\\x86-windows\\bin -Encoding utf8" } @@ -77,7 +106,8 @@ jobs: "configure-preset": "stage", "cache-variables": { - "CMAKE_PREFIX_PATH": "${sourceDir}/staging-deps;${sourceDir}/staging-deps/opt/opendaq" + "CMAKE_PREFIX_PATH": "${sourceDir}/staging-deps;${sourceDir}/staging-deps/opt/opendaq", + "CPACK_OPENDAQ_META_FIELDS": "dependencies/opendaq/revision=${{ needs.resolve.outputs.opendaq-ref }}" } }, { diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5f8b1c0..141e39b 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -8,7 +8,7 @@ include(FetchContent) FetchContent_Declare( opendaq-cmake-utils GIT_REPOSITORY https://github.com/openDAQ/opendaq-cmake-utils.git - GIT_TAG v1.1.1 + GIT_TAG jira/TBBAS-3548-staging-metadata-fields GIT_PROGRESS ON ) FetchContent_MakeAvailable(opendaq-cmake-utils)