diff --git a/.fullsend/config.yaml b/.fullsend/config.yaml deleted file mode 100644 index 5ad1358c36f..00000000000 --- a/.fullsend/config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# fullsend per-repo configuration -# https://github.com/fullsend-ai/fullsend -# -# This file configures fullsend for per-repo installation mode. -# See https://fullsend.sh/docs/guides/infrastructure/layered-config-reference -version: "1" -roles: - - fullsend - - triage - - coder - - fix - - review - - prioritize -allowed_remote_resources: - - https://raw.githubusercontent.com/fullsend-ai/fullsend/ - - https://raw.githubusercontent.com/fullsend-ai/agents/ -create_issues: - allow_targets: - repos: - - konflux-ci/devlake - - fullsend-ai/fullsend diff --git a/.github/workflows/fullsend.yaml b/.github/workflows/fullsend.yaml deleted file mode 100644 index 9a10f59f3bd..00000000000 --- a/.github/workflows/fullsend.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# This file is managed by fullsend. Do not edit it directly. -# Upstream: https://github.com/fullsend-ai/fullsend/blob/main/internal/scaffold/fullsend-repo/.github/workflows/fullsend.yaml ---- -# fullsend shim workflow (per-repo installation mode) -# Routes events to agent workflows via reusable-dispatch.yml. -# All agent execution happens in this repo's context — no external -# config repo is needed. -# -# Security: pull_request_target runs the BASE branch version of this workflow, -# preventing PRs from modifying it to exfiltrate credentials. -# This shim never checks out PR code, so it is not vulnerable to "pwn request" -# attacks. -# -# Routing: this shim forwards the raw event context to reusable-dispatch.yml, -# which determines the stage and runs the agent inline (ADR 62). -# Adding a new stage requires only a job in reusable-dispatch.yml — zero changes to this repo. -# -# Concurrency: per-role cancel-in-progress groups live in reusable-dispatch.yml -# stage jobs with -agent- suffix. Roles operate independently (#2452). -name: fullsend - -on: - issues: - types: [opened, edited, labeled] - issue_comment: - types: [created] - pull_request_target: - types: [opened, synchronize, ready_for_review, closed, labeled, unlabeled] - pull_request_review: - types: [submitted] - -permissions: {} - -jobs: - dispatch: - if: >- - (github.event_name != 'pull_request_target' && github.event_name != 'pull_request_review' - || github.event.pull_request.head.ref != 'fullsend/scaffold-install') - && (github.event_name != 'issue_comment' - || (startsWith(github.event.comment.body, '/fs-') - && github.event.comment.user.type != 'Bot')) - permissions: - actions: write - id-token: write - contents: write - issues: write - packages: read - pull-requests: write - uses: fullsend-ai/fullsend/.github/workflows/reusable-dispatch.yml@d5f36921ac754705619f38c637ef692873809fbc # v0.43.0 - with: - event_action: ${{ github.event.action }} - install_mode: per-repo - mint_url: ${{ vars.FULLSEND_MINT_URL }} - gcp_region: ${{ vars.FULLSEND_GCP_REGION }} - project_number: ${{ vars.FULLSEND_PROJECT_NUMBER }} - runner_image: ubuntu-24.04 - secrets: - FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} - FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} - OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} - OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} - - stop-fix: - # Job-level if: is intentionally coarse — it only screens for the - # /fs-fix-stop command on a PR from a non-bot. The authoritative - # authorization decision (collaborator permission API + PR-author escape - # hatch) is made in the step below, so a maintainer whose author_association - # is not MEMBER (e.g. private org membership) is not filtered out (ADR 0054). - if: >- - github.event_name == 'issue_comment' - && github.event.issue.pull_request - && github.event.comment.user.type != 'Bot' - && github.event.comment.body == '/fs-fix-stop' - runs-on: ubuntu-24.04 - permissions: - contents: read - issues: write - pull-requests: write - steps: - - name: Add fullsend-no-fix label and notify - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.issue.number }} - REPO: ${{ github.repository }} - COMMENT_USER_LOGIN: ${{ github.event.comment.user.login }} - ISSUE_USER_LOGIN: ${{ github.event.issue.user.login }} - run: | - set -euo pipefail - # ADR 0054: authorize via the collaborator permission API - # (admin|maintain|write), not author_association — the latter grants - # contributor status to anyone with a single merged PR (issue #5421). - # Mirrors has_repo_permission() in dispatch.yml; keep the two in sync. - # The PR author may always stop the fix agent on their own PR. - authorized=false - if [[ -n "$COMMENT_USER_LOGIN" && "$COMMENT_USER_LOGIN" == "$ISSUE_USER_LOGIN" ]]; then - authorized=true - else - if api_err=$(mktemp); then - if role=$(gh api "repos/$REPO/collaborators/$COMMENT_USER_LOGIN/permission" \ - --jq '.role_name' 2>"$api_err"); then - case "$role" in - admin|maintain|write) authorized=true ;; - esac - else - echo "::warning::Permission API call failed for $COMMENT_USER_LOGIN: $(cat "$api_err")" - fi - rm -f "$api_err" - else - echo "::warning::Failed to create temp file for permission check of $COMMENT_USER_LOGIN" - fi - fi - if [[ "$authorized" != "true" ]]; then - echo "::notice::User $COMMENT_USER_LOGIN is not authorized to stop the fix agent (requires write access or PR authorship)" - exit 0 - fi - gh label create "fullsend-no-fix" --repo "$REPO" \ - --description "Skip bot-triggered fix agent runs" --color "FBCA04" \ - --force 2>/dev/null || true - gh pr edit "$PR_NUMBER" --repo "$REPO" \ - --add-label "fullsend-no-fix" - gh pr comment "$PR_NUMBER" --repo "$REPO" \ - --body "Fix agent disabled for this PR. Remove the \`fullsend-no-fix\` label or use \`/fs-fix\` to re-engage." diff --git a/.github/workflows/prioritize.yml b/.github/workflows/prioritize.yml deleted file mode 100644 index 3e89e64a5ce..00000000000 --- a/.github/workflows/prioritize.yml +++ /dev/null @@ -1,50 +0,0 @@ -# This file is managed by fullsend. Do not edit it directly. -# Upstream: https://github.com/fullsend-ai/fullsend/blob/main/internal/scaffold/fullsend-repo/.github/workflows/prioritize.yml ---- -# fullsend-stage: prioritize -name: Prioritize - -permissions: - actions: write - contents: read - id-token: write - issues: write - -on: - workflow_dispatch: - inputs: - event_type: - required: true - type: string - source_repo: - required: true - type: string - event_payload: - required: true - type: string - project_number: - description: GitHub Projects V2 project number for RICE scoring - required: false - type: string - -concurrency: - group: fullsend-prioritize-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number }} - cancel-in-progress: true - -jobs: - prioritize: - uses: fullsend-ai/fullsend/.github/workflows/reusable-prioritize.yml@d5f36921ac754705619f38c637ef692873809fbc # v0.43.0 - with: - event_type: ${{ inputs.event_type }} - source_repo: ${{ inputs.source_repo }} - event_payload: ${{ inputs.event_payload }} - mint_url: ${{ vars.FULLSEND_MINT_URL }} - gcp_region: ${{ vars.FULLSEND_GCP_REGION }} - project_number: ${{ inputs.project_number || vars.FULLSEND_PROJECT_NUMBER }} - install_mode: per-repo - runner_image: ubuntu-24.04 - secrets: - FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} - FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} - OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} - OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} diff --git a/.licenserc.yaml b/.licenserc.yaml index c3bfed675d5..f051bef0adf 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -57,6 +57,8 @@ header: - ".pr_agent.toml" - ".gitleaks.toml" - ".github/workflows/fullsend.yaml" + - ".github/workflows/prioritize.yml" + - ".fullsend/**" - "CLAUDE.md" - "CODEOWNERS" - ".tekton/**" diff --git a/.tekton/devlake-backend-d41aa-pull-request.yaml b/.tekton/devlake-backend-pull-request.yaml similarity index 67% rename from .tekton/devlake-backend-d41aa-pull-request.yaml rename to .tekton/devlake-backend-pull-request.yaml index 53a349d3c6c..4d977b34053 100644 --- a/.tekton/devlake-backend-d41aa-pull-request.yaml +++ b/.tekton/devlake-backend-pull-request.yaml @@ -2,20 +2,20 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: annotations: - build.appstudio.openshift.io/repo: https://github.com/konflux-ci/devlake?rev={{revision}} + build.appstudio.openshift.io/repo: https://github.com/RH-DevProd/devlake?rev={{revision}} build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/cancel-in-progress: "true" pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch == "main" - creationTimestamp: + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "main" labels: appstudio.openshift.io/application: devlake - appstudio.openshift.io/component: devlake-backend-d41aa + appstudio.openshift.io/component: devlake-backend pipelines.appstudio.openshift.io/type: build - name: devlake-backend-d41aa-on-pull-request - namespace: rh-konflux-devprod-tenant + name: devlake-backend-on-pull-request + namespace: dprod-tenant spec: params: - name: git-url @@ -23,19 +23,22 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/konflux-ci/konflux-devprod/devlake-backend:on-pr-{{revision}} + value: quay.io/redhat-user-workloads/dprod-tenant/devlake-backend:on-pr-{{revision}} - name: image-expires-after value: 5d + - name: build-platforms + value: + - linux/x86_64 - name: dockerfile - value: backend/Containerfile + value: Containerfile - name: path-context value: backend pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -48,17 +51,15 @@ spec: name: output-image type: string - default: . - description: Path to the source code of an application's component from where to build image. + description: Path to the source code of an application's component from where + to build image. name: path-context type: string - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter path-context + description: Path to the Dockerfile inside the context specified by parameter + path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -72,23 +73,35 @@ spec: name: prefetch-input type: string - default: "" - description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. name: image-expires-after type: string - default: "false" description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string - default: docker - description: The format for the resulting image's mediaType. Valid values are oci or docker. + description: The format for the resulting image's mediaType. Valid values are + oci or docker. name: buildah-format type: string - - default: - - "BUILDPLATFORM=linux/amd64" + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: "true" + description: Use the package registry proxy when prefetching dependencies + name: enable-package-registry-proxy + - default: . + description: Target directories in component's source code to scan with SAST + tools. Multiple values should be separated with commas. + name: sast-target-dirs + type: string + - default: [] description: Array of --build-arg values ("arg=value" strings) for buildah name: build-args type: array @@ -97,13 +110,32 @@ spec: name: build-args-file type: string - default: "false" - description: Whether to enable privileged mode, should be used only with remote VMs + description: Whether to enable privileged mode, should be used only with remote + VMs name: privileged-nested type: string - - name: enable-cache-proxy - default: 'false' - description: Enable cache proxy configuration + - default: "" + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. + On its own, it does not change file timestamps inside the layers (set rewrite-timestamp + to "true" for that). Leave empty to keep the actual build time. + name: source-date-epoch + type: string + - default: "false" + description: When "true", clamp file modification times in the image layers + to at most source-date-epoch. Does nothing unless source-date-epoch is set. + name: rewrite-timestamp + type: string + - default: "false" + description: When "true", omit the build history (history timestamps, layer + metadata, etc.) from the resulting image. + name: omit-history type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -120,12 +152,6 @@ spec: tasks: - name: init params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -133,7 +159,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:ebf06778aeacbbeb081f9231eafbdfdb8e380ad04e211d7ed80ae9101e37fd82 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -154,15 +180,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:306b69e6db435ad4a7cf258b6219d9b998eb37da44f5e9ac882ac86a08109154 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" workspaces: - name: basic-auth workspace: git-auth @@ -170,6 +191,8 @@ spec: params: - name: input value: $(params.prefetch-input) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) - name: SOURCE_ARTIFACT value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - name: ociStorage @@ -183,7 +206,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:c664a6df6514b59c3ce53570b0994b45af66ecc89ba2a8e41834eae0622addf6 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -192,12 +215,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container - computeResources: - limits: - memory: 12Gi - requests: - memory: 4Gi + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -216,69 +239,64 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: HTTP_PROXY value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.8@sha256:8a984185d22d3180ecd575dbf6b6a63edb1b3ec754be0b8e5e4247a9eae9112b + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-image-index params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:05d3d8a5ded44c51b074a56a408ddf5d65c56b4c15e110abb1a99e3aff269d49 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-source-image params: - name: BINARY_IMAGE @@ -296,15 +314,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - input: $(params.build-source-image) operator: in values: @@ -322,7 +336,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e3a55ccdf1091b4a35507f9ee2d1918d8e89a5f96babcb5486b491226da03d6f + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -331,7 +345,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: roxctl-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -342,9 +361,9 @@ spec: taskRef: params: - name: name - value: clair-scan + value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:dadfea7633d82e4303ba73d5e9c7e2bc16834bde0fd7688880453b26452067eb + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -353,7 +372,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -364,7 +388,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:204fd3914d83c7b60e8eee72b5a944337720c79a3e660e7c994435456dcf7175 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -379,10 +403,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -390,7 +414,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:a70272ae12f6d7f0da2902158e1bcee756877aa8f71fd1a22ef9afd8b177fb41 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -399,86 +423,25 @@ spec: operator: in values: - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: + - matrix: params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:76efc0119a10bc8a420dbbb0cdab9ef8eafd263f6827498d2b644e450e93f446 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-coverity-check + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - coverity-availability-check - taskRef: - params: - - name: name - value: sast-coverity-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:9d0bc704bca6b6faad37b2ce8106b96e0cef35e1f647d037a878bf416589de9d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - input: $(tasks.coverity-availability-check.results.STATUS) - operator: in - values: - - success - - name: coverity-availability-check runAfter: - build-image-index taskRef: params: - name: name - value: coverity-availability-check + value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:267d5bc069a0323f41e24732ddfd1057e5c639e853d1e620c67505fab78f1301 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -493,10 +456,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -504,7 +467,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:e7a51575f9188a1461d4520da25aaa4efdd3b896c97dc750941fa22840e55c13 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -519,10 +482,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -530,7 +493,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:1818a5b3e4fa86c838ae71226a157241967d1f19c5ed377e4b2fddad7a3ceefe + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -552,7 +515,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:510b6d2a3b188adeb716e49566b57d611ab36bd69a2794b5ddfc11dbf014c2ca + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -575,7 +538,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles @@ -592,7 +555,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:3d016c63bcab64ab82da762a52b013c0bcb534e9523b8c0e073cc3a0c02f0cac + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -607,7 +570,7 @@ spec: - name: netrc optional: true taskRunSpecs: - - pipelineTaskName: build-container + - pipelineTaskName: build-images stepSpecs: - name: build computeResources: @@ -618,7 +581,7 @@ spec: memory: 16Gi cpu: "6" taskRunTemplate: - serviceAccountName: build-pipeline-devlake-backend-d41aa + serviceAccountName: build-pipeline-devlake-backend workspaces: - name: git-auth secret: diff --git a/.tekton/devlake-backend-d41aa-push.yaml b/.tekton/devlake-backend-push.yaml similarity index 68% rename from .tekton/devlake-backend-d41aa-push.yaml rename to .tekton/devlake-backend-push.yaml index 792280ecd02..01ff14e6de4 100644 --- a/.tekton/devlake-backend-d41aa-push.yaml +++ b/.tekton/devlake-backend-push.yaml @@ -2,19 +2,19 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: annotations: - build.appstudio.openshift.io/repo: https://github.com/konflux-ci/devlake?rev={{revision}} + build.appstudio.openshift.io/repo: https://github.com/RH-DevProd/devlake?rev={{revision}} build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/cancel-in-progress: "false" pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch == "main" - creationTimestamp: + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "main" labels: appstudio.openshift.io/application: devlake - appstudio.openshift.io/component: devlake-backend-d41aa + appstudio.openshift.io/component: devlake-backend pipelines.appstudio.openshift.io/type: build - name: devlake-backend-d41aa-on-push - namespace: rh-konflux-devprod-tenant + name: devlake-backend-on-push + namespace: dprod-tenant spec: params: - name: git-url @@ -22,17 +22,20 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/konflux-ci/konflux-devprod/devlake-backend:{{revision}} + value: quay.io/redhat-user-workloads/dprod-tenant/devlake-backend:{{revision}} + - name: build-platforms + value: + - linux/x86_64 - name: dockerfile - value: backend/Containerfile + value: Containerfile - name: path-context value: backend pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -45,17 +48,15 @@ spec: name: output-image type: string - default: . - description: Path to the source code of an application's component from where to build image. + description: Path to the source code of an application's component from where + to build image. name: path-context type: string - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter path-context + description: Path to the Dockerfile inside the context specified by parameter + path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -69,23 +70,35 @@ spec: name: prefetch-input type: string - default: "" - description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. name: image-expires-after type: string - default: "false" description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string - default: docker - description: The format for the resulting image's mediaType. Valid values are oci or docker. + description: The format for the resulting image's mediaType. Valid values are + oci or docker. name: buildah-format type: string - - default: - - "BUILDPLATFORM=linux/amd64" + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: "true" + description: Use the package registry proxy when prefetching dependencies + name: enable-package-registry-proxy + - default: . + description: Target directories in component's source code to scan with SAST + tools. Multiple values should be separated with commas. + name: sast-target-dirs + type: string + - default: [] description: Array of --build-arg values ("arg=value" strings) for buildah name: build-args type: array @@ -94,13 +107,32 @@ spec: name: build-args-file type: string - default: "false" - description: Whether to enable privileged mode, should be used only with remote VMs + description: Whether to enable privileged mode, should be used only with remote + VMs name: privileged-nested type: string - - name: enable-cache-proxy - default: 'false' - description: Enable cache proxy configuration + - default: "" + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. + On its own, it does not change file timestamps inside the layers (set rewrite-timestamp + to "true" for that). Leave empty to keep the actual build time. + name: source-date-epoch + type: string + - default: "false" + description: When "true", clamp file modification times in the image layers + to at most source-date-epoch. Does nothing unless source-date-epoch is set. + name: rewrite-timestamp + type: string + - default: "false" + description: When "true", omit the build history (history timestamps, layer + metadata, etc.) from the resulting image. + name: omit-history type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -117,12 +149,6 @@ spec: tasks: - name: init params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -130,7 +156,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:ebf06778aeacbbeb081f9231eafbdfdb8e380ad04e211d7ed80ae9101e37fd82 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -151,15 +177,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:306b69e6db435ad4a7cf258b6219d9b998eb37da44f5e9ac882ac86a08109154 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" workspaces: - name: basic-auth workspace: git-auth @@ -167,6 +188,8 @@ spec: params: - name: input value: $(params.prefetch-input) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) - name: SOURCE_ARTIFACT value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - name: ociStorage @@ -180,7 +203,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:c664a6df6514b59c3ce53570b0994b45af66ecc89ba2a8e41834eae0622addf6 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -189,7 +212,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -208,69 +236,64 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: HTTP_PROXY value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.8@sha256:8a984185d22d3180ecd575dbf6b6a63edb1b3ec754be0b8e5e4247a9eae9112b + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-image-index params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:05d3d8a5ded44c51b074a56a408ddf5d65c56b4c15e110abb1a99e3aff269d49 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-source-image params: - name: BINARY_IMAGE @@ -288,15 +311,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - input: $(params.build-source-image) operator: in values: @@ -314,7 +333,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e3a55ccdf1091b4a35507f9ee2d1918d8e89a5f96babcb5486b491226da03d6f + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -323,7 +342,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: roxctl-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -334,9 +358,9 @@ spec: taskRef: params: - name: name - value: clair-scan + value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:dadfea7633d82e4303ba73d5e9c7e2bc16834bde0fd7688880453b26452067eb + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -345,7 +369,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -356,7 +385,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:204fd3914d83c7b60e8eee72b5a944337720c79a3e660e7c994435456dcf7175 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -371,10 +400,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -382,7 +411,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:a70272ae12f6d7f0da2902158e1bcee756877aa8f71fd1a22ef9afd8b177fb41 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -391,86 +420,25 @@ spec: operator: in values: - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: + - matrix: params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:76efc0119a10bc8a420dbbb0cdab9ef8eafd263f6827498d2b644e450e93f446 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-coverity-check + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - coverity-availability-check - taskRef: - params: - - name: name - value: sast-coverity-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:9d0bc704bca6b6faad37b2ce8106b96e0cef35e1f647d037a878bf416589de9d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - input: $(tasks.coverity-availability-check.results.STATUS) - operator: in - values: - - success - - name: coverity-availability-check runAfter: - build-image-index taskRef: params: - name: name - value: coverity-availability-check + value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:267d5bc069a0323f41e24732ddfd1057e5c639e853d1e620c67505fab78f1301 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -485,10 +453,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -496,7 +464,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:e7a51575f9188a1461d4520da25aaa4efdd3b896c97dc750941fa22840e55c13 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -511,10 +479,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -522,7 +490,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:1818a5b3e4fa86c838ae71226a157241967d1f19c5ed377e4b2fddad7a3ceefe + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -544,7 +512,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:510b6d2a3b188adeb716e49566b57d611ab36bd69a2794b5ddfc11dbf014c2ca + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -567,7 +535,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles @@ -584,7 +552,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:3d016c63bcab64ab82da762a52b013c0bcb534e9523b8c0e073cc3a0c02f0cac + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -599,7 +567,7 @@ spec: - name: netrc optional: true taskRunSpecs: - - pipelineTaskName: build-container + - pipelineTaskName: build-images stepSpecs: - name: build computeResources: @@ -610,7 +578,7 @@ spec: memory: 16Gi cpu: "6" taskRunTemplate: - serviceAccountName: build-pipeline-devlake-backend-d41aa + serviceAccountName: build-pipeline-devlake-backend workspaces: - name: git-auth secret: diff --git a/.tekton/devlake-frontend-ba03c-pull-request.yaml b/.tekton/devlake-frontend-pull-request.yaml similarity index 67% rename from .tekton/devlake-frontend-ba03c-pull-request.yaml rename to .tekton/devlake-frontend-pull-request.yaml index c3dbb65ca75..765e650875b 100644 --- a/.tekton/devlake-frontend-ba03c-pull-request.yaml +++ b/.tekton/devlake-frontend-pull-request.yaml @@ -2,20 +2,20 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: annotations: - build.appstudio.openshift.io/repo: https://github.com/konflux-ci/devlake?rev={{revision}} + build.appstudio.openshift.io/repo: https://github.com/RH-DevProd/devlake?rev={{revision}} build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/cancel-in-progress: "true" pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch == "main" && ( "config-ui/***".pathChanged() || ".tekton/devlake-frontend-ba03c-pull-request.yaml".pathChanged() || "config-ui/frontend".pathChanged() ) - creationTimestamp: + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "main" && ( "config-ui/***".pathChanged() || ".tekton/devlake-frontend-pull-request.yaml".pathChanged() ) labels: appstudio.openshift.io/application: devlake - appstudio.openshift.io/component: devlake-frontend-ba03c + appstudio.openshift.io/component: devlake-frontend pipelines.appstudio.openshift.io/type: build - name: devlake-frontend-ba03c-on-pull-request - namespace: rh-konflux-devprod-tenant + name: devlake-frontend-on-pull-request + namespace: dprod-tenant spec: params: - name: git-url @@ -23,19 +23,22 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/konflux-ci/konflux-devprod/devlake-frontend:on-pr-{{revision}} + value: quay.io/redhat-user-workloads/dprod-tenant/devlake-frontend:on-pr-{{revision}} - name: image-expires-after value: 5d - - name: dockerfile - value: config-ui/Containerfile + - name: build-platforms + value: + - linux/x86_64 - name: path-context value: config-ui + - name: dockerfile + value: Containerfile pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -48,17 +51,15 @@ spec: name: output-image type: string - default: . - description: Path to the source code of an application's component from where to build image. + description: Path to the source code of an application's component from where + to build image. name: path-context type: string - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter path-context + description: Path to the Dockerfile inside the context specified by parameter + path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -72,21 +73,34 @@ spec: name: prefetch-input type: string - default: "" - description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. name: image-expires-after type: string - default: "false" description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string - default: docker - description: The format for the resulting image's mediaType. Valid values are oci or docker. + description: The format for the resulting image's mediaType. Valid values are + oci or docker. name: buildah-format type: string + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: "true" + description: Use the package registry proxy when prefetching dependencies + name: enable-package-registry-proxy + - default: . + description: Target directories in component's source code to scan with SAST + tools. Multiple values should be separated with commas. + name: sast-target-dirs + type: string - default: [] description: Array of --build-arg values ("arg=value" strings) for buildah name: build-args @@ -96,13 +110,32 @@ spec: name: build-args-file type: string - default: "false" - description: Whether to enable privileged mode, should be used only with remote VMs + description: Whether to enable privileged mode, should be used only with remote + VMs name: privileged-nested type: string - - name: enable-cache-proxy - default: 'false' - description: Enable cache proxy configuration + - default: "" + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. + On its own, it does not change file timestamps inside the layers (set rewrite-timestamp + to "true" for that). Leave empty to keep the actual build time. + name: source-date-epoch + type: string + - default: "false" + description: When "true", clamp file modification times in the image layers + to at most source-date-epoch. Does nothing unless source-date-epoch is set. + name: rewrite-timestamp + type: string + - default: "false" + description: When "true", omit the build history (history timestamps, layer + metadata, etc.) from the resulting image. + name: omit-history type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -119,12 +152,6 @@ spec: tasks: - name: init params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -132,7 +159,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:ebf06778aeacbbeb081f9231eafbdfdb8e380ad04e211d7ed80ae9101e37fd82 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -153,15 +180,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:306b69e6db435ad4a7cf258b6219d9b998eb37da44f5e9ac882ac86a08109154 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" workspaces: - name: basic-auth workspace: git-auth @@ -169,6 +191,8 @@ spec: params: - name: input value: $(params.prefetch-input) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) - name: SOURCE_ARTIFACT value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - name: ociStorage @@ -182,7 +206,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:c664a6df6514b59c3ce53570b0994b45af66ecc89ba2a8e41834eae0622addf6 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -191,7 +215,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -210,69 +239,64 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: HTTP_PROXY value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.8@sha256:8a984185d22d3180ecd575dbf6b6a63edb1b3ec754be0b8e5e4247a9eae9112b + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-image-index params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:05d3d8a5ded44c51b074a56a408ddf5d65c56b4c15e110abb1a99e3aff269d49 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-source-image params: - name: BINARY_IMAGE @@ -290,15 +314,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - input: $(params.build-source-image) operator: in values: @@ -316,7 +336,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e3a55ccdf1091b4a35507f9ee2d1918d8e89a5f96babcb5486b491226da03d6f + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -325,7 +345,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: roxctl-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -336,9 +361,9 @@ spec: taskRef: params: - name: name - value: clair-scan + value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:dadfea7633d82e4303ba73d5e9c7e2bc16834bde0fd7688880453b26452067eb + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -347,7 +372,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -358,7 +388,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:204fd3914d83c7b60e8eee72b5a944337720c79a3e660e7c994435456dcf7175 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -373,10 +403,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -384,7 +414,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:a70272ae12f6d7f0da2902158e1bcee756877aa8f71fd1a22ef9afd8b177fb41 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -393,86 +423,25 @@ spec: operator: in values: - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: + - matrix: params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:76efc0119a10bc8a420dbbb0cdab9ef8eafd263f6827498d2b644e450e93f446 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-coverity-check + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - coverity-availability-check - taskRef: - params: - - name: name - value: sast-coverity-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:9d0bc704bca6b6faad37b2ce8106b96e0cef35e1f647d037a878bf416589de9d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - input: $(tasks.coverity-availability-check.results.STATUS) - operator: in - values: - - success - - name: coverity-availability-check runAfter: - build-image-index taskRef: params: - name: name - value: coverity-availability-check + value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:267d5bc069a0323f41e24732ddfd1057e5c639e853d1e620c67505fab78f1301 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -487,10 +456,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -498,7 +467,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:e7a51575f9188a1461d4520da25aaa4efdd3b896c97dc750941fa22840e55c13 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -513,10 +482,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -524,7 +493,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:1818a5b3e4fa86c838ae71226a157241967d1f19c5ed377e4b2fddad7a3ceefe + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -546,7 +515,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:510b6d2a3b188adeb716e49566b57d611ab36bd69a2794b5ddfc11dbf014c2ca + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -569,7 +538,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles @@ -586,7 +555,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:3d016c63bcab64ab82da762a52b013c0bcb534e9523b8c0e073cc3a0c02f0cac + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -601,7 +570,7 @@ spec: - name: netrc optional: true taskRunTemplate: - serviceAccountName: build-pipeline-devlake-frontend-ba03c + serviceAccountName: build-pipeline-devlake-frontend workspaces: - name: git-auth secret: diff --git a/.tekton/devlake-frontend-ba03c-push.yaml b/.tekton/devlake-frontend-push.yaml similarity index 67% rename from .tekton/devlake-frontend-ba03c-push.yaml rename to .tekton/devlake-frontend-push.yaml index c8402983ce3..24008dd21a9 100644 --- a/.tekton/devlake-frontend-ba03c-push.yaml +++ b/.tekton/devlake-frontend-push.yaml @@ -2,19 +2,19 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: annotations: - build.appstudio.openshift.io/repo: https://github.com/konflux-ci/devlake?rev={{revision}} + build.appstudio.openshift.io/repo: https://github.com/RH-DevProd/devlake?rev={{revision}} build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/cancel-in-progress: "false" pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch == "main" && ( "config-ui/***".pathChanged() || ".tekton/devlake-frontend-ba03c-push.yaml".pathChanged() || "config-ui/frontend".pathChanged() ) - creationTimestamp: + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "main" && ( "config-ui/***".pathChanged() || ".tekton/devlake-frontend-push.yaml".pathChanged() ) labels: appstudio.openshift.io/application: devlake - appstudio.openshift.io/component: devlake-frontend-ba03c + appstudio.openshift.io/component: devlake-frontend pipelines.appstudio.openshift.io/type: build - name: devlake-frontend-ba03c-on-push - namespace: rh-konflux-devprod-tenant + name: devlake-frontend-on-push + namespace: dprod-tenant spec: params: - name: git-url @@ -22,17 +22,20 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/konflux-ci/konflux-devprod/devlake-frontend:{{revision}} - - name: dockerfile - value: config-ui/Containerfile + value: quay.io/redhat-user-workloads/dprod-tenant/devlake-frontend:{{revision}} + - name: build-platforms + value: + - linux/x86_64 - name: path-context value: config-ui + - name: dockerfile + value: Containerfile pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -45,17 +48,15 @@ spec: name: output-image type: string - default: . - description: Path to the source code of an application's component from where to build image. + description: Path to the source code of an application's component from where + to build image. name: path-context type: string - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter path-context + description: Path to the Dockerfile inside the context specified by parameter + path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -69,21 +70,34 @@ spec: name: prefetch-input type: string - default: "" - description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. name: image-expires-after type: string - default: "false" description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string - default: docker - description: The format for the resulting image's mediaType. Valid values are oci or docker. + description: The format for the resulting image's mediaType. Valid values are + oci or docker. name: buildah-format type: string + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: "true" + description: Use the package registry proxy when prefetching dependencies + name: enable-package-registry-proxy + - default: . + description: Target directories in component's source code to scan with SAST + tools. Multiple values should be separated with commas. + name: sast-target-dirs + type: string - default: [] description: Array of --build-arg values ("arg=value" strings) for buildah name: build-args @@ -93,13 +107,32 @@ spec: name: build-args-file type: string - default: "false" - description: Whether to enable privileged mode, should be used only with remote VMs + description: Whether to enable privileged mode, should be used only with remote + VMs name: privileged-nested type: string - - name: enable-cache-proxy - default: 'false' - description: Enable cache proxy configuration + - default: "" + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. + On its own, it does not change file timestamps inside the layers (set rewrite-timestamp + to "true" for that). Leave empty to keep the actual build time. + name: source-date-epoch + type: string + - default: "false" + description: When "true", clamp file modification times in the image layers + to at most source-date-epoch. Does nothing unless source-date-epoch is set. + name: rewrite-timestamp + type: string + - default: "false" + description: When "true", omit the build history (history timestamps, layer + metadata, etc.) from the resulting image. + name: omit-history type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -116,12 +149,6 @@ spec: tasks: - name: init params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -129,7 +156,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:ebf06778aeacbbeb081f9231eafbdfdb8e380ad04e211d7ed80ae9101e37fd82 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -150,15 +177,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:306b69e6db435ad4a7cf258b6219d9b998eb37da44f5e9ac882ac86a08109154 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" workspaces: - name: basic-auth workspace: git-auth @@ -166,6 +188,8 @@ spec: params: - name: input value: $(params.prefetch-input) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) - name: SOURCE_ARTIFACT value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - name: ociStorage @@ -179,7 +203,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:c664a6df6514b59c3ce53570b0994b45af66ecc89ba2a8e41834eae0622addf6 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -188,7 +212,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -207,69 +236,64 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: HTTP_PROXY value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.8@sha256:8a984185d22d3180ecd575dbf6b6a63edb1b3ec754be0b8e5e4247a9eae9112b + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-image-index params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:05d3d8a5ded44c51b074a56a408ddf5d65c56b4c15e110abb1a99e3aff269d49 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - name: build-source-image params: - name: BINARY_IMAGE @@ -287,15 +311,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - input: $(params.build-source-image) operator: in values: @@ -313,7 +333,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e3a55ccdf1091b4a35507f9ee2d1918d8e89a5f96babcb5486b491226da03d6f + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -322,7 +342,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: roxctl-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -333,9 +358,9 @@ spec: taskRef: params: - name: name - value: clair-scan + value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:dadfea7633d82e4303ba73d5e9c7e2bc16834bde0fd7688880453b26452067eb + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -344,7 +369,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -355,7 +385,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:204fd3914d83c7b60e8eee72b5a944337720c79a3e660e7c994435456dcf7175 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -370,10 +400,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -381,7 +411,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:a70272ae12f6d7f0da2902158e1bcee756877aa8f71fd1a22ef9afd8b177fb41 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -390,86 +420,25 @@ spec: operator: in values: - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: + - matrix: params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:76efc0119a10bc8a420dbbb0cdab9ef8eafd263f6827498d2b644e450e93f446 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-coverity-check + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - coverity-availability-check - taskRef: - params: - - name: name - value: sast-coverity-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:9d0bc704bca6b6faad37b2ce8106b96e0cef35e1f647d037a878bf416589de9d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - input: $(tasks.coverity-availability-check.results.STATUS) - operator: in - values: - - success - - name: coverity-availability-check runAfter: - build-image-index taskRef: params: - name: name - value: coverity-availability-check + value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:267d5bc069a0323f41e24732ddfd1057e5c639e853d1e620c67505fab78f1301 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -484,10 +453,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -495,7 +464,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:e7a51575f9188a1461d4520da25aaa4efdd3b896c97dc750941fa22840e55c13 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -510,10 +479,10 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -521,7 +490,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:1818a5b3e4fa86c838ae71226a157241967d1f19c5ed377e4b2fddad7a3ceefe + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -543,7 +512,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:510b6d2a3b188adeb716e49566b57d611ab36bd69a2794b5ddfc11dbf014c2ca + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -566,7 +535,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles @@ -583,7 +552,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:3d016c63bcab64ab82da762a52b013c0bcb534e9523b8c0e073cc3a0c02f0cac + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -598,7 +567,7 @@ spec: - name: netrc optional: true taskRunTemplate: - serviceAccountName: build-pipeline-devlake-frontend-ba03c + serviceAccountName: build-pipeline-devlake-frontend workspaces: - name: git-auth secret: diff --git a/backend/helpers/pluginhelper/api/graphql_async_client.go b/backend/helpers/pluginhelper/api/graphql_async_client.go index 32dac66a9eb..61a1c9fa3ca 100644 --- a/backend/helpers/pluginhelper/api/graphql_async_client.go +++ b/backend/helpers/pluginhelper/api/graphql_async_client.go @@ -20,12 +20,14 @@ package api import ( "context" "fmt" + "strconv" + "sync" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/core/utils" - "sync" - "time" "github.com/merico-ai/graphql" ) @@ -47,30 +49,52 @@ type GraphqlAsyncClient struct { getRateCost func(q interface{}) int } +// defaultRateLimitConst is the generic fallback rate limit for GraphQL requests. +// It is used as the initial remaining quota when dynamic rate limit +// information is unavailable from the provider. +const defaultRateLimitConst = 1000 + // CreateAsyncGraphqlClient creates a new GraphqlAsyncClient func CreateAsyncGraphqlClient( taskCtx plugin.TaskContext, graphqlClient *graphql.Client, logger log.Logger, getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), + opts ...func(*GraphqlAsyncClient), ) (*GraphqlAsyncClient, errors.Error) { ctxWithCancel, cancel := context.WithCancel(taskCtx.GetContext()) + graphqlAsyncClient := &GraphqlAsyncClient{ ctx: ctxWithCancel, cancel: cancel, client: graphqlClient, logger: logger, rateExhaustCond: sync.NewCond(&sync.Mutex{}), - rateRemaining: 0, + rateRemaining: defaultRateLimitConst, getRateRemaining: getRateRemaining, } + // apply options + for _, opt := range opts { + opt(graphqlAsyncClient) + } + + // Env config wins over everything, only if explicitly set + if rateLimit := resolveRateLimit(taskCtx, logger); rateLimit != -1 { + logger.Info("GRAPHQL_RATE_LIMIT env override applied: %d (was %d)", rateLimit, graphqlAsyncClient.rateRemaining) + graphqlAsyncClient.rateRemaining = rateLimit + } + if getRateRemaining != nil { rateRemaining, resetAt, err := getRateRemaining(taskCtx.GetContext(), graphqlClient, logger) if err != nil { - panic(err) + graphqlAsyncClient.logger.Info("failed to fetch initial graphql rate limit, fallback to default: %v", err) + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) + } else { + graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) } - graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) + } else { + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) } // load retry/timeout from configuration @@ -115,6 +139,10 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese apiClient.rateExhaustCond.Signal() } go func() { + if apiClient.getRateRemaining == nil { + return + } + nextDuring := 3 * time.Minute if resetAt != nil && resetAt.After(time.Now()) { nextDuring = time.Until(*resetAt) @@ -126,7 +154,15 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese case <-time.After(nextDuring): newRateRemaining, newResetAt, err := apiClient.getRateRemaining(apiClient.ctx, apiClient.client, apiClient.logger) if err != nil { - panic(err) + apiClient.logger.Info("failed to update graphql rate limit, will retry next cycle: %v", err) + // Floor the reused value so Signal() always fires; prevents deadlock when + // rateRemaining is 0 and the rate-limit endpoint keeps erroring (e.g. GHE). + fallback := apiClient.rateRemaining + if fallback < defaultRateLimitConst { + fallback = defaultRateLimitConst + } + apiClient.updateRateRemaining(fallback, nil) + return } apiClient.updateRateRemaining(newRateRemaining, newResetAt) } @@ -218,3 +254,25 @@ func (apiClient *GraphqlAsyncClient) Wait() { func (apiClient *GraphqlAsyncClient) Release() { apiClient.cancel() } + +// WithFallbackRateLimit sets the initial/fallback rate limit used when +// rate limit information cannot be fetched dynamically. +// This value may be overridden later by getRateRemaining. +func WithFallbackRateLimit(limit int) func(*GraphqlAsyncClient) { + return func(c *GraphqlAsyncClient) { + if limit > 0 { + c.rateRemaining = limit + } + } +} + +// resolveRateLimit returns -1 if GRAPHQL_RATE_LIMIT is not set or invalid +func resolveRateLimit(taskCtx plugin.TaskContext, logger log.Logger) int { + if v := taskCtx.GetConfig("GRAPHQL_RATE_LIMIT"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil { + return parsed + } + logger.Warn(nil, "invalid GRAPHQL_RATE_LIMIT, using default") + } + return -1 +} diff --git a/backend/pkg/gcshelper/client.go b/backend/pkg/gcshelper/client.go index f66ab87a975..b3552ef71b7 100644 --- a/backend/pkg/gcshelper/client.go +++ b/backend/pkg/gcshelper/client.go @@ -31,8 +31,11 @@ import ( "google.golang.org/api/option" ) -// OpenshiftCIBucketName is the public GCS bucket that stores Prow job results. -const OpenshiftCIBucketName = "test-platform-results" +// OpenshiftCIBucketName is the anonymously readable GCS bucket that stores +// Prow job artifacts. OpenShift CI cut over from test-platform-results to +// test-platform-results-public around 2026-09-17; the old bucket no longer +// allows unauthenticated listing. +const OpenshiftCIBucketName = "test-platform-results-public" // maxFileSize caps individual GCS object reads at 10 MB. const maxFileSize = 10 * 1024 * 1024 diff --git a/backend/pkg/gcshelper/client_test.go b/backend/pkg/gcshelper/client_test.go new file mode 100644 index 00000000000..964ff170ba4 --- /dev/null +++ b/backend/pkg/gcshelper/client_test.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcshelper + +import "testing" + +func TestOpenshiftCIBucketNameIsPublic(t *testing.T) { + // Prow view_url values since 2026-09-17 use this bucket. Unauthenticated + // listing of the previous name (test-platform-results) is denied. + if OpenshiftCIBucketName != "test-platform-results-public" { + t.Fatalf("OpenshiftCIBucketName = %q, want test-platform-results-public", OpenshiftCIBucketName) + } +} diff --git a/backend/plugins/codecov/models/migrationscripts/20261007000000_fix_commits_primary_key.go b/backend/plugins/codecov/models/migrationscripts/20261007000000_fix_commits_primary_key.go new file mode 100644 index 00000000000..563882ee1ce --- /dev/null +++ b/backend/plugins/codecov/models/migrationscripts/20261007000000_fix_commits_primary_key.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +type fixCommitsPrimaryKey struct{} + +// Up deduplicates _tool_codecov_commits and adds the missing composite +// primary key (connection_id, repo_id, commit_sha). +// +// Migration 20260615 dropped the old auto_increment id and called +// AutoMigrateTables, but MySQL's AutoMigrate does not reliably add a +// composite PK to an existing table. The result: no PK, so +// CreateOrUpdate always inserts, causing ~32x row duplication. +func (*fixCommitsPrimaryKey) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + logger := basicRes.GetLogger() + + // 1. Clean up leftover tables from a previous interrupted run (before any checks) + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_dedup`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] failed to drop leftover _dedup table") + } + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_old`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] failed to drop leftover _old table") + } + + // 2. Check if PK already exists (idempotent) + rows, err := db.RawCursor(` + SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = '_tool_codecov_commits' + AND CONSTRAINT_TYPE = 'PRIMARY KEY' + `) + if err == nil && rows != nil { + defer rows.Close() + if rows.Next() { + var pkCount int + if scanErr := rows.Scan(&pkCount); scanErr == nil && pkCount > 0 { + logger.Info("[fix-commits-pk] PK already exists, skipping") + return nil + } + } + } + + // 3. Deduplicate — keep exactly one row per (connection_id, repo_id, commit_sha). + // Uses ROW_NUMBER with a deterministic tie-breaker (updated_at DESC, created_at DESC) + // to handle duplicates that share the same updated_at timestamp. + logger.Info("[fix-commits-pk] Deduplicating _tool_codecov_commits") + + err = db.Exec(`CREATE TABLE _tool_codecov_commits_dedup LIKE _tool_codecov_commits`) + if err != nil { + return errors.Default.Wrap(err, "failed to create dedup table") + } + + err = db.Exec(` + INSERT INTO _tool_codecov_commits_dedup + (created_at, updated_at, _raw_data_params, _raw_data_table, + _raw_data_id, _raw_data_remark, connection_id, repo_id, + commit_sha, branch, commit_timestamp, message, author, parent_sha) + SELECT t.created_at, t.updated_at, t._raw_data_params, t._raw_data_table, + t._raw_data_id, t._raw_data_remark, t.connection_id, t.repo_id, + t.commit_sha, t.branch, t.commit_timestamp, t.message, t.author, t.parent_sha + FROM ( + SELECT *, ROW_NUMBER() OVER ( + PARTITION BY connection_id, repo_id, commit_sha + ORDER BY updated_at DESC, created_at DESC + ) AS rn + FROM _tool_codecov_commits + ) t WHERE t.rn = 1 + `) + if err != nil { + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_dedup`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] also failed to clean up _dedup table after INSERT error") + } + return errors.Default.Wrap(err, "failed to copy unique rows") + } + + // 4. Add PK on the replacement BEFORE swapping — validates uniqueness + // and avoids exposing a keyless table if ALTER fails. + err = db.Exec(` + ALTER TABLE _tool_codecov_commits_dedup + ADD PRIMARY KEY (connection_id, repo_id, commit_sha) + `) + if err != nil { + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_dedup`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] also failed to clean up _dedup table after PK error") + } + return errors.Default.Wrap(err, "failed to add primary key on dedup table") + } + + // 5. Atomic rename: swap dedup into service + err = db.Exec(` + RENAME TABLE _tool_codecov_commits TO _tool_codecov_commits_old, + _tool_codecov_commits_dedup TO _tool_codecov_commits + `) + if err != nil { + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_dedup`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] also failed to clean up _dedup table after RENAME error") + } + return errors.Default.Wrap(err, "failed to rename tables") + } + + // 6. Drop old bloated table + if dropErr := db.Exec(`DROP TABLE IF EXISTS _tool_codecov_commits_old`); dropErr != nil { + logger.Warn(dropErr, "[fix-commits-pk] could not drop _old table — clean up manually") + } + + logger.Info("[fix-commits-pk] Done — duplicates removed, PK added") + return nil +} + +func (*fixCommitsPrimaryKey) Version() uint64 { + return 20261007000000 +} + +func (*fixCommitsPrimaryKey) Name() string { + return "Codecov fix missing primary key on commits table and remove duplicates" +} diff --git a/backend/plugins/codecov/models/migrationscripts/register.go b/backend/plugins/codecov/models/migrationscripts/register.go index d858ddfc406..c357d52af43 100644 --- a/backend/plugins/codecov/models/migrationscripts/register.go +++ b/backend/plugins/codecov/models/migrationscripts/register.go @@ -41,5 +41,6 @@ func All() []plugin.MigrationScript { new(addRepoConfigTable), new(dedupRawTables), new(addServiceToConnections), + new(fixCommitsPrimaryKey), } } diff --git a/backend/plugins/codecov/tasks/commit_coverage_collector.go b/backend/plugins/codecov/tasks/commit_coverage_collector.go index 00e4cafb635..a1fb49b5166 100644 --- a/backend/plugins/codecov/tasks/commit_coverage_collector.go +++ b/backend/plugins/codecov/tasks/commit_coverage_collector.go @@ -68,9 +68,14 @@ func CollectCommitCoverage(taskCtx plugin.SubTaskContext) errors.Error { logger.Info("[Codecov] CommitCoverage: Using default 90 days from %s", startDate.Format("2006-01-02")) } - // Get commits filtered by sync policy + // Get distinct commits filtered by sync policy. + // The commits table may lack a PK (pre-migration), causing duplicate rows. + // Only commit_sha is needed downstream for the API iterator. var commits []models.CodecovCommit - err = db.All(&commits, dal.Where("connection_id = ? AND repo_id = ? AND commit_timestamp >= ?", data.Options.ConnectionId, data.Options.FullName, startDate)) + err = db.All(&commits, + dal.Select("DISTINCT connection_id, repo_id, commit_sha"), + dal.Where("connection_id = ? AND repo_id = ? AND commit_timestamp >= ?", data.Options.ConnectionId, data.Options.FullName, startDate), + ) if err != nil { return err } diff --git a/backend/plugins/github/tasks/api_client.go b/backend/plugins/github/tasks/api_client.go index 42181ff139e..c5be8a0f22b 100644 --- a/backend/plugins/github/tasks/api_client.go +++ b/backend/plugins/github/tasks/api_client.go @@ -26,7 +26,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "github.com/apache/incubator-devlake/plugins/github/token" ) func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnection) (*api.ApiAsyncClient, errors.Error) { @@ -35,40 +34,10 @@ func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnec return nil, err } - logger := taskCtx.GetLogger() - db := taskCtx.GetDal() - encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) - - // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. - var tp *token.TokenProvider - if connection.RefreshToken != "" { - tp = token.NewTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { - tp = token.NewAppInstallationTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } - if tp != nil { - // Wrap the transport - baseTransport := apiClient.GetClient().Transport - if baseTransport == nil { - baseTransport = http.DefaultTransport - } - - rt := token.NewRefreshRoundTripper(baseTransport, tp) - apiClient.GetClient().Transport = rt - logger.Info("Installed token refresh round tripper for connection %d (authMethod=%s)", - connection.ID, connection.AuthMethod) - } - - // Persist the freshly minted token so the DB has a correctly encrypted value. - // PrepareApiClient (called by NewApiClientFromConnection) mints the token - // in-memory but does not persist it; without this, the DB may contain a stale - // or corrupted token that breaks GET /connections. - if connection.AuthMethod == models.AppKey && connection.Token != "" { - if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { - logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) - } else { - logger.Info("Persisted initial token for connection %d", connection.ID) - } + // inject the shared auth layer + _, err = CreateAuthenticatedHttpClient(taskCtx, connection, apiClient.GetClient()) + if err != nil { + return nil, err } // create rate limit calculator diff --git a/backend/plugins/github/tasks/http_client.go b/backend/plugins/github/tasks/http_client.go new file mode 100644 index 00000000000..33ef6a3df6b --- /dev/null +++ b/backend/plugins/github/tasks/http_client.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/github/token" +) + +func CreateAuthenticatedHttpClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + baseClient *http.Client, +) (*http.Client, errors.Error) { + + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) + + if baseClient == nil { + baseClient = &http.Client{} + } + + // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. + var tp *token.TokenProvider + if connection.RefreshToken != "" { + tp = token.NewTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { + tp = token.NewAppInstallationTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } + + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + if tp != nil { + baseClient.Transport = token.NewRefreshRoundTripper(baseTransport, tp) + logger.Info( + "Installed token refresh round tripper for connection %d (authMethod=%s)", + connection.ID, + connection.AuthMethod, + ) + + } else if connection.Token != "" { + baseClient.Transport = token.NewStaticRoundTripper( + baseTransport, + connection.Token, + ) + logger.Info( + "Installed static token round tripper for connection %d", + connection.ID, + ) + } + + // Persist the freshly minted token so the DB has a correctly encrypted value. + // PrepareApiClient (called by NewApiClientFromConnection) mints the token + // in-memory but does not persist it; without this, the DB may contain a stale + // or corrupted token that breaks GET /connections. + if connection.AuthMethod == models.AppKey && connection.Token != "" { + if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { + logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) + } else { + logger.Info("Persisted initial token for connection %d", connection.ID) + } + } + + return baseClient, nil +} diff --git a/backend/plugins/github/token/round_tripper.go b/backend/plugins/github/token/round_tripper.go index 8868572dae6..8f05f838d0d 100644 --- a/backend/plugins/github/token/round_tripper.go +++ b/backend/plugins/github/token/round_tripper.go @@ -19,6 +19,8 @@ package token import ( "net/http" + "strings" + "sync/atomic" ) // RefreshRoundTripper is an HTTP transport middleware that automatically manages OAuth token refreshes. @@ -93,3 +95,37 @@ func (rt *RefreshRoundTripper) roundTripWithRetry(req *http.Request, refreshAtte return resp, nil } + +// StaticRoundTripper is an HTTP transport that injects a fixed bearer token. +// Unlike RefreshRoundTripper, it does NOT attempt refresh or retries. +type StaticRoundTripper struct { + base http.RoundTripper + tokens []string + idx atomic.Uint64 +} + +func NewStaticRoundTripper(base http.RoundTripper, rawToken string) *StaticRoundTripper { + if base == nil { + base = http.DefaultTransport + } + parts := strings.Split(rawToken, ",") + tokens := make([]string, 0, len(parts)) + for _, t := range parts { + if t = strings.TrimSpace(t); t != "" { + tokens = append(tokens, t) + } + } + if len(tokens) == 0 { + tokens = []string{rawToken} + } + return &StaticRoundTripper{base: base, tokens: tokens} +} + +func (rt *StaticRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // always overrides headers put by SetupAuthentication, to make sure the token is always injected + // Add(1)-1 yields a 0-based sequence (0, 1, 2, ...) so rotation starts at tokens[0]. + tok := rt.tokens[(rt.idx.Add(1)-1)%uint64(len(rt.tokens))] + reqClone := req.Clone(req.Context()) + reqClone.Header.Set("Authorization", "Bearer "+tok) + return rt.base.RoundTrip(reqClone) +} diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go index 3efbe10a272..f56c77644bf 100644 --- a/backend/plugins/github_graphql/impl/impl.go +++ b/backend/plugins/github_graphql/impl/impl.go @@ -20,10 +20,7 @@ package impl import ( "context" "fmt" - "net/http" - "net/url" "reflect" - "strings" "time" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -39,7 +36,6 @@ import ( "github.com/apache/incubator-devlake/plugins/github_graphql/model/migrationscripts" "github.com/apache/incubator-devlake/plugins/github_graphql/tasks" "github.com/merico-ai/graphql" - "golang.org/x/oauth2" ) // make sure interface is implemented @@ -180,46 +176,10 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return nil, err } - tokens := strings.Split(connection.Token, ",") - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: tokens[0]}, - ) - oauthContext := taskCtx.GetContext() - proxy := connection.GetProxy() - if proxy != "" { - pu, err := url.Parse(proxy) - if err != nil { - return nil, errors.Convert(err) - } - if pu.Scheme == "http" || pu.Scheme == "socks5" { - proxyClient := &http.Client{ - Transport: &http.Transport{Proxy: http.ProxyURL(pu)}, - } - oauthContext = context.WithValue( - taskCtx.GetContext(), - oauth2.HTTPClient, - proxyClient, - ) - logger.Debug("Proxy set in oauthContext to %s", proxy) - } else { - return nil, errors.BadInput.New("Unsupported scheme set in proxy") - } - } - - httpClient := oauth2.NewClient(oauthContext, src) - endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) - if err != nil { - return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) - } - - // github.com and github enterprise have different graphql endpoints - endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql - if endpoint.Hostname() != "api.github.com" { - // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql - endpoint.Path = "/api/graphql" - } - client := graphql.NewClient(endpoint.String(), httpClient) - graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + graphqlClient, err := tasks.CreateGraphqlClient( + taskCtx, + connection, + apiClient.ApiClient.GetClient(), func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) { var query GraphQueryRateLimit dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil)) @@ -230,8 +190,7 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`) } if query.RateLimit == nil { - logger.Info(`github graphql rate limit are disabled, fallback to 5000req/hour`) - return 5000, nil, nil + return 0, nil, errors.Default.New("rate limit unavailable") } logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`, query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt) diff --git a/backend/plugins/github_graphql/tasks/graphql_client.go b/backend/plugins/github_graphql/tasks/graphql_client.go new file mode 100644 index 00000000000..9c248e15cb6 --- /dev/null +++ b/backend/plugins/github_graphql/tasks/graphql_client.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/merico-ai/graphql" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +func CreateGraphqlClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + httpClient *http.Client, + getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), +) (*helper.GraphqlAsyncClient, errors.Error) { + // Build endpoint + endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) + } + // github.com and github enterprise have different graphql endpoints + if endpoint.Hostname() == "api.github.com" { + // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/graphql" + } else { + // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/api/graphql" + } + + gqlClient := graphql.NewClient(endpoint.String(), httpClient) + + return helper.CreateAsyncGraphqlClient( + taskCtx, + gqlClient, + taskCtx.GetLogger(), + getRateRemaining, + // GitHub GraphQL default fallback aligns with GitHub's standard rate limit (~5000) + helper.WithFallbackRateLimit(5000), + ) +} diff --git a/backend/plugins/testregistry/tasks/gcs_client.go b/backend/plugins/testregistry/tasks/gcs_client.go index b6a30c52d93..c51ebc65ce6 100644 --- a/backend/plugins/testregistry/tasks/gcs_client.go +++ b/backend/plugins/testregistry/tasks/gcs_client.go @@ -37,7 +37,8 @@ type GCSBucket struct { bkt *storage.BucketHandle } -// NewGCSBucketClient creates a new GCS client for the Openshift CI bucket. +// NewGCSBucketClient creates a new GCS client for the Openshift CI public bucket +// (test-platform-results-public). func NewGCSBucketClient(ctx context.Context) (*GCSBucket, errors.Error) { inner, err := gcshelper.New(ctx, gcshelper.OpenshiftCIBucketName) if err != nil { diff --git a/docs/upstream-diffs.md b/docs/upstream-diffs.md index 5c7f3ece521..4f702a2a65d 100644 --- a/docs/upstream-diffs.md +++ b/docs/upstream-diffs.md @@ -349,6 +349,38 @@ fix. and `Execute` worker-error logging. Watch for upstream changes around `IgnoreQueryErrors` / `isIgnorableGraphqlQueryError`. +## github/github_graphql: graphql client token refresh (cherry-pick) + +**Files:** +- `backend/helpers/pluginhelper/api/graphql_async_client.go` +- `backend/plugins/github/tasks/api_client.go` +- `backend/plugins/github/tasks/http_client.go` (new) +- `backend/plugins/github/token/round_tripper.go` +- `backend/plugins/github_graphql/impl/impl.go` +- `backend/plugins/github_graphql/tasks/graphql_client.go` (new) +- `env.example` + +**Reason:** `updateRateRemaining` in `graphql_async_client.go` called `panic(err)` +when the GitHub rate-limit API returned an error (e.g. 401 from an expired token). +Because the panic fired inside a background goroutine, Go killed the entire process, +causing ~80 container restarts in 4 days on the prod instance. The upstream fix +replaces the panic with a logged fallback + retry, adds `StaticRoundTripper` for +PAT token rotation, shares the REST client's authenticated HTTP transport with the +GraphQL client (enabling GitHub App token auto-refresh), and introduces a +hierarchical rate-limit fallback (`GRAPHQL_RATE_LIMIT` env → `WithFallbackRateLimit` +→ default 1000). + +**Upstream status:** Merged +**Upstream PR:** https://github.com/apache/devlake/pull/8791 +**Upstream commit:** `61d66a1339d35709868105cd85153c60b824f290` +**Cherry-picked as:** konflux-ci/devlake#162 +**Owner:** @psturc + +**Rebase notes:** Clean cherry-pick, no conflicts. On next upstream rebase this +entry can be removed — the changes will arrive via the normal merge. Watch for +upstream changes to `graphql_async_client.go`, `api_client.go`, or +`round_tripper.go`. + ## ci: unit-test Codecov upload on a separate Ubuntu job **Files:** diff --git a/env.example b/env.example index 9d688f67219..b363a5afc78 100755 --- a/env.example +++ b/env.example @@ -40,6 +40,8 @@ PUSH_API_ALLOWED_TABLES= NOTIFICATION_ENDPOINT= NOTIFICATION_SECRET= +# Default fallback rate limit for GraphQL clients (used if not dynamically fetched) +GRAPHQL_RATE_LIMIT= API_TIMEOUT=120s API_RETRY=3 API_REQUESTS_PER_HOUR=10000