diff --git a/eng/common/pipelines/templates/stages/archetype-eval.yml b/eng/common/pipelines/templates/stages/archetype-eval.yml index 8ee61d9878cd..128cc16c70ba 100644 --- a/eng/common/pipelines/templates/stages/archetype-eval.yml +++ b/eng/common/pipelines/templates/stages/archetype-eval.yml @@ -76,7 +76,7 @@ extends: parameters: # The checkbox controls compile-time egress; the account variable gates upload at runtime. # Keep trust policy in YAML, without hardcoding DevOps definition IDs in shared code. - ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}: + ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}: AllowAzureStorage: true stages: - stage: Prepare diff --git a/eng/common/pipelines/templates/steps/eval-publish-results.yml b/eng/common/pipelines/templates/steps/eval-publish-results.yml index 00d8c1c869c8..06ebaff1b49b 100644 --- a/eng/common/pipelines/templates/steps/eval-publish-results.yml +++ b/eng/common/pipelines/templates/steps/eval-publish-results.yml @@ -38,10 +38,10 @@ steps: # Pipeline-owned account configuration is a runtime value, not a template parameter. # Missing configuration skips the task; PRs/foreign repos/non-main never get this task. - - ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}: + - ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}: - task: AzureCLI@2 displayName: 'Publish saved build bundle directly to Blob' - condition: and(succeeded(), eq(variables['EvalSummaryComplete'], 'true'), ne(variables['EvalStorageAccountName'], ''), eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) + condition: and(succeeded(), eq(variables['EvalSummaryComplete'], 'true'), ne(variables['EvalStorageAccountName'], ''), eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) timeoutInMinutes: 20 env: EVAL_STORAGE_ACCOUNT_NAME: $(EvalStorageAccountName) diff --git a/eng/common/scripts/eval/README.md b/eng/common/scripts/eval/README.md index 0fd3b593a885..cccbca480958 100644 --- a/eng/common/scripts/eval/README.md +++ b/eng/common/scripts/eval/README.md @@ -67,11 +67,17 @@ publishing to another account does not redirect the reader. Service-connection authorization, approved branch checks and existing RBAC remain required. Publishing policy stays in YAML: organization `https://dev.azure.com/azure-sdk/`, -project `internal`, repository `Azure/azure-sdk-tools`, exact `refs/heads/main`, and -a CI, scheduled or manual reason. There is no hardcoded definition-ID allowlist or -variable template in shared code. Feature branches, other repositories, PRs, pull -refs and public-project runs do not get the credential-bearing task. At runtime it -also requires a complete summary and a configured storage account. +project `internal`, an Azure-owned repository whose name does not end in `-pr`, +exact `refs/heads/main`, and a CI, scheduled or manual reason. There is no per-repo +or definition-ID allowlist in shared code. New Azure SDK repos do not need another +shared-template edit. Personal forks, private `-pr` repos, feature branches, PRs, +pull refs and public-project runs do not get the credential-bearing task. At runtime +it also requires a complete summary and a configured storage account. + +Each consuming pipeline must opt in through its repo-local entrypoint. A repo-local +1ES redirect must expose the existing `AllowAzureStorage` hook. The pipeline owner +must authorize the publisher service connection and configure storage before uploads +can run. This support does not enable the separate TypeSpec authoring benchmarks. There is no separate automatic-publication, feature-branch override or network-access flag. Publication requests the documented diff --git a/eng/common/scripts/eval/publisher/CHANGELOG.md b/eng/common/scripts/eval/publisher/CHANGELOG.md index 3bf724917e3a..bb351914a2ac 100644 --- a/eng/common/scripts/eval/publisher/CHANGELOG.md +++ b/eng/common/scripts/eval/publisher/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Allow opted-in Azure-owned repositories to use the shared publisher and storage-egress policy without a per-repo allowlist, preserving private `-pr` exclusions, trusted-run checks, owner authorization and completeness checks. - Exclude older JUnit from summaries and selected test files when it does not match the latest verified timeline attempt; keep missing results incomplete. - Keep shared publisher tests independent of Tools-only CI/Pester/redirect files, while retaining those checks in Tools and testing consumer copies. - Pass the create-only wildcard as one `--if-none-match=*` argument so Linux cannot expand it into working-directory file names. @@ -21,4 +22,4 @@ - Remove optional local publication status files; log confirmed uploads before attempting warning-only dashboard notifications. - Use Node 24.14.1+ and pinned pnpm 12.6.0 for the publisher, with a portable SHA512 lock and frozen installs isolated from consuming workspaces. - Enforce required entries and byte/trial/entry budgets during construction, including merged JSONL, instead of decoding the new ZIP again. -- Validate exactly one Summary-selected attempt per expected shard and parse JSONL strings directly without repeated Buffer conversions. \ No newline at end of file +- Validate exactly one Summary-selected attempt per expected shard and parse JSONL strings directly without repeated Buffer conversions. diff --git a/eng/common/scripts/eval/publisher/test/publisher.test.ts b/eng/common/scripts/eval/publisher/test/publisher.test.ts index e566a87e1e28..f18fdf6d5a1a 100644 --- a/eng/common/scripts/eval/publisher/test/publisher.test.ts +++ b/eng/common/scripts/eval/publisher/test/publisher.test.ts @@ -47,6 +47,24 @@ function run(script, args, env = {}, preload = []) { return child; } +test("specs bundles preserve pipeline identity and completed failing evaluations", async (t) => { + const f = await constructionFixture(t); + const specsManifest = { ...manifest, repo: "Azure/azure-rest-api-specs", pipeline: "azure-rest-api-specs - eval-skills", + pipelineDefinitionId: "8286", branch: "refs/heads/main", sourceVersion: "c".repeat(40) }; + const failedTrial = { ...trial, status: "success", error: null, gradeResult: { passed: false } }; + await writeFile(join(f.artifact, "results.jsonl"), JSON.stringify(failedTrial) + "\n"); + await writeFile(join(f.junit, "0.junit.xml"), ''); + + const saved = await prepareBundle({ ...f, manifest: specsManifest }); + const decoded = validateBundle(await readFile(f.outputPath)); + + assert.deepEqual(decoded.manifest, specsManifest); + assert.equal(saved.blobName, "v1/azure-sdk/internal/8286/1001/1/dashboard-bundle.zip"); + assert.equal(saved.shards, 1); assert.equal(decoded.trials, 1); + assert.deepEqual(JSON.parse(Buffer.from(decoded.entries["results.jsonl"]).toString("utf8").trim()), failedTrial); + assert.match(Buffer.from(decoded.entries["junit/0-0.xml"]).toString("utf8"), / saved timeline evidence -> token-free summary -> one schema-v1 ZIP", async (t) => { const f = await fixture(t), scripts = resolve(import.meta.dirname, "../.."); const downloads = join(f.root, "downloads"); await mkdir(downloads); @@ -342,20 +360,33 @@ test("one checkbox gates trusted-run egress; owner configuration additionally ga const evaluate = (expression, variables, enabled = true, succeeds = true) => runInNewContext(expression.replace(/\bin\(/g, "oneOf("), { variables, parameters: { publishDashboardResults: enabled }, succeeded: () => succeeds, and: (...values) => values.every(Boolean), eq: (a, b) => String(a ?? "").toLowerCase() === String(b ?? "").toLowerCase(), + not: value => !value, + startsWith: (value, prefix) => String(value ?? "").toLowerCase().startsWith(String(prefix).toLowerCase()), + endsWith: (value, suffix) => String(value ?? "").toLowerCase().endsWith(String(suffix).toLowerCase()), ne: (a, b) => String(a ?? "").toLowerCase() !== String(b ?? "").toLowerCase(), oneOf: (value, ...items) => items.some(item => String(value ?? "").toLowerCase() === String(item).toLowerCase()), }); const base = { "System.CollectionUri": "https://dev.azure.com/azure-sdk/", "System.TeamProject": "internal", "Build.Repository.Name": "Azure/azure-sdk-tools", "Build.SourceBranch": "refs/heads/main", "Build.Reason": "Manual", "EvalSummaryComplete": "true", "EvalStorageAccountName": "evaltestsummary" }; - assert.equal(evaluate(networkGate, base), true); assert.equal(evaluate(runtime, base), true); - assert.equal(evaluate(networkGate, base, false), false); assert.equal(evaluate(runtime, base, true, false), false); - for (const account of [undefined, ""]) assert.equal(evaluate(runtime, { ...base, EvalStorageAccountName: account }), false); - assert.equal(evaluate(runtime, { ...base, EvalSummaryComplete: "false" }), false); - for (const [key, value] of [["System.CollectionUri", "https://dev.azure.com/other/"], ["System.TeamProject", "public"], - ["Build.Repository.Name", "Azure/azure-rest-api-specs"], ["Build.Reason", "PullRequest"], ["Build.Reason", "ResourceTrigger"], - ...["refs/pull/1/merge", "refs/heads/feature", "refs/heads/feature/main", "refs/tags/main", ""].map(branch => ["Build.SourceBranch", branch])]) { - const variables = { ...base, [key]: value }; - assert.equal(evaluate(networkGate, variables), false); assert.equal(evaluate(runtime, variables), false); + for (const repository of ["Azure/azure-sdk-tools", "Azure/azure-rest-api-specs", "Azure/azure-sdk-for-python", "Azure/new-sdk-repository"]) { + const trusted = { ...base, "Build.Repository.Name": repository }; + for (const reason of ["Manual", "Schedule", "IndividualCI", "BatchedCI"]) { + const variables = { ...trusted, "Build.Reason": reason }; + assert.equal(evaluate(networkGate, variables), true, `${repository}: ${reason} egress`); + assert.equal(evaluate(runtime, variables), true, `${repository}: ${reason} upload`); + } + assert.equal(evaluate(networkGate, trusted, false), false); + assert.equal(evaluate(runtime, trusted, true, false), false); + for (const account of [undefined, ""]) assert.equal(evaluate(runtime, { ...trusted, EvalStorageAccountName: account }), false); + for (const complete of [undefined, "", "false"]) assert.equal(evaluate(runtime, { ...trusted, EvalSummaryComplete: complete }), false); + for (const [key, value] of [["System.CollectionUri", "https://dev.azure.com/other/"], ["System.TeamProject", "public"], + ["Build.Repository.Name", "example-org/azure-rest-api-specs"], ["Build.Repository.Name", "Azure/azure-rest-api-specs-pr"], + ["Build.Repository.Name", "Azure/azure-sdk-for-python-pr"], ["Build.Repository.Name", "example-org/new-sdk-repository"], + ["Build.Reason", "PullRequest"], ["Build.Reason", "ResourceTrigger"], + ...["refs/pull/1/merge", "refs/heads/feature", "refs/heads/feature/main", "refs/tags/main", ""].map(branch => ["Build.SourceBranch", branch])]) { + const variables = { ...trusted, [key]: value }; + assert.equal(evaluate(networkGate, variables), false); assert.equal(evaluate(runtime, variables), false); + } } for (const id of ["8255", "9999"]) assert.equal(evaluate(networkGate, { ...base, "System.DefinitionId": id }), true, "Definition IDs are archive identity, not shared policy"); });