Sync eng/common directory with azure-sdk-tools for PR 17441

eng/common/pipelines/templates/stages/archetype-eval.yml

@@ -76,7 +76,7 @@ extends:
   parameters:
     # The checkbox controls compile-time egress; the account variable gates upload at runtime.
     # Keep trust policy in YAML, without hardcoding DevOps definition IDs in shared code.
-    ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}:
+    ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}:
       AllowAzureStorage: true
     stages:
       - stage: Prepare

eng/common/pipelines/templates/steps/eval-publish-results.yml

@@ -38,10 +38,10 @@ steps:
 
   # Pipeline-owned account configuration is a runtime value, not a template parameter.
   # Missing configuration skips the task; PRs/foreign repos/non-main never get this task.
-  - ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}:
+  - ${{ if and(parameters.publishDashboardResults, eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual')) }}:
     - task: AzureCLI@2
       displayName: 'Publish saved build bundle directly to Blob'
-      condition: and(succeeded(), eq(variables['EvalSummaryComplete'], 'true'), ne(variables['EvalStorageAccountName'], ''), eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Repository.Name'], 'Azure/azure-sdk-tools'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual'))
+      condition: and(succeeded(), eq(variables['EvalSummaryComplete'], 'true'), ne(variables['EvalStorageAccountName'], ''), eq(variables['System.CollectionUri'], 'https://dev.azure.com/azure-sdk/'), eq(variables['System.TeamProject'], 'internal'), startsWith(variables['Build.Repository.Name'], 'Azure/'), not(endsWith(variables['Build.Repository.Name'], '-pr')), eq(variables['Build.SourceBranch'], 'refs/heads/main'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Schedule', 'Manual'))
       timeoutInMinutes: 20
       env:
         EVAL_STORAGE_ACCOUNT_NAME: $(EvalStorageAccountName)

eng/common/scripts/eval/README.md

@@ -67,11 +67,17 @@ publishing to another account does not redirect the reader. Service-connection
 authorization, approved branch checks and existing RBAC remain required.
 
 Publishing policy stays in YAML: organization `https://dev.azure.com/azure-sdk/`,
-project `internal`, repository `Azure/azure-sdk-tools`, exact `refs/heads/main`, and
-a CI, scheduled or manual reason. There is no hardcoded definition-ID allowlist or
-variable template in shared code. Feature branches, other repositories, PRs, pull
-refs and public-project runs do not get the credential-bearing task. At runtime it
-also requires a complete summary and a configured storage account.
+project `internal`, an Azure-owned repository whose name does not end in `-pr`,
+exact `refs/heads/main`, and a CI, scheduled or manual reason. There is no per-repo
+or definition-ID allowlist in shared code. New Azure SDK repos do not need another
+shared-template edit. Personal forks, private `-pr` repos, feature branches, PRs,
+pull refs and public-project runs do not get the credential-bearing task. At runtime
+it also requires a complete summary and a configured storage account.
+
+Each consuming pipeline must opt in through its repo-local entrypoint. A repo-local
+1ES redirect must expose the existing `AllowAzureStorage` hook. The pipeline owner
+must authorize the publisher service connection and configure storage before uploads
+can run. This support does not enable the separate TypeSpec authoring benchmarks.
 
 There is no separate automatic-publication, feature-branch override or network-access
 flag. Publication requests the documented

eng/common/scripts/eval/publisher/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Allow opted-in Azure-owned repositories to use the shared publisher and storage-egress policy without a per-repo allowlist, preserving private `-pr` exclusions, trusted-run checks, owner authorization and completeness checks.
 - Exclude older JUnit from summaries and selected test files when it does not match the latest verified timeline attempt; keep missing results incomplete.
 - Keep shared publisher tests independent of Tools-only CI/Pester/redirect files, while retaining those checks in Tools and testing consumer copies.
 - Pass the create-only wildcard as one `--if-none-match=*` argument so Linux cannot expand it into working-directory file names.
@@ -21,4 +22,4 @@
 - Remove optional local publication status files; log confirmed uploads before attempting warning-only dashboard notifications.
 - Use Node 24.14.1+ and pinned pnpm 12.6.0 for the publisher, with a portable SHA512 lock and frozen installs isolated from consuming workspaces.
 - Enforce required entries and byte/trial/entry budgets during construction, including merged JSONL, instead of decoding the new ZIP again.
-- Validate exactly one Summary-selected attempt per expected shard and parse JSONL strings directly without repeated Buffer conversions.
+- Validate exactly one Summary-selected attempt per expected shard and parse JSONL strings directly without repeated Buffer conversions.

eng/common/scripts/eval/publisher/test/publisher.test.ts

@@ -47,6 +47,24 @@ function run(script, args, env = {}, preload = []) {
     return child;
 }
 
+test("specs bundles preserve pipeline identity and completed failing evaluations", async (t) => {
+    const f = await constructionFixture(t);
+    const specsManifest = { ...manifest, repo: "Azure/azure-rest-api-specs", pipeline: "azure-rest-api-specs - eval-skills",
+        pipelineDefinitionId: "8286", branch: "refs/heads/main", sourceVersion: "c".repeat(40) };
+    const failedTrial = { ...trial, status: "success", error: null, gradeResult: { passed: false } };
+    await writeFile(join(f.artifact, "results.jsonl"), JSON.stringify(failedTrial) + "\n");
+    await writeFile(join(f.junit, "0.junit.xml"), '<testsuite><testcase name="synthetic"><failure message="Evaluation failed"/></testcase></testsuite>');
+
+    const saved = await prepareBundle({ ...f, manifest: specsManifest });
+    const decoded = validateBundle(await readFile(f.outputPath));
+
+    assert.deepEqual(decoded.manifest, specsManifest);
+    assert.equal(saved.blobName, "v1/azure-sdk/internal/8286/1001/1/dashboard-bundle.zip");
+    assert.equal(saved.shards, 1); assert.equal(decoded.trials, 1);
+    assert.deepEqual(JSON.parse(Buffer.from(decoded.entries["results.jsonl"]).toString("utf8").trim()), failedTrial);
+    assert.match(Buffer.from(decoded.entries["junit/0-0.xml"]).toString("utf8"), /<failure/);
+});
+
 test("shared shards -> saved timeline evidence -> token-free summary -> one schema-v1 ZIP", async (t) => {
     const f = await fixture(t), scripts = resolve(import.meta.dirname, "../..");
     const downloads = join(f.root, "downloads"); await mkdir(downloads);
@@ -342,20 +360,33 @@ test("one checkbox gates trusted-run egress; owner configuration additionally ga
     const evaluate = (expression, variables, enabled = true, succeeds = true) => runInNewContext(expression.replace(/\bin\(/g, "oneOf("), {
         variables, parameters: { publishDashboardResults: enabled }, succeeded: () => succeeds,
         and: (...values) => values.every(Boolean), eq: (a, b) => String(a ?? "").toLowerCase() === String(b ?? "").toLowerCase(),
+        not: value => !value,
+        startsWith: (value, prefix) => String(value ?? "").toLowerCase().startsWith(String(prefix).toLowerCase()),
+        endsWith: (value, suffix) => String(value ?? "").toLowerCase().endsWith(String(suffix).toLowerCase()),
         ne: (a, b) => String(a ?? "").toLowerCase() !== String(b ?? "").toLowerCase(),
         oneOf: (value, ...items) => items.some(item => String(value ?? "").toLowerCase() === String(item).toLowerCase()),
     });
     const base = { "System.CollectionUri": "https://dev.azure.com/azure-sdk/", "System.TeamProject": "internal", "Build.Repository.Name": "Azure/azure-sdk-tools",
         "Build.SourceBranch": "refs/heads/main", "Build.Reason": "Manual", "EvalSummaryComplete": "true", "EvalStorageAccountName": "evaltestsummary" };
-    assert.equal(evaluate(networkGate, base), true); assert.equal(evaluate(runtime, base), true);
-    assert.equal(evaluate(networkGate, base, false), false); assert.equal(evaluate(runtime, base, true, false), false);
-    for (const account of [undefined, ""]) assert.equal(evaluate(runtime, { ...base, EvalStorageAccountName: account }), false);
-    assert.equal(evaluate(runtime, { ...base, EvalSummaryComplete: "false" }), false);
-    for (const [key, value] of [["System.CollectionUri", "https://dev.azure.com/other/"], ["System.TeamProject", "public"],
-        ["Build.Repository.Name", "Azure/azure-rest-api-specs"], ["Build.Reason", "PullRequest"], ["Build.Reason", "ResourceTrigger"],
-        ...["refs/pull/1/merge", "refs/heads/feature", "refs/heads/feature/main", "refs/tags/main", ""].map(branch => ["Build.SourceBranch", branch])]) {
-        const variables = { ...base, [key]: value };
-        assert.equal(evaluate(networkGate, variables), false); assert.equal(evaluate(runtime, variables), false);
+    for (const repository of ["Azure/azure-sdk-tools", "Azure/azure-rest-api-specs", "Azure/azure-sdk-for-python", "Azure/new-sdk-repository"]) {
+        const trusted = { ...base, "Build.Repository.Name": repository };
+        for (const reason of ["Manual", "Schedule", "IndividualCI", "BatchedCI"]) {
+            const variables = { ...trusted, "Build.Reason": reason };
+            assert.equal(evaluate(networkGate, variables), true, `${repository}: ${reason} egress`);
+            assert.equal(evaluate(runtime, variables), true, `${repository}: ${reason} upload`);
+        }
+        assert.equal(evaluate(networkGate, trusted, false), false);
+        assert.equal(evaluate(runtime, trusted, true, false), false);
+        for (const account of [undefined, ""]) assert.equal(evaluate(runtime, { ...trusted, EvalStorageAccountName: account }), false);
+        for (const complete of [undefined, "", "false"]) assert.equal(evaluate(runtime, { ...trusted, EvalSummaryComplete: complete }), false);
+        for (const [key, value] of [["System.CollectionUri", "https://dev.azure.com/other/"], ["System.TeamProject", "public"],
+            ["Build.Repository.Name", "example-org/azure-rest-api-specs"], ["Build.Repository.Name", "Azure/azure-rest-api-specs-pr"],
+            ["Build.Repository.Name", "Azure/azure-sdk-for-python-pr"], ["Build.Repository.Name", "example-org/new-sdk-repository"],
+            ["Build.Reason", "PullRequest"], ["Build.Reason", "ResourceTrigger"],
+            ...["refs/pull/1/merge", "refs/heads/feature", "refs/heads/feature/main", "refs/tags/main", ""].map(branch => ["Build.SourceBranch", branch])]) {
+            const variables = { ...trusted, [key]: value };
+            assert.equal(evaluate(networkGate, variables), false); assert.equal(evaluate(runtime, variables), false);
+        }
     }
     for (const id of ["8255", "9999"]) assert.equal(evaluate(networkGate, { ...base, "System.DefinitionId": id }), true, "Definition IDs are archive identity, not shared policy");
 });