diff --git a/eng/pipelines/pullrequest.yml b/eng/pipelines/pullrequest.yml index 012187e2d707..2adb1dc9fdfb 100644 --- a/eng/pipelines/pullrequest.yml +++ b/eng/pipelines/pullrequest.yml @@ -29,6 +29,12 @@ extends: BuildTargetingString: "*" TestProxy: true TestTimeOutInMinutes: 180 + # This storage feature branch needs the same native toolchain as sdk/storage/ci.yml. + # The generic PR pipeline does not inherit that service pipeline's parameters. + InstallMsRustToolchain: true + MsRustWorkingDirectory: $(Build.SourcesDirectory)/sdk/storage/azure-storage-extensions-transfer + MsRustToolchainFeed: https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust-org-test/nuget/v3/index.json + MsRustAdditionalTargets: aarch64-pc-windows-msvc # we still exclude cosmos, because of the implicit dependency on the cosmos emulator. Within a pullrequest pipeline we do not # currently have the ability to inject steps from a different ci.yml, which is required to start the cosmos emulator. # As such, the pullrequest pipeline itself will not invoke the cosmos tests, but the triggered `python - cosmos - ci` public build WILL. diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index 289549a717a8..cba90953e563 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -72,6 +72,15 @@ parameters: - name: DisableAzureResourceCreation type: boolean default: false + - name: InstallMsRustToolchain + type: boolean + default: false + - name: MsRustWorkingDirectory + type: string + default: '' + - name: MsRustToolchainFeed + type: string + default: '' jobs: - job: @@ -174,6 +183,13 @@ jobs: } displayName: 'Check for compiled dev requirements' + - ${{ if parameters.InstallMsRustToolchain }}: + - template: /eng/pipelines/templates/steps/install-msrust-toolchain.yml + parameters: + ToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + WorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + Condition: and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true')) + - pwsh: | Write-Host "##vso[task.setvariable variable=CIBW_ARCHS]native" Write-Host "##vso[task.setvariable variable=CIBW_SKIP]pp* *musllinux*" @@ -181,9 +197,16 @@ jobs: # CIBW_ENVIRONMENT_PASS_LINUX, not CIBW_ENVIRONMENT: the latter would clobber the package's # own [tool.cibuildwheel].environment (CFLAGS). PIP_INDEX_URL is set by the time the test step # runs (use-python-version.yml, then auth-dev-feed.yml inside build-test.yml). - Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL" + $passVariables = @('PIP_INDEX_URL') + if ($env:RUST_ENVIRONMENT_PASS) { + $passVariables += $env:RUST_ENVIRONMENT_PASS.Split(' ') + } + Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]$($passVariables -join ' ')" displayName: 'Configure cibuildwheel for dev requirement builds' condition: and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true')) + env: + ${{ if parameters.InstallMsRustToolchain }}: + RUST_ENVIRONMENT_PASS: MSRUSTUP_ACCESS_TOKEN MSRUSTUP_FEED_URL CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_INDEX CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER # Download the Python versions selected by cibuildwheel from the approved feed. # This test job builds only for the current machine's architecture, not ARM64. @@ -207,6 +230,11 @@ jobs: CoverageArg: $(CoverageArg) EnvVars: AZURE_RUN_MODE: 'Live' #Record, Playback + ${{ if parameters.InstallMsRustToolchain }}: + MSRUSTUP_ACCESS_TOKEN: $(System.AccessToken) + MSRUSTUP_FEED_URL: $(MSRUSTUP_FEED_URL) + CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN) + CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER) ${{ insert }}: ${{ parameters.EnvVars }} PythonVersion: $(PythonVersion) OSVmImage: $(OSVmImage) diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 27a3533fb26f..5e2a00691f8b 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -106,6 +106,16 @@ parameters: - name: PersistOidcToken type: boolean default: false + # Opt in when test requirements build a Rust extension from source. + - name: InstallMsRustToolchain + type: boolean + default: false + - name: MsRustWorkingDirectory + type: string + default: '' + - name: MsRustToolchainFeed + type: string + default: '' extends: template: /eng/pipelines/templates/stages/1es-redirect.yml @@ -159,6 +169,9 @@ extends: CheckEnv: ${{ parameters.CheckEnv }} UseFederatedAuth: ${{ and(parameters.UseFederatedAuth, ne(matrixType, 'fixed')) }} PersistOidcToken: ${{ parameters.PersistOidcToken }} + InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }} + MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }} MatrixConfigs: - ${{ if eq(matrixType, 'fixed') }}: - ${{ parameters.FixedAccountMatrixConfigs }} diff --git a/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py b/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py index 04bed9e492e4..8dbf79de4909 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py +++ b/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py @@ -98,9 +98,18 @@ def run_logged( # Stream live, no capturing return subprocess.run(cmd, cwd=cwd, check=check, text=True, stderr=subprocess.STDOUT, env=env) else: - # Capture merged output but don't print unless there's a failure + # Native build tools emit UTF-8 even when Windows defaults to a legacy code page. + # Preserve unexpected bytes as escapes rather than masking the command's exit status. return subprocess.run( - cmd, cwd=cwd, check=check, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env + cmd, + cwd=cwd, + check=check, + text=True, + encoding="utf-8", + errors="backslashreplace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, ) except subprocess.CalledProcessError as e: logger.error(f"Command failed: {' '.join(cmd)}") diff --git a/eng/tools/azure-sdk-tools/tests/test_logging_config.py b/eng/tools/azure-sdk-tools/tests/test_logging_config.py index 89b4966b179c..a360f1e0e998 100644 --- a/eng/tools/azure-sdk-tools/tests/test_logging_config.py +++ b/eng/tools/azure-sdk-tools/tests/test_logging_config.py @@ -82,3 +82,27 @@ def test_run_logged_failure_preserved(stream, tmp_path, caplog): assert "Command failed:" in caplog.text if not stream: assert "build failed" in caplog.text + + +@pytest.mark.parametrize("returncode", [0, 7]) +def test_run_logged_decodes_native_build_output(returncode, tmp_path, caplog, monkeypatch): + monkeypatch.setattr("locale.getencoding", lambda: "cp1252", raising=False) + command = [ + sys.executable, + "-c", + "import sys; sys.stdout.buffer.write(b'build \\xe2\\x9c\\x8d\\n'); " + "sys.stderr.buffer.write(b'diagnostic \\xff\\n'); " + f"raise SystemExit({returncode})", + ] + if returncode: + with pytest.raises(subprocess.CalledProcessError) as error: + run_logged(command, cwd=str(tmp_path), check=True, should_stream_to_console=False) + assert error.value.returncode == returncode + output = error.value.stdout + assert output.strip() in caplog.text + else: + result = run_logged(command, cwd=str(tmp_path), check=True, should_stream_to_console=False) + assert result.returncode == 0 + output = result.stdout + assert "build \u270d\n" in output + assert "diagnostic \\xff\n" in output diff --git a/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py b/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py index b83730a96c72..05c74737b39a 100644 --- a/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py +++ b/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py @@ -14,7 +14,6 @@ import pytest import yaml - REPO_ROOT = Path(__file__).resolve().parents[4] TEMPLATES = REPO_ROOT / "eng" / "pipelines" / "templates" RUST_GATE = "${{ if parameters.InstallMsRustToolchain }}" @@ -53,6 +52,26 @@ def test_analysis_checkout_has_history_for_api_comparison(): } +def test_storage_tests_use_the_same_rust_feed_as_wheel_builds(): + storage = REPO_ROOT / "sdk" / "storage" + tests = yaml.safe_load((storage / "tests.yml").read_text())["extends"]["parameters"] + build = yaml.safe_load((storage / "ci.yml").read_text())["extends"]["parameters"] + for name in RUST_PARAMETERS[:3]: + assert tests[name] == build[name] + assert tests["InstallMsRustToolchain"] is True + directory = tests["MsRustWorkingDirectory"].replace("$(Build.SourcesDirectory)", str(REPO_ROOT)) + assert (Path(directory) / "rust-toolchain.toml").is_file() + + +def test_storage_feature_pr_builds_use_the_service_rust_configuration(): + pullrequest = yaml.safe_load((REPO_ROOT / "eng" / "pipelines" / "pullrequest.yml").read_text()) + storage = yaml.safe_load((REPO_ROOT / "sdk" / "storage" / "ci.yml").read_text()) + parameters = pullrequest["extends"]["parameters"] + assert parameters["InstallMsRustToolchain"] is True + for name in RUST_PARAMETERS: + assert parameters[name] == storage["extends"]["parameters"][name] + + def test_rust_support_is_disabled_by_default_and_forwarded(): archetype = load_template("stages/archetype-sdk-client.yml") jobs = load_template("jobs/ci.yml") @@ -153,6 +172,77 @@ def test_dev_requirements_share_selected_target_provisioning(tmp_path): assert "variable=AZURESDK_PYTHON_ARM64_LIBS]" not in result.stdout +def test_live_test_rust_support_is_opt_in_and_forwarded(): + archetype = load_template("stages/archetype-sdk-tests.yml") + live = load_template("jobs/live.tests.yml") + parameters = RUST_PARAMETERS[:3] + for template in (archetype, live): + defaults = {parameter["name"]: parameter.get("default") for parameter in template["parameters"]} + assert defaults["InstallMsRustToolchain"] is False + assert all(defaults[name] == "" for name in parameters[1:]) + + stages = archetype["extends"]["parameters"]["stages"][0] + clouds = next(iter(stages.values()))[0] + cloud_gate = next(iter(clouds.values()))[0] + supported_gate = next(iter(cloud_gate.values()))[0] + stage = next(iter(supported_gate.values()))[0] + matrix_types = stage["jobs"][0] + matrix_gate = next(iter(matrix_types.values()))[0] + matrix = next(iter(matrix_gate.values()))[0] + forwarded = matrix["parameters"]["AdditionalParameters"] + assert all(forwarded[name] == "${{ parameters." + name + " }}" for name in parameters) + + steps = live["jobs"][0]["steps"] + install = next(step[RUST_GATE][0] for step in steps if isinstance(step, dict) and RUST_GATE in step) + assert install["template"].endswith("install-msrust-toolchain.yml") + assert install["parameters"] == { + "ToolchainFeed": "${{ parameters.MsRustToolchainFeed }}", + "WorkingDirectory": "${{ parameters.MsRustWorkingDirectory }}", + "Condition": "and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true'))", + } + test = next( + step for step in steps if isinstance(step, dict) and step.get("template", "").endswith("build-test.yml") + ) + credentials = test["parameters"]["EnvVars"][RUST_GATE] + build = load_template("steps/build-package-artifacts.yml") + generate = next(step for step in build["steps"] if step.get("displayName") == "Generate Packages") + assert credentials == generate["env"][RUST_GATE] + assert "${{ insert }}" in test["parameters"]["EnvVars"] + assert steps.index(next(step for step in steps if isinstance(step, dict) and RUST_GATE in step)) < steps.index(test) + + run_tests = load_template("steps/build-test.yml")["steps"] + federated = next(step for step in run_tests if "${{ if eq('true', parameters.UseFederatedAuth) }}" in step) + non_federated = next(step for step in run_tests if "${{ else }}" in step) + for step in federated["${{ if eq('true', parameters.UseFederatedAuth) }}"] + non_federated["${{ else }}"]: + environment = step["env"] + assert ( + environment.get("${{ insert }}") if isinstance(environment, dict) else environment + ) == "${{ parameters.EnvVars }}" + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_live_test_container_forwarding_preserves_non_rust_builds(enabled): + steps = load_template("jobs/live.tests.yml")["jobs"][0]["steps"] + configure = next( + step + for step in steps + if isinstance(step, dict) and step.get("displayName") == "Configure cibuildwheel for dev requirement builds" + ) + build_steps = load_template("steps/build-package-artifacts.yml")["steps"] + build_forwarding = next( + step for step in build_steps if step.get("displayName") == "Forward build environment to cibuildwheel" + ) + assert configure["env"] == build_forwarding["env"] + extra = configure["env"][RUST_GATE]["RUST_ENVIRONMENT_PASS"] if enabled else "" + result = run_powershell(configure["pwsh"], RUST_ENVIRONMENT_PASS=extra) + assert result.returncode == 0, result.stderr + expected = "PIP_INDEX_URL" + (" " + extra if enabled else "") + assert f"variable=CIBW_ENVIRONMENT_PASS_LINUX]{expected}\n" in result.stdout + assert "variable=CIBW_ARCHS]native" in result.stdout + assert "variable=CIBW_SKIP]pp* *musllinux*" in result.stdout + assert "variable=CIBW_TEST_SKIP]*" in result.stdout + + @pytest.mark.parametrize("sign_binaries", [False, True]) def test_platform_selection_uses_artifact_opt_in(tmp_path, sign_binaries): package_info = {"Name": "example-native-package", "ArtifactDetails": {"signBinaries": sign_binaries}} @@ -252,10 +342,12 @@ def test_build_powershell_scripts_parse(): "steps/install-msrust-toolchain.yml", "steps/build-package-artifacts.yml", "steps/resolve-build-platforms.yml", + "jobs/live.tests.yml", ): template = load_template(template_path) - for step in template["steps"]: - if "pwsh" not in step: + steps = template["jobs"][0]["steps"] if "jobs" in template else template["steps"] + for step in steps: + if not isinstance(step, dict) or "pwsh" not in step: continue # Azure expands template expressions and the command-valued PIP_EXE macro before parsing. script = re.sub(r"\$\{\{.*?\}\}", "example", step["pwsh"]) diff --git a/sdk/storage/tests.yml b/sdk/storage/tests.yml index 864ebeb5f6a3..fff0e59c19a2 100644 --- a/sdk/storage/tests.yml +++ b/sdk/storage/tests.yml @@ -16,6 +16,10 @@ extends: # All 5 service stages run in parallel, and with no parallel limits the tests may get throttled. MaxParallel: 2 ServiceDirectory: storage + # Blob test requirements build the native transfer extension. + InstallMsRustToolchain: true + MsRustWorkingDirectory: $(Build.SourcesDirectory)/sdk/storage/azure-storage-extensions-transfer + MsRustToolchainFeed: https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust-org-test/nuget/v3/index.json InjectedPackages: $(InjectedPackages) Packages: ${{ parameters.Services }} MatrixReplace: