Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions eng/pipelines/pullrequest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ extends:
BuildTargetingString: "*"
TestProxy: true
TestTimeOutInMinutes: 180
# This storage feature branch needs the same native toolchain as sdk/storage/ci.yml.
# The generic PR pipeline does not inherit that service pipeline's parameters.
InstallMsRustToolchain: true
MsRustWorkingDirectory: $(Build.SourcesDirectory)/sdk/storage/azure-storage-extensions-transfer
MsRustToolchainFeed: https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust-org-test/nuget/v3/index.json
MsRustAdditionalTargets: aarch64-pc-windows-msvc
# we still exclude cosmos, because of the implicit dependency on the cosmos emulator. Within a pullrequest pipeline we do not
# currently have the ability to inject steps from a different ci.yml, which is required to start the cosmos emulator.
# As such, the pullrequest pipeline itself will not invoke the cosmos tests, but the triggered `python - cosmos - ci` public build WILL.
Expand Down
30 changes: 29 additions & 1 deletion eng/pipelines/templates/jobs/live.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ parameters:
- name: DisableAzureResourceCreation
type: boolean
default: false
- name: InstallMsRustToolchain
type: boolean
default: false
- name: MsRustWorkingDirectory
type: string
default: ''
- name: MsRustToolchainFeed
type: string
default: ''

jobs:
- job:
Expand Down Expand Up @@ -174,16 +183,30 @@ jobs:
}
displayName: 'Check for compiled dev requirements'

- ${{ if parameters.InstallMsRustToolchain }}:
- template: /eng/pipelines/templates/steps/install-msrust-toolchain.yml
parameters:
ToolchainFeed: ${{ parameters.MsRustToolchainFeed }}
WorkingDirectory: ${{ parameters.MsRustWorkingDirectory }}
Condition: and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true'))

- pwsh: |
Write-Host "##vso[task.setvariable variable=CIBW_ARCHS]native"
Write-Host "##vso[task.setvariable variable=CIBW_SKIP]pp* *musllinux*"
Write-Host "##vso[task.setvariable variable=CIBW_TEST_SKIP]*"
# CIBW_ENVIRONMENT_PASS_LINUX, not CIBW_ENVIRONMENT: the latter would clobber the package's
# own [tool.cibuildwheel].environment (CFLAGS). PIP_INDEX_URL is set by the time the test step
# runs (use-python-version.yml, then auth-dev-feed.yml inside build-test.yml).
Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL"
$passVariables = @('PIP_INDEX_URL')
if ($env:RUST_ENVIRONMENT_PASS) {
$passVariables += $env:RUST_ENVIRONMENT_PASS.Split(' ')
}
Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]$($passVariables -join ' ')"
displayName: 'Configure cibuildwheel for dev requirement builds'
condition: and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true'))
env:
${{ if parameters.InstallMsRustToolchain }}:
RUST_ENVIRONMENT_PASS: MSRUSTUP_ACCESS_TOKEN MSRUSTUP_FEED_URL CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_INDEX CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER

# Download the Python versions selected by cibuildwheel from the approved feed.
# This test job builds only for the current machine's architecture, not ARM64.
Expand All @@ -207,6 +230,11 @@ jobs:
CoverageArg: $(CoverageArg)
EnvVars:
AZURE_RUN_MODE: 'Live' #Record, Playback
${{ if parameters.InstallMsRustToolchain }}:
MSRUSTUP_ACCESS_TOKEN: $(System.AccessToken)
MSRUSTUP_FEED_URL: $(MSRUSTUP_FEED_URL)
CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN)
CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER)
${{ insert }}: ${{ parameters.EnvVars }}
PythonVersion: $(PythonVersion)
OSVmImage: $(OSVmImage)
Expand Down
13 changes: 13 additions & 0 deletions eng/pipelines/templates/stages/archetype-sdk-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,16 @@ parameters:
- name: PersistOidcToken
type: boolean
default: false
# Opt in when test requirements build a Rust extension from source.
- name: InstallMsRustToolchain
type: boolean
default: false
- name: MsRustWorkingDirectory
type: string
default: ''
- name: MsRustToolchainFeed
type: string
default: ''

extends:
template: /eng/pipelines/templates/stages/1es-redirect.yml
Expand Down Expand Up @@ -159,6 +169,9 @@ extends:
CheckEnv: ${{ parameters.CheckEnv }}
UseFederatedAuth: ${{ and(parameters.UseFederatedAuth, ne(matrixType, 'fixed')) }}
PersistOidcToken: ${{ parameters.PersistOidcToken }}
InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }}
MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }}
MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }}
MatrixConfigs:
- ${{ if eq(matrixType, 'fixed') }}:
- ${{ parameters.FixedAccountMatrixConfigs }}
Expand Down
98 changes: 95 additions & 3 deletions eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import pytest
import yaml


REPO_ROOT = Path(__file__).resolve().parents[4]
TEMPLATES = REPO_ROOT / "eng" / "pipelines" / "templates"
RUST_GATE = "${{ if parameters.InstallMsRustToolchain }}"
Expand Down Expand Up @@ -53,6 +52,26 @@ def test_analysis_checkout_has_history_for_api_comparison():
}


def test_storage_tests_use_the_same_rust_feed_as_wheel_builds():
storage = REPO_ROOT / "sdk" / "storage"
tests = yaml.safe_load((storage / "tests.yml").read_text())["extends"]["parameters"]
build = yaml.safe_load((storage / "ci.yml").read_text())["extends"]["parameters"]
for name in RUST_PARAMETERS[:3]:
assert tests[name] == build[name]
assert tests["InstallMsRustToolchain"] is True
directory = tests["MsRustWorkingDirectory"].replace("$(Build.SourcesDirectory)", str(REPO_ROOT))
assert (Path(directory) / "rust-toolchain.toml").is_file()


def test_storage_feature_pr_builds_use_the_service_rust_configuration():
pullrequest = yaml.safe_load((REPO_ROOT / "eng" / "pipelines" / "pullrequest.yml").read_text())
storage = yaml.safe_load((REPO_ROOT / "sdk" / "storage" / "ci.yml").read_text())
parameters = pullrequest["extends"]["parameters"]
assert parameters["InstallMsRustToolchain"] is True
for name in RUST_PARAMETERS:
assert parameters[name] == storage["extends"]["parameters"][name]


def test_rust_support_is_disabled_by_default_and_forwarded():
archetype = load_template("stages/archetype-sdk-client.yml")
jobs = load_template("jobs/ci.yml")
Expand Down Expand Up @@ -153,6 +172,77 @@ def test_dev_requirements_share_selected_target_provisioning(tmp_path):
assert "variable=AZURESDK_PYTHON_ARM64_LIBS]" not in result.stdout


def test_live_test_rust_support_is_opt_in_and_forwarded():
archetype = load_template("stages/archetype-sdk-tests.yml")
live = load_template("jobs/live.tests.yml")
parameters = RUST_PARAMETERS[:3]
for template in (archetype, live):
defaults = {parameter["name"]: parameter.get("default") for parameter in template["parameters"]}
assert defaults["InstallMsRustToolchain"] is False
assert all(defaults[name] == "" for name in parameters[1:])

stages = archetype["extends"]["parameters"]["stages"][0]
clouds = next(iter(stages.values()))[0]
cloud_gate = next(iter(clouds.values()))[0]
supported_gate = next(iter(cloud_gate.values()))[0]
stage = next(iter(supported_gate.values()))[0]
matrix_types = stage["jobs"][0]
matrix_gate = next(iter(matrix_types.values()))[0]
matrix = next(iter(matrix_gate.values()))[0]
forwarded = matrix["parameters"]["AdditionalParameters"]
assert all(forwarded[name] == "${{ parameters." + name + " }}" for name in parameters)

steps = live["jobs"][0]["steps"]
install = next(step[RUST_GATE][0] for step in steps if isinstance(step, dict) and RUST_GATE in step)
assert install["template"].endswith("install-msrust-toolchain.yml")
assert install["parameters"] == {
"ToolchainFeed": "${{ parameters.MsRustToolchainFeed }}",
"WorkingDirectory": "${{ parameters.MsRustWorkingDirectory }}",
"Condition": "and(succeeded(), eq(variables['CIBUILDWHEEL_DEV_REQ'], 'true'))",
}
test = next(
step for step in steps if isinstance(step, dict) and step.get("template", "").endswith("build-test.yml")
)
credentials = test["parameters"]["EnvVars"][RUST_GATE]
build = load_template("steps/build-package-artifacts.yml")
generate = next(step for step in build["steps"] if step.get("displayName") == "Generate Packages")
assert credentials == generate["env"][RUST_GATE]
assert "${{ insert }}" in test["parameters"]["EnvVars"]
assert steps.index(next(step for step in steps if isinstance(step, dict) and RUST_GATE in step)) < steps.index(test)

run_tests = load_template("steps/build-test.yml")["steps"]
federated = next(step for step in run_tests if "${{ if eq('true', parameters.UseFederatedAuth) }}" in step)
non_federated = next(step for step in run_tests if "${{ else }}" in step)
for step in federated["${{ if eq('true', parameters.UseFederatedAuth) }}"] + non_federated["${{ else }}"]:
environment = step["env"]
assert (
environment.get("${{ insert }}") if isinstance(environment, dict) else environment
) == "${{ parameters.EnvVars }}"


@pytest.mark.parametrize("enabled", [False, True])
def test_live_test_container_forwarding_preserves_non_rust_builds(enabled):
steps = load_template("jobs/live.tests.yml")["jobs"][0]["steps"]
configure = next(
step
for step in steps
if isinstance(step, dict) and step.get("displayName") == "Configure cibuildwheel for dev requirement builds"
)
build_steps = load_template("steps/build-package-artifacts.yml")["steps"]
build_forwarding = next(
step for step in build_steps if step.get("displayName") == "Forward build environment to cibuildwheel"
)
assert configure["env"] == build_forwarding["env"]
extra = configure["env"][RUST_GATE]["RUST_ENVIRONMENT_PASS"] if enabled else ""
result = run_powershell(configure["pwsh"], RUST_ENVIRONMENT_PASS=extra)
assert result.returncode == 0, result.stderr
expected = "PIP_INDEX_URL" + (" " + extra if enabled else "")
assert f"variable=CIBW_ENVIRONMENT_PASS_LINUX]{expected}\n" in result.stdout
assert "variable=CIBW_ARCHS]native" in result.stdout
assert "variable=CIBW_SKIP]pp* *musllinux*" in result.stdout
assert "variable=CIBW_TEST_SKIP]*" in result.stdout


@pytest.mark.parametrize("sign_binaries", [False, True])
def test_platform_selection_uses_artifact_opt_in(tmp_path, sign_binaries):
package_info = {"Name": "example-native-package", "ArtifactDetails": {"signBinaries": sign_binaries}}
Expand Down Expand Up @@ -252,10 +342,12 @@ def test_build_powershell_scripts_parse():
"steps/install-msrust-toolchain.yml",
"steps/build-package-artifacts.yml",
"steps/resolve-build-platforms.yml",
"jobs/live.tests.yml",
):
template = load_template(template_path)
for step in template["steps"]:
if "pwsh" not in step:
steps = template["jobs"][0]["steps"] if "jobs" in template else template["steps"]
for step in steps:
if not isinstance(step, dict) or "pwsh" not in step:
continue
# Azure expands template expressions and the command-valued PIP_EXE macro before parsing.
script = re.sub(r"\$\{\{.*?\}\}", "example", step["pwsh"])
Expand Down
4 changes: 4 additions & 0 deletions sdk/storage/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ extends:
# All 5 service stages run in parallel, and with no parallel limits the tests may get throttled.
MaxParallel: 2
ServiceDirectory: storage
# Blob test requirements build the native transfer extension.
InstallMsRustToolchain: true
MsRustWorkingDirectory: $(Build.SourcesDirectory)/sdk/storage/azure-storage-extensions-transfer
MsRustToolchainFeed: https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust-org-test/nuget/v3/index.json
InjectedPackages: $(InjectedPackages)
Packages: ${{ parameters.Services }}
MatrixReplace:
Expand Down
Loading